af_smc.c 50.0 KB
Newer Older
U
Ursula Braun 已提交
1 2 3 4 5 6 7 8
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
 *  applies to SOCK_STREAM sockets only
 *  offers an alternative communication option for TCP-protocol sockets
 *  applicable with RoCE-cards only
 *
9 10 11
 *  Initial restrictions:
 *    - support for alternate links postponed
 *
12
 *  Copyright IBM Corp. 2016, 2018
U
Ursula Braun 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 *              based on prototype from Frank Blaschka
 */

#define KMSG_COMPONENT "smc"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/socket.h>
23
#include <linux/workqueue.h>
24
#include <linux/in.h>
25
#include <linux/sched/signal.h>
26
#include <linux/if_vlan.h>
27

U
Ursula Braun 已提交
28
#include <net/sock.h>
29
#include <net/tcp.h>
30
#include <net/smc.h>
31
#include <asm/ioctls.h>
U
Ursula Braun 已提交
32 33

#include "smc.h"
34
#include "smc_clc.h"
U
Ursula Braun 已提交
35
#include "smc_llc.h"
36
#include "smc_cdc.h"
37
#include "smc_core.h"
38
#include "smc_ib.h"
39
#include "smc_ism.h"
40
#include "smc_pnet.h"
U
Ursula Braun 已提交
41
#include "smc_tx.h"
U
Ursula Braun 已提交
42
#include "smc_rx.h"
43
#include "smc_close.h"
U
Ursula Braun 已提交
44

45 46 47 48
static DEFINE_MUTEX(smc_create_lgr_pending);	/* serialize link group
						 * creation
						 */

49
static void smc_tcp_listen_work(struct work_struct *);
50
static void smc_connect_work(struct work_struct *);
51

U
Ursula Braun 已提交
52 53 54 55 56 57 58
static void smc_set_keepalive(struct sock *sk, int val)
{
	struct smc_sock *smc = smc_sk(sk);

	smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
}

59 60 61 62
static struct smc_hashinfo smc_v4_hashinfo = {
	.lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
};

63 64 65 66
static struct smc_hashinfo smc_v6_hashinfo = {
	.lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
};

67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
int smc_hash_sk(struct sock *sk)
{
	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
	struct hlist_head *head;

	head = &h->ht;

	write_lock_bh(&h->lock);
	sk_add_node(sk, head);
	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
	write_unlock_bh(&h->lock);

	return 0;
}
EXPORT_SYMBOL_GPL(smc_hash_sk);

void smc_unhash_sk(struct sock *sk)
{
	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;

	write_lock_bh(&h->lock);
	if (sk_del_node_init(sk))
		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
	write_unlock_bh(&h->lock);
}
EXPORT_SYMBOL_GPL(smc_unhash_sk);

struct proto smc_proto = {
U
Ursula Braun 已提交
95 96 97
	.name		= "SMC",
	.owner		= THIS_MODULE,
	.keepalive	= smc_set_keepalive,
98 99
	.hash		= smc_hash_sk,
	.unhash		= smc_unhash_sk,
U
Ursula Braun 已提交
100
	.obj_size	= sizeof(struct smc_sock),
101
	.h.smc_hash	= &smc_v4_hashinfo,
102
	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
U
Ursula Braun 已提交
103
};
104
EXPORT_SYMBOL_GPL(smc_proto);
U
Ursula Braun 已提交
105

106 107 108 109 110 111 112 113 114 115 116 117
struct proto smc_proto6 = {
	.name		= "SMC6",
	.owner		= THIS_MODULE,
	.keepalive	= smc_set_keepalive,
	.hash		= smc_hash_sk,
	.unhash		= smc_unhash_sk,
	.obj_size	= sizeof(struct smc_sock),
	.h.smc_hash	= &smc_v6_hashinfo,
	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
};
EXPORT_SYMBOL_GPL(smc_proto6);

U
Ursula Braun 已提交
118 119 120 121
static int smc_release(struct socket *sock)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
122
	int rc = 0;
U
Ursula Braun 已提交
123 124 125 126 127

	if (!sk)
		goto out;

	smc = smc_sk(sk);
128 129 130 131 132 133

	/* cleanup for a dangling non-blocking connect */
	flush_work(&smc->connect_work);
	kfree(smc->connect_info);
	smc->connect_info = NULL;

134 135 136 137 138 139 140
	if (sk->sk_state == SMC_LISTEN)
		/* smc_close_non_accepted() is called and acquires
		 * sock lock for child sockets again
		 */
		lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
	else
		lock_sock(sk);
U
Ursula Braun 已提交
141

142
	if (!smc->use_fallback) {
143 144 145 146
		rc = smc_close_active(smc);
		sock_set_flag(sk, SOCK_DEAD);
		sk->sk_shutdown |= SHUTDOWN_MASK;
	}
U
Ursula Braun 已提交
147 148 149 150
	if (smc->clcsock) {
		sock_release(smc->clcsock);
		smc->clcsock = NULL;
	}
151
	if (smc->use_fallback) {
152 153
		if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
			sock_put(sk); /* passive closing */
154 155 156
		sk->sk_state = SMC_CLOSED;
		sk->sk_state_change(sk);
	}
U
Ursula Braun 已提交
157 158 159 160

	/* detach socket */
	sock_orphan(sk);
	sock->sk = NULL;
161
	if (!smc->use_fallback && sk->sk_state == SMC_CLOSED)
162
		smc_conn_free(&smc->conn);
U
Ursula Braun 已提交
163 164
	release_sock(sk);

165 166
	sk->sk_prot->unhash(sk);
	sock_put(sk); /* final sock_put */
U
Ursula Braun 已提交
167
out:
168
	return rc;
U
Ursula Braun 已提交
169 170 171 172 173 174 175 176 177 178 179 180
}

static void smc_destruct(struct sock *sk)
{
	if (sk->sk_state != SMC_CLOSED)
		return;
	if (!sock_flag(sk, SOCK_DEAD))
		return;

	sk_refcnt_debug_dec(sk);
}

181 182
static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
				   int protocol)
U
Ursula Braun 已提交
183 184
{
	struct smc_sock *smc;
185
	struct proto *prot;
U
Ursula Braun 已提交
186 187
	struct sock *sk;

188 189
	prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
	sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
U
Ursula Braun 已提交
190 191 192 193 194 195
	if (!sk)
		return NULL;

	sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
	sk->sk_state = SMC_INIT;
	sk->sk_destruct = smc_destruct;
196
	sk->sk_protocol = protocol;
U
Ursula Braun 已提交
197
	smc = smc_sk(sk);
198
	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
199
	INIT_WORK(&smc->connect_work, smc_connect_work);
200
	INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
201 202
	INIT_LIST_HEAD(&smc->accept_q);
	spin_lock_init(&smc->accept_q_lock);
203
	spin_lock_init(&smc->conn.send_lock);
204
	sk->sk_prot->hash(sk);
205
	sk_refcnt_debug_inc(sk);
U
Ursula Braun 已提交
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225

	return sk;
}

static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
		    int addr_len)
{
	struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc;

	smc = smc_sk(sk);

	/* replicate tests from inet_bind(), to be safe wrt. future changes */
	rc = -EINVAL;
	if (addr_len < sizeof(struct sockaddr_in))
		goto out;

	rc = -EAFNOSUPPORT;
226 227 228 229
	if (addr->sin_family != AF_INET &&
	    addr->sin_family != AF_INET6 &&
	    addr->sin_family != AF_UNSPEC)
		goto out;
U
Ursula Braun 已提交
230
	/* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
231 232
	if (addr->sin_family == AF_UNSPEC &&
	    addr->sin_addr.s_addr != htonl(INADDR_ANY))
U
Ursula Braun 已提交
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
		goto out;

	lock_sock(sk);

	/* Check if socket is already active */
	rc = -EINVAL;
	if (sk->sk_state != SMC_INIT)
		goto out_rel;

	smc->clcsock->sk->sk_reuse = sk->sk_reuse;
	rc = kernel_bind(smc->clcsock, uaddr, addr_len);

out_rel:
	release_sock(sk);
out:
	return rc;
}

static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
				   unsigned long mask)
{
	/* options we don't get control via setsockopt for */
	nsk->sk_type = osk->sk_type;
	nsk->sk_sndbuf = osk->sk_sndbuf;
	nsk->sk_rcvbuf = osk->sk_rcvbuf;
	nsk->sk_sndtimeo = osk->sk_sndtimeo;
	nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
	nsk->sk_mark = osk->sk_mark;
	nsk->sk_priority = osk->sk_priority;
	nsk->sk_rcvlowat = osk->sk_rcvlowat;
	nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
	nsk->sk_err = osk->sk_err;

	nsk->sk_flags &= ~mask;
	nsk->sk_flags |= osk->sk_flags & mask;
}

#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
			     (1UL << SOCK_KEEPOPEN) | \
			     (1UL << SOCK_LINGER) | \
			     (1UL << SOCK_BROADCAST) | \
			     (1UL << SOCK_TIMESTAMP) | \
			     (1UL << SOCK_DBG) | \
			     (1UL << SOCK_RCVTSTAMP) | \
			     (1UL << SOCK_RCVTSTAMPNS) | \
			     (1UL << SOCK_LOCALROUTE) | \
			     (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
			     (1UL << SOCK_RXQ_OVFL) | \
			     (1UL << SOCK_WIFI_STATUS) | \
			     (1UL << SOCK_NOFCS) | \
			     (1UL << SOCK_FILTER_LOCKED))
/* copy only relevant settings and flags of SOL_SOCKET level from smc to
 * clc socket (since smc is not called for these options from net/core)
 */
static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
{
	smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
}

#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
			     (1UL << SOCK_KEEPOPEN) | \
			     (1UL << SOCK_LINGER) | \
			     (1UL << SOCK_DBG))
/* copy only settings and flags relevant for smc from clc to smc socket */
static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
{
	smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
}

302 303 304
/* register a new rmb, optionally send confirm_rkey msg to register with peer */
static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
		       bool conf_rkey)
K
Karsten Graul 已提交
305 306
{
	/* register memory region for new rmb */
307 308
	if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
		rmb_desc->regerr = 1;
K
Karsten Graul 已提交
309
		return -EFAULT;
310
	}
311 312 313 314 315 316 317
	if (!conf_rkey)
		return 0;
	/* exchange confirm_rkey msg with peer */
	if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
		rmb_desc->regerr = 1;
		return -EFAULT;
	}
K
Karsten Graul 已提交
318 319 320
	return 0;
}

321
static int smc_clnt_conf_first_link(struct smc_sock *smc)
U
Ursula Braun 已提交
322
{
323
	struct net *net = sock_net(smc->clcsock->sk);
U
Ursula Braun 已提交
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
	struct smc_link_group *lgr = smc->conn.lgr;
	struct smc_link *link;
	int rest;
	int rc;

	link = &lgr->lnk[SMC_SINGLE_LINK];
	/* receive CONFIRM LINK request from server over RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(
		&link->llc_confirm,
		SMC_LLC_WAIT_FIRST_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
		return rc;
	}

342 343 344
	if (link->llc_confirm_rc)
		return SMC_CLC_DECL_RMBE_EC;

U
Ursula Braun 已提交
345 346
	rc = smc_ib_modify_qp_rts(link);
	if (rc)
347
		return SMC_CLC_DECL_ERR_RDYLNK;
U
Ursula Braun 已提交
348 349

	smc_wr_remember_qp_attr(link);
350

351
	if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
352
		return SMC_CLC_DECL_ERR_REGRMB;
353

U
Ursula Braun 已提交
354
	/* send CONFIRM LINK response over RoCE fabric */
355
	rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
U
Ursula Braun 已提交
356
	if (rc < 0)
357
		return SMC_CLC_DECL_TIMEOUT_CL;
U
Ursula Braun 已提交
358

359 360 361 362 363 364 365 366 367 368 369 370 371 372
	/* receive ADD LINK request from server over RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(&link->llc_add,
							 SMC_LLC_WAIT_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
		return rc;
	}

	/* send add link reject message, only one link supported for now */
	rc = smc_llc_send_add_link(link,
				   link->smcibdev->mac[link->ibport - 1],
373
				   link->gid, SMC_LLC_RESP);
374
	if (rc < 0)
375
		return SMC_CLC_DECL_TIMEOUT_AL;
376

377
	smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
378

379
	return 0;
U
Ursula Braun 已提交
380 381
}

382 383
static void smcr_conn_save_peer_info(struct smc_sock *smc,
				     struct smc_clc_msg_accept_confirm *clc)
384
{
385 386
	int bufsize = smc_uncompress_bufsize(clc->rmbe_size);

387
	smc->conn.peer_rmbe_idx = clc->rmbe_idx;
388
	smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
389
	smc->conn.peer_rmbe_size = bufsize;
U
Ursula Braun 已提交
390
	atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
391
	smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
392 393
}

394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
static void smcd_conn_save_peer_info(struct smc_sock *smc,
				     struct smc_clc_msg_accept_confirm *clc)
{
	int bufsize = smc_uncompress_bufsize(clc->dmbe_size);

	smc->conn.peer_rmbe_idx = clc->dmbe_idx;
	smc->conn.peer_token = clc->token;
	/* msg header takes up space in the buffer */
	smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
	atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
	smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
}

static void smc_conn_save_peer_info(struct smc_sock *smc,
				    struct smc_clc_msg_accept_confirm *clc)
{
	if (smc->conn.lgr->is_smcd)
		smcd_conn_save_peer_info(smc, clc);
	else
		smcr_conn_save_peer_info(smc, clc);
}

416 417 418 419 420 421 422 423 424 425
static void smc_link_save_peer_info(struct smc_link *link,
				    struct smc_clc_msg_accept_confirm *clc)
{
	link->peer_qpn = ntoh24(clc->qpn);
	memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
	memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
	link->peer_psn = ntoh24(clc->psn);
	link->peer_mtu = clc->qp_mtu;
}

426
/* fall back during connect */
427
static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
428
{
429
	smc->use_fallback = true;
430
	smc->fallback_rsn = reason_code;
431 432 433 434 435
	smc_copy_sock_settings_to_clc(smc);
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;
	return 0;
}
436

437 438 439 440
/* decline and fall back during connect */
static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
{
	int rc;
441

442 443 444
	if (reason_code < 0) { /* error, fallback is not possible */
		if (smc->sk.sk_state == SMC_INIT)
			sock_put(&smc->sk); /* passive closing */
445
		return reason_code;
446
	}
447
	if (reason_code != SMC_CLC_DECL_PEERDECL) {
448
		rc = smc_clc_send_decline(smc, reason_code);
449 450 451
		if (rc < 0) {
			if (smc->sk.sk_state == SMC_INIT)
				sock_put(&smc->sk); /* passive closing */
452
			return rc;
453
		}
U
Ursula Braun 已提交
454
	}
455
	return smc_connect_fallback(smc, reason_code);
456
}
U
Ursula Braun 已提交
457

458 459 460 461 462 463 464 465 466 467 468 469 470 471
/* abort connecting */
static int smc_connect_abort(struct smc_sock *smc, int reason_code,
			     int local_contact)
{
	if (local_contact == SMC_FIRST_CONTACT)
		smc_lgr_forget(smc->conn.lgr);
	mutex_unlock(&smc_create_lgr_pending);
	smc_conn_free(&smc->conn);
	return reason_code;
}

/* check if there is a rdma device available for this connection. */
/* called for connect and listen */
static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
472
			  u8 *ibport, unsigned short vlan_id, u8 gid[])
473 474
{
	int reason_code = 0;
475 476 477 478 479

	/* PNET table look up: search active ib_device and port
	 * within same PNETID that also contains the ethernet device
	 * used for the internal TCP socket
	 */
480 481
	smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id,
				    gid);
482
	if (!(*ibdev))
483
		reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
484 485 486 487

	return reason_code;
}

488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
/* check if there is an ISM device available for this connection. */
/* called for connect and listen */
static int smc_check_ism(struct smc_sock *smc, struct smcd_dev **ismdev)
{
	/* Find ISM device with same PNETID as connecting interface  */
	smc_pnet_find_ism_resource(smc->clcsock->sk, ismdev);
	if (!(*ismdev))
		return SMC_CLC_DECL_CNFERR; /* configuration error */
	return 0;
}

/* Check for VLAN ID and register it on ISM device just for CLC handshake */
static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
				      struct smcd_dev *ismdev,
				      unsigned short vlan_id)
{
	if (vlan_id && smc_ism_get_vlan(ismdev, vlan_id))
		return SMC_CLC_DECL_CNFERR;
	return 0;
}

/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 * used, the VLAN ID will be registered again during the connection setup.
 */
static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
					struct smcd_dev *ismdev,
					unsigned short vlan_id)
{
	if (!is_smcd)
		return 0;
	if (vlan_id && smc_ism_put_vlan(ismdev, vlan_id))
		return SMC_CLC_DECL_CNFERR;
	return 0;
}

523
/* CLC handshake during connect */
524
static int smc_connect_clc(struct smc_sock *smc, int smc_type,
525
			   struct smc_clc_msg_accept_confirm *aclc,
526
			   struct smc_ib_device *ibdev, u8 ibport,
527
			   u8 gid[], struct smcd_dev *ismdev)
528 529
{
	int rc = 0;
530 531

	/* do inband token exchange */
532
	rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev);
533 534
	if (rc)
		return rc;
535
	/* receive SMC Accept CLC message */
536 537 538 539 540 541 542 543 544 545 546
	return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT);
}

/* setup for RDMA connection of client */
static int smc_connect_rdma(struct smc_sock *smc,
			    struct smc_clc_msg_accept_confirm *aclc,
			    struct smc_ib_device *ibdev, u8 ibport)
{
	int local_contact = SMC_FIRST_CONTACT;
	struct smc_link *link;
	int reason_code = 0;
547

548
	mutex_lock(&smc_create_lgr_pending);
549 550
	local_contact = smc_conn_create(smc, false, aclc->hdr.flag, ibdev,
					ibport, &aclc->lcl, NULL, 0);
551
	if (local_contact < 0) {
552
		if (local_contact == -ENOMEM)
553
			reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
554
		else if (local_contact == -ENOLINK)
555
			reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
556 557
		else
			reason_code = SMC_CLC_DECL_INTERR; /* other error */
558
		return smc_connect_abort(smc, reason_code, 0);
559 560
	}
	link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
561

562
	smc_conn_save_peer_info(smc, aclc);
U
Ursula Braun 已提交
563

564
	/* create send buffer and rmb */
565
	if (smc_buf_create(smc, false))
566
		return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
U
Ursula Braun 已提交
567

568
	if (local_contact == SMC_FIRST_CONTACT)
569
		smc_link_save_peer_info(link, aclc);
570

571
	if (smc_rmb_rtoken_handling(&smc->conn, aclc))
572
		return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
573
					 local_contact);
574

575 576 577
	smc_close_init(smc);
	smc_rx_init(smc);

578
	if (local_contact == SMC_FIRST_CONTACT) {
579
		if (smc_ib_ready_link(link))
580
			return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
581
						 local_contact);
582
	} else {
583 584
		if (!smc->conn.rmb_desc->reused &&
		    smc_reg_rmb(link, smc->conn.rmb_desc, true))
585
			return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
586
						 local_contact);
587
	}
588
	smc_rmb_sync_sg_for_device(&smc->conn);
589

590 591 592 593 594
	reason_code = smc_clc_send_confirm(smc);
	if (reason_code)
		return smc_connect_abort(smc, reason_code, local_contact);

	smc_tx_init(smc);
595

U
Ursula Braun 已提交
596 597
	if (local_contact == SMC_FIRST_CONTACT) {
		/* QP confirmation over RoCE fabric */
598
		reason_code = smc_clnt_conf_first_link(smc);
599 600 601
		if (reason_code)
			return smc_connect_abort(smc, reason_code,
						 local_contact);
U
Ursula Braun 已提交
602
	}
603
	mutex_unlock(&smc_create_lgr_pending);
U
Ursula Braun 已提交
604

605
	smc_copy_sock_settings_to_clc(smc);
606 607
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;
608

609 610
	return 0;
}
611

612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
/* setup for ISM connection of client */
static int smc_connect_ism(struct smc_sock *smc,
			   struct smc_clc_msg_accept_confirm *aclc,
			   struct smcd_dev *ismdev)
{
	int local_contact = SMC_FIRST_CONTACT;
	int rc = 0;

	mutex_lock(&smc_create_lgr_pending);
	local_contact = smc_conn_create(smc, true, aclc->hdr.flag, NULL, 0,
					NULL, ismdev, aclc->gid);
	if (local_contact < 0)
		return smc_connect_abort(smc, SMC_CLC_DECL_MEM, 0);

	/* Create send and receive buffers */
	if (smc_buf_create(smc, true))
		return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);

	smc_conn_save_peer_info(smc, aclc);
	smc_close_init(smc);
	smc_rx_init(smc);
	smc_tx_init(smc);

	rc = smc_clc_send_confirm(smc);
	if (rc)
		return smc_connect_abort(smc, rc, local_contact);
	mutex_unlock(&smc_create_lgr_pending);

	smc_copy_sock_settings_to_clc(smc);
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;

	return 0;
}

647 648 649
/* perform steps before actually connecting */
static int __smc_connect(struct smc_sock *smc)
{
650
	bool ism_supported = false, rdma_supported = false;
651 652
	struct smc_clc_msg_accept_confirm aclc;
	struct smc_ib_device *ibdev;
653
	struct smcd_dev *ismdev;
654
	u8 gid[SMC_GID_SIZE];
655 656
	unsigned short vlan;
	int smc_type;
657 658
	int rc = 0;
	u8 ibport;
659

660 661 662
	sock_hold(&smc->sk); /* sock put in passive closing */

	if (smc->use_fallback)
663
		return smc_connect_fallback(smc, smc->fallback_rsn);
664 665 666

	/* if peer has not signalled SMC-capability, fall back */
	if (!tcp_sk(smc->clcsock->sk)->syn_smc)
667
		return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
668 669 670 671 672

	/* IPSec connections opt out of SMC-R optimizations */
	if (using_ipsec(smc))
		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);

673 674 675 676 677 678 679 680 681 682 683 684 685
	/* check for VLAN ID */
	if (smc_vlan_by_tcpsk(smc->clcsock, &vlan))
		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);

	/* check if there is an ism device available */
	if (!smc_check_ism(smc, &ismdev) &&
	    !smc_connect_ism_vlan_setup(smc, ismdev, vlan)) {
		/* ISM is supported for this connection */
		ism_supported = true;
		smc_type = SMC_TYPE_D;
	}

	/* check if there is a rdma device available */
686
	if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) {
687 688 689 690 691 692 693 694 695 696
		/* RDMA is supported for this connection */
		rdma_supported = true;
		if (ism_supported)
			smc_type = SMC_TYPE_B; /* both */
		else
			smc_type = SMC_TYPE_R; /* only RDMA */
	}

	/* if neither ISM nor RDMA are supported, fallback */
	if (!rdma_supported && !ism_supported)
697
		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
698 699

	/* perform CLC handshake */
700
	rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev);
701 702
	if (rc) {
		smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
703
		return smc_connect_decline_fallback(smc, rc);
704
	}
705

706 707 708 709 710 711
	/* depending on previous steps, connect using rdma or ism */
	if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
		rc = smc_connect_rdma(smc, &aclc, ibdev, ibport);
	else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
		rc = smc_connect_ism(smc, &aclc, ismdev);
	else
712
		rc = SMC_CLC_DECL_MODEUNSUPP;
713 714
	if (rc) {
		smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
715
		return smc_connect_decline_fallback(smc, rc);
716
	}
717

718
	smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
719
	return 0;
720 721
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
static void smc_connect_work(struct work_struct *work)
{
	struct smc_sock *smc = container_of(work, struct smc_sock,
					    connect_work);
	int rc;

	lock_sock(&smc->sk);
	rc = kernel_connect(smc->clcsock, &smc->connect_info->addr,
			    smc->connect_info->alen, smc->connect_info->flags);
	if (smc->clcsock->sk->sk_err) {
		smc->sk.sk_err = smc->clcsock->sk->sk_err;
		goto out;
	}
	if (rc < 0) {
		smc->sk.sk_err = -rc;
		goto out;
	}

	rc = __smc_connect(smc);
	if (rc < 0)
		smc->sk.sk_err = -rc;

out:
745 746 747 748
	if (smc->sk.sk_err)
		smc->sk.sk_state_change(&smc->sk);
	else
		smc->sk.sk_write_space(&smc->sk);
749 750 751 752 753
	kfree(smc->connect_info);
	smc->connect_info = NULL;
	release_sock(&smc->sk);
}

U
Ursula Braun 已提交
754 755 756 757 758 759 760 761 762 763 764 765
static int smc_connect(struct socket *sock, struct sockaddr *addr,
		       int alen, int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EINVAL;

	smc = smc_sk(sk);

	/* separate smc parameter checking to be safe */
	if (alen < sizeof(addr->sa_family))
		goto out_err;
766
	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
U
Ursula Braun 已提交
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
		goto out_err;

	lock_sock(sk);
	switch (sk->sk_state) {
	default:
		goto out;
	case SMC_ACTIVE:
		rc = -EISCONN;
		goto out;
	case SMC_INIT:
		rc = 0;
		break;
	}

	smc_copy_sock_settings_to_clc(smc);
U
Ursula Braun 已提交
782
	tcp_sk(smc->clcsock->sk)->syn_smc = 1;
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
	if (flags & O_NONBLOCK) {
		if (smc->connect_info) {
			rc = -EALREADY;
			goto out;
		}
		smc->connect_info = kzalloc(alen + 2 * sizeof(int), GFP_KERNEL);
		if (!smc->connect_info) {
			rc = -ENOMEM;
			goto out;
		}
		smc->connect_info->alen = alen;
		smc->connect_info->flags = flags ^ O_NONBLOCK;
		memcpy(&smc->connect_info->addr, addr, alen);
		schedule_work(&smc->connect_work);
		rc = -EINPROGRESS;
	} else {
		rc = kernel_connect(smc->clcsock, addr, alen, flags);
		if (rc)
			goto out;
U
Ursula Braun 已提交
802

803 804 805 806 807 808
		rc = __smc_connect(smc);
		if (rc < 0)
			goto out;
		else
			rc = 0; /* success cases including fallback */
	}
U
Ursula Braun 已提交
809 810 811 812 813 814 815 816 817

out:
	release_sock(sk);
out_err:
	return rc;
}

static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
{
818 819
	struct socket *new_clcsock = NULL;
	struct sock *lsk = &lsmc->sk;
U
Ursula Braun 已提交
820 821 822
	struct sock *new_sk;
	int rc;

823
	release_sock(lsk);
824
	new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
U
Ursula Braun 已提交
825 826
	if (!new_sk) {
		rc = -ENOMEM;
827
		lsk->sk_err = ENOMEM;
U
Ursula Braun 已提交
828
		*new_smc = NULL;
829
		lock_sock(lsk);
U
Ursula Braun 已提交
830 831 832 833 834
		goto out;
	}
	*new_smc = smc_sk(new_sk);

	rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
835
	lock_sock(lsk);
836
	if  (rc < 0)
837
		lsk->sk_err = -rc;
838
	if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
839 840 841 842
		if (new_clcsock)
			sock_release(new_clcsock);
		new_sk->sk_state = SMC_CLOSED;
		sock_set_flag(new_sk, SOCK_DEAD);
843
		new_sk->sk_prot->unhash(new_sk);
844
		sock_put(new_sk); /* final */
U
Ursula Braun 已提交
845 846 847 848 849 850 851 852 853
		*new_smc = NULL;
		goto out;
	}

	(*new_smc)->clcsock = new_clcsock;
out:
	return rc;
}

854 855 856 857 858 859 860
/* add a just created sock to the accept queue of the listen sock as
 * candidate for a following socket accept call from user space
 */
static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
{
	struct smc_sock *par = smc_sk(parent);

861
	sock_hold(sk); /* sock_put in smc_accept_unlink () */
862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
	spin_lock(&par->accept_q_lock);
	list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
	spin_unlock(&par->accept_q_lock);
	sk_acceptq_added(parent);
}

/* remove a socket from the accept queue of its parental listening socket */
static void smc_accept_unlink(struct sock *sk)
{
	struct smc_sock *par = smc_sk(sk)->listen_smc;

	spin_lock(&par->accept_q_lock);
	list_del_init(&smc_sk(sk)->accept_q);
	spin_unlock(&par->accept_q_lock);
	sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
877
	sock_put(sk); /* sock_hold in smc_accept_enqueue */
878 879 880 881 882
}

/* remove a sock from the accept queue to bind it to a new socket created
 * for a socket accept call from user space
 */
883 884
struct sock *smc_accept_dequeue(struct sock *parent,
				struct socket *new_sock)
885 886 887 888 889 890 891 892 893
{
	struct smc_sock *isk, *n;
	struct sock *new_sk;

	list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
		new_sk = (struct sock *)isk;

		smc_accept_unlink(new_sk);
		if (new_sk->sk_state == SMC_CLOSED) {
894 895 896 897
			if (isk->clcsock) {
				sock_release(isk->clcsock);
				isk->clcsock = NULL;
			}
898
			new_sk->sk_prot->unhash(new_sk);
899
			sock_put(new_sk); /* final */
900 901 902 903 904 905 906 907 908 909
			continue;
		}
		if (new_sock)
			sock_graft(new_sk, new_sock);
		return new_sk;
	}
	return NULL;
}

/* clean up for a created but never accepted sock */
910
void smc_close_non_accepted(struct sock *sk)
911 912 913
{
	struct smc_sock *smc = smc_sk(sk);

914 915 916 917
	lock_sock(sk);
	if (!sk->sk_lingertime)
		/* wait for peer closing */
		sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
918
	if (!smc->use_fallback) {
919
		smc_close_active(smc);
920 921 922
		sock_set_flag(sk, SOCK_DEAD);
		sk->sk_shutdown |= SHUTDOWN_MASK;
	}
923 924 925 926 927 928 929
	if (smc->clcsock) {
		struct socket *tcp;

		tcp = smc->clcsock;
		smc->clcsock = NULL;
		sock_release(tcp);
	}
930
	if (smc->use_fallback) {
931 932 933 934 935
		sock_put(sk); /* passive closing */
		sk->sk_state = SMC_CLOSED;
	} else {
		if (sk->sk_state == SMC_CLOSED)
			smc_conn_free(&smc->conn);
936 937
	}
	release_sock(sk);
938 939
	sk->sk_prot->unhash(sk);
	sock_put(sk); /* final sock_put */
940 941
}

U
Ursula Braun 已提交
942 943
static int smc_serv_conf_first_link(struct smc_sock *smc)
{
944
	struct net *net = sock_net(smc->clcsock->sk);
U
Ursula Braun 已提交
945 946 947 948 949 950
	struct smc_link_group *lgr = smc->conn.lgr;
	struct smc_link *link;
	int rest;
	int rc;

	link = &lgr->lnk[SMC_SINGLE_LINK];
951

952
	if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
953
		return SMC_CLC_DECL_ERR_REGRMB;
954

U
Ursula Braun 已提交
955
	/* send CONFIRM LINK request to client over the RoCE fabric */
956
	rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
U
Ursula Braun 已提交
957
	if (rc < 0)
958
		return SMC_CLC_DECL_TIMEOUT_CL;
U
Ursula Braun 已提交
959 960 961 962 963 964 965 966 967 968

	/* receive CONFIRM LINK response from client over the RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(
		&link->llc_confirm_resp,
		SMC_LLC_WAIT_FIRST_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
969
		return rc;
U
Ursula Braun 已提交
970 971
	}

972 973 974
	if (link->llc_confirm_resp_rc)
		return SMC_CLC_DECL_RMBE_EC;

975 976 977
	/* send ADD LINK request to client over the RoCE fabric */
	rc = smc_llc_send_add_link(link,
				   link->smcibdev->mac[link->ibport - 1],
978
				   link->gid, SMC_LLC_REQ);
979
	if (rc < 0)
980
		return SMC_CLC_DECL_TIMEOUT_AL;
981 982 983 984 985 986 987 988 989 990 991 992

	/* receive ADD LINK response from client over the RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
							 SMC_LLC_WAIT_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
		return rc;
	}

993
	smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
994

995
	return 0;
U
Ursula Braun 已提交
996 997
}

998 999
/* listen worker: finish */
static void smc_listen_out(struct smc_sock *new_smc)
1000 1001 1002 1003
{
	struct smc_sock *lsmc = new_smc->listen_smc;
	struct sock *newsmcsk = &new_smc->sk;

1004 1005 1006 1007 1008
	lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
	if (lsmc->sk.sk_state == SMC_LISTEN) {
		smc_accept_enqueue(&lsmc->sk, newsmcsk);
	} else { /* no longer listening */
		smc_close_non_accepted(newsmcsk);
U
Ursula Braun 已提交
1009
	}
1010
	release_sock(&lsmc->sk);
U
Ursula Braun 已提交
1011

1012 1013 1014 1015
	/* Wake up accept */
	lsmc->sk.sk_data_ready(&lsmc->sk);
	sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
}
1016

1017 1018 1019 1020
/* listen worker: finish in state connected */
static void smc_listen_out_connected(struct smc_sock *new_smc)
{
	struct sock *newsmcsk = &new_smc->sk;
1021

1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
	sk_refcnt_debug_inc(newsmcsk);
	if (newsmcsk->sk_state == SMC_INIT)
		newsmcsk->sk_state = SMC_ACTIVE;

	smc_listen_out(new_smc);
}

/* listen worker: finish in error state */
static void smc_listen_out_err(struct smc_sock *new_smc)
{
	struct sock *newsmcsk = &new_smc->sk;

	if (newsmcsk->sk_state == SMC_INIT)
		sock_put(&new_smc->sk); /* passive closing */
	newsmcsk->sk_state = SMC_CLOSED;
	smc_conn_free(&new_smc->conn);

	smc_listen_out(new_smc);
}

/* listen worker: decline and fall back if possible */
static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
			       int local_contact)
{
	/* RDMA setup failed, switch back to TCP */
	if (local_contact == SMC_FIRST_CONTACT)
		smc_lgr_forget(new_smc->conn.lgr);
	if (reason_code < 0) { /* error, no fallback possible */
		smc_listen_out_err(new_smc);
		return;
	}
	smc_conn_free(&new_smc->conn);
	new_smc->use_fallback = true;
1055 1056
	new_smc->fallback_rsn = reason_code;
	if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1057 1058 1059 1060
		if (smc_clc_send_decline(new_smc, reason_code) < 0) {
			smc_listen_out_err(new_smc);
			return;
		}
1061
	}
1062 1063 1064 1065 1066 1067 1068 1069 1070
	smc_listen_out_connected(new_smc);
}

/* listen worker: check prefixes */
static int smc_listen_rdma_check(struct smc_sock *new_smc,
				 struct smc_clc_msg_proposal *pclc)
{
	struct smc_clc_msg_proposal_prefix *pclc_prfx;
	struct socket *newclcsock = new_smc->clcsock;
1071

1072
	pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1073 1074
	if (smc_clc_prfx_match(newclcsock, pclc_prfx))
		return SMC_CLC_DECL_CNFERR;
1075

1076 1077
	return 0;
}
1078

1079 1080 1081 1082 1083 1084
/* listen worker: initialize connection and buffers */
static int smc_listen_rdma_init(struct smc_sock *new_smc,
				struct smc_clc_msg_proposal *pclc,
				struct smc_ib_device *ibdev, u8 ibport,
				int *local_contact)
{
1085
	/* allocate connection / link group */
1086 1087
	*local_contact = smc_conn_create(new_smc, false, 0, ibdev, ibport,
					 &pclc->lcl, NULL, 0);
1088 1089 1090 1091
	if (*local_contact < 0) {
		if (*local_contact == -ENOMEM)
			return SMC_CLC_DECL_MEM;/* insufficient memory*/
		return SMC_CLC_DECL_INTERR; /* other error */
1092
	}
1093

1094
	/* create send buffer and rmb */
1095
	if (smc_buf_create(new_smc, false))
1096
		return SMC_CLC_DECL_MEM;
1097

1098 1099 1100
	return 0;
}

1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
/* listen worker: initialize connection and buffers for SMC-D */
static int smc_listen_ism_init(struct smc_sock *new_smc,
			       struct smc_clc_msg_proposal *pclc,
			       struct smcd_dev *ismdev,
			       int *local_contact)
{
	struct smc_clc_msg_smcd *pclc_smcd;

	pclc_smcd = smc_get_clc_msg_smcd(pclc);
	*local_contact = smc_conn_create(new_smc, true, 0, NULL, 0, NULL,
					 ismdev, pclc_smcd->gid);
	if (*local_contact < 0) {
		if (*local_contact == -ENOMEM)
			return SMC_CLC_DECL_MEM;/* insufficient memory*/
		return SMC_CLC_DECL_INTERR; /* other error */
	}

	/* Check if peer can be reached via ISM device */
	if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
			    new_smc->conn.lgr->vlan_id,
			    new_smc->conn.lgr->smcd)) {
		if (*local_contact == SMC_FIRST_CONTACT)
			smc_lgr_forget(new_smc->conn.lgr);
		smc_conn_free(&new_smc->conn);
		return SMC_CLC_DECL_CNFERR;
	}

	/* Create send and receive buffers */
	if (smc_buf_create(new_smc, true)) {
		if (*local_contact == SMC_FIRST_CONTACT)
			smc_lgr_forget(new_smc->conn.lgr);
		smc_conn_free(&new_smc->conn);
		return SMC_CLC_DECL_MEM;
	}

	return 0;
}

1139 1140 1141 1142
/* listen worker: register buffers */
static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
{
	struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1143

1144
	if (local_contact != SMC_FIRST_CONTACT) {
K
Karsten Graul 已提交
1145
		if (!new_smc->conn.rmb_desc->reused) {
1146
			if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1147
				return SMC_CLC_DECL_ERR_REGRMB;
1148 1149
		}
	}
1150
	smc_rmb_sync_sg_for_device(&new_smc->conn);
1151

1152 1153 1154 1155
	return 0;
}

/* listen worker: finish RDMA setup */
1156 1157 1158
static int smc_listen_rdma_finish(struct smc_sock *new_smc,
				  struct smc_clc_msg_accept_confirm *cclc,
				  int local_contact)
1159 1160 1161
{
	struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
	int reason_code = 0;
1162

1163
	if (local_contact == SMC_FIRST_CONTACT)
1164
		smc_link_save_peer_info(link, cclc);
1165

1166
	if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1167
		reason_code = SMC_CLC_DECL_ERR_RTOK;
1168
		goto decline;
1169 1170 1171
	}

	if (local_contact == SMC_FIRST_CONTACT) {
1172
		if (smc_ib_ready_link(link)) {
1173
			reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1174
			goto decline;
1175
		}
U
Ursula Braun 已提交
1176 1177
		/* QP confirmation over RoCE fabric */
		reason_code = smc_serv_conf_first_link(new_smc);
1178 1179
		if (reason_code)
			goto decline;
1180
	}
1181
	return 0;
1182

1183
decline:
1184
	mutex_unlock(&smc_create_lgr_pending);
1185
	smc_listen_decline(new_smc, reason_code, local_contact);
1186
	return reason_code;
1187
}
U
Ursula Braun 已提交
1188

1189 1190 1191 1192 1193 1194 1195 1196 1197
/* setup for RDMA connection of server */
static void smc_listen_work(struct work_struct *work)
{
	struct smc_sock *new_smc = container_of(work, struct smc_sock,
						smc_listen_work);
	struct socket *newclcsock = new_smc->clcsock;
	struct smc_clc_msg_accept_confirm cclc;
	struct smc_clc_msg_proposal *pclc;
	struct smc_ib_device *ibdev;
1198 1199
	bool ism_supported = false;
	struct smcd_dev *ismdev;
1200 1201
	u8 buf[SMC_CLC_MAX_LEN];
	int local_contact = 0;
1202
	unsigned short vlan;
1203 1204 1205 1206 1207 1208 1209
	int reason_code = 0;
	int rc = 0;
	u8 ibport;

	if (new_smc->use_fallback) {
		smc_listen_out_connected(new_smc);
		return;
1210 1211
	}

1212 1213 1214
	/* check if peer is smc capable */
	if (!tcp_sk(newclcsock->sk)->syn_smc) {
		new_smc->use_fallback = true;
1215
		new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1216 1217 1218
		smc_listen_out_connected(new_smc);
		return;
	}
1219

1220 1221 1222 1223 1224 1225 1226 1227 1228
	/* do inband token exchange -
	 * wait for and receive SMC Proposal CLC message
	 */
	pclc = (struct smc_clc_msg_proposal *)&buf;
	reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
				       SMC_CLC_PROPOSAL);
	if (reason_code) {
		smc_listen_decline(new_smc, reason_code, 0);
		return;
1229 1230
	}

1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
	/* IPSec connections opt out of SMC-R optimizations */
	if (using_ipsec(new_smc)) {
		smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0);
		return;
	}

	mutex_lock(&smc_create_lgr_pending);
	smc_close_init(new_smc);
	smc_rx_init(new_smc);
	smc_tx_init(new_smc);

1242 1243 1244 1245 1246 1247 1248
	/* check if ISM is available */
	if ((pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) &&
	    !smc_check_ism(new_smc, &ismdev) &&
	    !smc_listen_ism_init(new_smc, pclc, ismdev, &local_contact)) {
		ism_supported = true;
	}

1249
	/* check if RDMA is available */
1250 1251
	if (!ism_supported &&
	    ((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) ||
1252 1253
	     smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) ||
	     smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) ||
1254 1255 1256 1257
	     smc_listen_rdma_check(new_smc, pclc) ||
	     smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
				  &local_contact) ||
	     smc_listen_rdma_reg(new_smc, local_contact))) {
1258 1259
		/* SMC not supported, decline */
		mutex_unlock(&smc_create_lgr_pending);
1260 1261
		smc_listen_decline(new_smc, SMC_CLC_DECL_MODEUNSUPP,
				   local_contact);
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
		return;
	}

	/* send SMC Accept CLC message */
	rc = smc_clc_send_accept(new_smc, local_contact);
	if (rc) {
		mutex_unlock(&smc_create_lgr_pending);
		smc_listen_decline(new_smc, rc, local_contact);
		return;
	}

	/* receive SMC Confirm CLC message */
	reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
				       SMC_CLC_CONFIRM);
	if (reason_code) {
		mutex_unlock(&smc_create_lgr_pending);
		smc_listen_decline(new_smc, reason_code, local_contact);
		return;
	}

	/* finish worker */
1283 1284 1285 1286
	if (!ism_supported) {
		if (smc_listen_rdma_finish(new_smc, &cclc, local_contact))
			return;
	}
1287
	smc_conn_save_peer_info(new_smc, &cclc);
1288
	mutex_unlock(&smc_create_lgr_pending);
1289
	smc_listen_out_connected(new_smc);
1290 1291 1292 1293 1294 1295
}

static void smc_tcp_listen_work(struct work_struct *work)
{
	struct smc_sock *lsmc = container_of(work, struct smc_sock,
					     tcp_listen_work);
1296
	struct sock *lsk = &lsmc->sk;
1297 1298 1299
	struct smc_sock *new_smc;
	int rc = 0;

1300 1301
	lock_sock(lsk);
	while (lsk->sk_state == SMC_LISTEN) {
1302 1303 1304 1305 1306 1307 1308
		rc = smc_clcsock_accept(lsmc, &new_smc);
		if (rc)
			goto out;
		if (!new_smc)
			continue;

		new_smc->listen_smc = lsmc;
1309
		new_smc->use_fallback = lsmc->use_fallback;
1310
		new_smc->fallback_rsn = lsmc->fallback_rsn;
1311
		sock_hold(lsk); /* sock_put in smc_listen_work */
1312 1313
		INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
		smc_copy_sock_settings_to_smc(new_smc);
1314 1315
		new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
		new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1316 1317 1318
		sock_hold(&new_smc->sk); /* sock_put in passive closing */
		if (!schedule_work(&new_smc->smc_listen_work))
			sock_put(&new_smc->sk);
1319 1320 1321
	}

out:
1322
	release_sock(lsk);
1323
	sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1324 1325
}

U
Ursula Braun 已提交
1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
static int smc_listen(struct socket *sock, int backlog)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc;

	smc = smc_sk(sk);
	lock_sock(sk);

	rc = -EINVAL;
	if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
		goto out;

	rc = 0;
	if (sk->sk_state == SMC_LISTEN) {
		sk->sk_max_ack_backlog = backlog;
		goto out;
	}
	/* some socket options are handled in core, so we could not apply
	 * them to the clc socket -- copy smc socket options to clc socket
	 */
	smc_copy_sock_settings_to_clc(smc);
1348 1349
	if (!smc->use_fallback)
		tcp_sk(smc->clcsock->sk)->syn_smc = 1;
U
Ursula Braun 已提交
1350 1351 1352 1353 1354 1355 1356

	rc = kernel_listen(smc->clcsock, backlog);
	if (rc)
		goto out;
	sk->sk_max_ack_backlog = backlog;
	sk->sk_ack_backlog = 0;
	sk->sk_state = SMC_LISTEN;
1357
	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
1358 1359 1360
	sock_hold(sk); /* sock_hold in tcp_listen_worker */
	if (!schedule_work(&smc->tcp_listen_work))
		sock_put(sk);
U
Ursula Braun 已提交
1361 1362 1363 1364 1365 1366 1367

out:
	release_sock(sk);
	return rc;
}

static int smc_accept(struct socket *sock, struct socket *new_sock,
1368
		      int flags, bool kern)
U
Ursula Braun 已提交
1369
{
1370 1371
	struct sock *sk = sock->sk, *nsk;
	DECLARE_WAITQUEUE(wait, current);
U
Ursula Braun 已提交
1372
	struct smc_sock *lsmc;
1373 1374
	long timeo;
	int rc = 0;
U
Ursula Braun 已提交
1375 1376

	lsmc = smc_sk(sk);
1377
	sock_hold(sk); /* sock_put below */
U
Ursula Braun 已提交
1378 1379 1380 1381
	lock_sock(sk);

	if (lsmc->sk.sk_state != SMC_LISTEN) {
		rc = -EINVAL;
1382
		release_sock(sk);
U
Ursula Braun 已提交
1383 1384 1385
		goto out;
	}

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
	/* Wait for an incoming connection */
	timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
	add_wait_queue_exclusive(sk_sleep(sk), &wait);
	while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
		set_current_state(TASK_INTERRUPTIBLE);
		if (!timeo) {
			rc = -EAGAIN;
			break;
		}
		release_sock(sk);
		timeo = schedule_timeout(timeo);
		/* wakeup by sk_data_ready in smc_listen_work() */
		sched_annotate_sleep();
		lock_sock(sk);
		if (signal_pending(current)) {
			rc = sock_intr_errno(timeo);
			break;
		}
	}
	set_current_state(TASK_RUNNING);
	remove_wait_queue(sk_sleep(sk), &wait);
U
Ursula Braun 已提交
1407

1408 1409
	if (!rc)
		rc = sock_error(nsk);
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426
	release_sock(sk);
	if (rc)
		goto out;

	if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
		/* wait till data arrives on the socket */
		timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
								MSEC_PER_SEC);
		if (smc_sk(nsk)->use_fallback) {
			struct sock *clcsk = smc_sk(nsk)->clcsock->sk;

			lock_sock(clcsk);
			if (skb_queue_empty(&clcsk->sk_receive_queue))
				sk_wait_data(clcsk, &timeo, NULL);
			release_sock(clcsk);
		} else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
			lock_sock(nsk);
1427
			smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1428 1429 1430
			release_sock(nsk);
		}
	}
U
Ursula Braun 已提交
1431 1432

out:
1433
	sock_put(sk); /* sock_hold above */
U
Ursula Braun 已提交
1434 1435 1436 1437
	return rc;
}

static int smc_getname(struct socket *sock, struct sockaddr *addr,
1438
		       int peer)
U
Ursula Braun 已提交
1439 1440 1441
{
	struct smc_sock *smc;

1442 1443
	if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
	    (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
U
Ursula Braun 已提交
1444 1445 1446 1447
		return -ENOTCONN;

	smc = smc_sk(sock->sk);

1448
	return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
U
Ursula Braun 已提交
1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
}

static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EPIPE;

	smc = smc_sk(sk);
	lock_sock(sk);
1459 1460 1461
	if ((sk->sk_state != SMC_ACTIVE) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
	    (sk->sk_state != SMC_INIT))
U
Ursula Braun 已提交
1462
		goto out;
1463 1464 1465 1466

	if (msg->msg_flags & MSG_FASTOPEN) {
		if (sk->sk_state == SMC_INIT) {
			smc->use_fallback = true;
1467
			smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1468 1469 1470 1471 1472 1473
		} else {
			rc = -EINVAL;
			goto out;
		}
	}

U
Ursula Braun 已提交
1474 1475 1476
	if (smc->use_fallback)
		rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
	else
U
Ursula Braun 已提交
1477
		rc = smc_tx_sendmsg(smc, msg, len);
U
Ursula Braun 已提交
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
out:
	release_sock(sk);
	return rc;
}

static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
		       int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -ENOTCONN;

	smc = smc_sk(sk);
	lock_sock(sk);
1492 1493 1494
	if ((sk->sk_state == SMC_INIT) ||
	    (sk->sk_state == SMC_LISTEN) ||
	    (sk->sk_state == SMC_CLOSED))
U
Ursula Braun 已提交
1495 1496
		goto out;

1497 1498 1499 1500 1501
	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
		rc = 0;
		goto out;
	}

S
Stefan Raspl 已提交
1502
	if (smc->use_fallback) {
U
Ursula Braun 已提交
1503
		rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
S
Stefan Raspl 已提交
1504 1505 1506 1507
	} else {
		msg->msg_namelen = 0;
		rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
	}
1508

U
Ursula Braun 已提交
1509 1510 1511 1512 1513
out:
	release_sock(sk);
	return rc;
}

A
Al Viro 已提交
1514
static __poll_t smc_accept_poll(struct sock *parent)
1515
{
U
Ursula Braun 已提交
1516
	struct smc_sock *isk = smc_sk(parent);
A
Al Viro 已提交
1517
	__poll_t mask = 0;
1518

U
Ursula Braun 已提交
1519 1520
	spin_lock(&isk->accept_q_lock);
	if (!list_empty(&isk->accept_q))
1521
		mask = EPOLLIN | EPOLLRDNORM;
U
Ursula Braun 已提交
1522
	spin_unlock(&isk->accept_q_lock);
1523

U
Ursula Braun 已提交
1524
	return mask;
1525 1526
}

1527 1528
static __poll_t smc_poll(struct file *file, struct socket *sock,
			     poll_table *wait)
U
Ursula Braun 已提交
1529 1530
{
	struct sock *sk = sock->sk;
1531
	__poll_t mask = 0;
U
Ursula Braun 已提交
1532 1533
	struct smc_sock *smc;

U
Ursula Braun 已提交
1534
	if (!sk)
1535
		return EPOLLNVAL;
U
Ursula Braun 已提交
1536

U
Ursula Braun 已提交
1537
	smc = smc_sk(sock->sk);
1538
	if (smc->use_fallback) {
1539
		/* delegate to CLC child sock */
1540
		mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1541
		sk->sk_err = smc->clcsock->sk->sk_err;
1542
		if (sk->sk_err)
1543
			mask |= EPOLLERR;
U
Ursula Braun 已提交
1544
	} else {
1545
		if (sk->sk_state != SMC_CLOSED)
1546
			sock_poll_wait(file, sock, wait);
1547
		if (sk->sk_err)
1548
			mask |= EPOLLERR;
1549 1550
		if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
		    (sk->sk_state == SMC_CLOSED))
1551
			mask |= EPOLLHUP;
U
Ursula Braun 已提交
1552 1553 1554 1555 1556 1557
		if (sk->sk_state == SMC_LISTEN) {
			/* woken up by sk_data_ready in smc_listen_work() */
			mask = smc_accept_poll(sk);
		} else {
			if (atomic_read(&smc->conn.sndbuf_space) ||
			    sk->sk_shutdown & SEND_SHUTDOWN) {
1558
				mask |= EPOLLOUT | EPOLLWRNORM;
U
Ursula Braun 已提交
1559 1560 1561 1562 1563
			} else {
				sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
			}
			if (atomic_read(&smc->conn.bytes_to_rcv))
1564
				mask |= EPOLLIN | EPOLLRDNORM;
U
Ursula Braun 已提交
1565
			if (sk->sk_shutdown & RCV_SHUTDOWN)
1566
				mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
U
Ursula Braun 已提交
1567
			if (sk->sk_state == SMC_APPCLOSEWAIT1)
1568
				mask |= EPOLLIN;
1569 1570
			if (smc->conn.urg_state == SMC_URG_VALID)
				mask |= EPOLLPRI;
U
Ursula Braun 已提交
1571
		}
U
Ursula Braun 已提交
1572 1573 1574 1575 1576 1577 1578 1579 1580 1581
	}

	return mask;
}

static int smc_shutdown(struct socket *sock, int how)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EINVAL;
1582
	int rc1 = 0;
U
Ursula Braun 已提交
1583 1584 1585 1586

	smc = smc_sk(sk);

	if ((how < SHUT_RD) || (how > SHUT_RDWR))
1587
		return rc;
U
Ursula Braun 已提交
1588 1589 1590 1591

	lock_sock(sk);

	rc = -ENOTCONN;
1592
	if ((sk->sk_state != SMC_ACTIVE) &&
1593 1594 1595 1596 1597
	    (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
	    (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT2) &&
	    (sk->sk_state != SMC_APPFINCLOSEWAIT))
U
Ursula Braun 已提交
1598 1599 1600 1601 1602 1603
		goto out;
	if (smc->use_fallback) {
		rc = kernel_sock_shutdown(smc->clcsock, how);
		sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
		if (sk->sk_shutdown == SHUTDOWN_MASK)
			sk->sk_state = SMC_CLOSED;
1604
		goto out;
U
Ursula Braun 已提交
1605
	}
1606 1607 1608 1609 1610 1611 1612 1613
	switch (how) {
	case SHUT_RDWR:		/* shutdown in both directions */
		rc = smc_close_active(smc);
		break;
	case SHUT_WR:
		rc = smc_close_shutdown_write(smc);
		break;
	case SHUT_RD:
1614 1615
		rc = 0;
		/* nothing more to do because peer is not involved */
1616 1617
		break;
	}
1618 1619
	if (smc->clcsock)
		rc1 = kernel_sock_shutdown(smc->clcsock, how);
1620 1621
	/* map sock_shutdown_cmd constants to sk_shutdown value range */
	sk->sk_shutdown |= how + 1;
U
Ursula Braun 已提交
1622 1623 1624

out:
	release_sock(sk);
1625
	return rc ? rc : rc1;
U
Ursula Braun 已提交
1626 1627 1628 1629 1630 1631 1632
}

static int smc_setsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, unsigned int optlen)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
1633
	int val, rc;
U
Ursula Braun 已提交
1634 1635 1636 1637 1638 1639

	smc = smc_sk(sk);

	/* generic setsockopts reaching us here always apply to the
	 * CLC socket
	 */
1640 1641 1642 1643 1644 1645 1646 1647 1648
	rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
					   optval, optlen);
	if (smc->clcsock->sk->sk_err) {
		sk->sk_err = smc->clcsock->sk->sk_err;
		sk->sk_error_report(sk);
	}
	if (rc)
		return rc;

1649
	if (optlen < sizeof(int))
1650
		return -EINVAL;
1651 1652
	if (get_user(val, (int __user *)optval))
		return -EFAULT;
1653

1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
	lock_sock(sk);
	switch (optname) {
	case TCP_ULP:
	case TCP_FASTOPEN:
	case TCP_FASTOPEN_CONNECT:
	case TCP_FASTOPEN_KEY:
	case TCP_FASTOPEN_NO_COOKIE:
		/* option not supported by SMC */
		if (sk->sk_state == SMC_INIT) {
			smc->use_fallback = true;
1664
			smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1665 1666 1667 1668 1669
		} else {
			if (!smc->use_fallback)
				rc = -EINVAL;
		}
		break;
1670 1671
	case TCP_NODELAY:
		if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1672
			if (val && !smc->use_fallback)
1673 1674 1675 1676 1677 1678
				mod_delayed_work(system_wq, &smc->conn.tx_work,
						 0);
		}
		break;
	case TCP_CORK:
		if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1679
			if (!val && !smc->use_fallback)
1680 1681 1682 1683
				mod_delayed_work(system_wq, &smc->conn.tx_work,
						 0);
		}
		break;
1684 1685 1686
	case TCP_DEFER_ACCEPT:
		smc->sockopt_defer_accept = val;
		break;
1687 1688 1689 1690 1691 1692
	default:
		break;
	}
	release_sock(sk);

	return rc;
U
Ursula Braun 已提交
1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
}

static int smc_getsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, int __user *optlen)
{
	struct smc_sock *smc;

	smc = smc_sk(sock->sk);
	/* socket options apply to the CLC socket */
	return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
					     optval, optlen);
}

static int smc_ioctl(struct socket *sock, unsigned int cmd,
		     unsigned long arg)
{
S
Stefan Raspl 已提交
1709 1710
	union smc_host_cursor cons, urg;
	struct smc_connection *conn;
U
Ursula Braun 已提交
1711
	struct smc_sock *smc;
1712
	int answ;
U
Ursula Braun 已提交
1713 1714

	smc = smc_sk(sock->sk);
S
Stefan Raspl 已提交
1715
	conn = &smc->conn;
1716
	lock_sock(&smc->sk);
1717
	if (smc->use_fallback) {
1718 1719
		if (!smc->clcsock) {
			release_sock(&smc->sk);
1720
			return -EBADF;
1721 1722 1723 1724
		}
		answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
		release_sock(&smc->sk);
		return answ;
1725 1726 1727
	}
	switch (cmd) {
	case SIOCINQ: /* same as FIONREAD */
1728 1729
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
1730
			return -EINVAL;
1731
		}
1732 1733 1734 1735 1736
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = atomic_read(&smc->conn.bytes_to_rcv);
1737 1738 1739
		break;
	case SIOCOUTQ:
		/* output queue size (not send + not acked) */
1740 1741
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
1742
			return -EINVAL;
1743
		}
1744 1745 1746 1747 1748
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = smc->conn.sndbuf_desc->len -
1749 1750 1751 1752
					atomic_read(&smc->conn.sndbuf_space);
		break;
	case SIOCOUTQNSD:
		/* output queue size (not send only) */
1753 1754
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
1755
			return -EINVAL;
1756
		}
1757 1758 1759 1760 1761
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = smc_tx_prepared_sends(&smc->conn);
1762
		break;
S
Stefan Raspl 已提交
1763
	case SIOCATMARK:
1764 1765
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
S
Stefan Raspl 已提交
1766
			return -EINVAL;
1767
		}
S
Stefan Raspl 已提交
1768 1769 1770 1771
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED) {
			answ = 0;
		} else {
1772 1773
			smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
			smc_curs_copy(&urg, &conn->urg_curs, conn);
S
Stefan Raspl 已提交
1774 1775 1776 1777
			answ = smc_curs_diff(conn->rmb_desc->len,
					     &cons, &urg) == 1;
		}
		break;
1778
	default:
1779
		release_sock(&smc->sk);
1780 1781
		return -ENOIOCTLCMD;
	}
1782
	release_sock(&smc->sk);
1783 1784

	return put_user(answ, (int __user *)arg);
U
Ursula Braun 已提交
1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
}

static ssize_t smc_sendpage(struct socket *sock, struct page *page,
			    int offset, size_t size, int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EPIPE;

	smc = smc_sk(sk);
	lock_sock(sk);
S
Stefan Raspl 已提交
1796 1797
	if (sk->sk_state != SMC_ACTIVE) {
		release_sock(sk);
U
Ursula Braun 已提交
1798
		goto out;
S
Stefan Raspl 已提交
1799 1800
	}
	release_sock(sk);
U
Ursula Braun 已提交
1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
	if (smc->use_fallback)
		rc = kernel_sendpage(smc->clcsock, page, offset,
				     size, flags);
	else
		rc = sock_no_sendpage(sock, page, offset, size, flags);

out:
	return rc;
}

S
Stefan Raspl 已提交
1811 1812 1813 1814 1815 1816
/* Map the affected portions of the rmbe into an spd, note the number of bytes
 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
 * updates till whenever a respective page has been fully processed.
 * Note that subsequent recv() calls have to wait till all splice() processing
 * completed.
 */
U
Ursula Braun 已提交
1817 1818
static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
			       struct pipe_inode_info *pipe, size_t len,
S
Stefan Raspl 已提交
1819
			       unsigned int flags)
U
Ursula Braun 已提交
1820 1821 1822 1823 1824 1825 1826
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -ENOTCONN;

	smc = smc_sk(sk);
	lock_sock(sk);
S
Stefan Raspl 已提交
1827 1828 1829 1830 1831 1832 1833 1834

	if (sk->sk_state == SMC_INIT ||
	    sk->sk_state == SMC_LISTEN ||
	    sk->sk_state == SMC_CLOSED)
		goto out;

	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
		rc = 0;
U
Ursula Braun 已提交
1835
		goto out;
S
Stefan Raspl 已提交
1836 1837
	}

U
Ursula Braun 已提交
1838 1839 1840 1841
	if (smc->use_fallback) {
		rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
						    pipe, len, flags);
	} else {
S
Stefan Raspl 已提交
1842 1843 1844 1845 1846 1847 1848 1849 1850
		if (*ppos) {
			rc = -ESPIPE;
			goto out;
		}
		if (flags & SPLICE_F_NONBLOCK)
			flags = MSG_DONTWAIT;
		else
			flags = 0;
		rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
U
Ursula Braun 已提交
1851 1852 1853
	}
out:
	release_sock(sk);
S
Stefan Raspl 已提交
1854

U
Ursula Braun 已提交
1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867
	return rc;
}

/* must look like tcp */
static const struct proto_ops smc_sock_ops = {
	.family		= PF_SMC,
	.owner		= THIS_MODULE,
	.release	= smc_release,
	.bind		= smc_bind,
	.connect	= smc_connect,
	.socketpair	= sock_no_socketpair,
	.accept		= smc_accept,
	.getname	= smc_getname,
1868
	.poll		= smc_poll,
U
Ursula Braun 已提交
1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
	.ioctl		= smc_ioctl,
	.listen		= smc_listen,
	.shutdown	= smc_shutdown,
	.setsockopt	= smc_setsockopt,
	.getsockopt	= smc_getsockopt,
	.sendmsg	= smc_sendmsg,
	.recvmsg	= smc_recvmsg,
	.mmap		= sock_no_mmap,
	.sendpage	= smc_sendpage,
	.splice_read	= smc_splice_read,
};

static int smc_create(struct net *net, struct socket *sock, int protocol,
		      int kern)
{
1884
	int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
U
Ursula Braun 已提交
1885 1886 1887 1888 1889 1890 1891 1892 1893
	struct smc_sock *smc;
	struct sock *sk;
	int rc;

	rc = -ESOCKTNOSUPPORT;
	if (sock->type != SOCK_STREAM)
		goto out;

	rc = -EPROTONOSUPPORT;
1894
	if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
U
Ursula Braun 已提交
1895 1896 1897 1898
		goto out;

	rc = -ENOBUFS;
	sock->ops = &smc_sock_ops;
1899
	sk = smc_sock_alloc(net, sock, protocol);
U
Ursula Braun 已提交
1900 1901 1902 1903 1904
	if (!sk)
		goto out;

	/* create internal TCP socket for CLC handshake and fallback */
	smc = smc_sk(sk);
1905
	smc->use_fallback = false; /* assume rdma capability first */
1906
	smc->fallback_rsn = 0;
1907 1908
	rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
			      &smc->clcsock);
1909
	if (rc) {
U
Ursula Braun 已提交
1910
		sk_common_release(sk);
1911 1912
		goto out;
	}
U
Ursula Braun 已提交
1913 1914
	smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
	smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
U
Ursula Braun 已提交
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929

out:
	return rc;
}

static const struct net_proto_family smc_sock_family_ops = {
	.family	= PF_SMC,
	.owner	= THIS_MODULE,
	.create	= smc_create,
};

static int __init smc_init(void)
{
	int rc;

1930 1931 1932 1933
	rc = smc_pnet_init();
	if (rc)
		return rc;

U
Ursula Braun 已提交
1934 1935 1936 1937 1938 1939
	rc = smc_llc_init();
	if (rc) {
		pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
		goto out_pnet;
	}

1940 1941 1942 1943 1944 1945
	rc = smc_cdc_init();
	if (rc) {
		pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
		goto out_pnet;
	}

U
Ursula Braun 已提交
1946 1947
	rc = proto_register(&smc_proto, 1);
	if (rc) {
1948
		pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
1949
		goto out_pnet;
U
Ursula Braun 已提交
1950 1951
	}

1952 1953 1954 1955 1956 1957
	rc = proto_register(&smc_proto6, 1);
	if (rc) {
		pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
		goto out_proto;
	}

U
Ursula Braun 已提交
1958 1959 1960
	rc = sock_register(&smc_sock_family_ops);
	if (rc) {
		pr_err("%s: sock_register fails with %d\n", __func__, rc);
1961
		goto out_proto6;
U
Ursula Braun 已提交
1962
	}
1963
	INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
1964
	INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
U
Ursula Braun 已提交
1965

1966 1967 1968 1969 1970 1971
	rc = smc_ib_register_client();
	if (rc) {
		pr_err("%s: ib_register fails with %d\n", __func__, rc);
		goto out_sock;
	}

U
Ursula Braun 已提交
1972
	static_branch_enable(&tcp_have_smc);
U
Ursula Braun 已提交
1973 1974
	return 0;

1975 1976
out_sock:
	sock_unregister(PF_SMC);
1977 1978
out_proto6:
	proto_unregister(&smc_proto6);
U
Ursula Braun 已提交
1979 1980
out_proto:
	proto_unregister(&smc_proto);
1981 1982
out_pnet:
	smc_pnet_exit();
U
Ursula Braun 已提交
1983 1984 1985 1986 1987
	return rc;
}

static void __exit smc_exit(void)
{
1988
	smc_core_exit();
U
Ursula Braun 已提交
1989
	static_branch_disable(&tcp_have_smc);
1990
	smc_ib_unregister_client();
U
Ursula Braun 已提交
1991
	sock_unregister(PF_SMC);
1992
	proto_unregister(&smc_proto6);
U
Ursula Braun 已提交
1993
	proto_unregister(&smc_proto);
1994
	smc_pnet_exit();
U
Ursula Braun 已提交
1995 1996 1997 1998 1999 2000 2001 2002 2003
}

module_init(smc_init);
module_exit(smc_exit);

MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("smc socket address family");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NETPROTO(PF_SMC);