xprtsock.c 78.7 KB
Newer Older
1 2 3 4 5
/*
 * linux/net/sunrpc/xprtsock.c
 *
 * Client-side transport implementation for sockets.
 *
6 7
 * TCP callback races fixes (C) 1998 Red Hat
 * TCP send fixes (C) 1998 Red Hat
8 9 10 11 12 13
 * TCP NFS related read + write fixes
 *  (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
 *
 * Rewrite of larges part of the code in order to stabilize TCP stuff.
 * Fix behaviour when socket buffer is full.
 *  (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no>
14 15
 *
 * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com>
16 17 18
 *
 * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005.
 *   <gilles.quillard@bull.net>
19 20 21
 */

#include <linux/types.h>
22
#include <linux/string.h>
23
#include <linux/slab.h>
24
#include <linux/module.h>
25 26 27 28 29 30 31
#include <linux/capability.h>
#include <linux/pagemap.h>
#include <linux/errno.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/net.h>
#include <linux/mm.h>
32
#include <linux/un.h>
33 34 35
#include <linux/udp.h>
#include <linux/tcp.h>
#include <linux/sunrpc/clnt.h>
36
#include <linux/sunrpc/addr.h>
37
#include <linux/sunrpc/sched.h>
38
#include <linux/sunrpc/svcsock.h>
39
#include <linux/sunrpc/xprtsock.h>
40
#include <linux/file.h>
41
#ifdef CONFIG_SUNRPC_BACKCHANNEL
R
Ricardo Labiaga 已提交
42 43
#include <linux/sunrpc/bc_xprt.h>
#endif
44 45 46 47 48 49

#include <net/sock.h>
#include <net/checksum.h>
#include <net/udp.h>
#include <net/tcp.h>

50 51
#include <trace/events/sunrpc.h>

52
#include "sunrpc.h"
53 54 55

static void xs_close(struct rpc_xprt *xprt);

56 57 58
/*
 * xprtsock tunables
 */
59 60 61
static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE;
static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE;
static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE;
62

63 64
static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT;
static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT;
65

66
#define XS_TCP_LINGER_TO	(15U * HZ)
67
static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;
68

69 70 71 72 73 74 75 76 77
/*
 * We can register our own files under /proc/sys/sunrpc by
 * calling register_sysctl_table() again.  The files in that
 * directory become the union of all files registered there.
 *
 * We simply need to make sure that we don't collide with
 * someone else's file names!
 */

J
Jeff Layton 已提交
78
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
79 80 81

static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE;
static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE;
82
static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT;
83 84 85 86 87 88 89 90 91
static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT;
static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT;

static struct ctl_table_header *sunrpc_table_header;

/*
 * FIXME: changing the UDP slot table size should also resize the UDP
 *        socket buffers for existing UDP transports
 */
92
static struct ctl_table xs_tunables_table[] = {
93 94 95 96 97
	{
		.procname	= "udp_slot_table_entries",
		.data		= &xprt_udp_slot_table_entries,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
98
		.proc_handler	= proc_dointvec_minmax,
99 100 101 102 103 104 105 106
		.extra1		= &min_slot_table_size,
		.extra2		= &max_slot_table_size
	},
	{
		.procname	= "tcp_slot_table_entries",
		.data		= &xprt_tcp_slot_table_entries,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
107
		.proc_handler	= proc_dointvec_minmax,
108 109 110
		.extra1		= &min_slot_table_size,
		.extra2		= &max_slot_table_size
	},
111 112 113 114 115 116 117 118 119
	{
		.procname	= "tcp_max_slot_table_entries",
		.data		= &xprt_max_tcp_slot_table_entries,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= &min_slot_table_size,
		.extra2		= &max_tcp_slot_table_limit
	},
120 121 122 123 124
	{
		.procname	= "min_resvport",
		.data		= &xprt_min_resvport,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
125
		.proc_handler	= proc_dointvec_minmax,
126 127 128 129 130 131 132 133
		.extra1		= &xprt_min_resvport_limit,
		.extra2		= &xprt_max_resvport_limit
	},
	{
		.procname	= "max_resvport",
		.data		= &xprt_max_resvport,
		.maxlen		= sizeof(unsigned int),
		.mode		= 0644,
134
		.proc_handler	= proc_dointvec_minmax,
135 136 137
		.extra1		= &xprt_min_resvport_limit,
		.extra2		= &xprt_max_resvport_limit
	},
138 139 140 141 142
	{
		.procname	= "tcp_fin_timeout",
		.data		= &xs_tcp_fin_timeout,
		.maxlen		= sizeof(xs_tcp_fin_timeout),
		.mode		= 0644,
143
		.proc_handler	= proc_dointvec_jiffies,
144
	},
145
	{ },
146 147
};

148
static struct ctl_table sunrpc_table[] = {
149 150 151 152 153
	{
		.procname	= "sunrpc",
		.mode		= 0555,
		.child		= xs_tunables_table
	},
154
	{ },
155 156 157 158
};

#endif

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
/*
 * Wait duration for a reply from the RPC portmapper.
 */
#define XS_BIND_TO		(60U * HZ)

/*
 * Delay if a UDP socket connect error occurs.  This is most likely some
 * kind of resource problem on the local host.
 */
#define XS_UDP_REEST_TO		(2U * HZ)

/*
 * The reestablish timeout allows clients to delay for a bit before attempting
 * to reconnect to a server that just dropped our connection.
 *
 * We implement an exponential backoff when trying to reestablish a TCP
 * transport connection with the server.  Some servers like to drop a TCP
 * connection when they are overworked, so we start with a short timeout and
 * increase over time if the server is down or not responding.
 */
#define XS_TCP_INIT_REEST_TO	(3U * HZ)
#define XS_TCP_MAX_REEST_TO	(5U * 60 * HZ)

/*
 * TCP idle timeout; client drops the transport socket if it is idle
 * for this long.  Note that we also timeout UDP sockets to prevent
 * holding port numbers when there is no RPC traffic.
 */
#define XS_IDLE_DISC_TO		(5U * 60 * HZ)

J
Jeff Layton 已提交
189
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
190
# undef  RPC_DEBUG_DATA
191
# define RPCDBG_FACILITY	RPCDBG_TRANS
192 193 194
#endif

#ifdef RPC_DEBUG_DATA
195
static void xs_pktdump(char *msg, u32 *packet, unsigned int count)
196
{
197 198
	u8 *buf = (u8 *) packet;
	int j;
199

200
	dprintk("RPC:       %s\n", msg);
201 202 203 204 205 206 207 208 209 210 211 212
	for (j = 0; j < count && j < 128; j += 4) {
		if (!(j & 31)) {
			if (j)
				dprintk("\n");
			dprintk("0x%04x ", j);
		}
		dprintk("%02x%02x%02x%02x ",
			buf[j], buf[j+1], buf[j+2], buf[j+3]);
	}
	dprintk("\n");
}
#else
213
static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count)
214 215 216 217 218
{
	/* NOP */
}
#endif

219 220 221 222 223
static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)
{
	return (struct rpc_xprt *) sk->sk_user_data;
}

224 225 226 227 228
static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt)
{
	return (struct sockaddr *) &xprt->addr;
}

229 230 231 232 233
static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt)
{
	return (struct sockaddr_un *) &xprt->addr;
}

234
static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt)
235
{
236 237 238 239 240 241 242 243
	return (struct sockaddr_in *) &xprt->addr;
}

static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt)
{
	return (struct sockaddr_in6 *) &xprt->addr;
}

244
static void xs_format_common_peer_addresses(struct rpc_xprt *xprt)
245
{
246
	struct sockaddr *sap = xs_addr(xprt);
247 248
	struct sockaddr_in6 *sin6;
	struct sockaddr_in *sin;
249
	struct sockaddr_un *sun;
250
	char buf[128];
251

252
	switch (sap->sa_family) {
253 254 255 256 257 258
	case AF_LOCAL:
		sun = xs_addr_un(xprt);
		strlcpy(buf, sun->sun_path, sizeof(buf));
		xprt->address_strings[RPC_DISPLAY_ADDR] =
						kstrdup(buf, GFP_KERNEL);
		break;
259
	case AF_INET:
260 261 262
		(void)rpc_ntop(sap, buf, sizeof(buf));
		xprt->address_strings[RPC_DISPLAY_ADDR] =
						kstrdup(buf, GFP_KERNEL);
263
		sin = xs_addr_in(xprt);
264
		snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));
265 266
		break;
	case AF_INET6:
267 268 269
		(void)rpc_ntop(sap, buf, sizeof(buf));
		xprt->address_strings[RPC_DISPLAY_ADDR] =
						kstrdup(buf, GFP_KERNEL);
270
		sin6 = xs_addr_in6(xprt);
271
		snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);
272 273 274
		break;
	default:
		BUG();
275
	}
276

277
	xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);
278 279
}

280
static void xs_format_common_peer_ports(struct rpc_xprt *xprt)
281
{
282 283
	struct sockaddr *sap = xs_addr(xprt);
	char buf[128];
284

285
	snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap));
286
	xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);
287

288
	snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap));
289 290
	xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
}
291

292 293 294
static void xs_format_peer_addresses(struct rpc_xprt *xprt,
				     const char *protocol,
				     const char *netid)
295
{
296 297
	xprt->address_strings[RPC_DISPLAY_PROTO] = protocol;
	xprt->address_strings[RPC_DISPLAY_NETID] = netid;
298
	xs_format_common_peer_addresses(xprt);
299
	xs_format_common_peer_ports(xprt);
300
}
301

302
static void xs_update_peer_port(struct rpc_xprt *xprt)
303
{
304 305
	kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
	kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
306

307
	xs_format_common_peer_ports(xprt);
308 309 310 311
}

static void xs_free_peer_addresses(struct rpc_xprt *xprt)
{
312 313 314 315 316 317 318 319 320 321
	unsigned int i;

	for (i = 0; i < RPC_DISPLAY_MAX; i++)
		switch (i) {
		case RPC_DISPLAY_PROTO:
		case RPC_DISPLAY_NETID:
			continue;
		default:
			kfree(xprt->address_strings[i]);
		}
322 323
}

324 325
#define XS_SENDMSG_FLAGS	(MSG_DONTWAIT | MSG_NOSIGNAL)

T
Trond Myklebust 已提交
326
static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, unsigned int base, int more)
327 328 329 330
{
	struct msghdr msg = {
		.msg_name	= addr,
		.msg_namelen	= addrlen,
T
Trond Myklebust 已提交
331 332 333 334 335
		.msg_flags	= XS_SENDMSG_FLAGS | (more ? MSG_MORE : 0),
	};
	struct kvec iov = {
		.iov_base	= vec->iov_base + base,
		.iov_len	= vec->iov_len - base,
336 337
	};

T
Trond Myklebust 已提交
338
	if (iov.iov_len != 0)
339 340 341 342
		return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len);
	return kernel_sendmsg(sock, &msg, NULL, 0, 0);
}

343
static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, bool zerocopy, int *sent_p)
344
{
345 346
	ssize_t (*do_sendpage)(struct socket *sock, struct page *page,
			int offset, size_t size, int flags);
T
Trond Myklebust 已提交
347 348
	struct page **ppage;
	unsigned int remainder;
349
	int err;
T
Trond Myklebust 已提交
350 351 352 353 354

	remainder = xdr->page_len - base;
	base += xdr->page_base;
	ppage = xdr->pages + (base >> PAGE_SHIFT);
	base &= ~PAGE_MASK;
355 356 357
	do_sendpage = sock->ops->sendpage;
	if (!zerocopy)
		do_sendpage = sock_no_sendpage;
T
Trond Myklebust 已提交
358 359 360
	for(;;) {
		unsigned int len = min_t(unsigned int, PAGE_SIZE - base, remainder);
		int flags = XS_SENDMSG_FLAGS;
361

T
Trond Myklebust 已提交
362 363 364
		remainder -= len;
		if (remainder != 0 || more)
			flags |= MSG_MORE;
365
		err = do_sendpage(sock, *ppage, base, len, flags);
T
Trond Myklebust 已提交
366 367
		if (remainder == 0 || err != len)
			break;
368
		*sent_p += err;
T
Trond Myklebust 已提交
369 370 371
		ppage++;
		base = 0;
	}
372 373 374 375 376
	if (err > 0) {
		*sent_p += err;
		err = 0;
	}
	return err;
377 378
}

379 380 381 382 383 384 385
/**
 * xs_sendpages - write pages directly to a socket
 * @sock: socket to send on
 * @addr: UDP only -- address of destination
 * @addrlen: UDP only -- length of destination address
 * @xdr: buffer containing this request
 * @base: starting position in the buffer
386
 * @zerocopy: true if it is safe to use sendpage()
387
 * @sent_p: return the total number of bytes successfully queued for sending
388
 *
389
 */
390
static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xdr, unsigned int base, bool zerocopy, int *sent_p)
391
{
T
Trond Myklebust 已提交
392
	unsigned int remainder = xdr->len - base;
393 394
	int err = 0;
	int sent = 0;
395

396
	if (unlikely(!sock))
397
		return -ENOTSOCK;
398 399

	clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags);
T
Trond Myklebust 已提交
400 401 402 403
	if (base != 0) {
		addr = NULL;
		addrlen = 0;
	}
404

T
Trond Myklebust 已提交
405 406 407 408 409
	if (base < xdr->head[0].iov_len || addr != NULL) {
		unsigned int len = xdr->head[0].iov_len - base;
		remainder -= len;
		err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0);
		if (remainder == 0 || err != len)
410
			goto out;
411
		*sent_p += err;
412 413
		base = 0;
	} else
T
Trond Myklebust 已提交
414
		base -= xdr->head[0].iov_len;
415

T
Trond Myklebust 已提交
416 417 418
	if (base < xdr->page_len) {
		unsigned int len = xdr->page_len - base;
		remainder -= len;
419 420 421
		err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent);
		*sent_p += sent;
		if (remainder == 0 || sent != len)
422 423
			goto out;
		base = 0;
T
Trond Myklebust 已提交
424 425 426 427
	} else
		base -= xdr->page_len;

	if (base >= xdr->tail[0].iov_len)
428
		return 0;
T
Trond Myklebust 已提交
429
	err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0);
430
out:
431 432 433 434 435
	if (err > 0) {
		*sent_p += err;
		err = 0;
	}
	return err;
436 437
}

438 439 440 441 442 443 444 445
static void xs_nospace_callback(struct rpc_task *task)
{
	struct sock_xprt *transport = container_of(task->tk_rqstp->rq_xprt, struct sock_xprt, xprt);

	transport->inet->sk_write_pending--;
	clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
}

446
/**
447 448
 * xs_nospace - place task on wait queue if transmit was incomplete
 * @task: task to put to sleep
449
 *
450
 */
451
static int xs_nospace(struct rpc_task *task)
452
{
453 454
	struct rpc_rqst *req = task->tk_rqstp;
	struct rpc_xprt *xprt = req->rq_xprt;
455
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
456
	struct sock *sk = transport->inet;
457
	int ret = -EAGAIN;
458

459
	dprintk("RPC: %5u xmit incomplete (%u left of %u)\n",
460 461 462
			task->tk_pid, req->rq_slen - req->rq_bytes_sent,
			req->rq_slen);

463 464 465 466 467 468 469 470 471 472 473
	/* Protect against races with write_space */
	spin_lock_bh(&xprt->transport_lock);

	/* Don't race with disconnect */
	if (xprt_connected(xprt)) {
		if (test_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags)) {
			/*
			 * Notify TCP that we're limited by the application
			 * window size
			 */
			set_bit(SOCK_NOSPACE, &transport->sock->flags);
474
			sk->sk_write_pending++;
475 476 477 478 479
			/* ...and wait for more buffer space */
			xprt_wait_for_buffer_space(task, xs_nospace_callback);
		}
	} else {
		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
480
		ret = -ENOTCONN;
481
	}
482

483
	spin_unlock_bh(&xprt->transport_lock);
484 485 486

	/* Race breaker in case memory is freed before above code is called */
	sk->sk_write_space(sk);
487
	return ret;
488 489
}

490 491 492 493 494 495 496 497 498 499
/*
 * Construct a stream transport record marker in @buf.
 */
static inline void xs_encode_stream_record_marker(struct xdr_buf *buf)
{
	u32 reclen = buf->len - sizeof(rpc_fraghdr);
	rpc_fraghdr *base = buf->head[0].iov_base;
	*base = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | reclen);
}

500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
/**
 * xs_local_send_request - write an RPC request to an AF_LOCAL socket
 * @task: RPC task that manages the state of an RPC request
 *
 * Return values:
 *        0:	The request has been sent
 *   EAGAIN:	The socket was blocked, please call again later to
 *		complete the request
 * ENOTCONN:	Caller needs to invoke connect logic then call again
 *    other:	Some other error occured, the request was not sent
 */
static int xs_local_send_request(struct rpc_task *task)
{
	struct rpc_rqst *req = task->tk_rqstp;
	struct rpc_xprt *xprt = req->rq_xprt;
	struct sock_xprt *transport =
				container_of(xprt, struct sock_xprt, xprt);
	struct xdr_buf *xdr = &req->rq_snd_buf;
	int status;
519
	int sent = 0;
520 521 522 523 524 525

	xs_encode_stream_record_marker(&req->rq_snd_buf);

	xs_pktdump("packet data:",
			req->rq_svec->iov_base, req->rq_svec->iov_len);

526 527
	status = xs_sendpages(transport->sock, NULL, 0, xdr, req->rq_bytes_sent,
			      true, &sent);
528 529
	dprintk("RPC:       %s(%u) = %d\n",
			__func__, xdr->len - req->rq_bytes_sent, status);
530 531 532
	if (likely(sent > 0) || status == 0) {
		req->rq_bytes_sent += sent;
		req->rq_xmit_bytes_sent += sent;
533 534 535 536 537 538 539 540
		if (likely(req->rq_bytes_sent >= req->rq_slen)) {
			req->rq_bytes_sent = 0;
			return 0;
		}
		status = -EAGAIN;
	}

	switch (status) {
541
	case -ENOBUFS:
542 543 544 545 546 547 548 549 550 551 552 553 554 555
	case -EAGAIN:
		status = xs_nospace(task);
		break;
	default:
		dprintk("RPC:       sendmsg returned unrecognized error %d\n",
			-status);
	case -EPIPE:
		xs_close(xprt);
		status = -ENOTCONN;
	}

	return status;
}

556 557 558 559 560 561 562 563 564
/**
 * xs_udp_send_request - write an RPC request to a UDP socket
 * @task: address of RPC task that manages the state of an RPC request
 *
 * Return values:
 *        0:	The request has been sent
 *   EAGAIN:	The socket was blocked, please call again later to
 *		complete the request
 * ENOTCONN:	Caller needs to invoke connect logic then call again
L
Lucas De Marchi 已提交
565
 *    other:	Some other error occurred, the request was not sent
566 567 568 569 570
 */
static int xs_udp_send_request(struct rpc_task *task)
{
	struct rpc_rqst *req = task->tk_rqstp;
	struct rpc_xprt *xprt = req->rq_xprt;
571
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
572
	struct xdr_buf *xdr = &req->rq_snd_buf;
573
	int sent = 0;
574
	int status;
575

576
	xs_pktdump("packet data:",
577 578 579
				req->rq_svec->iov_base,
				req->rq_svec->iov_len);

580 581
	if (!xprt_bound(xprt))
		return -ENOTCONN;
582 583
	status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen,
			      xdr, req->rq_bytes_sent, true, &sent);
584

585
	dprintk("RPC:       xs_udp_send_request(%u) = %d\n",
586
			xdr->len - req->rq_bytes_sent, status);
587

588 589 590 591
	/* firewall is blocking us, don't return -EAGAIN or we end up looping */
	if (status == -EPERM)
		goto process_status;

592 593 594
	if (sent > 0 || status == 0) {
		req->rq_xmit_bytes_sent += sent;
		if (sent >= req->rq_slen)
595 596
			return 0;
		/* Still some bytes left; set up for a retry later. */
597
		status = -EAGAIN;
598
	}
599

600
process_status:
601
	switch (status) {
602 603 604 605
	case -ENOTSOCK:
		status = -ENOTCONN;
		/* Should we call xs_close() here? */
		break;
606
	case -EAGAIN:
607
		status = xs_nospace(task);
608
		break;
609 610 611
	default:
		dprintk("RPC:       sendmsg returned unrecognized error %d\n",
			-status);
612
	case -ENETUNREACH:
613
	case -ENOBUFS:
614
	case -EPIPE:
615
	case -ECONNREFUSED:
616
	case -EPERM:
617
		/* When the server has died, an ICMP port unreachable message
618
		 * prompts ECONNREFUSED. */
619
		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
620
	}
621

622
	return status;
623 624
}

625 626 627 628 629
/**
 * xs_tcp_shutdown - gracefully shut down a TCP socket
 * @xprt: transport
 *
 * Initiates a graceful shutdown of the TCP socket by calling the
630
 * equivalent of shutdown(SHUT_RDWR);
631 632 633 634 635 636
 */
static void xs_tcp_shutdown(struct rpc_xprt *xprt)
{
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
	struct socket *sock = transport->sock;

637
	if (sock != NULL) {
638
		kernel_sock_shutdown(sock, SHUT_RDWR);
639 640
		trace_rpc_socket_shutdown(xprt, sock);
	}
641 642
}

643
/**
644
 * xs_tcp_send_request - write an RPC request to a TCP socket
645 646 647
 * @task: address of RPC task that manages the state of an RPC request
 *
 * Return values:
648 649 650 651
 *        0:	The request has been sent
 *   EAGAIN:	The socket was blocked, please call again later to
 *		complete the request
 * ENOTCONN:	Caller needs to invoke connect logic then call again
L
Lucas De Marchi 已提交
652
 *    other:	Some other error occurred, the request was not sent
653 654
 *
 * XXX: In the case of soft timeouts, should we eventually give up
655
 *	if sendmsg is not able to make progress?
656
 */
657
static int xs_tcp_send_request(struct rpc_task *task)
658 659 660
{
	struct rpc_rqst *req = task->tk_rqstp;
	struct rpc_xprt *xprt = req->rq_xprt;
661
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
662
	struct xdr_buf *xdr = &req->rq_snd_buf;
663
	bool zerocopy = true;
664
	int status;
665
	int sent;
666

667
	xs_encode_stream_record_marker(&req->rq_snd_buf);
668

669 670 671
	xs_pktdump("packet data:",
				req->rq_svec->iov_base,
				req->rq_svec->iov_len);
672 673 674 675 676 677
	/* Don't use zero copy if this is a resend. If the RPC call
	 * completes while the socket holds a reference to the pages,
	 * then we may end up resending corrupted data.
	 */
	if (task->tk_flags & RPC_TASK_SENT)
		zerocopy = false;
678 679 680

	/* Continue transmitting the packet/record. We must be careful
	 * to cope with writespace callbacks arriving _after_ we have
681
	 * called sendmsg(). */
682
	while (1) {
683 684 685
		sent = 0;
		status = xs_sendpages(transport->sock, NULL, 0, xdr,
				      req->rq_bytes_sent, zerocopy, &sent);
686

687
		dprintk("RPC:       xs_tcp_send_request(%u) = %d\n",
688
				xdr->len - req->rq_bytes_sent, status);
689

690
		if (unlikely(sent == 0 && status < 0))
691 692
			break;

693 694
		/* If we've sent the entire packet, immediately
		 * reset the count of bytes sent. */
695 696
		req->rq_bytes_sent += sent;
		req->rq_xmit_bytes_sent += sent;
697 698 699 700
		if (likely(req->rq_bytes_sent >= req->rq_slen)) {
			req->rq_bytes_sent = 0;
			return 0;
		}
701

702
		if (sent != 0)
703
			continue;
704
		status = -EAGAIN;
705
		break;
706 707
	}

708
	switch (status) {
709 710 711 712
	case -ENOTSOCK:
		status = -ENOTCONN;
		/* Should we call xs_close() here? */
		break;
713
	case -ENOBUFS:
714
	case -EAGAIN:
715
		status = xs_nospace(task);
716
		break;
717 718 719
	default:
		dprintk("RPC:       sendmsg returned unrecognized error %d\n",
			-status);
720
	case -ECONNRESET:
721 722
		xs_tcp_shutdown(xprt);
	case -ECONNREFUSED:
723
	case -ENOTCONN:
724
	case -EADDRINUSE:
725
	case -EPIPE:
726
		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);
727
	}
728

729 730 731
	return status;
}

732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
/**
 * xs_tcp_release_xprt - clean up after a tcp transmission
 * @xprt: transport
 * @task: rpc task
 *
 * This cleans up if an error causes us to abort the transmission of a request.
 * In this case, the socket may need to be reset in order to avoid confusing
 * the server.
 */
static void xs_tcp_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
{
	struct rpc_rqst *req;

	if (task != xprt->snd_task)
		return;
	if (task == NULL)
		goto out_release;
	req = task->tk_rqstp;
750 751
	if (req == NULL)
		goto out_release;
752 753 754 755
	if (req->rq_bytes_sent == 0)
		goto out_release;
	if (req->rq_bytes_sent == req->rq_snd_buf.len)
		goto out_release;
756
	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
757 758 759 760
out_release:
	xprt_release_xprt(xprt, task);
}

761 762 763 764 765
static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)
{
	transport->old_data_ready = sk->sk_data_ready;
	transport->old_state_change = sk->sk_state_change;
	transport->old_write_space = sk->sk_write_space;
766
	transport->old_error_report = sk->sk_error_report;
767 768 769 770 771 772 773
}

static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)
{
	sk->sk_data_ready = transport->old_data_ready;
	sk->sk_state_change = transport->old_state_change;
	sk->sk_write_space = transport->old_write_space;
774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797
	sk->sk_error_report = transport->old_error_report;
}

/**
 * xs_error_report - callback to handle TCP socket state errors
 * @sk: socket
 *
 * Note: we don't call sock_error() since there may be a rpc_task
 * using the socket, and so we don't want to clear sk->sk_err.
 */
static void xs_error_report(struct sock *sk)
{
	struct rpc_xprt *xprt;
	int err;

	read_lock_bh(&sk->sk_callback_lock);
	if (!(xprt = xprt_from_sock(sk)))
		goto out;

	err = -sk->sk_err;
	if (err == 0)
		goto out;
	dprintk("RPC:       xs_error_report client %p, error=%d...\n",
			xprt, -err);
798
	trace_rpc_socket_error(xprt, sk->sk_socket, err);
799 800 801
	xprt_wake_pending_tasks(xprt, err);
 out:
	read_unlock_bh(&sk->sk_callback_lock);
802 803
}

804 805 806 807 808 809 810 811
static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt)
{
	smp_mb__before_atomic();
	clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
	clear_bit(XPRT_CLOSING, &xprt->state);
	smp_mb__after_atomic();
}

812
static void xs_reset_transport(struct sock_xprt *transport)
813
{
814 815
	struct socket *sock = transport->sock;
	struct sock *sk = transport->inet;
816
	struct rpc_xprt *xprt = &transport->xprt;
817

818 819
	if (sk == NULL)
		return;
820

821
	write_lock_bh(&sk->sk_callback_lock);
822 823
	transport->inet = NULL;
	transport->sock = NULL;
824

825
	sk->sk_user_data = NULL;
826 827

	xs_restore_old_callbacks(transport, sk);
828
	write_unlock_bh(&sk->sk_callback_lock);
829
	xs_sock_reset_connection_flags(xprt);
830

831
	trace_rpc_socket_close(xprt, sock);
832
	sock_release(sock);
833 834 835 836 837 838 839 840
}

/**
 * xs_close - close a socket
 * @xprt: transport
 *
 * This is used when all requests are complete; ie, no DRC state remains
 * on the server we want to save.
841 842 843
 *
 * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with
 * xs_reset_transport() zeroing the socket from underneath a writer.
844 845 846 847 848 849 850 851
 */
static void xs_close(struct rpc_xprt *xprt)
{
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);

	dprintk("RPC:       xs_close xprt %p\n", xprt);

	xs_reset_transport(transport);
852
	xprt->reestablish_timeout = 0;
853

854
	xprt_disconnect_done(xprt);
855 856
}

857 858
static void xs_tcp_close(struct rpc_xprt *xprt)
{
859
	xs_tcp_shutdown(xprt);
860 861
}

862 863 864 865 866 867
static void xs_xprt_free(struct rpc_xprt *xprt)
{
	xs_free_peer_addresses(xprt);
	xprt_free(xprt);
}

868 869 870 871 872 873
/**
 * xs_destroy - prepare to shutdown a transport
 * @xprt: doomed transport
 *
 */
static void xs_destroy(struct rpc_xprt *xprt)
874
{
875
	dprintk("RPC:       xs_destroy xprt %p\n", xprt);
876

T
Trond Myklebust 已提交
877
	xs_close(xprt);
878
	xs_xprt_free(xprt);
T
Trond Myklebust 已提交
879
	module_put(THIS_MODULE);
880 881
}

882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903
static int xs_local_copy_to_xdr(struct xdr_buf *xdr, struct sk_buff *skb)
{
	struct xdr_skb_reader desc = {
		.skb		= skb,
		.offset		= sizeof(rpc_fraghdr),
		.count		= skb->len - sizeof(rpc_fraghdr),
	};

	if (xdr_partial_copy_from_skb(xdr, 0, &desc, xdr_skb_read_bits) < 0)
		return -1;
	if (desc.count)
		return -1;
	return 0;
}

/**
 * xs_local_data_ready - "data ready" callback for AF_LOCAL sockets
 * @sk: socket with data to read
 * @len: how much data to read
 *
 * Currently this assumes we can read the whole reply in a single gulp.
 */
904
static void xs_local_data_ready(struct sock *sk)
905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960
{
	struct rpc_task *task;
	struct rpc_xprt *xprt;
	struct rpc_rqst *rovr;
	struct sk_buff *skb;
	int err, repsize, copied;
	u32 _xid;
	__be32 *xp;

	read_lock_bh(&sk->sk_callback_lock);
	dprintk("RPC:       %s...\n", __func__);
	xprt = xprt_from_sock(sk);
	if (xprt == NULL)
		goto out;

	skb = skb_recv_datagram(sk, 0, 1, &err);
	if (skb == NULL)
		goto out;

	repsize = skb->len - sizeof(rpc_fraghdr);
	if (repsize < 4) {
		dprintk("RPC:       impossible RPC reply size %d\n", repsize);
		goto dropit;
	}

	/* Copy the XID from the skb... */
	xp = skb_header_pointer(skb, sizeof(rpc_fraghdr), sizeof(_xid), &_xid);
	if (xp == NULL)
		goto dropit;

	/* Look up and lock the request corresponding to the given XID */
	spin_lock(&xprt->transport_lock);
	rovr = xprt_lookup_rqst(xprt, *xp);
	if (!rovr)
		goto out_unlock;
	task = rovr->rq_task;

	copied = rovr->rq_private_buf.buflen;
	if (copied > repsize)
		copied = repsize;

	if (xs_local_copy_to_xdr(&rovr->rq_private_buf, skb)) {
		dprintk("RPC:       sk_buff copy failed\n");
		goto out_unlock;
	}

	xprt_complete_rqst(task, copied);

 out_unlock:
	spin_unlock(&xprt->transport_lock);
 dropit:
	skb_free_datagram(sk, skb);
 out:
	read_unlock_bh(&sk->sk_callback_lock);
}

961 962 963 964 965
/**
 * xs_udp_data_ready - "data ready" callback for UDP sockets
 * @sk: socket with data to read
 * @len: how much data to read
 *
966
 */
967
static void xs_udp_data_ready(struct sock *sk)
968
{
969 970
	struct rpc_task *task;
	struct rpc_xprt *xprt;
971
	struct rpc_rqst *rovr;
972
	struct sk_buff *skb;
973
	int err, repsize, copied;
974 975
	u32 _xid;
	__be32 *xp;
976

E
Eric Dumazet 已提交
977
	read_lock_bh(&sk->sk_callback_lock);
978
	dprintk("RPC:       xs_udp_data_ready...\n");
979
	if (!(xprt = xprt_from_sock(sk)))
980 981 982 983 984 985 986
		goto out;

	if ((skb = skb_recv_datagram(sk, 0, 1, &err)) == NULL)
		goto out;

	repsize = skb->len - sizeof(struct udphdr);
	if (repsize < 4) {
987
		dprintk("RPC:       impossible RPC reply size %d!\n", repsize);
988 989 990 991 992 993 994 995 996 997
		goto dropit;
	}

	/* Copy the XID from the skb... */
	xp = skb_header_pointer(skb, sizeof(struct udphdr),
				sizeof(_xid), &_xid);
	if (xp == NULL)
		goto dropit;

	/* Look up and lock the request corresponding to the given XID */
C
Chuck Lever 已提交
998
	spin_lock(&xprt->transport_lock);
999 1000 1001 1002 1003 1004 1005 1006 1007
	rovr = xprt_lookup_rqst(xprt, *xp);
	if (!rovr)
		goto out_unlock;
	task = rovr->rq_task;

	if ((copied = rovr->rq_private_buf.buflen) > repsize)
		copied = repsize;

	/* Suck it into the iovec, verify checksum if not done by hw. */
1008 1009
	if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) {
		UDPX_INC_STATS_BH(sk, UDP_MIB_INERRORS);
1010
		goto out_unlock;
1011 1012 1013
	}

	UDPX_INC_STATS_BH(sk, UDP_MIB_INDATAGRAMS);
1014

1015
	xprt_adjust_cwnd(xprt, task, copied);
1016
	xprt_complete_rqst(task, copied);
1017 1018

 out_unlock:
C
Chuck Lever 已提交
1019
	spin_unlock(&xprt->transport_lock);
1020 1021 1022
 dropit:
	skb_free_datagram(sk, skb);
 out:
E
Eric Dumazet 已提交
1023
	read_unlock_bh(&sk->sk_callback_lock);
1024 1025
}

1026 1027 1028 1029 1030 1031 1032 1033 1034
/*
 * Helper function to force a TCP close if the server is sending
 * junk and/or it has put us in CLOSE_WAIT
 */
static void xs_tcp_force_close(struct rpc_xprt *xprt)
{
	xprt_force_disconnect(xprt);
}

1035
static inline void xs_tcp_read_fraghdr(struct rpc_xprt *xprt, struct xdr_skb_reader *desc)
1036
{
1037
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1038 1039 1040
	size_t len, used;
	char *p;

1041 1042
	p = ((char *) &transport->tcp_fraghdr) + transport->tcp_offset;
	len = sizeof(transport->tcp_fraghdr) - transport->tcp_offset;
1043
	used = xdr_skb_read_bits(desc, p, len);
1044
	transport->tcp_offset += used;
1045 1046
	if (used != len)
		return;
1047

1048 1049
	transport->tcp_reclen = ntohl(transport->tcp_fraghdr);
	if (transport->tcp_reclen & RPC_LAST_STREAM_FRAGMENT)
1050
		transport->tcp_flags |= TCP_RCV_LAST_FRAG;
1051
	else
1052
		transport->tcp_flags &= ~TCP_RCV_LAST_FRAG;
1053
	transport->tcp_reclen &= RPC_FRAGMENT_SIZE_MASK;
1054

1055
	transport->tcp_flags &= ~TCP_RCV_COPY_FRAGHDR;
1056
	transport->tcp_offset = 0;
1057

1058
	/* Sanity check of the record length */
1059
	if (unlikely(transport->tcp_reclen < 8)) {
1060
		dprintk("RPC:       invalid TCP record fragment length\n");
1061
		xs_tcp_force_close(xprt);
1062
		return;
1063
	}
1064
	dprintk("RPC:       reading TCP record fragment of length %d\n",
1065
			transport->tcp_reclen);
1066 1067
}

1068
static void xs_tcp_check_fraghdr(struct sock_xprt *transport)
1069
{
1070
	if (transport->tcp_offset == transport->tcp_reclen) {
1071
		transport->tcp_flags |= TCP_RCV_COPY_FRAGHDR;
1072
		transport->tcp_offset = 0;
1073 1074 1075
		if (transport->tcp_flags & TCP_RCV_LAST_FRAG) {
			transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
			transport->tcp_flags |= TCP_RCV_COPY_XID;
1076
			transport->tcp_copied = 0;
1077 1078 1079 1080
		}
	}
}

1081
static inline void xs_tcp_read_xid(struct sock_xprt *transport, struct xdr_skb_reader *desc)
1082 1083 1084 1085
{
	size_t len, used;
	char *p;

1086
	len = sizeof(transport->tcp_xid) - transport->tcp_offset;
1087
	dprintk("RPC:       reading XID (%Zu bytes)\n", len);
1088
	p = ((char *) &transport->tcp_xid) + transport->tcp_offset;
1089
	used = xdr_skb_read_bits(desc, p, len);
1090
	transport->tcp_offset += used;
1091 1092
	if (used != len)
		return;
1093
	transport->tcp_flags &= ~TCP_RCV_COPY_XID;
1094
	transport->tcp_flags |= TCP_RCV_READ_CALLDIR;
1095
	transport->tcp_copied = 4;
1096 1097 1098
	dprintk("RPC:       reading %s XID %08x\n",
			(transport->tcp_flags & TCP_RPC_REPLY) ? "reply for"
							      : "request with",
1099 1100
			ntohl(transport->tcp_xid));
	xs_tcp_check_fraghdr(transport);
1101 1102
}

1103 1104
static inline void xs_tcp_read_calldir(struct sock_xprt *transport,
				       struct xdr_skb_reader *desc)
1105
{
1106 1107
	size_t len, used;
	u32 offset;
1108
	char *p;
1109 1110 1111 1112 1113 1114 1115 1116

	/*
	 * We want transport->tcp_offset to be 8 at the end of this routine
	 * (4 bytes for the xid and 4 bytes for the call/reply flag).
	 * When this function is called for the first time,
	 * transport->tcp_offset is 4 (after having already read the xid).
	 */
	offset = transport->tcp_offset - sizeof(transport->tcp_xid);
1117
	len = sizeof(transport->tcp_calldir) - offset;
1118
	dprintk("RPC:       reading CALL/REPLY flag (%Zu bytes)\n", len);
1119 1120
	p = ((char *) &transport->tcp_calldir) + offset;
	used = xdr_skb_read_bits(desc, p, len);
1121 1122 1123
	transport->tcp_offset += used;
	if (used != len)
		return;
1124 1125 1126 1127 1128
	transport->tcp_flags &= ~TCP_RCV_READ_CALLDIR;
	/*
	 * We don't yet have the XDR buffer, so we will write the calldir
	 * out after we get the buffer from the 'struct rpc_rqst'
	 */
1129 1130 1131 1132
	switch (ntohl(transport->tcp_calldir)) {
	case RPC_REPLY:
		transport->tcp_flags |= TCP_RCV_COPY_CALLDIR;
		transport->tcp_flags |= TCP_RCV_COPY_DATA;
1133
		transport->tcp_flags |= TCP_RPC_REPLY;
1134 1135 1136 1137
		break;
	case RPC_CALL:
		transport->tcp_flags |= TCP_RCV_COPY_CALLDIR;
		transport->tcp_flags |= TCP_RCV_COPY_DATA;
1138
		transport->tcp_flags &= ~TCP_RPC_REPLY;
1139 1140 1141
		break;
	default:
		dprintk("RPC:       invalid request message type\n");
1142
		xs_tcp_force_close(&transport->xprt);
1143
	}
1144 1145 1146
	xs_tcp_check_fraghdr(transport);
}

R
Ricardo Labiaga 已提交
1147 1148 1149
static inline void xs_tcp_read_common(struct rpc_xprt *xprt,
				     struct xdr_skb_reader *desc,
				     struct rpc_rqst *req)
1150
{
R
Ricardo Labiaga 已提交
1151 1152
	struct sock_xprt *transport =
				container_of(xprt, struct sock_xprt, xprt);
1153 1154 1155 1156 1157
	struct xdr_buf *rcvbuf;
	size_t len;
	ssize_t r;

	rcvbuf = &req->rq_private_buf;
1158 1159 1160 1161 1162 1163

	if (transport->tcp_flags & TCP_RCV_COPY_CALLDIR) {
		/*
		 * Save the RPC direction in the XDR buffer
		 */
		memcpy(rcvbuf->head[0].iov_base + transport->tcp_copied,
1164 1165 1166
			&transport->tcp_calldir,
			sizeof(transport->tcp_calldir));
		transport->tcp_copied += sizeof(transport->tcp_calldir);
1167
		transport->tcp_flags &= ~TCP_RCV_COPY_CALLDIR;
1168 1169 1170
	}

	len = desc->count;
1171
	if (len > transport->tcp_reclen - transport->tcp_offset) {
1172
		struct xdr_skb_reader my_desc;
1173

1174
		len = transport->tcp_reclen - transport->tcp_offset;
1175 1176
		memcpy(&my_desc, desc, sizeof(my_desc));
		my_desc.count = len;
1177
		r = xdr_partial_copy_from_skb(rcvbuf, transport->tcp_copied,
1178
					  &my_desc, xdr_skb_read_bits);
1179 1180 1181
		desc->count -= r;
		desc->offset += r;
	} else
1182
		r = xdr_partial_copy_from_skb(rcvbuf, transport->tcp_copied,
1183
					  desc, xdr_skb_read_bits);
1184 1185

	if (r > 0) {
1186 1187
		transport->tcp_copied += r;
		transport->tcp_offset += r;
1188 1189 1190 1191 1192
	}
	if (r != len) {
		/* Error when copying to the receive buffer,
		 * usually because we weren't able to allocate
		 * additional buffer pages. All we can do now
1193
		 * is turn off TCP_RCV_COPY_DATA, so the request
1194 1195 1196 1197 1198
		 * will not receive any additional updates,
		 * and time out.
		 * Any remaining data from this record will
		 * be discarded.
		 */
1199
		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
1200
		dprintk("RPC:       XID %08x truncated request\n",
1201
				ntohl(transport->tcp_xid));
1202 1203 1204 1205
		dprintk("RPC:       xprt = %p, tcp_copied = %lu, "
				"tcp_offset = %u, tcp_reclen = %u\n",
				xprt, transport->tcp_copied,
				transport->tcp_offset, transport->tcp_reclen);
R
Ricardo Labiaga 已提交
1206
		return;
1207 1208
	}

1209
	dprintk("RPC:       XID %08x read %Zd bytes\n",
1210
			ntohl(transport->tcp_xid), r);
1211 1212 1213
	dprintk("RPC:       xprt = %p, tcp_copied = %lu, tcp_offset = %u, "
			"tcp_reclen = %u\n", xprt, transport->tcp_copied,
			transport->tcp_offset, transport->tcp_reclen);
1214 1215

	if (transport->tcp_copied == req->rq_private_buf.buflen)
1216
		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
1217
	else if (transport->tcp_offset == transport->tcp_reclen) {
1218 1219
		if (transport->tcp_flags & TCP_RCV_LAST_FRAG)
			transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
1220
	}
R
Ricardo Labiaga 已提交
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
}

/*
 * Finds the request corresponding to the RPC xid and invokes the common
 * tcp read code to read the data.
 */
static inline int xs_tcp_read_reply(struct rpc_xprt *xprt,
				    struct xdr_skb_reader *desc)
{
	struct sock_xprt *transport =
				container_of(xprt, struct sock_xprt, xprt);
	struct rpc_rqst *req;

	dprintk("RPC:       read reply XID %08x\n", ntohl(transport->tcp_xid));

	/* Find and lock the request corresponding to this xid */
	spin_lock(&xprt->transport_lock);
	req = xprt_lookup_rqst(xprt, transport->tcp_xid);
	if (!req) {
		dprintk("RPC:       XID %08x request not found!\n",
				ntohl(transport->tcp_xid));
		spin_unlock(&xprt->transport_lock);
		return -1;
	}

	xs_tcp_read_common(xprt, desc, req);

1248
	if (!(transport->tcp_flags & TCP_RCV_COPY_DATA))
1249
		xprt_complete_rqst(req->rq_task, transport->tcp_copied);
R
Ricardo Labiaga 已提交
1250

C
Chuck Lever 已提交
1251
	spin_unlock(&xprt->transport_lock);
R
Ricardo Labiaga 已提交
1252 1253 1254
	return 0;
}

1255
#if defined(CONFIG_SUNRPC_BACKCHANNEL)
R
Ricardo Labiaga 已提交
1256 1257 1258 1259 1260 1261 1262
/*
 * Obtains an rpc_rqst previously allocated and invokes the common
 * tcp read code to read the data.  The result is placed in the callback
 * queue.
 * If we're unable to obtain the rpc_rqst we schedule the closing of the
 * connection and return -1.
 */
1263
static int xs_tcp_read_callback(struct rpc_xprt *xprt,
R
Ricardo Labiaga 已提交
1264 1265 1266 1267 1268 1269
				       struct xdr_skb_reader *desc)
{
	struct sock_xprt *transport =
				container_of(xprt, struct sock_xprt, xprt);
	struct rpc_rqst *req;

1270 1271 1272
	/* Look up and lock the request corresponding to the given XID */
	spin_lock(&xprt->transport_lock);
	req = xprt_lookup_bc_request(xprt, transport->tcp_xid);
R
Ricardo Labiaga 已提交
1273
	if (req == NULL) {
1274
		spin_unlock(&xprt->transport_lock);
R
Ricardo Labiaga 已提交
1275 1276 1277 1278 1279 1280 1281 1282
		printk(KERN_WARNING "Callback slot table overflowed\n");
		xprt_force_disconnect(xprt);
		return -1;
	}

	dprintk("RPC:       read callback  XID %08x\n", ntohl(req->rq_xid));
	xs_tcp_read_common(xprt, desc, req);

1283 1284 1285
	if (!(transport->tcp_flags & TCP_RCV_COPY_DATA))
		xprt_complete_bc_request(req, transport->tcp_copied);
	spin_unlock(&xprt->transport_lock);
R
Ricardo Labiaga 已提交
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305

	return 0;
}

static inline int _xs_tcp_read_data(struct rpc_xprt *xprt,
					struct xdr_skb_reader *desc)
{
	struct sock_xprt *transport =
				container_of(xprt, struct sock_xprt, xprt);

	return (transport->tcp_flags & TCP_RPC_REPLY) ?
		xs_tcp_read_reply(xprt, desc) :
		xs_tcp_read_callback(xprt, desc);
}
#else
static inline int _xs_tcp_read_data(struct rpc_xprt *xprt,
					struct xdr_skb_reader *desc)
{
	return xs_tcp_read_reply(xprt, desc);
}
1306
#endif /* CONFIG_SUNRPC_BACKCHANNEL */
R
Ricardo Labiaga 已提交
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326

/*
 * Read data off the transport.  This can be either an RPC_CALL or an
 * RPC_REPLY.  Relay the processing to helper functions.
 */
static void xs_tcp_read_data(struct rpc_xprt *xprt,
				    struct xdr_skb_reader *desc)
{
	struct sock_xprt *transport =
				container_of(xprt, struct sock_xprt, xprt);

	if (_xs_tcp_read_data(xprt, desc) == 0)
		xs_tcp_check_fraghdr(transport);
	else {
		/*
		 * The transport_lock protects the request handling.
		 * There's no need to hold it to update the tcp_flags.
		 */
		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
	}
1327 1328
}

1329
static inline void xs_tcp_read_discard(struct sock_xprt *transport, struct xdr_skb_reader *desc)
1330 1331 1332
{
	size_t len;

1333
	len = transport->tcp_reclen - transport->tcp_offset;
1334 1335 1336 1337
	if (len > desc->count)
		len = desc->count;
	desc->count -= len;
	desc->offset += len;
1338
	transport->tcp_offset += len;
1339
	dprintk("RPC:       discarded %Zu bytes\n", len);
1340
	xs_tcp_check_fraghdr(transport);
1341 1342
}

1343
static int xs_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, unsigned int offset, size_t len)
1344 1345
{
	struct rpc_xprt *xprt = rd_desc->arg.data;
1346
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1347
	struct xdr_skb_reader desc = {
1348 1349 1350
		.skb	= skb,
		.offset	= offset,
		.count	= len,
1351
	};
1352

1353
	dprintk("RPC:       xs_tcp_data_recv started\n");
1354
	do {
1355
		trace_xs_tcp_data_recv(transport);
1356 1357
		/* Read in a new fragment marker if necessary */
		/* Can we ever really expect to get completely empty fragments? */
1358
		if (transport->tcp_flags & TCP_RCV_COPY_FRAGHDR) {
1359
			xs_tcp_read_fraghdr(xprt, &desc);
1360 1361 1362
			continue;
		}
		/* Read in the xid if necessary */
1363
		if (transport->tcp_flags & TCP_RCV_COPY_XID) {
1364
			xs_tcp_read_xid(transport, &desc);
1365 1366
			continue;
		}
1367
		/* Read in the call/reply flag */
1368
		if (transport->tcp_flags & TCP_RCV_READ_CALLDIR) {
1369 1370 1371
			xs_tcp_read_calldir(transport, &desc);
			continue;
		}
1372
		/* Read in the request data */
1373
		if (transport->tcp_flags & TCP_RCV_COPY_DATA) {
R
Ricardo Labiaga 已提交
1374
			xs_tcp_read_data(xprt, &desc);
1375 1376 1377
			continue;
		}
		/* Skip over any trailing bytes on short reads */
1378
		xs_tcp_read_discard(transport, &desc);
1379
	} while (desc.count);
1380
	trace_xs_tcp_data_recv(transport);
1381
	dprintk("RPC:       xs_tcp_data_recv done\n");
1382 1383 1384
	return len - desc.count;
}

1385 1386 1387 1388 1389 1390
/**
 * xs_tcp_data_ready - "data ready" callback for TCP sockets
 * @sk: socket with data to read
 * @bytes: how much data to read
 *
 */
1391
static void xs_tcp_data_ready(struct sock *sk)
1392 1393 1394
{
	struct rpc_xprt *xprt;
	read_descriptor_t rd_desc;
1395
	int read;
1396
	unsigned long total = 0;
1397

1398 1399
	dprintk("RPC:       xs_tcp_data_ready...\n");

E
Eric Dumazet 已提交
1400
	read_lock_bh(&sk->sk_callback_lock);
1401 1402
	if (!(xprt = xprt_from_sock(sk))) {
		read = 0;
1403
		goto out;
1404
	}
1405 1406 1407 1408 1409 1410
	/* Any data means we had a useful conversation, so
	 * the we don't need to delay the next reconnect
	 */
	if (xprt->reestablish_timeout)
		xprt->reestablish_timeout = 0;

1411
	/* We use rd_desc to pass struct xprt to xs_tcp_data_recv */
1412
	rd_desc.arg.data = xprt;
1413 1414 1415
	do {
		rd_desc.count = 65536;
		read = tcp_read_sock(sk, &rd_desc, xs_tcp_data_recv);
1416 1417
		if (read > 0)
			total += read;
1418
	} while (read > 0);
1419
out:
1420
	trace_xs_tcp_data_ready(xprt, read, total);
E
Eric Dumazet 已提交
1421
	read_unlock_bh(&sk->sk_callback_lock);
1422 1423
}

1424 1425 1426
static void xs_sock_mark_closed(struct rpc_xprt *xprt)
{
	xs_sock_reset_connection_flags(xprt);
1427 1428 1429 1430
	/* Mark transport as closed and wake up all pending tasks */
	xprt_disconnect_done(xprt);
}

1431 1432 1433 1434 1435 1436
/**
 * xs_tcp_state_change - callback to handle TCP socket state changes
 * @sk: socket whose state has changed
 *
 */
static void xs_tcp_state_change(struct sock *sk)
1437
{
1438
	struct rpc_xprt *xprt;
1439

E
Eric Dumazet 已提交
1440
	read_lock_bh(&sk->sk_callback_lock);
1441 1442
	if (!(xprt = xprt_from_sock(sk)))
		goto out;
1443
	dprintk("RPC:       xs_tcp_state_change client %p...\n", xprt);
1444
	dprintk("RPC:       state %x conn %d dead %d zapped %d sk_shutdown %d\n",
1445 1446
			sk->sk_state, xprt_connected(xprt),
			sock_flag(sk, SOCK_DEAD),
1447 1448
			sock_flag(sk, SOCK_ZAPPED),
			sk->sk_shutdown);
1449

1450
	trace_rpc_socket_state_change(xprt, sk->sk_socket);
1451 1452
	switch (sk->sk_state) {
	case TCP_ESTABLISHED:
E
Eric Dumazet 已提交
1453
		spin_lock(&xprt->transport_lock);
1454
		if (!xprt_test_and_set_connected(xprt)) {
1455 1456 1457
			struct sock_xprt *transport = container_of(xprt,
					struct sock_xprt, xprt);

1458
			/* Reset TCP record info */
1459 1460 1461
			transport->tcp_offset = 0;
			transport->tcp_reclen = 0;
			transport->tcp_copied = 0;
1462 1463
			transport->tcp_flags =
				TCP_RCV_COPY_FRAGHDR | TCP_RCV_COPY_XID;
1464
			xprt->connect_cookie++;
1465

1466
			xprt_wake_pending_tasks(xprt, -EAGAIN);
1467
		}
E
Eric Dumazet 已提交
1468
		spin_unlock(&xprt->transport_lock);
1469
		break;
1470 1471
	case TCP_FIN_WAIT1:
		/* The client initiated a shutdown of the socket */
1472
		xprt->connect_cookie++;
1473
		xprt->reestablish_timeout = 0;
1474
		set_bit(XPRT_CLOSING, &xprt->state);
1475
		smp_mb__before_atomic();
1476
		clear_bit(XPRT_CONNECTED, &xprt->state);
1477
		clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
1478
		smp_mb__after_atomic();
1479
		break;
1480
	case TCP_CLOSE_WAIT:
1481
		/* The server initiated a shutdown of the socket */
1482
		xprt->connect_cookie++;
1483
		clear_bit(XPRT_CONNECTED, &xprt->state);
1484
		xs_tcp_force_close(xprt);
1485 1486 1487 1488 1489 1490 1491
	case TCP_CLOSING:
		/*
		 * If the server closed down the connection, make sure that
		 * we back off before reconnecting
		 */
		if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
			xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
1492 1493
		break;
	case TCP_LAST_ACK:
1494
		set_bit(XPRT_CLOSING, &xprt->state);
1495
		smp_mb__before_atomic();
1496
		clear_bit(XPRT_CONNECTED, &xprt->state);
1497
		smp_mb__after_atomic();
1498 1499
		break;
	case TCP_CLOSE:
1500
		xs_sock_mark_closed(xprt);
1501 1502
	}
 out:
E
Eric Dumazet 已提交
1503
	read_unlock_bh(&sk->sk_callback_lock);
1504 1505
}

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
static void xs_write_space(struct sock *sk)
{
	struct socket *sock;
	struct rpc_xprt *xprt;

	if (unlikely(!(sock = sk->sk_socket)))
		return;
	clear_bit(SOCK_NOSPACE, &sock->flags);

	if (unlikely(!(xprt = xprt_from_sock(sk))))
		return;
	if (test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags) == 0)
		return;

	xprt_write_space(xprt);
}

1523
/**
1524 1525
 * xs_udp_write_space - callback invoked when socket buffer space
 *                             becomes available
1526 1527
 * @sk: socket whose state has changed
 *
1528 1529
 * Called when more output buffer space is available for this socket.
 * We try not to wake our writers until they can make "significant"
1530
 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
1531 1532
 * with a bunch of small requests.
 */
1533
static void xs_udp_write_space(struct sock *sk)
1534
{
E
Eric Dumazet 已提交
1535
	read_lock_bh(&sk->sk_callback_lock);
1536

1537
	/* from net/core/sock.c:sock_def_write_space */
1538 1539
	if (sock_writeable(sk))
		xs_write_space(sk);
1540

E
Eric Dumazet 已提交
1541
	read_unlock_bh(&sk->sk_callback_lock);
1542
}
1543

1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
/**
 * xs_tcp_write_space - callback invoked when socket buffer space
 *                             becomes available
 * @sk: socket whose state has changed
 *
 * Called when more output buffer space is available for this socket.
 * We try not to wake our writers until they can make "significant"
 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
 * with a bunch of small requests.
 */
static void xs_tcp_write_space(struct sock *sk)
{
E
Eric Dumazet 已提交
1556
	read_lock_bh(&sk->sk_callback_lock);
1557 1558

	/* from net/core/stream.c:sk_stream_write_space */
1559
	if (sk_stream_is_writeable(sk))
1560
		xs_write_space(sk);
1561

E
Eric Dumazet 已提交
1562
	read_unlock_bh(&sk->sk_callback_lock);
1563 1564
}

1565
static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt)
1566
{
1567 1568
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
	struct sock *sk = transport->inet;
1569

1570
	if (transport->rcvsize) {
1571
		sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
1572
		sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;
1573
	}
1574
	if (transport->sndsize) {
1575
		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
1576
		sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;
1577 1578 1579 1580
		sk->sk_write_space(sk);
	}
}

1581
/**
1582
 * xs_udp_set_buffer_size - set send and receive limits
1583
 * @xprt: generic transport
1584 1585
 * @sndsize: requested size of send buffer, in bytes
 * @rcvsize: requested size of receive buffer, in bytes
1586
 *
1587
 * Set socket send and receive buffer size limits.
1588
 */
1589
static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize)
1590
{
1591 1592 1593
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);

	transport->sndsize = 0;
1594
	if (sndsize)
1595 1596
		transport->sndsize = sndsize + 1024;
	transport->rcvsize = 0;
1597
	if (rcvsize)
1598
		transport->rcvsize = rcvsize + 1024;
1599 1600

	xs_udp_do_set_buffer_size(xprt);
1601 1602
}

1603 1604 1605 1606 1607 1608
/**
 * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport
 * @task: task that timed out
 *
 * Adjust the congestion window after a retransmit timeout has occurred.
 */
1609
static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task)
1610
{
1611
	xprt_adjust_cwnd(xprt, task, -ETIMEDOUT);
1612 1613
}

1614 1615 1616
static unsigned short xs_get_random_port(void)
{
	unsigned short range = xprt_max_resvport - xprt_min_resvport;
1617
	unsigned short rand = (unsigned short) prandom_u32() % range;
1618 1619 1620
	return rand + xprt_min_resvport;
}

1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
/**
 * xs_set_reuseaddr_port - set the socket's port and address reuse options
 * @sock: socket
 *
 * Note that this function has to be called on all sockets that share the
 * same port, and it must be called before binding.
 */
static void xs_sock_set_reuseport(struct socket *sock)
{
	char opt = 1;

	kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
}

static unsigned short xs_sock_getport(struct socket *sock)
{
	struct sockaddr_storage buf;
	int buflen;
	unsigned short port = 0;

	if (kernel_getsockname(sock, (struct sockaddr *)&buf, &buflen) < 0)
		goto out;
	switch (buf.ss_family) {
	case AF_INET6:
		port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port);
		break;
	case AF_INET:
		port = ntohs(((struct sockaddr_in *)&buf)->sin_port);
	}
out:
	return port;
}

1654 1655 1656 1657 1658 1659 1660 1661
/**
 * xs_set_port - reset the port number in the remote endpoint address
 * @xprt: generic transport
 * @port: new port number
 *
 */
static void xs_set_port(struct rpc_xprt *xprt, unsigned short port)
{
1662
	dprintk("RPC:       setting port for xprt %p to %u\n", xprt, port);
1663

1664 1665
	rpc_set_port(xs_addr(xprt), port);
	xs_update_peer_port(xprt);
1666 1667
}

1668 1669 1670 1671 1672 1673
static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)
{
	if (transport->srcport == 0)
		transport->srcport = xs_sock_getport(sock);
}

1674
static unsigned short xs_get_srcport(struct sock_xprt *transport)
1675
{
1676
	unsigned short port = transport->srcport;
1677 1678 1679 1680 1681 1682

	if (port == 0 && transport->xprt.resvport)
		port = xs_get_random_port();
	return port;
}

1683
static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port)
1684
{
1685 1686
	if (transport->srcport != 0)
		transport->srcport = 0;
1687 1688 1689 1690 1691 1692
	if (!transport->xprt.resvport)
		return 0;
	if (port <= xprt_min_resvport || port > xprt_max_resvport)
		return xprt_max_resvport;
	return --port;
}
P
Pavel Emelyanov 已提交
1693
static int xs_bind(struct sock_xprt *transport, struct socket *sock)
1694
{
P
Pavel Emelyanov 已提交
1695
	struct sockaddr_storage myaddr;
1696
	int err, nloop = 0;
1697
	unsigned short port = xs_get_srcport(transport);
1698
	unsigned short last;
1699

1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
	/*
	 * If we are asking for any ephemeral port (i.e. port == 0 &&
	 * transport->xprt.resvport == 0), don't bind.  Let the local
	 * port selection happen implicitly when the socket is used
	 * (for example at connect time).
	 *
	 * This ensures that we can continue to establish TCP
	 * connections even when all local ephemeral ports are already
	 * a part of some TCP connection.  This makes no difference
	 * for UDP sockets, but also doens't harm them.
	 *
	 * If we're asking for any reserved port (i.e. port == 0 &&
	 * transport->xprt.resvport == 1) xs_get_srcport above will
	 * ensure that port is non-zero and we will bind as needed.
	 */
	if (port == 0)
		return 0;

P
Pavel Emelyanov 已提交
1718
	memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen);
1719
	do {
P
Pavel Emelyanov 已提交
1720 1721 1722
		rpc_set_port((struct sockaddr *)&myaddr, port);
		err = kernel_bind(sock, (struct sockaddr *)&myaddr,
				transport->xprt.addrlen);
1723
		if (err == 0) {
1724
			transport->srcport = port;
1725
			break;
1726
		}
1727
		last = port;
1728
		port = xs_next_srcport(transport, port);
1729 1730 1731
		if (port > last)
			nloop++;
	} while (err == -EADDRINUSE && nloop != 2);
1732

1733
	if (myaddr.ss_family == AF_INET)
P
Pavel Emelyanov 已提交
1734 1735 1736 1737 1738 1739 1740
		dprintk("RPC:       %s %pI4:%u: %s (%d)\n", __func__,
				&((struct sockaddr_in *)&myaddr)->sin_addr,
				port, err ? "failed" : "ok", err);
	else
		dprintk("RPC:       %s %pI6:%u: %s (%d)\n", __func__,
				&((struct sockaddr_in6 *)&myaddr)->sin6_addr,
				port, err ? "failed" : "ok", err);
1741 1742 1743
	return err;
}

1744 1745 1746 1747 1748
/*
 * We don't support autobind on AF_LOCAL sockets
 */
static void xs_local_rpcbind(struct rpc_task *task)
{
1749 1750 1751
	rcu_read_lock();
	xprt_set_bound(rcu_dereference(task->tk_client->cl_xprt));
	rcu_read_unlock();
1752 1753 1754 1755 1756
}

static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port)
{
}
P
Pavel Emelyanov 已提交
1757

1758 1759 1760 1761
#ifdef CONFIG_DEBUG_LOCK_ALLOC
static struct lock_class_key xs_key[2];
static struct lock_class_key xs_slock_key[2];

1762 1763 1764 1765 1766 1767 1768 1769
static inline void xs_reclassify_socketu(struct socket *sock)
{
	struct sock *sk = sock->sk;

	sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC",
		&xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]);
}

1770
static inline void xs_reclassify_socket4(struct socket *sock)
1771 1772
{
	struct sock *sk = sock->sk;
1773 1774 1775 1776

	sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC",
		&xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]);
}
1777

1778 1779 1780
static inline void xs_reclassify_socket6(struct socket *sock)
{
	struct sock *sk = sock->sk;
1781

1782 1783
	sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC",
		&xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]);
1784
}
1785 1786 1787

static inline void xs_reclassify_socket(int family, struct socket *sock)
{
1788 1789 1790 1791
	WARN_ON_ONCE(sock_owned_by_user(sock->sk));
	if (sock_owned_by_user(sock->sk))
		return;

1792
	switch (family) {
1793 1794 1795
	case AF_LOCAL:
		xs_reclassify_socketu(sock);
		break;
1796
	case AF_INET:
1797
		xs_reclassify_socket4(sock);
1798 1799
		break;
	case AF_INET6:
1800
		xs_reclassify_socket6(sock);
1801 1802
		break;
	}
1803
}
1804
#else
1805 1806 1807 1808
static inline void xs_reclassify_socketu(struct socket *sock)
{
}

1809 1810 1811 1812 1813
static inline void xs_reclassify_socket4(struct socket *sock)
{
}

static inline void xs_reclassify_socket6(struct socket *sock)
1814 1815
{
}
1816 1817 1818 1819

static inline void xs_reclassify_socket(int family, struct socket *sock)
{
}
1820 1821
#endif

1822 1823 1824 1825
static void xs_dummy_setup_socket(struct work_struct *work)
{
}

1826
static struct socket *xs_create_sock(struct rpc_xprt *xprt,
1827 1828
		struct sock_xprt *transport, int family, int type,
		int protocol, bool reuseport)
1829 1830 1831 1832
{
	struct socket *sock;
	int err;

1833
	err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1);
1834 1835 1836 1837 1838
	if (err < 0) {
		dprintk("RPC:       can't create %d transport socket (%d).\n",
				protocol, -err);
		goto out;
	}
1839
	xs_reclassify_socket(family, sock);
1840

1841 1842 1843
	if (reuseport)
		xs_sock_set_reuseport(sock);

1844 1845
	err = xs_bind(transport, sock);
	if (err) {
1846 1847 1848 1849 1850 1851 1852 1853 1854
		sock_release(sock);
		goto out;
	}

	return sock;
out:
	return ERR_PTR(err);
}

1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
static int xs_local_finish_connecting(struct rpc_xprt *xprt,
				      struct socket *sock)
{
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
									xprt);

	if (!transport->inet) {
		struct sock *sk = sock->sk;

		write_lock_bh(&sk->sk_callback_lock);

		xs_save_old_callbacks(transport, sk);

		sk->sk_user_data = xprt;
		sk->sk_data_ready = xs_local_data_ready;
		sk->sk_write_space = xs_udp_write_space;
1871
		sk->sk_error_report = xs_error_report;
1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894
		sk->sk_allocation = GFP_ATOMIC;

		xprt_clear_connected(xprt);

		/* Reset to new socket */
		transport->sock = sock;
		transport->inet = sk;

		write_unlock_bh(&sk->sk_callback_lock);
	}

	/* Tell the socket layer to start connecting... */
	xprt->stat.connect_count++;
	xprt->stat.connect_start = jiffies;
	return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0);
}

/**
 * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint
 * @xprt: RPC transport to connect
 * @transport: socket transport to connect
 * @create_sock: function to create a socket of the correct type
 */
1895
static int xs_local_setup_socket(struct sock_xprt *transport)
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913
{
	struct rpc_xprt *xprt = &transport->xprt;
	struct socket *sock;
	int status = -EIO;

	status = __sock_create(xprt->xprt_net, AF_LOCAL,
					SOCK_STREAM, 0, &sock, 1);
	if (status < 0) {
		dprintk("RPC:       can't create AF_LOCAL "
			"transport socket (%d).\n", -status);
		goto out;
	}
	xs_reclassify_socketu(sock);

	dprintk("RPC:       worker connecting xprt %p via AF_LOCAL to %s\n",
			xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);

	status = xs_local_finish_connecting(xprt, sock);
1914
	trace_rpc_socket_connect(xprt, sock, status);
1915 1916 1917 1918 1919
	switch (status) {
	case 0:
		dprintk("RPC:       xprt %p connected to %s\n",
				xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
		xprt_set_connected(xprt);
1920
	case -ENOBUFS:
1921 1922 1923 1924 1925
		break;
	case -ENOENT:
		dprintk("RPC:       xprt %p: socket %s does not exist\n",
				xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
		break;
1926 1927 1928 1929
	case -ECONNREFUSED:
		dprintk("RPC:       xprt %p: connection refused for %s\n",
				xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
		break;
1930 1931 1932 1933 1934 1935 1936 1937 1938
	default:
		printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n",
				__func__, -status,
				xprt->address_strings[RPC_DISPLAY_ADDR]);
	}

out:
	xprt_clear_connecting(xprt);
	xprt_wake_pending_tasks(xprt, status);
1939 1940 1941
	return status;
}

1942
static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task)
1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962
{
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
	int ret;

	 if (RPC_IS_ASYNC(task)) {
		/*
		 * We want the AF_LOCAL connect to be resolved in the
		 * filesystem namespace of the process making the rpc
		 * call.  Thus we connect synchronously.
		 *
		 * If we want to support asynchronous AF_LOCAL calls,
		 * we'll need to figure out how to pass a namespace to
		 * connect.
		 */
		rpc_exit(task, -ENOTCONN);
		return;
	}
	ret = xs_local_setup_socket(transport);
	if (ret && !RPC_IS_SOFTCONN(task))
		msleep_interruptible(15000);
1963 1964
}

M
Mel Gorman 已提交
1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
#ifdef CONFIG_SUNRPC_SWAP
static void xs_set_memalloc(struct rpc_xprt *xprt)
{
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
			xprt);

	if (xprt->swapper)
		sk_set_memalloc(transport->inet);
}

/**
 * xs_swapper - Tag this transport as being used for swap.
 * @xprt: transport to tag
 * @enable: enable/disable
 *
 */
int xs_swapper(struct rpc_xprt *xprt, int enable)
{
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
			xprt);
	int err = 0;

	if (enable) {
		xprt->swapper++;
		xs_set_memalloc(xprt);
	} else if (xprt->swapper) {
		xprt->swapper--;
		sk_clear_memalloc(transport->inet);
	}

	return err;
}
EXPORT_SYMBOL_GPL(xs_swapper);
#else
static void xs_set_memalloc(struct rpc_xprt *xprt)
{
}
#endif

2004 2005 2006 2007 2008 2009 2010 2011 2012
static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
{
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);

	if (!transport->inet) {
		struct sock *sk = sock->sk;

		write_lock_bh(&sk->sk_callback_lock);

2013 2014
		xs_save_old_callbacks(transport, sk);

2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
		sk->sk_user_data = xprt;
		sk->sk_data_ready = xs_udp_data_ready;
		sk->sk_write_space = xs_udp_write_space;
		sk->sk_allocation = GFP_ATOMIC;

		xprt_set_connected(xprt);

		/* Reset to new socket */
		transport->sock = sock;
		transport->inet = sk;

M
Mel Gorman 已提交
2026 2027
		xs_set_memalloc(xprt);

2028 2029 2030 2031 2032
		write_unlock_bh(&sk->sk_callback_lock);
	}
	xs_udp_do_set_buffer_size(xprt);
}

2033
static void xs_udp_setup_socket(struct work_struct *work)
2034
{
2035 2036
	struct sock_xprt *transport =
		container_of(work, struct sock_xprt, connect_worker.work);
2037
	struct rpc_xprt *xprt = &transport->xprt;
2038
	struct socket *sock = transport->sock;
2039
	int status = -EIO;
2040

2041
	sock = xs_create_sock(xprt, transport,
2042 2043
			xs_addr(xprt)->sa_family, SOCK_DGRAM,
			IPPROTO_UDP, false);
2044
	if (IS_ERR(sock))
2045
		goto out;
2046

C
Chuck Lever 已提交
2047 2048 2049 2050 2051
	dprintk("RPC:       worker connecting xprt %p via %s to "
				"%s (port %s)\n", xprt,
			xprt->address_strings[RPC_DISPLAY_PROTO],
			xprt->address_strings[RPC_DISPLAY_ADDR],
			xprt->address_strings[RPC_DISPLAY_PORT]);
2052 2053

	xs_udp_finish_connecting(xprt, sock);
2054
	trace_rpc_socket_connect(xprt, sock, 0);
2055 2056
	status = 0;
out:
2057
	xprt_unlock_connect(xprt, transport);
2058
	xprt_clear_connecting(xprt);
2059
	xprt_wake_pending_tasks(xprt, status);
2060 2061
}

2062
static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
2063
{
2064
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2065
	int ret = -ENOTCONN;
2066

2067
	if (!transport->inet) {
2068
		struct sock *sk = sock->sk;
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
		unsigned int keepidle = xprt->timeout->to_initval / HZ;
		unsigned int keepcnt = xprt->timeout->to_retries + 1;
		unsigned int opt_on = 1;

		/* TCP Keepalive options */
		kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,
				(char *)&opt_on, sizeof(opt_on));
		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE,
				(char *)&keepidle, sizeof(keepidle));
		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL,
				(char *)&keepidle, sizeof(keepidle));
		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT,
				(char *)&keepcnt, sizeof(keepcnt));
2082 2083 2084

		write_lock_bh(&sk->sk_callback_lock);

2085 2086
		xs_save_old_callbacks(transport, sk);

2087 2088 2089 2090
		sk->sk_user_data = xprt;
		sk->sk_data_ready = xs_tcp_data_ready;
		sk->sk_state_change = xs_tcp_state_change;
		sk->sk_write_space = xs_tcp_write_space;
2091
		sk->sk_error_report = xs_error_report;
2092
		sk->sk_allocation = GFP_ATOMIC;
2093 2094 2095 2096

		/* socket options */
		sock_reset_flag(sk, SOCK_LINGER);
		tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF;
2097 2098 2099 2100

		xprt_clear_connected(xprt);

		/* Reset to new socket */
2101 2102
		transport->sock = sock;
		transport->inet = sk;
2103 2104 2105 2106

		write_unlock_bh(&sk->sk_callback_lock);
	}

2107
	if (!xprt_bound(xprt))
2108
		goto out;
2109

M
Mel Gorman 已提交
2110 2111
	xs_set_memalloc(xprt);

2112
	/* Tell the socket layer to start connecting... */
2113 2114
	xprt->stat.connect_count++;
	xprt->stat.connect_start = jiffies;
2115 2116 2117
	ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);
	switch (ret) {
	case 0:
2118
		xs_set_srcport(transport, sock);
2119 2120 2121 2122 2123 2124 2125
	case -EINPROGRESS:
		/* SYN_SENT! */
		if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
			xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
	}
out:
	return ret;
2126 2127
}

2128
/**
2129 2130 2131 2132
 * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint
 * @xprt: RPC transport to connect
 * @transport: socket transport to connect
 * @create_sock: function to create a socket of the correct type
2133 2134
 *
 * Invoked by a work queue tasklet.
2135
 */
2136
static void xs_tcp_setup_socket(struct work_struct *work)
2137
{
2138 2139
	struct sock_xprt *transport =
		container_of(work, struct sock_xprt, connect_worker.work);
2140
	struct socket *sock = transport->sock;
2141
	struct rpc_xprt *xprt = &transport->xprt;
2142
	int status = -EIO;
2143

2144
	if (!sock) {
2145
		sock = xs_create_sock(xprt, transport,
2146 2147
				xs_addr(xprt)->sa_family, SOCK_STREAM,
				IPPROTO_TCP, true);
2148 2149
		if (IS_ERR(sock)) {
			status = PTR_ERR(sock);
2150 2151
			goto out;
		}
2152
	}
2153

C
Chuck Lever 已提交
2154 2155 2156 2157 2158
	dprintk("RPC:       worker connecting xprt %p via %s to "
				"%s (port %s)\n", xprt,
			xprt->address_strings[RPC_DISPLAY_PROTO],
			xprt->address_strings[RPC_DISPLAY_ADDR],
			xprt->address_strings[RPC_DISPLAY_PORT]);
2159

2160
	status = xs_tcp_finish_connecting(xprt, sock);
2161
	trace_rpc_socket_connect(xprt, sock, status);
2162 2163 2164
	dprintk("RPC:       %p connect status %d connected %d sock state %d\n",
			xprt, -status, xprt_connected(xprt),
			sock->sk->sk_state);
2165
	switch (status) {
2166 2167 2168 2169 2170 2171 2172
	default:
		printk("%s: connect returned unhandled error %d\n",
			__func__, status);
	case -EADDRNOTAVAIL:
		/* We're probably in TIME_WAIT. Get rid of existing socket,
		 * and retry
		 */
2173
		xs_tcp_force_close(xprt);
2174
		break;
2175 2176 2177
	case 0:
	case -EINPROGRESS:
	case -EALREADY:
2178
		xprt_unlock_connect(xprt, transport);
2179 2180
		xprt_clear_connecting(xprt);
		return;
2181 2182 2183 2184
	case -EINVAL:
		/* Happens, for instance, if the user specified a link
		 * local IPv6 address without a scope-id.
		 */
2185 2186 2187
	case -ECONNREFUSED:
	case -ECONNRESET:
	case -ENETUNREACH:
2188
	case -EADDRINUSE:
2189
	case -ENOBUFS:
2190
		/* retry with existing socket, after a delay */
2191
		xs_tcp_force_close(xprt);
2192
		goto out;
2193
	}
2194
	status = -EAGAIN;
2195
out:
2196
	xprt_unlock_connect(xprt, transport);
2197
	xprt_clear_connecting(xprt);
2198
	xprt_wake_pending_tasks(xprt, status);
2199
}
2200

2201 2202
/**
 * xs_connect - connect a socket to a remote endpoint
2203
 * @xprt: pointer to transport structure
2204 2205 2206
 * @task: address of RPC task that manages state of connect request
 *
 * TCP: If the remote end dropped the connection, delay reconnecting.
2207 2208 2209 2210 2211 2212 2213
 *
 * UDP socket connects are synchronous, but we use a work queue anyway
 * to guarantee that even unprivileged user processes can set up a
 * socket on a privileged port.
 *
 * If a UDP socket connect fails, the delay behavior here prevents
 * retry floods (hard mounts).
2214
 */
2215
static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
2216
{
2217
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2218

2219 2220
	WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));

2221 2222 2223
	/* Start by resetting any existing state */
	xs_reset_transport(transport);

2224
	if (transport->sock != NULL && !RPC_IS_SOFTCONN(task)) {
2225 2226
		dprintk("RPC:       xs_connect delayed xprt %p for %lu "
				"seconds\n",
2227
				xprt, xprt->reestablish_timeout / HZ);
2228 2229 2230
		queue_delayed_work(rpciod_workqueue,
				   &transport->connect_worker,
				   xprt->reestablish_timeout);
2231
		xprt->reestablish_timeout <<= 1;
2232 2233
		if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
			xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2234 2235
		if (xprt->reestablish_timeout > XS_TCP_MAX_REEST_TO)
			xprt->reestablish_timeout = XS_TCP_MAX_REEST_TO;
2236
	} else {
2237
		dprintk("RPC:       xs_connect scheduled xprt %p\n", xprt);
2238 2239
		queue_delayed_work(rpciod_workqueue,
				   &transport->connect_worker, 0);
2240 2241 2242
	}
}

2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
/**
 * xs_local_print_stats - display AF_LOCAL socket-specifc stats
 * @xprt: rpc_xprt struct containing statistics
 * @seq: output file
 *
 */
static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
{
	long idle_time = 0;

	if (xprt_connected(xprt))
		idle_time = (long)(jiffies - xprt->last_used) / HZ;

	seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu "
2257
			"%llu %llu %lu %llu %llu\n",
2258 2259 2260 2261 2262 2263 2264 2265
			xprt->stat.bind_count,
			xprt->stat.connect_count,
			xprt->stat.connect_time,
			idle_time,
			xprt->stat.sends,
			xprt->stat.recvs,
			xprt->stat.bad_xids,
			xprt->stat.req_u,
2266 2267 2268 2269
			xprt->stat.bklog_u,
			xprt->stat.max_slots,
			xprt->stat.sending_u,
			xprt->stat.pending_u);
2270 2271
}

2272 2273 2274 2275 2276 2277 2278 2279
/**
 * xs_udp_print_stats - display UDP socket-specifc stats
 * @xprt: rpc_xprt struct containing statistics
 * @seq: output file
 *
 */
static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
{
2280 2281
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);

2282 2283
	seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu "
			"%lu %llu %llu\n",
2284
			transport->srcport,
2285 2286 2287 2288 2289
			xprt->stat.bind_count,
			xprt->stat.sends,
			xprt->stat.recvs,
			xprt->stat.bad_xids,
			xprt->stat.req_u,
2290 2291 2292 2293
			xprt->stat.bklog_u,
			xprt->stat.max_slots,
			xprt->stat.sending_u,
			xprt->stat.pending_u);
2294 2295 2296 2297 2298 2299 2300 2301 2302 2303
}

/**
 * xs_tcp_print_stats - display TCP socket-specifc stats
 * @xprt: rpc_xprt struct containing statistics
 * @seq: output file
 *
 */
static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
{
2304
	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2305 2306 2307 2308 2309
	long idle_time = 0;

	if (xprt_connected(xprt))
		idle_time = (long)(jiffies - xprt->last_used) / HZ;

2310 2311
	seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu "
			"%llu %llu %lu %llu %llu\n",
2312
			transport->srcport,
2313 2314 2315 2316 2317 2318 2319 2320
			xprt->stat.bind_count,
			xprt->stat.connect_count,
			xprt->stat.connect_time,
			idle_time,
			xprt->stat.sends,
			xprt->stat.recvs,
			xprt->stat.bad_xids,
			xprt->stat.req_u,
2321 2322 2323 2324
			xprt->stat.bklog_u,
			xprt->stat.max_slots,
			xprt->stat.sending_u,
			xprt->stat.pending_u);
2325 2326
}

2327 2328 2329 2330 2331
/*
 * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason
 * we allocate pages instead doing a kmalloc like rpc_malloc is because we want
 * to use the server side send routines.
 */
2332
static void *bc_malloc(struct rpc_task *task, size_t size)
2333 2334 2335 2336
{
	struct page *page;
	struct rpc_buffer *buf;

2337 2338 2339
	WARN_ON_ONCE(size > PAGE_SIZE - sizeof(struct rpc_buffer));
	if (size > PAGE_SIZE - sizeof(struct rpc_buffer))
		return NULL;
2340

2341
	page = alloc_page(GFP_KERNEL);
2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353
	if (!page)
		return NULL;

	buf = page_address(page);
	buf->len = PAGE_SIZE;

	return buf->data;
}

/*
 * Free the space allocated in the bc_alloc routine
 */
2354
static void bc_free(void *buffer)
2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379
{
	struct rpc_buffer *buf;

	if (!buffer)
		return;

	buf = container_of(buffer, struct rpc_buffer, data);
	free_page((unsigned long)buf);
}

/*
 * Use the svc_sock to send the callback. Must be called with svsk->sk_mutex
 * held. Borrows heavily from svc_tcp_sendto and xs_tcp_send_request.
 */
static int bc_sendto(struct rpc_rqst *req)
{
	int len;
	struct xdr_buf *xbufp = &req->rq_snd_buf;
	struct rpc_xprt *xprt = req->rq_xprt;
	struct sock_xprt *transport =
				container_of(xprt, struct sock_xprt, xprt);
	struct socket *sock = transport->sock;
	unsigned long headoff;
	unsigned long tailoff;

2380
	xs_encode_stream_record_marker(xbufp);
2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447

	tailoff = (unsigned long)xbufp->tail[0].iov_base & ~PAGE_MASK;
	headoff = (unsigned long)xbufp->head[0].iov_base & ~PAGE_MASK;
	len = svc_send_common(sock, xbufp,
			      virt_to_page(xbufp->head[0].iov_base), headoff,
			      xbufp->tail[0].iov_base, tailoff);

	if (len != xbufp->len) {
		printk(KERN_NOTICE "Error sending entire callback!\n");
		len = -EAGAIN;
	}

	return len;
}

/*
 * The send routine. Borrows from svc_send
 */
static int bc_send_request(struct rpc_task *task)
{
	struct rpc_rqst *req = task->tk_rqstp;
	struct svc_xprt	*xprt;
	u32                     len;

	dprintk("sending request with xid: %08x\n", ntohl(req->rq_xid));
	/*
	 * Get the server socket associated with this callback xprt
	 */
	xprt = req->rq_xprt->bc_xprt;

	/*
	 * Grab the mutex to serialize data as the connection is shared
	 * with the fore channel
	 */
	if (!mutex_trylock(&xprt->xpt_mutex)) {
		rpc_sleep_on(&xprt->xpt_bc_pending, task, NULL);
		if (!mutex_trylock(&xprt->xpt_mutex))
			return -EAGAIN;
		rpc_wake_up_queued_task(&xprt->xpt_bc_pending, task);
	}
	if (test_bit(XPT_DEAD, &xprt->xpt_flags))
		len = -ENOTCONN;
	else
		len = bc_sendto(req);
	mutex_unlock(&xprt->xpt_mutex);

	if (len > 0)
		len = 0;

	return len;
}

/*
 * The close routine. Since this is client initiated, we do nothing
 */

static void bc_close(struct rpc_xprt *xprt)
{
}

/*
 * The xprt destroy routine. Again, because this connection is client
 * initiated, we do nothing
 */

static void bc_destroy(struct rpc_xprt *xprt)
{
2448 2449 2450 2451
	dprintk("RPC:       bc_destroy xprt %p\n", xprt);

	xs_xprt_free(xprt);
	module_put(THIS_MODULE);
2452 2453
}

2454 2455 2456
static struct rpc_xprt_ops xs_local_ops = {
	.reserve_xprt		= xprt_reserve_xprt,
	.release_xprt		= xs_tcp_release_xprt,
2457
	.alloc_slot		= xprt_alloc_slot,
2458 2459
	.rpcbind		= xs_local_rpcbind,
	.set_port		= xs_local_set_port,
2460
	.connect		= xs_local_connect,
2461 2462 2463 2464 2465
	.buf_alloc		= rpc_malloc,
	.buf_free		= rpc_free,
	.send_request		= xs_local_send_request,
	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
	.close			= xs_close,
T
Trond Myklebust 已提交
2466
	.destroy		= xs_destroy,
2467 2468 2469
	.print_stats		= xs_local_print_stats,
};

2470
static struct rpc_xprt_ops xs_udp_ops = {
2471
	.set_buffer_size	= xs_udp_set_buffer_size,
2472
	.reserve_xprt		= xprt_reserve_xprt_cong,
2473
	.release_xprt		= xprt_release_xprt_cong,
2474
	.alloc_slot		= xprt_alloc_slot,
2475
	.rpcbind		= rpcb_getport_async,
2476
	.set_port		= xs_set_port,
2477
	.connect		= xs_connect,
2478 2479
	.buf_alloc		= rpc_malloc,
	.buf_free		= rpc_free,
2480
	.send_request		= xs_udp_send_request,
2481
	.set_retrans_timeout	= xprt_set_retrans_timeout_rtt,
2482
	.timer			= xs_udp_timer,
2483
	.release_request	= xprt_release_rqst_cong,
2484 2485
	.close			= xs_close,
	.destroy		= xs_destroy,
2486
	.print_stats		= xs_udp_print_stats,
2487 2488 2489
};

static struct rpc_xprt_ops xs_tcp_ops = {
2490
	.reserve_xprt		= xprt_reserve_xprt,
2491
	.release_xprt		= xs_tcp_release_xprt,
2492
	.alloc_slot		= xprt_lock_and_alloc_slot,
2493
	.rpcbind		= rpcb_getport_async,
2494
	.set_port		= xs_set_port,
2495
	.connect		= xs_connect,
2496 2497
	.buf_alloc		= rpc_malloc,
	.buf_free		= rpc_free,
2498
	.send_request		= xs_tcp_send_request,
2499
	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
2500
	.close			= xs_tcp_close,
2501
	.destroy		= xs_destroy,
2502
	.print_stats		= xs_tcp_print_stats,
2503 2504
};

2505 2506 2507 2508 2509 2510 2511
/*
 * The rpc_xprt_ops for the server backchannel
 */

static struct rpc_xprt_ops bc_tcp_ops = {
	.reserve_xprt		= xprt_reserve_xprt,
	.release_xprt		= xprt_release_xprt,
2512
	.alloc_slot		= xprt_alloc_slot,
2513 2514 2515 2516 2517 2518 2519 2520 2521
	.buf_alloc		= bc_malloc,
	.buf_free		= bc_free,
	.send_request		= bc_send_request,
	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
	.close			= bc_close,
	.destroy		= bc_destroy,
	.print_stats		= xs_tcp_print_stats,
};

2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533
static int xs_init_anyaddr(const int family, struct sockaddr *sap)
{
	static const struct sockaddr_in sin = {
		.sin_family		= AF_INET,
		.sin_addr.s_addr	= htonl(INADDR_ANY),
	};
	static const struct sockaddr_in6 sin6 = {
		.sin6_family		= AF_INET6,
		.sin6_addr		= IN6ADDR_ANY_INIT,
	};

	switch (family) {
2534 2535
	case AF_LOCAL:
		break;
2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548
	case AF_INET:
		memcpy(sap, &sin, sizeof(sin));
		break;
	case AF_INET6:
		memcpy(sap, &sin6, sizeof(sin6));
		break;
	default:
		dprintk("RPC:       %s: Bad address family\n", __func__);
		return -EAFNOSUPPORT;
	}
	return 0;
}

2549
static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args,
2550 2551
				      unsigned int slot_table_size,
				      unsigned int max_slot_table_size)
2552 2553
{
	struct rpc_xprt *xprt;
2554
	struct sock_xprt *new;
2555

2556
	if (args->addrlen > sizeof(xprt->addr)) {
2557
		dprintk("RPC:       xs_setup_xprt: address too large\n");
2558 2559 2560
		return ERR_PTR(-EBADF);
	}

2561 2562
	xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size,
			max_slot_table_size);
2563
	if (xprt == NULL) {
2564 2565
		dprintk("RPC:       xs_setup_xprt: couldn't allocate "
				"rpc_xprt\n");
2566 2567 2568
		return ERR_PTR(-ENOMEM);
	}

2569
	new = container_of(xprt, struct sock_xprt, xprt);
2570 2571
	memcpy(&xprt->addr, args->dstaddr, args->addrlen);
	xprt->addrlen = args->addrlen;
2572
	if (args->srcaddr)
2573
		memcpy(&new->srcaddr, args->srcaddr, args->addrlen);
2574 2575 2576 2577
	else {
		int err;
		err = xs_init_anyaddr(args->dstaddr->sa_family,
					(struct sockaddr *)&new->srcaddr);
2578 2579
		if (err != 0) {
			xprt_free(xprt);
2580
			return ERR_PTR(err);
2581
		}
2582
	}
2583 2584 2585 2586

	return xprt;
}

2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605
static const struct rpc_timeout xs_local_default_timeout = {
	.to_initval = 10 * HZ,
	.to_maxval = 10 * HZ,
	.to_retries = 2,
};

/**
 * xs_setup_local - Set up transport to use an AF_LOCAL socket
 * @args: rpc transport creation arguments
 *
 * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP
 */
static struct rpc_xprt *xs_setup_local(struct xprt_create *args)
{
	struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr;
	struct sock_xprt *transport;
	struct rpc_xprt *xprt;
	struct rpc_xprt *ret;

2606 2607
	xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
			xprt_max_tcp_slot_table_entries);
2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
	if (IS_ERR(xprt))
		return xprt;
	transport = container_of(xprt, struct sock_xprt, xprt);

	xprt->prot = 0;
	xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32);
	xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;

	xprt->bind_timeout = XS_BIND_TO;
	xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
	xprt->idle_timeout = XS_IDLE_DISC_TO;

	xprt->ops = &xs_local_ops;
	xprt->timeout = &xs_local_default_timeout;

2623 2624 2625
	INIT_DELAYED_WORK(&transport->connect_worker,
			xs_dummy_setup_socket);

2626 2627 2628 2629 2630 2631 2632 2633 2634 2635
	switch (sun->sun_family) {
	case AF_LOCAL:
		if (sun->sun_path[0] != '/') {
			dprintk("RPC:       bad AF_LOCAL address: %s\n",
					sun->sun_path);
			ret = ERR_PTR(-EINVAL);
			goto out_err;
		}
		xprt_set_bound(xprt);
		xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL);
2636 2637 2638
		ret = ERR_PTR(xs_local_setup_socket(transport));
		if (ret)
			goto out_err;
2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651
		break;
	default:
		ret = ERR_PTR(-EAFNOSUPPORT);
		goto out_err;
	}

	dprintk("RPC:       set up xprt to %s via AF_LOCAL\n",
			xprt->address_strings[RPC_DISPLAY_ADDR]);

	if (try_module_get(THIS_MODULE))
		return xprt;
	ret = ERR_PTR(-EINVAL);
out_err:
2652
	xs_xprt_free(xprt);
2653 2654 2655
	return ret;
}

2656 2657 2658 2659 2660 2661 2662
static const struct rpc_timeout xs_udp_default_timeout = {
	.to_initval = 5 * HZ,
	.to_maxval = 30 * HZ,
	.to_increment = 5 * HZ,
	.to_retries = 5,
};

2663 2664
/**
 * xs_setup_udp - Set up transport to use a UDP socket
2665
 * @args: rpc transport creation arguments
2666 2667
 *
 */
2668
static struct rpc_xprt *xs_setup_udp(struct xprt_create *args)
2669
{
2670
	struct sockaddr *addr = args->dstaddr;
2671
	struct rpc_xprt *xprt;
2672
	struct sock_xprt *transport;
2673
	struct rpc_xprt *ret;
2674

2675 2676
	xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries,
			xprt_udp_slot_table_entries);
2677 2678
	if (IS_ERR(xprt))
		return xprt;
2679
	transport = container_of(xprt, struct sock_xprt, xprt);
2680

2681
	xprt->prot = IPPROTO_UDP;
2682
	xprt->tsh_size = 0;
2683 2684 2685
	/* XXX: header size can vary due to auth type, IPv6, etc. */
	xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);

2686 2687 2688
	xprt->bind_timeout = XS_BIND_TO;
	xprt->reestablish_timeout = XS_UDP_REEST_TO;
	xprt->idle_timeout = XS_IDLE_DISC_TO;
2689

2690
	xprt->ops = &xs_udp_ops;
2691

2692
	xprt->timeout = &xs_udp_default_timeout;
2693

2694 2695 2696 2697 2698 2699
	switch (addr->sa_family) {
	case AF_INET:
		if (((struct sockaddr_in *)addr)->sin_port != htons(0))
			xprt_set_bound(xprt);

		INIT_DELAYED_WORK(&transport->connect_worker,
2700
					xs_udp_setup_socket);
2701
		xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP);
2702 2703 2704 2705 2706 2707
		break;
	case AF_INET6:
		if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
			xprt_set_bound(xprt);

		INIT_DELAYED_WORK(&transport->connect_worker,
2708
					xs_udp_setup_socket);
2709
		xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6);
2710 2711
		break;
	default:
2712 2713
		ret = ERR_PTR(-EAFNOSUPPORT);
		goto out_err;
2714 2715
	}

C
Chuck Lever 已提交
2716 2717 2718 2719 2720 2721 2722 2723 2724
	if (xprt_bound(xprt))
		dprintk("RPC:       set up xprt to %s (port %s) via %s\n",
				xprt->address_strings[RPC_DISPLAY_ADDR],
				xprt->address_strings[RPC_DISPLAY_PORT],
				xprt->address_strings[RPC_DISPLAY_PROTO]);
	else
		dprintk("RPC:       set up xprt to %s (autobind) via %s\n",
				xprt->address_strings[RPC_DISPLAY_ADDR],
				xprt->address_strings[RPC_DISPLAY_PROTO]);
2725

2726 2727
	if (try_module_get(THIS_MODULE))
		return xprt;
2728 2729
	ret = ERR_PTR(-EINVAL);
out_err:
2730
	xs_xprt_free(xprt);
2731
	return ret;
2732 2733
}

2734 2735 2736 2737 2738 2739
static const struct rpc_timeout xs_tcp_default_timeout = {
	.to_initval = 60 * HZ,
	.to_maxval = 60 * HZ,
	.to_retries = 2,
};

2740 2741
/**
 * xs_setup_tcp - Set up transport to use a TCP socket
2742
 * @args: rpc transport creation arguments
2743 2744
 *
 */
2745
static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args)
2746
{
2747
	struct sockaddr *addr = args->dstaddr;
2748
	struct rpc_xprt *xprt;
2749
	struct sock_xprt *transport;
2750
	struct rpc_xprt *ret;
2751 2752 2753 2754
	unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries;

	if (args->flags & XPRT_CREATE_INFINITE_SLOTS)
		max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT;
2755

2756
	xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
2757
			max_slot_table_size);
2758 2759
	if (IS_ERR(xprt))
		return xprt;
2760
	transport = container_of(xprt, struct sock_xprt, xprt);
2761

2762
	xprt->prot = IPPROTO_TCP;
2763 2764
	xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32);
	xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
2765

2766 2767 2768
	xprt->bind_timeout = XS_BIND_TO;
	xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
	xprt->idle_timeout = XS_IDLE_DISC_TO;
2769

2770
	xprt->ops = &xs_tcp_ops;
2771
	xprt->timeout = &xs_tcp_default_timeout;
2772

2773 2774 2775 2776 2777
	switch (addr->sa_family) {
	case AF_INET:
		if (((struct sockaddr_in *)addr)->sin_port != htons(0))
			xprt_set_bound(xprt);

2778
		INIT_DELAYED_WORK(&transport->connect_worker,
2779
					xs_tcp_setup_socket);
2780
		xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP);
2781 2782 2783 2784 2785
		break;
	case AF_INET6:
		if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
			xprt_set_bound(xprt);

2786
		INIT_DELAYED_WORK(&transport->connect_worker,
2787
					xs_tcp_setup_socket);
2788
		xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6);
2789 2790
		break;
	default:
2791 2792
		ret = ERR_PTR(-EAFNOSUPPORT);
		goto out_err;
2793 2794
	}

C
Chuck Lever 已提交
2795 2796 2797 2798 2799 2800 2801 2802 2803 2804
	if (xprt_bound(xprt))
		dprintk("RPC:       set up xprt to %s (port %s) via %s\n",
				xprt->address_strings[RPC_DISPLAY_ADDR],
				xprt->address_strings[RPC_DISPLAY_PORT],
				xprt->address_strings[RPC_DISPLAY_PROTO]);
	else
		dprintk("RPC:       set up xprt to %s (autobind) via %s\n",
				xprt->address_strings[RPC_DISPLAY_ADDR],
				xprt->address_strings[RPC_DISPLAY_PROTO]);

2805 2806
	if (try_module_get(THIS_MODULE))
		return xprt;
2807 2808
	ret = ERR_PTR(-EINVAL);
out_err:
2809
	xs_xprt_free(xprt);
2810
	return ret;
2811
}
2812

2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823
/**
 * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket
 * @args: rpc transport creation arguments
 *
 */
static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args)
{
	struct sockaddr *addr = args->dstaddr;
	struct rpc_xprt *xprt;
	struct sock_xprt *transport;
	struct svc_sock *bc_sock;
2824
	struct rpc_xprt *ret;
2825

2826 2827
	xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
			xprt_tcp_slot_table_entries);
2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854
	if (IS_ERR(xprt))
		return xprt;
	transport = container_of(xprt, struct sock_xprt, xprt);

	xprt->prot = IPPROTO_TCP;
	xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32);
	xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
	xprt->timeout = &xs_tcp_default_timeout;

	/* backchannel */
	xprt_set_bound(xprt);
	xprt->bind_timeout = 0;
	xprt->reestablish_timeout = 0;
	xprt->idle_timeout = 0;

	xprt->ops = &bc_tcp_ops;

	switch (addr->sa_family) {
	case AF_INET:
		xs_format_peer_addresses(xprt, "tcp",
					 RPCBIND_NETID_TCP);
		break;
	case AF_INET6:
		xs_format_peer_addresses(xprt, "tcp",
				   RPCBIND_NETID_TCP6);
		break;
	default:
2855 2856
		ret = ERR_PTR(-EAFNOSUPPORT);
		goto out_err;
2857 2858
	}

2859 2860 2861 2862
	dprintk("RPC:       set up xprt to %s (port %s) via %s\n",
			xprt->address_strings[RPC_DISPLAY_ADDR],
			xprt->address_strings[RPC_DISPLAY_PORT],
			xprt->address_strings[RPC_DISPLAY_PROTO]);
2863

2864 2865
	/*
	 * Once we've associated a backchannel xprt with a connection,
W
Weng Meiling 已提交
2866 2867 2868
	 * we want to keep it around as long as the connection lasts,
	 * in case we need to start using it for a backchannel again;
	 * this reference won't be dropped until bc_xprt is destroyed.
2869 2870 2871 2872 2873 2874 2875 2876
	 */
	xprt_get(xprt);
	args->bc_xprt->xpt_bc_xprt = xprt;
	xprt->bc_xprt = args->bc_xprt;
	bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt);
	transport->sock = bc_sock->sk_sock;
	transport->inet = bc_sock->sk_sk;

2877 2878 2879 2880 2881 2882 2883 2884
	/*
	 * Since we don't want connections for the backchannel, we set
	 * the xprt status to connected
	 */
	xprt_set_connected(xprt);

	if (try_module_get(THIS_MODULE))
		return xprt;
2885 2886

	args->bc_xprt->xpt_bc_xprt = NULL;
2887
	xprt_put(xprt);
2888 2889
	ret = ERR_PTR(-EINVAL);
out_err:
2890
	xs_xprt_free(xprt);
2891
	return ret;
2892 2893
}

2894 2895 2896 2897 2898 2899 2900 2901
static struct xprt_class	xs_local_transport = {
	.list		= LIST_HEAD_INIT(xs_local_transport.list),
	.name		= "named UNIX socket",
	.owner		= THIS_MODULE,
	.ident		= XPRT_TRANSPORT_LOCAL,
	.setup		= xs_setup_local,
};

2902 2903 2904 2905
static struct xprt_class	xs_udp_transport = {
	.list		= LIST_HEAD_INIT(xs_udp_transport.list),
	.name		= "udp",
	.owner		= THIS_MODULE,
2906
	.ident		= XPRT_TRANSPORT_UDP,
2907 2908 2909 2910 2911 2912 2913
	.setup		= xs_setup_udp,
};

static struct xprt_class	xs_tcp_transport = {
	.list		= LIST_HEAD_INIT(xs_tcp_transport.list),
	.name		= "tcp",
	.owner		= THIS_MODULE,
2914
	.ident		= XPRT_TRANSPORT_TCP,
2915 2916 2917
	.setup		= xs_setup_tcp,
};

2918 2919 2920 2921 2922 2923 2924 2925
static struct xprt_class	xs_bc_tcp_transport = {
	.list		= LIST_HEAD_INIT(xs_bc_tcp_transport.list),
	.name		= "tcp NFSv4.1 backchannel",
	.owner		= THIS_MODULE,
	.ident		= XPRT_TRANSPORT_BC_TCP,
	.setup		= xs_setup_bc_tcp,
};

2926
/**
2927
 * init_socket_xprt - set up xprtsock's sysctls, register with RPC client
2928 2929 2930 2931
 *
 */
int init_socket_xprt(void)
{
J
Jeff Layton 已提交
2932
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
2933
	if (!sunrpc_table_header)
2934
		sunrpc_table_header = register_sysctl_table(sunrpc_table);
2935 2936
#endif

2937
	xprt_register_transport(&xs_local_transport);
2938 2939
	xprt_register_transport(&xs_udp_transport);
	xprt_register_transport(&xs_tcp_transport);
2940
	xprt_register_transport(&xs_bc_tcp_transport);
2941

2942 2943 2944 2945
	return 0;
}

/**
2946
 * cleanup_socket_xprt - remove xprtsock's sysctls, unregister
2947 2948 2949 2950
 *
 */
void cleanup_socket_xprt(void)
{
J
Jeff Layton 已提交
2951
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
2952 2953 2954 2955 2956
	if (sunrpc_table_header) {
		unregister_sysctl_table(sunrpc_table_header);
		sunrpc_table_header = NULL;
	}
#endif
2957

2958
	xprt_unregister_transport(&xs_local_transport);
2959 2960
	xprt_unregister_transport(&xs_udp_transport);
	xprt_unregister_transport(&xs_tcp_transport);
2961
	xprt_unregister_transport(&xs_bc_tcp_transport);
2962
}
2963

2964 2965
static int param_set_uint_minmax(const char *val,
		const struct kernel_param *kp,
2966 2967
		unsigned int min, unsigned int max)
{
D
Daniel Walter 已提交
2968
	unsigned int num;
2969 2970 2971 2972
	int ret;

	if (!val)
		return -EINVAL;
D
Daniel Walter 已提交
2973
	ret = kstrtouint(val, 0, &num);
2974 2975 2976 2977 2978 2979
	if (ret == -EINVAL || num < min || num > max)
		return -EINVAL;
	*((unsigned int *)kp->arg) = num;
	return 0;
}

2980
static int param_set_portnr(const char *val, const struct kernel_param *kp)
2981 2982 2983 2984 2985 2986
{
	return param_set_uint_minmax(val, kp,
			RPC_MIN_RESVPORT,
			RPC_MAX_RESVPORT);
}

2987 2988 2989 2990 2991
static struct kernel_param_ops param_ops_portnr = {
	.set = param_set_portnr,
	.get = param_get_uint,
};

2992 2993 2994 2995 2996 2997
#define param_check_portnr(name, p) \
	__param_check(name, p, unsigned int);

module_param_named(min_resvport, xprt_min_resvport, portnr, 0644);
module_param_named(max_resvport, xprt_max_resvport, portnr, 0644);

2998 2999
static int param_set_slot_table_size(const char *val,
				     const struct kernel_param *kp)
3000 3001 3002 3003 3004 3005
{
	return param_set_uint_minmax(val, kp,
			RPC_MIN_SLOT_TABLE,
			RPC_MAX_SLOT_TABLE);
}

3006 3007 3008 3009 3010
static struct kernel_param_ops param_ops_slot_table_size = {
	.set = param_set_slot_table_size,
	.get = param_get_uint,
};

3011 3012 3013
#define param_check_slot_table_size(name, p) \
	__param_check(name, p, unsigned int);

3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029
static int param_set_max_slot_table_size(const char *val,
				     const struct kernel_param *kp)
{
	return param_set_uint_minmax(val, kp,
			RPC_MIN_SLOT_TABLE,
			RPC_MAX_SLOT_TABLE_LIMIT);
}

static struct kernel_param_ops param_ops_max_slot_table_size = {
	.set = param_set_max_slot_table_size,
	.get = param_get_uint,
};

#define param_check_max_slot_table_size(name, p) \
	__param_check(name, p, unsigned int);

3030 3031
module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries,
		   slot_table_size, 0644);
3032 3033
module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries,
		   max_slot_table_size, 0644);
3034 3035 3036
module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries,
		   slot_table_size, 0644);