xprt.c 29.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12
/*
 *  linux/net/sunrpc/xprt.c
 *
 *  This is a generic RPC call interface supporting congestion avoidance,
 *  and asynchronous calls.
 *
 *  The interface works like this:
 *
 *  -	When a process places a call, it allocates a request slot if
 *	one is available. Otherwise, it sleeps on the backlog queue
 *	(xprt_reserve).
 *  -	Next, the caller puts together the RPC message, stuffs it into
13 14 15
 *	the request struct, and calls xprt_transmit().
 *  -	xprt_transmit sends the message and installs the caller on the
 *	transport's wait list. At the same time, it installs a timer that
L
Linus Torvalds 已提交
16 17
 *	is run after the packet's timeout has expired.
 *  -	When a packet arrives, the data_ready handler walks the list of
18
 *	pending requests for that transport. If a matching XID is found, the
L
Linus Torvalds 已提交
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
 *	caller is woken up, and the timer removed.
 *  -	When no reply arrives within the timeout interval, the timer is
 *	fired by the kernel and runs xprt_timer(). It either adjusts the
 *	timeout values (minor timeout) or wakes up the caller with a status
 *	of -ETIMEDOUT.
 *  -	When the caller receives a notification from RPC that a reply arrived,
 *	it should release the RPC slot, and process the reply.
 *	If the call timed out, it may choose to retry the operation by
 *	adjusting the initial timeout value, and simply calling rpc_call
 *	again.
 *
 *  Support for async RPC is done through a set of RPC-specific scheduling
 *  primitives that `transparently' work for processes as well as async
 *  tasks that rely on callbacks.
 *
 *  Copyright (C) 1995-1997, Olaf Kirch <okir@monad.swb.de>
35 36
 *
 *  Transport switch API copyright (C) 2005, Chuck Lever <cel@netapp.com>
L
Linus Torvalds 已提交
37 38
 */

39 40
#include <linux/module.h>

L
Linus Torvalds 已提交
41
#include <linux/types.h>
42
#include <linux/interrupt.h>
L
Linus Torvalds 已提交
43
#include <linux/workqueue.h>
44
#include <linux/net.h>
L
Linus Torvalds 已提交
45

46
#include <linux/sunrpc/clnt.h>
47
#include <linux/sunrpc/metrics.h>
L
Linus Torvalds 已提交
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64

/*
 * Local variables
 */

#ifdef RPC_DEBUG
# define RPCDBG_FACILITY	RPCDBG_XPRT
#endif

/*
 * Local functions
 */
static void	xprt_request_init(struct rpc_task *, struct rpc_xprt *);
static inline void	do_xprt_reserve(struct rpc_task *);
static void	xprt_connect_status(struct rpc_task *task);
static int      __xprt_get_cong(struct rpc_xprt *, struct rpc_task *);

J
Jiri Slaby 已提交
65
static DEFINE_SPINLOCK(xprt_list_lock);
66 67
static LIST_HEAD(xprt_list);

68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
/*
 * The transport code maintains an estimate on the maximum number of out-
 * standing RPC requests, using a smoothed version of the congestion
 * avoidance implemented in 44BSD. This is basically the Van Jacobson
 * congestion algorithm: If a retransmit occurs, the congestion window is
 * halved; otherwise, it is incremented by 1/cwnd when
 *
 *	-	a reply is received and
 *	-	a full number of requests are outstanding and
 *	-	the congestion window hasn't been updated recently.
 */
#define RPC_CWNDSHIFT		(8U)
#define RPC_CWNDSCALE		(1U << RPC_CWNDSHIFT)
#define RPC_INITCWND		RPC_CWNDSCALE
#define RPC_MAXCWND(xprt)	((xprt)->max_reqs << RPC_CWNDSHIFT)

#define RPCXPRT_CONGESTED(xprt) ((xprt)->cong >= (xprt)->cwnd)
L
Linus Torvalds 已提交
85

86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
/**
 * xprt_register_transport - register a transport implementation
 * @transport: transport to register
 *
 * If a transport implementation is loaded as a kernel module, it can
 * call this interface to make itself known to the RPC client.
 *
 * Returns:
 * 0:		transport successfully registered
 * -EEXIST:	transport already registered
 * -EINVAL:	transport module being unloaded
 */
int xprt_register_transport(struct xprt_class *transport)
{
	struct xprt_class *t;
	int result;

	result = -EEXIST;
	spin_lock(&xprt_list_lock);
	list_for_each_entry(t, &xprt_list, list) {
		/* don't register the same transport class twice */
107
		if (t->ident == transport->ident)
108 109 110
			goto out;
	}

111 112 113 114
	list_add_tail(&transport->list, &xprt_list);
	printk(KERN_INFO "RPC: Registered %s transport module.\n",
	       transport->name);
	result = 0;
115 116 117 118 119 120 121 122 123

out:
	spin_unlock(&xprt_list_lock);
	return result;
}
EXPORT_SYMBOL_GPL(xprt_register_transport);

/**
 * xprt_unregister_transport - unregister a transport implementation
124
 * @transport: transport to unregister
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
 *
 * Returns:
 * 0:		transport successfully unregistered
 * -ENOENT:	transport never registered
 */
int xprt_unregister_transport(struct xprt_class *transport)
{
	struct xprt_class *t;
	int result;

	result = 0;
	spin_lock(&xprt_list_lock);
	list_for_each_entry(t, &xprt_list, list) {
		if (t == transport) {
			printk(KERN_INFO
				"RPC: Unregistered %s transport module.\n",
				transport->name);
			list_del_init(&transport->list);
			goto out;
		}
	}
	result = -ENOENT;

out:
	spin_unlock(&xprt_list_lock);
	return result;
}
EXPORT_SYMBOL_GPL(xprt_unregister_transport);

154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
/**
 * xprt_reserve_xprt - serialize write access to transports
 * @task: task that is requesting access to the transport
 *
 * This prevents mixing the payload of separate requests, and prevents
 * transport connects from colliding with writes.  No congestion control
 * is provided.
 */
int xprt_reserve_xprt(struct rpc_task *task)
{
	struct rpc_xprt	*xprt = task->tk_xprt;
	struct rpc_rqst *req = task->tk_rqstp;

	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
		if (task == xprt->snd_task)
			return 1;
		if (task == NULL)
			return 0;
		goto out_sleep;
	}
	xprt->snd_task = task;
	if (req) {
		req->rq_bytes_sent = 0;
		req->rq_ntrans++;
	}
	return 1;

out_sleep:
182
	dprintk("RPC: %5u failed to lock transport %p\n",
183 184 185 186
			task->tk_pid, xprt);
	task->tk_timeout = 0;
	task->tk_status = -EAGAIN;
	if (req && req->rq_ntrans)
187
		rpc_sleep_on(&xprt->resend, task, NULL);
188
	else
189
		rpc_sleep_on(&xprt->sending, task, NULL);
190 191
	return 0;
}
192
EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
193

194 195 196 197 198 199 200 201
static void xprt_clear_locked(struct rpc_xprt *xprt)
{
	xprt->snd_task = NULL;
	if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state) || xprt->shutdown) {
		smp_mb__before_clear_bit();
		clear_bit(XPRT_LOCKED, &xprt->state);
		smp_mb__after_clear_bit();
	} else
202
		queue_work(rpciod_workqueue, &xprt->task_cleanup);
203 204
}

L
Linus Torvalds 已提交
205
/*
206 207 208 209 210 211
 * xprt_reserve_xprt_cong - serialize write access to transports
 * @task: task that is requesting access to the transport
 *
 * Same as xprt_reserve_xprt, but Van Jacobson congestion control is
 * integrated into the decision of whether a request is allowed to be
 * woken up and given access to the transport.
L
Linus Torvalds 已提交
212
 */
213
int xprt_reserve_xprt_cong(struct rpc_task *task)
L
Linus Torvalds 已提交
214
{
215
	struct rpc_xprt	*xprt = task->tk_xprt;
L
Linus Torvalds 已提交
216 217
	struct rpc_rqst *req = task->tk_rqstp;

218
	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
L
Linus Torvalds 已提交
219 220 221 222
		if (task == xprt->snd_task)
			return 1;
		goto out_sleep;
	}
223
	if (__xprt_get_cong(xprt, task)) {
L
Linus Torvalds 已提交
224 225 226 227 228 229 230
		xprt->snd_task = task;
		if (req) {
			req->rq_bytes_sent = 0;
			req->rq_ntrans++;
		}
		return 1;
	}
231
	xprt_clear_locked(xprt);
L
Linus Torvalds 已提交
232
out_sleep:
233
	dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
L
Linus Torvalds 已提交
234 235 236
	task->tk_timeout = 0;
	task->tk_status = -EAGAIN;
	if (req && req->rq_ntrans)
237
		rpc_sleep_on(&xprt->resend, task, NULL);
L
Linus Torvalds 已提交
238
	else
239
		rpc_sleep_on(&xprt->sending, task, NULL);
L
Linus Torvalds 已提交
240 241
	return 0;
}
242
EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
L
Linus Torvalds 已提交
243

244
static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)
L
Linus Torvalds 已提交
245 246 247
{
	int retval;

C
Chuck Lever 已提交
248
	spin_lock_bh(&xprt->transport_lock);
249
	retval = xprt->ops->reserve_xprt(task);
C
Chuck Lever 已提交
250
	spin_unlock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
251 252 253
	return retval;
}

254
static void __xprt_lock_write_next(struct rpc_xprt *xprt)
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
{
	struct rpc_task *task;
	struct rpc_rqst *req;

	if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
		return;

	task = rpc_wake_up_next(&xprt->resend);
	if (!task) {
		task = rpc_wake_up_next(&xprt->sending);
		if (!task)
			goto out_unlock;
	}

	req = task->tk_rqstp;
	xprt->snd_task = task;
	if (req) {
		req->rq_bytes_sent = 0;
		req->rq_ntrans++;
	}
	return;

out_unlock:
278
	xprt_clear_locked(xprt);
279 280 281
}

static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt)
L
Linus Torvalds 已提交
282 283 284
{
	struct rpc_task *task;

285
	if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
L
Linus Torvalds 已提交
286
		return;
287
	if (RPCXPRT_CONGESTED(xprt))
L
Linus Torvalds 已提交
288 289 290 291 292 293 294
		goto out_unlock;
	task = rpc_wake_up_next(&xprt->resend);
	if (!task) {
		task = rpc_wake_up_next(&xprt->sending);
		if (!task)
			goto out_unlock;
	}
295
	if (__xprt_get_cong(xprt, task)) {
L
Linus Torvalds 已提交
296 297 298 299 300 301 302 303 304
		struct rpc_rqst *req = task->tk_rqstp;
		xprt->snd_task = task;
		if (req) {
			req->rq_bytes_sent = 0;
			req->rq_ntrans++;
		}
		return;
	}
out_unlock:
305
	xprt_clear_locked(xprt);
L
Linus Torvalds 已提交
306 307
}

308 309 310 311 312 313
/**
 * xprt_release_xprt - allow other requests to use a transport
 * @xprt: transport with other tasks potentially waiting
 * @task: task that is releasing access to the transport
 *
 * Note that "task" can be NULL.  No congestion control is provided.
L
Linus Torvalds 已提交
314
 */
315
void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
L
Linus Torvalds 已提交
316 317
{
	if (xprt->snd_task == task) {
318
		xprt_clear_locked(xprt);
L
Linus Torvalds 已提交
319 320 321
		__xprt_lock_write_next(xprt);
	}
}
322
EXPORT_SYMBOL_GPL(xprt_release_xprt);
L
Linus Torvalds 已提交
323

324 325 326 327 328 329 330 331 332 333 334
/**
 * xprt_release_xprt_cong - allow other requests to use a transport
 * @xprt: transport with other tasks potentially waiting
 * @task: task that is releasing access to the transport
 *
 * Note that "task" can be NULL.  Another task is awoken to use the
 * transport if the transport's congestion window allows it.
 */
void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
{
	if (xprt->snd_task == task) {
335
		xprt_clear_locked(xprt);
336 337 338
		__xprt_lock_write_next_cong(xprt);
	}
}
339
EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
340 341

static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)
L
Linus Torvalds 已提交
342
{
C
Chuck Lever 已提交
343
	spin_lock_bh(&xprt->transport_lock);
344
	xprt->ops->release_xprt(xprt, task);
C
Chuck Lever 已提交
345
	spin_unlock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
346 347 348 349 350 351 352 353 354 355 356 357 358
}

/*
 * Van Jacobson congestion avoidance. Check if the congestion window
 * overflowed. Put the task to sleep if this is the case.
 */
static int
__xprt_get_cong(struct rpc_xprt *xprt, struct rpc_task *task)
{
	struct rpc_rqst *req = task->tk_rqstp;

	if (req->rq_cong)
		return 1;
359
	dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n",
L
Linus Torvalds 已提交
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
			task->tk_pid, xprt->cong, xprt->cwnd);
	if (RPCXPRT_CONGESTED(xprt))
		return 0;
	req->rq_cong = 1;
	xprt->cong += RPC_CWNDSCALE;
	return 1;
}

/*
 * Adjust the congestion window, and wake up the next task
 * that has been sleeping due to congestion
 */
static void
__xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
{
	if (!req->rq_cong)
		return;
	req->rq_cong = 0;
	xprt->cong -= RPC_CWNDSCALE;
379
	__xprt_lock_write_next_cong(xprt);
L
Linus Torvalds 已提交
380 381
}

382 383 384 385 386 387 388 389 390 391
/**
 * xprt_release_rqst_cong - housekeeping when request is complete
 * @task: RPC request that recently completed
 *
 * Useful for transports that require congestion control.
 */
void xprt_release_rqst_cong(struct rpc_task *task)
{
	__xprt_put_cong(task->tk_xprt, task->tk_rqstp);
}
392
EXPORT_SYMBOL_GPL(xprt_release_rqst_cong);
393

394 395 396 397 398
/**
 * xprt_adjust_cwnd - adjust transport congestion window
 * @task: recently completed RPC request used to adjust window
 * @result: result code of completed RPC request
 *
L
Linus Torvalds 已提交
399 400
 * We use a time-smoothed congestion estimator to avoid heavy oscillation.
 */
401
void xprt_adjust_cwnd(struct rpc_task *task, int result)
L
Linus Torvalds 已提交
402
{
403 404 405
	struct rpc_rqst *req = task->tk_rqstp;
	struct rpc_xprt *xprt = task->tk_xprt;
	unsigned long cwnd = xprt->cwnd;
L
Linus Torvalds 已提交
406 407 408 409 410 411 412

	if (result >= 0 && cwnd <= xprt->cong) {
		/* The (cwnd >> 1) term makes sure
		 * the result gets rounded properly. */
		cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd;
		if (cwnd > RPC_MAXCWND(xprt))
			cwnd = RPC_MAXCWND(xprt);
413
		__xprt_lock_write_next_cong(xprt);
L
Linus Torvalds 已提交
414 415 416 417 418
	} else if (result == -ETIMEDOUT) {
		cwnd >>= 1;
		if (cwnd < RPC_CWNDSCALE)
			cwnd = RPC_CWNDSCALE;
	}
419
	dprintk("RPC:       cong %ld, cwnd was %ld, now %ld\n",
L
Linus Torvalds 已提交
420 421
			xprt->cong, xprt->cwnd, cwnd);
	xprt->cwnd = cwnd;
422
	__xprt_put_cong(xprt, req);
L
Linus Torvalds 已提交
423
}
424
EXPORT_SYMBOL_GPL(xprt_adjust_cwnd);
L
Linus Torvalds 已提交
425

426 427 428 429 430 431 432 433 434 435 436 437 438
/**
 * xprt_wake_pending_tasks - wake all tasks on a transport's pending queue
 * @xprt: transport with waiting tasks
 * @status: result code to plant in each task before waking it
 *
 */
void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status)
{
	if (status < 0)
		rpc_wake_up_status(&xprt->pending, status);
	else
		rpc_wake_up(&xprt->pending);
}
439
EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks);
440

441 442 443
/**
 * xprt_wait_for_buffer_space - wait for transport output buffer to clear
 * @task: task to be put to sleep
R
Randy Dunlap 已提交
444
 * @action: function pointer to be executed after wait
445
 */
446
void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)
447 448 449 450 451
{
	struct rpc_rqst *req = task->tk_rqstp;
	struct rpc_xprt *xprt = req->rq_xprt;

	task->tk_timeout = req->rq_timeout;
452
	rpc_sleep_on(&xprt->pending, task, action);
453
}
454
EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
455 456 457 458 459 460 461 462 463 464 465 466 467 468

/**
 * xprt_write_space - wake the task waiting for transport output buffer space
 * @xprt: transport with waiting tasks
 *
 * Can be called in a soft IRQ context, so xprt_write_space never sleeps.
 */
void xprt_write_space(struct rpc_xprt *xprt)
{
	if (unlikely(xprt->shutdown))
		return;

	spin_lock_bh(&xprt->transport_lock);
	if (xprt->snd_task) {
469 470
		dprintk("RPC:       write space: waking waiting task on "
				"xprt %p\n", xprt);
471
		rpc_wake_up_queued_task(&xprt->pending, xprt->snd_task);
472 473 474
	}
	spin_unlock_bh(&xprt->transport_lock);
}
475
EXPORT_SYMBOL_GPL(xprt_write_space);
476

477 478 479 480 481 482 483 484 485 486 487 488
/**
 * xprt_set_retrans_timeout_def - set a request's retransmit timeout
 * @task: task whose timeout is to be set
 *
 * Set a request's retransmit timeout based on the transport's
 * default timeout parameters.  Used by transports that don't adjust
 * the retransmit timeout based on round-trip time estimation.
 */
void xprt_set_retrans_timeout_def(struct rpc_task *task)
{
	task->tk_timeout = task->tk_rqstp->rq_timeout;
}
489
EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_def);
490 491 492 493

/*
 * xprt_set_retrans_timeout_rtt - set a request's retransmit timeout
 * @task: task whose timeout is to be set
494
 *
495 496 497 498 499
 * Set a request's retransmit timeout using the RTT estimator.
 */
void xprt_set_retrans_timeout_rtt(struct rpc_task *task)
{
	int timer = task->tk_msg.rpc_proc->p_timer;
500 501
	struct rpc_clnt *clnt = task->tk_client;
	struct rpc_rtt *rtt = clnt->cl_rtt;
502
	struct rpc_rqst *req = task->tk_rqstp;
503
	unsigned long max_timeout = clnt->cl_timeout->to_maxval;
504 505 506 507 508 509

	task->tk_timeout = rpc_calc_rto(rtt, timer);
	task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
	if (task->tk_timeout > max_timeout || task->tk_timeout == 0)
		task->tk_timeout = max_timeout;
}
510
EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_rtt);
511

L
Linus Torvalds 已提交
512 513
static void xprt_reset_majortimeo(struct rpc_rqst *req)
{
514
	const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
L
Linus Torvalds 已提交
515 516 517 518 519 520 521 522 523 524 525

	req->rq_majortimeo = req->rq_timeout;
	if (to->to_exponential)
		req->rq_majortimeo <<= to->to_retries;
	else
		req->rq_majortimeo += to->to_increment * to->to_retries;
	if (req->rq_majortimeo > to->to_maxval || req->rq_majortimeo == 0)
		req->rq_majortimeo = to->to_maxval;
	req->rq_majortimeo += jiffies;
}

526 527 528 529
/**
 * xprt_adjust_timeout - adjust timeout values for next retransmit
 * @req: RPC request containing parameters to use for the adjustment
 *
L
Linus Torvalds 已提交
530 531 532 533
 */
int xprt_adjust_timeout(struct rpc_rqst *req)
{
	struct rpc_xprt *xprt = req->rq_xprt;
534
	const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
L
Linus Torvalds 已提交
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
	int status = 0;

	if (time_before(jiffies, req->rq_majortimeo)) {
		if (to->to_exponential)
			req->rq_timeout <<= 1;
		else
			req->rq_timeout += to->to_increment;
		if (to->to_maxval && req->rq_timeout >= to->to_maxval)
			req->rq_timeout = to->to_maxval;
		req->rq_retries++;
	} else {
		req->rq_timeout = to->to_initval;
		req->rq_retries = 0;
		xprt_reset_majortimeo(req);
		/* Reset the RTT counters == "slow start" */
C
Chuck Lever 已提交
550
		spin_lock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
551
		rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval);
C
Chuck Lever 已提交
552
		spin_unlock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
553 554 555 556 557 558 559 560 561 562
		status = -ETIMEDOUT;
	}

	if (req->rq_timeout == 0) {
		printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0!\n");
		req->rq_timeout = 5 * HZ;
	}
	return status;
}

563
static void xprt_autoclose(struct work_struct *work)
L
Linus Torvalds 已提交
564
{
565 566
	struct rpc_xprt *xprt =
		container_of(work, struct rpc_xprt, task_cleanup);
L
Linus Torvalds 已提交
567

568
	xprt->ops->close(xprt);
569
	clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
L
Linus Torvalds 已提交
570 571 572
	xprt_release_write(xprt, NULL);
}

573
/**
574
 * xprt_disconnect_done - mark a transport as disconnected
575 576
 * @xprt: transport to flag for disconnect
 *
L
Linus Torvalds 已提交
577
 */
578
void xprt_disconnect_done(struct rpc_xprt *xprt)
L
Linus Torvalds 已提交
579
{
580
	dprintk("RPC:       disconnected transport %p\n", xprt);
C
Chuck Lever 已提交
581
	spin_lock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
582
	xprt_clear_connected(xprt);
583
	xprt_wake_pending_tasks(xprt, -ENOTCONN);
C
Chuck Lever 已提交
584
	spin_unlock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
585
}
586
EXPORT_SYMBOL_GPL(xprt_disconnect_done);
L
Linus Torvalds 已提交
587

588 589 590 591 592 593 594 595 596 597 598 599 600
/**
 * xprt_force_disconnect - force a transport to disconnect
 * @xprt: transport to disconnect
 *
 */
void xprt_force_disconnect(struct rpc_xprt *xprt)
{
	/* Don't race with the test_bit() in xprt_clear_locked() */
	spin_lock_bh(&xprt->transport_lock);
	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
	/* Try to schedule an autoclose RPC call */
	if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
		queue_work(rpciod_workqueue, &xprt->task_cleanup);
601
	xprt_wake_pending_tasks(xprt, -ENOTCONN);
602 603 604
	spin_unlock_bh(&xprt->transport_lock);
}

605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
/**
 * xprt_conditional_disconnect - force a transport to disconnect
 * @xprt: transport to disconnect
 * @cookie: 'connection cookie'
 *
 * This attempts to break the connection if and only if 'cookie' matches
 * the current transport 'connection cookie'. It ensures that we don't
 * try to break the connection more than once when we need to retransmit
 * a batch of RPC requests.
 *
 */
void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
{
	/* Don't race with the test_bit() in xprt_clear_locked() */
	spin_lock_bh(&xprt->transport_lock);
	if (cookie != xprt->connect_cookie)
		goto out;
	if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
		goto out;
	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
	/* Try to schedule an autoclose RPC call */
	if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
		queue_work(rpciod_workqueue, &xprt->task_cleanup);
	xprt_wake_pending_tasks(xprt, -ENOTCONN);
out:
	spin_unlock_bh(&xprt->transport_lock);
}

L
Linus Torvalds 已提交
633 634 635 636 637
static void
xprt_init_autodisconnect(unsigned long data)
{
	struct rpc_xprt *xprt = (struct rpc_xprt *)data;

C
Chuck Lever 已提交
638
	spin_lock(&xprt->transport_lock);
L
Linus Torvalds 已提交
639 640
	if (!list_empty(&xprt->recv) || xprt->shutdown)
		goto out_abort;
641
	if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
L
Linus Torvalds 已提交
642
		goto out_abort;
C
Chuck Lever 已提交
643
	spin_unlock(&xprt->transport_lock);
644
	if (xprt_connecting(xprt))
L
Linus Torvalds 已提交
645 646
		xprt_release_write(xprt, NULL);
	else
647
		queue_work(rpciod_workqueue, &xprt->task_cleanup);
L
Linus Torvalds 已提交
648 649
	return;
out_abort:
C
Chuck Lever 已提交
650
	spin_unlock(&xprt->transport_lock);
L
Linus Torvalds 已提交
651 652
}

653 654 655
/**
 * xprt_connect - schedule a transport connect operation
 * @task: RPC task that is requesting the connect
L
Linus Torvalds 已提交
656 657 658 659 660 661
 *
 */
void xprt_connect(struct rpc_task *task)
{
	struct rpc_xprt	*xprt = task->tk_xprt;

662
	dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid,
L
Linus Torvalds 已提交
663 664
			xprt, (xprt_connected(xprt) ? "is" : "is not"));

665
	if (!xprt_bound(xprt)) {
666
		task->tk_status = -EAGAIN;
L
Linus Torvalds 已提交
667 668 669 670 671
		return;
	}
	if (!xprt_lock_write(xprt, task))
		return;
	if (xprt_connected(xprt))
672 673 674 675
		xprt_release_write(xprt, task);
	else {
		if (task->tk_rqstp)
			task->tk_rqstp->rq_bytes_sent = 0;
L
Linus Torvalds 已提交
676

677
		task->tk_timeout = xprt->connect_timeout;
678
		rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
679
		xprt->stat.connect_start = jiffies;
680
		xprt->ops->connect(task);
L
Linus Torvalds 已提交
681 682 683 684
	}
	return;
}

685
static void xprt_connect_status(struct rpc_task *task)
L
Linus Torvalds 已提交
686 687 688
{
	struct rpc_xprt	*xprt = task->tk_xprt;

689
	if (task->tk_status == 0) {
690 691
		xprt->stat.connect_count++;
		xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start;
692
		dprintk("RPC: %5u xprt_connect_status: connection established\n",
L
Linus Torvalds 已提交
693 694 695 696 697 698
				task->tk_pid);
		return;
	}

	switch (task->tk_status) {
	case -ENOTCONN:
699
		dprintk("RPC: %5u xprt_connect_status: connection broken\n",
700 701
				task->tk_pid);
		break;
L
Linus Torvalds 已提交
702
	case -ETIMEDOUT:
703 704
		dprintk("RPC: %5u xprt_connect_status: connect attempt timed "
				"out\n", task->tk_pid);
L
Linus Torvalds 已提交
705 706
		break;
	default:
707 708 709
		dprintk("RPC: %5u xprt_connect_status: error %d connecting to "
				"server %s\n", task->tk_pid, -task->tk_status,
				task->tk_client->cl_server);
710 711
		xprt_release_write(xprt, task);
		task->tk_status = -EIO;
L
Linus Torvalds 已提交
712 713 714
	}
}

715 716 717 718 719
/**
 * xprt_lookup_rqst - find an RPC request corresponding to an XID
 * @xprt: transport on which the original request was transmitted
 * @xid: RPC XID of incoming reply
 *
L
Linus Torvalds 已提交
720
 */
721
struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
L
Linus Torvalds 已提交
722 723 724 725 726
{
	struct list_head *pos;

	list_for_each(pos, &xprt->recv) {
		struct rpc_rqst *entry = list_entry(pos, struct rpc_rqst, rq_list);
727 728
		if (entry->rq_xid == xid)
			return entry;
L
Linus Torvalds 已提交
729
	}
730 731 732

	dprintk("RPC:       xprt_lookup_rqst did not find xid %08x\n",
			ntohl(xid));
733 734
	xprt->stat.bad_xids++;
	return NULL;
L
Linus Torvalds 已提交
735
}
736
EXPORT_SYMBOL_GPL(xprt_lookup_rqst);
L
Linus Torvalds 已提交
737

738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
/**
 * xprt_update_rtt - update an RPC client's RTT state after receiving a reply
 * @task: RPC request that recently completed
 *
 */
void xprt_update_rtt(struct rpc_task *task)
{
	struct rpc_rqst *req = task->tk_rqstp;
	struct rpc_rtt *rtt = task->tk_client->cl_rtt;
	unsigned timer = task->tk_msg.rpc_proc->p_timer;

	if (timer) {
		if (req->rq_ntrans == 1)
			rpc_update_rtt(rtt, timer,
					(long)jiffies - req->rq_xtime);
		rpc_set_timeo(rtt, timer, req->rq_ntrans - 1);
	}
}
756
EXPORT_SYMBOL_GPL(xprt_update_rtt);
757

758 759
/**
 * xprt_complete_rqst - called when reply processing is complete
760
 * @task: RPC request that recently completed
761 762
 * @copied: actual number of bytes received from the transport
 *
763
 * Caller holds transport lock.
L
Linus Torvalds 已提交
764
 */
765
void xprt_complete_rqst(struct rpc_task *task, int copied)
L
Linus Torvalds 已提交
766
{
767
	struct rpc_rqst *req = task->tk_rqstp;
768
	struct rpc_xprt *xprt = req->rq_xprt;
L
Linus Torvalds 已提交
769

770 771
	dprintk("RPC: %5u xid %08x complete (%d bytes received)\n",
			task->tk_pid, ntohl(req->rq_xid), copied);
L
Linus Torvalds 已提交
772

773
	xprt->stat.recvs++;
774 775
	task->tk_rtt = (long)jiffies - req->rq_xtime;

L
Linus Torvalds 已提交
776
	list_del_init(&req->rq_list);
777
	req->rq_private_buf.len = copied;
778 779
	/* Ensure all writes are done before we update req->rq_received */
	smp_wmb();
780
	req->rq_received = copied;
781
	rpc_wake_up_queued_task(&xprt->pending, task);
L
Linus Torvalds 已提交
782
}
783
EXPORT_SYMBOL_GPL(xprt_complete_rqst);
L
Linus Torvalds 已提交
784

785
static void xprt_timer(struct rpc_task *task)
L
Linus Torvalds 已提交
786
{
787
	struct rpc_rqst *req = task->tk_rqstp;
L
Linus Torvalds 已提交
788 789
	struct rpc_xprt *xprt = req->rq_xprt;

790 791
	if (task->tk_status != -ETIMEDOUT)
		return;
792
	dprintk("RPC: %5u xprt_timer\n", task->tk_pid);
L
Linus Torvalds 已提交
793

794
	spin_lock_bh(&xprt->transport_lock);
795 796 797
	if (!req->rq_received) {
		if (xprt->ops->timer)
			xprt->ops->timer(task);
798 799 800
	} else
		task->tk_status = 0;
	spin_unlock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
801 802
}

803 804 805 806
/**
 * xprt_prepare_transmit - reserve the transport before sending a request
 * @task: RPC task about to send a request
 *
L
Linus Torvalds 已提交
807
 */
808
int xprt_prepare_transmit(struct rpc_task *task)
L
Linus Torvalds 已提交
809 810 811 812 813
{
	struct rpc_rqst	*req = task->tk_rqstp;
	struct rpc_xprt	*xprt = req->rq_xprt;
	int err = 0;

814
	dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);
L
Linus Torvalds 已提交
815

C
Chuck Lever 已提交
816
	spin_lock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
817 818 819 820
	if (req->rq_received && !req->rq_bytes_sent) {
		err = req->rq_received;
		goto out_unlock;
	}
821
	if (!xprt->ops->reserve_xprt(task)) {
L
Linus Torvalds 已提交
822 823 824 825 826 827 828 829 830
		err = -EAGAIN;
		goto out_unlock;
	}

	if (!xprt_connected(xprt)) {
		err = -ENOTCONN;
		goto out_unlock;
	}
out_unlock:
C
Chuck Lever 已提交
831
	spin_unlock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
832 833 834
	return err;
}

835
void xprt_end_transmit(struct rpc_task *task)
836
{
837
	xprt_release_write(task->tk_xprt, task);
838 839
}

840 841 842 843 844 845 846
/**
 * xprt_transmit - send an RPC request on a transport
 * @task: controlling RPC task
 *
 * We have to copy the iovec because sendmsg fiddles with its contents.
 */
void xprt_transmit(struct rpc_task *task)
L
Linus Torvalds 已提交
847 848 849
{
	struct rpc_rqst	*req = task->tk_rqstp;
	struct rpc_xprt	*xprt = req->rq_xprt;
850
	int status;
L
Linus Torvalds 已提交
851

852
	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
L
Linus Torvalds 已提交
853 854 855

	if (!req->rq_received) {
		if (list_empty(&req->rq_list)) {
C
Chuck Lever 已提交
856
			spin_lock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
857 858 859 860 861
			/* Update the softirq receive buffer */
			memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
					sizeof(req->rq_private_buf));
			/* Add request to the receive list */
			list_add_tail(&req->rq_list, &xprt->recv);
C
Chuck Lever 已提交
862
			spin_unlock_bh(&xprt->transport_lock);
L
Linus Torvalds 已提交
863
			xprt_reset_majortimeo(req);
864 865
			/* Turn off autodisconnect */
			del_singleshot_timer_sync(&xprt->timer);
L
Linus Torvalds 已提交
866 867 868 869
		}
	} else if (!req->rq_bytes_sent)
		return;

870
	req->rq_connect_cookie = xprt->connect_cookie;
871
	req->rq_xtime = jiffies;
872
	status = xprt->ops->send_request(task);
873
	if (status == 0) {
874
		dprintk("RPC: %5u xmit complete\n", task->tk_pid);
875
		spin_lock_bh(&xprt->transport_lock);
876

877
		xprt->ops->set_retrans_timeout(task);
878 879 880 881 882

		xprt->stat.sends++;
		xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs;
		xprt->stat.bklog_u += xprt->backlog.qlen;

883 884 885 886
		/* Don't race with disconnect */
		if (!xprt_connected(xprt))
			task->tk_status = -ENOTCONN;
		else if (!req->rq_received)
887
			rpc_sleep_on(&xprt->pending, task, xprt_timer);
888 889 890
		spin_unlock_bh(&xprt->transport_lock);
		return;
	}
L
Linus Torvalds 已提交
891 892 893 894 895 896

	/* Note: at this point, task->tk_sleeping has not yet been set,
	 *	 hence there is no danger of the waking up task being put on
	 *	 schedq, and being picked up by a parallel run of rpciod().
	 */
	task->tk_status = status;
897
	if (status == -ECONNREFUSED)
898
		rpc_sleep_on(&xprt->sending, task, NULL);
L
Linus Torvalds 已提交
899 900
}

901
static inline void do_xprt_reserve(struct rpc_task *task)
L
Linus Torvalds 已提交
902 903 904 905 906 907 908 909 910 911 912 913 914
{
	struct rpc_xprt	*xprt = task->tk_xprt;

	task->tk_status = 0;
	if (task->tk_rqstp)
		return;
	if (!list_empty(&xprt->free)) {
		struct rpc_rqst	*req = list_entry(xprt->free.next, struct rpc_rqst, rq_list);
		list_del_init(&req->rq_list);
		task->tk_rqstp = req;
		xprt_request_init(task, xprt);
		return;
	}
915
	dprintk("RPC:       waiting for request slot\n");
L
Linus Torvalds 已提交
916 917
	task->tk_status = -EAGAIN;
	task->tk_timeout = 0;
918
	rpc_sleep_on(&xprt->backlog, task, NULL);
L
Linus Torvalds 已提交
919 920
}

921 922 923 924 925 926 927 928
/**
 * xprt_reserve - allocate an RPC request slot
 * @task: RPC task requesting a slot allocation
 *
 * If no more slots are available, place the task on the transport's
 * backlog queue.
 */
void xprt_reserve(struct rpc_task *task)
L
Linus Torvalds 已提交
929 930 931 932
{
	struct rpc_xprt	*xprt = task->tk_xprt;

	task->tk_status = -EIO;
933 934 935
	spin_lock(&xprt->reserve_lock);
	do_xprt_reserve(task);
	spin_unlock(&xprt->reserve_lock);
L
Linus Torvalds 已提交
936 937
}

938
static inline __be32 xprt_alloc_xid(struct rpc_xprt *xprt)
L
Linus Torvalds 已提交
939 940 941 942 943 944
{
	return xprt->xid++;
}

static inline void xprt_init_xid(struct rpc_xprt *xprt)
{
945
	xprt->xid = net_random();
L
Linus Torvalds 已提交
946 947
}

948
static void xprt_request_init(struct rpc_task *task, struct rpc_xprt *xprt)
L
Linus Torvalds 已提交
949 950 951
{
	struct rpc_rqst	*req = task->tk_rqstp;

952
	req->rq_timeout = task->tk_client->cl_timeout->to_initval;
L
Linus Torvalds 已提交
953 954
	req->rq_task	= task;
	req->rq_xprt    = xprt;
955
	req->rq_buffer  = NULL;
L
Linus Torvalds 已提交
956
	req->rq_xid     = xprt_alloc_xid(xprt);
957
	req->rq_release_snd_buf = NULL;
958
	xprt_reset_majortimeo(req);
959
	dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,
L
Linus Torvalds 已提交
960 961 962
			req, ntohl(req->rq_xid));
}

963 964 965 966
/**
 * xprt_release - release an RPC request slot
 * @task: task which is finished with the slot
 *
L
Linus Torvalds 已提交
967
 */
968
void xprt_release(struct rpc_task *task)
L
Linus Torvalds 已提交
969 970 971 972 973 974
{
	struct rpc_xprt	*xprt = task->tk_xprt;
	struct rpc_rqst	*req;

	if (!(req = task->tk_rqstp))
		return;
975
	rpc_count_iostats(task);
C
Chuck Lever 已提交
976
	spin_lock_bh(&xprt->transport_lock);
977
	xprt->ops->release_xprt(xprt, task);
978 979
	if (xprt->ops->release_request)
		xprt->ops->release_request(task);
L
Linus Torvalds 已提交
980 981 982
	if (!list_empty(&req->rq_list))
		list_del(&req->rq_list);
	xprt->last_used = jiffies;
983
	if (list_empty(&xprt->recv))
984
		mod_timer(&xprt->timer,
985
				xprt->last_used + xprt->idle_timeout);
C
Chuck Lever 已提交
986
	spin_unlock_bh(&xprt->transport_lock);
987
	xprt->ops->buf_free(req->rq_buffer);
L
Linus Torvalds 已提交
988
	task->tk_rqstp = NULL;
989 990
	if (req->rq_release_snd_buf)
		req->rq_release_snd_buf(req);
L
Linus Torvalds 已提交
991 992
	memset(req, 0, sizeof(*req));	/* mark unused */

993
	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
L
Linus Torvalds 已提交
994

C
Chuck Lever 已提交
995
	spin_lock(&xprt->reserve_lock);
L
Linus Torvalds 已提交
996
	list_add(&req->rq_list, &xprt->free);
997
	rpc_wake_up_next(&xprt->backlog);
C
Chuck Lever 已提交
998
	spin_unlock(&xprt->reserve_lock);
L
Linus Torvalds 已提交
999 1000
}

1001 1002
/**
 * xprt_create_transport - create an RPC transport
1003
 * @args: rpc transport creation arguments
1004 1005
 *
 */
1006
struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
1007 1008 1009
{
	struct rpc_xprt	*xprt;
	struct rpc_rqst	*req;
1010
	struct xprt_class *t;
1011

1012 1013
	spin_lock(&xprt_list_lock);
	list_for_each_entry(t, &xprt_list, list) {
1014
		if (t->ident == args->ident) {
1015 1016 1017
			spin_unlock(&xprt_list_lock);
			goto found;
		}
1018
	}
1019
	spin_unlock(&xprt_list_lock);
1020
	printk(KERN_ERR "RPC: transport (%d) not supported\n", args->ident);
1021 1022 1023 1024
	return ERR_PTR(-EIO);

found:
	xprt = t->setup(args);
1025
	if (IS_ERR(xprt)) {
1026
		dprintk("RPC:       xprt_create_transport: failed, %ld\n",
1027 1028
				-PTR_ERR(xprt));
		return xprt;
1029 1030
	}

1031
	kref_init(&xprt->kref);
1032 1033 1034 1035 1036
	spin_lock_init(&xprt->transport_lock);
	spin_lock_init(&xprt->reserve_lock);

	INIT_LIST_HEAD(&xprt->free);
	INIT_LIST_HEAD(&xprt->recv);
1037
	INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
1038 1039
	setup_timer(&xprt->timer, xprt_init_autodisconnect,
			(unsigned long)xprt);
1040 1041
	xprt->last_used = jiffies;
	xprt->cwnd = RPC_INITCWND;
1042
	xprt->bind_index = 0;
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055

	rpc_init_wait_queue(&xprt->binding, "xprt_binding");
	rpc_init_wait_queue(&xprt->pending, "xprt_pending");
	rpc_init_wait_queue(&xprt->sending, "xprt_sending");
	rpc_init_wait_queue(&xprt->resend, "xprt_resend");
	rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog");

	/* initialize free list */
	for (req = &xprt->slot[xprt->max_reqs-1]; req >= &xprt->slot[0]; req--)
		list_add(&req->rq_list, &xprt->free);

	xprt_init_xid(xprt);

1056
	dprintk("RPC:       created transport %p with %u slots\n", xprt,
1057 1058 1059 1060 1061
			xprt->max_reqs);

	return xprt;
}

1062 1063
/**
 * xprt_destroy - destroy an RPC transport, killing off all requests.
1064
 * @kref: kref for the transport to destroy
1065
 *
L
Linus Torvalds 已提交
1066
 */
1067
static void xprt_destroy(struct kref *kref)
L
Linus Torvalds 已提交
1068
{
1069 1070
	struct rpc_xprt *xprt = container_of(kref, struct rpc_xprt, kref);

1071
	dprintk("RPC:       destroying transport %p\n", xprt);
1072 1073
	xprt->shutdown = 1;
	del_timer_sync(&xprt->timer);
1074

1075 1076 1077 1078 1079
	rpc_destroy_wait_queue(&xprt->binding);
	rpc_destroy_wait_queue(&xprt->pending);
	rpc_destroy_wait_queue(&xprt->sending);
	rpc_destroy_wait_queue(&xprt->resend);
	rpc_destroy_wait_queue(&xprt->backlog);
1080 1081 1082
	/*
	 * Tear down transport state and free the rpc_xprt
	 */
1083
	xprt->ops->destroy(xprt);
1084
}
L
Linus Torvalds 已提交
1085

1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
/**
 * xprt_put - release a reference to an RPC transport.
 * @xprt: pointer to the transport
 *
 */
void xprt_put(struct rpc_xprt *xprt)
{
	kref_put(&xprt->kref, xprt_destroy);
}

/**
 * xprt_get - return a reference to an RPC transport.
 * @xprt: pointer to the transport
 *
 */
struct rpc_xprt *xprt_get(struct rpc_xprt *xprt)
{
	kref_get(&xprt->kref);
	return xprt;
L
Linus Torvalds 已提交
1105
}