svc.c 24.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 * linux/net/sunrpc/svc.c
 *
 * High-level RPC service routines
 *
 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
7 8 9 10
 *
 * Multiple threads pools and NUMAisation
 * Copyright (c) 2006 Silicon Graphics, Inc.
 * by Greg Banks <gnb@melbourne.sgi.com>
L
Linus Torvalds 已提交
11 12 13 14 15 16 17 18
 */

#include <linux/linkage.h>
#include <linux/sched.h>
#include <linux/errno.h>
#include <linux/net.h>
#include <linux/in.h>
#include <linux/mm.h>
19 20
#include <linux/interrupt.h>
#include <linux/module.h>
21
#include <linux/sched.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27 28 29 30

#include <linux/sunrpc/types.h>
#include <linux/sunrpc/xdr.h>
#include <linux/sunrpc/stats.h>
#include <linux/sunrpc/svcsock.h>
#include <linux/sunrpc/clnt.h>

#define RPCDBG_FACILITY	RPCDBG_SVCDSP

31 32
#define svc_serv_is_pooled(serv)    ((serv)->sv_function)

33 34 35 36
/*
 * Mode for mapping cpus to pools.
 */
enum {
37
	SVC_POOL_AUTO = -1,	/* choose one of the others */
38 39 40 41 42
	SVC_POOL_GLOBAL,	/* no mapping, just a single global pool
				 * (legacy & UP mode) */
	SVC_POOL_PERCPU,	/* one pool per cpu */
	SVC_POOL_PERNODE	/* one pool per numa node */
};
43
#define SVC_POOL_DEFAULT	SVC_POOL_GLOBAL
44 45 46 47 48 49

/*
 * Structure for mapping cpus to pools and vice versa.
 * Setup once during sunrpc initialisation.
 */
static struct svc_pool_map {
50
	int count;			/* How many svc_servs use us */
51 52 53 54 55 56 57
	int mode;			/* Note: int not enum to avoid
					 * warnings about "enumeration value
					 * not handled in switch" */
	unsigned int npools;
	unsigned int *pool_to;		/* maps pool id to cpu or node */
	unsigned int *to_pool;		/* maps cpu or node to pool id */
} svc_pool_map = {
58 59
	.count = 0,
	.mode = SVC_POOL_DEFAULT
60
};
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */

static int
param_set_pool_mode(const char *val, struct kernel_param *kp)
{
	int *ip = (int *)kp->arg;
	struct svc_pool_map *m = &svc_pool_map;
	int err;

	mutex_lock(&svc_pool_map_mutex);

	err = -EBUSY;
	if (m->count)
		goto out;

	err = 0;
	if (!strncmp(val, "auto", 4))
		*ip = SVC_POOL_AUTO;
	else if (!strncmp(val, "global", 6))
		*ip = SVC_POOL_GLOBAL;
	else if (!strncmp(val, "percpu", 6))
		*ip = SVC_POOL_PERCPU;
	else if (!strncmp(val, "pernode", 7))
		*ip = SVC_POOL_PERNODE;
	else
		err = -EINVAL;

out:
	mutex_unlock(&svc_pool_map_mutex);
	return err;
}

static int
param_get_pool_mode(char *buf, struct kernel_param *kp)
{
	int *ip = (int *)kp->arg;

	switch (*ip)
	{
	case SVC_POOL_AUTO:
		return strlcpy(buf, "auto", 20);
	case SVC_POOL_GLOBAL:
		return strlcpy(buf, "global", 20);
	case SVC_POOL_PERCPU:
		return strlcpy(buf, "percpu", 20);
	case SVC_POOL_PERNODE:
		return strlcpy(buf, "pernode", 20);
	default:
		return sprintf(buf, "%d", *ip);
	}
}
112

113 114
module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
		 &svc_pool_map.mode, 0644);
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176

/*
 * Detect best pool mapping mode heuristically,
 * according to the machine's topology.
 */
static int
svc_pool_map_choose_mode(void)
{
	unsigned int node;

	if (num_online_nodes() > 1) {
		/*
		 * Actually have multiple NUMA nodes,
		 * so split pools on NUMA node boundaries
		 */
		return SVC_POOL_PERNODE;
	}

	node = any_online_node(node_online_map);
	if (nr_cpus_node(node) > 2) {
		/*
		 * Non-trivial SMP, or CONFIG_NUMA on
		 * non-NUMA hardware, e.g. with a generic
		 * x86_64 kernel on Xeons.  In this case we
		 * want to divide the pools on cpu boundaries.
		 */
		return SVC_POOL_PERCPU;
	}

	/* default: one global pool */
	return SVC_POOL_GLOBAL;
}

/*
 * Allocate the to_pool[] and pool_to[] arrays.
 * Returns 0 on success or an errno.
 */
static int
svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
{
	m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
	if (!m->to_pool)
		goto fail;
	m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
	if (!m->pool_to)
		goto fail_free;

	return 0;

fail_free:
	kfree(m->to_pool);
fail:
	return -ENOMEM;
}

/*
 * Initialise the pool map for SVC_POOL_PERCPU mode.
 * Returns number of pools or <0 on error.
 */
static int
svc_pool_map_init_percpu(struct svc_pool_map *m)
{
177
	unsigned int maxpools = nr_cpu_ids;
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
	unsigned int pidx = 0;
	unsigned int cpu;
	int err;

	err = svc_pool_map_alloc_arrays(m, maxpools);
	if (err)
		return err;

	for_each_online_cpu(cpu) {
		BUG_ON(pidx > maxpools);
		m->to_pool[cpu] = pidx;
		m->pool_to[pidx] = cpu;
		pidx++;
	}
	/* cpus brought online later all get mapped to pool0, sorry */

	return pidx;
};


/*
 * Initialise the pool map for SVC_POOL_PERNODE mode.
 * Returns number of pools or <0 on error.
 */
static int
svc_pool_map_init_pernode(struct svc_pool_map *m)
{
205
	unsigned int maxpools = nr_node_ids;
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
	unsigned int pidx = 0;
	unsigned int node;
	int err;

	err = svc_pool_map_alloc_arrays(m, maxpools);
	if (err)
		return err;

	for_each_node_with_cpus(node) {
		/* some architectures (e.g. SN2) have cpuless nodes */
		BUG_ON(pidx > maxpools);
		m->to_pool[node] = pidx;
		m->pool_to[pidx] = node;
		pidx++;
	}
	/* nodes brought online later all get mapped to pool0, sorry */

	return pidx;
}


/*
228 229 230
 * Add a reference to the global map of cpus to pools (and
 * vice versa).  Initialise the map if we're the first user.
 * Returns the number of pools.
231 232
 */
static unsigned int
233
svc_pool_map_get(void)
234 235 236 237
{
	struct svc_pool_map *m = &svc_pool_map;
	int npools = -1;

238 239 240 241
	mutex_lock(&svc_pool_map_mutex);

	if (m->count++) {
		mutex_unlock(&svc_pool_map_mutex);
242
		return m->npools;
243
	}
244

245 246
	if (m->mode == SVC_POOL_AUTO)
		m->mode = svc_pool_map_choose_mode();
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263

	switch (m->mode) {
	case SVC_POOL_PERCPU:
		npools = svc_pool_map_init_percpu(m);
		break;
	case SVC_POOL_PERNODE:
		npools = svc_pool_map_init_pernode(m);
		break;
	}

	if (npools < 0) {
		/* default, or memory allocation failure */
		npools = 1;
		m->mode = SVC_POOL_GLOBAL;
	}
	m->npools = npools;

264
	mutex_unlock(&svc_pool_map_mutex);
265 266 267
	return m->npools;
}

268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293

/*
 * Drop a reference to the global map of cpus to pools.
 * When the last reference is dropped, the map data is
 * freed; this allows the sysadmin to change the pool
 * mode using the pool_mode module option without
 * rebooting or re-loading sunrpc.ko.
 */
static void
svc_pool_map_put(void)
{
	struct svc_pool_map *m = &svc_pool_map;

	mutex_lock(&svc_pool_map_mutex);

	if (!--m->count) {
		m->mode = SVC_POOL_DEFAULT;
		kfree(m->to_pool);
		kfree(m->pool_to);
		m->npools = 0;
	}

	mutex_unlock(&svc_pool_map_mutex);
}


294 295 296 297 298 299 300 301 302 303 304 305 306 307
/*
 * Set the current thread's cpus_allowed mask so that it
 * will only run on cpus in the given pool.
 *
 * Returns 1 and fills in oldmask iff a cpumask was applied.
 */
static inline int
svc_pool_map_set_cpumask(unsigned int pidx, cpumask_t *oldmask)
{
	struct svc_pool_map *m = &svc_pool_map;
	unsigned int node; /* or cpu */

	/*
	 * The caller checks for sv_nrpools > 1, which
308
	 * implies that we've been initialized.
309
	 */
310
	BUG_ON(m->count == 0);
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340

	switch (m->mode)
	{
	default:
		return 0;
	case SVC_POOL_PERCPU:
		node = m->pool_to[pidx];
		*oldmask = current->cpus_allowed;
		set_cpus_allowed(current, cpumask_of_cpu(node));
		return 1;
	case SVC_POOL_PERNODE:
		node = m->pool_to[pidx];
		*oldmask = current->cpus_allowed;
		set_cpus_allowed(current, node_to_cpumask(node));
		return 1;
	}
}

/*
 * Use the mapping mode to choose a pool for a given CPU.
 * Used when enqueueing an incoming RPC.  Always returns
 * a non-NULL pool pointer.
 */
struct svc_pool *
svc_pool_for_cpu(struct svc_serv *serv, int cpu)
{
	struct svc_pool_map *m = &svc_pool_map;
	unsigned int pidx = 0;

	/*
341
	 * An uninitialised map happens in a pure client when
342 343 344
	 * lockd is brought up, so silently treat it the
	 * same as SVC_POOL_GLOBAL.
	 */
345 346 347 348 349 350 351 352 353
	if (svc_serv_is_pooled(serv)) {
		switch (m->mode) {
		case SVC_POOL_PERCPU:
			pidx = m->to_pool[cpu];
			break;
		case SVC_POOL_PERNODE:
			pidx = m->to_pool[cpu_to_node(cpu)];
			break;
		}
354 355 356 357 358
	}
	return &serv->sv_pools[pidx % serv->sv_nrpools];
}


L
Linus Torvalds 已提交
359 360 361
/*
 * Create an RPC service
 */
362 363
static struct svc_serv *
__svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
364
	   void (*shutdown)(struct svc_serv *serv))
L
Linus Torvalds 已提交
365 366 367 368
{
	struct svc_serv	*serv;
	int vers;
	unsigned int xdrsize;
369
	unsigned int i;
L
Linus Torvalds 已提交
370

371
	if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
L
Linus Torvalds 已提交
372
		return NULL;
373
	serv->sv_name      = prog->pg_name;
L
Linus Torvalds 已提交
374 375 376
	serv->sv_program   = prog;
	serv->sv_nrthreads = 1;
	serv->sv_stats     = prog->pg_stats;
377 378 379 380
	if (bufsize > RPCSVC_MAXPAYLOAD)
		bufsize = RPCSVC_MAXPAYLOAD;
	serv->sv_max_payload = bufsize? bufsize : 4096;
	serv->sv_max_mesg  = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
381
	serv->sv_shutdown  = shutdown;
L
Linus Torvalds 已提交
382
	xdrsize = 0;
383 384 385 386 387 388 389 390 391 392 393 394
	while (prog) {
		prog->pg_lovers = prog->pg_nvers-1;
		for (vers=0; vers<prog->pg_nvers ; vers++)
			if (prog->pg_vers[vers]) {
				prog->pg_hivers = vers;
				if (prog->pg_lovers > vers)
					prog->pg_lovers = vers;
				if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
					xdrsize = prog->pg_vers[vers]->vs_xdrsize;
			}
		prog = prog->pg_next;
	}
L
Linus Torvalds 已提交
395 396 397
	serv->sv_xdrsize   = xdrsize;
	INIT_LIST_HEAD(&serv->sv_tempsocks);
	INIT_LIST_HEAD(&serv->sv_permsocks);
398
	init_timer(&serv->sv_temptimer);
L
Linus Torvalds 已提交
399 400
	spin_lock_init(&serv->sv_lock);

401
	serv->sv_nrpools = npools;
402
	serv->sv_pools =
403
		kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
404 405 406 407 408 409 410 411 412
			GFP_KERNEL);
	if (!serv->sv_pools) {
		kfree(serv);
		return NULL;
	}

	for (i = 0; i < serv->sv_nrpools; i++) {
		struct svc_pool *pool = &serv->sv_pools[i];

413
		dprintk("svc: initialising pool %u for %s\n",
414 415 416 417 418
				i, serv->sv_name);

		pool->sp_id = i;
		INIT_LIST_HEAD(&pool->sp_threads);
		INIT_LIST_HEAD(&pool->sp_sockets);
419
		INIT_LIST_HEAD(&pool->sp_all_threads);
420 421 422 423
		spin_lock_init(&pool->sp_lock);
	}


L
Linus Torvalds 已提交
424 425 426 427 428 429
	/* Remove any stale portmap registrations */
	svc_register(serv, 0, 0);

	return serv;
}

430 431 432 433 434 435 436 437 438 439 440 441 442
struct svc_serv *
svc_create(struct svc_program *prog, unsigned int bufsize,
		void (*shutdown)(struct svc_serv *serv))
{
	return __svc_create(prog, bufsize, /*npools*/1, shutdown);
}

struct svc_serv *
svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
		void (*shutdown)(struct svc_serv *serv),
		  svc_thread_fn func, int sig, struct module *mod)
{
	struct svc_serv *serv;
443
	unsigned int npools = svc_pool_map_get();
444

445
	serv = __svc_create(prog, bufsize, npools, shutdown);
446 447 448 449 450 451 452 453 454 455

	if (serv != NULL) {
		serv->sv_function = func;
		serv->sv_kill_signal = sig;
		serv->sv_module = mod;
	}

	return serv;
}

L
Linus Torvalds 已提交
456
/*
457
 * Destroy an RPC service.  Should be called with the BKL held
L
Linus Torvalds 已提交
458 459 460 461 462
 */
void
svc_destroy(struct svc_serv *serv)
{
	struct svc_sock	*svsk;
463
	struct svc_sock *tmp;
L
Linus Torvalds 已提交
464

465
	dprintk("svc: svc_destroy(%s, %d)\n",
L
Linus Torvalds 已提交
466 467 468 469 470 471 472 473 474 475 476
				serv->sv_program->pg_name,
				serv->sv_nrthreads);

	if (serv->sv_nrthreads) {
		if (--(serv->sv_nrthreads) != 0) {
			svc_sock_update_bufs(serv);
			return;
		}
	} else
		printk("svc_destroy: no threads for serv=%p!\n", serv);

477 478
	del_timer_sync(&serv->sv_temptimer);

479 480 481
	list_for_each_entry_safe(svsk, tmp, &serv->sv_tempsocks, sk_list)
		svc_force_close_socket(svsk);

482 483 484
	if (serv->sv_shutdown)
		serv->sv_shutdown(serv);

485 486 487 488 489
	list_for_each_entry_safe(svsk, tmp, &serv->sv_permsocks, sk_list)
		svc_force_close_socket(svsk);

	BUG_ON(!list_empty(&serv->sv_permsocks));
	BUG_ON(!list_empty(&serv->sv_tempsocks));
490

L
Linus Torvalds 已提交
491 492
	cache_clean_deferred(serv);

493 494 495
	if (svc_serv_is_pooled(serv))
		svc_pool_map_put();

L
Linus Torvalds 已提交
496 497
	/* Unregister service with the portmapper */
	svc_register(serv, 0, 0);
498
	kfree(serv->sv_pools);
L
Linus Torvalds 已提交
499 500 501 502 503 504 505 506 507 508 509 510
	kfree(serv);
}

/*
 * Allocate an RPC server's buffer space.
 * We allocate pages and place them in rq_argpages.
 */
static int
svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
{
	int pages;
	int arghi;
511

512 513 514
	pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
				       * We assume one is at most one page
				       */
L
Linus Torvalds 已提交
515
	arghi = 0;
516
	BUG_ON(pages > RPCSVC_MAXPAGES);
L
Linus Torvalds 已提交
517 518 519 520
	while (pages) {
		struct page *p = alloc_page(GFP_KERNEL);
		if (!p)
			break;
521
		rqstp->rq_pages[arghi++] = p;
L
Linus Torvalds 已提交
522 523 524 525 526 527 528 529 530 531 532
		pages--;
	}
	return ! pages;
}

/*
 * Release an RPC server buffer
 */
static void
svc_release_buffer(struct svc_rqst *rqstp)
{
533 534 535 536
	int i;
	for (i=0; i<ARRAY_SIZE(rqstp->rq_pages); i++)
		if (rqstp->rq_pages[i])
			put_page(rqstp->rq_pages[i]);
L
Linus Torvalds 已提交
537 538 539
}

/*
540
 * Create a thread in the given pool.  Caller must hold BKL.
541 542
 * On a NUMA or SMP machine, with a multi-pool serv, the thread
 * will be restricted to run on the cpus belonging to the pool.
L
Linus Torvalds 已提交
543
 */
544 545 546
static int
__svc_create_thread(svc_thread_fn func, struct svc_serv *serv,
		    struct svc_pool *pool)
L
Linus Torvalds 已提交
547 548 549
{
	struct svc_rqst	*rqstp;
	int		error = -ENOMEM;
550 551
	int		have_oldmask = 0;
	cpumask_t	oldmask;
L
Linus Torvalds 已提交
552

553
	rqstp = kzalloc(sizeof(*rqstp), GFP_KERNEL);
L
Linus Torvalds 已提交
554 555 556 557 558
	if (!rqstp)
		goto out;

	init_waitqueue_head(&rqstp->rq_wait);

559 560
	if (!(rqstp->rq_argp = kmalloc(serv->sv_xdrsize, GFP_KERNEL))
	 || !(rqstp->rq_resp = kmalloc(serv->sv_xdrsize, GFP_KERNEL))
561
	 || !svc_init_buffer(rqstp, serv->sv_max_mesg))
L
Linus Torvalds 已提交
562 563 564
		goto out_thread;

	serv->sv_nrthreads++;
565 566
	spin_lock_bh(&pool->sp_lock);
	pool->sp_nrthreads++;
567
	list_add(&rqstp->rq_all, &pool->sp_all_threads);
568
	spin_unlock_bh(&pool->sp_lock);
L
Linus Torvalds 已提交
569
	rqstp->rq_server = serv;
570
	rqstp->rq_pool = pool;
571 572 573 574

	if (serv->sv_nrpools > 1)
		have_oldmask = svc_pool_map_set_cpumask(pool->sp_id, &oldmask);

L
Linus Torvalds 已提交
575
	error = kernel_thread((int (*)(void *)) func, rqstp, 0);
576 577 578 579

	if (have_oldmask)
		set_cpus_allowed(current, oldmask);

L
Linus Torvalds 已提交
580 581 582 583 584 585 586 587 588 589 590 591 592
	if (error < 0)
		goto out_thread;
	svc_sock_update_bufs(serv);
	error = 0;
out:
	return error;

out_thread:
	svc_exit_thread(rqstp);
	goto out;
}

/*
593 594 595 596 597 598 599 600
 * Create a thread in the default pool.  Caller must hold BKL.
 */
int
svc_create_thread(svc_thread_fn func, struct svc_serv *serv)
{
	return __svc_create_thread(func, serv, &serv->sv_pools[0]);
}

601 602 603 604 605 606 607 608 609
/*
 * Choose a pool in which to create a new thread, for svc_set_num_threads
 */
static inline struct svc_pool *
choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
{
	if (pool != NULL)
		return pool;

610
	return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
}

/*
 * Choose a thread to kill, for svc_set_num_threads
 */
static inline struct task_struct *
choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
{
	unsigned int i;
	struct task_struct *task = NULL;

	if (pool != NULL) {
		spin_lock_bh(&pool->sp_lock);
	} else {
		/* choose a pool in round-robin fashion */
626 627
		for (i = 0; i < serv->sv_nrpools; i++) {
			pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
628
			spin_lock_bh(&pool->sp_lock);
629 630
			if (!list_empty(&pool->sp_all_threads))
				goto found_pool;
631
			spin_unlock_bh(&pool->sp_lock);
632
		}
633 634 635 636 637 638 639 640 641 642 643 644 645 646
		return NULL;
	}

found_pool:
	if (!list_empty(&pool->sp_all_threads)) {
		struct svc_rqst *rqstp;

		/*
		 * Remove from the pool->sp_all_threads list
		 * so we don't try to kill it again.
		 */
		rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
		list_del_init(&rqstp->rq_all);
		task = rqstp->rq_task;
647
	}
648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
	spin_unlock_bh(&pool->sp_lock);

	return task;
}

/*
 * Create or destroy enough new threads to make the number
 * of threads the given number.  If `pool' is non-NULL, applies
 * only to threads in that pool, otherwise round-robins between
 * all pools.  Must be called with a svc_get() reference and
 * the BKL held.
 *
 * Destroying threads relies on the service threads filling in
 * rqstp->rq_task, which only the nfs ones do.  Assumes the serv
 * has been created using svc_create_pooled().
 *
 * Based on code that used to be in nfsd_svc() but tweaked
 * to be pool-aware.
 */
int
svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
{
	struct task_struct *victim;
	int error = 0;
	unsigned int state = serv->sv_nrthreads-1;

	if (pool == NULL) {
		/* The -1 assumes caller has done a svc_get() */
		nrservs -= (serv->sv_nrthreads-1);
	} else {
		spin_lock_bh(&pool->sp_lock);
		nrservs -= pool->sp_nrthreads;
		spin_unlock_bh(&pool->sp_lock);
	}

	/* create new threads */
	while (nrservs > 0) {
		nrservs--;
		__module_get(serv->sv_module);
		error = __svc_create_thread(serv->sv_function, serv,
					    choose_pool(serv, pool, &state));
		if (error < 0) {
			module_put(serv->sv_module);
			break;
		}
	}
	/* destroy old threads */
	while (nrservs < 0 &&
	       (victim = choose_victim(serv, pool, &state)) != NULL) {
		send_sig(serv->sv_kill_signal, victim, 1);
		nrservs++;
	}

	return error;
}

704 705
/*
 * Called from a server thread as it's exiting.  Caller must hold BKL.
L
Linus Torvalds 已提交
706 707 708 709 710
 */
void
svc_exit_thread(struct svc_rqst *rqstp)
{
	struct svc_serv	*serv = rqstp->rq_server;
711
	struct svc_pool	*pool = rqstp->rq_pool;
L
Linus Torvalds 已提交
712 713

	svc_release_buffer(rqstp);
J
Jesper Juhl 已提交
714 715 716
	kfree(rqstp->rq_resp);
	kfree(rqstp->rq_argp);
	kfree(rqstp->rq_auth_data);
717 718 719

	spin_lock_bh(&pool->sp_lock);
	pool->sp_nrthreads--;
720
	list_del(&rqstp->rq_all);
721 722
	spin_unlock_bh(&pool->sp_lock);

L
Linus Torvalds 已提交
723 724 725 726 727 728 729 730 731
	kfree(rqstp);

	/* Release the server */
	if (serv)
		svc_destroy(serv);
}

/*
 * Register an RPC service with the local portmapper.
732
 * To unregister a service, call this routine with
L
Linus Torvalds 已提交
733 734 735 736 737 738 739 740 741 742 743 744
 * proto and port == 0.
 */
int
svc_register(struct svc_serv *serv, int proto, unsigned short port)
{
	struct svc_program	*progp;
	unsigned long		flags;
	int			i, error = 0, dummy;

	if (!port)
		clear_thread_flag(TIF_SIGPENDING);

745 746 747 748 749
	for (progp = serv->sv_program; progp; progp = progp->pg_next) {
		for (i = 0; i < progp->pg_nvers; i++) {
			if (progp->pg_vers[i] == NULL)
				continue;

750
			dprintk("svc: svc_register(%s, %s, %d, %d)%s\n",
751 752 753 754 755 756 757 758 759 760
					progp->pg_name,
					proto == IPPROTO_UDP?  "udp" : "tcp",
					port,
					i,
					progp->pg_vers[i]->vs_hidden?
						" (but not telling portmap)" : "");

			if (progp->pg_vers[i]->vs_hidden)
				continue;

761
			error = rpcb_register(progp->pg_prog, i, proto, port, &dummy);
762 763 764 765 766 767
			if (error < 0)
				break;
			if (port && !dummy) {
				error = -EACCES;
				break;
			}
L
Linus Torvalds 已提交
768 769 770 771 772 773 774 775 776 777 778 779
		}
	}

	if (!port) {
		spin_lock_irqsave(&current->sighand->siglock, flags);
		recalc_sigpending();
		spin_unlock_irqrestore(&current->sighand->siglock, flags);
	}

	return error;
}

780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
/*
 * Printk the given error with the address of the client that caused it.
 */
static int
__attribute__ ((format (printf, 2, 3)))
svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
{
	va_list args;
	int 	r;
	char 	buf[RPC_MAX_ADDRBUFLEN];

	if (!net_ratelimit())
		return 0;

	printk(KERN_WARNING "svc: %s: ",
		svc_print_addr(rqstp, buf, sizeof(buf)));

	va_start(args, fmt);
	r = vprintk(fmt, args);
	va_end(args);

	return r;
}

L
Linus Torvalds 已提交
804 805 806 807
/*
 * Process the RPC request.
 */
int
808
svc_process(struct svc_rqst *rqstp)
L
Linus Torvalds 已提交
809 810 811 812 813 814
{
	struct svc_program	*progp;
	struct svc_version	*versp = NULL;	/* compiler food */
	struct svc_procedure	*procp = NULL;
	struct kvec *		argv = &rqstp->rq_arg.head[0];
	struct kvec *		resv = &rqstp->rq_res.head[0];
815
	struct svc_serv		*serv = rqstp->rq_server;
L
Linus Torvalds 已提交
816
	kxdrproc_t		xdr;
817 818 819
	__be32			*statp;
	u32			dir, prog, vers, proc;
	__be32			auth_stat, rpc_stat;
L
Linus Torvalds 已提交
820
	int			auth_res;
821
	__be32			*reply_statp;
L
Linus Torvalds 已提交
822 823 824 825 826 827 828

	rpc_stat = rpc_success;

	if (argv->iov_len < 6*4)
		goto err_short_len;

	/* setup response xdr_buf.
829
	 * Initially it has just one page
L
Linus Torvalds 已提交
830
	 */
831
	rqstp->rq_resused = 1;
L
Linus Torvalds 已提交
832 833
	resv->iov_base = page_address(rqstp->rq_respages[0]);
	resv->iov_len = 0;
834
	rqstp->rq_res.pages = rqstp->rq_respages + 1;
L
Linus Torvalds 已提交
835 836 837
	rqstp->rq_res.len = 0;
	rqstp->rq_res.page_base = 0;
	rqstp->rq_res.page_len = 0;
838
	rqstp->rq_res.buflen = PAGE_SIZE;
839
	rqstp->rq_res.tail[0].iov_base = NULL;
L
Linus Torvalds 已提交
840
	rqstp->rq_res.tail[0].iov_len = 0;
841
	/* Will be turned off only in gss privacy case: */
842
	rqstp->rq_splice_ok = 1;
L
Linus Torvalds 已提交
843 844
	/* tcp needs a space for the record length... */
	if (rqstp->rq_prot == IPPROTO_TCP)
A
Alexey Dobriyan 已提交
845
		svc_putnl(resv, 0);
L
Linus Torvalds 已提交
846 847 848 849

	rqstp->rq_xid = svc_getu32(argv);
	svc_putu32(resv, rqstp->rq_xid);

A
Alexey Dobriyan 已提交
850 851
	dir  = svc_getnl(argv);
	vers = svc_getnl(argv);
L
Linus Torvalds 已提交
852 853

	/* First words of reply: */
A
Alexey Dobriyan 已提交
854
	svc_putnl(resv, 1);		/* REPLY */
L
Linus Torvalds 已提交
855 856 857 858 859 860 861

	if (dir != 0)		/* direction != CALL */
		goto err_bad_dir;
	if (vers != 2)		/* RPC version number */
		goto err_bad_rpc;

	/* Save position in case we later decide to reject: */
862
	reply_statp = resv->iov_base + resv->iov_len;
L
Linus Torvalds 已提交
863

A
Alexey Dobriyan 已提交
864
	svc_putnl(resv, 0);		/* ACCEPT */
L
Linus Torvalds 已提交
865

A
Alexey Dobriyan 已提交
866 867 868
	rqstp->rq_prog = prog = svc_getnl(argv);	/* program number */
	rqstp->rq_vers = vers = svc_getnl(argv);	/* version number */
	rqstp->rq_proc = proc = svc_getnl(argv);	/* procedure number */
L
Linus Torvalds 已提交
869 870

	progp = serv->sv_program;
871 872 873 874 875

	for (progp = serv->sv_program; progp; progp = progp->pg_next)
		if (prog == progp->pg_prog)
			break;

L
Linus Torvalds 已提交
876 877 878 879 880 881 882
	/*
	 * Decode auth data, and add verifier to reply buffer.
	 * We do this before anything else in order to get a decent
	 * auth verifier.
	 */
	auth_res = svc_authenticate(rqstp, &auth_stat);
	/* Also give the program a chance to reject this call: */
883
	if (auth_res == SVC_OK && progp) {
L
Linus Torvalds 已提交
884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902
		auth_stat = rpc_autherr_badcred;
		auth_res = progp->pg_authenticate(rqstp);
	}
	switch (auth_res) {
	case SVC_OK:
		break;
	case SVC_GARBAGE:
		rpc_stat = rpc_garbage_args;
		goto err_bad;
	case SVC_SYSERR:
		rpc_stat = rpc_system_err;
		goto err_bad;
	case SVC_DENIED:
		goto err_bad_auth;
	case SVC_DROP:
		goto dropit;
	case SVC_COMPLETE:
		goto sendit;
	}
903

904
	if (progp == NULL)
L
Linus Torvalds 已提交
905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921
		goto err_bad_prog;

	if (vers >= progp->pg_nvers ||
	  !(versp = progp->pg_vers[vers]))
		goto err_bad_vers;

	procp = versp->vs_proc + proc;
	if (proc >= versp->vs_nproc || !procp->pc_func)
		goto err_bad_proc;
	rqstp->rq_server   = serv;
	rqstp->rq_procinfo = procp;

	/* Syntactic check complete */
	serv->sv_stats->rpccnt++;

	/* Build the reply header. */
	statp = resv->iov_base +resv->iov_len;
A
Alexey Dobriyan 已提交
922
	svc_putnl(resv, RPC_SUCCESS);
L
Linus Torvalds 已提交
923 924 925 926 927 928 929 930

	/* Bump per-procedure stats counter */
	procp->pc_count++;

	/* Initialize storage for argp and resp */
	memset(rqstp->rq_argp, 0, procp->pc_argsize);
	memset(rqstp->rq_resp, 0, procp->pc_ressize);

931
	/* un-reserve some of the out-queue now that we have a
L
Linus Torvalds 已提交
932 933 934
	 * better idea of reply size
	 */
	if (procp->pc_xdrressize)
935
		svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
L
Linus Torvalds 已提交
936 937 938 939 940 941 942 943 944 945 946

	/* Call the function that processes the request. */
	if (!versp->vs_dispatch) {
		/* Decode arguments */
		xdr = procp->pc_decode;
		if (xdr && !xdr(rqstp, argv->iov_base, rqstp->rq_argp))
			goto err_garbage;

		*statp = procp->pc_func(rqstp, rqstp->rq_argp, rqstp->rq_resp);

		/* Encode reply */
947 948 949 950 951
		if (*statp == rpc_drop_reply) {
			if (procp->pc_release)
				procp->pc_release(rqstp, NULL, rqstp->rq_resp);
			goto dropit;
		}
L
Linus Torvalds 已提交
952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
		if (*statp == rpc_success && (xdr = procp->pc_encode)
		 && !xdr(rqstp, resv->iov_base+resv->iov_len, rqstp->rq_resp)) {
			dprintk("svc: failed to encode reply\n");
			/* serv->sv_stats->rpcsystemerr++; */
			*statp = rpc_system_err;
		}
	} else {
		dprintk("svc: calling dispatcher\n");
		if (!versp->vs_dispatch(rqstp, statp)) {
			/* Release reply info */
			if (procp->pc_release)
				procp->pc_release(rqstp, NULL, rqstp->rq_resp);
			goto dropit;
		}
	}

	/* Check RPC status result */
	if (*statp != rpc_success)
		resv->iov_len = ((void*)statp)  - resv->iov_base + 4;

	/* Release reply info */
	if (procp->pc_release)
		procp->pc_release(rqstp, NULL, rqstp->rq_resp);

	if (procp->pc_encode == NULL)
		goto dropit;

 sendit:
	if (svc_authorise(rqstp))
		goto dropit;
	return svc_send(rqstp);

 dropit:
	svc_authorise(rqstp);	/* doesn't hurt to call this twice */
	dprintk("svc: svc_process dropit\n");
	svc_drop(rqstp);
	return 0;

err_short_len:
991 992
	svc_printk(rqstp, "short len %Zd, dropping request\n",
			argv->iov_len);
993

L
Linus Torvalds 已提交
994 995 996
	goto dropit;			/* drop request */

err_bad_dir:
997
	svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
998

L
Linus Torvalds 已提交
999 1000 1001 1002 1003
	serv->sv_stats->rpcbadfmt++;
	goto dropit;			/* drop request */

err_bad_rpc:
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1004 1005 1006 1007
	svc_putnl(resv, 1);	/* REJECT */
	svc_putnl(resv, 0);	/* RPC_MISMATCH */
	svc_putnl(resv, 2);	/* Only RPCv2 supported */
	svc_putnl(resv, 2);
L
Linus Torvalds 已提交
1008 1009 1010 1011 1012 1013
	goto sendit;

err_bad_auth:
	dprintk("svc: authentication failed (%d)\n", ntohl(auth_stat));
	serv->sv_stats->rpcbadauth++;
	/* Restore write pointer to location of accept status: */
1014
	xdr_ressize_check(rqstp, reply_statp);
A
Alexey Dobriyan 已提交
1015 1016 1017
	svc_putnl(resv, 1);	/* REJECT */
	svc_putnl(resv, 1);	/* AUTH_ERROR */
	svc_putnl(resv, ntohl(auth_stat));	/* status */
L
Linus Torvalds 已提交
1018 1019 1020
	goto sendit;

err_bad_prog:
1021
	dprintk("svc: unknown program %d\n", prog);
L
Linus Torvalds 已提交
1022
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1023
	svc_putnl(resv, RPC_PROG_UNAVAIL);
L
Linus Torvalds 已提交
1024 1025 1026
	goto sendit;

err_bad_vers:
1027
	svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
1028 1029
		       vers, prog, progp->pg_name);

L
Linus Torvalds 已提交
1030
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1031 1032 1033
	svc_putnl(resv, RPC_PROG_MISMATCH);
	svc_putnl(resv, progp->pg_lovers);
	svc_putnl(resv, progp->pg_hivers);
L
Linus Torvalds 已提交
1034 1035 1036
	goto sendit;

err_bad_proc:
1037
	svc_printk(rqstp, "unknown procedure (%d)\n", proc);
1038

L
Linus Torvalds 已提交
1039
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1040
	svc_putnl(resv, RPC_PROC_UNAVAIL);
L
Linus Torvalds 已提交
1041 1042 1043
	goto sendit;

err_garbage:
1044
	svc_printk(rqstp, "failed to decode args\n");
1045

L
Linus Torvalds 已提交
1046 1047 1048
	rpc_stat = rpc_garbage_args;
err_bad:
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1049
	svc_putnl(resv, ntohl(rpc_stat));
L
Linus Torvalds 已提交
1050 1051
	goto sendit;
}
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061

/*
 * Return (transport-specific) limit on the rpc payload.
 */
u32 svc_max_payload(const struct svc_rqst *rqstp)
{
	int max = RPCSVC_MAXPAYLOAD_TCP;

	if (rqstp->rq_sock->sk_sock->type == SOCK_DGRAM)
		max = RPCSVC_MAXPAYLOAD_UDP;
1062 1063
	if (rqstp->rq_server->sv_max_payload < max)
		max = rqstp->rq_server->sv_max_payload;
1064 1065 1066
	return max;
}
EXPORT_SYMBOL_GPL(svc_max_payload);