svc.c 28.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 * linux/net/sunrpc/svc.c
 *
 * High-level RPC service routines
 *
 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
7 8 9 10
 *
 * Multiple threads pools and NUMAisation
 * Copyright (c) 2006 Silicon Graphics, Inc.
 * by Greg Banks <gnb@melbourne.sgi.com>
L
Linus Torvalds 已提交
11 12 13 14 15 16 17 18
 */

#include <linux/linkage.h>
#include <linux/sched.h>
#include <linux/errno.h>
#include <linux/net.h>
#include <linux/in.h>
#include <linux/mm.h>
19 20
#include <linux/interrupt.h>
#include <linux/module.h>
21
#include <linux/kthread.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27 28 29 30

#include <linux/sunrpc/types.h>
#include <linux/sunrpc/xdr.h>
#include <linux/sunrpc/stats.h>
#include <linux/sunrpc/svcsock.h>
#include <linux/sunrpc/clnt.h>

#define RPCDBG_FACILITY	RPCDBG_SVCDSP

31 32
static void svc_unregister(const struct svc_serv *serv);

33 34
#define svc_serv_is_pooled(serv)    ((serv)->sv_function)

35 36 37 38
/*
 * Mode for mapping cpus to pools.
 */
enum {
39
	SVC_POOL_AUTO = -1,	/* choose one of the others */
40 41 42 43 44
	SVC_POOL_GLOBAL,	/* no mapping, just a single global pool
				 * (legacy & UP mode) */
	SVC_POOL_PERCPU,	/* one pool per cpu */
	SVC_POOL_PERNODE	/* one pool per numa node */
};
45
#define SVC_POOL_DEFAULT	SVC_POOL_GLOBAL
46 47 48 49 50 51

/*
 * Structure for mapping cpus to pools and vice versa.
 * Setup once during sunrpc initialisation.
 */
static struct svc_pool_map {
52
	int count;			/* How many svc_servs use us */
53 54 55 56 57 58 59
	int mode;			/* Note: int not enum to avoid
					 * warnings about "enumeration value
					 * not handled in switch" */
	unsigned int npools;
	unsigned int *pool_to;		/* maps pool id to cpu or node */
	unsigned int *to_pool;		/* maps cpu or node to pool id */
} svc_pool_map = {
60 61
	.count = 0,
	.mode = SVC_POOL_DEFAULT
62
};
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */

static int
param_set_pool_mode(const char *val, struct kernel_param *kp)
{
	int *ip = (int *)kp->arg;
	struct svc_pool_map *m = &svc_pool_map;
	int err;

	mutex_lock(&svc_pool_map_mutex);

	err = -EBUSY;
	if (m->count)
		goto out;

	err = 0;
	if (!strncmp(val, "auto", 4))
		*ip = SVC_POOL_AUTO;
	else if (!strncmp(val, "global", 6))
		*ip = SVC_POOL_GLOBAL;
	else if (!strncmp(val, "percpu", 6))
		*ip = SVC_POOL_PERCPU;
	else if (!strncmp(val, "pernode", 7))
		*ip = SVC_POOL_PERNODE;
	else
		err = -EINVAL;

out:
	mutex_unlock(&svc_pool_map_mutex);
	return err;
}

static int
param_get_pool_mode(char *buf, struct kernel_param *kp)
{
	int *ip = (int *)kp->arg;

	switch (*ip)
	{
	case SVC_POOL_AUTO:
		return strlcpy(buf, "auto", 20);
	case SVC_POOL_GLOBAL:
		return strlcpy(buf, "global", 20);
	case SVC_POOL_PERCPU:
		return strlcpy(buf, "percpu", 20);
	case SVC_POOL_PERNODE:
		return strlcpy(buf, "pernode", 20);
	default:
		return sprintf(buf, "%d", *ip);
	}
}
114

115 116
module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
		 &svc_pool_map.mode, 0644);
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178

/*
 * Detect best pool mapping mode heuristically,
 * according to the machine's topology.
 */
static int
svc_pool_map_choose_mode(void)
{
	unsigned int node;

	if (num_online_nodes() > 1) {
		/*
		 * Actually have multiple NUMA nodes,
		 * so split pools on NUMA node boundaries
		 */
		return SVC_POOL_PERNODE;
	}

	node = any_online_node(node_online_map);
	if (nr_cpus_node(node) > 2) {
		/*
		 * Non-trivial SMP, or CONFIG_NUMA on
		 * non-NUMA hardware, e.g. with a generic
		 * x86_64 kernel on Xeons.  In this case we
		 * want to divide the pools on cpu boundaries.
		 */
		return SVC_POOL_PERCPU;
	}

	/* default: one global pool */
	return SVC_POOL_GLOBAL;
}

/*
 * Allocate the to_pool[] and pool_to[] arrays.
 * Returns 0 on success or an errno.
 */
static int
svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
{
	m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
	if (!m->to_pool)
		goto fail;
	m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
	if (!m->pool_to)
		goto fail_free;

	return 0;

fail_free:
	kfree(m->to_pool);
fail:
	return -ENOMEM;
}

/*
 * Initialise the pool map for SVC_POOL_PERCPU mode.
 * Returns number of pools or <0 on error.
 */
static int
svc_pool_map_init_percpu(struct svc_pool_map *m)
{
179
	unsigned int maxpools = nr_cpu_ids;
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
	unsigned int pidx = 0;
	unsigned int cpu;
	int err;

	err = svc_pool_map_alloc_arrays(m, maxpools);
	if (err)
		return err;

	for_each_online_cpu(cpu) {
		BUG_ON(pidx > maxpools);
		m->to_pool[cpu] = pidx;
		m->pool_to[pidx] = cpu;
		pidx++;
	}
	/* cpus brought online later all get mapped to pool0, sorry */

	return pidx;
};


/*
 * Initialise the pool map for SVC_POOL_PERNODE mode.
 * Returns number of pools or <0 on error.
 */
static int
svc_pool_map_init_pernode(struct svc_pool_map *m)
{
207
	unsigned int maxpools = nr_node_ids;
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
	unsigned int pidx = 0;
	unsigned int node;
	int err;

	err = svc_pool_map_alloc_arrays(m, maxpools);
	if (err)
		return err;

	for_each_node_with_cpus(node) {
		/* some architectures (e.g. SN2) have cpuless nodes */
		BUG_ON(pidx > maxpools);
		m->to_pool[node] = pidx;
		m->pool_to[pidx] = node;
		pidx++;
	}
	/* nodes brought online later all get mapped to pool0, sorry */

	return pidx;
}


/*
230 231 232
 * Add a reference to the global map of cpus to pools (and
 * vice versa).  Initialise the map if we're the first user.
 * Returns the number of pools.
233 234
 */
static unsigned int
235
svc_pool_map_get(void)
236 237 238 239
{
	struct svc_pool_map *m = &svc_pool_map;
	int npools = -1;

240 241 242 243
	mutex_lock(&svc_pool_map_mutex);

	if (m->count++) {
		mutex_unlock(&svc_pool_map_mutex);
244
		return m->npools;
245
	}
246

247 248
	if (m->mode == SVC_POOL_AUTO)
		m->mode = svc_pool_map_choose_mode();
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265

	switch (m->mode) {
	case SVC_POOL_PERCPU:
		npools = svc_pool_map_init_percpu(m);
		break;
	case SVC_POOL_PERNODE:
		npools = svc_pool_map_init_pernode(m);
		break;
	}

	if (npools < 0) {
		/* default, or memory allocation failure */
		npools = 1;
		m->mode = SVC_POOL_GLOBAL;
	}
	m->npools = npools;

266
	mutex_unlock(&svc_pool_map_mutex);
267 268 269
	return m->npools;
}

270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295

/*
 * Drop a reference to the global map of cpus to pools.
 * When the last reference is dropped, the map data is
 * freed; this allows the sysadmin to change the pool
 * mode using the pool_mode module option without
 * rebooting or re-loading sunrpc.ko.
 */
static void
svc_pool_map_put(void)
{
	struct svc_pool_map *m = &svc_pool_map;

	mutex_lock(&svc_pool_map_mutex);

	if (!--m->count) {
		m->mode = SVC_POOL_DEFAULT;
		kfree(m->to_pool);
		kfree(m->pool_to);
		m->npools = 0;
	}

	mutex_unlock(&svc_pool_map_mutex);
}


296
/*
297
 * Set the given thread's cpus_allowed mask so that it
298 299
 * will only run on cpus in the given pool.
 */
300 301
static inline void
svc_pool_map_set_cpumask(struct task_struct *task, unsigned int pidx)
302 303
{
	struct svc_pool_map *m = &svc_pool_map;
304
	unsigned int node = m->pool_to[pidx];
305 306 307

	/*
	 * The caller checks for sv_nrpools > 1, which
308
	 * implies that we've been initialized.
309
	 */
310
	BUG_ON(m->count == 0);
311

312
	switch (m->mode) {
313
	case SVC_POOL_PERCPU:
314
	{
315
		set_cpus_allowed_ptr(task, &cpumask_of_cpu(node));
316
		break;
317
	}
318
	case SVC_POOL_PERNODE:
319 320
	{
		node_to_cpumask_ptr(nodecpumask, node);
321 322
		set_cpus_allowed_ptr(task, nodecpumask);
		break;
323
	}
324
	}
325 326 327 328 329 330 331 332 333 334 335 336 337 338
}

/*
 * Use the mapping mode to choose a pool for a given CPU.
 * Used when enqueueing an incoming RPC.  Always returns
 * a non-NULL pool pointer.
 */
struct svc_pool *
svc_pool_for_cpu(struct svc_serv *serv, int cpu)
{
	struct svc_pool_map *m = &svc_pool_map;
	unsigned int pidx = 0;

	/*
339
	 * An uninitialised map happens in a pure client when
340 341 342
	 * lockd is brought up, so silently treat it the
	 * same as SVC_POOL_GLOBAL.
	 */
343 344 345 346 347 348 349 350 351
	if (svc_serv_is_pooled(serv)) {
		switch (m->mode) {
		case SVC_POOL_PERCPU:
			pidx = m->to_pool[cpu];
			break;
		case SVC_POOL_PERNODE:
			pidx = m->to_pool[cpu_to_node(cpu)];
			break;
		}
352 353 354 355 356
	}
	return &serv->sv_pools[pidx % serv->sv_nrpools];
}


L
Linus Torvalds 已提交
357 358 359
/*
 * Create an RPC service
 */
360 361
static struct svc_serv *
__svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
362
	   sa_family_t family, void (*shutdown)(struct svc_serv *serv))
L
Linus Torvalds 已提交
363 364
{
	struct svc_serv	*serv;
365
	unsigned int vers;
L
Linus Torvalds 已提交
366
	unsigned int xdrsize;
367
	unsigned int i;
L
Linus Torvalds 已提交
368

369
	if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
L
Linus Torvalds 已提交
370
		return NULL;
371
	serv->sv_family    = family;
372
	serv->sv_name      = prog->pg_name;
L
Linus Torvalds 已提交
373 374 375
	serv->sv_program   = prog;
	serv->sv_nrthreads = 1;
	serv->sv_stats     = prog->pg_stats;
376 377 378 379
	if (bufsize > RPCSVC_MAXPAYLOAD)
		bufsize = RPCSVC_MAXPAYLOAD;
	serv->sv_max_payload = bufsize? bufsize : 4096;
	serv->sv_max_mesg  = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
380
	serv->sv_shutdown  = shutdown;
L
Linus Torvalds 已提交
381
	xdrsize = 0;
382 383 384 385 386 387 388 389 390 391 392 393
	while (prog) {
		prog->pg_lovers = prog->pg_nvers-1;
		for (vers=0; vers<prog->pg_nvers ; vers++)
			if (prog->pg_vers[vers]) {
				prog->pg_hivers = vers;
				if (prog->pg_lovers > vers)
					prog->pg_lovers = vers;
				if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
					xdrsize = prog->pg_vers[vers]->vs_xdrsize;
			}
		prog = prog->pg_next;
	}
L
Linus Torvalds 已提交
394 395 396
	serv->sv_xdrsize   = xdrsize;
	INIT_LIST_HEAD(&serv->sv_tempsocks);
	INIT_LIST_HEAD(&serv->sv_permsocks);
397
	init_timer(&serv->sv_temptimer);
L
Linus Torvalds 已提交
398 399
	spin_lock_init(&serv->sv_lock);

400
	serv->sv_nrpools = npools;
401
	serv->sv_pools =
402
		kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
403 404 405 406 407 408 409 410 411
			GFP_KERNEL);
	if (!serv->sv_pools) {
		kfree(serv);
		return NULL;
	}

	for (i = 0; i < serv->sv_nrpools; i++) {
		struct svc_pool *pool = &serv->sv_pools[i];

412
		dprintk("svc: initialising pool %u for %s\n",
413 414 415 416 417
				i, serv->sv_name);

		pool->sp_id = i;
		INIT_LIST_HEAD(&pool->sp_threads);
		INIT_LIST_HEAD(&pool->sp_sockets);
418
		INIT_LIST_HEAD(&pool->sp_all_threads);
419 420 421
		spin_lock_init(&pool->sp_lock);
	}

L
Linus Torvalds 已提交
422
	/* Remove any stale portmap registrations */
423
	svc_unregister(serv);
L
Linus Torvalds 已提交
424 425 426 427

	return serv;
}

428 429
struct svc_serv *
svc_create(struct svc_program *prog, unsigned int bufsize,
430
		sa_family_t family, void (*shutdown)(struct svc_serv *serv))
431
{
432
	return __svc_create(prog, bufsize, /*npools*/1, family, shutdown);
433
}
434
EXPORT_SYMBOL_GPL(svc_create);
435 436 437

struct svc_serv *
svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
438
		  sa_family_t family, void (*shutdown)(struct svc_serv *serv),
439
		  svc_thread_fn func, struct module *mod)
440 441
{
	struct svc_serv *serv;
442
	unsigned int npools = svc_pool_map_get();
443

444
	serv = __svc_create(prog, bufsize, npools, family, shutdown);
445 446 447 448 449 450 451 452

	if (serv != NULL) {
		serv->sv_function = func;
		serv->sv_module = mod;
	}

	return serv;
}
453
EXPORT_SYMBOL_GPL(svc_create_pooled);
454

L
Linus Torvalds 已提交
455
/*
456 457
 * Destroy an RPC service. Should be called with appropriate locking to
 * protect the sv_nrthreads, sv_permsocks and sv_tempsocks.
L
Linus Torvalds 已提交
458 459 460 461
 */
void
svc_destroy(struct svc_serv *serv)
{
462
	dprintk("svc: svc_destroy(%s, %d)\n",
L
Linus Torvalds 已提交
463 464 465 466 467 468 469 470 471 472 473
				serv->sv_program->pg_name,
				serv->sv_nrthreads);

	if (serv->sv_nrthreads) {
		if (--(serv->sv_nrthreads) != 0) {
			svc_sock_update_bufs(serv);
			return;
		}
	} else
		printk("svc_destroy: no threads for serv=%p!\n", serv);

474 475
	del_timer_sync(&serv->sv_temptimer);

476
	svc_close_all(&serv->sv_tempsocks);
477

478 479 480
	if (serv->sv_shutdown)
		serv->sv_shutdown(serv);

481
	svc_close_all(&serv->sv_permsocks);
482 483 484

	BUG_ON(!list_empty(&serv->sv_permsocks));
	BUG_ON(!list_empty(&serv->sv_tempsocks));
485

L
Linus Torvalds 已提交
486 487
	cache_clean_deferred(serv);

488 489 490
	if (svc_serv_is_pooled(serv))
		svc_pool_map_put();

491
	svc_unregister(serv);
492
	kfree(serv->sv_pools);
L
Linus Torvalds 已提交
493 494
	kfree(serv);
}
495
EXPORT_SYMBOL_GPL(svc_destroy);
L
Linus Torvalds 已提交
496 497 498 499 500 501 502 503

/*
 * Allocate an RPC server's buffer space.
 * We allocate pages and place them in rq_argpages.
 */
static int
svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
{
504
	unsigned int pages, arghi;
505

506 507 508
	pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
				       * We assume one is at most one page
				       */
L
Linus Torvalds 已提交
509
	arghi = 0;
510
	BUG_ON(pages > RPCSVC_MAXPAGES);
L
Linus Torvalds 已提交
511 512 513 514
	while (pages) {
		struct page *p = alloc_page(GFP_KERNEL);
		if (!p)
			break;
515
		rqstp->rq_pages[arghi++] = p;
L
Linus Torvalds 已提交
516 517
		pages--;
	}
518
	return pages == 0;
L
Linus Torvalds 已提交
519 520 521 522 523 524 525 526
}

/*
 * Release an RPC server buffer
 */
static void
svc_release_buffer(struct svc_rqst *rqstp)
{
527 528 529
	unsigned int i;

	for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
530 531
		if (rqstp->rq_pages[i])
			put_page(rqstp->rq_pages[i]);
L
Linus Torvalds 已提交
532 533
}

534 535
struct svc_rqst *
svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool)
L
Linus Torvalds 已提交
536 537 538
{
	struct svc_rqst	*rqstp;

539
	rqstp = kzalloc(sizeof(*rqstp), GFP_KERNEL);
L
Linus Torvalds 已提交
540
	if (!rqstp)
541
		goto out_enomem;
L
Linus Torvalds 已提交
542 543 544 545

	init_waitqueue_head(&rqstp->rq_wait);

	serv->sv_nrthreads++;
546 547
	spin_lock_bh(&pool->sp_lock);
	pool->sp_nrthreads++;
548
	list_add(&rqstp->rq_all, &pool->sp_all_threads);
549
	spin_unlock_bh(&pool->sp_lock);
L
Linus Torvalds 已提交
550
	rqstp->rq_server = serv;
551
	rqstp->rq_pool = pool;
552

553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569
	rqstp->rq_argp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
	if (!rqstp->rq_argp)
		goto out_thread;

	rqstp->rq_resp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
	if (!rqstp->rq_resp)
		goto out_thread;

	if (!svc_init_buffer(rqstp, serv->sv_max_mesg))
		goto out_thread;

	return rqstp;
out_thread:
	svc_exit_thread(rqstp);
out_enomem:
	return ERR_PTR(-ENOMEM);
}
570
EXPORT_SYMBOL_GPL(svc_prepare_thread);
571

572 573 574 575 576 577 578 579 580
/*
 * Choose a pool in which to create a new thread, for svc_set_num_threads
 */
static inline struct svc_pool *
choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
{
	if (pool != NULL)
		return pool;

581
	return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
}

/*
 * Choose a thread to kill, for svc_set_num_threads
 */
static inline struct task_struct *
choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
{
	unsigned int i;
	struct task_struct *task = NULL;

	if (pool != NULL) {
		spin_lock_bh(&pool->sp_lock);
	} else {
		/* choose a pool in round-robin fashion */
597 598
		for (i = 0; i < serv->sv_nrpools; i++) {
			pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
599
			spin_lock_bh(&pool->sp_lock);
600 601
			if (!list_empty(&pool->sp_all_threads))
				goto found_pool;
602
			spin_unlock_bh(&pool->sp_lock);
603
		}
604 605 606 607 608 609 610 611 612 613 614 615 616 617
		return NULL;
	}

found_pool:
	if (!list_empty(&pool->sp_all_threads)) {
		struct svc_rqst *rqstp;

		/*
		 * Remove from the pool->sp_all_threads list
		 * so we don't try to kill it again.
		 */
		rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
		list_del_init(&rqstp->rq_all);
		task = rqstp->rq_task;
618
	}
619 620 621 622 623 624 625 626 627 628
	spin_unlock_bh(&pool->sp_lock);

	return task;
}

/*
 * Create or destroy enough new threads to make the number
 * of threads the given number.  If `pool' is non-NULL, applies
 * only to threads in that pool, otherwise round-robins between
 * all pools.  Must be called with a svc_get() reference and
629
 * the BKL or another lock to protect access to svc_serv fields.
630 631 632 633 634 635 636 637 638 639 640
 *
 * Destroying threads relies on the service threads filling in
 * rqstp->rq_task, which only the nfs ones do.  Assumes the serv
 * has been created using svc_create_pooled().
 *
 * Based on code that used to be in nfsd_svc() but tweaked
 * to be pool-aware.
 */
int
svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
{
641 642 643
	struct svc_rqst	*rqstp;
	struct task_struct *task;
	struct svc_pool *chosen_pool;
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
	int error = 0;
	unsigned int state = serv->sv_nrthreads-1;

	if (pool == NULL) {
		/* The -1 assumes caller has done a svc_get() */
		nrservs -= (serv->sv_nrthreads-1);
	} else {
		spin_lock_bh(&pool->sp_lock);
		nrservs -= pool->sp_nrthreads;
		spin_unlock_bh(&pool->sp_lock);
	}

	/* create new threads */
	while (nrservs > 0) {
		nrservs--;
659 660 661 662 663 664 665 666
		chosen_pool = choose_pool(serv, pool, &state);

		rqstp = svc_prepare_thread(serv, chosen_pool);
		if (IS_ERR(rqstp)) {
			error = PTR_ERR(rqstp);
			break;
		}

667
		__module_get(serv->sv_module);
668 669 670
		task = kthread_create(serv->sv_function, rqstp, serv->sv_name);
		if (IS_ERR(task)) {
			error = PTR_ERR(task);
671
			module_put(serv->sv_module);
672
			svc_exit_thread(rqstp);
673 674
			break;
		}
675 676 677 678 679 680 681

		rqstp->rq_task = task;
		if (serv->sv_nrpools > 1)
			svc_pool_map_set_cpumask(task, chosen_pool->sp_id);

		svc_sock_update_bufs(serv);
		wake_up_process(task);
682 683 684
	}
	/* destroy old threads */
	while (nrservs < 0 &&
685
	       (task = choose_victim(serv, pool, &state)) != NULL) {
686
		send_sig(SIGINT, task, 1);
687 688 689 690 691
		nrservs++;
	}

	return error;
}
692
EXPORT_SYMBOL_GPL(svc_set_num_threads);
693

694
/*
695 696
 * Called from a server thread as it's exiting. Caller must hold the BKL or
 * the "service mutex", whichever is appropriate for the service.
L
Linus Torvalds 已提交
697 698 699 700 701
 */
void
svc_exit_thread(struct svc_rqst *rqstp)
{
	struct svc_serv	*serv = rqstp->rq_server;
702
	struct svc_pool	*pool = rqstp->rq_pool;
L
Linus Torvalds 已提交
703 704

	svc_release_buffer(rqstp);
J
Jesper Juhl 已提交
705 706 707
	kfree(rqstp->rq_resp);
	kfree(rqstp->rq_argp);
	kfree(rqstp->rq_auth_data);
708 709 710

	spin_lock_bh(&pool->sp_lock);
	pool->sp_nrthreads--;
711
	list_del(&rqstp->rq_all);
712 713
	spin_unlock_bh(&pool->sp_lock);

L
Linus Torvalds 已提交
714 715 716 717 718 719
	kfree(rqstp);

	/* Release the server */
	if (serv)
		svc_destroy(serv);
}
720
EXPORT_SYMBOL_GPL(svc_exit_thread);
L
Linus Torvalds 已提交
721

722
#ifdef CONFIG_SUNRPC_REGISTER_V4
723

L
Linus Torvalds 已提交
724
/*
725 726
 * Register an "inet" protocol family netid with the local
 * rpcbind daemon via an rpcbind v4 SET request.
727
 *
728 729
 * No netconfig infrastructure is available in the kernel, so
 * we map IP_ protocol numbers to netids by hand.
730
 *
731 732
 * Returns zero on success; a negative errno value is returned
 * if any error occurs.
L
Linus Torvalds 已提交
733
 */
734 735 736
static int __svc_rpcb_register4(const u32 program, const u32 version,
				const unsigned short protocol,
				const unsigned short port)
737 738 739 740 741 742
{
	struct sockaddr_in sin = {
		.sin_family		= AF_INET,
		.sin_addr.s_addr	= htonl(INADDR_ANY),
		.sin_port		= htons(port),
	};
743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
	char *netid;

	switch (protocol) {
	case IPPROTO_UDP:
		netid = RPCBIND_NETID_UDP;
		break;
	case IPPROTO_TCP:
		netid = RPCBIND_NETID_TCP;
		break;
	default:
		return -EPROTONOSUPPORT;
	}

	return rpcb_v4_register(program, version,
				(struct sockaddr *)&sin, netid);
}

/*
 * Register an "inet6" protocol family netid with the local
 * rpcbind daemon via an rpcbind v4 SET request.
 *
 * No netconfig infrastructure is available in the kernel, so
 * we map IP_ protocol numbers to netids by hand.
 *
 * Returns zero on success; a negative errno value is returned
 * if any error occurs.
 */
static int __svc_rpcb_register6(const u32 program, const u32 version,
				const unsigned short protocol,
				const unsigned short port)
{
774 775 776 777 778 779 780
	struct sockaddr_in6 sin6 = {
		.sin6_family		= AF_INET6,
		.sin6_addr		= IN6ADDR_ANY_INIT,
		.sin6_port		= htons(port),
	};
	char *netid;

781 782 783
	switch (protocol) {
	case IPPROTO_UDP:
		netid = RPCBIND_NETID_UDP6;
784
		break;
785
	case IPPROTO_TCP:
786 787 788
		netid = RPCBIND_NETID_TCP6;
		break;
	default:
789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
		return -EPROTONOSUPPORT;
	}

	return rpcb_v4_register(program, version,
				(struct sockaddr *)&sin6, netid);
}

/*
 * Register a kernel RPC service via rpcbind version 4.
 *
 * Returns zero on success; a negative errno value is returned
 * if any error occurs.
 */
static int __svc_register(const u32 program, const u32 version,
			  const sa_family_t family,
			  const unsigned short protocol,
			  const unsigned short port)
{
	int error;

	switch (family) {
	case AF_INET:
		return __svc_rpcb_register4(program, version,
						protocol, port);
	case AF_INET6:
		error = __svc_rpcb_register6(program, version,
						protocol, port);
		if (error < 0)
			return error;

		/*
		 * Work around bug in some versions of Linux rpcbind
		 * which don't allow registration of both inet and
		 * inet6 netids.
		 *
		 * Error return ignored for now.
		 */
		__svc_rpcb_register4(program, version,
						protocol, port);
		return 0;
829 830
	}

831
	return -EAFNOSUPPORT;
832
}
833 834 835 836 837 838 839 840 841

#else	/* CONFIG_SUNRPC_REGISTER_V4 */

/*
 * Register a kernel RPC service via rpcbind version 2.
 *
 * Returns zero on success; a negative errno value is returned
 * if any error occurs.
 */
842 843 844 845 846 847 848 849 850 851
static int __svc_register(const u32 program, const u32 version,
			  sa_family_t family,
			  const unsigned short protocol,
			  const unsigned short port)
{
	if (family != AF_INET)
		return -EAFNOSUPPORT;

	return rpcb_register(program, version, protocol, port);
}
852 853

#endif /* CONFIG_SUNRPC_REGISTER_V4 */
854 855 856 857 858 859 860 861 862 863 864

/**
 * svc_register - register an RPC service with the local portmapper
 * @serv: svc_serv struct for the service to register
 * @proto: transport protocol number to advertise
 * @port: port to advertise
 *
 * Service is registered for any address in serv's address family
 */
int svc_register(const struct svc_serv *serv, const unsigned short proto,
		 const unsigned short port)
L
Linus Torvalds 已提交
865 866
{
	struct svc_program	*progp;
867
	unsigned int		i;
868
	int			error = 0;
L
Linus Torvalds 已提交
869

870
	BUG_ON(proto == 0 && port == 0);
L
Linus Torvalds 已提交
871

872 873 874 875 876
	for (progp = serv->sv_program; progp; progp = progp->pg_next) {
		for (i = 0; i < progp->pg_nvers; i++) {
			if (progp->pg_vers[i] == NULL)
				continue;

877
			dprintk("svc: svc_register(%sv%d, %s, %u, %u)%s\n",
878
					progp->pg_name,
879
					i,
880 881
					proto == IPPROTO_UDP?  "udp" : "tcp",
					port,
882
					serv->sv_family,
883 884 885 886 887 888
					progp->pg_vers[i]->vs_hidden?
						" (but not telling portmap)" : "");

			if (progp->pg_vers[i]->vs_hidden)
				continue;

889 890
			error = __svc_register(progp->pg_prog, i,
						serv->sv_family, proto, port);
891 892
			if (error < 0)
				break;
L
Linus Torvalds 已提交
893 894 895
		}
	}

896 897 898
	return error;
}

C
Chuck Lever 已提交
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
#ifdef CONFIG_SUNRPC_REGISTER_V4

static void __svc_unregister(const u32 program, const u32 version,
			     const char *progname)
{
	struct sockaddr_in6 sin6 = {
		.sin6_family		= AF_INET6,
		.sin6_addr		= IN6ADDR_ANY_INIT,
		.sin6_port		= 0,
	};
	int error;

	error = rpcb_v4_register(program, version,
				(struct sockaddr *)&sin6, "");
	dprintk("svc: %s(%sv%u), error %d\n",
			__func__, progname, version, error);
}

#else	/* CONFIG_SUNRPC_REGISTER_V4 */

static void __svc_unregister(const u32 program, const u32 version,
			     const char *progname)
{
	int error;

	error = rpcb_register(program, version, 0, 0);
	dprintk("svc: %s(%sv%u), error %d\n",
			__func__, progname, version, error);
}

#endif	/* CONFIG_SUNRPC_REGISTER_V4 */

931
/*
C
Chuck Lever 已提交
932 933 934
 * All netids, bind addresses and ports registered for [program, version]
 * are removed from the local rpcbind database (if the service is not
 * hidden) to make way for a new instance of the service.
935
 *
C
Chuck Lever 已提交
936 937
 * The result of unregistration is reported via dprintk for those who want
 * verification of the result, but is otherwise not important.
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953
 */
static void svc_unregister(const struct svc_serv *serv)
{
	struct svc_program *progp;
	unsigned long flags;
	unsigned int i;

	clear_thread_flag(TIF_SIGPENDING);

	for (progp = serv->sv_program; progp; progp = progp->pg_next) {
		for (i = 0; i < progp->pg_nvers; i++) {
			if (progp->pg_vers[i] == NULL)
				continue;
			if (progp->pg_vers[i]->vs_hidden)
				continue;

C
Chuck Lever 已提交
954
			__svc_unregister(progp->pg_prog, i, progp->pg_name);
955
		}
L
Linus Torvalds 已提交
956 957
	}

958 959 960
	spin_lock_irqsave(&current->sighand->siglock, flags);
	recalc_sigpending();
	spin_unlock_irqrestore(&current->sighand->siglock, flags);
L
Linus Torvalds 已提交
961 962
}

963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
/*
 * Printk the given error with the address of the client that caused it.
 */
static int
__attribute__ ((format (printf, 2, 3)))
svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
{
	va_list args;
	int 	r;
	char 	buf[RPC_MAX_ADDRBUFLEN];

	if (!net_ratelimit())
		return 0;

	printk(KERN_WARNING "svc: %s: ",
		svc_print_addr(rqstp, buf, sizeof(buf)));

	va_start(args, fmt);
	r = vprintk(fmt, args);
	va_end(args);

	return r;
}

L
Linus Torvalds 已提交
987 988 989 990
/*
 * Process the RPC request.
 */
int
991
svc_process(struct svc_rqst *rqstp)
L
Linus Torvalds 已提交
992 993 994 995 996 997
{
	struct svc_program	*progp;
	struct svc_version	*versp = NULL;	/* compiler food */
	struct svc_procedure	*procp = NULL;
	struct kvec *		argv = &rqstp->rq_arg.head[0];
	struct kvec *		resv = &rqstp->rq_res.head[0];
998
	struct svc_serv		*serv = rqstp->rq_server;
L
Linus Torvalds 已提交
999
	kxdrproc_t		xdr;
1000 1001 1002
	__be32			*statp;
	u32			dir, prog, vers, proc;
	__be32			auth_stat, rpc_stat;
L
Linus Torvalds 已提交
1003
	int			auth_res;
1004
	__be32			*reply_statp;
L
Linus Torvalds 已提交
1005 1006 1007 1008 1009 1010 1011

	rpc_stat = rpc_success;

	if (argv->iov_len < 6*4)
		goto err_short_len;

	/* setup response xdr_buf.
1012
	 * Initially it has just one page
L
Linus Torvalds 已提交
1013
	 */
1014
	rqstp->rq_resused = 1;
L
Linus Torvalds 已提交
1015 1016
	resv->iov_base = page_address(rqstp->rq_respages[0]);
	resv->iov_len = 0;
1017
	rqstp->rq_res.pages = rqstp->rq_respages + 1;
L
Linus Torvalds 已提交
1018 1019 1020
	rqstp->rq_res.len = 0;
	rqstp->rq_res.page_base = 0;
	rqstp->rq_res.page_len = 0;
1021
	rqstp->rq_res.buflen = PAGE_SIZE;
1022
	rqstp->rq_res.tail[0].iov_base = NULL;
L
Linus Torvalds 已提交
1023
	rqstp->rq_res.tail[0].iov_len = 0;
1024
	/* Will be turned off only in gss privacy case: */
1025
	rqstp->rq_splice_ok = 1;
T
Tom Tucker 已提交
1026 1027 1028

	/* Setup reply header */
	rqstp->rq_xprt->xpt_ops->xpo_prep_reply_hdr(rqstp);
L
Linus Torvalds 已提交
1029 1030 1031 1032

	rqstp->rq_xid = svc_getu32(argv);
	svc_putu32(resv, rqstp->rq_xid);

A
Alexey Dobriyan 已提交
1033 1034
	dir  = svc_getnl(argv);
	vers = svc_getnl(argv);
L
Linus Torvalds 已提交
1035 1036

	/* First words of reply: */
A
Alexey Dobriyan 已提交
1037
	svc_putnl(resv, 1);		/* REPLY */
L
Linus Torvalds 已提交
1038 1039 1040 1041 1042 1043 1044

	if (dir != 0)		/* direction != CALL */
		goto err_bad_dir;
	if (vers != 2)		/* RPC version number */
		goto err_bad_rpc;

	/* Save position in case we later decide to reject: */
1045
	reply_statp = resv->iov_base + resv->iov_len;
L
Linus Torvalds 已提交
1046

A
Alexey Dobriyan 已提交
1047
	svc_putnl(resv, 0);		/* ACCEPT */
L
Linus Torvalds 已提交
1048

A
Alexey Dobriyan 已提交
1049 1050 1051
	rqstp->rq_prog = prog = svc_getnl(argv);	/* program number */
	rqstp->rq_vers = vers = svc_getnl(argv);	/* version number */
	rqstp->rq_proc = proc = svc_getnl(argv);	/* procedure number */
L
Linus Torvalds 已提交
1052 1053

	progp = serv->sv_program;
1054 1055 1056 1057 1058

	for (progp = serv->sv_program; progp; progp = progp->pg_next)
		if (prog == progp->pg_prog)
			break;

L
Linus Torvalds 已提交
1059 1060 1061 1062 1063 1064 1065
	/*
	 * Decode auth data, and add verifier to reply buffer.
	 * We do this before anything else in order to get a decent
	 * auth verifier.
	 */
	auth_res = svc_authenticate(rqstp, &auth_stat);
	/* Also give the program a chance to reject this call: */
1066
	if (auth_res == SVC_OK && progp) {
L
Linus Torvalds 已提交
1067 1068 1069 1070 1071 1072 1073
		auth_stat = rpc_autherr_badcred;
		auth_res = progp->pg_authenticate(rqstp);
	}
	switch (auth_res) {
	case SVC_OK:
		break;
	case SVC_GARBAGE:
1074
		goto err_garbage;
L
Linus Torvalds 已提交
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
	case SVC_SYSERR:
		rpc_stat = rpc_system_err;
		goto err_bad;
	case SVC_DENIED:
		goto err_bad_auth;
	case SVC_DROP:
		goto dropit;
	case SVC_COMPLETE:
		goto sendit;
	}
1085

1086
	if (progp == NULL)
L
Linus Torvalds 已提交
1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
		goto err_bad_prog;

	if (vers >= progp->pg_nvers ||
	  !(versp = progp->pg_vers[vers]))
		goto err_bad_vers;

	procp = versp->vs_proc + proc;
	if (proc >= versp->vs_nproc || !procp->pc_func)
		goto err_bad_proc;
	rqstp->rq_server   = serv;
	rqstp->rq_procinfo = procp;

	/* Syntactic check complete */
	serv->sv_stats->rpccnt++;

	/* Build the reply header. */
	statp = resv->iov_base +resv->iov_len;
A
Alexey Dobriyan 已提交
1104
	svc_putnl(resv, RPC_SUCCESS);
L
Linus Torvalds 已提交
1105 1106 1107 1108 1109 1110 1111 1112

	/* Bump per-procedure stats counter */
	procp->pc_count++;

	/* Initialize storage for argp and resp */
	memset(rqstp->rq_argp, 0, procp->pc_argsize);
	memset(rqstp->rq_resp, 0, procp->pc_ressize);

1113
	/* un-reserve some of the out-queue now that we have a
L
Linus Torvalds 已提交
1114 1115 1116
	 * better idea of reply size
	 */
	if (procp->pc_xdrressize)
1117
		svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
L
Linus Torvalds 已提交
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128

	/* Call the function that processes the request. */
	if (!versp->vs_dispatch) {
		/* Decode arguments */
		xdr = procp->pc_decode;
		if (xdr && !xdr(rqstp, argv->iov_base, rqstp->rq_argp))
			goto err_garbage;

		*statp = procp->pc_func(rqstp, rqstp->rq_argp, rqstp->rq_resp);

		/* Encode reply */
1129 1130 1131 1132 1133
		if (*statp == rpc_drop_reply) {
			if (procp->pc_release)
				procp->pc_release(rqstp, NULL, rqstp->rq_resp);
			goto dropit;
		}
L
Linus Torvalds 已提交
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
		if (*statp == rpc_success && (xdr = procp->pc_encode)
		 && !xdr(rqstp, resv->iov_base+resv->iov_len, rqstp->rq_resp)) {
			dprintk("svc: failed to encode reply\n");
			/* serv->sv_stats->rpcsystemerr++; */
			*statp = rpc_system_err;
		}
	} else {
		dprintk("svc: calling dispatcher\n");
		if (!versp->vs_dispatch(rqstp, statp)) {
			/* Release reply info */
			if (procp->pc_release)
				procp->pc_release(rqstp, NULL, rqstp->rq_resp);
			goto dropit;
		}
	}

	/* Check RPC status result */
	if (*statp != rpc_success)
		resv->iov_len = ((void*)statp)  - resv->iov_base + 4;

	/* Release reply info */
	if (procp->pc_release)
		procp->pc_release(rqstp, NULL, rqstp->rq_resp);

	if (procp->pc_encode == NULL)
		goto dropit;

 sendit:
	if (svc_authorise(rqstp))
		goto dropit;
	return svc_send(rqstp);

 dropit:
	svc_authorise(rqstp);	/* doesn't hurt to call this twice */
	dprintk("svc: svc_process dropit\n");
	svc_drop(rqstp);
	return 0;

err_short_len:
1173 1174
	svc_printk(rqstp, "short len %Zd, dropping request\n",
			argv->iov_len);
1175

L
Linus Torvalds 已提交
1176 1177 1178
	goto dropit;			/* drop request */

err_bad_dir:
1179
	svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
1180

L
Linus Torvalds 已提交
1181 1182 1183 1184 1185
	serv->sv_stats->rpcbadfmt++;
	goto dropit;			/* drop request */

err_bad_rpc:
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1186 1187 1188 1189
	svc_putnl(resv, 1);	/* REJECT */
	svc_putnl(resv, 0);	/* RPC_MISMATCH */
	svc_putnl(resv, 2);	/* Only RPCv2 supported */
	svc_putnl(resv, 2);
L
Linus Torvalds 已提交
1190 1191 1192 1193 1194 1195
	goto sendit;

err_bad_auth:
	dprintk("svc: authentication failed (%d)\n", ntohl(auth_stat));
	serv->sv_stats->rpcbadauth++;
	/* Restore write pointer to location of accept status: */
1196
	xdr_ressize_check(rqstp, reply_statp);
A
Alexey Dobriyan 已提交
1197 1198 1199
	svc_putnl(resv, 1);	/* REJECT */
	svc_putnl(resv, 1);	/* AUTH_ERROR */
	svc_putnl(resv, ntohl(auth_stat));	/* status */
L
Linus Torvalds 已提交
1200 1201 1202
	goto sendit;

err_bad_prog:
1203
	dprintk("svc: unknown program %d\n", prog);
L
Linus Torvalds 已提交
1204
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1205
	svc_putnl(resv, RPC_PROG_UNAVAIL);
L
Linus Torvalds 已提交
1206 1207 1208
	goto sendit;

err_bad_vers:
1209
	svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
1210 1211
		       vers, prog, progp->pg_name);

L
Linus Torvalds 已提交
1212
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1213 1214 1215
	svc_putnl(resv, RPC_PROG_MISMATCH);
	svc_putnl(resv, progp->pg_lovers);
	svc_putnl(resv, progp->pg_hivers);
L
Linus Torvalds 已提交
1216 1217 1218
	goto sendit;

err_bad_proc:
1219
	svc_printk(rqstp, "unknown procedure (%d)\n", proc);
1220

L
Linus Torvalds 已提交
1221
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1222
	svc_putnl(resv, RPC_PROC_UNAVAIL);
L
Linus Torvalds 已提交
1223 1224 1225
	goto sendit;

err_garbage:
1226
	svc_printk(rqstp, "failed to decode args\n");
1227

L
Linus Torvalds 已提交
1228 1229 1230
	rpc_stat = rpc_garbage_args;
err_bad:
	serv->sv_stats->rpcbadfmt++;
A
Alexey Dobriyan 已提交
1231
	svc_putnl(resv, ntohl(rpc_stat));
L
Linus Torvalds 已提交
1232 1233
	goto sendit;
}
1234
EXPORT_SYMBOL_GPL(svc_process);
1235 1236 1237 1238 1239 1240

/*
 * Return (transport-specific) limit on the rpc payload.
 */
u32 svc_max_payload(const struct svc_rqst *rqstp)
{
1241
	u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
1242

1243 1244
	if (rqstp->rq_server->sv_max_payload < max)
		max = rqstp->rq_server->sv_max_payload;
1245 1246 1247
	return max;
}
EXPORT_SYMBOL_GPL(svc_max_payload);