rcutorture.c 51.2 KB
Newer Older
1
/*
2
 * Read-Copy Update module-based torture test facility
3 4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
15 16
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
17
 *
18
 * Copyright (C) IBM Corporation, 2005, 2006
19 20
 *
 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
J
Josh Triplett 已提交
21
 *	  Josh Triplett <josh@joshtriplett.org>
22 23 24 25 26 27 28 29 30 31 32 33 34 35
 *
 * See also:  Documentation/RCU/torture.txt
 */
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/err.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/rcupdate.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
A
Arun Sharma 已提交
36
#include <linux/atomic.h>
37 38 39 40 41
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/moduleparam.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
42
#include <linux/reboot.h>
43
#include <linux/freezer.h>
44 45 46
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/stat.h>
47
#include <linux/srcu.h>
48
#include <linux/slab.h>
49
#include <linux/trace_clock.h>
50
#include <asm/byteorder.h>
51
#include <linux/torture.h>
52
#include <linux/vmalloc.h>
53 54

MODULE_LICENSE("GPL");
J
Josh Triplett 已提交
55
MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
56

57

58 59 60 61 62 63 64
torture_param(int, cbflood_inter_holdoff, HZ,
	      "Holdoff between floods (jiffies)");
torture_param(int, cbflood_intra_holdoff, 1,
	      "Holdoff between bursts (jiffies)");
torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable");
torture_param(int, cbflood_n_per_burst, 20000,
	      "# callbacks per burst in flood");
65 66 67 68
torture_param(int, fqs_duration, 0,
	      "Duration of fqs bursts (us), 0 to disable");
torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
69
torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
70 71 72
torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
torture_param(bool, gp_normal, false,
	     "Use normal (non-expedited) GP wait primitives");
73
torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
torture_param(int, n_barrier_cbs, 0,
	     "# of callbacks/kthreads for barrier testing");
torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
torture_param(int, nreaders, -1, "Number of RCU reader threads");
torture_param(int, object_debug, 0,
	     "Enable debug-object double call_rcu() testing");
torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
torture_param(int, onoff_interval, 0,
	     "Time between CPU hotplugs (s), 0=disable");
torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
torture_param(int, stall_cpu_holdoff, 10,
	     "Time to wait before starting stall (s).");
torture_param(int, stat_interval, 60,
	     "Number of seconds between stats printk()s");
torture_param(int, stutter, 5, "Number of seconds to run/halt test");
torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
torture_param(int, test_boost_duration, 4,
	     "Duration of each boost test, seconds.");
torture_param(int, test_boost_interval, 7,
	     "Interval between boost tests, seconds.");
torture_param(bool, test_no_idle_hz, true,
	     "Test support for tickless idle CPUs");
99 100
torture_param(bool, verbose, true,
	     "Enable verbose debugging printk()s");
101

102
static char *torture_type = "rcu";
103
module_param(torture_type, charp, 0444);
104
MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)");
105 106

static int nrealreaders;
107
static int ncbflooders;
108
static struct task_struct *writer_task;
109
static struct task_struct **fakewriter_tasks;
110 111
static struct task_struct **reader_tasks;
static struct task_struct *stats_task;
112
static struct task_struct **cbflood_task;
113
static struct task_struct *fqs_task;
114
static struct task_struct *boost_tasks[NR_CPUS];
115
static struct task_struct *stall_task;
116 117
static struct task_struct **barrier_cbs_tasks;
static struct task_struct *barrier_task;
118 119 120 121 122 123 124

#define RCU_TORTURE_PIPE_LEN 10

struct rcu_torture {
	struct rcu_head rtort_rcu;
	int rtort_pipe_count;
	struct list_head rtort_free;
125
	int rtort_mbtest;
126 127 128
};

static LIST_HEAD(rcu_torture_freelist);
129
static struct rcu_torture __rcu *rcu_torture_current;
130
static unsigned long rcu_torture_current_version;
131 132
static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
static DEFINE_SPINLOCK(rcu_torture_lock);
133 134 135 136
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1],
		      rcu_torture_count) = { 0 };
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1],
		      rcu_torture_batch) = { 0 };
137
static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
138 139 140 141 142
static atomic_t n_rcu_torture_alloc;
static atomic_t n_rcu_torture_alloc_fail;
static atomic_t n_rcu_torture_free;
static atomic_t n_rcu_torture_mberror;
static atomic_t n_rcu_torture_error;
143
static long n_rcu_torture_barrier_error;
144 145 146 147
static long n_rcu_torture_boost_ktrerror;
static long n_rcu_torture_boost_rterror;
static long n_rcu_torture_boost_failure;
static long n_rcu_torture_boosts;
148
static long n_rcu_torture_timers;
149 150
static long n_barrier_attempts;
static long n_barrier_successes;
151
static atomic_long_t n_cbfloods;
152
static struct list_head rcu_torture_removed;
153

154 155 156 157 158 159
static int rcu_torture_writer_state;
#define RTWS_FIXED_DELAY	0
#define RTWS_DELAY		1
#define RTWS_REPLACE		2
#define RTWS_DEF_FREE		3
#define RTWS_EXP_SYNC		4
160 161
#define RTWS_COND_GET		5
#define RTWS_COND_SYNC		6
162 163 164
#define RTWS_SYNC		7
#define RTWS_STUTTER		8
#define RTWS_STOPPING		9
165

166 167 168 169 170 171
#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
#define RCUTORTURE_RUNNABLE_INIT 1
#else
#define RCUTORTURE_RUNNABLE_INIT 0
#endif
int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
172 173
module_param(rcutorture_runnable, int, 0444);
MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot");
174

175
#if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
176
#define rcu_can_boost() 1
177
#else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
178
#define rcu_can_boost() 0
179
#endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
180

181 182 183 184 185 186 187 188 189 190 191 192 193 194
#ifdef CONFIG_RCU_TRACE
static u64 notrace rcu_trace_clock_local(void)
{
	u64 ts = trace_clock_local();
	unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC);
	return ts;
}
#else /* #ifdef CONFIG_RCU_TRACE */
static u64 notrace rcu_trace_clock_local(void)
{
	return 0ULL;
}
#endif /* #else #ifdef CONFIG_RCU_TRACE */

195
static unsigned long boost_starttime;	/* jiffies of next boost test start. */
196
static DEFINE_MUTEX(boost_mutex);	/* protect setting boost_starttime */
197
					/*  and boost task create/destroy. */
198
static atomic_t barrier_cbs_count;	/* Barrier callbacks registered. */
199
static bool barrier_phase;		/* Test phase. */
200 201 202
static atomic_t barrier_cbs_invoked;	/* Barrier callbacks invoked. */
static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
203

204 205 206
/*
 * Allocate an element from the rcu_tortures pool.
 */
A
Adrian Bunk 已提交
207
static struct rcu_torture *
208 209 210 211
rcu_torture_alloc(void)
{
	struct list_head *p;

212
	spin_lock_bh(&rcu_torture_lock);
213 214
	if (list_empty(&rcu_torture_freelist)) {
		atomic_inc(&n_rcu_torture_alloc_fail);
215
		spin_unlock_bh(&rcu_torture_lock);
216 217 218 219 220
		return NULL;
	}
	atomic_inc(&n_rcu_torture_alloc);
	p = rcu_torture_freelist.next;
	list_del_init(p);
221
	spin_unlock_bh(&rcu_torture_lock);
222 223 224 225 226 227 228 229 230 231
	return container_of(p, struct rcu_torture, rtort_free);
}

/*
 * Free an element to the rcu_tortures pool.
 */
static void
rcu_torture_free(struct rcu_torture *p)
{
	atomic_inc(&n_rcu_torture_free);
232
	spin_lock_bh(&rcu_torture_lock);
233
	list_add_tail(&p->rtort_free, &rcu_torture_freelist);
234
	spin_unlock_bh(&rcu_torture_lock);
235 236
}

237 238 239 240 241
/*
 * Operations vector for selecting different types of tests.
 */

struct rcu_torture_ops {
242
	int ttype;
243 244
	void (*init)(void);
	int (*readlock)(void);
245
	void (*read_delay)(struct torture_random_state *rrsp);
246 247
	void (*readunlock)(int idx);
	int (*completed)(void);
248
	void (*deferred_free)(struct rcu_torture *p);
249
	void (*sync)(void);
250
	void (*exp_sync)(void);
251 252
	unsigned long (*get_state)(void);
	void (*cond_sync)(unsigned long oldstate);
253
	void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
254
	void (*cb_barrier)(void);
255
	void (*fqs)(void);
256
	void (*stats)(void);
257
	int irq_capable;
258
	int can_boost;
259
	const char *name;
260
};
261 262

static struct rcu_torture_ops *cur_ops;
263 264 265 266 267

/*
 * Definitions for rcu torture testing.
 */

268
static int rcu_torture_read_lock(void) __acquires(RCU)
269 270 271 272 273
{
	rcu_read_lock();
	return 0;
}

274
static void rcu_read_delay(struct torture_random_state *rrsp)
275
{
276 277
	const unsigned long shortdelay_us = 200;
	const unsigned long longdelay_ms = 50;
278

279 280 281
	/* We want a short delay sometimes to make a reader delay the grace
	 * period, and we want a long delay occasionally to trigger
	 * force_quiescent_state. */
282

283
	if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
284
		mdelay(longdelay_ms);
285
	if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
286
		udelay(shortdelay_us);
287
#ifdef CONFIG_PREEMPT
288 289
	if (!preempt_count() &&
	    !(torture_random(rrsp) % (nrealreaders * 20000)))
290 291
		preempt_schedule();  /* No QS if preempt_disable() in effect */
#endif
292 293
}

294
static void rcu_torture_read_unlock(int idx) __releases(RCU)
295 296 297 298 299 300 301 302 303
{
	rcu_read_unlock();
}

static int rcu_torture_completed(void)
{
	return rcu_batches_completed();
}

304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
/*
 * Update callback in the pipe.  This should be invoked after a grace period.
 */
static bool
rcu_torture_pipe_update_one(struct rcu_torture *rp)
{
	int i;

	i = rp->rtort_pipe_count;
	if (i > RCU_TORTURE_PIPE_LEN)
		i = RCU_TORTURE_PIPE_LEN;
	atomic_inc(&rcu_torture_wcount[i]);
	if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
		rp->rtort_mbtest = 0;
		return true;
	}
	return false;
}

/*
 * Update all callbacks in the pipe.  Suitable for synchronous grace-period
 * primitives.
 */
static void
rcu_torture_pipe_update(struct rcu_torture *old_rp)
{
	struct rcu_torture *rp;
	struct rcu_torture *rp1;

	if (old_rp)
		list_add(&old_rp->rtort_free, &rcu_torture_removed);
	list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
		if (rcu_torture_pipe_update_one(rp)) {
			list_del(&rp->rtort_free);
			rcu_torture_free(rp);
		}
	}
}

343 344 345 346 347
static void
rcu_torture_cb(struct rcu_head *p)
{
	struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);

348
	if (torture_must_stop_irq()) {
349 350 351 352
		/* Test is ending, just drop callbacks on the floor. */
		/* The next initialization will pick up the pieces. */
		return;
	}
353
	if (rcu_torture_pipe_update_one(rp))
354
		rcu_torture_free(rp);
355
	else
356
		cur_ops->deferred_free(rp);
357 358
}

359 360 361 362 363
static int rcu_no_completed(void)
{
	return 0;
}

364 365 366 367 368
static void rcu_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu(&p->rtort_rcu, rcu_torture_cb);
}

369 370 371 372 373
static void rcu_sync_torture_init(void)
{
	INIT_LIST_HEAD(&rcu_torture_removed);
}

374
static struct rcu_torture_ops rcu_ops = {
375
	.ttype		= RCU_FLAVOR,
376 377 378 379 380
	.init		= rcu_sync_torture_init,
	.readlock	= rcu_torture_read_lock,
	.read_delay	= rcu_read_delay,
	.readunlock	= rcu_torture_read_unlock,
	.completed	= rcu_torture_completed,
381
	.deferred_free	= rcu_torture_deferred_free,
382
	.sync		= synchronize_rcu,
383
	.exp_sync	= synchronize_rcu_expedited,
384 385
	.get_state	= get_state_synchronize_rcu,
	.cond_sync	= cond_synchronize_rcu,
386 387
	.call		= call_rcu,
	.cb_barrier	= rcu_barrier,
388
	.fqs		= rcu_force_quiescent_state,
389 390
	.stats		= NULL,
	.irq_capable	= 1,
391
	.can_boost	= rcu_can_boost(),
392
	.name		= "rcu"
393 394
};

395 396 397 398
/*
 * Definitions for rcu_bh torture testing.
 */

399
static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
400 401 402 403 404
{
	rcu_read_lock_bh();
	return 0;
}

405
static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
{
	rcu_read_unlock_bh();
}

static int rcu_bh_torture_completed(void)
{
	return rcu_batches_completed_bh();
}

static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
}

static struct rcu_torture_ops rcu_bh_ops = {
421
	.ttype		= RCU_BH_FLAVOR,
422
	.init		= rcu_sync_torture_init,
423 424 425 426 427
	.readlock	= rcu_bh_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= rcu_bh_torture_read_unlock,
	.completed	= rcu_bh_torture_completed,
	.deferred_free	= rcu_bh_torture_deferred_free,
428
	.sync		= synchronize_rcu_bh,
429
	.exp_sync	= synchronize_rcu_bh_expedited,
430
	.call		= call_rcu_bh,
431
	.cb_barrier	= rcu_barrier_bh,
432
	.fqs		= rcu_bh_force_quiescent_state,
433 434 435
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "rcu_bh"
436 437
};

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
/*
 * Don't even think about trying any of these in real life!!!
 * The names includes "busted", and they really means it!
 * The only purpose of these functions is to provide a buggy RCU
 * implementation to make sure that rcutorture correctly emits
 * buggy-RCU error messages.
 */
static void rcu_busted_torture_deferred_free(struct rcu_torture *p)
{
	/* This is a deliberate bug for testing purposes only! */
	rcu_torture_cb(&p->rtort_rcu);
}

static void synchronize_rcu_busted(void)
{
	/* This is a deliberate bug for testing purposes only! */
}

static void
call_rcu_busted(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
{
	/* This is a deliberate bug for testing purposes only! */
	func(head);
}

static struct rcu_torture_ops rcu_busted_ops = {
464
	.ttype		= INVALID_RCU_FLAVOR,
465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
	.init		= rcu_sync_torture_init,
	.readlock	= rcu_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= rcu_torture_read_unlock,
	.completed	= rcu_no_completed,
	.deferred_free	= rcu_busted_torture_deferred_free,
	.sync		= synchronize_rcu_busted,
	.exp_sync	= synchronize_rcu_busted,
	.call		= call_rcu_busted,
	.cb_barrier	= NULL,
	.fqs		= NULL,
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "rcu_busted"
};

481 482 483 484
/*
 * Definitions for srcu torture testing.
 */

485
DEFINE_STATIC_SRCU(srcu_ctl);
486

487
static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
488 489 490 491
{
	return srcu_read_lock(&srcu_ctl);
}

492
static void srcu_read_delay(struct torture_random_state *rrsp)
493 494 495 496 497 498 499
{
	long delay;
	const long uspertick = 1000000 / HZ;
	const long longdelay = 10;

	/* We want there to be long-running readers, but not all the time. */

500 501
	delay = torture_random(rrsp) %
		(nrealreaders * 2 * longdelay * uspertick);
502 503
	if (!delay)
		schedule_timeout_interruptible(longdelay);
504 505
	else
		rcu_read_delay(rrsp);
506 507
}

508
static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
509 510 511 512 513 514 515 516 517
{
	srcu_read_unlock(&srcu_ctl, idx);
}

static int srcu_torture_completed(void)
{
	return srcu_batches_completed(&srcu_ctl);
}

518 519 520 521 522
static void srcu_torture_deferred_free(struct rcu_torture *rp)
{
	call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb);
}

523 524 525 526 527
static void srcu_torture_synchronize(void)
{
	synchronize_srcu(&srcu_ctl);
}

528 529 530 531 532 533 534 535 536 537 538
static void srcu_torture_call(struct rcu_head *head,
			      void (*func)(struct rcu_head *head))
{
	call_srcu(&srcu_ctl, head, func);
}

static void srcu_torture_barrier(void)
{
	srcu_barrier(&srcu_ctl);
}

539
static void srcu_torture_stats(void)
540 541 542 543
{
	int cpu;
	int idx = srcu_ctl.completed & 0x1;

544 545
	pr_alert("%s%s per-CPU(idx=%d):",
		 torture_type, TORTURE_FLAG, idx);
546
	for_each_possible_cpu(cpu) {
547 548 549 550
		long c0, c1;

		c0 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx];
		c1 = (long)per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx];
551
		pr_cont(" %d(%ld,%ld)", cpu, c0, c1);
552
	}
553
	pr_cont("\n");
554 555
}

556 557 558 559 560
static void srcu_torture_synchronize_expedited(void)
{
	synchronize_srcu_expedited(&srcu_ctl);
}

561
static struct rcu_torture_ops srcu_ops = {
562
	.ttype		= SRCU_FLAVOR,
563
	.init		= rcu_sync_torture_init,
564 565 566 567
	.readlock	= srcu_torture_read_lock,
	.read_delay	= srcu_read_delay,
	.readunlock	= srcu_torture_read_unlock,
	.completed	= srcu_torture_completed,
568
	.deferred_free	= srcu_torture_deferred_free,
569
	.sync		= srcu_torture_synchronize,
570
	.exp_sync	= srcu_torture_synchronize_expedited,
571 572
	.call		= srcu_torture_call,
	.cb_barrier	= srcu_torture_barrier,
573 574
	.stats		= srcu_torture_stats,
	.name		= "srcu"
575 576
};

577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
/*
 * Definitions for sched torture testing.
 */

static int sched_torture_read_lock(void)
{
	preempt_disable();
	return 0;
}

static void sched_torture_read_unlock(int idx)
{
	preempt_enable();
}

592 593 594 595 596
static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
}

597
static struct rcu_torture_ops sched_ops = {
598
	.ttype		= RCU_SCHED_FLAVOR,
599 600 601 602
	.init		= rcu_sync_torture_init,
	.readlock	= sched_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= sched_torture_read_unlock,
603
	.completed	= rcu_no_completed,
604
	.deferred_free	= rcu_sched_torture_deferred_free,
605
	.sync		= synchronize_sched,
606 607
	.exp_sync	= synchronize_sched_expedited,
	.call		= call_rcu_sched,
608
	.cb_barrier	= rcu_barrier_sched,
609
	.fqs		= rcu_sched_force_quiescent_state,
610 611 612
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "sched"
613 614
};

615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
#ifdef CONFIG_TASKS_RCU

/*
 * Definitions for RCU-tasks torture testing.
 */

static int tasks_torture_read_lock(void)
{
	return 0;
}

static void tasks_torture_read_unlock(int idx)
{
}

static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb);
}

static struct rcu_torture_ops tasks_ops = {
	.ttype		= RCU_TASKS_FLAVOR,
	.init		= rcu_sync_torture_init,
	.readlock	= tasks_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= tasks_torture_read_unlock,
	.completed	= rcu_no_completed,
	.deferred_free	= rcu_tasks_torture_deferred_free,
	.sync		= synchronize_rcu_tasks,
	.exp_sync	= synchronize_rcu_tasks,
	.call		= call_rcu_tasks,
	.cb_barrier	= rcu_barrier_tasks,
	.fqs		= NULL,
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "tasks"
};

#define RCUTORTURE_TASKS_OPS &tasks_ops,

#else /* #ifdef CONFIG_TASKS_RCU */

#define RCUTORTURE_TASKS_OPS

#endif /* #else #ifdef CONFIG_TASKS_RCU */

661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
/*
 * RCU torture priority-boost testing.  Runs one real-time thread per
 * CPU for moderate bursts, repeatedly registering RCU callbacks and
 * spinning waiting for them to be invoked.  If a given callback takes
 * too long to be invoked, we assume that priority inversion has occurred.
 */

struct rcu_boost_inflight {
	struct rcu_head rcu;
	int inflight;
};

static void rcu_torture_boost_cb(struct rcu_head *head)
{
	struct rcu_boost_inflight *rbip =
		container_of(head, struct rcu_boost_inflight, rcu);

	smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */
	rbip->inflight = 0;
}

static int rcu_torture_boost(void *arg)
{
	unsigned long call_rcu_time;
	unsigned long endtime;
	unsigned long oldstarttime;
	struct rcu_boost_inflight rbi = { .inflight = 0 };
	struct sched_param sp;

690
	VERBOSE_TOROUT_STRING("rcu_torture_boost started");
691 692 693 694

	/* Set real-time priority. */
	sp.sched_priority = 1;
	if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
695
		VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!");
696 697 698
		n_rcu_torture_boost_rterror++;
	}

699
	init_rcu_head_on_stack(&rbi.rcu);
700 701 702 703
	/* Each pass through the following loop does one boost-test cycle. */
	do {
		/* Wait for the next test interval. */
		oldstarttime = boost_starttime;
704
		while (ULONG_CMP_LT(jiffies, oldstarttime)) {
705
			schedule_timeout_interruptible(oldstarttime - jiffies);
706
			stutter_wait("rcu_torture_boost");
707
			if (torture_must_stop())
708 709 710 711 712 713
				goto checkwait;
		}

		/* Do one boost-test interval. */
		endtime = oldstarttime + test_boost_duration * HZ;
		call_rcu_time = jiffies;
714
		while (ULONG_CMP_LT(jiffies, endtime)) {
715 716 717 718 719 720 721
			/* If we don't have a callback in flight, post one. */
			if (!rbi.inflight) {
				smp_mb(); /* RCU core before ->inflight = 1. */
				rbi.inflight = 1;
				call_rcu(&rbi.rcu, rcu_torture_boost_cb);
				if (jiffies - call_rcu_time >
					 test_boost_duration * HZ - HZ / 2) {
722
					VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
723 724 725 726
					n_rcu_torture_boost_failure++;
				}
				call_rcu_time = jiffies;
			}
727
			cond_resched_rcu_qs();
728
			stutter_wait("rcu_torture_boost");
729
			if (torture_must_stop())
730 731 732 733 734 735 736 737 738 739
				goto checkwait;
		}

		/*
		 * Set the start time of the next test interval.
		 * Yes, this is vulnerable to long delays, but such
		 * delays simply cause a false negative for the next
		 * interval.  Besides, we are running at RT priority,
		 * so delays should be relatively rare.
		 */
740 741
		while (oldstarttime == boost_starttime &&
		       !kthread_should_stop()) {
742 743 744 745 746 747 748 749 750 751 752
			if (mutex_trylock(&boost_mutex)) {
				boost_starttime = jiffies +
						  test_boost_interval * HZ;
				n_rcu_torture_boosts++;
				mutex_unlock(&boost_mutex);
				break;
			}
			schedule_timeout_uninterruptible(1);
		}

		/* Go do the stutter. */
753
checkwait:	stutter_wait("rcu_torture_boost");
754
	} while (!torture_must_stop());
755 756

	/* Clean up and exit. */
757 758
	while (!kthread_should_stop() || rbi.inflight) {
		torture_shutdown_absorb("rcu_torture_boost");
759
		schedule_timeout_uninterruptible(1);
760
	}
761
	smp_mb(); /* order accesses to ->inflight before stack-frame death. */
762
	destroy_rcu_head_on_stack(&rbi.rcu);
763
	torture_kthread_stopping("rcu_torture_boost");
764 765 766
	return 0;
}

767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
static void rcu_torture_cbflood_cb(struct rcu_head *rhp)
{
}

/*
 * RCU torture callback-flood kthread.  Repeatedly induces bursts of calls
 * to call_rcu() or analogous, increasing the probability of occurrence
 * of callback-overflow corner cases.
 */
static int
rcu_torture_cbflood(void *arg)
{
	int err = 1;
	int i;
	int j;
	struct rcu_head *rhp;

	if (cbflood_n_per_burst > 0 &&
	    cbflood_inter_holdoff > 0 &&
	    cbflood_intra_holdoff > 0 &&
	    cur_ops->call &&
	    cur_ops->cb_barrier) {
		rhp = vmalloc(sizeof(*rhp) *
			      cbflood_n_burst * cbflood_n_per_burst);
		err = !rhp;
	}
	if (err) {
		VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM");
		while (!torture_must_stop())
			schedule_timeout_interruptible(HZ);
		return 0;
	}
	VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
	do {
		schedule_timeout_interruptible(cbflood_inter_holdoff);
		atomic_long_inc(&n_cbfloods);
		WARN_ON(signal_pending(current));
		for (i = 0; i < cbflood_n_burst; i++) {
			for (j = 0; j < cbflood_n_per_burst; j++) {
				cur_ops->call(&rhp[i * cbflood_n_per_burst + j],
					      rcu_torture_cbflood_cb);
			}
			schedule_timeout_interruptible(cbflood_intra_holdoff);
			WARN_ON(signal_pending(current));
		}
		cur_ops->cb_barrier();
		stutter_wait("rcu_torture_cbflood");
	} while (!torture_must_stop());
	torture_kthread_stopping("rcu_torture_cbflood");
	return 0;
}

819 820 821 822 823 824 825 826 827 828 829
/*
 * RCU torture force-quiescent-state kthread.  Repeatedly induces
 * bursts of calls to force_quiescent_state(), increasing the probability
 * of occurrence of some important types of race conditions.
 */
static int
rcu_torture_fqs(void *arg)
{
	unsigned long fqs_resume_time;
	int fqs_burst_remaining;

830
	VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
831 832
	do {
		fqs_resume_time = jiffies + fqs_stutter * HZ;
833 834
		while (ULONG_CMP_LT(jiffies, fqs_resume_time) &&
		       !kthread_should_stop()) {
835 836 837
			schedule_timeout_interruptible(1);
		}
		fqs_burst_remaining = fqs_duration;
838 839
		while (fqs_burst_remaining > 0 &&
		       !kthread_should_stop()) {
840 841 842 843
			cur_ops->fqs();
			udelay(fqs_holdoff);
			fqs_burst_remaining -= fqs_holdoff;
		}
844
		stutter_wait("rcu_torture_fqs");
845
	} while (!torture_must_stop());
846
	torture_kthread_stopping("rcu_torture_fqs");
847 848 849
	return 0;
}

850 851 852 853 854 855 856 857
/*
 * RCU torture writer kthread.  Repeatedly substitutes a new structure
 * for that pointed to by rcu_torture_current, freeing the old structure
 * after a series of grace periods (the "pipeline").
 */
static int
rcu_torture_writer(void *arg)
{
858 859
	unsigned long gp_snap;
	bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
860
	bool gp_sync1 = gp_sync;
861 862 863
	int i;
	struct rcu_torture *rp;
	struct rcu_torture *old_rp;
864
	static DEFINE_TORTURE_RANDOM(rand);
865 866
	int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC,
			   RTWS_COND_GET, RTWS_SYNC };
867
	int nsynctypes = 0;
868

869
	VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
870

871
	/* Initialize synctype[] array.  If none set, take default. */
872 873
	if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync)
		gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true;
874 875 876 877 878 879 880 881 882 883 884 885
	if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync)
		synctype[nsynctypes++] = RTWS_COND_GET;
	else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync))
		pr_alert("rcu_torture_writer: gp_cond without primitives.\n");
	if (gp_exp1 && cur_ops->exp_sync)
		synctype[nsynctypes++] = RTWS_EXP_SYNC;
	else if (gp_exp && !cur_ops->exp_sync)
		pr_alert("rcu_torture_writer: gp_exp without primitives.\n");
	if (gp_normal1 && cur_ops->deferred_free)
		synctype[nsynctypes++] = RTWS_DEF_FREE;
	else if (gp_normal && !cur_ops->deferred_free)
		pr_alert("rcu_torture_writer: gp_normal without primitives.\n");
886 887 888 889
	if (gp_sync1 && cur_ops->sync)
		synctype[nsynctypes++] = RTWS_SYNC;
	else if (gp_sync && !cur_ops->sync)
		pr_alert("rcu_torture_writer: gp_sync without primitives.\n");
890 891
	if (WARN_ONCE(nsynctypes == 0,
		      "rcu_torture_writer: No update-side primitives.\n")) {
892 893 894 895 896
		/*
		 * No updates primitives, so don't try updating.
		 * The resulting test won't be testing much, hence the
		 * above WARN_ONCE().
		 */
897 898 899 900
		rcu_torture_writer_state = RTWS_STOPPING;
		torture_kthread_stopping("rcu_torture_writer");
	}

901
	do {
902
		rcu_torture_writer_state = RTWS_FIXED_DELAY;
903
		schedule_timeout_uninterruptible(1);
904 905
		rp = rcu_torture_alloc();
		if (rp == NULL)
906 907
			continue;
		rp->rtort_pipe_count = 0;
908
		rcu_torture_writer_state = RTWS_DELAY;
909
		udelay(torture_random(&rand) & 0x3ff);
910
		rcu_torture_writer_state = RTWS_REPLACE;
911 912
		old_rp = rcu_dereference_check(rcu_torture_current,
					       current == writer_task);
913
		rp->rtort_mbtest = 1;
914
		rcu_assign_pointer(rcu_torture_current, rp);
915
		smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
916
		if (old_rp) {
917 918 919 920 921
			i = old_rp->rtort_pipe_count;
			if (i > RCU_TORTURE_PIPE_LEN)
				i = RCU_TORTURE_PIPE_LEN;
			atomic_inc(&rcu_torture_wcount[i]);
			old_rp->rtort_pipe_count++;
922 923
			switch (synctype[torture_random(&rand) % nsynctypes]) {
			case RTWS_DEF_FREE:
924
				rcu_torture_writer_state = RTWS_DEF_FREE;
925
				cur_ops->deferred_free(old_rp);
926 927
				break;
			case RTWS_EXP_SYNC:
928
				rcu_torture_writer_state = RTWS_EXP_SYNC;
929
				cur_ops->exp_sync();
930 931 932 933 934 935 936 937 938 939 940 941 942
				rcu_torture_pipe_update(old_rp);
				break;
			case RTWS_COND_GET:
				rcu_torture_writer_state = RTWS_COND_GET;
				gp_snap = cur_ops->get_state();
				i = torture_random(&rand) % 16;
				if (i != 0)
					schedule_timeout_interruptible(i);
				udelay(torture_random(&rand) % 1000);
				rcu_torture_writer_state = RTWS_COND_SYNC;
				cur_ops->cond_sync(gp_snap);
				rcu_torture_pipe_update(old_rp);
				break;
943 944 945 946 947
			case RTWS_SYNC:
				rcu_torture_writer_state = RTWS_SYNC;
				cur_ops->sync();
				rcu_torture_pipe_update(old_rp);
				break;
948 949 950
			default:
				WARN_ON_ONCE(1);
				break;
951
			}
952
		}
953
		rcutorture_record_progress(++rcu_torture_current_version);
954
		rcu_torture_writer_state = RTWS_STUTTER;
955
		stutter_wait("rcu_torture_writer");
956
	} while (!torture_must_stop());
957
	rcu_torture_writer_state = RTWS_STOPPING;
958
	torture_kthread_stopping("rcu_torture_writer");
959 960 961
	return 0;
}

962 963 964 965 966 967 968
/*
 * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
 * delay between calls.
 */
static int
rcu_torture_fakewriter(void *arg)
{
969
	DEFINE_TORTURE_RANDOM(rand);
970

971
	VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
972
	set_user_nice(current, MAX_NICE);
973 974

	do {
975 976
		schedule_timeout_uninterruptible(1 + torture_random(&rand)%10);
		udelay(torture_random(&rand) & 0x3ff);
977
		if (cur_ops->cb_barrier != NULL &&
978
		    torture_random(&rand) % (nfakewriters * 8) == 0) {
979
			cur_ops->cb_barrier();
980
		} else if (gp_normal == gp_exp) {
981
			if (torture_random(&rand) & 0x80)
982 983 984 985
				cur_ops->sync();
			else
				cur_ops->exp_sync();
		} else if (gp_normal) {
986
			cur_ops->sync();
987 988 989
		} else {
			cur_ops->exp_sync();
		}
990
		stutter_wait("rcu_torture_fakewriter");
991
	} while (!torture_must_stop());
992

993
	torture_kthread_stopping("rcu_torture_fakewriter");
994 995 996
	return 0;
}

997
static void rcutorture_trace_dump(void)
998 999 1000 1001 1002 1003 1004 1005 1006 1007
{
	static atomic_t beenhere = ATOMIC_INIT(0);

	if (atomic_read(&beenhere))
		return;
	if (atomic_xchg(&beenhere, 1) != 0)
		return;
	ftrace_dump(DUMP_ALL);
}

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
/*
 * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
 * incrementing the corresponding element of the pipeline array.  The
 * counter in the element should never be greater than 1, otherwise, the
 * RCU implementation is broken.
 */
static void rcu_torture_timer(unsigned long unused)
{
	int idx;
	int completed;
1018
	int completed_end;
1019
	static DEFINE_TORTURE_RANDOM(rand);
1020 1021 1022
	static DEFINE_SPINLOCK(rand_lock);
	struct rcu_torture *p;
	int pipe_count;
1023
	unsigned long long ts;
1024 1025 1026

	idx = cur_ops->readlock();
	completed = cur_ops->completed();
1027
	ts = rcu_trace_clock_local();
1028 1029 1030 1031
	p = rcu_dereference_check(rcu_torture_current,
				  rcu_read_lock_bh_held() ||
				  rcu_read_lock_sched_held() ||
				  srcu_read_lock_held(&srcu_ctl));
1032 1033 1034 1035 1036 1037 1038 1039
	if (p == NULL) {
		/* Leave because rcu_torture_writer is not yet underway */
		cur_ops->readunlock(idx);
		return;
	}
	if (p->rtort_mbtest == 0)
		atomic_inc(&n_rcu_torture_mberror);
	spin_lock(&rand_lock);
1040
	cur_ops->read_delay(&rand);
1041 1042 1043 1044 1045 1046 1047 1048
	n_rcu_torture_timers++;
	spin_unlock(&rand_lock);
	preempt_disable();
	pipe_count = p->rtort_pipe_count;
	if (pipe_count > RCU_TORTURE_PIPE_LEN) {
		/* Should not happen, but... */
		pipe_count = RCU_TORTURE_PIPE_LEN;
	}
1049 1050 1051 1052
	completed_end = cur_ops->completed();
	if (pipe_count > 1) {
		do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
					  completed, completed_end);
1053
		rcutorture_trace_dump();
1054
	}
R
Rusty Russell 已提交
1055
	__this_cpu_inc(rcu_torture_count[pipe_count]);
1056
	completed = completed_end - completed;
1057 1058 1059 1060
	if (completed > RCU_TORTURE_PIPE_LEN) {
		/* Should not happen, but... */
		completed = RCU_TORTURE_PIPE_LEN;
	}
R
Rusty Russell 已提交
1061
	__this_cpu_inc(rcu_torture_batch[completed]);
1062 1063 1064 1065
	preempt_enable();
	cur_ops->readunlock(idx);
}

1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
/*
 * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 * incrementing the corresponding element of the pipeline array.  The
 * counter in the element should never be greater than 1, otherwise, the
 * RCU implementation is broken.
 */
static int
rcu_torture_reader(void *arg)
{
	int completed;
1076
	int completed_end;
1077
	int idx;
1078
	DEFINE_TORTURE_RANDOM(rand);
1079 1080
	struct rcu_torture *p;
	int pipe_count;
1081
	struct timer_list t;
1082
	unsigned long long ts;
1083

1084
	VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
1085
	set_user_nice(current, MAX_NICE);
1086
	if (irqreader && cur_ops->irq_capable)
1087
		setup_timer_on_stack(&t, rcu_torture_timer, 0);
1088

1089
	do {
1090
		if (irqreader && cur_ops->irq_capable) {
1091
			if (!timer_pending(&t))
1092
				mod_timer(&t, jiffies + 1);
1093
		}
1094 1095
		idx = cur_ops->readlock();
		completed = cur_ops->completed();
1096
		ts = rcu_trace_clock_local();
1097 1098 1099 1100
		p = rcu_dereference_check(rcu_torture_current,
					  rcu_read_lock_bh_held() ||
					  rcu_read_lock_sched_held() ||
					  srcu_read_lock_held(&srcu_ctl));
1101 1102
		if (p == NULL) {
			/* Wait for rcu_torture_writer to get underway */
1103
			cur_ops->readunlock(idx);
1104 1105 1106
			schedule_timeout_interruptible(HZ);
			continue;
		}
1107 1108
		if (p->rtort_mbtest == 0)
			atomic_inc(&n_rcu_torture_mberror);
1109
		cur_ops->read_delay(&rand);
1110 1111 1112 1113 1114 1115
		preempt_disable();
		pipe_count = p->rtort_pipe_count;
		if (pipe_count > RCU_TORTURE_PIPE_LEN) {
			/* Should not happen, but... */
			pipe_count = RCU_TORTURE_PIPE_LEN;
		}
1116 1117 1118 1119
		completed_end = cur_ops->completed();
		if (pipe_count > 1) {
			do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
						  ts, completed, completed_end);
1120
			rcutorture_trace_dump();
1121
		}
R
Rusty Russell 已提交
1122
		__this_cpu_inc(rcu_torture_count[pipe_count]);
1123
		completed = completed_end - completed;
1124 1125 1126 1127
		if (completed > RCU_TORTURE_PIPE_LEN) {
			/* Should not happen, but... */
			completed = RCU_TORTURE_PIPE_LEN;
		}
R
Rusty Russell 已提交
1128
		__this_cpu_inc(rcu_torture_batch[completed]);
1129
		preempt_enable();
1130
		cur_ops->readunlock(idx);
1131
		cond_resched_rcu_qs();
1132
		stutter_wait("rcu_torture_reader");
1133
	} while (!torture_must_stop());
1134
	if (irqreader && cur_ops->irq_capable) {
1135
		del_timer_sync(&t);
1136 1137
		destroy_timer_on_stack(&t);
	}
1138
	torture_kthread_stopping("rcu_torture_reader");
1139 1140 1141 1142
	return 0;
}

/*
1143 1144 1145 1146 1147 1148
 * Print torture statistics.  Caller must ensure that there is only
 * one call to this function at a given time!!!  This is normally
 * accomplished by relying on the module system to only have one copy
 * of the module loaded, and then by giving the rcu_torture_stats
 * kthread full control (or the init/cleanup functions when rcu_torture_stats
 * thread is not running).
1149
 */
1150
static void
1151
rcu_torture_stats_print(void)
1152 1153 1154 1155 1156
{
	int cpu;
	int i;
	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1157
	static unsigned long rtcv_snap = ULONG_MAX;
1158

1159
	for_each_possible_cpu(cpu) {
1160 1161 1162 1163 1164 1165 1166 1167 1168
		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
			pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
			batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
		}
	}
	for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
		if (pipesummary[i] != 0)
			break;
	}
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
		rcu_torture_current,
		rcu_torture_current_version,
		list_empty(&rcu_torture_freelist),
		atomic_read(&n_rcu_torture_alloc),
		atomic_read(&n_rcu_torture_alloc_fail),
		atomic_read(&n_rcu_torture_free));
	pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ",
		atomic_read(&n_rcu_torture_mberror),
		n_rcu_torture_boost_ktrerror,
		n_rcu_torture_boost_rterror);
	pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
		n_rcu_torture_boost_failure,
		n_rcu_torture_boosts,
		n_rcu_torture_timers);
	torture_onoff_stats();
1187
	pr_cont("barrier: %ld/%ld:%ld ",
1188 1189 1190
		n_barrier_successes,
		n_barrier_attempts,
		n_rcu_torture_barrier_error);
1191
	pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods));
1192 1193

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1194
	if (atomic_read(&n_rcu_torture_mberror) != 0 ||
1195
	    n_rcu_torture_barrier_error != 0 ||
1196 1197
	    n_rcu_torture_boost_ktrerror != 0 ||
	    n_rcu_torture_boost_rterror != 0 ||
1198 1199
	    n_rcu_torture_boost_failure != 0 ||
	    i > 1) {
1200
		pr_cont("%s", "!!! ");
1201
		atomic_inc(&n_rcu_torture_error);
1202
		WARN_ON_ONCE(1);
1203
	}
1204
	pr_cont("Reader Pipe: ");
1205
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1206 1207 1208 1209 1210
		pr_cont(" %ld", pipesummary[i]);
	pr_cont("\n");

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("Reader Batch: ");
1211
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1212 1213 1214 1215 1216
		pr_cont(" %ld", batchsummary[i]);
	pr_cont("\n");

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("Free-Block Circulation: ");
1217
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1218
		pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
1219
	}
1220 1221
	pr_cont("\n");

1222
	if (cur_ops->stats)
1223
		cur_ops->stats();
1224 1225 1226 1227 1228 1229 1230 1231
	if (rtcv_snap == rcu_torture_current_version &&
	    rcu_torture_current != NULL) {
		int __maybe_unused flags;
		unsigned long __maybe_unused gpnum;
		unsigned long __maybe_unused completed;

		rcutorture_get_gp_data(cur_ops->ttype,
				       &flags, &gpnum, &completed);
1232 1233 1234
		pr_alert("??? Writer stall state %d g%lu c%lu f%#x\n",
			 rcu_torture_writer_state,
			 gpnum, completed, flags);
1235
		show_rcu_gp_kthreads();
1236
		rcutorture_trace_dump();
1237 1238
	}
	rtcv_snap = rcu_torture_current_version;
1239 1240 1241 1242 1243 1244 1245 1246 1247
}

/*
 * Periodically prints torture statistics, if periodic statistics printing
 * was specified via the stat_interval module parameter.
 */
static int
rcu_torture_stats(void *arg)
{
1248
	VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
1249 1250 1251
	do {
		schedule_timeout_interruptible(stat_interval * HZ);
		rcu_torture_stats_print();
1252
		torture_shutdown_absorb("rcu_torture_stats");
1253
	} while (!torture_must_stop());
1254
	torture_kthread_stopping("rcu_torture_stats");
1255 1256 1257
	return 0;
}

1258
static inline void
1259
rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
1260
{
1261 1262 1263 1264 1265 1266 1267
	pr_alert("%s" TORTURE_FLAG
		 "--- %s: nreaders=%d nfakewriters=%d "
		 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
		 "shuffle_interval=%d stutter=%d irqreader=%d "
		 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
		 "test_boost=%d/%d test_boost_interval=%d "
		 "test_boost_duration=%d shutdown_secs=%d "
1268 1269
		 "stall_cpu=%d stall_cpu_holdoff=%d "
		 "n_barrier_cbs=%d "
1270 1271 1272 1273 1274 1275
		 "onoff_interval=%d onoff_holdoff=%d\n",
		 torture_type, tag, nrealreaders, nfakewriters,
		 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
		 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
		 test_boost, cur_ops->can_boost,
		 test_boost_interval, test_boost_duration, shutdown_secs,
1276 1277
		 stall_cpu, stall_cpu_holdoff,
		 n_barrier_cbs,
1278
		 onoff_interval, onoff_holdoff);
1279 1280
}

1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
static void rcutorture_booster_cleanup(int cpu)
{
	struct task_struct *t;

	if (boost_tasks[cpu] == NULL)
		return;
	mutex_lock(&boost_mutex);
	t = boost_tasks[cpu];
	boost_tasks[cpu] = NULL;
	mutex_unlock(&boost_mutex);

	/* This must be outside of the mutex, otherwise deadlock! */
1293
	torture_stop_kthread(rcu_torture_boost, t);
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304
}

static int rcutorture_booster_init(int cpu)
{
	int retval;

	if (boost_tasks[cpu] != NULL)
		return 0;  /* Already created, nothing more to do. */

	/* Don't allow time recalculation while creating a new task. */
	mutex_lock(&boost_mutex);
1305
	VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
E
Eric Dumazet 已提交
1306 1307 1308
	boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
						  cpu_to_node(cpu),
						  "rcu_torture_boost");
1309 1310
	if (IS_ERR(boost_tasks[cpu])) {
		retval = PTR_ERR(boost_tasks[cpu]);
1311
		VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
		n_rcu_torture_boost_ktrerror++;
		boost_tasks[cpu] = NULL;
		mutex_unlock(&boost_mutex);
		return retval;
	}
	kthread_bind(boost_tasks[cpu], cpu);
	wake_up_process(boost_tasks[cpu]);
	mutex_unlock(&boost_mutex);
	return 0;
}

1323 1324 1325 1326
/*
 * CPU-stall kthread.  It waits as specified by stall_cpu_holdoff, then
 * induces a CPU stall for the time specified by stall_cpu.
 */
1327
static int rcu_torture_stall(void *args)
1328 1329 1330
{
	unsigned long stop_at;

1331
	VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
1332
	if (stall_cpu_holdoff > 0) {
1333
		VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
1334
		schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1335
		VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
1336 1337 1338 1339
	}
	if (!kthread_should_stop()) {
		stop_at = get_seconds() + stall_cpu;
		/* RCU CPU stall is expected behavior in following code. */
1340
		pr_alert("rcu_torture_stall start.\n");
1341 1342 1343 1344 1345 1346
		rcu_read_lock();
		preempt_disable();
		while (ULONG_CMP_LT(get_seconds(), stop_at))
			continue;  /* Induce RCU CPU stall warning. */
		preempt_enable();
		rcu_read_unlock();
1347
		pr_alert("rcu_torture_stall end.\n");
1348
	}
1349
	torture_shutdown_absorb("rcu_torture_stall");
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
	while (!kthread_should_stop())
		schedule_timeout_interruptible(10 * HZ);
	return 0;
}

/* Spawn CPU-stall kthread, if stall_cpu specified. */
static int __init rcu_torture_stall_init(void)
{
	if (stall_cpu <= 0)
		return 0;
1360
	return torture_create_kthread(rcu_torture_stall, NULL, stall_task);
1361 1362
}

1363
/* Callback function for RCU barrier testing. */
1364
static void rcu_torture_barrier_cbf(struct rcu_head *rcu)
1365 1366 1367 1368 1369 1370 1371 1372
{
	atomic_inc(&barrier_cbs_invoked);
}

/* kthread function to register callbacks used to test RCU barriers. */
static int rcu_torture_barrier_cbs(void *arg)
{
	long myid = (long)arg;
1373
	bool lastphase = 0;
1374
	bool newphase;
1375 1376 1377
	struct rcu_head rcu;

	init_rcu_head_on_stack(&rcu);
1378
	VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
1379
	set_user_nice(current, MAX_NICE);
1380 1381
	do {
		wait_event(barrier_cbs_wq[myid],
1382 1383
			   (newphase =
			    ACCESS_ONCE(barrier_phase)) != lastphase ||
1384
			   torture_must_stop());
1385
		lastphase = newphase;
1386
		smp_mb(); /* ensure barrier_phase load before ->call(). */
1387
		if (torture_must_stop())
1388 1389 1390 1391
			break;
		cur_ops->call(&rcu, rcu_torture_barrier_cbf);
		if (atomic_dec_and_test(&barrier_cbs_count))
			wake_up(&barrier_wq);
1392
	} while (!torture_must_stop());
1393 1394
	if (cur_ops->cb_barrier != NULL)
		cur_ops->cb_barrier();
1395
	destroy_rcu_head_on_stack(&rcu);
1396
	torture_kthread_stopping("rcu_torture_barrier_cbs");
1397 1398 1399 1400 1401 1402 1403 1404
	return 0;
}

/* kthread function to drive and coordinate RCU barrier testing. */
static int rcu_torture_barrier(void *arg)
{
	int i;

1405
	VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
1406 1407 1408
	do {
		atomic_set(&barrier_cbs_invoked, 0);
		atomic_set(&barrier_cbs_count, n_barrier_cbs);
1409 1410
		smp_mb(); /* Ensure barrier_phase after prior assignments. */
		barrier_phase = !barrier_phase;
1411 1412 1413 1414
		for (i = 0; i < n_barrier_cbs; i++)
			wake_up(&barrier_cbs_wq[i]);
		wait_event(barrier_wq,
			   atomic_read(&barrier_cbs_count) == 0 ||
1415 1416
			   torture_must_stop());
		if (torture_must_stop())
1417 1418
			break;
		n_barrier_attempts++;
1419
		cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
1420 1421 1422 1423 1424 1425
		if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
			n_rcu_torture_barrier_error++;
			WARN_ON_ONCE(1);
		}
		n_barrier_successes++;
		schedule_timeout_interruptible(HZ / 10);
1426
	} while (!torture_must_stop());
1427
	torture_kthread_stopping("rcu_torture_barrier");
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
	return 0;
}

/* Initialize RCU barrier testing. */
static int rcu_torture_barrier_init(void)
{
	int i;
	int ret;

	if (n_barrier_cbs == 0)
		return 0;
	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
1440 1441 1442 1443 1444 1445
		pr_alert("%s" TORTURE_FLAG
			 " Call or barrier ops missing for %s,\n",
			 torture_type, cur_ops->name);
		pr_alert("%s" TORTURE_FLAG
			 " RCU barrier testing omitted from run.\n",
			 torture_type);
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
		return 0;
	}
	atomic_set(&barrier_cbs_count, 0);
	atomic_set(&barrier_cbs_invoked, 0);
	barrier_cbs_tasks =
		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
			GFP_KERNEL);
	barrier_cbs_wq =
		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
			GFP_KERNEL);
1456
	if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
1457 1458 1459
		return -ENOMEM;
	for (i = 0; i < n_barrier_cbs; i++) {
		init_waitqueue_head(&barrier_cbs_wq[i]);
1460 1461 1462 1463
		ret = torture_create_kthread(rcu_torture_barrier_cbs,
					     (void *)(long)i,
					     barrier_cbs_tasks[i]);
		if (ret)
1464 1465
			return ret;
	}
1466
	return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task);
1467 1468 1469 1470 1471 1472 1473
}

/* Clean up after RCU barrier testing. */
static void rcu_torture_barrier_cleanup(void)
{
	int i;

1474
	torture_stop_kthread(rcu_torture_barrier, barrier_task);
1475
	if (barrier_cbs_tasks != NULL) {
1476 1477 1478
		for (i = 0; i < n_barrier_cbs; i++)
			torture_stop_kthread(rcu_torture_barrier_cbs,
					     barrier_cbs_tasks[i]);
1479 1480 1481 1482 1483 1484 1485 1486 1487
		kfree(barrier_cbs_tasks);
		barrier_cbs_tasks = NULL;
	}
	if (barrier_cbs_wq != NULL) {
		kfree(barrier_cbs_wq);
		barrier_cbs_wq = NULL;
	}
}

1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
static int rcutorture_cpu_notify(struct notifier_block *self,
				 unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;

	switch (action) {
	case CPU_ONLINE:
	case CPU_DOWN_FAILED:
		(void)rcutorture_booster_init(cpu);
		break;
	case CPU_DOWN_PREPARE:
		rcutorture_booster_cleanup(cpu);
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block rcutorture_cpu_nb = {
	.notifier_call = rcutorture_cpu_notify,
};

1511 1512 1513 1514 1515
static void
rcu_torture_cleanup(void)
{
	int i;

1516
	rcutorture_record_test_transition();
1517
	if (torture_cleanup()) {
1518 1519 1520 1521
		if (cur_ops->cb_barrier != NULL)
			cur_ops->cb_barrier();
		return;
	}
1522

1523
	rcu_torture_barrier_cleanup();
1524 1525
	torture_stop_kthread(rcu_torture_stall, stall_task);
	torture_stop_kthread(rcu_torture_writer, writer_task);
1526

1527
	if (reader_tasks) {
1528 1529 1530
		for (i = 0; i < nrealreaders; i++)
			torture_stop_kthread(rcu_torture_reader,
					     reader_tasks[i]);
1531 1532 1533 1534
		kfree(reader_tasks);
	}
	rcu_torture_current = NULL;

1535
	if (fakewriter_tasks) {
1536
		for (i = 0; i < nfakewriters; i++) {
1537 1538
			torture_stop_kthread(rcu_torture_fakewriter,
					     fakewriter_tasks[i]);
1539 1540 1541 1542 1543
		}
		kfree(fakewriter_tasks);
		fakewriter_tasks = NULL;
	}

1544 1545
	torture_stop_kthread(rcu_torture_stats, stats_task);
	torture_stop_kthread(rcu_torture_fqs, fqs_task);
1546 1547
	for (i = 0; i < ncbflooders; i++)
		torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]);
1548 1549 1550 1551 1552 1553
	if ((test_boost == 1 && cur_ops->can_boost) ||
	    test_boost == 2) {
		unregister_cpu_notifier(&rcutorture_cpu_nb);
		for_each_possible_cpu(i)
			rcutorture_booster_cleanup(i);
	}
1554

1555
	/* Wait for all RCU callbacks to fire.  */
1556 1557 1558

	if (cur_ops->cb_barrier != NULL)
		cur_ops->cb_barrier();
1559 1560

	rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1561

1562
	if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
1563
		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
1564
	else if (torture_onoff_failures())
1565 1566
		rcu_torture_print_module_parms(cur_ops,
					       "End of test: RCU_HOTPLUG");
1567
	else
1568
		rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
1569 1570
}

1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
static void rcu_torture_leak_cb(struct rcu_head *rhp)
{
}

static void rcu_torture_err_cb(struct rcu_head *rhp)
{
	/*
	 * This -might- happen due to race conditions, but is unlikely.
	 * The scenario that leads to this happening is that the
	 * first of the pair of duplicate callbacks is queued,
	 * someone else starts a grace period that includes that
	 * callback, then the second of the pair must wait for the
	 * next grace period.  Unlikely, but can happen.  If it
	 * does happen, the debug-objects subsystem won't have splatted.
	 */
	pr_alert("rcutorture: duplicated callback was invoked.\n");
}
#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */

/*
 * Verify that double-free causes debug-objects to complain, but only
 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.  Otherwise, say that the test
 * cannot be carried out.
 */
static void rcu_test_debug_objects(void)
{
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	struct rcu_head rh1;
	struct rcu_head rh2;

	init_rcu_head_on_stack(&rh1);
	init_rcu_head_on_stack(&rh2);
	pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");

	/* Try to queue the rh2 pair of callbacks for the same grace period. */
	preempt_disable(); /* Prevent preemption from interrupting test. */
	rcu_read_lock(); /* Make it impossible to finish a grace period. */
	call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */
	local_irq_disable(); /* Make it harder to start a new grace period. */
	call_rcu(&rh2, rcu_torture_leak_cb);
	call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
	local_irq_enable();
	rcu_read_unlock();
	preempt_enable();

	/* Wait for them all to get done so we can safely return. */
	rcu_barrier();
	pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
	destroy_rcu_head_on_stack(&rh1);
	destroy_rcu_head_on_stack(&rh2);
#else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
	pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
#endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
}

1627
static int __init
1628 1629 1630 1631 1632
rcu_torture_init(void)
{
	int i;
	int cpu;
	int firsterr = 0;
1633
	static struct rcu_torture_ops *torture_ops[] = {
1634
		&rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &sched_ops,
1635
		RCUTORTURE_TASKS_OPS
1636
	};
1637

1638 1639
	if (!torture_init_begin(torture_type, verbose, &rcutorture_runnable))
		return -EBUSY;
1640

1641
	/* Process args and tell the world that the torturer is on the job. */
1642
	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1643
		cur_ops = torture_ops[i];
1644
		if (strcmp(torture_type, cur_ops->name) == 0)
1645 1646
			break;
	}
1647
	if (i == ARRAY_SIZE(torture_ops)) {
1648 1649 1650
		pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
			 torture_type);
		pr_alert("rcu-torture types:");
1651
		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1652 1653
			pr_alert(" %s", torture_ops[i]->name);
		pr_alert("\n");
1654
		torture_init_end();
1655
		return -EINVAL;
1656
	}
1657
	if (cur_ops->fqs == NULL && fqs_duration != 0) {
1658
		pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
1659 1660
		fqs_duration = 0;
	}
1661
	if (cur_ops->init)
1662 1663
		cur_ops->init(); /* no "goto unwind" prior to this point!!! */

1664
	if (nreaders >= 0) {
1665
		nrealreaders = nreaders;
1666 1667 1668 1669 1670
	} else {
		nrealreaders = num_online_cpus() - 1;
		if (nrealreaders <= 0)
			nrealreaders = 1;
	}
1671
	rcu_torture_print_module_parms(cur_ops, "Start of test");
1672 1673 1674 1675

	/* Set up the freelist. */

	INIT_LIST_HEAD(&rcu_torture_freelist);
1676
	for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1677
		rcu_tortures[i].rtort_mbtest = 0;
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688
		list_add_tail(&rcu_tortures[i].rtort_free,
			      &rcu_torture_freelist);
	}

	/* Initialize the statistics so that each run gets its own numbers. */

	rcu_torture_current = NULL;
	rcu_torture_current_version = 0;
	atomic_set(&n_rcu_torture_alloc, 0);
	atomic_set(&n_rcu_torture_alloc_fail, 0);
	atomic_set(&n_rcu_torture_free, 0);
1689 1690
	atomic_set(&n_rcu_torture_mberror, 0);
	atomic_set(&n_rcu_torture_error, 0);
1691
	n_rcu_torture_barrier_error = 0;
1692 1693 1694 1695
	n_rcu_torture_boost_ktrerror = 0;
	n_rcu_torture_boost_rterror = 0;
	n_rcu_torture_boost_failure = 0;
	n_rcu_torture_boosts = 0;
1696 1697
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
		atomic_set(&rcu_torture_wcount[i], 0);
1698
	for_each_possible_cpu(cpu) {
1699 1700 1701 1702 1703 1704 1705 1706
		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
			per_cpu(rcu_torture_count, cpu)[i] = 0;
			per_cpu(rcu_torture_batch, cpu)[i] = 0;
		}
	}

	/* Start up the kthreads. */

1707 1708 1709
	firsterr = torture_create_kthread(rcu_torture_writer, NULL,
					  writer_task);
	if (firsterr)
1710
		goto unwind;
1711
	fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
1712
				   GFP_KERNEL);
1713
	if (fakewriter_tasks == NULL) {
1714
		VERBOSE_TOROUT_ERRSTRING("out of memory");
1715 1716 1717 1718
		firsterr = -ENOMEM;
		goto unwind;
	}
	for (i = 0; i < nfakewriters; i++) {
1719 1720 1721
		firsterr = torture_create_kthread(rcu_torture_fakewriter,
						  NULL, fakewriter_tasks[i]);
		if (firsterr)
1722 1723
			goto unwind;
	}
1724
	reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
1725 1726
			       GFP_KERNEL);
	if (reader_tasks == NULL) {
1727
		VERBOSE_TOROUT_ERRSTRING("out of memory");
1728 1729 1730 1731
		firsterr = -ENOMEM;
		goto unwind;
	}
	for (i = 0; i < nrealreaders; i++) {
1732 1733 1734
		firsterr = torture_create_kthread(rcu_torture_reader, NULL,
						  reader_tasks[i]);
		if (firsterr)
1735 1736 1737
			goto unwind;
	}
	if (stat_interval > 0) {
1738 1739 1740
		firsterr = torture_create_kthread(rcu_torture_stats, NULL,
						  stats_task);
		if (firsterr)
1741 1742
			goto unwind;
	}
1743
	if (test_no_idle_hz) {
1744 1745
		firsterr = torture_shuffle_init(shuffle_interval * HZ);
		if (firsterr)
1746 1747
			goto unwind;
	}
1748 1749 1750
	if (stutter < 0)
		stutter = 0;
	if (stutter) {
1751 1752
		firsterr = torture_stutter_init(stutter * HZ);
		if (firsterr)
1753 1754
			goto unwind;
	}
1755 1756 1757
	if (fqs_duration < 0)
		fqs_duration = 0;
	if (fqs_duration) {
1758
		/* Create the fqs thread */
1759 1760
		firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
						  fqs_task);
1761
		if (firsterr)
1762 1763
			goto unwind;
	}
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775
	if (test_boost_interval < 1)
		test_boost_interval = 1;
	if (test_boost_duration < 2)
		test_boost_duration = 2;
	if ((test_boost == 1 && cur_ops->can_boost) ||
	    test_boost == 2) {

		boost_starttime = jiffies + test_boost_interval * HZ;
		register_cpu_notifier(&rcutorture_cpu_nb);
		for_each_possible_cpu(i) {
			if (cpu_is_offline(i))
				continue;  /* Heuristic: CPU can go offline. */
1776 1777
			firsterr = rcutorture_booster_init(i);
			if (firsterr)
1778 1779 1780
				goto unwind;
		}
	}
1781 1782
	firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
	if (firsterr)
1783
		goto unwind;
1784 1785
	firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ);
	if (firsterr)
1786
		goto unwind;
1787 1788
	firsterr = rcu_torture_stall_init();
	if (firsterr)
1789
		goto unwind;
1790 1791
	firsterr = rcu_torture_barrier_init();
	if (firsterr)
1792
		goto unwind;
1793 1794
	if (object_debug)
		rcu_test_debug_objects();
1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
	if (cbflood_n_burst > 0) {
		/* Create the cbflood threads */
		ncbflooders = (num_online_cpus() + 3) / 4;
		cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task),
				       GFP_KERNEL);
		if (!cbflood_task) {
			VERBOSE_TOROUT_ERRSTRING("out of memory");
			firsterr = -ENOMEM;
			goto unwind;
		}
		for (i = 0; i < ncbflooders; i++) {
			firsterr = torture_create_kthread(rcu_torture_cbflood,
							  NULL,
							  cbflood_task[i]);
			if (firsterr)
				goto unwind;
		}
	}
1813
	rcutorture_record_test_transition();
1814
	torture_init_end();
1815 1816 1817
	return 0;

unwind:
1818
	torture_init_end();
1819 1820 1821 1822 1823 1824
	rcu_torture_cleanup();
	return firsterr;
}

module_init(rcu_torture_init);
module_exit(rcu_torture_cleanup);