rcutorture.c 54.3 KB
Newer Older
1
/*
2
 * Read-Copy Update module-based torture test facility
3 4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
15 16
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
17
 *
18
 * Copyright (C) IBM Corporation, 2005, 2006
19 20
 *
 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
J
Josh Triplett 已提交
21
 *	  Josh Triplett <josh@joshtriplett.org>
22 23 24 25 26 27 28 29 30 31 32 33 34 35
 *
 * See also:  Documentation/RCU/torture.txt
 */
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/err.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/rcupdate.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
A
Arun Sharma 已提交
36
#include <linux/atomic.h>
37 38 39 40 41
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/moduleparam.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
42
#include <linux/reboot.h>
43
#include <linux/freezer.h>
44 45 46
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/stat.h>
47
#include <linux/srcu.h>
48
#include <linux/slab.h>
49
#include <linux/trace_clock.h>
50
#include <asm/byteorder.h>
51
#include <linux/torture.h>
52
#include <linux/vmalloc.h>
53 54

MODULE_LICENSE("GPL");
J
Josh Triplett 已提交
55
MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
56

57

58 59 60 61 62 63 64
torture_param(int, cbflood_inter_holdoff, HZ,
	      "Holdoff between floods (jiffies)");
torture_param(int, cbflood_intra_holdoff, 1,
	      "Holdoff between bursts (jiffies)");
torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable");
torture_param(int, cbflood_n_per_burst, 20000,
	      "# callbacks per burst in flood");
65 66 67 68
torture_param(int, fqs_duration, 0,
	      "Duration of fqs bursts (us), 0 to disable");
torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
69
torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
70 71 72
torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
torture_param(bool, gp_normal, false,
	     "Use normal (non-expedited) GP wait primitives");
73
torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
torture_param(int, n_barrier_cbs, 0,
	     "# of callbacks/kthreads for barrier testing");
torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
torture_param(int, nreaders, -1, "Number of RCU reader threads");
torture_param(int, object_debug, 0,
	     "Enable debug-object double call_rcu() testing");
torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
torture_param(int, onoff_interval, 0,
	     "Time between CPU hotplugs (s), 0=disable");
torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
torture_param(int, stall_cpu_holdoff, 10,
	     "Time to wait before starting stall (s).");
torture_param(int, stat_interval, 60,
	     "Number of seconds between stats printk()s");
torture_param(int, stutter, 5, "Number of seconds to run/halt test");
torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
torture_param(int, test_boost_duration, 4,
	     "Duration of each boost test, seconds.");
torture_param(int, test_boost_interval, 7,
	     "Interval between boost tests, seconds.");
torture_param(bool, test_no_idle_hz, true,
	     "Test support for tickless idle CPUs");
99 100
torture_param(bool, verbose, true,
	     "Enable verbose debugging printk()s");
101

102
static char *torture_type = "rcu";
103
module_param(torture_type, charp, 0444);
104
MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)");
105 106

static int nrealreaders;
107
static int ncbflooders;
108
static struct task_struct *writer_task;
109
static struct task_struct **fakewriter_tasks;
110 111
static struct task_struct **reader_tasks;
static struct task_struct *stats_task;
112
static struct task_struct **cbflood_task;
113
static struct task_struct *fqs_task;
114
static struct task_struct *boost_tasks[NR_CPUS];
115
static struct task_struct *stall_task;
116 117
static struct task_struct **barrier_cbs_tasks;
static struct task_struct *barrier_task;
118 119 120 121 122 123 124

#define RCU_TORTURE_PIPE_LEN 10

struct rcu_torture {
	struct rcu_head rtort_rcu;
	int rtort_pipe_count;
	struct list_head rtort_free;
125
	int rtort_mbtest;
126 127 128
};

static LIST_HEAD(rcu_torture_freelist);
129
static struct rcu_torture __rcu *rcu_torture_current;
130
static unsigned long rcu_torture_current_version;
131 132
static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
static DEFINE_SPINLOCK(rcu_torture_lock);
133 134 135 136
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1],
		      rcu_torture_count) = { 0 };
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1],
		      rcu_torture_batch) = { 0 };
137
static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
138 139 140 141 142
static atomic_t n_rcu_torture_alloc;
static atomic_t n_rcu_torture_alloc_fail;
static atomic_t n_rcu_torture_free;
static atomic_t n_rcu_torture_mberror;
static atomic_t n_rcu_torture_error;
143
static long n_rcu_torture_barrier_error;
144 145 146 147
static long n_rcu_torture_boost_ktrerror;
static long n_rcu_torture_boost_rterror;
static long n_rcu_torture_boost_failure;
static long n_rcu_torture_boosts;
148
static long n_rcu_torture_timers;
149 150
static long n_barrier_attempts;
static long n_barrier_successes;
151
static atomic_long_t n_cbfloods;
152
static struct list_head rcu_torture_removed;
153

154 155 156 157 158 159
static int rcu_torture_writer_state;
#define RTWS_FIXED_DELAY	0
#define RTWS_DELAY		1
#define RTWS_REPLACE		2
#define RTWS_DEF_FREE		3
#define RTWS_EXP_SYNC		4
160 161
#define RTWS_COND_GET		5
#define RTWS_COND_SYNC		6
162 163 164
#define RTWS_SYNC		7
#define RTWS_STUTTER		8
#define RTWS_STOPPING		9
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
static const char * const rcu_torture_writer_state_names[] = {
	"RTWS_FIXED_DELAY",
	"RTWS_DELAY",
	"RTWS_REPLACE",
	"RTWS_DEF_FREE",
	"RTWS_EXP_SYNC",
	"RTWS_COND_GET",
	"RTWS_COND_SYNC",
	"RTWS_SYNC",
	"RTWS_STUTTER",
	"RTWS_STOPPING",
};

static const char *rcu_torture_writer_state_getname(void)
{
	unsigned int i = READ_ONCE(rcu_torture_writer_state);

	if (i >= ARRAY_SIZE(rcu_torture_writer_state_names))
		return "???";
	return rcu_torture_writer_state_names[i];
}
186

187 188 189 190 191
#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
#define RCUTORTURE_RUNNABLE_INIT 1
#else
#define RCUTORTURE_RUNNABLE_INIT 0
#endif
192 193 194
static int torture_runnable = RCUTORTURE_RUNNABLE_INIT;
module_param(torture_runnable, int, 0444);
MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot");
195

196
#if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
197
#define rcu_can_boost() 1
198
#else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
199
#define rcu_can_boost() 0
200
#endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
201

202 203 204 205 206 207 208 209 210 211 212 213 214 215
#ifdef CONFIG_RCU_TRACE
static u64 notrace rcu_trace_clock_local(void)
{
	u64 ts = trace_clock_local();
	unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC);
	return ts;
}
#else /* #ifdef CONFIG_RCU_TRACE */
static u64 notrace rcu_trace_clock_local(void)
{
	return 0ULL;
}
#endif /* #else #ifdef CONFIG_RCU_TRACE */

216
static unsigned long boost_starttime;	/* jiffies of next boost test start. */
217
static DEFINE_MUTEX(boost_mutex);	/* protect setting boost_starttime */
218
					/*  and boost task create/destroy. */
219
static atomic_t barrier_cbs_count;	/* Barrier callbacks registered. */
220
static bool barrier_phase;		/* Test phase. */
221 222 223
static atomic_t barrier_cbs_invoked;	/* Barrier callbacks invoked. */
static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
224

225 226 227
/*
 * Allocate an element from the rcu_tortures pool.
 */
A
Adrian Bunk 已提交
228
static struct rcu_torture *
229 230 231 232
rcu_torture_alloc(void)
{
	struct list_head *p;

233
	spin_lock_bh(&rcu_torture_lock);
234 235
	if (list_empty(&rcu_torture_freelist)) {
		atomic_inc(&n_rcu_torture_alloc_fail);
236
		spin_unlock_bh(&rcu_torture_lock);
237 238 239 240 241
		return NULL;
	}
	atomic_inc(&n_rcu_torture_alloc);
	p = rcu_torture_freelist.next;
	list_del_init(p);
242
	spin_unlock_bh(&rcu_torture_lock);
243 244 245 246 247 248 249 250 251 252
	return container_of(p, struct rcu_torture, rtort_free);
}

/*
 * Free an element to the rcu_tortures pool.
 */
static void
rcu_torture_free(struct rcu_torture *p)
{
	atomic_inc(&n_rcu_torture_free);
253
	spin_lock_bh(&rcu_torture_lock);
254
	list_add_tail(&p->rtort_free, &rcu_torture_freelist);
255
	spin_unlock_bh(&rcu_torture_lock);
256 257
}

258 259 260 261 262
/*
 * Operations vector for selecting different types of tests.
 */

struct rcu_torture_ops {
263
	int ttype;
264
	void (*init)(void);
265
	void (*cleanup)(void);
266
	int (*readlock)(void);
267
	void (*read_delay)(struct torture_random_state *rrsp);
268
	void (*readunlock)(int idx);
269
	unsigned long (*started)(void);
270
	unsigned long (*completed)(void);
271
	void (*deferred_free)(struct rcu_torture *p);
272
	void (*sync)(void);
273
	void (*exp_sync)(void);
274 275
	unsigned long (*get_state)(void);
	void (*cond_sync)(unsigned long oldstate);
276
	call_rcu_func_t call;
277
	void (*cb_barrier)(void);
278
	void (*fqs)(void);
279
	void (*stats)(void);
280
	int irq_capable;
281
	int can_boost;
282
	const char *name;
283
};
284 285

static struct rcu_torture_ops *cur_ops;
286 287 288 289 290

/*
 * Definitions for rcu torture testing.
 */

291
static int rcu_torture_read_lock(void) __acquires(RCU)
292 293 294 295 296
{
	rcu_read_lock();
	return 0;
}

297
static void rcu_read_delay(struct torture_random_state *rrsp)
298
{
299 300
	const unsigned long shortdelay_us = 200;
	const unsigned long longdelay_ms = 50;
301

302 303 304
	/* We want a short delay sometimes to make a reader delay the grace
	 * period, and we want a long delay occasionally to trigger
	 * force_quiescent_state. */
305

306
	if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
307
		mdelay(longdelay_ms);
308
	if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
309
		udelay(shortdelay_us);
310
#ifdef CONFIG_PREEMPT
311 312
	if (!preempt_count() &&
	    !(torture_random(rrsp) % (nrealreaders * 20000)))
313 314
		preempt_schedule();  /* No QS if preempt_disable() in effect */
#endif
315 316
}

317
static void rcu_torture_read_unlock(int idx) __releases(RCU)
318 319 320 321
{
	rcu_read_unlock();
}

322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
/*
 * Update callback in the pipe.  This should be invoked after a grace period.
 */
static bool
rcu_torture_pipe_update_one(struct rcu_torture *rp)
{
	int i;

	i = rp->rtort_pipe_count;
	if (i > RCU_TORTURE_PIPE_LEN)
		i = RCU_TORTURE_PIPE_LEN;
	atomic_inc(&rcu_torture_wcount[i]);
	if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
		rp->rtort_mbtest = 0;
		return true;
	}
	return false;
}

/*
 * Update all callbacks in the pipe.  Suitable for synchronous grace-period
 * primitives.
 */
static void
rcu_torture_pipe_update(struct rcu_torture *old_rp)
{
	struct rcu_torture *rp;
	struct rcu_torture *rp1;

	if (old_rp)
		list_add(&old_rp->rtort_free, &rcu_torture_removed);
	list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
		if (rcu_torture_pipe_update_one(rp)) {
			list_del(&rp->rtort_free);
			rcu_torture_free(rp);
		}
	}
}

361 362 363 364 365
static void
rcu_torture_cb(struct rcu_head *p)
{
	struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);

366
	if (torture_must_stop_irq()) {
367 368 369 370
		/* Test is ending, just drop callbacks on the floor. */
		/* The next initialization will pick up the pieces. */
		return;
	}
371
	if (rcu_torture_pipe_update_one(rp))
372
		rcu_torture_free(rp);
373
	else
374
		cur_ops->deferred_free(rp);
375 376
}

377
static unsigned long rcu_no_completed(void)
378 379 380 381
{
	return 0;
}

382 383 384 385 386
static void rcu_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu(&p->rtort_rcu, rcu_torture_cb);
}

387 388 389 390 391
static void rcu_sync_torture_init(void)
{
	INIT_LIST_HEAD(&rcu_torture_removed);
}

392
static struct rcu_torture_ops rcu_ops = {
393
	.ttype		= RCU_FLAVOR,
394 395 396 397
	.init		= rcu_sync_torture_init,
	.readlock	= rcu_torture_read_lock,
	.read_delay	= rcu_read_delay,
	.readunlock	= rcu_torture_read_unlock,
398
	.started	= rcu_batches_started,
399
	.completed	= rcu_batches_completed,
400
	.deferred_free	= rcu_torture_deferred_free,
401
	.sync		= synchronize_rcu,
402
	.exp_sync	= synchronize_rcu_expedited,
403 404
	.get_state	= get_state_synchronize_rcu,
	.cond_sync	= cond_synchronize_rcu,
405 406
	.call		= call_rcu,
	.cb_barrier	= rcu_barrier,
407
	.fqs		= rcu_force_quiescent_state,
408 409
	.stats		= NULL,
	.irq_capable	= 1,
410
	.can_boost	= rcu_can_boost(),
411
	.name		= "rcu"
412 413
};

414 415 416 417
/*
 * Definitions for rcu_bh torture testing.
 */

418
static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
419 420 421 422 423
{
	rcu_read_lock_bh();
	return 0;
}

424
static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
425 426 427 428 429 430 431 432 433 434
{
	rcu_read_unlock_bh();
}

static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
}

static struct rcu_torture_ops rcu_bh_ops = {
435
	.ttype		= RCU_BH_FLAVOR,
436
	.init		= rcu_sync_torture_init,
437 438 439
	.readlock	= rcu_bh_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= rcu_bh_torture_read_unlock,
440
	.started	= rcu_batches_started_bh,
441
	.completed	= rcu_batches_completed_bh,
442
	.deferred_free	= rcu_bh_torture_deferred_free,
443
	.sync		= synchronize_rcu_bh,
444
	.exp_sync	= synchronize_rcu_bh_expedited,
445
	.call		= call_rcu_bh,
446
	.cb_barrier	= rcu_barrier_bh,
447
	.fqs		= rcu_bh_force_quiescent_state,
448 449 450
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "rcu_bh"
451 452
};

453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/*
 * Don't even think about trying any of these in real life!!!
 * The names includes "busted", and they really means it!
 * The only purpose of these functions is to provide a buggy RCU
 * implementation to make sure that rcutorture correctly emits
 * buggy-RCU error messages.
 */
static void rcu_busted_torture_deferred_free(struct rcu_torture *p)
{
	/* This is a deliberate bug for testing purposes only! */
	rcu_torture_cb(&p->rtort_rcu);
}

static void synchronize_rcu_busted(void)
{
	/* This is a deliberate bug for testing purposes only! */
}

static void
472
call_rcu_busted(struct rcu_head *head, rcu_callback_t func)
473 474 475 476 477 478
{
	/* This is a deliberate bug for testing purposes only! */
	func(head);
}

static struct rcu_torture_ops rcu_busted_ops = {
479
	.ttype		= INVALID_RCU_FLAVOR,
480 481 482 483
	.init		= rcu_sync_torture_init,
	.readlock	= rcu_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= rcu_torture_read_unlock,
484
	.started	= rcu_no_completed,
485 486 487 488 489 490 491 492 493 494 495 496
	.completed	= rcu_no_completed,
	.deferred_free	= rcu_busted_torture_deferred_free,
	.sync		= synchronize_rcu_busted,
	.exp_sync	= synchronize_rcu_busted,
	.call		= call_rcu_busted,
	.cb_barrier	= NULL,
	.fqs		= NULL,
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "rcu_busted"
};

497 498 499 500
/*
 * Definitions for srcu torture testing.
 */

501
DEFINE_STATIC_SRCU(srcu_ctl);
502 503
static struct srcu_struct srcu_ctld;
static struct srcu_struct *srcu_ctlp = &srcu_ctl;
504

505
static int srcu_torture_read_lock(void) __acquires(srcu_ctlp)
506
{
507
	return srcu_read_lock(srcu_ctlp);
508 509
}

510
static void srcu_read_delay(struct torture_random_state *rrsp)
511 512 513 514 515 516 517
{
	long delay;
	const long uspertick = 1000000 / HZ;
	const long longdelay = 10;

	/* We want there to be long-running readers, but not all the time. */

518 519
	delay = torture_random(rrsp) %
		(nrealreaders * 2 * longdelay * uspertick);
520 521
	if (!delay)
		schedule_timeout_interruptible(longdelay);
522 523
	else
		rcu_read_delay(rrsp);
524 525
}

526
static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp)
527
{
528
	srcu_read_unlock(srcu_ctlp, idx);
529 530
}

531
static unsigned long srcu_torture_completed(void)
532
{
533
	return srcu_batches_completed(srcu_ctlp);
534 535
}

536 537
static void srcu_torture_deferred_free(struct rcu_torture *rp)
{
538
	call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb);
539 540
}

541 542
static void srcu_torture_synchronize(void)
{
543
	synchronize_srcu(srcu_ctlp);
544 545
}

546
static void srcu_torture_call(struct rcu_head *head,
547
			      rcu_callback_t func)
548
{
549
	call_srcu(srcu_ctlp, head, func);
550 551 552 553
}

static void srcu_torture_barrier(void)
{
554
	srcu_barrier(srcu_ctlp);
555 556
}

557
static void srcu_torture_stats(void)
558 559
{
	int cpu;
560
	int idx = srcu_ctlp->completed & 0x1;
561

562 563
	pr_alert("%s%s per-CPU(idx=%d):",
		 torture_type, TORTURE_FLAG, idx);
564
	for_each_possible_cpu(cpu) {
565 566
		long c0, c1;

567 568
		c0 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[!idx];
		c1 = (long)per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu)->c[idx];
569
		pr_cont(" %d(%ld,%ld)", cpu, c0, c1);
570
	}
571
	pr_cont("\n");
572 573
}

574 575
static void srcu_torture_synchronize_expedited(void)
{
576
	synchronize_srcu_expedited(srcu_ctlp);
577 578
}

579
static struct rcu_torture_ops srcu_ops = {
580
	.ttype		= SRCU_FLAVOR,
581
	.init		= rcu_sync_torture_init,
582 583 584
	.readlock	= srcu_torture_read_lock,
	.read_delay	= srcu_read_delay,
	.readunlock	= srcu_torture_read_unlock,
585
	.started	= NULL,
586
	.completed	= srcu_torture_completed,
587
	.deferred_free	= srcu_torture_deferred_free,
588
	.sync		= srcu_torture_synchronize,
589
	.exp_sync	= srcu_torture_synchronize_expedited,
590 591
	.call		= srcu_torture_call,
	.cb_barrier	= srcu_torture_barrier,
592 593
	.stats		= srcu_torture_stats,
	.name		= "srcu"
594 595
};

596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
static void srcu_torture_init(void)
{
	rcu_sync_torture_init();
	WARN_ON(init_srcu_struct(&srcu_ctld));
	srcu_ctlp = &srcu_ctld;
}

static void srcu_torture_cleanup(void)
{
	cleanup_srcu_struct(&srcu_ctld);
	srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */
}

/* As above, but dynamically allocated. */
static struct rcu_torture_ops srcud_ops = {
	.ttype		= SRCU_FLAVOR,
	.init		= srcu_torture_init,
	.cleanup	= srcu_torture_cleanup,
	.readlock	= srcu_torture_read_lock,
	.read_delay	= srcu_read_delay,
	.readunlock	= srcu_torture_read_unlock,
	.started	= NULL,
	.completed	= srcu_torture_completed,
	.deferred_free	= srcu_torture_deferred_free,
	.sync		= srcu_torture_synchronize,
	.exp_sync	= srcu_torture_synchronize_expedited,
	.call		= srcu_torture_call,
	.cb_barrier	= srcu_torture_barrier,
	.stats		= srcu_torture_stats,
	.name		= "srcud"
};

628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
/*
 * Definitions for sched torture testing.
 */

static int sched_torture_read_lock(void)
{
	preempt_disable();
	return 0;
}

static void sched_torture_read_unlock(int idx)
{
	preempt_enable();
}

643 644 645 646 647
static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
}

648
static struct rcu_torture_ops sched_ops = {
649
	.ttype		= RCU_SCHED_FLAVOR,
650 651 652 653
	.init		= rcu_sync_torture_init,
	.readlock	= sched_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= sched_torture_read_unlock,
654
	.started	= rcu_batches_started_sched,
655
	.completed	= rcu_batches_completed_sched,
656
	.deferred_free	= rcu_sched_torture_deferred_free,
657
	.sync		= synchronize_sched,
658
	.exp_sync	= synchronize_sched_expedited,
659 660
	.get_state	= get_state_synchronize_sched,
	.cond_sync	= cond_synchronize_sched,
661
	.call		= call_rcu_sched,
662
	.cb_barrier	= rcu_barrier_sched,
663
	.fqs		= rcu_sched_force_quiescent_state,
664 665 666
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "sched"
667 668
};

669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
#ifdef CONFIG_TASKS_RCU

/*
 * Definitions for RCU-tasks torture testing.
 */

static int tasks_torture_read_lock(void)
{
	return 0;
}

static void tasks_torture_read_unlock(int idx)
{
}

static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb);
}

static struct rcu_torture_ops tasks_ops = {
	.ttype		= RCU_TASKS_FLAVOR,
	.init		= rcu_sync_torture_init,
	.readlock	= tasks_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= tasks_torture_read_unlock,
695
	.started	= rcu_no_completed,
696 697 698 699 700 701 702 703 704 705 706 707 708 709
	.completed	= rcu_no_completed,
	.deferred_free	= rcu_tasks_torture_deferred_free,
	.sync		= synchronize_rcu_tasks,
	.exp_sync	= synchronize_rcu_tasks,
	.call		= call_rcu_tasks,
	.cb_barrier	= rcu_barrier_tasks,
	.fqs		= NULL,
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "tasks"
};

#define RCUTORTURE_TASKS_OPS &tasks_ops,

710 711 712 713 714
static bool __maybe_unused torturing_tasks(void)
{
	return cur_ops == &tasks_ops;
}

715 716 717 718
#else /* #ifdef CONFIG_TASKS_RCU */

#define RCUTORTURE_TASKS_OPS

719
static bool __maybe_unused torturing_tasks(void)
720 721 722 723
{
	return false;
}

724 725
#endif /* #else #ifdef CONFIG_TASKS_RCU */

726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
/*
 * RCU torture priority-boost testing.  Runs one real-time thread per
 * CPU for moderate bursts, repeatedly registering RCU callbacks and
 * spinning waiting for them to be invoked.  If a given callback takes
 * too long to be invoked, we assume that priority inversion has occurred.
 */

struct rcu_boost_inflight {
	struct rcu_head rcu;
	int inflight;
};

static void rcu_torture_boost_cb(struct rcu_head *head)
{
	struct rcu_boost_inflight *rbip =
		container_of(head, struct rcu_boost_inflight, rcu);

743 744
	/* Ensure RCU-core accesses precede clearing ->inflight */
	smp_store_release(&rbip->inflight, 0);
745 746 747 748 749 750 751 752 753 754
}

static int rcu_torture_boost(void *arg)
{
	unsigned long call_rcu_time;
	unsigned long endtime;
	unsigned long oldstarttime;
	struct rcu_boost_inflight rbi = { .inflight = 0 };
	struct sched_param sp;

755
	VERBOSE_TOROUT_STRING("rcu_torture_boost started");
756 757 758 759

	/* Set real-time priority. */
	sp.sched_priority = 1;
	if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
760
		VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!");
761 762 763
		n_rcu_torture_boost_rterror++;
	}

764
	init_rcu_head_on_stack(&rbi.rcu);
765 766 767 768
	/* Each pass through the following loop does one boost-test cycle. */
	do {
		/* Wait for the next test interval. */
		oldstarttime = boost_starttime;
769
		while (ULONG_CMP_LT(jiffies, oldstarttime)) {
770
			schedule_timeout_interruptible(oldstarttime - jiffies);
771
			stutter_wait("rcu_torture_boost");
772
			if (torture_must_stop())
773 774 775 776 777 778
				goto checkwait;
		}

		/* Do one boost-test interval. */
		endtime = oldstarttime + test_boost_duration * HZ;
		call_rcu_time = jiffies;
779
		while (ULONG_CMP_LT(jiffies, endtime)) {
780
			/* If we don't have a callback in flight, post one. */
781 782 783
			if (!smp_load_acquire(&rbi.inflight)) {
				/* RCU core before ->inflight = 1. */
				smp_store_release(&rbi.inflight, 1);
784 785 786
				call_rcu(&rbi.rcu, rcu_torture_boost_cb);
				if (jiffies - call_rcu_time >
					 test_boost_duration * HZ - HZ / 2) {
787
					VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
788 789 790 791
					n_rcu_torture_boost_failure++;
				}
				call_rcu_time = jiffies;
			}
792
			stutter_wait("rcu_torture_boost");
793
			if (torture_must_stop())
794 795 796 797 798 799 800 801 802 803
				goto checkwait;
		}

		/*
		 * Set the start time of the next test interval.
		 * Yes, this is vulnerable to long delays, but such
		 * delays simply cause a false negative for the next
		 * interval.  Besides, we are running at RT priority,
		 * so delays should be relatively rare.
		 */
804 805
		while (oldstarttime == boost_starttime &&
		       !kthread_should_stop()) {
806 807 808 809 810 811 812 813 814 815 816
			if (mutex_trylock(&boost_mutex)) {
				boost_starttime = jiffies +
						  test_boost_interval * HZ;
				n_rcu_torture_boosts++;
				mutex_unlock(&boost_mutex);
				break;
			}
			schedule_timeout_uninterruptible(1);
		}

		/* Go do the stutter. */
817
checkwait:	stutter_wait("rcu_torture_boost");
818
	} while (!torture_must_stop());
819 820

	/* Clean up and exit. */
821
	while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) {
822
		torture_shutdown_absorb("rcu_torture_boost");
823
		schedule_timeout_uninterruptible(1);
824
	}
825
	destroy_rcu_head_on_stack(&rbi.rcu);
826
	torture_kthread_stopping("rcu_torture_boost");
827 828 829
	return 0;
}

830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857
static void rcu_torture_cbflood_cb(struct rcu_head *rhp)
{
}

/*
 * RCU torture callback-flood kthread.  Repeatedly induces bursts of calls
 * to call_rcu() or analogous, increasing the probability of occurrence
 * of callback-overflow corner cases.
 */
static int
rcu_torture_cbflood(void *arg)
{
	int err = 1;
	int i;
	int j;
	struct rcu_head *rhp;

	if (cbflood_n_per_burst > 0 &&
	    cbflood_inter_holdoff > 0 &&
	    cbflood_intra_holdoff > 0 &&
	    cur_ops->call &&
	    cur_ops->cb_barrier) {
		rhp = vmalloc(sizeof(*rhp) *
			      cbflood_n_burst * cbflood_n_per_burst);
		err = !rhp;
	}
	if (err) {
		VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM");
858
		goto wait_for_stop;
859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
	}
	VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
	do {
		schedule_timeout_interruptible(cbflood_inter_holdoff);
		atomic_long_inc(&n_cbfloods);
		WARN_ON(signal_pending(current));
		for (i = 0; i < cbflood_n_burst; i++) {
			for (j = 0; j < cbflood_n_per_burst; j++) {
				cur_ops->call(&rhp[i * cbflood_n_per_burst + j],
					      rcu_torture_cbflood_cb);
			}
			schedule_timeout_interruptible(cbflood_intra_holdoff);
			WARN_ON(signal_pending(current));
		}
		cur_ops->cb_barrier();
		stutter_wait("rcu_torture_cbflood");
	} while (!torture_must_stop());
876
	vfree(rhp);
877
wait_for_stop:
878 879 880 881
	torture_kthread_stopping("rcu_torture_cbflood");
	return 0;
}

882 883 884 885 886 887 888 889 890 891 892
/*
 * RCU torture force-quiescent-state kthread.  Repeatedly induces
 * bursts of calls to force_quiescent_state(), increasing the probability
 * of occurrence of some important types of race conditions.
 */
static int
rcu_torture_fqs(void *arg)
{
	unsigned long fqs_resume_time;
	int fqs_burst_remaining;

893
	VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
894 895
	do {
		fqs_resume_time = jiffies + fqs_stutter * HZ;
896 897
		while (ULONG_CMP_LT(jiffies, fqs_resume_time) &&
		       !kthread_should_stop()) {
898 899 900
			schedule_timeout_interruptible(1);
		}
		fqs_burst_remaining = fqs_duration;
901 902
		while (fqs_burst_remaining > 0 &&
		       !kthread_should_stop()) {
903 904 905 906
			cur_ops->fqs();
			udelay(fqs_holdoff);
			fqs_burst_remaining -= fqs_holdoff;
		}
907
		stutter_wait("rcu_torture_fqs");
908
	} while (!torture_must_stop());
909
	torture_kthread_stopping("rcu_torture_fqs");
910 911 912
	return 0;
}

913 914 915 916 917 918 919 920
/*
 * RCU torture writer kthread.  Repeatedly substitutes a new structure
 * for that pointed to by rcu_torture_current, freeing the old structure
 * after a series of grace periods (the "pipeline").
 */
static int
rcu_torture_writer(void *arg)
{
921 922
	bool can_expedite = !rcu_gp_is_expedited();
	int expediting = 0;
923 924
	unsigned long gp_snap;
	bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
925
	bool gp_sync1 = gp_sync;
926 927 928
	int i;
	struct rcu_torture *rp;
	struct rcu_torture *old_rp;
929
	static DEFINE_TORTURE_RANDOM(rand);
930 931
	int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC,
			   RTWS_COND_GET, RTWS_SYNC };
932
	int nsynctypes = 0;
933

934
	VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
935 936 937 938 939 940
	pr_alert("%s" TORTURE_FLAG
		 " Grace periods expedited from boot/sysfs for %s,\n",
		 torture_type, cur_ops->name);
	pr_alert("%s" TORTURE_FLAG
		 " Testing of dynamic grace-period expediting diabled.\n",
		 torture_type);
941

942
	/* Initialize synctype[] array.  If none set, take default. */
943
	if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1)
944
		gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true;
945 946 947 948 949 950 951 952 953 954 955 956
	if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync)
		synctype[nsynctypes++] = RTWS_COND_GET;
	else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync))
		pr_alert("rcu_torture_writer: gp_cond without primitives.\n");
	if (gp_exp1 && cur_ops->exp_sync)
		synctype[nsynctypes++] = RTWS_EXP_SYNC;
	else if (gp_exp && !cur_ops->exp_sync)
		pr_alert("rcu_torture_writer: gp_exp without primitives.\n");
	if (gp_normal1 && cur_ops->deferred_free)
		synctype[nsynctypes++] = RTWS_DEF_FREE;
	else if (gp_normal && !cur_ops->deferred_free)
		pr_alert("rcu_torture_writer: gp_normal without primitives.\n");
957 958 959 960
	if (gp_sync1 && cur_ops->sync)
		synctype[nsynctypes++] = RTWS_SYNC;
	else if (gp_sync && !cur_ops->sync)
		pr_alert("rcu_torture_writer: gp_sync without primitives.\n");
961 962
	if (WARN_ONCE(nsynctypes == 0,
		      "rcu_torture_writer: No update-side primitives.\n")) {
963 964 965 966 967
		/*
		 * No updates primitives, so don't try updating.
		 * The resulting test won't be testing much, hence the
		 * above WARN_ONCE().
		 */
968 969 970 971
		rcu_torture_writer_state = RTWS_STOPPING;
		torture_kthread_stopping("rcu_torture_writer");
	}

972
	do {
973
		rcu_torture_writer_state = RTWS_FIXED_DELAY;
974
		schedule_timeout_uninterruptible(1);
975 976
		rp = rcu_torture_alloc();
		if (rp == NULL)
977 978
			continue;
		rp->rtort_pipe_count = 0;
979
		rcu_torture_writer_state = RTWS_DELAY;
980
		udelay(torture_random(&rand) & 0x3ff);
981
		rcu_torture_writer_state = RTWS_REPLACE;
982 983
		old_rp = rcu_dereference_check(rcu_torture_current,
					       current == writer_task);
984
		rp->rtort_mbtest = 1;
985
		rcu_assign_pointer(rcu_torture_current, rp);
986
		smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
987
		if (old_rp) {
988 989 990 991 992
			i = old_rp->rtort_pipe_count;
			if (i > RCU_TORTURE_PIPE_LEN)
				i = RCU_TORTURE_PIPE_LEN;
			atomic_inc(&rcu_torture_wcount[i]);
			old_rp->rtort_pipe_count++;
993 994
			switch (synctype[torture_random(&rand) % nsynctypes]) {
			case RTWS_DEF_FREE:
995
				rcu_torture_writer_state = RTWS_DEF_FREE;
996
				cur_ops->deferred_free(old_rp);
997 998
				break;
			case RTWS_EXP_SYNC:
999
				rcu_torture_writer_state = RTWS_EXP_SYNC;
1000
				cur_ops->exp_sync();
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
				rcu_torture_pipe_update(old_rp);
				break;
			case RTWS_COND_GET:
				rcu_torture_writer_state = RTWS_COND_GET;
				gp_snap = cur_ops->get_state();
				i = torture_random(&rand) % 16;
				if (i != 0)
					schedule_timeout_interruptible(i);
				udelay(torture_random(&rand) % 1000);
				rcu_torture_writer_state = RTWS_COND_SYNC;
				cur_ops->cond_sync(gp_snap);
				rcu_torture_pipe_update(old_rp);
				break;
1014 1015 1016 1017 1018
			case RTWS_SYNC:
				rcu_torture_writer_state = RTWS_SYNC;
				cur_ops->sync();
				rcu_torture_pipe_update(old_rp);
				break;
1019 1020 1021
			default:
				WARN_ON_ONCE(1);
				break;
1022
			}
1023
		}
1024
		rcutorture_record_progress(++rcu_torture_current_version);
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
		/* Cycle through nesting levels of rcu_expedite_gp() calls. */
		if (can_expedite &&
		    !(torture_random(&rand) & 0xff & (!!expediting - 1))) {
			WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
			if (expediting >= 0)
				rcu_expedite_gp();
			else
				rcu_unexpedite_gp();
			if (++expediting > 3)
				expediting = -expediting;
		}
1036
		rcu_torture_writer_state = RTWS_STUTTER;
1037
		stutter_wait("rcu_torture_writer");
1038
	} while (!torture_must_stop());
1039 1040 1041 1042 1043 1044
	/* Reset expediting back to unexpedited. */
	if (expediting > 0)
		expediting = -expediting;
	while (can_expedite && expediting++ < 0)
		rcu_unexpedite_gp();
	WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
1045
	rcu_torture_writer_state = RTWS_STOPPING;
1046
	torture_kthread_stopping("rcu_torture_writer");
1047 1048 1049
	return 0;
}

1050 1051 1052 1053 1054 1055 1056
/*
 * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
 * delay between calls.
 */
static int
rcu_torture_fakewriter(void *arg)
{
1057
	DEFINE_TORTURE_RANDOM(rand);
1058

1059
	VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1060
	set_user_nice(current, MAX_NICE);
1061 1062

	do {
1063 1064
		schedule_timeout_uninterruptible(1 + torture_random(&rand)%10);
		udelay(torture_random(&rand) & 0x3ff);
1065
		if (cur_ops->cb_barrier != NULL &&
1066
		    torture_random(&rand) % (nfakewriters * 8) == 0) {
1067
			cur_ops->cb_barrier();
1068
		} else if (gp_normal == gp_exp) {
1069
			if (torture_random(&rand) & 0x80)
1070 1071 1072 1073
				cur_ops->sync();
			else
				cur_ops->exp_sync();
		} else if (gp_normal) {
1074
			cur_ops->sync();
1075 1076 1077
		} else {
			cur_ops->exp_sync();
		}
1078
		stutter_wait("rcu_torture_fakewriter");
1079
	} while (!torture_must_stop());
1080

1081
	torture_kthread_stopping("rcu_torture_fakewriter");
1082 1083 1084
	return 0;
}

1085
static void rcutorture_trace_dump(void)
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
{
	static atomic_t beenhere = ATOMIC_INIT(0);

	if (atomic_read(&beenhere))
		return;
	if (atomic_xchg(&beenhere, 1) != 0)
		return;
	ftrace_dump(DUMP_ALL);
}

1096 1097 1098 1099 1100 1101 1102 1103 1104
/*
 * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
 * incrementing the corresponding element of the pipeline array.  The
 * counter in the element should never be greater than 1, otherwise, the
 * RCU implementation is broken.
 */
static void rcu_torture_timer(unsigned long unused)
{
	int idx;
1105
	unsigned long started;
1106
	unsigned long completed;
1107
	static DEFINE_TORTURE_RANDOM(rand);
1108 1109 1110
	static DEFINE_SPINLOCK(rand_lock);
	struct rcu_torture *p;
	int pipe_count;
1111
	unsigned long long ts;
1112 1113

	idx = cur_ops->readlock();
1114 1115 1116 1117
	if (cur_ops->started)
		started = cur_ops->started();
	else
		started = cur_ops->completed();
1118
	ts = rcu_trace_clock_local();
1119 1120 1121
	p = rcu_dereference_check(rcu_torture_current,
				  rcu_read_lock_bh_held() ||
				  rcu_read_lock_sched_held() ||
1122 1123
				  srcu_read_lock_held(srcu_ctlp) ||
				  torturing_tasks());
1124 1125 1126 1127 1128 1129 1130 1131
	if (p == NULL) {
		/* Leave because rcu_torture_writer is not yet underway */
		cur_ops->readunlock(idx);
		return;
	}
	if (p->rtort_mbtest == 0)
		atomic_inc(&n_rcu_torture_mberror);
	spin_lock(&rand_lock);
1132
	cur_ops->read_delay(&rand);
1133 1134 1135 1136 1137 1138 1139 1140
	n_rcu_torture_timers++;
	spin_unlock(&rand_lock);
	preempt_disable();
	pipe_count = p->rtort_pipe_count;
	if (pipe_count > RCU_TORTURE_PIPE_LEN) {
		/* Should not happen, but... */
		pipe_count = RCU_TORTURE_PIPE_LEN;
	}
1141
	completed = cur_ops->completed();
1142 1143
	if (pipe_count > 1) {
		do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
1144
					  started, completed);
1145
		rcutorture_trace_dump();
1146
	}
R
Rusty Russell 已提交
1147
	__this_cpu_inc(rcu_torture_count[pipe_count]);
1148 1149 1150
	completed = completed - started;
	if (cur_ops->started)
		completed++;
1151 1152 1153 1154
	if (completed > RCU_TORTURE_PIPE_LEN) {
		/* Should not happen, but... */
		completed = RCU_TORTURE_PIPE_LEN;
	}
R
Rusty Russell 已提交
1155
	__this_cpu_inc(rcu_torture_batch[completed]);
1156 1157 1158 1159
	preempt_enable();
	cur_ops->readunlock(idx);
}

1160 1161 1162 1163 1164 1165 1166 1167 1168
/*
 * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 * incrementing the corresponding element of the pipeline array.  The
 * counter in the element should never be greater than 1, otherwise, the
 * RCU implementation is broken.
 */
static int
rcu_torture_reader(void *arg)
{
1169
	unsigned long started;
1170
	unsigned long completed;
1171
	int idx;
1172
	DEFINE_TORTURE_RANDOM(rand);
1173 1174
	struct rcu_torture *p;
	int pipe_count;
1175
	struct timer_list t;
1176
	unsigned long long ts;
1177

1178
	VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
1179
	set_user_nice(current, MAX_NICE);
1180
	if (irqreader && cur_ops->irq_capable)
1181
		setup_timer_on_stack(&t, rcu_torture_timer, 0);
1182

1183
	do {
1184
		if (irqreader && cur_ops->irq_capable) {
1185
			if (!timer_pending(&t))
1186
				mod_timer(&t, jiffies + 1);
1187
		}
1188
		idx = cur_ops->readlock();
1189 1190 1191 1192
		if (cur_ops->started)
			started = cur_ops->started();
		else
			started = cur_ops->completed();
1193
		ts = rcu_trace_clock_local();
1194 1195 1196
		p = rcu_dereference_check(rcu_torture_current,
					  rcu_read_lock_bh_held() ||
					  rcu_read_lock_sched_held() ||
1197 1198
					  srcu_read_lock_held(srcu_ctlp) ||
					  torturing_tasks());
1199 1200
		if (p == NULL) {
			/* Wait for rcu_torture_writer to get underway */
1201
			cur_ops->readunlock(idx);
1202 1203 1204
			schedule_timeout_interruptible(HZ);
			continue;
		}
1205 1206
		if (p->rtort_mbtest == 0)
			atomic_inc(&n_rcu_torture_mberror);
1207
		cur_ops->read_delay(&rand);
1208 1209 1210 1211 1212 1213
		preempt_disable();
		pipe_count = p->rtort_pipe_count;
		if (pipe_count > RCU_TORTURE_PIPE_LEN) {
			/* Should not happen, but... */
			pipe_count = RCU_TORTURE_PIPE_LEN;
		}
1214
		completed = cur_ops->completed();
1215 1216
		if (pipe_count > 1) {
			do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
1217
						  ts, started, completed);
1218
			rcutorture_trace_dump();
1219
		}
R
Rusty Russell 已提交
1220
		__this_cpu_inc(rcu_torture_count[pipe_count]);
1221 1222 1223
		completed = completed - started;
		if (cur_ops->started)
			completed++;
1224 1225 1226 1227
		if (completed > RCU_TORTURE_PIPE_LEN) {
			/* Should not happen, but... */
			completed = RCU_TORTURE_PIPE_LEN;
		}
R
Rusty Russell 已提交
1228
		__this_cpu_inc(rcu_torture_batch[completed]);
1229
		preempt_enable();
1230
		cur_ops->readunlock(idx);
1231
		stutter_wait("rcu_torture_reader");
1232
	} while (!torture_must_stop());
1233
	if (irqreader && cur_ops->irq_capable) {
1234
		del_timer_sync(&t);
1235 1236
		destroy_timer_on_stack(&t);
	}
1237
	torture_kthread_stopping("rcu_torture_reader");
1238 1239 1240 1241
	return 0;
}

/*
1242 1243 1244 1245 1246 1247
 * Print torture statistics.  Caller must ensure that there is only
 * one call to this function at a given time!!!  This is normally
 * accomplished by relying on the module system to only have one copy
 * of the module loaded, and then by giving the rcu_torture_stats
 * kthread full control (or the init/cleanup functions when rcu_torture_stats
 * thread is not running).
1248
 */
1249
static void
1250
rcu_torture_stats_print(void)
1251 1252 1253 1254 1255
{
	int cpu;
	int i;
	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1256
	static unsigned long rtcv_snap = ULONG_MAX;
1257

1258
	for_each_possible_cpu(cpu) {
1259 1260 1261 1262 1263 1264 1265 1266 1267
		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
			pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
			batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
		}
	}
	for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
		if (pipesummary[i] != 0)
			break;
	}
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
		rcu_torture_current,
		rcu_torture_current_version,
		list_empty(&rcu_torture_freelist),
		atomic_read(&n_rcu_torture_alloc),
		atomic_read(&n_rcu_torture_alloc_fail),
		atomic_read(&n_rcu_torture_free));
	pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ",
		atomic_read(&n_rcu_torture_mberror),
		n_rcu_torture_boost_ktrerror,
		n_rcu_torture_boost_rterror);
	pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
		n_rcu_torture_boost_failure,
		n_rcu_torture_boosts,
		n_rcu_torture_timers);
	torture_onoff_stats();
1286
	pr_cont("barrier: %ld/%ld:%ld ",
1287 1288 1289
		n_barrier_successes,
		n_barrier_attempts,
		n_rcu_torture_barrier_error);
1290
	pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods));
1291 1292

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1293
	if (atomic_read(&n_rcu_torture_mberror) != 0 ||
1294
	    n_rcu_torture_barrier_error != 0 ||
1295 1296
	    n_rcu_torture_boost_ktrerror != 0 ||
	    n_rcu_torture_boost_rterror != 0 ||
1297 1298
	    n_rcu_torture_boost_failure != 0 ||
	    i > 1) {
1299
		pr_cont("%s", "!!! ");
1300
		atomic_inc(&n_rcu_torture_error);
1301
		WARN_ON_ONCE(1);
1302
	}
1303
	pr_cont("Reader Pipe: ");
1304
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1305 1306 1307 1308 1309
		pr_cont(" %ld", pipesummary[i]);
	pr_cont("\n");

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("Reader Batch: ");
1310
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1311 1312 1313 1314 1315
		pr_cont(" %ld", batchsummary[i]);
	pr_cont("\n");

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("Free-Block Circulation: ");
1316
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1317
		pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
1318
	}
1319 1320
	pr_cont("\n");

1321
	if (cur_ops->stats)
1322
		cur_ops->stats();
1323 1324 1325 1326 1327 1328 1329 1330
	if (rtcv_snap == rcu_torture_current_version &&
	    rcu_torture_current != NULL) {
		int __maybe_unused flags;
		unsigned long __maybe_unused gpnum;
		unsigned long __maybe_unused completed;

		rcutorture_get_gp_data(cur_ops->ttype,
				       &flags, &gpnum, &completed);
1331 1332
		pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x\n",
			 rcu_torture_writer_state_getname(),
1333 1334
			 rcu_torture_writer_state,
			 gpnum, completed, flags);
1335
		show_rcu_gp_kthreads();
1336
		rcutorture_trace_dump();
1337 1338
	}
	rtcv_snap = rcu_torture_current_version;
1339 1340 1341 1342 1343 1344 1345 1346 1347
}

/*
 * Periodically prints torture statistics, if periodic statistics printing
 * was specified via the stat_interval module parameter.
 */
static int
rcu_torture_stats(void *arg)
{
1348
	VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
1349 1350 1351
	do {
		schedule_timeout_interruptible(stat_interval * HZ);
		rcu_torture_stats_print();
1352
		torture_shutdown_absorb("rcu_torture_stats");
1353
	} while (!torture_must_stop());
1354
	torture_kthread_stopping("rcu_torture_stats");
1355 1356 1357
	return 0;
}

1358
static inline void
1359
rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
1360
{
1361 1362 1363 1364 1365 1366 1367
	pr_alert("%s" TORTURE_FLAG
		 "--- %s: nreaders=%d nfakewriters=%d "
		 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
		 "shuffle_interval=%d stutter=%d irqreader=%d "
		 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
		 "test_boost=%d/%d test_boost_interval=%d "
		 "test_boost_duration=%d shutdown_secs=%d "
1368 1369
		 "stall_cpu=%d stall_cpu_holdoff=%d "
		 "n_barrier_cbs=%d "
1370 1371 1372 1373 1374 1375
		 "onoff_interval=%d onoff_holdoff=%d\n",
		 torture_type, tag, nrealreaders, nfakewriters,
		 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
		 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
		 test_boost, cur_ops->can_boost,
		 test_boost_interval, test_boost_duration, shutdown_secs,
1376 1377
		 stall_cpu, stall_cpu_holdoff,
		 n_barrier_cbs,
1378
		 onoff_interval, onoff_holdoff);
1379 1380
}

1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
static void rcutorture_booster_cleanup(int cpu)
{
	struct task_struct *t;

	if (boost_tasks[cpu] == NULL)
		return;
	mutex_lock(&boost_mutex);
	t = boost_tasks[cpu];
	boost_tasks[cpu] = NULL;
	mutex_unlock(&boost_mutex);

	/* This must be outside of the mutex, otherwise deadlock! */
1393
	torture_stop_kthread(rcu_torture_boost, t);
1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
}

static int rcutorture_booster_init(int cpu)
{
	int retval;

	if (boost_tasks[cpu] != NULL)
		return 0;  /* Already created, nothing more to do. */

	/* Don't allow time recalculation while creating a new task. */
	mutex_lock(&boost_mutex);
1405
	VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
E
Eric Dumazet 已提交
1406 1407 1408
	boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
						  cpu_to_node(cpu),
						  "rcu_torture_boost");
1409 1410
	if (IS_ERR(boost_tasks[cpu])) {
		retval = PTR_ERR(boost_tasks[cpu]);
1411
		VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422
		n_rcu_torture_boost_ktrerror++;
		boost_tasks[cpu] = NULL;
		mutex_unlock(&boost_mutex);
		return retval;
	}
	kthread_bind(boost_tasks[cpu], cpu);
	wake_up_process(boost_tasks[cpu]);
	mutex_unlock(&boost_mutex);
	return 0;
}

1423 1424 1425 1426
/*
 * CPU-stall kthread.  It waits as specified by stall_cpu_holdoff, then
 * induces a CPU stall for the time specified by stall_cpu.
 */
1427
static int rcu_torture_stall(void *args)
1428 1429 1430
{
	unsigned long stop_at;

1431
	VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
1432
	if (stall_cpu_holdoff > 0) {
1433
		VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
1434
		schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1435
		VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
1436 1437 1438 1439
	}
	if (!kthread_should_stop()) {
		stop_at = get_seconds() + stall_cpu;
		/* RCU CPU stall is expected behavior in following code. */
1440
		pr_alert("rcu_torture_stall start.\n");
1441 1442 1443 1444 1445 1446
		rcu_read_lock();
		preempt_disable();
		while (ULONG_CMP_LT(get_seconds(), stop_at))
			continue;  /* Induce RCU CPU stall warning. */
		preempt_enable();
		rcu_read_unlock();
1447
		pr_alert("rcu_torture_stall end.\n");
1448
	}
1449
	torture_shutdown_absorb("rcu_torture_stall");
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
	while (!kthread_should_stop())
		schedule_timeout_interruptible(10 * HZ);
	return 0;
}

/* Spawn CPU-stall kthread, if stall_cpu specified. */
static int __init rcu_torture_stall_init(void)
{
	if (stall_cpu <= 0)
		return 0;
1460
	return torture_create_kthread(rcu_torture_stall, NULL, stall_task);
1461 1462
}

1463
/* Callback function for RCU barrier testing. */
1464
static void rcu_torture_barrier_cbf(struct rcu_head *rcu)
1465 1466 1467 1468 1469 1470 1471 1472
{
	atomic_inc(&barrier_cbs_invoked);
}

/* kthread function to register callbacks used to test RCU barriers. */
static int rcu_torture_barrier_cbs(void *arg)
{
	long myid = (long)arg;
1473
	bool lastphase = 0;
1474
	bool newphase;
1475 1476 1477
	struct rcu_head rcu;

	init_rcu_head_on_stack(&rcu);
1478
	VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
1479
	set_user_nice(current, MAX_NICE);
1480 1481
	do {
		wait_event(barrier_cbs_wq[myid],
1482
			   (newphase =
1483
			    smp_load_acquire(&barrier_phase)) != lastphase ||
1484
			   torture_must_stop());
1485
		lastphase = newphase;
1486
		if (torture_must_stop())
1487
			break;
1488 1489 1490 1491
		/*
		 * The above smp_load_acquire() ensures barrier_phase load
		 * is ordered before the folloiwng ->call().
		 */
1492 1493 1494
		cur_ops->call(&rcu, rcu_torture_barrier_cbf);
		if (atomic_dec_and_test(&barrier_cbs_count))
			wake_up(&barrier_wq);
1495
	} while (!torture_must_stop());
1496 1497
	if (cur_ops->cb_barrier != NULL)
		cur_ops->cb_barrier();
1498
	destroy_rcu_head_on_stack(&rcu);
1499
	torture_kthread_stopping("rcu_torture_barrier_cbs");
1500 1501 1502 1503 1504 1505 1506 1507
	return 0;
}

/* kthread function to drive and coordinate RCU barrier testing. */
static int rcu_torture_barrier(void *arg)
{
	int i;

1508
	VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
1509 1510 1511
	do {
		atomic_set(&barrier_cbs_invoked, 0);
		atomic_set(&barrier_cbs_count, n_barrier_cbs);
1512 1513
		/* Ensure barrier_phase ordered after prior assignments. */
		smp_store_release(&barrier_phase, !barrier_phase);
1514 1515 1516 1517
		for (i = 0; i < n_barrier_cbs; i++)
			wake_up(&barrier_cbs_wq[i]);
		wait_event(barrier_wq,
			   atomic_read(&barrier_cbs_count) == 0 ||
1518 1519
			   torture_must_stop());
		if (torture_must_stop())
1520 1521
			break;
		n_barrier_attempts++;
1522
		cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
1523 1524
		if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
			n_rcu_torture_barrier_error++;
1525 1526 1527
			pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
			       atomic_read(&barrier_cbs_invoked),
			       n_barrier_cbs);
1528 1529 1530 1531
			WARN_ON_ONCE(1);
		}
		n_barrier_successes++;
		schedule_timeout_interruptible(HZ / 10);
1532
	} while (!torture_must_stop());
1533
	torture_kthread_stopping("rcu_torture_barrier");
1534 1535 1536 1537 1538 1539 1540 1541 1542
	return 0;
}

/* Initialize RCU barrier testing. */
static int rcu_torture_barrier_init(void)
{
	int i;
	int ret;

1543
	if (n_barrier_cbs <= 0)
1544 1545
		return 0;
	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
1546 1547 1548 1549 1550 1551
		pr_alert("%s" TORTURE_FLAG
			 " Call or barrier ops missing for %s,\n",
			 torture_type, cur_ops->name);
		pr_alert("%s" TORTURE_FLAG
			 " RCU barrier testing omitted from run.\n",
			 torture_type);
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561
		return 0;
	}
	atomic_set(&barrier_cbs_count, 0);
	atomic_set(&barrier_cbs_invoked, 0);
	barrier_cbs_tasks =
		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
			GFP_KERNEL);
	barrier_cbs_wq =
		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
			GFP_KERNEL);
1562
	if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
1563 1564 1565
		return -ENOMEM;
	for (i = 0; i < n_barrier_cbs; i++) {
		init_waitqueue_head(&barrier_cbs_wq[i]);
1566 1567 1568 1569
		ret = torture_create_kthread(rcu_torture_barrier_cbs,
					     (void *)(long)i,
					     barrier_cbs_tasks[i]);
		if (ret)
1570 1571
			return ret;
	}
1572
	return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task);
1573 1574 1575 1576 1577 1578 1579
}

/* Clean up after RCU barrier testing. */
static void rcu_torture_barrier_cleanup(void)
{
	int i;

1580
	torture_stop_kthread(rcu_torture_barrier, barrier_task);
1581
	if (barrier_cbs_tasks != NULL) {
1582 1583 1584
		for (i = 0; i < n_barrier_cbs; i++)
			torture_stop_kthread(rcu_torture_barrier_cbs,
					     barrier_cbs_tasks[i]);
1585 1586 1587 1588 1589 1590 1591 1592 1593
		kfree(barrier_cbs_tasks);
		barrier_cbs_tasks = NULL;
	}
	if (barrier_cbs_wq != NULL) {
		kfree(barrier_cbs_wq);
		barrier_cbs_wq = NULL;
	}
}

1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
static int rcutorture_cpu_notify(struct notifier_block *self,
				 unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;

	switch (action) {
	case CPU_ONLINE:
	case CPU_DOWN_FAILED:
		(void)rcutorture_booster_init(cpu);
		break;
	case CPU_DOWN_PREPARE:
		rcutorture_booster_cleanup(cpu);
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block rcutorture_cpu_nb = {
	.notifier_call = rcutorture_cpu_notify,
};

1617 1618 1619 1620 1621
static void
rcu_torture_cleanup(void)
{
	int i;

1622
	rcutorture_record_test_transition();
1623
	if (torture_cleanup_begin()) {
1624 1625 1626 1627
		if (cur_ops->cb_barrier != NULL)
			cur_ops->cb_barrier();
		return;
	}
1628

1629
	rcu_torture_barrier_cleanup();
1630 1631
	torture_stop_kthread(rcu_torture_stall, stall_task);
	torture_stop_kthread(rcu_torture_writer, writer_task);
1632

1633
	if (reader_tasks) {
1634 1635 1636
		for (i = 0; i < nrealreaders; i++)
			torture_stop_kthread(rcu_torture_reader,
					     reader_tasks[i]);
1637 1638 1639 1640
		kfree(reader_tasks);
	}
	rcu_torture_current = NULL;

1641
	if (fakewriter_tasks) {
1642
		for (i = 0; i < nfakewriters; i++) {
1643 1644
			torture_stop_kthread(rcu_torture_fakewriter,
					     fakewriter_tasks[i]);
1645 1646 1647 1648 1649
		}
		kfree(fakewriter_tasks);
		fakewriter_tasks = NULL;
	}

1650 1651
	torture_stop_kthread(rcu_torture_stats, stats_task);
	torture_stop_kthread(rcu_torture_fqs, fqs_task);
1652 1653
	for (i = 0; i < ncbflooders; i++)
		torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]);
1654 1655 1656 1657 1658 1659
	if ((test_boost == 1 && cur_ops->can_boost) ||
	    test_boost == 2) {
		unregister_cpu_notifier(&rcutorture_cpu_nb);
		for_each_possible_cpu(i)
			rcutorture_booster_cleanup(i);
	}
1660

1661 1662 1663 1664
	/*
	 * Wait for all RCU callbacks to fire, then do flavor-specific
	 * cleanup operations.
	 */
1665 1666
	if (cur_ops->cb_barrier != NULL)
		cur_ops->cb_barrier();
1667 1668
	if (cur_ops->cleanup != NULL)
		cur_ops->cleanup();
1669 1670

	rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1671

1672
	if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
1673
		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
1674
	else if (torture_onoff_failures())
1675 1676
		rcu_torture_print_module_parms(cur_ops,
					       "End of test: RCU_HOTPLUG");
1677
	else
1678
		rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
1679
	torture_cleanup_end();
1680 1681
}

1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
static void rcu_torture_leak_cb(struct rcu_head *rhp)
{
}

static void rcu_torture_err_cb(struct rcu_head *rhp)
{
	/*
	 * This -might- happen due to race conditions, but is unlikely.
	 * The scenario that leads to this happening is that the
	 * first of the pair of duplicate callbacks is queued,
	 * someone else starts a grace period that includes that
	 * callback, then the second of the pair must wait for the
	 * next grace period.  Unlikely, but can happen.  If it
	 * does happen, the debug-objects subsystem won't have splatted.
	 */
	pr_alert("rcutorture: duplicated callback was invoked.\n");
}
#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */

/*
 * Verify that double-free causes debug-objects to complain, but only
 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.  Otherwise, say that the test
 * cannot be carried out.
 */
static void rcu_test_debug_objects(void)
{
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	struct rcu_head rh1;
	struct rcu_head rh2;

	init_rcu_head_on_stack(&rh1);
	init_rcu_head_on_stack(&rh2);
	pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");

	/* Try to queue the rh2 pair of callbacks for the same grace period. */
	preempt_disable(); /* Prevent preemption from interrupting test. */
	rcu_read_lock(); /* Make it impossible to finish a grace period. */
	call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */
	local_irq_disable(); /* Make it harder to start a new grace period. */
	call_rcu(&rh2, rcu_torture_leak_cb);
	call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
	local_irq_enable();
	rcu_read_unlock();
	preempt_enable();

	/* Wait for them all to get done so we can safely return. */
	rcu_barrier();
	pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
	destroy_rcu_head_on_stack(&rh1);
	destroy_rcu_head_on_stack(&rh2);
#else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
	pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
#endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
}

1738
static int __init
1739 1740 1741 1742 1743
rcu_torture_init(void)
{
	int i;
	int cpu;
	int firsterr = 0;
1744
	static struct rcu_torture_ops *torture_ops[] = {
1745 1746
		&rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops,
		&sched_ops, RCUTORTURE_TASKS_OPS
1747
	};
1748

1749
	if (!torture_init_begin(torture_type, verbose, &torture_runnable))
1750
		return -EBUSY;
1751

1752
	/* Process args and tell the world that the torturer is on the job. */
1753
	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1754
		cur_ops = torture_ops[i];
1755
		if (strcmp(torture_type, cur_ops->name) == 0)
1756 1757
			break;
	}
1758
	if (i == ARRAY_SIZE(torture_ops)) {
1759 1760 1761
		pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
			 torture_type);
		pr_alert("rcu-torture types:");
1762
		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1763 1764
			pr_alert(" %s", torture_ops[i]->name);
		pr_alert("\n");
1765 1766
		firsterr = -EINVAL;
		goto unwind;
1767
	}
1768
	if (cur_ops->fqs == NULL && fqs_duration != 0) {
1769
		pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
1770 1771
		fqs_duration = 0;
	}
1772
	if (cur_ops->init)
1773
		cur_ops->init();
1774

1775
	if (nreaders >= 0) {
1776
		nrealreaders = nreaders;
1777
	} else {
1778
		nrealreaders = num_online_cpus() - 2 - nreaders;
1779 1780 1781
		if (nrealreaders <= 0)
			nrealreaders = 1;
	}
1782
	rcu_torture_print_module_parms(cur_ops, "Start of test");
1783 1784 1785 1786

	/* Set up the freelist. */

	INIT_LIST_HEAD(&rcu_torture_freelist);
1787
	for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1788
		rcu_tortures[i].rtort_mbtest = 0;
1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
		list_add_tail(&rcu_tortures[i].rtort_free,
			      &rcu_torture_freelist);
	}

	/* Initialize the statistics so that each run gets its own numbers. */

	rcu_torture_current = NULL;
	rcu_torture_current_version = 0;
	atomic_set(&n_rcu_torture_alloc, 0);
	atomic_set(&n_rcu_torture_alloc_fail, 0);
	atomic_set(&n_rcu_torture_free, 0);
1800 1801
	atomic_set(&n_rcu_torture_mberror, 0);
	atomic_set(&n_rcu_torture_error, 0);
1802
	n_rcu_torture_barrier_error = 0;
1803 1804 1805 1806
	n_rcu_torture_boost_ktrerror = 0;
	n_rcu_torture_boost_rterror = 0;
	n_rcu_torture_boost_failure = 0;
	n_rcu_torture_boosts = 0;
1807 1808
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
		atomic_set(&rcu_torture_wcount[i], 0);
1809
	for_each_possible_cpu(cpu) {
1810 1811 1812 1813 1814 1815 1816 1817
		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
			per_cpu(rcu_torture_count, cpu)[i] = 0;
			per_cpu(rcu_torture_batch, cpu)[i] = 0;
		}
	}

	/* Start up the kthreads. */

1818 1819 1820
	firsterr = torture_create_kthread(rcu_torture_writer, NULL,
					  writer_task);
	if (firsterr)
1821
		goto unwind;
1822 1823 1824 1825 1826 1827 1828 1829 1830
	if (nfakewriters > 0) {
		fakewriter_tasks = kzalloc(nfakewriters *
					   sizeof(fakewriter_tasks[0]),
					   GFP_KERNEL);
		if (fakewriter_tasks == NULL) {
			VERBOSE_TOROUT_ERRSTRING("out of memory");
			firsterr = -ENOMEM;
			goto unwind;
		}
1831 1832
	}
	for (i = 0; i < nfakewriters; i++) {
1833 1834 1835
		firsterr = torture_create_kthread(rcu_torture_fakewriter,
						  NULL, fakewriter_tasks[i]);
		if (firsterr)
1836 1837
			goto unwind;
	}
1838
	reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
1839 1840
			       GFP_KERNEL);
	if (reader_tasks == NULL) {
1841
		VERBOSE_TOROUT_ERRSTRING("out of memory");
1842 1843 1844 1845
		firsterr = -ENOMEM;
		goto unwind;
	}
	for (i = 0; i < nrealreaders; i++) {
1846 1847 1848
		firsterr = torture_create_kthread(rcu_torture_reader, NULL,
						  reader_tasks[i]);
		if (firsterr)
1849 1850 1851
			goto unwind;
	}
	if (stat_interval > 0) {
1852 1853 1854
		firsterr = torture_create_kthread(rcu_torture_stats, NULL,
						  stats_task);
		if (firsterr)
1855 1856
			goto unwind;
	}
1857
	if (test_no_idle_hz && shuffle_interval > 0) {
1858 1859
		firsterr = torture_shuffle_init(shuffle_interval * HZ);
		if (firsterr)
1860 1861
			goto unwind;
	}
1862 1863 1864
	if (stutter < 0)
		stutter = 0;
	if (stutter) {
1865 1866
		firsterr = torture_stutter_init(stutter * HZ);
		if (firsterr)
1867 1868
			goto unwind;
	}
1869 1870 1871
	if (fqs_duration < 0)
		fqs_duration = 0;
	if (fqs_duration) {
1872
		/* Create the fqs thread */
1873 1874
		firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
						  fqs_task);
1875
		if (firsterr)
1876 1877
			goto unwind;
	}
1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
	if (test_boost_interval < 1)
		test_boost_interval = 1;
	if (test_boost_duration < 2)
		test_boost_duration = 2;
	if ((test_boost == 1 && cur_ops->can_boost) ||
	    test_boost == 2) {

		boost_starttime = jiffies + test_boost_interval * HZ;
		register_cpu_notifier(&rcutorture_cpu_nb);
		for_each_possible_cpu(i) {
			if (cpu_is_offline(i))
				continue;  /* Heuristic: CPU can go offline. */
1890 1891
			firsterr = rcutorture_booster_init(i);
			if (firsterr)
1892 1893 1894
				goto unwind;
		}
	}
1895 1896
	firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
	if (firsterr)
1897
		goto unwind;
1898 1899
	firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ);
	if (firsterr)
1900
		goto unwind;
1901 1902
	firsterr = rcu_torture_stall_init();
	if (firsterr)
1903
		goto unwind;
1904 1905
	firsterr = rcu_torture_barrier_init();
	if (firsterr)
1906
		goto unwind;
1907 1908
	if (object_debug)
		rcu_test_debug_objects();
1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926
	if (cbflood_n_burst > 0) {
		/* Create the cbflood threads */
		ncbflooders = (num_online_cpus() + 3) / 4;
		cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task),
				       GFP_KERNEL);
		if (!cbflood_task) {
			VERBOSE_TOROUT_ERRSTRING("out of memory");
			firsterr = -ENOMEM;
			goto unwind;
		}
		for (i = 0; i < ncbflooders; i++) {
			firsterr = torture_create_kthread(rcu_torture_cbflood,
							  NULL,
							  cbflood_task[i]);
			if (firsterr)
				goto unwind;
		}
	}
1927
	rcutorture_record_test_transition();
1928
	torture_init_end();
1929 1930 1931
	return 0;

unwind:
1932
	torture_init_end();
1933 1934 1935 1936 1937 1938
	rcu_torture_cleanup();
	return firsterr;
}

module_init(rcu_torture_init);
module_exit(rcu_torture_cleanup);