rcutorture.c 54.3 KB
Newer Older
1
/*
2
 * Read-Copy Update module-based torture test facility
3 4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
15 16
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
17
 *
18
 * Copyright (C) IBM Corporation, 2005, 2006
19 20
 *
 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
J
Josh Triplett 已提交
21
 *	  Josh Triplett <josh@joshtriplett.org>
22 23 24 25 26 27 28 29 30 31 32 33 34
 *
 * See also:  Documentation/RCU/torture.txt
 */
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/err.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/rcupdate.h>
#include <linux/interrupt.h>
35
#include <linux/sched/signal.h>
36
#include <uapi/linux/sched/types.h>
A
Arun Sharma 已提交
37
#include <linux/atomic.h>
38 39 40 41 42
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/moduleparam.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
43
#include <linux/reboot.h>
44
#include <linux/freezer.h>
45 46 47
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/stat.h>
48
#include <linux/srcu.h>
49
#include <linux/slab.h>
50
#include <linux/trace_clock.h>
51
#include <asm/byteorder.h>
52
#include <linux/torture.h>
53
#include <linux/vmalloc.h>
54
#include <linux/sched/debug.h>
55

56 57
#include "rcu.h"

58
MODULE_LICENSE("GPL");
J
Josh Triplett 已提交
59
MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
60

61

62 63 64 65 66 67 68
torture_param(int, cbflood_inter_holdoff, HZ,
	      "Holdoff between floods (jiffies)");
torture_param(int, cbflood_intra_holdoff, 1,
	      "Holdoff between bursts (jiffies)");
torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable");
torture_param(int, cbflood_n_per_burst, 20000,
	      "# callbacks per burst in flood");
69 70 71 72
torture_param(int, fqs_duration, 0,
	      "Duration of fqs bursts (us), 0 to disable");
torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
73
torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
74 75 76
torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
torture_param(bool, gp_normal, false,
	     "Use normal (non-expedited) GP wait primitives");
77
torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
torture_param(int, n_barrier_cbs, 0,
	     "# of callbacks/kthreads for barrier testing");
torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
torture_param(int, nreaders, -1, "Number of RCU reader threads");
torture_param(int, object_debug, 0,
	     "Enable debug-object double call_rcu() testing");
torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
torture_param(int, onoff_interval, 0,
	     "Time between CPU hotplugs (s), 0=disable");
torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
torture_param(int, stall_cpu_holdoff, 10,
	     "Time to wait before starting stall (s).");
93
torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");
94 95 96 97 98 99 100 101 102 103
torture_param(int, stat_interval, 60,
	     "Number of seconds between stats printk()s");
torture_param(int, stutter, 5, "Number of seconds to run/halt test");
torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
torture_param(int, test_boost_duration, 4,
	     "Duration of each boost test, seconds.");
torture_param(int, test_boost_interval, 7,
	     "Interval between boost tests, seconds.");
torture_param(bool, test_no_idle_hz, true,
	     "Test support for tickless idle CPUs");
104 105
torture_param(bool, verbose, true,
	     "Enable verbose debugging printk()s");
106

107
static char *torture_type = "rcu";
108
module_param(torture_type, charp, 0444);
109
MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)");
110 111

static int nrealreaders;
112
static int ncbflooders;
113
static struct task_struct *writer_task;
114
static struct task_struct **fakewriter_tasks;
115 116
static struct task_struct **reader_tasks;
static struct task_struct *stats_task;
117
static struct task_struct **cbflood_task;
118
static struct task_struct *fqs_task;
119
static struct task_struct *boost_tasks[NR_CPUS];
120
static struct task_struct *stall_task;
121 122
static struct task_struct **barrier_cbs_tasks;
static struct task_struct *barrier_task;
123 124 125 126 127 128 129

#define RCU_TORTURE_PIPE_LEN 10

struct rcu_torture {
	struct rcu_head rtort_rcu;
	int rtort_pipe_count;
	struct list_head rtort_free;
130
	int rtort_mbtest;
131 132 133
};

static LIST_HEAD(rcu_torture_freelist);
134
static struct rcu_torture __rcu *rcu_torture_current;
135
static unsigned long rcu_torture_current_version;
136 137
static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
static DEFINE_SPINLOCK(rcu_torture_lock);
138 139
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count);
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch);
140
static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
141 142 143 144 145
static atomic_t n_rcu_torture_alloc;
static atomic_t n_rcu_torture_alloc_fail;
static atomic_t n_rcu_torture_free;
static atomic_t n_rcu_torture_mberror;
static atomic_t n_rcu_torture_error;
146
static long n_rcu_torture_barrier_error;
147 148 149 150
static long n_rcu_torture_boost_ktrerror;
static long n_rcu_torture_boost_rterror;
static long n_rcu_torture_boost_failure;
static long n_rcu_torture_boosts;
151
static long n_rcu_torture_timers;
152 153
static long n_barrier_attempts;
static long n_barrier_successes;
154
static atomic_long_t n_cbfloods;
155
static struct list_head rcu_torture_removed;
156

157 158 159 160 161 162
static int rcu_torture_writer_state;
#define RTWS_FIXED_DELAY	0
#define RTWS_DELAY		1
#define RTWS_REPLACE		2
#define RTWS_DEF_FREE		3
#define RTWS_EXP_SYNC		4
163 164
#define RTWS_COND_GET		5
#define RTWS_COND_SYNC		6
165 166 167
#define RTWS_SYNC		7
#define RTWS_STUTTER		8
#define RTWS_STOPPING		9
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
static const char * const rcu_torture_writer_state_names[] = {
	"RTWS_FIXED_DELAY",
	"RTWS_DELAY",
	"RTWS_REPLACE",
	"RTWS_DEF_FREE",
	"RTWS_EXP_SYNC",
	"RTWS_COND_GET",
	"RTWS_COND_SYNC",
	"RTWS_SYNC",
	"RTWS_STUTTER",
	"RTWS_STOPPING",
};

static const char *rcu_torture_writer_state_getname(void)
{
	unsigned int i = READ_ONCE(rcu_torture_writer_state);

	if (i >= ARRAY_SIZE(rcu_torture_writer_state_names))
		return "???";
	return rcu_torture_writer_state_names[i];
}
189

190
static int torture_runnable = IS_ENABLED(MODULE);
191 192
module_param(torture_runnable, int, 0444);
MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot");
193

194
#if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
195
#define rcu_can_boost() 1
196
#else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
197
#define rcu_can_boost() 0
198
#endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
199

200 201 202 203
#ifdef CONFIG_RCU_TRACE
static u64 notrace rcu_trace_clock_local(void)
{
	u64 ts = trace_clock_local();
204 205

	(void)do_div(ts, NSEC_PER_USEC);
206 207 208 209 210 211 212 213 214
	return ts;
}
#else /* #ifdef CONFIG_RCU_TRACE */
static u64 notrace rcu_trace_clock_local(void)
{
	return 0ULL;
}
#endif /* #else #ifdef CONFIG_RCU_TRACE */

215
static unsigned long boost_starttime;	/* jiffies of next boost test start. */
216
static DEFINE_MUTEX(boost_mutex);	/* protect setting boost_starttime */
217
					/*  and boost task create/destroy. */
218
static atomic_t barrier_cbs_count;	/* Barrier callbacks registered. */
219
static bool barrier_phase;		/* Test phase. */
220 221 222
static atomic_t barrier_cbs_invoked;	/* Barrier callbacks invoked. */
static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
223

224 225 226
/*
 * Allocate an element from the rcu_tortures pool.
 */
A
Adrian Bunk 已提交
227
static struct rcu_torture *
228 229 230 231
rcu_torture_alloc(void)
{
	struct list_head *p;

232
	spin_lock_bh(&rcu_torture_lock);
233 234
	if (list_empty(&rcu_torture_freelist)) {
		atomic_inc(&n_rcu_torture_alloc_fail);
235
		spin_unlock_bh(&rcu_torture_lock);
236 237 238 239 240
		return NULL;
	}
	atomic_inc(&n_rcu_torture_alloc);
	p = rcu_torture_freelist.next;
	list_del_init(p);
241
	spin_unlock_bh(&rcu_torture_lock);
242 243 244 245 246 247 248 249 250 251
	return container_of(p, struct rcu_torture, rtort_free);
}

/*
 * Free an element to the rcu_tortures pool.
 */
static void
rcu_torture_free(struct rcu_torture *p)
{
	atomic_inc(&n_rcu_torture_free);
252
	spin_lock_bh(&rcu_torture_lock);
253
	list_add_tail(&p->rtort_free, &rcu_torture_freelist);
254
	spin_unlock_bh(&rcu_torture_lock);
255 256
}

257 258 259 260 261
/*
 * Operations vector for selecting different types of tests.
 */

struct rcu_torture_ops {
262
	int ttype;
263
	void (*init)(void);
264
	void (*cleanup)(void);
265
	int (*readlock)(void);
266
	void (*read_delay)(struct torture_random_state *rrsp);
267
	void (*readunlock)(int idx);
268
	unsigned long (*started)(void);
269
	unsigned long (*completed)(void);
270
	void (*deferred_free)(struct rcu_torture *p);
271
	void (*sync)(void);
272
	void (*exp_sync)(void);
273 274
	unsigned long (*get_state)(void);
	void (*cond_sync)(unsigned long oldstate);
275
	call_rcu_func_t call;
276
	void (*cb_barrier)(void);
277
	void (*fqs)(void);
278
	void (*stats)(void);
279
	int irq_capable;
280
	int can_boost;
281
	const char *name;
282
};
283 284

static struct rcu_torture_ops *cur_ops;
285 286 287 288 289

/*
 * Definitions for rcu torture testing.
 */

290
static int rcu_torture_read_lock(void) __acquires(RCU)
291 292 293 294 295
{
	rcu_read_lock();
	return 0;
}

296
static void rcu_read_delay(struct torture_random_state *rrsp)
297
{
298 299
	unsigned long started;
	unsigned long completed;
300 301
	const unsigned long shortdelay_us = 200;
	const unsigned long longdelay_ms = 50;
302
	unsigned long long ts;
303

304 305 306
	/* We want a short delay sometimes to make a reader delay the grace
	 * period, and we want a long delay occasionally to trigger
	 * force_quiescent_state. */
307

308 309 310
	if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
		started = cur_ops->completed();
		ts = rcu_trace_clock_local();
311
		mdelay(longdelay_ms);
312 313 314 315
		completed = cur_ops->completed();
		do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
					  started, completed);
	}
316
	if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
317
		udelay(shortdelay_us);
318
#ifdef CONFIG_PREEMPT
319 320
	if (!preempt_count() &&
	    !(torture_random(rrsp) % (nrealreaders * 20000)))
321 322
		preempt_schedule();  /* No QS if preempt_disable() in effect */
#endif
323 324
}

325
static void rcu_torture_read_unlock(int idx) __releases(RCU)
326 327 328 329
{
	rcu_read_unlock();
}

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
/*
 * Update callback in the pipe.  This should be invoked after a grace period.
 */
static bool
rcu_torture_pipe_update_one(struct rcu_torture *rp)
{
	int i;

	i = rp->rtort_pipe_count;
	if (i > RCU_TORTURE_PIPE_LEN)
		i = RCU_TORTURE_PIPE_LEN;
	atomic_inc(&rcu_torture_wcount[i]);
	if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
		rp->rtort_mbtest = 0;
		return true;
	}
	return false;
}

/*
 * Update all callbacks in the pipe.  Suitable for synchronous grace-period
 * primitives.
 */
static void
rcu_torture_pipe_update(struct rcu_torture *old_rp)
{
	struct rcu_torture *rp;
	struct rcu_torture *rp1;

	if (old_rp)
		list_add(&old_rp->rtort_free, &rcu_torture_removed);
	list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
		if (rcu_torture_pipe_update_one(rp)) {
			list_del(&rp->rtort_free);
			rcu_torture_free(rp);
		}
	}
}

369 370 371 372 373
static void
rcu_torture_cb(struct rcu_head *p)
{
	struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);

374
	if (torture_must_stop_irq()) {
375 376 377 378
		/* Test is ending, just drop callbacks on the floor. */
		/* The next initialization will pick up the pieces. */
		return;
	}
379
	if (rcu_torture_pipe_update_one(rp))
380
		rcu_torture_free(rp);
381
	else
382
		cur_ops->deferred_free(rp);
383 384
}

385
static unsigned long rcu_no_completed(void)
386 387 388 389
{
	return 0;
}

390 391 392 393 394
static void rcu_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu(&p->rtort_rcu, rcu_torture_cb);
}

395 396 397 398 399
static void rcu_sync_torture_init(void)
{
	INIT_LIST_HEAD(&rcu_torture_removed);
}

400
static struct rcu_torture_ops rcu_ops = {
401
	.ttype		= RCU_FLAVOR,
402 403 404 405
	.init		= rcu_sync_torture_init,
	.readlock	= rcu_torture_read_lock,
	.read_delay	= rcu_read_delay,
	.readunlock	= rcu_torture_read_unlock,
406
	.started	= rcu_batches_started,
407
	.completed	= rcu_batches_completed,
408
	.deferred_free	= rcu_torture_deferred_free,
409
	.sync		= synchronize_rcu,
410
	.exp_sync	= synchronize_rcu_expedited,
411 412
	.get_state	= get_state_synchronize_rcu,
	.cond_sync	= cond_synchronize_rcu,
413 414
	.call		= call_rcu,
	.cb_barrier	= rcu_barrier,
415
	.fqs		= rcu_force_quiescent_state,
416 417
	.stats		= NULL,
	.irq_capable	= 1,
418
	.can_boost	= rcu_can_boost(),
419
	.name		= "rcu"
420 421
};

422 423 424 425
/*
 * Definitions for rcu_bh torture testing.
 */

426
static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
427 428 429 430 431
{
	rcu_read_lock_bh();
	return 0;
}

432
static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
433 434 435 436 437 438 439 440 441 442
{
	rcu_read_unlock_bh();
}

static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
}

static struct rcu_torture_ops rcu_bh_ops = {
443
	.ttype		= RCU_BH_FLAVOR,
444
	.init		= rcu_sync_torture_init,
445 446 447
	.readlock	= rcu_bh_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= rcu_bh_torture_read_unlock,
448
	.started	= rcu_batches_started_bh,
449
	.completed	= rcu_batches_completed_bh,
450
	.deferred_free	= rcu_bh_torture_deferred_free,
451
	.sync		= synchronize_rcu_bh,
452
	.exp_sync	= synchronize_rcu_bh_expedited,
453
	.call		= call_rcu_bh,
454
	.cb_barrier	= rcu_barrier_bh,
455
	.fqs		= rcu_bh_force_quiescent_state,
456 457 458
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "rcu_bh"
459 460
};

461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
/*
 * Don't even think about trying any of these in real life!!!
 * The names includes "busted", and they really means it!
 * The only purpose of these functions is to provide a buggy RCU
 * implementation to make sure that rcutorture correctly emits
 * buggy-RCU error messages.
 */
static void rcu_busted_torture_deferred_free(struct rcu_torture *p)
{
	/* This is a deliberate bug for testing purposes only! */
	rcu_torture_cb(&p->rtort_rcu);
}

static void synchronize_rcu_busted(void)
{
	/* This is a deliberate bug for testing purposes only! */
}

static void
480
call_rcu_busted(struct rcu_head *head, rcu_callback_t func)
481 482 483 484 485 486
{
	/* This is a deliberate bug for testing purposes only! */
	func(head);
}

static struct rcu_torture_ops rcu_busted_ops = {
487
	.ttype		= INVALID_RCU_FLAVOR,
488 489 490 491
	.init		= rcu_sync_torture_init,
	.readlock	= rcu_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= rcu_torture_read_unlock,
492
	.started	= rcu_no_completed,
493 494 495 496 497 498 499 500 501
	.completed	= rcu_no_completed,
	.deferred_free	= rcu_busted_torture_deferred_free,
	.sync		= synchronize_rcu_busted,
	.exp_sync	= synchronize_rcu_busted,
	.call		= call_rcu_busted,
	.cb_barrier	= NULL,
	.fqs		= NULL,
	.stats		= NULL,
	.irq_capable	= 1,
502
	.name		= "busted"
503 504
};

505 506 507 508
/*
 * Definitions for srcu torture testing.
 */

509
DEFINE_STATIC_SRCU(srcu_ctl);
510 511
static struct srcu_struct srcu_ctld;
static struct srcu_struct *srcu_ctlp = &srcu_ctl;
512

513
static int srcu_torture_read_lock(void) __acquires(srcu_ctlp)
514
{
515
	return srcu_read_lock(srcu_ctlp);
516 517
}

518
static void srcu_read_delay(struct torture_random_state *rrsp)
519 520 521 522 523 524 525
{
	long delay;
	const long uspertick = 1000000 / HZ;
	const long longdelay = 10;

	/* We want there to be long-running readers, but not all the time. */

526 527
	delay = torture_random(rrsp) %
		(nrealreaders * 2 * longdelay * uspertick);
528
	if (!delay && in_task())
529
		schedule_timeout_interruptible(longdelay);
530 531
	else
		rcu_read_delay(rrsp);
532 533
}

534
static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp)
535
{
536
	srcu_read_unlock(srcu_ctlp, idx);
537 538
}

539
static unsigned long srcu_torture_completed(void)
540
{
541
	return srcu_batches_completed(srcu_ctlp);
542 543
}

544 545
static void srcu_torture_deferred_free(struct rcu_torture *rp)
{
546
	call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb);
547 548
}

549 550
static void srcu_torture_synchronize(void)
{
551
	synchronize_srcu(srcu_ctlp);
552 553
}

554
static void srcu_torture_call(struct rcu_head *head,
555
			      rcu_callback_t func)
556
{
557
	call_srcu(srcu_ctlp, head, func);
558 559 560 561
}

static void srcu_torture_barrier(void)
{
562
	srcu_barrier(srcu_ctlp);
563 564
}

565
static void srcu_torture_stats(void)
566
{
567
	srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG);
568 569
}

570 571
static void srcu_torture_synchronize_expedited(void)
{
572
	synchronize_srcu_expedited(srcu_ctlp);
573 574
}

575
static struct rcu_torture_ops srcu_ops = {
576
	.ttype		= SRCU_FLAVOR,
577
	.init		= rcu_sync_torture_init,
578 579 580
	.readlock	= srcu_torture_read_lock,
	.read_delay	= srcu_read_delay,
	.readunlock	= srcu_torture_read_unlock,
581
	.started	= NULL,
582
	.completed	= srcu_torture_completed,
583
	.deferred_free	= srcu_torture_deferred_free,
584
	.sync		= srcu_torture_synchronize,
585
	.exp_sync	= srcu_torture_synchronize_expedited,
586 587
	.call		= srcu_torture_call,
	.cb_barrier	= srcu_torture_barrier,
588
	.stats		= srcu_torture_stats,
589
	.irq_capable	= 1,
590
	.name		= "srcu"
591 592
};

593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
static void srcu_torture_init(void)
{
	rcu_sync_torture_init();
	WARN_ON(init_srcu_struct(&srcu_ctld));
	srcu_ctlp = &srcu_ctld;
}

static void srcu_torture_cleanup(void)
{
	cleanup_srcu_struct(&srcu_ctld);
	srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */
}

/* As above, but dynamically allocated. */
static struct rcu_torture_ops srcud_ops = {
	.ttype		= SRCU_FLAVOR,
	.init		= srcu_torture_init,
	.cleanup	= srcu_torture_cleanup,
	.readlock	= srcu_torture_read_lock,
	.read_delay	= srcu_read_delay,
	.readunlock	= srcu_torture_read_unlock,
	.started	= NULL,
	.completed	= srcu_torture_completed,
	.deferred_free	= srcu_torture_deferred_free,
	.sync		= srcu_torture_synchronize,
	.exp_sync	= srcu_torture_synchronize_expedited,
	.call		= srcu_torture_call,
	.cb_barrier	= srcu_torture_barrier,
	.stats		= srcu_torture_stats,
622
	.irq_capable	= 1,
623 624 625
	.name		= "srcud"
};

626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
/*
 * Definitions for sched torture testing.
 */

static int sched_torture_read_lock(void)
{
	preempt_disable();
	return 0;
}

static void sched_torture_read_unlock(int idx)
{
	preempt_enable();
}

641 642 643 644 645
static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
}

646
static struct rcu_torture_ops sched_ops = {
647
	.ttype		= RCU_SCHED_FLAVOR,
648 649 650 651
	.init		= rcu_sync_torture_init,
	.readlock	= sched_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= sched_torture_read_unlock,
652
	.started	= rcu_batches_started_sched,
653
	.completed	= rcu_batches_completed_sched,
654
	.deferred_free	= rcu_sched_torture_deferred_free,
655
	.sync		= synchronize_sched,
656
	.exp_sync	= synchronize_sched_expedited,
657 658
	.get_state	= get_state_synchronize_sched,
	.cond_sync	= cond_synchronize_sched,
659
	.call		= call_rcu_sched,
660
	.cb_barrier	= rcu_barrier_sched,
661
	.fqs		= rcu_sched_force_quiescent_state,
662 663 664
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "sched"
665 666
};

667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
/*
 * Definitions for RCU-tasks torture testing.
 */

static int tasks_torture_read_lock(void)
{
	return 0;
}

static void tasks_torture_read_unlock(int idx)
{
}

static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
{
	call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb);
}

static struct rcu_torture_ops tasks_ops = {
	.ttype		= RCU_TASKS_FLAVOR,
	.init		= rcu_sync_torture_init,
	.readlock	= tasks_torture_read_lock,
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
	.readunlock	= tasks_torture_read_unlock,
691
	.started	= rcu_no_completed,
692 693 694 695 696 697 698 699 700 701 702 703
	.completed	= rcu_no_completed,
	.deferred_free	= rcu_tasks_torture_deferred_free,
	.sync		= synchronize_rcu_tasks,
	.exp_sync	= synchronize_rcu_tasks,
	.call		= call_rcu_tasks,
	.cb_barrier	= rcu_barrier_tasks,
	.fqs		= NULL,
	.stats		= NULL,
	.irq_capable	= 1,
	.name		= "tasks"
};

704 705 706 707 708
static bool __maybe_unused torturing_tasks(void)
{
	return cur_ops == &tasks_ops;
}

709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
/*
 * RCU torture priority-boost testing.  Runs one real-time thread per
 * CPU for moderate bursts, repeatedly registering RCU callbacks and
 * spinning waiting for them to be invoked.  If a given callback takes
 * too long to be invoked, we assume that priority inversion has occurred.
 */

struct rcu_boost_inflight {
	struct rcu_head rcu;
	int inflight;
};

static void rcu_torture_boost_cb(struct rcu_head *head)
{
	struct rcu_boost_inflight *rbip =
		container_of(head, struct rcu_boost_inflight, rcu);

726 727
	/* Ensure RCU-core accesses precede clearing ->inflight */
	smp_store_release(&rbip->inflight, 0);
728 729 730 731 732 733 734 735 736 737
}

static int rcu_torture_boost(void *arg)
{
	unsigned long call_rcu_time;
	unsigned long endtime;
	unsigned long oldstarttime;
	struct rcu_boost_inflight rbi = { .inflight = 0 };
	struct sched_param sp;

738
	VERBOSE_TOROUT_STRING("rcu_torture_boost started");
739 740 741 742

	/* Set real-time priority. */
	sp.sched_priority = 1;
	if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
743
		VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!");
744 745 746
		n_rcu_torture_boost_rterror++;
	}

747
	init_rcu_head_on_stack(&rbi.rcu);
748 749 750 751
	/* Each pass through the following loop does one boost-test cycle. */
	do {
		/* Wait for the next test interval. */
		oldstarttime = boost_starttime;
752
		while (ULONG_CMP_LT(jiffies, oldstarttime)) {
753
			schedule_timeout_interruptible(oldstarttime - jiffies);
754
			stutter_wait("rcu_torture_boost");
755
			if (torture_must_stop())
756 757 758 759 760 761
				goto checkwait;
		}

		/* Do one boost-test interval. */
		endtime = oldstarttime + test_boost_duration * HZ;
		call_rcu_time = jiffies;
762
		while (ULONG_CMP_LT(jiffies, endtime)) {
763
			/* If we don't have a callback in flight, post one. */
764 765 766
			if (!smp_load_acquire(&rbi.inflight)) {
				/* RCU core before ->inflight = 1. */
				smp_store_release(&rbi.inflight, 1);
767 768 769
				call_rcu(&rbi.rcu, rcu_torture_boost_cb);
				if (jiffies - call_rcu_time >
					 test_boost_duration * HZ - HZ / 2) {
770
					VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
771 772 773 774
					n_rcu_torture_boost_failure++;
				}
				call_rcu_time = jiffies;
			}
775
			stutter_wait("rcu_torture_boost");
776
			if (torture_must_stop())
777 778 779 780 781 782 783 784 785 786
				goto checkwait;
		}

		/*
		 * Set the start time of the next test interval.
		 * Yes, this is vulnerable to long delays, but such
		 * delays simply cause a false negative for the next
		 * interval.  Besides, we are running at RT priority,
		 * so delays should be relatively rare.
		 */
787 788
		while (oldstarttime == boost_starttime &&
		       !kthread_should_stop()) {
789 790 791 792 793 794 795 796 797 798 799
			if (mutex_trylock(&boost_mutex)) {
				boost_starttime = jiffies +
						  test_boost_interval * HZ;
				n_rcu_torture_boosts++;
				mutex_unlock(&boost_mutex);
				break;
			}
			schedule_timeout_uninterruptible(1);
		}

		/* Go do the stutter. */
800
checkwait:	stutter_wait("rcu_torture_boost");
801
	} while (!torture_must_stop());
802 803

	/* Clean up and exit. */
804
	while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) {
805
		torture_shutdown_absorb("rcu_torture_boost");
806
		schedule_timeout_uninterruptible(1);
807
	}
808
	destroy_rcu_head_on_stack(&rbi.rcu);
809
	torture_kthread_stopping("rcu_torture_boost");
810 811 812
	return 0;
}

813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
static void rcu_torture_cbflood_cb(struct rcu_head *rhp)
{
}

/*
 * RCU torture callback-flood kthread.  Repeatedly induces bursts of calls
 * to call_rcu() or analogous, increasing the probability of occurrence
 * of callback-overflow corner cases.
 */
static int
rcu_torture_cbflood(void *arg)
{
	int err = 1;
	int i;
	int j;
	struct rcu_head *rhp;

	if (cbflood_n_per_burst > 0 &&
	    cbflood_inter_holdoff > 0 &&
	    cbflood_intra_holdoff > 0 &&
	    cur_ops->call &&
	    cur_ops->cb_barrier) {
		rhp = vmalloc(sizeof(*rhp) *
			      cbflood_n_burst * cbflood_n_per_burst);
		err = !rhp;
	}
	if (err) {
		VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM");
841
		goto wait_for_stop;
842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
	}
	VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
	do {
		schedule_timeout_interruptible(cbflood_inter_holdoff);
		atomic_long_inc(&n_cbfloods);
		WARN_ON(signal_pending(current));
		for (i = 0; i < cbflood_n_burst; i++) {
			for (j = 0; j < cbflood_n_per_burst; j++) {
				cur_ops->call(&rhp[i * cbflood_n_per_burst + j],
					      rcu_torture_cbflood_cb);
			}
			schedule_timeout_interruptible(cbflood_intra_holdoff);
			WARN_ON(signal_pending(current));
		}
		cur_ops->cb_barrier();
		stutter_wait("rcu_torture_cbflood");
	} while (!torture_must_stop());
859
	vfree(rhp);
860
wait_for_stop:
861 862 863 864
	torture_kthread_stopping("rcu_torture_cbflood");
	return 0;
}

865 866 867 868 869 870 871 872 873 874 875
/*
 * RCU torture force-quiescent-state kthread.  Repeatedly induces
 * bursts of calls to force_quiescent_state(), increasing the probability
 * of occurrence of some important types of race conditions.
 */
static int
rcu_torture_fqs(void *arg)
{
	unsigned long fqs_resume_time;
	int fqs_burst_remaining;

876
	VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
877 878
	do {
		fqs_resume_time = jiffies + fqs_stutter * HZ;
879 880
		while (ULONG_CMP_LT(jiffies, fqs_resume_time) &&
		       !kthread_should_stop()) {
881 882 883
			schedule_timeout_interruptible(1);
		}
		fqs_burst_remaining = fqs_duration;
884 885
		while (fqs_burst_remaining > 0 &&
		       !kthread_should_stop()) {
886 887 888 889
			cur_ops->fqs();
			udelay(fqs_holdoff);
			fqs_burst_remaining -= fqs_holdoff;
		}
890
		stutter_wait("rcu_torture_fqs");
891
	} while (!torture_must_stop());
892
	torture_kthread_stopping("rcu_torture_fqs");
893 894 895
	return 0;
}

896 897 898 899 900 901 902 903
/*
 * RCU torture writer kthread.  Repeatedly substitutes a new structure
 * for that pointed to by rcu_torture_current, freeing the old structure
 * after a series of grace periods (the "pipeline").
 */
static int
rcu_torture_writer(void *arg)
{
904
	bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal();
905
	int expediting = 0;
906 907
	unsigned long gp_snap;
	bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
908
	bool gp_sync1 = gp_sync;
909 910 911
	int i;
	struct rcu_torture *rp;
	struct rcu_torture *old_rp;
912
	static DEFINE_TORTURE_RANDOM(rand);
913 914
	int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC,
			   RTWS_COND_GET, RTWS_SYNC };
915
	int nsynctypes = 0;
916

917
	VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
918 919
	if (!can_expedite) {
		pr_alert("%s" TORTURE_FLAG
920
			 " GP expediting controlled from boot/sysfs for %s,\n",
921 922 923 924 925
			 torture_type, cur_ops->name);
		pr_alert("%s" TORTURE_FLAG
			 " Disabled dynamic grace-period expediting.\n",
			 torture_type);
	}
926

927
	/* Initialize synctype[] array.  If none set, take default. */
928
	if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1)
929
		gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true;
930 931 932 933 934 935 936 937 938 939 940 941
	if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync)
		synctype[nsynctypes++] = RTWS_COND_GET;
	else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync))
		pr_alert("rcu_torture_writer: gp_cond without primitives.\n");
	if (gp_exp1 && cur_ops->exp_sync)
		synctype[nsynctypes++] = RTWS_EXP_SYNC;
	else if (gp_exp && !cur_ops->exp_sync)
		pr_alert("rcu_torture_writer: gp_exp without primitives.\n");
	if (gp_normal1 && cur_ops->deferred_free)
		synctype[nsynctypes++] = RTWS_DEF_FREE;
	else if (gp_normal && !cur_ops->deferred_free)
		pr_alert("rcu_torture_writer: gp_normal without primitives.\n");
942 943 944 945
	if (gp_sync1 && cur_ops->sync)
		synctype[nsynctypes++] = RTWS_SYNC;
	else if (gp_sync && !cur_ops->sync)
		pr_alert("rcu_torture_writer: gp_sync without primitives.\n");
946 947
	if (WARN_ONCE(nsynctypes == 0,
		      "rcu_torture_writer: No update-side primitives.\n")) {
948 949 950 951 952
		/*
		 * No updates primitives, so don't try updating.
		 * The resulting test won't be testing much, hence the
		 * above WARN_ONCE().
		 */
953 954 955 956
		rcu_torture_writer_state = RTWS_STOPPING;
		torture_kthread_stopping("rcu_torture_writer");
	}

957
	do {
958
		rcu_torture_writer_state = RTWS_FIXED_DELAY;
959
		schedule_timeout_uninterruptible(1);
960 961
		rp = rcu_torture_alloc();
		if (rp == NULL)
962 963
			continue;
		rp->rtort_pipe_count = 0;
964
		rcu_torture_writer_state = RTWS_DELAY;
965
		udelay(torture_random(&rand) & 0x3ff);
966
		rcu_torture_writer_state = RTWS_REPLACE;
967 968
		old_rp = rcu_dereference_check(rcu_torture_current,
					       current == writer_task);
969
		rp->rtort_mbtest = 1;
970
		rcu_assign_pointer(rcu_torture_current, rp);
971
		smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
972
		if (old_rp) {
973 974 975 976 977
			i = old_rp->rtort_pipe_count;
			if (i > RCU_TORTURE_PIPE_LEN)
				i = RCU_TORTURE_PIPE_LEN;
			atomic_inc(&rcu_torture_wcount[i]);
			old_rp->rtort_pipe_count++;
978 979
			switch (synctype[torture_random(&rand) % nsynctypes]) {
			case RTWS_DEF_FREE:
980
				rcu_torture_writer_state = RTWS_DEF_FREE;
981
				cur_ops->deferred_free(old_rp);
982 983
				break;
			case RTWS_EXP_SYNC:
984
				rcu_torture_writer_state = RTWS_EXP_SYNC;
985
				cur_ops->exp_sync();
986 987 988 989 990 991 992 993 994 995 996 997 998
				rcu_torture_pipe_update(old_rp);
				break;
			case RTWS_COND_GET:
				rcu_torture_writer_state = RTWS_COND_GET;
				gp_snap = cur_ops->get_state();
				i = torture_random(&rand) % 16;
				if (i != 0)
					schedule_timeout_interruptible(i);
				udelay(torture_random(&rand) % 1000);
				rcu_torture_writer_state = RTWS_COND_SYNC;
				cur_ops->cond_sync(gp_snap);
				rcu_torture_pipe_update(old_rp);
				break;
999 1000 1001 1002 1003
			case RTWS_SYNC:
				rcu_torture_writer_state = RTWS_SYNC;
				cur_ops->sync();
				rcu_torture_pipe_update(old_rp);
				break;
1004 1005 1006
			default:
				WARN_ON_ONCE(1);
				break;
1007
			}
1008
		}
1009
		rcutorture_record_progress(++rcu_torture_current_version);
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
		/* Cycle through nesting levels of rcu_expedite_gp() calls. */
		if (can_expedite &&
		    !(torture_random(&rand) & 0xff & (!!expediting - 1))) {
			WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
			if (expediting >= 0)
				rcu_expedite_gp();
			else
				rcu_unexpedite_gp();
			if (++expediting > 3)
				expediting = -expediting;
		}
1021
		rcu_torture_writer_state = RTWS_STUTTER;
1022
		stutter_wait("rcu_torture_writer");
1023
	} while (!torture_must_stop());
1024 1025 1026 1027 1028 1029
	/* Reset expediting back to unexpedited. */
	if (expediting > 0)
		expediting = -expediting;
	while (can_expedite && expediting++ < 0)
		rcu_unexpedite_gp();
	WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
1030
	rcu_torture_writer_state = RTWS_STOPPING;
1031
	torture_kthread_stopping("rcu_torture_writer");
1032 1033 1034
	return 0;
}

1035 1036 1037 1038 1039 1040 1041
/*
 * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
 * delay between calls.
 */
static int
rcu_torture_fakewriter(void *arg)
{
1042
	DEFINE_TORTURE_RANDOM(rand);
1043

1044
	VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1045
	set_user_nice(current, MAX_NICE);
1046 1047

	do {
1048 1049
		schedule_timeout_uninterruptible(1 + torture_random(&rand)%10);
		udelay(torture_random(&rand) & 0x3ff);
1050
		if (cur_ops->cb_barrier != NULL &&
1051
		    torture_random(&rand) % (nfakewriters * 8) == 0) {
1052
			cur_ops->cb_barrier();
1053
		} else if (gp_normal == gp_exp) {
1054
			if (torture_random(&rand) & 0x80)
1055 1056 1057 1058
				cur_ops->sync();
			else
				cur_ops->exp_sync();
		} else if (gp_normal) {
1059
			cur_ops->sync();
1060 1061 1062
		} else {
			cur_ops->exp_sync();
		}
1063
		stutter_wait("rcu_torture_fakewriter");
1064
	} while (!torture_must_stop());
1065

1066
	torture_kthread_stopping("rcu_torture_fakewriter");
1067 1068 1069
	return 0;
}

1070 1071 1072 1073 1074
static void rcu_torture_timer_cb(struct rcu_head *rhp)
{
	kfree(rhp);
}

1075 1076 1077 1078 1079 1080 1081 1082 1083
/*
 * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
 * incrementing the corresponding element of the pipeline array.  The
 * counter in the element should never be greater than 1, otherwise, the
 * RCU implementation is broken.
 */
static void rcu_torture_timer(unsigned long unused)
{
	int idx;
1084
	unsigned long started;
1085
	unsigned long completed;
1086
	static DEFINE_TORTURE_RANDOM(rand);
1087 1088 1089
	static DEFINE_SPINLOCK(rand_lock);
	struct rcu_torture *p;
	int pipe_count;
1090
	unsigned long long ts;
1091 1092

	idx = cur_ops->readlock();
1093 1094 1095 1096
	if (cur_ops->started)
		started = cur_ops->started();
	else
		started = cur_ops->completed();
1097
	ts = rcu_trace_clock_local();
1098 1099 1100
	p = rcu_dereference_check(rcu_torture_current,
				  rcu_read_lock_bh_held() ||
				  rcu_read_lock_sched_held() ||
1101 1102
				  srcu_read_lock_held(srcu_ctlp) ||
				  torturing_tasks());
1103 1104 1105 1106 1107 1108 1109 1110
	if (p == NULL) {
		/* Leave because rcu_torture_writer is not yet underway */
		cur_ops->readunlock(idx);
		return;
	}
	if (p->rtort_mbtest == 0)
		atomic_inc(&n_rcu_torture_mberror);
	spin_lock(&rand_lock);
1111
	cur_ops->read_delay(&rand);
1112 1113 1114 1115 1116 1117 1118 1119
	n_rcu_torture_timers++;
	spin_unlock(&rand_lock);
	preempt_disable();
	pipe_count = p->rtort_pipe_count;
	if (pipe_count > RCU_TORTURE_PIPE_LEN) {
		/* Should not happen, but... */
		pipe_count = RCU_TORTURE_PIPE_LEN;
	}
1120
	completed = cur_ops->completed();
1121 1122
	if (pipe_count > 1) {
		do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
1123
					  started, completed);
1124
		rcu_ftrace_dump(DUMP_ALL);
1125
	}
R
Rusty Russell 已提交
1126
	__this_cpu_inc(rcu_torture_count[pipe_count]);
1127 1128 1129
	completed = completed - started;
	if (cur_ops->started)
		completed++;
1130 1131 1132 1133
	if (completed > RCU_TORTURE_PIPE_LEN) {
		/* Should not happen, but... */
		completed = RCU_TORTURE_PIPE_LEN;
	}
R
Rusty Russell 已提交
1134
	__this_cpu_inc(rcu_torture_batch[completed]);
1135 1136
	preempt_enable();
	cur_ops->readunlock(idx);
1137 1138 1139 1140 1141 1142 1143 1144

	/* Test call_rcu() invocation from interrupt handler. */
	if (cur_ops->call) {
		struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT);

		if (rhp)
			cur_ops->call(rhp, rcu_torture_timer_cb);
	}
1145 1146
}

1147 1148 1149 1150 1151 1152 1153 1154 1155
/*
 * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 * incrementing the corresponding element of the pipeline array.  The
 * counter in the element should never be greater than 1, otherwise, the
 * RCU implementation is broken.
 */
static int
rcu_torture_reader(void *arg)
{
1156
	unsigned long started;
1157
	unsigned long completed;
1158
	int idx;
1159
	DEFINE_TORTURE_RANDOM(rand);
1160 1161
	struct rcu_torture *p;
	int pipe_count;
1162
	struct timer_list t;
1163
	unsigned long long ts;
1164

1165
	VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
1166
	set_user_nice(current, MAX_NICE);
1167
	if (irqreader && cur_ops->irq_capable)
1168
		setup_timer_on_stack(&t, rcu_torture_timer, 0);
1169

1170
	do {
1171
		if (irqreader && cur_ops->irq_capable) {
1172
			if (!timer_pending(&t))
1173
				mod_timer(&t, jiffies + 1);
1174
		}
1175
		idx = cur_ops->readlock();
1176 1177 1178 1179
		if (cur_ops->started)
			started = cur_ops->started();
		else
			started = cur_ops->completed();
1180
		ts = rcu_trace_clock_local();
1181 1182 1183
		p = rcu_dereference_check(rcu_torture_current,
					  rcu_read_lock_bh_held() ||
					  rcu_read_lock_sched_held() ||
1184 1185
					  srcu_read_lock_held(srcu_ctlp) ||
					  torturing_tasks());
1186 1187
		if (p == NULL) {
			/* Wait for rcu_torture_writer to get underway */
1188
			cur_ops->readunlock(idx);
1189 1190 1191
			schedule_timeout_interruptible(HZ);
			continue;
		}
1192 1193
		if (p->rtort_mbtest == 0)
			atomic_inc(&n_rcu_torture_mberror);
1194
		cur_ops->read_delay(&rand);
1195 1196 1197 1198 1199 1200
		preempt_disable();
		pipe_count = p->rtort_pipe_count;
		if (pipe_count > RCU_TORTURE_PIPE_LEN) {
			/* Should not happen, but... */
			pipe_count = RCU_TORTURE_PIPE_LEN;
		}
1201
		completed = cur_ops->completed();
1202 1203
		if (pipe_count > 1) {
			do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
1204
						  ts, started, completed);
1205
			rcu_ftrace_dump(DUMP_ALL);
1206
		}
R
Rusty Russell 已提交
1207
		__this_cpu_inc(rcu_torture_count[pipe_count]);
1208 1209 1210
		completed = completed - started;
		if (cur_ops->started)
			completed++;
1211 1212 1213 1214
		if (completed > RCU_TORTURE_PIPE_LEN) {
			/* Should not happen, but... */
			completed = RCU_TORTURE_PIPE_LEN;
		}
R
Rusty Russell 已提交
1215
		__this_cpu_inc(rcu_torture_batch[completed]);
1216
		preempt_enable();
1217
		cur_ops->readunlock(idx);
1218
		stutter_wait("rcu_torture_reader");
1219
	} while (!torture_must_stop());
1220
	if (irqreader && cur_ops->irq_capable) {
1221
		del_timer_sync(&t);
1222 1223
		destroy_timer_on_stack(&t);
	}
1224
	torture_kthread_stopping("rcu_torture_reader");
1225 1226 1227 1228
	return 0;
}

/*
1229 1230 1231 1232 1233 1234
 * Print torture statistics.  Caller must ensure that there is only
 * one call to this function at a given time!!!  This is normally
 * accomplished by relying on the module system to only have one copy
 * of the module loaded, and then by giving the rcu_torture_stats
 * kthread full control (or the init/cleanup functions when rcu_torture_stats
 * thread is not running).
1235
 */
1236
static void
1237
rcu_torture_stats_print(void)
1238 1239 1240 1241 1242
{
	int cpu;
	int i;
	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1243
	static unsigned long rtcv_snap = ULONG_MAX;
1244
	static bool splatted;
1245
	struct task_struct *wtp;
1246

1247
	for_each_possible_cpu(cpu) {
1248 1249 1250 1251 1252 1253 1254 1255 1256
		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
			pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
			batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
		}
	}
	for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
		if (pipesummary[i] != 0)
			break;
	}
1257 1258 1259 1260 1261 1262 1263 1264 1265

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
		rcu_torture_current,
		rcu_torture_current_version,
		list_empty(&rcu_torture_freelist),
		atomic_read(&n_rcu_torture_alloc),
		atomic_read(&n_rcu_torture_alloc_fail),
		atomic_read(&n_rcu_torture_free));
1266
	pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ",
1267
		atomic_read(&n_rcu_torture_mberror),
1268
		n_rcu_torture_barrier_error,
1269 1270 1271 1272 1273 1274 1275
		n_rcu_torture_boost_ktrerror,
		n_rcu_torture_boost_rterror);
	pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
		n_rcu_torture_boost_failure,
		n_rcu_torture_boosts,
		n_rcu_torture_timers);
	torture_onoff_stats();
1276
	pr_cont("barrier: %ld/%ld:%ld ",
1277 1278 1279
		n_barrier_successes,
		n_barrier_attempts,
		n_rcu_torture_barrier_error);
1280
	pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods));
1281 1282

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1283
	if (atomic_read(&n_rcu_torture_mberror) != 0 ||
1284
	    n_rcu_torture_barrier_error != 0 ||
1285 1286
	    n_rcu_torture_boost_ktrerror != 0 ||
	    n_rcu_torture_boost_rterror != 0 ||
1287 1288
	    n_rcu_torture_boost_failure != 0 ||
	    i > 1) {
1289
		pr_cont("%s", "!!! ");
1290
		atomic_inc(&n_rcu_torture_error);
1291
		WARN_ON_ONCE(1);
1292
	}
1293
	pr_cont("Reader Pipe: ");
1294
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1295 1296 1297 1298 1299
		pr_cont(" %ld", pipesummary[i]);
	pr_cont("\n");

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("Reader Batch: ");
1300
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1301 1302 1303 1304 1305
		pr_cont(" %ld", batchsummary[i]);
	pr_cont("\n");

	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	pr_cont("Free-Block Circulation: ");
1306
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1307
		pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
1308
	}
1309 1310
	pr_cont("\n");

1311
	if (cur_ops->stats)
1312
		cur_ops->stats();
1313 1314
	if (rtcv_snap == rcu_torture_current_version &&
	    rcu_torture_current != NULL) {
1315 1316 1317
		int __maybe_unused flags = 0;
		unsigned long __maybe_unused gpnum = 0;
		unsigned long __maybe_unused completed = 0;
1318 1319 1320

		rcutorture_get_gp_data(cur_ops->ttype,
				       &flags, &gpnum, &completed);
1321 1322
		srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp,
					&flags, &gpnum, &completed);
1323
		wtp = READ_ONCE(writer_task);
1324
		pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx cpu %d\n",
1325
			 rcu_torture_writer_state_getname(),
1326
			 rcu_torture_writer_state,
1327
			 gpnum, completed, flags,
1328 1329
			 wtp == NULL ? ~0UL : wtp->state,
			 wtp == NULL ? -1 : (int)task_cpu(wtp));
1330 1331 1332 1333
		if (!splatted && wtp) {
			sched_show_task(wtp);
			splatted = true;
		}
1334
		show_rcu_gp_kthreads();
1335
		rcu_ftrace_dump(DUMP_ALL);
1336 1337
	}
	rtcv_snap = rcu_torture_current_version;
1338 1339 1340 1341 1342 1343 1344 1345 1346
}

/*
 * Periodically prints torture statistics, if periodic statistics printing
 * was specified via the stat_interval module parameter.
 */
static int
rcu_torture_stats(void *arg)
{
1347
	VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
1348 1349 1350
	do {
		schedule_timeout_interruptible(stat_interval * HZ);
		rcu_torture_stats_print();
1351
		torture_shutdown_absorb("rcu_torture_stats");
1352
	} while (!torture_must_stop());
1353
	torture_kthread_stopping("rcu_torture_stats");
1354 1355 1356
	return 0;
}

1357
static inline void
1358
rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
1359
{
1360 1361 1362 1363 1364 1365 1366
	pr_alert("%s" TORTURE_FLAG
		 "--- %s: nreaders=%d nfakewriters=%d "
		 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
		 "shuffle_interval=%d stutter=%d irqreader=%d "
		 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
		 "test_boost=%d/%d test_boost_interval=%d "
		 "test_boost_duration=%d shutdown_secs=%d "
1367
		 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "
1368
		 "n_barrier_cbs=%d "
1369 1370 1371 1372 1373 1374
		 "onoff_interval=%d onoff_holdoff=%d\n",
		 torture_type, tag, nrealreaders, nfakewriters,
		 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
		 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
		 test_boost, cur_ops->can_boost,
		 test_boost_interval, test_boost_duration, shutdown_secs,
1375
		 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff,
1376
		 n_barrier_cbs,
1377
		 onoff_interval, onoff_holdoff);
1378 1379
}

1380
static int rcutorture_booster_cleanup(unsigned int cpu)
1381 1382 1383 1384
{
	struct task_struct *t;

	if (boost_tasks[cpu] == NULL)
1385
		return 0;
1386 1387 1388 1389 1390 1391
	mutex_lock(&boost_mutex);
	t = boost_tasks[cpu];
	boost_tasks[cpu] = NULL;
	mutex_unlock(&boost_mutex);

	/* This must be outside of the mutex, otherwise deadlock! */
1392
	torture_stop_kthread(rcu_torture_boost, t);
1393
	return 0;
1394 1395
}

1396
static int rcutorture_booster_init(unsigned int cpu)
1397 1398 1399 1400 1401 1402 1403 1404
{
	int retval;

	if (boost_tasks[cpu] != NULL)
		return 0;  /* Already created, nothing more to do. */

	/* Don't allow time recalculation while creating a new task. */
	mutex_lock(&boost_mutex);
1405
	VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
E
Eric Dumazet 已提交
1406 1407 1408
	boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
						  cpu_to_node(cpu),
						  "rcu_torture_boost");
1409 1410
	if (IS_ERR(boost_tasks[cpu])) {
		retval = PTR_ERR(boost_tasks[cpu]);
1411
		VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422
		n_rcu_torture_boost_ktrerror++;
		boost_tasks[cpu] = NULL;
		mutex_unlock(&boost_mutex);
		return retval;
	}
	kthread_bind(boost_tasks[cpu], cpu);
	wake_up_process(boost_tasks[cpu]);
	mutex_unlock(&boost_mutex);
	return 0;
}

1423 1424 1425 1426
/*
 * CPU-stall kthread.  It waits as specified by stall_cpu_holdoff, then
 * induces a CPU stall for the time specified by stall_cpu.
 */
1427
static int rcu_torture_stall(void *args)
1428 1429 1430
{
	unsigned long stop_at;

1431
	VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
1432
	if (stall_cpu_holdoff > 0) {
1433
		VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
1434
		schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1435
		VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
1436 1437 1438 1439 1440
	}
	if (!kthread_should_stop()) {
		stop_at = get_seconds() + stall_cpu;
		/* RCU CPU stall is expected behavior in following code. */
		rcu_read_lock();
1441 1442 1443 1444 1445 1446
		if (stall_cpu_irqsoff)
			local_irq_disable();
		else
			preempt_disable();
		pr_alert("rcu_torture_stall start on CPU %d.\n",
			 smp_processor_id());
1447 1448
		while (ULONG_CMP_LT(get_seconds(), stop_at))
			continue;  /* Induce RCU CPU stall warning. */
1449 1450 1451 1452
		if (stall_cpu_irqsoff)
			local_irq_enable();
		else
			preempt_enable();
1453
		rcu_read_unlock();
1454
		pr_alert("rcu_torture_stall end.\n");
1455
	}
1456
	torture_shutdown_absorb("rcu_torture_stall");
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
	while (!kthread_should_stop())
		schedule_timeout_interruptible(10 * HZ);
	return 0;
}

/* Spawn CPU-stall kthread, if stall_cpu specified. */
static int __init rcu_torture_stall_init(void)
{
	if (stall_cpu <= 0)
		return 0;
1467
	return torture_create_kthread(rcu_torture_stall, NULL, stall_task);
1468 1469
}

1470
/* Callback function for RCU barrier testing. */
1471
static void rcu_torture_barrier_cbf(struct rcu_head *rcu)
1472 1473 1474 1475 1476 1477 1478 1479
{
	atomic_inc(&barrier_cbs_invoked);
}

/* kthread function to register callbacks used to test RCU barriers. */
static int rcu_torture_barrier_cbs(void *arg)
{
	long myid = (long)arg;
1480
	bool lastphase = 0;
1481
	bool newphase;
1482 1483 1484
	struct rcu_head rcu;

	init_rcu_head_on_stack(&rcu);
1485
	VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
1486
	set_user_nice(current, MAX_NICE);
1487 1488
	do {
		wait_event(barrier_cbs_wq[myid],
1489
			   (newphase =
1490
			    smp_load_acquire(&barrier_phase)) != lastphase ||
1491
			   torture_must_stop());
1492
		lastphase = newphase;
1493
		if (torture_must_stop())
1494
			break;
1495 1496
		/*
		 * The above smp_load_acquire() ensures barrier_phase load
P
Paul E. McKenney 已提交
1497
		 * is ordered before the following ->call().
1498
		 */
1499
		local_irq_disable(); /* Just to test no-irq call_rcu(). */
1500
		cur_ops->call(&rcu, rcu_torture_barrier_cbf);
1501
		local_irq_enable();
1502 1503
		if (atomic_dec_and_test(&barrier_cbs_count))
			wake_up(&barrier_wq);
1504
	} while (!torture_must_stop());
1505 1506
	if (cur_ops->cb_barrier != NULL)
		cur_ops->cb_barrier();
1507
	destroy_rcu_head_on_stack(&rcu);
1508
	torture_kthread_stopping("rcu_torture_barrier_cbs");
1509 1510 1511 1512 1513 1514 1515 1516
	return 0;
}

/* kthread function to drive and coordinate RCU barrier testing. */
static int rcu_torture_barrier(void *arg)
{
	int i;

1517
	VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
1518 1519 1520
	do {
		atomic_set(&barrier_cbs_invoked, 0);
		atomic_set(&barrier_cbs_count, n_barrier_cbs);
1521 1522
		/* Ensure barrier_phase ordered after prior assignments. */
		smp_store_release(&barrier_phase, !barrier_phase);
1523 1524 1525 1526
		for (i = 0; i < n_barrier_cbs; i++)
			wake_up(&barrier_cbs_wq[i]);
		wait_event(barrier_wq,
			   atomic_read(&barrier_cbs_count) == 0 ||
1527 1528
			   torture_must_stop());
		if (torture_must_stop())
1529 1530
			break;
		n_barrier_attempts++;
1531
		cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
1532 1533
		if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
			n_rcu_torture_barrier_error++;
1534 1535 1536
			pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
			       atomic_read(&barrier_cbs_invoked),
			       n_barrier_cbs);
1537 1538 1539 1540
			WARN_ON_ONCE(1);
		}
		n_barrier_successes++;
		schedule_timeout_interruptible(HZ / 10);
1541
	} while (!torture_must_stop());
1542
	torture_kthread_stopping("rcu_torture_barrier");
1543 1544 1545 1546 1547 1548 1549 1550 1551
	return 0;
}

/* Initialize RCU barrier testing. */
static int rcu_torture_barrier_init(void)
{
	int i;
	int ret;

1552
	if (n_barrier_cbs <= 0)
1553 1554
		return 0;
	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
1555 1556 1557 1558 1559 1560
		pr_alert("%s" TORTURE_FLAG
			 " Call or barrier ops missing for %s,\n",
			 torture_type, cur_ops->name);
		pr_alert("%s" TORTURE_FLAG
			 " RCU barrier testing omitted from run.\n",
			 torture_type);
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
		return 0;
	}
	atomic_set(&barrier_cbs_count, 0);
	atomic_set(&barrier_cbs_invoked, 0);
	barrier_cbs_tasks =
		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
			GFP_KERNEL);
	barrier_cbs_wq =
		kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
			GFP_KERNEL);
1571
	if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
1572 1573 1574
		return -ENOMEM;
	for (i = 0; i < n_barrier_cbs; i++) {
		init_waitqueue_head(&barrier_cbs_wq[i]);
1575 1576 1577 1578
		ret = torture_create_kthread(rcu_torture_barrier_cbs,
					     (void *)(long)i,
					     barrier_cbs_tasks[i]);
		if (ret)
1579 1580
			return ret;
	}
1581
	return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task);
1582 1583 1584 1585 1586 1587 1588
}

/* Clean up after RCU barrier testing. */
static void rcu_torture_barrier_cleanup(void)
{
	int i;

1589
	torture_stop_kthread(rcu_torture_barrier, barrier_task);
1590
	if (barrier_cbs_tasks != NULL) {
1591 1592 1593
		for (i = 0; i < n_barrier_cbs; i++)
			torture_stop_kthread(rcu_torture_barrier_cbs,
					     barrier_cbs_tasks[i]);
1594 1595 1596 1597 1598 1599 1600 1601 1602
		kfree(barrier_cbs_tasks);
		barrier_cbs_tasks = NULL;
	}
	if (barrier_cbs_wq != NULL) {
		kfree(barrier_cbs_wq);
		barrier_cbs_wq = NULL;
	}
}

1603
static enum cpuhp_state rcutor_hp;
1604

1605 1606 1607 1608 1609
static void
rcu_torture_cleanup(void)
{
	int i;

1610
	rcutorture_record_test_transition();
1611
	if (torture_cleanup_begin()) {
1612 1613 1614 1615
		if (cur_ops->cb_barrier != NULL)
			cur_ops->cb_barrier();
		return;
	}
1616

1617
	rcu_torture_barrier_cleanup();
1618 1619
	torture_stop_kthread(rcu_torture_stall, stall_task);
	torture_stop_kthread(rcu_torture_writer, writer_task);
1620

1621
	if (reader_tasks) {
1622 1623 1624
		for (i = 0; i < nrealreaders; i++)
			torture_stop_kthread(rcu_torture_reader,
					     reader_tasks[i]);
1625 1626 1627 1628
		kfree(reader_tasks);
	}
	rcu_torture_current = NULL;

1629
	if (fakewriter_tasks) {
1630
		for (i = 0; i < nfakewriters; i++) {
1631 1632
			torture_stop_kthread(rcu_torture_fakewriter,
					     fakewriter_tasks[i]);
1633 1634 1635 1636 1637
		}
		kfree(fakewriter_tasks);
		fakewriter_tasks = NULL;
	}

1638 1639
	torture_stop_kthread(rcu_torture_stats, stats_task);
	torture_stop_kthread(rcu_torture_fqs, fqs_task);
1640 1641
	for (i = 0; i < ncbflooders; i++)
		torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]);
1642
	if ((test_boost == 1 && cur_ops->can_boost) ||
1643 1644
	    test_boost == 2)
		cpuhp_remove_state(rcutor_hp);
1645

1646 1647 1648 1649
	/*
	 * Wait for all RCU callbacks to fire, then do flavor-specific
	 * cleanup operations.
	 */
1650 1651
	if (cur_ops->cb_barrier != NULL)
		cur_ops->cb_barrier();
1652 1653
	if (cur_ops->cleanup != NULL)
		cur_ops->cleanup();
1654 1655

	rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1656

1657
	if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
1658
		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
1659
	else if (torture_onoff_failures())
1660 1661
		rcu_torture_print_module_parms(cur_ops,
					       "End of test: RCU_HOTPLUG");
1662
	else
1663
		rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
1664
	torture_cleanup_end();
1665 1666
}

1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
static void rcu_torture_leak_cb(struct rcu_head *rhp)
{
}

static void rcu_torture_err_cb(struct rcu_head *rhp)
{
	/*
	 * This -might- happen due to race conditions, but is unlikely.
	 * The scenario that leads to this happening is that the
	 * first of the pair of duplicate callbacks is queued,
	 * someone else starts a grace period that includes that
	 * callback, then the second of the pair must wait for the
	 * next grace period.  Unlikely, but can happen.  If it
	 * does happen, the debug-objects subsystem won't have splatted.
	 */
	pr_alert("rcutorture: duplicated callback was invoked.\n");
}
#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */

/*
 * Verify that double-free causes debug-objects to complain, but only
 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.  Otherwise, say that the test
 * cannot be carried out.
 */
static void rcu_test_debug_objects(void)
{
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	struct rcu_head rh1;
	struct rcu_head rh2;

	init_rcu_head_on_stack(&rh1);
	init_rcu_head_on_stack(&rh2);
	pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");

	/* Try to queue the rh2 pair of callbacks for the same grace period. */
	preempt_disable(); /* Prevent preemption from interrupting test. */
	rcu_read_lock(); /* Make it impossible to finish a grace period. */
	call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */
	local_irq_disable(); /* Make it harder to start a new grace period. */
	call_rcu(&rh2, rcu_torture_leak_cb);
	call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
	local_irq_enable();
	rcu_read_unlock();
	preempt_enable();

	/* Wait for them all to get done so we can safely return. */
	rcu_barrier();
	pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
	destroy_rcu_head_on_stack(&rh1);
	destroy_rcu_head_on_stack(&rh2);
#else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
	pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
#endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
}

1723
static int __init
1724 1725 1726 1727 1728
rcu_torture_init(void)
{
	int i;
	int cpu;
	int firsterr = 0;
1729
	static struct rcu_torture_ops *torture_ops[] = {
1730
		&rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops,
1731
		&sched_ops, &tasks_ops,
1732
	};
1733

1734
	if (!torture_init_begin(torture_type, verbose, &torture_runnable))
1735
		return -EBUSY;
1736

1737
	/* Process args and tell the world that the torturer is on the job. */
1738
	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1739
		cur_ops = torture_ops[i];
1740
		if (strcmp(torture_type, cur_ops->name) == 0)
1741 1742
			break;
	}
1743
	if (i == ARRAY_SIZE(torture_ops)) {
1744 1745 1746
		pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
			 torture_type);
		pr_alert("rcu-torture types:");
1747
		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1748 1749
			pr_alert(" %s", torture_ops[i]->name);
		pr_alert("\n");
1750 1751
		firsterr = -EINVAL;
		goto unwind;
1752
	}
1753
	if (cur_ops->fqs == NULL && fqs_duration != 0) {
1754
		pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
1755 1756
		fqs_duration = 0;
	}
1757
	if (cur_ops->init)
1758
		cur_ops->init();
1759

1760
	if (nreaders >= 0) {
1761
		nrealreaders = nreaders;
1762
	} else {
1763
		nrealreaders = num_online_cpus() - 2 - nreaders;
1764 1765 1766
		if (nrealreaders <= 0)
			nrealreaders = 1;
	}
1767
	rcu_torture_print_module_parms(cur_ops, "Start of test");
1768 1769 1770 1771

	/* Set up the freelist. */

	INIT_LIST_HEAD(&rcu_torture_freelist);
1772
	for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1773
		rcu_tortures[i].rtort_mbtest = 0;
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784
		list_add_tail(&rcu_tortures[i].rtort_free,
			      &rcu_torture_freelist);
	}

	/* Initialize the statistics so that each run gets its own numbers. */

	rcu_torture_current = NULL;
	rcu_torture_current_version = 0;
	atomic_set(&n_rcu_torture_alloc, 0);
	atomic_set(&n_rcu_torture_alloc_fail, 0);
	atomic_set(&n_rcu_torture_free, 0);
1785 1786
	atomic_set(&n_rcu_torture_mberror, 0);
	atomic_set(&n_rcu_torture_error, 0);
1787
	n_rcu_torture_barrier_error = 0;
1788 1789 1790 1791
	n_rcu_torture_boost_ktrerror = 0;
	n_rcu_torture_boost_rterror = 0;
	n_rcu_torture_boost_failure = 0;
	n_rcu_torture_boosts = 0;
1792 1793
	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
		atomic_set(&rcu_torture_wcount[i], 0);
1794
	for_each_possible_cpu(cpu) {
1795 1796 1797 1798 1799 1800 1801 1802
		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
			per_cpu(rcu_torture_count, cpu)[i] = 0;
			per_cpu(rcu_torture_batch, cpu)[i] = 0;
		}
	}

	/* Start up the kthreads. */

1803 1804 1805
	firsterr = torture_create_kthread(rcu_torture_writer, NULL,
					  writer_task);
	if (firsterr)
1806
		goto unwind;
1807 1808 1809 1810 1811 1812 1813 1814 1815
	if (nfakewriters > 0) {
		fakewriter_tasks = kzalloc(nfakewriters *
					   sizeof(fakewriter_tasks[0]),
					   GFP_KERNEL);
		if (fakewriter_tasks == NULL) {
			VERBOSE_TOROUT_ERRSTRING("out of memory");
			firsterr = -ENOMEM;
			goto unwind;
		}
1816 1817
	}
	for (i = 0; i < nfakewriters; i++) {
1818 1819 1820
		firsterr = torture_create_kthread(rcu_torture_fakewriter,
						  NULL, fakewriter_tasks[i]);
		if (firsterr)
1821 1822
			goto unwind;
	}
1823
	reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
1824 1825
			       GFP_KERNEL);
	if (reader_tasks == NULL) {
1826
		VERBOSE_TOROUT_ERRSTRING("out of memory");
1827 1828 1829 1830
		firsterr = -ENOMEM;
		goto unwind;
	}
	for (i = 0; i < nrealreaders; i++) {
1831 1832 1833
		firsterr = torture_create_kthread(rcu_torture_reader, NULL,
						  reader_tasks[i]);
		if (firsterr)
1834 1835 1836
			goto unwind;
	}
	if (stat_interval > 0) {
1837 1838 1839
		firsterr = torture_create_kthread(rcu_torture_stats, NULL,
						  stats_task);
		if (firsterr)
1840 1841
			goto unwind;
	}
1842
	if (test_no_idle_hz && shuffle_interval > 0) {
1843 1844
		firsterr = torture_shuffle_init(shuffle_interval * HZ);
		if (firsterr)
1845 1846
			goto unwind;
	}
1847 1848 1849
	if (stutter < 0)
		stutter = 0;
	if (stutter) {
1850 1851
		firsterr = torture_stutter_init(stutter * HZ);
		if (firsterr)
1852 1853
			goto unwind;
	}
1854 1855 1856
	if (fqs_duration < 0)
		fqs_duration = 0;
	if (fqs_duration) {
1857
		/* Create the fqs thread */
1858 1859
		firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
						  fqs_task);
1860
		if (firsterr)
1861 1862
			goto unwind;
	}
1863 1864 1865 1866 1867 1868 1869 1870
	if (test_boost_interval < 1)
		test_boost_interval = 1;
	if (test_boost_duration < 2)
		test_boost_duration = 2;
	if ((test_boost == 1 && cur_ops->can_boost) ||
	    test_boost == 2) {

		boost_starttime = jiffies + test_boost_interval * HZ;
1871 1872 1873 1874 1875 1876 1877

		firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE",
					     rcutorture_booster_init,
					     rcutorture_booster_cleanup);
		if (firsterr < 0)
			goto unwind;
		rcutor_hp = firsterr;
1878
	}
1879 1880
	firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
	if (firsterr)
1881
		goto unwind;
1882 1883
	firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ);
	if (firsterr)
1884
		goto unwind;
1885 1886
	firsterr = rcu_torture_stall_init();
	if (firsterr)
1887
		goto unwind;
1888 1889
	firsterr = rcu_torture_barrier_init();
	if (firsterr)
1890
		goto unwind;
1891 1892
	if (object_debug)
		rcu_test_debug_objects();
1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910
	if (cbflood_n_burst > 0) {
		/* Create the cbflood threads */
		ncbflooders = (num_online_cpus() + 3) / 4;
		cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task),
				       GFP_KERNEL);
		if (!cbflood_task) {
			VERBOSE_TOROUT_ERRSTRING("out of memory");
			firsterr = -ENOMEM;
			goto unwind;
		}
		for (i = 0; i < ncbflooders; i++) {
			firsterr = torture_create_kthread(rcu_torture_cbflood,
							  NULL,
							  cbflood_task[i]);
			if (firsterr)
				goto unwind;
		}
	}
1911
	rcutorture_record_test_transition();
1912
	torture_init_end();
1913 1914 1915
	return 0;

unwind:
1916
	torture_init_end();
1917 1918 1919 1920 1921 1922
	rcu_torture_cleanup();
	return firsterr;
}

module_init(rcu_torture_init);
module_exit(rcu_torture_cleanup);