Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/kernel/rcutorture.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@freedesktop.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25 #include <linux/types.h> 26 #include <linux/kernel.h> 27 #include <linux/init.h> 28 #include <linux/module.h> 29 #include <linux/kthread.h> 30 #include <linux/err.h> 31 #include <linux/spinlock.h> 32 #include <linux/smp.h> 33 #include <linux/rcupdate.h> 34 #include <linux/interrupt.h> 35 #include <linux/sched.h> 36 #include <linux/atomic.h> 37 #include <linux/bitops.h> 38 #include <linux/completion.h> 39 #include <linux/moduleparam.h> 40 #include <linux/percpu.h> 41 #include <linux/notifier.h> 42 #include <linux/reboot.h> 43 #include <linux/freezer.h> 44 #include <linux/cpu.h> 45 #include <linux/delay.h> 46 #include <linux/stat.h> 47 #include <linux/srcu.h> 48 #include <linux/slab.h> 49 #include <asm/byteorder.h> 50 51 MODULE_LICENSE("GPL"); 52 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>"); 53 54 static int nreaders = -1; /* # reader threads, defaults to 2*ncpus */ 55 static int nfakewriters = 4; /* # fake writer threads */ 56 static int stat_interval = 60; /* Interval between stats, in seconds. */ 57 /* Zero means "only at end of test". */ 58 static bool verbose; /* Print more debug info. */ 59 static bool test_no_idle_hz = true; 60 /* Test RCU support for tickless idle CPUs. */ 61 static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/ 62 static int stutter = 5; /* Start/stop testing interval (in sec) */ 63 static int irqreader = 1; /* RCU readers from irq (timers). */ 64 static int fqs_duration; /* Duration of bursts (us), 0 to disable. */ 65 static int fqs_holdoff; /* Hold time within burst (us). */ 66 static int fqs_stutter = 3; /* Wait time between bursts (s). */ 67 static int n_barrier_cbs; /* Number of callbacks to test RCU barriers. */ 68 static int onoff_interval; /* Wait time between CPU hotplugs, 0=disable. */ 69 static int onoff_holdoff; /* Seconds after boot before CPU hotplugs. */ 70 static int shutdown_secs; /* Shutdown time (s). <=0 for no shutdown. */ 71 static int stall_cpu; /* CPU-stall duration (s). 0 for no stall. */ 72 static int stall_cpu_holdoff = 10; /* Time to wait until stall (s). */ 73 static int test_boost = 1; /* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */ 74 static int test_boost_interval = 7; /* Interval between boost tests, seconds. */ 75 static int test_boost_duration = 4; /* Duration of each boost test, seconds. */ 76 static char *torture_type = "rcu"; /* What RCU implementation to torture. */ 77 78 module_param(nreaders, int, 0444); 79 MODULE_PARM_DESC(nreaders, "Number of RCU reader threads"); 80 module_param(nfakewriters, int, 0444); 81 MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads"); 82 module_param(stat_interval, int, 0644); 83 MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s"); 84 module_param(verbose, bool, 0444); 85 MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s"); 86 module_param(test_no_idle_hz, bool, 0444); 87 MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs"); 88 module_param(shuffle_interval, int, 0444); 89 MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles"); 90 module_param(stutter, int, 0444); 91 MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test"); 92 module_param(irqreader, int, 0444); 93 MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers"); 94 module_param(fqs_duration, int, 0444); 95 MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)"); 96 module_param(fqs_holdoff, int, 0444); 97 MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)"); 98 module_param(fqs_stutter, int, 0444); 99 MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)"); 100 module_param(n_barrier_cbs, int, 0444); 101 MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing"); 102 module_param(onoff_interval, int, 0444); 103 MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable"); 104 module_param(onoff_holdoff, int, 0444); 105 MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (s)"); 106 module_param(shutdown_secs, int, 0444); 107 MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable."); 108 module_param(stall_cpu, int, 0444); 109 MODULE_PARM_DESC(stall_cpu, "Stall duration (s), zero to disable."); 110 module_param(stall_cpu_holdoff, int, 0444); 111 MODULE_PARM_DESC(stall_cpu_holdoff, "Time to wait before starting stall (s)."); 112 module_param(test_boost, int, 0444); 113 MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 114 module_param(test_boost_interval, int, 0444); 115 MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds."); 116 module_param(test_boost_duration, int, 0444); 117 MODULE_PARM_DESC(test_boost_duration, "Duration of each boost test, seconds."); 118 module_param(torture_type, charp, 0444); 119 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)"); 120 121 #define TORTURE_FLAG "-torture:" 122 #define PRINTK_STRING(s) \ 123 do { pr_alert("%s" TORTURE_FLAG s "\n", torture_type); } while (0) 124 #define VERBOSE_PRINTK_STRING(s) \ 125 do { if (verbose) pr_alert("%s" TORTURE_FLAG s "\n", torture_type); } while (0) 126 #define VERBOSE_PRINTK_ERRSTRING(s) \ 127 do { if (verbose) pr_alert("%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0) 128 129 static char printk_buf[4096]; 130 131 static int nrealreaders; 132 static struct task_struct *writer_task; 133 static struct task_struct **fakewriter_tasks; 134 static struct task_struct **reader_tasks; 135 static struct task_struct *stats_task; 136 static struct task_struct *shuffler_task; 137 static struct task_struct *stutter_task; 138 static struct task_struct *fqs_task; 139 static struct task_struct *boost_tasks[NR_CPUS]; 140 static struct task_struct *shutdown_task; 141 #ifdef CONFIG_HOTPLUG_CPU 142 static struct task_struct *onoff_task; 143 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 144 static struct task_struct *stall_task; 145 static struct task_struct **barrier_cbs_tasks; 146 static struct task_struct *barrier_task; 147 148 #define RCU_TORTURE_PIPE_LEN 10 149 150 struct rcu_torture { 151 struct rcu_head rtort_rcu; 152 int rtort_pipe_count; 153 struct list_head rtort_free; 154 int rtort_mbtest; 155 }; 156 157 static LIST_HEAD(rcu_torture_freelist); 158 static struct rcu_torture __rcu *rcu_torture_current; 159 static unsigned long rcu_torture_current_version; 160 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 161 static DEFINE_SPINLOCK(rcu_torture_lock); 162 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) = 163 { 0 }; 164 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) = 165 { 0 }; 166 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 167 static atomic_t n_rcu_torture_alloc; 168 static atomic_t n_rcu_torture_alloc_fail; 169 static atomic_t n_rcu_torture_free; 170 static atomic_t n_rcu_torture_mberror; 171 static atomic_t n_rcu_torture_error; 172 static long n_rcu_torture_barrier_error; 173 static long n_rcu_torture_boost_ktrerror; 174 static long n_rcu_torture_boost_rterror; 175 static long n_rcu_torture_boost_failure; 176 static long n_rcu_torture_boosts; 177 static long n_rcu_torture_timers; 178 static long n_offline_attempts; 179 static long n_offline_successes; 180 static unsigned long sum_offline; 181 static int min_offline = -1; 182 static int max_offline; 183 static long n_online_attempts; 184 static long n_online_successes; 185 static unsigned long sum_online; 186 static int min_online = -1; 187 static int max_online; 188 static long n_barrier_attempts; 189 static long n_barrier_successes; 190 static struct list_head rcu_torture_removed; 191 static cpumask_var_t shuffle_tmp_mask; 192 193 static int stutter_pause_test; 194 195 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 196 #define RCUTORTURE_RUNNABLE_INIT 1 197 #else 198 #define RCUTORTURE_RUNNABLE_INIT 0 199 #endif 200 int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT; 201 module_param(rcutorture_runnable, int, 0444); 202 MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot"); 203 204 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) 205 #define rcu_can_boost() 1 206 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 207 #define rcu_can_boost() 0 208 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ 209 210 static unsigned long shutdown_time; /* jiffies to system shutdown. */ 211 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 212 DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 213 /* and boost task create/destroy. */ 214 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 215 static bool barrier_phase; /* Test phase. */ 216 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 217 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 218 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 219 220 /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ 221 222 #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ 223 #define FULLSTOP_SHUTDOWN 1 /* System shutdown with rcutorture running. */ 224 #define FULLSTOP_RMMOD 2 /* Normal rmmod of rcutorture. */ 225 static int fullstop = FULLSTOP_RMMOD; 226 /* 227 * Protect fullstop transitions and spawning of kthreads. 228 */ 229 static DEFINE_MUTEX(fullstop_mutex); 230 231 /* Forward reference. */ 232 static void rcu_torture_cleanup(void); 233 234 /* 235 * Detect and respond to a system shutdown. 236 */ 237 static int 238 rcutorture_shutdown_notify(struct notifier_block *unused1, 239 unsigned long unused2, void *unused3) 240 { 241 mutex_lock(&fullstop_mutex); 242 if (fullstop == FULLSTOP_DONTSTOP) 243 fullstop = FULLSTOP_SHUTDOWN; 244 else 245 pr_warn(/* but going down anyway, so... */ 246 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n"); 247 mutex_unlock(&fullstop_mutex); 248 return NOTIFY_DONE; 249 } 250 251 /* 252 * Absorb kthreads into a kernel function that won't return, so that 253 * they won't ever access module text or data again. 254 */ 255 static void rcutorture_shutdown_absorb(char *title) 256 { 257 if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 258 pr_notice( 259 "rcutorture thread %s parking due to system shutdown\n", 260 title); 261 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); 262 } 263 } 264 265 /* 266 * Allocate an element from the rcu_tortures pool. 267 */ 268 static struct rcu_torture * 269 rcu_torture_alloc(void) 270 { 271 struct list_head *p; 272 273 spin_lock_bh(&rcu_torture_lock); 274 if (list_empty(&rcu_torture_freelist)) { 275 atomic_inc(&n_rcu_torture_alloc_fail); 276 spin_unlock_bh(&rcu_torture_lock); 277 return NULL; 278 } 279 atomic_inc(&n_rcu_torture_alloc); 280 p = rcu_torture_freelist.next; 281 list_del_init(p); 282 spin_unlock_bh(&rcu_torture_lock); 283 return container_of(p, struct rcu_torture, rtort_free); 284 } 285 286 /* 287 * Free an element to the rcu_tortures pool. 288 */ 289 static void 290 rcu_torture_free(struct rcu_torture *p) 291 { 292 atomic_inc(&n_rcu_torture_free); 293 spin_lock_bh(&rcu_torture_lock); 294 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 295 spin_unlock_bh(&rcu_torture_lock); 296 } 297 298 struct rcu_random_state { 299 unsigned long rrs_state; 300 long rrs_count; 301 }; 302 303 #define RCU_RANDOM_MULT 39916801 /* prime */ 304 #define RCU_RANDOM_ADD 479001701 /* prime */ 305 #define RCU_RANDOM_REFRESH 10000 306 307 #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 } 308 309 /* 310 * Crude but fast random-number generator. Uses a linear congruential 311 * generator, with occasional help from cpu_clock(). 312 */ 313 static unsigned long 314 rcu_random(struct rcu_random_state *rrsp) 315 { 316 if (--rrsp->rrs_count < 0) { 317 rrsp->rrs_state += (unsigned long)local_clock(); 318 rrsp->rrs_count = RCU_RANDOM_REFRESH; 319 } 320 rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD; 321 return swahw32(rrsp->rrs_state); 322 } 323 324 static void 325 rcu_stutter_wait(char *title) 326 { 327 while (stutter_pause_test || !rcutorture_runnable) { 328 if (rcutorture_runnable) 329 schedule_timeout_interruptible(1); 330 else 331 schedule_timeout_interruptible(round_jiffies_relative(HZ)); 332 rcutorture_shutdown_absorb(title); 333 } 334 } 335 336 /* 337 * Operations vector for selecting different types of tests. 338 */ 339 340 struct rcu_torture_ops { 341 void (*init)(void); 342 int (*readlock)(void); 343 void (*read_delay)(struct rcu_random_state *rrsp); 344 void (*readunlock)(int idx); 345 int (*completed)(void); 346 void (*deferred_free)(struct rcu_torture *p); 347 void (*sync)(void); 348 void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 349 void (*cb_barrier)(void); 350 void (*fqs)(void); 351 int (*stats)(char *page); 352 int irq_capable; 353 int can_boost; 354 char *name; 355 }; 356 357 static struct rcu_torture_ops *cur_ops; 358 359 /* 360 * Definitions for rcu torture testing. 361 */ 362 363 static int rcu_torture_read_lock(void) __acquires(RCU) 364 { 365 rcu_read_lock(); 366 return 0; 367 } 368 369 static void rcu_read_delay(struct rcu_random_state *rrsp) 370 { 371 const unsigned long shortdelay_us = 200; 372 const unsigned long longdelay_ms = 50; 373 374 /* We want a short delay sometimes to make a reader delay the grace 375 * period, and we want a long delay occasionally to trigger 376 * force_quiescent_state. */ 377 378 if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 379 mdelay(longdelay_ms); 380 if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 381 udelay(shortdelay_us); 382 #ifdef CONFIG_PREEMPT 383 if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000))) 384 preempt_schedule(); /* No QS if preempt_disable() in effect */ 385 #endif 386 } 387 388 static void rcu_torture_read_unlock(int idx) __releases(RCU) 389 { 390 rcu_read_unlock(); 391 } 392 393 static int rcu_torture_completed(void) 394 { 395 return rcu_batches_completed(); 396 } 397 398 static void 399 rcu_torture_cb(struct rcu_head *p) 400 { 401 int i; 402 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 403 404 if (fullstop != FULLSTOP_DONTSTOP) { 405 /* Test is ending, just drop callbacks on the floor. */ 406 /* The next initialization will pick up the pieces. */ 407 return; 408 } 409 i = rp->rtort_pipe_count; 410 if (i > RCU_TORTURE_PIPE_LEN) 411 i = RCU_TORTURE_PIPE_LEN; 412 atomic_inc(&rcu_torture_wcount[i]); 413 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 414 rp->rtort_mbtest = 0; 415 rcu_torture_free(rp); 416 } else { 417 cur_ops->deferred_free(rp); 418 } 419 } 420 421 static int rcu_no_completed(void) 422 { 423 return 0; 424 } 425 426 static void rcu_torture_deferred_free(struct rcu_torture *p) 427 { 428 call_rcu(&p->rtort_rcu, rcu_torture_cb); 429 } 430 431 static struct rcu_torture_ops rcu_ops = { 432 .init = NULL, 433 .readlock = rcu_torture_read_lock, 434 .read_delay = rcu_read_delay, 435 .readunlock = rcu_torture_read_unlock, 436 .completed = rcu_torture_completed, 437 .deferred_free = rcu_torture_deferred_free, 438 .sync = synchronize_rcu, 439 .call = call_rcu, 440 .cb_barrier = rcu_barrier, 441 .fqs = rcu_force_quiescent_state, 442 .stats = NULL, 443 .irq_capable = 1, 444 .can_boost = rcu_can_boost(), 445 .name = "rcu" 446 }; 447 448 static void rcu_sync_torture_deferred_free(struct rcu_torture *p) 449 { 450 int i; 451 struct rcu_torture *rp; 452 struct rcu_torture *rp1; 453 454 cur_ops->sync(); 455 list_add(&p->rtort_free, &rcu_torture_removed); 456 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 457 i = rp->rtort_pipe_count; 458 if (i > RCU_TORTURE_PIPE_LEN) 459 i = RCU_TORTURE_PIPE_LEN; 460 atomic_inc(&rcu_torture_wcount[i]); 461 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 462 rp->rtort_mbtest = 0; 463 list_del(&rp->rtort_free); 464 rcu_torture_free(rp); 465 } 466 } 467 } 468 469 static void rcu_sync_torture_init(void) 470 { 471 INIT_LIST_HEAD(&rcu_torture_removed); 472 } 473 474 static struct rcu_torture_ops rcu_sync_ops = { 475 .init = rcu_sync_torture_init, 476 .readlock = rcu_torture_read_lock, 477 .read_delay = rcu_read_delay, 478 .readunlock = rcu_torture_read_unlock, 479 .completed = rcu_torture_completed, 480 .deferred_free = rcu_sync_torture_deferred_free, 481 .sync = synchronize_rcu, 482 .call = NULL, 483 .cb_barrier = NULL, 484 .fqs = rcu_force_quiescent_state, 485 .stats = NULL, 486 .irq_capable = 1, 487 .can_boost = rcu_can_boost(), 488 .name = "rcu_sync" 489 }; 490 491 static struct rcu_torture_ops rcu_expedited_ops = { 492 .init = rcu_sync_torture_init, 493 .readlock = rcu_torture_read_lock, 494 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 495 .readunlock = rcu_torture_read_unlock, 496 .completed = rcu_no_completed, 497 .deferred_free = rcu_sync_torture_deferred_free, 498 .sync = synchronize_rcu_expedited, 499 .call = NULL, 500 .cb_barrier = NULL, 501 .fqs = rcu_force_quiescent_state, 502 .stats = NULL, 503 .irq_capable = 1, 504 .can_boost = rcu_can_boost(), 505 .name = "rcu_expedited" 506 }; 507 508 /* 509 * Definitions for rcu_bh torture testing. 510 */ 511 512 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 513 { 514 rcu_read_lock_bh(); 515 return 0; 516 } 517 518 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 519 { 520 rcu_read_unlock_bh(); 521 } 522 523 static int rcu_bh_torture_completed(void) 524 { 525 return rcu_batches_completed_bh(); 526 } 527 528 static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 529 { 530 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 531 } 532 533 static struct rcu_torture_ops rcu_bh_ops = { 534 .init = NULL, 535 .readlock = rcu_bh_torture_read_lock, 536 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 537 .readunlock = rcu_bh_torture_read_unlock, 538 .completed = rcu_bh_torture_completed, 539 .deferred_free = rcu_bh_torture_deferred_free, 540 .sync = synchronize_rcu_bh, 541 .call = call_rcu_bh, 542 .cb_barrier = rcu_barrier_bh, 543 .fqs = rcu_bh_force_quiescent_state, 544 .stats = NULL, 545 .irq_capable = 1, 546 .name = "rcu_bh" 547 }; 548 549 static struct rcu_torture_ops rcu_bh_sync_ops = { 550 .init = rcu_sync_torture_init, 551 .readlock = rcu_bh_torture_read_lock, 552 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 553 .readunlock = rcu_bh_torture_read_unlock, 554 .completed = rcu_bh_torture_completed, 555 .deferred_free = rcu_sync_torture_deferred_free, 556 .sync = synchronize_rcu_bh, 557 .call = NULL, 558 .cb_barrier = NULL, 559 .fqs = rcu_bh_force_quiescent_state, 560 .stats = NULL, 561 .irq_capable = 1, 562 .name = "rcu_bh_sync" 563 }; 564 565 static struct rcu_torture_ops rcu_bh_expedited_ops = { 566 .init = rcu_sync_torture_init, 567 .readlock = rcu_bh_torture_read_lock, 568 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 569 .readunlock = rcu_bh_torture_read_unlock, 570 .completed = rcu_bh_torture_completed, 571 .deferred_free = rcu_sync_torture_deferred_free, 572 .sync = synchronize_rcu_bh_expedited, 573 .call = NULL, 574 .cb_barrier = NULL, 575 .fqs = rcu_bh_force_quiescent_state, 576 .stats = NULL, 577 .irq_capable = 1, 578 .name = "rcu_bh_expedited" 579 }; 580 581 /* 582 * Definitions for srcu torture testing. 583 */ 584 585 DEFINE_STATIC_SRCU(srcu_ctl); 586 587 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl) 588 { 589 return srcu_read_lock(&srcu_ctl); 590 } 591 592 static void srcu_read_delay(struct rcu_random_state *rrsp) 593 { 594 long delay; 595 const long uspertick = 1000000 / HZ; 596 const long longdelay = 10; 597 598 /* We want there to be long-running readers, but not all the time. */ 599 600 delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); 601 if (!delay) 602 schedule_timeout_interruptible(longdelay); 603 else 604 rcu_read_delay(rrsp); 605 } 606 607 static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl) 608 { 609 srcu_read_unlock(&srcu_ctl, idx); 610 } 611 612 static int srcu_torture_completed(void) 613 { 614 return srcu_batches_completed(&srcu_ctl); 615 } 616 617 static void srcu_torture_deferred_free(struct rcu_torture *rp) 618 { 619 call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb); 620 } 621 622 static void srcu_torture_synchronize(void) 623 { 624 synchronize_srcu(&srcu_ctl); 625 } 626 627 static void srcu_torture_call(struct rcu_head *head, 628 void (*func)(struct rcu_head *head)) 629 { 630 call_srcu(&srcu_ctl, head, func); 631 } 632 633 static void srcu_torture_barrier(void) 634 { 635 srcu_barrier(&srcu_ctl); 636 } 637 638 static int srcu_torture_stats(char *page) 639 { 640 int cnt = 0; 641 int cpu; 642 int idx = srcu_ctl.completed & 0x1; 643 644 cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):", 645 torture_type, TORTURE_FLAG, idx); 646 for_each_possible_cpu(cpu) { 647 cnt += sprintf(&page[cnt], " %d(%lu,%lu)", cpu, 648 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx], 649 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]); 650 } 651 cnt += sprintf(&page[cnt], "\n"); 652 return cnt; 653 } 654 655 static struct rcu_torture_ops srcu_ops = { 656 .init = rcu_sync_torture_init, 657 .readlock = srcu_torture_read_lock, 658 .read_delay = srcu_read_delay, 659 .readunlock = srcu_torture_read_unlock, 660 .completed = srcu_torture_completed, 661 .deferred_free = srcu_torture_deferred_free, 662 .sync = srcu_torture_synchronize, 663 .call = srcu_torture_call, 664 .cb_barrier = srcu_torture_barrier, 665 .stats = srcu_torture_stats, 666 .name = "srcu" 667 }; 668 669 static struct rcu_torture_ops srcu_sync_ops = { 670 .init = rcu_sync_torture_init, 671 .readlock = srcu_torture_read_lock, 672 .read_delay = srcu_read_delay, 673 .readunlock = srcu_torture_read_unlock, 674 .completed = srcu_torture_completed, 675 .deferred_free = rcu_sync_torture_deferred_free, 676 .sync = srcu_torture_synchronize, 677 .call = NULL, 678 .cb_barrier = NULL, 679 .stats = srcu_torture_stats, 680 .name = "srcu_sync" 681 }; 682 683 static int srcu_torture_read_lock_raw(void) __acquires(&srcu_ctl) 684 { 685 return srcu_read_lock_raw(&srcu_ctl); 686 } 687 688 static void srcu_torture_read_unlock_raw(int idx) __releases(&srcu_ctl) 689 { 690 srcu_read_unlock_raw(&srcu_ctl, idx); 691 } 692 693 static struct rcu_torture_ops srcu_raw_ops = { 694 .init = rcu_sync_torture_init, 695 .readlock = srcu_torture_read_lock_raw, 696 .read_delay = srcu_read_delay, 697 .readunlock = srcu_torture_read_unlock_raw, 698 .completed = srcu_torture_completed, 699 .deferred_free = srcu_torture_deferred_free, 700 .sync = srcu_torture_synchronize, 701 .call = NULL, 702 .cb_barrier = NULL, 703 .stats = srcu_torture_stats, 704 .name = "srcu_raw" 705 }; 706 707 static struct rcu_torture_ops srcu_raw_sync_ops = { 708 .init = rcu_sync_torture_init, 709 .readlock = srcu_torture_read_lock_raw, 710 .read_delay = srcu_read_delay, 711 .readunlock = srcu_torture_read_unlock_raw, 712 .completed = srcu_torture_completed, 713 .deferred_free = rcu_sync_torture_deferred_free, 714 .sync = srcu_torture_synchronize, 715 .call = NULL, 716 .cb_barrier = NULL, 717 .stats = srcu_torture_stats, 718 .name = "srcu_raw_sync" 719 }; 720 721 static void srcu_torture_synchronize_expedited(void) 722 { 723 synchronize_srcu_expedited(&srcu_ctl); 724 } 725 726 static struct rcu_torture_ops srcu_expedited_ops = { 727 .init = rcu_sync_torture_init, 728 .readlock = srcu_torture_read_lock, 729 .read_delay = srcu_read_delay, 730 .readunlock = srcu_torture_read_unlock, 731 .completed = srcu_torture_completed, 732 .deferred_free = rcu_sync_torture_deferred_free, 733 .sync = srcu_torture_synchronize_expedited, 734 .call = NULL, 735 .cb_barrier = NULL, 736 .stats = srcu_torture_stats, 737 .name = "srcu_expedited" 738 }; 739 740 /* 741 * Definitions for sched torture testing. 742 */ 743 744 static int sched_torture_read_lock(void) 745 { 746 preempt_disable(); 747 return 0; 748 } 749 750 static void sched_torture_read_unlock(int idx) 751 { 752 preempt_enable(); 753 } 754 755 static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 756 { 757 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 758 } 759 760 static struct rcu_torture_ops sched_ops = { 761 .init = rcu_sync_torture_init, 762 .readlock = sched_torture_read_lock, 763 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 764 .readunlock = sched_torture_read_unlock, 765 .completed = rcu_no_completed, 766 .deferred_free = rcu_sched_torture_deferred_free, 767 .sync = synchronize_sched, 768 .cb_barrier = rcu_barrier_sched, 769 .fqs = rcu_sched_force_quiescent_state, 770 .stats = NULL, 771 .irq_capable = 1, 772 .name = "sched" 773 }; 774 775 static struct rcu_torture_ops sched_sync_ops = { 776 .init = rcu_sync_torture_init, 777 .readlock = sched_torture_read_lock, 778 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 779 .readunlock = sched_torture_read_unlock, 780 .completed = rcu_no_completed, 781 .deferred_free = rcu_sync_torture_deferred_free, 782 .sync = synchronize_sched, 783 .cb_barrier = NULL, 784 .fqs = rcu_sched_force_quiescent_state, 785 .stats = NULL, 786 .name = "sched_sync" 787 }; 788 789 static struct rcu_torture_ops sched_expedited_ops = { 790 .init = rcu_sync_torture_init, 791 .readlock = sched_torture_read_lock, 792 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 793 .readunlock = sched_torture_read_unlock, 794 .completed = rcu_no_completed, 795 .deferred_free = rcu_sync_torture_deferred_free, 796 .sync = synchronize_sched_expedited, 797 .cb_barrier = NULL, 798 .fqs = rcu_sched_force_quiescent_state, 799 .stats = NULL, 800 .irq_capable = 1, 801 .name = "sched_expedited" 802 }; 803 804 /* 805 * RCU torture priority-boost testing. Runs one real-time thread per 806 * CPU for moderate bursts, repeatedly registering RCU callbacks and 807 * spinning waiting for them to be invoked. If a given callback takes 808 * too long to be invoked, we assume that priority inversion has occurred. 809 */ 810 811 struct rcu_boost_inflight { 812 struct rcu_head rcu; 813 int inflight; 814 }; 815 816 static void rcu_torture_boost_cb(struct rcu_head *head) 817 { 818 struct rcu_boost_inflight *rbip = 819 container_of(head, struct rcu_boost_inflight, rcu); 820 821 smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */ 822 rbip->inflight = 0; 823 } 824 825 static int rcu_torture_boost(void *arg) 826 { 827 unsigned long call_rcu_time; 828 unsigned long endtime; 829 unsigned long oldstarttime; 830 struct rcu_boost_inflight rbi = { .inflight = 0 }; 831 struct sched_param sp; 832 833 VERBOSE_PRINTK_STRING("rcu_torture_boost started"); 834 835 /* Set real-time priority. */ 836 sp.sched_priority = 1; 837 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) { 838 VERBOSE_PRINTK_STRING("rcu_torture_boost RT prio failed!"); 839 n_rcu_torture_boost_rterror++; 840 } 841 842 init_rcu_head_on_stack(&rbi.rcu); 843 /* Each pass through the following loop does one boost-test cycle. */ 844 do { 845 /* Wait for the next test interval. */ 846 oldstarttime = boost_starttime; 847 while (ULONG_CMP_LT(jiffies, oldstarttime)) { 848 schedule_timeout_uninterruptible(1); 849 rcu_stutter_wait("rcu_torture_boost"); 850 if (kthread_should_stop() || 851 fullstop != FULLSTOP_DONTSTOP) 852 goto checkwait; 853 } 854 855 /* Do one boost-test interval. */ 856 endtime = oldstarttime + test_boost_duration * HZ; 857 call_rcu_time = jiffies; 858 while (ULONG_CMP_LT(jiffies, endtime)) { 859 /* If we don't have a callback in flight, post one. */ 860 if (!rbi.inflight) { 861 smp_mb(); /* RCU core before ->inflight = 1. */ 862 rbi.inflight = 1; 863 call_rcu(&rbi.rcu, rcu_torture_boost_cb); 864 if (jiffies - call_rcu_time > 865 test_boost_duration * HZ - HZ / 2) { 866 VERBOSE_PRINTK_STRING("rcu_torture_boost boosting failed"); 867 n_rcu_torture_boost_failure++; 868 } 869 call_rcu_time = jiffies; 870 } 871 cond_resched(); 872 rcu_stutter_wait("rcu_torture_boost"); 873 if (kthread_should_stop() || 874 fullstop != FULLSTOP_DONTSTOP) 875 goto checkwait; 876 } 877 878 /* 879 * Set the start time of the next test interval. 880 * Yes, this is vulnerable to long delays, but such 881 * delays simply cause a false negative for the next 882 * interval. Besides, we are running at RT priority, 883 * so delays should be relatively rare. 884 */ 885 while (oldstarttime == boost_starttime && 886 !kthread_should_stop()) { 887 if (mutex_trylock(&boost_mutex)) { 888 boost_starttime = jiffies + 889 test_boost_interval * HZ; 890 n_rcu_torture_boosts++; 891 mutex_unlock(&boost_mutex); 892 break; 893 } 894 schedule_timeout_uninterruptible(1); 895 } 896 897 /* Go do the stutter. */ 898 checkwait: rcu_stutter_wait("rcu_torture_boost"); 899 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 900 901 /* Clean up and exit. */ 902 VERBOSE_PRINTK_STRING("rcu_torture_boost task stopping"); 903 rcutorture_shutdown_absorb("rcu_torture_boost"); 904 while (!kthread_should_stop() || rbi.inflight) 905 schedule_timeout_uninterruptible(1); 906 smp_mb(); /* order accesses to ->inflight before stack-frame death. */ 907 destroy_rcu_head_on_stack(&rbi.rcu); 908 return 0; 909 } 910 911 /* 912 * RCU torture force-quiescent-state kthread. Repeatedly induces 913 * bursts of calls to force_quiescent_state(), increasing the probability 914 * of occurrence of some important types of race conditions. 915 */ 916 static int 917 rcu_torture_fqs(void *arg) 918 { 919 unsigned long fqs_resume_time; 920 int fqs_burst_remaining; 921 922 VERBOSE_PRINTK_STRING("rcu_torture_fqs task started"); 923 do { 924 fqs_resume_time = jiffies + fqs_stutter * HZ; 925 while (ULONG_CMP_LT(jiffies, fqs_resume_time) && 926 !kthread_should_stop()) { 927 schedule_timeout_interruptible(1); 928 } 929 fqs_burst_remaining = fqs_duration; 930 while (fqs_burst_remaining > 0 && 931 !kthread_should_stop()) { 932 cur_ops->fqs(); 933 udelay(fqs_holdoff); 934 fqs_burst_remaining -= fqs_holdoff; 935 } 936 rcu_stutter_wait("rcu_torture_fqs"); 937 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 938 VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping"); 939 rcutorture_shutdown_absorb("rcu_torture_fqs"); 940 while (!kthread_should_stop()) 941 schedule_timeout_uninterruptible(1); 942 return 0; 943 } 944 945 /* 946 * RCU torture writer kthread. Repeatedly substitutes a new structure 947 * for that pointed to by rcu_torture_current, freeing the old structure 948 * after a series of grace periods (the "pipeline"). 949 */ 950 static int 951 rcu_torture_writer(void *arg) 952 { 953 int i; 954 long oldbatch = rcu_batches_completed(); 955 struct rcu_torture *rp; 956 struct rcu_torture *old_rp; 957 static DEFINE_RCU_RANDOM(rand); 958 959 VERBOSE_PRINTK_STRING("rcu_torture_writer task started"); 960 set_user_nice(current, 19); 961 962 do { 963 schedule_timeout_uninterruptible(1); 964 rp = rcu_torture_alloc(); 965 if (rp == NULL) 966 continue; 967 rp->rtort_pipe_count = 0; 968 udelay(rcu_random(&rand) & 0x3ff); 969 old_rp = rcu_dereference_check(rcu_torture_current, 970 current == writer_task); 971 rp->rtort_mbtest = 1; 972 rcu_assign_pointer(rcu_torture_current, rp); 973 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 974 if (old_rp) { 975 i = old_rp->rtort_pipe_count; 976 if (i > RCU_TORTURE_PIPE_LEN) 977 i = RCU_TORTURE_PIPE_LEN; 978 atomic_inc(&rcu_torture_wcount[i]); 979 old_rp->rtort_pipe_count++; 980 cur_ops->deferred_free(old_rp); 981 } 982 rcutorture_record_progress(++rcu_torture_current_version); 983 oldbatch = cur_ops->completed(); 984 rcu_stutter_wait("rcu_torture_writer"); 985 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 986 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping"); 987 rcutorture_shutdown_absorb("rcu_torture_writer"); 988 while (!kthread_should_stop()) 989 schedule_timeout_uninterruptible(1); 990 return 0; 991 } 992 993 /* 994 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 995 * delay between calls. 996 */ 997 static int 998 rcu_torture_fakewriter(void *arg) 999 { 1000 DEFINE_RCU_RANDOM(rand); 1001 1002 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started"); 1003 set_user_nice(current, 19); 1004 1005 do { 1006 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10); 1007 udelay(rcu_random(&rand) & 0x3ff); 1008 if (cur_ops->cb_barrier != NULL && 1009 rcu_random(&rand) % (nfakewriters * 8) == 0) 1010 cur_ops->cb_barrier(); 1011 else 1012 cur_ops->sync(); 1013 rcu_stutter_wait("rcu_torture_fakewriter"); 1014 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 1015 1016 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping"); 1017 rcutorture_shutdown_absorb("rcu_torture_fakewriter"); 1018 while (!kthread_should_stop()) 1019 schedule_timeout_uninterruptible(1); 1020 return 0; 1021 } 1022 1023 void rcutorture_trace_dump(void) 1024 { 1025 static atomic_t beenhere = ATOMIC_INIT(0); 1026 1027 if (atomic_read(&beenhere)) 1028 return; 1029 if (atomic_xchg(&beenhere, 1) != 0) 1030 return; 1031 do_trace_rcu_torture_read(cur_ops->name, (struct rcu_head *)~0UL); 1032 ftrace_dump(DUMP_ALL); 1033 } 1034 1035 /* 1036 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 1037 * incrementing the corresponding element of the pipeline array. The 1038 * counter in the element should never be greater than 1, otherwise, the 1039 * RCU implementation is broken. 1040 */ 1041 static void rcu_torture_timer(unsigned long unused) 1042 { 1043 int idx; 1044 int completed; 1045 static DEFINE_RCU_RANDOM(rand); 1046 static DEFINE_SPINLOCK(rand_lock); 1047 struct rcu_torture *p; 1048 int pipe_count; 1049 1050 idx = cur_ops->readlock(); 1051 completed = cur_ops->completed(); 1052 p = rcu_dereference_check(rcu_torture_current, 1053 rcu_read_lock_bh_held() || 1054 rcu_read_lock_sched_held() || 1055 srcu_read_lock_held(&srcu_ctl)); 1056 if (p == NULL) { 1057 /* Leave because rcu_torture_writer is not yet underway */ 1058 cur_ops->readunlock(idx); 1059 return; 1060 } 1061 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu); 1062 if (p->rtort_mbtest == 0) 1063 atomic_inc(&n_rcu_torture_mberror); 1064 spin_lock(&rand_lock); 1065 cur_ops->read_delay(&rand); 1066 n_rcu_torture_timers++; 1067 spin_unlock(&rand_lock); 1068 preempt_disable(); 1069 pipe_count = p->rtort_pipe_count; 1070 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1071 /* Should not happen, but... */ 1072 pipe_count = RCU_TORTURE_PIPE_LEN; 1073 } 1074 if (pipe_count > 1) 1075 rcutorture_trace_dump(); 1076 __this_cpu_inc(rcu_torture_count[pipe_count]); 1077 completed = cur_ops->completed() - completed; 1078 if (completed > RCU_TORTURE_PIPE_LEN) { 1079 /* Should not happen, but... */ 1080 completed = RCU_TORTURE_PIPE_LEN; 1081 } 1082 __this_cpu_inc(rcu_torture_batch[completed]); 1083 preempt_enable(); 1084 cur_ops->readunlock(idx); 1085 } 1086 1087 /* 1088 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 1089 * incrementing the corresponding element of the pipeline array. The 1090 * counter in the element should never be greater than 1, otherwise, the 1091 * RCU implementation is broken. 1092 */ 1093 static int 1094 rcu_torture_reader(void *arg) 1095 { 1096 int completed; 1097 int idx; 1098 DEFINE_RCU_RANDOM(rand); 1099 struct rcu_torture *p; 1100 int pipe_count; 1101 struct timer_list t; 1102 1103 VERBOSE_PRINTK_STRING("rcu_torture_reader task started"); 1104 set_user_nice(current, 19); 1105 if (irqreader && cur_ops->irq_capable) 1106 setup_timer_on_stack(&t, rcu_torture_timer, 0); 1107 1108 do { 1109 if (irqreader && cur_ops->irq_capable) { 1110 if (!timer_pending(&t)) 1111 mod_timer(&t, jiffies + 1); 1112 } 1113 idx = cur_ops->readlock(); 1114 completed = cur_ops->completed(); 1115 p = rcu_dereference_check(rcu_torture_current, 1116 rcu_read_lock_bh_held() || 1117 rcu_read_lock_sched_held() || 1118 srcu_read_lock_held(&srcu_ctl)); 1119 if (p == NULL) { 1120 /* Wait for rcu_torture_writer to get underway */ 1121 cur_ops->readunlock(idx); 1122 schedule_timeout_interruptible(HZ); 1123 continue; 1124 } 1125 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu); 1126 if (p->rtort_mbtest == 0) 1127 atomic_inc(&n_rcu_torture_mberror); 1128 cur_ops->read_delay(&rand); 1129 preempt_disable(); 1130 pipe_count = p->rtort_pipe_count; 1131 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 1132 /* Should not happen, but... */ 1133 pipe_count = RCU_TORTURE_PIPE_LEN; 1134 } 1135 if (pipe_count > 1) 1136 rcutorture_trace_dump(); 1137 __this_cpu_inc(rcu_torture_count[pipe_count]); 1138 completed = cur_ops->completed() - completed; 1139 if (completed > RCU_TORTURE_PIPE_LEN) { 1140 /* Should not happen, but... */ 1141 completed = RCU_TORTURE_PIPE_LEN; 1142 } 1143 __this_cpu_inc(rcu_torture_batch[completed]); 1144 preempt_enable(); 1145 cur_ops->readunlock(idx); 1146 schedule(); 1147 rcu_stutter_wait("rcu_torture_reader"); 1148 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 1149 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping"); 1150 rcutorture_shutdown_absorb("rcu_torture_reader"); 1151 if (irqreader && cur_ops->irq_capable) 1152 del_timer_sync(&t); 1153 while (!kthread_should_stop()) 1154 schedule_timeout_uninterruptible(1); 1155 return 0; 1156 } 1157 1158 /* 1159 * Create an RCU-torture statistics message in the specified buffer. 1160 */ 1161 static int 1162 rcu_torture_printk(char *page) 1163 { 1164 int cnt = 0; 1165 int cpu; 1166 int i; 1167 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1168 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 1169 1170 for_each_possible_cpu(cpu) { 1171 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1172 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 1173 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 1174 } 1175 } 1176 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 1177 if (pipesummary[i] != 0) 1178 break; 1179 } 1180 cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG); 1181 cnt += sprintf(&page[cnt], 1182 "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 1183 rcu_torture_current, 1184 rcu_torture_current_version, 1185 list_empty(&rcu_torture_freelist), 1186 atomic_read(&n_rcu_torture_alloc), 1187 atomic_read(&n_rcu_torture_alloc_fail), 1188 atomic_read(&n_rcu_torture_free)); 1189 cnt += sprintf(&page[cnt], "rtmbe: %d rtbke: %ld rtbre: %ld ", 1190 atomic_read(&n_rcu_torture_mberror), 1191 n_rcu_torture_boost_ktrerror, 1192 n_rcu_torture_boost_rterror); 1193 cnt += sprintf(&page[cnt], "rtbf: %ld rtb: %ld nt: %ld ", 1194 n_rcu_torture_boost_failure, 1195 n_rcu_torture_boosts, 1196 n_rcu_torture_timers); 1197 cnt += sprintf(&page[cnt], 1198 "onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ", 1199 n_online_successes, n_online_attempts, 1200 n_offline_successes, n_offline_attempts, 1201 min_online, max_online, 1202 min_offline, max_offline, 1203 sum_online, sum_offline, HZ); 1204 cnt += sprintf(&page[cnt], "barrier: %ld/%ld:%ld", 1205 n_barrier_successes, 1206 n_barrier_attempts, 1207 n_rcu_torture_barrier_error); 1208 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG); 1209 if (atomic_read(&n_rcu_torture_mberror) != 0 || 1210 n_rcu_torture_barrier_error != 0 || 1211 n_rcu_torture_boost_ktrerror != 0 || 1212 n_rcu_torture_boost_rterror != 0 || 1213 n_rcu_torture_boost_failure != 0 || 1214 i > 1) { 1215 cnt += sprintf(&page[cnt], "!!! "); 1216 atomic_inc(&n_rcu_torture_error); 1217 WARN_ON_ONCE(1); 1218 } 1219 cnt += sprintf(&page[cnt], "Reader Pipe: "); 1220 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1221 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]); 1222 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG); 1223 cnt += sprintf(&page[cnt], "Reader Batch: "); 1224 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1225 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]); 1226 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG); 1227 cnt += sprintf(&page[cnt], "Free-Block Circulation: "); 1228 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1229 cnt += sprintf(&page[cnt], " %d", 1230 atomic_read(&rcu_torture_wcount[i])); 1231 } 1232 cnt += sprintf(&page[cnt], "\n"); 1233 if (cur_ops->stats) 1234 cnt += cur_ops->stats(&page[cnt]); 1235 return cnt; 1236 } 1237 1238 /* 1239 * Print torture statistics. Caller must ensure that there is only 1240 * one call to this function at a given time!!! This is normally 1241 * accomplished by relying on the module system to only have one copy 1242 * of the module loaded, and then by giving the rcu_torture_stats 1243 * kthread full control (or the init/cleanup functions when rcu_torture_stats 1244 * thread is not running). 1245 */ 1246 static void 1247 rcu_torture_stats_print(void) 1248 { 1249 int cnt; 1250 1251 cnt = rcu_torture_printk(printk_buf); 1252 pr_alert("%s", printk_buf); 1253 } 1254 1255 /* 1256 * Periodically prints torture statistics, if periodic statistics printing 1257 * was specified via the stat_interval module parameter. 1258 * 1259 * No need to worry about fullstop here, since this one doesn't reference 1260 * volatile state or register callbacks. 1261 */ 1262 static int 1263 rcu_torture_stats(void *arg) 1264 { 1265 VERBOSE_PRINTK_STRING("rcu_torture_stats task started"); 1266 do { 1267 schedule_timeout_interruptible(stat_interval * HZ); 1268 rcu_torture_stats_print(); 1269 rcutorture_shutdown_absorb("rcu_torture_stats"); 1270 } while (!kthread_should_stop()); 1271 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping"); 1272 return 0; 1273 } 1274 1275 static int rcu_idle_cpu; /* Force all torture tasks off this CPU */ 1276 1277 /* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case 1278 * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs. 1279 */ 1280 static void rcu_torture_shuffle_tasks(void) 1281 { 1282 int i; 1283 1284 cpumask_setall(shuffle_tmp_mask); 1285 get_online_cpus(); 1286 1287 /* No point in shuffling if there is only one online CPU (ex: UP) */ 1288 if (num_online_cpus() == 1) { 1289 put_online_cpus(); 1290 return; 1291 } 1292 1293 if (rcu_idle_cpu != -1) 1294 cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask); 1295 1296 set_cpus_allowed_ptr(current, shuffle_tmp_mask); 1297 1298 if (reader_tasks) { 1299 for (i = 0; i < nrealreaders; i++) 1300 if (reader_tasks[i]) 1301 set_cpus_allowed_ptr(reader_tasks[i], 1302 shuffle_tmp_mask); 1303 } 1304 1305 if (fakewriter_tasks) { 1306 for (i = 0; i < nfakewriters; i++) 1307 if (fakewriter_tasks[i]) 1308 set_cpus_allowed_ptr(fakewriter_tasks[i], 1309 shuffle_tmp_mask); 1310 } 1311 1312 if (writer_task) 1313 set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask); 1314 1315 if (stats_task) 1316 set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask); 1317 1318 if (rcu_idle_cpu == -1) 1319 rcu_idle_cpu = num_online_cpus() - 1; 1320 else 1321 rcu_idle_cpu--; 1322 1323 put_online_cpus(); 1324 } 1325 1326 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the 1327 * system to become idle at a time and cut off its timer ticks. This is meant 1328 * to test the support for such tickless idle CPU in RCU. 1329 */ 1330 static int 1331 rcu_torture_shuffle(void *arg) 1332 { 1333 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started"); 1334 do { 1335 schedule_timeout_interruptible(shuffle_interval * HZ); 1336 rcu_torture_shuffle_tasks(); 1337 rcutorture_shutdown_absorb("rcu_torture_shuffle"); 1338 } while (!kthread_should_stop()); 1339 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping"); 1340 return 0; 1341 } 1342 1343 /* Cause the rcutorture test to "stutter", starting and stopping all 1344 * threads periodically. 1345 */ 1346 static int 1347 rcu_torture_stutter(void *arg) 1348 { 1349 VERBOSE_PRINTK_STRING("rcu_torture_stutter task started"); 1350 do { 1351 schedule_timeout_interruptible(stutter * HZ); 1352 stutter_pause_test = 1; 1353 if (!kthread_should_stop()) 1354 schedule_timeout_interruptible(stutter * HZ); 1355 stutter_pause_test = 0; 1356 rcutorture_shutdown_absorb("rcu_torture_stutter"); 1357 } while (!kthread_should_stop()); 1358 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping"); 1359 return 0; 1360 } 1361 1362 static inline void 1363 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag) 1364 { 1365 pr_alert("%s" TORTURE_FLAG 1366 "--- %s: nreaders=%d nfakewriters=%d " 1367 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1368 "shuffle_interval=%d stutter=%d irqreader=%d " 1369 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 1370 "test_boost=%d/%d test_boost_interval=%d " 1371 "test_boost_duration=%d shutdown_secs=%d " 1372 "stall_cpu=%d stall_cpu_holdoff=%d " 1373 "n_barrier_cbs=%d " 1374 "onoff_interval=%d onoff_holdoff=%d\n", 1375 torture_type, tag, nrealreaders, nfakewriters, 1376 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1377 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 1378 test_boost, cur_ops->can_boost, 1379 test_boost_interval, test_boost_duration, shutdown_secs, 1380 stall_cpu, stall_cpu_holdoff, 1381 n_barrier_cbs, 1382 onoff_interval, onoff_holdoff); 1383 } 1384 1385 static struct notifier_block rcutorture_shutdown_nb = { 1386 .notifier_call = rcutorture_shutdown_notify, 1387 }; 1388 1389 static void rcutorture_booster_cleanup(int cpu) 1390 { 1391 struct task_struct *t; 1392 1393 if (boost_tasks[cpu] == NULL) 1394 return; 1395 mutex_lock(&boost_mutex); 1396 VERBOSE_PRINTK_STRING("Stopping rcu_torture_boost task"); 1397 t = boost_tasks[cpu]; 1398 boost_tasks[cpu] = NULL; 1399 mutex_unlock(&boost_mutex); 1400 1401 /* This must be outside of the mutex, otherwise deadlock! */ 1402 kthread_stop(t); 1403 boost_tasks[cpu] = NULL; 1404 } 1405 1406 static int rcutorture_booster_init(int cpu) 1407 { 1408 int retval; 1409 1410 if (boost_tasks[cpu] != NULL) 1411 return 0; /* Already created, nothing more to do. */ 1412 1413 /* Don't allow time recalculation while creating a new task. */ 1414 mutex_lock(&boost_mutex); 1415 VERBOSE_PRINTK_STRING("Creating rcu_torture_boost task"); 1416 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL, 1417 cpu_to_node(cpu), 1418 "rcu_torture_boost"); 1419 if (IS_ERR(boost_tasks[cpu])) { 1420 retval = PTR_ERR(boost_tasks[cpu]); 1421 VERBOSE_PRINTK_STRING("rcu_torture_boost task create failed"); 1422 n_rcu_torture_boost_ktrerror++; 1423 boost_tasks[cpu] = NULL; 1424 mutex_unlock(&boost_mutex); 1425 return retval; 1426 } 1427 kthread_bind(boost_tasks[cpu], cpu); 1428 wake_up_process(boost_tasks[cpu]); 1429 mutex_unlock(&boost_mutex); 1430 return 0; 1431 } 1432 1433 /* 1434 * Cause the rcutorture test to shutdown the system after the test has 1435 * run for the time specified by the shutdown_secs module parameter. 1436 */ 1437 static int 1438 rcu_torture_shutdown(void *arg) 1439 { 1440 long delta; 1441 unsigned long jiffies_snap; 1442 1443 VERBOSE_PRINTK_STRING("rcu_torture_shutdown task started"); 1444 jiffies_snap = ACCESS_ONCE(jiffies); 1445 while (ULONG_CMP_LT(jiffies_snap, shutdown_time) && 1446 !kthread_should_stop()) { 1447 delta = shutdown_time - jiffies_snap; 1448 if (verbose) 1449 pr_alert("%s" TORTURE_FLAG 1450 "rcu_torture_shutdown task: %lu jiffies remaining\n", 1451 torture_type, delta); 1452 schedule_timeout_interruptible(delta); 1453 jiffies_snap = ACCESS_ONCE(jiffies); 1454 } 1455 if (kthread_should_stop()) { 1456 VERBOSE_PRINTK_STRING("rcu_torture_shutdown task stopping"); 1457 return 0; 1458 } 1459 1460 /* OK, shut down the system. */ 1461 1462 VERBOSE_PRINTK_STRING("rcu_torture_shutdown task shutting down system"); 1463 shutdown_task = NULL; /* Avoid self-kill deadlock. */ 1464 rcu_torture_cleanup(); /* Get the success/failure message. */ 1465 kernel_power_off(); /* Shut down the system. */ 1466 return 0; 1467 } 1468 1469 #ifdef CONFIG_HOTPLUG_CPU 1470 1471 /* 1472 * Execute random CPU-hotplug operations at the interval specified 1473 * by the onoff_interval. 1474 */ 1475 static int __cpuinit 1476 rcu_torture_onoff(void *arg) 1477 { 1478 int cpu; 1479 unsigned long delta; 1480 int maxcpu = -1; 1481 DEFINE_RCU_RANDOM(rand); 1482 int ret; 1483 unsigned long starttime; 1484 1485 VERBOSE_PRINTK_STRING("rcu_torture_onoff task started"); 1486 for_each_online_cpu(cpu) 1487 maxcpu = cpu; 1488 WARN_ON(maxcpu < 0); 1489 if (onoff_holdoff > 0) { 1490 VERBOSE_PRINTK_STRING("rcu_torture_onoff begin holdoff"); 1491 schedule_timeout_interruptible(onoff_holdoff * HZ); 1492 VERBOSE_PRINTK_STRING("rcu_torture_onoff end holdoff"); 1493 } 1494 while (!kthread_should_stop()) { 1495 cpu = (rcu_random(&rand) >> 4) % (maxcpu + 1); 1496 if (cpu_online(cpu) && cpu_is_hotpluggable(cpu)) { 1497 if (verbose) 1498 pr_alert("%s" TORTURE_FLAG 1499 "rcu_torture_onoff task: offlining %d\n", 1500 torture_type, cpu); 1501 starttime = jiffies; 1502 n_offline_attempts++; 1503 ret = cpu_down(cpu); 1504 if (ret) { 1505 if (verbose) 1506 pr_alert("%s" TORTURE_FLAG 1507 "rcu_torture_onoff task: offline %d failed: errno %d\n", 1508 torture_type, cpu, ret); 1509 } else { 1510 if (verbose) 1511 pr_alert("%s" TORTURE_FLAG 1512 "rcu_torture_onoff task: offlined %d\n", 1513 torture_type, cpu); 1514 n_offline_successes++; 1515 delta = jiffies - starttime; 1516 sum_offline += delta; 1517 if (min_offline < 0) { 1518 min_offline = delta; 1519 max_offline = delta; 1520 } 1521 if (min_offline > delta) 1522 min_offline = delta; 1523 if (max_offline < delta) 1524 max_offline = delta; 1525 } 1526 } else if (cpu_is_hotpluggable(cpu)) { 1527 if (verbose) 1528 pr_alert("%s" TORTURE_FLAG 1529 "rcu_torture_onoff task: onlining %d\n", 1530 torture_type, cpu); 1531 starttime = jiffies; 1532 n_online_attempts++; 1533 if (cpu_up(cpu) == 0) { 1534 if (verbose) 1535 pr_alert("%s" TORTURE_FLAG 1536 "rcu_torture_onoff task: onlined %d\n", 1537 torture_type, cpu); 1538 n_online_successes++; 1539 delta = jiffies - starttime; 1540 sum_online += delta; 1541 if (min_online < 0) { 1542 min_online = delta; 1543 max_online = delta; 1544 } 1545 if (min_online > delta) 1546 min_online = delta; 1547 if (max_online < delta) 1548 max_online = delta; 1549 } 1550 } 1551 schedule_timeout_interruptible(onoff_interval * HZ); 1552 } 1553 VERBOSE_PRINTK_STRING("rcu_torture_onoff task stopping"); 1554 return 0; 1555 } 1556 1557 static int __cpuinit 1558 rcu_torture_onoff_init(void) 1559 { 1560 int ret; 1561 1562 if (onoff_interval <= 0) 1563 return 0; 1564 onoff_task = kthread_run(rcu_torture_onoff, NULL, "rcu_torture_onoff"); 1565 if (IS_ERR(onoff_task)) { 1566 ret = PTR_ERR(onoff_task); 1567 onoff_task = NULL; 1568 return ret; 1569 } 1570 return 0; 1571 } 1572 1573 static void rcu_torture_onoff_cleanup(void) 1574 { 1575 if (onoff_task == NULL) 1576 return; 1577 VERBOSE_PRINTK_STRING("Stopping rcu_torture_onoff task"); 1578 kthread_stop(onoff_task); 1579 onoff_task = NULL; 1580 } 1581 1582 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 1583 1584 static int 1585 rcu_torture_onoff_init(void) 1586 { 1587 return 0; 1588 } 1589 1590 static void rcu_torture_onoff_cleanup(void) 1591 { 1592 } 1593 1594 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 1595 1596 /* 1597 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 1598 * induces a CPU stall for the time specified by stall_cpu. 1599 */ 1600 static int __cpuinit rcu_torture_stall(void *args) 1601 { 1602 unsigned long stop_at; 1603 1604 VERBOSE_PRINTK_STRING("rcu_torture_stall task started"); 1605 if (stall_cpu_holdoff > 0) { 1606 VERBOSE_PRINTK_STRING("rcu_torture_stall begin holdoff"); 1607 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 1608 VERBOSE_PRINTK_STRING("rcu_torture_stall end holdoff"); 1609 } 1610 if (!kthread_should_stop()) { 1611 stop_at = get_seconds() + stall_cpu; 1612 /* RCU CPU stall is expected behavior in following code. */ 1613 pr_alert("rcu_torture_stall start.\n"); 1614 rcu_read_lock(); 1615 preempt_disable(); 1616 while (ULONG_CMP_LT(get_seconds(), stop_at)) 1617 continue; /* Induce RCU CPU stall warning. */ 1618 preempt_enable(); 1619 rcu_read_unlock(); 1620 pr_alert("rcu_torture_stall end.\n"); 1621 } 1622 rcutorture_shutdown_absorb("rcu_torture_stall"); 1623 while (!kthread_should_stop()) 1624 schedule_timeout_interruptible(10 * HZ); 1625 return 0; 1626 } 1627 1628 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 1629 static int __init rcu_torture_stall_init(void) 1630 { 1631 int ret; 1632 1633 if (stall_cpu <= 0) 1634 return 0; 1635 stall_task = kthread_run(rcu_torture_stall, NULL, "rcu_torture_stall"); 1636 if (IS_ERR(stall_task)) { 1637 ret = PTR_ERR(stall_task); 1638 stall_task = NULL; 1639 return ret; 1640 } 1641 return 0; 1642 } 1643 1644 /* Clean up after the CPU-stall kthread, if one was spawned. */ 1645 static void rcu_torture_stall_cleanup(void) 1646 { 1647 if (stall_task == NULL) 1648 return; 1649 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stall_task."); 1650 kthread_stop(stall_task); 1651 stall_task = NULL; 1652 } 1653 1654 /* Callback function for RCU barrier testing. */ 1655 void rcu_torture_barrier_cbf(struct rcu_head *rcu) 1656 { 1657 atomic_inc(&barrier_cbs_invoked); 1658 } 1659 1660 /* kthread function to register callbacks used to test RCU barriers. */ 1661 static int rcu_torture_barrier_cbs(void *arg) 1662 { 1663 long myid = (long)arg; 1664 bool lastphase = 0; 1665 struct rcu_head rcu; 1666 1667 init_rcu_head_on_stack(&rcu); 1668 VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task started"); 1669 set_user_nice(current, 19); 1670 do { 1671 wait_event(barrier_cbs_wq[myid], 1672 barrier_phase != lastphase || 1673 kthread_should_stop() || 1674 fullstop != FULLSTOP_DONTSTOP); 1675 lastphase = barrier_phase; 1676 smp_mb(); /* ensure barrier_phase load before ->call(). */ 1677 if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP) 1678 break; 1679 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 1680 if (atomic_dec_and_test(&barrier_cbs_count)) 1681 wake_up(&barrier_wq); 1682 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 1683 VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task stopping"); 1684 rcutorture_shutdown_absorb("rcu_torture_barrier_cbs"); 1685 while (!kthread_should_stop()) 1686 schedule_timeout_interruptible(1); 1687 cur_ops->cb_barrier(); 1688 destroy_rcu_head_on_stack(&rcu); 1689 return 0; 1690 } 1691 1692 /* kthread function to drive and coordinate RCU barrier testing. */ 1693 static int rcu_torture_barrier(void *arg) 1694 { 1695 int i; 1696 1697 VERBOSE_PRINTK_STRING("rcu_torture_barrier task starting"); 1698 do { 1699 atomic_set(&barrier_cbs_invoked, 0); 1700 atomic_set(&barrier_cbs_count, n_barrier_cbs); 1701 smp_mb(); /* Ensure barrier_phase after prior assignments. */ 1702 barrier_phase = !barrier_phase; 1703 for (i = 0; i < n_barrier_cbs; i++) 1704 wake_up(&barrier_cbs_wq[i]); 1705 wait_event(barrier_wq, 1706 atomic_read(&barrier_cbs_count) == 0 || 1707 kthread_should_stop() || 1708 fullstop != FULLSTOP_DONTSTOP); 1709 if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP) 1710 break; 1711 n_barrier_attempts++; 1712 cur_ops->cb_barrier(); 1713 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 1714 n_rcu_torture_barrier_error++; 1715 WARN_ON_ONCE(1); 1716 } 1717 n_barrier_successes++; 1718 schedule_timeout_interruptible(HZ / 10); 1719 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 1720 VERBOSE_PRINTK_STRING("rcu_torture_barrier task stopping"); 1721 rcutorture_shutdown_absorb("rcu_torture_barrier"); 1722 while (!kthread_should_stop()) 1723 schedule_timeout_interruptible(1); 1724 return 0; 1725 } 1726 1727 /* Initialize RCU barrier testing. */ 1728 static int rcu_torture_barrier_init(void) 1729 { 1730 int i; 1731 int ret; 1732 1733 if (n_barrier_cbs == 0) 1734 return 0; 1735 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 1736 pr_alert("%s" TORTURE_FLAG 1737 " Call or barrier ops missing for %s,\n", 1738 torture_type, cur_ops->name); 1739 pr_alert("%s" TORTURE_FLAG 1740 " RCU barrier testing omitted from run.\n", 1741 torture_type); 1742 return 0; 1743 } 1744 atomic_set(&barrier_cbs_count, 0); 1745 atomic_set(&barrier_cbs_invoked, 0); 1746 barrier_cbs_tasks = 1747 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]), 1748 GFP_KERNEL); 1749 barrier_cbs_wq = 1750 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]), 1751 GFP_KERNEL); 1752 if (barrier_cbs_tasks == NULL || barrier_cbs_wq == 0) 1753 return -ENOMEM; 1754 for (i = 0; i < n_barrier_cbs; i++) { 1755 init_waitqueue_head(&barrier_cbs_wq[i]); 1756 barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs, 1757 (void *)(long)i, 1758 "rcu_torture_barrier_cbs"); 1759 if (IS_ERR(barrier_cbs_tasks[i])) { 1760 ret = PTR_ERR(barrier_cbs_tasks[i]); 1761 VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier_cbs"); 1762 barrier_cbs_tasks[i] = NULL; 1763 return ret; 1764 } 1765 } 1766 barrier_task = kthread_run(rcu_torture_barrier, NULL, 1767 "rcu_torture_barrier"); 1768 if (IS_ERR(barrier_task)) { 1769 ret = PTR_ERR(barrier_task); 1770 VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier"); 1771 barrier_task = NULL; 1772 } 1773 return 0; 1774 } 1775 1776 /* Clean up after RCU barrier testing. */ 1777 static void rcu_torture_barrier_cleanup(void) 1778 { 1779 int i; 1780 1781 if (barrier_task != NULL) { 1782 VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier task"); 1783 kthread_stop(barrier_task); 1784 barrier_task = NULL; 1785 } 1786 if (barrier_cbs_tasks != NULL) { 1787 for (i = 0; i < n_barrier_cbs; i++) { 1788 if (barrier_cbs_tasks[i] != NULL) { 1789 VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier_cbs task"); 1790 kthread_stop(barrier_cbs_tasks[i]); 1791 barrier_cbs_tasks[i] = NULL; 1792 } 1793 } 1794 kfree(barrier_cbs_tasks); 1795 barrier_cbs_tasks = NULL; 1796 } 1797 if (barrier_cbs_wq != NULL) { 1798 kfree(barrier_cbs_wq); 1799 barrier_cbs_wq = NULL; 1800 } 1801 } 1802 1803 static int rcutorture_cpu_notify(struct notifier_block *self, 1804 unsigned long action, void *hcpu) 1805 { 1806 long cpu = (long)hcpu; 1807 1808 switch (action) { 1809 case CPU_ONLINE: 1810 case CPU_DOWN_FAILED: 1811 (void)rcutorture_booster_init(cpu); 1812 break; 1813 case CPU_DOWN_PREPARE: 1814 rcutorture_booster_cleanup(cpu); 1815 break; 1816 default: 1817 break; 1818 } 1819 return NOTIFY_OK; 1820 } 1821 1822 static struct notifier_block rcutorture_cpu_nb = { 1823 .notifier_call = rcutorture_cpu_notify, 1824 }; 1825 1826 static void 1827 rcu_torture_cleanup(void) 1828 { 1829 int i; 1830 1831 mutex_lock(&fullstop_mutex); 1832 rcutorture_record_test_transition(); 1833 if (fullstop == FULLSTOP_SHUTDOWN) { 1834 pr_warn(/* but going down anyway, so... */ 1835 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n"); 1836 mutex_unlock(&fullstop_mutex); 1837 schedule_timeout_uninterruptible(10); 1838 if (cur_ops->cb_barrier != NULL) 1839 cur_ops->cb_barrier(); 1840 return; 1841 } 1842 fullstop = FULLSTOP_RMMOD; 1843 mutex_unlock(&fullstop_mutex); 1844 unregister_reboot_notifier(&rcutorture_shutdown_nb); 1845 rcu_torture_barrier_cleanup(); 1846 rcu_torture_stall_cleanup(); 1847 if (stutter_task) { 1848 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task"); 1849 kthread_stop(stutter_task); 1850 } 1851 stutter_task = NULL; 1852 if (shuffler_task) { 1853 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task"); 1854 kthread_stop(shuffler_task); 1855 free_cpumask_var(shuffle_tmp_mask); 1856 } 1857 shuffler_task = NULL; 1858 1859 if (writer_task) { 1860 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task"); 1861 kthread_stop(writer_task); 1862 } 1863 writer_task = NULL; 1864 1865 if (reader_tasks) { 1866 for (i = 0; i < nrealreaders; i++) { 1867 if (reader_tasks[i]) { 1868 VERBOSE_PRINTK_STRING( 1869 "Stopping rcu_torture_reader task"); 1870 kthread_stop(reader_tasks[i]); 1871 } 1872 reader_tasks[i] = NULL; 1873 } 1874 kfree(reader_tasks); 1875 reader_tasks = NULL; 1876 } 1877 rcu_torture_current = NULL; 1878 1879 if (fakewriter_tasks) { 1880 for (i = 0; i < nfakewriters; i++) { 1881 if (fakewriter_tasks[i]) { 1882 VERBOSE_PRINTK_STRING( 1883 "Stopping rcu_torture_fakewriter task"); 1884 kthread_stop(fakewriter_tasks[i]); 1885 } 1886 fakewriter_tasks[i] = NULL; 1887 } 1888 kfree(fakewriter_tasks); 1889 fakewriter_tasks = NULL; 1890 } 1891 1892 if (stats_task) { 1893 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task"); 1894 kthread_stop(stats_task); 1895 } 1896 stats_task = NULL; 1897 1898 if (fqs_task) { 1899 VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task"); 1900 kthread_stop(fqs_task); 1901 } 1902 fqs_task = NULL; 1903 if ((test_boost == 1 && cur_ops->can_boost) || 1904 test_boost == 2) { 1905 unregister_cpu_notifier(&rcutorture_cpu_nb); 1906 for_each_possible_cpu(i) 1907 rcutorture_booster_cleanup(i); 1908 } 1909 if (shutdown_task != NULL) { 1910 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shutdown task"); 1911 kthread_stop(shutdown_task); 1912 } 1913 shutdown_task = NULL; 1914 rcu_torture_onoff_cleanup(); 1915 1916 /* Wait for all RCU callbacks to fire. */ 1917 1918 if (cur_ops->cb_barrier != NULL) 1919 cur_ops->cb_barrier(); 1920 1921 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1922 1923 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 1924 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 1925 else if (n_online_successes != n_online_attempts || 1926 n_offline_successes != n_offline_attempts) 1927 rcu_torture_print_module_parms(cur_ops, 1928 "End of test: RCU_HOTPLUG"); 1929 else 1930 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 1931 } 1932 1933 static int __init 1934 rcu_torture_init(void) 1935 { 1936 int i; 1937 int cpu; 1938 int firsterr = 0; 1939 int retval; 1940 static struct rcu_torture_ops *torture_ops[] = 1941 { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops, 1942 &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops, 1943 &srcu_ops, &srcu_sync_ops, &srcu_expedited_ops, 1944 &srcu_raw_ops, &srcu_raw_sync_ops, 1945 &sched_ops, &sched_sync_ops, &sched_expedited_ops, }; 1946 1947 mutex_lock(&fullstop_mutex); 1948 1949 /* Process args and tell the world that the torturer is on the job. */ 1950 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1951 cur_ops = torture_ops[i]; 1952 if (strcmp(torture_type, cur_ops->name) == 0) 1953 break; 1954 } 1955 if (i == ARRAY_SIZE(torture_ops)) { 1956 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 1957 torture_type); 1958 pr_alert("rcu-torture types:"); 1959 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1960 pr_alert(" %s", torture_ops[i]->name); 1961 pr_alert("\n"); 1962 mutex_unlock(&fullstop_mutex); 1963 return -EINVAL; 1964 } 1965 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1966 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 1967 fqs_duration = 0; 1968 } 1969 if (cur_ops->init) 1970 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 1971 1972 if (nreaders >= 0) 1973 nrealreaders = nreaders; 1974 else 1975 nrealreaders = 2 * num_online_cpus(); 1976 rcu_torture_print_module_parms(cur_ops, "Start of test"); 1977 fullstop = FULLSTOP_DONTSTOP; 1978 1979 /* Set up the freelist. */ 1980 1981 INIT_LIST_HEAD(&rcu_torture_freelist); 1982 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1983 rcu_tortures[i].rtort_mbtest = 0; 1984 list_add_tail(&rcu_tortures[i].rtort_free, 1985 &rcu_torture_freelist); 1986 } 1987 1988 /* Initialize the statistics so that each run gets its own numbers. */ 1989 1990 rcu_torture_current = NULL; 1991 rcu_torture_current_version = 0; 1992 atomic_set(&n_rcu_torture_alloc, 0); 1993 atomic_set(&n_rcu_torture_alloc_fail, 0); 1994 atomic_set(&n_rcu_torture_free, 0); 1995 atomic_set(&n_rcu_torture_mberror, 0); 1996 atomic_set(&n_rcu_torture_error, 0); 1997 n_rcu_torture_barrier_error = 0; 1998 n_rcu_torture_boost_ktrerror = 0; 1999 n_rcu_torture_boost_rterror = 0; 2000 n_rcu_torture_boost_failure = 0; 2001 n_rcu_torture_boosts = 0; 2002 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2003 atomic_set(&rcu_torture_wcount[i], 0); 2004 for_each_possible_cpu(cpu) { 2005 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2006 per_cpu(rcu_torture_count, cpu)[i] = 0; 2007 per_cpu(rcu_torture_batch, cpu)[i] = 0; 2008 } 2009 } 2010 2011 /* Start up the kthreads. */ 2012 2013 VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task"); 2014 writer_task = kthread_create(rcu_torture_writer, NULL, 2015 "rcu_torture_writer"); 2016 if (IS_ERR(writer_task)) { 2017 firsterr = PTR_ERR(writer_task); 2018 VERBOSE_PRINTK_ERRSTRING("Failed to create writer"); 2019 writer_task = NULL; 2020 goto unwind; 2021 } 2022 wake_up_process(writer_task); 2023 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]), 2024 GFP_KERNEL); 2025 if (fakewriter_tasks == NULL) { 2026 VERBOSE_PRINTK_ERRSTRING("out of memory"); 2027 firsterr = -ENOMEM; 2028 goto unwind; 2029 } 2030 for (i = 0; i < nfakewriters; i++) { 2031 VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task"); 2032 fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL, 2033 "rcu_torture_fakewriter"); 2034 if (IS_ERR(fakewriter_tasks[i])) { 2035 firsterr = PTR_ERR(fakewriter_tasks[i]); 2036 VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter"); 2037 fakewriter_tasks[i] = NULL; 2038 goto unwind; 2039 } 2040 } 2041 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 2042 GFP_KERNEL); 2043 if (reader_tasks == NULL) { 2044 VERBOSE_PRINTK_ERRSTRING("out of memory"); 2045 firsterr = -ENOMEM; 2046 goto unwind; 2047 } 2048 for (i = 0; i < nrealreaders; i++) { 2049 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task"); 2050 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL, 2051 "rcu_torture_reader"); 2052 if (IS_ERR(reader_tasks[i])) { 2053 firsterr = PTR_ERR(reader_tasks[i]); 2054 VERBOSE_PRINTK_ERRSTRING("Failed to create reader"); 2055 reader_tasks[i] = NULL; 2056 goto unwind; 2057 } 2058 } 2059 if (stat_interval > 0) { 2060 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task"); 2061 stats_task = kthread_run(rcu_torture_stats, NULL, 2062 "rcu_torture_stats"); 2063 if (IS_ERR(stats_task)) { 2064 firsterr = PTR_ERR(stats_task); 2065 VERBOSE_PRINTK_ERRSTRING("Failed to create stats"); 2066 stats_task = NULL; 2067 goto unwind; 2068 } 2069 } 2070 if (test_no_idle_hz) { 2071 rcu_idle_cpu = num_online_cpus() - 1; 2072 2073 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { 2074 firsterr = -ENOMEM; 2075 VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask"); 2076 goto unwind; 2077 } 2078 2079 /* Create the shuffler thread */ 2080 shuffler_task = kthread_run(rcu_torture_shuffle, NULL, 2081 "rcu_torture_shuffle"); 2082 if (IS_ERR(shuffler_task)) { 2083 free_cpumask_var(shuffle_tmp_mask); 2084 firsterr = PTR_ERR(shuffler_task); 2085 VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler"); 2086 shuffler_task = NULL; 2087 goto unwind; 2088 } 2089 } 2090 if (stutter < 0) 2091 stutter = 0; 2092 if (stutter) { 2093 /* Create the stutter thread */ 2094 stutter_task = kthread_run(rcu_torture_stutter, NULL, 2095 "rcu_torture_stutter"); 2096 if (IS_ERR(stutter_task)) { 2097 firsterr = PTR_ERR(stutter_task); 2098 VERBOSE_PRINTK_ERRSTRING("Failed to create stutter"); 2099 stutter_task = NULL; 2100 goto unwind; 2101 } 2102 } 2103 if (fqs_duration < 0) 2104 fqs_duration = 0; 2105 if (fqs_duration) { 2106 /* Create the stutter thread */ 2107 fqs_task = kthread_run(rcu_torture_fqs, NULL, 2108 "rcu_torture_fqs"); 2109 if (IS_ERR(fqs_task)) { 2110 firsterr = PTR_ERR(fqs_task); 2111 VERBOSE_PRINTK_ERRSTRING("Failed to create fqs"); 2112 fqs_task = NULL; 2113 goto unwind; 2114 } 2115 } 2116 if (test_boost_interval < 1) 2117 test_boost_interval = 1; 2118 if (test_boost_duration < 2) 2119 test_boost_duration = 2; 2120 if ((test_boost == 1 && cur_ops->can_boost) || 2121 test_boost == 2) { 2122 2123 boost_starttime = jiffies + test_boost_interval * HZ; 2124 register_cpu_notifier(&rcutorture_cpu_nb); 2125 for_each_possible_cpu(i) { 2126 if (cpu_is_offline(i)) 2127 continue; /* Heuristic: CPU can go offline. */ 2128 retval = rcutorture_booster_init(i); 2129 if (retval < 0) { 2130 firsterr = retval; 2131 goto unwind; 2132 } 2133 } 2134 } 2135 if (shutdown_secs > 0) { 2136 shutdown_time = jiffies + shutdown_secs * HZ; 2137 shutdown_task = kthread_create(rcu_torture_shutdown, NULL, 2138 "rcu_torture_shutdown"); 2139 if (IS_ERR(shutdown_task)) { 2140 firsterr = PTR_ERR(shutdown_task); 2141 VERBOSE_PRINTK_ERRSTRING("Failed to create shutdown"); 2142 shutdown_task = NULL; 2143 goto unwind; 2144 } 2145 wake_up_process(shutdown_task); 2146 } 2147 i = rcu_torture_onoff_init(); 2148 if (i != 0) { 2149 firsterr = i; 2150 goto unwind; 2151 } 2152 register_reboot_notifier(&rcutorture_shutdown_nb); 2153 i = rcu_torture_stall_init(); 2154 if (i != 0) { 2155 firsterr = i; 2156 goto unwind; 2157 } 2158 retval = rcu_torture_barrier_init(); 2159 if (retval != 0) { 2160 firsterr = retval; 2161 goto unwind; 2162 } 2163 rcutorture_record_test_transition(); 2164 mutex_unlock(&fullstop_mutex); 2165 return 0; 2166 2167 unwind: 2168 mutex_unlock(&fullstop_mutex); 2169 rcu_torture_cleanup(); 2170 return firsterr; 2171 } 2172 2173 module_init(rcu_torture_init); 2174 module_exit(rcu_torture_cleanup);
Cache object: 7fc17e412dd0137f5a399a82d47fcd0e
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]