1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Common functions for in-kernel torture tests.
4 *
5 * Copyright (C) IBM Corporation, 2014
6 *
7 * Author: Paul E. McKenney <paulmck@linux.ibm.com>
8 * Based on kernel/rcu/torture.c.
9 */
10
11 #define pr_fmt(fmt) fmt
12
13 #include <linux/types.h>
14 #include <linux/kernel.h>
15 #include <linux/init.h>
16 #include <linux/module.h>
17 #include <linux/kthread.h>
18 #include <linux/err.h>
19 #include <linux/spinlock.h>
20 #include <linux/smp.h>
21 #include <linux/interrupt.h>
22 #include <linux/sched.h>
23 #include <linux/sched/clock.h>
24 #include <linux/atomic.h>
25 #include <linux/bitops.h>
26 #include <linux/completion.h>
27 #include <linux/moduleparam.h>
28 #include <linux/percpu.h>
29 #include <linux/notifier.h>
30 #include <linux/reboot.h>
31 #include <linux/freezer.h>
32 #include <linux/cpu.h>
33 #include <linux/delay.h>
34 #include <linux/stat.h>
35 #include <linux/slab.h>
36 #include <linux/trace_clock.h>
37 #include <linux/ktime.h>
38 #include <asm/byteorder.h>
39 #include <linux/torture.h>
40 #include <linux/sched/rt.h>
41 #include "rcu/rcu.h"
42
43 MODULE_LICENSE("GPL");
44 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
45
46 static bool disable_onoff_at_boot;
47 module_param(disable_onoff_at_boot, bool, 0444);
48
49 static bool ftrace_dump_at_shutdown;
50 module_param(ftrace_dump_at_shutdown, bool, 0444);
51
52 static int verbose_sleep_frequency;
53 module_param(verbose_sleep_frequency, int, 0444);
54
55 static int verbose_sleep_duration = 1;
56 module_param(verbose_sleep_duration, int, 0444);
57
58 static int random_shuffle;
59 module_param(random_shuffle, int, 0444);
60
61 static char *torture_type;
62 static int verbose;
63
64 /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */
65 #define FULLSTOP_DONTSTOP 0 /* Normal operation. */
66 #define FULLSTOP_SHUTDOWN 1 /* System shutdown with torture running. */
67 #define FULLSTOP_RMMOD 2 /* Normal rmmod of torture. */
68 static int fullstop = FULLSTOP_RMMOD;
69 static DEFINE_MUTEX(fullstop_mutex);
70
71 static atomic_t verbose_sleep_counter;
72
73 /*
74 * Sleep if needed from VERBOSE_TOROUT*().
75 */
verbose_torout_sleep(void)76 void verbose_torout_sleep(void)
77 {
78 if (verbose_sleep_frequency > 0 &&
79 verbose_sleep_duration > 0 &&
80 !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency))
81 schedule_timeout_uninterruptible(verbose_sleep_duration);
82 }
83 EXPORT_SYMBOL_GPL(verbose_torout_sleep);
84
85 /*
86 * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit
87 * nanosecond random fuzz. This function and its friends desynchronize
88 * testing from the timer wheel.
89 */
torture_hrtimeout_ns(ktime_t baset_ns,u32 fuzzt_ns,struct torture_random_state * trsp)90 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp)
91 {
92 ktime_t hto = baset_ns;
93
94 if (trsp)
95 hto += torture_random(trsp) % fuzzt_ns;
96 set_current_state(TASK_IDLE);
97 return schedule_hrtimeout(&hto, HRTIMER_MODE_REL);
98 }
99 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns);
100
101 /*
102 * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit
103 * nanosecond (not microsecond!) random fuzz.
104 */
torture_hrtimeout_us(u32 baset_us,u32 fuzzt_ns,struct torture_random_state * trsp)105 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp)
106 {
107 ktime_t baset_ns = baset_us * NSEC_PER_USEC;
108
109 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
110 }
111 EXPORT_SYMBOL_GPL(torture_hrtimeout_us);
112
113 /*
114 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
115 * microsecond (not millisecond!) random fuzz.
116 */
torture_hrtimeout_ms(u32 baset_ms,u32 fuzzt_us,struct torture_random_state * trsp)117 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp)
118 {
119 ktime_t baset_ns = baset_ms * NSEC_PER_MSEC;
120 u32 fuzzt_ns;
121
122 if ((u32)~0U / NSEC_PER_USEC < fuzzt_us)
123 fuzzt_ns = (u32)~0U;
124 else
125 fuzzt_ns = fuzzt_us * NSEC_PER_USEC;
126 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
127 }
128 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms);
129
130 /*
131 * Schedule a high-resolution-timer sleep in jiffies, with an
132 * implied one-jiffy random fuzz. This is intended to replace calls to
133 * schedule_timeout_interruptible() and friends.
134 */
torture_hrtimeout_jiffies(u32 baset_j,struct torture_random_state * trsp)135 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp)
136 {
137 ktime_t baset_ns = jiffies_to_nsecs(baset_j);
138
139 return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp);
140 }
141 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies);
142
143 /*
144 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
145 * millisecond (not second!) random fuzz.
146 */
torture_hrtimeout_s(u32 baset_s,u32 fuzzt_ms,struct torture_random_state * trsp)147 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp)
148 {
149 ktime_t baset_ns = baset_s * NSEC_PER_SEC;
150 u32 fuzzt_ns;
151
152 if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms)
153 fuzzt_ns = (u32)~0U;
154 else
155 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC;
156 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
157 }
158 EXPORT_SYMBOL_GPL(torture_hrtimeout_s);
159
160 #ifdef CONFIG_HOTPLUG_CPU
161
162 /*
163 * Variables for online-offline handling. Only present if CPU hotplug
164 * is enabled, otherwise does nothing.
165 */
166
167 static struct task_struct *onoff_task;
168 static long onoff_holdoff;
169 static long onoff_interval;
170 static torture_ofl_func *onoff_f;
171 static long n_offline_attempts;
172 static long n_offline_successes;
173 static unsigned long sum_offline;
174 static int min_offline = -1;
175 static int max_offline;
176 static long n_online_attempts;
177 static long n_online_successes;
178 static unsigned long sum_online;
179 static int min_online = -1;
180 static int max_online;
181
182 static int torture_online_cpus = NR_CPUS;
183
184 /*
185 * Some torture testing leverages confusion as to the number of online
186 * CPUs. This function returns the torture-testing view of this number,
187 * which allows torture tests to load-balance appropriately.
188 */
torture_num_online_cpus(void)189 int torture_num_online_cpus(void)
190 {
191 return READ_ONCE(torture_online_cpus);
192 }
193 EXPORT_SYMBOL_GPL(torture_num_online_cpus);
194
195 /*
196 * Attempt to take a CPU offline. Return false if the CPU is already
197 * offline or if it is not subject to CPU-hotplug operations. The
198 * caller can detect other failures by looking at the statistics.
199 */
torture_offline(int cpu,long * n_offl_attempts,long * n_offl_successes,unsigned long * sum_offl,int * min_offl,int * max_offl)200 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
201 unsigned long *sum_offl, int *min_offl, int *max_offl)
202 {
203 unsigned long delta;
204 int ret;
205 char *s;
206 unsigned long starttime;
207
208 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
209 return false;
210 if (num_online_cpus() <= 1)
211 return false; /* Can't offline the last CPU. */
212
213 if (verbose > 1)
214 pr_alert("%s" TORTURE_FLAG
215 "torture_onoff task: offlining %d\n",
216 torture_type, cpu);
217 starttime = jiffies;
218 (*n_offl_attempts)++;
219 ret = remove_cpu(cpu);
220 if (ret) {
221 s = "";
222 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
223 // PCI probe frequently disables hotplug during boot.
224 (*n_offl_attempts)--;
225 s = " (-EBUSY forgiven during boot)";
226 }
227 if (verbose)
228 pr_alert("%s" TORTURE_FLAG
229 "torture_onoff task: offline %d failed%s: errno %d\n",
230 torture_type, cpu, s, ret);
231 } else {
232 if (verbose > 1)
233 pr_alert("%s" TORTURE_FLAG
234 "torture_onoff task: offlined %d\n",
235 torture_type, cpu);
236 if (onoff_f)
237 onoff_f();
238 (*n_offl_successes)++;
239 delta = jiffies - starttime;
240 *sum_offl += delta;
241 if (*min_offl < 0) {
242 *min_offl = delta;
243 *max_offl = delta;
244 }
245 if (*min_offl > delta)
246 *min_offl = delta;
247 if (*max_offl < delta)
248 *max_offl = delta;
249 WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1);
250 WARN_ON_ONCE(torture_online_cpus <= 0);
251 }
252
253 return true;
254 }
255 EXPORT_SYMBOL_GPL(torture_offline);
256
257 /*
258 * Attempt to bring a CPU online. Return false if the CPU is already
259 * online or if it is not subject to CPU-hotplug operations. The
260 * caller can detect other failures by looking at the statistics.
261 */
torture_online(int cpu,long * n_onl_attempts,long * n_onl_successes,unsigned long * sum_onl,int * min_onl,int * max_onl)262 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
263 unsigned long *sum_onl, int *min_onl, int *max_onl)
264 {
265 unsigned long delta;
266 int ret;
267 char *s;
268 unsigned long starttime;
269
270 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
271 return false;
272
273 if (verbose > 1)
274 pr_alert("%s" TORTURE_FLAG
275 "torture_onoff task: onlining %d\n",
276 torture_type, cpu);
277 starttime = jiffies;
278 (*n_onl_attempts)++;
279 ret = add_cpu(cpu);
280 if (ret) {
281 s = "";
282 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
283 // PCI probe frequently disables hotplug during boot.
284 (*n_onl_attempts)--;
285 s = " (-EBUSY forgiven during boot)";
286 }
287 if (verbose)
288 pr_alert("%s" TORTURE_FLAG
289 "torture_onoff task: online %d failed%s: errno %d\n",
290 torture_type, cpu, s, ret);
291 } else {
292 if (verbose > 1)
293 pr_alert("%s" TORTURE_FLAG
294 "torture_onoff task: onlined %d\n",
295 torture_type, cpu);
296 (*n_onl_successes)++;
297 delta = jiffies - starttime;
298 *sum_onl += delta;
299 if (*min_onl < 0) {
300 *min_onl = delta;
301 *max_onl = delta;
302 }
303 if (*min_onl > delta)
304 *min_onl = delta;
305 if (*max_onl < delta)
306 *max_onl = delta;
307 WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1);
308 }
309
310 return true;
311 }
312 EXPORT_SYMBOL_GPL(torture_online);
313
314 /*
315 * Get everything online at the beginning and ends of tests.
316 */
torture_online_all(char * phase)317 static void torture_online_all(char *phase)
318 {
319 int cpu;
320 int ret;
321
322 for_each_possible_cpu(cpu) {
323 if (cpu_online(cpu))
324 continue;
325 ret = add_cpu(cpu);
326 if (ret && verbose) {
327 pr_alert("%s" TORTURE_FLAG
328 "%s: %s online %d: errno %d\n",
329 __func__, phase, torture_type, cpu, ret);
330 }
331 }
332 }
333
334 /*
335 * Execute random CPU-hotplug operations at the interval specified
336 * by the onoff_interval.
337 */
338 static int
torture_onoff(void * arg)339 torture_onoff(void *arg)
340 {
341 int cpu;
342 int maxcpu = -1;
343 DEFINE_TORTURE_RANDOM(rand);
344
345 VERBOSE_TOROUT_STRING("torture_onoff task started");
346 for_each_online_cpu(cpu)
347 maxcpu = cpu;
348 WARN_ON(maxcpu < 0);
349 torture_online_all("Initial");
350 if (maxcpu == 0) {
351 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
352 goto stop;
353 }
354
355 if (onoff_holdoff > 0) {
356 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
357 torture_hrtimeout_jiffies(onoff_holdoff, &rand);
358 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
359 }
360 while (!torture_must_stop()) {
361 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
362 torture_hrtimeout_jiffies(HZ / 10, &rand);
363 continue;
364 }
365 cpu = torture_random(&rand) % (maxcpu + 1);
366 if (!torture_offline(cpu,
367 &n_offline_attempts, &n_offline_successes,
368 &sum_offline, &min_offline, &max_offline))
369 torture_online(cpu,
370 &n_online_attempts, &n_online_successes,
371 &sum_online, &min_online, &max_online);
372 torture_hrtimeout_jiffies(onoff_interval, &rand);
373 }
374
375 stop:
376 torture_kthread_stopping("torture_onoff");
377 torture_online_all("Final");
378 return 0;
379 }
380
381 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
382
383 /*
384 * Initiate online-offline handling.
385 */
torture_onoff_init(long ooholdoff,long oointerval,torture_ofl_func * f)386 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f)
387 {
388 #ifdef CONFIG_HOTPLUG_CPU
389 onoff_holdoff = ooholdoff;
390 onoff_interval = oointerval;
391 onoff_f = f;
392 if (onoff_interval <= 0)
393 return 0;
394 return torture_create_kthread(torture_onoff, NULL, onoff_task);
395 #else /* #ifdef CONFIG_HOTPLUG_CPU */
396 return 0;
397 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
398 }
399 EXPORT_SYMBOL_GPL(torture_onoff_init);
400
401 /*
402 * Clean up after online/offline testing.
403 */
torture_onoff_cleanup(void)404 static void torture_onoff_cleanup(void)
405 {
406 #ifdef CONFIG_HOTPLUG_CPU
407 if (onoff_task == NULL)
408 return;
409 VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
410 kthread_stop(onoff_task);
411 onoff_task = NULL;
412 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
413 }
414
415 /*
416 * Print online/offline testing statistics.
417 */
torture_onoff_stats(void)418 void torture_onoff_stats(void)
419 {
420 #ifdef CONFIG_HOTPLUG_CPU
421 pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
422 n_online_successes, n_online_attempts,
423 n_offline_successes, n_offline_attempts,
424 min_online, max_online,
425 min_offline, max_offline,
426 sum_online, sum_offline, HZ);
427 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
428 }
429 EXPORT_SYMBOL_GPL(torture_onoff_stats);
430
431 /*
432 * Were all the online/offline operations successful?
433 */
torture_onoff_failures(void)434 bool torture_onoff_failures(void)
435 {
436 #ifdef CONFIG_HOTPLUG_CPU
437 return n_online_successes != n_online_attempts ||
438 n_offline_successes != n_offline_attempts;
439 #else /* #ifdef CONFIG_HOTPLUG_CPU */
440 return false;
441 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
442 }
443 EXPORT_SYMBOL_GPL(torture_onoff_failures);
444
445 #define TORTURE_RANDOM_MULT 39916801 /* prime */
446 #define TORTURE_RANDOM_ADD 479001701 /* prime */
447 #define TORTURE_RANDOM_REFRESH 10000
448
449 /*
450 * Crude but fast random-number generator. Uses a linear congruential
451 * generator, with occasional help from cpu_clock().
452 */
453 unsigned long
torture_random(struct torture_random_state * trsp)454 torture_random(struct torture_random_state *trsp)
455 {
456 if (--trsp->trs_count < 0) {
457 trsp->trs_state += (unsigned long)local_clock() + raw_smp_processor_id();
458 trsp->trs_count = TORTURE_RANDOM_REFRESH;
459 }
460 trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
461 TORTURE_RANDOM_ADD;
462 return swahw32(trsp->trs_state);
463 }
464 EXPORT_SYMBOL_GPL(torture_random);
465
466 /*
467 * Variables for shuffling. The idea is to ensure that each CPU stays
468 * idle for an extended period to test interactions with dyntick idle,
469 * as well as interactions with any per-CPU variables.
470 */
471 struct shuffle_task {
472 struct list_head st_l;
473 struct task_struct *st_t;
474 };
475
476 static long shuffle_interval; /* In jiffies. */
477 static struct task_struct *shuffler_task;
478 static cpumask_var_t shuffle_tmp_mask;
479 static int shuffle_idle_cpu; /* Force all torture tasks off this CPU */
480 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
481 static DEFINE_MUTEX(shuffle_task_mutex);
482
483 /*
484 * Register a task to be shuffled. If there is no memory, just splat
485 * and don't bother registering.
486 */
torture_shuffle_task_register(struct task_struct * tp)487 void torture_shuffle_task_register(struct task_struct *tp)
488 {
489 struct shuffle_task *stp;
490
491 if (WARN_ON_ONCE(tp == NULL))
492 return;
493 stp = kmalloc(sizeof(*stp), GFP_KERNEL);
494 if (WARN_ON_ONCE(stp == NULL))
495 return;
496 stp->st_t = tp;
497 mutex_lock(&shuffle_task_mutex);
498 list_add(&stp->st_l, &shuffle_task_list);
499 mutex_unlock(&shuffle_task_mutex);
500 }
501 EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
502
503 /*
504 * Unregister all tasks, for example, at the end of the torture run.
505 */
torture_shuffle_task_unregister_all(void)506 static void torture_shuffle_task_unregister_all(void)
507 {
508 struct shuffle_task *stp;
509 struct shuffle_task *p;
510
511 mutex_lock(&shuffle_task_mutex);
512 list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
513 list_del(&stp->st_l);
514 kfree(stp);
515 }
516 mutex_unlock(&shuffle_task_mutex);
517 }
518
519 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle.
520 * A special case is when shuffle_idle_cpu = -1, in which case we allow
521 * the tasks to run on all CPUs.
522 */
torture_shuffle_tasks(void)523 static void torture_shuffle_tasks(void)
524 {
525 DEFINE_TORTURE_RANDOM(rand);
526 struct shuffle_task *stp;
527
528 cpumask_setall(shuffle_tmp_mask);
529 cpus_read_lock();
530
531 /* No point in shuffling if there is only one online CPU (ex: UP) */
532 if (num_online_cpus() == 1) {
533 cpus_read_unlock();
534 return;
535 }
536
537 /* Advance to the next CPU. Upon overflow, don't idle any CPUs. */
538 shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
539 if (shuffle_idle_cpu >= nr_cpu_ids)
540 shuffle_idle_cpu = -1;
541 else
542 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
543
544 mutex_lock(&shuffle_task_mutex);
545 list_for_each_entry(stp, &shuffle_task_list, st_l) {
546 if (!random_shuffle || torture_random(&rand) & 0x1)
547 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
548 }
549 mutex_unlock(&shuffle_task_mutex);
550
551 cpus_read_unlock();
552 }
553
554 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
555 * system to become idle at a time and cut off its timer ticks. This is meant
556 * to test the support for such tickless idle CPU in RCU.
557 */
torture_shuffle(void * arg)558 static int torture_shuffle(void *arg)
559 {
560 DEFINE_TORTURE_RANDOM(rand);
561
562 VERBOSE_TOROUT_STRING("torture_shuffle task started");
563 do {
564 torture_hrtimeout_jiffies(shuffle_interval, &rand);
565 torture_shuffle_tasks();
566 torture_shutdown_absorb("torture_shuffle");
567 } while (!torture_must_stop());
568 torture_kthread_stopping("torture_shuffle");
569 return 0;
570 }
571
572 /*
573 * Start the shuffler, with shuffint in jiffies.
574 */
torture_shuffle_init(long shuffint)575 int torture_shuffle_init(long shuffint)
576 {
577 shuffle_interval = shuffint;
578
579 shuffle_idle_cpu = -1;
580
581 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
582 TOROUT_ERRSTRING("Failed to alloc mask");
583 return -ENOMEM;
584 }
585
586 /* Create the shuffler thread */
587 return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
588 }
589 EXPORT_SYMBOL_GPL(torture_shuffle_init);
590
591 /*
592 * Stop the shuffling.
593 */
torture_shuffle_cleanup(void)594 static void torture_shuffle_cleanup(void)
595 {
596 torture_shuffle_task_unregister_all();
597 if (shuffler_task) {
598 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
599 kthread_stop(shuffler_task);
600 free_cpumask_var(shuffle_tmp_mask);
601 }
602 shuffler_task = NULL;
603 }
604
605 /*
606 * Variables for auto-shutdown. This allows "lights out" torture runs
607 * to be fully scripted.
608 */
609 static struct task_struct *shutdown_task;
610 static ktime_t shutdown_time; /* time to system shutdown. */
611 static void (*torture_shutdown_hook)(void);
612
613 /*
614 * Absorb kthreads into a kernel function that won't return, so that
615 * they won't ever access module text or data again.
616 */
torture_shutdown_absorb(const char * title)617 void torture_shutdown_absorb(const char *title)
618 {
619 while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
620 pr_notice("torture thread %s parking due to system shutdown\n",
621 title);
622 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
623 }
624 }
625 EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
626
627 /*
628 * Cause the torture test to shutdown the system after the test has
629 * run for the time specified by the shutdown_secs parameter.
630 */
torture_shutdown(void * arg)631 static int torture_shutdown(void *arg)
632 {
633 ktime_t ktime_snap;
634
635 VERBOSE_TOROUT_STRING("torture_shutdown task started");
636 ktime_snap = ktime_get();
637 while (ktime_before(ktime_snap, shutdown_time) &&
638 !torture_must_stop()) {
639 if (verbose)
640 pr_alert("%s" TORTURE_FLAG
641 "torture_shutdown task: %llu ms remaining\n",
642 torture_type,
643 ktime_ms_delta(shutdown_time, ktime_snap));
644 set_current_state(TASK_INTERRUPTIBLE);
645 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
646 ktime_snap = ktime_get();
647 }
648 if (torture_must_stop()) {
649 torture_kthread_stopping("torture_shutdown");
650 return 0;
651 }
652
653 /* OK, shut down the system. */
654
655 VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
656 shutdown_task = NULL; /* Avoid self-kill deadlock. */
657 if (torture_shutdown_hook)
658 torture_shutdown_hook();
659 else
660 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
661 if (ftrace_dump_at_shutdown)
662 rcu_ftrace_dump(DUMP_ALL);
663 kernel_power_off(); /* Shut down the system. */
664 return 0;
665 }
666
667 /*
668 * Start up the shutdown task.
669 */
torture_shutdown_init(int ssecs,void (* cleanup)(void))670 int torture_shutdown_init(int ssecs, void (*cleanup)(void))
671 {
672 torture_shutdown_hook = cleanup;
673 if (ssecs > 0) {
674 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
675 return torture_create_kthread(torture_shutdown, NULL,
676 shutdown_task);
677 }
678 return 0;
679 }
680 EXPORT_SYMBOL_GPL(torture_shutdown_init);
681
682 /*
683 * Detect and respond to a system shutdown.
684 */
torture_shutdown_notify(struct notifier_block * unused1,unsigned long unused2,void * unused3)685 static int torture_shutdown_notify(struct notifier_block *unused1,
686 unsigned long unused2, void *unused3)
687 {
688 mutex_lock(&fullstop_mutex);
689 if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
690 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
691 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
692 } else {
693 pr_warn("Concurrent rmmod and shutdown illegal!\n");
694 }
695 mutex_unlock(&fullstop_mutex);
696 return NOTIFY_DONE;
697 }
698
699 static struct notifier_block torture_shutdown_nb = {
700 .notifier_call = torture_shutdown_notify,
701 };
702
703 /*
704 * Shut down the shutdown task. Say what??? Heh! This can happen if
705 * the torture module gets an rmmod before the shutdown time arrives. ;-)
706 */
torture_shutdown_cleanup(void)707 static void torture_shutdown_cleanup(void)
708 {
709 unregister_reboot_notifier(&torture_shutdown_nb);
710 if (shutdown_task != NULL) {
711 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
712 kthread_stop(shutdown_task);
713 }
714 shutdown_task = NULL;
715 }
716
717 /*
718 * Variables for stuttering, which means to periodically pause and
719 * restart testing in order to catch bugs that appear when load is
720 * suddenly applied to or removed from the system.
721 */
722 static struct task_struct *stutter_task;
723 static int stutter_pause_test;
724 static int stutter;
725 static int stutter_gap;
726
727 /*
728 * Block until the stutter interval ends. This must be called periodically
729 * by all running kthreads that need to be subject to stuttering.
730 */
stutter_wait(const char * title)731 bool stutter_wait(const char *title)
732 {
733 unsigned int i = 0;
734 bool ret = false;
735 int spt;
736
737 cond_resched_tasks_rcu_qs();
738 spt = READ_ONCE(stutter_pause_test);
739 for (; spt; spt = READ_ONCE(stutter_pause_test)) {
740 if (!ret && !rt_task(current)) {
741 sched_set_normal(current, MAX_NICE);
742 ret = true;
743 }
744 if (spt == 1) {
745 torture_hrtimeout_jiffies(1, NULL);
746 } else if (spt == 2) {
747 while (READ_ONCE(stutter_pause_test)) {
748 if (!(i++ & 0xffff))
749 torture_hrtimeout_us(10, 0, NULL);
750 cond_resched();
751 }
752 } else {
753 torture_hrtimeout_jiffies(round_jiffies_relative(HZ), NULL);
754 }
755 torture_shutdown_absorb(title);
756 }
757 return ret;
758 }
759 EXPORT_SYMBOL_GPL(stutter_wait);
760
761 /*
762 * Cause the torture test to "stutter", starting and stopping all
763 * threads periodically.
764 */
torture_stutter(void * arg)765 static int torture_stutter(void *arg)
766 {
767 DEFINE_TORTURE_RANDOM(rand);
768 int wtime;
769
770 VERBOSE_TOROUT_STRING("torture_stutter task started");
771 do {
772 if (!torture_must_stop() && stutter > 1) {
773 wtime = stutter;
774 if (stutter > 2) {
775 WRITE_ONCE(stutter_pause_test, 1);
776 wtime = stutter - 3;
777 torture_hrtimeout_jiffies(wtime, &rand);
778 wtime = 2;
779 }
780 WRITE_ONCE(stutter_pause_test, 2);
781 torture_hrtimeout_jiffies(wtime, NULL);
782 }
783 WRITE_ONCE(stutter_pause_test, 0);
784 if (!torture_must_stop())
785 torture_hrtimeout_jiffies(stutter_gap, NULL);
786 torture_shutdown_absorb("torture_stutter");
787 } while (!torture_must_stop());
788 torture_kthread_stopping("torture_stutter");
789 return 0;
790 }
791
792 /*
793 * Initialize and kick off the torture_stutter kthread.
794 */
torture_stutter_init(const int s,const int sgap)795 int torture_stutter_init(const int s, const int sgap)
796 {
797 stutter = s;
798 stutter_gap = sgap;
799 return torture_create_kthread(torture_stutter, NULL, stutter_task);
800 }
801 EXPORT_SYMBOL_GPL(torture_stutter_init);
802
803 /*
804 * Cleanup after the torture_stutter kthread.
805 */
torture_stutter_cleanup(void)806 static void torture_stutter_cleanup(void)
807 {
808 if (!stutter_task)
809 return;
810 VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
811 kthread_stop(stutter_task);
812 stutter_task = NULL;
813 }
814
815 /*
816 * Initialize torture module. Please note that this is -not- invoked via
817 * the usual module_init() mechanism, but rather by an explicit call from
818 * the client torture module. This call must be paired with a later
819 * torture_init_end().
820 *
821 * The runnable parameter points to a flag that controls whether or not
822 * the test is currently runnable. If there is no such flag, pass in NULL.
823 */
torture_init_begin(char * ttype,int v)824 bool torture_init_begin(char *ttype, int v)
825 {
826 mutex_lock(&fullstop_mutex);
827 if (torture_type != NULL) {
828 pr_alert("%s: Refusing %s init: %s running.\n",
829 __func__, ttype, torture_type);
830 pr_alert("%s: One torture test at a time!\n", __func__);
831 mutex_unlock(&fullstop_mutex);
832 return false;
833 }
834 torture_type = ttype;
835 verbose = v;
836 fullstop = FULLSTOP_DONTSTOP;
837 return true;
838 }
839 EXPORT_SYMBOL_GPL(torture_init_begin);
840
841 /*
842 * Tell the torture module that initialization is complete.
843 */
torture_init_end(void)844 void torture_init_end(void)
845 {
846 mutex_unlock(&fullstop_mutex);
847 register_reboot_notifier(&torture_shutdown_nb);
848 }
849 EXPORT_SYMBOL_GPL(torture_init_end);
850
851 /*
852 * Clean up torture module. Please note that this is -not- invoked via
853 * the usual module_exit() mechanism, but rather by an explicit call from
854 * the client torture module. Returns true if a race with system shutdown
855 * is detected, otherwise, all kthreads started by functions in this file
856 * will be shut down.
857 *
858 * This must be called before the caller starts shutting down its own
859 * kthreads.
860 *
861 * Both torture_cleanup_begin() and torture_cleanup_end() must be paired,
862 * in order to correctly perform the cleanup. They are separated because
863 * threads can still need to reference the torture_type type, thus nullify
864 * only after completing all other relevant calls.
865 */
torture_cleanup_begin(void)866 bool torture_cleanup_begin(void)
867 {
868 mutex_lock(&fullstop_mutex);
869 if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
870 pr_warn("Concurrent rmmod and shutdown illegal!\n");
871 mutex_unlock(&fullstop_mutex);
872 schedule_timeout_uninterruptible(10);
873 return true;
874 }
875 WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
876 mutex_unlock(&fullstop_mutex);
877 torture_shutdown_cleanup();
878 torture_shuffle_cleanup();
879 torture_stutter_cleanup();
880 torture_onoff_cleanup();
881 return false;
882 }
883 EXPORT_SYMBOL_GPL(torture_cleanup_begin);
884
torture_cleanup_end(void)885 void torture_cleanup_end(void)
886 {
887 mutex_lock(&fullstop_mutex);
888 torture_type = NULL;
889 mutex_unlock(&fullstop_mutex);
890 }
891 EXPORT_SYMBOL_GPL(torture_cleanup_end);
892
893 /*
894 * Is it time for the current torture test to stop?
895 */
torture_must_stop(void)896 bool torture_must_stop(void)
897 {
898 return torture_must_stop_irq() || kthread_should_stop();
899 }
900 EXPORT_SYMBOL_GPL(torture_must_stop);
901
902 /*
903 * Is it time for the current torture test to stop? This is the irq-safe
904 * version, hence no check for kthread_should_stop().
905 */
torture_must_stop_irq(void)906 bool torture_must_stop_irq(void)
907 {
908 return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
909 }
910 EXPORT_SYMBOL_GPL(torture_must_stop_irq);
911
912 /*
913 * Each kthread must wait for kthread_should_stop() before returning from
914 * its top-level function, otherwise segfaults ensue. This function
915 * prints a "stopping" message and waits for kthread_should_stop(), and
916 * should be called from all torture kthreads immediately prior to
917 * returning.
918 */
torture_kthread_stopping(char * title)919 void torture_kthread_stopping(char *title)
920 {
921 char buf[128];
922
923 snprintf(buf, sizeof(buf), "%s is stopping", title);
924 VERBOSE_TOROUT_STRING(buf);
925 while (!kthread_should_stop()) {
926 torture_shutdown_absorb(title);
927 schedule_timeout_uninterruptible(HZ / 20);
928 }
929 }
930 EXPORT_SYMBOL_GPL(torture_kthread_stopping);
931
932 /*
933 * Create a generic torture kthread that is immediately runnable. If you
934 * need the kthread to be stopped so that you can do something to it before
935 * it starts, you will need to open-code your own.
936 */
_torture_create_kthread(int (* fn)(void * arg),void * arg,char * s,char * m,char * f,struct task_struct ** tp,void (* cbf)(struct task_struct * tp))937 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
938 char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp))
939 {
940 int ret = 0;
941
942 VERBOSE_TOROUT_STRING(m);
943 *tp = kthread_create(fn, arg, "%s", s);
944 if (IS_ERR(*tp)) {
945 ret = PTR_ERR(*tp);
946 TOROUT_ERRSTRING(f);
947 *tp = NULL;
948 return ret;
949 }
950
951 if (cbf)
952 cbf(*tp);
953
954 wake_up_process(*tp); // Process is sleeping, so ordering provided.
955 torture_shuffle_task_register(*tp);
956 return ret;
957 }
958 EXPORT_SYMBOL_GPL(_torture_create_kthread);
959
960 /*
961 * Stop a generic kthread, emitting a message.
962 */
_torture_stop_kthread(char * m,struct task_struct ** tp)963 void _torture_stop_kthread(char *m, struct task_struct **tp)
964 {
965 if (*tp == NULL)
966 return;
967 VERBOSE_TOROUT_STRING(m);
968 kthread_stop(*tp);
969 *tp = NULL;
970 }
971 EXPORT_SYMBOL_GPL(_torture_stop_kthread);
972
