1 // SPDX-License-Identifier: GPL-2.0+
41 #define SCALE_FLAG "-ref-scale: "
78 	      "Holdoff time before test start (s)");
83 // Number of readers, with -1 defaulting to about 75% of the CPUs.
84 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
88 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
150 	for (i = nloops; i >= 0; i--) {  in ref_rcu_read_section()
160 	for (i = nloops; i >= 0; i--) {  in ref_rcu_delay_section()
188 	for (i = nloops; i >= 0; i--) {  in srcu_ref_scale_read_section()
199 	for (i = nloops; i >= 0; i--) {  in srcu_ref_scale_delay_section()
221 	for (i = nloops; i >= 0; i--)  in rcu_tasks_ref_scale_read_section()
229 	for (i = nloops; i >= 0; i--)  in rcu_tasks_ref_scale_delay_section()
237 	.name		= "rcu-tasks"
255 	for (i = nloops; i >= 0; i--) {  in rcu_trace_ref_scale_read_section()
265 	for (i = nloops; i >= 0; i--) {  in rcu_trace_ref_scale_delay_section()
276 	.name		= "rcu-trace"
294 	for (i = nloops; i >= 0; i--) {  in ref_refcnt_section()
304 	for (i = nloops; i >= 0; i--) {  in ref_refcnt_delay_section()
331 	for (i = nloops; i >= 0; i--) {  in ref_rwlock_section()
341 	for (i = nloops; i >= 0; i--) {  in ref_rwlock_delay_section()
368 	for (i = nloops; i >= 0; i--) {  in ref_rwsem_section()
378 	for (i = nloops; i >= 0; i--) {  in ref_rwsem_delay_section()
400 	for (i = nloops; i >= 0; i--) {  in ref_lock_section()
412 	for (i = nloops; i >= 0; i--) {  in ref_lock_delay_section()
426 // Definitions for global irq-save spinlock
434 	for (i = nloops; i >= 0; i--) {  in ref_lock_irq_section()
447 	for (i = nloops; i >= 0; i--) {  in ref_lock_irq_delay_section()
458 	.name		= "lock-irq"
461 // Definitions acquire-release.
470 	for (i = nloops; i >= 0; i--) {  in ref_acqrel_section()
483 	for (i = nloops; i >= 0; i--) {  in ref_acqrel_delay_section()
505 	for (i = nloops; i >= 0; i--)  in ref_clock_section()
517 	for (i = nloops; i >= 0; i--) {  in ref_clock_delay_section()
537 	for (i = nloops; i >= 0; i--)  in ref_jiffies_section()
549 	for (i = nloops; i >= 0; i--) {  in ref_jiffies_delay_section()
584 // Conditionally acquire an explicit in-structure reference count.
587 	return atomic_inc_not_zero(&rtsp->rts_refctr);  in typesafe_ref_acquire()
590 // Unconditionally release an explicit in-structure reference count.
593 	if (!atomic_dec_return(&rtsp->rts_refctr)) {  in typesafe_ref_release()
594 		WRITE_ONCE(rtsp->a, rtsp->a + 1);  in typesafe_ref_release()
600 // Unconditionally acquire an explicit in-structure spinlock.
603 	spin_lock(&rtsp->rts_lock);  in typesafe_lock_acquire()
607 // Unconditionally release an explicit in-structure spinlock.
610 	spin_unlock(&rtsp->rts_lock);  in typesafe_lock_release()
614 // Unconditionally acquire an explicit in-structure sequence lock.
617 	*start = read_seqbegin(&rtsp->rts_seqlock);  in typesafe_seqlock_acquire()
621 // Conditionally release an explicit in-structure sequence lock.  Return
625 	return !read_seqretry(&rtsp->rts_seqlock, start);  in typesafe_seqlock_release()
628 // Do a read-side critical section with the specified delay in
640 	for (i = nloops; i >= 0; i--) {  in typesafe_delay_section()
647 		a = READ_ONCE(rtsp->a);  in typesafe_delay_section()
652 		if (a != READ_ONCE(rtsp->a)) {  in typesafe_delay_section()
658 		// Remember, seqlock read-side release can fail.  in typesafe_delay_section()
663 		b = READ_ONCE(rtsp->a);  in typesafe_delay_section()
664 		WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);  in typesafe_delay_section()
665 		b = rtsp->b;  in typesafe_delay_section()
671 // Because the acquisition and release methods are expensive, there
688 	atomic_set(&rtsp->rts_refctr, 1);  in typesafe_alloc_one()
689 	WRITE_ONCE(rtsp->a, rtsp->a + 1);  in typesafe_alloc_one()
690 	WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);  in typesafe_alloc_one()
694 // Slab-allocator constructor for refscale_typesafe structures created
700 	spin_lock_init(&rtsp->rts_lock);  in refscale_typesafe_ctor()
701 	seqlock_init(&rtsp->rts_seqlock);  in refscale_typesafe_ctor()
703 	rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));  in refscale_typesafe_ctor()
723 		si = -si * nr_cpu_ids;  in typesafe_init()
797 		cur_ops->readsection(loops);  in rcu_scale_one_reader()
799 		cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);  in rcu_scale_one_reader()
802 // Reader kthread.  Repeatedly does empty RCU read-side
803 // critical section, minimizing update-side interference.
823 	wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||  in ref_scale_reader()
832 	WRITE_ONCE(rt->start_reader, 0);  in ref_scale_reader()
840 	// To reduce noise, do an initial cache-warming invocation, check  in ref_scale_reader()
853 	duration = ktime_get_mono_fast_ns() - start;  in ref_scale_reader()
856 	rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;  in ref_scale_reader()
857 	// To reduce runtime-skew noise, do maintain-load invocations until  in ref_scale_reader()
884 		rt->last_duration_ns = 0;  in reset_readers()
901 	sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",  in process_durations()
906 		sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);  in process_durations()
916 		sum += rt->last_duration_ns;  in process_durations()
990 	pr_alert("Runs\tTime(ns)\n");  in main_func()
1028 …"--- %s:  verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_…  in ref_scale_print_module_parms()
1055 	// Do scale-type-specific cleanup operations.  in ref_scale_cleanup()
1056 	if (cur_ops->cleanup != NULL)  in ref_scale_cleanup()
1057 		cur_ops->cleanup();  in ref_scale_cleanup()
1072 	return -EINVAL;  in ref_scale_shutdown()
1087 		return -EBUSY;  in ref_scale_init()
1091 		if (strcmp(scale_type, cur_ops->name) == 0)  in ref_scale_init()
1095 		pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);  in ref_scale_init()
1096 		pr_alert("rcu-scale types:");  in ref_scale_init()
1098 			pr_cont(" %s", scale_ops[i]->name);  in ref_scale_init()
1100 		firsterr = -EINVAL;  in ref_scale_init()
1104 	if (cur_ops->init)  in ref_scale_init()
1105 		if (!cur_ops->init()) {  in ref_scale_init()
1106 			firsterr = -EUCLEAN;  in ref_scale_init()
1135 		firsterr = -ENOMEM;  in ref_scale_init()