2556 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)  in rcu_torture_fwd_cb_hist()  argument
2563 	for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)  in rcu_torture_fwd_cb_hist()
2564 		if (rfp->n_launders_hist[i].n_launders > 0)  in rcu_torture_fwd_cb_hist()
2567 		 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat);  in rcu_torture_fwd_cb_hist()
2568 	gps_old = rfp->rcu_launder_gp_seq_start;  in rcu_torture_fwd_cb_hist()
2570 		gps = rfp->n_launders_hist[j].launder_gp_seq;  in rcu_torture_fwd_cb_hist()
2573 			rfp->n_launders_hist[j].n_launders,  in rcu_torture_fwd_cb_hist()
2587 	struct rcu_fwd *rfp = rfcp->rfc_rfp;  in rcu_torture_fwd_cb_cr()  local
2591 	spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
2592 	rfcpp = rfp->rcu_fwd_cb_tail;  in rcu_torture_fwd_cb_cr()
2593 	rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;  in rcu_torture_fwd_cb_cr()
2595 	WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);  in rcu_torture_fwd_cb_cr()
2596 	i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));  in rcu_torture_fwd_cb_cr()
2597 	if (i >= ARRAY_SIZE(rfp->n_launders_hist))  in rcu_torture_fwd_cb_cr()
2598 		i = ARRAY_SIZE(rfp->n_launders_hist) - 1;  in rcu_torture_fwd_cb_cr()
2599 	rfp->n_launders_hist[i].n_launders++;  in rcu_torture_fwd_cb_cr()
2600 	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();  in rcu_torture_fwd_cb_cr()
2601 	spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
2621 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cbfree()  argument
2628 		spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2629 		rfcp = rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
2631 			spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2634 		rfp->rcu_fwd_cb_head = rfcp->rfc_next;  in rcu_torture_fwd_prog_cbfree()
2635 		if (!rfp->rcu_fwd_cb_head)  in rcu_torture_fwd_prog_cbfree()
2636 			rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
2637 		spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2651 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,  in rcu_torture_fwd_prog_nr()  argument
2665 	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_nr()
2685 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_nr()
2686 	stopat = rfp->rcu_fwd_startat + dur;  in rcu_torture_fwd_prog_nr()
2705 			 rfp->rcu_fwd_id, dur, cver, gps);  in rcu_torture_fwd_prog_nr()
2710 		pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_nr()
2723 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cr()  argument
2739 	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_cr()
2748 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_cr()
2749 	stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;  in rcu_torture_fwd_prog_cr()
2751 	rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread  in rcu_torture_fwd_prog_cr()
2755 	for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)  in rcu_torture_fwd_prog_cr()
2756 		rfp->n_launders_hist[i].n_launders = 0;  in rcu_torture_fwd_prog_cr()
2759 	rfp->rcu_launder_gp_seq_start = gps;  in rcu_torture_fwd_prog_cr()
2764 		rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);  in rcu_torture_fwd_prog_cr()
2772 			rfp->rcu_fwd_cb_head = rfcpn;  in rcu_torture_fwd_prog_cr()
2784 			rfcp->rfc_rfp = rfp;  in rcu_torture_fwd_prog_cr()
2798 	n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);  in rcu_torture_fwd_prog_cr()
2801 	pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_cr()
2803 	(void)rcu_torture_fwd_prog_cbfree(rfp);  in rcu_torture_fwd_prog_cr()
2810 			 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,  in rcu_torture_fwd_prog_cr()
2816 		rcu_torture_fwd_cb_hist(rfp);  in rcu_torture_fwd_prog_cr()
2834 	struct rcu_fwd *rfp;  in rcutorture_oom_notify()  local
2837 	rfp = rcu_fwds;  in rcutorture_oom_notify()
2838 	if (!rfp) {  in rcutorture_oom_notify()
2845 		rcu_torture_fwd_cb_hist(&rfp[i]);  in rcutorture_oom_notify()
2846 		rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2);  in rcutorture_oom_notify()
2852 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);  in rcutorture_oom_notify()
2857 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);  in rcutorture_oom_notify()
2862 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);  in rcutorture_oom_notify()
2882 	struct rcu_fwd *rfp = args;  in rcu_torture_fwd_prog()  local
2891 		if (!rfp->rcu_fwd_id) {  in rcu_torture_fwd_prog()
2905 		pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog()
2906 		if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id)  in rcu_torture_fwd_prog()
2907 			rcu_torture_fwd_prog_cr(rfp);  in rcu_torture_fwd_prog()
2911 		      torture_num_online_cpus() > rfp->rcu_fwd_id)))  in rcu_torture_fwd_prog()
2912 			rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);  in rcu_torture_fwd_prog()
2919 	if (!rfp->rcu_fwd_id) {  in rcu_torture_fwd_prog()
2932 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_init()  local
2961 	rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL);  in rcu_torture_fwd_prog_init()
2963 	if (!rfp || !fwd_prog_tasks) {  in rcu_torture_fwd_prog_init()
2964 		kfree(rfp);  in rcu_torture_fwd_prog_init()
2971 		spin_lock_init(&rfp[i].rcu_fwd_lock);  in rcu_torture_fwd_prog_init()
2972 		rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head;  in rcu_torture_fwd_prog_init()
2973 		rfp[i].rcu_fwd_id = i;  in rcu_torture_fwd_prog_init()
2976 	rcu_fwds = rfp;  in rcu_torture_fwd_prog_init()
2992 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_cleanup()  local
3000 	rfp = rcu_fwds;  in rcu_torture_fwd_prog_cleanup()
3003 	kfree(rfp);  in rcu_torture_fwd_prog_cleanup()