2511 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)  in rcu_torture_fwd_cb_hist()  argument
2518 	for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)  in rcu_torture_fwd_cb_hist()
2519 		if (rfp->n_launders_hist[i].n_launders > 0)  in rcu_torture_fwd_cb_hist()
2522 		 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat);  in rcu_torture_fwd_cb_hist()
2523 	gps_old = rfp->rcu_launder_gp_seq_start;  in rcu_torture_fwd_cb_hist()
2525 		gps = rfp->n_launders_hist[j].launder_gp_seq;  in rcu_torture_fwd_cb_hist()
2528 			rfp->n_launders_hist[j].n_launders,  in rcu_torture_fwd_cb_hist()
2542 	struct rcu_fwd *rfp = rfcp->rfc_rfp;  in rcu_torture_fwd_cb_cr()  local
2546 	spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
2547 	rfcpp = rfp->rcu_fwd_cb_tail;  in rcu_torture_fwd_cb_cr()
2548 	rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;  in rcu_torture_fwd_cb_cr()
2550 	WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);  in rcu_torture_fwd_cb_cr()
2551 	i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));  in rcu_torture_fwd_cb_cr()
2552 	if (i >= ARRAY_SIZE(rfp->n_launders_hist))  in rcu_torture_fwd_cb_cr()
2553 		i = ARRAY_SIZE(rfp->n_launders_hist) - 1;  in rcu_torture_fwd_cb_cr()
2554 	rfp->n_launders_hist[i].n_launders++;  in rcu_torture_fwd_cb_cr()
2555 	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();  in rcu_torture_fwd_cb_cr()
2556 	spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
2576 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cbfree()  argument
2583 		spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2584 		rfcp = rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
2586 			spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2589 		rfp->rcu_fwd_cb_head = rfcp->rfc_next;  in rcu_torture_fwd_prog_cbfree()
2590 		if (!rfp->rcu_fwd_cb_head)  in rcu_torture_fwd_prog_cbfree()
2591 			rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
2592 		spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2606 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,  in rcu_torture_fwd_prog_nr()  argument
2620 	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_nr()
2640 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_nr()
2641 	stopat = rfp->rcu_fwd_startat + dur;  in rcu_torture_fwd_prog_nr()
2660 			 rfp->rcu_fwd_id, dur, cver, gps);  in rcu_torture_fwd_prog_nr()
2665 		pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_nr()
2678 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cr()  argument
2694 	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_cr()
2703 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_cr()
2704 	stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;  in rcu_torture_fwd_prog_cr()
2706 	rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread  in rcu_torture_fwd_prog_cr()
2710 	for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)  in rcu_torture_fwd_prog_cr()
2711 		rfp->n_launders_hist[i].n_launders = 0;  in rcu_torture_fwd_prog_cr()
2714 	rfp->rcu_launder_gp_seq_start = gps;  in rcu_torture_fwd_prog_cr()
2719 		rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);  in rcu_torture_fwd_prog_cr()
2727 			rfp->rcu_fwd_cb_head = rfcpn;  in rcu_torture_fwd_prog_cr()
2739 			rfcp->rfc_rfp = rfp;  in rcu_torture_fwd_prog_cr()
2753 	n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);  in rcu_torture_fwd_prog_cr()
2756 	pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_cr()
2758 	(void)rcu_torture_fwd_prog_cbfree(rfp);  in rcu_torture_fwd_prog_cr()
2765 			 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,  in rcu_torture_fwd_prog_cr()
2771 		rcu_torture_fwd_cb_hist(rfp);  in rcu_torture_fwd_prog_cr()
2789 	struct rcu_fwd *rfp;  in rcutorture_oom_notify()  local
2792 	rfp = rcu_fwds;  in rcutorture_oom_notify()
2793 	if (!rfp) {  in rcutorture_oom_notify()
2800 		rcu_torture_fwd_cb_hist(&rfp[i]);  in rcutorture_oom_notify()
2801 		rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2);  in rcutorture_oom_notify()
2807 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);  in rcutorture_oom_notify()
2812 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);  in rcutorture_oom_notify()
2817 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);  in rcutorture_oom_notify()
2837 	struct rcu_fwd *rfp = args;  in rcu_torture_fwd_prog()  local
2846 		if (!rfp->rcu_fwd_id) {  in rcu_torture_fwd_prog()
2860 		pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog()
2861 		if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id)  in rcu_torture_fwd_prog()
2862 			rcu_torture_fwd_prog_cr(rfp);  in rcu_torture_fwd_prog()
2866 		      torture_num_online_cpus() > rfp->rcu_fwd_id)))  in rcu_torture_fwd_prog()
2867 			rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);  in rcu_torture_fwd_prog()
2874 	if (!rfp->rcu_fwd_id) {  in rcu_torture_fwd_prog()
2887 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_init()  local
2916 	rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL);  in rcu_torture_fwd_prog_init()
2918 	if (!rfp || !fwd_prog_tasks) {  in rcu_torture_fwd_prog_init()
2919 		kfree(rfp);  in rcu_torture_fwd_prog_init()
2926 		spin_lock_init(&rfp[i].rcu_fwd_lock);  in rcu_torture_fwd_prog_init()
2927 		rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head;  in rcu_torture_fwd_prog_init()
2928 		rfp[i].rcu_fwd_id = i;  in rcu_torture_fwd_prog_init()
2931 	rcu_fwds = rfp;  in rcu_torture_fwd_prog_init()
2947 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_cleanup()  local
2955 	rfp = rcu_fwds;  in rcu_torture_fwd_prog_cleanup()
2958 	kfree(rfp);  in rcu_torture_fwd_prog_cleanup()