1807 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)  in rcu_torture_fwd_cb_hist()  argument
1814 	for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)  in rcu_torture_fwd_cb_hist()
1815 		if (rfp->n_launders_hist[i].n_launders > 0)  in rcu_torture_fwd_cb_hist()
1818 		 __func__, jiffies - rfp->rcu_fwd_startat);  in rcu_torture_fwd_cb_hist()
1819 	gps_old = rfp->rcu_launder_gp_seq_start;  in rcu_torture_fwd_cb_hist()
1821 		gps = rfp->n_launders_hist[j].launder_gp_seq;  in rcu_torture_fwd_cb_hist()
1824 			rfp->n_launders_hist[j].n_launders,  in rcu_torture_fwd_cb_hist()
1838 	struct rcu_fwd *rfp = rfcp->rfc_rfp;  in rcu_torture_fwd_cb_cr()  local
1842 	spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
1843 	rfcpp = rfp->rcu_fwd_cb_tail;  in rcu_torture_fwd_cb_cr()
1844 	rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;  in rcu_torture_fwd_cb_cr()
1846 	WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);  in rcu_torture_fwd_cb_cr()
1847 	i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));  in rcu_torture_fwd_cb_cr()
1848 	if (i >= ARRAY_SIZE(rfp->n_launders_hist))  in rcu_torture_fwd_cb_cr()
1849 		i = ARRAY_SIZE(rfp->n_launders_hist) - 1;  in rcu_torture_fwd_cb_cr()
1850 	rfp->n_launders_hist[i].n_launders++;  in rcu_torture_fwd_cb_cr()
1851 	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();  in rcu_torture_fwd_cb_cr()
1852 	spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
1872 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cbfree()  argument
1879 		spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
1880 		rfcp = rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
1882 			spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
1885 		rfp->rcu_fwd_cb_head = rfcp->rfc_next;  in rcu_torture_fwd_prog_cbfree()
1886 		if (!rfp->rcu_fwd_cb_head)  in rcu_torture_fwd_prog_cbfree()
1887 			rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
1888 		spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
1902 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,  in rcu_torture_fwd_prog_nr()  argument
1933 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_nr()
1934 	stopat = rfp->rcu_fwd_startat + dur;  in rcu_torture_fwd_prog_nr()
1969 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cr()  argument
1993 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_cr()
1994 	stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;  in rcu_torture_fwd_prog_cr()
1996 	rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread  in rcu_torture_fwd_prog_cr()
2000 	for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)  in rcu_torture_fwd_prog_cr()
2001 		rfp->n_launders_hist[i].n_launders = 0;  in rcu_torture_fwd_prog_cr()
2004 	rfp->rcu_launder_gp_seq_start = gps;  in rcu_torture_fwd_prog_cr()
2009 		rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);  in rcu_torture_fwd_prog_cr()
2017 			rfp->rcu_fwd_cb_head = rfcpn;  in rcu_torture_fwd_prog_cr()
2029 			rfcp->rfc_rfp = rfp;  in rcu_torture_fwd_prog_cr()
2040 	n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);  in rcu_torture_fwd_prog_cr()
2044 	(void)rcu_torture_fwd_prog_cbfree(rfp);  in rcu_torture_fwd_prog_cr()
2051 			 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,  in rcu_torture_fwd_prog_cr()
2055 		rcu_torture_fwd_cb_hist(rfp);  in rcu_torture_fwd_prog_cr()
2070 	struct rcu_fwd *rfp;  in rcutorture_oom_notify()  local
2073 	rfp = rcu_fwds;  in rcutorture_oom_notify()
2074 	if (!rfp) {  in rcutorture_oom_notify()
2080 	rcu_torture_fwd_cb_hist(rfp);  in rcutorture_oom_notify()
2081 	rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp->rcu_fwd_startat)) / 2);  in rcutorture_oom_notify()
2085 		__func__, rcu_torture_fwd_prog_cbfree(rfp));  in rcutorture_oom_notify()
2088 		__func__, rcu_torture_fwd_prog_cbfree(rfp));  in rcutorture_oom_notify()
2091 		__func__, rcu_torture_fwd_prog_cbfree(rfp));  in rcutorture_oom_notify()
2106 	struct rcu_fwd *rfp = args;  in rcu_torture_fwd_prog()  local
2119 			rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);  in rcu_torture_fwd_prog()
2121 			rcu_torture_fwd_prog_cr(rfp);  in rcu_torture_fwd_prog()
2136 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_init()  local
2156 	rfp = kzalloc(sizeof(*rfp), GFP_KERNEL);  in rcu_torture_fwd_prog_init()
2157 	if (!rfp)  in rcu_torture_fwd_prog_init()
2159 	spin_lock_init(&rfp->rcu_fwd_lock);  in rcu_torture_fwd_prog_init()
2160 	rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_init()
2162 	rcu_fwds = rfp;  in rcu_torture_fwd_prog_init()
2165 	return torture_create_kthread(rcu_torture_fwd_prog, rfp, fwd_prog_task);  in rcu_torture_fwd_prog_init()
2170 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_cleanup()  local
2173 	rfp = rcu_fwds;  in rcu_torture_fwd_prog_cleanup()
2178 	kfree(rfp);  in rcu_torture_fwd_prog_cleanup()