2108 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)  in rcu_torture_fwd_cb_hist()  argument
2115 	for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)  in rcu_torture_fwd_cb_hist()
2116 		if (rfp->n_launders_hist[i].n_launders > 0)  in rcu_torture_fwd_cb_hist()
2119 		 __func__, jiffies - rfp->rcu_fwd_startat);  in rcu_torture_fwd_cb_hist()
2120 	gps_old = rfp->rcu_launder_gp_seq_start;  in rcu_torture_fwd_cb_hist()
2122 		gps = rfp->n_launders_hist[j].launder_gp_seq;  in rcu_torture_fwd_cb_hist()
2125 			rfp->n_launders_hist[j].n_launders,  in rcu_torture_fwd_cb_hist()
2139 	struct rcu_fwd *rfp = rfcp->rfc_rfp;  in rcu_torture_fwd_cb_cr()  local
2143 	spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
2144 	rfcpp = rfp->rcu_fwd_cb_tail;  in rcu_torture_fwd_cb_cr()
2145 	rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;  in rcu_torture_fwd_cb_cr()
2147 	WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);  in rcu_torture_fwd_cb_cr()
2148 	i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));  in rcu_torture_fwd_cb_cr()
2149 	if (i >= ARRAY_SIZE(rfp->n_launders_hist))  in rcu_torture_fwd_cb_cr()
2150 		i = ARRAY_SIZE(rfp->n_launders_hist) - 1;  in rcu_torture_fwd_cb_cr()
2151 	rfp->n_launders_hist[i].n_launders++;  in rcu_torture_fwd_cb_cr()
2152 	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();  in rcu_torture_fwd_cb_cr()
2153 	spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_cb_cr()
2173 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cbfree()  argument
2180 		spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2181 		rfcp = rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
2183 			spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2186 		rfp->rcu_fwd_cb_head = rfcp->rfc_next;  in rcu_torture_fwd_prog_cbfree()
2187 		if (!rfp->rcu_fwd_cb_head)  in rcu_torture_fwd_prog_cbfree()
2188 			rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_cbfree()
2189 		spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);  in rcu_torture_fwd_prog_cbfree()
2203 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,  in rcu_torture_fwd_prog_nr()  argument
2236 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_nr()
2237 	stopat = rfp->rcu_fwd_startat + dur;  in rcu_torture_fwd_prog_nr()
2272 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)  in rcu_torture_fwd_prog_cr()  argument
2296 	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);  in rcu_torture_fwd_prog_cr()
2297 	stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;  in rcu_torture_fwd_prog_cr()
2299 	rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread  in rcu_torture_fwd_prog_cr()
2303 	for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)  in rcu_torture_fwd_prog_cr()
2304 		rfp->n_launders_hist[i].n_launders = 0;  in rcu_torture_fwd_prog_cr()
2307 	rfp->rcu_launder_gp_seq_start = gps;  in rcu_torture_fwd_prog_cr()
2312 		rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);  in rcu_torture_fwd_prog_cr()
2320 			rfp->rcu_fwd_cb_head = rfcpn;  in rcu_torture_fwd_prog_cr()
2332 			rfcp->rfc_rfp = rfp;  in rcu_torture_fwd_prog_cr()
2343 	n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);  in rcu_torture_fwd_prog_cr()
2347 	(void)rcu_torture_fwd_prog_cbfree(rfp);  in rcu_torture_fwd_prog_cr()
2354 			 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,  in rcu_torture_fwd_prog_cr()
2358 		rcu_torture_fwd_cb_hist(rfp);  in rcu_torture_fwd_prog_cr()
2373 	struct rcu_fwd *rfp;  in rcutorture_oom_notify()  local
2376 	rfp = rcu_fwds;  in rcutorture_oom_notify()
2377 	if (!rfp) {  in rcutorture_oom_notify()
2383 	rcu_torture_fwd_cb_hist(rfp);  in rcutorture_oom_notify()
2384 	rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp->rcu_fwd_startat)) / 2);  in rcutorture_oom_notify()
2388 		__func__, rcu_torture_fwd_prog_cbfree(rfp));  in rcutorture_oom_notify()
2391 		__func__, rcu_torture_fwd_prog_cbfree(rfp));  in rcutorture_oom_notify()
2394 		__func__, rcu_torture_fwd_prog_cbfree(rfp));  in rcutorture_oom_notify()
2410 	struct rcu_fwd *rfp = args;  in rcu_torture_fwd_prog()  local
2423 			rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);  in rcu_torture_fwd_prog()
2425 			rcu_torture_fwd_prog_cr(rfp);  in rcu_torture_fwd_prog()
2441 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_init()  local
2461 	rfp = kzalloc(sizeof(*rfp), GFP_KERNEL);  in rcu_torture_fwd_prog_init()
2462 	if (!rfp)  in rcu_torture_fwd_prog_init()
2464 	spin_lock_init(&rfp->rcu_fwd_lock);  in rcu_torture_fwd_prog_init()
2465 	rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;  in rcu_torture_fwd_prog_init()
2467 	rcu_fwds = rfp;  in rcu_torture_fwd_prog_init()
2470 	return torture_create_kthread(rcu_torture_fwd_prog, rfp, fwd_prog_task);  in rcu_torture_fwd_prog_init()
2475 	struct rcu_fwd *rfp;  in rcu_torture_fwd_prog_cleanup()  local
2478 	rfp = rcu_fwds;  in rcu_torture_fwd_prog_cleanup()
2483 	kfree(rfp);  in rcu_torture_fwd_prog_cleanup()