17 typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop);
247 	struct task_struct *t;  in rcu_spawn_tasks_kthread_generic()  local
249 	t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname);  in rcu_spawn_tasks_kthread_generic()
250 …if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavio…  in rcu_spawn_tasks_kthread_generic()
295 static void exit_tasks_rcu_finish_trace(struct task_struct *t);
306 	struct task_struct *g, *t;  in rcu_tasks_wait_gp()  local
322 	for_each_process_thread(g, t)  in rcu_tasks_wait_gp()
323 		rtp->pertask_func(t, &holdouts);  in rcu_tasks_wait_gp()
441 static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop)  in rcu_tasks_pertask()  argument
443 	if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) {  in rcu_tasks_pertask()
444 		get_task_struct(t);  in rcu_tasks_pertask()
445 		t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);  in rcu_tasks_pertask()
446 		WRITE_ONCE(t->rcu_tasks_holdout, true);  in rcu_tasks_pertask()
447 		list_add(&t->rcu_tasks_holdout_list, hop);  in rcu_tasks_pertask()
465 static void check_holdout_task(struct task_struct *t,  in check_holdout_task()  argument
470 	if (!READ_ONCE(t->rcu_tasks_holdout) ||  in check_holdout_task()
471 	    t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) ||  in check_holdout_task()
472 	    !READ_ONCE(t->on_rq) ||  in check_holdout_task()
474 	     !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {  in check_holdout_task()
475 		WRITE_ONCE(t->rcu_tasks_holdout, false);  in check_holdout_task()
476 		list_del_init(&t->rcu_tasks_holdout_list);  in check_holdout_task()
477 		put_task_struct(t);  in check_holdout_task()
480 	rcu_request_urgent_qs_task(t);  in check_holdout_task()
487 	cpu = task_cpu(t);  in check_holdout_task()
489 		 t, ".I"[is_idle_task(t)],  in check_holdout_task()
491 		 t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout,  in check_holdout_task()
492 		 t->rcu_tasks_idle_cpu, cpu);  in check_holdout_task()
493 	sched_show_task(t);  in check_holdout_task()
500 	struct task_struct *t, *t1;  in check_all_holdout_tasks()  local
502 	list_for_each_entry_safe(t, t1, hop, rcu_tasks_holdout_list) {  in check_all_holdout_tasks()
503 		check_holdout_task(t, needreport, firstreport);  in check_all_holdout_tasks()
627 	struct task_struct *t = current;  in exit_tasks_rcu_finish()  local
630 	__srcu_read_unlock(&tasks_rcu_exit_srcu, t->rcu_tasks_idx);  in exit_tasks_rcu_finish()
632 	exit_tasks_rcu_finish_trace(t);  in exit_tasks_rcu_finish()
851 void rcu_read_unlock_trace_special(struct task_struct *t, int nesting)  in rcu_read_unlock_trace_special()  argument
853 	int nq = READ_ONCE(t->trc_reader_special.b.need_qs);  in rcu_read_unlock_trace_special()
856 	    t->trc_reader_special.b.need_mb)  in rcu_read_unlock_trace_special()
860 		WRITE_ONCE(t->trc_reader_special.b.need_qs, false);  in rcu_read_unlock_trace_special()
861 	WRITE_ONCE(t->trc_reader_nesting, nesting);  in rcu_read_unlock_trace_special()
868 static void trc_add_holdout(struct task_struct *t, struct list_head *bhp)  in trc_add_holdout()  argument
870 	if (list_empty(&t->trc_holdout_list)) {  in trc_add_holdout()
871 		get_task_struct(t);  in trc_add_holdout()
872 		list_add(&t->trc_holdout_list, bhp);  in trc_add_holdout()
877 static void trc_del_holdout(struct task_struct *t)  in trc_del_holdout()  argument
879 	if (!list_empty(&t->trc_holdout_list)) {  in trc_del_holdout()
880 		list_del_init(&t->trc_holdout_list);  in trc_del_holdout()
881 		put_task_struct(t);  in trc_del_holdout()
888 	struct task_struct *t = current;  in trc_read_check_handler()  local
892 	if (unlikely(texp != t)) {  in trc_read_check_handler()
900 	if (likely(!READ_ONCE(t->trc_reader_nesting))) {  in trc_read_check_handler()
905 		WRITE_ONCE(t->trc_reader_checked, true);  in trc_read_check_handler()
909 	if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) {  in trc_read_check_handler()
914 	WRITE_ONCE(t->trc_reader_checked, true);  in trc_read_check_handler()
919 	WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs));  in trc_read_check_handler()
920 	WRITE_ONCE(t->trc_reader_special.b.need_qs, true);  in trc_read_check_handler()
931 static bool trc_inspect_reader(struct task_struct *t, void *arg)  in trc_inspect_reader()  argument
933 	int cpu = task_cpu(t);  in trc_inspect_reader()
937 	if (task_curr(t)) {  in trc_inspect_reader()
938 		WARN_ON_ONCE(ofl && !is_idle_task(t));  in trc_inspect_reader()
949 		    !rcu_dynticks_zero_in_eqs(cpu, &t->trc_reader_nesting))  in trc_inspect_reader()
957 		in_qs = likely(!t->trc_reader_nesting);  in trc_inspect_reader()
962 	t->trc_reader_checked = true;  in trc_inspect_reader()
971 	WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs));  in trc_inspect_reader()
972 	WRITE_ONCE(t->trc_reader_special.b.need_qs, true);  in trc_inspect_reader()
977 static void trc_wait_for_one_reader(struct task_struct *t,  in trc_wait_for_one_reader()  argument
983 	if (smp_load_acquire(&t->trc_ipi_to_cpu) != -1) // Order IPI  in trc_wait_for_one_reader()
987 	if (t == current) {  in trc_wait_for_one_reader()
988 		t->trc_reader_checked = true;  in trc_wait_for_one_reader()
989 		WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting));  in trc_wait_for_one_reader()
994 	get_task_struct(t);  in trc_wait_for_one_reader()
995 	if (try_invoke_on_locked_down_task(t, trc_inspect_reader, NULL)) {  in trc_wait_for_one_reader()
996 		put_task_struct(t);  in trc_wait_for_one_reader()
999 	put_task_struct(t);  in trc_wait_for_one_reader()
1008 	trc_add_holdout(t, bhp);  in trc_wait_for_one_reader()
1009 	if (task_curr(t) &&  in trc_wait_for_one_reader()
1012 		cpu = task_cpu(t);  in trc_wait_for_one_reader()
1015 		if (per_cpu(trc_ipi_to_cpu, cpu) || t->trc_ipi_to_cpu >= 0)  in trc_wait_for_one_reader()
1020 		t->trc_ipi_to_cpu = cpu;  in trc_wait_for_one_reader()
1023 					     trc_read_check_handler, t, 0)) {  in trc_wait_for_one_reader()
1028 			t->trc_ipi_to_cpu = cpu;  in trc_wait_for_one_reader()
1055 static void rcu_tasks_trace_pertask(struct task_struct *t,  in rcu_tasks_trace_pertask()  argument
1060 	if (unlikely(t == NULL))  in rcu_tasks_trace_pertask()
1063 	WRITE_ONCE(t->trc_reader_special.b.need_qs, false);  in rcu_tasks_trace_pertask()
1064 	WRITE_ONCE(t->trc_reader_checked, false);  in rcu_tasks_trace_pertask()
1065 	t->trc_ipi_to_cpu = -1;  in rcu_tasks_trace_pertask()
1066 	trc_wait_for_one_reader(t, hop);  in rcu_tasks_trace_pertask()
1090 static void show_stalled_task_trace(struct task_struct *t, bool *firstreport)  in show_stalled_task_trace()  argument
1099 	cpu = task_cpu(t);  in show_stalled_task_trace()
1101 		 t->pid,  in show_stalled_task_trace()
1102 		 ".I"[READ_ONCE(t->trc_ipi_to_cpu) > 0],  in show_stalled_task_trace()
1103 		 ".i"[is_idle_task(t)],  in show_stalled_task_trace()
1105 		 READ_ONCE(t->trc_reader_nesting),  in show_stalled_task_trace()
1106 		 " N"[!!READ_ONCE(t->trc_reader_special.b.need_qs)],  in show_stalled_task_trace()
1108 	sched_show_task(t);  in show_stalled_task_trace()
1125 	struct task_struct *g, *t;  in check_all_holdout_tasks_trace()  local
1130 	list_for_each_entry_safe(t, g, hop, trc_holdout_list) {  in check_all_holdout_tasks_trace()
1132 		if (READ_ONCE(t->trc_ipi_to_cpu) == -1 &&  in check_all_holdout_tasks_trace()
1133 		    !READ_ONCE(t->trc_reader_checked))  in check_all_holdout_tasks_trace()
1134 			trc_wait_for_one_reader(t, hop);  in check_all_holdout_tasks_trace()
1137 		if (READ_ONCE(t->trc_reader_checked))  in check_all_holdout_tasks_trace()
1138 			trc_del_holdout(t);  in check_all_holdout_tasks_trace()
1140 			show_stalled_task_trace(t, firstreport);  in check_all_holdout_tasks_trace()
1157 	struct task_struct *g, *t;  in rcu_tasks_trace_postgp()  local
1177 		for_each_process_thread(g, t)  in rcu_tasks_trace_postgp()
1178 			if (READ_ONCE(t->trc_reader_special.b.need_qs))  in rcu_tasks_trace_postgp()
1179 				trc_add_holdout(t, &holdouts);  in rcu_tasks_trace_postgp()
1182 		list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) {  in rcu_tasks_trace_postgp()
1183 			if (READ_ONCE(t->trc_reader_special.b.need_qs))  in rcu_tasks_trace_postgp()
1184 				show_stalled_task_trace(t, &firstreport);  in rcu_tasks_trace_postgp()
1185 			trc_del_holdout(t); // Release task_struct reference.  in rcu_tasks_trace_postgp()
1197 static void exit_tasks_rcu_finish_trace(struct task_struct *t)  in exit_tasks_rcu_finish_trace()  argument
1199 	WRITE_ONCE(t->trc_reader_checked, true);  in exit_tasks_rcu_finish_trace()
1200 	WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting));  in exit_tasks_rcu_finish_trace()
1201 	WRITE_ONCE(t->trc_reader_nesting, 0);  in exit_tasks_rcu_finish_trace()
1202 	if (WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)))  in exit_tasks_rcu_finish_trace()
1203 		rcu_read_unlock_trace_special(t, 0);  in exit_tasks_rcu_finish_trace()
1304 static void exit_tasks_rcu_finish_trace(struct task_struct *t) { }  in exit_tasks_rcu_finish_trace()  argument