17 typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop);
247 	struct task_struct *t;  in rcu_spawn_tasks_kthread_generic()  local
249 	t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname);  in rcu_spawn_tasks_kthread_generic()
250 …if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavio…  in rcu_spawn_tasks_kthread_generic()
295 static void exit_tasks_rcu_finish_trace(struct task_struct *t);
306 	struct task_struct *g, *t;  in rcu_tasks_wait_gp()  local
322 	for_each_process_thread(g, t)  in rcu_tasks_wait_gp()
323 		rtp->pertask_func(t, &holdouts);  in rcu_tasks_wait_gp()
406 static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop)  in rcu_tasks_pertask()  argument
408 	if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) {  in rcu_tasks_pertask()
409 		get_task_struct(t);  in rcu_tasks_pertask()
410 		t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);  in rcu_tasks_pertask()
411 		WRITE_ONCE(t->rcu_tasks_holdout, true);  in rcu_tasks_pertask()
412 		list_add(&t->rcu_tasks_holdout_list, hop);  in rcu_tasks_pertask()
430 static void check_holdout_task(struct task_struct *t,  in check_holdout_task()  argument
435 	if (!READ_ONCE(t->rcu_tasks_holdout) ||  in check_holdout_task()
436 	    t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) ||  in check_holdout_task()
437 	    !READ_ONCE(t->on_rq) ||  in check_holdout_task()
439 	     !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {  in check_holdout_task()
440 		WRITE_ONCE(t->rcu_tasks_holdout, false);  in check_holdout_task()
441 		list_del_init(&t->rcu_tasks_holdout_list);  in check_holdout_task()
442 		put_task_struct(t);  in check_holdout_task()
445 	rcu_request_urgent_qs_task(t);  in check_holdout_task()
452 	cpu = task_cpu(t);  in check_holdout_task()
454 		 t, ".I"[is_idle_task(t)],  in check_holdout_task()
456 		 t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout,  in check_holdout_task()
457 		 t->rcu_tasks_idle_cpu, cpu);  in check_holdout_task()
458 	sched_show_task(t);  in check_holdout_task()
465 	struct task_struct *t, *t1;  in check_all_holdout_tasks()  local
467 	list_for_each_entry_safe(t, t1, hop, rcu_tasks_holdout_list) {  in check_all_holdout_tasks()
468 		check_holdout_task(t, needreport, firstreport);  in check_all_holdout_tasks()
592 	struct task_struct *t = current;  in exit_tasks_rcu_finish()  local
595 	__srcu_read_unlock(&tasks_rcu_exit_srcu, t->rcu_tasks_idx);  in exit_tasks_rcu_finish()
597 	exit_tasks_rcu_finish_trace(t);  in exit_tasks_rcu_finish()
776 void rcu_read_unlock_trace_special(struct task_struct *t, int nesting)  in rcu_read_unlock_trace_special()  argument
778 	int nq = t->trc_reader_special.b.need_qs;  in rcu_read_unlock_trace_special()
781 	    t->trc_reader_special.b.need_mb)  in rcu_read_unlock_trace_special()
785 		WRITE_ONCE(t->trc_reader_special.b.need_qs, false);  in rcu_read_unlock_trace_special()
786 	WRITE_ONCE(t->trc_reader_nesting, nesting);  in rcu_read_unlock_trace_special()
793 static void trc_add_holdout(struct task_struct *t, struct list_head *bhp)  in trc_add_holdout()  argument
795 	if (list_empty(&t->trc_holdout_list)) {  in trc_add_holdout()
796 		get_task_struct(t);  in trc_add_holdout()
797 		list_add(&t->trc_holdout_list, bhp);  in trc_add_holdout()
802 static void trc_del_holdout(struct task_struct *t)  in trc_del_holdout()  argument
804 	if (!list_empty(&t->trc_holdout_list)) {  in trc_del_holdout()
805 		list_del_init(&t->trc_holdout_list);  in trc_del_holdout()
806 		put_task_struct(t);  in trc_del_holdout()
813 	struct task_struct *t = current;  in trc_read_check_handler()  local
817 	if (unlikely(texp != t)) {  in trc_read_check_handler()
825 	if (likely(!t->trc_reader_nesting)) {  in trc_read_check_handler()
830 		WRITE_ONCE(t->trc_reader_checked, true);  in trc_read_check_handler()
834 	if (unlikely(t->trc_reader_nesting < 0)) {  in trc_read_check_handler()
839 	WRITE_ONCE(t->trc_reader_checked, true);  in trc_read_check_handler()
844 	WARN_ON_ONCE(t->trc_reader_special.b.need_qs);  in trc_read_check_handler()
845 	WRITE_ONCE(t->trc_reader_special.b.need_qs, true);  in trc_read_check_handler()
856 static bool trc_inspect_reader(struct task_struct *t, void *arg)  in trc_inspect_reader()  argument
858 	int cpu = task_cpu(t);  in trc_inspect_reader()
862 	if (task_curr(t)) {  in trc_inspect_reader()
863 		WARN_ON_ONCE(ofl && !is_idle_task(t));  in trc_inspect_reader()
874 		    !rcu_dynticks_zero_in_eqs(cpu, &t->trc_reader_nesting))  in trc_inspect_reader()
881 		in_qs = likely(!t->trc_reader_nesting);  in trc_inspect_reader()
886 	t->trc_reader_checked = true;  in trc_inspect_reader()
887 	trc_del_holdout(t);  in trc_inspect_reader()
896 	WARN_ON_ONCE(t->trc_reader_special.b.need_qs);  in trc_inspect_reader()
897 	WRITE_ONCE(t->trc_reader_special.b.need_qs, true);  in trc_inspect_reader()
902 static void trc_wait_for_one_reader(struct task_struct *t,  in trc_wait_for_one_reader()  argument
908 	if (smp_load_acquire(&t->trc_ipi_to_cpu) != -1) // Order IPI  in trc_wait_for_one_reader()
912 	if (t == current) {  in trc_wait_for_one_reader()
913 		t->trc_reader_checked = true;  in trc_wait_for_one_reader()
914 		trc_del_holdout(t);  in trc_wait_for_one_reader()
915 		WARN_ON_ONCE(t->trc_reader_nesting);  in trc_wait_for_one_reader()
920 	get_task_struct(t);  in trc_wait_for_one_reader()
921 	if (try_invoke_on_locked_down_task(t, trc_inspect_reader, NULL)) {  in trc_wait_for_one_reader()
922 		put_task_struct(t);  in trc_wait_for_one_reader()
925 	put_task_struct(t);  in trc_wait_for_one_reader()
928 	trc_add_holdout(t, bhp);  in trc_wait_for_one_reader()
929 	if (task_curr(t) &&  in trc_wait_for_one_reader()
932 		cpu = task_cpu(t);  in trc_wait_for_one_reader()
935 		if (per_cpu(trc_ipi_to_cpu, cpu) || t->trc_ipi_to_cpu >= 0)  in trc_wait_for_one_reader()
940 		t->trc_ipi_to_cpu = cpu;  in trc_wait_for_one_reader()
943 					     trc_read_check_handler, t, 0)) {  in trc_wait_for_one_reader()
948 			t->trc_ipi_to_cpu = cpu;  in trc_wait_for_one_reader()
975 static void rcu_tasks_trace_pertask(struct task_struct *t,  in rcu_tasks_trace_pertask()  argument
978 	WRITE_ONCE(t->trc_reader_special.b.need_qs, false);  in rcu_tasks_trace_pertask()
979 	WRITE_ONCE(t->trc_reader_checked, false);  in rcu_tasks_trace_pertask()
980 	t->trc_ipi_to_cpu = -1;  in rcu_tasks_trace_pertask()
981 	trc_wait_for_one_reader(t, hop);  in rcu_tasks_trace_pertask()
1005 static void show_stalled_task_trace(struct task_struct *t, bool *firstreport)  in show_stalled_task_trace()  argument
1014 	cpu = task_cpu(t);  in show_stalled_task_trace()
1016 		 t->pid,  in show_stalled_task_trace()
1017 		 ".I"[READ_ONCE(t->trc_ipi_to_cpu) > 0],  in show_stalled_task_trace()
1018 		 ".i"[is_idle_task(t)],  in show_stalled_task_trace()
1020 		 t->trc_reader_nesting,  in show_stalled_task_trace()
1021 		 " N"[!!t->trc_reader_special.b.need_qs],  in show_stalled_task_trace()
1023 	sched_show_task(t);  in show_stalled_task_trace()
1040 	struct task_struct *g, *t;  in check_all_holdout_tasks_trace()  local
1045 	list_for_each_entry_safe(t, g, hop, trc_holdout_list) {  in check_all_holdout_tasks_trace()
1047 		if (READ_ONCE(t->trc_ipi_to_cpu) == -1 &&  in check_all_holdout_tasks_trace()
1048 		    !READ_ONCE(t->trc_reader_checked))  in check_all_holdout_tasks_trace()
1049 			trc_wait_for_one_reader(t, hop);  in check_all_holdout_tasks_trace()
1052 		if (READ_ONCE(t->trc_reader_checked))  in check_all_holdout_tasks_trace()
1053 			trc_del_holdout(t);  in check_all_holdout_tasks_trace()
1055 			show_stalled_task_trace(t, firstreport);  in check_all_holdout_tasks_trace()
1072 	struct task_struct *g, *t;  in rcu_tasks_trace_postgp()  local
1092 		for_each_process_thread(g, t)  in rcu_tasks_trace_postgp()
1093 			if (READ_ONCE(t->trc_reader_special.b.need_qs))  in rcu_tasks_trace_postgp()
1094 				trc_add_holdout(t, &holdouts);  in rcu_tasks_trace_postgp()
1097 		list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) {  in rcu_tasks_trace_postgp()
1098 			if (READ_ONCE(t->trc_reader_special.b.need_qs))  in rcu_tasks_trace_postgp()
1099 				show_stalled_task_trace(t, &firstreport);  in rcu_tasks_trace_postgp()
1100 			trc_del_holdout(t); // Release task_struct reference.  in rcu_tasks_trace_postgp()
1112 static void exit_tasks_rcu_finish_trace(struct task_struct *t)  in exit_tasks_rcu_finish_trace()  argument
1114 	WRITE_ONCE(t->trc_reader_checked, true);  in exit_tasks_rcu_finish_trace()
1115 	WARN_ON_ONCE(t->trc_reader_nesting);  in exit_tasks_rcu_finish_trace()
1116 	WRITE_ONCE(t->trc_reader_nesting, 0);  in exit_tasks_rcu_finish_trace()
1117 	if (WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)))  in exit_tasks_rcu_finish_trace()
1118 		rcu_read_unlock_trace_special(t, 0);  in exit_tasks_rcu_finish_trace()
1219 static void exit_tasks_rcu_finish_trace(struct task_struct *t) { }  in exit_tasks_rcu_finish_trace()  argument