1524 static void task_numa_assign(struct task_numa_env *env,  in task_numa_assign()  argument
1527 	struct rq *rq = cpu_rq(env->dst_cpu);  in task_numa_assign()
1537 	if (env->best_cpu != -1) {  in task_numa_assign()
1538 		rq = cpu_rq(env->best_cpu);  in task_numa_assign()
1542 	if (env->best_task)  in task_numa_assign()
1543 		put_task_struct(env->best_task);  in task_numa_assign()
1547 	env->best_task = p;  in task_numa_assign()
1548 	env->best_imp = imp;  in task_numa_assign()
1549 	env->best_cpu = env->dst_cpu;  in task_numa_assign()
1553 				struct task_numa_env *env)  in load_too_imbalanced()  argument
1566 	src_capacity = env->src_stats.compute_capacity;  in load_too_imbalanced()
1567 	dst_capacity = env->dst_stats.compute_capacity;  in load_too_imbalanced()
1571 	orig_src_load = env->src_stats.load;  in load_too_imbalanced()
1572 	orig_dst_load = env->dst_stats.load;  in load_too_imbalanced()
1593 static void task_numa_compare(struct task_numa_env *env,  in task_numa_compare()  argument
1596 	struct rq *dst_rq = cpu_rq(env->dst_cpu);  in task_numa_compare()
1600 	long imp = env->p->numa_group ? groupimp : taskimp;  in task_numa_compare()
1602 	int dist = env->dist;  in task_numa_compare()
1616 	if (cur == env->p)  in task_numa_compare()
1620 		if (maymove && moveimp >= env->best_imp)  in task_numa_compare()
1634 	if (!cpumask_test_cpu(env->src_cpu, &cur->cpus_allowed))  in task_numa_compare()
1641 	if (cur->numa_group == env->p->numa_group) {  in task_numa_compare()
1642 		imp = taskimp + task_weight(cur, env->src_nid, dist) -  in task_numa_compare()
1643 		      task_weight(cur, env->dst_nid, dist);  in task_numa_compare()
1655 		if (cur->numa_group && env->p->numa_group)  in task_numa_compare()
1656 			imp += group_weight(cur, env->src_nid, dist) -  in task_numa_compare()
1657 			       group_weight(cur, env->dst_nid, dist);  in task_numa_compare()
1659 			imp += task_weight(cur, env->src_nid, dist) -  in task_numa_compare()
1660 			       task_weight(cur, env->dst_nid, dist);  in task_numa_compare()
1663 	if (maymove && moveimp > imp && moveimp > env->best_imp) {  in task_numa_compare()
1675 	if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)  in task_numa_compare()
1681 	load = task_h_load(env->p) - task_h_load(cur);  in task_numa_compare()
1685 	dst_load = env->dst_stats.load + load;  in task_numa_compare()
1686 	src_load = env->src_stats.load - load;  in task_numa_compare()
1688 	if (load_too_imbalanced(src_load, dst_load, env))  in task_numa_compare()
1702 		env->dst_cpu = select_idle_sibling(env->p, env->src_cpu,  in task_numa_compare()
1703 						   env->dst_cpu);  in task_numa_compare()
1707 	task_numa_assign(env, cur, imp);  in task_numa_compare()
1712 static void task_numa_find_cpu(struct task_numa_env *env,  in task_numa_find_cpu()  argument
1719 	load = task_h_load(env->p);  in task_numa_find_cpu()
1720 	dst_load = env->dst_stats.load + load;  in task_numa_find_cpu()
1721 	src_load = env->src_stats.load - load;  in task_numa_find_cpu()
1727 	maymove = !load_too_imbalanced(src_load, dst_load, env);  in task_numa_find_cpu()
1729 	for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {  in task_numa_find_cpu()
1731 		if (!cpumask_test_cpu(cpu, &env->p->cpus_allowed))  in task_numa_find_cpu()
1734 		env->dst_cpu = cpu;  in task_numa_find_cpu()
1735 		task_numa_compare(env, taskimp, groupimp, maymove);  in task_numa_find_cpu()
1741 	struct task_numa_env env = {  in task_numa_migrate()  local
1768 	sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));  in task_numa_migrate()
1770 		env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;  in task_numa_migrate()
1784 	env.dst_nid = p->numa_preferred_nid;  in task_numa_migrate()
1785 	dist = env.dist = node_distance(env.src_nid, env.dst_nid);  in task_numa_migrate()
1786 	taskweight = task_weight(p, env.src_nid, dist);  in task_numa_migrate()
1787 	groupweight = group_weight(p, env.src_nid, dist);  in task_numa_migrate()
1788 	update_numa_stats(&env.src_stats, env.src_nid);  in task_numa_migrate()
1789 	taskimp = task_weight(p, env.dst_nid, dist) - taskweight;  in task_numa_migrate()
1790 	groupimp = group_weight(p, env.dst_nid, dist) - groupweight;  in task_numa_migrate()
1791 	update_numa_stats(&env.dst_stats, env.dst_nid);  in task_numa_migrate()
1794 	task_numa_find_cpu(&env, taskimp, groupimp);  in task_numa_migrate()
1803 	if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) {  in task_numa_migrate()
1805 			if (nid == env.src_nid || nid == p->numa_preferred_nid)  in task_numa_migrate()
1808 			dist = node_distance(env.src_nid, env.dst_nid);  in task_numa_migrate()
1810 						dist != env.dist) {  in task_numa_migrate()
1811 				taskweight = task_weight(p, env.src_nid, dist);  in task_numa_migrate()
1812 				groupweight = group_weight(p, env.src_nid, dist);  in task_numa_migrate()
1821 			env.dist = dist;  in task_numa_migrate()
1822 			env.dst_nid = nid;  in task_numa_migrate()
1823 			update_numa_stats(&env.dst_stats, env.dst_nid);  in task_numa_migrate()
1824 			task_numa_find_cpu(&env, taskimp, groupimp);  in task_numa_migrate()
1837 		if (env.best_cpu == -1)  in task_numa_migrate()
1838 			nid = env.src_nid;  in task_numa_migrate()
1840 			nid = cpu_to_node(env.best_cpu);  in task_numa_migrate()
1847 	if (env.best_cpu == -1)  in task_numa_migrate()
1850 	best_rq = cpu_rq(env.best_cpu);  in task_numa_migrate()
1851 	if (env.best_task == NULL) {  in task_numa_migrate()
1852 		ret = migrate_task_to(p, env.best_cpu);  in task_numa_migrate()
1855 			trace_sched_stick_numa(p, env.src_cpu, env.best_cpu);  in task_numa_migrate()
1859 	ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);  in task_numa_migrate()
1863 		trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task));  in task_numa_migrate()
1864 	put_task_struct(env.best_task);  in task_numa_migrate()
6956 static int task_hot(struct task_struct *p, struct lb_env *env)  in task_hot()  argument
6960 	lockdep_assert_held(&env->src_rq->lock);  in task_hot()
6971 	if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&  in task_hot()
6981 	delta = rq_clock_task(env->src_rq) - p->se.exec_start;  in task_hot()
6992 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)  in migrate_degrades_locality()  argument
7001 	if (!p->numa_faults || !(env->sd->flags & SD_NUMA))  in migrate_degrades_locality()
7004 	src_nid = cpu_to_node(env->src_cpu);  in migrate_degrades_locality()
7005 	dst_nid = cpu_to_node(env->dst_cpu);  in migrate_degrades_locality()
7012 		if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)  in migrate_degrades_locality()
7023 	if (env->idle == CPU_IDLE)  in migrate_degrades_locality()
7040 					     struct lb_env *env)  in migrate_degrades_locality()  argument
7050 int can_migrate_task(struct task_struct *p, struct lb_env *env)  in can_migrate_task()  argument
7054 	lockdep_assert_held(&env->src_rq->lock);  in can_migrate_task()
7063 	if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))  in can_migrate_task()
7066 	if (!cpumask_test_cpu(env->dst_cpu, &p->cpus_allowed)) {  in can_migrate_task()
7071 		env->flags |= LBF_SOME_PINNED;  in can_migrate_task()
7081 		if (env->idle == CPU_NEWLY_IDLE || (env->flags & LBF_DST_PINNED))  in can_migrate_task()
7085 		for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {  in can_migrate_task()
7087 				env->flags |= LBF_DST_PINNED;  in can_migrate_task()
7088 				env->new_dst_cpu = cpu;  in can_migrate_task()
7097 	env->flags &= ~LBF_ALL_PINNED;  in can_migrate_task()
7099 	if (task_running(env->src_rq, p)) {  in can_migrate_task()
7110 	tsk_cache_hot = migrate_degrades_locality(p, env);  in can_migrate_task()
7112 		tsk_cache_hot = task_hot(p, env);  in can_migrate_task()
7115 	    env->sd->nr_balance_failed > env->sd->cache_nice_tries) {  in can_migrate_task()
7117 			schedstat_inc(env->sd->lb_hot_gained[env->idle]);  in can_migrate_task()
7130 static void detach_task(struct task_struct *p, struct lb_env *env)  in detach_task()  argument
7132 	lockdep_assert_held(&env->src_rq->lock);  in detach_task()
7135 	deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);  in detach_task()
7136 	set_task_cpu(p, env->dst_cpu);  in detach_task()
7145 static struct task_struct *detach_one_task(struct lb_env *env)  in detach_one_task()  argument
7149 	lockdep_assert_held(&env->src_rq->lock);  in detach_one_task()
7152 			&env->src_rq->cfs_tasks, se.group_node) {  in detach_one_task()
7153 		if (!can_migrate_task(p, env))  in detach_one_task()
7156 		detach_task(p, env);  in detach_one_task()
7164 		schedstat_inc(env->sd->lb_gained[env->idle]);  in detach_one_task()
7178 static int detach_tasks(struct lb_env *env)  in detach_tasks()  argument
7180 	struct list_head *tasks = &env->src_rq->cfs_tasks;  in detach_tasks()
7185 	lockdep_assert_held(&env->src_rq->lock);  in detach_tasks()
7187 	if (env->imbalance <= 0)  in detach_tasks()
7195 		if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)  in detach_tasks()
7200 		env->loop++;  in detach_tasks()
7202 		if (env->loop > env->loop_max)  in detach_tasks()
7206 		if (env->loop > env->loop_break) {  in detach_tasks()
7207 			env->loop_break += sched_nr_migrate_break;  in detach_tasks()
7208 			env->flags |= LBF_NEED_BREAK;  in detach_tasks()
7212 		if (!can_migrate_task(p, env))  in detach_tasks()
7217 		if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed)  in detach_tasks()
7220 		if ((load / 2) > env->imbalance)  in detach_tasks()
7223 		detach_task(p, env);  in detach_tasks()
7224 		list_add(&p->se.group_node, &env->tasks);  in detach_tasks()
7227 		env->imbalance -= load;  in detach_tasks()
7235 		if (env->idle == CPU_NEWLY_IDLE)  in detach_tasks()
7243 		if (env->imbalance <= 0)  in detach_tasks()
7256 	schedstat_add(env->sd->lb_gained[env->idle], detached);  in detach_tasks()
7292 static void attach_tasks(struct lb_env *env)  in attach_tasks()  argument
7294 	struct list_head *tasks = &env->tasks;  in attach_tasks()
7298 	rq_lock(env->dst_rq, &rf);  in attach_tasks()
7299 	update_rq_clock(env->dst_rq);  in attach_tasks()
7305 		attach_task(env->dst_rq, p);  in attach_tasks()
7308 	rq_unlock(env->dst_rq, &rf);  in attach_tasks()
7752 group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs)  in group_has_capacity()  argument
7758 			(sgs->group_util * env->sd->imbalance_pct))  in group_has_capacity()
7773 group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs)  in group_is_overloaded()  argument
7779 			(sgs->group_util * env->sd->imbalance_pct))  in group_is_overloaded()
7840 static inline void update_sg_lb_stats(struct lb_env *env,  in update_sg_lb_stats()  argument
7850 	for_each_cpu_and(i, sched_group_span(group), env->cpus) {  in update_sg_lb_stats()
7853 		if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false))  in update_sg_lb_stats()
7854 			env->flags |= LBF_NOHZ_AGAIN;  in update_sg_lb_stats()
7891 	sgs->group_no_capacity = group_is_overloaded(env, sgs);  in update_sg_lb_stats()
7908 static bool update_sd_pick_busiest(struct lb_env *env,  in update_sd_pick_busiest()  argument
7924 	if (!(env->sd->flags & SD_ASYM_CPUCAPACITY))  in update_sd_pick_busiest()
7939 	if (!(env->sd->flags & SD_ASYM_PACKING))  in update_sd_pick_busiest()
7943 	if (env->idle == CPU_NOT_IDLE)  in update_sd_pick_busiest()
7951 	    sched_asym_prefer(env->dst_cpu, sg->asym_prefer_cpu)) {  in update_sd_pick_busiest()
7999 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)  in update_sd_lb_stats()  argument
8001 	struct sched_domain *child = env->sd->child;  in update_sd_lb_stats()
8002 	struct sched_group *sg = env->sd->groups;  in update_sd_lb_stats()
8012 	if (env->idle == CPU_NEWLY_IDLE && READ_ONCE(nohz.has_blocked))  in update_sd_lb_stats()
8013 		env->flags |= LBF_NOHZ_STATS;  in update_sd_lb_stats()
8016 	load_idx = get_sd_load_idx(env->sd, env->idle);  in update_sd_lb_stats()
8022 		local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));  in update_sd_lb_stats()
8027 			if (env->idle != CPU_NEWLY_IDLE ||  in update_sd_lb_stats()
8029 				update_group_capacity(env->sd, env->dst_cpu);  in update_sd_lb_stats()
8032 		update_sg_lb_stats(env, sg, load_idx, local_group, sgs,  in update_sd_lb_stats()
8049 		    group_has_capacity(env, local) &&  in update_sd_lb_stats()
8055 		if (update_sd_pick_busiest(env, sds, sg, sgs)) {  in update_sd_lb_stats()
8067 	} while (sg != env->sd->groups);  in update_sd_lb_stats()
8070 	if ((env->flags & LBF_NOHZ_AGAIN) &&  in update_sd_lb_stats()
8071 	    cpumask_subset(nohz.idle_cpus_mask, sched_domain_span(env->sd))) {  in update_sd_lb_stats()
8078 	if (env->sd->flags & SD_NUMA)  in update_sd_lb_stats()
8079 		env->fbq_type = fbq_classify_group(&sds->busiest_stat);  in update_sd_lb_stats()
8081 	if (!env->sd->parent) {  in update_sd_lb_stats()
8083 		if (env->dst_rq->rd->overload != overload)  in update_sd_lb_stats()
8084 			env->dst_rq->rd->overload = overload;  in update_sd_lb_stats()
8111 static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)  in check_asym_packing()  argument
8115 	if (!(env->sd->flags & SD_ASYM_PACKING))  in check_asym_packing()
8118 	if (env->idle == CPU_NOT_IDLE)  in check_asym_packing()
8125 	if (sched_asym_prefer(busiest_cpu, env->dst_cpu))  in check_asym_packing()
8128 	env->imbalance = DIV_ROUND_CLOSEST(  in check_asym_packing()
8143 void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)  in fix_small_imbalance()  argument
8154 		local->load_per_task = cpu_avg_load_per_task(env->dst_cpu);  in fix_small_imbalance()
8164 		env->imbalance = busiest->load_per_task;  in fix_small_imbalance()
8202 		env->imbalance = busiest->load_per_task;  in fix_small_imbalance()
8211 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)  in calculate_imbalance()  argument
8236 		env->imbalance = 0;  in calculate_imbalance()
8237 		return fix_small_imbalance(env, sds);  in calculate_imbalance()
8264 	env->imbalance = min(  in calculate_imbalance()
8275 	if (env->imbalance < busiest->load_per_task)  in calculate_imbalance()
8276 		return fix_small_imbalance(env, sds);  in calculate_imbalance()
8292 static struct sched_group *find_busiest_group(struct lb_env *env)  in find_busiest_group()  argument
8303 	update_sd_lb_stats(env, &sds);  in find_busiest_group()
8308 	if (check_asym_packing(env, &sds))  in find_busiest_group()
8331 	if (env->idle != CPU_NOT_IDLE && group_has_capacity(env, local) &&  in find_busiest_group()
8349 	if (env->idle == CPU_IDLE) {  in find_busiest_group()
8366 				env->sd->imbalance_pct * local->avg_load)  in find_busiest_group()
8372 	calculate_imbalance(env, &sds);  in find_busiest_group()
8373 	return env->imbalance ? sds.busiest : NULL;  in find_busiest_group()
8376 	env->imbalance = 0;  in find_busiest_group()
8383 static struct rq *find_busiest_queue(struct lb_env *env,  in find_busiest_queue()  argument
8390 	for_each_cpu_and(i, sched_group_span(group), env->cpus) {  in find_busiest_queue()
8416 		if (rt > env->fbq_type)  in find_busiest_queue()
8428 		if (rq->nr_running == 1 && wl > env->imbalance &&  in find_busiest_queue()
8429 		    !check_cpu_capacity(rq, env->sd))  in find_busiest_queue()
8459 static int need_active_balance(struct lb_env *env)  in need_active_balance()  argument
8461 	struct sched_domain *sd = env->sd;  in need_active_balance()
8463 	if (env->idle == CPU_NEWLY_IDLE) {  in need_active_balance()
8471 		    sched_asym_prefer(env->dst_cpu, env->src_cpu))  in need_active_balance()
8481 	if ((env->idle != CPU_NOT_IDLE) &&  in need_active_balance()
8482 	    (env->src_rq->cfs.h_nr_running == 1)) {  in need_active_balance()
8483 		if ((check_cpu_capacity(env->src_rq, sd)) &&  in need_active_balance()
8484 		    (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))  in need_active_balance()
8493 static int should_we_balance(struct lb_env *env)  in should_we_balance()  argument
8495 	struct sched_group *sg = env->sd->groups;  in should_we_balance()
8502 	if (!cpumask_test_cpu(env->dst_cpu, env->cpus))  in should_we_balance()
8509 	if (env->idle == CPU_NEWLY_IDLE)  in should_we_balance()
8513 	for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) {  in should_we_balance()
8528 	return balance_cpu == env->dst_cpu;  in should_we_balance()
8546 	struct lb_env env = {  in load_balance()  local
8555 		.tasks		= LIST_HEAD_INIT(env.tasks),  in load_balance()
8563 	if (!should_we_balance(&env)) {  in load_balance()
8568 	group = find_busiest_group(&env);  in load_balance()
8574 	busiest = find_busiest_queue(&env, group);  in load_balance()
8580 	BUG_ON(busiest == env.dst_rq);  in load_balance()
8582 	schedstat_add(sd->lb_imbalance[idle], env.imbalance);  in load_balance()
8584 	env.src_cpu = busiest->cpu;  in load_balance()
8585 	env.src_rq = busiest;  in load_balance()
8595 		env.flags |= LBF_ALL_PINNED;  in load_balance()
8596 		env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);  in load_balance()
8606 		cur_ld_moved = detach_tasks(&env);  in load_balance()
8619 			attach_tasks(&env);  in load_balance()
8625 		if (env.flags & LBF_NEED_BREAK) {  in load_balance()
8626 			env.flags &= ~LBF_NEED_BREAK;  in load_balance()
8649 		if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {  in load_balance()
8652 			cpumask_clear_cpu(env.dst_cpu, env.cpus);  in load_balance()
8654 			env.dst_rq	 = cpu_rq(env.new_dst_cpu);  in load_balance()
8655 			env.dst_cpu	 = env.new_dst_cpu;  in load_balance()
8656 			env.flags	&= ~LBF_DST_PINNED;  in load_balance()
8657 			env.loop	 = 0;  in load_balance()
8658 			env.loop_break	 = sched_nr_migrate_break;  in load_balance()
8673 			if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)  in load_balance()
8678 		if (unlikely(env.flags & LBF_ALL_PINNED)) {  in load_balance()
8688 			if (!cpumask_subset(cpus, env.dst_grpmask)) {  in load_balance()
8689 				env.loop = 0;  in load_balance()
8690 				env.loop_break = sched_nr_migrate_break;  in load_balance()
8708 		if (need_active_balance(&env)) {  in load_balance()
8721 				env.flags |= LBF_ALL_PINNED;  in load_balance()
8789 	if (((env.flags & LBF_ALL_PINNED) &&  in load_balance()
8877 		struct lb_env env = {  in active_load_balance_cpu_stop()  local
8896 		p = detach_one_task(&env);  in active_load_balance_cpu_stop()