170 	struct padata_instance *pinst = ps->pinst;  in padata_do_parallel()  local
180 	if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)  in padata_do_parallel()
198 	if ((pinst->flags & PADATA_RESET))  in padata_do_parallel()
214 		queue_work(pinst->parallel_wq, &pw->pw_work);  in padata_do_parallel()
276 	struct padata_instance *pinst = pd->ps->pinst;  in padata_reorder()  local
313 		queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work);  in padata_reorder()
330 		queue_work(pinst->serial_wq, &pd->reorder_work);  in padata_reorder()
411 static int padata_setup_cpumasks(struct padata_instance *pinst)  in padata_setup_cpumasks()  argument
421 	cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu);  in padata_setup_cpumasks()
422 	err = apply_workqueue_attrs(pinst->parallel_wq, attrs);  in padata_setup_cpumasks()
555 	struct padata_instance *pinst = ps->pinst;  in padata_alloc_pd()  local
577 	cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask);  in padata_alloc_pd()
578 	cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask);  in padata_alloc_pd()
611 static void __padata_start(struct padata_instance *pinst)  in __padata_start()  argument
613 	pinst->flags |= PADATA_INIT;  in __padata_start()
616 static void __padata_stop(struct padata_instance *pinst)  in __padata_stop()  argument
618 	if (!(pinst->flags & PADATA_INIT))  in __padata_stop()
621 	pinst->flags &= ~PADATA_INIT;  in __padata_stop()
641 static int padata_replace(struct padata_instance *pinst)  in padata_replace()  argument
646 	pinst->flags |= PADATA_RESET;  in padata_replace()
648 	list_for_each_entry(ps, &pinst->pslist, list) {  in padata_replace()
656 	list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)  in padata_replace()
660 	pinst->flags &= ~PADATA_RESET;  in padata_replace()
666 static bool padata_validate_cpumask(struct padata_instance *pinst,  in padata_validate_cpumask()  argument
670 		pinst->flags |= PADATA_INVALID;  in padata_validate_cpumask()
674 	pinst->flags &= ~PADATA_INVALID;  in padata_validate_cpumask()
678 static int __padata_set_cpumasks(struct padata_instance *pinst,  in __padata_set_cpumasks()  argument
685 	valid = padata_validate_cpumask(pinst, pcpumask);  in __padata_set_cpumasks()
687 		__padata_stop(pinst);  in __padata_set_cpumasks()
691 	valid = padata_validate_cpumask(pinst, cbcpumask);  in __padata_set_cpumasks()
693 		__padata_stop(pinst);  in __padata_set_cpumasks()
696 	cpumask_copy(pinst->cpumask.pcpu, pcpumask);  in __padata_set_cpumasks()
697 	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);  in __padata_set_cpumasks()
699 	err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst);  in __padata_set_cpumasks()
702 		__padata_start(pinst);  in __padata_set_cpumasks()
717 int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,  in padata_set_cpumask()  argument
724 	mutex_lock(&pinst->lock);  in padata_set_cpumask()
728 		serial_mask = pinst->cpumask.cbcpu;  in padata_set_cpumask()
732 		parallel_mask = pinst->cpumask.pcpu;  in padata_set_cpumask()
739 	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask);  in padata_set_cpumask()
742 	mutex_unlock(&pinst->lock);  in padata_set_cpumask()
751 static int __padata_add_cpu(struct padata_instance *pinst, int cpu)  in __padata_add_cpu()  argument
756 		err = padata_replace(pinst);  in __padata_add_cpu()
758 		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&  in __padata_add_cpu()
759 		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_add_cpu()
760 			__padata_start(pinst);  in __padata_add_cpu()
766 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)  in __padata_remove_cpu()  argument
771 		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||  in __padata_remove_cpu()
772 		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_remove_cpu()
773 			__padata_stop(pinst);  in __padata_remove_cpu()
775 		err = padata_replace(pinst);  in __padata_remove_cpu()
781 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)  in pinst_has_cpu()  argument
783 	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||  in pinst_has_cpu()
784 		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);  in pinst_has_cpu()
789 	struct padata_instance *pinst;  in padata_cpu_online()  local
792 	pinst = hlist_entry_safe(node, struct padata_instance, cpu_online_node);  in padata_cpu_online()
793 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_online()
796 	mutex_lock(&pinst->lock);  in padata_cpu_online()
797 	ret = __padata_add_cpu(pinst, cpu);  in padata_cpu_online()
798 	mutex_unlock(&pinst->lock);  in padata_cpu_online()
804 	struct padata_instance *pinst;  in padata_cpu_dead()  local
807 	pinst = hlist_entry_safe(node, struct padata_instance, cpu_dead_node);  in padata_cpu_dead()
808 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_dead()
811 	mutex_lock(&pinst->lock);  in padata_cpu_dead()
812 	ret = __padata_remove_cpu(pinst, cpu);  in padata_cpu_dead()
813 	mutex_unlock(&pinst->lock);  in padata_cpu_dead()
820 static void __padata_free(struct padata_instance *pinst)  in __padata_free()  argument
824 					    &pinst->cpu_dead_node);  in __padata_free()
825 	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpu_online_node);  in __padata_free()
828 	WARN_ON(!list_empty(&pinst->pslist));  in __padata_free()
830 	free_cpumask_var(pinst->cpumask.pcpu);  in __padata_free()
831 	free_cpumask_var(pinst->cpumask.cbcpu);  in __padata_free()
832 	destroy_workqueue(pinst->serial_wq);  in __padata_free()
833 	destroy_workqueue(pinst->parallel_wq);  in __padata_free()
834 	kfree(pinst);  in __padata_free()
844 	struct padata_instance *pinst = kobj2pinst(kobj);  in padata_sysfs_release()  local
845 	__padata_free(pinst);  in padata_sysfs_release()
855 static ssize_t show_cpumask(struct padata_instance *pinst,  in show_cpumask()  argument
861 	mutex_lock(&pinst->lock);  in show_cpumask()
863 		cpumask = pinst->cpumask.cbcpu;  in show_cpumask()
865 		cpumask = pinst->cpumask.pcpu;  in show_cpumask()
869 	mutex_unlock(&pinst->lock);  in show_cpumask()
873 static ssize_t store_cpumask(struct padata_instance *pinst,  in store_cpumask()  argument
891 	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);  in store_cpumask()
925 	struct padata_instance *pinst;  in padata_sysfs_show()  local
929 	pinst = kobj2pinst(kobj);  in padata_sysfs_show()
932 		ret = pentry->show(pinst, attr, buf);  in padata_sysfs_show()
940 	struct padata_instance *pinst;  in padata_sysfs_store()  local
944 	pinst = kobj2pinst(kobj);  in padata_sysfs_store()
947 		ret = pentry->store(pinst, attr, buf, count);  in padata_sysfs_store()
971 	struct padata_instance *pinst;  in padata_alloc()  local
973 	pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);  in padata_alloc()
974 	if (!pinst)  in padata_alloc()
977 	pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0,  in padata_alloc()
979 	if (!pinst->parallel_wq)  in padata_alloc()
984 	pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM |  in padata_alloc()
986 	if (!pinst->serial_wq)  in padata_alloc()
989 	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))  in padata_alloc()
991 	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {  in padata_alloc()
992 		free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
996 	INIT_LIST_HEAD(&pinst->pslist);  in padata_alloc()
998 	cpumask_copy(pinst->cpumask.pcpu, cpu_possible_mask);  in padata_alloc()
999 	cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask);  in padata_alloc()
1001 	if (padata_setup_cpumasks(pinst))  in padata_alloc()
1004 	__padata_start(pinst);  in padata_alloc()
1006 	kobject_init(&pinst->kobj, &padata_attr_type);  in padata_alloc()
1007 	mutex_init(&pinst->lock);  in padata_alloc()
1011 						    &pinst->cpu_online_node);  in padata_alloc()
1013 						    &pinst->cpu_dead_node);  in padata_alloc()
1018 	return pinst;  in padata_alloc()
1021 	free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
1022 	free_cpumask_var(pinst->cpumask.cbcpu);  in padata_alloc()
1024 	destroy_workqueue(pinst->serial_wq);  in padata_alloc()
1027 	destroy_workqueue(pinst->parallel_wq);  in padata_alloc()
1029 	kfree(pinst);  in padata_alloc()
1040 void padata_free(struct padata_instance *pinst)  in padata_free()  argument
1042 	kobject_put(&pinst->kobj);  in padata_free()
1053 struct padata_shell *padata_alloc_shell(struct padata_instance *pinst)  in padata_alloc_shell()  argument
1062 	ps->pinst = pinst;  in padata_alloc_shell()
1071 	mutex_lock(&pinst->lock);  in padata_alloc_shell()
1073 	list_add(&ps->list, &pinst->pslist);  in padata_alloc_shell()
1074 	mutex_unlock(&pinst->lock);  in padata_alloc_shell()
1095 	mutex_lock(&ps->pinst->lock);  in padata_free_shell()
1098 	mutex_unlock(&ps->pinst->lock);  in padata_free_shell()