104 int padata_do_parallel(struct padata_instance *pinst,  in padata_do_parallel()  argument
113 	pd = rcu_dereference_bh(pinst->pd);  in padata_do_parallel()
116 	if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)  in padata_do_parallel()
123 	if ((pinst->flags & PADATA_RESET))  in padata_do_parallel()
142 	queue_work_on(target_cpu, pinst->wq, &queue->work);  in padata_do_parallel()
217 	struct padata_instance *pinst = pd->pinst;  in padata_reorder()  local
261 		queue_work_on(cb_cpu, pinst->wq, &squeue->work);  in padata_reorder()
272 			&& !(pinst->flags & PADATA_RESET))  in padata_reorder()
312 		struct padata_instance *pinst;  in padata_reorder_timer()  local
317 		pinst = pd->pinst;  in padata_reorder_timer()
319 		queue_work_on(target_cpu, pinst->wq, &pqueue->reorder_work);  in padata_reorder_timer()
396 		queue_work_on(cpu, pd->pinst->wq, &pqueue->reorder_work);  in padata_do_serial()
467 static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,  in padata_alloc_pd()  argument
493 	pd->pinst = pinst;  in padata_alloc_pd()
542 static void __padata_start(struct padata_instance *pinst)  in __padata_start()  argument
544 	pinst->flags |= PADATA_INIT;  in __padata_start()
547 static void __padata_stop(struct padata_instance *pinst)  in __padata_stop()  argument
549 	if (!(pinst->flags & PADATA_INIT))  in __padata_stop()
552 	pinst->flags &= ~PADATA_INIT;  in __padata_stop()
557 	padata_flush_queues(pinst->pd);  in __padata_stop()
562 static void padata_replace(struct padata_instance *pinst,  in padata_replace()  argument
565 	struct parallel_data *pd_old = pinst->pd;  in padata_replace()
568 	pinst->flags |= PADATA_RESET;  in padata_replace()
570 	rcu_assign_pointer(pinst->pd, pd_new);  in padata_replace()
583 		blocking_notifier_call_chain(&pinst->cpumask_change_notifier,  in padata_replace()
587 	pinst->flags &= ~PADATA_RESET;  in padata_replace()
597 int padata_register_cpumask_notifier(struct padata_instance *pinst,  in padata_register_cpumask_notifier()  argument
600 	return blocking_notifier_chain_register(&pinst->cpumask_change_notifier,  in padata_register_cpumask_notifier()
612 int padata_unregister_cpumask_notifier(struct padata_instance *pinst,  in padata_unregister_cpumask_notifier()  argument
616 		&pinst->cpumask_change_notifier,  in padata_unregister_cpumask_notifier()
623 static bool padata_validate_cpumask(struct padata_instance *pinst,  in padata_validate_cpumask()  argument
627 		pinst->flags |= PADATA_INVALID;  in padata_validate_cpumask()
631 	pinst->flags &= ~PADATA_INVALID;  in padata_validate_cpumask()
635 static int __padata_set_cpumasks(struct padata_instance *pinst,  in __padata_set_cpumasks()  argument
642 	valid = padata_validate_cpumask(pinst, pcpumask);  in __padata_set_cpumasks()
644 		__padata_stop(pinst);  in __padata_set_cpumasks()
648 	valid = padata_validate_cpumask(pinst, cbcpumask);  in __padata_set_cpumasks()
650 		__padata_stop(pinst);  in __padata_set_cpumasks()
653 	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);  in __padata_set_cpumasks()
657 	cpumask_copy(pinst->cpumask.pcpu, pcpumask);  in __padata_set_cpumasks()
658 	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);  in __padata_set_cpumasks()
660 	padata_replace(pinst, pd);  in __padata_set_cpumasks()
663 		__padata_start(pinst);  in __padata_set_cpumasks()
677 int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,  in padata_set_cpumask()  argument
683 	mutex_lock(&pinst->lock);  in padata_set_cpumask()
688 		serial_mask = pinst->cpumask.cbcpu;  in padata_set_cpumask()
692 		parallel_mask = pinst->cpumask.pcpu;  in padata_set_cpumask()
699 	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask);  in padata_set_cpumask()
703 	mutex_unlock(&pinst->lock);  in padata_set_cpumask()
714 int padata_start(struct padata_instance *pinst)  in padata_start()  argument
718 	mutex_lock(&pinst->lock);  in padata_start()
720 	if (pinst->flags & PADATA_INVALID)  in padata_start()
723 	 __padata_start(pinst);  in padata_start()
725 	mutex_unlock(&pinst->lock);  in padata_start()
736 void padata_stop(struct padata_instance *pinst)  in padata_stop()  argument
738 	mutex_lock(&pinst->lock);  in padata_stop()
739 	__padata_stop(pinst);  in padata_stop()
740 	mutex_unlock(&pinst->lock);  in padata_stop()
746 static int __padata_add_cpu(struct padata_instance *pinst, int cpu)  in __padata_add_cpu()  argument
751 		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,  in __padata_add_cpu()
752 				     pinst->cpumask.cbcpu);  in __padata_add_cpu()
756 		padata_replace(pinst, pd);  in __padata_add_cpu()
758 		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&  in __padata_add_cpu()
759 		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_add_cpu()
760 			__padata_start(pinst);  in __padata_add_cpu()
766 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)  in __padata_remove_cpu()  argument
772 		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||  in __padata_remove_cpu()
773 		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_remove_cpu()
774 			__padata_stop(pinst);  in __padata_remove_cpu()
776 		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,  in __padata_remove_cpu()
777 				     pinst->cpumask.cbcpu);  in __padata_remove_cpu()
781 		padata_replace(pinst, pd);  in __padata_remove_cpu()
801 int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask)  in padata_remove_cpu()  argument
808 	mutex_lock(&pinst->lock);  in padata_remove_cpu()
812 		cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu);  in padata_remove_cpu()
814 		cpumask_clear_cpu(cpu, pinst->cpumask.pcpu);  in padata_remove_cpu()
816 	err = __padata_remove_cpu(pinst, cpu);  in padata_remove_cpu()
819 	mutex_unlock(&pinst->lock);  in padata_remove_cpu()
825 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)  in pinst_has_cpu()  argument
827 	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||  in pinst_has_cpu()
828 		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);  in pinst_has_cpu()
833 	struct padata_instance *pinst;  in padata_cpu_online()  local
836 	pinst = hlist_entry_safe(node, struct padata_instance, node);  in padata_cpu_online()
837 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_online()
840 	mutex_lock(&pinst->lock);  in padata_cpu_online()
841 	ret = __padata_add_cpu(pinst, cpu);  in padata_cpu_online()
842 	mutex_unlock(&pinst->lock);  in padata_cpu_online()
848 	struct padata_instance *pinst;  in padata_cpu_prep_down()  local
851 	pinst = hlist_entry_safe(node, struct padata_instance, node);  in padata_cpu_prep_down()
852 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_prep_down()
855 	mutex_lock(&pinst->lock);  in padata_cpu_prep_down()
856 	ret = __padata_remove_cpu(pinst, cpu);  in padata_cpu_prep_down()
857 	mutex_unlock(&pinst->lock);  in padata_cpu_prep_down()
864 static void __padata_free(struct padata_instance *pinst)  in __padata_free()  argument
867 	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->node);  in __padata_free()
870 	padata_stop(pinst);  in __padata_free()
871 	padata_free_pd(pinst->pd);  in __padata_free()
872 	free_cpumask_var(pinst->cpumask.pcpu);  in __padata_free()
873 	free_cpumask_var(pinst->cpumask.cbcpu);  in __padata_free()
874 	kfree(pinst);  in __padata_free()
884 	struct padata_instance *pinst = kobj2pinst(kobj);  in padata_sysfs_release()  local
885 	__padata_free(pinst);  in padata_sysfs_release()
895 static ssize_t show_cpumask(struct padata_instance *pinst,  in show_cpumask()  argument
901 	mutex_lock(&pinst->lock);  in show_cpumask()
903 		cpumask = pinst->cpumask.cbcpu;  in show_cpumask()
905 		cpumask = pinst->cpumask.pcpu;  in show_cpumask()
909 	mutex_unlock(&pinst->lock);  in show_cpumask()
913 static ssize_t store_cpumask(struct padata_instance *pinst,  in store_cpumask()  argument
931 	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);  in store_cpumask()
964 	struct padata_instance *pinst;  in padata_sysfs_show()  local
968 	pinst = kobj2pinst(kobj);  in padata_sysfs_show()
971 		ret = pentry->show(pinst, attr, buf);  in padata_sysfs_show()
979 	struct padata_instance *pinst;  in padata_sysfs_store()  local
983 	pinst = kobj2pinst(kobj);  in padata_sysfs_store()
986 		ret = pentry->store(pinst, attr, buf, count);  in padata_sysfs_store()
1016 	struct padata_instance *pinst;  in padata_alloc()  local
1019 	pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);  in padata_alloc()
1020 	if (!pinst)  in padata_alloc()
1023 	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))  in padata_alloc()
1025 	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {  in padata_alloc()
1026 		free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
1029 	if (!padata_validate_cpumask(pinst, pcpumask) ||  in padata_alloc()
1030 	    !padata_validate_cpumask(pinst, cbcpumask))  in padata_alloc()
1033 	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);  in padata_alloc()
1037 	rcu_assign_pointer(pinst->pd, pd);  in padata_alloc()
1039 	pinst->wq = wq;  in padata_alloc()
1041 	cpumask_copy(pinst->cpumask.pcpu, pcpumask);  in padata_alloc()
1042 	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);  in padata_alloc()
1044 	pinst->flags = 0;  in padata_alloc()
1046 	BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier);  in padata_alloc()
1047 	kobject_init(&pinst->kobj, &padata_attr_type);  in padata_alloc()
1048 	mutex_init(&pinst->lock);  in padata_alloc()
1051 	cpuhp_state_add_instance_nocalls_cpuslocked(hp_online, &pinst->node);  in padata_alloc()
1053 	return pinst;  in padata_alloc()
1056 	free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
1057 	free_cpumask_var(pinst->cpumask.cbcpu);  in padata_alloc()
1059 	kfree(pinst);  in padata_alloc()
1085 void padata_free(struct padata_instance *pinst)  in padata_free()  argument
1087 	kobject_put(&pinst->kobj);  in padata_free()