101 int padata_do_parallel(struct padata_instance *pinst,  in padata_do_parallel()  argument
110 	pd = rcu_dereference_bh(pinst->pd);  in padata_do_parallel()
113 	if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)  in padata_do_parallel()
131 	if ((pinst->flags & PADATA_RESET))  in padata_do_parallel()
151 	queue_work(pinst->parallel_wq, &queue->work);  in padata_do_parallel()
216 	struct padata_instance *pinst = pd->pinst;  in padata_reorder()  local
250 		queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work);  in padata_reorder()
268 		queue_work(pinst->serial_wq, &pd->reorder_work);  in padata_reorder()
365 	err = apply_workqueue_attrs(pd->pinst->parallel_wq, attrs);  in padata_setup_cpumasks()
417 static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,  in padata_alloc_pd()  argument
435 	pd->pinst = pinst;  in padata_alloc_pd()
492 static void __padata_start(struct padata_instance *pinst)  in __padata_start()  argument
494 	pinst->flags |= PADATA_INIT;  in __padata_start()
497 static void __padata_stop(struct padata_instance *pinst)  in __padata_stop()  argument
499 	if (!(pinst->flags & PADATA_INIT))  in __padata_stop()
502 	pinst->flags &= ~PADATA_INIT;  in __padata_stop()
507 	padata_flush_queues(pinst->pd);  in __padata_stop()
512 static void padata_replace(struct padata_instance *pinst,  in padata_replace()  argument
515 	struct parallel_data *pd_old = pinst->pd;  in padata_replace()
518 	pinst->flags |= PADATA_RESET;  in padata_replace()
520 	rcu_assign_pointer(pinst->pd, pd_new);  in padata_replace()
533 		blocking_notifier_call_chain(&pinst->cpumask_change_notifier,  in padata_replace()
537 	pinst->flags &= ~PADATA_RESET;  in padata_replace()
547 int padata_register_cpumask_notifier(struct padata_instance *pinst,  in padata_register_cpumask_notifier()  argument
550 	return blocking_notifier_chain_register(&pinst->cpumask_change_notifier,  in padata_register_cpumask_notifier()
562 int padata_unregister_cpumask_notifier(struct padata_instance *pinst,  in padata_unregister_cpumask_notifier()  argument
566 		&pinst->cpumask_change_notifier,  in padata_unregister_cpumask_notifier()
573 static bool padata_validate_cpumask(struct padata_instance *pinst,  in padata_validate_cpumask()  argument
577 		pinst->flags |= PADATA_INVALID;  in padata_validate_cpumask()
581 	pinst->flags &= ~PADATA_INVALID;  in padata_validate_cpumask()
585 static int __padata_set_cpumasks(struct padata_instance *pinst,  in __padata_set_cpumasks()  argument
592 	valid = padata_validate_cpumask(pinst, pcpumask);  in __padata_set_cpumasks()
594 		__padata_stop(pinst);  in __padata_set_cpumasks()
598 	valid = padata_validate_cpumask(pinst, cbcpumask);  in __padata_set_cpumasks()
600 		__padata_stop(pinst);  in __padata_set_cpumasks()
603 	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);  in __padata_set_cpumasks()
607 	cpumask_copy(pinst->cpumask.pcpu, pcpumask);  in __padata_set_cpumasks()
608 	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);  in __padata_set_cpumasks()
610 	padata_replace(pinst, pd);  in __padata_set_cpumasks()
613 		__padata_start(pinst);  in __padata_set_cpumasks()
627 int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,  in padata_set_cpumask()  argument
633 	mutex_lock(&pinst->lock);  in padata_set_cpumask()
638 		serial_mask = pinst->cpumask.cbcpu;  in padata_set_cpumask()
642 		parallel_mask = pinst->cpumask.pcpu;  in padata_set_cpumask()
649 	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask);  in padata_set_cpumask()
653 	mutex_unlock(&pinst->lock);  in padata_set_cpumask()
664 int padata_start(struct padata_instance *pinst)  in padata_start()  argument
668 	mutex_lock(&pinst->lock);  in padata_start()
670 	if (pinst->flags & PADATA_INVALID)  in padata_start()
673 	__padata_start(pinst);  in padata_start()
675 	mutex_unlock(&pinst->lock);  in padata_start()
686 void padata_stop(struct padata_instance *pinst)  in padata_stop()  argument
688 	mutex_lock(&pinst->lock);  in padata_stop()
689 	__padata_stop(pinst);  in padata_stop()
690 	mutex_unlock(&pinst->lock);  in padata_stop()
696 static int __padata_add_cpu(struct padata_instance *pinst, int cpu)  in __padata_add_cpu()  argument
701 		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,  in __padata_add_cpu()
702 				     pinst->cpumask.cbcpu);  in __padata_add_cpu()
706 		padata_replace(pinst, pd);  in __padata_add_cpu()
708 		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&  in __padata_add_cpu()
709 		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_add_cpu()
710 			__padata_start(pinst);  in __padata_add_cpu()
716 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)  in __padata_remove_cpu()  argument
722 		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||  in __padata_remove_cpu()
723 		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))  in __padata_remove_cpu()
724 			__padata_stop(pinst);  in __padata_remove_cpu()
726 		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,  in __padata_remove_cpu()
727 				     pinst->cpumask.cbcpu);  in __padata_remove_cpu()
731 		padata_replace(pinst, pd);  in __padata_remove_cpu()
751 int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask)  in padata_remove_cpu()  argument
758 	mutex_lock(&pinst->lock);  in padata_remove_cpu()
762 		cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu);  in padata_remove_cpu()
764 		cpumask_clear_cpu(cpu, pinst->cpumask.pcpu);  in padata_remove_cpu()
766 	err = __padata_remove_cpu(pinst, cpu);  in padata_remove_cpu()
769 	mutex_unlock(&pinst->lock);  in padata_remove_cpu()
775 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)  in pinst_has_cpu()  argument
777 	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||  in pinst_has_cpu()
778 		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);  in pinst_has_cpu()
783 	struct padata_instance *pinst;  in padata_cpu_online()  local
786 	pinst = hlist_entry_safe(node, struct padata_instance, node);  in padata_cpu_online()
787 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_online()
790 	mutex_lock(&pinst->lock);  in padata_cpu_online()
791 	ret = __padata_add_cpu(pinst, cpu);  in padata_cpu_online()
792 	mutex_unlock(&pinst->lock);  in padata_cpu_online()
798 	struct padata_instance *pinst;  in padata_cpu_prep_down()  local
801 	pinst = hlist_entry_safe(node, struct padata_instance, node);  in padata_cpu_prep_down()
802 	if (!pinst_has_cpu(pinst, cpu))  in padata_cpu_prep_down()
805 	mutex_lock(&pinst->lock);  in padata_cpu_prep_down()
806 	ret = __padata_remove_cpu(pinst, cpu);  in padata_cpu_prep_down()
807 	mutex_unlock(&pinst->lock);  in padata_cpu_prep_down()
814 static void __padata_free(struct padata_instance *pinst)  in __padata_free()  argument
817 	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->node);  in __padata_free()
820 	padata_stop(pinst);  in __padata_free()
821 	padata_free_pd(pinst->pd);  in __padata_free()
822 	free_cpumask_var(pinst->cpumask.pcpu);  in __padata_free()
823 	free_cpumask_var(pinst->cpumask.cbcpu);  in __padata_free()
824 	destroy_workqueue(pinst->serial_wq);  in __padata_free()
825 	destroy_workqueue(pinst->parallel_wq);  in __padata_free()
826 	kfree(pinst);  in __padata_free()
836 	struct padata_instance *pinst = kobj2pinst(kobj);  in padata_sysfs_release()  local
837 	__padata_free(pinst);  in padata_sysfs_release()
847 static ssize_t show_cpumask(struct padata_instance *pinst,  in show_cpumask()  argument
853 	mutex_lock(&pinst->lock);  in show_cpumask()
855 		cpumask = pinst->cpumask.cbcpu;  in show_cpumask()
857 		cpumask = pinst->cpumask.pcpu;  in show_cpumask()
861 	mutex_unlock(&pinst->lock);  in show_cpumask()
865 static ssize_t store_cpumask(struct padata_instance *pinst,  in store_cpumask()  argument
883 	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);  in store_cpumask()
917 	struct padata_instance *pinst;  in padata_sysfs_show()  local
921 	pinst = kobj2pinst(kobj);  in padata_sysfs_show()
924 		ret = pentry->show(pinst, attr, buf);  in padata_sysfs_show()
932 	struct padata_instance *pinst;  in padata_sysfs_store()  local
936 	pinst = kobj2pinst(kobj);  in padata_sysfs_store()
939 		ret = pentry->store(pinst, attr, buf, count);  in padata_sysfs_store()
967 	struct padata_instance *pinst;  in padata_alloc()  local
970 	pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);  in padata_alloc()
971 	if (!pinst)  in padata_alloc()
974 	pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0,  in padata_alloc()
976 	if (!pinst->parallel_wq)  in padata_alloc()
981 	pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM |  in padata_alloc()
983 	if (!pinst->serial_wq)  in padata_alloc()
986 	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))  in padata_alloc()
988 	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {  in padata_alloc()
989 		free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
992 	if (!padata_validate_cpumask(pinst, pcpumask) ||  in padata_alloc()
993 	    !padata_validate_cpumask(pinst, cbcpumask))  in padata_alloc()
996 	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);  in padata_alloc()
1000 	rcu_assign_pointer(pinst->pd, pd);  in padata_alloc()
1002 	cpumask_copy(pinst->cpumask.pcpu, pcpumask);  in padata_alloc()
1003 	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);  in padata_alloc()
1005 	pinst->flags = 0;  in padata_alloc()
1007 	BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier);  in padata_alloc()
1008 	kobject_init(&pinst->kobj, &padata_attr_type);  in padata_alloc()
1009 	mutex_init(&pinst->lock);  in padata_alloc()
1012 	cpuhp_state_add_instance_nocalls_cpuslocked(hp_online, &pinst->node);  in padata_alloc()
1017 	return pinst;  in padata_alloc()
1020 	free_cpumask_var(pinst->cpumask.pcpu);  in padata_alloc()
1021 	free_cpumask_var(pinst->cpumask.cbcpu);  in padata_alloc()
1023 	destroy_workqueue(pinst->serial_wq);  in padata_alloc()
1026 	destroy_workqueue(pinst->parallel_wq);  in padata_alloc()
1028 	kfree(pinst);  in padata_alloc()
1051 void padata_free(struct padata_instance *pinst)  in padata_free()  argument
1053 	kobject_put(&pinst->kobj);  in padata_free()