38 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)  in padata_index_to_cpu()  argument
42 	target_cpu = cpumask_first(pd->cpumask.pcpu);  in padata_index_to_cpu()
44 		target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu);  in padata_index_to_cpu()
49 static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr)  in padata_cpu_hash()  argument
55 	int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);  in padata_cpu_hash()
57 	return padata_index_to_cpu(pd, cpu_index);  in padata_cpu_hash()
106 	struct parallel_data *pd;  in padata_do_parallel()  local
110 	pd = rcu_dereference_bh(pinst->pd);  in padata_do_parallel()
116 	if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) {  in padata_do_parallel()
117 		if (!cpumask_weight(pd->cpumask.cbcpu))  in padata_do_parallel()
121 		cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu);  in padata_do_parallel()
123 		cpu = cpumask_first(pd->cpumask.cbcpu);  in padata_do_parallel()
125 			cpu = cpumask_next(cpu, pd->cpumask.cbcpu);  in padata_do_parallel()
134 	if (atomic_read(&pd->refcnt) >= MAX_OBJ_NUM)  in padata_do_parallel()
138 	atomic_inc(&pd->refcnt);  in padata_do_parallel()
139 	padata->pd = pd;  in padata_do_parallel()
142 	padata->seq_nr = atomic_inc_return(&pd->seq_nr);  in padata_do_parallel()
143 	target_cpu = padata_cpu_hash(pd, padata->seq_nr);  in padata_do_parallel()
145 	queue = per_cpu_ptr(pd->pqueue, target_cpu);  in padata_do_parallel()
172 static struct padata_priv *padata_find_next(struct parallel_data *pd,  in padata_find_next()  argument
178 	int cpu = pd->cpu;  in padata_find_next()
180 	next_queue = per_cpu_ptr(pd->pqueue, cpu);  in padata_find_next()
195 	if (padata->seq_nr != pd->processed) {  in padata_find_next()
202 		atomic_dec(&pd->reorder_objects);  in padata_find_next()
203 		++pd->processed;  in padata_find_next()
204 		pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false);  in padata_find_next()
211 static void padata_reorder(struct parallel_data *pd)  in padata_reorder()  argument
216 	struct padata_instance *pinst = pd->pinst;  in padata_reorder()
229 	if (!spin_trylock_bh(&pd->lock))  in padata_reorder()
233 		padata = padata_find_next(pd, true);  in padata_reorder()
244 		squeue = per_cpu_ptr(pd->squeue, cb_cpu);  in padata_reorder()
253 	spin_unlock_bh(&pd->lock);  in padata_reorder()
265 	next_queue = per_cpu_ptr(pd->pqueue, pd->cpu);  in padata_reorder()
267 	    padata_find_next(pd, false))  in padata_reorder()
268 		queue_work(pinst->serial_wq, &pd->reorder_work);  in padata_reorder()
273 	struct parallel_data *pd;  in invoke_padata_reorder()  local
276 	pd = container_of(work, struct parallel_data, reorder_work);  in invoke_padata_reorder()
277 	padata_reorder(pd);  in invoke_padata_reorder()
284 	struct parallel_data *pd;  in padata_serial_worker()  local
289 	pd = squeue->pd;  in padata_serial_worker()
304 		atomic_dec(&pd->refcnt);  in padata_serial_worker()
319 	struct parallel_data *pd = padata->pd;  in padata_do_serial()  local
320 	struct padata_parallel_queue *pqueue = per_cpu_ptr(pd->pqueue,  in padata_do_serial()
330 	atomic_inc(&pd->reorder_objects);  in padata_do_serial()
340 	padata_reorder(pd);  in padata_do_serial()
344 static int padata_setup_cpumasks(struct parallel_data *pd,  in padata_setup_cpumasks()  argument
351 	if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))  in padata_setup_cpumasks()
353 	cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_online_mask);  in padata_setup_cpumasks()
355 	if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL))  in padata_setup_cpumasks()
357 	cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_online_mask);  in padata_setup_cpumasks()
364 	cpumask_copy(attrs->cpumask, pd->cpumask.pcpu);  in padata_setup_cpumasks()
365 	err = apply_workqueue_attrs(pd->pinst->parallel_wq, attrs);  in padata_setup_cpumasks()
373 	free_cpumask_var(pd->cpumask.cbcpu);  in padata_setup_cpumasks()
375 	free_cpumask_var(pd->cpumask.pcpu);  in padata_setup_cpumasks()
387 static void padata_init_squeues(struct parallel_data *pd)  in padata_init_squeues()  argument
392 	for_each_cpu(cpu, pd->cpumask.cbcpu) {  in padata_init_squeues()
393 		squeue = per_cpu_ptr(pd->squeue, cpu);  in padata_init_squeues()
394 		squeue->pd = pd;  in padata_init_squeues()
401 static void padata_init_pqueues(struct parallel_data *pd)  in padata_init_pqueues()  argument
406 	for_each_cpu(cpu, pd->cpumask.pcpu) {  in padata_init_pqueues()
407 		pqueue = per_cpu_ptr(pd->pqueue, cpu);  in padata_init_pqueues()
421 	struct parallel_data *pd;  in padata_alloc_pd()  local
423 	pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL);  in padata_alloc_pd()
424 	if (!pd)  in padata_alloc_pd()
427 	pd->pqueue = alloc_percpu(struct padata_parallel_queue);  in padata_alloc_pd()
428 	if (!pd->pqueue)  in padata_alloc_pd()
431 	pd->squeue = alloc_percpu(struct padata_serial_queue);  in padata_alloc_pd()
432 	if (!pd->squeue)  in padata_alloc_pd()
435 	pd->pinst = pinst;  in padata_alloc_pd()
436 	if (padata_setup_cpumasks(pd, pcpumask, cbcpumask) < 0)  in padata_alloc_pd()
439 	padata_init_pqueues(pd);  in padata_alloc_pd()
440 	padata_init_squeues(pd);  in padata_alloc_pd()
441 	atomic_set(&pd->seq_nr, -1);  in padata_alloc_pd()
442 	atomic_set(&pd->reorder_objects, 0);  in padata_alloc_pd()
443 	atomic_set(&pd->refcnt, 0);  in padata_alloc_pd()
444 	spin_lock_init(&pd->lock);  in padata_alloc_pd()
445 	pd->cpu = cpumask_first(pd->cpumask.pcpu);  in padata_alloc_pd()
446 	INIT_WORK(&pd->reorder_work, invoke_padata_reorder);  in padata_alloc_pd()
448 	return pd;  in padata_alloc_pd()
451 	free_percpu(pd->squeue);  in padata_alloc_pd()
453 	free_percpu(pd->pqueue);  in padata_alloc_pd()
455 	kfree(pd);  in padata_alloc_pd()
460 static void padata_free_pd(struct parallel_data *pd)  in padata_free_pd()  argument
462 	free_cpumask_var(pd->cpumask.pcpu);  in padata_free_pd()
463 	free_cpumask_var(pd->cpumask.cbcpu);  in padata_free_pd()
464 	free_percpu(pd->pqueue);  in padata_free_pd()
465 	free_percpu(pd->squeue);  in padata_free_pd()
466 	kfree(pd);  in padata_free_pd()
470 static void padata_flush_queues(struct parallel_data *pd)  in padata_flush_queues()  argument
476 	for_each_cpu(cpu, pd->cpumask.pcpu) {  in padata_flush_queues()
477 		pqueue = per_cpu_ptr(pd->pqueue, cpu);  in padata_flush_queues()
481 	if (atomic_read(&pd->reorder_objects))  in padata_flush_queues()
482 		padata_reorder(pd);  in padata_flush_queues()
484 	for_each_cpu(cpu, pd->cpumask.cbcpu) {  in padata_flush_queues()
485 		squeue = per_cpu_ptr(pd->squeue, cpu);  in padata_flush_queues()
489 	BUG_ON(atomic_read(&pd->refcnt) != 0);  in padata_flush_queues()
507 	padata_flush_queues(pinst->pd);  in __padata_stop()
515 	struct parallel_data *pd_old = pinst->pd;  in padata_replace()
520 	rcu_assign_pointer(pinst->pd, pd_new);  in padata_replace()
590 	struct parallel_data *pd;  in __padata_set_cpumasks()  local
603 	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);  in __padata_set_cpumasks()
604 	if (!pd)  in __padata_set_cpumasks()
610 	padata_replace(pinst, pd);  in __padata_set_cpumasks()
698 	struct parallel_data *pd;  in __padata_add_cpu()  local
701 		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,  in __padata_add_cpu()
703 		if (!pd)  in __padata_add_cpu()
706 		padata_replace(pinst, pd);  in __padata_add_cpu()
718 	struct parallel_data *pd = NULL;  in __padata_remove_cpu()  local
726 		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,  in __padata_remove_cpu()
728 		if (!pd)  in __padata_remove_cpu()
731 		padata_replace(pinst, pd);  in __padata_remove_cpu()
733 		cpumask_clear_cpu(cpu, pd->cpumask.cbcpu);  in __padata_remove_cpu()
734 		cpumask_clear_cpu(cpu, pd->cpumask.pcpu);  in __padata_remove_cpu()
821 	padata_free_pd(pinst->pd);  in __padata_free()
968 	struct parallel_data *pd = NULL;  in padata_alloc()  local
996 	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);  in padata_alloc()
997 	if (!pd)  in padata_alloc()
1000 	rcu_assign_pointer(pinst->pd, pd);  in padata_alloc()