Lines Matching full:pd
47 static void padata_free_pd(struct parallel_data *pd);
50 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) in padata_index_to_cpu() argument
54 target_cpu = cpumask_first(pd->cpumask.pcpu); in padata_index_to_cpu()
56 target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu); in padata_index_to_cpu()
61 static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr) in padata_cpu_hash() argument
67 int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu); in padata_cpu_hash()
69 return padata_index_to_cpu(pd, cpu_index); in padata_cpu_hash()
172 struct parallel_data *pd; in padata_do_parallel() local
177 pd = rcu_dereference_bh(ps->pd); in padata_do_parallel()
183 if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) { in padata_do_parallel()
184 if (!cpumask_weight(pd->cpumask.cbcpu)) in padata_do_parallel()
188 cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu); in padata_do_parallel()
190 cpu = cpumask_first(pd->cpumask.cbcpu); in padata_do_parallel()
192 cpu = cpumask_next(cpu, pd->cpumask.cbcpu); in padata_do_parallel()
201 refcount_inc(&pd->refcnt); in padata_do_parallel()
202 padata->pd = pd; in padata_do_parallel()
206 padata->seq_nr = ++pd->seq_nr; in padata_do_parallel()
238 static struct padata_priv *padata_find_next(struct parallel_data *pd, in padata_find_next() argument
243 int cpu = pd->cpu; in padata_find_next()
245 reorder = per_cpu_ptr(pd->reorder_list, cpu); in padata_find_next()
259 if (padata->seq_nr != pd->processed) { in padata_find_next()
266 ++pd->processed; in padata_find_next()
267 pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false); in padata_find_next()
274 static void padata_reorder(struct parallel_data *pd) in padata_reorder() argument
276 struct padata_instance *pinst = pd->ps->pinst; in padata_reorder()
292 if (!spin_trylock_bh(&pd->lock)) in padata_reorder()
296 padata = padata_find_next(pd, true); in padata_reorder()
307 squeue = per_cpu_ptr(pd->squeue, cb_cpu); in padata_reorder()
316 spin_unlock_bh(&pd->lock); in padata_reorder()
322 * Ensure reorder queue is read after pd->lock is dropped so we see in padata_reorder()
328 reorder = per_cpu_ptr(pd->reorder_list, pd->cpu); in padata_reorder()
329 if (!list_empty(&reorder->list) && padata_find_next(pd, false)) in padata_reorder()
330 queue_work(pinst->serial_wq, &pd->reorder_work); in padata_reorder()
335 struct parallel_data *pd; in invoke_padata_reorder() local
338 pd = container_of(work, struct parallel_data, reorder_work); in invoke_padata_reorder()
339 padata_reorder(pd); in invoke_padata_reorder()
346 struct parallel_data *pd; in padata_serial_worker() local
352 pd = squeue->pd; in padata_serial_worker()
373 if (refcount_sub_and_test(cnt, &pd->refcnt)) in padata_serial_worker()
374 padata_free_pd(pd); in padata_serial_worker()
387 struct parallel_data *pd = padata->pd; in padata_do_serial() local
388 int hashed_cpu = padata_cpu_hash(pd, padata->seq_nr); in padata_do_serial()
389 struct padata_list *reorder = per_cpu_ptr(pd->reorder_list, hashed_cpu); in padata_do_serial()
402 * with the trylock of pd->lock in padata_reorder. Pairs with smp_mb in padata_do_serial()
407 padata_reorder(pd); in padata_do_serial()
420 /* Restrict parallel_wq workers to pd->cpumask.pcpu. */ in padata_setup_cpumasks()
527 static void padata_init_squeues(struct parallel_data *pd) in padata_init_squeues() argument
532 for_each_cpu(cpu, pd->cpumask.cbcpu) { in padata_init_squeues()
533 squeue = per_cpu_ptr(pd->squeue, cpu); in padata_init_squeues()
534 squeue->pd = pd; in padata_init_squeues()
541 static void padata_init_reorder_list(struct parallel_data *pd) in padata_init_reorder_list() argument
546 for_each_cpu(cpu, pd->cpumask.pcpu) { in padata_init_reorder_list()
547 list = per_cpu_ptr(pd->reorder_list, cpu); in padata_init_reorder_list()
556 struct parallel_data *pd; in padata_alloc_pd() local
558 pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL); in padata_alloc_pd()
559 if (!pd) in padata_alloc_pd()
562 pd->reorder_list = alloc_percpu(struct padata_list); in padata_alloc_pd()
563 if (!pd->reorder_list) in padata_alloc_pd()
566 pd->squeue = alloc_percpu(struct padata_serial_queue); in padata_alloc_pd()
567 if (!pd->squeue) in padata_alloc_pd()
570 pd->ps = ps; in padata_alloc_pd()
572 if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL)) in padata_alloc_pd()
574 if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) in padata_alloc_pd()
577 cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask); in padata_alloc_pd()
578 cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask); in padata_alloc_pd()
580 padata_init_reorder_list(pd); in padata_alloc_pd()
581 padata_init_squeues(pd); in padata_alloc_pd()
582 pd->seq_nr = -1; in padata_alloc_pd()
583 refcount_set(&pd->refcnt, 1); in padata_alloc_pd()
584 spin_lock_init(&pd->lock); in padata_alloc_pd()
585 pd->cpu = cpumask_first(pd->cpumask.pcpu); in padata_alloc_pd()
586 INIT_WORK(&pd->reorder_work, invoke_padata_reorder); in padata_alloc_pd()
588 return pd; in padata_alloc_pd()
591 free_cpumask_var(pd->cpumask.pcpu); in padata_alloc_pd()
593 free_percpu(pd->squeue); in padata_alloc_pd()
595 free_percpu(pd->reorder_list); in padata_alloc_pd()
597 kfree(pd); in padata_alloc_pd()
602 static void padata_free_pd(struct parallel_data *pd) in padata_free_pd() argument
604 free_cpumask_var(pd->cpumask.pcpu); in padata_free_pd()
605 free_cpumask_var(pd->cpumask.cbcpu); in padata_free_pd()
606 free_percpu(pd->reorder_list); in padata_free_pd()
607 free_percpu(pd->squeue); in padata_free_pd()
608 kfree(pd); in padata_free_pd()
635 ps->opd = rcu_dereference_protected(ps->pd, 1); in padata_replace_one()
636 rcu_assign_pointer(ps->pd, pd_new); in padata_replace_one()
1055 struct parallel_data *pd; in padata_alloc_shell() local
1065 pd = padata_alloc_pd(ps); in padata_alloc_shell()
1068 if (!pd) in padata_alloc_shell()
1072 RCU_INIT_POINTER(ps->pd, pd); in padata_alloc_shell()
1097 padata_free_pd(rcu_dereference_protected(ps->pd, 1)); in padata_free_shell()