Lines Matching refs:kc
206 static void wake(struct dm_kcopyd_client *kc) in wake() argument
208 queue_work(kc->kcopyd_wq, &kc->kcopyd_work); in wake()
241 static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl) in kcopyd_put_pages() argument
248 if (kc->nr_free_pages >= kc->nr_reserved_pages) in kcopyd_put_pages()
251 pl->next = kc->pages; in kcopyd_put_pages()
252 kc->pages = pl; in kcopyd_put_pages()
253 kc->nr_free_pages++; in kcopyd_put_pages()
260 static int kcopyd_get_pages(struct dm_kcopyd_client *kc, in kcopyd_get_pages() argument
271 pl = kc->pages; in kcopyd_get_pages()
274 kc->pages = pl->next; in kcopyd_get_pages()
275 kc->nr_free_pages--; in kcopyd_get_pages()
285 kcopyd_put_pages(kc, *pages); in kcopyd_get_pages()
306 static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages) in client_reserve_pages() argument
322 kc->nr_reserved_pages += nr_pages; in client_reserve_pages()
323 kcopyd_put_pages(kc, pl); in client_reserve_pages()
328 static void client_free_pages(struct dm_kcopyd_client *kc) in client_free_pages() argument
330 BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages); in client_free_pages()
331 drop_pages(kc->pages); in client_free_pages()
332 kc->pages = NULL; in client_free_pages()
333 kc->nr_free_pages = kc->nr_reserved_pages = 0; in client_free_pages()
342 struct dm_kcopyd_client *kc; member
412 struct dm_kcopyd_client *kc) in pop_io_job() argument
437 struct dm_kcopyd_client *kc) in pop() argument
442 spin_lock_irqsave(&kc->job_lock, flags); in pop()
445 if (jobs == &kc->io_jobs) in pop()
446 job = pop_io_job(jobs, kc); in pop()
452 spin_unlock_irqrestore(&kc->job_lock, flags); in pop()
460 struct dm_kcopyd_client *kc = job->kc; in push() local
462 spin_lock_irqsave(&kc->job_lock, flags); in push()
464 spin_unlock_irqrestore(&kc->job_lock, flags); in push()
471 struct dm_kcopyd_client *kc = job->kc; in push_head() local
473 spin_lock_irqsave(&kc->job_lock, flags); in push_head()
475 spin_unlock_irqrestore(&kc->job_lock, flags); in push_head()
493 struct dm_kcopyd_client *kc = job->kc; in run_complete_job() local
496 kcopyd_put_pages(kc, job->pages); in run_complete_job()
503 mempool_free(job, &kc->job_pool); in run_complete_job()
507 if (atomic_dec_and_test(&kc->nr_jobs)) in run_complete_job()
508 wake_up(&kc->destroyq); in run_complete_job()
518 struct dm_kcopyd_client *kc = job->kc; in complete_io() local
520 io_job_finish(kc->throttle); in complete_io()
529 push(&kc->complete_jobs, job); in complete_io()
530 wake(kc); in complete_io()
536 push(&kc->complete_jobs, job); in complete_io()
540 push(&kc->io_jobs, job); in complete_io()
543 wake(kc); in complete_io()
561 .client = job->kc->io_client, in run_io_job()
574 io_job_start(job->kc->throttle); in run_io_job()
589 r = kcopyd_get_pages(job->kc, nr_pages, &job->pages); in run_pages_job()
592 push(&job->kc->io_jobs, job); in run_pages_job()
607 static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc, in process_jobs() argument
613 while ((job = pop(jobs, kc))) { in process_jobs()
623 push(&kc->complete_jobs, job); in process_jobs()
624 wake(kc); in process_jobs()
648 struct dm_kcopyd_client *kc = container_of(work, in do_work() local
660 spin_lock_irqsave(&kc->job_lock, flags); in do_work()
661 list_splice_tail_init(&kc->callback_jobs, &kc->complete_jobs); in do_work()
662 spin_unlock_irqrestore(&kc->job_lock, flags); in do_work()
665 process_jobs(&kc->complete_jobs, kc, run_complete_job); in do_work()
666 process_jobs(&kc->pages_jobs, kc, run_pages_job); in do_work()
667 process_jobs(&kc->io_jobs, kc, run_io_job); in do_work()
678 struct dm_kcopyd_client *kc = job->kc; in dispatch_job() local
679 atomic_inc(&kc->nr_jobs); in dispatch_job()
681 push(&kc->callback_jobs, job); in dispatch_job()
683 push(&kc->io_jobs, job); in dispatch_job()
685 push(&kc->pages_jobs, job); in dispatch_job()
686 wake(kc); in dispatch_job()
697 struct dm_kcopyd_client *kc = job->kc; in segment_complete() local
717 if (count > kc->sub_job_size) in segment_complete()
718 count = kc->sub_job_size; in segment_complete()
753 push(&kc->complete_jobs, job); in segment_complete()
754 wake(kc); in segment_complete()
765 atomic_inc(&master_job->kc->nr_jobs); in split_job()
774 void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, in dm_kcopyd_copy() argument
785 job = mempool_alloc(&kc->job_pool, GFP_NOIO); in dm_kcopyd_copy()
791 job->kc = kc; in dm_kcopyd_copy()
845 if (job->source.count <= kc->sub_job_size) in dm_kcopyd_copy()
854 void dm_kcopyd_zero(struct dm_kcopyd_client *kc, in dm_kcopyd_zero() argument
858 dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context); in dm_kcopyd_zero()
862 void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc, in dm_kcopyd_prepare_callback() argument
867 job = mempool_alloc(&kc->job_pool, GFP_NOIO); in dm_kcopyd_prepare_callback()
870 job->kc = kc; in dm_kcopyd_prepare_callback()
875 atomic_inc(&kc->nr_jobs); in dm_kcopyd_prepare_callback()
884 struct dm_kcopyd_client *kc = job->kc; in dm_kcopyd_do_callback() local
889 push(&kc->callback_jobs, job); in dm_kcopyd_do_callback()
890 wake(kc); in dm_kcopyd_do_callback()
913 struct dm_kcopyd_client *kc; in dm_kcopyd_client_create() local
915 kc = kzalloc(sizeof(*kc), GFP_KERNEL); in dm_kcopyd_client_create()
916 if (!kc) in dm_kcopyd_client_create()
919 spin_lock_init(&kc->job_lock); in dm_kcopyd_client_create()
920 INIT_LIST_HEAD(&kc->callback_jobs); in dm_kcopyd_client_create()
921 INIT_LIST_HEAD(&kc->complete_jobs); in dm_kcopyd_client_create()
922 INIT_LIST_HEAD(&kc->io_jobs); in dm_kcopyd_client_create()
923 INIT_LIST_HEAD(&kc->pages_jobs); in dm_kcopyd_client_create()
924 kc->throttle = throttle; in dm_kcopyd_client_create()
926 r = mempool_init_slab_pool(&kc->job_pool, MIN_JOBS, _job_cache); in dm_kcopyd_client_create()
930 INIT_WORK(&kc->kcopyd_work, do_work); in dm_kcopyd_client_create()
931 kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0); in dm_kcopyd_client_create()
932 if (!kc->kcopyd_wq) { in dm_kcopyd_client_create()
937 kc->sub_job_size = dm_get_kcopyd_subjob_size(); in dm_kcopyd_client_create()
938 reserve_pages = DIV_ROUND_UP(kc->sub_job_size << SECTOR_SHIFT, PAGE_SIZE); in dm_kcopyd_client_create()
940 kc->pages = NULL; in dm_kcopyd_client_create()
941 kc->nr_reserved_pages = kc->nr_free_pages = 0; in dm_kcopyd_client_create()
942 r = client_reserve_pages(kc, reserve_pages); in dm_kcopyd_client_create()
946 kc->io_client = dm_io_client_create(); in dm_kcopyd_client_create()
947 if (IS_ERR(kc->io_client)) { in dm_kcopyd_client_create()
948 r = PTR_ERR(kc->io_client); in dm_kcopyd_client_create()
952 init_waitqueue_head(&kc->destroyq); in dm_kcopyd_client_create()
953 atomic_set(&kc->nr_jobs, 0); in dm_kcopyd_client_create()
955 return kc; in dm_kcopyd_client_create()
958 client_free_pages(kc); in dm_kcopyd_client_create()
960 destroy_workqueue(kc->kcopyd_wq); in dm_kcopyd_client_create()
962 mempool_exit(&kc->job_pool); in dm_kcopyd_client_create()
964 kfree(kc); in dm_kcopyd_client_create()
970 void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc) in dm_kcopyd_client_destroy() argument
973 wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs)); in dm_kcopyd_client_destroy()
975 BUG_ON(!list_empty(&kc->callback_jobs)); in dm_kcopyd_client_destroy()
976 BUG_ON(!list_empty(&kc->complete_jobs)); in dm_kcopyd_client_destroy()
977 BUG_ON(!list_empty(&kc->io_jobs)); in dm_kcopyd_client_destroy()
978 BUG_ON(!list_empty(&kc->pages_jobs)); in dm_kcopyd_client_destroy()
979 destroy_workqueue(kc->kcopyd_wq); in dm_kcopyd_client_destroy()
980 dm_io_client_destroy(kc->io_client); in dm_kcopyd_client_destroy()
981 client_free_pages(kc); in dm_kcopyd_client_destroy()
982 mempool_exit(&kc->job_pool); in dm_kcopyd_client_destroy()
983 kfree(kc); in dm_kcopyd_client_destroy()