40 MODULE_PARM_DESC(kcopyd_subjob_size_kb, "Sub-job size for dm-kcopyd clients");
347 	 * Error state of the job.
367 	 * Set this to ensure you are notified when the job has
374 	 * These fields are only used if the job has been split
408  * Functions to push and pop a job onto the head of a given job
414 	struct kcopyd_job *job;  in pop_io_job()  local
420 	list_for_each_entry(job, jobs, list) {  in pop_io_job()
421 		if (job->op == REQ_OP_READ ||  in pop_io_job()
422 		    !(job->flags & BIT(DM_KCOPYD_WRITE_SEQ))) {  in pop_io_job()
423 			list_del(&job->list);  in pop_io_job()
424 			return job;  in pop_io_job()
427 		if (job->write_offset == job->master_job->write_offset) {  in pop_io_job()
428 			job->master_job->write_offset += job->source.count;  in pop_io_job()
429 			list_del(&job->list);  in pop_io_job()
430 			return job;  in pop_io_job()
440 	struct kcopyd_job *job = NULL;  in pop()  local
446 			job = pop_io_job(jobs, kc);  in pop()
448 			job = list_entry(jobs->next, struct kcopyd_job, list);  in pop()
449 			list_del(&job->list);  in pop()
454 	return job;  in pop()
457 static void push(struct list_head *jobs, struct kcopyd_job *job)  in push()  argument
460 	struct dm_kcopyd_client *kc = job->kc;  in push()
463 	list_add_tail(&job->list, jobs);  in push()
468 static void push_head(struct list_head *jobs, struct kcopyd_job *job)  in push_head()  argument
470 	struct dm_kcopyd_client *kc = job->kc;  in push_head()
473 	list_add(&job->list, jobs);  in push_head()
479  * job list.
486 static int run_complete_job(struct kcopyd_job *job)  in run_complete_job()  argument
488 	void *context = job->context;  in run_complete_job()
489 	int read_err = job->read_err;  in run_complete_job()
490 	unsigned long write_err = job->write_err;  in run_complete_job()
491 	dm_kcopyd_notify_fn fn = job->fn;  in run_complete_job()
492 	struct dm_kcopyd_client *kc = job->kc;  in run_complete_job()
494 	if (job->pages && job->pages != &zero_page_list)  in run_complete_job()
495 		kcopyd_put_pages(kc, job->pages);  in run_complete_job()
497 	 * If this is the master job, the sub jobs have already  in run_complete_job()
500 	if (job->master_job == job) {  in run_complete_job()
501 		mutex_destroy(&job->lock);  in run_complete_job()
502 		mempool_free(job, &kc->job_pool);  in run_complete_job()
516 	struct kcopyd_job *job = (struct kcopyd_job *) context;  in complete_io()  local
517 	struct dm_kcopyd_client *kc = job->kc;  in complete_io()
522 		if (op_is_write(job->op))  in complete_io()
523 			job->write_err |= error;  in complete_io()
525 			job->read_err = 1;  in complete_io()
527 		if (!(job->flags & BIT(DM_KCOPYD_IGNORE_ERROR))) {  in complete_io()
528 			push(&kc->complete_jobs, job);  in complete_io()
534 	if (op_is_write(job->op))  in complete_io()
535 		push(&kc->complete_jobs, job);  in complete_io()
538 		job->op = REQ_OP_WRITE;  in complete_io()
539 		push(&kc->io_jobs, job);  in complete_io()
547  * a particular job.
549 static int run_io_job(struct kcopyd_job *job)  in run_io_job()  argument
553 		.bi_opf = job->op,  in run_io_job()
555 		.mem.ptr.pl = job->pages,  in run_io_job()
558 		.notify.context = job,  in run_io_job()
559 		.client = job->kc->io_client,  in run_io_job()
566 	if (job->flags & BIT(DM_KCOPYD_WRITE_SEQ) &&  in run_io_job()
567 	    job->master_job->write_err) {  in run_io_job()
568 		job->write_err = job->master_job->write_err;  in run_io_job()
572 	io_job_start(job->kc->throttle);  in run_io_job()
574 	if (job->op == REQ_OP_READ)  in run_io_job()
575 		r = dm_io(&io_req, 1, &job->source, NULL);  in run_io_job()
577 		r = dm_io(&io_req, job->num_dests, job->dests, NULL);  in run_io_job()
582 static int run_pages_job(struct kcopyd_job *job)  in run_pages_job()  argument
585 	unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9);  in run_pages_job()
587 	r = kcopyd_get_pages(job->kc, nr_pages, &job->pages);  in run_pages_job()
589 		/* this job is ready for io */  in run_pages_job()
590 		push(&job->kc->io_jobs, job);  in run_pages_job()
608 	struct kcopyd_job *job;  in process_jobs()  local
611 	while ((job = pop(jobs, kc))) {  in process_jobs()
613 		r = fn(job);  in process_jobs()
616 			/* error this rogue job */  in process_jobs()
617 			if (op_is_write(job->op))  in process_jobs()
618 				job->write_err = (unsigned long) -1L;  in process_jobs()
620 				job->read_err = 1;  in process_jobs()
621 			push(&kc->complete_jobs, job);  in process_jobs()
628 			 * We couldn't service this job ATM, so  in process_jobs()
629 			 * push this job back onto the list.  in process_jobs()
631 			push_head(jobs, job);  in process_jobs()
669  * If we are copying a small region we just dispatch a single job
673 static void dispatch_job(struct kcopyd_job *job)  in dispatch_job()  argument
675 	struct dm_kcopyd_client *kc = job->kc;  in dispatch_job()
677 	if (unlikely(!job->source.count))  in dispatch_job()
678 		push(&kc->callback_jobs, job);  in dispatch_job()
679 	else if (job->pages == &zero_page_list)  in dispatch_job()
680 		push(&kc->io_jobs, job);  in dispatch_job()
682 		push(&kc->pages_jobs, job);  in dispatch_job()
693 	struct kcopyd_job *job = sub_job->master_job;  in segment_complete()  local
694 	struct dm_kcopyd_client *kc = job->kc;  in segment_complete()
696 	mutex_lock(&job->lock);  in segment_complete()
700 		job->read_err = 1;  in segment_complete()
703 		job->write_err |= write_err;  in segment_complete()
708 	if ((!job->read_err && !job->write_err) ||  in segment_complete()
709 	    job->flags & BIT(DM_KCOPYD_IGNORE_ERROR)) {  in segment_complete()
711 		progress = job->progress;  in segment_complete()
712 		count = job->source.count - progress;  in segment_complete()
717 			job->progress += count;  in segment_complete()
720 	mutex_unlock(&job->lock);  in segment_complete()
725 		*sub_job = *job;  in segment_complete()
730 		for (i = 0; i < job->num_dests; i++) {  in segment_complete()
739 	} else if (atomic_dec_and_test(&job->sub_jobs)) {  in segment_complete()
750 		push(&kc->complete_jobs, job);  in segment_complete()
775 	struct kcopyd_job *job;  in dm_kcopyd_copy()  local
779 	 * Allocate an array of jobs consisting of one master job  in dm_kcopyd_copy()
782 	job = mempool_alloc(&kc->job_pool, GFP_NOIO);  in dm_kcopyd_copy()
783 	mutex_init(&job->lock);  in dm_kcopyd_copy()
788 	job->kc = kc;  in dm_kcopyd_copy()
789 	job->flags = flags;  in dm_kcopyd_copy()
790 	job->read_err = 0;  in dm_kcopyd_copy()
791 	job->write_err = 0;  in dm_kcopyd_copy()
793 	job->num_dests = num_dests;  in dm_kcopyd_copy()
794 	memcpy(&job->dests, dests, sizeof(*dests) * num_dests);  in dm_kcopyd_copy()
801 	if (!(job->flags & BIT(DM_KCOPYD_WRITE_SEQ))) {  in dm_kcopyd_copy()
802 		for (i = 0; i < job->num_dests; i++) {  in dm_kcopyd_copy()
804 				job->flags |= BIT(DM_KCOPYD_WRITE_SEQ);  in dm_kcopyd_copy()
813 	if (job->flags & BIT(DM_KCOPYD_WRITE_SEQ) &&  in dm_kcopyd_copy()
814 	    job->flags & BIT(DM_KCOPYD_IGNORE_ERROR))  in dm_kcopyd_copy()
815 		job->flags &= ~BIT(DM_KCOPYD_IGNORE_ERROR);  in dm_kcopyd_copy()
818 		job->source = *from;  in dm_kcopyd_copy()
819 		job->pages = NULL;  in dm_kcopyd_copy()
820 		job->op = REQ_OP_READ;  in dm_kcopyd_copy()
822 		memset(&job->source, 0, sizeof job->source);  in dm_kcopyd_copy()
823 		job->source.count = job->dests[0].count;  in dm_kcopyd_copy()
824 		job->pages = &zero_page_list;  in dm_kcopyd_copy()
829 		job->op = REQ_OP_WRITE_ZEROES;  in dm_kcopyd_copy()
830 		for (i = 0; i < job->num_dests; i++)  in dm_kcopyd_copy()
831 			if (!bdev_write_zeroes_sectors(job->dests[i].bdev)) {  in dm_kcopyd_copy()
832 				job->op = REQ_OP_WRITE;  in dm_kcopyd_copy()
837 	job->fn = fn;  in dm_kcopyd_copy()
838 	job->context = context;  in dm_kcopyd_copy()
839 	job->master_job = job;  in dm_kcopyd_copy()
840 	job->write_offset = 0;  in dm_kcopyd_copy()
842 	if (job->source.count <= kc->sub_job_size)  in dm_kcopyd_copy()
843 		dispatch_job(job);  in dm_kcopyd_copy()
845 		job->progress = 0;  in dm_kcopyd_copy()
846 		split_job(job);  in dm_kcopyd_copy()
862 	struct kcopyd_job *job;  in dm_kcopyd_prepare_callback()  local
864 	job = mempool_alloc(&kc->job_pool, GFP_NOIO);  in dm_kcopyd_prepare_callback()
866 	memset(job, 0, sizeof(struct kcopyd_job));  in dm_kcopyd_prepare_callback()
867 	job->kc = kc;  in dm_kcopyd_prepare_callback()
868 	job->fn = fn;  in dm_kcopyd_prepare_callback()
869 	job->context = context;  in dm_kcopyd_prepare_callback()
870 	job->master_job = job;  in dm_kcopyd_prepare_callback()
874 	return job;  in dm_kcopyd_prepare_callback()
880 	struct kcopyd_job *job = j;  in dm_kcopyd_do_callback()  local
881 	struct dm_kcopyd_client *kc = job->kc;  in dm_kcopyd_do_callback()
883 	job->read_err = read_err;  in dm_kcopyd_do_callback()
884 	job->write_err = write_err;  in dm_kcopyd_do_callback()
886 	push(&kc->callback_jobs, job);  in dm_kcopyd_do_callback()
892  * Cancels a kcopyd job, eg. someone might be deactivating a
896 int kcopyd_cancel(struct kcopyd_job *job, int block)