40 MODULE_PARM_DESC(kcopyd_subjob_size_kb, "Sub-job size for dm-kcopyd clients");
347 	 * Error state of the job.
367 	 * Set this to ensure you are notified when the job has
374 	 * These fields are only used if the job has been split
408  * Functions to push and pop a job onto the head of a given job
414 	struct kcopyd_job *job;  in pop_io_job()  local
420 	list_for_each_entry(job, jobs, list) {  in pop_io_job()
421 		if (job->rw == READ || !test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags)) {  in pop_io_job()
422 			list_del(&job->list);  in pop_io_job()
423 			return job;  in pop_io_job()
426 		if (job->write_offset == job->master_job->write_offset) {  in pop_io_job()
427 			job->master_job->write_offset += job->source.count;  in pop_io_job()
428 			list_del(&job->list);  in pop_io_job()
429 			return job;  in pop_io_job()
439 	struct kcopyd_job *job = NULL;  in pop()  local
446 			job = pop_io_job(jobs, kc);  in pop()
448 			job = list_entry(jobs->next, struct kcopyd_job, list);  in pop()
449 			list_del(&job->list);  in pop()
454 	return job;  in pop()
457 static void push(struct list_head *jobs, struct kcopyd_job *job)  in push()  argument
460 	struct dm_kcopyd_client *kc = job->kc;  in push()
463 	list_add_tail(&job->list, jobs);  in push()
468 static void push_head(struct list_head *jobs, struct kcopyd_job *job)  in push_head()  argument
471 	struct dm_kcopyd_client *kc = job->kc;  in push_head()
474 	list_add(&job->list, jobs);  in push_head()
480  * job list.
487 static int run_complete_job(struct kcopyd_job *job)  in run_complete_job()  argument
489 	void *context = job->context;  in run_complete_job()
490 	int read_err = job->read_err;  in run_complete_job()
491 	unsigned long write_err = job->write_err;  in run_complete_job()
492 	dm_kcopyd_notify_fn fn = job->fn;  in run_complete_job()
493 	struct dm_kcopyd_client *kc = job->kc;  in run_complete_job()
495 	if (job->pages && job->pages != &zero_page_list)  in run_complete_job()
496 		kcopyd_put_pages(kc, job->pages);  in run_complete_job()
498 	 * If this is the master job, the sub jobs have already  in run_complete_job()
501 	if (job->master_job == job) {  in run_complete_job()
502 		mutex_destroy(&job->lock);  in run_complete_job()
503 		mempool_free(job, &kc->job_pool);  in run_complete_job()
517 	struct kcopyd_job *job = (struct kcopyd_job *) context;  in complete_io()  local
518 	struct dm_kcopyd_client *kc = job->kc;  in complete_io()
523 		if (op_is_write(job->rw))  in complete_io()
524 			job->write_err |= error;  in complete_io()
526 			job->read_err = 1;  in complete_io()
528 		if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {  in complete_io()
529 			push(&kc->complete_jobs, job);  in complete_io()
535 	if (op_is_write(job->rw))  in complete_io()
536 		push(&kc->complete_jobs, job);  in complete_io()
539 		job->rw = WRITE;  in complete_io()
540 		push(&kc->io_jobs, job);  in complete_io()
548  * a particular job.
550 static int run_io_job(struct kcopyd_job *job)  in run_io_job()  argument
554 		.bi_op = job->rw,  in run_io_job()
557 		.mem.ptr.pl = job->pages,  in run_io_job()
560 		.notify.context = job,  in run_io_job()
561 		.client = job->kc->io_client,  in run_io_job()
568 	if (test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags) &&  in run_io_job()
569 	    job->master_job->write_err) {  in run_io_job()
570 		job->write_err = job->master_job->write_err;  in run_io_job()
574 	io_job_start(job->kc->throttle);  in run_io_job()
576 	if (job->rw == READ)  in run_io_job()
577 		r = dm_io(&io_req, 1, &job->source, NULL);  in run_io_job()
579 		r = dm_io(&io_req, job->num_dests, job->dests, NULL);  in run_io_job()
584 static int run_pages_job(struct kcopyd_job *job)  in run_pages_job()  argument
587 	unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9);  in run_pages_job()
589 	r = kcopyd_get_pages(job->kc, nr_pages, &job->pages);  in run_pages_job()
591 		/* this job is ready for io */  in run_pages_job()
592 		push(&job->kc->io_jobs, job);  in run_pages_job()
610 	struct kcopyd_job *job;  in process_jobs()  local
613 	while ((job = pop(jobs, kc))) {  in process_jobs()
615 		r = fn(job);  in process_jobs()
618 			/* error this rogue job */  in process_jobs()
619 			if (op_is_write(job->rw))  in process_jobs()
620 				job->write_err = (unsigned long) -1L;  in process_jobs()
622 				job->read_err = 1;  in process_jobs()
623 			push(&kc->complete_jobs, job);  in process_jobs()
630 			 * We couldn't service this job ATM, so  in process_jobs()
631 			 * push this job back onto the list.  in process_jobs()
633 			push_head(jobs, job);  in process_jobs()
672  * If we are copying a small region we just dispatch a single job
676 static void dispatch_job(struct kcopyd_job *job)  in dispatch_job()  argument
678 	struct dm_kcopyd_client *kc = job->kc;  in dispatch_job()
680 	if (unlikely(!job->source.count))  in dispatch_job()
681 		push(&kc->callback_jobs, job);  in dispatch_job()
682 	else if (job->pages == &zero_page_list)  in dispatch_job()
683 		push(&kc->io_jobs, job);  in dispatch_job()
685 		push(&kc->pages_jobs, job);  in dispatch_job()
696 	struct kcopyd_job *job = sub_job->master_job;  in segment_complete()  local
697 	struct dm_kcopyd_client *kc = job->kc;  in segment_complete()
699 	mutex_lock(&job->lock);  in segment_complete()
703 		job->read_err = 1;  in segment_complete()
706 		job->write_err |= write_err;  in segment_complete()
711 	if ((!job->read_err && !job->write_err) ||  in segment_complete()
712 	    test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {  in segment_complete()
714 		progress = job->progress;  in segment_complete()
715 		count = job->source.count - progress;  in segment_complete()
720 			job->progress += count;  in segment_complete()
723 	mutex_unlock(&job->lock);  in segment_complete()
728 		*sub_job = *job;  in segment_complete()
733 		for (i = 0; i < job->num_dests; i++) {  in segment_complete()
742 	} else if (atomic_dec_and_test(&job->sub_jobs)) {  in segment_complete()
753 		push(&kc->complete_jobs, job);  in segment_complete()
778 	struct kcopyd_job *job;  in dm_kcopyd_copy()  local
782 	 * Allocate an array of jobs consisting of one master job  in dm_kcopyd_copy()
785 	job = mempool_alloc(&kc->job_pool, GFP_NOIO);  in dm_kcopyd_copy()
786 	mutex_init(&job->lock);  in dm_kcopyd_copy()
791 	job->kc = kc;  in dm_kcopyd_copy()
792 	job->flags = flags;  in dm_kcopyd_copy()
793 	job->read_err = 0;  in dm_kcopyd_copy()
794 	job->write_err = 0;  in dm_kcopyd_copy()
796 	job->num_dests = num_dests;  in dm_kcopyd_copy()
797 	memcpy(&job->dests, dests, sizeof(*dests) * num_dests);  in dm_kcopyd_copy()
804 	if (!test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags)) {  in dm_kcopyd_copy()
805 		for (i = 0; i < job->num_dests; i++) {  in dm_kcopyd_copy()
807 				set_bit(DM_KCOPYD_WRITE_SEQ, &job->flags);  in dm_kcopyd_copy()
816 	if (test_bit(DM_KCOPYD_WRITE_SEQ, &job->flags) &&  in dm_kcopyd_copy()
817 	    test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags))  in dm_kcopyd_copy()
818 		clear_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags);  in dm_kcopyd_copy()
821 		job->source = *from;  in dm_kcopyd_copy()
822 		job->pages = NULL;  in dm_kcopyd_copy()
823 		job->rw = READ;  in dm_kcopyd_copy()
825 		memset(&job->source, 0, sizeof job->source);  in dm_kcopyd_copy()
826 		job->source.count = job->dests[0].count;  in dm_kcopyd_copy()
827 		job->pages = &zero_page_list;  in dm_kcopyd_copy()
832 		job->rw = REQ_OP_WRITE_ZEROES;  in dm_kcopyd_copy()
833 		for (i = 0; i < job->num_dests; i++)  in dm_kcopyd_copy()
834 			if (!bdev_write_zeroes_sectors(job->dests[i].bdev)) {  in dm_kcopyd_copy()
835 				job->rw = WRITE;  in dm_kcopyd_copy()
840 	job->fn = fn;  in dm_kcopyd_copy()
841 	job->context = context;  in dm_kcopyd_copy()
842 	job->master_job = job;  in dm_kcopyd_copy()
843 	job->write_offset = 0;  in dm_kcopyd_copy()
845 	if (job->source.count <= kc->sub_job_size)  in dm_kcopyd_copy()
846 		dispatch_job(job);  in dm_kcopyd_copy()
848 		job->progress = 0;  in dm_kcopyd_copy()
849 		split_job(job);  in dm_kcopyd_copy()
865 	struct kcopyd_job *job;  in dm_kcopyd_prepare_callback()  local
867 	job = mempool_alloc(&kc->job_pool, GFP_NOIO);  in dm_kcopyd_prepare_callback()
869 	memset(job, 0, sizeof(struct kcopyd_job));  in dm_kcopyd_prepare_callback()
870 	job->kc = kc;  in dm_kcopyd_prepare_callback()
871 	job->fn = fn;  in dm_kcopyd_prepare_callback()
872 	job->context = context;  in dm_kcopyd_prepare_callback()
873 	job->master_job = job;  in dm_kcopyd_prepare_callback()
877 	return job;  in dm_kcopyd_prepare_callback()
883 	struct kcopyd_job *job = j;  in dm_kcopyd_do_callback()  local
884 	struct dm_kcopyd_client *kc = job->kc;  in dm_kcopyd_do_callback()
886 	job->read_err = read_err;  in dm_kcopyd_do_callback()
887 	job->write_err = write_err;  in dm_kcopyd_do_callback()
889 	push(&kc->callback_jobs, job);  in dm_kcopyd_do_callback()
895  * Cancels a kcopyd job, eg. someone might be deactivating a
899 int kcopyd_cancel(struct kcopyd_job *job, int block)