1552 static struct page **to_addr_page(struct raid5_percpu *percpu, int i)  in to_addr_page()  argument
1554 	return percpu->scribble + i * percpu->scribble_obj_size;  in to_addr_page()
1559 				 struct raid5_percpu *percpu, int i)  in to_addr_conv()  argument
1561 	return (void *) (to_addr_page(percpu, i) + sh->disks + 2);  in to_addr_conv()
1568 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)  in to_addr_offs()  argument
1570 	return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);  in to_addr_offs()
1574 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute5()  argument
1577 	struct page **xor_srcs = to_addr_page(percpu, 0);  in ops_run_compute5()
1578 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_compute5()
1604 			  ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));  in ops_run_compute5()
1671 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_1()  argument
1674 	struct page **blocks = to_addr_page(percpu, 0);  in ops_run_compute6_1()
1675 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_1()
1711 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1726 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1735 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_2()  argument
1746 	struct page **blocks = to_addr_page(percpu, 0);  in ops_run_compute6_2()
1747 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_2()
1793 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1821 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1829 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1837 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1870 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor5()  argument
1874 	struct page **xor_srcs = to_addr_page(percpu, 0);  in ops_run_prexor5()
1875 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_prexor5()
1904 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor5()
1912 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor6()  argument
1915 	struct page **blocks = to_addr_page(percpu, 0);  in ops_run_prexor6()
1916 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_prexor6()
1926 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor6()
2053 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct5()  argument
2086 	xor_srcs = to_addr_page(percpu, j);  in ops_run_reconstruct5()
2087 	off_srcs = to_addr_offs(sh, percpu);  in ops_run_reconstruct5()
2129 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2133 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2151 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct6()  argument
2180 	blocks = to_addr_page(percpu, j);  in ops_run_reconstruct6()
2181 	offs = to_addr_offs(sh, percpu);  in ops_run_reconstruct6()
2199 				  head_sh, to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2202 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2225 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_check_p()  argument
2232 	struct page **xor_srcs = to_addr_page(percpu, 0);  in ops_run_check_p()
2233 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_check_p()
2256 			  to_addr_conv(sh, percpu, 0));  in ops_run_check_p()
2266 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)  in ops_run_check_pq()  argument
2268 	struct page **srcs = to_addr_page(percpu, 0);  in ops_run_check_pq()
2269 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_check_pq()
2283 			  sh, to_addr_conv(sh, percpu, 0));  in ops_run_check_pq()
2286 			   &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);  in ops_run_check_pq()
2295 	struct raid5_percpu *percpu;  in raid_run_ops()  local
2297 	local_lock(&conf->percpu->lock);  in raid_run_ops()
2298 	percpu = this_cpu_ptr(conf->percpu);  in raid_run_ops()
2306 			tx = ops_run_compute5(sh, percpu);  in raid_run_ops()
2309 				tx = ops_run_compute6_1(sh, percpu);  in raid_run_ops()
2311 				tx = ops_run_compute6_2(sh, percpu);  in raid_run_ops()
2320 			tx = ops_run_prexor5(sh, percpu, tx);  in raid_run_ops()
2322 			tx = ops_run_prexor6(sh, percpu, tx);  in raid_run_ops()
2326 		tx = ops_run_partial_parity(sh, percpu, tx);  in raid_run_ops()
2335 			ops_run_reconstruct5(sh, percpu, tx);  in raid_run_ops()
2337 			ops_run_reconstruct6(sh, percpu, tx);  in raid_run_ops()
2342 			ops_run_check_p(sh, percpu);  in raid_run_ops()
2344 			ops_run_check_pq(sh, percpu, 0);  in raid_run_ops()
2346 			ops_run_check_pq(sh, percpu, 1);  in raid_run_ops()
2358 	local_unlock(&conf->percpu->lock);  in raid_run_ops()
2472 static int scribble_alloc(struct raid5_percpu *percpu,  in scribble_alloc()  argument
2490 	kvfree(percpu->scribble);  in scribble_alloc()
2492 	percpu->scribble = scribble;  in scribble_alloc()
2493 	percpu->scribble_obj_size = obj_size;  in scribble_alloc()
2514 		struct raid5_percpu *percpu;  in resize_chunks()  local
2516 		percpu = per_cpu_ptr(conf->percpu, cpu);  in resize_chunks()
2517 		err = scribble_alloc(percpu, new_disks,  in resize_chunks()
7358 static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)  in free_scratch_buffer()  argument
7360 	safe_put_page(percpu->spare_page);  in free_scratch_buffer()
7361 	percpu->spare_page = NULL;  in free_scratch_buffer()
7362 	kvfree(percpu->scribble);  in free_scratch_buffer()
7363 	percpu->scribble = NULL;  in free_scratch_buffer()
7366 static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)  in alloc_scratch_buffer()  argument
7368 	if (conf->level == 6 && !percpu->spare_page) {  in alloc_scratch_buffer()
7369 		percpu->spare_page = alloc_page(GFP_KERNEL);  in alloc_scratch_buffer()
7370 		if (!percpu->spare_page)  in alloc_scratch_buffer()
7374 	if (scribble_alloc(percpu,  in alloc_scratch_buffer()
7380 		free_scratch_buffer(conf, percpu);  in alloc_scratch_buffer()
7384 	local_lock_init(&percpu->lock);  in alloc_scratch_buffer()
7392 	free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));  in raid456_cpu_dead()
7398 	if (!conf->percpu)  in raid5_free_percpu()
7402 	free_percpu(conf->percpu);  in raid5_free_percpu()
7428 	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);  in raid456_cpu_up_prepare()  local
7430 	if (alloc_scratch_buffer(conf, percpu)) {  in raid456_cpu_up_prepare()
7442 	conf->percpu = alloc_percpu(struct raid5_percpu);  in raid5_alloc_percpu()
7443 	if (!conf->percpu)  in raid5_alloc_percpu()