354 static int _alloc_read_4_write(struct ore_io_state *ios)  in _alloc_read_4_write()  argument
356 	struct ore_layout *layout = ios->layout;  in _alloc_read_4_write()
361 	unsigned sgs_per_dev = ios->sp2d->pages_in_unit + 2;  in _alloc_read_4_write()
363 	ret = _ore_get_io_state(layout, ios->oc,  in _alloc_read_4_write()
365 				sgs_per_dev, 0, &ios->ios_read_4_write);  in _alloc_read_4_write()
372 static int _add_to_r4w(struct ore_io_state *ios, struct ore_striping_info *si,  in _add_to_r4w()  argument
379 			  (ios->layout->group_width * ios->layout->mirrors_p1));  in _add_to_r4w()
383 	if (!ios->ios_read_4_write) {  in _add_to_r4w()
384 		int ret = _alloc_read_4_write(ios);  in _add_to_r4w()
390 	read_ios = ios->ios_read_4_write;  in _add_to_r4w()
391 	read_ios->numdevs = ios->layout->group_width * ios->layout->mirrors_p1;  in _add_to_r4w()
396 					   ios->sp2d->pages_in_unit);  in _add_to_r4w()
399 				     ios->sp2d->pages_in_unit);  in _add_to_r4w()
423 static int _add_to_r4w_first_page(struct ore_io_state *ios, struct page *page)  in _add_to_r4w_first_page()  argument
428 	ore_calc_stripe_info(ios->layout, ios->offset, 0, &si);  in _add_to_r4w_first_page()
436 	return _add_to_r4w(ios, &si, page, pg_len);  in _add_to_r4w_first_page()
440 static int _add_to_r4w_last_page(struct ore_io_state *ios, u64 *offset)  in _add_to_r4w_last_page()  argument
446 	ore_calc_stripe_info(ios->layout, *offset, 0, &si);  in _add_to_r4w_last_page()
450 	page = ios->sp2d->_1p_stripes[p].pages[c];  in _add_to_r4w_last_page()
460 	return _add_to_r4w(ios, &si, page, pg_len);  in _add_to_r4w_last_page()
463 static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)  in _mark_read4write_pages_uptodate()  argument
469 	for (d = 0; d < ios->numdevs; d++) {  in _mark_read4write_pages_uptodate()
470 		struct bio *bio = ios->per_dev[d].bio;  in _mark_read4write_pages_uptodate()
504 static int _read_4_write_first_stripe(struct ore_io_state *ios)  in _read_4_write_first_stripe()  argument
507 	struct __stripe_pages_2d *sp2d = ios->sp2d;  in _read_4_write_first_stripe()
508 	u64 offset = ios->si.first_stripe_start;  in _read_4_write_first_stripe()
511 	if (offset == ios->offset) /* Go to start collect $200 */  in _read_4_write_first_stripe()
518 		   offset, ios->offset, min_p, max_p);  in _read_4_write_first_stripe()
521 		ore_calc_stripe_info(ios->layout, offset, 0, &read_si);  in _read_4_write_first_stripe()
530 				if (ios->offset % PAGE_SIZE)  in _read_4_write_first_stripe()
532 					_add_to_r4w_first_page(ios, *pp);  in _read_4_write_first_stripe()
537 			*pp = ios->r4w->get_page(ios->private, offset,  in _read_4_write_first_stripe()
543 				_add_to_r4w(ios, &read_si, *pp, PAGE_SIZE);  in _read_4_write_first_stripe()
557 static int _read_4_write_last_stripe(struct ore_io_state *ios)  in _read_4_write_last_stripe()  argument
560 	struct __stripe_pages_2d *sp2d = ios->sp2d;  in _read_4_write_last_stripe()
563 	unsigned bytes_in_stripe = ios->si.bytes_in_stripe;  in _read_4_write_last_stripe()
566 	offset = ios->offset + ios->length;  in _read_4_write_last_stripe()
568 		_add_to_r4w_last_page(ios, &offset);  in _read_4_write_last_stripe()
576 	ore_calc_stripe_info(ios->layout, offset, 0, &read_si);  in _read_4_write_last_stripe()
597 			page = ios->r4w->get_page(ios->private, offset,  in _read_4_write_last_stripe()
606 				_add_to_r4w(ios, &read_si, page, PAGE_SIZE);  in _read_4_write_last_stripe()
613 			ore_calc_stripe_info(ios->layout, offset, 0, &read_si);  in _read_4_write_last_stripe()
624 static int _read_4_write_execute(struct ore_io_state *ios)  in _read_4_write_execute()  argument
630 	ios_read = ios->ios_read_4_write;  in _read_4_write_execute()
637 	ios_read->pages = ios->pages;  in _read_4_write_execute()
654 	ios->ios_read_4_write = NULL; /* Might need a reuse at last stripe */  in _read_4_write_execute()
659 int _ore_add_parity_unit(struct ore_io_state *ios,  in _ore_add_parity_unit()  argument
664 	if (ios->reading) {  in _ore_add_parity_unit()
665 		if (per_dev->cur_sg >= ios->sgs_per_dev) {  in _ore_add_parity_unit()
667 				per_dev->cur_sg, ios->sgs_per_dev);  in _ore_add_parity_unit()
672 		struct __stripe_pages_2d *sp2d = ios->sp2d;  in _ore_add_parity_unit()
673 		struct page **pages = ios->parity_pages + ios->cur_par_page;  in _ore_add_parity_unit()
688 				_read_4_write_first_stripe(ios);  in _ore_add_parity_unit()
692 			_read_4_write_last_stripe(ios);  in _ore_add_parity_unit()
693 		_read_4_write_execute(ios);  in _ore_add_parity_unit()
700 			++(ios->cur_par_page);  in _ore_add_parity_unit()
706 		ret = _ore_add_stripe_unit(ios,  &array_start, 0, pages,  in _ore_add_parity_unit()
713 			_sp2d_reset(sp2d, ios->r4w, ios->private);  in _ore_add_parity_unit()
719 int _ore_post_alloc_raid_stuff(struct ore_io_state *ios)  in _ore_post_alloc_raid_stuff()  argument
721 	if (ios->parity_pages) {  in _ore_post_alloc_raid_stuff()
722 		struct ore_layout *layout = ios->layout;  in _ore_post_alloc_raid_stuff()
726 				layout->parity, &ios->sp2d)) {  in _ore_post_alloc_raid_stuff()
733 void _ore_free_raid_stuff(struct ore_io_state *ios)  in _ore_free_raid_stuff()  argument
735 	if (ios->sp2d) { /* writing and raid */  in _ore_free_raid_stuff()
738 		for (i = 0; i < ios->cur_par_page; i++) {  in _ore_free_raid_stuff()
739 			struct page *page = ios->parity_pages[i];  in _ore_free_raid_stuff()
744 		if (ios->extra_part_alloc)  in _ore_free_raid_stuff()
745 			kfree(ios->parity_pages);  in _ore_free_raid_stuff()
747 		_sp2d_reset(ios->sp2d, ios->r4w, ios->private);  in _ore_free_raid_stuff()
748 		_sp2d_free(ios->sp2d);  in _ore_free_raid_stuff()
751 		if (ios->extra_part_alloc)  in _ore_free_raid_stuff()
752 			kfree(ios->per_dev[0].sglist);  in _ore_free_raid_stuff()
754 	if (ios->ios_read_4_write)  in _ore_free_raid_stuff()
755 		ore_put_io_state(ios->ios_read_4_write);  in _ore_free_raid_stuff()