Lines Matching refs:sdio
158 static inline unsigned dio_pages_present(struct dio_submit *sdio) in dio_pages_present() argument
160 return sdio->tail - sdio->head; in dio_pages_present()
166 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
170 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
171 &sdio->from); in dio_refill_pages()
173 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
184 sdio->head = 0; in dio_refill_pages()
185 sdio->tail = 1; in dio_refill_pages()
186 sdio->from = 0; in dio_refill_pages()
187 sdio->to = PAGE_SIZE; in dio_refill_pages()
192 iov_iter_advance(sdio->iter, ret); in dio_refill_pages()
193 ret += sdio->from; in dio_refill_pages()
194 sdio->head = 0; in dio_refill_pages()
195 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
196 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
209 struct dio_submit *sdio) in dio_get_page() argument
211 if (dio_pages_present(sdio) == 0) { in dio_get_page()
214 ret = dio_refill_pages(dio, sdio); in dio_get_page()
217 BUG_ON(dio_pages_present(sdio) == 0); in dio_get_page()
219 return dio->pages[sdio->head]; in dio_get_page()
428 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
450 sdio->bio = bio; in dio_bio_alloc()
451 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
461 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
463 struct bio *bio = sdio->bio; in dio_bio_submit()
477 if (sdio->submit_io) { in dio_bio_submit()
478 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
483 sdio->bio = NULL; in dio_bio_submit()
484 sdio->boundary = 0; in dio_bio_submit()
485 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
491 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
493 while (sdio->head < sdio->tail) in dio_cleanup()
494 put_page(dio->pages[sdio->head++]); in dio_cleanup()
591 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
595 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
609 sdio->reap_counter = 0; in dio_bio_reap()
673 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
681 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
689 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
690 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
691 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
692 sdio->blkfactor; in get_more_blocks()
716 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
731 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
737 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
740 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
741 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); in dio_new_bio()
743 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
744 sdio->boundary = 0; in dio_new_bio()
756 static inline int dio_bio_add_page(struct dio_submit *sdio) in dio_bio_add_page() argument
760 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
761 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
762 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
766 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
767 sdio->pages_in_io--; in dio_bio_add_page()
768 get_page(sdio->cur_page); in dio_bio_add_page()
769 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
770 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
788 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
793 if (sdio->bio) { in dio_send_cur_page()
794 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
795 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
796 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
812 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
814 dio_bio_submit(dio, sdio); in dio_send_cur_page()
817 if (sdio->bio == NULL) { in dio_send_cur_page()
818 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
823 if (dio_bio_add_page(sdio) != 0) { in dio_send_cur_page()
824 dio_bio_submit(dio, sdio); in dio_send_cur_page()
825 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
827 ret = dio_bio_add_page(sdio); in dio_send_cur_page()
853 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
869 if (sdio->cur_page == page && in submit_page_section()
870 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
871 sdio->cur_page_block + in submit_page_section()
872 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
873 sdio->cur_page_len += len; in submit_page_section()
880 if (sdio->cur_page) { in submit_page_section()
881 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
882 put_page(sdio->cur_page); in submit_page_section()
883 sdio->cur_page = NULL; in submit_page_section()
889 sdio->cur_page = page; in submit_page_section()
890 sdio->cur_page_offset = offset; in submit_page_section()
891 sdio->cur_page_len = len; in submit_page_section()
892 sdio->cur_page_block = blocknr; in submit_page_section()
893 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
899 if (sdio->boundary) { in submit_page_section()
900 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
901 if (sdio->bio) in submit_page_section()
902 dio_bio_submit(dio, sdio); in submit_page_section()
903 put_page(sdio->cur_page); in submit_page_section()
904 sdio->cur_page = NULL; in submit_page_section()
918 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
926 sdio->start_zero_done = 1; in dio_zero_block()
927 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
930 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
931 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
943 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
946 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
947 sdio->next_block_for_io, map_bh)) in dio_zero_block()
950 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
969 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
972 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
973 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
976 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
980 page = dio_get_page(dio, sdio); in do_direct_IO()
985 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
986 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
987 sdio->head++; in do_direct_IO()
994 if (sdio->blocks_available == 0) { in do_direct_IO()
1001 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
1009 sdio->blocks_available = in do_direct_IO()
1011 sdio->next_block_for_io = in do_direct_IO()
1012 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
1020 if (!sdio->blkfactor) in do_direct_IO()
1023 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
1024 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
1038 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
1039 sdio->blocks_available -= dio_remainder; in do_direct_IO()
1058 if (sdio->block_in_file >= in do_direct_IO()
1065 sdio->block_in_file++; in do_direct_IO()
1076 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1077 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1083 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1087 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1093 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1094 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1095 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1098 sdio->next_block_for_io, in do_direct_IO()
1104 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1106 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1109 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1111 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1112 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1184 struct dio_submit sdio = { 0, }; in do_blockdev_direct_IO() local
1301 sdio.blkbits = blkbits; in do_blockdev_direct_IO()
1302 sdio.blkfactor = i_blkbits - blkbits; in do_blockdev_direct_IO()
1303 sdio.block_in_file = offset >> blkbits; in do_blockdev_direct_IO()
1305 sdio.get_block = get_block; in do_blockdev_direct_IO()
1307 sdio.submit_io = submit_io; in do_blockdev_direct_IO()
1308 sdio.final_block_in_bio = -1; in do_blockdev_direct_IO()
1309 sdio.next_block_for_io = -1; in do_blockdev_direct_IO()
1317 sdio.iter = iter; in do_blockdev_direct_IO()
1318 sdio.final_block_in_request = end >> blkbits; in do_blockdev_direct_IO()
1324 if (unlikely(sdio.blkfactor)) in do_blockdev_direct_IO()
1325 sdio.pages_in_io = 2; in do_blockdev_direct_IO()
1327 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); in do_blockdev_direct_IO()
1331 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1333 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1346 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1348 if (sdio.cur_page) { in do_blockdev_direct_IO()
1351 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1354 put_page(sdio.cur_page); in do_blockdev_direct_IO()
1355 sdio.cur_page = NULL; in do_blockdev_direct_IO()
1357 if (sdio.bio) in do_blockdev_direct_IO()
1358 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1366 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()