Lines Matching full:io

18 #include <linux/dm-io.h>
20 #define DM_MSG_PREFIX "io"
30 * Aligning 'struct io' reduces the number of bits required to store
33 struct io { struct
84 * We need to keep track of which region a bio is doing io for.
86 * ensure the 'struct io' pointer is aligned so enough low bits are
90 static void store_io_and_region_in_bio(struct bio *bio, struct io *io, in store_io_and_region_in_bio() argument
93 if (unlikely(!IS_ALIGNED((unsigned long)io, DM_IO_MAX_REGIONS))) { in store_io_and_region_in_bio()
94 DMCRIT("Unaligned struct io pointer %p", io); in store_io_and_region_in_bio()
98 bio->bi_private = (void *)((unsigned long)io | region); in store_io_and_region_in_bio()
101 static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io, in retrieve_io_and_region_from_bio() argument
106 *io = (void *)(val & -(unsigned long)DM_IO_MAX_REGIONS); in retrieve_io_and_region_from_bio()
111 * We need an io object to keep track of the number of bios that
112 * have been dispatched for a particular io.
114 static void complete_io(struct io *io) in complete_io() argument
116 unsigned long error_bits = io->error_bits; in complete_io()
117 io_notify_fn fn = io->callback; in complete_io()
118 void *context = io->context; in complete_io()
120 if (io->vma_invalidate_size) in complete_io()
121 invalidate_kernel_vmap_range(io->vma_invalidate_address, in complete_io()
122 io->vma_invalidate_size); in complete_io()
124 mempool_free(io, &io->client->pool); in complete_io()
128 static void dec_count(struct io *io, unsigned int region, blk_status_t error) in dec_count() argument
131 set_bit(region, &io->error_bits); in dec_count()
133 if (atomic_dec_and_test(&io->count)) in dec_count()
134 complete_io(io); in dec_count()
139 struct io *io; in endio() local
147 * The bio destructor in bio_put() may use the io object. in endio()
149 retrieve_io_and_region_from_bio(bio, &io, &region); in endio()
154 dec_count(io, region, error); in endio()
159 * destination page for io.
294 * IO routines that accept a list of pages.
298 struct io *io) in do_region() argument
320 atomic_inc(&io->count); in do_region()
321 dec_count(io, region, BLK_STS_NOTSUPP); in do_region()
344 &io->client->bios); in do_region()
347 store_io_and_region_in_bio(bio, io, region); in do_region()
367 atomic_inc(&io->count); in do_region()
374 struct io *io, int sync) in dispatch_io() argument
391 do_region(opf, i, where + i, dp, io); in dispatch_io()
396 * the io being completed too early. in dispatch_io()
398 dec_count(io, 0, 0); in dispatch_io()
418 struct io *io; in sync_io() local
428 io = mempool_alloc(&client->pool, GFP_NOIO); in sync_io()
429 io->error_bits = 0; in sync_io()
430 atomic_set(&io->count, 1); /* see dispatch_io() */ in sync_io()
431 io->client = client; in sync_io()
432 io->callback = sync_io_complete; in sync_io()
433 io->context = &sio; in sync_io()
435 io->vma_invalidate_address = dp->vma_invalidate_address; in sync_io()
436 io->vma_invalidate_size = dp->vma_invalidate_size; in sync_io()
438 dispatch_io(opf, num_regions, where, dp, io, 1); in sync_io()
452 struct io *io; in async_io() local
460 io = mempool_alloc(&client->pool, GFP_NOIO); in async_io()
461 io->error_bits = 0; in async_io()
462 atomic_set(&io->count, 1); /* see dispatch_io() */ in async_io()
463 io->client = client; in async_io()
464 io->callback = fn; in async_io()
465 io->context = context; in async_io()
467 io->vma_invalidate_address = dp->vma_invalidate_address; in async_io()
468 io->vma_invalidate_size = dp->vma_invalidate_size; in async_io()
470 dispatch_io(opf, num_regions, where, dp, io, 0); in async_io()
533 _dm_io_cache = KMEM_CACHE(io, 0); in dm_io_init()