107 static inline unsigned int bs_bio_slab_size(struct bio_set *bs)  in bs_bio_slab_size()  argument
109 	return bs->front_pad + sizeof(struct bio) + bs->back_pad;  in bs_bio_slab_size()
112 static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs)  in bio_find_or_create_slab()  argument
114 	unsigned int size = bs_bio_slab_size(bs);  in bio_find_or_create_slab()
130 static void bio_put_slab(struct bio_set *bs)  in bio_put_slab()  argument
133 	unsigned int slab_size = bs_bio_slab_size(bs);  in bio_put_slab()
141 	WARN_ON_ONCE(bslab->slab != bs->bio_slab);  in bio_put_slab()
225 	struct bio_set *bs = bio->bi_pool;  in bio_free()  local
230 	if (bs) {  in bio_free()
231 		bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);  in bio_free()
237 		p -= bs->front_pad;  in bio_free()
239 		mempool_free(p, &bs->bio_pool);  in bio_free()
347 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);  in bio_alloc_rescue()  local
351 		spin_lock(&bs->rescue_lock);  in bio_alloc_rescue()
352 		bio = bio_list_pop(&bs->rescue_list);  in bio_alloc_rescue()
353 		spin_unlock(&bs->rescue_lock);  in bio_alloc_rescue()
362 static void punt_bios_to_rescuer(struct bio_set *bs)  in punt_bios_to_rescuer()  argument
367 	if (WARN_ON_ONCE(!bs->rescue_workqueue))  in punt_bios_to_rescuer()
384 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
389 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
392 	spin_lock(&bs->rescue_lock);  in punt_bios_to_rescuer()
393 	bio_list_merge(&bs->rescue_list, &punt);  in punt_bios_to_rescuer()
394 	spin_unlock(&bs->rescue_lock);  in punt_bios_to_rescuer()
396 	queue_work(bs->rescue_workqueue, &bs->rescue_work);  in punt_bios_to_rescuer()
432 			     struct bio_set *bs)  in bio_alloc_bioset()  argument
439 	if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_iovecs > 0))  in bio_alloc_bioset()
463 	    bs->rescue_workqueue)  in bio_alloc_bioset()
466 	p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
468 		punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
470 		p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
475 	bio = p + bs->front_pad;  in bio_alloc_bioset()
479 		bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);  in bio_alloc_bioset()
481 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
483 			bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);  in bio_alloc_bioset()
495 	bio->bi_pool = bs;  in bio_alloc_bioset()
499 	mempool_free(p, &bs->bio_pool);  in bio_alloc_bioset()
641 	struct bio_set *bs;  in bio_cpu_dead()  local
643 	bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead);  in bio_cpu_dead()
644 	if (bs->cache) {  in bio_cpu_dead()
645 		struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);  in bio_cpu_dead()
652 static void bio_alloc_cache_destroy(struct bio_set *bs)  in bio_alloc_cache_destroy()  argument
656 	if (!bs->cache)  in bio_alloc_cache_destroy()
659 	cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);  in bio_alloc_cache_destroy()
663 		cache = per_cpu_ptr(bs->cache, cpu);  in bio_alloc_cache_destroy()
666 	free_percpu(bs->cache);  in bio_alloc_cache_destroy()
744 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)  in bio_clone_fast()  argument
748 	b = bio_alloc_bioset(gfp_mask, 0, bs);  in bio_clone_fast()
1513 		      gfp_t gfp, struct bio_set *bs)  in bio_split()  argument
1524 	split = bio_clone_fast(bio, gfp, bs);  in bio_split()
1586 void bioset_exit(struct bio_set *bs)  in bioset_exit()  argument
1588 	bio_alloc_cache_destroy(bs);  in bioset_exit()
1589 	if (bs->rescue_workqueue)  in bioset_exit()
1590 		destroy_workqueue(bs->rescue_workqueue);  in bioset_exit()
1591 	bs->rescue_workqueue = NULL;  in bioset_exit()
1593 	mempool_exit(&bs->bio_pool);  in bioset_exit()
1594 	mempool_exit(&bs->bvec_pool);  in bioset_exit()
1596 	bioset_integrity_free(bs);  in bioset_exit()
1597 	if (bs->bio_slab)  in bioset_exit()
1598 		bio_put_slab(bs);  in bioset_exit()
1599 	bs->bio_slab = NULL;  in bioset_exit()
1624 int bioset_init(struct bio_set *bs,  in bioset_init()  argument
1629 	bs->front_pad = front_pad;  in bioset_init()
1631 		bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);  in bioset_init()
1633 		bs->back_pad = 0;  in bioset_init()
1635 	spin_lock_init(&bs->rescue_lock);  in bioset_init()
1636 	bio_list_init(&bs->rescue_list);  in bioset_init()
1637 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);  in bioset_init()
1639 	bs->bio_slab = bio_find_or_create_slab(bs);  in bioset_init()
1640 	if (!bs->bio_slab)  in bioset_init()
1643 	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))  in bioset_init()
1647 	    biovec_init_pool(&bs->bvec_pool, pool_size))  in bioset_init()
1651 		bs->rescue_workqueue = alloc_workqueue("bioset",  in bioset_init()
1653 		if (!bs->rescue_workqueue)  in bioset_init()
1657 		bs->cache = alloc_percpu(struct bio_alloc_cache);  in bioset_init()
1658 		if (!bs->cache)  in bioset_init()
1660 		cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);  in bioset_init()
1665 	bioset_exit(bs);  in bioset_init()
1674 int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)  in bioset_init_from_src()  argument
1684 	return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);  in bioset_init_from_src()
1703 			    struct bio_set *bs)  in bio_alloc_kiocb()  argument
1709 		return bio_alloc_bioset(GFP_KERNEL, nr_vecs, bs);  in bio_alloc_kiocb()
1711 	cache = per_cpu_ptr(bs->cache, get_cpu());  in bio_alloc_kiocb()
1717 		bio->bi_pool = bs;  in bio_alloc_kiocb()
1722 	bio = bio_alloc_bioset(GFP_KERNEL, nr_vecs, bs);  in bio_alloc_kiocb()