118 static void bio_put_slab(struct bio_set *bs)  in bio_put_slab()  argument
126 		if (bs->bio_slab == bio_slabs[i].slab) {  in bio_put_slab()
241 	struct bio_set *bs = bio->bi_pool;  in bio_free()  local
246 	if (bs) {  in bio_free()
247 		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));  in bio_free()
253 		p -= bs->front_pad;  in bio_free()
255 		mempool_free(p, &bs->bio_pool);  in bio_free()
339 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);  in bio_alloc_rescue()  local
343 		spin_lock(&bs->rescue_lock);  in bio_alloc_rescue()
344 		bio = bio_list_pop(&bs->rescue_list);  in bio_alloc_rescue()
345 		spin_unlock(&bs->rescue_lock);  in bio_alloc_rescue()
354 static void punt_bios_to_rescuer(struct bio_set *bs)  in punt_bios_to_rescuer()  argument
359 	if (WARN_ON_ONCE(!bs->rescue_workqueue))  in punt_bios_to_rescuer()
376 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
381 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
384 	spin_lock(&bs->rescue_lock);  in punt_bios_to_rescuer()
385 	bio_list_merge(&bs->rescue_list, &punt);  in punt_bios_to_rescuer()
386 	spin_unlock(&bs->rescue_lock);  in punt_bios_to_rescuer()
388 	queue_work(bs->rescue_workqueue, &bs->rescue_work);  in punt_bios_to_rescuer()
427 			     struct bio_set *bs)  in bio_alloc_bioset()  argument
436 	if (!bs) {  in bio_alloc_bioset()
447 		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&  in bio_alloc_bioset()
474 		    bs->rescue_workqueue)  in bio_alloc_bioset()
477 		p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
479 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
481 			p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
484 		front_pad = bs->front_pad;  in bio_alloc_bioset()
497 		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
499 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
501 			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
512 	bio->bi_pool = bs;  in bio_alloc_bioset()
518 	mempool_free(p, &bs->bio_pool);  in bio_alloc_bioset()
605 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)  in bio_clone_fast()  argument
609 	b = bio_alloc_bioset(gfp_mask, 0, bs);  in bio_clone_fast()
1837 		      gfp_t gfp, struct bio_set *bs)  in bio_split()  argument
1844 	split = bio_clone_fast(bio, gfp, bs);  in bio_split()
1904 void bioset_exit(struct bio_set *bs)  in bioset_exit()  argument
1906 	if (bs->rescue_workqueue)  in bioset_exit()
1907 		destroy_workqueue(bs->rescue_workqueue);  in bioset_exit()
1908 	bs->rescue_workqueue = NULL;  in bioset_exit()
1910 	mempool_exit(&bs->bio_pool);  in bioset_exit()
1911 	mempool_exit(&bs->bvec_pool);  in bioset_exit()
1913 	bioset_integrity_free(bs);  in bioset_exit()
1914 	if (bs->bio_slab)  in bioset_exit()
1915 		bio_put_slab(bs);  in bioset_exit()
1916 	bs->bio_slab = NULL;  in bioset_exit()
1941 int bioset_init(struct bio_set *bs,  in bioset_init()  argument
1948 	bs->front_pad = front_pad;  in bioset_init()
1950 	spin_lock_init(&bs->rescue_lock);  in bioset_init()
1951 	bio_list_init(&bs->rescue_list);  in bioset_init()
1952 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);  in bioset_init()
1954 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);  in bioset_init()
1955 	if (!bs->bio_slab)  in bioset_init()
1958 	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))  in bioset_init()
1962 	    biovec_init_pool(&bs->bvec_pool, pool_size))  in bioset_init()
1968 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);  in bioset_init()
1969 	if (!bs->rescue_workqueue)  in bioset_init()
1974 	bioset_exit(bs);  in bioset_init()
1983 int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)  in bioset_init_from_src()  argument
1993 	return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);  in bioset_init_from_src()