130 static void bio_put_slab(struct bio_set *bs)  in bio_put_slab()  argument
138 		if (bs->bio_slab == bio_slabs[i].slab) {  in bio_put_slab()
253 	struct bio_set *bs = bio->bi_pool;  in bio_free()  local
258 	if (bs) {  in bio_free()
259 		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));  in bio_free()
265 		p -= bs->front_pad;  in bio_free()
267 		mempool_free(p, &bs->bio_pool);  in bio_free()
351 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);  in bio_alloc_rescue()  local
355 		spin_lock(&bs->rescue_lock);  in bio_alloc_rescue()
356 		bio = bio_list_pop(&bs->rescue_list);  in bio_alloc_rescue()
357 		spin_unlock(&bs->rescue_lock);  in bio_alloc_rescue()
366 static void punt_bios_to_rescuer(struct bio_set *bs)  in punt_bios_to_rescuer()  argument
371 	if (WARN_ON_ONCE(!bs->rescue_workqueue))  in punt_bios_to_rescuer()
388 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
393 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
396 	spin_lock(&bs->rescue_lock);  in punt_bios_to_rescuer()
397 	bio_list_merge(&bs->rescue_list, &punt);  in punt_bios_to_rescuer()
398 	spin_unlock(&bs->rescue_lock);  in punt_bios_to_rescuer()
400 	queue_work(bs->rescue_workqueue, &bs->rescue_work);  in punt_bios_to_rescuer()
439 			     struct bio_set *bs)  in bio_alloc_bioset()  argument
448 	if (!bs) {  in bio_alloc_bioset()
459 		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&  in bio_alloc_bioset()
486 		    bs->rescue_workqueue)  in bio_alloc_bioset()
489 		p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
491 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
493 			p = mempool_alloc(&bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
496 		front_pad = bs->front_pad;  in bio_alloc_bioset()
509 		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
511 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
513 			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);  in bio_alloc_bioset()
524 	bio->bi_pool = bs;  in bio_alloc_bioset()
530 	mempool_free(p, &bs->bio_pool);  in bio_alloc_bioset()
624 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)  in bio_clone_fast()  argument
628 	b = bio_alloc_bioset(gfp_mask, 0, bs);  in bio_clone_fast()
1793 		      gfp_t gfp, struct bio_set *bs)  in bio_split()  argument
1800 	split = bio_clone_fast(bio, gfp, bs);  in bio_split()
1864 void bioset_exit(struct bio_set *bs)  in bioset_exit()  argument
1866 	if (bs->rescue_workqueue)  in bioset_exit()
1867 		destroy_workqueue(bs->rescue_workqueue);  in bioset_exit()
1868 	bs->rescue_workqueue = NULL;  in bioset_exit()
1870 	mempool_exit(&bs->bio_pool);  in bioset_exit()
1871 	mempool_exit(&bs->bvec_pool);  in bioset_exit()
1873 	bioset_integrity_free(bs);  in bioset_exit()
1874 	if (bs->bio_slab)  in bioset_exit()
1875 		bio_put_slab(bs);  in bioset_exit()
1876 	bs->bio_slab = NULL;  in bioset_exit()
1901 int bioset_init(struct bio_set *bs,  in bioset_init()  argument
1908 	bs->front_pad = front_pad;  in bioset_init()
1910 	spin_lock_init(&bs->rescue_lock);  in bioset_init()
1911 	bio_list_init(&bs->rescue_list);  in bioset_init()
1912 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);  in bioset_init()
1914 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);  in bioset_init()
1915 	if (!bs->bio_slab)  in bioset_init()
1918 	if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))  in bioset_init()
1922 	    biovec_init_pool(&bs->bvec_pool, pool_size))  in bioset_init()
1928 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);  in bioset_init()
1929 	if (!bs->rescue_workqueue)  in bioset_init()
1934 	bioset_exit(bs);  in bioset_init()
1943 int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)  in bioset_init_from_src()  argument
1953 	return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);  in bioset_init_from_src()