263 static void btt_debugfs_init(struct btt *btt)  in btt_debugfs_init()  argument
268 	btt->debugfs_dir = debugfs_create_dir(dev_name(&btt->nd_btt->dev),  in btt_debugfs_init()
270 	if (IS_ERR_OR_NULL(btt->debugfs_dir))  in btt_debugfs_init()
273 	list_for_each_entry(arena, &btt->arena_list, list) {  in btt_debugfs_init()
274 		arena_debugfs_init(arena, btt->debugfs_dir, i);  in btt_debugfs_init()
744 static struct arena_info *alloc_arena(struct btt *btt, size_t size,  in alloc_arena()  argument
754 	arena->nd_btt = btt->nd_btt;  in alloc_arena()
755 	arena->sector_size = btt->sector_size;  in alloc_arena()
763 	arena->external_lbasize = btt->lbasize;  in alloc_arena()
767 	arena->version_major = btt->nd_btt->version_major;  in alloc_arena()
768 	arena->version_minor = btt->nd_btt->version_minor;  in alloc_arena()
801 static void free_arenas(struct btt *btt)  in free_arenas()  argument
805 	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {  in free_arenas()
845 static int discover_arenas(struct btt *btt)  in discover_arenas()  argument
850 	size_t remaining = btt->rawsize;  in discover_arenas()
861 		arena = alloc_arena(btt, 0, 0, 0);  in discover_arenas()
872 		if (!nd_btt_arena_is_valid(btt->nd_btt, super)) {  in discover_arenas()
873 			if (remaining == btt->rawsize) {  in discover_arenas()
874 				btt->init_state = INIT_NOTFOUND;  in discover_arenas()
907 		list_add_tail(&arena->list, &btt->arena_list);  in discover_arenas()
917 	btt->num_arenas = num_arenas;  in discover_arenas()
918 	btt->nlba = cur_nlba;  in discover_arenas()
919 	btt->init_state = INIT_READY;  in discover_arenas()
926 	free_arenas(btt);  in discover_arenas()
932 static int create_arenas(struct btt *btt)  in create_arenas()  argument
934 	size_t remaining = btt->rawsize;  in create_arenas()
945 		arena = alloc_arena(btt, arena_size, btt->nlba, cur_off);  in create_arenas()
947 			free_arenas(btt);  in create_arenas()
950 		btt->nlba += arena->external_nlba;  in create_arenas()
956 		list_add_tail(&arena->list, &btt->arena_list);  in create_arenas()
1024 static int btt_meta_init(struct btt *btt)  in btt_meta_init()  argument
1029 	mutex_lock(&btt->init_lock);  in btt_meta_init()
1030 	list_for_each_entry(arena, &btt->arena_list, list) {  in btt_meta_init()
1048 	btt->init_state = INIT_READY;  in btt_meta_init()
1051 	mutex_unlock(&btt->init_lock);  in btt_meta_init()
1055 static u32 btt_meta_size(struct btt *btt)  in btt_meta_size()  argument
1057 	return btt->lbasize - btt->sector_size;  in btt_meta_size()
1067 static int lba_to_arena(struct btt *btt, sector_t sector, __u32 *premap,  in lba_to_arena()  argument
1071 	__u64 lba = div_u64(sector << SECTOR_SHIFT, btt->sector_size);  in lba_to_arena()
1073 	list_for_each_entry(arena_list, &btt->arena_list, list) {  in lba_to_arena()
1140 static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip,  in btt_rw_integrity()  argument
1143 	unsigned int len = btt_meta_size(btt);  in btt_rw_integrity()
1150 	meta_nsoff = to_namespace_offset(arena, postmap) + btt->sector_size;  in btt_rw_integrity()
1189 static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip,  in btt_rw_integrity()  argument
1196 static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip,  in btt_read_pg()  argument
1208 		lane = nd_region_acquire_lane(btt->nd_region);  in btt_read_pg()
1210 		ret = lba_to_arena(btt, sector, &premap, &arena);  in btt_read_pg()
1214 		cur_len = min(btt->sector_size, len);  in btt_read_pg()
1272 			ret = btt_rw_integrity(btt, bip, arena, postmap, READ);  in btt_read_pg()
1278 		nd_region_release_lane(btt->nd_region, lane);  in btt_read_pg()
1282 		sector += btt->sector_size >> SECTOR_SHIFT;  in btt_read_pg()
1290 	nd_region_release_lane(btt->nd_region, lane);  in btt_read_pg()
1299 static bool btt_is_badblock(struct btt *btt, struct arena_info *arena,  in btt_is_badblock()  argument
1306 	return is_bad_pmem(btt->phys_bb, phys_sector, arena->internal_lbasize);  in btt_is_badblock()
1309 static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,  in btt_write_pg()  argument
1324 		lane = nd_region_acquire_lane(btt->nd_region);  in btt_write_pg()
1326 		ret = lba_to_arena(btt, sector, &premap, &arena);  in btt_write_pg()
1329 		cur_len = min(btt->sector_size, len);  in btt_write_pg()
1336 		if (btt_is_badblock(btt, arena, arena->freelist[lane].block))  in btt_write_pg()
1341 			nd_region_release_lane(btt->nd_region, lane);  in btt_write_pg()
1369 			ret = btt_rw_integrity(btt, bip, arena, new_postmap,  in btt_write_pg()
1402 		nd_region_release_lane(btt->nd_region, lane);  in btt_write_pg()
1412 		sector += btt->sector_size >> SECTOR_SHIFT;  in btt_write_pg()
1420 	nd_region_release_lane(btt->nd_region, lane);  in btt_write_pg()
1424 static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip,  in btt_do_bvec()  argument
1431 		ret = btt_read_pg(btt, bip, page, off, sector, len);  in btt_do_bvec()
1435 		ret = btt_write_pg(btt, bip, sector, page, off, len);  in btt_do_bvec()
1444 	struct btt *btt = q->queuedata;  in btt_make_request()  local
1458 		if (len > PAGE_SIZE || len < btt->sector_size ||  in btt_make_request()
1459 				len % btt->sector_size) {  in btt_make_request()
1460 			dev_err_ratelimited(&btt->nd_btt->dev,  in btt_make_request()
1466 		err = btt_do_bvec(btt, bip, bvec.bv_page, len, bvec.bv_offset,  in btt_make_request()
1469 			dev_err(&btt->nd_btt->dev,  in btt_make_request()
1488 	struct btt *btt = bdev->bd_disk->private_data;  in btt_rw_page()  local
1493 	rc = btt_do_bvec(btt, NULL, page, len, 0, op, sector);  in btt_rw_page()
1517 static int btt_blk_init(struct btt *btt)  in btt_blk_init()  argument
1519 	struct nd_btt *nd_btt = btt->nd_btt;  in btt_blk_init()
1523 	btt->btt_queue = blk_alloc_queue(GFP_KERNEL);  in btt_blk_init()
1524 	if (!btt->btt_queue)  in btt_blk_init()
1527 	btt->btt_disk = alloc_disk(0);  in btt_blk_init()
1528 	if (!btt->btt_disk) {  in btt_blk_init()
1529 		blk_cleanup_queue(btt->btt_queue);  in btt_blk_init()
1533 	nvdimm_namespace_disk_name(ndns, btt->btt_disk->disk_name);  in btt_blk_init()
1534 	btt->btt_disk->first_minor = 0;  in btt_blk_init()
1535 	btt->btt_disk->fops = &btt_fops;  in btt_blk_init()
1536 	btt->btt_disk->private_data = btt;  in btt_blk_init()
1537 	btt->btt_disk->queue = btt->btt_queue;  in btt_blk_init()
1538 	btt->btt_disk->flags = GENHD_FL_EXT_DEVT;  in btt_blk_init()
1539 	btt->btt_disk->queue->backing_dev_info->capabilities |=  in btt_blk_init()
1542 	blk_queue_make_request(btt->btt_queue, btt_make_request);  in btt_blk_init()
1543 	blk_queue_logical_block_size(btt->btt_queue, btt->sector_size);  in btt_blk_init()
1544 	blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX);  in btt_blk_init()
1545 	blk_queue_flag_set(QUEUE_FLAG_NONROT, btt->btt_queue);  in btt_blk_init()
1546 	btt->btt_queue->queuedata = btt;  in btt_blk_init()
1548 	if (btt_meta_size(btt)) {  in btt_blk_init()
1549 		int rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt));  in btt_blk_init()
1552 			del_gendisk(btt->btt_disk);  in btt_blk_init()
1553 			put_disk(btt->btt_disk);  in btt_blk_init()
1554 			blk_cleanup_queue(btt->btt_queue);  in btt_blk_init()
1558 	set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9);  in btt_blk_init()
1559 	device_add_disk(&btt->nd_btt->dev, btt->btt_disk);  in btt_blk_init()
1560 	btt->nd_btt->size = btt->nlba * (u64)btt->sector_size;  in btt_blk_init()
1561 	revalidate_disk(btt->btt_disk);  in btt_blk_init()
1566 static void btt_blk_cleanup(struct btt *btt)  in btt_blk_cleanup()  argument
1568 	del_gendisk(btt->btt_disk);  in btt_blk_cleanup()
1569 	put_disk(btt->btt_disk);  in btt_blk_cleanup()
1570 	blk_cleanup_queue(btt->btt_queue);  in btt_blk_cleanup()
1590 static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,  in btt_init()
1594 	struct btt *btt;  in btt_init()  local
1598 	btt = devm_kzalloc(dev, sizeof(struct btt), GFP_KERNEL);  in btt_init()
1599 	if (!btt)  in btt_init()
1602 	btt->nd_btt = nd_btt;  in btt_init()
1603 	btt->rawsize = rawsize;  in btt_init()
1604 	btt->lbasize = lbasize;  in btt_init()
1605 	btt->sector_size = ((lbasize >= 4096) ? 4096 : 512);  in btt_init()
1606 	INIT_LIST_HEAD(&btt->arena_list);  in btt_init()
1607 	mutex_init(&btt->init_lock);  in btt_init()
1608 	btt->nd_region = nd_region;  in btt_init()
1610 	btt->phys_bb = &nsio->bb;  in btt_init()
1612 	ret = discover_arenas(btt);  in btt_init()
1618 	if (btt->init_state != INIT_READY && nd_region->ro) {  in btt_init()
1622 	} else if (btt->init_state != INIT_READY) {  in btt_init()
1623 		btt->num_arenas = (rawsize / ARENA_MAX_SIZE) +  in btt_init()
1626 				btt->num_arenas, rawsize);  in btt_init()
1628 		ret = create_arenas(btt);  in btt_init()
1634 		ret = btt_meta_init(btt);  in btt_init()
1641 	ret = btt_blk_init(btt);  in btt_init()
1647 	btt_debugfs_init(btt);  in btt_init()
1649 	return btt;  in btt_init()
1661 static void btt_fini(struct btt *btt)  in btt_fini()  argument
1663 	if (btt) {  in btt_fini()
1664 		btt_blk_cleanup(btt);  in btt_fini()
1665 		free_arenas(btt);  in btt_fini()
1666 		debugfs_remove_recursive(btt->debugfs_dir);  in btt_fini()
1675 	struct btt *btt;  in nvdimm_namespace_attach_btt()  local
1703 	btt = btt_init(nd_btt, rawsize, nd_btt->lbasize, nd_btt->uuid,  in nvdimm_namespace_attach_btt()
1705 	if (!btt)  in nvdimm_namespace_attach_btt()
1707 	nd_btt->btt = btt;  in nvdimm_namespace_attach_btt()
1715 	struct btt *btt = nd_btt->btt;  in nvdimm_namespace_detach_btt()  local
1717 	btt_fini(btt);  in nvdimm_namespace_detach_btt()
1718 	nd_btt->btt = NULL;  in nvdimm_namespace_detach_btt()