98 static inline sector_t *get_node(struct dm_table *t,  in get_node()  argument
101 	return t->index[l] + (n * KEYS_PER_NODE);  in get_node()
108 static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)  in high()  argument
110 	for (; l < t->depth - 1; l++)  in high()
113 	if (n >= t->counts[l])  in high()
116 	return get_node(t, l, n)[KEYS_PER_NODE - 1];  in high()
123 static int setup_btree_index(unsigned int l, struct dm_table *t)  in setup_btree_index()  argument
128 	for (n = 0U; n < t->counts[l]; n++) {  in setup_btree_index()
129 		node = get_node(t, l, n);  in setup_btree_index()
132 			node[k] = high(t, l + 1, get_child(n, k));  in setup_btree_index()
160 static int alloc_targets(struct dm_table *t, unsigned int num)  in alloc_targets()  argument
178 	vfree(t->highs);  in alloc_targets()
180 	t->num_allocated = num;  in alloc_targets()
181 	t->highs = n_highs;  in alloc_targets()
182 	t->targets = n_targets;  in alloc_targets()
190 	struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL);  in dm_table_create()  local
192 	if (!t)  in dm_table_create()
195 	INIT_LIST_HEAD(&t->devices);  in dm_table_create()
196 	INIT_LIST_HEAD(&t->target_callbacks);  in dm_table_create()
204 		kfree(t);  in dm_table_create()
208 	if (alloc_targets(t, num_targets)) {  in dm_table_create()
209 		kfree(t);  in dm_table_create()
213 	t->type = DM_TYPE_NONE;  in dm_table_create()
214 	t->mode = mode;  in dm_table_create()
215 	t->md = md;  in dm_table_create()
216 	*result = t;  in dm_table_create()
234 void dm_table_destroy(struct dm_table *t)  in dm_table_destroy()  argument
238 	if (!t)  in dm_table_destroy()
242 	if (t->depth >= 2)  in dm_table_destroy()
243 		vfree(t->index[t->depth - 2]);  in dm_table_destroy()
246 	for (i = 0; i < t->num_targets; i++) {  in dm_table_destroy()
247 		struct dm_target *tgt = t->targets + i;  in dm_table_destroy()
255 	vfree(t->highs);  in dm_table_destroy()
258 	free_devices(&t->devices, t->md);  in dm_table_destroy()
260 	dm_free_md_mempools(t->mempools);  in dm_table_destroy()
262 	kfree(t);  in dm_table_destroy()
435 	struct dm_table *t = ti->table;  in dm_get_device()  local
437 	BUG_ON(!t);  in dm_get_device()
443 	dd = find_device(&t->devices, dev);  in dm_get_device()
449 		if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) {  in dm_get_device()
455 		list_add(&dd->list, &t->devices);  in dm_get_device()
459 		r = upgrade_mode(dd, mode, t->md);  in dm_get_device()
711 int dm_table_add_target(struct dm_table *t, const char *type,  in dm_table_add_target()  argument
718 	if (t->singleton) {  in dm_table_add_target()
720 		      dm_device_name(t->md), t->targets->type->name);  in dm_table_add_target()
724 	BUG_ON(t->num_targets >= t->num_allocated);  in dm_table_add_target()
726 	tgt = t->targets + t->num_targets;  in dm_table_add_target()
730 		DMERR("%s: zero-length target", dm_device_name(t->md));  in dm_table_add_target()
736 		DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);  in dm_table_add_target()
741 		if (t->num_targets) {  in dm_table_add_target()
745 		t->singleton = true;  in dm_table_add_target()
748 	if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {  in dm_table_add_target()
753 	if (t->immutable_target_type) {  in dm_table_add_target()
754 		if (t->immutable_target_type != tgt->type) {  in dm_table_add_target()
759 		if (t->num_targets) {  in dm_table_add_target()
763 		t->immutable_target_type = tgt->type;  in dm_table_add_target()
767 		t->integrity_added = 1;  in dm_table_add_target()
769 	tgt->table = t;  in dm_table_add_target()
777 	if (!adjoin(t, tgt)) {  in dm_table_add_target()
793 	t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;  in dm_table_add_target()
797 		       dm_device_name(t->md), type);  in dm_table_add_target()
802 	DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error);  in dm_table_add_target()
879 void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)  in dm_table_set_type()  argument
881 	t->type = type;  in dm_table_set_type()
891 static bool dm_table_supports_dax(struct dm_table *t)  in dm_table_supports_dax()  argument
897 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_supports_dax()
898 		ti = dm_table_get_target(t, i);  in dm_table_supports_dax()
911 static bool dm_table_does_not_support_partial_completion(struct dm_table *t);
932 static int dm_table_determine_type(struct dm_table *t)  in dm_table_determine_type()  argument
938 	struct list_head *devices = dm_table_get_devices(t);  in dm_table_determine_type()
939 	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);  in dm_table_determine_type()
941 	if (t->type != DM_TYPE_NONE) {  in dm_table_determine_type()
943 		if (t->type == DM_TYPE_BIO_BASED) {  in dm_table_determine_type()
947 		BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);  in dm_table_determine_type()
948 		BUG_ON(t->type == DM_TYPE_NVME_BIO_BASED);  in dm_table_determine_type()
952 	for (i = 0; i < t->num_targets; i++) {  in dm_table_determine_type()
953 		tgt = t->targets + i;  in dm_table_determine_type()
983 		t->type = DM_TYPE_BIO_BASED;  in dm_table_determine_type()
984 		if (dm_table_supports_dax(t) ||  in dm_table_determine_type()
986 			t->type = DM_TYPE_DAX_BIO_BASED;  in dm_table_determine_type()
989 			tgt = dm_table_get_immutable_target(t);  in dm_table_determine_type()
990 			if (tgt && !tgt->max_io_len && dm_table_does_not_support_partial_completion(t)) {  in dm_table_determine_type()
991 				t->type = DM_TYPE_NVME_BIO_BASED;  in dm_table_determine_type()
994 				t->type = DM_TYPE_NVME_BIO_BASED;  in dm_table_determine_type()
1006 	t->type = DM_TYPE_REQUEST_BASED;  in dm_table_determine_type()
1015 	if (t->num_targets > 1) {  in dm_table_determine_type()
1017 		      t->type == DM_TYPE_NVME_BIO_BASED ? "nvme bio-based" : "request-based");  in dm_table_determine_type()
1023 		struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);  in dm_table_determine_type()
1027 			t->type = live_table->type;  in dm_table_determine_type()
1028 			t->all_blk_mq = live_table->all_blk_mq;  in dm_table_determine_type()
1030 		dm_put_live_table(t->md, srcu_idx);  in dm_table_determine_type()
1034 	tgt = dm_table_get_immutable_target(t);  in dm_table_determine_type()
1053 	t->all_blk_mq = v.mq_count > 0;  in dm_table_determine_type()
1055 	if (!t->all_blk_mq &&  in dm_table_determine_type()
1056 	    (t->type == DM_TYPE_MQ_REQUEST_BASED || t->type == DM_TYPE_NVME_BIO_BASED)) {  in dm_table_determine_type()
1064 enum dm_queue_mode dm_table_get_type(struct dm_table *t)  in dm_table_get_type()  argument
1066 	return t->type;  in dm_table_get_type()
1069 struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)  in dm_table_get_immutable_target_type()  argument
1071 	return t->immutable_target_type;  in dm_table_get_immutable_target_type()
1074 struct dm_target *dm_table_get_immutable_target(struct dm_table *t)  in dm_table_get_immutable_target()  argument
1077 	if (t->num_targets > 1 ||  in dm_table_get_immutable_target()
1078 	    !dm_target_is_immutable(t->targets[0].type))  in dm_table_get_immutable_target()
1081 	return t->targets;  in dm_table_get_immutable_target()
1084 struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)  in dm_table_get_wildcard_target()  argument
1089 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_get_wildcard_target()
1090 		ti = dm_table_get_target(t, i);  in dm_table_get_wildcard_target()
1098 bool dm_table_bio_based(struct dm_table *t)  in dm_table_bio_based()  argument
1100 	return __table_type_bio_based(dm_table_get_type(t));  in dm_table_bio_based()
1103 bool dm_table_request_based(struct dm_table *t)  in dm_table_request_based()  argument
1105 	return __table_type_request_based(dm_table_get_type(t));  in dm_table_request_based()
1108 bool dm_table_all_blk_mq_devices(struct dm_table *t)  in dm_table_all_blk_mq_devices()  argument
1110 	return t->all_blk_mq;  in dm_table_all_blk_mq_devices()
1113 static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)  in dm_table_alloc_md_mempools()  argument
1115 	enum dm_queue_mode type = dm_table_get_type(t);  in dm_table_alloc_md_mempools()
1127 		for (i = 0; i < t->num_targets; i++) {  in dm_table_alloc_md_mempools()
1128 			ti = t->targets + i;  in dm_table_alloc_md_mempools()
1133 	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported,  in dm_table_alloc_md_mempools()
1135 	if (!t->mempools)  in dm_table_alloc_md_mempools()
1141 void dm_table_free_md_mempools(struct dm_table *t)  in dm_table_free_md_mempools()  argument
1143 	dm_free_md_mempools(t->mempools);  in dm_table_free_md_mempools()
1144 	t->mempools = NULL;  in dm_table_free_md_mempools()
1147 struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t)  in dm_table_get_md_mempools()  argument
1149 	return t->mempools;  in dm_table_get_md_mempools()
1152 static int setup_indexes(struct dm_table *t)  in setup_indexes()  argument
1159 	for (i = t->depth - 2; i >= 0; i--) {  in setup_indexes()
1160 		t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);  in setup_indexes()
1161 		total += t->counts[i];  in setup_indexes()
1169 	for (i = t->depth - 2; i >= 0; i--) {  in setup_indexes()
1170 		t->index[i] = indexes;  in setup_indexes()
1171 		indexes += (KEYS_PER_NODE * t->counts[i]);  in setup_indexes()
1172 		setup_btree_index(i, t);  in setup_indexes()
1181 static int dm_table_build_index(struct dm_table *t)  in dm_table_build_index()  argument
1187 	leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);  in dm_table_build_index()
1188 	t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);  in dm_table_build_index()
1191 	t->counts[t->depth - 1] = leaf_nodes;  in dm_table_build_index()
1192 	t->index[t->depth - 1] = t->highs;  in dm_table_build_index()
1194 	if (t->depth >= 2)  in dm_table_build_index()
1195 		r = setup_indexes(t);  in dm_table_build_index()
1209 static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t)  in dm_table_get_integrity_disk()  argument
1211 	struct list_head *devices = dm_table_get_devices(t);  in dm_table_get_integrity_disk()
1216 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_get_integrity_disk()
1217 		struct dm_target *ti = dm_table_get_target(t, i);  in dm_table_get_integrity_disk()
1237 		       dm_device_name(t->md),  in dm_table_get_integrity_disk()
1253 static int dm_table_register_integrity(struct dm_table *t)  in dm_table_register_integrity()  argument
1255 	struct mapped_device *md = t->md;  in dm_table_register_integrity()
1259 	if (t->integrity_added)  in dm_table_register_integrity()
1262 	template_disk = dm_table_get_integrity_disk(t);  in dm_table_register_integrity()
1267 		t->integrity_supported = true;  in dm_table_register_integrity()
1284 		       dm_device_name(t->md),  in dm_table_register_integrity()
1290 	t->integrity_supported = true;  in dm_table_register_integrity()
1298 int dm_table_complete(struct dm_table *t)  in dm_table_complete()  argument
1302 	r = dm_table_determine_type(t);  in dm_table_complete()
1308 	r = dm_table_build_index(t);  in dm_table_complete()
1314 	r = dm_table_register_integrity(t);  in dm_table_complete()
1320 	r = dm_table_alloc_md_mempools(t, t->md);  in dm_table_complete()
1328 void dm_table_event_callback(struct dm_table *t,  in dm_table_event_callback()  argument
1332 	t->event_fn = fn;  in dm_table_event_callback()
1333 	t->event_context = context;  in dm_table_event_callback()
1337 void dm_table_event(struct dm_table *t)  in dm_table_event()  argument
1346 	if (t->event_fn)  in dm_table_event()
1347 		t->event_fn(t->event_context);  in dm_table_event()
1352 sector_t dm_table_get_size(struct dm_table *t)  in dm_table_get_size()  argument
1354 	return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;  in dm_table_get_size()
1358 struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)  in dm_table_get_target()  argument
1360 	if (index >= t->num_targets)  in dm_table_get_target()
1363 	return t->targets + index;  in dm_table_get_target()
1372 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)  in dm_table_find_target()  argument
1377 	for (l = 0; l < t->depth; l++) {  in dm_table_find_target()
1379 		node = get_node(t, l, n);  in dm_table_find_target()
1386 	return &t->targets[(KEYS_PER_NODE * n) + k];  in dm_table_find_target()
1434 static bool dm_table_supports_zoned_model(struct dm_table *t,  in dm_table_supports_zoned_model()  argument
1440 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_supports_zoned_model()
1441 		ti = dm_table_get_target(t, i);  in dm_table_supports_zoned_model()
1464 static bool dm_table_matches_zone_sectors(struct dm_table *t,  in dm_table_matches_zone_sectors()  argument
1470 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_matches_zone_sectors()
1471 		ti = dm_table_get_target(t, i);  in dm_table_matches_zone_sectors()
1612 static void dm_table_verify_integrity(struct dm_table *t)  in dm_table_verify_integrity()  argument
1616 	if (t->integrity_added)  in dm_table_verify_integrity()
1619 	if (t->integrity_supported) {  in dm_table_verify_integrity()
1624 		template_disk = dm_table_get_integrity_disk(t);  in dm_table_verify_integrity()
1626 		    blk_integrity_compare(dm_disk(t->md), template_disk) >= 0)  in dm_table_verify_integrity()
1630 	if (integrity_profile_exists(dm_disk(t->md))) {  in dm_table_verify_integrity()
1632 		       dm_device_name(t->md));  in dm_table_verify_integrity()
1633 		blk_integrity_unregister(dm_disk(t->md));  in dm_table_verify_integrity()
1646 static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)  in dm_table_supports_flush()  argument
1657 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_supports_flush()
1658 		ti = dm_table_get_target(t, i);  in dm_table_supports_flush()
1688 static int dm_table_supports_dax_write_cache(struct dm_table *t)  in dm_table_supports_dax_write_cache()  argument
1693 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_supports_dax_write_cache()
1694 		ti = dm_table_get_target(t, i);  in dm_table_supports_dax_write_cache()
1729 static bool dm_table_all_devices_attribute(struct dm_table *t,  in dm_table_all_devices_attribute()  argument
1735 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_all_devices_attribute()
1736 		ti = dm_table_get_target(t, i);  in dm_table_all_devices_attribute()
1755 static bool dm_table_does_not_support_partial_completion(struct dm_table *t)  in dm_table_does_not_support_partial_completion()  argument
1757 	return dm_table_all_devices_attribute(t, device_no_partial_completion);  in dm_table_does_not_support_partial_completion()
1768 static bool dm_table_supports_write_same(struct dm_table *t)  in dm_table_supports_write_same()  argument
1773 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_supports_write_same()
1774 		ti = dm_table_get_target(t, i);  in dm_table_supports_write_same()
1795 static bool dm_table_supports_write_zeroes(struct dm_table *t)  in dm_table_supports_write_zeroes()  argument
1800 	while (i < dm_table_get_num_targets(t)) {  in dm_table_supports_write_zeroes()
1801 		ti = dm_table_get_target(t, i++);  in dm_table_supports_write_zeroes()
1822 static bool dm_table_supports_discards(struct dm_table *t)  in dm_table_supports_discards()  argument
1827 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_supports_discards()
1828 		ti = dm_table_get_target(t, i);  in dm_table_supports_discards()
1856 static bool dm_table_supports_secure_erase(struct dm_table *t)  in dm_table_supports_secure_erase()  argument
1861 	for (i = 0; i < dm_table_get_num_targets(t); i++) {  in dm_table_supports_secure_erase()
1862 		ti = dm_table_get_target(t, i);  in dm_table_supports_secure_erase()
1875 void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,  in dm_table_set_restrictions()  argument
1885 	if (!dm_table_supports_discards(t)) {  in dm_table_set_restrictions()
1896 	if (dm_table_supports_secure_erase(t))  in dm_table_set_restrictions()
1899 	if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_WC))) {  in dm_table_set_restrictions()
1901 		if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_FUA)))  in dm_table_set_restrictions()
1906 	if (dm_table_supports_dax(t))  in dm_table_set_restrictions()
1911 	if (dm_table_supports_dax_write_cache(t))  in dm_table_set_restrictions()
1912 		dax_write_cache(t->md->dax_dev, true);  in dm_table_set_restrictions()
1915 	if (dm_table_all_devices_attribute(t, device_is_nonrot))  in dm_table_set_restrictions()
1920 	if (!dm_table_supports_write_same(t))  in dm_table_set_restrictions()
1922 	if (!dm_table_supports_write_zeroes(t))  in dm_table_set_restrictions()
1925 	if (dm_table_all_devices_attribute(t, queue_supports_sg_merge))  in dm_table_set_restrictions()
1930 	dm_table_verify_integrity(t);  in dm_table_set_restrictions()
1938 	if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random))  in dm_table_set_restrictions()
1942 unsigned int dm_table_get_num_targets(struct dm_table *t)  in dm_table_get_num_targets()  argument
1944 	return t->num_targets;  in dm_table_get_num_targets()
1947 struct list_head *dm_table_get_devices(struct dm_table *t)  in dm_table_get_devices()  argument
1949 	return &t->devices;  in dm_table_get_devices()
1952 fmode_t dm_table_get_mode(struct dm_table *t)  in dm_table_get_mode()  argument
1954 	return t->mode;  in dm_table_get_mode()
1964 static void suspend_targets(struct dm_table *t, enum suspend_mode mode)  in suspend_targets()  argument
1966 	int i = t->num_targets;  in suspend_targets()
1967 	struct dm_target *ti = t->targets;  in suspend_targets()
1969 	lockdep_assert_held(&t->md->suspend_lock);  in suspend_targets()
1990 void dm_table_presuspend_targets(struct dm_table *t)  in dm_table_presuspend_targets()  argument
1992 	if (!t)  in dm_table_presuspend_targets()
1995 	suspend_targets(t, PRESUSPEND);  in dm_table_presuspend_targets()
1998 void dm_table_presuspend_undo_targets(struct dm_table *t)  in dm_table_presuspend_undo_targets()  argument
2000 	if (!t)  in dm_table_presuspend_undo_targets()
2003 	suspend_targets(t, PRESUSPEND_UNDO);  in dm_table_presuspend_undo_targets()
2006 void dm_table_postsuspend_targets(struct dm_table *t)  in dm_table_postsuspend_targets()  argument
2008 	if (!t)  in dm_table_postsuspend_targets()
2011 	suspend_targets(t, POSTSUSPEND);  in dm_table_postsuspend_targets()
2014 int dm_table_resume_targets(struct dm_table *t)  in dm_table_resume_targets()  argument
2018 	lockdep_assert_held(&t->md->suspend_lock);  in dm_table_resume_targets()
2020 	for (i = 0; i < t->num_targets; i++) {  in dm_table_resume_targets()
2021 		struct dm_target *ti = t->targets + i;  in dm_table_resume_targets()
2029 			      dm_device_name(t->md), ti->type->name, r);  in dm_table_resume_targets()
2034 	for (i = 0; i < t->num_targets; i++) {  in dm_table_resume_targets()
2035 		struct dm_target *ti = t->targets + i;  in dm_table_resume_targets()
2044 void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb)  in dm_table_add_target_callbacks()  argument
2046 	list_add(&cb->list, &t->target_callbacks);  in dm_table_add_target_callbacks()
2050 int dm_table_any_congested(struct dm_table *t, int bdi_bits)  in dm_table_any_congested()  argument
2053 	struct list_head *devices = dm_table_get_devices(t);  in dm_table_any_congested()
2065 				     dm_device_name(t->md),  in dm_table_any_congested()
2069 	list_for_each_entry(cb, &t->target_callbacks, list)  in dm_table_any_congested()
2076 struct mapped_device *dm_table_get_md(struct dm_table *t)  in dm_table_get_md()  argument
2078 	return t->md;  in dm_table_get_md()
2082 void dm_table_run_md_queue_async(struct dm_table *t)  in dm_table_run_md_queue_async()  argument
2088 	if (!dm_table_request_based(t))  in dm_table_run_md_queue_async()
2091 	md = dm_table_get_md(t);  in dm_table_run_md_queue_async()