450 	int r;  in __superblock_all_zeroes()  local
459 	r = dm_bm_read_lock(bm, THIN_SUPERBLOCK_LOCATION, NULL, &b);  in __superblock_all_zeroes()
460 	if (r)  in __superblock_all_zeroes()
461 		return r;  in __superblock_all_zeroes()
517 	int r;  in save_sm_roots()  local
520 	r = dm_sm_root_size(pmd->metadata_sm, &len);  in save_sm_roots()
521 	if (r < 0)  in save_sm_roots()
522 		return r;  in save_sm_roots()
524 	r = dm_sm_copy_root(pmd->metadata_sm, &pmd->metadata_space_map_root, len);  in save_sm_roots()
525 	if (r < 0)  in save_sm_roots()
526 		return r;  in save_sm_roots()
528 	r = dm_sm_root_size(pmd->data_sm, &len);  in save_sm_roots()
529 	if (r < 0)  in save_sm_roots()
530 		return r;  in save_sm_roots()
549 	int r;  in __write_initial_superblock()  local
557 	r = dm_sm_commit(pmd->data_sm);  in __write_initial_superblock()
558 	if (r < 0)  in __write_initial_superblock()
559 		return r;  in __write_initial_superblock()
561 	r = dm_tm_pre_commit(pmd->tm);  in __write_initial_superblock()
562 	if (r < 0)  in __write_initial_superblock()
563 		return r;  in __write_initial_superblock()
565 	r = save_sm_roots(pmd);  in __write_initial_superblock()
566 	if (r < 0)  in __write_initial_superblock()
567 		return r;  in __write_initial_superblock()
569 	r = superblock_lock_zero(pmd, &sblock);  in __write_initial_superblock()
570 	if (r)  in __write_initial_superblock()
571 		return r;  in __write_initial_superblock()
595 	int r;  in __format_metadata()  local
597 	r = dm_tm_create_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __format_metadata()
599 	if (r < 0) {  in __format_metadata()
601 		return r;  in __format_metadata()
607 		r = PTR_ERR(pmd->data_sm);  in __format_metadata()
614 		r = -ENOMEM;  in __format_metadata()
620 	r = dm_btree_empty(&pmd->info, &pmd->root);  in __format_metadata()
621 	if (r < 0)  in __format_metadata()
624 	r = dm_btree_empty(&pmd->details_info, &pmd->details_root);  in __format_metadata()
625 	if (r < 0) {  in __format_metadata()
630 	r = __write_initial_superblock(pmd);  in __format_metadata()
631 	if (r)  in __format_metadata()
644 	return r;  in __format_metadata()
677 	int r;  in __open_metadata()  local
681 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
683 	if (r < 0) {  in __open_metadata()
685 		return r;  in __open_metadata()
695 		r = -EINVAL;  in __open_metadata()
699 	r = __check_incompat_features(disk_super, pmd);  in __open_metadata()
700 	if (r < 0)  in __open_metadata()
703 	r = dm_tm_open_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
707 	if (r < 0) {  in __open_metadata()
716 		r = PTR_ERR(pmd->data_sm);  in __open_metadata()
723 		r = -ENOMEM;  in __open_metadata()
740 	return r;  in __open_metadata()
745 	int r, unformatted;  in __open_or_format_metadata()  local
747 	r = __superblock_all_zeroes(pmd->bm, &unformatted);  in __open_or_format_metadata()
748 	if (r)  in __open_or_format_metadata()
749 		return r;  in __open_or_format_metadata()
759 	int r;  in __create_persistent_data_objects()  local
765 		r = PTR_ERR(pmd->bm);  in __create_persistent_data_objects()
767 		return r;  in __create_persistent_data_objects()
770 	r = __open_or_format_metadata(pmd, format_device);  in __create_persistent_data_objects()
771 	if (r) {  in __create_persistent_data_objects()
776 	return r;  in __create_persistent_data_objects()
790 	int r;  in __begin_transaction()  local
798 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __begin_transaction()
800 	if (r)  in __begin_transaction()
801 		return r;  in __begin_transaction()
817 	int r;  in __write_changed_details()  local
834 		r = dm_btree_insert(&pmd->details_info, pmd->details_root,  in __write_changed_details()
836 		if (r)  in __write_changed_details()
837 			return r;  in __write_changed_details()
852 	int r;  in __commit_transaction()  local
866 		r = pmd->pre_commit_fn(pmd->pre_commit_context);  in __commit_transaction()
867 		if (r < 0) {  in __commit_transaction()
869 			return r;  in __commit_transaction()
873 	r = __write_changed_details(pmd);  in __commit_transaction()
874 	if (r < 0)  in __commit_transaction()
875 		return r;  in __commit_transaction()
877 	r = dm_sm_commit(pmd->data_sm);  in __commit_transaction()
878 	if (r < 0)  in __commit_transaction()
879 		return r;  in __commit_transaction()
881 	r = dm_tm_pre_commit(pmd->tm);  in __commit_transaction()
882 	if (r < 0)  in __commit_transaction()
883 		return r;  in __commit_transaction()
885 	r = save_sm_roots(pmd);  in __commit_transaction()
886 	if (r < 0)  in __commit_transaction()
887 		return r;  in __commit_transaction()
889 	r = superblock_lock(pmd, &sblock);  in __commit_transaction()
890 	if (r)  in __commit_transaction()
891 		return r;  in __commit_transaction()
907 	int r;  in __set_metadata_reserve()  local
911 	r = dm_sm_get_nr_blocks(pmd->metadata_sm, &total);  in __set_metadata_reserve()
912 	if (r) {  in __set_metadata_reserve()
923 	int r;  in dm_pool_metadata_open()  local
942 	r = __create_persistent_data_objects(pmd, format_device);  in dm_pool_metadata_open()
943 	if (r) {  in dm_pool_metadata_open()
945 		return ERR_PTR(r);  in dm_pool_metadata_open()
948 	r = __begin_transaction(pmd);  in dm_pool_metadata_open()
949 	if (r < 0) {  in dm_pool_metadata_open()
952 		return ERR_PTR(r);  in dm_pool_metadata_open()
962 	int r;  in dm_pool_metadata_close()  local
985 		r = __commit_transaction(pmd);  in dm_pool_metadata_close()
986 		if (r < 0)  in dm_pool_metadata_close()
988 			       __func__, r);  in dm_pool_metadata_close()
1007 	int r, changed = 0;  in __open_device()  local
1031 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __open_device()
1033 	if (r) {  in __open_device()
1034 		if (r != -ENODATA || !create)  in __open_device()
1035 			return r;  in __open_device()
1074 	int r;  in __create_thin()  local
1080 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_thin()
1082 	if (!r)  in __create_thin()
1088 	r = dm_btree_empty(&pmd->bl_info, &dev_root);  in __create_thin()
1089 	if (r)  in __create_thin()
1090 		return r;  in __create_thin()
1097 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_thin()
1098 	if (r) {  in __create_thin()
1100 		return r;  in __create_thin()
1103 	r = __open_device(pmd, dev, 1, &td);  in __create_thin()
1104 	if (r) {  in __create_thin()
1107 		return r;  in __create_thin()
1111 	return r;  in __create_thin()
1116 	int r = -EINVAL;  in dm_pool_create_thin()  local
1120 		r = __create_thin(pmd, dev);  in dm_pool_create_thin()
1123 	return r;  in dm_pool_create_thin()
1130 	int r;  in __set_snapshot_details()  local
1133 	r = __open_device(pmd, origin, 0, &td);  in __set_snapshot_details()
1134 	if (r)  in __set_snapshot_details()
1135 		return r;  in __set_snapshot_details()
1150 	int r;  in __create_snap()  local
1157 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_snap()
1159 	if (!r)  in __create_snap()
1163 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &key, &value);  in __create_snap()
1164 	if (r)  in __create_snap()
1165 		return r;  in __create_snap()
1175 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_snap()
1176 	if (r) {  in __create_snap()
1178 		return r;  in __create_snap()
1183 	r = __open_device(pmd, dev, 1, &td);  in __create_snap()
1184 	if (r)  in __create_snap()
1187 	r = __set_snapshot_details(pmd, td, origin, pmd->time);  in __create_snap()
1190 	if (r)  in __create_snap()
1199 	return r;  in __create_snap()
1206 	int r = -EINVAL;  in dm_pool_create_snap()  local
1210 		r = __create_snap(pmd, dev, origin);  in dm_pool_create_snap()
1213 	return r;  in dm_pool_create_snap()
1218 	int r;  in __delete_device()  local
1223 	r = __open_device(pmd, dev, 0, &td);  in __delete_device()
1224 	if (r)  in __delete_device()
1225 		return r;  in __delete_device()
1234 	r = dm_btree_remove(&pmd->details_info, pmd->details_root,  in __delete_device()
1236 	if (r)  in __delete_device()
1237 		return r;  in __delete_device()
1239 	r = dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);  in __delete_device()
1240 	if (r)  in __delete_device()
1241 		return r;  in __delete_device()
1249 	int r = -EINVAL;  in dm_pool_delete_thin_device()  local
1253 		r = __delete_device(pmd, dev);  in dm_pool_delete_thin_device()
1256 	return r;  in dm_pool_delete_thin_device()
1263 	int r = -EINVAL;  in dm_pool_set_metadata_transaction_id()  local
1276 	r = 0;  in dm_pool_set_metadata_transaction_id()
1281 	return r;  in dm_pool_set_metadata_transaction_id()
1287 	int r = -EINVAL;  in dm_pool_get_metadata_transaction_id()  local
1292 		r = 0;  in dm_pool_get_metadata_transaction_id()
1296 	return r;  in dm_pool_get_metadata_transaction_id()
1301 	int r, inc;  in __reserve_metadata_snap()  local
1310 	r = __commit_transaction(pmd);  in __reserve_metadata_snap()
1311 	if (r < 0) {  in __reserve_metadata_snap()
1313 		       __func__, r);  in __reserve_metadata_snap()
1314 		return r;  in __reserve_metadata_snap()
1321 	r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION,  in __reserve_metadata_snap()
1323 	if (r)  in __reserve_metadata_snap()
1324 		return r;  in __reserve_metadata_snap()
1357 	r = superblock_lock(pmd, &sblock);  in __reserve_metadata_snap()
1358 	if (r) {  in __reserve_metadata_snap()
1360 		return r;  in __reserve_metadata_snap()
1371 	int r = -EINVAL;  in dm_pool_reserve_metadata_snap()  local
1375 		r = __reserve_metadata_snap(pmd);  in dm_pool_reserve_metadata_snap()
1378 	return r;  in dm_pool_reserve_metadata_snap()
1383 	int r;  in __release_metadata_snap()  local
1388 	r = superblock_lock(pmd, &sblock);  in __release_metadata_snap()
1389 	if (r)  in __release_metadata_snap()
1390 		return r;  in __release_metadata_snap()
1403 	r = dm_tm_read_lock(pmd->tm, held_root, &sb_validator, &copy);  in __release_metadata_snap()
1404 	if (r)  in __release_metadata_snap()
1405 		return r;  in __release_metadata_snap()
1419 	int r = -EINVAL;  in dm_pool_release_metadata_snap()  local
1423 		r = __release_metadata_snap(pmd);  in dm_pool_release_metadata_snap()
1426 	return r;  in dm_pool_release_metadata_snap()
1432 	int r;  in __get_metadata_snap()  local
1436 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __get_metadata_snap()
1438 	if (r)  in __get_metadata_snap()
1439 		return r;  in __get_metadata_snap()
1452 	int r = -EINVAL;  in dm_pool_get_metadata_snap()  local
1456 		r = __get_metadata_snap(pmd, result);  in dm_pool_get_metadata_snap()
1459 	return r;  in dm_pool_get_metadata_snap()
1465 	int r = -EINVAL;  in dm_pool_open_thin_device()  local
1469 		r = __open_device(pmd, dev, 0, td);  in dm_pool_open_thin_device()
1472 	return r;  in dm_pool_open_thin_device()
1516 	int r;  in __find_block()  local
1527 	r = dm_btree_lookup(info, pmd->root, keys, &value);  in __find_block()
1528 	if (!r)  in __find_block()
1531 	return r;  in __find_block()
1537 	int r;  in dm_thin_find_block()  local
1546 	r = __find_block(td, block, can_issue_io, result);  in dm_thin_find_block()
1549 	return r;  in dm_thin_find_block()
1556 	int r;  in __find_next_mapped_block()  local
1561 	r = dm_btree_lookup_next(&pmd->info, pmd->root, keys, vblock, &value);  in __find_next_mapped_block()
1562 	if (!r)  in __find_next_mapped_block()
1565 	return r;  in __find_next_mapped_block()
1573 	int r;  in __find_mapped_range()  local
1580 	r = __find_next_mapped_block(td, begin, &begin, &lookup);  in __find_mapped_range()
1581 	if (r)  in __find_mapped_range()
1582 		return r;  in __find_mapped_range()
1594 		r = __find_block(td, begin, true, &lookup);  in __find_mapped_range()
1595 		if (r) {  in __find_mapped_range()
1596 			if (r == -ENODATA)  in __find_mapped_range()
1599 				return r;  in __find_mapped_range()
1619 	int r = -EINVAL;  in dm_thin_find_mapped_range()  local
1624 		r = __find_mapped_range(td, begin, end, thin_begin, thin_end,  in dm_thin_find_mapped_range()
1629 	return r;  in dm_thin_find_mapped_range()
1635 	int r, inserted;  in __insert()  local
1643 	r = dm_btree_insert_notify(&pmd->info, pmd->root, keys, &value,  in __insert()
1645 	if (r)  in __insert()
1646 		return r;  in __insert()
1658 	int r = -EINVAL;  in dm_thin_insert_block()  local
1662 		r = __insert(td, block, data_block);  in dm_thin_insert_block()
1665 	return r;  in dm_thin_insert_block()
1670 	int r;  in __remove_range()  local
1680 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, keys, &value);  in __remove_range()
1681 	if (r)  in __remove_range()
1682 		return r;  in __remove_range()
1690 	r = dm_btree_remove(&pmd->tl_info, pmd->root, keys, &pmd->root);  in __remove_range()
1691 	if (r)  in __remove_range()
1692 		return r;  in __remove_range()
1699 		r = dm_btree_lookup_next(&pmd->bl_info, mapping_root, &begin, &begin, &value);  in __remove_range()
1700 		if (r == -ENODATA)  in __remove_range()
1703 		if (r)  in __remove_range()
1704 			return r;  in __remove_range()
1709 		r = dm_btree_remove_leaves(&pmd->bl_info, mapping_root, &begin, end, &mapping_root, &count);  in __remove_range()
1710 		if (r)  in __remove_range()
1711 			return r;  in __remove_range()
1730 	int r = -EINVAL;  in dm_thin_remove_range()  local
1734 		r = __remove_range(td, begin, end);  in dm_thin_remove_range()
1737 	return r;  in dm_thin_remove_range()
1742 	int r;  in dm_pool_block_is_shared()  local
1746 	r = dm_sm_get_count(pmd->data_sm, b, &ref_count);  in dm_pool_block_is_shared()
1747 	if (!r)  in dm_pool_block_is_shared()
1751 	return r;  in dm_pool_block_is_shared()
1756 	int r = 0;  in dm_pool_inc_data_range()  local
1759 	r = dm_sm_inc_blocks(pmd->data_sm, b, e);  in dm_pool_inc_data_range()
1762 	return r;  in dm_pool_inc_data_range()
1767 	int r = 0;  in dm_pool_dec_data_range()  local
1770 	r = dm_sm_dec_blocks(pmd->data_sm, b, e);  in dm_pool_dec_data_range()
1773 	return r;  in dm_pool_dec_data_range()
1778 	int r;  in dm_thin_changed_this_transaction()  local
1781 	r = td->changed;  in dm_thin_changed_this_transaction()
1784 	return r;  in dm_thin_changed_this_transaction()
1789 	bool r = false;  in dm_pool_changed_this_transaction()  local
1795 			r = td->changed;  in dm_pool_changed_this_transaction()
1801 	return r;  in dm_pool_changed_this_transaction()
1806 	bool r;  in dm_thin_aborted_changes()  local
1809 	r = td->aborted_with_changes;  in dm_thin_aborted_changes()
1812 	return r;  in dm_thin_aborted_changes()
1817 	int r = -EINVAL;  in dm_pool_alloc_data_block()  local
1821 		r = dm_sm_new_block(pmd->data_sm, result);  in dm_pool_alloc_data_block()
1824 	return r;  in dm_pool_alloc_data_block()
1829 	int r = -EINVAL;  in dm_pool_commit_metadata()  local
1839 	r = __commit_transaction(pmd);  in dm_pool_commit_metadata()
1840 	if (r < 0)  in dm_pool_commit_metadata()
1846 	r = __begin_transaction(pmd);  in dm_pool_commit_metadata()
1849 	return r;  in dm_pool_commit_metadata()
1862 	int r = -EINVAL;  in dm_pool_abort_metadata()  local
1870 	r = __create_persistent_data_objects(pmd, false);  in dm_pool_abort_metadata()
1871 	if (r)  in dm_pool_abort_metadata()
1877 	return r;  in dm_pool_abort_metadata()
1882 	int r = -EINVAL;  in dm_pool_get_free_block_count()  local
1886 		r = dm_sm_get_nr_free(pmd->data_sm, result);  in dm_pool_get_free_block_count()
1889 	return r;  in dm_pool_get_free_block_count()
1895 	int r = -EINVAL;  in dm_pool_get_free_metadata_block_count()  local
1899 		r = dm_sm_get_nr_free(pmd->metadata_sm, result);  in dm_pool_get_free_metadata_block_count()
1901 	if (!r) {  in dm_pool_get_free_metadata_block_count()
1909 	return r;  in dm_pool_get_free_metadata_block_count()
1915 	int r = -EINVAL;  in dm_pool_get_metadata_dev_size()  local
1919 		r = dm_sm_get_nr_blocks(pmd->metadata_sm, result);  in dm_pool_get_metadata_dev_size()
1922 	return r;  in dm_pool_get_metadata_dev_size()
1927 	int r = -EINVAL;  in dm_pool_get_data_dev_size()  local
1931 		r = dm_sm_get_nr_blocks(pmd->data_sm, result);  in dm_pool_get_data_dev_size()
1934 	return r;  in dm_pool_get_data_dev_size()
1939 	int r = -EINVAL;  in dm_thin_get_mapped_count()  local
1945 		r = 0;  in dm_thin_get_mapped_count()
1949 	return r;  in dm_thin_get_mapped_count()
1954 	int r;  in __highest_block()  local
1959 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &td->id, &value_le);  in __highest_block()
1960 	if (r)  in __highest_block()
1961 		return r;  in __highest_block()
1971 	int r = -EINVAL;  in dm_thin_get_highest_mapped_block()  local
1976 		r = __highest_block(td, result);  in dm_thin_get_highest_mapped_block()
1979 	return r;  in dm_thin_get_highest_mapped_block()
1984 	int r;  in __resize_space_map()  local
1987 	r = dm_sm_get_nr_blocks(sm, &old_count);  in __resize_space_map()
1988 	if (r)  in __resize_space_map()
1989 		return r;  in __resize_space_map()
2004 	int r = -EINVAL;  in dm_pool_resize_data_dev()  local
2008 		r = __resize_space_map(pmd->data_sm, new_count);  in dm_pool_resize_data_dev()
2011 	return r;  in dm_pool_resize_data_dev()
2016 	int r = -EINVAL;  in dm_pool_resize_metadata_dev()  local
2020 		r = __resize_space_map(pmd->metadata_sm, new_count);  in dm_pool_resize_metadata_dev()
2021 		if (!r)  in dm_pool_resize_metadata_dev()
2026 	return r;  in dm_pool_resize_metadata_dev()
2048 	int r = -EINVAL;  in dm_pool_register_metadata_threshold()  local
2052 		r = dm_sm_register_threshold_callback(pmd->metadata_sm,  in dm_pool_register_metadata_threshold()
2057 	return r;  in dm_pool_register_metadata_threshold()
2072 	int r = -EINVAL;  in dm_pool_metadata_set_needs_check()  local
2082 	r = superblock_lock(pmd, &sblock);  in dm_pool_metadata_set_needs_check()
2083 	if (r) {  in dm_pool_metadata_set_needs_check()
2094 	return r;  in dm_pool_metadata_set_needs_check()