427 	int r;  in __superblock_all_zeroes()  local
436 	r = dm_bm_read_lock(bm, THIN_SUPERBLOCK_LOCATION, NULL, &b);  in __superblock_all_zeroes()
437 	if (r)  in __superblock_all_zeroes()
438 		return r;  in __superblock_all_zeroes()
494 	int r;  in save_sm_roots()  local
497 	r = dm_sm_root_size(pmd->metadata_sm, &len);  in save_sm_roots()
498 	if (r < 0)  in save_sm_roots()
499 		return r;  in save_sm_roots()
501 	r = dm_sm_copy_root(pmd->metadata_sm, &pmd->metadata_space_map_root, len);  in save_sm_roots()
502 	if (r < 0)  in save_sm_roots()
503 		return r;  in save_sm_roots()
505 	r = dm_sm_root_size(pmd->data_sm, &len);  in save_sm_roots()
506 	if (r < 0)  in save_sm_roots()
507 		return r;  in save_sm_roots()
526 	int r;  in __write_initial_superblock()  local
534 	r = dm_sm_commit(pmd->data_sm);  in __write_initial_superblock()
535 	if (r < 0)  in __write_initial_superblock()
536 		return r;  in __write_initial_superblock()
538 	r = dm_tm_pre_commit(pmd->tm);  in __write_initial_superblock()
539 	if (r < 0)  in __write_initial_superblock()
540 		return r;  in __write_initial_superblock()
542 	r = save_sm_roots(pmd);  in __write_initial_superblock()
543 	if (r < 0)  in __write_initial_superblock()
544 		return r;  in __write_initial_superblock()
546 	r = superblock_lock_zero(pmd, &sblock);  in __write_initial_superblock()
547 	if (r)  in __write_initial_superblock()
548 		return r;  in __write_initial_superblock()
572 	int r;  in __format_metadata()  local
574 	r = dm_tm_create_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __format_metadata()
576 	if (r < 0) {  in __format_metadata()
578 		return r;  in __format_metadata()
584 		r = PTR_ERR(pmd->data_sm);  in __format_metadata()
591 		r = -ENOMEM;  in __format_metadata()
597 	r = dm_btree_empty(&pmd->info, &pmd->root);  in __format_metadata()
598 	if (r < 0)  in __format_metadata()
601 	r = dm_btree_empty(&pmd->details_info, &pmd->details_root);  in __format_metadata()
602 	if (r < 0) {  in __format_metadata()
607 	r = __write_initial_superblock(pmd);  in __format_metadata()
608 	if (r)  in __format_metadata()
621 	return r;  in __format_metadata()
654 	int r;  in __open_metadata()  local
658 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
660 	if (r < 0) {  in __open_metadata()
662 		return r;  in __open_metadata()
672 		r = -EINVAL;  in __open_metadata()
676 	r = __check_incompat_features(disk_super, pmd);  in __open_metadata()
677 	if (r < 0)  in __open_metadata()
680 	r = dm_tm_open_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
684 	if (r < 0) {  in __open_metadata()
693 		r = PTR_ERR(pmd->data_sm);  in __open_metadata()
700 		r = -ENOMEM;  in __open_metadata()
717 	return r;  in __open_metadata()
722 	int r, unformatted;  in __open_or_format_metadata()  local
724 	r = __superblock_all_zeroes(pmd->bm, &unformatted);  in __open_or_format_metadata()
725 	if (r)  in __open_or_format_metadata()
726 		return r;  in __open_or_format_metadata()
736 	int r;  in __create_persistent_data_objects()  local
742 		r = PTR_ERR(pmd->bm);  in __create_persistent_data_objects()
744 		return r;  in __create_persistent_data_objects()
747 	r = __open_or_format_metadata(pmd, format_device);  in __create_persistent_data_objects()
748 	if (r) {  in __create_persistent_data_objects()
753 	return r;  in __create_persistent_data_objects()
767 	int r;  in __begin_transaction()  local
775 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __begin_transaction()
777 	if (r)  in __begin_transaction()
778 		return r;  in __begin_transaction()
794 	int r;  in __write_changed_details()  local
811 		r = dm_btree_insert(&pmd->details_info, pmd->details_root,  in __write_changed_details()
813 		if (r)  in __write_changed_details()
814 			return r;  in __write_changed_details()
829 	int r;  in __commit_transaction()  local
843 		r = pmd->pre_commit_fn(pmd->pre_commit_context);  in __commit_transaction()
844 		if (r < 0) {  in __commit_transaction()
846 			return r;  in __commit_transaction()
850 	r = __write_changed_details(pmd);  in __commit_transaction()
851 	if (r < 0)  in __commit_transaction()
852 		return r;  in __commit_transaction()
854 	r = dm_sm_commit(pmd->data_sm);  in __commit_transaction()
855 	if (r < 0)  in __commit_transaction()
856 		return r;  in __commit_transaction()
858 	r = dm_tm_pre_commit(pmd->tm);  in __commit_transaction()
859 	if (r < 0)  in __commit_transaction()
860 		return r;  in __commit_transaction()
862 	r = save_sm_roots(pmd);  in __commit_transaction()
863 	if (r < 0)  in __commit_transaction()
864 		return r;  in __commit_transaction()
866 	r = superblock_lock(pmd, &sblock);  in __commit_transaction()
867 	if (r)  in __commit_transaction()
868 		return r;  in __commit_transaction()
884 	int r;  in __set_metadata_reserve()  local
888 	r = dm_sm_get_nr_blocks(pmd->metadata_sm, &total);  in __set_metadata_reserve()
889 	if (r) {  in __set_metadata_reserve()
900 	int r;  in dm_pool_metadata_open()  local
919 	r = __create_persistent_data_objects(pmd, format_device);  in dm_pool_metadata_open()
920 	if (r) {  in dm_pool_metadata_open()
922 		return ERR_PTR(r);  in dm_pool_metadata_open()
925 	r = __begin_transaction(pmd);  in dm_pool_metadata_open()
926 	if (r < 0) {  in dm_pool_metadata_open()
929 		return ERR_PTR(r);  in dm_pool_metadata_open()
939 	int r;  in dm_pool_metadata_close()  local
962 		r = __commit_transaction(pmd);  in dm_pool_metadata_close()
963 		if (r < 0)  in dm_pool_metadata_close()
965 			       __func__, r);  in dm_pool_metadata_close()
984 	int r, changed = 0;  in __open_device()  local
1008 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __open_device()
1010 	if (r) {  in __open_device()
1011 		if (r != -ENODATA || !create)  in __open_device()
1012 			return r;  in __open_device()
1051 	int r;  in __create_thin()  local
1057 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_thin()
1059 	if (!r)  in __create_thin()
1065 	r = dm_btree_empty(&pmd->bl_info, &dev_root);  in __create_thin()
1066 	if (r)  in __create_thin()
1067 		return r;  in __create_thin()
1074 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_thin()
1075 	if (r) {  in __create_thin()
1077 		return r;  in __create_thin()
1080 	r = __open_device(pmd, dev, 1, &td);  in __create_thin()
1081 	if (r) {  in __create_thin()
1084 		return r;  in __create_thin()
1088 	return r;  in __create_thin()
1093 	int r = -EINVAL;  in dm_pool_create_thin()  local
1097 		r = __create_thin(pmd, dev);  in dm_pool_create_thin()
1100 	return r;  in dm_pool_create_thin()
1107 	int r;  in __set_snapshot_details()  local
1110 	r = __open_device(pmd, origin, 0, &td);  in __set_snapshot_details()
1111 	if (r)  in __set_snapshot_details()
1112 		return r;  in __set_snapshot_details()
1127 	int r;  in __create_snap()  local
1134 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_snap()
1136 	if (!r)  in __create_snap()
1140 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &key, &value);  in __create_snap()
1141 	if (r)  in __create_snap()
1142 		return r;  in __create_snap()
1152 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_snap()
1153 	if (r) {  in __create_snap()
1155 		return r;  in __create_snap()
1160 	r = __open_device(pmd, dev, 1, &td);  in __create_snap()
1161 	if (r)  in __create_snap()
1164 	r = __set_snapshot_details(pmd, td, origin, pmd->time);  in __create_snap()
1167 	if (r)  in __create_snap()
1176 	return r;  in __create_snap()
1183 	int r = -EINVAL;  in dm_pool_create_snap()  local
1187 		r = __create_snap(pmd, dev, origin);  in dm_pool_create_snap()
1190 	return r;  in dm_pool_create_snap()
1195 	int r;  in __delete_device()  local
1200 	r = __open_device(pmd, dev, 0, &td);  in __delete_device()
1201 	if (r)  in __delete_device()
1202 		return r;  in __delete_device()
1211 	r = dm_btree_remove(&pmd->details_info, pmd->details_root,  in __delete_device()
1213 	if (r)  in __delete_device()
1214 		return r;  in __delete_device()
1216 	r = dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);  in __delete_device()
1217 	if (r)  in __delete_device()
1218 		return r;  in __delete_device()
1226 	int r = -EINVAL;  in dm_pool_delete_thin_device()  local
1230 		r = __delete_device(pmd, dev);  in dm_pool_delete_thin_device()
1233 	return r;  in dm_pool_delete_thin_device()
1240 	int r = -EINVAL;  in dm_pool_set_metadata_transaction_id()  local
1253 	r = 0;  in dm_pool_set_metadata_transaction_id()
1258 	return r;  in dm_pool_set_metadata_transaction_id()
1264 	int r = -EINVAL;  in dm_pool_get_metadata_transaction_id()  local
1269 		r = 0;  in dm_pool_get_metadata_transaction_id()
1273 	return r;  in dm_pool_get_metadata_transaction_id()
1278 	int r, inc;  in __reserve_metadata_snap()  local
1287 	r = __commit_transaction(pmd);  in __reserve_metadata_snap()
1288 	if (r < 0) {  in __reserve_metadata_snap()
1290 		       __func__, r);  in __reserve_metadata_snap()
1291 		return r;  in __reserve_metadata_snap()
1298 	r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION,  in __reserve_metadata_snap()
1300 	if (r)  in __reserve_metadata_snap()
1301 		return r;  in __reserve_metadata_snap()
1334 	r = superblock_lock(pmd, &sblock);  in __reserve_metadata_snap()
1335 	if (r) {  in __reserve_metadata_snap()
1337 		return r;  in __reserve_metadata_snap()
1348 	int r = -EINVAL;  in dm_pool_reserve_metadata_snap()  local
1352 		r = __reserve_metadata_snap(pmd);  in dm_pool_reserve_metadata_snap()
1355 	return r;  in dm_pool_reserve_metadata_snap()
1360 	int r;  in __release_metadata_snap()  local
1365 	r = superblock_lock(pmd, &sblock);  in __release_metadata_snap()
1366 	if (r)  in __release_metadata_snap()
1367 		return r;  in __release_metadata_snap()
1380 	r = dm_tm_read_lock(pmd->tm, held_root, &sb_validator, &copy);  in __release_metadata_snap()
1381 	if (r)  in __release_metadata_snap()
1382 		return r;  in __release_metadata_snap()
1396 	int r = -EINVAL;  in dm_pool_release_metadata_snap()  local
1400 		r = __release_metadata_snap(pmd);  in dm_pool_release_metadata_snap()
1403 	return r;  in dm_pool_release_metadata_snap()
1409 	int r;  in __get_metadata_snap()  local
1413 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __get_metadata_snap()
1415 	if (r)  in __get_metadata_snap()
1416 		return r;  in __get_metadata_snap()
1429 	int r = -EINVAL;  in dm_pool_get_metadata_snap()  local
1433 		r = __get_metadata_snap(pmd, result);  in dm_pool_get_metadata_snap()
1436 	return r;  in dm_pool_get_metadata_snap()
1442 	int r = -EINVAL;  in dm_pool_open_thin_device()  local
1446 		r = __open_device(pmd, dev, 0, td);  in dm_pool_open_thin_device()
1449 	return r;  in dm_pool_open_thin_device()
1493 	int r;  in __find_block()  local
1504 	r = dm_btree_lookup(info, pmd->root, keys, &value);  in __find_block()
1505 	if (!r)  in __find_block()
1508 	return r;  in __find_block()
1514 	int r;  in dm_thin_find_block()  local
1523 	r = __find_block(td, block, can_issue_io, result);  in dm_thin_find_block()
1526 	return r;  in dm_thin_find_block()
1533 	int r;  in __find_next_mapped_block()  local
1538 	r = dm_btree_lookup_next(&pmd->info, pmd->root, keys, vblock, &value);  in __find_next_mapped_block()
1539 	if (!r)  in __find_next_mapped_block()
1542 	return r;  in __find_next_mapped_block()
1550 	int r;  in __find_mapped_range()  local
1557 	r = __find_next_mapped_block(td, begin, &begin, &lookup);  in __find_mapped_range()
1558 	if (r)  in __find_mapped_range()
1559 		return r;  in __find_mapped_range()
1571 		r = __find_block(td, begin, true, &lookup);  in __find_mapped_range()
1572 		if (r) {  in __find_mapped_range()
1573 			if (r == -ENODATA)  in __find_mapped_range()
1576 				return r;  in __find_mapped_range()
1596 	int r = -EINVAL;  in dm_thin_find_mapped_range()  local
1601 		r = __find_mapped_range(td, begin, end, thin_begin, thin_end,  in dm_thin_find_mapped_range()
1606 	return r;  in dm_thin_find_mapped_range()
1612 	int r, inserted;  in __insert()  local
1620 	r = dm_btree_insert_notify(&pmd->info, pmd->root, keys, &value,  in __insert()
1622 	if (r)  in __insert()
1623 		return r;  in __insert()
1635 	int r = -EINVAL;  in dm_thin_insert_block()  local
1639 		r = __insert(td, block, data_block);  in dm_thin_insert_block()
1642 	return r;  in dm_thin_insert_block()
1647 	int r;  in __remove()  local
1651 	r = dm_btree_remove(&pmd->info, pmd->root, keys, &pmd->root);  in __remove()
1652 	if (r)  in __remove()
1653 		return r;  in __remove()
1663 	int r;  in __remove_range()  local
1673 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, keys, &value);  in __remove_range()
1674 	if (r)  in __remove_range()
1675 		return r;  in __remove_range()
1683 	r = dm_btree_remove(&pmd->tl_info, pmd->root, keys, &pmd->root);  in __remove_range()
1684 	if (r)  in __remove_range()
1685 		return r;  in __remove_range()
1692 		r = dm_btree_lookup_next(&pmd->bl_info, mapping_root, &begin, &begin, &value);  in __remove_range()
1693 		if (r == -ENODATA)  in __remove_range()
1696 		if (r)  in __remove_range()
1697 			return r;  in __remove_range()
1702 		r = dm_btree_remove_leaves(&pmd->bl_info, mapping_root, &begin, end, &mapping_root, &count);  in __remove_range()
1703 		if (r)  in __remove_range()
1704 			return r;  in __remove_range()
1722 	int r = -EINVAL;  in dm_thin_remove_block()  local
1726 		r = __remove(td, block);  in dm_thin_remove_block()
1729 	return r;  in dm_thin_remove_block()
1735 	int r = -EINVAL;  in dm_thin_remove_range()  local
1739 		r = __remove_range(td, begin, end);  in dm_thin_remove_range()
1742 	return r;  in dm_thin_remove_range()
1747 	int r;  in dm_pool_block_is_shared()  local
1751 	r = dm_sm_get_count(pmd->data_sm, b, &ref_count);  in dm_pool_block_is_shared()
1752 	if (!r)  in dm_pool_block_is_shared()
1756 	return r;  in dm_pool_block_is_shared()
1761 	int r = 0;  in dm_pool_inc_data_range()  local
1765 		r = dm_sm_inc_block(pmd->data_sm, b);  in dm_pool_inc_data_range()
1766 		if (r)  in dm_pool_inc_data_range()
1771 	return r;  in dm_pool_inc_data_range()
1776 	int r = 0;  in dm_pool_dec_data_range()  local
1780 		r = dm_sm_dec_block(pmd->data_sm, b);  in dm_pool_dec_data_range()
1781 		if (r)  in dm_pool_dec_data_range()
1786 	return r;  in dm_pool_dec_data_range()
1791 	int r;  in dm_thin_changed_this_transaction()  local
1794 	r = td->changed;  in dm_thin_changed_this_transaction()
1797 	return r;  in dm_thin_changed_this_transaction()
1802 	bool r = false;  in dm_pool_changed_this_transaction()  local
1808 			r = td->changed;  in dm_pool_changed_this_transaction()
1814 	return r;  in dm_pool_changed_this_transaction()
1819 	bool r;  in dm_thin_aborted_changes()  local
1822 	r = td->aborted_with_changes;  in dm_thin_aborted_changes()
1825 	return r;  in dm_thin_aborted_changes()
1830 	int r = -EINVAL;  in dm_pool_alloc_data_block()  local
1834 		r = dm_sm_new_block(pmd->data_sm, result);  in dm_pool_alloc_data_block()
1837 	return r;  in dm_pool_alloc_data_block()
1842 	int r = -EINVAL;  in dm_pool_commit_metadata()  local
1852 	r = __commit_transaction(pmd);  in dm_pool_commit_metadata()
1853 	if (r < 0)  in dm_pool_commit_metadata()
1859 	r = __begin_transaction(pmd);  in dm_pool_commit_metadata()
1862 	return r;  in dm_pool_commit_metadata()
1875 	int r = -EINVAL;  in dm_pool_abort_metadata()  local
1883 	r = __create_persistent_data_objects(pmd, false);  in dm_pool_abort_metadata()
1884 	if (r)  in dm_pool_abort_metadata()
1890 	return r;  in dm_pool_abort_metadata()
1895 	int r = -EINVAL;  in dm_pool_get_free_block_count()  local
1899 		r = dm_sm_get_nr_free(pmd->data_sm, result);  in dm_pool_get_free_block_count()
1902 	return r;  in dm_pool_get_free_block_count()
1908 	int r = -EINVAL;  in dm_pool_get_free_metadata_block_count()  local
1912 		r = dm_sm_get_nr_free(pmd->metadata_sm, result);  in dm_pool_get_free_metadata_block_count()
1914 	if (!r) {  in dm_pool_get_free_metadata_block_count()
1922 	return r;  in dm_pool_get_free_metadata_block_count()
1928 	int r = -EINVAL;  in dm_pool_get_metadata_dev_size()  local
1932 		r = dm_sm_get_nr_blocks(pmd->metadata_sm, result);  in dm_pool_get_metadata_dev_size()
1935 	return r;  in dm_pool_get_metadata_dev_size()
1940 	int r = -EINVAL;  in dm_pool_get_data_dev_size()  local
1944 		r = dm_sm_get_nr_blocks(pmd->data_sm, result);  in dm_pool_get_data_dev_size()
1947 	return r;  in dm_pool_get_data_dev_size()
1952 	int r = -EINVAL;  in dm_thin_get_mapped_count()  local
1958 		r = 0;  in dm_thin_get_mapped_count()
1962 	return r;  in dm_thin_get_mapped_count()
1967 	int r;  in __highest_block()  local
1972 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &td->id, &value_le);  in __highest_block()
1973 	if (r)  in __highest_block()
1974 		return r;  in __highest_block()
1984 	int r = -EINVAL;  in dm_thin_get_highest_mapped_block()  local
1989 		r = __highest_block(td, result);  in dm_thin_get_highest_mapped_block()
1992 	return r;  in dm_thin_get_highest_mapped_block()
1997 	int r;  in __resize_space_map()  local
2000 	r = dm_sm_get_nr_blocks(sm, &old_count);  in __resize_space_map()
2001 	if (r)  in __resize_space_map()
2002 		return r;  in __resize_space_map()
2017 	int r = -EINVAL;  in dm_pool_resize_data_dev()  local
2021 		r = __resize_space_map(pmd->data_sm, new_count);  in dm_pool_resize_data_dev()
2024 	return r;  in dm_pool_resize_data_dev()
2029 	int r = -EINVAL;  in dm_pool_resize_metadata_dev()  local
2033 		r = __resize_space_map(pmd->metadata_sm, new_count);  in dm_pool_resize_metadata_dev()
2034 		if (!r)  in dm_pool_resize_metadata_dev()
2039 	return r;  in dm_pool_resize_metadata_dev()
2061 	int r;  in dm_pool_register_metadata_threshold()  local
2064 	r = dm_sm_register_threshold_callback(pmd->metadata_sm, threshold, fn, context);  in dm_pool_register_metadata_threshold()
2067 	return r;  in dm_pool_register_metadata_threshold()
2082 	int r = -EINVAL;  in dm_pool_metadata_set_needs_check()  local
2092 	r = superblock_lock(pmd, &sblock);  in dm_pool_metadata_set_needs_check()
2093 	if (r) {  in dm_pool_metadata_set_needs_check()
2104 	return r;  in dm_pool_metadata_set_needs_check()