419 	int r;  in __superblock_all_zeroes()  local
428 	r = dm_bm_read_lock(bm, THIN_SUPERBLOCK_LOCATION, NULL, &b);  in __superblock_all_zeroes()
429 	if (r)  in __superblock_all_zeroes()
430 		return r;  in __superblock_all_zeroes()
486 	int r;  in save_sm_roots()  local
489 	r = dm_sm_root_size(pmd->metadata_sm, &len);  in save_sm_roots()
490 	if (r < 0)  in save_sm_roots()
491 		return r;  in save_sm_roots()
493 	r = dm_sm_copy_root(pmd->metadata_sm, &pmd->metadata_space_map_root, len);  in save_sm_roots()
494 	if (r < 0)  in save_sm_roots()
495 		return r;  in save_sm_roots()
497 	r = dm_sm_root_size(pmd->data_sm, &len);  in save_sm_roots()
498 	if (r < 0)  in save_sm_roots()
499 		return r;  in save_sm_roots()
518 	int r;  in __write_initial_superblock()  local
526 	r = dm_sm_commit(pmd->data_sm);  in __write_initial_superblock()
527 	if (r < 0)  in __write_initial_superblock()
528 		return r;  in __write_initial_superblock()
530 	r = dm_tm_pre_commit(pmd->tm);  in __write_initial_superblock()
531 	if (r < 0)  in __write_initial_superblock()
532 		return r;  in __write_initial_superblock()
534 	r = save_sm_roots(pmd);  in __write_initial_superblock()
535 	if (r < 0)  in __write_initial_superblock()
536 		return r;  in __write_initial_superblock()
538 	r = superblock_lock_zero(pmd, &sblock);  in __write_initial_superblock()
539 	if (r)  in __write_initial_superblock()
540 		return r;  in __write_initial_superblock()
564 	int r;  in __format_metadata()  local
566 	r = dm_tm_create_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __format_metadata()
568 	if (r < 0) {  in __format_metadata()
570 		return r;  in __format_metadata()
576 		r = PTR_ERR(pmd->data_sm);  in __format_metadata()
583 		r = -ENOMEM;  in __format_metadata()
589 	r = dm_btree_empty(&pmd->info, &pmd->root);  in __format_metadata()
590 	if (r < 0)  in __format_metadata()
593 	r = dm_btree_empty(&pmd->details_info, &pmd->details_root);  in __format_metadata()
594 	if (r < 0) {  in __format_metadata()
599 	r = __write_initial_superblock(pmd);  in __format_metadata()
600 	if (r)  in __format_metadata()
613 	return r;  in __format_metadata()
646 	int r;  in __open_metadata()  local
650 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
652 	if (r < 0) {  in __open_metadata()
654 		return r;  in __open_metadata()
664 		r = -EINVAL;  in __open_metadata()
668 	r = __check_incompat_features(disk_super, pmd);  in __open_metadata()
669 	if (r < 0)  in __open_metadata()
672 	r = dm_tm_open_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
676 	if (r < 0) {  in __open_metadata()
685 		r = PTR_ERR(pmd->data_sm);  in __open_metadata()
692 		r = -ENOMEM;  in __open_metadata()
709 	return r;  in __open_metadata()
714 	int r, unformatted;  in __open_or_format_metadata()  local
716 	r = __superblock_all_zeroes(pmd->bm, &unformatted);  in __open_or_format_metadata()
717 	if (r)  in __open_or_format_metadata()
718 		return r;  in __open_or_format_metadata()
728 	int r;  in __create_persistent_data_objects()  local
737 	r = __open_or_format_metadata(pmd, format_device);  in __create_persistent_data_objects()
738 	if (r)  in __create_persistent_data_objects()
741 	return r;  in __create_persistent_data_objects()
755 	int r;  in __begin_transaction()  local
763 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __begin_transaction()
765 	if (r)  in __begin_transaction()
766 		return r;  in __begin_transaction()
782 	int r;  in __write_changed_details()  local
799 		r = dm_btree_insert(&pmd->details_info, pmd->details_root,  in __write_changed_details()
801 		if (r)  in __write_changed_details()
802 			return r;  in __write_changed_details()
817 	int r;  in __commit_transaction()  local
829 	r = __write_changed_details(pmd);  in __commit_transaction()
830 	if (r < 0)  in __commit_transaction()
831 		return r;  in __commit_transaction()
833 	r = dm_sm_commit(pmd->data_sm);  in __commit_transaction()
834 	if (r < 0)  in __commit_transaction()
835 		return r;  in __commit_transaction()
837 	r = dm_tm_pre_commit(pmd->tm);  in __commit_transaction()
838 	if (r < 0)  in __commit_transaction()
839 		return r;  in __commit_transaction()
841 	r = save_sm_roots(pmd);  in __commit_transaction()
842 	if (r < 0)  in __commit_transaction()
843 		return r;  in __commit_transaction()
845 	r = superblock_lock(pmd, &sblock);  in __commit_transaction()
846 	if (r)  in __commit_transaction()
847 		return r;  in __commit_transaction()
863 	int r;  in __set_metadata_reserve()  local
867 	r = dm_sm_get_nr_blocks(pmd->metadata_sm, &total);  in __set_metadata_reserve()
868 	if (r) {  in __set_metadata_reserve()
879 	int r;  in dm_pool_metadata_open()  local
896 	r = __create_persistent_data_objects(pmd, format_device);  in dm_pool_metadata_open()
897 	if (r) {  in dm_pool_metadata_open()
899 		return ERR_PTR(r);  in dm_pool_metadata_open()
902 	r = __begin_transaction(pmd);  in dm_pool_metadata_open()
903 	if (r < 0) {  in dm_pool_metadata_open()
906 		return ERR_PTR(r);  in dm_pool_metadata_open()
916 	int r;  in dm_pool_metadata_close()  local
938 		r = __commit_transaction(pmd);  in dm_pool_metadata_close()
939 		if (r < 0)  in dm_pool_metadata_close()
941 			       __func__, r);  in dm_pool_metadata_close()
959 	int r, changed = 0;  in __open_device()  local
983 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __open_device()
985 	if (r) {  in __open_device()
986 		if (r != -ENODATA || !create)  in __open_device()
987 			return r;  in __open_device()
1026 	int r;  in __create_thin()  local
1033 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_thin()
1035 	if (!r)  in __create_thin()
1041 	r = dm_btree_empty(&pmd->bl_info, &dev_root);  in __create_thin()
1042 	if (r)  in __create_thin()
1043 		return r;  in __create_thin()
1050 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_thin()
1051 	if (r) {  in __create_thin()
1053 		return r;  in __create_thin()
1056 	r = __open_device(pmd, dev, 1, &td);  in __create_thin()
1057 	if (r) {  in __create_thin()
1060 		return r;  in __create_thin()
1064 	return r;  in __create_thin()
1069 	int r = -EINVAL;  in dm_pool_create_thin()  local
1073 		r = __create_thin(pmd, dev);  in dm_pool_create_thin()
1076 	return r;  in dm_pool_create_thin()
1083 	int r;  in __set_snapshot_details()  local
1086 	r = __open_device(pmd, origin, 0, &td);  in __set_snapshot_details()
1087 	if (r)  in __set_snapshot_details()
1088 		return r;  in __set_snapshot_details()
1103 	int r;  in __create_snap()  local
1111 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_snap()
1113 	if (!r)  in __create_snap()
1117 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &key, &value);  in __create_snap()
1118 	if (r)  in __create_snap()
1119 		return r;  in __create_snap()
1129 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_snap()
1130 	if (r) {  in __create_snap()
1132 		return r;  in __create_snap()
1137 	r = __open_device(pmd, dev, 1, &td);  in __create_snap()
1138 	if (r)  in __create_snap()
1141 	r = __set_snapshot_details(pmd, td, origin, pmd->time);  in __create_snap()
1144 	if (r)  in __create_snap()
1153 	return r;  in __create_snap()
1160 	int r = -EINVAL;  in dm_pool_create_snap()  local
1164 		r = __create_snap(pmd, dev, origin);  in dm_pool_create_snap()
1167 	return r;  in dm_pool_create_snap()
1172 	int r;  in __delete_device()  local
1177 	r = __open_device(pmd, dev, 0, &td);  in __delete_device()
1178 	if (r)  in __delete_device()
1179 		return r;  in __delete_device()
1188 	r = dm_btree_remove(&pmd->details_info, pmd->details_root,  in __delete_device()
1190 	if (r)  in __delete_device()
1191 		return r;  in __delete_device()
1193 	r = dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);  in __delete_device()
1194 	if (r)  in __delete_device()
1195 		return r;  in __delete_device()
1203 	int r = -EINVAL;  in dm_pool_delete_thin_device()  local
1207 		r = __delete_device(pmd, dev);  in dm_pool_delete_thin_device()
1210 	return r;  in dm_pool_delete_thin_device()
1217 	int r = -EINVAL;  in dm_pool_set_metadata_transaction_id()  local
1230 	r = 0;  in dm_pool_set_metadata_transaction_id()
1235 	return r;  in dm_pool_set_metadata_transaction_id()
1241 	int r = -EINVAL;  in dm_pool_get_metadata_transaction_id()  local
1246 		r = 0;  in dm_pool_get_metadata_transaction_id()
1250 	return r;  in dm_pool_get_metadata_transaction_id()
1255 	int r, inc;  in __reserve_metadata_snap()  local
1264 	r = __commit_transaction(pmd);  in __reserve_metadata_snap()
1265 	if (r < 0) {  in __reserve_metadata_snap()
1267 		       __func__, r);  in __reserve_metadata_snap()
1268 		return r;  in __reserve_metadata_snap()
1275 	r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION,  in __reserve_metadata_snap()
1277 	if (r)  in __reserve_metadata_snap()
1278 		return r;  in __reserve_metadata_snap()
1311 	r = superblock_lock(pmd, &sblock);  in __reserve_metadata_snap()
1312 	if (r) {  in __reserve_metadata_snap()
1314 		return r;  in __reserve_metadata_snap()
1325 	int r = -EINVAL;  in dm_pool_reserve_metadata_snap()  local
1329 		r = __reserve_metadata_snap(pmd);  in dm_pool_reserve_metadata_snap()
1332 	return r;  in dm_pool_reserve_metadata_snap()
1337 	int r;  in __release_metadata_snap()  local
1342 	r = superblock_lock(pmd, &sblock);  in __release_metadata_snap()
1343 	if (r)  in __release_metadata_snap()
1344 		return r;  in __release_metadata_snap()
1357 	r = dm_tm_read_lock(pmd->tm, held_root, &sb_validator, &copy);  in __release_metadata_snap()
1358 	if (r)  in __release_metadata_snap()
1359 		return r;  in __release_metadata_snap()
1373 	int r = -EINVAL;  in dm_pool_release_metadata_snap()  local
1377 		r = __release_metadata_snap(pmd);  in dm_pool_release_metadata_snap()
1380 	return r;  in dm_pool_release_metadata_snap()
1386 	int r;  in __get_metadata_snap()  local
1390 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __get_metadata_snap()
1392 	if (r)  in __get_metadata_snap()
1393 		return r;  in __get_metadata_snap()
1406 	int r = -EINVAL;  in dm_pool_get_metadata_snap()  local
1410 		r = __get_metadata_snap(pmd, result);  in dm_pool_get_metadata_snap()
1413 	return r;  in dm_pool_get_metadata_snap()
1419 	int r = -EINVAL;  in dm_pool_open_thin_device()  local
1423 		r = __open_device(pmd, dev, 0, td);  in dm_pool_open_thin_device()
1426 	return r;  in dm_pool_open_thin_device()
1470 	int r;  in __find_block()  local
1481 	r = dm_btree_lookup(info, pmd->root, keys, &value);  in __find_block()
1482 	if (!r)  in __find_block()
1485 	return r;  in __find_block()
1491 	int r;  in dm_thin_find_block()  local
1500 	r = __find_block(td, block, can_issue_io, result);  in dm_thin_find_block()
1503 	return r;  in dm_thin_find_block()
1510 	int r;  in __find_next_mapped_block()  local
1515 	r = dm_btree_lookup_next(&pmd->info, pmd->root, keys, vblock, &value);  in __find_next_mapped_block()
1516 	if (!r)  in __find_next_mapped_block()
1519 	return r;  in __find_next_mapped_block()
1527 	int r;  in __find_mapped_range()  local
1534 	r = __find_next_mapped_block(td, begin, &begin, &lookup);  in __find_mapped_range()
1535 	if (r)  in __find_mapped_range()
1536 		return r;  in __find_mapped_range()
1548 		r = __find_block(td, begin, true, &lookup);  in __find_mapped_range()
1549 		if (r) {  in __find_mapped_range()
1550 			if (r == -ENODATA)  in __find_mapped_range()
1553 				return r;  in __find_mapped_range()
1573 	int r = -EINVAL;  in dm_thin_find_mapped_range()  local
1578 		r = __find_mapped_range(td, begin, end, thin_begin, thin_end,  in dm_thin_find_mapped_range()
1583 	return r;  in dm_thin_find_mapped_range()
1589 	int r, inserted;  in __insert()  local
1597 	r = dm_btree_insert_notify(&pmd->info, pmd->root, keys, &value,  in __insert()
1599 	if (r)  in __insert()
1600 		return r;  in __insert()
1612 	int r = -EINVAL;  in dm_thin_insert_block()  local
1616 		r = __insert(td, block, data_block);  in dm_thin_insert_block()
1619 	return r;  in dm_thin_insert_block()
1624 	int r;  in __remove()  local
1628 	r = dm_btree_remove(&pmd->info, pmd->root, keys, &pmd->root);  in __remove()
1629 	if (r)  in __remove()
1630 		return r;  in __remove()
1640 	int r;  in __remove_range()  local
1650 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, keys, &value);  in __remove_range()
1651 	if (r)  in __remove_range()
1652 		return r;  in __remove_range()
1660 	r = dm_btree_remove(&pmd->tl_info, pmd->root, keys, &pmd->root);  in __remove_range()
1661 	if (r)  in __remove_range()
1662 		return r;  in __remove_range()
1669 		r = dm_btree_lookup_next(&pmd->bl_info, mapping_root, &begin, &begin, &value);  in __remove_range()
1670 		if (r == -ENODATA)  in __remove_range()
1673 		if (r)  in __remove_range()
1674 			return r;  in __remove_range()
1679 		r = dm_btree_remove_leaves(&pmd->bl_info, mapping_root, &begin, end, &mapping_root, &count);  in __remove_range()
1680 		if (r)  in __remove_range()
1681 			return r;  in __remove_range()
1699 	int r = -EINVAL;  in dm_thin_remove_block()  local
1703 		r = __remove(td, block);  in dm_thin_remove_block()
1706 	return r;  in dm_thin_remove_block()
1712 	int r = -EINVAL;  in dm_thin_remove_range()  local
1716 		r = __remove_range(td, begin, end);  in dm_thin_remove_range()
1719 	return r;  in dm_thin_remove_range()
1724 	int r;  in dm_pool_block_is_shared()  local
1728 	r = dm_sm_get_count(pmd->data_sm, b, &ref_count);  in dm_pool_block_is_shared()
1729 	if (!r)  in dm_pool_block_is_shared()
1733 	return r;  in dm_pool_block_is_shared()
1738 	int r = 0;  in dm_pool_inc_data_range()  local
1742 		r = dm_sm_inc_block(pmd->data_sm, b);  in dm_pool_inc_data_range()
1743 		if (r)  in dm_pool_inc_data_range()
1748 	return r;  in dm_pool_inc_data_range()
1753 	int r = 0;  in dm_pool_dec_data_range()  local
1757 		r = dm_sm_dec_block(pmd->data_sm, b);  in dm_pool_dec_data_range()
1758 		if (r)  in dm_pool_dec_data_range()
1763 	return r;  in dm_pool_dec_data_range()
1768 	int r;  in dm_thin_changed_this_transaction()  local
1771 	r = td->changed;  in dm_thin_changed_this_transaction()
1774 	return r;  in dm_thin_changed_this_transaction()
1779 	bool r = false;  in dm_pool_changed_this_transaction()  local
1785 			r = td->changed;  in dm_pool_changed_this_transaction()
1791 	return r;  in dm_pool_changed_this_transaction()
1796 	bool r;  in dm_thin_aborted_changes()  local
1799 	r = td->aborted_with_changes;  in dm_thin_aborted_changes()
1802 	return r;  in dm_thin_aborted_changes()
1807 	int r = -EINVAL;  in dm_pool_alloc_data_block()  local
1811 		r = dm_sm_new_block(pmd->data_sm, result);  in dm_pool_alloc_data_block()
1814 	return r;  in dm_pool_alloc_data_block()
1819 	int r = -EINVAL;  in dm_pool_commit_metadata()  local
1829 	r = __commit_transaction(pmd);  in dm_pool_commit_metadata()
1830 	if (r < 0)  in dm_pool_commit_metadata()
1836 	r = __begin_transaction(pmd);  in dm_pool_commit_metadata()
1839 	return r;  in dm_pool_commit_metadata()
1852 	int r = -EINVAL;  in dm_pool_abort_metadata()  local
1860 	r = __create_persistent_data_objects(pmd, false);  in dm_pool_abort_metadata()
1861 	if (r)  in dm_pool_abort_metadata()
1867 	return r;  in dm_pool_abort_metadata()
1872 	int r = -EINVAL;  in dm_pool_get_free_block_count()  local
1876 		r = dm_sm_get_nr_free(pmd->data_sm, result);  in dm_pool_get_free_block_count()
1879 	return r;  in dm_pool_get_free_block_count()
1885 	int r = -EINVAL;  in dm_pool_get_free_metadata_block_count()  local
1889 		r = dm_sm_get_nr_free(pmd->metadata_sm, result);  in dm_pool_get_free_metadata_block_count()
1891 	if (!r) {  in dm_pool_get_free_metadata_block_count()
1899 	return r;  in dm_pool_get_free_metadata_block_count()
1905 	int r = -EINVAL;  in dm_pool_get_metadata_dev_size()  local
1909 		r = dm_sm_get_nr_blocks(pmd->metadata_sm, result);  in dm_pool_get_metadata_dev_size()
1912 	return r;  in dm_pool_get_metadata_dev_size()
1917 	int r = -EINVAL;  in dm_pool_get_data_dev_size()  local
1921 		r = dm_sm_get_nr_blocks(pmd->data_sm, result);  in dm_pool_get_data_dev_size()
1924 	return r;  in dm_pool_get_data_dev_size()
1929 	int r = -EINVAL;  in dm_thin_get_mapped_count()  local
1935 		r = 0;  in dm_thin_get_mapped_count()
1939 	return r;  in dm_thin_get_mapped_count()
1944 	int r;  in __highest_block()  local
1949 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &td->id, &value_le);  in __highest_block()
1950 	if (r)  in __highest_block()
1951 		return r;  in __highest_block()
1961 	int r = -EINVAL;  in dm_thin_get_highest_mapped_block()  local
1966 		r = __highest_block(td, result);  in dm_thin_get_highest_mapped_block()
1969 	return r;  in dm_thin_get_highest_mapped_block()
1974 	int r;  in __resize_space_map()  local
1977 	r = dm_sm_get_nr_blocks(sm, &old_count);  in __resize_space_map()
1978 	if (r)  in __resize_space_map()
1979 		return r;  in __resize_space_map()
1994 	int r = -EINVAL;  in dm_pool_resize_data_dev()  local
1998 		r = __resize_space_map(pmd->data_sm, new_count);  in dm_pool_resize_data_dev()
2001 	return r;  in dm_pool_resize_data_dev()
2006 	int r = -EINVAL;  in dm_pool_resize_metadata_dev()  local
2010 		r = __resize_space_map(pmd->metadata_sm, new_count);  in dm_pool_resize_metadata_dev()
2011 		if (!r)  in dm_pool_resize_metadata_dev()
2016 	return r;  in dm_pool_resize_metadata_dev()
2038 	int r;  in dm_pool_register_metadata_threshold()  local
2041 	r = dm_sm_register_threshold_callback(pmd->metadata_sm, threshold, fn, context);  in dm_pool_register_metadata_threshold()
2044 	return r;  in dm_pool_register_metadata_threshold()
2049 	int r = -EINVAL;  in dm_pool_metadata_set_needs_check()  local
2059 	r = superblock_lock(pmd, &sblock);  in dm_pool_metadata_set_needs_check()
2060 	if (r) {  in dm_pool_metadata_set_needs_check()
2071 	return r;  in dm_pool_metadata_set_needs_check()