386 	int r;  in __superblock_all_zeroes()  local
395 	r = dm_bm_read_lock(bm, THIN_SUPERBLOCK_LOCATION, NULL, &b);  in __superblock_all_zeroes()
396 	if (r)  in __superblock_all_zeroes()
397 		return r;  in __superblock_all_zeroes()
453 	int r;  in save_sm_roots()  local
456 	r = dm_sm_root_size(pmd->metadata_sm, &len);  in save_sm_roots()
457 	if (r < 0)  in save_sm_roots()
458 		return r;  in save_sm_roots()
460 	r = dm_sm_copy_root(pmd->metadata_sm, &pmd->metadata_space_map_root, len);  in save_sm_roots()
461 	if (r < 0)  in save_sm_roots()
462 		return r;  in save_sm_roots()
464 	r = dm_sm_root_size(pmd->data_sm, &len);  in save_sm_roots()
465 	if (r < 0)  in save_sm_roots()
466 		return r;  in save_sm_roots()
485 	int r;  in __write_initial_superblock()  local
493 	r = dm_sm_commit(pmd->data_sm);  in __write_initial_superblock()
494 	if (r < 0)  in __write_initial_superblock()
495 		return r;  in __write_initial_superblock()
497 	r = dm_tm_pre_commit(pmd->tm);  in __write_initial_superblock()
498 	if (r < 0)  in __write_initial_superblock()
499 		return r;  in __write_initial_superblock()
501 	r = save_sm_roots(pmd);  in __write_initial_superblock()
502 	if (r < 0)  in __write_initial_superblock()
503 		return r;  in __write_initial_superblock()
505 	r = superblock_lock_zero(pmd, &sblock);  in __write_initial_superblock()
506 	if (r)  in __write_initial_superblock()
507 		return r;  in __write_initial_superblock()
531 	int r;  in __format_metadata()  local
533 	r = dm_tm_create_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __format_metadata()
535 	if (r < 0) {  in __format_metadata()
537 		return r;  in __format_metadata()
543 		r = PTR_ERR(pmd->data_sm);  in __format_metadata()
550 		r = -ENOMEM;  in __format_metadata()
556 	r = dm_btree_empty(&pmd->info, &pmd->root);  in __format_metadata()
557 	if (r < 0)  in __format_metadata()
560 	r = dm_btree_empty(&pmd->details_info, &pmd->details_root);  in __format_metadata()
561 	if (r < 0) {  in __format_metadata()
566 	r = __write_initial_superblock(pmd);  in __format_metadata()
567 	if (r)  in __format_metadata()
580 	return r;  in __format_metadata()
613 	int r;  in __open_metadata()  local
617 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
619 	if (r < 0) {  in __open_metadata()
621 		return r;  in __open_metadata()
631 		r = -EINVAL;  in __open_metadata()
635 	r = __check_incompat_features(disk_super, pmd);  in __open_metadata()
636 	if (r < 0)  in __open_metadata()
639 	r = dm_tm_open_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
643 	if (r < 0) {  in __open_metadata()
652 		r = PTR_ERR(pmd->data_sm);  in __open_metadata()
659 		r = -ENOMEM;  in __open_metadata()
676 	return r;  in __open_metadata()
681 	int r, unformatted;  in __open_or_format_metadata()  local
683 	r = __superblock_all_zeroes(pmd->bm, &unformatted);  in __open_or_format_metadata()
684 	if (r)  in __open_or_format_metadata()
685 		return r;  in __open_or_format_metadata()
695 	int r;  in __create_persistent_data_objects()  local
704 	r = __open_or_format_metadata(pmd, format_device);  in __create_persistent_data_objects()
705 	if (r)  in __create_persistent_data_objects()
708 	return r;  in __create_persistent_data_objects()
722 	int r;  in __begin_transaction()  local
730 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __begin_transaction()
732 	if (r)  in __begin_transaction()
733 		return r;  in __begin_transaction()
749 	int r;  in __write_changed_details()  local
766 		r = dm_btree_insert(&pmd->details_info, pmd->details_root,  in __write_changed_details()
768 		if (r)  in __write_changed_details()
769 			return r;  in __write_changed_details()
784 	int r;  in __commit_transaction()  local
793 	r = __write_changed_details(pmd);  in __commit_transaction()
794 	if (r < 0)  in __commit_transaction()
795 		return r;  in __commit_transaction()
797 	r = dm_sm_commit(pmd->data_sm);  in __commit_transaction()
798 	if (r < 0)  in __commit_transaction()
799 		return r;  in __commit_transaction()
801 	r = dm_tm_pre_commit(pmd->tm);  in __commit_transaction()
802 	if (r < 0)  in __commit_transaction()
803 		return r;  in __commit_transaction()
805 	r = save_sm_roots(pmd);  in __commit_transaction()
806 	if (r < 0)  in __commit_transaction()
807 		return r;  in __commit_transaction()
809 	r = superblock_lock(pmd, &sblock);  in __commit_transaction()
810 	if (r)  in __commit_transaction()
811 		return r;  in __commit_transaction()
827 	int r;  in __set_metadata_reserve()  local
831 	r = dm_sm_get_nr_blocks(pmd->metadata_sm, &total);  in __set_metadata_reserve()
832 	if (r) {  in __set_metadata_reserve()
843 	int r;  in dm_pool_metadata_open()  local
859 	r = __create_persistent_data_objects(pmd, format_device);  in dm_pool_metadata_open()
860 	if (r) {  in dm_pool_metadata_open()
862 		return ERR_PTR(r);  in dm_pool_metadata_open()
865 	r = __begin_transaction(pmd);  in dm_pool_metadata_open()
866 	if (r < 0) {  in dm_pool_metadata_open()
869 		return ERR_PTR(r);  in dm_pool_metadata_open()
879 	int r;  in dm_pool_metadata_close()  local
901 		r = __commit_transaction(pmd);  in dm_pool_metadata_close()
902 		if (r < 0)  in dm_pool_metadata_close()
904 			       __func__, r);  in dm_pool_metadata_close()
923 	int r, changed = 0;  in __open_device()  local
947 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __open_device()
949 	if (r) {  in __open_device()
950 		if (r != -ENODATA || !create)  in __open_device()
951 			return r;  in __open_device()
990 	int r;  in __create_thin()  local
997 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_thin()
999 	if (!r)  in __create_thin()
1005 	r = dm_btree_empty(&pmd->bl_info, &dev_root);  in __create_thin()
1006 	if (r)  in __create_thin()
1007 		return r;  in __create_thin()
1014 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_thin()
1015 	if (r) {  in __create_thin()
1017 		return r;  in __create_thin()
1020 	r = __open_device(pmd, dev, 1, &td);  in __create_thin()
1021 	if (r) {  in __create_thin()
1024 		return r;  in __create_thin()
1028 	return r;  in __create_thin()
1033 	int r = -EINVAL;  in dm_pool_create_thin()  local
1037 		r = __create_thin(pmd, dev);  in dm_pool_create_thin()
1040 	return r;  in dm_pool_create_thin()
1047 	int r;  in __set_snapshot_details()  local
1050 	r = __open_device(pmd, origin, 0, &td);  in __set_snapshot_details()
1051 	if (r)  in __set_snapshot_details()
1052 		return r;  in __set_snapshot_details()
1067 	int r;  in __create_snap()  local
1075 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_snap()
1077 	if (!r)  in __create_snap()
1081 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &key, &value);  in __create_snap()
1082 	if (r)  in __create_snap()
1083 		return r;  in __create_snap()
1093 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_snap()
1094 	if (r) {  in __create_snap()
1096 		return r;  in __create_snap()
1101 	r = __open_device(pmd, dev, 1, &td);  in __create_snap()
1102 	if (r)  in __create_snap()
1105 	r = __set_snapshot_details(pmd, td, origin, pmd->time);  in __create_snap()
1108 	if (r)  in __create_snap()
1117 	return r;  in __create_snap()
1124 	int r = -EINVAL;  in dm_pool_create_snap()  local
1128 		r = __create_snap(pmd, dev, origin);  in dm_pool_create_snap()
1131 	return r;  in dm_pool_create_snap()
1136 	int r;  in __delete_device()  local
1141 	r = __open_device(pmd, dev, 0, &td);  in __delete_device()
1142 	if (r)  in __delete_device()
1143 		return r;  in __delete_device()
1152 	r = dm_btree_remove(&pmd->details_info, pmd->details_root,  in __delete_device()
1154 	if (r)  in __delete_device()
1155 		return r;  in __delete_device()
1157 	r = dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);  in __delete_device()
1158 	if (r)  in __delete_device()
1159 		return r;  in __delete_device()
1167 	int r = -EINVAL;  in dm_pool_delete_thin_device()  local
1171 		r = __delete_device(pmd, dev);  in dm_pool_delete_thin_device()
1174 	return r;  in dm_pool_delete_thin_device()
1181 	int r = -EINVAL;  in dm_pool_set_metadata_transaction_id()  local
1194 	r = 0;  in dm_pool_set_metadata_transaction_id()
1199 	return r;  in dm_pool_set_metadata_transaction_id()
1205 	int r = -EINVAL;  in dm_pool_get_metadata_transaction_id()  local
1210 		r = 0;  in dm_pool_get_metadata_transaction_id()
1214 	return r;  in dm_pool_get_metadata_transaction_id()
1219 	int r, inc;  in __reserve_metadata_snap()  local
1234 	r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION,  in __reserve_metadata_snap()
1236 	if (r)  in __reserve_metadata_snap()
1237 		return r;  in __reserve_metadata_snap()
1270 	r = superblock_lock(pmd, &sblock);  in __reserve_metadata_snap()
1271 	if (r) {  in __reserve_metadata_snap()
1273 		return r;  in __reserve_metadata_snap()
1284 	int r = -EINVAL;  in dm_pool_reserve_metadata_snap()  local
1288 		r = __reserve_metadata_snap(pmd);  in dm_pool_reserve_metadata_snap()
1291 	return r;  in dm_pool_reserve_metadata_snap()
1296 	int r;  in __release_metadata_snap()  local
1301 	r = superblock_lock(pmd, &sblock);  in __release_metadata_snap()
1302 	if (r)  in __release_metadata_snap()
1303 		return r;  in __release_metadata_snap()
1316 	r = dm_tm_read_lock(pmd->tm, held_root, &sb_validator, &copy);  in __release_metadata_snap()
1317 	if (r)  in __release_metadata_snap()
1318 		return r;  in __release_metadata_snap()
1332 	int r = -EINVAL;  in dm_pool_release_metadata_snap()  local
1336 		r = __release_metadata_snap(pmd);  in dm_pool_release_metadata_snap()
1339 	return r;  in dm_pool_release_metadata_snap()
1345 	int r;  in __get_metadata_snap()  local
1349 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __get_metadata_snap()
1351 	if (r)  in __get_metadata_snap()
1352 		return r;  in __get_metadata_snap()
1365 	int r = -EINVAL;  in dm_pool_get_metadata_snap()  local
1369 		r = __get_metadata_snap(pmd, result);  in dm_pool_get_metadata_snap()
1372 	return r;  in dm_pool_get_metadata_snap()
1378 	int r = -EINVAL;  in dm_pool_open_thin_device()  local
1382 		r = __open_device(pmd, dev, 0, td);  in dm_pool_open_thin_device()
1385 	return r;  in dm_pool_open_thin_device()
1429 	int r;  in __find_block()  local
1440 	r = dm_btree_lookup(info, pmd->root, keys, &value);  in __find_block()
1441 	if (!r)  in __find_block()
1444 	return r;  in __find_block()
1450 	int r;  in dm_thin_find_block()  local
1459 	r = __find_block(td, block, can_issue_io, result);  in dm_thin_find_block()
1462 	return r;  in dm_thin_find_block()
1469 	int r;  in __find_next_mapped_block()  local
1474 	r = dm_btree_lookup_next(&pmd->info, pmd->root, keys, vblock, &value);  in __find_next_mapped_block()
1475 	if (!r)  in __find_next_mapped_block()
1478 	return r;  in __find_next_mapped_block()
1486 	int r;  in __find_mapped_range()  local
1493 	r = __find_next_mapped_block(td, begin, &begin, &lookup);  in __find_mapped_range()
1494 	if (r)  in __find_mapped_range()
1495 		return r;  in __find_mapped_range()
1507 		r = __find_block(td, begin, true, &lookup);  in __find_mapped_range()
1508 		if (r) {  in __find_mapped_range()
1509 			if (r == -ENODATA)  in __find_mapped_range()
1512 				return r;  in __find_mapped_range()
1532 	int r = -EINVAL;  in dm_thin_find_mapped_range()  local
1537 		r = __find_mapped_range(td, begin, end, thin_begin, thin_end,  in dm_thin_find_mapped_range()
1542 	return r;  in dm_thin_find_mapped_range()
1548 	int r, inserted;  in __insert()  local
1556 	r = dm_btree_insert_notify(&pmd->info, pmd->root, keys, &value,  in __insert()
1558 	if (r)  in __insert()
1559 		return r;  in __insert()
1571 	int r = -EINVAL;  in dm_thin_insert_block()  local
1575 		r = __insert(td, block, data_block);  in dm_thin_insert_block()
1578 	return r;  in dm_thin_insert_block()
1583 	int r;  in __remove()  local
1587 	r = dm_btree_remove(&pmd->info, pmd->root, keys, &pmd->root);  in __remove()
1588 	if (r)  in __remove()
1589 		return r;  in __remove()
1599 	int r;  in __remove_range()  local
1609 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, keys, &value);  in __remove_range()
1610 	if (r)  in __remove_range()
1611 		return r;  in __remove_range()
1619 	r = dm_btree_remove(&pmd->tl_info, pmd->root, keys, &pmd->root);  in __remove_range()
1620 	if (r)  in __remove_range()
1621 		return r;  in __remove_range()
1628 		r = dm_btree_lookup_next(&pmd->bl_info, mapping_root, &begin, &begin, &value);  in __remove_range()
1629 		if (r == -ENODATA)  in __remove_range()
1632 		if (r)  in __remove_range()
1633 			return r;  in __remove_range()
1638 		r = dm_btree_remove_leaves(&pmd->bl_info, mapping_root, &begin, end, &mapping_root, &count);  in __remove_range()
1639 		if (r)  in __remove_range()
1640 			return r;  in __remove_range()
1658 	int r = -EINVAL;  in dm_thin_remove_block()  local
1662 		r = __remove(td, block);  in dm_thin_remove_block()
1665 	return r;  in dm_thin_remove_block()
1671 	int r = -EINVAL;  in dm_thin_remove_range()  local
1675 		r = __remove_range(td, begin, end);  in dm_thin_remove_range()
1678 	return r;  in dm_thin_remove_range()
1683 	int r;  in dm_pool_block_is_used()  local
1687 	r = dm_sm_get_count(pmd->data_sm, b, &ref_count);  in dm_pool_block_is_used()
1688 	if (!r)  in dm_pool_block_is_used()
1692 	return r;  in dm_pool_block_is_used()
1697 	int r = 0;  in dm_pool_inc_data_range()  local
1701 		r = dm_sm_inc_block(pmd->data_sm, b);  in dm_pool_inc_data_range()
1702 		if (r)  in dm_pool_inc_data_range()
1707 	return r;  in dm_pool_inc_data_range()
1712 	int r = 0;  in dm_pool_dec_data_range()  local
1716 		r = dm_sm_dec_block(pmd->data_sm, b);  in dm_pool_dec_data_range()
1717 		if (r)  in dm_pool_dec_data_range()
1722 	return r;  in dm_pool_dec_data_range()
1727 	int r;  in dm_thin_changed_this_transaction()  local
1730 	r = td->changed;  in dm_thin_changed_this_transaction()
1733 	return r;  in dm_thin_changed_this_transaction()
1738 	bool r = false;  in dm_pool_changed_this_transaction()  local
1744 			r = td->changed;  in dm_pool_changed_this_transaction()
1750 	return r;  in dm_pool_changed_this_transaction()
1755 	bool r;  in dm_thin_aborted_changes()  local
1758 	r = td->aborted_with_changes;  in dm_thin_aborted_changes()
1761 	return r;  in dm_thin_aborted_changes()
1766 	int r = -EINVAL;  in dm_pool_alloc_data_block()  local
1770 		r = dm_sm_new_block(pmd->data_sm, result);  in dm_pool_alloc_data_block()
1773 	return r;  in dm_pool_alloc_data_block()
1778 	int r = -EINVAL;  in dm_pool_commit_metadata()  local
1784 	r = __commit_transaction(pmd);  in dm_pool_commit_metadata()
1785 	if (r <= 0)  in dm_pool_commit_metadata()
1791 	r = __begin_transaction(pmd);  in dm_pool_commit_metadata()
1794 	return r;  in dm_pool_commit_metadata()
1807 	int r = -EINVAL;  in dm_pool_abort_metadata()  local
1815 	r = __create_persistent_data_objects(pmd, false);  in dm_pool_abort_metadata()
1816 	if (r)  in dm_pool_abort_metadata()
1822 	return r;  in dm_pool_abort_metadata()
1827 	int r = -EINVAL;  in dm_pool_get_free_block_count()  local
1831 		r = dm_sm_get_nr_free(pmd->data_sm, result);  in dm_pool_get_free_block_count()
1834 	return r;  in dm_pool_get_free_block_count()
1840 	int r = -EINVAL;  in dm_pool_get_free_metadata_block_count()  local
1844 		r = dm_sm_get_nr_free(pmd->metadata_sm, result);  in dm_pool_get_free_metadata_block_count()
1846 	if (!r) {  in dm_pool_get_free_metadata_block_count()
1854 	return r;  in dm_pool_get_free_metadata_block_count()
1860 	int r = -EINVAL;  in dm_pool_get_metadata_dev_size()  local
1864 		r = dm_sm_get_nr_blocks(pmd->metadata_sm, result);  in dm_pool_get_metadata_dev_size()
1867 	return r;  in dm_pool_get_metadata_dev_size()
1872 	int r = -EINVAL;  in dm_pool_get_data_dev_size()  local
1876 		r = dm_sm_get_nr_blocks(pmd->data_sm, result);  in dm_pool_get_data_dev_size()
1879 	return r;  in dm_pool_get_data_dev_size()
1884 	int r = -EINVAL;  in dm_thin_get_mapped_count()  local
1890 		r = 0;  in dm_thin_get_mapped_count()
1894 	return r;  in dm_thin_get_mapped_count()
1899 	int r;  in __highest_block()  local
1904 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &td->id, &value_le);  in __highest_block()
1905 	if (r)  in __highest_block()
1906 		return r;  in __highest_block()
1916 	int r = -EINVAL;  in dm_thin_get_highest_mapped_block()  local
1921 		r = __highest_block(td, result);  in dm_thin_get_highest_mapped_block()
1924 	return r;  in dm_thin_get_highest_mapped_block()
1929 	int r;  in __resize_space_map()  local
1932 	r = dm_sm_get_nr_blocks(sm, &old_count);  in __resize_space_map()
1933 	if (r)  in __resize_space_map()
1934 		return r;  in __resize_space_map()
1949 	int r = -EINVAL;  in dm_pool_resize_data_dev()  local
1953 		r = __resize_space_map(pmd->data_sm, new_count);  in dm_pool_resize_data_dev()
1956 	return r;  in dm_pool_resize_data_dev()
1961 	int r = -EINVAL;  in dm_pool_resize_metadata_dev()  local
1965 		r = __resize_space_map(pmd->metadata_sm, new_count);  in dm_pool_resize_metadata_dev()
1966 		if (!r)  in dm_pool_resize_metadata_dev()
1971 	return r;  in dm_pool_resize_metadata_dev()
1993 	int r;  in dm_pool_register_metadata_threshold()  local
1996 	r = dm_sm_register_threshold_callback(pmd->metadata_sm, threshold, fn, context);  in dm_pool_register_metadata_threshold()
1999 	return r;  in dm_pool_register_metadata_threshold()
2004 	int r;  in dm_pool_metadata_set_needs_check()  local
2011 	r = superblock_lock(pmd, &sblock);  in dm_pool_metadata_set_needs_check()
2012 	if (r) {  in dm_pool_metadata_set_needs_check()
2023 	return r;  in dm_pool_metadata_set_needs_check()