514 static inline void init_inc_context(struct inc_context *ic)  in init_inc_context()  argument
516 	ic->bitmap_block = NULL;  in init_inc_context()
517 	ic->bitmap = NULL;  in init_inc_context()
518 	ic->overflow_leaf = NULL;  in init_inc_context()
521 static inline void exit_inc_context(struct ll_disk *ll, struct inc_context *ic)  in exit_inc_context()  argument
523 	if (ic->bitmap_block)  in exit_inc_context()
524 		dm_tm_unlock(ll->tm, ic->bitmap_block);  in exit_inc_context()
525 	if (ic->overflow_leaf)  in exit_inc_context()
526 		dm_tm_unlock(ll->tm, ic->overflow_leaf);  in exit_inc_context()
529 static inline void reset_inc_context(struct ll_disk *ll, struct inc_context *ic)  in reset_inc_context()  argument
531 	exit_inc_context(ll, ic);  in reset_inc_context()
532 	init_inc_context(ic);  in reset_inc_context()
545 static int __sm_ll_inc_overflow(struct ll_disk *ll, dm_block_t b, struct inc_context *ic)  in __sm_ll_inc_overflow()  argument
557 	reset_inc_context(ll, ic);  in __sm_ll_inc_overflow()
560 				     b, &index, &ll->ref_count_root, &ic->overflow_leaf);  in __sm_ll_inc_overflow()
564 	n = dm_block_data(ic->overflow_leaf);  in __sm_ll_inc_overflow()
578 static int sm_ll_inc_overflow(struct ll_disk *ll, dm_block_t b, struct inc_context *ic)  in sm_ll_inc_overflow()  argument
588 	if (ic->overflow_leaf) {  in sm_ll_inc_overflow()
589 		n = dm_block_data(ic->overflow_leaf);  in sm_ll_inc_overflow()
600 	return __sm_ll_inc_overflow(ll, b, ic);  in sm_ll_inc_overflow()
603 static inline int shadow_bitmap(struct ll_disk *ll, struct inc_context *ic)  in shadow_bitmap()  argument
606 	r = dm_tm_shadow_block(ll->tm, le64_to_cpu(ic->ie_disk.blocknr),  in shadow_bitmap()
607 			       &dm_sm_bitmap_validator, &ic->bitmap_block, &inc);  in shadow_bitmap()
612 	ic->ie_disk.blocknr = cpu_to_le64(dm_block_location(ic->bitmap_block));  in shadow_bitmap()
613 	ic->bitmap = dm_bitmap_data(ic->bitmap_block);  in shadow_bitmap()
622 static inline int ensure_bitmap(struct ll_disk *ll, struct inc_context *ic)  in ensure_bitmap()  argument
624 	if (!ic->bitmap_block) {  in ensure_bitmap()
625 		int r = dm_bm_write_lock(dm_tm_get_bm(ll->tm), le64_to_cpu(ic->ie_disk.blocknr),  in ensure_bitmap()
626 					 &dm_sm_bitmap_validator, &ic->bitmap_block);  in ensure_bitmap()
631 		ic->bitmap = dm_bitmap_data(ic->bitmap_block);  in ensure_bitmap()
643 				   struct inc_context *ic)  in sm_ll_inc_bitmap()  argument
655 		r = ensure_bitmap(ll, ic);  in sm_ll_inc_bitmap()
659 		old = sm_lookup_bitmap(ic->bitmap, bit);  in sm_ll_inc_bitmap()
663 			sm_set_bitmap(ic->bitmap, bit, 1);  in sm_ll_inc_bitmap()
666 			le32_add_cpu(&ic->ie_disk.nr_free, -1);  in sm_ll_inc_bitmap()
667 			if (le32_to_cpu(ic->ie_disk.none_free_before) == bit)  in sm_ll_inc_bitmap()
668 				ic->ie_disk.none_free_before = cpu_to_le32(bit + 1);  in sm_ll_inc_bitmap()
673 			sm_set_bitmap(ic->bitmap, bit, 2);  in sm_ll_inc_bitmap()
678 			sm_set_bitmap(ic->bitmap, bit, 3);  in sm_ll_inc_bitmap()
679 			reset_inc_context(ll, ic);  in sm_ll_inc_bitmap()
695 			r = sm_ll_inc_overflow(ll, b, ic);  in sm_ll_inc_bitmap()
713 	struct inc_context ic;  in __sm_ll_inc()  local
717 	init_inc_context(&ic);  in __sm_ll_inc()
720 	r = ll->load_ie(ll, index, &ic.ie_disk);  in __sm_ll_inc()
724 	r = shadow_bitmap(ll, &ic);  in __sm_ll_inc()
729 	r = sm_ll_inc_bitmap(ll, b, bit, bit_end, nr_allocations, new_b, &ic);  in __sm_ll_inc()
731 	exit_inc_context(ll, &ic);  in __sm_ll_inc()
736 	return ll->save_ie(ll, index, &ic.ie_disk);  in __sm_ll_inc()
755 				struct inc_context *ic)  in __sm_ll_del_overflow()  argument
757 	reset_inc_context(ll, ic);  in __sm_ll_del_overflow()
763 				struct inc_context *ic, uint32_t *old_rc)  in __sm_ll_dec_overflow()  argument
771 	reset_inc_context(ll, ic);  in __sm_ll_dec_overflow()
773 				     b, &index, &ll->ref_count_root, &ic->overflow_leaf);  in __sm_ll_dec_overflow()
777 	n = dm_block_data(ic->overflow_leaf);  in __sm_ll_dec_overflow()
789 		return __sm_ll_del_overflow(ll, b, ic);  in __sm_ll_dec_overflow()
798 			      struct inc_context *ic, uint32_t *old_rc)  in sm_ll_dec_overflow()  argument
803 	if (ic->overflow_leaf) {  in sm_ll_dec_overflow()
809 		n = dm_block_data(ic->overflow_leaf);  in sm_ll_dec_overflow()
821 				return __sm_ll_del_overflow(ll, b, ic);  in sm_ll_dec_overflow()
827 	return __sm_ll_dec_overflow(ll, b, ic, old_rc);  in sm_ll_dec_overflow()
835 				   struct inc_context *ic,  in sm_ll_dec_bitmap()  argument
847 		r = ensure_bitmap(ll, ic);  in sm_ll_dec_bitmap()
851 		old = sm_lookup_bitmap(ic->bitmap, bit);  in sm_ll_dec_bitmap()
859 			sm_set_bitmap(ic->bitmap, bit, 0);  in sm_ll_dec_bitmap()
862 			le32_add_cpu(&ic->ie_disk.nr_free, 1);  in sm_ll_dec_bitmap()
863 			ic->ie_disk.none_free_before =  in sm_ll_dec_bitmap()
864 				cpu_to_le32(min(le32_to_cpu(ic->ie_disk.none_free_before), bit));  in sm_ll_dec_bitmap()
869 			sm_set_bitmap(ic->bitmap, bit, 1);  in sm_ll_dec_bitmap()
873 			r = sm_ll_dec_overflow(ll, b, ic, &old);  in sm_ll_dec_bitmap()
878 				r = ensure_bitmap(ll, ic);  in sm_ll_dec_bitmap()
882 				sm_set_bitmap(ic->bitmap, bit, 2);  in sm_ll_dec_bitmap()
897 	struct inc_context ic;  in __sm_ll_dec()  local
900 	init_inc_context(&ic);  in __sm_ll_dec()
903 	r = ll->load_ie(ll, index, &ic.ie_disk);  in __sm_ll_dec()
907 	r = shadow_bitmap(ll, &ic);  in __sm_ll_dec()
912 	r = sm_ll_dec_bitmap(ll, b, bit, bit_end, &ic, nr_allocations, new_b);  in __sm_ll_dec()
913 	exit_inc_context(ll, &ic);  in __sm_ll_dec()
918 	return ll->save_ie(ll, index, &ic.ie_disk);  in __sm_ll_dec()