181 static int add_bop(struct sm_metadata *smm, enum block_op_type type, dm_block_t b, dm_block_t e)  in add_bop()  argument
183 	int r = brb_push(&smm->uncommitted, type, b, e);  in add_bop()
192 static int commit_bop(struct sm_metadata *smm, struct block_op *op)  in commit_bop()  argument
199 		r = sm_ll_inc(&smm->ll, op->b, op->e, &nr_allocations);  in commit_bop()
203 		r = sm_ll_dec(&smm->ll, op->b, op->e, &nr_allocations);  in commit_bop()
210 static void in(struct sm_metadata *smm)  in in()  argument
212 	smm->recursion_count++;  in in()
215 static int apply_bops(struct sm_metadata *smm)  in apply_bops()  argument
219 	while (!brb_empty(&smm->uncommitted)) {  in apply_bops()
222 		r = brb_peek(&smm->uncommitted, &bop);  in apply_bops()
228 		r = commit_bop(smm, &bop);  in apply_bops()
232 		brb_pop(&smm->uncommitted);  in apply_bops()
238 static int out(struct sm_metadata *smm)  in out()  argument
245 	if (!smm->recursion_count) {  in out()
250 	if (smm->recursion_count == 1)  in out()
251 		r = apply_bops(smm);  in out()
253 	smm->recursion_count--;  in out()
268 static int recursing(struct sm_metadata *smm)  in recursing()  argument
270 	return smm->recursion_count;  in recursing()
275 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_destroy()  local
277 	kfree(smm);  in sm_metadata_destroy()
282 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_blocks()  local
284 	*count = smm->ll.nr_blocks;  in sm_metadata_get_nr_blocks()
291 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_free()  local
293 	*count = smm->old_ll.nr_blocks - smm->old_ll.nr_allocated -  in sm_metadata_get_nr_free()
294 		 smm->allocated_this_transaction;  in sm_metadata_get_nr_free()
304 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_count()  local
311 	for (i = smm->uncommitted.begin;  in sm_metadata_get_count()
312 	     i != smm->uncommitted.end;  in sm_metadata_get_count()
313 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_get_count()
314 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_get_count()
330 	r = sm_ll_lookup(&smm->ll, b, result);  in sm_metadata_get_count()
344 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_count_is_more_than_one()  local
351 	for (i = smm->uncommitted.begin;  in sm_metadata_count_is_more_than_one()
352 	     i != smm->uncommitted.end;  in sm_metadata_count_is_more_than_one()
353 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_count_is_more_than_one()
355 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_count_is_more_than_one()
376 	r = sm_ll_lookup_bitmap(&smm->ll, b, &rc);  in sm_metadata_count_is_more_than_one()
396 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_set_count()  local
398 	if (smm->recursion_count) {  in sm_metadata_set_count()
403 	in(smm);  in sm_metadata_set_count()
404 	r = sm_ll_insert(&smm->ll, b, count, &nr_allocations);  in sm_metadata_set_count()
405 	r2 = out(smm);  in sm_metadata_set_count()
414 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_inc_blocks()  local
416 	if (recursing(smm)) {  in sm_metadata_inc_blocks()
417 		r = add_bop(smm, BOP_INC, b, e);  in sm_metadata_inc_blocks()
421 		in(smm);  in sm_metadata_inc_blocks()
422 		r = sm_ll_inc(&smm->ll, b, e, &nr_allocations);  in sm_metadata_inc_blocks()
423 		r2 = out(smm);  in sm_metadata_inc_blocks()
433 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_dec_blocks()  local
435 	if (recursing(smm))  in sm_metadata_dec_blocks()
436 		r = add_bop(smm, BOP_DEC, b, e);  in sm_metadata_dec_blocks()
438 		in(smm);  in sm_metadata_dec_blocks()
439 		r = sm_ll_dec(&smm->ll, b, e, &nr_allocations);  in sm_metadata_dec_blocks()
440 		r2 = out(smm);  in sm_metadata_dec_blocks()
450 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block_()  local
455 	r = sm_ll_find_common_free_block(&smm->old_ll, &smm->ll, smm->begin, smm->ll.nr_blocks, b);  in sm_metadata_new_block_()
461 		r = sm_ll_find_common_free_block(&smm->old_ll, &smm->ll, 0, smm->begin, b);  in sm_metadata_new_block_()
467 	smm->begin = *b + 1;  in sm_metadata_new_block_()
469 	if (recursing(smm))  in sm_metadata_new_block_()
470 		r = add_bop(smm, BOP_INC, *b, *b + 1);  in sm_metadata_new_block_()
472 		in(smm);  in sm_metadata_new_block_()
473 		r = sm_ll_inc(&smm->ll, *b, *b + 1, &nr_allocations);  in sm_metadata_new_block_()
474 		r2 = out(smm);  in sm_metadata_new_block_()
478 		smm->allocated_this_transaction++;  in sm_metadata_new_block_()
486 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block()  local
500 	check_threshold(&smm->threshold, count);  in sm_metadata_new_block()
508 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_commit()  local
510 	r = sm_ll_commit(&smm->ll);  in sm_metadata_commit()
514 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in sm_metadata_commit()
515 	smm->allocated_this_transaction = 0;  in sm_metadata_commit()
525 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_register_threshold_callback()  local
527 	set_threshold(&smm->threshold, threshold, fn, context);  in sm_metadata_register_threshold_callback()
541 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_copy_root()  local
544 	root_le.nr_blocks = cpu_to_le64(smm->ll.nr_blocks);  in sm_metadata_copy_root()
545 	root_le.nr_allocated = cpu_to_le64(smm->ll.nr_allocated);  in sm_metadata_copy_root()
546 	root_le.bitmap_root = cpu_to_le64(smm->ll.bitmap_root);  in sm_metadata_copy_root()
547 	root_le.ref_count_root = cpu_to_le64(smm->ll.ref_count_root);  in sm_metadata_copy_root()
595 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_blocks()  local
597 	*count = smm->ll.nr_blocks;  in sm_bootstrap_get_nr_blocks()
604 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_free()  local
606 	*count = smm->ll.nr_blocks - smm->begin;  in sm_bootstrap_get_nr_free()
614 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_count()  local
616 	*result = (b < smm->begin) ? 1 : 0;  in sm_bootstrap_get_count()
639 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_new_block()  local
644 	if (smm->begin == smm->ll.nr_blocks)  in sm_bootstrap_new_block()
647 	*b = smm->begin++;  in sm_bootstrap_new_block()
655 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_inc_blocks()  local
657 	r = add_bop(smm, BOP_INC, b, e);  in sm_bootstrap_inc_blocks()
667 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_dec_blocks()  local
669 	r = add_bop(smm, BOP_DEC, b, e);  in sm_bootstrap_dec_blocks()
718 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_extend()  local
719 	dm_block_t old_len = smm->ll.nr_blocks;  in sm_metadata_extend()
724 	smm->begin = old_len;  in sm_metadata_extend()
730 	r = sm_ll_extend(&smm->ll, extra_blocks);  in sm_metadata_extend()
739 		r = add_bop(smm, BOP_INC, old_len, smm->begin);  in sm_metadata_extend()
743 		old_len = smm->begin;  in sm_metadata_extend()
745 		r = apply_bops(smm);  in sm_metadata_extend()
751 		r = sm_ll_commit(&smm->ll);  in sm_metadata_extend()
755 	} while (old_len != smm->begin);  in sm_metadata_extend()
769 	struct sm_metadata *smm;  in dm_sm_metadata_init()  local
771 	smm = kmalloc(sizeof(*smm), GFP_KERNEL);  in dm_sm_metadata_init()
772 	if (!smm)  in dm_sm_metadata_init()
775 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_init()
777 	return &smm->sm;  in dm_sm_metadata_init()
786 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_create()  local
788 	smm->begin = superblock + 1;  in dm_sm_metadata_create()
789 	smm->recursion_count = 0;  in dm_sm_metadata_create()
790 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_create()
791 	brb_init(&smm->uncommitted);  in dm_sm_metadata_create()
792 	threshold_init(&smm->threshold);  in dm_sm_metadata_create()
794 	memcpy(&smm->sm, &bootstrap_ops, sizeof(smm->sm));  in dm_sm_metadata_create()
796 	r = sm_ll_new_metadata(&smm->ll, tm);  in dm_sm_metadata_create()
800 		r = sm_ll_extend(&smm->ll, nr_blocks);  in dm_sm_metadata_create()
802 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_create()
810 	r = add_bop(smm, BOP_INC, superblock, smm->begin);  in dm_sm_metadata_create()
814 	r = apply_bops(smm);  in dm_sm_metadata_create()
828 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_open()  local
830 	r = sm_ll_open_metadata(&smm->ll, tm, root_le, len);  in dm_sm_metadata_open()
834 	smm->begin = 0;  in dm_sm_metadata_open()
835 	smm->recursion_count = 0;  in dm_sm_metadata_open()
836 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_open()
837 	brb_init(&smm->uncommitted);  in dm_sm_metadata_open()
838 	threshold_init(&smm->threshold);  in dm_sm_metadata_open()
840 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in dm_sm_metadata_open()