181 static int add_bop(struct sm_metadata *smm, enum block_op_type type, dm_block_t b)  in add_bop()  argument
183 	int r = brb_push(&smm->uncommitted, type, b);  in add_bop()
193 static int commit_bop(struct sm_metadata *smm, struct block_op *op)  in commit_bop()  argument
200 		r = sm_ll_inc(&smm->ll, op->block, &ev);  in commit_bop()
204 		r = sm_ll_dec(&smm->ll, op->block, &ev);  in commit_bop()
211 static void in(struct sm_metadata *smm)  in in()  argument
213 	smm->recursion_count++;  in in()
216 static int apply_bops(struct sm_metadata *smm)  in apply_bops()  argument
220 	while (!brb_empty(&smm->uncommitted)) {  in apply_bops()
223 		r = brb_peek(&smm->uncommitted, &bop);  in apply_bops()
229 		r = commit_bop(smm, &bop);  in apply_bops()
233 		brb_pop(&smm->uncommitted);  in apply_bops()
239 static int out(struct sm_metadata *smm)  in out()  argument
246 	if (!smm->recursion_count) {  in out()
251 	if (smm->recursion_count == 1)  in out()
252 		r = apply_bops(smm);  in out()
254 	smm->recursion_count--;  in out()
269 static int recursing(struct sm_metadata *smm)  in recursing()  argument
271 	return smm->recursion_count;  in recursing()
276 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_destroy()  local
278 	kfree(smm);  in sm_metadata_destroy()
283 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_blocks()  local
285 	*count = smm->ll.nr_blocks;  in sm_metadata_get_nr_blocks()
292 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_free()  local
294 	*count = smm->old_ll.nr_blocks - smm->old_ll.nr_allocated -  in sm_metadata_get_nr_free()
295 		 smm->allocated_this_transaction;  in sm_metadata_get_nr_free()
305 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_count()  local
312 	for (i = smm->uncommitted.begin;  in sm_metadata_get_count()
313 	     i != smm->uncommitted.end;  in sm_metadata_get_count()
314 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_get_count()
315 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_get_count()
331 	r = sm_ll_lookup(&smm->ll, b, result);  in sm_metadata_get_count()
345 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_count_is_more_than_one()  local
352 	for (i = smm->uncommitted.begin;  in sm_metadata_count_is_more_than_one()
353 	     i != smm->uncommitted.end;  in sm_metadata_count_is_more_than_one()
354 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_count_is_more_than_one()
356 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_count_is_more_than_one()
377 	r = sm_ll_lookup_bitmap(&smm->ll, b, &rc);  in sm_metadata_count_is_more_than_one()
397 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_set_count()  local
399 	if (smm->recursion_count) {  in sm_metadata_set_count()
404 	in(smm);  in sm_metadata_set_count()
405 	r = sm_ll_insert(&smm->ll, b, count, &ev);  in sm_metadata_set_count()
406 	r2 = out(smm);  in sm_metadata_set_count()
415 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_inc_block()  local
417 	if (recursing(smm))  in sm_metadata_inc_block()
418 		r = add_bop(smm, BOP_INC, b);  in sm_metadata_inc_block()
420 		in(smm);  in sm_metadata_inc_block()
421 		r = sm_ll_inc(&smm->ll, b, &ev);  in sm_metadata_inc_block()
422 		r2 = out(smm);  in sm_metadata_inc_block()
432 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_dec_block()  local
434 	if (recursing(smm))  in sm_metadata_dec_block()
435 		r = add_bop(smm, BOP_DEC, b);  in sm_metadata_dec_block()
437 		in(smm);  in sm_metadata_dec_block()
438 		r = sm_ll_dec(&smm->ll, b, &ev);  in sm_metadata_dec_block()
439 		r2 = out(smm);  in sm_metadata_dec_block()
449 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block_()  local
454 	r = sm_ll_find_common_free_block(&smm->old_ll, &smm->ll, smm->begin, smm->ll.nr_blocks, b);  in sm_metadata_new_block_()
458 	smm->begin = *b + 1;  in sm_metadata_new_block_()
460 	if (recursing(smm))  in sm_metadata_new_block_()
461 		r = add_bop(smm, BOP_INC, *b);  in sm_metadata_new_block_()
463 		in(smm);  in sm_metadata_new_block_()
464 		r = sm_ll_inc(&smm->ll, *b, &ev);  in sm_metadata_new_block_()
465 		r2 = out(smm);  in sm_metadata_new_block_()
469 		smm->allocated_this_transaction++;  in sm_metadata_new_block_()
477 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block()  local
491 	check_threshold(&smm->threshold, count);  in sm_metadata_new_block()
499 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_commit()  local
501 	r = sm_ll_commit(&smm->ll);  in sm_metadata_commit()
505 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in sm_metadata_commit()
506 	smm->begin = 0;  in sm_metadata_commit()
507 	smm->allocated_this_transaction = 0;  in sm_metadata_commit()
517 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_register_threshold_callback()  local
519 	set_threshold(&smm->threshold, threshold, fn, context);  in sm_metadata_register_threshold_callback()
533 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_copy_root()  local
536 	root_le.nr_blocks = cpu_to_le64(smm->ll.nr_blocks);  in sm_metadata_copy_root()
537 	root_le.nr_allocated = cpu_to_le64(smm->ll.nr_allocated);  in sm_metadata_copy_root()
538 	root_le.bitmap_root = cpu_to_le64(smm->ll.bitmap_root);  in sm_metadata_copy_root()
539 	root_le.ref_count_root = cpu_to_le64(smm->ll.ref_count_root);  in sm_metadata_copy_root()
587 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_blocks()  local
589 	*count = smm->ll.nr_blocks;  in sm_bootstrap_get_nr_blocks()
596 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_free()  local
598 	*count = smm->ll.nr_blocks - smm->begin;  in sm_bootstrap_get_nr_free()
606 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_count()  local
608 	*result = (b < smm->begin) ? 1 : 0;  in sm_bootstrap_get_count()
631 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_new_block()  local
636 	if (smm->begin == smm->ll.nr_blocks)  in sm_bootstrap_new_block()
639 	*b = smm->begin++;  in sm_bootstrap_new_block()
646 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_inc_block()  local
648 	return add_bop(smm, BOP_INC, b);  in sm_bootstrap_inc_block()
653 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_dec_block()  local
655 	return add_bop(smm, BOP_DEC, b);  in sm_bootstrap_dec_block()
700 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_extend()  local
701 	dm_block_t old_len = smm->ll.nr_blocks;  in sm_metadata_extend()
706 	smm->begin = old_len;  in sm_metadata_extend()
712 	r = sm_ll_extend(&smm->ll, extra_blocks);  in sm_metadata_extend()
721 		for (i = old_len; !r && i < smm->begin; i++)  in sm_metadata_extend()
722 			r = add_bop(smm, BOP_INC, i);  in sm_metadata_extend()
727 		old_len = smm->begin;  in sm_metadata_extend()
729 		r = apply_bops(smm);  in sm_metadata_extend()
735 		r = sm_ll_commit(&smm->ll);  in sm_metadata_extend()
739 	} while (old_len != smm->begin);  in sm_metadata_extend()
753 	struct sm_metadata *smm;  in dm_sm_metadata_init()  local
755 	smm = kmalloc(sizeof(*smm), GFP_KERNEL);  in dm_sm_metadata_init()
756 	if (!smm)  in dm_sm_metadata_init()
759 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_init()
761 	return &smm->sm;  in dm_sm_metadata_init()
771 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_create()  local
773 	smm->begin = superblock + 1;  in dm_sm_metadata_create()
774 	smm->recursion_count = 0;  in dm_sm_metadata_create()
775 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_create()
776 	brb_init(&smm->uncommitted);  in dm_sm_metadata_create()
777 	threshold_init(&smm->threshold);  in dm_sm_metadata_create()
779 	memcpy(&smm->sm, &bootstrap_ops, sizeof(smm->sm));  in dm_sm_metadata_create()
781 	r = sm_ll_new_metadata(&smm->ll, tm);  in dm_sm_metadata_create()
785 		r = sm_ll_extend(&smm->ll, nr_blocks);  in dm_sm_metadata_create()
787 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_create()
795 	for (i = superblock; !r && i < smm->begin; i++)  in dm_sm_metadata_create()
796 		r = add_bop(smm, BOP_INC, i);  in dm_sm_metadata_create()
801 	r = apply_bops(smm);  in dm_sm_metadata_create()
815 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_open()  local
817 	r = sm_ll_open_metadata(&smm->ll, tm, root_le, len);  in dm_sm_metadata_open()
821 	smm->begin = 0;  in dm_sm_metadata_open()
822 	smm->recursion_count = 0;  in dm_sm_metadata_open()
823 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_open()
824 	brb_init(&smm->uncommitted);  in dm_sm_metadata_open()
825 	threshold_init(&smm->threshold);  in dm_sm_metadata_open()
827 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in dm_sm_metadata_open()