23 	struct bkey *k, *next;  in bch_dump_bset()  local
25 	for (k = i->start; k < bset_bkey_last(i); k = next) {  in bch_dump_bset()
26 		next = bkey_next(k);  in bch_dump_bset()
29 		       (unsigned int) ((u64 *) k - i->d), i->keys);  in bch_dump_bset()
32 			b->ops->key_dump(b, k);  in bch_dump_bset()
34 			pr_err("%llu:%llu\n", KEY_INODE(k), KEY_OFFSET(k));  in bch_dump_bset()
37 		    bkey_cmp(k, b->ops->is_extents ?  in bch_dump_bset()
58 	struct bkey *k;  in __bch_count_data()  local
61 		for_each_key(b, k, &iter)  in __bch_count_data()
62 			ret += KEY_SIZE(k);  in __bch_count_data()
69 	struct bkey *k, *p = NULL;  in __bch_check_keys()  local
73 	for_each_key(b, k, &iter) {  in __bch_check_keys()
76 			if (p && bkey_cmp(&START_KEY(p), &START_KEY(k)) > 0)  in __bch_check_keys()
79 			if (bch_ptr_invalid(b, k))  in __bch_check_keys()
83 			if (p && bkey_cmp(p, &START_KEY(k)) > 0)  in __bch_check_keys()
86 			if (bch_ptr_bad(b, k))  in __bch_check_keys()
90 			if (p && !bkey_cmp(p, k))  in __bch_check_keys()
93 		p = k;  in __bch_check_keys()
113 	struct bkey *k = iter->data->k, *next = bkey_next(k);  in bch_btree_iter_next_check()  local
116 	    bkey_cmp(k, iter->b->ops->is_extents ?  in bch_btree_iter_next_check()
160 	struct bkey *k = l->keys;  in bch_keylist_pop()  local
162 	if (k == l->top)  in bch_keylist_pop()
165 	while (bkey_next(k) != l->top)  in bch_keylist_pop()
166 		k = bkey_next(k);  in bch_keylist_pop()
168 	return l->top = k;  in bch_keylist_pop()
195 bool __bch_cut_front(const struct bkey *where, struct bkey *k)  in __bch_cut_front()  argument
199 	if (bkey_cmp(where, &START_KEY(k)) <= 0)  in __bch_cut_front()
202 	if (bkey_cmp(where, k) < 0)  in __bch_cut_front()
203 		len = KEY_OFFSET(k) - KEY_OFFSET(where);  in __bch_cut_front()
205 		bkey_copy_key(k, where);  in __bch_cut_front()
207 	for (i = 0; i < KEY_PTRS(k); i++)  in __bch_cut_front()
208 		SET_PTR_OFFSET(k, i, PTR_OFFSET(k, i) + KEY_SIZE(k) - len);  in __bch_cut_front()
210 	BUG_ON(len > KEY_SIZE(k));  in __bch_cut_front()
211 	SET_KEY_SIZE(k, len);  in __bch_cut_front()
215 bool __bch_cut_back(const struct bkey *where, struct bkey *k)  in __bch_cut_back()  argument
219 	if (bkey_cmp(where, k) >= 0)  in __bch_cut_back()
222 	BUG_ON(KEY_INODE(where) != KEY_INODE(k));  in __bch_cut_back()
224 	if (bkey_cmp(where, &START_KEY(k)) > 0)  in __bch_cut_back()
225 		len = KEY_OFFSET(where) - KEY_START(k);  in __bch_cut_back()
227 	bkey_copy_key(k, where);  in __bch_cut_back()
229 	BUG_ON(len > KEY_SIZE(k));  in __bch_cut_back()
230 	SET_KEY_SIZE(k, len);  in __bch_cut_back()
536 static unsigned int bkey_to_cacheline(struct bset_tree *t, struct bkey *k)  in bkey_to_cacheline()  argument
538 	return ((void *) k - (void *) t->data) / BSET_CACHELINE;  in bkey_to_cacheline()
543 					 struct bkey *k)  in bkey_to_cacheline_offset()  argument
545 	return (u64 *) k - (u64 *) cacheline_to_bkey(t, cacheline, 0);  in bkey_to_cacheline_offset()
588 static inline unsigned int bfloat_mantissa(const struct bkey *k,  in bfloat_mantissa()  argument
591 	const uint64_t *p = &k->low - (f->exponent >> 6);  in bfloat_mantissa()
698 	struct bkey *prev = NULL, *k = t->data->start;  in bch_bset_build_written_tree()  local
720 		while (bkey_to_cacheline(t, k) < cacheline)  in bch_bset_build_written_tree()
721 			prev = k, k = bkey_next(k);  in bch_bset_build_written_tree()
724 		t->tree[j].m = bkey_to_cacheline_offset(t, cacheline++, k);  in bch_bset_build_written_tree()
727 	while (bkey_next(k) != bset_bkey_last(t->data))  in bch_bset_build_written_tree()
728 		k = bkey_next(k);  in bch_bset_build_written_tree()
730 	t->end = *k;  in bch_bset_build_written_tree()
742 void bch_bset_fix_invalidated_key(struct btree_keys *b, struct bkey *k)  in bch_bset_fix_invalidated_key()  argument
748 		if (k < bset_bkey_last(t->data))  in bch_bset_fix_invalidated_key()
756 	inorder = bkey_to_cacheline(t, k);  in bch_bset_fix_invalidated_key()
758 	if (k == t->data->start)  in bch_bset_fix_invalidated_key()
761 	if (bkey_next(k) == bset_bkey_last(t->data)) {  in bch_bset_fix_invalidated_key()
762 		t->end = *k;  in bch_bset_fix_invalidated_key()
770 	    k == tree_to_bkey(t, j))  in bch_bset_fix_invalidated_key()
780 	    k == tree_to_prev_bkey(t, j))  in bch_bset_fix_invalidated_key()
790 				      struct bkey *k)  in bch_bset_fix_lookup_table()  argument
792 	unsigned int shift = bkey_u64s(k);  in bch_bset_fix_lookup_table()
793 	unsigned int j = bkey_to_cacheline(t, k);  in bch_bset_fix_lookup_table()
805 	       table_to_bkey(t, j) <= k)  in bch_bset_fix_lookup_table()
816 			k = table_to_bkey(t, j - 1);  in bch_bset_fix_lookup_table()
818 			while (k < cacheline_to_bkey(t, j, 0))  in bch_bset_fix_lookup_table()
819 				k = bkey_next(k);  in bch_bset_fix_lookup_table()
821 			t->prev[j] = bkey_to_cacheline_offset(t, j, k);  in bch_bset_fix_lookup_table()
830 	for (k = table_to_bkey(t, t->size - 1);  in bch_bset_fix_lookup_table()
831 	     k != bset_bkey_last(t->data);  in bch_bset_fix_lookup_table()
832 	     k = bkey_next(k))  in bch_bset_fix_lookup_table()
833 		if (t->size == bkey_to_cacheline(t, k)) {  in bch_bset_fix_lookup_table()
835 				bkey_to_cacheline_offset(t, t->size, k);  in bch_bset_fix_lookup_table()
883 unsigned int bch_btree_insert_key(struct btree_keys *b, struct bkey *k,  in bch_btree_insert_key()  argument
891 	BUG_ON(b->ops->is_extents && !KEY_SIZE(k));  in bch_btree_insert_key()
894 				? PRECEDING_KEY(&START_KEY(k))  in bch_btree_insert_key()
895 				: PRECEDING_KEY(k));  in bch_btree_insert_key()
897 	if (b->ops->insert_fixup(b, k, &iter, replace_key))  in bch_btree_insert_key()
903 	       bkey_cmp(k, b->ops->is_extents ? &START_KEY(m) : m) > 0)  in bch_btree_insert_key()
909 	    bch_bkey_try_merge(b, prev, k))  in bch_btree_insert_key()
914 	    KEY_PTRS(m) == KEY_PTRS(k) && !KEY_SIZE(m))  in bch_btree_insert_key()
919 	    bch_bkey_try_merge(b, k, m))  in bch_btree_insert_key()
922 	bch_bset_insert(b, m, k);  in bch_btree_insert_key()
923 copy:	bkey_copy(m, k);  in bch_btree_insert_key()
1103 	return bkey_cmp(l.k, r.k) > 0;  in btree_iter_cmp()
1111 void bch_btree_iter_push(struct btree_iter *iter, struct bkey *k,  in bch_btree_iter_push()  argument
1114 	if (k != end)  in bch_btree_iter_push()
1116 				 ((struct btree_iter_set) { k, end }),  in bch_btree_iter_push()
1159 		ret = iter->data->k;  in __bch_btree_iter_next()
1160 		iter->data->k = bkey_next(iter->data->k);  in __bch_btree_iter_next()
1162 		if (iter->data->k > iter->data->end) {  in __bch_btree_iter_next()
1164 			iter->data->k = iter->data->end;  in __bch_btree_iter_next()
1167 		if (iter->data->k == iter->data->end)  in __bch_btree_iter_next()
1219 	struct bkey *k, *last = NULL;  in btree_mergesort()  local
1220 	BKEY_PADDED(k) tmp;  in btree_mergesort()
1231 			k = b->ops->sort_fixup(iter, &tmp.k);  in btree_mergesort()
1233 			k = NULL;  in btree_mergesort()
1235 		if (!k)  in btree_mergesort()
1236 			k = __bch_btree_iter_next(iter, b->ops->sort_cmp);  in btree_mergesort()
1238 		if (bad(b, k))  in btree_mergesort()
1243 			bkey_copy(last, k);  in btree_mergesort()
1244 		} else if (!bch_bkey_try_merge(b, last, k)) {  in btree_mergesort()
1246 			bkey_copy(last, k);  in btree_mergesort()