152 static inline void tag_set(struct radix_tree_node *node, unsigned int tag,  in tag_set()  argument
155 	__set_bit(offset, node->tags[tag]);  in tag_set()
158 static inline void tag_clear(struct radix_tree_node *node, unsigned int tag,  in tag_clear()  argument
161 	__clear_bit(offset, node->tags[tag]);  in tag_clear()
164 static inline int tag_get(const struct radix_tree_node *node, unsigned int tag,  in tag_get()  argument
167 	return test_bit(offset, node->tags[tag]);  in tag_get()
170 static inline void root_tag_set(struct radix_tree_root *root, unsigned tag)  in root_tag_set()  argument
172 	root->gfp_mask |= (__force gfp_t)(1 << (tag + ROOT_TAG_SHIFT));  in root_tag_set()
175 static inline void root_tag_clear(struct radix_tree_root *root, unsigned tag)  in root_tag_clear()  argument
177 	root->gfp_mask &= (__force gfp_t)~(1 << (tag + ROOT_TAG_SHIFT));  in root_tag_clear()
185 static inline int root_tag_get(const struct radix_tree_root *root, unsigned tag)  in root_tag_get()  argument
187 	return (__force int)root->gfp_mask & (1 << (tag + ROOT_TAG_SHIFT));  in root_tag_get()
205 							unsigned int tag)  in any_tag_set()  argument
209 		if (node->tags[tag][idx])  in any_tag_set()
215 static inline void all_tag_set(struct radix_tree_node *node, unsigned int tag)  in all_tag_set()  argument
217 	bitmap_fill(node->tags[tag], RADIX_TREE_MAP_SIZE);  in all_tag_set()
232 radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag,  in radix_tree_find_next_bit()  argument
235 	const unsigned long *addr = node->tags[tag];  in radix_tree_find_next_bit()
625 	int tag;  in radix_tree_extend()  local
650 			for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {  in radix_tree_extend()
651 				if (root_tag_get(root, tag))  in radix_tree_extend()
652 					tag_set(node, tag, 0);  in radix_tree_extend()
902 	unsigned i, n, tag, offset, tags = 0;  in insert_entries()  local
925 				for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in insert_entries()
926 					if (tag_get(node, tag, offset + i))  in insert_entries()
927 						tags |= 1 << tag;  in insert_entries()
937 			for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in insert_entries()
938 				if (tags & (1 << tag))  in insert_entries()
939 					tag_clear(node, tag, offset + i);  in insert_entries()
942 			for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in insert_entries()
943 				if (tags & (1 << tag))  in insert_entries()
944 					tag_set(node, tag, offset);  in insert_entries()
1140 				unsigned int tag, unsigned int offset)  in node_tag_get()  argument
1143 		return tag_get(node, tag, offset);  in node_tag_get()
1144 	return root_tag_get(root, tag);  in node_tag_get()
1303 	unsigned n, tag, tags = 0;  in radix_tree_split()  local
1313 	for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in radix_tree_split()
1314 		if (tag_get(parent, tag, offset))  in radix_tree_split()
1315 			tags |= 1 << tag;  in radix_tree_split()
1321 		for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in radix_tree_split()
1322 			if (tags & (1 << tag))  in radix_tree_split()
1323 				tag_set(parent, tag, end);  in radix_tree_split()
1352 				for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in radix_tree_split()
1353 					if (tags & (1 << tag))  in radix_tree_split()
1354 						tag_set(node, tag, offset);  in radix_tree_split()
1366 		for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in radix_tree_split()
1367 			if (tags & (1 << tag))  in radix_tree_split()
1368 				tag_set(node, tag, offset);  in radix_tree_split()
1395 				unsigned int tag, unsigned int offset)  in node_tag_set()  argument
1398 		if (tag_get(node, tag, offset))  in node_tag_set()
1400 		tag_set(node, tag, offset);  in node_tag_set()
1405 	if (!root_tag_get(root, tag))  in node_tag_set()
1406 		root_tag_set(root, tag);  in node_tag_set()
1423 			unsigned long index, unsigned int tag)  in radix_tree_tag_set()  argument
1438 		if (!tag_get(parent, tag, offset))  in radix_tree_tag_set()
1439 			tag_set(parent, tag, offset);  in radix_tree_tag_set()
1443 	if (!root_tag_get(root, tag))  in radix_tree_tag_set()
1444 		root_tag_set(root, tag);  in radix_tree_tag_set()
1457 			const struct radix_tree_iter *iter, unsigned int tag)  in radix_tree_iter_tag_set()  argument
1459 	node_tag_set(root, iter->node, tag, iter_offset(iter));  in radix_tree_iter_tag_set()
1464 				unsigned int tag, unsigned int offset)  in node_tag_clear()  argument
1467 		if (!tag_get(node, tag, offset))  in node_tag_clear()
1469 		tag_clear(node, tag, offset);  in node_tag_clear()
1470 		if (any_tag_set(node, tag))  in node_tag_clear()
1478 	if (root_tag_get(root, tag))  in node_tag_clear()
1479 		root_tag_clear(root, tag);  in node_tag_clear()
1497 			unsigned long index, unsigned int tag)  in radix_tree_tag_clear()  argument
1515 		node_tag_clear(root, parent, tag, offset);  in radix_tree_tag_clear()
1528 			const struct radix_tree_iter *iter, unsigned int tag)  in radix_tree_iter_tag_clear()  argument
1530 	node_tag_clear(root, iter->node, tag, iter_offset(iter));  in radix_tree_iter_tag_clear()
1549 			unsigned long index, unsigned int tag)  in radix_tree_tag_get()  argument
1554 	if (!root_tag_get(root, tag))  in radix_tree_tag_get()
1567 		if (!tag_get(parent, tag, offset))  in radix_tree_tag_get()
1588 				unsigned tag)  in set_iter_tags()  argument
1598 	iter->tags = node->tags[tag][tag_long] >> tag_bit;  in set_iter_tags()
1604 			iter->tags |= node->tags[tag][tag_long + 1] <<  in set_iter_tags()
1631 	unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;  in __radix_tree_next_slot()  local
1647 			offset = radix_tree_find_next_bit(node, tag, 0);  in __radix_tree_next_slot()
1652 			set_iter_tags(iter, node, offset, tag);  in __radix_tree_next_slot()
1714 	unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;  in radix_tree_next_chunk()  local
1718 	if ((flags & RADIX_TREE_ITER_TAGGED) && !root_tag_get(root, tag))  in radix_tree_next_chunk()
1756 				!tag_get(node, tag, offset) : !child) {  in radix_tree_next_chunk()
1762 				offset = radix_tree_find_next_bit(node, tag,  in radix_tree_next_chunk()
1796 		set_iter_tags(iter, node, offset, tag);  in radix_tree_next_chunk()
1907 		unsigned int tag)  in radix_tree_gang_lookup_tag()  argument
1916 	radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {  in radix_tree_gang_lookup_tag()
1948 		unsigned int max_items, unsigned int tag)  in radix_tree_gang_lookup_tag_slot()  argument
1957 	radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {  in radix_tree_gang_lookup_tag_slot()
1990 	int tag;  in __radix_tree_delete()  local
1995 		for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in __radix_tree_delete()
1996 			node_tag_clear(root, node, tag, offset);  in __radix_tree_delete()
2076 		unsigned int tag, offset = get_slot_offset(node, slot);  in radix_tree_clear_tags()  local
2077 		for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)  in radix_tree_clear_tags()
2078 			node_tag_clear(root, node, tag, offset);  in radix_tree_clear_tags()
2089 int radix_tree_tagged(const struct radix_tree_root *root, unsigned int tag)  in radix_tree_tagged()  argument
2091 	return root_tag_get(root, tag);  in radix_tree_tagged()