42 	struct cache *cache;  member
116 struct cache {  struct
122 	struct cache *next_local;      /* next cache of >= level */  argument
137 static const char *cache_type_string(const struct cache *cache)  in cache_type_string()  argument
139 	return cache_type_info[cache->type].name;  in cache_type_string()
142 static void cache_init(struct cache *cache, int type, int level,  in cache_init()  argument
145 	cache->type = type;  in cache_init()
146 	cache->level = level;  in cache_init()
147 	cache->ofnode = of_node_get(ofnode);  in cache_init()
148 	INIT_LIST_HEAD(&cache->list);  in cache_init()
149 	list_add(&cache->list, &cache_list);  in cache_init()
152 static struct cache *new_cache(int type, int level, struct device_node *ofnode)  in new_cache()
154 	struct cache *cache;  in new_cache()  local
156 	cache = kzalloc(sizeof(*cache), GFP_KERNEL);  in new_cache()
157 	if (cache)  in new_cache()
158 		cache_init(cache, type, level, ofnode);  in new_cache()
160 	return cache;  in new_cache()
163 static void release_cache_debugcheck(struct cache *cache)  in release_cache_debugcheck()  argument
165 	struct cache *iter;  in release_cache_debugcheck()
168 		WARN_ONCE(iter->next_local == cache,  in release_cache_debugcheck()
172 			  cache->ofnode,  in release_cache_debugcheck()
173 			  cache_type_string(cache));  in release_cache_debugcheck()
176 static void release_cache(struct cache *cache)  in release_cache()  argument
178 	if (!cache)  in release_cache()
181 	pr_debug("freeing L%d %s cache for %pOF\n", cache->level,  in release_cache()
182 		 cache_type_string(cache), cache->ofnode);  in release_cache()
184 	release_cache_debugcheck(cache);  in release_cache()
185 	list_del(&cache->list);  in release_cache()
186 	of_node_put(cache->ofnode);  in release_cache()
187 	kfree(cache);  in release_cache()
190 static void cache_cpu_set(struct cache *cache, int cpu)  in cache_cpu_set()  argument
192 	struct cache *next = cache;  in cache_cpu_set()
204 static int cache_size(const struct cache *cache, unsigned int *ret)  in cache_size()  argument
209 	propname = cache_type_info[cache->type].size_prop;  in cache_size()
211 	cache_size = of_get_property(cache->ofnode, propname, NULL);  in cache_size()
219 static int cache_size_kb(const struct cache *cache, unsigned int *ret)  in cache_size_kb()  argument
223 	if (cache_size(cache, &size))  in cache_size_kb()
231 static int cache_get_line_size(const struct cache *cache, unsigned int *ret)  in cache_get_line_size()  argument
236 	lim = ARRAY_SIZE(cache_type_info[cache->type].line_size_props);  in cache_get_line_size()
241 		propname = cache_type_info[cache->type].line_size_props[i];  in cache_get_line_size()
242 		line_size = of_get_property(cache->ofnode, propname, NULL);  in cache_get_line_size()
254 static int cache_nr_sets(const struct cache *cache, unsigned int *ret)  in cache_nr_sets()  argument
259 	propname = cache_type_info[cache->type].nr_sets_prop;  in cache_nr_sets()
261 	nr_sets = of_get_property(cache->ofnode, propname, NULL);  in cache_nr_sets()
269 static int cache_associativity(const struct cache *cache, unsigned int *ret)  in cache_associativity()  argument
275 	if (cache_nr_sets(cache, &nr_sets))  in cache_associativity()
286 	if (cache_get_line_size(cache, &line_size))  in cache_associativity()
288 	if (cache_size(cache, &size))  in cache_associativity()
301 static struct cache *cache_find_first_sibling(struct cache *cache)  in cache_find_first_sibling()  argument
303 	struct cache *iter;  in cache_find_first_sibling()
305 	if (cache->type == CACHE_TYPE_UNIFIED ||  in cache_find_first_sibling()
306 	    cache->type == CACHE_TYPE_UNIFIED_D)  in cache_find_first_sibling()
307 		return cache;  in cache_find_first_sibling()
310 		if (iter->ofnode == cache->ofnode && iter->next_local == cache)  in cache_find_first_sibling()
313 	return cache;  in cache_find_first_sibling()
317 static struct cache *cache_lookup_by_node(const struct device_node *node)  in cache_lookup_by_node()
319 	struct cache *cache = NULL;  in cache_lookup_by_node()  local
320 	struct cache *iter;  in cache_lookup_by_node()
325 		cache = cache_find_first_sibling(iter);  in cache_lookup_by_node()
329 	return cache;  in cache_lookup_by_node()
353 static struct cache *cache_do_one_devnode_unified(struct device_node *node, int level)  in cache_do_one_devnode_unified()
360 static struct cache *cache_do_one_devnode_split(struct device_node *node,  in cache_do_one_devnode_split()
363 	struct cache *dcache, *icache;  in cache_do_one_devnode_split()
383 static struct cache *cache_do_one_devnode(struct device_node *node, int level)  in cache_do_one_devnode()
385 	struct cache *cache;  in cache_do_one_devnode()  local
388 		cache = cache_do_one_devnode_unified(node, level);  in cache_do_one_devnode()
390 		cache = cache_do_one_devnode_split(node, level);  in cache_do_one_devnode()
392 	return cache;  in cache_do_one_devnode()
395 static struct cache *cache_lookup_or_instantiate(struct device_node *node,  in cache_lookup_or_instantiate()
398 	struct cache *cache;  in cache_lookup_or_instantiate()  local
400 	cache = cache_lookup_by_node(node);  in cache_lookup_or_instantiate()
402 	WARN_ONCE(cache && cache->level != level,  in cache_lookup_or_instantiate()
404 		  cache->level, level);  in cache_lookup_or_instantiate()
406 	if (!cache)  in cache_lookup_or_instantiate()
407 		cache = cache_do_one_devnode(node, level);  in cache_lookup_or_instantiate()
409 	return cache;  in cache_lookup_or_instantiate()
412 static void link_cache_lists(struct cache *smaller, struct cache *bigger)  in link_cache_lists()
423 static void do_subsidiary_caches_debugcheck(struct cache *cache)  in do_subsidiary_caches_debugcheck()  argument
425 	WARN_ON_ONCE(cache->level != 1);  in do_subsidiary_caches_debugcheck()
426 	WARN_ON_ONCE(!of_node_is_type(cache->ofnode, "cpu"));  in do_subsidiary_caches_debugcheck()
429 static void do_subsidiary_caches(struct cache *cache)  in do_subsidiary_caches()  argument
432 	int level = cache->level;  in do_subsidiary_caches()
434 	do_subsidiary_caches_debugcheck(cache);  in do_subsidiary_caches()
436 	while ((subcache_node = of_find_next_cache_node(cache->ofnode))) {  in do_subsidiary_caches()
437 		struct cache *subcache;  in do_subsidiary_caches()
445 		link_cache_lists(cache, subcache);  in do_subsidiary_caches()
446 		cache = subcache;  in do_subsidiary_caches()
450 static struct cache *cache_chain_instantiate(unsigned int cpu_id)  in cache_chain_instantiate()
453 	struct cache *cpu_cache = NULL;  in cache_chain_instantiate()
513 		 index->cache->level, cache_type_string(index->cache));  in cache_index_release()
527 static struct cache *index_kobj_to_cache(struct kobject *k)  in index_kobj_to_cache()
533 	return index->cache;  in index_kobj_to_cache()
539 	struct cache *cache;  in size_show()  local
541 	cache = index_kobj_to_cache(k);  in size_show()
543 	if (cache_size_kb(cache, &size_kb))  in size_show()
556 	struct cache *cache;  in line_size_show()  local
558 	cache = index_kobj_to_cache(k);  in line_size_show()
560 	if (cache_get_line_size(cache, &line_size))  in line_size_show()
572 	struct cache *cache;  in nr_sets_show()  local
574 	cache = index_kobj_to_cache(k);  in nr_sets_show()
576 	if (cache_nr_sets(cache, &nr_sets))  in nr_sets_show()
588 	struct cache *cache;  in associativity_show()  local
590 	cache = index_kobj_to_cache(k);  in associativity_show()
592 	if (cache_associativity(cache, &associativity))  in associativity_show()
603 	struct cache *cache;  in type_show()  local
605 	cache = index_kobj_to_cache(k);  in type_show()
607 	return sprintf(buf, "%s\n", cache_type_string(cache));  in type_show()
616 	struct cache *cache;  in level_show()  local
619 	cache = index->cache;  in level_show()
621 	return sprintf(buf, "%d\n", cache->level);  in level_show()
642 static const struct cpumask *get_big_core_shared_cpu_map(int cpu, struct cache *cache)  in get_big_core_shared_cpu_map()  argument
644 	if (cache->level == 1)  in get_big_core_shared_cpu_map()
647 	return &cache->shared_cpu_map;  in get_big_core_shared_cpu_map()
653 	struct cache *cache;  in shared_cpu_map_show()  local
658 	cache = index->cache;  in shared_cpu_map_show()
662 		mask = get_big_core_shared_cpu_map(cpu, cache);  in shared_cpu_map_show()
664 		mask  = &cache->shared_cpu_map;  in shared_cpu_map_show()
711 	struct cache *cache;  in cacheinfo_create_index_opt_attrs()  local
719 	cache = dir->cache;  in cacheinfo_create_index_opt_attrs()
720 	cache_type = cache_type_string(cache);  in cacheinfo_create_index_opt_attrs()
737 				 attr->attr.name, cache->ofnode,  in cacheinfo_create_index_opt_attrs()
743 				 attr->attr.name, cache->ofnode, cache_type);  in cacheinfo_create_index_opt_attrs()
749 static void cacheinfo_create_index_dir(struct cache *cache, int index,  in cacheinfo_create_index_dir()  argument
759 	index_dir->cache = cache;  in cacheinfo_create_index_dir()
775 				     struct cache *cache_list)  in cacheinfo_sysfs_populate()
778 	struct cache *cache;  in cacheinfo_sysfs_populate()  local
785 	cache = cache_list;  in cacheinfo_sysfs_populate()
786 	while (cache) {  in cacheinfo_sysfs_populate()
787 		cacheinfo_create_index_dir(cache, index, cache_dir);  in cacheinfo_sysfs_populate()
789 		cache = cache->next_local;  in cacheinfo_sysfs_populate()
795 	struct cache *cache;  in cacheinfo_cpu_online()  local
797 	cache = cache_chain_instantiate(cpu_id);  in cacheinfo_cpu_online()
798 	if (!cache)  in cacheinfo_cpu_online()
801 	cacheinfo_sysfs_populate(cpu_id, cache);  in cacheinfo_cpu_online()
809 static struct cache *cache_lookup_by_cpu(unsigned int cpu_id)  in cache_lookup_by_cpu()
812 	struct cache *cache;  in cache_lookup_by_cpu()  local
819 	cache = cache_lookup_by_node(cpu_node);  in cache_lookup_by_cpu()
822 	return cache;  in cache_lookup_by_cpu()
852 static void cache_cpu_clear(struct cache *cache, int cpu)  in cache_cpu_clear()  argument
854 	while (cache) {  in cache_cpu_clear()
855 		struct cache *next = cache->next_local;  in cache_cpu_clear()
857 		WARN_ONCE(!cpumask_test_cpu(cpu, &cache->shared_cpu_map),  in cache_cpu_clear()
859 			  cpu, cache->ofnode,  in cache_cpu_clear()
860 			  cache_type_string(cache));  in cache_cpu_clear()
862 		cpumask_clear_cpu(cpu, &cache->shared_cpu_map);  in cache_cpu_clear()
866 		if (cpumask_empty(&cache->shared_cpu_map))  in cache_cpu_clear()
867 			release_cache(cache);  in cache_cpu_clear()
869 		cache = next;  in cache_cpu_clear()
876 	struct cache *cache;  in cacheinfo_cpu_offline()  local
890 	cache = cache_lookup_by_cpu(cpu_id);  in cacheinfo_cpu_offline()
891 	if (cache)  in cacheinfo_cpu_offline()
892 		cache_cpu_clear(cache, cpu_id);  in cacheinfo_cpu_offline()