25 #define ci_cacheinfo(cpu)	(&per_cpu(ci_cpu_cacheinfo, cpu))  argument
26 #define cache_leaves(cpu)	(ci_cacheinfo(cpu)->num_leaves)  argument
27 #define per_cpu_cacheinfo(cpu)	(ci_cacheinfo(cpu)->info_list)  argument
28 #define per_cpu_cacheinfo_idx(cpu, idx)		\  argument
29 				(per_cpu_cacheinfo(cpu) + (idx))
31 struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu)  in get_cpu_cacheinfo()  argument
33 	return ci_cacheinfo(cpu);  in get_cpu_cacheinfo()
54 bool last_level_cache_is_valid(unsigned int cpu)  in last_level_cache_is_valid()  argument
58 	if (!cache_leaves(cpu))  in last_level_cache_is_valid()
61 	llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);  in last_level_cache_is_valid()
197 static int cache_setup_of_node(unsigned int cpu)  in cache_setup_of_node()  argument
203 	np = of_cpu_device_node_get(cpu);  in cache_setup_of_node()
205 		pr_err("Failed to find cpu%d device node\n", cpu);  in cache_setup_of_node()
209 	while (index < cache_leaves(cpu)) {  in cache_setup_of_node()
210 		this_leaf = per_cpu_cacheinfo_idx(cpu, index);  in cache_setup_of_node()
222 	if (index != cache_leaves(cpu)) /* not all OF nodes populated */  in cache_setup_of_node()
228 static inline int cache_setup_of_node(unsigned int cpu) { return 0; }  in cache_setup_of_node()  argument
231 int __weak cache_setup_acpi(unsigned int cpu)  in cache_setup_acpi()  argument
238 static int cache_setup_properties(unsigned int cpu)  in cache_setup_properties()  argument
243 		ret = cache_setup_of_node(cpu);  in cache_setup_properties()
245 		ret = cache_setup_acpi(cpu);  in cache_setup_properties()
250 static int cache_shared_cpu_map_setup(unsigned int cpu)  in cache_shared_cpu_map_setup()  argument
252 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);  in cache_shared_cpu_map_setup()
265 	if (!last_level_cache_is_valid(cpu)) {  in cache_shared_cpu_map_setup()
266 		ret = cache_setup_properties(cpu);  in cache_shared_cpu_map_setup()
271 	for (index = 0; index < cache_leaves(cpu); index++) {  in cache_shared_cpu_map_setup()
274 		this_leaf = per_cpu_cacheinfo_idx(cpu, index);  in cache_shared_cpu_map_setup()
276 		cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);  in cache_shared_cpu_map_setup()
280 			if (i == cpu || !sib_cpu_ci->info_list)  in cache_shared_cpu_map_setup()
285 				cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map);  in cache_shared_cpu_map_setup()
297 static void cache_shared_cpu_map_remove(unsigned int cpu)  in cache_shared_cpu_map_remove()  argument
302 	for (index = 0; index < cache_leaves(cpu); index++) {  in cache_shared_cpu_map_remove()
303 		this_leaf = per_cpu_cacheinfo_idx(cpu, index);  in cache_shared_cpu_map_remove()
308 			if (sibling == cpu || !sib_cpu_ci->info_list)  in cache_shared_cpu_map_remove()
312 			cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map);  in cache_shared_cpu_map_remove()
320 static void free_cache_attributes(unsigned int cpu)  in free_cache_attributes()  argument
322 	if (!per_cpu_cacheinfo(cpu))  in free_cache_attributes()
325 	cache_shared_cpu_map_remove(cpu);  in free_cache_attributes()
327 	kfree(per_cpu_cacheinfo(cpu));  in free_cache_attributes()
328 	per_cpu_cacheinfo(cpu) = NULL;  in free_cache_attributes()
329 	cache_leaves(cpu) = 0;  in free_cache_attributes()
332 int __weak init_cache_level(unsigned int cpu)  in init_cache_level()  argument
337 int __weak populate_cache_leaves(unsigned int cpu)  in populate_cache_leaves()  argument
342 int detect_cache_attributes(unsigned int cpu)  in detect_cache_attributes()  argument
352 	if (per_cpu_cacheinfo(cpu))  in detect_cache_attributes()
355 	if (init_cache_level(cpu) || !cache_leaves(cpu))  in detect_cache_attributes()
358 	per_cpu_cacheinfo(cpu) = kcalloc(cache_leaves(cpu),  in detect_cache_attributes()
360 	if (per_cpu_cacheinfo(cpu) == NULL) {  in detect_cache_attributes()
361 		cache_leaves(cpu) = 0;  in detect_cache_attributes()
369 	ret = populate_cache_leaves(cpu);  in detect_cache_attributes()
379 	ret = cache_shared_cpu_map_setup(cpu);  in detect_cache_attributes()
381 		pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu);  in detect_cache_attributes()
388 	free_cache_attributes(cpu);  in detect_cache_attributes()
394 #define per_cpu_cache_dev(cpu)	(per_cpu(ci_cache_dev, cpu))  argument
400 #define per_cpu_index_dev(cpu)	(per_cpu(ci_index_dev, cpu))  argument
401 #define per_cache_index_dev(cpu, idx)	((per_cpu_index_dev(cpu))[idx])  argument
610 static void cpu_cache_sysfs_exit(unsigned int cpu)  in cpu_cache_sysfs_exit()  argument
615 	if (per_cpu_index_dev(cpu)) {  in cpu_cache_sysfs_exit()
616 		for (i = 0; i < cache_leaves(cpu); i++) {  in cpu_cache_sysfs_exit()
617 			ci_dev = per_cache_index_dev(cpu, i);  in cpu_cache_sysfs_exit()
622 		kfree(per_cpu_index_dev(cpu));  in cpu_cache_sysfs_exit()
623 		per_cpu_index_dev(cpu) = NULL;  in cpu_cache_sysfs_exit()
625 	device_unregister(per_cpu_cache_dev(cpu));  in cpu_cache_sysfs_exit()
626 	per_cpu_cache_dev(cpu) = NULL;  in cpu_cache_sysfs_exit()
629 static int cpu_cache_sysfs_init(unsigned int cpu)  in cpu_cache_sysfs_init()  argument
631 	struct device *dev = get_cpu_device(cpu);  in cpu_cache_sysfs_init()
633 	if (per_cpu_cacheinfo(cpu) == NULL)  in cpu_cache_sysfs_init()
636 	per_cpu_cache_dev(cpu) = cpu_device_create(dev, NULL, NULL, "cache");  in cpu_cache_sysfs_init()
637 	if (IS_ERR(per_cpu_cache_dev(cpu)))  in cpu_cache_sysfs_init()
638 		return PTR_ERR(per_cpu_cache_dev(cpu));  in cpu_cache_sysfs_init()
641 	per_cpu_index_dev(cpu) = kcalloc(cache_leaves(cpu),  in cpu_cache_sysfs_init()
643 	if (unlikely(per_cpu_index_dev(cpu) == NULL))  in cpu_cache_sysfs_init()
649 	cpu_cache_sysfs_exit(cpu);  in cpu_cache_sysfs_init()
653 static int cache_add_dev(unsigned int cpu)  in cache_add_dev()  argument
661 	rc = cpu_cache_sysfs_init(cpu);  in cache_add_dev()
665 	parent = per_cpu_cache_dev(cpu);  in cache_add_dev()
666 	for (i = 0; i < cache_leaves(cpu); i++) {  in cache_add_dev()
667 		this_leaf = per_cpu_cacheinfo_idx(cpu, i);  in cache_add_dev()
679 		per_cache_index_dev(cpu, i) = ci_dev;  in cache_add_dev()
681 	cpumask_set_cpu(cpu, &cache_dev_map);  in cache_add_dev()
685 	cpu_cache_sysfs_exit(cpu);  in cache_add_dev()
689 static int cacheinfo_cpu_online(unsigned int cpu)  in cacheinfo_cpu_online()  argument
691 	int rc = detect_cache_attributes(cpu);  in cacheinfo_cpu_online()
695 	rc = cache_add_dev(cpu);  in cacheinfo_cpu_online()
697 		free_cache_attributes(cpu);  in cacheinfo_cpu_online()
701 static int cacheinfo_cpu_pre_down(unsigned int cpu)  in cacheinfo_cpu_pre_down()  argument
703 	if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map))  in cacheinfo_cpu_pre_down()
704 		cpu_cache_sysfs_exit(cpu);  in cacheinfo_cpu_pre_down()
706 	free_cache_attributes(cpu);  in cacheinfo_cpu_pre_down()