1 // SPDX-License-Identifier: GPL-2.0-only
3  * cpu_rmap.c: CPU affinity reverse-map support
13  * objects with CPU affinities.  This can be seen as a reverse-map of
14  * CPU affinity.  However, we do not assume that the object affinities
17  * CPU topology.
21  * alloc_cpu_rmap - allocate CPU affinity reverse-map
28 	unsigned int cpu;  in alloc_cpu_rmap()  local
39 	rmap = kzalloc(obj_offset + size * sizeof(rmap->obj[0]), flags);  in alloc_cpu_rmap()
43 	kref_init(&rmap->refcount);  in alloc_cpu_rmap()
44 	rmap->obj = (void **)((char *)rmap + obj_offset);  in alloc_cpu_rmap()
50 	 * any newly-hotplugged CPUs to have some object assigned.  in alloc_cpu_rmap()
52 	for_each_possible_cpu(cpu) {  in alloc_cpu_rmap()
53 		rmap->near[cpu].index = cpu % size;  in alloc_cpu_rmap()
54 		rmap->near[cpu].dist = CPU_RMAP_DIST_INF;  in alloc_cpu_rmap()
57 	rmap->size = size;  in alloc_cpu_rmap()
63  * cpu_rmap_release - internal reclaiming helper called from kref_put
73  * cpu_rmap_get - internal helper to get new ref on a cpu_rmap
74  * @rmap: reverse-map allocated with alloc_cpu_rmap()
78 	kref_get(&rmap->refcount);  in cpu_rmap_get()
82  * cpu_rmap_put - release ref on a cpu_rmap
83  * @rmap: reverse-map allocated with alloc_cpu_rmap()
87 	return kref_put(&rmap->refcount, cpu_rmap_release);  in cpu_rmap_put()
91 /* Reevaluate nearest object for given CPU, comparing with the given
94 static bool cpu_rmap_copy_neigh(struct cpu_rmap *rmap, unsigned int cpu,  in cpu_rmap_copy_neigh()  argument
100 		if (rmap->near[cpu].dist > dist &&  in cpu_rmap_copy_neigh()
101 		    rmap->near[neigh].dist <= dist) {  in cpu_rmap_copy_neigh()
102 			rmap->near[cpu].index = rmap->near[neigh].index;  in cpu_rmap_copy_neigh()
103 			rmap->near[cpu].dist = dist;  in cpu_rmap_copy_neigh()
114 	unsigned int cpu;  in debug_print_rmap()  local
118 	for_each_possible_cpu(cpu) {  in debug_print_rmap()
119 		index = rmap->near[cpu].index;  in debug_print_rmap()
120 		pr_info("cpu %d -> obj %u (distance %u)\n",  in debug_print_rmap()
121 			cpu, index, rmap->near[cpu].dist);  in debug_print_rmap()
132  * cpu_rmap_add - add object to a rmap
133  * @rmap: CPU rmap allocated with alloc_cpu_rmap()
142 	BUG_ON(rmap->used >= rmap->size);  in cpu_rmap_add()
143 	index = rmap->used++;  in cpu_rmap_add()
144 	rmap->obj[index] = obj;  in cpu_rmap_add()
150  * cpu_rmap_update - update CPU rmap following a change of object affinity
151  * @rmap: CPU rmap to update
153  * @affinity: New CPU affinity of object
159 	unsigned int cpu;  in cpu_rmap_update()  local
162 		return -ENOMEM;  in cpu_rmap_update()
167 	for_each_online_cpu(cpu) {  in cpu_rmap_update()
168 		if (rmap->near[cpu].index == index) {  in cpu_rmap_update()
169 			rmap->near[cpu].dist = CPU_RMAP_DIST_INF;  in cpu_rmap_update()
170 			cpumask_set_cpu(cpu, update_mask);  in cpu_rmap_update()
179 	for_each_cpu(cpu, affinity) {  in cpu_rmap_update()
180 		rmap->near[cpu].index = index;  in cpu_rmap_update()
181 		rmap->near[cpu].dist = 0;  in cpu_rmap_update()
183 			   cpumask_of_node(cpu_to_node(cpu)));  in cpu_rmap_update()
189 	for_each_cpu(cpu, update_mask) {  in cpu_rmap_update()
190 		if (cpu_rmap_copy_neigh(rmap, cpu,  in cpu_rmap_update()
191 					topology_sibling_cpumask(cpu), 1))  in cpu_rmap_update()
193 		if (cpu_rmap_copy_neigh(rmap, cpu,  in cpu_rmap_update()
194 					topology_core_cpumask(cpu), 2))  in cpu_rmap_update()
196 		if (cpu_rmap_copy_neigh(rmap, cpu,  in cpu_rmap_update()
197 					cpumask_of_node(cpu_to_node(cpu)), 3))  in cpu_rmap_update()
211 /* Glue between IRQ affinity notifiers and CPU rmaps */
220  * free_irq_cpu_rmap - free a CPU affinity reverse-map used for IRQs
221  * @rmap: Reverse-map allocated with alloc_irq_cpu_map(), or %NULL
233 	for (index = 0; index < rmap->used; index++) {  in free_irq_cpu_rmap()
234 		glue = rmap->obj[index];  in free_irq_cpu_rmap()
235 		irq_set_affinity_notifier(glue->notify.irq, NULL);  in free_irq_cpu_rmap()
243  * irq_cpu_rmap_notify - callback for IRQ subsystem when IRQ affinity updated
245  * @mask: cpu mask for new SMP affinity
256 	rc = cpu_rmap_update(glue->rmap, glue->index, mask);  in irq_cpu_rmap_notify()
262  * irq_cpu_rmap_release - reclaiming callback for IRQ subsystem
270 	cpu_rmap_put(glue->rmap);  in irq_cpu_rmap_release()
275  * irq_cpu_rmap_add - add an IRQ to a CPU affinity reverse-map
276  * @rmap: The reverse-map
279  * This adds an IRQ affinity notifier that will update the reverse-map
291 		return -ENOMEM;  in irq_cpu_rmap_add()
292 	glue->notify.notify = irq_cpu_rmap_notify;  in irq_cpu_rmap_add()
293 	glue->notify.release = irq_cpu_rmap_release;  in irq_cpu_rmap_add()
294 	glue->rmap = rmap;  in irq_cpu_rmap_add()
296 	glue->index = cpu_rmap_add(rmap, glue);  in irq_cpu_rmap_add()
297 	rc = irq_set_affinity_notifier(irq, &glue->notify);  in irq_cpu_rmap_add()
299 		cpu_rmap_put(glue->rmap);  in irq_cpu_rmap_add()