80 static struct apic_chip_data *apic_chip_data(struct irq_data *irqd)  in apic_chip_data()  argument
82 	if (!irqd)  in apic_chip_data()
85 	while (irqd->parent_data)  in apic_chip_data()
86 		irqd = irqd->parent_data;  in apic_chip_data()
88 	return irqd->chip_data;  in apic_chip_data()
91 struct irq_cfg *irqd_cfg(struct irq_data *irqd)  in irqd_cfg()  argument
93 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in irqd_cfg()
119 static void apic_update_irq_cfg(struct irq_data *irqd, unsigned int vector,  in apic_update_irq_cfg()  argument
122 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in apic_update_irq_cfg()
128 	irq_data_update_effective_affinity(irqd, cpumask_of(cpu));  in apic_update_irq_cfg()
129 	trace_vector_config(irqd->irq, vector, cpu,  in apic_update_irq_cfg()
133 static void apic_update_vector(struct irq_data *irqd, unsigned int newvec,  in apic_update_vector()  argument
136 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in apic_update_vector()
137 	struct irq_desc *desc = irq_data_to_desc(irqd);  in apic_update_vector()
138 	bool managed = irqd_affinity_is_managed(irqd);  in apic_update_vector()
142 	trace_vector_update(irqd->irq, newvec, newcpu, apicd->vector,  in apic_update_vector()
178 static void vector_assign_managed_shutdown(struct irq_data *irqd)  in vector_assign_managed_shutdown()  argument
182 	apic_update_irq_cfg(irqd, MANAGED_IRQ_SHUTDOWN_VECTOR, cpu);  in vector_assign_managed_shutdown()
185 static int reserve_managed_vector(struct irq_data *irqd)  in reserve_managed_vector()  argument
187 	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);  in reserve_managed_vector()
188 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in reserve_managed_vector()
196 	trace_vector_reserve_managed(irqd->irq, ret);  in reserve_managed_vector()
200 static void reserve_irq_vector_locked(struct irq_data *irqd)  in reserve_irq_vector_locked()  argument
202 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in reserve_irq_vector_locked()
207 	irqd_set_can_reserve(irqd);  in reserve_irq_vector_locked()
208 	trace_vector_reserve(irqd->irq, 0);  in reserve_irq_vector_locked()
209 	vector_assign_managed_shutdown(irqd);  in reserve_irq_vector_locked()
212 static int reserve_irq_vector(struct irq_data *irqd)  in reserve_irq_vector()  argument
217 	reserve_irq_vector_locked(irqd);  in reserve_irq_vector()
223 assign_vector_locked(struct irq_data *irqd, const struct cpumask *dest)  in assign_vector_locked()  argument
225 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in assign_vector_locked()
250 	trace_vector_alloc(irqd->irq, vector, resvd, vector);  in assign_vector_locked()
253 	apic_update_vector(irqd, vector, cpu);  in assign_vector_locked()
254 	apic_update_irq_cfg(irqd, vector, cpu);  in assign_vector_locked()
259 static int assign_irq_vector(struct irq_data *irqd, const struct cpumask *dest)  in assign_irq_vector()  argument
266 	ret = assign_vector_locked(irqd, vector_searchmask);  in assign_irq_vector()
271 static int assign_irq_vector_any_locked(struct irq_data *irqd)  in assign_irq_vector_any_locked()  argument
274 	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);  in assign_irq_vector_any_locked()
275 	int node = irq_data_get_node(irqd);  in assign_irq_vector_any_locked()
281 	if (!assign_vector_locked(irqd, vector_searchmask))  in assign_irq_vector_any_locked()
284 	if (!assign_vector_locked(irqd, cpumask_of_node(node)))  in assign_irq_vector_any_locked()
289 	if (!assign_vector_locked(irqd, vector_searchmask))  in assign_irq_vector_any_locked()
292 	return assign_vector_locked(irqd, cpu_online_mask);  in assign_irq_vector_any_locked()
296 assign_irq_vector_policy(struct irq_data *irqd, struct irq_alloc_info *info)  in assign_irq_vector_policy()  argument
298 	if (irqd_affinity_is_managed(irqd))  in assign_irq_vector_policy()
299 		return reserve_managed_vector(irqd);  in assign_irq_vector_policy()
301 		return assign_irq_vector(irqd, info->mask);  in assign_irq_vector_policy()
306 	return reserve_irq_vector(irqd);  in assign_irq_vector_policy()
310 assign_managed_vector(struct irq_data *irqd, const struct cpumask *dest)  in assign_managed_vector()  argument
312 	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);  in assign_managed_vector()
313 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in assign_managed_vector()
324 	trace_vector_alloc_managed(irqd->irq, vector, vector);  in assign_managed_vector()
327 	apic_update_vector(irqd, vector, cpu);  in assign_managed_vector()
328 	apic_update_irq_cfg(irqd, vector, cpu);  in assign_managed_vector()
332 static void clear_irq_vector(struct irq_data *irqd)  in clear_irq_vector()  argument
334 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in clear_irq_vector()
335 	bool managed = irqd_affinity_is_managed(irqd);  in clear_irq_vector()
343 	trace_vector_clear(irqd->irq, vector, apicd->cpu, apicd->prev_vector,  in clear_irq_vector()
362 static void x86_vector_deactivate(struct irq_domain *dom, struct irq_data *irqd)  in x86_vector_deactivate()  argument
364 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in x86_vector_deactivate()
367 	trace_vector_deactivate(irqd->irq, apicd->is_managed,  in x86_vector_deactivate()
378 	clear_irq_vector(irqd);  in x86_vector_deactivate()
380 		reserve_irq_vector_locked(irqd);  in x86_vector_deactivate()
382 		vector_assign_managed_shutdown(irqd);  in x86_vector_deactivate()
386 static int activate_reserved(struct irq_data *irqd)  in activate_reserved()  argument
388 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in activate_reserved()
391 	ret = assign_irq_vector_any_locked(irqd);  in activate_reserved()
401 		if (!irqd_can_reserve(irqd))  in activate_reserved()
407 static int activate_managed(struct irq_data *irqd)  in activate_managed()  argument
409 	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);  in activate_managed()
415 		pr_err("Managed startup for irq %u, but no CPU\n", irqd->irq);  in activate_managed()
419 	ret = assign_managed_vector(irqd, vector_searchmask);  in activate_managed()
426 		       irqd->irq);  in activate_managed()
431 static int x86_vector_activate(struct irq_domain *dom, struct irq_data *irqd,  in x86_vector_activate()  argument
434 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in x86_vector_activate()
438 	trace_vector_activate(irqd->irq, apicd->is_managed,  in x86_vector_activate()
446 	if (reserve || irqd_is_managed_and_shutdown(irqd))  in x86_vector_activate()
447 		vector_assign_managed_shutdown(irqd);  in x86_vector_activate()
449 		ret = activate_managed(irqd);  in x86_vector_activate()
451 		ret = activate_reserved(irqd);  in x86_vector_activate()
456 static void vector_free_reserved_and_managed(struct irq_data *irqd)  in vector_free_reserved_and_managed()  argument
458 	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);  in vector_free_reserved_and_managed()
459 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in vector_free_reserved_and_managed()
461 	trace_vector_teardown(irqd->irq, apicd->is_managed,  in vector_free_reserved_and_managed()
474 	struct irq_data *irqd;  in x86_vector_free_irqs()  local
479 		irqd = irq_domain_get_irq_data(x86_vector_domain, virq + i);  in x86_vector_free_irqs()
480 		if (irqd && irqd->chip_data) {  in x86_vector_free_irqs()
482 			clear_irq_vector(irqd);  in x86_vector_free_irqs()
483 			vector_free_reserved_and_managed(irqd);  in x86_vector_free_irqs()
484 			apicd = irqd->chip_data;  in x86_vector_free_irqs()
485 			irq_domain_reset_irq_data(irqd);  in x86_vector_free_irqs()
492 static bool vector_configure_legacy(unsigned int virq, struct irq_data *irqd,  in vector_configure_legacy()  argument
506 	if (irqd_is_activated(irqd)) {  in vector_configure_legacy()
508 		apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu);  in vector_configure_legacy()
512 		irqd_set_can_reserve(irqd);  in vector_configure_legacy()
513 		clear_irq_vector(irqd);  in vector_configure_legacy()
525 	struct irq_data *irqd;  in x86_vector_alloc_irqs()  local
536 		irqd = irq_domain_get_irq_data(domain, virq + i);  in x86_vector_alloc_irqs()
537 		BUG_ON(!irqd);  in x86_vector_alloc_irqs()
538 		node = irq_data_get_node(irqd);  in x86_vector_alloc_irqs()
539 		WARN_ON_ONCE(irqd->chip_data);  in x86_vector_alloc_irqs()
547 		irqd->chip = &lapic_controller;  in x86_vector_alloc_irqs()
548 		irqd->chip_data = apicd;  in x86_vector_alloc_irqs()
549 		irqd->hwirq = virq + i;  in x86_vector_alloc_irqs()
550 		irqd_set_single_target(irqd);  in x86_vector_alloc_irqs()
559 			if (!vector_configure_legacy(virq + i, irqd, apicd))  in x86_vector_alloc_irqs()
563 		err = assign_irq_vector_policy(irqd, info);  in x86_vector_alloc_irqs()
566 			irqd->chip_data = NULL;  in x86_vector_alloc_irqs()
581 				  struct irq_data *irqd, int ind)  in x86_vector_debug_show()  argument
587 	if (!irqd) {  in x86_vector_debug_show()
592 	irq = irqd->irq;  in x86_vector_debug_show()
599 	if (!irqd->chip_data) {  in x86_vector_debug_show()
605 	memcpy(&apicd, irqd->chip_data, sizeof(apicd));  in x86_vector_debug_show()
761 static int apic_set_affinity(struct irq_data *irqd,  in apic_set_affinity()  argument
764 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in apic_set_affinity()
775 	if (!irqd_is_activated(irqd) &&  in apic_set_affinity()
781 	if (irqd_affinity_is_managed(irqd))  in apic_set_affinity()
782 		err = assign_managed_vector(irqd, vector_searchmask);  in apic_set_affinity()
784 		err = assign_vector_locked(irqd, vector_searchmask);  in apic_set_affinity()
793 static int apic_retrigger_irq(struct irq_data *irqd)  in apic_retrigger_irq()  argument
795 	struct apic_chip_data *apicd = apic_chip_data(irqd);  in apic_retrigger_irq()
805 void apic_ack_irq(struct irq_data *irqd)  in apic_ack_irq()  argument
807 	irq_move_irq(irqd);  in apic_ack_irq()
811 void apic_ack_edge(struct irq_data *irqd)  in apic_ack_edge()  argument
813 	irq_complete_move(irqd_cfg(irqd));  in apic_ack_edge()
814 	apic_ack_irq(irqd);  in apic_ack_edge()
930 	struct irq_data *irqd;  in irq_force_complete_move()  local
942 	irqd = irq_domain_get_irq_data(x86_vector_domain,  in irq_force_complete_move()
944 	if (!irqd)  in irq_force_complete_move()
948 	apicd = apic_chip_data(irqd);  in irq_force_complete_move()
1008 			irqd->irq, vector);  in irq_force_complete_move()