17 	struct mic_device *mdev = dev;  in mic_thread_fn()  local
18 	struct mic_intr_info *intr_info = mdev->intr_info;  in mic_thread_fn()
19 	struct mic_irq_info *irq_info = &mdev->irq_info;  in mic_thread_fn()
21 	struct pci_dev *pdev = mdev->pdev;  in mic_thread_fn()
43 	struct mic_device *mdev = dev;  in mic_interrupt()  local
44 	struct mic_intr_info *intr_info = mdev->intr_info;  in mic_interrupt()
45 	struct mic_irq_info *irq_info = &mdev->irq_info;  in mic_interrupt()
47 	struct pci_dev *pdev = mdev->pdev;  in mic_interrupt()
51 	mask = mdev->ops->ack_interrupt(mdev);  in mic_interrupt()
71 static u16 mic_map_src_to_offset(struct mic_device *mdev,  in mic_map_src_to_offset()  argument
76 	if (intr_src >= mdev->intr_info->intr_len[type])  in mic_map_src_to_offset()
79 	return mdev->intr_info->intr_start_idx[type] + intr_src;  in mic_map_src_to_offset()
83 static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)  in mic_get_available_vector()  argument
86 	struct mic_irq_info *info = &mdev->irq_info;  in mic_get_available_vector()
107 static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,  in mic_register_intr_callback()  argument
122 	intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,  in mic_register_intr_callback()
129 	spin_lock(&mdev->irq_info.mic_thread_lock);  in mic_register_intr_callback()
130 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);  in mic_register_intr_callback()
131 	list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);  in mic_register_intr_callback()
132 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);  in mic_register_intr_callback()
133 	spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_register_intr_callback()
150 static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)  in mic_unregister_intr_callback()  argument
157 	spin_lock(&mdev->irq_info.mic_thread_lock);  in mic_unregister_intr_callback()
158 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);  in mic_unregister_intr_callback()
160 		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {  in mic_unregister_intr_callback()
164 				ida_simple_remove(&mdev->irq_info.cb_ida,  in mic_unregister_intr_callback()
168 					&mdev->irq_info.mic_intr_lock, flags);  in mic_unregister_intr_callback()
169 				spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_unregister_intr_callback()
174 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);  in mic_unregister_intr_callback()
175 	spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_unregister_intr_callback()
187 static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_msix()  argument
190 	int entry_size = sizeof(*mdev->irq_info.msix_entries);  in mic_setup_msix()
192 	mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,  in mic_setup_msix()
194 	if (!mdev->irq_info.msix_entries) {  in mic_setup_msix()
200 		mdev->irq_info.msix_entries[i].entry = i;  in mic_setup_msix()
202 	rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries,  in mic_setup_msix()
209 	mdev->irq_info.num_vectors = MIC_MIN_MSIX;  in mic_setup_msix()
210 	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *  in mic_setup_msix()
211 		mdev->irq_info.num_vectors), GFP_KERNEL);  in mic_setup_msix()
213 	if (!mdev->irq_info.mic_msi_map) {  in mic_setup_msix()
218 	dev_dbg(&mdev->pdev->dev,  in mic_setup_msix()
219 		"%d MSIx irqs setup\n", mdev->irq_info.num_vectors);  in mic_setup_msix()
224 	kfree(mdev->irq_info.msix_entries);  in mic_setup_msix()
226 	mdev->irq_info.num_vectors = 0;  in mic_setup_msix()
236 static int mic_setup_callbacks(struct mic_device *mdev)  in mic_setup_callbacks()  argument
240 	mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,  in mic_setup_callbacks()
241 					       sizeof(*mdev->irq_info.cb_list),  in mic_setup_callbacks()
243 	if (!mdev->irq_info.cb_list)  in mic_setup_callbacks()
247 		INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);  in mic_setup_callbacks()
248 	ida_init(&mdev->irq_info.cb_ida);  in mic_setup_callbacks()
249 	spin_lock_init(&mdev->irq_info.mic_intr_lock);  in mic_setup_callbacks()
250 	spin_lock_init(&mdev->irq_info.mic_thread_lock);  in mic_setup_callbacks()
260 static void mic_release_callbacks(struct mic_device *mdev)  in mic_release_callbacks()  argument
267 	spin_lock(&mdev->irq_info.mic_thread_lock);  in mic_release_callbacks()
268 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);  in mic_release_callbacks()
270 		if (list_empty(&mdev->irq_info.cb_list[i]))  in mic_release_callbacks()
273 		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {  in mic_release_callbacks()
276 			ida_simple_remove(&mdev->irq_info.cb_ida,  in mic_release_callbacks()
281 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);  in mic_release_callbacks()
282 	spin_unlock(&mdev->irq_info.mic_thread_lock);  in mic_release_callbacks()
283 	ida_destroy(&mdev->irq_info.cb_ida);  in mic_release_callbacks()
284 	kfree(mdev->irq_info.cb_list);  in mic_release_callbacks()
295 static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_msi()  argument
305 	mdev->irq_info.num_vectors = 1;  in mic_setup_msi()
306 	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *  in mic_setup_msi()
307 		mdev->irq_info.num_vectors), GFP_KERNEL);  in mic_setup_msi()
309 	if (!mdev->irq_info.mic_msi_map) {  in mic_setup_msi()
314 	rc = mic_setup_callbacks(mdev);  in mic_setup_msi()
321 				  0, "mic-msi", mdev);  in mic_setup_msi()
327 	dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);  in mic_setup_msi()
330 	mic_release_callbacks(mdev);  in mic_setup_msi()
332 	kfree(mdev->irq_info.mic_msi_map);  in mic_setup_msi()
335 	mdev->irq_info.num_vectors = 0;  in mic_setup_msi()
347 static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_intx()  argument
353 	rc = mic_setup_callbacks(mdev);  in mic_setup_intx()
360 				  IRQF_SHARED, "mic-intx", mdev);  in mic_setup_intx()
367 	mic_release_callbacks(mdev);  in mic_setup_intx()
381 int mic_next_db(struct mic_device *mdev)  in mic_next_db()  argument
385 	next_db = mdev->irq_info.next_avail_src %  in mic_next_db()
386 		mdev->intr_info->intr_len[MIC_INTR_DB];  in mic_next_db()
387 	mdev->irq_info.next_avail_src++;  in mic_next_db()
420 mic_request_threaded_irq(struct mic_device *mdev,  in mic_request_threaded_irq()  argument
431 	struct pci_dev *pdev = mdev->pdev;  in mic_request_threaded_irq()
433 	offset = mic_map_src_to_offset(mdev, intr_src, type);  in mic_request_threaded_irq()
435 		dev_err(&mdev->pdev->dev,  in mic_request_threaded_irq()
442 	if (mdev->irq_info.num_vectors > 1) {  in mic_request_threaded_irq()
443 		msix = mic_get_available_vector(mdev);  in mic_request_threaded_irq()
445 			dev_err(&mdev->pdev->dev,  in mic_request_threaded_irq()
454 			dev_dbg(&mdev->pdev->dev,  in mic_request_threaded_irq()
459 		mdev->irq_info.mic_msi_map[entry] |= BIT(offset);  in mic_request_threaded_irq()
460 		mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_request_threaded_irq()
463 		dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n",  in mic_request_threaded_irq()
466 		intr_cb = mic_register_intr_callback(mdev, offset, handler,  in mic_request_threaded_irq()
469 			dev_err(&mdev->pdev->dev,  in mic_request_threaded_irq()
477 			mdev->irq_info.mic_msi_map[entry] |= (1 << offset);  in mic_request_threaded_irq()
478 			mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_request_threaded_irq()
482 		dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n",  in mic_request_threaded_irq()
501 void mic_free_irq(struct mic_device *mdev,  in mic_free_irq()  argument
508 	struct pci_dev *pdev = mdev->pdev;  in mic_free_irq()
512 	if (mdev->irq_info.num_vectors > 1) {  in mic_free_irq()
513 		if (entry >= mdev->irq_info.num_vectors) {  in mic_free_irq()
514 			dev_warn(&mdev->pdev->dev,  in mic_free_irq()
516 				entry, mdev->irq_info.num_vectors);  in mic_free_irq()
519 		irq = mdev->irq_info.msix_entries[entry].vector;  in mic_free_irq()
521 		mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));  in mic_free_irq()
522 		mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_free_irq()
525 		dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq);  in mic_free_irq()
528 		src_id = mic_unregister_intr_callback(mdev, offset);  in mic_free_irq()
530 			dev_warn(&mdev->pdev->dev, "Error unregistering callback\n");  in mic_free_irq()
534 			mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));  in mic_free_irq()
535 			mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_free_irq()
538 		dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n",  in mic_free_irq()
551 int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)  in mic_setup_interrupts()  argument
555 	rc = mic_setup_msix(mdev, pdev);  in mic_setup_interrupts()
559 	rc = mic_setup_msi(mdev, pdev);  in mic_setup_interrupts()
563 	rc = mic_setup_intx(mdev, pdev);  in mic_setup_interrupts()
565 		dev_err(&mdev->pdev->dev, "no usable interrupts\n");  in mic_setup_interrupts()
569 	mdev->intr_ops->enable_interrupts(mdev);  in mic_setup_interrupts()
581 void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)  in mic_free_interrupts()  argument
585 	mdev->intr_ops->disable_interrupts(mdev);  in mic_free_interrupts()
586 	if (mdev->irq_info.num_vectors > 1) {  in mic_free_interrupts()
587 		for (i = 0; i < mdev->irq_info.num_vectors; i++) {  in mic_free_interrupts()
588 			if (mdev->irq_info.mic_msi_map[i])  in mic_free_interrupts()
590 					 mdev->irq_info.msix_entries[i].vector);  in mic_free_interrupts()
592 		kfree(mdev->irq_info.mic_msi_map);  in mic_free_interrupts()
593 		kfree(mdev->irq_info.msix_entries);  in mic_free_interrupts()
597 			free_irq(pdev->irq, mdev);  in mic_free_interrupts()
598 			kfree(mdev->irq_info.mic_msi_map);  in mic_free_interrupts()
601 			free_irq(pdev->irq, mdev);  in mic_free_interrupts()
603 		mic_release_callbacks(mdev);  in mic_free_interrupts()
618 void mic_intr_restore(struct mic_device *mdev)  in mic_intr_restore()  argument
621 	struct pci_dev *pdev = mdev->pdev;  in mic_intr_restore()
626 	for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {  in mic_intr_restore()
628 			if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))  in mic_intr_restore()
629 				mdev->intr_ops->program_msi_to_src_map(mdev,  in mic_intr_restore()