79 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)  in efx_wanted_parallelism()  argument
89 			netif_warn(efx, probe, efx->net_dev,  in efx_wanted_parallelism()
107 		netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,  in efx_wanted_parallelism()
117 	if (efx->type->sriov_wanted) {  in efx_wanted_parallelism()
118 		if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&  in efx_wanted_parallelism()
119 		    count > efx_vf_size(efx)) {  in efx_wanted_parallelism()
120 			netif_warn(efx, probe, efx->net_dev,  in efx_wanted_parallelism()
124 				   count, efx_vf_size(efx));  in efx_wanted_parallelism()
125 			count = efx_vf_size(efx);  in efx_wanted_parallelism()
133 static int efx_allocate_msix_channels(struct efx_nic *efx,  in efx_allocate_msix_channels()  argument
156 	vec_count = pci_msix_vec_count(efx->pci_dev);  in efx_allocate_msix_channels()
167 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
170 		efx->n_xdp_channels = 0;  in efx_allocate_msix_channels()
171 		efx->xdp_tx_per_channel = 0;  in efx_allocate_msix_channels()
172 		efx->xdp_tx_queue_count = 0;  in efx_allocate_msix_channels()
173 	} else if (n_channels + n_xdp_tx > efx->max_vis) {  in efx_allocate_msix_channels()
174 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
176 			  n_xdp_tx, n_channels, efx->max_vis);  in efx_allocate_msix_channels()
177 		efx->n_xdp_channels = 0;  in efx_allocate_msix_channels()
178 		efx->xdp_tx_per_channel = 0;  in efx_allocate_msix_channels()
179 		efx->xdp_tx_queue_count = 0;  in efx_allocate_msix_channels()
181 		efx->n_xdp_channels = n_xdp_ev;  in efx_allocate_msix_channels()
182 		efx->xdp_tx_per_channel = EFX_MAX_TXQ_PER_CHANNEL;  in efx_allocate_msix_channels()
183 		efx->xdp_tx_queue_count = n_xdp_tx;  in efx_allocate_msix_channels()
185 		netif_dbg(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
191 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
194 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
201 	efx->n_channels = n_channels;  in efx_allocate_msix_channels()
204 	n_channels -= efx->n_xdp_channels;  in efx_allocate_msix_channels()
207 		efx->n_tx_channels =  in efx_allocate_msix_channels()
209 			    efx->max_tx_channels);  in efx_allocate_msix_channels()
210 		efx->tx_channel_offset =  in efx_allocate_msix_channels()
211 			n_channels - efx->n_tx_channels;  in efx_allocate_msix_channels()
212 		efx->n_rx_channels =  in efx_allocate_msix_channels()
214 			    efx->n_tx_channels, 1U);  in efx_allocate_msix_channels()
216 		efx->n_tx_channels = min(n_channels, efx->max_tx_channels);  in efx_allocate_msix_channels()
217 		efx->tx_channel_offset = 0;  in efx_allocate_msix_channels()
218 		efx->n_rx_channels = n_channels;  in efx_allocate_msix_channels()
221 	efx->n_rx_channels = min(efx->n_rx_channels, parallelism);  in efx_allocate_msix_channels()
222 	efx->n_tx_channels = min(efx->n_tx_channels, parallelism);  in efx_allocate_msix_channels()
224 	efx->xdp_channel_offset = n_channels;  in efx_allocate_msix_channels()
226 	netif_dbg(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
228 		  efx->n_rx_channels);  in efx_allocate_msix_channels()
230 	return efx->n_channels;  in efx_allocate_msix_channels()
236 int efx_probe_interrupts(struct efx_nic *efx)  in efx_probe_interrupts()  argument
244 		if (efx->extra_channel_type[i])  in efx_probe_interrupts()
247 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {  in efx_probe_interrupts()
248 		unsigned int parallelism = efx_wanted_parallelism(efx);  in efx_probe_interrupts()
252 		rc = efx_allocate_msix_channels(efx, efx->max_channels,  in efx_probe_interrupts()
258 			rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,  in efx_probe_interrupts()
263 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
265 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)  in efx_probe_interrupts()
266 				efx->interrupt_mode = EFX_INT_MODE_MSI;  in efx_probe_interrupts()
270 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
273 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
279 			for (i = 0; i < efx->n_channels; i++)  in efx_probe_interrupts()
280 				efx_get_channel(efx, i)->irq =  in efx_probe_interrupts()
286 	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {  in efx_probe_interrupts()
287 		efx->n_channels = 1;  in efx_probe_interrupts()
288 		efx->n_rx_channels = 1;  in efx_probe_interrupts()
289 		efx->n_tx_channels = 1;  in efx_probe_interrupts()
290 		efx->n_xdp_channels = 0;  in efx_probe_interrupts()
291 		efx->xdp_channel_offset = efx->n_channels;  in efx_probe_interrupts()
292 		rc = pci_enable_msi(efx->pci_dev);  in efx_probe_interrupts()
294 			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;  in efx_probe_interrupts()
296 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
298 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)  in efx_probe_interrupts()
299 				efx->interrupt_mode = EFX_INT_MODE_LEGACY;  in efx_probe_interrupts()
306 	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {  in efx_probe_interrupts()
307 		efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);  in efx_probe_interrupts()
308 		efx->n_rx_channels = 1;  in efx_probe_interrupts()
309 		efx->n_tx_channels = 1;  in efx_probe_interrupts()
310 		efx->n_xdp_channels = 0;  in efx_probe_interrupts()
311 		efx->xdp_channel_offset = efx->n_channels;  in efx_probe_interrupts()
312 		efx->legacy_irq = efx->pci_dev->irq;  in efx_probe_interrupts()
316 	efx->n_extra_tx_channels = 0;  in efx_probe_interrupts()
317 	j = efx->xdp_channel_offset;  in efx_probe_interrupts()
319 		if (!efx->extra_channel_type[i])  in efx_probe_interrupts()
321 		if (j <= efx->tx_channel_offset + efx->n_tx_channels) {  in efx_probe_interrupts()
322 			efx->extra_channel_type[i]->handle_no_channel(efx);  in efx_probe_interrupts()
325 			efx_get_channel(efx, j)->type =  in efx_probe_interrupts()
326 				efx->extra_channel_type[i];  in efx_probe_interrupts()
327 			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))  in efx_probe_interrupts()
328 				efx->n_extra_tx_channels++;  in efx_probe_interrupts()
332 	rss_spread = efx->n_rx_channels;  in efx_probe_interrupts()
335 	if (efx->type->sriov_wanted) {  in efx_probe_interrupts()
336 		efx->rss_spread = ((rss_spread > 1 ||  in efx_probe_interrupts()
337 				    !efx->type->sriov_wanted(efx)) ?  in efx_probe_interrupts()
338 				   rss_spread : efx_vf_size(efx));  in efx_probe_interrupts()
342 	efx->rss_spread = rss_spread;  in efx_probe_interrupts()
348 void efx_set_interrupt_affinity(struct efx_nic *efx)  in efx_set_interrupt_affinity()  argument
353 	efx_for_each_channel(channel, efx) {  in efx_set_interrupt_affinity()
355 					   pcibus_to_node(efx->pci_dev->bus));  in efx_set_interrupt_affinity()
360 void efx_clear_interrupt_affinity(struct efx_nic *efx)  in efx_clear_interrupt_affinity()  argument
364 	efx_for_each_channel(channel, efx)  in efx_clear_interrupt_affinity()
369 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))  in efx_set_interrupt_affinity()  argument
374 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))  in efx_clear_interrupt_affinity()  argument
379 void efx_remove_interrupts(struct efx_nic *efx)  in efx_remove_interrupts()  argument
384 	efx_for_each_channel(channel, efx)  in efx_remove_interrupts()
386 	pci_disable_msi(efx->pci_dev);  in efx_remove_interrupts()
387 	pci_disable_msix(efx->pci_dev);  in efx_remove_interrupts()
390 	efx->legacy_irq = 0;  in efx_remove_interrupts()
404 	struct efx_nic *efx = channel->efx;  in efx_probe_eventq()  local
407 	netif_dbg(efx, probe, efx->net_dev,  in efx_probe_eventq()
413 	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);  in efx_probe_eventq()
423 	struct efx_nic *efx = channel->efx;  in efx_init_eventq()  local
428 	netif_dbg(efx, drv, efx->net_dev,  in efx_init_eventq()
433 		efx->type->push_irq_moderation(channel);  in efx_init_eventq()
443 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,  in efx_start_eventq()
469 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_fini_eventq()
478 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_remove_eventq()
508 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)  in efx_alloc_channel()  argument
519 	channel->efx = efx;  in efx_alloc_channel()
525 		tx_queue->efx = efx;  in efx_alloc_channel()
536 	rx_queue->efx = efx;  in efx_alloc_channel()
542 int efx_init_channels(struct efx_nic *efx)  in efx_init_channels()  argument
547 		efx->channel[i] = efx_alloc_channel(efx, i);  in efx_init_channels()
548 		if (!efx->channel[i])  in efx_init_channels()
550 		efx->msi_context[i].efx = efx;  in efx_init_channels()
551 		efx->msi_context[i].index = i;  in efx_init_channels()
555 	efx->interrupt_mode = min(efx->type->min_interrupt_mode,  in efx_init_channels()
558 	efx->max_channels = EFX_MAX_CHANNELS;  in efx_init_channels()
559 	efx->max_tx_channels = EFX_MAX_CHANNELS;  in efx_init_channels()
564 void efx_fini_channels(struct efx_nic *efx)  in efx_fini_channels()  argument
569 		if (efx->channel[i]) {  in efx_fini_channels()
570 			kfree(efx->channel[i]);  in efx_fini_channels()
571 			efx->channel[i] = NULL;  in efx_fini_channels()
623 	netif_dbg(channel->efx, probe, channel->efx->net_dev,  in efx_probe_channel()
657 	struct efx_nic *efx = channel->efx;  in efx_get_channel_name()  local
663 	if (number >= efx->xdp_channel_offset &&  in efx_get_channel_name()
664 	    !WARN_ON_ONCE(!efx->n_xdp_channels)) {  in efx_get_channel_name()
666 		number -= efx->xdp_channel_offset;  in efx_get_channel_name()
667 	} else if (efx->tx_channel_offset == 0) {  in efx_get_channel_name()
669 	} else if (number < efx->tx_channel_offset) {  in efx_get_channel_name()
673 		number -= efx->tx_channel_offset;  in efx_get_channel_name()
675 	snprintf(buf, len, "%s%s-%d", efx->name, type, number);  in efx_get_channel_name()
678 void efx_set_channel_names(struct efx_nic *efx)  in efx_set_channel_names()  argument
682 	efx_for_each_channel(channel, efx)  in efx_set_channel_names()
684 					efx->msi_context[channel->channel].name,  in efx_set_channel_names()
685 					sizeof(efx->msi_context[0].name));  in efx_set_channel_names()
688 int efx_probe_channels(struct efx_nic *efx)  in efx_probe_channels()  argument
694 	efx->next_buffer_table = 0;  in efx_probe_channels()
701 	efx_for_each_channel_rev(channel, efx) {  in efx_probe_channels()
704 			netif_err(efx, probe, efx->net_dev,  in efx_probe_channels()
710 	efx_set_channel_names(efx);  in efx_probe_channels()
715 	efx_remove_channels(efx);  in efx_probe_channels()
724 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_remove_channel()
735 void efx_remove_channels(struct efx_nic *efx)  in efx_remove_channels()  argument
739 	efx_for_each_channel(channel, efx)  in efx_remove_channels()
742 	kfree(efx->xdp_tx_queues);  in efx_remove_channels()
745 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)  in efx_realloc_channels()  argument
752 	rc = efx_check_disabled(efx);  in efx_realloc_channels()
759 	efx_for_each_channel(channel, efx) {  in efx_realloc_channels()
778 	efx_device_detach_sync(efx);  in efx_realloc_channels()
779 	efx_stop_all(efx);  in efx_realloc_channels()
780 	efx_soft_disable_interrupts(efx);  in efx_realloc_channels()
784 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
785 		channel = efx->channel[i];  in efx_realloc_channels()
796 	old_rxq_entries = efx->rxq_entries;  in efx_realloc_channels()
797 	old_txq_entries = efx->txq_entries;  in efx_realloc_channels()
798 	efx->rxq_entries = rxq_entries;  in efx_realloc_channels()
799 	efx->txq_entries = txq_entries;  in efx_realloc_channels()
800 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
801 		channel = efx->channel[i];  in efx_realloc_channels()
802 		efx->channel[i] = other_channel[i];  in efx_realloc_channels()
807 	efx->next_buffer_table = next_buffer_table;  in efx_realloc_channels()
809 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
810 		channel = efx->channel[i];  in efx_realloc_channels()
816 		efx_init_napi_channel(efx->channel[i]);  in efx_realloc_channels()
821 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
830 	rc2 = efx_soft_enable_interrupts(efx);  in efx_realloc_channels()
833 		netif_err(efx, drv, efx->net_dev,  in efx_realloc_channels()
835 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);  in efx_realloc_channels()
837 		efx_start_all(efx);  in efx_realloc_channels()
838 		efx_device_attach_if_not_resetting(efx);  in efx_realloc_channels()
844 	efx->rxq_entries = old_rxq_entries;  in efx_realloc_channels()
845 	efx->txq_entries = old_txq_entries;  in efx_realloc_channels()
846 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
847 		channel = efx->channel[i];  in efx_realloc_channels()
848 		efx->channel[i] = other_channel[i];  in efx_realloc_channels()
854 int efx_set_channels(struct efx_nic *efx)  in efx_set_channels()  argument
862 	efx->tx_channel_offset =  in efx_set_channels()
864 		efx->n_channels - efx->n_tx_channels : 0;  in efx_set_channels()
866 	if (efx->xdp_tx_queue_count) {  in efx_set_channels()
867 		EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);  in efx_set_channels()
870 		efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,  in efx_set_channels()
871 					     sizeof(*efx->xdp_tx_queues),  in efx_set_channels()
873 		if (!efx->xdp_tx_queues)  in efx_set_channels()
882 	efx_for_each_channel(channel, efx) {  in efx_set_channels()
883 		if (channel->channel < efx->n_rx_channels)  in efx_set_channels()
888 		if (channel->channel >= efx->tx_channel_offset) {  in efx_set_channels()
892 					netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is XDP %u, HW %u\n",  in efx_set_channels()
901 					if (xdp_queue_number < efx->xdp_tx_queue_count)  in efx_set_channels()
902 						efx->xdp_tx_queues[xdp_queue_number] = tx_queue;  in efx_set_channels()
908 					netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is HW %u\n",  in efx_set_channels()
916 	rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);  in efx_set_channels()
919 	return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);  in efx_set_channels()
924 	return channel->channel - channel->efx->tx_channel_offset <  in efx_default_channel_want_txqs()
925 		channel->efx->n_tx_channels;  in efx_default_channel_want_txqs()
932 int efx_soft_enable_interrupts(struct efx_nic *efx)  in efx_soft_enable_interrupts()  argument
937 	BUG_ON(efx->state == STATE_DISABLED);  in efx_soft_enable_interrupts()
939 	efx->irq_soft_enabled = true;  in efx_soft_enable_interrupts()
942 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
951 	efx_mcdi_mode_event(efx);  in efx_soft_enable_interrupts()
956 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
967 void efx_soft_disable_interrupts(struct efx_nic *efx)  in efx_soft_disable_interrupts()  argument
971 	if (efx->state == STATE_DISABLED)  in efx_soft_disable_interrupts()
974 	efx_mcdi_mode_poll(efx);  in efx_soft_disable_interrupts()
976 	efx->irq_soft_enabled = false;  in efx_soft_disable_interrupts()
979 	if (efx->legacy_irq)  in efx_soft_disable_interrupts()
980 		synchronize_irq(efx->legacy_irq);  in efx_soft_disable_interrupts()
982 	efx_for_each_channel(channel, efx) {  in efx_soft_disable_interrupts()
992 	efx_mcdi_flush_async(efx);  in efx_soft_disable_interrupts()
995 int efx_enable_interrupts(struct efx_nic *efx)  in efx_enable_interrupts()  argument
1001 	BUG_ON(efx->state == STATE_DISABLED);  in efx_enable_interrupts()
1003 	if (efx->eeh_disabled_legacy_irq) {  in efx_enable_interrupts()
1004 		enable_irq(efx->legacy_irq);  in efx_enable_interrupts()
1005 		efx->eeh_disabled_legacy_irq = false;  in efx_enable_interrupts()
1008 	efx->type->irq_enable_master(efx);  in efx_enable_interrupts()
1010 	efx_for_each_channel(channel, efx) {  in efx_enable_interrupts()
1018 	rc = efx_soft_enable_interrupts(efx);  in efx_enable_interrupts()
1026 	efx_for_each_channel(channel, efx) {  in efx_enable_interrupts()
1033 	efx->type->irq_disable_non_ev(efx);  in efx_enable_interrupts()
1038 void efx_disable_interrupts(struct efx_nic *efx)  in efx_disable_interrupts()  argument
1042 	efx_soft_disable_interrupts(efx);  in efx_disable_interrupts()
1044 	efx_for_each_channel(channel, efx) {  in efx_disable_interrupts()
1049 	efx->type->irq_disable_non_ev(efx);  in efx_disable_interrupts()
1052 void efx_start_channels(struct efx_nic *efx)  in efx_start_channels()  argument
1058 	efx_for_each_channel(channel, efx) {  in efx_start_channels()
1061 			atomic_inc(&efx->active_queues);  in efx_start_channels()
1066 			atomic_inc(&efx->active_queues);  in efx_start_channels()
1076 void efx_stop_channels(struct efx_nic *efx)  in efx_stop_channels()  argument
1084 	efx_for_each_channel(channel, efx) {  in efx_stop_channels()
1089 	efx_for_each_channel(channel, efx) {  in efx_stop_channels()
1102 	if (efx->type->fini_dmaq)  in efx_stop_channels()
1103 		rc = efx->type->fini_dmaq(efx);  in efx_stop_channels()
1106 		netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");  in efx_stop_channels()
1108 		netif_dbg(efx, drv, efx->net_dev,  in efx_stop_channels()
1112 	efx_for_each_channel(channel, efx) {  in efx_stop_channels()
1177 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)  in efx_update_irq_mod()  argument
1179 	int step = efx->irq_mod_step_us;  in efx_update_irq_mod()
1184 			efx->type->push_irq_moderation(channel);  in efx_update_irq_mod()
1188 		    efx->irq_rx_moderation_us) {  in efx_update_irq_mod()
1190 			efx->type->push_irq_moderation(channel);  in efx_update_irq_mod()
1207 	struct efx_nic *efx = channel->efx;  in efx_poll()  local
1213 	netif_vdbg(efx, intr, efx->net_dev,  in efx_poll()
1223 		    efx->irq_rx_adaptive &&  in efx_poll()
1225 			efx_update_irq_mod(efx, channel);  in efx_poll()
1250 	struct efx_nic *efx = channel->efx;  in efx_init_napi_channel()  local
1252 	channel->napi_dev = efx->net_dev;  in efx_init_napi_channel()
1257 void efx_init_napi(struct efx_nic *efx)  in efx_init_napi()  argument
1261 	efx_for_each_channel(channel, efx)  in efx_init_napi()
1273 void efx_fini_napi(struct efx_nic *efx)  in efx_fini_napi()  argument
1277 	efx_for_each_channel(channel, efx)  in efx_fini_napi()