157 	 * We need a channel per event queue, plus a VI per tx queue.  in efx_allocate_msix_channels()
274 			/* Fall back to single channel MSI */  in efx_siena_probe_interrupts()
365 	struct efx_channel *channel;  in efx_siena_set_interrupt_affinity()  local
373 	efx_for_each_channel(channel, efx) {  in efx_siena_set_interrupt_affinity()
377 		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));  in efx_siena_set_interrupt_affinity()
383 	struct efx_channel *channel;  in efx_siena_clear_interrupt_affinity()  local
385 	efx_for_each_channel(channel, efx)  in efx_siena_clear_interrupt_affinity()
386 		irq_set_affinity_hint(channel->irq, NULL);  in efx_siena_clear_interrupt_affinity()
402 	struct efx_channel *channel;  in efx_siena_remove_interrupts()  local
405 	efx_for_each_channel(channel, efx)  in efx_siena_remove_interrupts()
406 		channel->irq = 0;  in efx_siena_remove_interrupts()
419  * Event queue memory allocations are done only once.  If the channel
421  * errors during channel reset and also simplifies interrupt handling.
423 static int efx_probe_eventq(struct efx_channel *channel)  in efx_probe_eventq()  argument
425 	struct efx_nic *efx = channel->efx;  in efx_probe_eventq()
429 		  "chan %d create event queue\n", channel->channel);  in efx_probe_eventq()
436 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;  in efx_probe_eventq()
438 	return efx_nic_probe_eventq(channel);  in efx_probe_eventq()
441 /* Prepare channel's event queue */
442 static int efx_init_eventq(struct efx_channel *channel)  in efx_init_eventq()  argument
444 	struct efx_nic *efx = channel->efx;  in efx_init_eventq()
447 	EFX_WARN_ON_PARANOID(channel->eventq_init);  in efx_init_eventq()
450 		  "chan %d init event queue\n", channel->channel);  in efx_init_eventq()
452 	rc = efx_nic_init_eventq(channel);  in efx_init_eventq()
454 		efx->type->push_irq_moderation(channel);  in efx_init_eventq()
455 		channel->eventq_read_ptr = 0;  in efx_init_eventq()
456 		channel->eventq_init = true;  in efx_init_eventq()
462 void efx_siena_start_eventq(struct efx_channel *channel)  in efx_siena_start_eventq()  argument
464 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,  in efx_siena_start_eventq()
465 		  "chan %d start event queue\n", channel->channel);  in efx_siena_start_eventq()
468 	channel->enabled = true;  in efx_siena_start_eventq()
471 	napi_enable(&channel->napi_str);  in efx_siena_start_eventq()
472 	efx_nic_eventq_read_ack(channel);  in efx_siena_start_eventq()
476 void efx_siena_stop_eventq(struct efx_channel *channel)  in efx_siena_stop_eventq()  argument
478 	if (!channel->enabled)  in efx_siena_stop_eventq()
481 	napi_disable(&channel->napi_str);  in efx_siena_stop_eventq()
482 	channel->enabled = false;  in efx_siena_stop_eventq()
485 static void efx_fini_eventq(struct efx_channel *channel)  in efx_fini_eventq()  argument
487 	if (!channel->eventq_init)  in efx_fini_eventq()
490 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_fini_eventq()
491 		  "chan %d fini event queue\n", channel->channel);  in efx_fini_eventq()
493 	efx_nic_fini_eventq(channel);  in efx_fini_eventq()
494 	channel->eventq_init = false;  in efx_fini_eventq()
497 static void efx_remove_eventq(struct efx_channel *channel)  in efx_remove_eventq()  argument
499 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_remove_eventq()
500 		  "chan %d remove event queue\n", channel->channel);  in efx_remove_eventq()
502 	efx_nic_remove_eventq(channel);  in efx_remove_eventq()
507  * Channel handling
515 	struct efx_channel *channel;  in efx_filter_rfs_expire()  local
518 	channel = container_of(dwork, struct efx_channel, filter_work);  in efx_filter_rfs_expire()
519 	time = jiffies - channel->rfs_last_expiry;  in efx_filter_rfs_expire()
520 	quota = channel->rfs_filter_count * time / (30 * HZ);  in efx_filter_rfs_expire()
521 	if (quota >= 20 && __efx_siena_filter_rfs_expire(channel,  in efx_filter_rfs_expire()
522 					min(channel->rfs_filter_count, quota)))  in efx_filter_rfs_expire()
523 		channel->rfs_last_expiry += time;  in efx_filter_rfs_expire()
529 /* Allocate and initialise a channel structure. */
534 	struct efx_channel *channel;  in efx_alloc_channel()  local
537 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);  in efx_alloc_channel()
538 	if (!channel)  in efx_alloc_channel()
541 	channel->efx = efx;  in efx_alloc_channel()
542 	channel->channel = i;  in efx_alloc_channel()
543 	channel->type = &efx_default_channel_type;  in efx_alloc_channel()
546 		tx_queue = &channel->tx_queue[j];  in efx_alloc_channel()
550 		tx_queue->channel = channel;  in efx_alloc_channel()
554 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);  in efx_alloc_channel()
557 	rx_queue = &channel->rx_queue;  in efx_alloc_channel()
561 	return channel;  in efx_alloc_channel()
569 		efx->channel[i] = efx_alloc_channel(efx, i);  in efx_siena_init_channels()
570 		if (!efx->channel[i])  in efx_siena_init_channels()
591 		if (efx->channel[i]) {  in efx_siena_fini_channels()
592 			kfree(efx->channel[i]);  in efx_siena_fini_channels()
593 			efx->channel[i] = NULL;  in efx_siena_fini_channels()
597 /* Allocate and initialise a channel structure, copying parameters
598  * (but not resources) from an old channel structure.
605 	struct efx_channel *channel;  in efx_copy_channel()  local
608 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);  in efx_copy_channel()
609 	if (!channel)  in efx_copy_channel()
612 	*channel = *old_channel;  in efx_copy_channel()
614 	channel->napi_dev = NULL;  in efx_copy_channel()
615 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);  in efx_copy_channel()
616 	channel->napi_str.napi_id = 0;  in efx_copy_channel()
617 	channel->napi_str.state = 0;  in efx_copy_channel()
618 	memset(&channel->eventq, 0, sizeof(channel->eventq));  in efx_copy_channel()
621 		tx_queue = &channel->tx_queue[j];  in efx_copy_channel()
622 		if (tx_queue->channel)  in efx_copy_channel()
623 			tx_queue->channel = channel;  in efx_copy_channel()
629 	rx_queue = &channel->rx_queue;  in efx_copy_channel()
634 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);  in efx_copy_channel()
637 	return channel;  in efx_copy_channel()
640 static int efx_probe_channel(struct efx_channel *channel)  in efx_probe_channel()  argument
646 	netif_dbg(channel->efx, probe, channel->efx->net_dev,  in efx_probe_channel()
647 		  "creating channel %d\n", channel->channel);  in efx_probe_channel()
649 	rc = channel->type->pre_probe(channel);  in efx_probe_channel()
653 	rc = efx_probe_eventq(channel);  in efx_probe_channel()
657 	efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_probe_channel()
663 	efx_for_each_channel_rx_queue(rx_queue, channel) {  in efx_probe_channel()
669 	channel->rx_list = NULL;  in efx_probe_channel()
674 	efx_siena_remove_channel(channel);  in efx_probe_channel()
678 static void efx_get_channel_name(struct efx_channel *channel, char *buf,  in efx_get_channel_name()  argument
681 	struct efx_nic *efx = channel->efx;  in efx_get_channel_name()
685 	number = channel->channel;  in efx_get_channel_name()
704 	struct efx_channel *channel;  in efx_siena_set_channel_names()  local
706 	efx_for_each_channel(channel, efx)  in efx_siena_set_channel_names()
707 		channel->type->get_name(channel,  in efx_siena_set_channel_names()
708 					efx->msi_context[channel->channel].name,  in efx_siena_set_channel_names()
714 	struct efx_channel *channel;  in efx_siena_probe_channels()  local
725 	efx_for_each_channel_rev(channel, efx) {  in efx_siena_probe_channels()
726 		rc = efx_probe_channel(channel);  in efx_siena_probe_channels()
729 				  "failed to create channel %d\n",  in efx_siena_probe_channels()
730 				  channel->channel);  in efx_siena_probe_channels()
743 void efx_siena_remove_channel(struct efx_channel *channel)  in efx_siena_remove_channel()  argument
748 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_siena_remove_channel()
749 		  "destroy chan %d\n", channel->channel);  in efx_siena_remove_channel()
751 	efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_siena_remove_channel()
753 	efx_for_each_channel_tx_queue(tx_queue, channel)  in efx_siena_remove_channel()
755 	efx_remove_eventq(channel);  in efx_siena_remove_channel()
756 	channel->type->post_remove(channel);  in efx_siena_remove_channel()
761 	struct efx_channel *channel;  in efx_siena_remove_channels()  local
763 	efx_for_each_channel(channel, efx)  in efx_siena_remove_channels()
764 		efx_siena_remove_channel(channel);  in efx_siena_remove_channels()
776 		  "Channel %u TXQ %u is XDP %u, HW %u\n",  in efx_set_xdp_tx_queue()
777 		  tx_queue->channel->channel, tx_queue->label,  in efx_set_xdp_tx_queue()
786 	struct efx_channel *channel;  in efx_set_xdp_channels()  local
795 	efx_for_each_channel(channel, efx) {  in efx_set_xdp_channels()
796 		if (channel->channel < efx->tx_channel_offset)  in efx_set_xdp_channels()
799 		if (efx_channel_is_xdp_tx(channel)) {  in efx_set_xdp_channels()
800 			efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_set_xdp_channels()
808 			efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_set_xdp_channels()
811 					  "Channel %u TXQ %u is HW %u\n",  in efx_set_xdp_channels()
812 					  channel->channel, tx_queue->label,  in efx_set_xdp_channels()
818 			 * first one of the channel  in efx_set_xdp_channels()
823 				tx_queue = &channel->tx_queue[0];  in efx_set_xdp_channels()
850 static void efx_init_napi_channel(struct efx_channel *channel);
851 static void efx_fini_napi_channel(struct efx_channel *channel);
856 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;  in efx_siena_realloc_channels()  local
868 	efx_for_each_channel(channel, efx) {  in efx_siena_realloc_channels()
872 		if (channel->type->copy)  in efx_siena_realloc_channels()
875 					channel->eventq.index +  in efx_siena_realloc_channels()
876 					channel->eventq.entries);  in efx_siena_realloc_channels()
877 		efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_siena_realloc_channels()
881 		efx_for_each_channel_tx_queue(tx_queue, channel)  in efx_siena_realloc_channels()
894 		channel = efx->channel[i];  in efx_siena_realloc_channels()
895 		if (channel->type->copy)  in efx_siena_realloc_channels()
896 			channel = channel->type->copy(channel);  in efx_siena_realloc_channels()
897 		if (!channel) {  in efx_siena_realloc_channels()
901 		other_channel[i] = channel;  in efx_siena_realloc_channels()
904 	/* Swap entry counts and channel pointers */  in efx_siena_realloc_channels()
910 		swap(efx->channel[i], other_channel[i]);  in efx_siena_realloc_channels()
916 		channel = efx->channel[i];  in efx_siena_realloc_channels()
917 		if (!channel->type->copy)  in efx_siena_realloc_channels()
919 		rc = efx_probe_channel(channel);  in efx_siena_realloc_channels()
922 		efx_init_napi_channel(efx->channel[i]);  in efx_siena_realloc_channels()
927 	/* Destroy unused channel structures */  in efx_siena_realloc_channels()
929 		channel = other_channel[i];  in efx_siena_realloc_channels()
930 		if (channel && channel->type->copy) {  in efx_siena_realloc_channels()
931 			efx_fini_napi_channel(channel);  in efx_siena_realloc_channels()
932 			efx_siena_remove_channel(channel);  in efx_siena_realloc_channels()
933 			kfree(channel);  in efx_siena_realloc_channels()
941 			  "unable to restart interrupts on channel reallocation\n");  in efx_siena_realloc_channels()
954 		swap(efx->channel[i], other_channel[i]);  in efx_siena_realloc_channels()
960 	struct efx_channel *channel;  in efx_siena_set_channels()  local
974 	efx_for_each_channel(channel, efx) {  in efx_siena_set_channels()
975 		if (channel->channel < efx->n_rx_channels)  in efx_siena_set_channels()
976 			channel->rx_queue.core_index = channel->channel;  in efx_siena_set_channels()
978 			channel->rx_queue.core_index = -1;  in efx_siena_set_channels()
989 static bool efx_default_channel_want_txqs(struct efx_channel *channel)  in efx_default_channel_want_txqs()  argument
991 	return channel->channel - channel->efx->tx_channel_offset <  in efx_default_channel_want_txqs()
992 		channel->efx->n_tx_channels;  in efx_default_channel_want_txqs()
1001 	struct efx_channel *channel, *end_channel;  in efx_soft_enable_interrupts()  local
1009 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
1010 		if (!channel->type->keep_eventq) {  in efx_soft_enable_interrupts()
1011 			rc = efx_init_eventq(channel);  in efx_soft_enable_interrupts()
1015 		efx_siena_start_eventq(channel);  in efx_soft_enable_interrupts()
1022 	end_channel = channel;  in efx_soft_enable_interrupts()
1023 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
1024 		if (channel == end_channel)  in efx_soft_enable_interrupts()
1026 		efx_siena_stop_eventq(channel);  in efx_soft_enable_interrupts()
1027 		if (!channel->type->keep_eventq)  in efx_soft_enable_interrupts()
1028 			efx_fini_eventq(channel);  in efx_soft_enable_interrupts()
1036 	struct efx_channel *channel;  in efx_soft_disable_interrupts()  local
1049 	efx_for_each_channel(channel, efx) {  in efx_soft_disable_interrupts()
1050 		if (channel->irq)  in efx_soft_disable_interrupts()
1051 			synchronize_irq(channel->irq);  in efx_soft_disable_interrupts()
1053 		efx_siena_stop_eventq(channel);  in efx_soft_disable_interrupts()
1054 		if (!channel->type->keep_eventq)  in efx_soft_disable_interrupts()
1055 			efx_fini_eventq(channel);  in efx_soft_disable_interrupts()
1064 	struct efx_channel *channel, *end_channel;  in efx_siena_enable_interrupts()  local
1077 	efx_for_each_channel(channel, efx) {  in efx_siena_enable_interrupts()
1078 		if (channel->type->keep_eventq) {  in efx_siena_enable_interrupts()
1079 			rc = efx_init_eventq(channel);  in efx_siena_enable_interrupts()
1092 	end_channel = channel;  in efx_siena_enable_interrupts()
1093 	efx_for_each_channel(channel, efx) {  in efx_siena_enable_interrupts()
1094 		if (channel == end_channel)  in efx_siena_enable_interrupts()
1096 		if (channel->type->keep_eventq)  in efx_siena_enable_interrupts()
1097 			efx_fini_eventq(channel);  in efx_siena_enable_interrupts()
1107 	struct efx_channel *channel;  in efx_siena_disable_interrupts()  local
1111 	efx_for_each_channel(channel, efx) {  in efx_siena_disable_interrupts()
1112 		if (channel->type->keep_eventq)  in efx_siena_disable_interrupts()
1113 			efx_fini_eventq(channel);  in efx_siena_disable_interrupts()
1123 	struct efx_channel *channel;  in efx_siena_start_channels()  local
1125 	efx_for_each_channel_rev(channel, efx) {  in efx_siena_start_channels()
1126 		efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_siena_start_channels()
1131 		efx_for_each_channel_rx_queue(rx_queue, channel) {  in efx_siena_start_channels()
1134 			efx_siena_stop_eventq(channel);  in efx_siena_start_channels()
1136 			efx_siena_start_eventq(channel);  in efx_siena_start_channels()
1139 		WARN_ON(channel->rx_pkt_n_frags);  in efx_siena_start_channels()
1147 	struct efx_channel *channel;  in efx_siena_stop_channels()  local
1151 	efx_for_each_channel(channel, efx) {  in efx_siena_stop_channels()
1152 		efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_siena_stop_channels()
1156 	efx_for_each_channel(channel, efx) {  in efx_siena_stop_channels()
1163 		if (efx_channel_has_rx_queue(channel)) {  in efx_siena_stop_channels()
1164 			efx_siena_stop_eventq(channel);  in efx_siena_stop_channels()
1165 			efx_siena_start_eventq(channel);  in efx_siena_stop_channels()
1179 	efx_for_each_channel(channel, efx) {  in efx_siena_stop_channels()
1180 		efx_for_each_channel_rx_queue(rx_queue, channel)  in efx_siena_stop_channels()
1182 		efx_for_each_channel_tx_queue(tx_queue, channel)  in efx_siena_stop_channels()
1193 /* Process channel's event queue
1196  * single channel.  The caller must guarantee that this function will
1197  * never be concurrently called more than once on the same channel,
1200 static int efx_process_channel(struct efx_channel *channel, int budget)  in efx_process_channel()  argument
1206 	if (unlikely(!channel->enabled))  in efx_process_channel()
1210 	EFX_WARN_ON_PARANOID(channel->rx_list != NULL);  in efx_process_channel()
1212 	channel->rx_list = &rx_list;  in efx_process_channel()
1214 	efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_process_channel()
1219 	spent = efx_nic_process_eventq(channel, budget);  in efx_process_channel()
1220 	if (spent && efx_channel_has_rx_queue(channel)) {  in efx_process_channel()
1222 			efx_channel_get_rx_queue(channel);  in efx_process_channel()
1224 		efx_rx_flush_packet(channel);  in efx_process_channel()
1229 	efx_for_each_channel_tx_queue(tx_queue, channel) {  in efx_process_channel()
1238 	netif_receive_skb_list(channel->rx_list);  in efx_process_channel()
1239 	channel->rx_list = NULL;  in efx_process_channel()
1244 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)  in efx_update_irq_mod()  argument
1248 	if (channel->irq_mod_score < irq_adapt_low_thresh) {  in efx_update_irq_mod()
1249 		if (channel->irq_moderation_us > step) {  in efx_update_irq_mod()
1250 			channel->irq_moderation_us -= step;  in efx_update_irq_mod()
1251 			efx->type->push_irq_moderation(channel);  in efx_update_irq_mod()
1253 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {  in efx_update_irq_mod()
1254 		if (channel->irq_moderation_us <  in efx_update_irq_mod()
1256 			channel->irq_moderation_us += step;  in efx_update_irq_mod()
1257 			efx->type->push_irq_moderation(channel);  in efx_update_irq_mod()
1261 	channel->irq_count = 0;  in efx_update_irq_mod()
1262 	channel->irq_mod_score = 0;  in efx_update_irq_mod()
1272 	struct efx_channel *channel =  in efx_poll()  local
1274 	struct efx_nic *efx = channel->efx;  in efx_poll()
1281 		   "channel %d NAPI poll executing on CPU %d\n",  in efx_poll()
1282 		   channel->channel, raw_smp_processor_id());  in efx_poll()
1284 	spent = efx_process_channel(channel, budget);  in efx_poll()
1289 		if (efx_channel_has_rx_queue(channel) &&  in efx_poll()
1291 		    unlikely(++channel->irq_count == 1000)) {  in efx_poll()
1292 			efx_update_irq_mod(efx, channel);  in efx_poll()
1297 		time = jiffies - channel->rfs_last_expiry;  in efx_poll()
1299 		if (channel->rfs_filter_count * time >= 600 * HZ)  in efx_poll()
1300 			mod_delayed_work(system_wq, &channel->filter_work, 0);  in efx_poll()
1309 			efx_nic_eventq_read_ack(channel);  in efx_poll()
1315 static void efx_init_napi_channel(struct efx_channel *channel)  in efx_init_napi_channel()  argument
1317 	struct efx_nic *efx = channel->efx;  in efx_init_napi_channel()
1319 	channel->napi_dev = efx->net_dev;  in efx_init_napi_channel()
1320 	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);  in efx_init_napi_channel()
1325 	struct efx_channel *channel;  in efx_siena_init_napi()  local
1327 	efx_for_each_channel(channel, efx)  in efx_siena_init_napi()
1328 		efx_init_napi_channel(channel);  in efx_siena_init_napi()
1331 static void efx_fini_napi_channel(struct efx_channel *channel)  in efx_fini_napi_channel()  argument
1333 	if (channel->napi_dev)  in efx_fini_napi_channel()
1334 		netif_napi_del(&channel->napi_str);  in efx_fini_napi_channel()
1336 	channel->napi_dev = NULL;  in efx_fini_napi_channel()
1341 	struct efx_channel *channel;  in efx_siena_fini_napi()  local
1343 	efx_for_each_channel(channel, efx)  in efx_siena_fini_napi()
1344 		efx_fini_napi_channel(channel);  in efx_siena_fini_napi()
1351 static int efx_channel_dummy_op_int(struct efx_channel *channel)  in efx_channel_dummy_op_int()  argument
1356 void efx_siena_channel_dummy_op_void(struct efx_channel *channel)  in efx_siena_channel_dummy_op_void()  argument