196 static void ef4_remove_channel(struct ef4_channel *channel);
200 static void ef4_init_napi_channel(struct ef4_channel *channel);
202 static void ef4_fini_napi_channel(struct ef4_channel *channel);
238 static int ef4_process_channel(struct ef4_channel *channel, int budget)  in ef4_process_channel()  argument
243 	if (unlikely(!channel->enabled))  in ef4_process_channel()
246 	ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_process_channel()
251 	spent = ef4_nic_process_eventq(channel, budget);  in ef4_process_channel()
252 	if (spent && ef4_channel_has_rx_queue(channel)) {  in ef4_process_channel()
254 			ef4_channel_get_rx_queue(channel);  in ef4_process_channel()
256 		ef4_rx_flush_packet(channel);  in ef4_process_channel()
261 	ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_process_channel()
276 static void ef4_update_irq_mod(struct ef4_nic *efx, struct ef4_channel *channel)  in ef4_update_irq_mod()  argument
280 	if (channel->irq_mod_score < irq_adapt_low_thresh) {  in ef4_update_irq_mod()
281 		if (channel->irq_moderation_us > step) {  in ef4_update_irq_mod()
282 			channel->irq_moderation_us -= step;  in ef4_update_irq_mod()
283 			efx->type->push_irq_moderation(channel);  in ef4_update_irq_mod()
285 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {  in ef4_update_irq_mod()
286 		if (channel->irq_moderation_us <  in ef4_update_irq_mod()
288 			channel->irq_moderation_us += step;  in ef4_update_irq_mod()
289 			efx->type->push_irq_moderation(channel);  in ef4_update_irq_mod()
293 	channel->irq_count = 0;  in ef4_update_irq_mod()
294 	channel->irq_mod_score = 0;  in ef4_update_irq_mod()
299 	struct ef4_channel *channel =  in ef4_poll()  local
301 	struct ef4_nic *efx = channel->efx;  in ef4_poll()
306 		   channel->channel, raw_smp_processor_id());  in ef4_poll()
308 	spent = ef4_process_channel(channel, budget);  in ef4_poll()
311 		if (ef4_channel_has_rx_queue(channel) &&  in ef4_poll()
313 		    unlikely(++channel->irq_count == 1000)) {  in ef4_poll()
314 			ef4_update_irq_mod(efx, channel);  in ef4_poll()
317 		ef4_filter_rfs_expire(channel);  in ef4_poll()
325 		ef4_nic_eventq_read_ack(channel);  in ef4_poll()
336 static int ef4_probe_eventq(struct ef4_channel *channel)  in ef4_probe_eventq()  argument
338 	struct ef4_nic *efx = channel->efx;  in ef4_probe_eventq()
342 		  "chan %d create event queue\n", channel->channel);  in ef4_probe_eventq()
348 	channel->eventq_mask = max(entries, EF4_MIN_EVQ_SIZE) - 1;  in ef4_probe_eventq()
350 	return ef4_nic_probe_eventq(channel);  in ef4_probe_eventq()
354 static int ef4_init_eventq(struct ef4_channel *channel)  in ef4_init_eventq()  argument
356 	struct ef4_nic *efx = channel->efx;  in ef4_init_eventq()
359 	EF4_WARN_ON_PARANOID(channel->eventq_init);  in ef4_init_eventq()
362 		  "chan %d init event queue\n", channel->channel);  in ef4_init_eventq()
364 	rc = ef4_nic_init_eventq(channel);  in ef4_init_eventq()
366 		efx->type->push_irq_moderation(channel);  in ef4_init_eventq()
367 		channel->eventq_read_ptr = 0;  in ef4_init_eventq()
368 		channel->eventq_init = true;  in ef4_init_eventq()
374 void ef4_start_eventq(struct ef4_channel *channel)  in ef4_start_eventq()  argument
376 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,  in ef4_start_eventq()
377 		  "chan %d start event queue\n", channel->channel);  in ef4_start_eventq()
380 	channel->enabled = true;  in ef4_start_eventq()
383 	napi_enable(&channel->napi_str);  in ef4_start_eventq()
384 	ef4_nic_eventq_read_ack(channel);  in ef4_start_eventq()
388 void ef4_stop_eventq(struct ef4_channel *channel)  in ef4_stop_eventq()  argument
390 	if (!channel->enabled)  in ef4_stop_eventq()
393 	napi_disable(&channel->napi_str);  in ef4_stop_eventq()
394 	channel->enabled = false;  in ef4_stop_eventq()
397 static void ef4_fini_eventq(struct ef4_channel *channel)  in ef4_fini_eventq()  argument
399 	if (!channel->eventq_init)  in ef4_fini_eventq()
402 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in ef4_fini_eventq()
403 		  "chan %d fini event queue\n", channel->channel);  in ef4_fini_eventq()
405 	ef4_nic_fini_eventq(channel);  in ef4_fini_eventq()
406 	channel->eventq_init = false;  in ef4_fini_eventq()
409 static void ef4_remove_eventq(struct ef4_channel *channel)  in ef4_remove_eventq()  argument
411 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in ef4_remove_eventq()
412 		  "chan %d remove event queue\n", channel->channel);  in ef4_remove_eventq()
414 	ef4_nic_remove_eventq(channel);  in ef4_remove_eventq()
427 	struct ef4_channel *channel;  in ef4_alloc_channel()  local
432 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);  in ef4_alloc_channel()
433 	if (!channel)  in ef4_alloc_channel()
436 	channel->efx = efx;  in ef4_alloc_channel()
437 	channel->channel = i;  in ef4_alloc_channel()
438 	channel->type = &ef4_default_channel_type;  in ef4_alloc_channel()
441 		tx_queue = &channel->tx_queue[j];  in ef4_alloc_channel()
444 		tx_queue->channel = channel;  in ef4_alloc_channel()
447 	rx_queue = &channel->rx_queue;  in ef4_alloc_channel()
451 	return channel;  in ef4_alloc_channel()
460 	struct ef4_channel *channel;  in ef4_copy_channel()  local
465 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);  in ef4_copy_channel()
466 	if (!channel)  in ef4_copy_channel()
469 	*channel = *old_channel;  in ef4_copy_channel()
471 	channel->napi_dev = NULL;  in ef4_copy_channel()
472 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);  in ef4_copy_channel()
473 	channel->napi_str.napi_id = 0;  in ef4_copy_channel()
474 	channel->napi_str.state = 0;  in ef4_copy_channel()
475 	memset(&channel->eventq, 0, sizeof(channel->eventq));  in ef4_copy_channel()
478 		tx_queue = &channel->tx_queue[j];  in ef4_copy_channel()
479 		if (tx_queue->channel)  in ef4_copy_channel()
480 			tx_queue->channel = channel;  in ef4_copy_channel()
485 	rx_queue = &channel->rx_queue;  in ef4_copy_channel()
490 	return channel;  in ef4_copy_channel()
493 static int ef4_probe_channel(struct ef4_channel *channel)  in ef4_probe_channel()  argument
499 	netif_dbg(channel->efx, probe, channel->efx->net_dev,  in ef4_probe_channel()
500 		  "creating channel %d\n", channel->channel);  in ef4_probe_channel()
502 	rc = channel->type->pre_probe(channel);  in ef4_probe_channel()
506 	rc = ef4_probe_eventq(channel);  in ef4_probe_channel()
510 	ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_probe_channel()
516 	ef4_for_each_channel_rx_queue(rx_queue, channel) {  in ef4_probe_channel()
525 	ef4_remove_channel(channel);  in ef4_probe_channel()
530 ef4_get_channel_name(struct ef4_channel *channel, char *buf, size_t len)  in ef4_get_channel_name()  argument
532 	struct ef4_nic *efx = channel->efx;  in ef4_get_channel_name()
536 	number = channel->channel;  in ef4_get_channel_name()
539 	} else if (channel->channel < efx->tx_channel_offset) {  in ef4_get_channel_name()
550 	struct ef4_channel *channel;  in ef4_set_channel_names()  local
552 	ef4_for_each_channel(channel, efx)  in ef4_set_channel_names()
553 		channel->type->get_name(channel,  in ef4_set_channel_names()
554 					efx->msi_context[channel->channel].name,  in ef4_set_channel_names()
560 	struct ef4_channel *channel;  in ef4_probe_channels()  local
571 	ef4_for_each_channel_rev(channel, efx) {  in ef4_probe_channels()
572 		rc = ef4_probe_channel(channel);  in ef4_probe_channels()
576 				  channel->channel);  in ef4_probe_channels()
599 	struct ef4_channel *channel;  in ef4_start_datapath()  local
664 	ef4_for_each_channel(channel, efx) {  in ef4_start_datapath()
665 		ef4_for_each_channel_tx_queue(tx_queue, channel) {  in ef4_start_datapath()
670 		ef4_for_each_channel_rx_queue(rx_queue, channel) {  in ef4_start_datapath()
673 			ef4_stop_eventq(channel);  in ef4_start_datapath()
675 			ef4_start_eventq(channel);  in ef4_start_datapath()
678 		WARN_ON(channel->rx_pkt_n_frags);  in ef4_start_datapath()
687 	struct ef4_channel *channel;  in ef4_stop_datapath()  local
696 	ef4_for_each_channel(channel, efx) {  in ef4_stop_datapath()
697 		ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_stop_datapath()
701 	ef4_for_each_channel(channel, efx) {  in ef4_stop_datapath()
708 		if (ef4_channel_has_rx_queue(channel)) {  in ef4_stop_datapath()
709 			ef4_stop_eventq(channel);  in ef4_stop_datapath()
710 			ef4_start_eventq(channel);  in ef4_stop_datapath()
731 	ef4_for_each_channel(channel, efx) {  in ef4_stop_datapath()
732 		ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_stop_datapath()
734 		ef4_for_each_possible_channel_tx_queue(tx_queue, channel)  in ef4_stop_datapath()
739 static void ef4_remove_channel(struct ef4_channel *channel)  in ef4_remove_channel()  argument
744 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in ef4_remove_channel()
745 		  "destroy chan %d\n", channel->channel);  in ef4_remove_channel()
747 	ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_remove_channel()
749 	ef4_for_each_possible_channel_tx_queue(tx_queue, channel)  in ef4_remove_channel()
751 	ef4_remove_eventq(channel);  in ef4_remove_channel()
752 	channel->type->post_remove(channel);  in ef4_remove_channel()
757 	struct ef4_channel *channel;  in ef4_remove_channels()  local
759 	ef4_for_each_channel(channel, efx)  in ef4_remove_channels()
760 		ef4_remove_channel(channel);  in ef4_remove_channels()
766 	struct ef4_channel *other_channel[EF4_MAX_CHANNELS], *channel;  in ef4_realloc_channels()  local
778 	ef4_for_each_channel(channel, efx) {  in ef4_realloc_channels()
782 		if (channel->type->copy)  in ef4_realloc_channels()
785 					channel->eventq.index +  in ef4_realloc_channels()
786 					channel->eventq.entries);  in ef4_realloc_channels()
787 		ef4_for_each_channel_rx_queue(rx_queue, channel)  in ef4_realloc_channels()
791 		ef4_for_each_channel_tx_queue(tx_queue, channel)  in ef4_realloc_channels()
804 		channel = efx->channel[i];  in ef4_realloc_channels()
805 		if (channel->type->copy)  in ef4_realloc_channels()
806 			channel = channel->type->copy(channel);  in ef4_realloc_channels()
807 		if (!channel) {  in ef4_realloc_channels()
811 		other_channel[i] = channel;  in ef4_realloc_channels()
820 		channel = efx->channel[i];  in ef4_realloc_channels()
821 		efx->channel[i] = other_channel[i];  in ef4_realloc_channels()
822 		other_channel[i] = channel;  in ef4_realloc_channels()
829 		channel = efx->channel[i];  in ef4_realloc_channels()
830 		if (!channel->type->copy)  in ef4_realloc_channels()
832 		rc = ef4_probe_channel(channel);  in ef4_realloc_channels()
835 		ef4_init_napi_channel(efx->channel[i]);  in ef4_realloc_channels()
841 		channel = other_channel[i];  in ef4_realloc_channels()
842 		if (channel && channel->type->copy) {  in ef4_realloc_channels()
843 			ef4_fini_napi_channel(channel);  in ef4_realloc_channels()
844 			ef4_remove_channel(channel);  in ef4_realloc_channels()
845 			kfree(channel);  in ef4_realloc_channels()
866 		channel = efx->channel[i];  in ef4_realloc_channels()
867 		efx->channel[i] = other_channel[i];  in ef4_realloc_channels()
868 		other_channel[i] = channel;  in ef4_realloc_channels()
886 int ef4_channel_dummy_op_int(struct ef4_channel *channel)  in ef4_channel_dummy_op_int()  argument
891 void ef4_channel_dummy_op_void(struct ef4_channel *channel)  in ef4_channel_dummy_op_void()  argument
1469 	struct ef4_channel *channel, *end_channel;  in ef4_soft_enable_interrupts()  local
1477 	ef4_for_each_channel(channel, efx) {  in ef4_soft_enable_interrupts()
1478 		if (!channel->type->keep_eventq) {  in ef4_soft_enable_interrupts()
1479 			rc = ef4_init_eventq(channel);  in ef4_soft_enable_interrupts()
1483 		ef4_start_eventq(channel);  in ef4_soft_enable_interrupts()
1488 	end_channel = channel;  in ef4_soft_enable_interrupts()
1489 	ef4_for_each_channel(channel, efx) {  in ef4_soft_enable_interrupts()
1490 		if (channel == end_channel)  in ef4_soft_enable_interrupts()
1492 		ef4_stop_eventq(channel);  in ef4_soft_enable_interrupts()
1493 		if (!channel->type->keep_eventq)  in ef4_soft_enable_interrupts()
1494 			ef4_fini_eventq(channel);  in ef4_soft_enable_interrupts()
1502 	struct ef4_channel *channel;  in ef4_soft_disable_interrupts()  local
1513 	ef4_for_each_channel(channel, efx) {  in ef4_soft_disable_interrupts()
1514 		if (channel->irq)  in ef4_soft_disable_interrupts()
1515 			synchronize_irq(channel->irq);  in ef4_soft_disable_interrupts()
1517 		ef4_stop_eventq(channel);  in ef4_soft_disable_interrupts()
1518 		if (!channel->type->keep_eventq)  in ef4_soft_disable_interrupts()
1519 			ef4_fini_eventq(channel);  in ef4_soft_disable_interrupts()
1525 	struct ef4_channel *channel, *end_channel;  in ef4_enable_interrupts()  local
1537 	ef4_for_each_channel(channel, efx) {  in ef4_enable_interrupts()
1538 		if (channel->type->keep_eventq) {  in ef4_enable_interrupts()
1539 			rc = ef4_init_eventq(channel);  in ef4_enable_interrupts()
1552 	end_channel = channel;  in ef4_enable_interrupts()
1553 	ef4_for_each_channel(channel, efx) {  in ef4_enable_interrupts()
1554 		if (channel == end_channel)  in ef4_enable_interrupts()
1556 		if (channel->type->keep_eventq)  in ef4_enable_interrupts()
1557 			ef4_fini_eventq(channel);  in ef4_enable_interrupts()
1567 	struct ef4_channel *channel;  in ef4_disable_interrupts()  local
1571 	ef4_for_each_channel(channel, efx) {  in ef4_disable_interrupts()
1572 		if (channel->type->keep_eventq)  in ef4_disable_interrupts()
1573 			ef4_fini_eventq(channel);  in ef4_disable_interrupts()
1581 	struct ef4_channel *channel;  in ef4_remove_interrupts()  local
1584 	ef4_for_each_channel(channel, efx)  in ef4_remove_interrupts()
1585 		channel->irq = 0;  in ef4_remove_interrupts()
1595 	struct ef4_channel *channel;  in ef4_set_channels()  local
1606 	ef4_for_each_channel(channel, efx) {  in ef4_set_channels()
1607 		if (channel->channel < efx->n_rx_channels)  in ef4_set_channels()
1608 			channel->rx_queue.core_index = channel->channel;  in ef4_set_channels()
1610 			channel->rx_queue.core_index = -1;  in ef4_set_channels()
1612 		ef4_for_each_channel_tx_queue(tx_queue, channel)  in ef4_set_channels()
1702 		struct ef4_channel *channel;  in ef4_probe_filters()  local
1705 		ef4_for_each_channel(channel, efx) {  in ef4_probe_filters()
1706 			channel->rps_flow_id =  in ef4_probe_filters()
1708 					sizeof(*channel->rps_flow_id),  in ef4_probe_filters()
1710 			if (!channel->rps_flow_id)  in ef4_probe_filters()
1716 					channel->rps_flow_id[i] =  in ef4_probe_filters()
1721 			ef4_for_each_channel(channel, efx)  in ef4_probe_filters()
1722 				kfree(channel->rps_flow_id);  in ef4_probe_filters()
1740 	struct ef4_channel *channel;  in ef4_remove_filters()  local
1742 	ef4_for_each_channel(channel, efx)  in ef4_remove_filters()
1743 		kfree(channel->rps_flow_id);  in ef4_remove_filters()
1912 	struct ef4_channel *channel;  in ef4_init_irq_moderation()  local
1931 	ef4_for_each_channel(channel, efx) {  in ef4_init_irq_moderation()
1932 		if (ef4_channel_has_rx_queue(channel))  in ef4_init_irq_moderation()
1933 			channel->irq_moderation_us = rx_usecs;  in ef4_init_irq_moderation()
1934 		else if (ef4_channel_has_tx_queues(channel))  in ef4_init_irq_moderation()
1935 			channel->irq_moderation_us = tx_usecs;  in ef4_init_irq_moderation()
1956 		tx_channel = efx->channel[efx->tx_channel_offset];  in ef4_get_irq_moderation()
2019 static void ef4_init_napi_channel(struct ef4_channel *channel)  in ef4_init_napi_channel()  argument
2021 	struct ef4_nic *efx = channel->efx;  in ef4_init_napi_channel()
2023 	channel->napi_dev = efx->net_dev;  in ef4_init_napi_channel()
2024 	netif_napi_add(channel->napi_dev, &channel->napi_str,  in ef4_init_napi_channel()
2030 	struct ef4_channel *channel;  in ef4_init_napi()  local
2032 	ef4_for_each_channel(channel, efx)  in ef4_init_napi()
2033 		ef4_init_napi_channel(channel);  in ef4_init_napi()
2036 static void ef4_fini_napi_channel(struct ef4_channel *channel)  in ef4_fini_napi_channel()  argument
2038 	if (channel->napi_dev)  in ef4_fini_napi_channel()
2039 		netif_napi_del(&channel->napi_str);  in ef4_fini_napi_channel()
2041 	channel->napi_dev = NULL;  in ef4_fini_napi_channel()
2046 	struct ef4_channel *channel;  in ef4_fini_napi()  local
2048 	ef4_for_each_channel(channel, efx)  in ef4_fini_napi()
2049 		ef4_fini_napi_channel(channel);  in ef4_fini_napi()
2267 	struct ef4_channel *channel;  in ef4_register_netdev()  local
2305 	ef4_for_each_channel(channel, efx) {  in ef4_register_netdev()
2307 		ef4_for_each_channel_tx_queue(tx_queue, channel)  in ef4_register_netdev()
2671 		efx->channel[i] = ef4_alloc_channel(efx, i, NULL);  in ef4_init_struct()
2672 		if (!efx->channel[i])  in ef4_init_struct()
2701 		kfree(efx->channel[i]);  in ef4_fini_struct()
2714 	struct ef4_channel *channel;  in ef4_update_sw_stats()  local
2716 	ef4_for_each_channel(channel, efx)  in ef4_update_sw_stats()
2717 		n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;  in ef4_update_sw_stats()