89 static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,  in efx_write_buf_tbl()  argument
92 	efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,  in efx_write_buf_tbl()
103 int efx_farch_test_registers(struct efx_nic *efx,  in efx_farch_test_registers()  argument
116 		efx_reado(efx, &original, address);  in efx_farch_test_registers()
127 			efx_writeo(efx, &reg, address);  in efx_farch_test_registers()
128 			efx_reado(efx, &buf, address);  in efx_farch_test_registers()
137 			efx_writeo(efx, &reg, address);  in efx_farch_test_registers()
138 			efx_reado(efx, &buf, address);  in efx_farch_test_registers()
144 		efx_writeo(efx, &original, address);  in efx_farch_test_registers()
150 	netif_err(efx, hw, efx->net_dev,  in efx_farch_test_registers()
173 efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)  in efx_init_special_buffer()  argument
186 		netif_dbg(efx, probe, efx->net_dev,  in efx_init_special_buffer()
193 		efx_write_buf_tbl(efx, &buf_desc, index);  in efx_init_special_buffer()
199 efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)  in efx_fini_special_buffer()  argument
208 	netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n",  in efx_fini_special_buffer()
216 	efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);  in efx_fini_special_buffer()
228 static int efx_alloc_special_buffer(struct efx_nic *efx,  in efx_alloc_special_buffer()  argument
233 	struct siena_nic_data *nic_data = efx->nic_data;  in efx_alloc_special_buffer()
237 	if (efx_nic_alloc_buffer(efx, &buffer->buf, len, GFP_KERNEL))  in efx_alloc_special_buffer()
243 	buffer->index = efx->next_buffer_table;  in efx_alloc_special_buffer()
244 	efx->next_buffer_table += buffer->entries;  in efx_alloc_special_buffer()
246 	BUG_ON(efx_siena_sriov_enabled(efx) &&  in efx_alloc_special_buffer()
247 	       nic_data->vf_buftbl_base < efx->next_buffer_table);  in efx_alloc_special_buffer()
250 	netif_dbg(efx, probe, efx->net_dev,  in efx_alloc_special_buffer()
261 efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)  in efx_free_special_buffer()  argument
266 	netif_dbg(efx, hw, efx->net_dev,  in efx_free_special_buffer()
273 	efx_nic_free_buffer(efx, &buffer->buf);  in efx_free_special_buffer()
291 	efx_writed_page(tx_queue->efx, &reg,  in efx_farch_notify_tx_desc()
309 	efx_writeo_page(tx_queue->efx, &reg,  in efx_farch_push_tx_desc()
374 	struct efx_nic *efx = tx_queue->efx;  in efx_farch_tx_probe()  local
378 	return efx_alloc_special_buffer(efx, &tx_queue->txd,  in efx_farch_tx_probe()
385 	struct efx_nic *efx = tx_queue->efx;  in efx_farch_tx_init()  local
389 	efx_init_special_buffer(efx, &tx_queue->txd);  in efx_farch_tx_init()
409 	efx_writeo_table(efx, &reg, efx->type->txd_ptr_tbl_base,  in efx_farch_tx_init()
417 	efx_writeo_table(efx, &reg, FR_BZ_TX_PACE_TBL, tx_queue->queue);  in efx_farch_tx_init()
422 	struct efx_nic *efx = tx_queue->efx;  in efx_farch_flush_tx_queue()  local
431 	efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);  in efx_farch_flush_tx_queue()
436 	struct efx_nic *efx = tx_queue->efx;  in efx_farch_tx_fini()  local
441 	efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,  in efx_farch_tx_fini()
445 	efx_fini_special_buffer(efx, &tx_queue->txd);  in efx_farch_tx_fini()
451 	efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);  in efx_farch_tx_remove()
472 			     rx_queue->efx->type->rx_buffer_padding,  in efx_farch_build_rx_desc()
482 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_write()  local
496 	efx_writed_page(efx, &reg, FR_AZ_RX_DESC_UPD_DWORD_P0,  in efx_farch_rx_write()
502 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_probe()  local
506 	return efx_alloc_special_buffer(efx, &rx_queue->rxd,  in efx_farch_rx_probe()
513 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_init()  local
517 	jumbo_en = efx->rx_scatter;  in efx_farch_rx_init()
519 	netif_dbg(efx, hw, efx->net_dev,  in efx_farch_rx_init()
527 	efx_init_special_buffer(efx, &rx_queue->rxd);  in efx_farch_rx_init()
544 	efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,  in efx_farch_rx_init()
550 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_flush_rx_queue()  local
557 	efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);  in efx_farch_flush_rx_queue()
563 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_rx_fini()  local
567 	efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,  in efx_farch_rx_fini()
571 	efx_fini_special_buffer(efx, &rx_queue->rxd);  in efx_farch_rx_fini()
577 	efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);  in efx_farch_rx_remove()
589 static bool efx_farch_flush_wake(struct efx_nic *efx)  in efx_farch_flush_wake()  argument
594 	return (atomic_read(&efx->active_queues) == 0 ||  in efx_farch_flush_wake()
595 		(atomic_read(&efx->rxq_flush_outstanding) < EFX_RX_FLUSH_COUNT  in efx_farch_flush_wake()
596 		 && atomic_read(&efx->rxq_flush_pending) > 0));  in efx_farch_flush_wake()
599 static bool efx_check_tx_flush_complete(struct efx_nic *efx)  in efx_check_tx_flush_complete()  argument
606 	efx_for_each_channel(channel, efx) {  in efx_check_tx_flush_complete()
608 			efx_reado_table(efx, &txd_ptr_tbl,  in efx_check_tx_flush_complete()
614 				netif_dbg(efx, hw, efx->net_dev,  in efx_check_tx_flush_complete()
623 				netif_dbg(efx, hw, efx->net_dev,  in efx_check_tx_flush_complete()
643 static int efx_farch_do_flush(struct efx_nic *efx)  in efx_farch_do_flush()  argument
651 	efx_for_each_channel(channel, efx) {  in efx_farch_do_flush()
657 			atomic_inc(&efx->rxq_flush_pending);  in efx_farch_do_flush()
661 	while (timeout && atomic_read(&efx->active_queues) > 0) {  in efx_farch_do_flush()
666 		if (efx_siena_sriov_enabled(efx)) {  in efx_farch_do_flush()
667 			rc = efx_mcdi_flush_rxqs(efx);  in efx_farch_do_flush()
676 		efx_for_each_channel(channel, efx) {  in efx_farch_do_flush()
678 				if (atomic_read(&efx->rxq_flush_outstanding) >=  in efx_farch_do_flush()
684 					atomic_dec(&efx->rxq_flush_pending);  in efx_farch_do_flush()
685 					atomic_inc(&efx->rxq_flush_outstanding);  in efx_farch_do_flush()
692 		timeout = wait_event_timeout(efx->flush_wq,  in efx_farch_do_flush()
693 					     efx_farch_flush_wake(efx),  in efx_farch_do_flush()
697 	if (atomic_read(&efx->active_queues) &&  in efx_farch_do_flush()
698 	    !efx_check_tx_flush_complete(efx)) {  in efx_farch_do_flush()
699 		netif_err(efx, hw, efx->net_dev, "failed to flush %d queues "  in efx_farch_do_flush()
700 			  "(rx %d+%d)\n", atomic_read(&efx->active_queues),  in efx_farch_do_flush()
701 			  atomic_read(&efx->rxq_flush_outstanding),  in efx_farch_do_flush()
702 			  atomic_read(&efx->rxq_flush_pending));  in efx_farch_do_flush()
705 		atomic_set(&efx->active_queues, 0);  in efx_farch_do_flush()
706 		atomic_set(&efx->rxq_flush_pending, 0);  in efx_farch_do_flush()
707 		atomic_set(&efx->rxq_flush_outstanding, 0);  in efx_farch_do_flush()
713 int efx_farch_fini_dmaq(struct efx_nic *efx)  in efx_farch_fini_dmaq()  argument
721 	if (efx->state != STATE_RECOVERY) {  in efx_farch_fini_dmaq()
723 		if (efx->pci_dev->is_busmaster) {  in efx_farch_fini_dmaq()
724 			efx->type->prepare_flush(efx);  in efx_farch_fini_dmaq()
725 			rc = efx_farch_do_flush(efx);  in efx_farch_fini_dmaq()
726 			efx->type->finish_flush(efx);  in efx_farch_fini_dmaq()
729 		efx_for_each_channel(channel, efx) {  in efx_farch_fini_dmaq()
754 void efx_farch_finish_flr(struct efx_nic *efx)  in efx_farch_finish_flr()  argument
756 	atomic_set(&efx->rxq_flush_pending, 0);  in efx_farch_finish_flr()
757 	atomic_set(&efx->rxq_flush_outstanding, 0);  in efx_farch_finish_flr()
758 	atomic_set(&efx->active_queues, 0);  in efx_farch_finish_flr()
777 	struct efx_nic *efx = channel->efx;  in efx_farch_ev_read_ack()  local
785 	efx_writed(efx, &reg,  in efx_farch_ev_read_ack()
786 		   efx->type->evq_rptr_tbl_base +  in efx_farch_ev_read_ack()
791 void efx_farch_generate_event(struct efx_nic *efx, unsigned int evq,  in efx_farch_generate_event()  argument
803 	efx_writeo(efx, &drv_ev_reg, FR_AZ_DRV_EV);  in efx_farch_generate_event()
813 	efx_farch_generate_event(channel->efx, channel->channel, &event);  in efx_farch_magic_event()
827 	struct efx_nic *efx = channel->efx;  in efx_farch_handle_tx_event()  local
829 	if (unlikely(READ_ONCE(efx->reset_pending)))  in efx_farch_handle_tx_event()
845 		netif_tx_lock(efx->net_dev);  in efx_farch_handle_tx_event()
847 		netif_tx_unlock(efx->net_dev);  in efx_farch_handle_tx_event()
849 		efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR);  in efx_farch_handle_tx_event()
851 		netif_err(efx, tx_err, efx->net_dev,  in efx_farch_handle_tx_event()
863 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_handle_rx_not_ok()  local
896 	else if (!efx->loopback_selftest) {  in efx_farch_handle_rx_not_ok()
909 		netif_dbg(efx, rx_err, efx->net_dev,  in efx_farch_handle_rx_not_ok()
925 	if (efx->net_dev->features & NETIF_F_RXALL)  in efx_farch_handle_rx_not_ok()
943 	struct efx_nic *efx = rx_queue->efx;  in efx_farch_handle_rx_bad_index()  local
955 	netif_info(efx, rx_err, efx->net_dev,  in efx_farch_handle_rx_bad_index()
959 	efx_schedule_reset(efx, RESET_TYPE_DISABLE);  in efx_farch_handle_rx_bad_index()
979 	struct efx_nic *efx = channel->efx;  in efx_farch_handle_rx_event()  local
981 	if (unlikely(READ_ONCE(efx->reset_pending)))  in efx_farch_handle_rx_event()
1082 efx_farch_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)  in efx_farch_handle_tx_flush_done()  argument
1088 	if (qid < EFX_TXQ_TYPES * (efx->n_tx_channels + efx->n_extra_tx_channels)) {  in efx_farch_handle_tx_flush_done()
1089 		tx_queue = efx_get_tx_queue(efx, qid / EFX_TXQ_TYPES,  in efx_farch_handle_tx_flush_done()
1103 efx_farch_handle_rx_flush_done(struct efx_nic *efx, efx_qword_t *event)  in efx_farch_handle_rx_flush_done()  argument
1112 	if (qid >= efx->n_channels)  in efx_farch_handle_rx_flush_done()
1114 	channel = efx_get_channel(efx, qid);  in efx_farch_handle_rx_flush_done()
1120 		netif_info(efx, hw, efx->net_dev,  in efx_farch_handle_rx_flush_done()
1123 		atomic_inc(&efx->rxq_flush_pending);  in efx_farch_handle_rx_flush_done()
1128 	atomic_dec(&efx->rxq_flush_outstanding);  in efx_farch_handle_rx_flush_done()
1129 	if (efx_farch_flush_wake(efx))  in efx_farch_handle_rx_flush_done()
1130 		wake_up(&efx->flush_wq);  in efx_farch_handle_rx_flush_done()
1136 	struct efx_nic *efx = channel->efx;  in efx_farch_handle_drain_event()  local
1138 	WARN_ON(atomic_read(&efx->active_queues) == 0);  in efx_farch_handle_drain_event()
1139 	atomic_dec(&efx->active_queues);  in efx_farch_handle_drain_event()
1140 	if (efx_farch_flush_wake(efx))  in efx_farch_handle_drain_event()
1141 		wake_up(&efx->flush_wq);  in efx_farch_handle_drain_event()
1147 	struct efx_nic *efx = channel->efx;  in efx_farch_handle_generated_event()  local
1168 		netif_dbg(efx, hw, efx->net_dev, "channel %d received "  in efx_farch_handle_generated_event()
1177 	struct efx_nic *efx = channel->efx;  in efx_farch_handle_driver_event()  local
1186 		netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",  in efx_farch_handle_driver_event()
1188 		efx_farch_handle_tx_flush_done(efx, event);  in efx_farch_handle_driver_event()
1190 		efx_siena_sriov_tx_flush_done(efx, event);  in efx_farch_handle_driver_event()
1194 		netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",  in efx_farch_handle_driver_event()
1196 		efx_farch_handle_rx_flush_done(efx, event);  in efx_farch_handle_driver_event()
1198 		efx_siena_sriov_rx_flush_done(efx, event);  in efx_farch_handle_driver_event()
1202 		netif_dbg(efx, hw, efx->net_dev,  in efx_farch_handle_driver_event()
1207 		netif_vdbg(efx, hw, efx->net_dev,  in efx_farch_handle_driver_event()
1211 		netif_vdbg(efx, hw, efx->net_dev,  in efx_farch_handle_driver_event()
1216 		netif_vdbg(efx, hw, efx->net_dev,  in efx_farch_handle_driver_event()
1221 		netif_err(efx, rx_err, efx->net_dev,  in efx_farch_handle_driver_event()
1224 		atomic_inc(&efx->rx_reset);  in efx_farch_handle_driver_event()
1225 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);  in efx_farch_handle_driver_event()
1229 			netif_err(efx, rx_err, efx->net_dev,  in efx_farch_handle_driver_event()
1233 			efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR);  in efx_farch_handle_driver_event()
1237 			efx_siena_sriov_desc_fetch_err(efx, ev_sub_data);  in efx_farch_handle_driver_event()
1242 			netif_err(efx, tx_err, efx->net_dev,  in efx_farch_handle_driver_event()
1246 			efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR);  in efx_farch_handle_driver_event()
1250 			efx_siena_sriov_desc_fetch_err(efx, ev_sub_data);  in efx_farch_handle_driver_event()
1254 		netif_vdbg(efx, hw, efx->net_dev,  in efx_farch_handle_driver_event()
1264 	struct efx_nic *efx = channel->efx;  in efx_farch_ev_process()  local
1283 		netif_vdbg(channel->efx, intr, channel->efx->net_dev,  in efx_farch_ev_process()
1318 			if (efx->type->handle_global_event &&  in efx_farch_ev_process()
1319 			    efx->type->handle_global_event(channel, &event))  in efx_farch_ev_process()
1323 			netif_err(channel->efx, hw, channel->efx->net_dev,  in efx_farch_ev_process()
1338 	struct efx_nic *efx = channel->efx;  in efx_farch_ev_probe()  local
1342 	return efx_alloc_special_buffer(efx, &channel->eventq,  in efx_farch_ev_probe()
1349 	struct efx_nic *efx = channel->efx;  in efx_farch_ev_init()  local
1351 	netif_dbg(efx, hw, efx->net_dev,  in efx_farch_ev_init()
1360 	efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);  in efx_farch_ev_init()
1363 	efx_init_special_buffer(efx, &channel->eventq);  in efx_farch_ev_init()
1373 	efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,  in efx_farch_ev_init()
1382 	struct efx_nic *efx = channel->efx;  in efx_farch_ev_fini()  local
1386 	efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,  in efx_farch_ev_fini()
1388 	efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);  in efx_farch_ev_fini()
1391 	efx_fini_special_buffer(efx, &channel->eventq);  in efx_farch_ev_fini()
1397 	efx_free_special_buffer(channel->efx, &channel->eventq);  in efx_farch_ev_remove()
1421 static inline void efx_farch_interrupts(struct efx_nic *efx,  in efx_farch_interrupts()  argument
1427 			     FRF_AZ_KER_INT_LEVE_SEL, efx->irq_level,  in efx_farch_interrupts()
1430 	efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);  in efx_farch_interrupts()
1433 void efx_farch_irq_enable_master(struct efx_nic *efx)  in efx_farch_irq_enable_master()  argument
1435 	EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));  in efx_farch_irq_enable_master()
1438 	efx_farch_interrupts(efx, true, false);  in efx_farch_irq_enable_master()
1441 void efx_farch_irq_disable_master(struct efx_nic *efx)  in efx_farch_irq_disable_master()  argument
1444 	efx_farch_interrupts(efx, false, false);  in efx_farch_irq_disable_master()
1451 int efx_farch_irq_test_generate(struct efx_nic *efx)  in efx_farch_irq_test_generate()  argument
1453 	efx_farch_interrupts(efx, true, true);  in efx_farch_irq_test_generate()
1460 irqreturn_t efx_farch_fatal_interrupt(struct efx_nic *efx)  in efx_farch_fatal_interrupt()  argument
1462 	efx_oword_t *int_ker = efx->irq_status.addr;  in efx_farch_fatal_interrupt()
1466 	efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);  in efx_farch_fatal_interrupt()
1469 	netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status "  in efx_farch_fatal_interrupt()
1479 		efx_reado(efx, &reg, FR_AZ_MEM_STAT);  in efx_farch_fatal_interrupt()
1480 		netif_err(efx, hw, efx->net_dev,  in efx_farch_fatal_interrupt()
1486 	pci_clear_master(efx->pci_dev);  in efx_farch_fatal_interrupt()
1487 	efx_farch_irq_disable_master(efx);  in efx_farch_fatal_interrupt()
1490 	if (efx->int_error_count == 0 ||  in efx_farch_fatal_interrupt()
1491 	    time_after(jiffies, efx->int_error_expire)) {  in efx_farch_fatal_interrupt()
1492 		efx->int_error_count = 0;  in efx_farch_fatal_interrupt()
1493 		efx->int_error_expire =  in efx_farch_fatal_interrupt()
1496 	if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {  in efx_farch_fatal_interrupt()
1497 		netif_err(efx, hw, efx->net_dev,  in efx_farch_fatal_interrupt()
1499 		efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);  in efx_farch_fatal_interrupt()
1501 		netif_err(efx, hw, efx->net_dev,  in efx_farch_fatal_interrupt()
1504 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);  in efx_farch_fatal_interrupt()
1515 	struct efx_nic *efx = dev_id;  in efx_farch_legacy_interrupt()  local
1516 	bool soft_enabled = READ_ONCE(efx->irq_soft_enabled);  in efx_farch_legacy_interrupt()
1517 	efx_oword_t *int_ker = efx->irq_status.addr;  in efx_farch_legacy_interrupt()
1525 	efx_readd(efx, &reg, FR_BZ_INT_ISR0);  in efx_farch_legacy_interrupt()
1532 	if (EFX_DWORD_IS_ALL_ONES(reg) && efx_try_recovery(efx) &&  in efx_farch_legacy_interrupt()
1533 	    !efx->eeh_disabled_legacy_irq) {  in efx_farch_legacy_interrupt()
1534 		disable_irq_nosync(efx->legacy_irq);  in efx_farch_legacy_interrupt()
1535 		efx->eeh_disabled_legacy_irq = true;  in efx_farch_legacy_interrupt()
1539 	if (queues & (1U << efx->irq_level) && soft_enabled) {  in efx_farch_legacy_interrupt()
1542 			return efx_farch_fatal_interrupt(efx);  in efx_farch_legacy_interrupt()
1543 		efx->last_irq_cpu = raw_smp_processor_id();  in efx_farch_legacy_interrupt()
1547 		efx->irq_zero_count = 0;  in efx_farch_legacy_interrupt()
1551 			efx_for_each_channel(channel, efx) {  in efx_farch_legacy_interrupt()
1566 		if (efx->irq_zero_count++ == 0)  in efx_farch_legacy_interrupt()
1571 			efx_for_each_channel(channel, efx) {  in efx_farch_legacy_interrupt()
1583 		netif_vdbg(efx, intr, efx->net_dev,  in efx_farch_legacy_interrupt()
1600 	struct efx_nic *efx = context->efx;  in efx_farch_msi_interrupt()  local
1601 	efx_oword_t *int_ker = efx->irq_status.addr;  in efx_farch_msi_interrupt()
1604 	netif_vdbg(efx, intr, efx->net_dev,  in efx_farch_msi_interrupt()
1608 	if (!likely(READ_ONCE(efx->irq_soft_enabled)))  in efx_farch_msi_interrupt()
1612 	if (context->index == efx->irq_level) {  in efx_farch_msi_interrupt()
1615 			return efx_farch_fatal_interrupt(efx);  in efx_farch_msi_interrupt()
1616 		efx->last_irq_cpu = raw_smp_processor_id();  in efx_farch_msi_interrupt()
1620 	efx_schedule_channel_irq(efx->channel[context->index]);  in efx_farch_msi_interrupt()
1628 void efx_farch_rx_push_indir_table(struct efx_nic *efx)  in efx_farch_rx_push_indir_table()  argument
1633 	BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=  in efx_farch_rx_push_indir_table()
1638 				     efx->rss_context.rx_indir_table[i]);  in efx_farch_rx_push_indir_table()
1639 		efx_writed(efx, &dword,  in efx_farch_rx_push_indir_table()
1645 void efx_farch_rx_pull_indir_table(struct efx_nic *efx)  in efx_farch_rx_pull_indir_table()  argument
1650 	BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=  in efx_farch_rx_pull_indir_table()
1654 		efx_readd(efx, &dword,  in efx_farch_rx_pull_indir_table()
1657 		efx->rss_context.rx_indir_table[i] = EFX_DWORD_FIELD(dword, FRF_BZ_IT_QUEUE);  in efx_farch_rx_pull_indir_table()
1670 void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)  in efx_farch_dimension_resources()  argument
1675 	struct siena_nic_data *nic_data = efx->nic_data;  in efx_farch_dimension_resources()
1678 	total_tx_channels = efx->n_tx_channels + efx->n_extra_tx_channels;  in efx_farch_dimension_resources()
1682 	buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE +  in efx_farch_dimension_resources()
1684 		       efx->n_channels * EFX_MAX_EVQ_SIZE)  in efx_farch_dimension_resources()
1686 	vi_count = max(efx->n_channels, total_tx_channels * EFX_TXQ_TYPES);  in efx_farch_dimension_resources()
1689 	if (efx->type->sriov_wanted) {  in efx_farch_dimension_resources()
1690 		if (efx->type->sriov_wanted(efx)) {  in efx_farch_dimension_resources()
1703 					  efx_vf_size(efx));  in efx_farch_dimension_resources()
1705 				       (1024U - EFX_VI_BASE) >> efx->vi_scale);  in efx_farch_dimension_resources()
1707 			if (efx->vf_count > vf_limit) {  in efx_farch_dimension_resources()
1708 				netif_err(efx, probe, efx->net_dev,  in efx_farch_dimension_resources()
1710 					  efx->vf_count, vf_limit);  in efx_farch_dimension_resources()
1711 				efx->vf_count = vf_limit;  in efx_farch_dimension_resources()
1713 			vi_count += efx->vf_count * efx_vf_size(efx);  in efx_farch_dimension_resources()
1718 	efx->tx_dc_base = sram_lim_qw - vi_count * TX_DC_ENTRIES;  in efx_farch_dimension_resources()
1719 	efx->rx_dc_base = efx->tx_dc_base - vi_count * RX_DC_ENTRIES;  in efx_farch_dimension_resources()
1722 u32 efx_farch_fpga_ver(struct efx_nic *efx)  in efx_farch_fpga_ver()  argument
1725 	efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);  in efx_farch_fpga_ver()
1729 void efx_farch_init_common(struct efx_nic *efx)  in efx_farch_init_common()  argument
1734 	EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, efx->tx_dc_base);  in efx_farch_init_common()
1735 	efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);  in efx_farch_init_common()
1736 	EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, efx->rx_dc_base);  in efx_farch_init_common()
1737 	efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);  in efx_farch_init_common()
1742 	efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);  in efx_farch_init_common()
1749 	efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);  in efx_farch_init_common()
1751 	efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);  in efx_farch_init_common()
1756 			     EFX_INT_MODE_USE_MSI(efx),  in efx_farch_init_common()
1757 			     FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);  in efx_farch_init_common()
1758 	efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);  in efx_farch_init_common()
1760 	if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx))  in efx_farch_init_common()
1762 		efx->irq_level = 0x1f;  in efx_farch_init_common()
1765 		efx->irq_level = 0;  in efx_farch_init_common()
1779 	efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);  in efx_farch_init_common()
1784 	efx_reado(efx, &temp, FR_AZ_TX_RESERVED);  in efx_farch_init_common()
1798 	efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);  in efx_farch_init_common()
1808 	efx_writeo(efx, &temp, FR_BZ_TX_PACE);  in efx_farch_init_common()
1886 efx_farch_filter_table_clear_entry(struct efx_nic *efx,
1933 static void efx_farch_filter_push_rx_config(struct efx_nic *efx)  in efx_farch_filter_push_rx_config()  argument
1935 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_push_rx_config()
1939 	efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);  in efx_farch_filter_push_rx_config()
2001 			efx->rx_scatter);  in efx_farch_filter_push_rx_config()
2004 	efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);  in efx_farch_filter_push_rx_config()
2007 static void efx_farch_filter_push_tx_limits(struct efx_nic *efx)  in efx_farch_filter_push_tx_limits()  argument
2009 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_push_tx_limits()
2013 	efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG);  in efx_farch_filter_push_tx_limits()
2027 	efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG);  in efx_farch_filter_push_tx_limits()
2217 efx_farch_filter_init_rx_auto(struct efx_nic *efx,  in efx_farch_filter_init_rx_auto()  argument
2225 		       (efx_rss_enabled(efx) ? EFX_FILTER_FLAG_RX_RSS : 0) |  in efx_farch_filter_init_rx_auto()
2226 		       (efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0));  in efx_farch_filter_init_rx_auto()
2369 u32 efx_farch_filter_get_rx_id_limit(struct efx_nic *efx)  in efx_farch_filter_get_rx_id_limit()  argument
2371 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_get_rx_id_limit()
2385 s32 efx_farch_filter_insert(struct efx_nic *efx,  in efx_farch_filter_insert()  argument
2389 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_insert()
2409 	netif_vdbg(efx, hw, efx->net_dev,  in efx_farch_filter_insert()
2506 		efx_farch_filter_push_rx_config(efx);  in efx_farch_filter_insert()
2511 				efx_farch_filter_push_tx_limits(efx);  in efx_farch_filter_insert()
2513 				efx_farch_filter_push_rx_config(efx);  in efx_farch_filter_insert()
2516 		efx_writeo(efx, &filter,  in efx_farch_filter_insert()
2523 			efx_farch_filter_table_clear_entry(efx, table,  in efx_farch_filter_insert()
2527 	netif_vdbg(efx, hw, efx->net_dev,  in efx_farch_filter_insert()
2538 efx_farch_filter_table_clear_entry(struct efx_nic *efx,  in efx_farch_filter_table_clear_entry()  argument
2551 	efx_writeo(efx, &filter, table->offset + table->step * filter_idx);  in efx_farch_filter_table_clear_entry()
2562 			efx_farch_filter_push_tx_limits(efx);  in efx_farch_filter_table_clear_entry()
2564 			efx_farch_filter_push_rx_config(efx);  in efx_farch_filter_table_clear_entry()
2568 static int efx_farch_filter_remove(struct efx_nic *efx,  in efx_farch_filter_remove()  argument
2580 		efx_farch_filter_init_rx_auto(efx, spec);  in efx_farch_filter_remove()
2581 		efx_farch_filter_push_rx_config(efx);  in efx_farch_filter_remove()
2583 		efx_farch_filter_table_clear_entry(efx, table, filter_idx);  in efx_farch_filter_remove()
2589 int efx_farch_filter_remove_safe(struct efx_nic *efx,  in efx_farch_filter_remove_safe()  argument
2593 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_remove_safe()
2611 	rc = efx_farch_filter_remove(efx, table, filter_idx, priority);  in efx_farch_filter_remove_safe()
2617 int efx_farch_filter_get_safe(struct efx_nic *efx,  in efx_farch_filter_get_safe()  argument
2621 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_get_safe()
2652 efx_farch_filter_table_clear(struct efx_nic *efx,  in efx_farch_filter_table_clear()  argument
2656 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_table_clear()
2663 			efx_farch_filter_remove(efx, table,  in efx_farch_filter_table_clear()
2669 int efx_farch_filter_clear_rx(struct efx_nic *efx,  in efx_farch_filter_clear_rx()  argument
2672 	efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_IP,  in efx_farch_filter_clear_rx()
2674 	efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_MAC,  in efx_farch_filter_clear_rx()
2676 	efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_DEF,  in efx_farch_filter_clear_rx()
2681 u32 efx_farch_filter_count_rx_used(struct efx_nic *efx,  in efx_farch_filter_count_rx_used()  argument
2684 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_count_rx_used()
2708 s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx,  in efx_farch_filter_get_rx_ids()  argument
2712 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_get_rx_ids()
2743 void efx_farch_filter_table_restore(struct efx_nic *efx)  in efx_farch_filter_table_restore()  argument
2745 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_table_restore()
2764 			efx_writeo(efx, &filter,  in efx_farch_filter_table_restore()
2769 	efx_farch_filter_push_rx_config(efx);  in efx_farch_filter_table_restore()
2770 	efx_farch_filter_push_tx_limits(efx);  in efx_farch_filter_table_restore()
2775 void efx_farch_filter_table_remove(struct efx_nic *efx)  in efx_farch_filter_table_remove()  argument
2777 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_table_remove()
2787 int efx_farch_filter_table_probe(struct efx_nic *efx)  in efx_farch_filter_table_probe()  argument
2796 	efx->filter_state = state;  in efx_farch_filter_table_probe()
2845 			efx_farch_filter_init_rx_auto(efx, spec);  in efx_farch_filter_table_probe()
2850 	efx_farch_filter_push_rx_config(efx);  in efx_farch_filter_table_probe()
2855 	efx_farch_filter_table_remove(efx);  in efx_farch_filter_table_probe()
2860 void efx_farch_filter_update_rx_scatter(struct efx_nic *efx)  in efx_farch_filter_update_rx_scatter()  argument
2862 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_update_rx_scatter()
2878 			    efx->n_rx_channels)  in efx_farch_filter_update_rx_scatter()
2881 			if (efx->rx_scatter)  in efx_farch_filter_update_rx_scatter()
2893 			efx_writeo(efx, &filter,  in efx_farch_filter_update_rx_scatter()
2898 	efx_farch_filter_push_rx_config(efx);  in efx_farch_filter_update_rx_scatter()
2905 bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,  in efx_farch_filter_rfs_expire_one()  argument
2908 	struct efx_farch_filter_state *state = efx->filter_state;  in efx_farch_filter_rfs_expire_one()
2914 	spin_lock_bh(&efx->rps_hash_lock);  in efx_farch_filter_rfs_expire_one()
2922 		if (!efx->rps_hash_table) {  in efx_farch_filter_rfs_expire_one()
2928 			rule = efx_rps_hash_find(efx, &spec);  in efx_farch_filter_rfs_expire_one()
2938 		if (force || rps_may_expire_flow(efx->net_dev, spec.dmaq_id,  in efx_farch_filter_rfs_expire_one()
2942 			efx_rps_hash_del(efx, &spec);  in efx_farch_filter_rfs_expire_one()
2943 			efx_farch_filter_table_clear_entry(efx, table, index);  in efx_farch_filter_rfs_expire_one()
2948 	spin_unlock_bh(&efx->rps_hash_lock);  in efx_farch_filter_rfs_expire_one()
2955 void efx_farch_filter_sync_rx_mode(struct efx_nic *efx)  in efx_farch_filter_sync_rx_mode()  argument
2957 	struct net_device *net_dev = efx->net_dev;  in efx_farch_filter_sync_rx_mode()
2959 	union efx_multicast_hash *mc_hash = &efx->multicast_hash;  in efx_farch_filter_sync_rx_mode()
2963 	if (!efx_dev_registered(efx))  in efx_farch_filter_sync_rx_mode()
2968 	efx->unicast_filter = !(net_dev->flags & IFF_PROMISC);  in efx_farch_filter_sync_rx_mode()