83 static void igbvf_receive_skb(struct igbvf_adapter *adapter,  in igbvf_receive_skb()  argument
91 		if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&  in igbvf_receive_skb()
96 		if (test_bit(vid, adapter->active_vlans))  in igbvf_receive_skb()
100 	napi_gro_receive(&adapter->rx_ring->napi, skb);  in igbvf_receive_skb()
103 static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,  in igbvf_rx_checksum_adv()  argument
110 	    (adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))  in igbvf_rx_checksum_adv()
117 		adapter->hw_csum_err++;  in igbvf_rx_checksum_adv()
125 	adapter->hw_csum_good++;  in igbvf_rx_checksum_adv()
136 	struct igbvf_adapter *adapter = rx_ring->adapter;  in igbvf_alloc_rx_buffers()  local
137 	struct net_device *netdev = adapter->netdev;  in igbvf_alloc_rx_buffers()
138 	struct pci_dev *pdev = adapter->pdev;  in igbvf_alloc_rx_buffers()
148 	if (adapter->rx_ps_hdr_size)  in igbvf_alloc_rx_buffers()
149 		bufsz = adapter->rx_ps_hdr_size;  in igbvf_alloc_rx_buffers()
151 		bufsz = adapter->rx_buffer_len;  in igbvf_alloc_rx_buffers()
156 		if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {  in igbvf_alloc_rx_buffers()
160 					adapter->alloc_rx_buff_failed++;  in igbvf_alloc_rx_buffers()
184 				adapter->alloc_rx_buff_failed++;  in igbvf_alloc_rx_buffers()
202 		if (adapter->rx_ps_hdr_size) {  in igbvf_alloc_rx_buffers()
231 		writel(i, adapter->hw.hw_addr + rx_ring->tail);  in igbvf_alloc_rx_buffers()
244 static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,  in igbvf_clean_rx_irq()  argument
247 	struct igbvf_ring *rx_ring = adapter->rx_ring;  in igbvf_clean_rx_irq()
248 	struct net_device *netdev = adapter->netdev;  in igbvf_clean_rx_irq()
249 	struct pci_dev *pdev = adapter->pdev;  in igbvf_clean_rx_irq()
279 		if (hlen > adapter->rx_ps_hdr_size)  in igbvf_clean_rx_irq()
280 			hlen = adapter->rx_ps_hdr_size;  in igbvf_clean_rx_irq()
289 		if (!adapter->rx_ps_hdr_size) {  in igbvf_clean_rx_irq()
291 					 adapter->rx_buffer_len,  in igbvf_clean_rx_irq()
300 					 adapter->rx_ps_hdr_size,  in igbvf_clean_rx_irq()
317 			if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||  in igbvf_clean_rx_irq()
351 		igbvf_rx_checksum_adv(adapter, staterr, skb);  in igbvf_clean_rx_irq()
355 		igbvf_receive_skb(adapter, netdev, skb, staterr,  in igbvf_clean_rx_irq()
380 	adapter->total_rx_packets += total_packets;  in igbvf_clean_rx_irq()
381 	adapter->total_rx_bytes += total_bytes;  in igbvf_clean_rx_irq()
387 static void igbvf_put_txbuf(struct igbvf_adapter *adapter,  in igbvf_put_txbuf()  argument
392 			dma_unmap_page(&adapter->pdev->dev,  in igbvf_put_txbuf()
397 			dma_unmap_single(&adapter->pdev->dev,  in igbvf_put_txbuf()
417 int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,  in igbvf_setup_tx_resources()  argument
420 	struct pci_dev *pdev = adapter->pdev;  in igbvf_setup_tx_resources()
437 	tx_ring->adapter = adapter;  in igbvf_setup_tx_resources()
444 	dev_err(&adapter->pdev->dev,  in igbvf_setup_tx_resources()
456 int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,  in igbvf_setup_rx_resources()  argument
459 	struct pci_dev *pdev = adapter->pdev;  in igbvf_setup_rx_resources()
481 	rx_ring->adapter = adapter;  in igbvf_setup_rx_resources()
488 	dev_err(&adapter->pdev->dev,  in igbvf_setup_rx_resources()
499 	struct igbvf_adapter *adapter = tx_ring->adapter;  in igbvf_clean_tx_ring()  local
510 		igbvf_put_txbuf(adapter, buffer_info);  in igbvf_clean_tx_ring()
522 	writel(0, adapter->hw.hw_addr + tx_ring->head);  in igbvf_clean_tx_ring()
523 	writel(0, adapter->hw.hw_addr + tx_ring->tail);  in igbvf_clean_tx_ring()
534 	struct pci_dev *pdev = tx_ring->adapter->pdev;  in igbvf_free_tx_resources()
553 	struct igbvf_adapter *adapter = rx_ring->adapter;  in igbvf_clean_rx_ring()  local
555 	struct pci_dev *pdev = adapter->pdev;  in igbvf_clean_rx_ring()
566 			if (adapter->rx_ps_hdr_size) {  in igbvf_clean_rx_ring()
568 						 adapter->rx_ps_hdr_size,  in igbvf_clean_rx_ring()
572 						 adapter->rx_buffer_len,  in igbvf_clean_rx_ring()
605 	writel(0, adapter->hw.hw_addr + rx_ring->head);  in igbvf_clean_rx_ring()
606 	writel(0, adapter->hw.hw_addr + rx_ring->tail);  in igbvf_clean_rx_ring()
618 	struct pci_dev *pdev = rx_ring->adapter->pdev;  in igbvf_free_rx_resources()
644 static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,  in igbvf_update_itr()  argument
714 static void igbvf_set_itr(struct igbvf_adapter *adapter)  in igbvf_set_itr()  argument
718 	adapter->tx_ring->itr_range =  in igbvf_set_itr()
719 			igbvf_update_itr(adapter,  in igbvf_set_itr()
720 					 adapter->tx_ring->itr_val,  in igbvf_set_itr()
721 					 adapter->total_tx_packets,  in igbvf_set_itr()
722 					 adapter->total_tx_bytes);  in igbvf_set_itr()
725 	if (adapter->requested_itr == 3 &&  in igbvf_set_itr()
726 	    adapter->tx_ring->itr_range == lowest_latency)  in igbvf_set_itr()
727 		adapter->tx_ring->itr_range = low_latency;  in igbvf_set_itr()
729 	new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);  in igbvf_set_itr()
731 	if (new_itr != adapter->tx_ring->itr_val) {  in igbvf_set_itr()
732 		u32 current_itr = adapter->tx_ring->itr_val;  in igbvf_set_itr()
740 		adapter->tx_ring->itr_val = new_itr;  in igbvf_set_itr()
742 		adapter->tx_ring->set_itr = 1;  in igbvf_set_itr()
745 	adapter->rx_ring->itr_range =  in igbvf_set_itr()
746 			igbvf_update_itr(adapter, adapter->rx_ring->itr_val,  in igbvf_set_itr()
747 					 adapter->total_rx_packets,  in igbvf_set_itr()
748 					 adapter->total_rx_bytes);  in igbvf_set_itr()
749 	if (adapter->requested_itr == 3 &&  in igbvf_set_itr()
750 	    adapter->rx_ring->itr_range == lowest_latency)  in igbvf_set_itr()
751 		adapter->rx_ring->itr_range = low_latency;  in igbvf_set_itr()
753 	new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);  in igbvf_set_itr()
755 	if (new_itr != adapter->rx_ring->itr_val) {  in igbvf_set_itr()
756 		u32 current_itr = adapter->rx_ring->itr_val;  in igbvf_set_itr()
761 		adapter->rx_ring->itr_val = new_itr;  in igbvf_set_itr()
763 		adapter->rx_ring->set_itr = 1;  in igbvf_set_itr()
775 	struct igbvf_adapter *adapter = tx_ring->adapter;  in igbvf_clean_tx_irq()  local
776 	struct net_device *netdev = adapter->netdev;  in igbvf_clean_tx_irq()
820 			igbvf_put_txbuf(adapter, buffer_info);  in igbvf_clean_tx_irq()
842 		    !(test_bit(__IGBVF_DOWN, &adapter->state))) {  in igbvf_clean_tx_irq()
844 			++adapter->restart_queue;  in igbvf_clean_tx_irq()
856 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_msix_other()  local
857 	struct e1000_hw *hw = &adapter->hw;  in igbvf_msix_other()
859 	adapter->int_counter1++;  in igbvf_msix_other()
862 	if (!test_bit(__IGBVF_DOWN, &adapter->state))  in igbvf_msix_other()
863 		mod_timer(&adapter->watchdog_timer, jiffies + 1);  in igbvf_msix_other()
865 	ew32(EIMS, adapter->eims_other);  in igbvf_msix_other()
873 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_intr_msix_tx()  local
874 	struct e1000_hw *hw = &adapter->hw;  in igbvf_intr_msix_tx()
875 	struct igbvf_ring *tx_ring = adapter->tx_ring;  in igbvf_intr_msix_tx()
879 		       adapter->hw.hw_addr + tx_ring->itr_register);  in igbvf_intr_msix_tx()
880 		adapter->tx_ring->set_itr = 0;  in igbvf_intr_msix_tx()
883 	adapter->total_tx_bytes = 0;  in igbvf_intr_msix_tx()
884 	adapter->total_tx_packets = 0;  in igbvf_intr_msix_tx()
901 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_intr_msix_rx()  local
903 	adapter->int_counter0++;  in igbvf_intr_msix_rx()
908 	if (adapter->rx_ring->set_itr) {  in igbvf_intr_msix_rx()
909 		writel(adapter->rx_ring->itr_val,  in igbvf_intr_msix_rx()
910 		       adapter->hw.hw_addr + adapter->rx_ring->itr_register);  in igbvf_intr_msix_rx()
911 		adapter->rx_ring->set_itr = 0;  in igbvf_intr_msix_rx()
914 	if (napi_schedule_prep(&adapter->rx_ring->napi)) {  in igbvf_intr_msix_rx()
915 		adapter->total_rx_bytes = 0;  in igbvf_intr_msix_rx()
916 		adapter->total_rx_packets = 0;  in igbvf_intr_msix_rx()
917 		__napi_schedule(&adapter->rx_ring->napi);  in igbvf_intr_msix_rx()
925 static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,  in igbvf_assign_vector()  argument
928 	struct e1000_hw *hw = &adapter->hw;  in igbvf_assign_vector()
948 		adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);  in igbvf_assign_vector()
963 		adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);  in igbvf_assign_vector()
975 static void igbvf_configure_msix(struct igbvf_adapter *adapter)  in igbvf_configure_msix()  argument
978 	struct e1000_hw *hw = &adapter->hw;  in igbvf_configure_msix()
979 	struct igbvf_ring *tx_ring = adapter->tx_ring;  in igbvf_configure_msix()
980 	struct igbvf_ring *rx_ring = adapter->rx_ring;  in igbvf_configure_msix()
983 	adapter->eims_enable_mask = 0;  in igbvf_configure_msix()
985 	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);  in igbvf_configure_msix()
986 	adapter->eims_enable_mask |= tx_ring->eims_value;  in igbvf_configure_msix()
988 	igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);  in igbvf_configure_msix()
989 	adapter->eims_enable_mask |= rx_ring->eims_value;  in igbvf_configure_msix()
998 	adapter->eims_enable_mask = GENMASK(vector - 1, 0);  in igbvf_configure_msix()
999 	adapter->eims_other = BIT(vector - 1);  in igbvf_configure_msix()
1003 static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)  in igbvf_reset_interrupt_capability()  argument
1005 	if (adapter->msix_entries) {  in igbvf_reset_interrupt_capability()
1006 		pci_disable_msix(adapter->pdev);  in igbvf_reset_interrupt_capability()
1007 		kfree(adapter->msix_entries);  in igbvf_reset_interrupt_capability()
1008 		adapter->msix_entries = NULL;  in igbvf_reset_interrupt_capability()
1019 static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)  in igbvf_set_interrupt_capability()  argument
1025 	adapter->msix_entries = kcalloc(3, sizeof(struct msix_entry),  in igbvf_set_interrupt_capability()
1027 	if (adapter->msix_entries) {  in igbvf_set_interrupt_capability()
1029 			adapter->msix_entries[i].entry = i;  in igbvf_set_interrupt_capability()
1031 		err = pci_enable_msix_range(adapter->pdev,  in igbvf_set_interrupt_capability()
1032 					    adapter->msix_entries, 3, 3);  in igbvf_set_interrupt_capability()
1037 		dev_err(&adapter->pdev->dev,  in igbvf_set_interrupt_capability()
1039 		igbvf_reset_interrupt_capability(adapter);  in igbvf_set_interrupt_capability()
1050 static int igbvf_request_msix(struct igbvf_adapter *adapter)  in igbvf_request_msix()  argument
1052 	struct net_device *netdev = adapter->netdev;  in igbvf_request_msix()
1056 		sprintf(adapter->tx_ring->name, "%s-tx-0", netdev->name);  in igbvf_request_msix()
1057 		sprintf(adapter->rx_ring->name, "%s-rx-0", netdev->name);  in igbvf_request_msix()
1059 		memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);  in igbvf_request_msix()
1060 		memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);  in igbvf_request_msix()
1063 	err = request_irq(adapter->msix_entries[vector].vector,  in igbvf_request_msix()
1064 			  igbvf_intr_msix_tx, 0, adapter->tx_ring->name,  in igbvf_request_msix()
1069 	adapter->tx_ring->itr_register = E1000_EITR(vector);  in igbvf_request_msix()
1070 	adapter->tx_ring->itr_val = adapter->current_itr;  in igbvf_request_msix()
1073 	err = request_irq(adapter->msix_entries[vector].vector,  in igbvf_request_msix()
1074 			  igbvf_intr_msix_rx, 0, adapter->rx_ring->name,  in igbvf_request_msix()
1079 	adapter->rx_ring->itr_register = E1000_EITR(vector);  in igbvf_request_msix()
1080 	adapter->rx_ring->itr_val = adapter->current_itr;  in igbvf_request_msix()
1083 	err = request_irq(adapter->msix_entries[vector].vector,  in igbvf_request_msix()
1088 	igbvf_configure_msix(adapter);  in igbvf_request_msix()
1091 	free_irq(adapter->msix_entries[--vector].vector, netdev);  in igbvf_request_msix()
1093 	free_irq(adapter->msix_entries[--vector].vector, netdev);  in igbvf_request_msix()
1102 static int igbvf_alloc_queues(struct igbvf_adapter *adapter)  in igbvf_alloc_queues()  argument
1104 	struct net_device *netdev = adapter->netdev;  in igbvf_alloc_queues()
1106 	adapter->tx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);  in igbvf_alloc_queues()
1107 	if (!adapter->tx_ring)  in igbvf_alloc_queues()
1110 	adapter->rx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);  in igbvf_alloc_queues()
1111 	if (!adapter->rx_ring) {  in igbvf_alloc_queues()
1112 		kfree(adapter->tx_ring);  in igbvf_alloc_queues()
1116 	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);  in igbvf_alloc_queues()
1128 static int igbvf_request_irq(struct igbvf_adapter *adapter)  in igbvf_request_irq()  argument
1133 	if (adapter->msix_entries)  in igbvf_request_irq()
1134 		err = igbvf_request_msix(adapter);  in igbvf_request_irq()
1139 	dev_err(&adapter->pdev->dev,  in igbvf_request_irq()
1145 static void igbvf_free_irq(struct igbvf_adapter *adapter)  in igbvf_free_irq()  argument
1147 	struct net_device *netdev = adapter->netdev;  in igbvf_free_irq()
1150 	if (adapter->msix_entries) {  in igbvf_free_irq()
1152 			free_irq(adapter->msix_entries[vector].vector, netdev);  in igbvf_free_irq()
1160 static void igbvf_irq_disable(struct igbvf_adapter *adapter)  in igbvf_irq_disable()  argument
1162 	struct e1000_hw *hw = &adapter->hw;  in igbvf_irq_disable()
1166 	if (adapter->msix_entries)  in igbvf_irq_disable()
1174 static void igbvf_irq_enable(struct igbvf_adapter *adapter)  in igbvf_irq_enable()  argument
1176 	struct e1000_hw *hw = &adapter->hw;  in igbvf_irq_enable()
1178 	ew32(EIAC, adapter->eims_enable_mask);  in igbvf_irq_enable()
1179 	ew32(EIAM, adapter->eims_enable_mask);  in igbvf_irq_enable()
1180 	ew32(EIMS, adapter->eims_enable_mask);  in igbvf_irq_enable()
1191 	struct igbvf_adapter *adapter = rx_ring->adapter;  in igbvf_poll()  local
1192 	struct e1000_hw *hw = &adapter->hw;  in igbvf_poll()
1195 	igbvf_clean_rx_irq(adapter, &work_done, budget);  in igbvf_poll()
1204 		if (adapter->requested_itr & 3)  in igbvf_poll()
1205 			igbvf_set_itr(adapter);  in igbvf_poll()
1207 		if (!test_bit(__IGBVF_DOWN, &adapter->state))  in igbvf_poll()
1208 			ew32(EIMS, adapter->rx_ring->eims_value);  in igbvf_poll()
1220 static void igbvf_set_rlpml(struct igbvf_adapter *adapter)  in igbvf_set_rlpml()  argument
1223 	struct e1000_hw *hw = &adapter->hw;  in igbvf_set_rlpml()
1225 	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;  in igbvf_set_rlpml()
1237 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_vlan_rx_add_vid()  local
1238 	struct e1000_hw *hw = &adapter->hw;  in igbvf_vlan_rx_add_vid()
1243 		dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid);  in igbvf_vlan_rx_add_vid()
1250 	set_bit(vid, adapter->active_vlans);  in igbvf_vlan_rx_add_vid()
1257 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_vlan_rx_kill_vid()  local
1258 	struct e1000_hw *hw = &adapter->hw;  in igbvf_vlan_rx_kill_vid()
1263 		dev_err(&adapter->pdev->dev,  in igbvf_vlan_rx_kill_vid()
1271 	clear_bit(vid, adapter->active_vlans);  in igbvf_vlan_rx_kill_vid()
1275 static void igbvf_restore_vlan(struct igbvf_adapter *adapter)  in igbvf_restore_vlan()  argument
1279 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)  in igbvf_restore_vlan()
1280 		igbvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);  in igbvf_restore_vlan()
1289 static void igbvf_configure_tx(struct igbvf_adapter *adapter)  in igbvf_configure_tx()  argument
1291 	struct e1000_hw *hw = &adapter->hw;  in igbvf_configure_tx()
1292 	struct igbvf_ring *tx_ring = adapter->tx_ring;  in igbvf_configure_tx()
1325 	adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;  in igbvf_configure_tx()
1328 	adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;  in igbvf_configure_tx()
1335 static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)  in igbvf_setup_srrctl()  argument
1337 	struct e1000_hw *hw = &adapter->hw;  in igbvf_setup_srrctl()
1348 	srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>  in igbvf_setup_srrctl()
1351 	if (adapter->rx_buffer_len < 2048) {  in igbvf_setup_srrctl()
1352 		adapter->rx_ps_hdr_size = 0;  in igbvf_setup_srrctl()
1355 		adapter->rx_ps_hdr_size = 128;  in igbvf_setup_srrctl()
1356 		srrctl |= adapter->rx_ps_hdr_size <<  in igbvf_setup_srrctl()
1370 static void igbvf_configure_rx(struct igbvf_adapter *adapter)  in igbvf_configure_rx()  argument
1372 	struct e1000_hw *hw = &adapter->hw;  in igbvf_configure_rx()
1373 	struct igbvf_ring *rx_ring = adapter->rx_ring;  in igbvf_configure_rx()
1401 	igbvf_set_rlpml(adapter);  in igbvf_configure_rx()
1418 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_set_multi()  local
1419 	struct e1000_hw *hw = &adapter->hw;  in igbvf_set_multi()
1453 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_set_uni()  local
1454 	struct e1000_hw *hw = &adapter->hw;  in igbvf_set_uni()
1496 static void igbvf_configure(struct igbvf_adapter *adapter)  in igbvf_configure()  argument
1498 	igbvf_set_rx_mode(adapter->netdev);  in igbvf_configure()
1500 	igbvf_restore_vlan(adapter);  in igbvf_configure()
1502 	igbvf_configure_tx(adapter);  in igbvf_configure()
1503 	igbvf_setup_srrctl(adapter);  in igbvf_configure()
1504 	igbvf_configure_rx(adapter);  in igbvf_configure()
1505 	igbvf_alloc_rx_buffers(adapter->rx_ring,  in igbvf_configure()
1506 			       igbvf_desc_unused(adapter->rx_ring));  in igbvf_configure()
1517 static void igbvf_reset(struct igbvf_adapter *adapter)  in igbvf_reset()  argument
1519 	struct e1000_mac_info *mac = &adapter->hw.mac;  in igbvf_reset()
1520 	struct net_device *netdev = adapter->netdev;  in igbvf_reset()
1521 	struct e1000_hw *hw = &adapter->hw;  in igbvf_reset()
1527 		dev_info(&adapter->pdev->dev, "PF still resetting\n");  in igbvf_reset()
1533 	if (is_valid_ether_addr(adapter->hw.mac.addr)) {  in igbvf_reset()
1534 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);  in igbvf_reset()
1535 		memcpy(netdev->perm_addr, adapter->hw.mac.addr,  in igbvf_reset()
1539 	adapter->last_reset = jiffies;  in igbvf_reset()
1542 int igbvf_up(struct igbvf_adapter *adapter)  in igbvf_up()  argument
1544 	struct e1000_hw *hw = &adapter->hw;  in igbvf_up()
1547 	igbvf_configure(adapter);  in igbvf_up()
1549 	clear_bit(__IGBVF_DOWN, &adapter->state);  in igbvf_up()
1551 	napi_enable(&adapter->rx_ring->napi);  in igbvf_up()
1552 	if (adapter->msix_entries)  in igbvf_up()
1553 		igbvf_configure_msix(adapter);  in igbvf_up()
1557 	igbvf_irq_enable(adapter);  in igbvf_up()
1561 	mod_timer(&adapter->watchdog_timer, jiffies + 1);  in igbvf_up()
1566 void igbvf_down(struct igbvf_adapter *adapter)  in igbvf_down()  argument
1568 	struct net_device *netdev = adapter->netdev;  in igbvf_down()
1569 	struct e1000_hw *hw = &adapter->hw;  in igbvf_down()
1575 	set_bit(__IGBVF_DOWN, &adapter->state);  in igbvf_down()
1592 	napi_disable(&adapter->rx_ring->napi);  in igbvf_down()
1594 	igbvf_irq_disable(adapter);  in igbvf_down()
1596 	del_timer_sync(&adapter->watchdog_timer);  in igbvf_down()
1599 	igbvf_update_stats(adapter);  in igbvf_down()
1601 	adapter->link_speed = 0;  in igbvf_down()
1602 	adapter->link_duplex = 0;  in igbvf_down()
1604 	igbvf_reset(adapter);  in igbvf_down()
1605 	igbvf_clean_tx_ring(adapter->tx_ring);  in igbvf_down()
1606 	igbvf_clean_rx_ring(adapter->rx_ring);  in igbvf_down()
1609 void igbvf_reinit_locked(struct igbvf_adapter *adapter)  in igbvf_reinit_locked()  argument
1612 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))  in igbvf_reinit_locked()
1614 	igbvf_down(adapter);  in igbvf_reinit_locked()
1615 	igbvf_up(adapter);  in igbvf_reinit_locked()
1616 	clear_bit(__IGBVF_RESETTING, &adapter->state);  in igbvf_reinit_locked()
1627 static int igbvf_sw_init(struct igbvf_adapter *adapter)  in igbvf_sw_init()  argument
1629 	struct net_device *netdev = adapter->netdev;  in igbvf_sw_init()
1632 	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;  in igbvf_sw_init()
1633 	adapter->rx_ps_hdr_size = 0;  in igbvf_sw_init()
1634 	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;  in igbvf_sw_init()
1635 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;  in igbvf_sw_init()
1637 	adapter->tx_int_delay = 8;  in igbvf_sw_init()
1638 	adapter->tx_abs_int_delay = 32;  in igbvf_sw_init()
1639 	adapter->rx_int_delay = 0;  in igbvf_sw_init()
1640 	adapter->rx_abs_int_delay = 8;  in igbvf_sw_init()
1641 	adapter->requested_itr = 3;  in igbvf_sw_init()
1642 	adapter->current_itr = IGBVF_START_ITR;  in igbvf_sw_init()
1645 	adapter->ei->init_ops(&adapter->hw);  in igbvf_sw_init()
1647 	rc = adapter->hw.mac.ops.init_params(&adapter->hw);  in igbvf_sw_init()
1651 	rc = adapter->hw.mbx.ops.init_params(&adapter->hw);  in igbvf_sw_init()
1655 	igbvf_set_interrupt_capability(adapter);  in igbvf_sw_init()
1657 	if (igbvf_alloc_queues(adapter))  in igbvf_sw_init()
1660 	spin_lock_init(&adapter->tx_queue_lock);  in igbvf_sw_init()
1663 	igbvf_irq_disable(adapter);  in igbvf_sw_init()
1665 	spin_lock_init(&adapter->stats_lock);  in igbvf_sw_init()
1666 	spin_lock_init(&adapter->hw.mbx_lock);  in igbvf_sw_init()
1668 	set_bit(__IGBVF_DOWN, &adapter->state);  in igbvf_sw_init()
1672 static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)  in igbvf_initialize_last_counter_stats()  argument
1674 	struct e1000_hw *hw = &adapter->hw;  in igbvf_initialize_last_counter_stats()
1676 	adapter->stats.last_gprc = er32(VFGPRC);  in igbvf_initialize_last_counter_stats()
1677 	adapter->stats.last_gorc = er32(VFGORC);  in igbvf_initialize_last_counter_stats()
1678 	adapter->stats.last_gptc = er32(VFGPTC);  in igbvf_initialize_last_counter_stats()
1679 	adapter->stats.last_gotc = er32(VFGOTC);  in igbvf_initialize_last_counter_stats()
1680 	adapter->stats.last_mprc = er32(VFMPRC);  in igbvf_initialize_last_counter_stats()
1681 	adapter->stats.last_gotlbc = er32(VFGOTLBC);  in igbvf_initialize_last_counter_stats()
1682 	adapter->stats.last_gptlbc = er32(VFGPTLBC);  in igbvf_initialize_last_counter_stats()
1683 	adapter->stats.last_gorlbc = er32(VFGORLBC);  in igbvf_initialize_last_counter_stats()
1684 	adapter->stats.last_gprlbc = er32(VFGPRLBC);  in igbvf_initialize_last_counter_stats()
1686 	adapter->stats.base_gprc = er32(VFGPRC);  in igbvf_initialize_last_counter_stats()
1687 	adapter->stats.base_gorc = er32(VFGORC);  in igbvf_initialize_last_counter_stats()
1688 	adapter->stats.base_gptc = er32(VFGPTC);  in igbvf_initialize_last_counter_stats()
1689 	adapter->stats.base_gotc = er32(VFGOTC);  in igbvf_initialize_last_counter_stats()
1690 	adapter->stats.base_mprc = er32(VFMPRC);  in igbvf_initialize_last_counter_stats()
1691 	adapter->stats.base_gotlbc = er32(VFGOTLBC);  in igbvf_initialize_last_counter_stats()
1692 	adapter->stats.base_gptlbc = er32(VFGPTLBC);  in igbvf_initialize_last_counter_stats()
1693 	adapter->stats.base_gorlbc = er32(VFGORLBC);  in igbvf_initialize_last_counter_stats()
1694 	adapter->stats.base_gprlbc = er32(VFGPRLBC);  in igbvf_initialize_last_counter_stats()
1711 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_open()  local
1712 	struct e1000_hw *hw = &adapter->hw;  in igbvf_open()
1716 	if (test_bit(__IGBVF_TESTING, &adapter->state))  in igbvf_open()
1720 	err = igbvf_setup_tx_resources(adapter, adapter->tx_ring);  in igbvf_open()
1725 	err = igbvf_setup_rx_resources(adapter, adapter->rx_ring);  in igbvf_open()
1734 	igbvf_configure(adapter);  in igbvf_open()
1736 	err = igbvf_request_irq(adapter);  in igbvf_open()
1741 	clear_bit(__IGBVF_DOWN, &adapter->state);  in igbvf_open()
1743 	napi_enable(&adapter->rx_ring->napi);  in igbvf_open()
1748 	igbvf_irq_enable(adapter);  in igbvf_open()
1752 	mod_timer(&adapter->watchdog_timer, jiffies + 1);  in igbvf_open()
1757 	igbvf_free_rx_resources(adapter->rx_ring);  in igbvf_open()
1759 	igbvf_free_tx_resources(adapter->tx_ring);  in igbvf_open()
1761 	igbvf_reset(adapter);  in igbvf_open()
1779 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_close()  local
1781 	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));  in igbvf_close()
1782 	igbvf_down(adapter);  in igbvf_close()
1784 	igbvf_free_irq(adapter);  in igbvf_close()
1786 	igbvf_free_tx_resources(adapter->tx_ring);  in igbvf_close()
1787 	igbvf_free_rx_resources(adapter->rx_ring);  in igbvf_close()
1801 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_set_mac()  local
1802 	struct e1000_hw *hw = &adapter->hw;  in igbvf_set_mac()
1827 	if (current_counter < adapter->stats.last_##name) \
1828 		adapter->stats.name += 0x100000000LL; \
1829 	adapter->stats.last_##name = current_counter; \
1830 	adapter->stats.name &= 0xFFFFFFFF00000000LL; \
1831 	adapter->stats.name |= current_counter; \
1838 void igbvf_update_stats(struct igbvf_adapter *adapter)  in igbvf_update_stats()  argument
1840 	struct e1000_hw *hw = &adapter->hw;  in igbvf_update_stats()
1841 	struct pci_dev *pdev = adapter->pdev;  in igbvf_update_stats()
1846 	if (adapter->link_speed == 0)  in igbvf_update_stats()
1849 	if (test_bit(__IGBVF_RESETTING, &adapter->state))  in igbvf_update_stats()
1866 	adapter->netdev->stats.multicast = adapter->stats.mprc;  in igbvf_update_stats()
1869 static void igbvf_print_link_info(struct igbvf_adapter *adapter)  in igbvf_print_link_info()  argument
1871 	dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",  in igbvf_print_link_info()
1872 		 adapter->link_speed,  in igbvf_print_link_info()
1873 		 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");  in igbvf_print_link_info()
1876 static bool igbvf_has_link(struct igbvf_adapter *adapter)  in igbvf_has_link()  argument
1878 	struct e1000_hw *hw = &adapter->hw;  in igbvf_has_link()
1883 	if (test_bit(__IGBVF_DOWN, &adapter->state))  in igbvf_has_link()
1895 	if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))  in igbvf_has_link()
1896 		schedule_work(&adapter->reset_task);  in igbvf_has_link()
1907 	struct igbvf_adapter *adapter = from_timer(adapter, t, watchdog_timer);  in igbvf_watchdog()  local
1910 	schedule_work(&adapter->watchdog_task);  in igbvf_watchdog()
1915 	struct igbvf_adapter *adapter = container_of(work,  in igbvf_watchdog_task()  local
1918 	struct net_device *netdev = adapter->netdev;  in igbvf_watchdog_task()
1919 	struct e1000_mac_info *mac = &adapter->hw.mac;  in igbvf_watchdog_task()
1920 	struct igbvf_ring *tx_ring = adapter->tx_ring;  in igbvf_watchdog_task()
1921 	struct e1000_hw *hw = &adapter->hw;  in igbvf_watchdog_task()
1925 	link = igbvf_has_link(adapter);  in igbvf_watchdog_task()
1929 			mac->ops.get_link_up_info(&adapter->hw,  in igbvf_watchdog_task()
1930 						  &adapter->link_speed,  in igbvf_watchdog_task()
1931 						  &adapter->link_duplex);  in igbvf_watchdog_task()
1932 			igbvf_print_link_info(adapter);  in igbvf_watchdog_task()
1939 			adapter->link_speed = 0;  in igbvf_watchdog_task()
1940 			adapter->link_duplex = 0;  in igbvf_watchdog_task()
1941 			dev_info(&adapter->pdev->dev, "Link is Down\n");  in igbvf_watchdog_task()
1948 		igbvf_update_stats(adapter);  in igbvf_watchdog_task()
1958 			adapter->tx_timeout_count++;  in igbvf_watchdog_task()
1959 			schedule_work(&adapter->reset_task);  in igbvf_watchdog_task()
1964 	ew32(EICS, adapter->rx_ring->eims_value);  in igbvf_watchdog_task()
1967 	if (!test_bit(__IGBVF_DOWN, &adapter->state))  in igbvf_watchdog_task()
1968 		mod_timer(&adapter->watchdog_timer,  in igbvf_watchdog_task()
2121 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_maybe_stop_tx()  local
2124 	if (igbvf_desc_unused(adapter->tx_ring) >= size)  in igbvf_maybe_stop_tx()
2136 	if (igbvf_desc_unused(adapter->tx_ring) < size)  in igbvf_maybe_stop_tx()
2141 	++adapter->restart_queue;  in igbvf_maybe_stop_tx()
2148 static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,  in igbvf_tx_map_adv()  argument
2153 	struct pci_dev *pdev = adapter->pdev;  in igbvf_tx_map_adv()
2214 		igbvf_put_txbuf(adapter, buffer_info);  in igbvf_tx_map_adv()
2220 static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,  in igbvf_tx_queue_adv()  argument
2266 	tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);  in igbvf_tx_queue_adv()
2276 	writel(i, adapter->hw.hw_addr + tx_ring->tail);  in igbvf_tx_queue_adv()
2283 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_xmit_frame_ring_adv()  local
2290 	if (test_bit(__IGBVF_DOWN, &adapter->state)) {  in igbvf_xmit_frame_ring_adv()
2337 	count = igbvf_tx_map_adv(adapter, tx_ring, skb);  in igbvf_xmit_frame_ring_adv()
2340 		igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,  in igbvf_xmit_frame_ring_adv()
2356 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_xmit_frame()  local
2359 	if (test_bit(__IGBVF_DOWN, &adapter->state)) {  in igbvf_xmit_frame()
2364 	tx_ring = &adapter->tx_ring[0];  in igbvf_xmit_frame()
2376 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_tx_timeout()  local
2379 	adapter->tx_timeout_count++;  in igbvf_tx_timeout()
2380 	schedule_work(&adapter->reset_task);  in igbvf_tx_timeout()
2385 	struct igbvf_adapter *adapter;  in igbvf_reset_task()  local
2387 	adapter = container_of(work, struct igbvf_adapter, reset_task);  in igbvf_reset_task()
2389 	igbvf_reinit_locked(adapter);  in igbvf_reset_task()
2401 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_change_mtu()  local
2404 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))  in igbvf_change_mtu()
2407 	adapter->max_frame_size = max_frame;  in igbvf_change_mtu()
2409 		igbvf_down(adapter);  in igbvf_change_mtu()
2420 		adapter->rx_buffer_len = 1024;  in igbvf_change_mtu()
2422 		adapter->rx_buffer_len = 2048;  in igbvf_change_mtu()
2425 		adapter->rx_buffer_len = 16384;  in igbvf_change_mtu()
2427 		adapter->rx_buffer_len = PAGE_SIZE / 2;  in igbvf_change_mtu()
2433 		adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +  in igbvf_change_mtu()
2441 		igbvf_up(adapter);  in igbvf_change_mtu()
2443 		igbvf_reset(adapter);  in igbvf_change_mtu()
2445 	clear_bit(__IGBVF_RESETTING, &adapter->state);  in igbvf_change_mtu()
2461 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_suspend()  local
2466 		WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));  in igbvf_suspend()
2467 		igbvf_down(adapter);  in igbvf_suspend()
2468 		igbvf_free_irq(adapter);  in igbvf_suspend()
2478 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_resume()  local
2484 		err = igbvf_request_irq(adapter);  in igbvf_resume()
2489 	igbvf_reset(adapter);  in igbvf_resume()
2492 		igbvf_up(adapter);  in igbvf_resume()
2511 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_netpoll()  local
2513 	disable_irq(adapter->pdev->irq);  in igbvf_netpoll()
2515 	igbvf_clean_tx_irq(adapter->tx_ring);  in igbvf_netpoll()
2517 	enable_irq(adapter->pdev->irq);  in igbvf_netpoll()
2533 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_io_error_detected()  local
2541 		igbvf_down(adapter);  in igbvf_io_error_detected()
2558 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_io_slot_reset()  local
2567 	igbvf_reset(adapter);  in igbvf_io_slot_reset()
2583 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_io_resume()  local
2586 		if (igbvf_up(adapter)) {  in igbvf_io_resume()
2603 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_io_prepare()  local
2605 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))  in igbvf_io_prepare()
2607 	igbvf_down(adapter);  in igbvf_io_prepare()
2617 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_io_reset_done()  local
2619 	igbvf_up(adapter);  in igbvf_io_reset_done()
2620 	clear_bit(__IGBVF_RESETTING, &adapter->state);  in igbvf_io_reset_done()
2623 static void igbvf_print_device_info(struct igbvf_adapter *adapter)  in igbvf_print_device_info()  argument
2625 	struct e1000_hw *hw = &adapter->hw;  in igbvf_print_device_info()
2626 	struct net_device *netdev = adapter->netdev;  in igbvf_print_device_info()
2627 	struct pci_dev *pdev = adapter->pdev;  in igbvf_print_device_info()
2639 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_set_features()  local
2642 		adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;  in igbvf_set_features()
2644 		adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;  in igbvf_set_features()
2715 	struct igbvf_adapter *adapter;  in igbvf_probe()  local
2746 	adapter = netdev_priv(netdev);  in igbvf_probe()
2747 	hw = &adapter->hw;  in igbvf_probe()
2748 	adapter->netdev = netdev;  in igbvf_probe()
2749 	adapter->pdev = pdev;  in igbvf_probe()
2750 	adapter->ei = ei;  in igbvf_probe()
2751 	adapter->pba = ei->pba;  in igbvf_probe()
2752 	adapter->flags = ei->flags;  in igbvf_probe()
2753 	adapter->hw.back = adapter;  in igbvf_probe()
2754 	adapter->hw.mac.type = ei->mac;  in igbvf_probe()
2755 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);  in igbvf_probe()
2766 	adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),  in igbvf_probe()
2769 	if (!adapter->hw.hw_addr)  in igbvf_probe()
2773 		err = ei->get_variants(adapter);  in igbvf_probe()
2779 	err = igbvf_sw_init(adapter);  in igbvf_probe()
2790 	adapter->bd_number = cards_found++;  in igbvf_probe()
2836 		else if (is_zero_ether_addr(adapter->hw.mac.addr))  in igbvf_probe()
2839 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);  in igbvf_probe()
2847 		memcpy(adapter->hw.mac.addr, netdev->dev_addr,  in igbvf_probe()
2851 	timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);  in igbvf_probe()
2853 	INIT_WORK(&adapter->reset_task, igbvf_reset_task);  in igbvf_probe()
2854 	INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);  in igbvf_probe()
2857 	adapter->rx_ring->count = 1024;  in igbvf_probe()
2858 	adapter->tx_ring->count = 1024;  in igbvf_probe()
2861 	igbvf_reset(adapter);  in igbvf_probe()
2864 	if (adapter->hw.mac.type == e1000_vfadapt_i350)  in igbvf_probe()
2865 		adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;  in igbvf_probe()
2876 	igbvf_print_device_info(adapter);  in igbvf_probe()
2878 	igbvf_initialize_last_counter_stats(adapter);  in igbvf_probe()
2883 	netif_napi_del(&adapter->rx_ring->napi);  in igbvf_probe()
2884 	kfree(adapter->tx_ring);  in igbvf_probe()
2885 	kfree(adapter->rx_ring);  in igbvf_probe()
2887 	igbvf_reset_interrupt_capability(adapter);  in igbvf_probe()
2889 	iounmap(adapter->hw.hw_addr);  in igbvf_probe()
2912 	struct igbvf_adapter *adapter = netdev_priv(netdev);  in igbvf_remove()  local
2913 	struct e1000_hw *hw = &adapter->hw;  in igbvf_remove()
2918 	set_bit(__IGBVF_DOWN, &adapter->state);  in igbvf_remove()
2919 	del_timer_sync(&adapter->watchdog_timer);  in igbvf_remove()
2921 	cancel_work_sync(&adapter->reset_task);  in igbvf_remove()
2922 	cancel_work_sync(&adapter->watchdog_task);  in igbvf_remove()
2926 	igbvf_reset_interrupt_capability(adapter);  in igbvf_remove()
2931 	netif_napi_del(&adapter->rx_ring->napi);  in igbvf_remove()
2932 	kfree(adapter->tx_ring);  in igbvf_remove()
2933 	kfree(adapter->rx_ring);  in igbvf_remove()