23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff,  in aq_get_rxpages_xdp()  argument
38 	page_ref_inc(buff->rxdata.page);  in aq_get_rxpages_xdp()
321 		struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];  in aq_ring_tx_clean()  local
323 		if (likely(buff->is_mapped)) {  in aq_ring_tx_clean()
324 			if (unlikely(buff->is_sop)) {  in aq_ring_tx_clean()
325 				if (!buff->is_eop &&  in aq_ring_tx_clean()
326 				    buff->eop_index != 0xffffU &&  in aq_ring_tx_clean()
328 						buff->eop_index,  in aq_ring_tx_clean()
332 				dma_unmap_single(dev, buff->pa, buff->len,  in aq_ring_tx_clean()
335 				dma_unmap_page(dev, buff->pa, buff->len,  in aq_ring_tx_clean()
340 		if (likely(!buff->is_eop))  in aq_ring_tx_clean()
343 		if (buff->skb) {  in aq_ring_tx_clean()
346 			self->stats.tx.bytes += buff->skb->len;  in aq_ring_tx_clean()
348 			dev_kfree_skb_any(buff->skb);  in aq_ring_tx_clean()
349 		} else if (buff->xdpf) {  in aq_ring_tx_clean()
352 			self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf);  in aq_ring_tx_clean()
354 			xdp_return_frame_rx_napi(buff->xdpf);  in aq_ring_tx_clean()
358 		buff->skb = NULL;  in aq_ring_tx_clean()
359 		buff->xdpf = NULL;  in aq_ring_tx_clean()
360 		buff->pa = 0U;  in aq_ring_tx_clean()
361 		buff->eop_index = 0xffffU;  in aq_ring_tx_clean()
369 			   struct aq_ring_buff_s *buff,  in aq_rx_checksum()  argument
375 	if (unlikely(buff->is_cso_err)) {  in aq_rx_checksum()
382 	if (buff->is_ip_cso) {  in aq_rx_checksum()
388 	if (buff->is_udp_cso || buff->is_tcp_cso)  in aq_rx_checksum()
417 					struct aq_ring_buff_s *buff)  in aq_xdp_build_skb()  argument
430 	aq_get_rxpages_xdp(buff, xdp);  in aq_xdp_build_skb()
437 				       struct aq_ring_buff_s *buff)  in aq_xdp_run_prog()  argument
453 		return aq_xdp_build_skb(xdp, aq_nic->ndev, buff);  in aq_xdp_run_prog()
464 		skb = aq_xdp_build_skb(xdp, aq_nic->ndev, buff);  in aq_xdp_run_prog()
482 		aq_get_rxpages_xdp(buff, xdp);  in aq_xdp_run_prog()
491 		aq_get_rxpages_xdp(buff, xdp);  in aq_xdp_run_prog()
515 			       struct aq_ring_buff_s *buff,  in aq_add_rx_fragment()  argument
519 	struct aq_ring_buff_s *buff_ = buff;  in aq_add_rx_fragment()
542 		buff->is_ip_cso &= buff_->is_ip_cso;  in aq_add_rx_fragment()
543 		buff->is_udp_cso &= buff_->is_udp_cso;  in aq_add_rx_fragment()
544 		buff->is_tcp_cso &= buff_->is_tcp_cso;  in aq_add_rx_fragment()
545 		buff->is_cso_err |= buff_->is_cso_err;  in aq_add_rx_fragment()
566 		struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];  in __aq_ring_rx_clean()  local
574 		if (buff->is_cleaned)  in __aq_ring_rx_clean()
577 		if (!buff->is_eop) {  in __aq_ring_rx_clean()
579 			buff_ = buff;  in __aq_ring_rx_clean()
602 				buff->is_error |= buff_->is_error;  in __aq_ring_rx_clean()
603 				buff->is_cso_err |= buff_->is_cso_err;  in __aq_ring_rx_clean()
607 			if (buff->is_error ||  in __aq_ring_rx_clean()
608 			    (buff->is_lro && buff->is_cso_err)) {  in __aq_ring_rx_clean()
609 				buff_ = buff;  in __aq_ring_rx_clean()
628 		if (buff->is_error) {  in __aq_ring_rx_clean()
636 					      buff->rxdata.daddr,  in __aq_ring_rx_clean()
637 					      buff->rxdata.pg_off,  in __aq_ring_rx_clean()
638 					      buff->len, DMA_FROM_DEVICE);  in __aq_ring_rx_clean()
649 			buff->len -=  in __aq_ring_rx_clean()
651 						  aq_buf_vaddr(&buff->rxdata),  in __aq_ring_rx_clean()
652 						  buff->len);  in __aq_ring_rx_clean()
654 		hdr_len = buff->len;  in __aq_ring_rx_clean()
657 						  aq_buf_vaddr(&buff->rxdata),  in __aq_ring_rx_clean()
660 		memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata),  in __aq_ring_rx_clean()
663 		if (buff->len - hdr_len > 0) {  in __aq_ring_rx_clean()
664 			skb_add_rx_frag(skb, i++, buff->rxdata.page,  in __aq_ring_rx_clean()
665 					buff->rxdata.pg_off + hdr_len,  in __aq_ring_rx_clean()
666 					buff->len - hdr_len,  in __aq_ring_rx_clean()
668 			page_ref_inc(buff->rxdata.page);  in __aq_ring_rx_clean()
671 		if (!buff->is_eop) {  in __aq_ring_rx_clean()
672 			buff_ = buff;  in __aq_ring_rx_clean()
690 				buff->is_ip_cso &= buff_->is_ip_cso;  in __aq_ring_rx_clean()
691 				buff->is_udp_cso &= buff_->is_udp_cso;  in __aq_ring_rx_clean()
692 				buff->is_tcp_cso &= buff_->is_tcp_cso;  in __aq_ring_rx_clean()
693 				buff->is_cso_err |= buff_->is_cso_err;  in __aq_ring_rx_clean()
698 		if (buff->is_vlan)  in __aq_ring_rx_clean()
700 					       buff->vlan_rx_tag);  in __aq_ring_rx_clean()
704 		aq_rx_checksum(self, buff, skb);  in __aq_ring_rx_clean()
706 		skb_set_hash(skb, buff->rss_hash,  in __aq_ring_rx_clean()
707 			     buff->is_hash_l4 ? PKT_HASH_TYPE_L4 :  in __aq_ring_rx_clean()
742 		struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head];  in __aq_ring_xdp_clean()  local
750 		if (buff->is_cleaned)  in __aq_ring_xdp_clean()
753 		if (!buff->is_eop) {  in __aq_ring_xdp_clean()
754 			buff_ = buff;  in __aq_ring_xdp_clean()
770 				buff->is_error |= buff_->is_error;  in __aq_ring_xdp_clean()
771 				buff->is_cso_err |= buff_->is_cso_err;  in __aq_ring_xdp_clean()
778 			if (buff->is_error ||  in __aq_ring_xdp_clean()
779 			    (buff->is_lro && buff->is_cso_err)) {  in __aq_ring_xdp_clean()
780 				buff_ = buff;  in __aq_ring_xdp_clean()
799 		if (buff->is_error) {  in __aq_ring_xdp_clean()
807 					      buff->rxdata.daddr,  in __aq_ring_xdp_clean()
808 					      buff->rxdata.pg_off,  in __aq_ring_xdp_clean()
809 					      buff->len, DMA_FROM_DEVICE);  in __aq_ring_xdp_clean()
810 		hard_start = page_address(buff->rxdata.page) +  in __aq_ring_xdp_clean()
811 			     buff->rxdata.pg_off - rx_ring->page_offset;  in __aq_ring_xdp_clean()
814 			buff->len -=  in __aq_ring_xdp_clean()
816 						  aq_buf_vaddr(&buff->rxdata),  in __aq_ring_xdp_clean()
817 						  buff->len);  in __aq_ring_xdp_clean()
821 				 buff->len, false);  in __aq_ring_xdp_clean()
822 		if (!buff->is_eop) {  in __aq_ring_xdp_clean()
823 			if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) {  in __aq_ring_xdp_clean()
833 		skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff);  in __aq_ring_xdp_clean()
837 		if (buff->is_vlan)  in __aq_ring_xdp_clean()
839 					       buff->vlan_rx_tag);  in __aq_ring_xdp_clean()
841 		aq_rx_checksum(rx_ring, buff, skb);  in __aq_ring_xdp_clean()
843 		skb_set_hash(skb, buff->rss_hash,  in __aq_ring_xdp_clean()
844 			     buff->is_hash_l4 ? PKT_HASH_TYPE_L4 :  in __aq_ring_xdp_clean()
889 	struct aq_ring_buff_s *buff = NULL;  in aq_ring_rx_fill()  local
899 		buff = &self->buff_ring[self->sw_tail];  in aq_ring_rx_fill()
901 		buff->flags = 0U;  in aq_ring_rx_fill()
902 		buff->len = self->frame_max;  in aq_ring_rx_fill()
904 		err = aq_get_rxpages(self, buff);  in aq_ring_rx_fill()
908 		buff->pa = aq_buf_daddr(&buff->rxdata);  in aq_ring_rx_fill()
909 		buff = NULL;  in aq_ring_rx_fill()
923 		struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];  in aq_ring_rx_deinit()  local
925 		aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic));  in aq_ring_rx_deinit()