84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf)  in aq_get_rxpages()  argument
86 	unsigned int order = self->page_order;  in aq_get_rxpages()
87 	u16 page_offset = self->page_offset;  in aq_get_rxpages()
88 	u16 frame_max = self->frame_max;  in aq_get_rxpages()
89 	u16 tail_size = self->tail_size;  in aq_get_rxpages()
100 				u64_stats_update_begin(&self->stats.rx.syncp);  in aq_get_rxpages()
101 				self->stats.rx.pg_flips++;  in aq_get_rxpages()
102 				u64_stats_update_end(&self->stats.rx.syncp);  in aq_get_rxpages()
109 					       aq_nic_get_dev(self->aq_nic));  in aq_get_rxpages()
110 				u64_stats_update_begin(&self->stats.rx.syncp);  in aq_get_rxpages()
111 				self->stats.rx.pg_losts++;  in aq_get_rxpages()
112 				u64_stats_update_end(&self->stats.rx.syncp);  in aq_get_rxpages()
116 			u64_stats_update_begin(&self->stats.rx.syncp);  in aq_get_rxpages()
117 			self->stats.rx.pg_reuses++;  in aq_get_rxpages()
118 			u64_stats_update_end(&self->stats.rx.syncp);  in aq_get_rxpages()
123 		ret = aq_alloc_rxpages(&rxbuf->rxdata, self);  in aq_get_rxpages()
125 			u64_stats_update_begin(&self->stats.rx.syncp);  in aq_get_rxpages()
126 			self->stats.rx.alloc_fails++;  in aq_get_rxpages()
127 			u64_stats_update_end(&self->stats.rx.syncp);  in aq_get_rxpages()
135 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self,  in aq_ring_alloc()  argument
140 	self->buff_ring =  in aq_ring_alloc()
141 		kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL);  in aq_ring_alloc()
143 	if (!self->buff_ring) {  in aq_ring_alloc()
148 	self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic),  in aq_ring_alloc()
149 					   self->size * self->dx_size,  in aq_ring_alloc()
150 					   &self->dx_ring_pa, GFP_KERNEL);  in aq_ring_alloc()
151 	if (!self->dx_ring) {  in aq_ring_alloc()
158 		aq_ring_free(self);  in aq_ring_alloc()
159 		self = NULL;  in aq_ring_alloc()
162 	return self;  in aq_ring_alloc()
165 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self,  in aq_ring_tx_alloc()  argument
172 	self->aq_nic = aq_nic;  in aq_ring_tx_alloc()
173 	self->idx = idx;  in aq_ring_tx_alloc()
174 	self->size = aq_nic_cfg->txds;  in aq_ring_tx_alloc()
175 	self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size;  in aq_ring_tx_alloc()
177 	self = aq_ring_alloc(self, aq_nic);  in aq_ring_tx_alloc()
178 	if (!self) {  in aq_ring_tx_alloc()
185 		aq_ring_free(self);  in aq_ring_tx_alloc()
186 		self = NULL;  in aq_ring_tx_alloc()
189 	return self;  in aq_ring_tx_alloc()
192 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self,  in aq_ring_rx_alloc()  argument
199 	self->aq_nic = aq_nic;  in aq_ring_rx_alloc()
200 	self->idx = idx;  in aq_ring_rx_alloc()
201 	self->size = aq_nic_cfg->rxds;  in aq_ring_rx_alloc()
202 	self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size;  in aq_ring_rx_alloc()
203 	self->xdp_prog = aq_nic->xdp_prog;  in aq_ring_rx_alloc()
204 	self->frame_max = AQ_CFG_RX_FRAME_MAX;  in aq_ring_rx_alloc()
207 	if (READ_ONCE(self->xdp_prog)) {  in aq_ring_rx_alloc()
208 		self->page_offset = AQ_XDP_HEADROOM;  in aq_ring_rx_alloc()
209 		self->page_order = AQ_CFG_XDP_PAGEORDER;  in aq_ring_rx_alloc()
210 		self->tail_size = AQ_XDP_TAILROOM;  in aq_ring_rx_alloc()
212 		self->page_offset = 0;  in aq_ring_rx_alloc()
213 		self->page_order = fls(self->frame_max / PAGE_SIZE +  in aq_ring_rx_alloc()
214 				       (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1;  in aq_ring_rx_alloc()
215 		if (aq_nic_cfg->rxpageorder > self->page_order)  in aq_ring_rx_alloc()
216 			self->page_order = aq_nic_cfg->rxpageorder;  in aq_ring_rx_alloc()
217 		self->tail_size = 0;  in aq_ring_rx_alloc()
220 	self = aq_ring_alloc(self, aq_nic);  in aq_ring_rx_alloc()
221 	if (!self) {  in aq_ring_rx_alloc()
228 		aq_ring_free(self);  in aq_ring_rx_alloc()
229 		self = NULL;  in aq_ring_rx_alloc()
232 	return self;  in aq_ring_rx_alloc()
236 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic,  in aq_ring_hwts_rx_alloc()  argument
242 	memset(self, 0, sizeof(*self));  in aq_ring_hwts_rx_alloc()
244 	self->aq_nic = aq_nic;  in aq_ring_hwts_rx_alloc()
245 	self->idx = idx;  in aq_ring_hwts_rx_alloc()
246 	self->size = size;  in aq_ring_hwts_rx_alloc()
247 	self->dx_size = dx_size;  in aq_ring_hwts_rx_alloc()
249 	self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa,  in aq_ring_hwts_rx_alloc()
251 	if (!self->dx_ring) {  in aq_ring_hwts_rx_alloc()
252 		aq_ring_free(self);  in aq_ring_hwts_rx_alloc()
256 	return self;  in aq_ring_hwts_rx_alloc()
259 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type)  in aq_ring_init()  argument
261 	self->hw_head = 0;  in aq_ring_init()
262 	self->sw_head = 0;  in aq_ring_init()
263 	self->sw_tail = 0;  in aq_ring_init()
264 	self->ring_type = ring_type;  in aq_ring_init()
266 	if (self->ring_type == ATL_RING_RX)  in aq_ring_init()
267 		u64_stats_init(&self->stats.rx.syncp);  in aq_ring_init()
269 		u64_stats_init(&self->stats.tx.syncp);  in aq_ring_init()
314 bool aq_ring_tx_clean(struct aq_ring_s *self)  in aq_ring_tx_clean()  argument
316 	struct device *dev = aq_nic_get_dev(self->aq_nic);  in aq_ring_tx_clean()
320 	     budget && self->sw_head != self->hw_head; budget--) {  in aq_ring_tx_clean()
321 		struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];  in aq_ring_tx_clean()
327 				    (!aq_ring_dx_in_range(self->sw_head,  in aq_ring_tx_clean()
329 						self->hw_head)))  in aq_ring_tx_clean()
344 			u64_stats_update_begin(&self->stats.tx.syncp);  in aq_ring_tx_clean()
345 			++self->stats.tx.packets;  in aq_ring_tx_clean()
346 			self->stats.tx.bytes += buff->skb->len;  in aq_ring_tx_clean()
347 			u64_stats_update_end(&self->stats.tx.syncp);  in aq_ring_tx_clean()
350 			u64_stats_update_begin(&self->stats.tx.syncp);  in aq_ring_tx_clean()
351 			++self->stats.tx.packets;  in aq_ring_tx_clean()
352 			self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf);  in aq_ring_tx_clean()
353 			u64_stats_update_end(&self->stats.tx.syncp);  in aq_ring_tx_clean()
362 		self->sw_head = aq_ring_next_dx(self, self->sw_head);  in aq_ring_tx_clean()
368 static void aq_rx_checksum(struct aq_ring_s *self,  in aq_rx_checksum()  argument
372 	if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM))  in aq_rx_checksum()
376 		u64_stats_update_begin(&self->stats.rx.syncp);  in aq_rx_checksum()
377 		++self->stats.rx.errors;  in aq_rx_checksum()
378 		u64_stats_update_end(&self->stats.rx.syncp);  in aq_rx_checksum()
543 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi,  in __aq_ring_rx_clean()  argument
546 	struct net_device *ndev = aq_nic_get_ndev(self->aq_nic);  in __aq_ring_rx_clean()
549 	for (; (self->sw_head != self->hw_head) && budget;  in __aq_ring_rx_clean()
550 		self->sw_head = aq_ring_next_dx(self, self->sw_head),  in __aq_ring_rx_clean()
552 		struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];  in __aq_ring_rx_clean()
553 		bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self);  in __aq_ring_rx_clean()
569 				if (buff_->next >= self->size) {  in __aq_ring_rx_clean()
576 				buff_ = &self->buff_ring[next_];  in __aq_ring_rx_clean()
578 					aq_ring_dx_in_range(self->sw_head,  in __aq_ring_rx_clean()
580 							    self->hw_head);  in __aq_ring_rx_clean()
597 					if (buff_->next >= self->size) {  in __aq_ring_rx_clean()
602 					buff_ = &self->buff_ring[next_];  in __aq_ring_rx_clean()
607 				u64_stats_update_begin(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
608 				++self->stats.rx.errors;  in __aq_ring_rx_clean()
609 				u64_stats_update_end(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
615 			u64_stats_update_begin(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
616 			++self->stats.rx.errors;  in __aq_ring_rx_clean()
617 			u64_stats_update_end(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
621 		dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic),  in __aq_ring_rx_clean()
628 			u64_stats_update_begin(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
629 			self->stats.rx.skb_alloc_fails++;  in __aq_ring_rx_clean()
630 			u64_stats_update_end(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
636 				aq_ptp_extract_ts(self->aq_nic, skb,  in __aq_ring_rx_clean()
653 					self->frame_max);  in __aq_ring_rx_clean()
661 				buff_ = &self->buff_ring[next_];  in __aq_ring_rx_clean()
663 				dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic),  in __aq_ring_rx_clean()
672 						self->frame_max);  in __aq_ring_rx_clean()
690 		aq_rx_checksum(self, buff, skb);  in __aq_ring_rx_clean()
698 						: AQ_NIC_RING2QMAP(self->aq_nic,  in __aq_ring_rx_clean()
699 								   self->idx));  in __aq_ring_rx_clean()
701 		u64_stats_update_begin(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
702 		++self->stats.rx.packets;  in __aq_ring_rx_clean()
703 		self->stats.rx.bytes += skb->len;  in __aq_ring_rx_clean()
704 		u64_stats_update_end(&self->stats.rx.syncp);  in __aq_ring_rx_clean()
845 int aq_ring_rx_clean(struct aq_ring_s *self,  in aq_ring_rx_clean()  argument
851 		return __aq_ring_xdp_clean(self, napi, work_done, budget);  in aq_ring_rx_clean()
853 		return __aq_ring_rx_clean(self, napi, work_done, budget);  in aq_ring_rx_clean()
856 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic)  in aq_ring_hwts_rx_clean()  argument
859 	while (self->sw_head != self->hw_head) {  in aq_ring_hwts_rx_clean()
863 						self->dx_ring +  in aq_ring_hwts_rx_clean()
864 						(self->sw_head * self->dx_size),  in aq_ring_hwts_rx_clean()
865 						self->dx_size, &ns);  in aq_ring_hwts_rx_clean()
868 		self->sw_head = aq_ring_next_dx(self, self->sw_head);  in aq_ring_hwts_rx_clean()
873 int aq_ring_rx_fill(struct aq_ring_s *self)  in aq_ring_rx_fill()  argument
879 	if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES,  in aq_ring_rx_fill()
880 					   self->size / 2))  in aq_ring_rx_fill()
883 	for (i = aq_ring_avail_dx(self); i--;  in aq_ring_rx_fill()
884 		self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) {  in aq_ring_rx_fill()
885 		buff = &self->buff_ring[self->sw_tail];  in aq_ring_rx_fill()
888 		buff->len = self->frame_max;  in aq_ring_rx_fill()
890 		err = aq_get_rxpages(self, buff);  in aq_ring_rx_fill()
902 void aq_ring_rx_deinit(struct aq_ring_s *self)  in aq_ring_rx_deinit()  argument
904 	if (!self)  in aq_ring_rx_deinit()
907 	for (; self->sw_head != self->sw_tail;  in aq_ring_rx_deinit()
908 		self->sw_head = aq_ring_next_dx(self, self->sw_head)) {  in aq_ring_rx_deinit()
909 		struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];  in aq_ring_rx_deinit()
911 		aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic));  in aq_ring_rx_deinit()
915 void aq_ring_free(struct aq_ring_s *self)  in aq_ring_free()  argument
917 	if (!self)  in aq_ring_free()
920 	kfree(self->buff_ring);  in aq_ring_free()
922 	if (self->dx_ring)  in aq_ring_free()
923 		dma_free_coherent(aq_nic_get_dev(self->aq_nic),  in aq_ring_free()
924 				  self->size * self->dx_size, self->dx_ring,  in aq_ring_free()
925 				  self->dx_ring_pa);  in aq_ring_free()
928 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data)  in aq_ring_fill_stats_data()  argument
933 	if (self->ring_type == ATL_RING_RX) {  in aq_ring_fill_stats_data()
937 			start = u64_stats_fetch_begin_irq(&self->stats.rx.syncp);  in aq_ring_fill_stats_data()
938 			data[count] = self->stats.rx.packets;  in aq_ring_fill_stats_data()
939 			data[++count] = self->stats.rx.jumbo_packets;  in aq_ring_fill_stats_data()
940 			data[++count] = self->stats.rx.lro_packets;  in aq_ring_fill_stats_data()
941 			data[++count] = self->stats.rx.errors;  in aq_ring_fill_stats_data()
942 			data[++count] = self->stats.rx.alloc_fails;  in aq_ring_fill_stats_data()
943 			data[++count] = self->stats.rx.skb_alloc_fails;  in aq_ring_fill_stats_data()
944 			data[++count] = self->stats.rx.polls;  in aq_ring_fill_stats_data()
945 			data[++count] = self->stats.rx.pg_flips;  in aq_ring_fill_stats_data()
946 			data[++count] = self->stats.rx.pg_reuses;  in aq_ring_fill_stats_data()
947 			data[++count] = self->stats.rx.pg_losts;  in aq_ring_fill_stats_data()
948 			data[++count] = self->stats.rx.xdp_aborted;  in aq_ring_fill_stats_data()
949 			data[++count] = self->stats.rx.xdp_drop;  in aq_ring_fill_stats_data()
950 			data[++count] = self->stats.rx.xdp_pass;  in aq_ring_fill_stats_data()
951 			data[++count] = self->stats.rx.xdp_tx;  in aq_ring_fill_stats_data()
952 			data[++count] = self->stats.rx.xdp_invalid;  in aq_ring_fill_stats_data()
953 			data[++count] = self->stats.rx.xdp_redirect;  in aq_ring_fill_stats_data()
954 		} while (u64_stats_fetch_retry_irq(&self->stats.rx.syncp, start));  in aq_ring_fill_stats_data()
959 			start = u64_stats_fetch_begin_irq(&self->stats.tx.syncp);  in aq_ring_fill_stats_data()
960 			data[count] = self->stats.tx.packets;  in aq_ring_fill_stats_data()
961 			data[++count] = self->stats.tx.queue_restarts;  in aq_ring_fill_stats_data()
962 		} while (u64_stats_fetch_retry_irq(&self->stats.tx.syncp, start));  in aq_ring_fill_stats_data()