613 			     struct netsec_desc_ring *dring, u16 idx,  in netsec_set_rx_de()  argument
616 	struct netsec_de *de = dring->vaddr + DESC_SZ * idx;  in netsec_set_rx_de()
630 	dring->desc[idx].dma_addr = desc->dma_addr;  in netsec_set_rx_de()
631 	dring->desc[idx].addr = desc->addr;  in netsec_set_rx_de()
632 	dring->desc[idx].len = desc->len;  in netsec_set_rx_de()
637 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_clean_tx_dring()  local
640 	int tail = dring->tail;  in netsec_clean_tx_dring()
644 	spin_lock(&dring->lock);  in netsec_clean_tx_dring()
648 	entry = dring->vaddr + DESC_SZ * tail;  in netsec_clean_tx_dring()
657 		desc = &dring->desc[tail];  in netsec_clean_tx_dring()
692 		dring->tail = (tail + 1) % DESC_NUM;  in netsec_clean_tx_dring()
694 		tail = dring->tail;  in netsec_clean_tx_dring()
695 		entry = dring->vaddr + DESC_SZ * tail;  in netsec_clean_tx_dring()
702 	spin_unlock(&dring->lock);  in netsec_clean_tx_dring()
739 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_alloc_rx_data()  local
742 	page = page_pool_dev_alloc_pages(dring->page_pool);  in netsec_alloc_rx_data()
761 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_rx_fill()  local
765 		netsec_set_rx_de(priv, dring, idx, &dring->desc[idx]);  in netsec_rx_fill()
790 			     struct netsec_desc_ring *dring,  in netsec_set_tx_de()  argument
794 	int idx = dring->head;  in netsec_set_tx_de()
798 	de = dring->vaddr + (DESC_SZ * idx);  in netsec_set_tx_de()
816 	dring->desc[idx] = *desc;  in netsec_set_tx_de()
818 		dring->desc[idx].skb = buf;  in netsec_set_tx_de()
821 		dring->desc[idx].xdpf = buf;  in netsec_set_tx_de()
824 	dring->head = (dring->head + 1) % DESC_NUM;  in netsec_set_tx_de()
901 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_run_xdp()  local
922 			page_pool_put_page(dring->page_pool, page, sync, true);  in netsec_run_xdp()
932 			page_pool_put_page(dring->page_pool, page, sync, true);  in netsec_run_xdp()
944 		page_pool_put_page(dring->page_pool, page, sync, true);  in netsec_run_xdp()
953 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_process_rx()  local
963 	xdp_init_buff(&xdp, PAGE_SIZE, &dring->xdp_rxq);  in netsec_process_rx()
966 	dma_dir = page_pool_get_dma_dir(dring->page_pool);  in netsec_process_rx()
969 		u16 idx = dring->tail;  in netsec_process_rx()
970 		struct netsec_de *de = dring->vaddr + (DESC_SZ * idx);  in netsec_process_rx()
971 		struct netsec_desc *desc = &dring->desc[idx];  in netsec_process_rx()
1001 			dring->tail = (dring->tail + 1) % DESC_NUM;  in netsec_process_rx()
1041 			page_pool_put_page(dring->page_pool, page, pkt_len,  in netsec_process_rx()
1071 		dring->tail = (dring->tail + 1) % DESC_NUM;  in netsec_process_rx()
1101 static int netsec_desc_used(struct netsec_desc_ring *dring)  in netsec_desc_used()  argument
1105 	if (dring->head >= dring->tail)  in netsec_desc_used()
1106 		used = dring->head - dring->tail;  in netsec_desc_used()
1108 		used = dring->head + DESC_NUM - dring->tail;  in netsec_desc_used()
1115 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_check_stop_tx()  local
1126 		used = netsec_desc_used(dring);  in netsec_check_stop_tx()
1140 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_netdev_start_xmit()  local
1146 	spin_lock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1147 	filled = netsec_desc_used(dring);  in netsec_netdev_start_xmit()
1149 		spin_unlock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1178 		spin_unlock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1192 	netsec_set_tx_de(priv, dring, &tx_ctrl, &tx_desc, skb);  in netsec_netdev_start_xmit()
1193 	spin_unlock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1201 	struct netsec_desc_ring *dring = &priv->desc_ring[id];  in netsec_uninit_pkt_dring()  local
1205 	if (!dring->vaddr || !dring->desc)  in netsec_uninit_pkt_dring()
1208 		desc = &dring->desc[idx];  in netsec_uninit_pkt_dring()
1215 			page_pool_put_full_page(dring->page_pool, page, false);  in netsec_uninit_pkt_dring()
1225 		if (xdp_rxq_info_is_reg(&dring->xdp_rxq))  in netsec_uninit_pkt_dring()
1226 			xdp_rxq_info_unreg(&dring->xdp_rxq);  in netsec_uninit_pkt_dring()
1227 		page_pool_destroy(dring->page_pool);  in netsec_uninit_pkt_dring()
1230 	memset(dring->desc, 0, sizeof(struct netsec_desc) * DESC_NUM);  in netsec_uninit_pkt_dring()
1231 	memset(dring->vaddr, 0, DESC_SZ * DESC_NUM);  in netsec_uninit_pkt_dring()
1233 	dring->head = 0;  in netsec_uninit_pkt_dring()
1234 	dring->tail = 0;  in netsec_uninit_pkt_dring()
1242 	struct netsec_desc_ring *dring = &priv->desc_ring[id];  in netsec_free_dring()  local
1244 	if (dring->vaddr) {  in netsec_free_dring()
1246 				  dring->vaddr, dring->desc_dma);  in netsec_free_dring()
1247 		dring->vaddr = NULL;  in netsec_free_dring()
1250 	kfree(dring->desc);  in netsec_free_dring()
1251 	dring->desc = NULL;  in netsec_free_dring()
1256 	struct netsec_desc_ring *dring = &priv->desc_ring[id];  in netsec_alloc_dring()  local
1258 	dring->vaddr = dma_alloc_coherent(priv->dev, DESC_SZ * DESC_NUM,  in netsec_alloc_dring()
1259 					  &dring->desc_dma, GFP_KERNEL);  in netsec_alloc_dring()
1260 	if (!dring->vaddr)  in netsec_alloc_dring()
1263 	dring->desc = kcalloc(DESC_NUM, sizeof(*dring->desc), GFP_KERNEL);  in netsec_alloc_dring()
1264 	if (!dring->desc)  in netsec_alloc_dring()
1276 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_setup_tx_dring()  local
1282 		de = dring->vaddr + (DESC_SZ * i);  in netsec_setup_tx_dring()
1293 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_setup_rx_dring()  local
1308 	dring->page_pool = page_pool_create(&pp_params);  in netsec_setup_rx_dring()
1309 	if (IS_ERR(dring->page_pool)) {  in netsec_setup_rx_dring()
1310 		err = PTR_ERR(dring->page_pool);  in netsec_setup_rx_dring()
1311 		dring->page_pool = NULL;  in netsec_setup_rx_dring()
1315 	err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0, priv->napi.napi_id);  in netsec_setup_rx_dring()
1319 	err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_POOL,  in netsec_setup_rx_dring()
1320 					 dring->page_pool);  in netsec_setup_rx_dring()
1325 		struct netsec_desc *desc = &dring->desc[i];  in netsec_setup_rx_dring()