1 // SPDX-License-Identifier: GPL-2.0-only
18 #include <linux/dma-mapping.h>
46 	(HINIC_GET_RX_PKT_TYPE(be32_to_cpu((cqe)->offload_type)) == \
50  * hinic_rxq_clean_stats - Clean the statistics of specific queue
51  * @rxq: Logical Rx Queue
55 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in hinic_rxq_clean_stats()
57 	u64_stats_update_begin(&rxq_stats->syncp);  in hinic_rxq_clean_stats()
58 	rxq_stats->pkts  = 0;  in hinic_rxq_clean_stats()
59 	rxq_stats->bytes = 0;  in hinic_rxq_clean_stats()
60 	rxq_stats->errors = 0;  in hinic_rxq_clean_stats()
61 	rxq_stats->csum_errors = 0;  in hinic_rxq_clean_stats()
62 	rxq_stats->other_errors = 0;  in hinic_rxq_clean_stats()
63 	u64_stats_update_end(&rxq_stats->syncp);  in hinic_rxq_clean_stats()
67  * hinic_rxq_get_stats - get statistics of Rx Queue
68  * @rxq: Logical Rx Queue
73 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in hinic_rxq_get_stats()
76 	u64_stats_update_begin(&stats->syncp);  in hinic_rxq_get_stats()
78 		start = u64_stats_fetch_begin(&rxq_stats->syncp);  in hinic_rxq_get_stats()
79 		stats->pkts = rxq_stats->pkts;  in hinic_rxq_get_stats()
80 		stats->bytes = rxq_stats->bytes;  in hinic_rxq_get_stats()
81 		stats->errors = rxq_stats->csum_errors +  in hinic_rxq_get_stats()
82 				rxq_stats->other_errors;  in hinic_rxq_get_stats()
83 		stats->csum_errors = rxq_stats->csum_errors;  in hinic_rxq_get_stats()
84 		stats->other_errors = rxq_stats->other_errors;  in hinic_rxq_get_stats()
85 	} while (u64_stats_fetch_retry(&rxq_stats->syncp, start));  in hinic_rxq_get_stats()
86 	u64_stats_update_end(&stats->syncp);  in hinic_rxq_get_stats()
90  * rxq_stats_init - Initialize the statistics of specific queue
91  * @rxq: Logical Rx Queue
95 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in rxq_stats_init()
97 	u64_stats_init(&rxq_stats->syncp);  in rxq_stats_init()
104 	struct net_device *netdev = rxq->netdev;  in rx_csum()
109 	if (!(netdev->features & NETIF_F_RXCSUM))  in rx_csum()
113 		skb->ip_summed = CHECKSUM_UNNECESSARY;  in rx_csum()
117 			rxq->rxq_stats.csum_errors++;  in rx_csum()
118 		skb->ip_summed = CHECKSUM_NONE;  in rx_csum()
123  * rx_alloc_skb - allocate skb and map it to dma address
124  * @rxq: rx queue
132 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_alloc_skb()
133 	struct hinic_hwdev *hwdev = nic_dev->hwdev;  in rx_alloc_skb()
134 	struct hinic_hwif *hwif = hwdev->hwif;  in rx_alloc_skb()
135 	struct pci_dev *pdev = hwif->pdev;  in rx_alloc_skb()
140 	skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);  in rx_alloc_skb()
144 	addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,  in rx_alloc_skb()
146 	err = dma_mapping_error(&pdev->dev, addr);  in rx_alloc_skb()
148 		dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);  in rx_alloc_skb()
161  * rx_unmap_skb - unmap the dma address of the skb
162  * @rxq: rx queue
167 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_unmap_skb()
168 	struct hinic_hwdev *hwdev = nic_dev->hwdev;  in rx_unmap_skb()
169 	struct hinic_hwif *hwif = hwdev->hwif;  in rx_unmap_skb()
170 	struct pci_dev *pdev = hwif->pdev;  in rx_unmap_skb()
172 	dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,  in rx_unmap_skb()
177  * rx_free_skb - unmap and free skb
178  * @rxq: rx queue
190  * rx_alloc_pkts - allocate pkts in rx queue
191  * @rxq: rx queue
197 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_alloc_pkts()
206 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);  in rx_alloc_pkts()
209 	if (free_wqebbs > nic_dev->rx_weight)  in rx_alloc_pkts()
210 		free_wqebbs = nic_dev->rx_weight;  in rx_alloc_pkts()
217 		hinic_set_sge(&sge, dma_addr, skb->len);  in rx_alloc_pkts()
219 		rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,  in rx_alloc_pkts()
226 		hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);  in rx_alloc_pkts()
228 		hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);  in rx_alloc_pkts()
235 		hinic_rq_update(rxq->rq, prod_idx);  in rx_alloc_pkts()
242  * free_all_rx_skbs - free all skbs in rx queue
243  * @rxq: rx queue
247 	struct hinic_rq *rq = rxq->rq;  in free_all_rx_skbs()
252 	while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {  in free_all_rx_skbs()
256 		hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);  in free_all_rx_skbs()
258 		hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);  in free_all_rx_skbs()
260 		rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));  in free_all_rx_skbs()
265  * rx_recv_jumbo_pkt - Rx handler for jumbo pkt
266  * @rxq: rx queue
283 		rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,  in rx_recv_jumbo_pkt()
288 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);  in rx_recv_jumbo_pkt()
292 		prefetch(skb->data);  in rx_recv_jumbo_pkt()
297 		left_pkt_len -= curr_len;  in rx_recv_jumbo_pkt()
302 			skb_shinfo(head_skb)->frag_list = skb;  in rx_recv_jumbo_pkt()
304 			curr_skb->next = skb;  in rx_recv_jumbo_pkt()
306 		head_skb->len += skb->len;  in rx_recv_jumbo_pkt()
307 		head_skb->data_len += skb->len;  in rx_recv_jumbo_pkt()
308 		head_skb->truesize += skb->truesize;  in rx_recv_jumbo_pkt()
319 	struct net_device *netdev = nic_dev->netdev;  in hinic_copy_lp_data()
320 	u8 *lb_buf = nic_dev->lb_test_rx_buf;  in hinic_copy_lp_data()
321 	int lb_len = nic_dev->lb_pkt_len;  in hinic_copy_lp_data()
325 	if (nic_dev->lb_test_rx_idx == LP_PKT_CNT) {  in hinic_copy_lp_data()
326 		nic_dev->lb_test_rx_idx = 0;  in hinic_copy_lp_data()
330 	if (skb->len != nic_dev->lb_pkt_len) {  in hinic_copy_lp_data()
332 		nic_dev->lb_test_rx_idx++;  in hinic_copy_lp_data()
336 	pkt_offset = nic_dev->lb_test_rx_idx * lb_len;  in hinic_copy_lp_data()
338 	memcpy(lb_buf + pkt_offset, skb->data, frag_len);  in hinic_copy_lp_data()
340 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {  in hinic_copy_lp_data()
341 		frag_data = skb_frag_address(&skb_shinfo(skb)->frags[i]);  in hinic_copy_lp_data()
342 		frag_len = (int)skb_frag_size(&skb_shinfo(skb)->frags[i]);  in hinic_copy_lp_data()
346 	nic_dev->lb_test_rx_idx++;  in hinic_copy_lp_data()
350  * rxq_recv - Rx handler
351  * @rxq: rx queue
358 	struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);  in rxq_recv()
359 	struct net_device *netdev = rxq->netdev;  in rxq_recv()
361 	struct hinic_rq *rq = rxq->rq;  in rxq_recv()
381 		rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,  in rxq_recv()
389 		cqe = rq->cqe[ci];  in rxq_recv()
390 		status =  be32_to_cpu(cqe->status);  in rxq_recv()
391 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);  in rxq_recv()
397 		prefetch(skb->data);  in rxq_recv()
405 			num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -  in rxq_recv()
412 		offload_type = be32_to_cpu(cqe->offload_type);  in rxq_recv()
413 		vlan_len = be32_to_cpu(cqe->len);  in rxq_recv()
414 		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&  in rxq_recv()
420 		if (unlikely(nic_dev->flags & HINIC_LP_TEST))  in rxq_recv()
423 		skb_record_rx_queue(skb, qp->q_id);  in rxq_recv()
424 		skb->protocol = eth_type_trans(skb, rxq->netdev);  in rxq_recv()
426 		napi_gro_receive(&rxq->napi, skb);  in rxq_recv()
433 			rx_bytes += ((num_lro - 1) *  in rxq_recv()
437 			(u16)(pkt_len >> rxq->rx_buff_shift) +  in rxq_recv()
438 			((pkt_len & (rxq->buf_len - 1)) ? 1 : 0);  in rxq_recv()
441 		cqe->status = 0;  in rxq_recv()
447 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);  in rxq_recv()
451 	u64_stats_update_begin(&rxq->rxq_stats.syncp);  in rxq_recv()
452 	rxq->rxq_stats.pkts += pkts;  in rxq_recv()
453 	rxq->rxq_stats.bytes += rx_bytes;  in rxq_recv()
454 	u64_stats_update_end(&rxq->rxq_stats.syncp);  in rxq_recv()
462 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_poll()
463 	struct hinic_rq *rq = rxq->rq;  in rx_poll()
472 	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))  in rx_poll()
473 		hinic_hwdev_set_msix_state(nic_dev->hwdev,  in rx_poll()
474 					   rq->msix_entry,  in rx_poll()
482 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_add_napi()
484 	netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight);  in rx_add_napi()
485 	napi_enable(&rxq->napi);  in rx_add_napi()
490 	napi_disable(&rxq->napi);  in rx_del_napi()
491 	netif_napi_del(&rxq->napi);  in rx_del_napi()
497 	struct hinic_rq *rq = rxq->rq;  in rx_irq()
501 	nic_dev = netdev_priv(rxq->netdev);  in rx_irq()
502 	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))  in rx_irq()
503 		hinic_hwdev_set_msix_state(nic_dev->hwdev,  in rx_irq()
504 					   rq->msix_entry,  in rx_irq()
507 	nic_dev = netdev_priv(rxq->netdev);  in rx_irq()
508 	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);  in rx_irq()
510 	napi_schedule(&rxq->napi);  in rx_irq()
516 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_request_irq()
519 	struct hinic_hwdev *hwdev = nic_dev->hwdev;  in rx_request_irq()
520 	struct hinic_rq *rq = rxq->rq;  in rx_request_irq()
528 	hinic_hwdev_msix_set(hwdev, rq->msix_entry,  in rx_request_irq()
533 	intr_coal = &nic_dev->rx_intr_coalesce[qp->q_id];  in rx_request_irq()
534 	interrupt_info.msix_index = rq->msix_entry;  in rx_request_irq()
535 	interrupt_info.coalesce_timer_cnt = intr_coal->coalesce_timer_cfg;  in rx_request_irq()
536 	interrupt_info.pending_cnt = intr_coal->pending_limt;  in rx_request_irq()
537 	interrupt_info.resend_timer_cnt = intr_coal->resend_timer_cfg;  in rx_request_irq()
541 		netif_err(nic_dev, drv, rxq->netdev,  in rx_request_irq()
542 			  "Failed to set RX interrupt coalescing attribute\n");  in rx_request_irq()
546 	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);  in rx_request_irq()
550 	cpumask_set_cpu(qp->q_id % num_online_cpus(), &rq->affinity_mask);  in rx_request_irq()
551 	err = irq_set_affinity_hint(rq->irq, &rq->affinity_mask);  in rx_request_irq()
558 	free_irq(rq->irq, rxq);  in rx_request_irq()
566 	struct hinic_rq *rq = rxq->rq;  in rx_free_irq()
568 	irq_set_affinity_hint(rq->irq, NULL);  in rx_free_irq()
569 	free_irq(rq->irq, rxq);  in rx_free_irq()
574  * hinic_init_rxq - Initialize the Rx Queue
575  * @rxq: Logical Rx Queue
576  * @rq: Hardware Rx Queue to connect the Logical queue with
577  * @netdev: network device to connect the Logical queue with
579  * Return 0 - Success, negative - Failure
587 	rxq->netdev = netdev;  in hinic_init_rxq()
588 	rxq->rq = rq;  in hinic_init_rxq()
589 	rxq->buf_len = HINIC_RX_BUF_SZ;  in hinic_init_rxq()
590 	rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ);  in hinic_init_rxq()
594 	rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL,  in hinic_init_rxq()
595 				       "%s_rxq%d", netdev->name, qp->q_id);  in hinic_init_rxq()
596 	if (!rxq->irq_name)  in hinic_init_rxq()
597 		return -ENOMEM;  in hinic_init_rxq()
601 		err = -ENOMEM;  in hinic_init_rxq()
607 		netdev_err(netdev, "Failed to request Rx irq\n");  in hinic_init_rxq()
616 	devm_kfree(&netdev->dev, rxq->irq_name);  in hinic_init_rxq()
621  * hinic_clean_rxq - Clean the Rx Queue
622  * @rxq: Logical Rx Queue
626 	struct net_device *netdev = rxq->netdev;  in hinic_clean_rxq()
631 	devm_kfree(&netdev->dev, rxq->irq_name);  in hinic_clean_rxq()