1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
26 	iowrite32be(val, (mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
28 	ioread32be((mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
133 	tasklet_schedule(&q->tasklet);  in mlxsw_pci_queue_tasklet_schedule()
139 	return q->mem_item.buf + (elem_size * elem_index);  in __mlxsw_pci_queue_elem_get()
145 	return &q->elem_info[elem_index];  in mlxsw_pci_queue_elem_info_get()
151 	int index = q->producer_counter & (q->count - 1);  in mlxsw_pci_queue_elem_info_producer_get()
153 	if ((u16) (q->producer_counter - q->consumer_counter) == q->count)  in mlxsw_pci_queue_elem_info_producer_get()
161 	int index = q->consumer_counter & (q->count - 1);  in mlxsw_pci_queue_elem_info_consumer_get()
168 	return mlxsw_pci_queue_elem_info_get(q, elem_index)->elem;  in mlxsw_pci_queue_elem_get()
173 	return owner_bit != !!(q->consumer_counter & q->count);  in mlxsw_pci_elem_hw_owned()
180 	return &mlxsw_pci->queues[q_type];  in mlxsw_pci_queue_type_group_get()
189 	return queue_group->count;  in __mlxsw_pci_queue_count()
206 	return &mlxsw_pci->queues[q_type].q[q_num];  in __mlxsw_pci_queue_get()
240 			  DOORBELL(mlxsw_pci->doorbell_offset,  in __mlxsw_pci_queue_doorbell_set()
241 				   mlxsw_pci_doorbell_type_offset[q->type],  in __mlxsw_pci_queue_doorbell_set()
242 				   q->num), val);  in __mlxsw_pci_queue_doorbell_set()
250 			  DOORBELL(mlxsw_pci->doorbell_offset,  in __mlxsw_pci_queue_doorbell_arm_set()
251 				   mlxsw_pci_doorbell_arm_type_offset[q->type],  in __mlxsw_pci_queue_doorbell_arm_set()
252 				   q->num), val);  in __mlxsw_pci_queue_doorbell_arm_set()
259 	__mlxsw_pci_queue_doorbell_set(mlxsw_pci, q, q->producer_counter);  in mlxsw_pci_queue_doorbell_producer_ring()
267 				       q->consumer_counter + q->count);  in mlxsw_pci_queue_doorbell_consumer_ring()
275 	__mlxsw_pci_queue_doorbell_arm_set(mlxsw_pci, q, q->consumer_counter);  in mlxsw_pci_queue_doorbell_arm_consumer_ring()
281 	return q->mem_item.mapaddr + MLXSW_PCI_PAGE_SIZE * page_index;  in __mlxsw_pci_queue_page_get()
284 static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_sdq_init()  argument
291 	q->producer_counter = 0;  in mlxsw_pci_sdq_init()
292 	q->consumer_counter = 0;  in mlxsw_pci_sdq_init()
293 	tclass = q->num == MLXSW_PCI_SDQ_EMAD_INDEX ? MLXSW_PCI_SDQ_EMAD_TC :  in mlxsw_pci_sdq_init()
297 	mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num);  in mlxsw_pci_sdq_init()
298 	mlxsw_cmd_mbox_sw2hw_dq_sdq_tclass_set(mbox, tclass);  in mlxsw_pci_sdq_init()
299 	mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */  in mlxsw_pci_sdq_init()
303 		mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);  in mlxsw_pci_sdq_init()
306 	err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_sdq_init()
316 	mlxsw_cmd_hw2sw_sdq(mlxsw_pci->core, q->num);  in mlxsw_pci_sdq_fini()
323 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_wqe_frag_map()
326 	mapaddr = dma_map_single(&pdev->dev, frag_data, frag_len, direction);  in mlxsw_pci_wqe_frag_map()
327 	if (unlikely(dma_mapping_error(&pdev->dev, mapaddr))) {  in mlxsw_pci_wqe_frag_map()
328 		dev_err_ratelimited(&pdev->dev, "failed to dma map tx frag\n");  in mlxsw_pci_wqe_frag_map()
329 		return -EIO;  in mlxsw_pci_wqe_frag_map()
339 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_wqe_frag_unmap()
345 	dma_unmap_single(&pdev->dev, mapaddr, frag_len, direction);  in mlxsw_pci_wqe_frag_unmap()
352 	char *wqe = elem_info->elem;  in mlxsw_pci_rdq_skb_alloc()
358 		return -ENOMEM;  in mlxsw_pci_rdq_skb_alloc()
360 	err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,  in mlxsw_pci_rdq_skb_alloc()
365 	elem_info->u.rdq.skb = skb;  in mlxsw_pci_rdq_skb_alloc()
379 	skb = elem_info->u.rdq.skb;  in mlxsw_pci_rdq_skb_free()
380 	wqe = elem_info->elem;  in mlxsw_pci_rdq_skb_free()
386 static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_rdq_init()  argument
394 	q->producer_counter = 0;  in mlxsw_pci_rdq_init()
395 	q->consumer_counter = 0;  in mlxsw_pci_rdq_init()
400 	mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, sdq_count + q->num);  in mlxsw_pci_rdq_init()
401 	mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */  in mlxsw_pci_rdq_init()
405 		mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);  in mlxsw_pci_rdq_init()
408 	err = mlxsw_cmd_sw2hw_rdq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_rdq_init()
414 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_rdq_init()
421 		q->producer_counter++;  in mlxsw_pci_rdq_init()
428 	for (i--; i >= 0; i--) {  in mlxsw_pci_rdq_init()
432 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);  in mlxsw_pci_rdq_init()
443 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);  in mlxsw_pci_rdq_fini()
444 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_rdq_fini()
453 	q->u.cq.v = mlxsw_pci->max_cqe_ver;  in mlxsw_pci_cq_pre_init()
456 	if (q->u.cq.v == MLXSW_PCI_CQE_V2 &&  in mlxsw_pci_cq_pre_init()
457 	    q->num < mlxsw_pci->num_sdq_cqs)  in mlxsw_pci_cq_pre_init()
458 		q->u.cq.v = MLXSW_PCI_CQE_V1;  in mlxsw_pci_cq_pre_init()
461 static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_cq_init()  argument
467 	q->consumer_counter = 0;  in mlxsw_pci_cq_init()
469 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_cq_init()
472 		mlxsw_pci_cqe_owner_set(q->u.cq.v, elem, 1);  in mlxsw_pci_cq_init()
475 	if (q->u.cq.v == MLXSW_PCI_CQE_V1)  in mlxsw_pci_cq_init()
476 		mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox,  in mlxsw_pci_cq_init()
478 	else if (q->u.cq.v == MLXSW_PCI_CQE_V2)  in mlxsw_pci_cq_init()
479 		mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox,  in mlxsw_pci_cq_init()
482 	mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM);  in mlxsw_pci_cq_init()
483 	mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0);  in mlxsw_pci_cq_init()
484 	mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count));  in mlxsw_pci_cq_init()
488 		mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr);  in mlxsw_pci_cq_init()
490 	err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_cq_init()
501 	mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num);  in mlxsw_pci_cq_fini()
509 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_cqe_sdq_handle()
516 	spin_lock(&q->lock);  in mlxsw_pci_cqe_sdq_handle()
518 	tx_info = mlxsw_skb_cb(elem_info->u.sdq.skb)->tx_info;  in mlxsw_pci_cqe_sdq_handle()
519 	skb = elem_info->u.sdq.skb;  in mlxsw_pci_cqe_sdq_handle()
520 	wqe = elem_info->elem;  in mlxsw_pci_cqe_sdq_handle()
525 		     skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {  in mlxsw_pci_cqe_sdq_handle()
526 		mlxsw_core_ptp_transmitted(mlxsw_pci->core, skb,  in mlxsw_pci_cqe_sdq_handle()
533 	elem_info->u.sdq.skb = NULL;  in mlxsw_pci_cqe_sdq_handle()
535 	if (q->consumer_counter++ != consumer_counter_limit)  in mlxsw_pci_cqe_sdq_handle()
536 		dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in SDQ\n");  in mlxsw_pci_cqe_sdq_handle()
537 	spin_unlock(&q->lock);  in mlxsw_pci_cqe_sdq_handle()
546 		cb->rx_md_info.tx_port_is_lag = true;  in mlxsw_pci_cqe_rdq_md_tx_port_init()
547 		cb->rx_md_info.tx_lag_id = mlxsw_pci_cqe2_tx_lag_id_get(cqe);  in mlxsw_pci_cqe_rdq_md_tx_port_init()
548 		cb->rx_md_info.tx_lag_port_index =  in mlxsw_pci_cqe_rdq_md_tx_port_init()
551 		cb->rx_md_info.tx_port_is_lag = false;  in mlxsw_pci_cqe_rdq_md_tx_port_init()
552 		cb->rx_md_info.tx_sys_port =  in mlxsw_pci_cqe_rdq_md_tx_port_init()
556 	if (cb->rx_md_info.tx_sys_port != MLXSW_PCI_CQE2_TX_PORT_MULTI_PORT &&  in mlxsw_pci_cqe_rdq_md_tx_port_init()
557 	    cb->rx_md_info.tx_sys_port != MLXSW_PCI_CQE2_TX_PORT_INVALID)  in mlxsw_pci_cqe_rdq_md_tx_port_init()
558 		cb->rx_md_info.tx_port_valid = 1;  in mlxsw_pci_cqe_rdq_md_tx_port_init()
560 		cb->rx_md_info.tx_port_valid = 0;  in mlxsw_pci_cqe_rdq_md_tx_port_init()
567 	cb->rx_md_info.tx_congestion = mlxsw_pci_cqe2_mirror_cong_get(cqe);  in mlxsw_pci_cqe_rdq_md_init()
568 	if (cb->rx_md_info.tx_congestion != MLXSW_PCI_CQE2_MIRROR_CONG_INVALID)  in mlxsw_pci_cqe_rdq_md_init()
569 		cb->rx_md_info.tx_congestion_valid = 1;  in mlxsw_pci_cqe_rdq_md_init()
571 		cb->rx_md_info.tx_congestion_valid = 0;  in mlxsw_pci_cqe_rdq_md_init()
572 	cb->rx_md_info.tx_congestion <<= MLXSW_PCI_CQE2_MIRROR_CONG_SHIFT;  in mlxsw_pci_cqe_rdq_md_init()
574 	cb->rx_md_info.latency = mlxsw_pci_cqe2_mirror_latency_get(cqe);  in mlxsw_pci_cqe_rdq_md_init()
575 	if (cb->rx_md_info.latency != MLXSW_PCI_CQE2_MIRROR_LATENCY_INVALID)  in mlxsw_pci_cqe_rdq_md_init()
576 		cb->rx_md_info.latency_valid = 1;  in mlxsw_pci_cqe_rdq_md_init()
578 		cb->rx_md_info.latency_valid = 0;  in mlxsw_pci_cqe_rdq_md_init()
580 	cb->rx_md_info.tx_tc = mlxsw_pci_cqe2_mirror_tclass_get(cqe);  in mlxsw_pci_cqe_rdq_md_init()
581 	if (cb->rx_md_info.tx_tc != MLXSW_PCI_CQE2_MIRROR_TCLASS_INVALID)  in mlxsw_pci_cqe_rdq_md_init()
582 		cb->rx_md_info.tx_tc_valid = 1;  in mlxsw_pci_cqe_rdq_md_init()
584 		cb->rx_md_info.tx_tc_valid = 0;  in mlxsw_pci_cqe_rdq_md_init()
594 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_cqe_rdq_handle()
603 	skb = elem_info->u.rdq.skb;  in mlxsw_pci_cqe_rdq_handle()
604 	memcpy(wqe, elem_info->elem, MLXSW_PCI_WQE_SIZE);  in mlxsw_pci_cqe_rdq_handle()
606 	if (q->consumer_counter++ != consumer_counter_limit)  in mlxsw_pci_cqe_rdq_handle()
607 		dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in RDQ\n");  in mlxsw_pci_cqe_rdq_handle()
611 		dev_err_ratelimited(&pdev->dev, "Failed to alloc skb for RDQ\n");  in mlxsw_pci_cqe_rdq_handle()
633 		if (mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2)  in mlxsw_pci_cqe_rdq_handle()
635 		mlxsw_skb_cb(skb)->rx_md_info.cookie_index = cookie_index;  in mlxsw_pci_cqe_rdq_handle()
638 		   mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2) {  in mlxsw_pci_cqe_rdq_handle()
642 		   mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2) {  in mlxsw_pci_cqe_rdq_handle()
648 		byte_count -= ETH_FCS_LEN;  in mlxsw_pci_cqe_rdq_handle()
650 	mlxsw_core_skb_receive(mlxsw_pci->core, skb, &rx_info);  in mlxsw_pci_cqe_rdq_handle()
654 	q->producer_counter++;  in mlxsw_pci_cqe_rdq_handle()
666 	elem = elem_info->elem;  in mlxsw_pci_cq_sw_cqe_get()
667 	owner_bit = mlxsw_pci_cqe_owner_get(q->u.cq.v, elem);  in mlxsw_pci_cq_sw_cqe_get()
670 	q->consumer_counter++;  in mlxsw_pci_cq_sw_cqe_get()
678 	struct mlxsw_pci *mlxsw_pci = q->pci;  in mlxsw_pci_cq_tasklet()
681 	int credits = q->count >> 1;  in mlxsw_pci_cq_tasklet()
685 		u8 sendq = mlxsw_pci_cqe_sr_get(q->u.cq.v, cqe);  in mlxsw_pci_cq_tasklet()
686 		u8 dqn = mlxsw_pci_cqe_dqn_get(q->u.cq.v, cqe);  in mlxsw_pci_cq_tasklet()
689 		memcpy(ncqe, cqe, q->elem_size);  in mlxsw_pci_cq_tasklet()
698 			q->u.cq.comp_sdq_count++;  in mlxsw_pci_cq_tasklet()
704 						 wqe_counter, q->u.cq.v, ncqe);  in mlxsw_pci_cq_tasklet()
705 			q->u.cq.comp_rdq_count++;  in mlxsw_pci_cq_tasklet()
716 	return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT :  in mlxsw_pci_cq_elem_count()
722 	return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_SIZE :  in mlxsw_pci_cq_elem_size()
726 static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_eq_init()  argument
732 	q->consumer_counter = 0;  in mlxsw_pci_eq_init()
734 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_eq_init()
740 	mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */  in mlxsw_pci_eq_init()
741 	mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */  in mlxsw_pci_eq_init()
742 	mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count));  in mlxsw_pci_eq_init()
746 		mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr);  in mlxsw_pci_eq_init()
748 	err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_eq_init()
759 	mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num);  in mlxsw_pci_eq_fini()
764 	mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe);  in mlxsw_pci_eq_cmd_event()
765 	mlxsw_pci->cmd.comp.out_param =  in mlxsw_pci_eq_cmd_event()
768 	mlxsw_pci->cmd.wait_done = true;  in mlxsw_pci_eq_cmd_event()
769 	wake_up(&mlxsw_pci->cmd.wait);  in mlxsw_pci_eq_cmd_event()
779 	elem = elem_info->elem;  in mlxsw_pci_eq_sw_eqe_get()
783 	q->consumer_counter++;  in mlxsw_pci_eq_sw_eqe_get()
791 	struct mlxsw_pci *mlxsw_pci = q->pci;  in mlxsw_pci_eq_tasklet()
798 	int credits = q->count >> 1;  in mlxsw_pci_eq_tasklet()
808 		switch (q->num) {  in mlxsw_pci_eq_tasklet()
811 			q->u.eq.ev_cmd_count++;  in mlxsw_pci_eq_tasklet()
817 			q->u.eq.ev_comp_count++;  in mlxsw_pci_eq_tasklet()
820 			q->u.eq.ev_other_count++;  in mlxsw_pci_eq_tasklet()
843 	int (*init)(struct mlxsw_pci *mlxsw_pci, char *mbox,
889 static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_queue_init()  argument
893 	struct mlxsw_pci_mem_item *mem_item = &q->mem_item;  in mlxsw_pci_queue_init()
897 	q->num = q_num;  in mlxsw_pci_queue_init()
898 	if (q_ops->pre_init)  in mlxsw_pci_queue_init()
899 		q_ops->pre_init(mlxsw_pci, q);  in mlxsw_pci_queue_init()
901 	spin_lock_init(&q->lock);  in mlxsw_pci_queue_init()
902 	q->count = q_ops->elem_count_f ? q_ops->elem_count_f(q) :  in mlxsw_pci_queue_init()
903 					 q_ops->elem_count;  in mlxsw_pci_queue_init()
904 	q->elem_size = q_ops->elem_size_f ? q_ops->elem_size_f(q) :  in mlxsw_pci_queue_init()
905 					    q_ops->elem_size;  in mlxsw_pci_queue_init()
906 	q->type = q_ops->type;  in mlxsw_pci_queue_init()
907 	q->pci = mlxsw_pci;  in mlxsw_pci_queue_init()
909 	if (q_ops->tasklet)  in mlxsw_pci_queue_init()
910 		tasklet_setup(&q->tasklet, q_ops->tasklet);  in mlxsw_pci_queue_init()
912 	mem_item->size = MLXSW_PCI_AQ_SIZE;  in mlxsw_pci_queue_init()
913 	mem_item->buf = dma_alloc_coherent(&mlxsw_pci->pdev->dev,  in mlxsw_pci_queue_init()
914 					   mem_item->size, &mem_item->mapaddr,  in mlxsw_pci_queue_init()
916 	if (!mem_item->buf)  in mlxsw_pci_queue_init()
917 		return -ENOMEM;  in mlxsw_pci_queue_init()
919 	q->elem_info = kcalloc(q->count, sizeof(*q->elem_info), GFP_KERNEL);  in mlxsw_pci_queue_init()
920 	if (!q->elem_info) {  in mlxsw_pci_queue_init()
921 		err = -ENOMEM;  in mlxsw_pci_queue_init()
928 	for (i = 0; i < q->count; i++) {  in mlxsw_pci_queue_init()
932 		elem_info->elem =  in mlxsw_pci_queue_init()
933 			__mlxsw_pci_queue_elem_get(q, q->elem_size, i);  in mlxsw_pci_queue_init()
936 	mlxsw_cmd_mbox_zero(mbox);  in mlxsw_pci_queue_init()
937 	err = q_ops->init(mlxsw_pci, mbox, q);  in mlxsw_pci_queue_init()
943 	kfree(q->elem_info);  in mlxsw_pci_queue_init()
945 	dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,  in mlxsw_pci_queue_init()
946 			  mem_item->buf, mem_item->mapaddr);  in mlxsw_pci_queue_init()
954 	struct mlxsw_pci_mem_item *mem_item = &q->mem_item;  in mlxsw_pci_queue_fini()
956 	q_ops->fini(mlxsw_pci, q);  in mlxsw_pci_queue_fini()
957 	kfree(q->elem_info);  in mlxsw_pci_queue_fini()
958 	dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,  in mlxsw_pci_queue_fini()
959 			  mem_item->buf, mem_item->mapaddr);  in mlxsw_pci_queue_fini()
962 static int mlxsw_pci_queue_group_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_queue_group_init()  argument
970 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);  in mlxsw_pci_queue_group_init()
971 	queue_group->q = kcalloc(num_qs, sizeof(*queue_group->q), GFP_KERNEL);  in mlxsw_pci_queue_group_init()
972 	if (!queue_group->q)  in mlxsw_pci_queue_group_init()
973 		return -ENOMEM;  in mlxsw_pci_queue_group_init()
976 		err = mlxsw_pci_queue_init(mlxsw_pci, mbox, q_ops,  in mlxsw_pci_queue_group_init()
977 					   &queue_group->q[i], i);  in mlxsw_pci_queue_group_init()
981 	queue_group->count = num_qs;  in mlxsw_pci_queue_group_init()
986 	for (i--; i >= 0; i--)  in mlxsw_pci_queue_group_init()
987 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);  in mlxsw_pci_queue_group_init()
988 	kfree(queue_group->q);  in mlxsw_pci_queue_group_init()
998 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);  in mlxsw_pci_queue_group_fini()
999 	for (i = 0; i < queue_group->count; i++)  in mlxsw_pci_queue_group_fini()
1000 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);  in mlxsw_pci_queue_group_fini()
1001 	kfree(queue_group->q);  in mlxsw_pci_queue_group_fini()
1004 static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)  in mlxsw_pci_aqs_init()  argument
1006 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_aqs_init()
1018 	mlxsw_cmd_mbox_zero(mbox);  in mlxsw_pci_aqs_init()
1019 	err = mlxsw_cmd_query_aq_cap(mlxsw_pci->core, mbox);  in mlxsw_pci_aqs_init()
1023 	num_sdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_sdqs_get(mbox);  in mlxsw_pci_aqs_init()
1024 	sdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_sdq_sz_get(mbox);  in mlxsw_pci_aqs_init()
1025 	num_rdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_rdqs_get(mbox);  in mlxsw_pci_aqs_init()
1026 	rdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_rdq_sz_get(mbox);  in mlxsw_pci_aqs_init()
1027 	num_cqs = mlxsw_cmd_mbox_query_aq_cap_max_num_cqs_get(mbox);  in mlxsw_pci_aqs_init()
1028 	cq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cq_sz_get(mbox);  in mlxsw_pci_aqs_init()
1029 	cqv2_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cqv2_sz_get(mbox);  in mlxsw_pci_aqs_init()
1030 	num_eqs = mlxsw_cmd_mbox_query_aq_cap_max_num_eqs_get(mbox);  in mlxsw_pci_aqs_init()
1031 	eq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_eq_sz_get(mbox);  in mlxsw_pci_aqs_init()
1036 		dev_err(&pdev->dev, "Unsupported number of queues\n");  in mlxsw_pci_aqs_init()
1037 		return -EINVAL;  in mlxsw_pci_aqs_init()
1043 	    (mlxsw_pci->max_cqe_ver == MLXSW_PCI_CQE_V2 &&  in mlxsw_pci_aqs_init()
1046 		dev_err(&pdev->dev, "Unsupported number of async queue descriptors\n");  in mlxsw_pci_aqs_init()
1047 		return -EINVAL;  in mlxsw_pci_aqs_init()
1050 	mlxsw_pci->num_sdq_cqs = num_sdqs;  in mlxsw_pci_aqs_init()
1052 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops,  in mlxsw_pci_aqs_init()
1055 		dev_err(&pdev->dev, "Failed to initialize event queues\n");  in mlxsw_pci_aqs_init()
1059 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_cq_ops,  in mlxsw_pci_aqs_init()
1062 		dev_err(&pdev->dev, "Failed to initialize completion queues\n");  in mlxsw_pci_aqs_init()
1066 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_sdq_ops,  in mlxsw_pci_aqs_init()
1069 		dev_err(&pdev->dev, "Failed to initialize send descriptor queues\n");  in mlxsw_pci_aqs_init()
1073 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_rdq_ops,  in mlxsw_pci_aqs_init()
1076 		dev_err(&pdev->dev, "Failed to initialize receive descriptor queues\n");  in mlxsw_pci_aqs_init()
1081 	mlxsw_pci->cmd.nopoll = true;  in mlxsw_pci_aqs_init()
1095 	mlxsw_pci->cmd.nopoll = false;  in mlxsw_pci_aqs_fini()
1104 				     char *mbox, int index,  in mlxsw_pci_config_profile_swid_config()  argument
1109 	if (swid->used_type) {  in mlxsw_pci_config_profile_swid_config()
1111 			mbox, index, swid->type);  in mlxsw_pci_config_profile_swid_config()
1114 	if (swid->used_properties) {  in mlxsw_pci_config_profile_swid_config()
1116 			mbox, index, swid->properties);  in mlxsw_pci_config_profile_swid_config()
1119 	mlxsw_cmd_mbox_config_profile_swid_config_mask_set(mbox, index, mask);  in mlxsw_pci_config_profile_swid_config()
1130 	err = mlxsw_core_kvd_sizes_get(mlxsw_pci->core, profile,  in mlxsw_pci_profile_get_kvd_sizes()
1143 static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_config_profile()  argument
1150 	mlxsw_cmd_mbox_zero(mbox);  in mlxsw_pci_config_profile()
1152 	if (profile->used_max_vepa_channels) {  in mlxsw_pci_config_profile()
1154 			mbox, 1);  in mlxsw_pci_config_profile()
1156 			mbox, profile->max_vepa_channels);  in mlxsw_pci_config_profile()
1158 	if (profile->used_max_mid) {  in mlxsw_pci_config_profile()
1160 			mbox, 1);  in mlxsw_pci_config_profile()
1162 			mbox, profile->max_mid);  in mlxsw_pci_config_profile()
1164 	if (profile->used_max_pgt) {  in mlxsw_pci_config_profile()
1166 			mbox, 1);  in mlxsw_pci_config_profile()
1168 			mbox, profile->max_pgt);  in mlxsw_pci_config_profile()
1170 	if (profile->used_max_system_port) {  in mlxsw_pci_config_profile()
1172 			mbox, 1);  in mlxsw_pci_config_profile()
1174 			mbox, profile->max_system_port);  in mlxsw_pci_config_profile()
1176 	if (profile->used_max_vlan_groups) {  in mlxsw_pci_config_profile()
1178 			mbox, 1);  in mlxsw_pci_config_profile()
1180 			mbox, profile->max_vlan_groups);  in mlxsw_pci_config_profile()
1182 	if (profile->used_max_regions) {  in mlxsw_pci_config_profile()
1184 			mbox, 1);  in mlxsw_pci_config_profile()
1186 			mbox, profile->max_regions);  in mlxsw_pci_config_profile()
1188 	if (profile->used_flood_tables) {  in mlxsw_pci_config_profile()
1190 			mbox, 1);  in mlxsw_pci_config_profile()
1192 			mbox, profile->max_flood_tables);  in mlxsw_pci_config_profile()
1194 			mbox, profile->max_vid_flood_tables);  in mlxsw_pci_config_profile()
1196 			mbox, profile->max_fid_offset_flood_tables);  in mlxsw_pci_config_profile()
1198 			mbox, profile->fid_offset_flood_table_size);  in mlxsw_pci_config_profile()
1200 			mbox, profile->max_fid_flood_tables);  in mlxsw_pci_config_profile()
1202 			mbox, profile->fid_flood_table_size);  in mlxsw_pci_config_profile()
1204 	if (profile->used_flood_mode) {  in mlxsw_pci_config_profile()
1206 			mbox, 1);  in mlxsw_pci_config_profile()
1208 			mbox, profile->flood_mode);  in mlxsw_pci_config_profile()
1210 	if (profile->used_max_ib_mc) {  in mlxsw_pci_config_profile()
1212 			mbox, 1);  in mlxsw_pci_config_profile()
1214 			mbox, profile->max_ib_mc);  in mlxsw_pci_config_profile()
1216 	if (profile->used_max_pkey) {  in mlxsw_pci_config_profile()
1218 			mbox, 1);  in mlxsw_pci_config_profile()
1220 			mbox, profile->max_pkey);  in mlxsw_pci_config_profile()
1222 	if (profile->used_ar_sec) {  in mlxsw_pci_config_profile()
1224 			mbox, 1);  in mlxsw_pci_config_profile()
1226 			mbox, profile->ar_sec);  in mlxsw_pci_config_profile()
1228 	if (profile->used_adaptive_routing_group_cap) {  in mlxsw_pci_config_profile()
1230 			mbox, 1);  in mlxsw_pci_config_profile()
1232 			mbox, profile->adaptive_routing_group_cap);  in mlxsw_pci_config_profile()
1234 	if (profile->used_kvd_sizes && MLXSW_RES_VALID(res, KVD_SIZE)) {  in mlxsw_pci_config_profile()
1239 		mlxsw_cmd_mbox_config_profile_set_kvd_linear_size_set(mbox, 1);  in mlxsw_pci_config_profile()
1240 		mlxsw_cmd_mbox_config_profile_kvd_linear_size_set(mbox,  in mlxsw_pci_config_profile()
1242 		mlxsw_cmd_mbox_config_profile_set_kvd_hash_single_size_set(mbox,  in mlxsw_pci_config_profile()
1244 		mlxsw_cmd_mbox_config_profile_kvd_hash_single_size_set(mbox,  in mlxsw_pci_config_profile()
1247 								mbox, 1);  in mlxsw_pci_config_profile()
1248 		mlxsw_cmd_mbox_config_profile_kvd_hash_double_size_set(mbox,  in mlxsw_pci_config_profile()
1251 	if (profile->used_kvh_xlt_cache_mode) {  in mlxsw_pci_config_profile()
1253 			mbox, 1);  in mlxsw_pci_config_profile()
1255 			mbox, profile->kvh_xlt_cache_mode);  in mlxsw_pci_config_profile()
1259 		mlxsw_pci_config_profile_swid_config(mlxsw_pci, mbox, i,  in mlxsw_pci_config_profile()
1260 						     &profile->swid_config[i]);  in mlxsw_pci_config_profile()
1262 	if (mlxsw_pci->max_cqe_ver > MLXSW_PCI_CQE_V0) {  in mlxsw_pci_config_profile()
1263 		mlxsw_cmd_mbox_config_profile_set_cqe_version_set(mbox, 1);  in mlxsw_pci_config_profile()
1264 		mlxsw_cmd_mbox_config_profile_cqe_version_set(mbox, 1);  in mlxsw_pci_config_profile()
1267 	return mlxsw_cmd_config_profile_set(mlxsw_pci->core, mbox);  in mlxsw_pci_config_profile()
1272 					  char *mbox)  in mlxsw_pci_boardinfo_xm_process()  argument
1274 	int count = mlxsw_cmd_mbox_boardinfo_xm_num_local_ports_get(mbox);  in mlxsw_pci_boardinfo_xm_process()
1277 	if (!mlxsw_cmd_mbox_boardinfo_xm_exists_get(mbox))  in mlxsw_pci_boardinfo_xm_process()
1280 	bus_info->xm_exists = true;  in mlxsw_pci_boardinfo_xm_process()
1283 		dev_err(&mlxsw_pci->pdev->dev, "Invalid number of XM local ports\n");  in mlxsw_pci_boardinfo_xm_process()
1284 		return -EINVAL;  in mlxsw_pci_boardinfo_xm_process()
1286 	bus_info->xm_local_ports_count = count;  in mlxsw_pci_boardinfo_xm_process()
1288 		bus_info->xm_local_ports[i] =  in mlxsw_pci_boardinfo_xm_process()
1289 			mlxsw_cmd_mbox_boardinfo_xm_local_port_entry_get(mbox,  in mlxsw_pci_boardinfo_xm_process()
1294 static int mlxsw_pci_boardinfo(struct mlxsw_pci *mlxsw_pci, char *mbox)  in mlxsw_pci_boardinfo()  argument
1296 	struct mlxsw_bus_info *bus_info = &mlxsw_pci->bus_info;  in mlxsw_pci_boardinfo()
1299 	mlxsw_cmd_mbox_zero(mbox);  in mlxsw_pci_boardinfo()
1300 	err = mlxsw_cmd_boardinfo(mlxsw_pci->core, mbox);  in mlxsw_pci_boardinfo()
1303 	mlxsw_cmd_mbox_boardinfo_vsd_memcpy_from(mbox, bus_info->vsd);  in mlxsw_pci_boardinfo()
1304 	mlxsw_cmd_mbox_boardinfo_psid_memcpy_from(mbox, bus_info->psid);  in mlxsw_pci_boardinfo()
1306 	return mlxsw_pci_boardinfo_xm_process(mlxsw_pci, bus_info, mbox);  in mlxsw_pci_boardinfo()
1309 static int mlxsw_pci_fw_area_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_fw_area_init()  argument
1317 	mlxsw_pci->fw_area.items = kcalloc(num_pages, sizeof(*mem_item),  in mlxsw_pci_fw_area_init()
1319 	if (!mlxsw_pci->fw_area.items)  in mlxsw_pci_fw_area_init()
1320 		return -ENOMEM;  in mlxsw_pci_fw_area_init()
1321 	mlxsw_pci->fw_area.count = num_pages;  in mlxsw_pci_fw_area_init()
1323 	mlxsw_cmd_mbox_zero(mbox);  in mlxsw_pci_fw_area_init()
1325 		mem_item = &mlxsw_pci->fw_area.items[i];  in mlxsw_pci_fw_area_init()
1327 		mem_item->size = MLXSW_PCI_PAGE_SIZE;  in mlxsw_pci_fw_area_init()
1328 		mem_item->buf = dma_alloc_coherent(&mlxsw_pci->pdev->dev,  in mlxsw_pci_fw_area_init()
1329 						   mem_item->size,  in mlxsw_pci_fw_area_init()
1330 						   &mem_item->mapaddr, GFP_KERNEL);  in mlxsw_pci_fw_area_init()
1331 		if (!mem_item->buf) {  in mlxsw_pci_fw_area_init()
1332 			err = -ENOMEM;  in mlxsw_pci_fw_area_init()
1335 		mlxsw_cmd_mbox_map_fa_pa_set(mbox, nent, mem_item->mapaddr);  in mlxsw_pci_fw_area_init()
1336 		mlxsw_cmd_mbox_map_fa_log2size_set(mbox, nent, 0); /* 1 page */  in mlxsw_pci_fw_area_init()
1338 			err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);  in mlxsw_pci_fw_area_init()
1342 			mlxsw_cmd_mbox_zero(mbox);  in mlxsw_pci_fw_area_init()
1347 		err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);  in mlxsw_pci_fw_area_init()
1356 	for (i--; i >= 0; i--) {  in mlxsw_pci_fw_area_init()
1357 		mem_item = &mlxsw_pci->fw_area.items[i];  in mlxsw_pci_fw_area_init()
1359 		dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,  in mlxsw_pci_fw_area_init()
1360 				  mem_item->buf, mem_item->mapaddr);  in mlxsw_pci_fw_area_init()
1362 	kfree(mlxsw_pci->fw_area.items);  in mlxsw_pci_fw_area_init()
1371 	mlxsw_cmd_unmap_fa(mlxsw_pci->core);  in mlxsw_pci_fw_area_fini()
1373 	for (i = 0; i < mlxsw_pci->fw_area.count; i++) {  in mlxsw_pci_fw_area_fini()
1374 		mem_item = &mlxsw_pci->fw_area.items[i];  in mlxsw_pci_fw_area_fini()
1376 		dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,  in mlxsw_pci_fw_area_fini()
1377 				  mem_item->buf, mem_item->mapaddr);  in mlxsw_pci_fw_area_fini()
1379 	kfree(mlxsw_pci->fw_area.items);  in mlxsw_pci_fw_area_fini()
1396 				struct mlxsw_pci_mem_item *mbox)  in mlxsw_pci_mbox_alloc()  argument
1398 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_mbox_alloc()
1401 	mbox->size = MLXSW_CMD_MBOX_SIZE;  in mlxsw_pci_mbox_alloc()
1402 	mbox->buf = dma_alloc_coherent(&pdev->dev, MLXSW_CMD_MBOX_SIZE,  in mlxsw_pci_mbox_alloc()
1403 				       &mbox->mapaddr, GFP_KERNEL);  in mlxsw_pci_mbox_alloc()
1404 	if (!mbox->buf) {  in mlxsw_pci_mbox_alloc()
1405 		dev_err(&pdev->dev, "Failed allocating memory for mailbox\n");  in mlxsw_pci_mbox_alloc()
1406 		err = -ENOMEM;  in mlxsw_pci_mbox_alloc()
1413 				struct mlxsw_pci_mem_item *mbox)  in mlxsw_pci_mbox_free()  argument
1415 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_mbox_free()
1417 	dma_free_coherent(&pdev->dev, MLXSW_CMD_MBOX_SIZE, mbox->buf,  in mlxsw_pci_mbox_free()
1418 			  mbox->mapaddr);  in mlxsw_pci_mbox_free()
1441 	return -EBUSY;  in mlxsw_pci_sys_ready_wait()
1447 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_sw_reset()
1454 		dev_err(&pdev->dev, "Failed to reach system ready status before reset. Status is 0x%x\n",  in mlxsw_pci_sw_reset()
1460 	err = mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl);  in mlxsw_pci_sw_reset()
1466 		dev_err(&pdev->dev, "Failed to reach system ready status after reset. Status is 0x%x\n",  in mlxsw_pci_sw_reset()
1478 	err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX);  in mlxsw_pci_alloc_irq_vectors()
1480 		dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n");  in mlxsw_pci_alloc_irq_vectors()
1486 	pci_free_irq_vectors(mlxsw_pci->pdev);  in mlxsw_pci_free_irq_vectors()
1494 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_init()
1495 	char *mbox;  in mlxsw_pci_init()  local
1499 	mlxsw_pci->core = mlxsw_core;  in mlxsw_pci_init()
1501 	mbox = mlxsw_cmd_mbox_alloc();  in mlxsw_pci_init()
1502 	if (!mbox)  in mlxsw_pci_init()
1503 		return -ENOMEM;  in mlxsw_pci_init()
1505 	err = mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id);  in mlxsw_pci_init()
1511 		dev_err(&pdev->dev, "MSI-X init failed\n");  in mlxsw_pci_init()
1515 	err = mlxsw_cmd_query_fw(mlxsw_core, mbox);  in mlxsw_pci_init()
1519 	mlxsw_pci->bus_info.fw_rev.major =  in mlxsw_pci_init()
1520 		mlxsw_cmd_mbox_query_fw_fw_rev_major_get(mbox);  in mlxsw_pci_init()
1521 	mlxsw_pci->bus_info.fw_rev.minor =  in mlxsw_pci_init()
1522 		mlxsw_cmd_mbox_query_fw_fw_rev_minor_get(mbox);  in mlxsw_pci_init()
1523 	mlxsw_pci->bus_info.fw_rev.subminor =  in mlxsw_pci_init()
1524 		mlxsw_cmd_mbox_query_fw_fw_rev_subminor_get(mbox);  in mlxsw_pci_init()
1526 	if (mlxsw_cmd_mbox_query_fw_cmd_interface_rev_get(mbox) != 1) {  in mlxsw_pci_init()
1527 		dev_err(&pdev->dev, "Unsupported cmd interface revision ID queried from hw\n");  in mlxsw_pci_init()
1528 		err = -EINVAL;  in mlxsw_pci_init()
1531 	if (mlxsw_cmd_mbox_query_fw_doorbell_page_bar_get(mbox) != 0) {  in mlxsw_pci_init()
1532 		dev_err(&pdev->dev, "Unsupported doorbell page bar queried from hw\n");  in mlxsw_pci_init()
1533 		err = -EINVAL;  in mlxsw_pci_init()
1537 	mlxsw_pci->doorbell_offset =  in mlxsw_pci_init()
1538 		mlxsw_cmd_mbox_query_fw_doorbell_page_offset_get(mbox);  in mlxsw_pci_init()
1540 	if (mlxsw_cmd_mbox_query_fw_fr_rn_clk_bar_get(mbox) != 0) {  in mlxsw_pci_init()
1541 		dev_err(&pdev->dev, "Unsupported free running clock BAR queried from hw\n");  in mlxsw_pci_init()
1542 		err = -EINVAL;  in mlxsw_pci_init()
1546 	mlxsw_pci->free_running_clock_offset =  in mlxsw_pci_init()
1547 		mlxsw_cmd_mbox_query_fw_free_running_clock_offset_get(mbox);  in mlxsw_pci_init()
1549 	num_pages = mlxsw_cmd_mbox_query_fw_fw_pages_get(mbox);  in mlxsw_pci_init()
1550 	err = mlxsw_pci_fw_area_init(mlxsw_pci, mbox, num_pages);  in mlxsw_pci_init()
1554 	err = mlxsw_pci_boardinfo(mlxsw_pci, mbox);  in mlxsw_pci_init()
1558 	err = mlxsw_core_resources_query(mlxsw_core, mbox, res);  in mlxsw_pci_init()
1564 		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V2;  in mlxsw_pci_init()
1567 		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V1;  in mlxsw_pci_init()
1571 		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V0;  in mlxsw_pci_init()
1573 		dev_err(&pdev->dev, "Invalid supported CQE version combination reported\n");  in mlxsw_pci_init()
1577 	err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile, res);  in mlxsw_pci_init()
1581 	err = mlxsw_pci_aqs_init(mlxsw_pci, mbox);  in mlxsw_pci_init()
1587 			  mlxsw_pci->bus_info.device_kind, mlxsw_pci);  in mlxsw_pci_init()
1589 		dev_err(&pdev->dev, "IRQ request failed\n");  in mlxsw_pci_init()
1612 	mlxsw_cmd_mbox_free(mbox);  in mlxsw_pci_init()
1620 	free_irq(pci_irq_vector(mlxsw_pci->pdev, 0), mlxsw_pci);  in mlxsw_pci_fini()
1630 	u8 ctl_sdq_count = mlxsw_pci_sdq_count(mlxsw_pci) - 1;  in mlxsw_pci_sdq_pick()
1633 	if (tx_info->is_emad) {  in mlxsw_pci_sdq_pick()
1637 		sdqn = 1 + (tx_info->local_port % ctl_sdq_count);  in mlxsw_pci_sdq_pick()
1662 	if (skb_shinfo(skb)->nr_frags > MLXSW_PCI_WQE_SG_ENTRIES - 1) {  in mlxsw_pci_skb_transmit()
1669 	spin_lock_bh(&q->lock);  in mlxsw_pci_skb_transmit()
1673 		err = -EAGAIN;  in mlxsw_pci_skb_transmit()
1676 	mlxsw_skb_cb(skb)->tx_info = *tx_info;  in mlxsw_pci_skb_transmit()
1677 	elem_info->u.sdq.skb = skb;  in mlxsw_pci_skb_transmit()
1679 	wqe = elem_info->elem;  in mlxsw_pci_skb_transmit()
1681 	mlxsw_pci_wqe_lp_set(wqe, !!tx_info->is_emad);  in mlxsw_pci_skb_transmit()
1684 	err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,  in mlxsw_pci_skb_transmit()
1689 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {  in mlxsw_pci_skb_transmit()
1690 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];  in mlxsw_pci_skb_transmit()
1700 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))  in mlxsw_pci_skb_transmit()
1701 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;  in mlxsw_pci_skb_transmit()
1708 	q->producer_counter++;  in mlxsw_pci_skb_transmit()
1714 	for (; i >= 0; i--)  in mlxsw_pci_skb_transmit()
1717 	spin_unlock_bh(&q->lock);  in mlxsw_pci_skb_transmit()
1729 	bool evreq = mlxsw_pci->cmd.nopoll;  in mlxsw_pci_cmd_exec()
1731 	bool *p_wait_done = &mlxsw_pci->cmd.wait_done;  in mlxsw_pci_cmd_exec()
1736 	err = mutex_lock_interruptible(&mlxsw_pci->cmd.lock);  in mlxsw_pci_cmd_exec()
1741 		memcpy(mlxsw_pci->cmd.in_mbox.buf, in_mbox, in_mbox_size);  in mlxsw_pci_cmd_exec()
1742 		in_mapaddr = mlxsw_pci->cmd.in_mbox.mapaddr;  in mlxsw_pci_cmd_exec()
1748 		out_mapaddr = mlxsw_pci->cmd.out_mbox.mapaddr;  in mlxsw_pci_cmd_exec()
1779 		wait_event_timeout(mlxsw_pci->cmd.wait, *p_wait_done, timeout);  in mlxsw_pci_cmd_exec()
1780 		*p_status = mlxsw_pci->cmd.comp.status;  in mlxsw_pci_cmd_exec()
1786 			err = -EIO;  in mlxsw_pci_cmd_exec()
1788 		err = -ETIMEDOUT;  in mlxsw_pci_cmd_exec()
1794 		 * copy registers into mbox buffer.  in mlxsw_pci_cmd_exec()
1807 		memcpy(out_mbox, mlxsw_pci->cmd.out_mbox.buf, out_mbox_size);  in mlxsw_pci_cmd_exec()
1810 	mutex_unlock(&mlxsw_pci->cmd.lock);  in mlxsw_pci_cmd_exec()
1820 	frc_offset = mlxsw_pci->free_running_clock_offset;  in mlxsw_pci_read_frc_h()
1829 	frc_offset = mlxsw_pci->free_running_clock_offset;  in mlxsw_pci_read_frc_l()
1849 	mutex_init(&mlxsw_pci->cmd.lock);  in mlxsw_pci_cmd_init()
1850 	init_waitqueue_head(&mlxsw_pci->cmd.wait);  in mlxsw_pci_cmd_init()
1852 	err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);  in mlxsw_pci_cmd_init()
1856 	err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);  in mlxsw_pci_cmd_init()
1863 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);  in mlxsw_pci_cmd_init()
1865 	mutex_destroy(&mlxsw_pci->cmd.lock);  in mlxsw_pci_cmd_init()
1871 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);  in mlxsw_pci_cmd_fini()
1872 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);  in mlxsw_pci_cmd_fini()
1873 	mutex_destroy(&mlxsw_pci->cmd.lock);  in mlxsw_pci_cmd_fini()
1878 	const char *driver_name = pdev->driver->name;  in mlxsw_pci_probe()
1884 		return -ENOMEM;  in mlxsw_pci_probe()
1888 		dev_err(&pdev->dev, "pci_enable_device failed\n");  in mlxsw_pci_probe()
1894 		dev_err(&pdev->dev, "pci_request_regions failed\n");  in mlxsw_pci_probe()
1898 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));  in mlxsw_pci_probe()
1900 		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));  in mlxsw_pci_probe()
1902 			dev_err(&pdev->dev, "dma_set_mask failed\n");  in mlxsw_pci_probe()
1908 		dev_err(&pdev->dev, "invalid PCI region size\n");  in mlxsw_pci_probe()
1909 		err = -EINVAL;  in mlxsw_pci_probe()
1913 	mlxsw_pci->hw_addr = ioremap(pci_resource_start(pdev, 0),  in mlxsw_pci_probe()
1915 	if (!mlxsw_pci->hw_addr) {  in mlxsw_pci_probe()
1916 		dev_err(&pdev->dev, "ioremap failed\n");  in mlxsw_pci_probe()
1917 		err = -EIO;  in mlxsw_pci_probe()
1922 	mlxsw_pci->pdev = pdev;  in mlxsw_pci_probe()
1929 	mlxsw_pci->bus_info.device_kind = driver_name;  in mlxsw_pci_probe()
1930 	mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev);  in mlxsw_pci_probe()
1931 	mlxsw_pci->bus_info.dev = &pdev->dev;  in mlxsw_pci_probe()
1932 	mlxsw_pci->bus_info.read_frc_capable = true;  in mlxsw_pci_probe()
1933 	mlxsw_pci->id = id;  in mlxsw_pci_probe()
1935 	err = mlxsw_core_bus_device_register(&mlxsw_pci->bus_info,  in mlxsw_pci_probe()
1939 		dev_err(&pdev->dev, "cannot register bus device\n");  in mlxsw_pci_probe()
1948 	iounmap(mlxsw_pci->hw_addr);  in mlxsw_pci_probe()
1964 	mlxsw_core_bus_device_unregister(mlxsw_pci->core, false);  in mlxsw_pci_remove()
1966 	iounmap(mlxsw_pci->hw_addr);  in mlxsw_pci_remove()
1967 	pci_release_regions(mlxsw_pci->pdev);  in mlxsw_pci_remove()
1968 	pci_disable_device(mlxsw_pci->pdev);  in mlxsw_pci_remove()
1974 	pci_driver->probe = mlxsw_pci_probe;  in mlxsw_pci_driver_register()
1975 	pci_driver->remove = mlxsw_pci_remove;  in mlxsw_pci_driver_register()