25 #define mlxsw_pci_write32(mlxsw_pci, reg, val) \  argument
26 	iowrite32be(val, (mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
27 #define mlxsw_pci_read32(mlxsw_pci, reg) \  argument
28 	ioread32be((mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
82 	struct mlxsw_pci *pci;
102 struct mlxsw_pci {  struct
177 mlxsw_pci_queue_type_group_get(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_queue_type_group_get()  argument
180 	return &mlxsw_pci->queues[q_type];  in mlxsw_pci_queue_type_group_get()
183 static u8 __mlxsw_pci_queue_count(struct mlxsw_pci *mlxsw_pci,  in __mlxsw_pci_queue_count()  argument
188 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_type);  in __mlxsw_pci_queue_count()
192 static u8 mlxsw_pci_sdq_count(struct mlxsw_pci *mlxsw_pci)  in mlxsw_pci_sdq_count()  argument
194 	return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_SDQ);  in mlxsw_pci_sdq_count()
197 static u8 mlxsw_pci_cq_count(struct mlxsw_pci *mlxsw_pci)  in mlxsw_pci_cq_count()  argument
199 	return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ);  in mlxsw_pci_cq_count()
203 __mlxsw_pci_queue_get(struct mlxsw_pci *mlxsw_pci,  in __mlxsw_pci_queue_get()  argument
206 	return &mlxsw_pci->queues[q_type].q[q_num];  in __mlxsw_pci_queue_get()
209 static struct mlxsw_pci_queue *mlxsw_pci_sdq_get(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_sdq_get()  argument
212 	return __mlxsw_pci_queue_get(mlxsw_pci,  in mlxsw_pci_sdq_get()
216 static struct mlxsw_pci_queue *mlxsw_pci_rdq_get(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_rdq_get()  argument
219 	return __mlxsw_pci_queue_get(mlxsw_pci,  in mlxsw_pci_rdq_get()
223 static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_cq_get()  argument
226 	return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ, q_num);  in mlxsw_pci_cq_get()
229 static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_eq_get()  argument
232 	return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, q_num);  in mlxsw_pci_eq_get()
235 static void __mlxsw_pci_queue_doorbell_set(struct mlxsw_pci *mlxsw_pci,  in __mlxsw_pci_queue_doorbell_set()  argument
239 	mlxsw_pci_write32(mlxsw_pci,  in __mlxsw_pci_queue_doorbell_set()
240 			  DOORBELL(mlxsw_pci->doorbell_offset,  in __mlxsw_pci_queue_doorbell_set()
245 static void __mlxsw_pci_queue_doorbell_arm_set(struct mlxsw_pci *mlxsw_pci,  in __mlxsw_pci_queue_doorbell_arm_set()  argument
249 	mlxsw_pci_write32(mlxsw_pci,  in __mlxsw_pci_queue_doorbell_arm_set()
250 			  DOORBELL(mlxsw_pci->doorbell_offset,  in __mlxsw_pci_queue_doorbell_arm_set()
255 static void mlxsw_pci_queue_doorbell_producer_ring(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_queue_doorbell_producer_ring()  argument
259 	__mlxsw_pci_queue_doorbell_set(mlxsw_pci, q, q->producer_counter);  in mlxsw_pci_queue_doorbell_producer_ring()
262 static void mlxsw_pci_queue_doorbell_consumer_ring(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_queue_doorbell_consumer_ring()  argument
266 	__mlxsw_pci_queue_doorbell_set(mlxsw_pci, q,  in mlxsw_pci_queue_doorbell_consumer_ring()
271 mlxsw_pci_queue_doorbell_arm_consumer_ring(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_queue_doorbell_arm_consumer_ring()  argument
275 	__mlxsw_pci_queue_doorbell_arm_set(mlxsw_pci, q, q->consumer_counter);  in mlxsw_pci_queue_doorbell_arm_consumer_ring()
284 static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_sdq_init()  argument
303 	err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_sdq_init()
306 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_sdq_init()
310 static void mlxsw_pci_sdq_fini(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_sdq_fini()  argument
313 	mlxsw_cmd_hw2sw_sdq(mlxsw_pci->core, q->num);  in mlxsw_pci_sdq_fini()
316 static int mlxsw_pci_wqe_frag_map(struct mlxsw_pci *mlxsw_pci, char *wqe,  in mlxsw_pci_wqe_frag_map()  argument
320 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_wqe_frag_map()
333 static void mlxsw_pci_wqe_frag_unmap(struct mlxsw_pci *mlxsw_pci, char *wqe,  in mlxsw_pci_wqe_frag_unmap()  argument
336 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_wqe_frag_unmap()
345 static int mlxsw_pci_rdq_skb_alloc(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_rdq_skb_alloc()  argument
360 	err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,  in mlxsw_pci_rdq_skb_alloc()
373 static void mlxsw_pci_rdq_skb_free(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_rdq_skb_free()  argument
382 	mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);  in mlxsw_pci_rdq_skb_free()
386 static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_rdq_init()  argument
390 	u8 sdq_count = mlxsw_pci_sdq_count(mlxsw_pci);  in mlxsw_pci_rdq_init()
408 	err = mlxsw_cmd_sw2hw_rdq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_rdq_init()
412 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_rdq_init()
417 		err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);  in mlxsw_pci_rdq_init()
422 		mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_rdq_init()
430 		mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);  in mlxsw_pci_rdq_init()
432 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);  in mlxsw_pci_rdq_init()
437 static void mlxsw_pci_rdq_fini(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_rdq_fini()  argument
443 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);  in mlxsw_pci_rdq_fini()
446 		mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);  in mlxsw_pci_rdq_fini()
450 static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_cq_pre_init()  argument
453 	q->u.cq.v = mlxsw_pci->max_cqe_ver;  in mlxsw_pci_cq_pre_init()
457 	    q->num < mlxsw_pci->num_sdq_cqs)  in mlxsw_pci_cq_pre_init()
461 static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_cq_init()  argument
490 	err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_cq_init()
493 	mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_init()
494 	mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_init()
498 static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_cq_fini()  argument
501 	mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num);  in mlxsw_pci_cq_fini()
504 static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_cqe_sdq_handle()  argument
509 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_cqe_sdq_handle()
522 		mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);  in mlxsw_pci_cqe_sdq_handle()
526 		mlxsw_core_ptp_transmitted(mlxsw_pci->core, skb,  in mlxsw_pci_cqe_sdq_handle()
540 static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_cqe_rdq_handle()  argument
545 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_cqe_rdq_handle()
558 	mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);  in mlxsw_pci_cqe_rdq_handle()
579 	mlxsw_core_skb_receive(mlxsw_pci->core, skb, &rx_info);  in mlxsw_pci_cqe_rdq_handle()
582 	err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);  in mlxsw_pci_cqe_rdq_handle()
587 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_cqe_rdq_handle()
610 	struct mlxsw_pci *mlxsw_pci = q->pci;  in mlxsw_pci_cq_tasklet()  local
622 		mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_tasklet()
627 			sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn);  in mlxsw_pci_cq_tasklet()
628 			mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq,  in mlxsw_pci_cq_tasklet()
634 			rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn);  in mlxsw_pci_cq_tasklet()
635 			mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq,  in mlxsw_pci_cq_tasklet()
643 		mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_cq_tasklet()
658 static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_eq_init()  argument
680 	err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num);  in mlxsw_pci_eq_init()
683 	mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_init()
684 	mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_init()
688 static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_eq_fini()  argument
691 	mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num);  in mlxsw_pci_eq_fini()
694 static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe)  in mlxsw_pci_eq_cmd_event()  argument
696 	mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe);  in mlxsw_pci_eq_cmd_event()
697 	mlxsw_pci->cmd.comp.out_param =  in mlxsw_pci_eq_cmd_event()
700 	mlxsw_pci->cmd.wait_done = true;  in mlxsw_pci_eq_cmd_event()
701 	wake_up(&mlxsw_pci->cmd.wait);  in mlxsw_pci_eq_cmd_event()
723 	struct mlxsw_pci *mlxsw_pci = q->pci;  in mlxsw_pci_eq_tasklet()  local
724 	u8 cq_count = mlxsw_pci_cq_count(mlxsw_pci);  in mlxsw_pci_eq_tasklet()
742 			mlxsw_pci_eq_cmd_event(mlxsw_pci, eqe);  in mlxsw_pci_eq_tasklet()
758 		mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_tasklet()
759 		mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);  in mlxsw_pci_eq_tasklet()
765 		q = mlxsw_pci_cq_get(mlxsw_pci, cqn);  in mlxsw_pci_eq_tasklet()
773 	void (*pre_init)(struct mlxsw_pci *mlxsw_pci,
775 	int (*init)(struct mlxsw_pci *mlxsw_pci, char *mbox,
777 	void (*fini)(struct mlxsw_pci *mlxsw_pci,
821 static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_queue_init()  argument
831 		q_ops->pre_init(mlxsw_pci, q);  in mlxsw_pci_queue_init()
839 	q->pci = mlxsw_pci;  in mlxsw_pci_queue_init()
845 	mem_item->buf = pci_alloc_consistent(mlxsw_pci->pdev,  in mlxsw_pci_queue_init()
869 	err = q_ops->init(mlxsw_pci, mbox, q);  in mlxsw_pci_queue_init()
877 	pci_free_consistent(mlxsw_pci->pdev, mem_item->size,  in mlxsw_pci_queue_init()
882 static void mlxsw_pci_queue_fini(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_queue_fini()  argument
888 	q_ops->fini(mlxsw_pci, q);  in mlxsw_pci_queue_fini()
890 	pci_free_consistent(mlxsw_pci->pdev, mem_item->size,  in mlxsw_pci_queue_fini()
894 static int mlxsw_pci_queue_group_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_queue_group_init()  argument
902 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);  in mlxsw_pci_queue_group_init()
908 		err = mlxsw_pci_queue_init(mlxsw_pci, mbox, q_ops,  in mlxsw_pci_queue_group_init()
919 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);  in mlxsw_pci_queue_group_init()
924 static void mlxsw_pci_queue_group_fini(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_queue_group_fini()  argument
930 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);  in mlxsw_pci_queue_group_fini()
932 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);  in mlxsw_pci_queue_group_fini()
936 static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)  in mlxsw_pci_aqs_init()  argument
938 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_aqs_init()
951 	err = mlxsw_cmd_query_aq_cap(mlxsw_pci->core, mbox);  in mlxsw_pci_aqs_init()
974 	    (mlxsw_pci->max_cqe_ver == MLXSW_PCI_CQE_V2 &&  in mlxsw_pci_aqs_init()
981 	mlxsw_pci->num_sdq_cqs = num_sdqs;  in mlxsw_pci_aqs_init()
983 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops,  in mlxsw_pci_aqs_init()
990 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_cq_ops,  in mlxsw_pci_aqs_init()
997 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_sdq_ops,  in mlxsw_pci_aqs_init()
1004 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_rdq_ops,  in mlxsw_pci_aqs_init()
1012 	mlxsw_pci->cmd.nopoll = true;  in mlxsw_pci_aqs_init()
1016 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);  in mlxsw_pci_aqs_init()
1018 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);  in mlxsw_pci_aqs_init()
1020 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);  in mlxsw_pci_aqs_init()
1024 static void mlxsw_pci_aqs_fini(struct mlxsw_pci *mlxsw_pci)  in mlxsw_pci_aqs_fini()  argument
1026 	mlxsw_pci->cmd.nopoll = false;  in mlxsw_pci_aqs_fini()
1027 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_rdq_ops);  in mlxsw_pci_aqs_fini()
1028 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);  in mlxsw_pci_aqs_fini()
1029 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);  in mlxsw_pci_aqs_fini()
1030 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);  in mlxsw_pci_aqs_fini()
1034 mlxsw_pci_config_profile_swid_config(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_config_profile_swid_config()  argument
1054 mlxsw_pci_profile_get_kvd_sizes(const struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_profile_get_kvd_sizes()  argument
1061 	err = mlxsw_core_kvd_sizes_get(mlxsw_pci->core, profile,  in mlxsw_pci_profile_get_kvd_sizes()
1074 static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_config_profile()  argument
1166 		err = mlxsw_pci_profile_get_kvd_sizes(mlxsw_pci, profile, res);  in mlxsw_pci_config_profile()
1184 		mlxsw_pci_config_profile_swid_config(mlxsw_pci, mbox, i,  in mlxsw_pci_config_profile()
1187 	if (mlxsw_pci->max_cqe_ver > MLXSW_PCI_CQE_V0) {  in mlxsw_pci_config_profile()
1192 	return mlxsw_cmd_config_profile_set(mlxsw_pci->core, mbox);  in mlxsw_pci_config_profile()
1195 static int mlxsw_pci_boardinfo(struct mlxsw_pci *mlxsw_pci, char *mbox)  in mlxsw_pci_boardinfo()  argument
1197 	struct mlxsw_bus_info *bus_info = &mlxsw_pci->bus_info;  in mlxsw_pci_boardinfo()
1201 	err = mlxsw_cmd_boardinfo(mlxsw_pci->core, mbox);  in mlxsw_pci_boardinfo()
1209 static int mlxsw_pci_fw_area_init(struct mlxsw_pci *mlxsw_pci, char *mbox,  in mlxsw_pci_fw_area_init()  argument
1217 	mlxsw_pci->fw_area.items = kcalloc(num_pages, sizeof(*mem_item),  in mlxsw_pci_fw_area_init()
1219 	if (!mlxsw_pci->fw_area.items)  in mlxsw_pci_fw_area_init()
1221 	mlxsw_pci->fw_area.count = num_pages;  in mlxsw_pci_fw_area_init()
1225 		mem_item = &mlxsw_pci->fw_area.items[i];  in mlxsw_pci_fw_area_init()
1228 		mem_item->buf = pci_alloc_consistent(mlxsw_pci->pdev,  in mlxsw_pci_fw_area_init()
1238 			err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);  in mlxsw_pci_fw_area_init()
1247 		err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);  in mlxsw_pci_fw_area_init()
1257 		mem_item = &mlxsw_pci->fw_area.items[i];  in mlxsw_pci_fw_area_init()
1259 		pci_free_consistent(mlxsw_pci->pdev, mem_item->size,  in mlxsw_pci_fw_area_init()
1262 	kfree(mlxsw_pci->fw_area.items);  in mlxsw_pci_fw_area_init()
1266 static void mlxsw_pci_fw_area_fini(struct mlxsw_pci *mlxsw_pci)  in mlxsw_pci_fw_area_fini()  argument
1271 	mlxsw_cmd_unmap_fa(mlxsw_pci->core);  in mlxsw_pci_fw_area_fini()
1273 	for (i = 0; i < mlxsw_pci->fw_area.count; i++) {  in mlxsw_pci_fw_area_fini()
1274 		mem_item = &mlxsw_pci->fw_area.items[i];  in mlxsw_pci_fw_area_fini()
1276 		pci_free_consistent(mlxsw_pci->pdev, mem_item->size,  in mlxsw_pci_fw_area_fini()
1279 	kfree(mlxsw_pci->fw_area.items);  in mlxsw_pci_fw_area_fini()
1284 	struct mlxsw_pci *mlxsw_pci = dev_id;  in mlxsw_pci_eq_irq_handler()  local
1289 		q = mlxsw_pci_eq_get(mlxsw_pci, i);  in mlxsw_pci_eq_irq_handler()
1295 static int mlxsw_pci_mbox_alloc(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_mbox_alloc()  argument
1298 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_mbox_alloc()
1312 static void mlxsw_pci_mbox_free(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_mbox_free()  argument
1315 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_mbox_free()
1321 static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_sw_reset()  argument
1329 	err = mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl);  in mlxsw_pci_sw_reset()
1342 		u32 val = mlxsw_pci_read32(mlxsw_pci, FW_READY);  in mlxsw_pci_sw_reset()
1351 static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci)  in mlxsw_pci_alloc_irq_vectors()  argument
1355 	err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX);  in mlxsw_pci_alloc_irq_vectors()
1357 		dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n");  in mlxsw_pci_alloc_irq_vectors()
1361 static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci)  in mlxsw_pci_free_irq_vectors()  argument
1363 	pci_free_irq_vectors(mlxsw_pci->pdev);  in mlxsw_pci_free_irq_vectors()
1370 	struct mlxsw_pci *mlxsw_pci = bus_priv;  in mlxsw_pci_init()  local
1371 	struct pci_dev *pdev = mlxsw_pci->pdev;  in mlxsw_pci_init()
1376 	mutex_init(&mlxsw_pci->cmd.lock);  in mlxsw_pci_init()
1377 	init_waitqueue_head(&mlxsw_pci->cmd.wait);  in mlxsw_pci_init()
1379 	mlxsw_pci->core = mlxsw_core;  in mlxsw_pci_init()
1385 	err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);  in mlxsw_pci_init()
1389 	err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);  in mlxsw_pci_init()
1393 	err = mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id);  in mlxsw_pci_init()
1397 	err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci);  in mlxsw_pci_init()
1407 	mlxsw_pci->bus_info.fw_rev.major =  in mlxsw_pci_init()
1409 	mlxsw_pci->bus_info.fw_rev.minor =  in mlxsw_pci_init()
1411 	mlxsw_pci->bus_info.fw_rev.subminor =  in mlxsw_pci_init()
1425 	mlxsw_pci->doorbell_offset =  in mlxsw_pci_init()
1434 	mlxsw_pci->free_running_clock_offset =  in mlxsw_pci_init()
1438 	err = mlxsw_pci_fw_area_init(mlxsw_pci, mbox, num_pages);  in mlxsw_pci_init()
1442 	err = mlxsw_pci_boardinfo(mlxsw_pci, mbox);  in mlxsw_pci_init()
1452 		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V2;  in mlxsw_pci_init()
1455 		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V1;  in mlxsw_pci_init()
1459 		mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V0;  in mlxsw_pci_init()
1465 	err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile, res);  in mlxsw_pci_init()
1469 	err = mlxsw_pci_aqs_init(mlxsw_pci, mbox);  in mlxsw_pci_init()
1475 			  mlxsw_pci->bus_info.device_kind, mlxsw_pci);  in mlxsw_pci_init()
1484 	mlxsw_pci_aqs_fini(mlxsw_pci);  in mlxsw_pci_init()
1490 	mlxsw_pci_fw_area_fini(mlxsw_pci);  in mlxsw_pci_init()
1496 	mlxsw_pci_free_irq_vectors(mlxsw_pci);  in mlxsw_pci_init()
1499 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);  in mlxsw_pci_init()
1501 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);  in mlxsw_pci_init()
1509 	struct mlxsw_pci *mlxsw_pci = bus_priv;  in mlxsw_pci_fini()  local
1511 	free_irq(pci_irq_vector(mlxsw_pci->pdev, 0), mlxsw_pci);  in mlxsw_pci_fini()
1512 	mlxsw_pci_aqs_fini(mlxsw_pci);  in mlxsw_pci_fini()
1513 	mlxsw_pci_fw_area_fini(mlxsw_pci);  in mlxsw_pci_fini()
1514 	mlxsw_pci_free_irq_vectors(mlxsw_pci);  in mlxsw_pci_fini()
1515 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);  in mlxsw_pci_fini()
1516 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);  in mlxsw_pci_fini()
1520 mlxsw_pci_sdq_pick(struct mlxsw_pci *mlxsw_pci,  in mlxsw_pci_sdq_pick()  argument
1523 	u8 sdqn = tx_info->local_port % mlxsw_pci_sdq_count(mlxsw_pci);  in mlxsw_pci_sdq_pick()
1525 	return mlxsw_pci_sdq_get(mlxsw_pci, sdqn);  in mlxsw_pci_sdq_pick()
1531 	struct mlxsw_pci *mlxsw_pci = bus_priv;  in mlxsw_pci_skb_transmit_busy()  local
1532 	struct mlxsw_pci_queue *q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);  in mlxsw_pci_skb_transmit_busy()
1540 	struct mlxsw_pci *mlxsw_pci = bus_priv;  in mlxsw_pci_skb_transmit()  local
1553 	q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);  in mlxsw_pci_skb_transmit()
1569 	err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,  in mlxsw_pci_skb_transmit()
1577 		err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, i + 1,  in mlxsw_pci_skb_transmit()
1594 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);  in mlxsw_pci_skb_transmit()
1600 		mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);  in mlxsw_pci_skb_transmit()
1612 	struct mlxsw_pci *mlxsw_pci = bus_priv;  in mlxsw_pci_cmd_exec()  local
1614 	bool evreq = mlxsw_pci->cmd.nopoll;  in mlxsw_pci_cmd_exec()
1616 	bool *p_wait_done = &mlxsw_pci->cmd.wait_done;  in mlxsw_pci_cmd_exec()
1621 	err = mutex_lock_interruptible(&mlxsw_pci->cmd.lock);  in mlxsw_pci_cmd_exec()
1626 		memcpy(mlxsw_pci->cmd.in_mbox.buf, in_mbox, in_mbox_size);  in mlxsw_pci_cmd_exec()
1627 		in_mapaddr = mlxsw_pci->cmd.in_mbox.mapaddr;  in mlxsw_pci_cmd_exec()
1629 	mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_HI, upper_32_bits(in_mapaddr));  in mlxsw_pci_cmd_exec()
1630 	mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_LO, lower_32_bits(in_mapaddr));  in mlxsw_pci_cmd_exec()
1633 		out_mapaddr = mlxsw_pci->cmd.out_mbox.mapaddr;  in mlxsw_pci_cmd_exec()
1634 	mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_HI, upper_32_bits(out_mapaddr));  in mlxsw_pci_cmd_exec()
1635 	mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_LO, lower_32_bits(out_mapaddr));  in mlxsw_pci_cmd_exec()
1637 	mlxsw_pci_write32(mlxsw_pci, CIR_IN_MODIFIER, in_mod);  in mlxsw_pci_cmd_exec()
1638 	mlxsw_pci_write32(mlxsw_pci, CIR_TOKEN, 0);  in mlxsw_pci_cmd_exec()
1643 	mlxsw_pci_write32(mlxsw_pci, CIR_CTRL,  in mlxsw_pci_cmd_exec()
1654 			u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL);  in mlxsw_pci_cmd_exec()
1664 		wait_event_timeout(mlxsw_pci->cmd.wait, *p_wait_done, timeout);  in mlxsw_pci_cmd_exec()
1665 		*p_status = mlxsw_pci->cmd.comp.status;  in mlxsw_pci_cmd_exec()
1684 			tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,  in mlxsw_pci_cmd_exec()
1687 			tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,  in mlxsw_pci_cmd_exec()
1692 		memcpy(out_mbox, mlxsw_pci->cmd.out_mbox.buf, out_mbox_size);  in mlxsw_pci_cmd_exec()
1695 	mutex_unlock(&mlxsw_pci->cmd.lock);  in mlxsw_pci_cmd_exec()
1702 	struct mlxsw_pci *mlxsw_pci = bus_priv;  in mlxsw_pci_read_frc_h()  local
1705 	frc_offset = mlxsw_pci->free_running_clock_offset;  in mlxsw_pci_read_frc_h()
1706 	return mlxsw_pci_read32(mlxsw_pci, FREE_RUNNING_CLOCK_H(frc_offset));  in mlxsw_pci_read_frc_h()
1711 	struct mlxsw_pci *mlxsw_pci = bus_priv;  in mlxsw_pci_read_frc_l()  local
1714 	frc_offset = mlxsw_pci->free_running_clock_offset;  in mlxsw_pci_read_frc_l()
1715 	return mlxsw_pci_read32(mlxsw_pci, FREE_RUNNING_CLOCK_L(frc_offset));  in mlxsw_pci_read_frc_l()
1733 	struct mlxsw_pci *mlxsw_pci;  in mlxsw_pci_probe()  local
1736 	mlxsw_pci = kzalloc(sizeof(*mlxsw_pci), GFP_KERNEL);  in mlxsw_pci_probe()
1737 	if (!mlxsw_pci)  in mlxsw_pci_probe()
1773 	mlxsw_pci->hw_addr = ioremap(pci_resource_start(pdev, 0),  in mlxsw_pci_probe()
1775 	if (!mlxsw_pci->hw_addr) {  in mlxsw_pci_probe()
1782 	mlxsw_pci->pdev = pdev;  in mlxsw_pci_probe()
1783 	pci_set_drvdata(pdev, mlxsw_pci);  in mlxsw_pci_probe()
1785 	mlxsw_pci->bus_info.device_kind = driver_name;  in mlxsw_pci_probe()
1786 	mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev);  in mlxsw_pci_probe()
1787 	mlxsw_pci->bus_info.dev = &pdev->dev;  in mlxsw_pci_probe()
1788 	mlxsw_pci->bus_info.read_frc_capable = true;  in mlxsw_pci_probe()
1789 	mlxsw_pci->id = id;  in mlxsw_pci_probe()
1791 	err = mlxsw_core_bus_device_register(&mlxsw_pci->bus_info,  in mlxsw_pci_probe()
1792 					     &mlxsw_pci_bus, mlxsw_pci, false,  in mlxsw_pci_probe()
1802 	iounmap(mlxsw_pci->hw_addr);  in mlxsw_pci_probe()
1810 	kfree(mlxsw_pci);  in mlxsw_pci_probe()
1816 	struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);  in mlxsw_pci_remove()  local
1818 	mlxsw_core_bus_device_unregister(mlxsw_pci->core, false);  in mlxsw_pci_remove()
1819 	iounmap(mlxsw_pci->hw_addr);  in mlxsw_pci_remove()
1820 	pci_release_regions(mlxsw_pci->pdev);  in mlxsw_pci_remove()
1821 	pci_disable_device(mlxsw_pci->pdev);  in mlxsw_pci_remove()
1822 	kfree(mlxsw_pci);  in mlxsw_pci_remove()