Lines Matching refs:cptvf
28 static int init_worker_threads(struct otx_cptvf *cptvf) in init_worker_threads() argument
30 struct pci_dev *pdev = cptvf->pdev; in init_worker_threads()
38 if (cptvf->num_queues) { in init_worker_threads()
40 cptvf->num_queues); in init_worker_threads()
43 for (i = 0; i < cptvf->num_queues; i++) { in init_worker_threads()
46 cwqe_info->vq_wqe[i].cptvf = cptvf; in init_worker_threads()
48 cptvf->wqe_info = cwqe_info; in init_worker_threads()
53 static void cleanup_worker_threads(struct otx_cptvf *cptvf) in cleanup_worker_threads() argument
55 struct pci_dev *pdev = cptvf->pdev; in cleanup_worker_threads()
59 cwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info; in cleanup_worker_threads()
63 if (cptvf->num_queues) { in cleanup_worker_threads()
65 cptvf->num_queues); in cleanup_worker_threads()
68 for (i = 0; i < cptvf->num_queues; i++) in cleanup_worker_threads()
72 cptvf->wqe_info = NULL; in cleanup_worker_threads()
125 static int init_pending_queues(struct otx_cptvf *cptvf, u32 qlen, in init_pending_queues() argument
128 struct pci_dev *pdev = cptvf->pdev; in init_pending_queues()
134 ret = alloc_pending_queues(&cptvf->pqinfo, qlen, num_queues); in init_pending_queues()
143 static void cleanup_pending_queues(struct otx_cptvf *cptvf) in cleanup_pending_queues() argument
145 struct pci_dev *pdev = cptvf->pdev; in cleanup_pending_queues()
147 if (!cptvf->num_queues) in cleanup_pending_queues()
151 cptvf->num_queues); in cleanup_pending_queues()
152 free_pending_queues(&cptvf->pqinfo); in cleanup_pending_queues()
155 static void free_command_queues(struct otx_cptvf *cptvf, in free_command_queues() argument
160 struct pci_dev *pdev = cptvf->pdev; in free_command_queues()
164 for (i = 0; i < cptvf->num_queues; i++) { in free_command_queues()
185 static int alloc_command_queues(struct otx_cptvf *cptvf, in alloc_command_queues() argument
191 struct pci_dev *pdev = cptvf->pdev; in alloc_command_queues()
198 cptvf->qsize = min(qlen, cqinfo->qchunksize) * in alloc_command_queues()
206 for (i = 0; i < cptvf->num_queues; i++) { in alloc_command_queues()
258 free_command_queues(cptvf, cqinfo); in alloc_command_queues()
262 static int init_command_queues(struct otx_cptvf *cptvf, u32 qlen) in init_command_queues() argument
264 struct pci_dev *pdev = cptvf->pdev; in init_command_queues()
268 ret = alloc_command_queues(cptvf, &cptvf->cqinfo, qlen); in init_command_queues()
271 cptvf->num_queues); in init_command_queues()
277 static void cleanup_command_queues(struct otx_cptvf *cptvf) in cleanup_command_queues() argument
279 struct pci_dev *pdev = cptvf->pdev; in cleanup_command_queues()
281 if (!cptvf->num_queues) in cleanup_command_queues()
285 cptvf->num_queues); in cleanup_command_queues()
286 free_command_queues(cptvf, &cptvf->cqinfo); in cleanup_command_queues()
289 static void cptvf_sw_cleanup(struct otx_cptvf *cptvf) in cptvf_sw_cleanup() argument
291 cleanup_worker_threads(cptvf); in cptvf_sw_cleanup()
292 cleanup_pending_queues(cptvf); in cptvf_sw_cleanup()
293 cleanup_command_queues(cptvf); in cptvf_sw_cleanup()
296 static int cptvf_sw_init(struct otx_cptvf *cptvf, u32 qlen, u32 num_queues) in cptvf_sw_init() argument
298 struct pci_dev *pdev = cptvf->pdev; in cptvf_sw_init()
305 cptvf->num_queues = num_queues; in cptvf_sw_init()
307 ret = init_command_queues(cptvf, qlen); in cptvf_sw_init()
314 ret = init_pending_queues(cptvf, qlen, num_queues); in cptvf_sw_init()
322 ret = init_worker_threads(cptvf); in cptvf_sw_init()
330 cleanup_worker_threads(cptvf); in cptvf_sw_init()
331 cleanup_pending_queues(cptvf); in cptvf_sw_init()
334 cleanup_command_queues(cptvf); in cptvf_sw_init()
339 static void cptvf_free_irq_affinity(struct otx_cptvf *cptvf, int vec) in cptvf_free_irq_affinity() argument
341 irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); in cptvf_free_irq_affinity()
342 free_cpumask_var(cptvf->affinity_mask[vec]); in cptvf_free_irq_affinity()
345 static void cptvf_write_vq_ctl(struct otx_cptvf *cptvf, bool val) in cptvf_write_vq_ctl() argument
349 vqx_ctl.u = readq(cptvf->reg_base + OTX_CPT_VQX_CTL(0)); in cptvf_write_vq_ctl()
351 writeq(vqx_ctl.u, cptvf->reg_base + OTX_CPT_VQX_CTL(0)); in cptvf_write_vq_ctl()
354 void otx_cptvf_write_vq_doorbell(struct otx_cptvf *cptvf, u32 val) in otx_cptvf_write_vq_doorbell() argument
358 vqx_dbell.u = readq(cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0)); in otx_cptvf_write_vq_doorbell()
360 writeq(vqx_dbell.u, cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0)); in otx_cptvf_write_vq_doorbell()
363 static void cptvf_write_vq_inprog(struct otx_cptvf *cptvf, u8 val) in cptvf_write_vq_inprog() argument
367 vqx_inprg.u = readq(cptvf->reg_base + OTX_CPT_VQX_INPROG(0)); in cptvf_write_vq_inprog()
369 writeq(vqx_inprg.u, cptvf->reg_base + OTX_CPT_VQX_INPROG(0)); in cptvf_write_vq_inprog()
372 static void cptvf_write_vq_done_numwait(struct otx_cptvf *cptvf, u32 val) in cptvf_write_vq_done_numwait() argument
376 vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); in cptvf_write_vq_done_numwait()
378 writeq(vqx_dwait.u, cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); in cptvf_write_vq_done_numwait()
381 static u32 cptvf_read_vq_done_numwait(struct otx_cptvf *cptvf) in cptvf_read_vq_done_numwait() argument
385 vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); in cptvf_read_vq_done_numwait()
389 static void cptvf_write_vq_done_timewait(struct otx_cptvf *cptvf, u16 time) in cptvf_write_vq_done_timewait() argument
393 vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); in cptvf_write_vq_done_timewait()
395 writeq(vqx_dwait.u, cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); in cptvf_write_vq_done_timewait()
399 static u16 cptvf_read_vq_done_timewait(struct otx_cptvf *cptvf) in cptvf_read_vq_done_timewait() argument
403 vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0)); in cptvf_read_vq_done_timewait()
407 static void cptvf_enable_swerr_interrupts(struct otx_cptvf *cptvf) in cptvf_enable_swerr_interrupts() argument
411 vqx_misc_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); in cptvf_enable_swerr_interrupts()
414 writeq(vqx_misc_ena.u, cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); in cptvf_enable_swerr_interrupts()
417 static void cptvf_enable_mbox_interrupts(struct otx_cptvf *cptvf) in cptvf_enable_mbox_interrupts() argument
421 vqx_misc_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); in cptvf_enable_mbox_interrupts()
424 writeq(vqx_misc_ena.u, cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0)); in cptvf_enable_mbox_interrupts()
427 static void cptvf_enable_done_interrupts(struct otx_cptvf *cptvf) in cptvf_enable_done_interrupts() argument
431 vqx_done_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0)); in cptvf_enable_done_interrupts()
434 writeq(vqx_done_ena.u, cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0)); in cptvf_enable_done_interrupts()
437 static void cptvf_clear_dovf_intr(struct otx_cptvf *cptvf) in cptvf_clear_dovf_intr() argument
441 vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_dovf_intr()
444 writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_dovf_intr()
447 static void cptvf_clear_irde_intr(struct otx_cptvf *cptvf) in cptvf_clear_irde_intr() argument
451 vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_irde_intr()
454 writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_irde_intr()
457 static void cptvf_clear_nwrp_intr(struct otx_cptvf *cptvf) in cptvf_clear_nwrp_intr() argument
461 vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_nwrp_intr()
464 writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_nwrp_intr()
467 static void cptvf_clear_mbox_intr(struct otx_cptvf *cptvf) in cptvf_clear_mbox_intr() argument
471 vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_mbox_intr()
474 writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_mbox_intr()
477 static void cptvf_clear_swerr_intr(struct otx_cptvf *cptvf) in cptvf_clear_swerr_intr() argument
481 vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_swerr_intr()
484 writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_clear_swerr_intr()
487 static u64 cptvf_read_vf_misc_intr_status(struct otx_cptvf *cptvf) in cptvf_read_vf_misc_intr_status() argument
489 return readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0)); in cptvf_read_vf_misc_intr_status()
495 struct otx_cptvf *cptvf = arg; in cptvf_misc_intr_handler() local
496 struct pci_dev *pdev = cptvf->pdev; in cptvf_misc_intr_handler()
499 intr = cptvf_read_vf_misc_intr_status(cptvf); in cptvf_misc_intr_handler()
503 intr, cptvf->vfid); in cptvf_misc_intr_handler()
504 otx_cptvf_handle_mbox_intr(cptvf); in cptvf_misc_intr_handler()
505 cptvf_clear_mbox_intr(cptvf); in cptvf_misc_intr_handler()
507 cptvf_clear_dovf_intr(cptvf); in cptvf_misc_intr_handler()
509 otx_cptvf_write_vq_doorbell(cptvf, 0); in cptvf_misc_intr_handler()
512 intr, cptvf->vfid); in cptvf_misc_intr_handler()
514 cptvf_clear_irde_intr(cptvf); in cptvf_misc_intr_handler()
517 intr, cptvf->vfid); in cptvf_misc_intr_handler()
519 cptvf_clear_nwrp_intr(cptvf); in cptvf_misc_intr_handler()
522 intr, cptvf->vfid); in cptvf_misc_intr_handler()
524 cptvf_clear_swerr_intr(cptvf); in cptvf_misc_intr_handler()
527 intr, cptvf->vfid); in cptvf_misc_intr_handler()
530 cptvf->vfid); in cptvf_misc_intr_handler()
536 static inline struct otx_cptvf_wqe *get_cptvf_vq_wqe(struct otx_cptvf *cptvf, in get_cptvf_vq_wqe() argument
541 if (unlikely(qno >= cptvf->num_queues)) in get_cptvf_vq_wqe()
543 nwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info; in get_cptvf_vq_wqe()
548 static inline u32 cptvf_read_vq_done_count(struct otx_cptvf *cptvf) in cptvf_read_vq_done_count() argument
552 vqx_done.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE(0)); in cptvf_read_vq_done_count()
556 static inline void cptvf_write_vq_done_ack(struct otx_cptvf *cptvf, in cptvf_write_vq_done_ack() argument
561 vqx_dack_cnt.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0)); in cptvf_write_vq_done_ack()
563 writeq(vqx_dack_cnt.u, cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0)); in cptvf_write_vq_done_ack()
569 struct otx_cptvf *cptvf = (struct otx_cptvf *)cptvf_dev; in cptvf_done_intr_handler() local
570 struct pci_dev *pdev = cptvf->pdev; in cptvf_done_intr_handler()
572 u32 intr = cptvf_read_vq_done_count(cptvf); in cptvf_done_intr_handler()
581 cptvf_write_vq_done_ack(cptvf, intr); in cptvf_done_intr_handler()
582 wqe = get_cptvf_vq_wqe(cptvf, 0); in cptvf_done_intr_handler()
585 cptvf->vfid); in cptvf_done_intr_handler()
594 static void cptvf_set_irq_affinity(struct otx_cptvf *cptvf, int vec) in cptvf_set_irq_affinity() argument
596 struct pci_dev *pdev = cptvf->pdev; in cptvf_set_irq_affinity()
599 if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], in cptvf_set_irq_affinity()
603 cptvf->vfid); in cptvf_set_irq_affinity()
607 cpu = cptvf->vfid % num_online_cpus(); in cptvf_set_irq_affinity()
608 cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), in cptvf_set_irq_affinity()
609 cptvf->affinity_mask[vec]); in cptvf_set_irq_affinity()
611 cptvf->affinity_mask[vec]); in cptvf_set_irq_affinity()
614 static void cptvf_write_vq_saddr(struct otx_cptvf *cptvf, u64 val) in cptvf_write_vq_saddr() argument
619 writeq(vqx_saddr.u, cptvf->reg_base + OTX_CPT_VQX_SADDR(0)); in cptvf_write_vq_saddr()
622 static void cptvf_device_init(struct otx_cptvf *cptvf) in cptvf_device_init() argument
627 cptvf_write_vq_ctl(cptvf, 0); in cptvf_device_init()
629 otx_cptvf_write_vq_doorbell(cptvf, 0); in cptvf_device_init()
631 cptvf_write_vq_inprog(cptvf, 0); in cptvf_device_init()
633 base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); in cptvf_device_init()
634 cptvf_write_vq_saddr(cptvf, base_addr); in cptvf_device_init()
636 cptvf_write_vq_done_timewait(cptvf, OTX_CPT_TIMER_HOLD); in cptvf_device_init()
637 cptvf_write_vq_done_numwait(cptvf, OTX_CPT_COUNT_HOLD); in cptvf_device_init()
639 cptvf_write_vq_ctl(cptvf, 1); in cptvf_device_init()
641 cptvf->flags |= OTX_CPT_FLAG_DEVICE_READY; in cptvf_device_init()
648 struct otx_cptvf *cptvf = dev_get_drvdata(dev); in vf_type_show() local
651 switch (cptvf->vftype) { in vf_type_show()
671 struct otx_cptvf *cptvf = dev_get_drvdata(dev); in vf_engine_group_show() local
673 return sysfs_emit(buf, "%d\n", cptvf->vfgrp); in vf_engine_group_show()
680 struct otx_cptvf *cptvf = dev_get_drvdata(dev); in vf_engine_group_store() local
696 ret = otx_cptvf_send_vf_to_grp_msg(cptvf, val); in vf_engine_group_store()
707 struct otx_cptvf *cptvf = dev_get_drvdata(dev); in vf_coalesc_time_wait_show() local
710 cptvf_read_vq_done_timewait(cptvf)); in vf_coalesc_time_wait_show()
717 struct otx_cptvf *cptvf = dev_get_drvdata(dev); in vf_coalesc_num_wait_show() local
720 cptvf_read_vq_done_numwait(cptvf)); in vf_coalesc_num_wait_show()
727 struct otx_cptvf *cptvf = dev_get_drvdata(dev); in vf_coalesc_time_wait_store() local
739 cptvf_write_vq_done_timewait(cptvf, val); in vf_coalesc_time_wait_store()
747 struct otx_cptvf *cptvf = dev_get_drvdata(dev); in vf_coalesc_num_wait_store() local
759 cptvf_write_vq_done_numwait(cptvf, val); in vf_coalesc_num_wait_store()
784 struct otx_cptvf *cptvf; in otx_cptvf_probe() local
787 cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); in otx_cptvf_probe()
788 if (!cptvf) in otx_cptvf_probe()
791 pci_set_drvdata(pdev, cptvf); in otx_cptvf_probe()
792 cptvf->pdev = pdev; in otx_cptvf_probe()
811 cptvf->reg_base = pci_iomap(pdev, OTX_CPT_VF_PCI_CFG_BAR, 0); in otx_cptvf_probe()
812 if (!cptvf->reg_base) { in otx_cptvf_probe()
818 cptvf->node = dev_to_node(&pdev->dev); in otx_cptvf_probe()
829 cptvf); in otx_cptvf_probe()
836 cptvf_enable_mbox_interrupts(cptvf); in otx_cptvf_probe()
837 cptvf_enable_swerr_interrupts(cptvf); in otx_cptvf_probe()
840 err = otx_cptvf_check_pf_ready(cptvf); in otx_cptvf_probe()
845 cptvf->cqinfo.qchunksize = OTX_CPT_CMD_QCHUNK_SIZE; in otx_cptvf_probe()
846 err = cptvf_sw_init(cptvf, OTX_CPT_CMD_QLEN, OTX_CPT_NUM_QS_PER_VF); in otx_cptvf_probe()
852 err = otx_cptvf_send_vq_size_msg(cptvf); in otx_cptvf_probe()
857 cptvf_device_init(cptvf); in otx_cptvf_probe()
859 err = otx_cptvf_send_vf_to_grp_msg(cptvf, cptvf->vfgrp); in otx_cptvf_probe()
863 cptvf->priority = 1; in otx_cptvf_probe()
864 err = otx_cptvf_send_vf_priority_msg(cptvf); in otx_cptvf_probe()
870 cptvf); in otx_cptvf_probe()
877 cptvf_enable_done_interrupts(cptvf); in otx_cptvf_probe()
880 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in otx_cptvf_probe()
881 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in otx_cptvf_probe()
883 err = otx_cptvf_send_vf_up(cptvf); in otx_cptvf_probe()
889 cptvf->vftype == OTX_CPT_SE_TYPES ? OTX_CPT_SE : OTX_CPT_AE, in otx_cptvf_probe()
890 cptvf->vftype, 1, cptvf->num_vfs); in otx_cptvf_probe()
905 otx_cpt_crypto_exit(pdev, THIS_MODULE, cptvf->vftype); in otx_cptvf_probe()
907 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in otx_cptvf_probe()
908 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in otx_cptvf_probe()
910 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); in otx_cptvf_probe()
912 cptvf_sw_cleanup(cptvf); in otx_cptvf_probe()
914 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); in otx_cptvf_probe()
916 pci_free_irq_vectors(cptvf->pdev); in otx_cptvf_probe()
918 pci_iounmap(pdev, cptvf->reg_base); in otx_cptvf_probe()
931 struct otx_cptvf *cptvf = pci_get_drvdata(pdev); in otx_cptvf_remove() local
933 if (!cptvf) { in otx_cptvf_remove()
939 if (otx_cptvf_send_vf_down(cptvf)) { in otx_cptvf_remove()
943 otx_cpt_crypto_exit(pdev, THIS_MODULE, cptvf->vftype); in otx_cptvf_remove()
944 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in otx_cptvf_remove()
945 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in otx_cptvf_remove()
946 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); in otx_cptvf_remove()
947 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); in otx_cptvf_remove()
948 cptvf_sw_cleanup(cptvf); in otx_cptvf_remove()
949 pci_free_irq_vectors(cptvf->pdev); in otx_cptvf_remove()
950 pci_iounmap(pdev, cptvf->reg_base); in otx_cptvf_remove()