39 static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
43 static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)  in IQ_INSTR_MODE_64B()  argument
46 	    (struct octeon_instr_queue *)oct->instr_queue[iq_no];  in IQ_INSTR_MODE_64B()
50 #define IQ_INSTR_MODE_32B(oct, iq_no)  (!IQ_INSTR_MODE_64B(oct, iq_no))  argument
56 int octeon_init_instr_queue(struct octeon_device *oct,  in octeon_init_instr_queue()  argument
65 	int numa_node = dev_to_node(&oct->pci_dev->dev);  in octeon_init_instr_queue()
67 	if (OCTEON_CN6XXX(oct))  in octeon_init_instr_queue()
68 		conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn6xxx)));  in octeon_init_instr_queue()
69 	else if (OCTEON_CN23XX_PF(oct))  in octeon_init_instr_queue()
70 		conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_pf)));  in octeon_init_instr_queue()
71 	else if (OCTEON_CN23XX_VF(oct))  in octeon_init_instr_queue()
72 		conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_vf)));  in octeon_init_instr_queue()
75 		dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n",  in octeon_init_instr_queue()
76 			oct->chip_id);  in octeon_init_instr_queue()
82 	iq = oct->instr_queue[iq_no];  in octeon_init_instr_queue()
84 	iq->oct_dev = oct;  in octeon_init_instr_queue()
86 	iq->base_addr = lio_dma_alloc(oct, q_size, &iq->base_addr_dma);  in octeon_init_instr_queue()
88 		dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n",  in octeon_init_instr_queue()
105 		lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);  in octeon_init_instr_queue()
106 		dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n",  in octeon_init_instr_queue()
113 	dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %pad count: %d\n",  in octeon_init_instr_queue()
139 	oct->io_qmask.iq |= BIT_ULL(iq_no);  in octeon_init_instr_queue()
142 	oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);  in octeon_init_instr_queue()
145 	oct->fn_list.setup_iq_regs(oct, iq_no);  in octeon_init_instr_queue()
147 	oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db",  in octeon_init_instr_queue()
150 	if (!oct->check_db_wq[iq_no].wq) {  in octeon_init_instr_queue()
153 		lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);  in octeon_init_instr_queue()
154 		dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n",  in octeon_init_instr_queue()
159 	db_wq = &oct->check_db_wq[iq_no];  in octeon_init_instr_queue()
162 	db_wq->wk.ctxptr = oct;  in octeon_init_instr_queue()
169 int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)  in octeon_delete_instr_queue()  argument
172 	struct octeon_instr_queue *iq = oct->instr_queue[iq_no];  in octeon_delete_instr_queue()
174 	cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work);  in octeon_delete_instr_queue()
175 	destroy_workqueue(oct->check_db_wq[iq_no].wq);  in octeon_delete_instr_queue()
177 	if (OCTEON_CN6XXX(oct))  in octeon_delete_instr_queue()
179 		    CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn6xxx));  in octeon_delete_instr_queue()
180 	else if (OCTEON_CN23XX_PF(oct))  in octeon_delete_instr_queue()
182 		    CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_pf));  in octeon_delete_instr_queue()
183 	else if (OCTEON_CN23XX_VF(oct))  in octeon_delete_instr_queue()
185 		    CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_vf));  in octeon_delete_instr_queue()
191 		lio_dma_free(oct, (u32)q_size, iq->base_addr,  in octeon_delete_instr_queue()
193 		oct->io_qmask.iq &= ~(1ULL << iq_no);  in octeon_delete_instr_queue()
194 		vfree(oct->instr_queue[iq_no]);  in octeon_delete_instr_queue()
195 		oct->instr_queue[iq_no] = NULL;  in octeon_delete_instr_queue()
196 		oct->num_iqs--;  in octeon_delete_instr_queue()
203 int octeon_setup_iq(struct octeon_device *oct,  in octeon_setup_iq()  argument
211 	int numa_node = dev_to_node(&oct->pci_dev->dev);  in octeon_setup_iq()
213 	if (oct->instr_queue[iq_no]) {  in octeon_setup_iq()
214 		dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n",  in octeon_setup_iq()
216 		oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64;  in octeon_setup_iq()
217 		oct->instr_queue[iq_no]->app_ctx = app_ctx;  in octeon_setup_iq()
220 	oct->instr_queue[iq_no] =  in octeon_setup_iq()
222 	if (!oct->instr_queue[iq_no])  in octeon_setup_iq()
223 		oct->instr_queue[iq_no] =  in octeon_setup_iq()
225 	if (!oct->instr_queue[iq_no])  in octeon_setup_iq()
229 	oct->instr_queue[iq_no]->q_index = q_index;  in octeon_setup_iq()
230 	oct->instr_queue[iq_no]->app_ctx = app_ctx;  in octeon_setup_iq()
231 	oct->instr_queue[iq_no]->ifidx = ifidx;  in octeon_setup_iq()
233 	if (octeon_init_instr_queue(oct, txpciq, num_descs)) {  in octeon_setup_iq()
234 		vfree(oct->instr_queue[iq_no]);  in octeon_setup_iq()
235 		oct->instr_queue[iq_no] = NULL;  in octeon_setup_iq()
239 	oct->num_iqs++;  in octeon_setup_iq()
240 	if (oct->fn_list.enable_io_queues(oct)) {  in octeon_setup_iq()
241 		octeon_delete_instr_queue(oct, iq_no);  in octeon_setup_iq()
248 int lio_wait_for_instr_fetch(struct octeon_device *oct)  in lio_wait_for_instr_fetch()  argument
255 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {  in lio_wait_for_instr_fetch()
256 			if (!(oct->io_qmask.iq & BIT_ULL(i)))  in lio_wait_for_instr_fetch()
259 			    atomic_read(&oct->instr_queue[i]->instr_pending);  in lio_wait_for_instr_fetch()
261 				__check_db_timeout(oct, i);  in lio_wait_for_instr_fetch()
276 ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq)  in ring_doorbell()  argument
278 	if (atomic_read(&oct->status) == OCT_DEV_RUNNING) {  in ring_doorbell()
288 octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no)  in octeon_ring_doorbell_locked()  argument
292 	iq = oct->instr_queue[iq_no];  in octeon_ring_doorbell_locked()
295 		ring_doorbell(oct, iq);  in octeon_ring_doorbell_locked()
348 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,  in octeon_register_reqtype_free_fn()  argument
352 		dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n",  in octeon_register_reqtype_free_fn()
357 	reqtype_free_fn[oct->octeon_id][reqtype] = fn;  in octeon_register_reqtype_free_fn()
372 lio_process_iq_request_list(struct octeon_device *oct,  in lio_process_iq_request_list()  argument
375 	struct cavium_wq *cwq = &oct->dma_comp_wq;  in lio_process_iq_request_list()
398 			reqtype_free_fn[oct->octeon_id][reqtype](buf);  in lio_process_iq_request_list()
409 			spin_lock_irqsave(&oct->response_list  in lio_process_iq_request_list()
411 			atomic_inc(&oct->response_list  in lio_process_iq_request_list()
413 			list_add_tail(&sc->node, &oct->response_list  in lio_process_iq_request_list()
415 			spin_unlock_irqrestore(&oct->response_list  in lio_process_iq_request_list()
420 			dev_err(&oct->pci_dev->dev,  in lio_process_iq_request_list()
440 	if (atomic_read(&oct->response_list  in lio_process_iq_request_list()
449 octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,  in octeon_flush_iq()  argument
461 	iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);  in octeon_flush_iq()
470 				lio_process_iq_request_list(oct, iq,  in octeon_flush_iq()
475 				lio_process_iq_request_list(oct, iq, 0);  in octeon_flush_iq()
501 static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)  in __check_db_timeout()  argument
506 	if (!oct)  in __check_db_timeout()
509 	iq = oct->instr_queue[iq_no];  in __check_db_timeout()
523 	octeon_flush_iq(oct, iq, 0);  in __check_db_timeout()
534 	struct octeon_device *oct = (struct octeon_device *)wk->ctxptr;  in check_db_timeout()  local
536 	struct cavium_wq *db_wq = &oct->check_db_wq[iq_no];  in check_db_timeout()
539 	__check_db_timeout(oct, iq_no);  in check_db_timeout()
544 octeon_send_command(struct octeon_device *oct, u32 iq_no,  in octeon_send_command()  argument
550 	struct octeon_instr_queue *iq = oct->instr_queue[iq_no];  in octeon_send_command()
563 		INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);  in octeon_send_command()
564 		INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);  in octeon_send_command()
568 			ring_doorbell(oct, iq);  in octeon_send_command()
570 		INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);  in octeon_send_command()
584 octeon_prepare_soft_command(struct octeon_device *oct,  in octeon_prepare_soft_command()  argument
602 	oct_cfg = octeon_get_conf(oct);  in octeon_prepare_soft_command()
604 	if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {  in octeon_prepare_soft_command()
607 		ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;  in octeon_prepare_soft_command()
615 			oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;  in octeon_prepare_soft_command()
620 			oct->instr_queue[sc->iq_no]->txpciq.s.ctrl_qpg;  in octeon_prepare_soft_command()
639 			rdp->pcie_port = oct->pcie_port;  in octeon_prepare_soft_command()
676 			rdp->pcie_port = oct->pcie_port;  in octeon_prepare_soft_command()
690 int octeon_send_soft_command(struct octeon_device *oct,  in octeon_send_soft_command()  argument
699 	iq = oct->instr_queue[sc->iq_no];  in octeon_send_soft_command()
701 		dev_err(&oct->pci_dev->dev, "Soft commands are not allowed on Queue %d\n",  in octeon_send_soft_command()
703 		INCR_INSTRQUEUE_PKT_COUNT(oct, sc->iq_no, instr_dropped, 1);  in octeon_send_soft_command()
707 	if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {  in octeon_send_soft_command()
739 	return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,  in octeon_send_soft_command()
743 int octeon_setup_sc_buffer_pool(struct octeon_device *oct)  in octeon_setup_sc_buffer_pool()  argument
749 	INIT_LIST_HEAD(&oct->sc_buf_pool.head);  in octeon_setup_sc_buffer_pool()
750 	spin_lock_init(&oct->sc_buf_pool.lock);  in octeon_setup_sc_buffer_pool()
751 	atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0);  in octeon_setup_sc_buffer_pool()
755 			lio_dma_alloc(oct,  in octeon_setup_sc_buffer_pool()
759 			octeon_free_sc_buffer_pool(oct);  in octeon_setup_sc_buffer_pool()
766 		list_add_tail(&sc->node, &oct->sc_buf_pool.head);  in octeon_setup_sc_buffer_pool()
772 int octeon_free_sc_done_list(struct octeon_device *oct)  in octeon_free_sc_done_list()  argument
779 	done_sc_list = &oct->response_list[OCTEON_DONE_SC_LIST];  in octeon_free_sc_done_list()
780 	zombie_sc_list = &oct->response_list[OCTEON_ZOMBIE_SC_LIST];  in octeon_free_sc_done_list()
785 	sc_lists_lock = &oct->response_list[OCTEON_ORDERED_SC_LIST].lock;  in octeon_free_sc_done_list()
801 				octeon_free_soft_command(oct, sc);  in octeon_free_sc_done_list()
811 int octeon_free_sc_zombie_list(struct octeon_device *oct)  in octeon_free_sc_zombie_list()  argument
818 	zombie_sc_list = &oct->response_list[OCTEON_ZOMBIE_SC_LIST];  in octeon_free_sc_zombie_list()
819 	sc_lists_lock = &oct->response_list[OCTEON_ORDERED_SC_LIST].lock;  in octeon_free_sc_zombie_list()
827 		octeon_free_soft_command(oct, sc);  in octeon_free_sc_zombie_list()
835 int octeon_free_sc_buffer_pool(struct octeon_device *oct)  in octeon_free_sc_buffer_pool()  argument
840 	octeon_free_sc_zombie_list(oct);  in octeon_free_sc_buffer_pool()
842 	spin_lock_bh(&oct->sc_buf_pool.lock);  in octeon_free_sc_buffer_pool()
844 	list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) {  in octeon_free_sc_buffer_pool()
849 		lio_dma_free(oct, sc->size, sc, sc->dma_addr);  in octeon_free_sc_buffer_pool()
852 	INIT_LIST_HEAD(&oct->sc_buf_pool.head);  in octeon_free_sc_buffer_pool()
854 	spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_free_sc_buffer_pool()
859 struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct,  in octeon_alloc_soft_command()  argument
876 	spin_lock_bh(&oct->sc_buf_pool.lock);  in octeon_alloc_soft_command()
878 	if (list_empty(&oct->sc_buf_pool.head)) {  in octeon_alloc_soft_command()
879 		spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_alloc_soft_command()
883 	list_for_each(tmp, &oct->sc_buf_pool.head)  in octeon_alloc_soft_command()
888 	atomic_inc(&oct->sc_buf_pool.alloc_buf_count);  in octeon_alloc_soft_command()
890 	spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_alloc_soft_command()
930 void octeon_free_soft_command(struct octeon_device *oct,  in octeon_free_soft_command()  argument
933 	spin_lock_bh(&oct->sc_buf_pool.lock);  in octeon_free_soft_command()
935 	list_add_tail(&sc->node, &oct->sc_buf_pool.head);  in octeon_free_soft_command()
937 	atomic_dec(&oct->sc_buf_pool.alloc_buf_count);  in octeon_free_soft_command()
939 	spin_unlock_bh(&oct->sc_buf_pool.lock);  in octeon_free_soft_command()