1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
5 /* Copyright (c) 2020-2022, Alibaba Group. */
12 	u64 db_data = FIELD_PREP(ERDMA_CQDB_CI_MASK, cmdq->cq.ci) |  in arm_cmdq_cq()
14 		      FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cmdq->cq.cmdsn) |  in arm_cmdq_cq()
15 		      FIELD_PREP(ERDMA_CQDB_IDX_MASK, cmdq->cq.cmdsn);  in arm_cmdq_cq()
17 	*cmdq->cq.db_record = db_data;  in arm_cmdq_cq()
18 	writeq(db_data, dev->func_bar + ERDMA_CMDQ_CQDB_REG);  in arm_cmdq_cq()
20 	atomic64_inc(&cmdq->cq.armed_num);  in arm_cmdq_cq()
26 	u64 db_data = FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi);  in kick_cmdq_db()
28 	*cmdq->sq.db_record = db_data;  in kick_cmdq_db()
29 	writeq(db_data, dev->func_bar + ERDMA_CMDQ_SQDB_REG);  in kick_cmdq_db()
36 	spin_lock(&cmdq->lock);  in get_comp_wait()
37 	comp_idx = find_first_zero_bit(cmdq->comp_wait_bitmap,  in get_comp_wait()
38 				       cmdq->max_outstandings);  in get_comp_wait()
39 	if (comp_idx == cmdq->max_outstandings) {  in get_comp_wait()
40 		spin_unlock(&cmdq->lock);  in get_comp_wait()
41 		return ERR_PTR(-ENOMEM);  in get_comp_wait()
44 	__set_bit(comp_idx, cmdq->comp_wait_bitmap);  in get_comp_wait()
45 	spin_unlock(&cmdq->lock);  in get_comp_wait()
47 	return &cmdq->wait_pool[comp_idx];  in get_comp_wait()
55 	cmdq->wait_pool[comp_wait->ctx_id].cmd_status = ERDMA_CMD_STATUS_INIT;  in put_comp_wait()
56 	spin_lock(&cmdq->lock);  in put_comp_wait()
57 	used = __test_and_clear_bit(comp_wait->ctx_id, cmdq->comp_wait_bitmap);  in put_comp_wait()
58 	spin_unlock(&cmdq->lock);  in put_comp_wait()
68 	cmdq->wait_pool =  in erdma_cmdq_wait_res_init()
69 		devm_kcalloc(&dev->pdev->dev, cmdq->max_outstandings,  in erdma_cmdq_wait_res_init()
71 	if (!cmdq->wait_pool)  in erdma_cmdq_wait_res_init()
72 		return -ENOMEM;  in erdma_cmdq_wait_res_init()
74 	spin_lock_init(&cmdq->lock);  in erdma_cmdq_wait_res_init()
75 	cmdq->comp_wait_bitmap = devm_bitmap_zalloc(  in erdma_cmdq_wait_res_init()
76 		&dev->pdev->dev, cmdq->max_outstandings, GFP_KERNEL);  in erdma_cmdq_wait_res_init()
77 	if (!cmdq->comp_wait_bitmap)  in erdma_cmdq_wait_res_init()
78 		return -ENOMEM;  in erdma_cmdq_wait_res_init()
80 	for (i = 0; i < cmdq->max_outstandings; i++) {  in erdma_cmdq_wait_res_init()
81 		init_completion(&cmdq->wait_pool[i].wait_event);  in erdma_cmdq_wait_res_init()
82 		cmdq->wait_pool[i].ctx_id = i;  in erdma_cmdq_wait_res_init()
90 	struct erdma_cmdq *cmdq = &dev->cmdq;  in erdma_cmdq_sq_init()
91 	struct erdma_cmdq_sq *sq = &cmdq->sq;  in erdma_cmdq_sq_init()
94 	sq->wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE);  in erdma_cmdq_sq_init()
95 	sq->depth = cmdq->max_outstandings * sq->wqebb_cnt;  in erdma_cmdq_sq_init()
97 	buf_size = sq->depth << SQEBB_SHIFT;  in erdma_cmdq_sq_init()
99 	sq->qbuf =  in erdma_cmdq_sq_init()
100 		dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),  in erdma_cmdq_sq_init()
101 				   &sq->qbuf_dma_addr, GFP_KERNEL);  in erdma_cmdq_sq_init()
102 	if (!sq->qbuf)  in erdma_cmdq_sq_init()
103 		return -ENOMEM;  in erdma_cmdq_sq_init()
105 	sq->db_record = (u64 *)(sq->qbuf + buf_size);  in erdma_cmdq_sq_init()
107 	spin_lock_init(&sq->lock);  in erdma_cmdq_sq_init()
110 			  upper_32_bits(sq->qbuf_dma_addr));  in erdma_cmdq_sq_init()
112 			  lower_32_bits(sq->qbuf_dma_addr));  in erdma_cmdq_sq_init()
113 	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth);  in erdma_cmdq_sq_init()
115 			  sq->qbuf_dma_addr + buf_size);  in erdma_cmdq_sq_init()
122 	struct erdma_cmdq *cmdq = &dev->cmdq;  in erdma_cmdq_cq_init()
123 	struct erdma_cmdq_cq *cq = &cmdq->cq;  in erdma_cmdq_cq_init()
126 	cq->depth = cmdq->sq.depth;  in erdma_cmdq_cq_init()
127 	buf_size = cq->depth << CQE_SHIFT;  in erdma_cmdq_cq_init()
129 	cq->qbuf =  in erdma_cmdq_cq_init()
130 		dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),  in erdma_cmdq_cq_init()
131 				   &cq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);  in erdma_cmdq_cq_init()
132 	if (!cq->qbuf)  in erdma_cmdq_cq_init()
133 		return -ENOMEM;  in erdma_cmdq_cq_init()
135 	spin_lock_init(&cq->lock);  in erdma_cmdq_cq_init()
137 	cq->db_record = (u64 *)(cq->qbuf + buf_size);  in erdma_cmdq_cq_init()
139 	atomic64_set(&cq->armed_num, 0);  in erdma_cmdq_cq_init()
142 			  upper_32_bits(cq->qbuf_dma_addr));  in erdma_cmdq_cq_init()
144 			  lower_32_bits(cq->qbuf_dma_addr));  in erdma_cmdq_cq_init()
146 			  cq->qbuf_dma_addr + buf_size);  in erdma_cmdq_cq_init()
153 	struct erdma_cmdq *cmdq = &dev->cmdq;  in erdma_cmdq_eq_init()
154 	struct erdma_eq *eq = &cmdq->eq;  in erdma_cmdq_eq_init()
157 	eq->depth = cmdq->max_outstandings;  in erdma_cmdq_eq_init()
158 	buf_size = eq->depth << EQE_SHIFT;  in erdma_cmdq_eq_init()
160 	eq->qbuf =  in erdma_cmdq_eq_init()
161 		dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),  in erdma_cmdq_eq_init()
162 				   &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);  in erdma_cmdq_eq_init()
163 	if (!eq->qbuf)  in erdma_cmdq_eq_init()
164 		return -ENOMEM;  in erdma_cmdq_eq_init()
166 	spin_lock_init(&eq->lock);  in erdma_cmdq_eq_init()
167 	atomic64_set(&eq->event_num, 0);  in erdma_cmdq_eq_init()
169 	eq->db_addr =  in erdma_cmdq_eq_init()
170 		(u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG);  in erdma_cmdq_eq_init()
171 	eq->db_record = (u64 *)(eq->qbuf + buf_size);  in erdma_cmdq_eq_init()
174 			  upper_32_bits(eq->qbuf_dma_addr));  in erdma_cmdq_eq_init()
176 			  lower_32_bits(eq->qbuf_dma_addr));  in erdma_cmdq_eq_init()
177 	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth);  in erdma_cmdq_eq_init()
179 			  eq->qbuf_dma_addr + buf_size);  in erdma_cmdq_eq_init()
187 	struct erdma_cmdq *cmdq = &dev->cmdq;  in erdma_cmdq_init()
190 	cmdq->max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;  in erdma_cmdq_init()
191 	cmdq->use_event = false;  in erdma_cmdq_init()
193 	sema_init(&cmdq->credits, cmdq->max_outstandings);  in erdma_cmdq_init()
224 		dev_err(&dev->pdev->dev, "wait init done failed.\n");  in erdma_cmdq_init()
225 		err = -ETIMEDOUT;  in erdma_cmdq_init()
229 	set_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);  in erdma_cmdq_init()
234 	dma_free_coherent(&dev->pdev->dev,  in erdma_cmdq_init()
235 			  (cmdq->eq.depth << EQE_SHIFT) +  in erdma_cmdq_init()
237 			  cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr);  in erdma_cmdq_init()
240 	dma_free_coherent(&dev->pdev->dev,  in erdma_cmdq_init()
241 			  (cmdq->cq.depth << CQE_SHIFT) +  in erdma_cmdq_init()
243 			  cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);  in erdma_cmdq_init()
246 	dma_free_coherent(&dev->pdev->dev,  in erdma_cmdq_init()
247 			  (cmdq->sq.depth << SQEBB_SHIFT) +  in erdma_cmdq_init()
249 			  cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);  in erdma_cmdq_init()
257 	dev->cmdq.use_event = true;  in erdma_finish_cmdq_init()
258 	arm_cmdq_cq(&dev->cmdq);  in erdma_finish_cmdq_init()
263 	struct erdma_cmdq *cmdq = &dev->cmdq;  in erdma_cmdq_destroy()
265 	clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);  in erdma_cmdq_destroy()
267 	dma_free_coherent(&dev->pdev->dev,  in erdma_cmdq_destroy()
268 			  (cmdq->eq.depth << EQE_SHIFT) +  in erdma_cmdq_destroy()
270 			  cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr);  in erdma_cmdq_destroy()
271 	dma_free_coherent(&dev->pdev->dev,  in erdma_cmdq_destroy()
272 			  (cmdq->sq.depth << SQEBB_SHIFT) +  in erdma_cmdq_destroy()
274 			  cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);  in erdma_cmdq_destroy()
275 	dma_free_coherent(&dev->pdev->dev,  in erdma_cmdq_destroy()
276 			  (cmdq->cq.depth << CQE_SHIFT) +  in erdma_cmdq_destroy()
278 			  cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);  in erdma_cmdq_destroy()
283 	__be32 *cqe = get_queue_entry(cmdq->cq.qbuf, cmdq->cq.ci,  in get_next_valid_cmdq_cqe()
284 				      cmdq->cq.depth, CQE_SHIFT);  in get_next_valid_cmdq_cqe()
288 	return owner ^ !!(cmdq->cq.ci & cmdq->cq.depth) ? cqe : NULL;  in get_next_valid_cmdq_cqe()
297 	comp_wait->cmd_status = ERDMA_CMD_STATUS_ISSUED;  in push_cmdq_sqe()
298 	reinit_completion(&comp_wait->wait_event);  in push_cmdq_sqe()
299 	comp_wait->sq_pi = cmdq->sq.pi;  in push_cmdq_sqe()
301 	wqe = get_queue_entry(cmdq->sq.qbuf, cmdq->sq.pi, cmdq->sq.depth,  in push_cmdq_sqe()
305 	cmdq->sq.pi += cmdq->sq.wqebb_cnt;  in push_cmdq_sqe()
306 	hdr |= FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi) |  in push_cmdq_sqe()
308 			  comp_wait->ctx_id) |  in push_cmdq_sqe()
309 	       FIELD_PREP(ERDMA_CMD_HDR_WQEBB_CNT_MASK, cmdq->sq.wqebb_cnt - 1);  in push_cmdq_sqe()
326 		return -EAGAIN;  in erdma_poll_single_cmd_completion()
328 	cmdq->cq.ci++;  in erdma_poll_single_cmd_completion()
334 	sqe = get_queue_entry(cmdq->sq.qbuf, sqe_idx, cmdq->sq.depth,  in erdma_poll_single_cmd_completion()
337 	comp_wait = &cmdq->wait_pool[ctx_id];  in erdma_poll_single_cmd_completion()
338 	if (comp_wait->cmd_status != ERDMA_CMD_STATUS_ISSUED)  in erdma_poll_single_cmd_completion()
339 		return -EIO;  in erdma_poll_single_cmd_completion()
341 	comp_wait->cmd_status = ERDMA_CMD_STATUS_FINISHED;  in erdma_poll_single_cmd_completion()
342 	comp_wait->comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);  in erdma_poll_single_cmd_completion()
343 	cmdq->sq.ci += cmdq->sq.wqebb_cnt;  in erdma_poll_single_cmd_completion()
346 		comp_wait->comp_data[i] = __be32_to_cpu(*(cqe + 2 + i));  in erdma_poll_single_cmd_completion()
348 	if (cmdq->use_event)  in erdma_poll_single_cmd_completion()
349 		complete(&comp_wait->wait_event);  in erdma_poll_single_cmd_completion()
359 	spin_lock_irqsave(&cmdq->cq.lock, flags);  in erdma_polling_cmd_completions()
364 	for (comp_num = 0; comp_num < cmdq->max_outstandings; comp_num++)  in erdma_polling_cmd_completions()
368 	if (comp_num && cmdq->use_event)  in erdma_polling_cmd_completions()
371 	spin_unlock_irqrestore(&cmdq->cq.lock, flags);  in erdma_polling_cmd_completions()
378 	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state) ||  in erdma_cmdq_completion_handler()
379 	    !cmdq->use_event)  in erdma_cmdq_completion_handler()
382 	while (get_next_valid_eqe(&cmdq->eq)) {  in erdma_cmdq_completion_handler()
383 		cmdq->eq.ci++;  in erdma_cmdq_completion_handler()
388 		cmdq->cq.cmdsn++;  in erdma_cmdq_completion_handler()
392 	notify_eq(&cmdq->eq);  in erdma_cmdq_completion_handler()
402 		if (comp_ctx->cmd_status != ERDMA_CMD_STATUS_ISSUED)  in erdma_poll_cmd_completion()
406 			return -ETIME;  in erdma_poll_cmd_completion()
419 	wait_for_completion_timeout(&comp_ctx->wait_event,  in erdma_wait_cmd_completion()
422 	if (unlikely(comp_ctx->cmd_status != ERDMA_CMD_STATUS_FINISHED)) {  in erdma_wait_cmd_completion()
423 		spin_lock_irqsave(&cmdq->cq.lock, flags);  in erdma_wait_cmd_completion()
424 		comp_ctx->cmd_status = ERDMA_CMD_STATUS_TIMEOUT;  in erdma_wait_cmd_completion()
425 		spin_unlock_irqrestore(&cmdq->cq.lock, flags);  in erdma_wait_cmd_completion()
426 		return -ETIME;  in erdma_wait_cmd_completion()
444 	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state))  in erdma_post_cmd_wait()
445 		return -ENODEV;  in erdma_post_cmd_wait()
447 	down(&cmdq->credits);  in erdma_post_cmd_wait()
451 		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);  in erdma_post_cmd_wait()
452 		set_bit(ERDMA_CMDQ_STATE_CTX_ERR_BIT, &cmdq->state);  in erdma_post_cmd_wait()
453 		up(&cmdq->credits);  in erdma_post_cmd_wait()
457 	spin_lock(&cmdq->sq.lock);  in erdma_post_cmd_wait()
459 	spin_unlock(&cmdq->sq.lock);  in erdma_post_cmd_wait()
461 	if (cmdq->use_event)  in erdma_post_cmd_wait()
469 		set_bit(ERDMA_CMDQ_STATE_TIMEOUT_BIT, &cmdq->state);  in erdma_post_cmd_wait()
470 		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);  in erdma_post_cmd_wait()
474 	if (comp_wait->comp_status)  in erdma_post_cmd_wait()
475 		ret = -EIO;  in erdma_post_cmd_wait()
478 		*resp0 = *((u64 *)&comp_wait->comp_data[0]);  in erdma_post_cmd_wait()
479 		*resp1 = *((u64 *)&comp_wait->comp_data[2]);  in erdma_post_cmd_wait()
484 	up(&cmdq->credits);  in erdma_post_cmd_wait()