58 static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in alloc_cqn()  argument
60 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;  in alloc_cqn()
89 static void free_cqn(struct hns_roce_dev *hr_dev, unsigned long cqn)  in free_cqn()  argument
91 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;  in free_cqn()
103 static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev,  in hns_roce_create_cqc()  argument
107 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_create_cqc()
111 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);  in hns_roce_create_cqc()
117 	hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle);  in hns_roce_create_cqc()
119 	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC,  in hns_roce_create_cqc()
126 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);  in hns_roce_create_cqc()
131 static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in alloc_cqc()  argument
133 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;  in alloc_cqc()
134 	struct ib_device *ibdev = &hr_dev->ib_dev;  in alloc_cqc()
139 	ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts),  in alloc_cqc()
147 	ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn);  in alloc_cqc()
160 	ret = hns_roce_create_cqc(hr_dev, hr_cq, mtts, dma_handle);  in alloc_cqc()
169 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);  in alloc_cqc()
174 static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in free_cqc()  argument
176 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;  in free_cqc()
177 	struct device *dev = hr_dev->dev;  in free_cqc()
180 	ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_CQC,  in free_cqc()
189 	synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq);  in free_cqc()
196 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);  in free_cqc()
199 static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,  in alloc_cq_buf()  argument
202 	struct ib_device *ibdev = &hr_dev->ib_dev;  in alloc_cq_buf()
206 	buf_attr.page_shift = hr_dev->caps.cqe_buf_pg_sz + PAGE_SHIFT;  in alloc_cq_buf()
208 	buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num;  in alloc_cq_buf()
211 	ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,  in alloc_cq_buf()
212 				  hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT,  in alloc_cq_buf()
220 static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in free_cq_buf()  argument
222 	hns_roce_mtr_destroy(hr_dev, &hr_cq->mtr);  in free_cq_buf()
225 static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,  in alloc_cq_db()  argument
229 	bool has_db = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB;  in alloc_cq_db()
246 			err = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1);  in alloc_cq_db()
253 		hr_cq->db_reg = hr_dev->reg_base + hr_dev->odb_offset +  in alloc_cq_db()
254 				DB_REG_OFFSET * hr_dev->priv_uar.index;  in alloc_cq_db()
260 static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,  in free_cq_db()  argument
275 		hns_roce_free_db(hr_dev, &hr_cq->db);  in free_cq_db()
279 static int verify_cq_create_attr(struct hns_roce_dev *hr_dev,  in verify_cq_create_attr()  argument
282 	struct ib_device *ibdev = &hr_dev->ib_dev;  in verify_cq_create_attr()
284 	if (!attr->cqe || attr->cqe > hr_dev->caps.max_cqes) {  in verify_cq_create_attr()
286 			  attr->cqe, hr_dev->caps.max_cqes);  in verify_cq_create_attr()
290 	if (attr->comp_vector >= hr_dev->caps.num_comp_vectors) {  in verify_cq_create_attr()
292 			  attr->comp_vector, hr_dev->caps.num_comp_vectors);  in verify_cq_create_attr()
317 	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);  in set_cq_param()  local
319 	cq_entries = max(cq_entries, hr_dev->caps.min_cqes);  in set_cq_param()
333 	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);  in set_cqe_size()  local
336 		hr_cq->cqe_size = hr_dev->caps.cqe_sz;  in set_cqe_size()
343 			ibdev_err(&hr_dev->ib_dev,  in set_cqe_size()
359 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);  in hns_roce_create_cq()  local
362 	struct ib_device *ibdev = &hr_dev->ib_dev;  in hns_roce_create_cq()
369 	ret = verify_cq_create_attr(hr_dev, attr);  in hns_roce_create_cq()
385 	ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr);  in hns_roce_create_cq()
391 	ret = alloc_cq_db(hr_dev, hr_cq, udata, ucmd.db_addr, &resp);  in hns_roce_create_cq()
397 	ret = alloc_cqn(hr_dev, hr_cq);  in hns_roce_create_cq()
403 	ret = alloc_cqc(hr_dev, hr_cq);  in hns_roce_create_cq()
426 	free_cqc(hr_dev, hr_cq);  in hns_roce_create_cq()
428 	free_cqn(hr_dev, hr_cq->cqn);  in hns_roce_create_cq()
430 	free_cq_db(hr_dev, hr_cq, udata);  in hns_roce_create_cq()
432 	free_cq_buf(hr_dev, hr_cq);  in hns_roce_create_cq()
438 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);  in hns_roce_destroy_cq()  local
441 	free_cqc(hr_dev, hr_cq);  in hns_roce_destroy_cq()
442 	free_cqn(hr_dev, hr_cq->cqn);  in hns_roce_destroy_cq()
443 	free_cq_db(hr_dev, hr_cq, udata);  in hns_roce_destroy_cq()
444 	free_cq_buf(hr_dev, hr_cq);  in hns_roce_destroy_cq()
449 void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)  in hns_roce_cq_completion()  argument
454 	hr_cq = xa_load(&hr_dev->cq_table.array,  in hns_roce_cq_completion()
455 			cqn & (hr_dev->caps.num_cqs - 1));  in hns_roce_cq_completion()
457 		dev_warn(hr_dev->dev, "completion event for bogus CQ 0x%06x\n",  in hns_roce_cq_completion()
468 void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)  in hns_roce_cq_event()  argument
470 	struct device *dev = hr_dev->dev;  in hns_roce_cq_event()
475 	hr_cq = xa_load(&hr_dev->cq_table.array,  in hns_roce_cq_event()
476 			cqn & (hr_dev->caps.num_cqs - 1));  in hns_roce_cq_event()
504 void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev)  in hns_roce_init_cq_table()  argument
506 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;  in hns_roce_init_cq_table()
513 	reserved_from_bot = hr_dev->caps.reserved_cqs;  in hns_roce_init_cq_table()
522 		cq_table->bank[i].max = hr_dev->caps.num_cqs /  in hns_roce_init_cq_table()
527 void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev)  in hns_roce_cleanup_cq_table()  argument
532 		ida_destroy(&hr_dev->cq_table.bank[i].ida);  in hns_roce_cleanup_cq_table()