1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/ib_umem.h>
16 #include <rdma/uverbs_ioctl.h>
17
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21
create_qp_cmd(struct erdma_dev * dev,struct erdma_qp * qp)22 static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp)
23 {
24 struct erdma_cmdq_create_qp_req req;
25 struct erdma_pd *pd = to_epd(qp->ibqp.pd);
26 struct erdma_uqp *user_qp;
27 u64 resp0, resp1;
28 int err;
29
30 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
31 CMDQ_OPCODE_CREATE_QP);
32
33 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
34 ilog2(qp->attrs.sq_size)) |
35 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
36 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
37 ilog2(qp->attrs.rq_size)) |
38 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
39
40 if (rdma_is_kernel_res(&qp->ibqp.res)) {
41 u32 pgsz_range = ilog2(SZ_1M) - PAGE_SHIFT;
42
43 req.sq_cqn_mtt_cfg =
44 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
45 pgsz_range) |
46 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
47 req.rq_cqn_mtt_cfg =
48 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
49 pgsz_range) |
50 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
51
52 req.sq_mtt_cfg =
53 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
54 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
55 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
56 ERDMA_MR_INLINE_MTT);
57 req.rq_mtt_cfg = req.sq_mtt_cfg;
58
59 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
60 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
61 req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr +
62 (qp->attrs.sq_size << SQEBB_SHIFT);
63 req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr +
64 (qp->attrs.rq_size << RQE_SHIFT);
65 } else {
66 user_qp = &qp->user_qp;
67 req.sq_cqn_mtt_cfg = FIELD_PREP(
68 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
69 ilog2(user_qp->sq_mtt.page_size) - PAGE_SHIFT);
70 req.sq_cqn_mtt_cfg |=
71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
72
73 req.rq_cqn_mtt_cfg = FIELD_PREP(
74 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
75 ilog2(user_qp->rq_mtt.page_size) - PAGE_SHIFT);
76 req.rq_cqn_mtt_cfg |=
77 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
78
79 req.sq_mtt_cfg = user_qp->sq_mtt.page_offset;
80 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
81 user_qp->sq_mtt.mtt_nents) |
82 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
83 user_qp->sq_mtt.mtt_type);
84
85 req.rq_mtt_cfg = user_qp->rq_mtt.page_offset;
86 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
87 user_qp->rq_mtt.mtt_nents) |
88 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK,
89 user_qp->rq_mtt.mtt_type);
90
91 req.sq_buf_addr = user_qp->sq_mtt.mtt_entry[0];
92 req.rq_buf_addr = user_qp->rq_mtt.mtt_entry[0];
93
94 req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr;
95 req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr;
96 }
97
98 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
99 &resp1);
100 if (!err)
101 qp->attrs.cookie =
102 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
103
104 return err;
105 }
106
regmr_cmd(struct erdma_dev * dev,struct erdma_mr * mr)107 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
108 {
109 struct erdma_cmdq_reg_mr_req req;
110 struct erdma_pd *pd = to_epd(mr->ibmr.pd);
111 u64 *phy_addr;
112 int i;
113
114 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
115
116 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
117 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
118 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
119 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
120 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
121 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access) |
122 FIELD_PREP(ERDMA_CMD_REGMR_ACC_MODE_MASK, 0);
123 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
124 ilog2(mr->mem.page_size)) |
125 FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) |
126 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
127
128 if (mr->type == ERDMA_MR_TYPE_DMA)
129 goto post_cmd;
130
131 if (mr->type == ERDMA_MR_TYPE_NORMAL) {
132 req.start_va = mr->mem.va;
133 req.size = mr->mem.len;
134 }
135
136 if (mr->type == ERDMA_MR_TYPE_FRMR ||
137 mr->mem.mtt_type == ERDMA_MR_INDIRECT_MTT) {
138 phy_addr = req.phy_addr;
139 *phy_addr = mr->mem.mtt_entry[0];
140 } else {
141 phy_addr = req.phy_addr;
142 for (i = 0; i < mr->mem.mtt_nents; i++)
143 *phy_addr++ = mr->mem.mtt_entry[i];
144 }
145
146 post_cmd:
147 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
148 }
149
create_cq_cmd(struct erdma_dev * dev,struct erdma_cq * cq)150 static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq)
151 {
152 struct erdma_cmdq_create_cq_req req;
153 u32 page_size;
154 struct erdma_mem *mtt;
155
156 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
157 CMDQ_OPCODE_CREATE_CQ);
158
159 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
160 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
161 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
162
163 if (rdma_is_kernel_res(&cq->ibcq.res)) {
164 page_size = SZ_32M;
165 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
166 ilog2(page_size) - PAGE_SHIFT);
167 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
168 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
169
170 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
171 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK,
172 ERDMA_MR_INLINE_MTT);
173
174 req.first_page_offset = 0;
175 req.cq_db_info_addr =
176 cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT);
177 } else {
178 mtt = &cq->user_cq.qbuf_mtt;
179 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
180 ilog2(mtt->page_size) - PAGE_SHIFT);
181 if (mtt->mtt_nents == 1) {
182 req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf);
183 req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf);
184 } else {
185 req.qbuf_addr_l = lower_32_bits(mtt->mtt_entry[0]);
186 req.qbuf_addr_h = upper_32_bits(mtt->mtt_entry[0]);
187 }
188 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
189 mtt->mtt_nents);
190 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK,
191 mtt->mtt_type);
192
193 req.first_page_offset = mtt->page_offset;
194 req.cq_db_info_addr = cq->user_cq.db_info_dma_addr;
195 }
196
197 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
198 }
199
erdma_alloc_idx(struct erdma_resource_cb * res_cb)200 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
201 {
202 int idx;
203 unsigned long flags;
204
205 spin_lock_irqsave(&res_cb->lock, flags);
206 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
207 res_cb->next_alloc_idx);
208 if (idx == res_cb->max_cap) {
209 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
210 if (idx == res_cb->max_cap) {
211 res_cb->next_alloc_idx = 1;
212 spin_unlock_irqrestore(&res_cb->lock, flags);
213 return -ENOSPC;
214 }
215 }
216
217 set_bit(idx, res_cb->bitmap);
218 res_cb->next_alloc_idx = idx + 1;
219 spin_unlock_irqrestore(&res_cb->lock, flags);
220
221 return idx;
222 }
223
erdma_free_idx(struct erdma_resource_cb * res_cb,u32 idx)224 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
225 {
226 unsigned long flags;
227 u32 used;
228
229 spin_lock_irqsave(&res_cb->lock, flags);
230 used = __test_and_clear_bit(idx, res_cb->bitmap);
231 spin_unlock_irqrestore(&res_cb->lock, flags);
232 WARN_ON(!used);
233 }
234
235 static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext * uctx,void * address,u32 size,u8 mmap_flag,u64 * mmap_offset)236 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
237 u32 size, u8 mmap_flag, u64 *mmap_offset)
238 {
239 struct erdma_user_mmap_entry *entry =
240 kzalloc(sizeof(*entry), GFP_KERNEL);
241 int ret;
242
243 if (!entry)
244 return NULL;
245
246 entry->address = (u64)address;
247 entry->mmap_flag = mmap_flag;
248
249 size = PAGE_ALIGN(size);
250
251 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
252 size);
253 if (ret) {
254 kfree(entry);
255 return NULL;
256 }
257
258 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
259
260 return &entry->rdma_entry;
261 }
262
erdma_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * unused)263 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
264 struct ib_udata *unused)
265 {
266 struct erdma_dev *dev = to_edev(ibdev);
267
268 memset(attr, 0, sizeof(*attr));
269
270 attr->max_mr_size = dev->attrs.max_mr_size;
271 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
272 attr->vendor_part_id = dev->pdev->device;
273 attr->hw_ver = dev->pdev->revision;
274 attr->max_qp = dev->attrs.max_qp - 1;
275 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
276 attr->max_qp_rd_atom = dev->attrs.max_ord;
277 attr->max_qp_init_rd_atom = dev->attrs.max_ird;
278 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
279 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
280 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
281 ibdev->local_dma_lkey = dev->attrs.local_dma_key;
282 attr->max_send_sge = dev->attrs.max_send_sge;
283 attr->max_recv_sge = dev->attrs.max_recv_sge;
284 attr->max_sge_rd = dev->attrs.max_sge_rd;
285 attr->max_cq = dev->attrs.max_cq - 1;
286 attr->max_cqe = dev->attrs.max_cqe;
287 attr->max_mr = dev->attrs.max_mr;
288 attr->max_pd = dev->attrs.max_pd;
289 attr->max_mw = dev->attrs.max_mw;
290 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
291 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
292 attr->fw_ver = dev->attrs.fw_version;
293
294 if (dev->netdev)
295 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
296 dev->netdev->dev_addr);
297
298 return 0;
299 }
300
erdma_query_gid(struct ib_device * ibdev,u32 port,int idx,union ib_gid * gid)301 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
302 union ib_gid *gid)
303 {
304 struct erdma_dev *dev = to_edev(ibdev);
305
306 memset(gid, 0, sizeof(*gid));
307 ether_addr_copy(gid->raw, dev->attrs.peer_addr);
308
309 return 0;
310 }
311
erdma_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)312 int erdma_query_port(struct ib_device *ibdev, u32 port,
313 struct ib_port_attr *attr)
314 {
315 struct erdma_dev *dev = to_edev(ibdev);
316 struct net_device *ndev = dev->netdev;
317
318 memset(attr, 0, sizeof(*attr));
319
320 attr->gid_tbl_len = 1;
321 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
322 attr->max_msg_sz = -1;
323
324 if (!ndev)
325 goto out;
326
327 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
328 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
329 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
330 if (netif_running(ndev) && netif_carrier_ok(ndev))
331 dev->state = IB_PORT_ACTIVE;
332 else
333 dev->state = IB_PORT_DOWN;
334 attr->state = dev->state;
335
336 out:
337 if (dev->state == IB_PORT_ACTIVE)
338 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
339 else
340 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
341
342 return 0;
343 }
344
erdma_get_port_immutable(struct ib_device * ibdev,u32 port,struct ib_port_immutable * port_immutable)345 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
346 struct ib_port_immutable *port_immutable)
347 {
348 port_immutable->gid_tbl_len = 1;
349 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
350
351 return 0;
352 }
353
erdma_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)354 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
355 {
356 struct erdma_pd *pd = to_epd(ibpd);
357 struct erdma_dev *dev = to_edev(ibpd->device);
358 int pdn;
359
360 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
361 if (pdn < 0)
362 return pdn;
363
364 pd->pdn = pdn;
365
366 return 0;
367 }
368
erdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)369 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
370 {
371 struct erdma_pd *pd = to_epd(ibpd);
372 struct erdma_dev *dev = to_edev(ibpd->device);
373
374 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
375
376 return 0;
377 }
378
erdma_qp_validate_cap(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)379 static int erdma_qp_validate_cap(struct erdma_dev *dev,
380 struct ib_qp_init_attr *attrs)
381 {
382 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
383 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
384 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
385 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
386 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
387 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
388 return -EINVAL;
389 }
390
391 return 0;
392 }
393
erdma_qp_validate_attr(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)394 static int erdma_qp_validate_attr(struct erdma_dev *dev,
395 struct ib_qp_init_attr *attrs)
396 {
397 if (attrs->qp_type != IB_QPT_RC)
398 return -EOPNOTSUPP;
399
400 if (attrs->srq)
401 return -EOPNOTSUPP;
402
403 if (!attrs->send_cq || !attrs->recv_cq)
404 return -EOPNOTSUPP;
405
406 return 0;
407 }
408
free_kernel_qp(struct erdma_qp * qp)409 static void free_kernel_qp(struct erdma_qp *qp)
410 {
411 struct erdma_dev *dev = qp->dev;
412
413 vfree(qp->kern_qp.swr_tbl);
414 vfree(qp->kern_qp.rwr_tbl);
415
416 if (qp->kern_qp.sq_buf)
417 dma_free_coherent(
418 &dev->pdev->dev,
419 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
420 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
421
422 if (qp->kern_qp.rq_buf)
423 dma_free_coherent(
424 &dev->pdev->dev,
425 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
426 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
427 }
428
init_kernel_qp(struct erdma_dev * dev,struct erdma_qp * qp,struct ib_qp_init_attr * attrs)429 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
430 struct ib_qp_init_attr *attrs)
431 {
432 struct erdma_kqp *kqp = &qp->kern_qp;
433 int size;
434
435 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
436 kqp->sig_all = 1;
437
438 kqp->sq_pi = 0;
439 kqp->sq_ci = 0;
440 kqp->rq_pi = 0;
441 kqp->rq_ci = 0;
442 kqp->hw_sq_db =
443 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
444 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
445
446 kqp->swr_tbl = vmalloc(qp->attrs.sq_size * sizeof(u64));
447 kqp->rwr_tbl = vmalloc(qp->attrs.rq_size * sizeof(u64));
448 if (!kqp->swr_tbl || !kqp->rwr_tbl)
449 goto err_out;
450
451 size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
452 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
453 &kqp->sq_buf_dma_addr, GFP_KERNEL);
454 if (!kqp->sq_buf)
455 goto err_out;
456
457 size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
458 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
459 &kqp->rq_buf_dma_addr, GFP_KERNEL);
460 if (!kqp->rq_buf)
461 goto err_out;
462
463 kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT);
464 kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT);
465
466 return 0;
467
468 err_out:
469 free_kernel_qp(qp);
470 return -ENOMEM;
471 }
472
get_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem,u64 start,u64 len,int access,u64 virt,unsigned long req_page_size,u8 force_indirect_mtt)473 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
474 u64 start, u64 len, int access, u64 virt,
475 unsigned long req_page_size, u8 force_indirect_mtt)
476 {
477 struct ib_block_iter biter;
478 uint64_t *phy_addr = NULL;
479 int ret = 0;
480
481 mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
482 if (IS_ERR(mem->umem)) {
483 ret = PTR_ERR(mem->umem);
484 mem->umem = NULL;
485 return ret;
486 }
487
488 mem->va = virt;
489 mem->len = len;
490 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
491 mem->page_offset = start & (mem->page_size - 1);
492 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
493 mem->page_cnt = mem->mtt_nents;
494
495 if (mem->page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES ||
496 force_indirect_mtt) {
497 mem->mtt_type = ERDMA_MR_INDIRECT_MTT;
498 mem->mtt_buf =
499 alloc_pages_exact(MTT_SIZE(mem->page_cnt), GFP_KERNEL);
500 if (!mem->mtt_buf) {
501 ret = -ENOMEM;
502 goto error_ret;
503 }
504 phy_addr = mem->mtt_buf;
505 } else {
506 mem->mtt_type = ERDMA_MR_INLINE_MTT;
507 phy_addr = mem->mtt_entry;
508 }
509
510 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) {
511 *phy_addr = rdma_block_iter_dma_address(&biter);
512 phy_addr++;
513 }
514
515 if (mem->mtt_type == ERDMA_MR_INDIRECT_MTT) {
516 mem->mtt_entry[0] =
517 dma_map_single(&dev->pdev->dev, mem->mtt_buf,
518 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE);
519 if (dma_mapping_error(&dev->pdev->dev, mem->mtt_entry[0])) {
520 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt));
521 mem->mtt_buf = NULL;
522 ret = -ENOMEM;
523 goto error_ret;
524 }
525 }
526
527 return 0;
528
529 error_ret:
530 if (mem->umem) {
531 ib_umem_release(mem->umem);
532 mem->umem = NULL;
533 }
534
535 return ret;
536 }
537
put_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem)538 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
539 {
540 if (mem->mtt_buf) {
541 dma_unmap_single(&dev->pdev->dev, mem->mtt_entry[0],
542 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE);
543 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt));
544 }
545
546 if (mem->umem) {
547 ib_umem_release(mem->umem);
548 mem->umem = NULL;
549 }
550 }
551
erdma_map_user_dbrecords(struct erdma_ucontext * ctx,u64 dbrecords_va,struct erdma_user_dbrecords_page ** dbr_page,dma_addr_t * dma_addr)552 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
553 u64 dbrecords_va,
554 struct erdma_user_dbrecords_page **dbr_page,
555 dma_addr_t *dma_addr)
556 {
557 struct erdma_user_dbrecords_page *page = NULL;
558 int rv = 0;
559
560 mutex_lock(&ctx->dbrecords_page_mutex);
561
562 list_for_each_entry(page, &ctx->dbrecords_page_list, list)
563 if (page->va == (dbrecords_va & PAGE_MASK))
564 goto found;
565
566 page = kmalloc(sizeof(*page), GFP_KERNEL);
567 if (!page) {
568 rv = -ENOMEM;
569 goto out;
570 }
571
572 page->va = (dbrecords_va & PAGE_MASK);
573 page->refcnt = 0;
574
575 page->umem = ib_umem_get(ctx->ibucontext.device,
576 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
577 if (IS_ERR(page->umem)) {
578 rv = PTR_ERR(page->umem);
579 kfree(page);
580 goto out;
581 }
582
583 list_add(&page->list, &ctx->dbrecords_page_list);
584
585 found:
586 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
587 (dbrecords_va & ~PAGE_MASK);
588 *dbr_page = page;
589 page->refcnt++;
590
591 out:
592 mutex_unlock(&ctx->dbrecords_page_mutex);
593 return rv;
594 }
595
596 static void
erdma_unmap_user_dbrecords(struct erdma_ucontext * ctx,struct erdma_user_dbrecords_page ** dbr_page)597 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
598 struct erdma_user_dbrecords_page **dbr_page)
599 {
600 if (!ctx || !(*dbr_page))
601 return;
602
603 mutex_lock(&ctx->dbrecords_page_mutex);
604 if (--(*dbr_page)->refcnt == 0) {
605 list_del(&(*dbr_page)->list);
606 ib_umem_release((*dbr_page)->umem);
607 kfree(*dbr_page);
608 }
609
610 *dbr_page = NULL;
611 mutex_unlock(&ctx->dbrecords_page_mutex);
612 }
613
init_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx,u64 va,u32 len,u64 db_info_va)614 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
615 u64 va, u32 len, u64 db_info_va)
616 {
617 dma_addr_t db_info_dma_addr;
618 u32 rq_offset;
619 int ret;
620
621 if (len < (PAGE_ALIGN(qp->attrs.sq_size * SQEBB_SIZE) +
622 qp->attrs.rq_size * RQE_SIZE))
623 return -EINVAL;
624
625 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mtt, va,
626 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
627 (SZ_1M - SZ_4K), 1);
628 if (ret)
629 return ret;
630
631 rq_offset = PAGE_ALIGN(qp->attrs.sq_size << SQEBB_SHIFT);
632 qp->user_qp.rq_offset = rq_offset;
633
634 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset,
635 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
636 (SZ_1M - SZ_4K), 1);
637 if (ret)
638 goto put_sq_mtt;
639
640 ret = erdma_map_user_dbrecords(uctx, db_info_va,
641 &qp->user_qp.user_dbr_page,
642 &db_info_dma_addr);
643 if (ret)
644 goto put_rq_mtt;
645
646 qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr;
647 qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE;
648
649 return 0;
650
651 put_rq_mtt:
652 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt);
653
654 put_sq_mtt:
655 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt);
656
657 return ret;
658 }
659
free_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx)660 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
661 {
662 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt);
663 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt);
664 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
665 }
666
erdma_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)667 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
668 struct ib_udata *udata)
669 {
670 struct erdma_qp *qp = to_eqp(ibqp);
671 struct erdma_dev *dev = to_edev(ibqp->device);
672 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
673 udata, struct erdma_ucontext, ibucontext);
674 struct erdma_ureq_create_qp ureq;
675 struct erdma_uresp_create_qp uresp;
676 int ret;
677
678 ret = erdma_qp_validate_cap(dev, attrs);
679 if (ret)
680 goto err_out;
681
682 ret = erdma_qp_validate_attr(dev, attrs);
683 if (ret)
684 goto err_out;
685
686 qp->scq = to_ecq(attrs->send_cq);
687 qp->rcq = to_ecq(attrs->recv_cq);
688 qp->dev = dev;
689 qp->attrs.cc = dev->attrs.cc;
690
691 init_rwsem(&qp->state_lock);
692 kref_init(&qp->ref);
693 init_completion(&qp->safe_free);
694
695 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
696 XA_LIMIT(1, dev->attrs.max_qp - 1),
697 &dev->next_alloc_qpn, GFP_KERNEL);
698 if (ret < 0) {
699 ret = -ENOMEM;
700 goto err_out;
701 }
702
703 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
704 ERDMA_MAX_WQEBB_PER_SQE);
705 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
706
707 if (uctx) {
708 ret = ib_copy_from_udata(&ureq, udata,
709 min(sizeof(ureq), udata->inlen));
710 if (ret)
711 goto err_out_xa;
712
713 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
714 ureq.db_record_va);
715 if (ret)
716 goto err_out_xa;
717
718 memset(&uresp, 0, sizeof(uresp));
719
720 uresp.num_sqe = qp->attrs.sq_size;
721 uresp.num_rqe = qp->attrs.rq_size;
722 uresp.qp_id = QP_ID(qp);
723 uresp.rq_offset = qp->user_qp.rq_offset;
724
725 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
726 if (ret)
727 goto err_out_cmd;
728 } else {
729 init_kernel_qp(dev, qp, attrs);
730 }
731
732 qp->attrs.max_send_sge = attrs->cap.max_send_sge;
733 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
734 qp->attrs.state = ERDMA_QP_STATE_IDLE;
735
736 ret = create_qp_cmd(dev, qp);
737 if (ret)
738 goto err_out_cmd;
739
740 spin_lock_init(&qp->lock);
741
742 return 0;
743
744 err_out_cmd:
745 if (uctx)
746 free_user_qp(qp, uctx);
747 else
748 free_kernel_qp(qp);
749 err_out_xa:
750 xa_erase(&dev->qp_xa, QP_ID(qp));
751 err_out:
752 return ret;
753 }
754
erdma_create_stag(struct erdma_dev * dev,u32 * stag)755 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
756 {
757 int stag_idx;
758
759 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
760 if (stag_idx < 0)
761 return stag_idx;
762
763 /* For now, we always let key field be zero. */
764 *stag = (stag_idx << 8);
765
766 return 0;
767 }
768
erdma_get_dma_mr(struct ib_pd * ibpd,int acc)769 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
770 {
771 struct erdma_dev *dev = to_edev(ibpd->device);
772 struct erdma_mr *mr;
773 u32 stag;
774 int ret;
775
776 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
777 if (!mr)
778 return ERR_PTR(-ENOMEM);
779
780 ret = erdma_create_stag(dev, &stag);
781 if (ret)
782 goto out_free;
783
784 mr->type = ERDMA_MR_TYPE_DMA;
785
786 mr->ibmr.lkey = stag;
787 mr->ibmr.rkey = stag;
788 mr->ibmr.pd = ibpd;
789 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
790 ret = regmr_cmd(dev, mr);
791 if (ret)
792 goto out_remove_stag;
793
794 return &mr->ibmr;
795
796 out_remove_stag:
797 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
798 mr->ibmr.lkey >> 8);
799
800 out_free:
801 kfree(mr);
802
803 return ERR_PTR(ret);
804 }
805
erdma_ib_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)806 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
807 u32 max_num_sg)
808 {
809 struct erdma_mr *mr;
810 struct erdma_dev *dev = to_edev(ibpd->device);
811 int ret;
812 u32 stag;
813
814 if (mr_type != IB_MR_TYPE_MEM_REG)
815 return ERR_PTR(-EOPNOTSUPP);
816
817 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
818 return ERR_PTR(-EINVAL);
819
820 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
821 if (!mr)
822 return ERR_PTR(-ENOMEM);
823
824 ret = erdma_create_stag(dev, &stag);
825 if (ret)
826 goto out_free;
827
828 mr->type = ERDMA_MR_TYPE_FRMR;
829
830 mr->ibmr.lkey = stag;
831 mr->ibmr.rkey = stag;
832 mr->ibmr.pd = ibpd;
833 /* update it in FRMR. */
834 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
835 ERDMA_MR_ACC_RW;
836
837 mr->mem.page_size = PAGE_SIZE; /* update it later. */
838 mr->mem.page_cnt = max_num_sg;
839 mr->mem.mtt_type = ERDMA_MR_INDIRECT_MTT;
840 mr->mem.mtt_buf =
841 alloc_pages_exact(MTT_SIZE(mr->mem.page_cnt), GFP_KERNEL);
842 if (!mr->mem.mtt_buf) {
843 ret = -ENOMEM;
844 goto out_remove_stag;
845 }
846
847 mr->mem.mtt_entry[0] =
848 dma_map_single(&dev->pdev->dev, mr->mem.mtt_buf,
849 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE);
850 if (dma_mapping_error(&dev->pdev->dev, mr->mem.mtt_entry[0])) {
851 ret = -ENOMEM;
852 goto out_free_mtt;
853 }
854
855 ret = regmr_cmd(dev, mr);
856 if (ret)
857 goto out_dma_unmap;
858
859 return &mr->ibmr;
860
861 out_dma_unmap:
862 dma_unmap_single(&dev->pdev->dev, mr->mem.mtt_entry[0],
863 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE);
864 out_free_mtt:
865 free_pages_exact(mr->mem.mtt_buf, MTT_SIZE(mr->mem.page_cnt));
866
867 out_remove_stag:
868 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
869 mr->ibmr.lkey >> 8);
870
871 out_free:
872 kfree(mr);
873
874 return ERR_PTR(ret);
875 }
876
erdma_set_page(struct ib_mr * ibmr,u64 addr)877 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
878 {
879 struct erdma_mr *mr = to_emr(ibmr);
880
881 if (mr->mem.mtt_nents >= mr->mem.page_cnt)
882 return -1;
883
884 *((u64 *)mr->mem.mtt_buf + mr->mem.mtt_nents) = addr;
885 mr->mem.mtt_nents++;
886
887 return 0;
888 }
889
erdma_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)890 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
891 unsigned int *sg_offset)
892 {
893 struct erdma_mr *mr = to_emr(ibmr);
894 int num;
895
896 mr->mem.mtt_nents = 0;
897
898 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
899 erdma_set_page);
900
901 return num;
902 }
903
erdma_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 virt,int access,struct ib_udata * udata)904 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
905 u64 virt, int access, struct ib_udata *udata)
906 {
907 struct erdma_mr *mr = NULL;
908 struct erdma_dev *dev = to_edev(ibpd->device);
909 u32 stag;
910 int ret;
911
912 if (!len || len > dev->attrs.max_mr_size)
913 return ERR_PTR(-EINVAL);
914
915 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
916 if (!mr)
917 return ERR_PTR(-ENOMEM);
918
919 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
920 SZ_2G - SZ_4K, 0);
921 if (ret)
922 goto err_out_free;
923
924 ret = erdma_create_stag(dev, &stag);
925 if (ret)
926 goto err_out_put_mtt;
927
928 mr->ibmr.lkey = mr->ibmr.rkey = stag;
929 mr->ibmr.pd = ibpd;
930 mr->mem.va = virt;
931 mr->mem.len = len;
932 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
933 mr->valid = 1;
934 mr->type = ERDMA_MR_TYPE_NORMAL;
935
936 ret = regmr_cmd(dev, mr);
937 if (ret)
938 goto err_out_mr;
939
940 return &mr->ibmr;
941
942 err_out_mr:
943 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
944 mr->ibmr.lkey >> 8);
945
946 err_out_put_mtt:
947 put_mtt_entries(dev, &mr->mem);
948
949 err_out_free:
950 kfree(mr);
951
952 return ERR_PTR(ret);
953 }
954
erdma_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)955 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
956 {
957 struct erdma_mr *mr;
958 struct erdma_dev *dev = to_edev(ibmr->device);
959 struct erdma_cmdq_dereg_mr_req req;
960 int ret;
961
962 mr = to_emr(ibmr);
963
964 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
965 CMDQ_OPCODE_DEREG_MR);
966
967 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
968 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
969
970 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
971 if (ret)
972 return ret;
973
974 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
975
976 put_mtt_entries(dev, &mr->mem);
977
978 kfree(mr);
979 return 0;
980 }
981
erdma_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)982 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
983 {
984 struct erdma_cq *cq = to_ecq(ibcq);
985 struct erdma_dev *dev = to_edev(ibcq->device);
986 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
987 udata, struct erdma_ucontext, ibucontext);
988 int err;
989 struct erdma_cmdq_destroy_cq_req req;
990
991 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
992 CMDQ_OPCODE_DESTROY_CQ);
993 req.cqn = cq->cqn;
994
995 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
996 if (err)
997 return err;
998
999 if (rdma_is_kernel_res(&cq->ibcq.res)) {
1000 dma_free_coherent(&dev->pdev->dev,
1001 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1002 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1003 } else {
1004 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1005 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1006 }
1007
1008 xa_erase(&dev->cq_xa, cq->cqn);
1009
1010 return 0;
1011 }
1012
erdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)1013 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1014 {
1015 struct erdma_qp *qp = to_eqp(ibqp);
1016 struct erdma_dev *dev = to_edev(ibqp->device);
1017 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1018 udata, struct erdma_ucontext, ibucontext);
1019 struct erdma_qp_attrs qp_attrs;
1020 int err;
1021 struct erdma_cmdq_destroy_qp_req req;
1022
1023 down_write(&qp->state_lock);
1024 qp_attrs.state = ERDMA_QP_STATE_ERROR;
1025 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
1026 up_write(&qp->state_lock);
1027
1028 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1029 CMDQ_OPCODE_DESTROY_QP);
1030 req.qpn = QP_ID(qp);
1031
1032 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1033 if (err)
1034 return err;
1035
1036 erdma_qp_put(qp);
1037 wait_for_completion(&qp->safe_free);
1038
1039 if (rdma_is_kernel_res(&qp->ibqp.res)) {
1040 vfree(qp->kern_qp.swr_tbl);
1041 vfree(qp->kern_qp.rwr_tbl);
1042 dma_free_coherent(
1043 &dev->pdev->dev,
1044 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
1045 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
1046 dma_free_coherent(
1047 &dev->pdev->dev,
1048 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
1049 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
1050 } else {
1051 put_mtt_entries(dev, &qp->user_qp.sq_mtt);
1052 put_mtt_entries(dev, &qp->user_qp.rq_mtt);
1053 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1054 }
1055
1056 if (qp->cep)
1057 erdma_cep_put(qp->cep);
1058 xa_erase(&dev->qp_xa, QP_ID(qp));
1059
1060 return 0;
1061 }
1062
erdma_qp_get_ref(struct ib_qp * ibqp)1063 void erdma_qp_get_ref(struct ib_qp *ibqp)
1064 {
1065 erdma_qp_get(to_eqp(ibqp));
1066 }
1067
erdma_qp_put_ref(struct ib_qp * ibqp)1068 void erdma_qp_put_ref(struct ib_qp *ibqp)
1069 {
1070 erdma_qp_put(to_eqp(ibqp));
1071 }
1072
erdma_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)1073 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1074 {
1075 struct rdma_user_mmap_entry *rdma_entry;
1076 struct erdma_user_mmap_entry *entry;
1077 pgprot_t prot;
1078 int err;
1079
1080 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1081 if (!rdma_entry)
1082 return -EINVAL;
1083
1084 entry = to_emmap(rdma_entry);
1085
1086 switch (entry->mmap_flag) {
1087 case ERDMA_MMAP_IO_NC:
1088 /* map doorbell. */
1089 prot = pgprot_device(vma->vm_page_prot);
1090 break;
1091 default:
1092 return -EINVAL;
1093 }
1094
1095 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1096 prot, rdma_entry);
1097
1098 rdma_user_mmap_entry_put(rdma_entry);
1099 return err;
1100 }
1101
erdma_mmap_free(struct rdma_user_mmap_entry * rdma_entry)1102 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1103 {
1104 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1105
1106 kfree(entry);
1107 }
1108
1109 #define ERDMA_SDB_PAGE 0
1110 #define ERDMA_SDB_ENTRY 1
1111 #define ERDMA_SDB_SHARED 2
1112
alloc_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx)1113 static void alloc_db_resources(struct erdma_dev *dev,
1114 struct erdma_ucontext *ctx)
1115 {
1116 u32 bitmap_idx;
1117 struct erdma_devattr *attrs = &dev->attrs;
1118
1119 if (attrs->disable_dwqe)
1120 goto alloc_normal_db;
1121
1122 /* Try to alloc independent SDB page. */
1123 spin_lock(&dev->db_bitmap_lock);
1124 bitmap_idx = find_first_zero_bit(dev->sdb_page, attrs->dwqe_pages);
1125 if (bitmap_idx != attrs->dwqe_pages) {
1126 set_bit(bitmap_idx, dev->sdb_page);
1127 spin_unlock(&dev->db_bitmap_lock);
1128
1129 ctx->sdb_type = ERDMA_SDB_PAGE;
1130 ctx->sdb_idx = bitmap_idx;
1131 ctx->sdb_page_idx = bitmap_idx;
1132 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET +
1133 (bitmap_idx << PAGE_SHIFT);
1134 ctx->sdb_page_off = 0;
1135
1136 return;
1137 }
1138
1139 bitmap_idx = find_first_zero_bit(dev->sdb_entry, attrs->dwqe_entries);
1140 if (bitmap_idx != attrs->dwqe_entries) {
1141 set_bit(bitmap_idx, dev->sdb_entry);
1142 spin_unlock(&dev->db_bitmap_lock);
1143
1144 ctx->sdb_type = ERDMA_SDB_ENTRY;
1145 ctx->sdb_idx = bitmap_idx;
1146 ctx->sdb_page_idx = attrs->dwqe_pages +
1147 bitmap_idx / ERDMA_DWQE_TYPE1_CNT_PER_PAGE;
1148 ctx->sdb_page_off = bitmap_idx % ERDMA_DWQE_TYPE1_CNT_PER_PAGE;
1149
1150 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET +
1151 (ctx->sdb_page_idx << PAGE_SHIFT);
1152
1153 return;
1154 }
1155
1156 spin_unlock(&dev->db_bitmap_lock);
1157
1158 alloc_normal_db:
1159 ctx->sdb_type = ERDMA_SDB_SHARED;
1160 ctx->sdb_idx = 0;
1161 ctx->sdb_page_idx = ERDMA_SDB_SHARED_PAGE_INDEX;
1162 ctx->sdb_page_off = 0;
1163
1164 ctx->sdb = dev->func_bar_addr + (ctx->sdb_page_idx << PAGE_SHIFT);
1165 }
1166
erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext * uctx)1167 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1168 {
1169 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1170 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1171 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1172 }
1173
erdma_alloc_ucontext(struct ib_ucontext * ibctx,struct ib_udata * udata)1174 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1175 {
1176 struct erdma_ucontext *ctx = to_ectx(ibctx);
1177 struct erdma_dev *dev = to_edev(ibctx->device);
1178 int ret;
1179 struct erdma_uresp_alloc_ctx uresp = {};
1180
1181 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1182 ret = -ENOMEM;
1183 goto err_out;
1184 }
1185
1186 INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1187 mutex_init(&ctx->dbrecords_page_mutex);
1188
1189 alloc_db_resources(dev, ctx);
1190
1191 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1192 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1193
1194 if (udata->outlen < sizeof(uresp)) {
1195 ret = -EINVAL;
1196 goto err_out;
1197 }
1198
1199 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1200 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1201 if (!ctx->sq_db_mmap_entry) {
1202 ret = -ENOMEM;
1203 goto err_out;
1204 }
1205
1206 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1207 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1208 if (!ctx->rq_db_mmap_entry) {
1209 ret = -EINVAL;
1210 goto err_out;
1211 }
1212
1213 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1214 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1215 if (!ctx->cq_db_mmap_entry) {
1216 ret = -EINVAL;
1217 goto err_out;
1218 }
1219
1220 uresp.dev_id = dev->pdev->device;
1221 uresp.sdb_type = ctx->sdb_type;
1222 uresp.sdb_offset = ctx->sdb_page_off;
1223
1224 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1225 if (ret)
1226 goto err_out;
1227
1228 return 0;
1229
1230 err_out:
1231 erdma_uctx_user_mmap_entries_remove(ctx);
1232 atomic_dec(&dev->num_ctx);
1233 return ret;
1234 }
1235
erdma_dealloc_ucontext(struct ib_ucontext * ibctx)1236 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1237 {
1238 struct erdma_ucontext *ctx = to_ectx(ibctx);
1239 struct erdma_dev *dev = to_edev(ibctx->device);
1240
1241 spin_lock(&dev->db_bitmap_lock);
1242 if (ctx->sdb_type == ERDMA_SDB_PAGE)
1243 clear_bit(ctx->sdb_idx, dev->sdb_page);
1244 else if (ctx->sdb_type == ERDMA_SDB_ENTRY)
1245 clear_bit(ctx->sdb_idx, dev->sdb_entry);
1246
1247 erdma_uctx_user_mmap_entries_remove(ctx);
1248
1249 spin_unlock(&dev->db_bitmap_lock);
1250
1251 atomic_dec(&dev->num_ctx);
1252 }
1253
1254 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = {
1255 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE,
1256 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE,
1257 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR,
1258 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS,
1259 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING,
1260 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE,
1261 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR
1262 };
1263
erdma_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1264 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1265 struct ib_udata *udata)
1266 {
1267 struct erdma_qp_attrs new_attrs;
1268 enum erdma_qp_attr_mask erdma_attr_mask = 0;
1269 struct erdma_qp *qp = to_eqp(ibqp);
1270 int ret = 0;
1271
1272 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1273 return -EOPNOTSUPP;
1274
1275 memset(&new_attrs, 0, sizeof(new_attrs));
1276
1277 if (attr_mask & IB_QP_STATE) {
1278 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state];
1279
1280 erdma_attr_mask |= ERDMA_QP_ATTR_STATE;
1281 }
1282
1283 down_write(&qp->state_lock);
1284
1285 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask);
1286
1287 up_write(&qp->state_lock);
1288
1289 return ret;
1290 }
1291
erdma_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1292 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1293 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1294 {
1295 struct erdma_qp *qp;
1296 struct erdma_dev *dev;
1297
1298 if (ibqp && qp_attr && qp_init_attr) {
1299 qp = to_eqp(ibqp);
1300 dev = to_edev(ibqp->device);
1301 } else {
1302 return -EINVAL;
1303 }
1304
1305 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1306 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1307
1308 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1309 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1310 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1311 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1312
1313 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1314 qp_attr->max_rd_atomic = qp->attrs.irq_size;
1315 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1316
1317 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1318 IB_ACCESS_REMOTE_WRITE |
1319 IB_ACCESS_REMOTE_READ;
1320
1321 qp_init_attr->cap = qp_attr->cap;
1322
1323 return 0;
1324 }
1325
erdma_init_user_cq(struct erdma_ucontext * ctx,struct erdma_cq * cq,struct erdma_ureq_create_cq * ureq)1326 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1327 struct erdma_ureq_create_cq *ureq)
1328 {
1329 int ret;
1330 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1331
1332 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mtt, ureq->qbuf_va,
1333 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1334 1);
1335 if (ret)
1336 return ret;
1337
1338 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1339 &cq->user_cq.user_dbr_page,
1340 &cq->user_cq.db_info_dma_addr);
1341 if (ret)
1342 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1343
1344 return ret;
1345 }
1346
erdma_init_kernel_cq(struct erdma_cq * cq)1347 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1348 {
1349 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1350
1351 cq->kern_cq.qbuf =
1352 dma_alloc_coherent(&dev->pdev->dev,
1353 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1354 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1355 if (!cq->kern_cq.qbuf)
1356 return -ENOMEM;
1357
1358 cq->kern_cq.db_record =
1359 (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT));
1360 spin_lock_init(&cq->kern_cq.lock);
1361 /* use default cqdb addr */
1362 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1363
1364 return 0;
1365 }
1366
erdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)1367 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1368 struct ib_udata *udata)
1369 {
1370 struct erdma_cq *cq = to_ecq(ibcq);
1371 struct erdma_dev *dev = to_edev(ibcq->device);
1372 unsigned int depth = attr->cqe;
1373 int ret;
1374 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1375 udata, struct erdma_ucontext, ibucontext);
1376
1377 if (depth > dev->attrs.max_cqe)
1378 return -EINVAL;
1379
1380 depth = roundup_pow_of_two(depth);
1381 cq->ibcq.cqe = depth;
1382 cq->depth = depth;
1383 cq->assoc_eqn = attr->comp_vector + 1;
1384
1385 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1386 XA_LIMIT(1, dev->attrs.max_cq - 1),
1387 &dev->next_alloc_cqn, GFP_KERNEL);
1388 if (ret < 0)
1389 return ret;
1390
1391 if (!rdma_is_kernel_res(&ibcq->res)) {
1392 struct erdma_ureq_create_cq ureq;
1393 struct erdma_uresp_create_cq uresp;
1394
1395 ret = ib_copy_from_udata(&ureq, udata,
1396 min(udata->inlen, sizeof(ureq)));
1397 if (ret)
1398 goto err_out_xa;
1399
1400 ret = erdma_init_user_cq(ctx, cq, &ureq);
1401 if (ret)
1402 goto err_out_xa;
1403
1404 uresp.cq_id = cq->cqn;
1405 uresp.num_cqe = depth;
1406
1407 ret = ib_copy_to_udata(udata, &uresp,
1408 min(sizeof(uresp), udata->outlen));
1409 if (ret)
1410 goto err_free_res;
1411 } else {
1412 ret = erdma_init_kernel_cq(cq);
1413 if (ret)
1414 goto err_out_xa;
1415 }
1416
1417 ret = create_cq_cmd(dev, cq);
1418 if (ret)
1419 goto err_free_res;
1420
1421 return 0;
1422
1423 err_free_res:
1424 if (!rdma_is_kernel_res(&ibcq->res)) {
1425 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1426 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt);
1427 } else {
1428 dma_free_coherent(&dev->pdev->dev,
1429 WARPPED_BUFSIZE(depth << CQE_SHIFT),
1430 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1431 }
1432
1433 err_out_xa:
1434 xa_erase(&dev->cq_xa, cq->cqn);
1435
1436 return ret;
1437 }
1438
erdma_set_mtu(struct erdma_dev * dev,u32 mtu)1439 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
1440 {
1441 struct erdma_cmdq_config_mtu_req req;
1442
1443 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1444 CMDQ_OPCODE_CONF_MTU);
1445 req.mtu = mtu;
1446
1447 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1448 }
1449
erdma_port_event(struct erdma_dev * dev,enum ib_event_type reason)1450 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
1451 {
1452 struct ib_event event;
1453
1454 event.device = &dev->ibdev;
1455 event.element.port_num = 1;
1456 event.event = reason;
1457
1458 ib_dispatch_event(&event);
1459 }
1460