1 /*
2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <net/addrconf.h>
35 #include <linux/etherdevice.h>
36 #include <linux/mlx5/vport.h>
37
38 #include "mlx5_core.h"
39 #include "lib/mlx5.h"
40 #include "fpga/conn.h"
41
42 #define MLX5_FPGA_PKEY 0xFFFF
43 #define MLX5_FPGA_PKEY_INDEX 0 /* RoCE PKEY 0xFFFF is always at index 0 */
44 #define MLX5_FPGA_RECV_SIZE 2048
45 #define MLX5_FPGA_PORT_NUM 1
46 #define MLX5_FPGA_CQ_BUDGET 64
47
mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)48 static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn,
49 struct mlx5_fpga_dma_buf *buf)
50 {
51 struct device *dma_device;
52 int err = 0;
53
54 if (unlikely(!buf->sg[0].data))
55 goto out;
56
57 dma_device = mlx5_core_dma_dev(conn->fdev->mdev);
58 buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data,
59 buf->sg[0].size, buf->dma_dir);
60 err = dma_mapping_error(dma_device, buf->sg[0].dma_addr);
61 if (unlikely(err)) {
62 mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err);
63 err = -ENOMEM;
64 goto out;
65 }
66
67 if (!buf->sg[1].data)
68 goto out;
69
70 buf->sg[1].dma_addr = dma_map_single(dma_device, buf->sg[1].data,
71 buf->sg[1].size, buf->dma_dir);
72 err = dma_mapping_error(dma_device, buf->sg[1].dma_addr);
73 if (unlikely(err)) {
74 mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err);
75 dma_unmap_single(dma_device, buf->sg[0].dma_addr,
76 buf->sg[0].size, buf->dma_dir);
77 err = -ENOMEM;
78 }
79
80 out:
81 return err;
82 }
83
mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)84 static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn,
85 struct mlx5_fpga_dma_buf *buf)
86 {
87 struct device *dma_device;
88
89 dma_device = mlx5_core_dma_dev(conn->fdev->mdev);
90 if (buf->sg[1].data)
91 dma_unmap_single(dma_device, buf->sg[1].dma_addr,
92 buf->sg[1].size, buf->dma_dir);
93
94 if (likely(buf->sg[0].data))
95 dma_unmap_single(dma_device, buf->sg[0].dma_addr,
96 buf->sg[0].size, buf->dma_dir);
97 }
98
mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)99 static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn,
100 struct mlx5_fpga_dma_buf *buf)
101 {
102 struct mlx5_wqe_data_seg *data;
103 unsigned int ix;
104 int err = 0;
105
106 err = mlx5_fpga_conn_map_buf(conn, buf);
107 if (unlikely(err))
108 goto out;
109
110 if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) {
111 mlx5_fpga_conn_unmap_buf(conn, buf);
112 return -EBUSY;
113 }
114
115 ix = conn->qp.rq.pc & (conn->qp.rq.size - 1);
116 data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix);
117 data->byte_count = cpu_to_be32(buf->sg[0].size);
118 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key);
119 data->addr = cpu_to_be64(buf->sg[0].dma_addr);
120
121 conn->qp.rq.pc++;
122 conn->qp.rq.bufs[ix] = buf;
123
124 /* Make sure that descriptors are written before doorbell record. */
125 dma_wmb();
126 *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff);
127 out:
128 return err;
129 }
130
mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn * conn,void * wqe)131 static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe)
132 {
133 /* ensure wqe is visible to device before updating doorbell record */
134 dma_wmb();
135 *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc);
136 /* Make sure that doorbell record is visible before ringing */
137 wmb();
138 mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET);
139 }
140
mlx5_fpga_conn_post_send(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)141 static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn,
142 struct mlx5_fpga_dma_buf *buf)
143 {
144 struct mlx5_wqe_ctrl_seg *ctrl;
145 struct mlx5_wqe_data_seg *data;
146 unsigned int ix, sgi;
147 int size = 1;
148
149 ix = conn->qp.sq.pc & (conn->qp.sq.size - 1);
150
151 ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix);
152 data = (void *)(ctrl + 1);
153
154 for (sgi = 0; sgi < ARRAY_SIZE(buf->sg); sgi++) {
155 if (!buf->sg[sgi].data)
156 break;
157 data->byte_count = cpu_to_be32(buf->sg[sgi].size);
158 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key);
159 data->addr = cpu_to_be64(buf->sg[sgi].dma_addr);
160 data++;
161 size++;
162 }
163
164 ctrl->imm = 0;
165 ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
166 ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) |
167 MLX5_OPCODE_SEND);
168 ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.qpn << 8));
169
170 conn->qp.sq.pc++;
171 conn->qp.sq.bufs[ix] = buf;
172 mlx5_fpga_conn_notify_hw(conn, ctrl);
173 }
174
mlx5_fpga_conn_send(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)175 int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn,
176 struct mlx5_fpga_dma_buf *buf)
177 {
178 unsigned long flags;
179 int err;
180
181 if (!conn->qp.active)
182 return -ENOTCONN;
183
184 buf->dma_dir = DMA_TO_DEVICE;
185 err = mlx5_fpga_conn_map_buf(conn, buf);
186 if (err)
187 return err;
188
189 spin_lock_irqsave(&conn->qp.sq.lock, flags);
190
191 if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) {
192 list_add_tail(&buf->list, &conn->qp.sq.backlog);
193 goto out_unlock;
194 }
195
196 mlx5_fpga_conn_post_send(conn, buf);
197
198 out_unlock:
199 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
200 return err;
201 }
202
mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn * conn)203 static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn)
204 {
205 struct mlx5_fpga_dma_buf *buf;
206 int err;
207
208 buf = kzalloc(sizeof(*buf) + MLX5_FPGA_RECV_SIZE, 0);
209 if (!buf)
210 return -ENOMEM;
211
212 buf->sg[0].data = (void *)(buf + 1);
213 buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
214 buf->dma_dir = DMA_FROM_DEVICE;
215
216 err = mlx5_fpga_conn_post_recv(conn, buf);
217 if (err)
218 kfree(buf);
219
220 return err;
221 }
222
mlx5_fpga_conn_create_mkey(struct mlx5_core_dev * mdev,u32 pdn,struct mlx5_core_mkey * mkey)223 static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
224 struct mlx5_core_mkey *mkey)
225 {
226 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
227 void *mkc;
228 u32 *in;
229 int err;
230
231 in = kvzalloc(inlen, GFP_KERNEL);
232 if (!in)
233 return -ENOMEM;
234
235 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
236 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
237 MLX5_SET(mkc, mkc, lw, 1);
238 MLX5_SET(mkc, mkc, lr, 1);
239
240 MLX5_SET(mkc, mkc, pd, pdn);
241 MLX5_SET(mkc, mkc, length64, 1);
242 MLX5_SET(mkc, mkc, qpn, 0xffffff);
243
244 err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
245
246 kvfree(in);
247 return err;
248 }
249
mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn * conn,struct mlx5_cqe64 * cqe,u8 status)250 static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn,
251 struct mlx5_cqe64 *cqe, u8 status)
252 {
253 struct mlx5_fpga_dma_buf *buf;
254 int ix, err;
255
256 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1);
257 buf = conn->qp.rq.bufs[ix];
258 conn->qp.rq.bufs[ix] = NULL;
259 conn->qp.rq.cc++;
260
261 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
262 mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
263 buf, conn->fpga_qpn, status);
264 else
265 mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
266 buf, conn->fpga_qpn, status);
267
268 mlx5_fpga_conn_unmap_buf(conn, buf);
269
270 if (unlikely(status || !conn->qp.active)) {
271 conn->qp.active = false;
272 kfree(buf);
273 return;
274 }
275
276 buf->sg[0].size = be32_to_cpu(cqe->byte_cnt);
277 mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n",
278 buf->sg[0].size);
279 conn->recv_cb(conn->cb_arg, buf);
280
281 buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
282 err = mlx5_fpga_conn_post_recv(conn, buf);
283 if (unlikely(err)) {
284 mlx5_fpga_warn(conn->fdev,
285 "Failed to re-post recv buf: %d\n", err);
286 kfree(buf);
287 }
288 }
289
mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn * conn,struct mlx5_cqe64 * cqe,u8 status)290 static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn,
291 struct mlx5_cqe64 *cqe, u8 status)
292 {
293 struct mlx5_fpga_dma_buf *buf, *nextbuf;
294 unsigned long flags;
295 int ix;
296
297 spin_lock_irqsave(&conn->qp.sq.lock, flags);
298
299 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1);
300 buf = conn->qp.sq.bufs[ix];
301 conn->qp.sq.bufs[ix] = NULL;
302 conn->qp.sq.cc++;
303
304 /* Handle backlog still under the spinlock to ensure message post order */
305 if (unlikely(!list_empty(&conn->qp.sq.backlog))) {
306 if (likely(conn->qp.active)) {
307 nextbuf = list_first_entry(&conn->qp.sq.backlog,
308 struct mlx5_fpga_dma_buf, list);
309 list_del(&nextbuf->list);
310 mlx5_fpga_conn_post_send(conn, nextbuf);
311 }
312 }
313
314 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
315
316 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
317 mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
318 buf, conn->fpga_qpn, status);
319 else
320 mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
321 buf, conn->fpga_qpn, status);
322
323 mlx5_fpga_conn_unmap_buf(conn, buf);
324
325 if (likely(buf->complete))
326 buf->complete(conn, conn->fdev, buf, status);
327
328 if (unlikely(status))
329 conn->qp.active = false;
330 }
331
mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn * conn,struct mlx5_cqe64 * cqe)332 static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn,
333 struct mlx5_cqe64 *cqe)
334 {
335 u8 opcode, status = 0;
336
337 opcode = get_cqe_opcode(cqe);
338
339 switch (opcode) {
340 case MLX5_CQE_REQ_ERR:
341 status = ((struct mlx5_err_cqe *)cqe)->syndrome;
342 fallthrough;
343 case MLX5_CQE_REQ:
344 mlx5_fpga_conn_sq_cqe(conn, cqe, status);
345 break;
346
347 case MLX5_CQE_RESP_ERR:
348 status = ((struct mlx5_err_cqe *)cqe)->syndrome;
349 fallthrough;
350 case MLX5_CQE_RESP_SEND:
351 mlx5_fpga_conn_rq_cqe(conn, cqe, status);
352 break;
353 default:
354 mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n",
355 opcode);
356 }
357 }
358
mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn * conn)359 static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn)
360 {
361 mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT,
362 conn->fdev->conn_res.uar->map, conn->cq.wq.cc);
363 }
364
mlx5_fpga_conn_cqes(struct mlx5_fpga_conn * conn,unsigned int budget)365 static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn,
366 unsigned int budget)
367 {
368 struct mlx5_cqe64 *cqe;
369
370 while (budget) {
371 cqe = mlx5_cqwq_get_cqe(&conn->cq.wq);
372 if (!cqe)
373 break;
374
375 budget--;
376 mlx5_cqwq_pop(&conn->cq.wq);
377 mlx5_fpga_conn_handle_cqe(conn, cqe);
378 mlx5_cqwq_update_db_record(&conn->cq.wq);
379 }
380 if (!budget) {
381 tasklet_schedule(&conn->cq.tasklet);
382 return;
383 }
384
385 mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc);
386 /* ensure cq space is freed before enabling more cqes */
387 wmb();
388 mlx5_fpga_conn_arm_cq(conn);
389 }
390
mlx5_fpga_conn_cq_tasklet(struct tasklet_struct * t)391 static void mlx5_fpga_conn_cq_tasklet(struct tasklet_struct *t)
392 {
393 struct mlx5_fpga_conn *conn = from_tasklet(conn, t, cq.tasklet);
394
395 if (unlikely(!conn->qp.active))
396 return;
397 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
398 }
399
mlx5_fpga_conn_cq_complete(struct mlx5_core_cq * mcq,struct mlx5_eqe * eqe)400 static void mlx5_fpga_conn_cq_complete(struct mlx5_core_cq *mcq,
401 struct mlx5_eqe *eqe)
402 {
403 struct mlx5_fpga_conn *conn;
404
405 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq);
406 if (unlikely(!conn->qp.active))
407 return;
408 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
409 }
410
mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn * conn,int cq_size)411 static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
412 {
413 struct mlx5_fpga_device *fdev = conn->fdev;
414 struct mlx5_core_dev *mdev = fdev->mdev;
415 u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {0};
416 u32 out[MLX5_ST_SZ_DW(create_cq_out)];
417 struct mlx5_wq_param wqp;
418 struct mlx5_cqe64 *cqe;
419 int inlen, err, eqn;
420 unsigned int irqn;
421 void *cqc, *in;
422 __be64 *pas;
423 u32 i;
424
425 cq_size = roundup_pow_of_two(cq_size);
426 MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(cq_size));
427
428 wqp.buf_numa_node = mdev->priv.numa_node;
429 wqp.db_numa_node = mdev->priv.numa_node;
430
431 err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq,
432 &conn->cq.wq_ctrl);
433 if (err)
434 return err;
435
436 for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) {
437 cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i);
438 cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK;
439 }
440
441 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
442 sizeof(u64) * conn->cq.wq_ctrl.buf.npages;
443 in = kvzalloc(inlen, GFP_KERNEL);
444 if (!in) {
445 err = -ENOMEM;
446 goto err_cqwq;
447 }
448
449 err = mlx5_vector2eqn(mdev, smp_processor_id(), &eqn, &irqn);
450 if (err) {
451 kvfree(in);
452 goto err_cqwq;
453 }
454
455 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
456 MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size));
457 MLX5_SET(cqc, cqc, c_eqn, eqn);
458 MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index);
459 MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.buf.page_shift -
460 MLX5_ADAPTER_PAGE_SHIFT);
461 MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma);
462
463 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
464 mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.buf, pas);
465
466 err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen, out, sizeof(out));
467 kvfree(in);
468
469 if (err)
470 goto err_cqwq;
471
472 conn->cq.mcq.cqe_sz = 64;
473 conn->cq.mcq.set_ci_db = conn->cq.wq_ctrl.db.db;
474 conn->cq.mcq.arm_db = conn->cq.wq_ctrl.db.db + 1;
475 *conn->cq.mcq.set_ci_db = 0;
476 *conn->cq.mcq.arm_db = 0;
477 conn->cq.mcq.vector = 0;
478 conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete;
479 conn->cq.mcq.irqn = irqn;
480 conn->cq.mcq.uar = fdev->conn_res.uar;
481 tasklet_setup(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet);
482
483 mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn);
484
485 goto out;
486
487 err_cqwq:
488 mlx5_wq_destroy(&conn->cq.wq_ctrl);
489 out:
490 return err;
491 }
492
mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn * conn)493 static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn)
494 {
495 tasklet_disable(&conn->cq.tasklet);
496 tasklet_kill(&conn->cq.tasklet);
497 mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq);
498 mlx5_wq_destroy(&conn->cq.wq_ctrl);
499 }
500
mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn * conn,void * qpc)501 static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc)
502 {
503 struct mlx5_fpga_device *fdev = conn->fdev;
504 struct mlx5_core_dev *mdev = fdev->mdev;
505 struct mlx5_wq_param wqp;
506
507 wqp.buf_numa_node = mdev->priv.numa_node;
508 wqp.db_numa_node = mdev->priv.numa_node;
509
510 return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq,
511 &conn->qp.wq_ctrl);
512 }
513
mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn * conn,unsigned int tx_size,unsigned int rx_size)514 static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn,
515 unsigned int tx_size, unsigned int rx_size)
516 {
517 struct mlx5_fpga_device *fdev = conn->fdev;
518 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
519 struct mlx5_core_dev *mdev = fdev->mdev;
520 u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {};
521 void *in = NULL, *qpc;
522 int err, inlen;
523
524 conn->qp.rq.pc = 0;
525 conn->qp.rq.cc = 0;
526 conn->qp.rq.size = roundup_pow_of_two(rx_size);
527 conn->qp.sq.pc = 0;
528 conn->qp.sq.cc = 0;
529 conn->qp.sq.size = roundup_pow_of_two(tx_size);
530
531 MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
532 MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size));
533 MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size));
534 err = mlx5_fpga_conn_create_wq(conn, temp_qpc);
535 if (err)
536 goto out;
537
538 conn->qp.rq.bufs = kvcalloc(conn->qp.rq.size,
539 sizeof(conn->qp.rq.bufs[0]),
540 GFP_KERNEL);
541 if (!conn->qp.rq.bufs) {
542 err = -ENOMEM;
543 goto err_wq;
544 }
545
546 conn->qp.sq.bufs = kvcalloc(conn->qp.sq.size,
547 sizeof(conn->qp.sq.bufs[0]),
548 GFP_KERNEL);
549 if (!conn->qp.sq.bufs) {
550 err = -ENOMEM;
551 goto err_rq_bufs;
552 }
553
554 inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
555 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) *
556 conn->qp.wq_ctrl.buf.npages;
557 in = kvzalloc(inlen, GFP_KERNEL);
558 if (!in) {
559 err = -ENOMEM;
560 goto err_sq_bufs;
561 }
562
563 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
564 MLX5_SET(qpc, qpc, uar_page, fdev->conn_res.uar->index);
565 MLX5_SET(qpc, qpc, log_page_size,
566 conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
567 MLX5_SET(qpc, qpc, fre, 1);
568 MLX5_SET(qpc, qpc, rlky, 1);
569 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
570 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
571 MLX5_SET(qpc, qpc, pd, fdev->conn_res.pdn);
572 MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
573 MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size));
574 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
575 MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size));
576 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
577 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
578 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
579 if (MLX5_CAP_GEN(mdev, cqe_version) == 1)
580 MLX5_SET(qpc, qpc, user_index, 0xFFFFFF);
581
582 mlx5_fill_page_frag_array(&conn->qp.wq_ctrl.buf,
583 (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas));
584
585 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
586 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
587 if (err)
588 goto err_sq_bufs;
589
590 conn->qp.qpn = MLX5_GET(create_qp_out, out, qpn);
591 mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.qpn);
592
593 goto out;
594
595 err_sq_bufs:
596 kvfree(conn->qp.sq.bufs);
597 err_rq_bufs:
598 kvfree(conn->qp.rq.bufs);
599 err_wq:
600 mlx5_wq_destroy(&conn->qp.wq_ctrl);
601 out:
602 kvfree(in);
603 return err;
604 }
605
mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn * conn)606 static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn)
607 {
608 int ix;
609
610 for (ix = 0; ix < conn->qp.rq.size; ix++) {
611 if (!conn->qp.rq.bufs[ix])
612 continue;
613 mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]);
614 kfree(conn->qp.rq.bufs[ix]);
615 conn->qp.rq.bufs[ix] = NULL;
616 }
617 }
618
mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn * conn)619 static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn)
620 {
621 struct mlx5_fpga_dma_buf *buf, *temp;
622 int ix;
623
624 for (ix = 0; ix < conn->qp.sq.size; ix++) {
625 buf = conn->qp.sq.bufs[ix];
626 if (!buf)
627 continue;
628 conn->qp.sq.bufs[ix] = NULL;
629 mlx5_fpga_conn_unmap_buf(conn, buf);
630 if (!buf->complete)
631 continue;
632 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
633 }
634 list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) {
635 mlx5_fpga_conn_unmap_buf(conn, buf);
636 if (!buf->complete)
637 continue;
638 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
639 }
640 }
641
mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn * conn)642 static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn)
643 {
644 struct mlx5_core_dev *dev = conn->fdev->mdev;
645 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
646
647 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
648 MLX5_SET(destroy_qp_in, in, qpn, conn->qp.qpn);
649 mlx5_cmd_exec_in(dev, destroy_qp, in);
650
651 mlx5_fpga_conn_free_recv_bufs(conn);
652 mlx5_fpga_conn_flush_send_bufs(conn);
653 kvfree(conn->qp.sq.bufs);
654 kvfree(conn->qp.rq.bufs);
655 mlx5_wq_destroy(&conn->qp.wq_ctrl);
656 }
657
mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn * conn)658 static int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn)
659 {
660 struct mlx5_core_dev *mdev = conn->fdev->mdev;
661 u32 in[MLX5_ST_SZ_DW(qp_2rst_in)] = {};
662
663 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.qpn);
664
665 MLX5_SET(qp_2rst_in, in, opcode, MLX5_CMD_OP_2RST_QP);
666 MLX5_SET(qp_2rst_in, in, qpn, conn->qp.qpn);
667
668 return mlx5_cmd_exec_in(mdev, qp_2rst, in);
669 }
670
mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn * conn)671 static int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn)
672 {
673 u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {};
674 struct mlx5_fpga_device *fdev = conn->fdev;
675 struct mlx5_core_dev *mdev = fdev->mdev;
676 u32 *qpc;
677
678 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.qpn);
679
680 qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc);
681
682 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
683 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
684 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
685 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
686 MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn);
687 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
688 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
689 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
690
691 MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP);
692 MLX5_SET(rst2init_qp_in, in, qpn, conn->qp.qpn);
693
694 return mlx5_cmd_exec_in(mdev, rst2init_qp, in);
695 }
696
mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn * conn)697 static int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn)
698 {
699 u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {};
700 struct mlx5_fpga_device *fdev = conn->fdev;
701 struct mlx5_core_dev *mdev = fdev->mdev;
702 u32 *qpc;
703
704 mlx5_fpga_dbg(conn->fdev, "QP RTR\n");
705
706 qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc);
707
708 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_1K_BYTES);
709 MLX5_SET(qpc, qpc, log_msg_max, (u8)MLX5_CAP_GEN(mdev, log_max_msg));
710 MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn);
711 MLX5_SET(qpc, qpc, next_rcv_psn,
712 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn));
713 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
714 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
715 ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32),
716 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32));
717 MLX5_SET(qpc, qpc, primary_address_path.udp_sport,
718 MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port));
719 MLX5_SET(qpc, qpc, primary_address_path.src_addr_index,
720 conn->qp.sgid_index);
721 MLX5_SET(qpc, qpc, primary_address_path.hop_limit, 0);
722 memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip),
723 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip),
724 MLX5_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip));
725
726 MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP);
727 MLX5_SET(init2rtr_qp_in, in, qpn, conn->qp.qpn);
728
729 return mlx5_cmd_exec_in(mdev, init2rtr_qp, in);
730 }
731
mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn * conn)732 static int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn)
733 {
734 struct mlx5_fpga_device *fdev = conn->fdev;
735 u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {};
736 struct mlx5_core_dev *mdev = fdev->mdev;
737 u32 *qpc;
738
739 mlx5_fpga_dbg(conn->fdev, "QP RTS\n");
740
741 qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc);
742
743 MLX5_SET(qpc, qpc, log_ack_req_freq, 8);
744 MLX5_SET(qpc, qpc, min_rnr_nak, 0x12);
745 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x12); /* ~1.07s */
746 MLX5_SET(qpc, qpc, next_send_psn,
747 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn));
748 MLX5_SET(qpc, qpc, retry_count, 7);
749 MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */
750
751 MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP);
752 MLX5_SET(rtr2rts_qp_in, in, qpn, conn->qp.qpn);
753 MLX5_SET(rtr2rts_qp_in, in, opt_param_mask, MLX5_QP_OPTPAR_RNR_TIMEOUT);
754
755 return mlx5_cmd_exec_in(mdev, rtr2rts_qp, in);
756 }
757
mlx5_fpga_conn_connect(struct mlx5_fpga_conn * conn)758 static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn)
759 {
760 struct mlx5_fpga_device *fdev = conn->fdev;
761 int err;
762
763 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE);
764 err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
765 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc);
766 if (err) {
767 mlx5_fpga_err(fdev, "Failed to activate FPGA RC QP: %d\n", err);
768 goto out;
769 }
770
771 err = mlx5_fpga_conn_reset_qp(conn);
772 if (err) {
773 mlx5_fpga_err(fdev, "Failed to change QP state to reset\n");
774 goto err_fpga_qp;
775 }
776
777 err = mlx5_fpga_conn_init_qp(conn);
778 if (err) {
779 mlx5_fpga_err(fdev, "Failed to modify QP from RESET to INIT\n");
780 goto err_fpga_qp;
781 }
782 conn->qp.active = true;
783
784 while (!mlx5_fpga_conn_post_recv_buf(conn))
785 ;
786
787 err = mlx5_fpga_conn_rtr_qp(conn);
788 if (err) {
789 mlx5_fpga_err(fdev, "Failed to change QP state from INIT to RTR\n");
790 goto err_recv_bufs;
791 }
792
793 err = mlx5_fpga_conn_rts_qp(conn);
794 if (err) {
795 mlx5_fpga_err(fdev, "Failed to change QP state from RTR to RTS\n");
796 goto err_recv_bufs;
797 }
798 goto out;
799
800 err_recv_bufs:
801 mlx5_fpga_conn_free_recv_bufs(conn);
802 err_fpga_qp:
803 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
804 if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
805 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc))
806 mlx5_fpga_err(fdev, "Failed to revert FPGA QP to INIT\n");
807 out:
808 return err;
809 }
810
mlx5_fpga_conn_create(struct mlx5_fpga_device * fdev,struct mlx5_fpga_conn_attr * attr,enum mlx5_ifc_fpga_qp_type qp_type)811 struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev,
812 struct mlx5_fpga_conn_attr *attr,
813 enum mlx5_ifc_fpga_qp_type qp_type)
814 {
815 struct mlx5_fpga_conn *ret, *conn;
816 u8 *remote_mac, *remote_ip;
817 int err;
818
819 if (!attr->recv_cb)
820 return ERR_PTR(-EINVAL);
821
822 conn = kzalloc(sizeof(*conn), GFP_KERNEL);
823 if (!conn)
824 return ERR_PTR(-ENOMEM);
825
826 conn->fdev = fdev;
827 INIT_LIST_HEAD(&conn->qp.sq.backlog);
828
829 spin_lock_init(&conn->qp.sq.lock);
830
831 conn->recv_cb = attr->recv_cb;
832 conn->cb_arg = attr->cb_arg;
833
834 remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32);
835 err = mlx5_query_mac_address(fdev->mdev, remote_mac);
836 if (err) {
837 mlx5_fpga_err(fdev, "Failed to query local MAC: %d\n", err);
838 ret = ERR_PTR(err);
839 goto err;
840 }
841
842 /* Build Modified EUI-64 IPv6 address from the MAC address */
843 remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip);
844 remote_ip[0] = 0xfe;
845 remote_ip[1] = 0x80;
846 addrconf_addr_eui48(&remote_ip[8], remote_mac);
847
848 err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index);
849 if (err) {
850 mlx5_fpga_err(fdev, "Failed to allocate SGID: %d\n", err);
851 ret = ERR_PTR(err);
852 goto err;
853 }
854
855 err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index,
856 MLX5_ROCE_VERSION_2,
857 MLX5_ROCE_L3_TYPE_IPV6,
858 remote_ip, remote_mac, true, 0,
859 MLX5_FPGA_PORT_NUM);
860 if (err) {
861 mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err);
862 ret = ERR_PTR(err);
863 goto err_rsvd_gid;
864 }
865 mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index);
866
867 /* Allow for one cqe per rx/tx wqe, plus one cqe for the next wqe,
868 * created during processing of the cqe
869 */
870 err = mlx5_fpga_conn_create_cq(conn,
871 (attr->tx_size + attr->rx_size) * 2);
872 if (err) {
873 mlx5_fpga_err(fdev, "Failed to create CQ: %d\n", err);
874 ret = ERR_PTR(err);
875 goto err_gid;
876 }
877
878 mlx5_fpga_conn_arm_cq(conn);
879
880 err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size);
881 if (err) {
882 mlx5_fpga_err(fdev, "Failed to create QP: %d\n", err);
883 ret = ERR_PTR(err);
884 goto err_cq;
885 }
886
887 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
888 MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type);
889 MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC);
890 MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q);
891 MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0);
892 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1);
893 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0);
894 MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY);
895 MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.qpn);
896 MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7);
897 MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7);
898
899 err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc,
900 &conn->fpga_qpn);
901 if (err) {
902 mlx5_fpga_err(fdev, "Failed to create FPGA RC QP: %d\n", err);
903 ret = ERR_PTR(err);
904 goto err_qp;
905 }
906
907 err = mlx5_fpga_conn_connect(conn);
908 if (err) {
909 ret = ERR_PTR(err);
910 goto err_conn;
911 }
912
913 mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn);
914 ret = conn;
915 goto out;
916
917 err_conn:
918 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
919 err_qp:
920 mlx5_fpga_conn_destroy_qp(conn);
921 err_cq:
922 mlx5_fpga_conn_destroy_cq(conn);
923 err_gid:
924 mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL,
925 NULL, false, 0, MLX5_FPGA_PORT_NUM);
926 err_rsvd_gid:
927 mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index);
928 err:
929 kfree(conn);
930 out:
931 return ret;
932 }
933
mlx5_fpga_conn_destroy(struct mlx5_fpga_conn * conn)934 void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn)
935 {
936 conn->qp.active = false;
937 tasklet_disable(&conn->cq.tasklet);
938 synchronize_irq(conn->cq.mcq.irqn);
939
940 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
941 mlx5_fpga_conn_destroy_qp(conn);
942 mlx5_fpga_conn_destroy_cq(conn);
943
944 mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0,
945 NULL, NULL, false, 0, MLX5_FPGA_PORT_NUM);
946 mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index);
947 kfree(conn);
948 }
949
mlx5_fpga_conn_device_init(struct mlx5_fpga_device * fdev)950 int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev)
951 {
952 int err;
953
954 err = mlx5_nic_vport_enable_roce(fdev->mdev);
955 if (err) {
956 mlx5_fpga_err(fdev, "Failed to enable RoCE: %d\n", err);
957 goto out;
958 }
959
960 fdev->conn_res.uar = mlx5_get_uars_page(fdev->mdev);
961 if (IS_ERR(fdev->conn_res.uar)) {
962 err = PTR_ERR(fdev->conn_res.uar);
963 mlx5_fpga_err(fdev, "get_uars_page failed, %d\n", err);
964 goto err_roce;
965 }
966 mlx5_fpga_dbg(fdev, "Allocated UAR index %u\n",
967 fdev->conn_res.uar->index);
968
969 err = mlx5_core_alloc_pd(fdev->mdev, &fdev->conn_res.pdn);
970 if (err) {
971 mlx5_fpga_err(fdev, "alloc pd failed, %d\n", err);
972 goto err_uar;
973 }
974 mlx5_fpga_dbg(fdev, "Allocated PD %u\n", fdev->conn_res.pdn);
975
976 err = mlx5_fpga_conn_create_mkey(fdev->mdev, fdev->conn_res.pdn,
977 &fdev->conn_res.mkey);
978 if (err) {
979 mlx5_fpga_err(fdev, "create mkey failed, %d\n", err);
980 goto err_dealloc_pd;
981 }
982 mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey.key);
983
984 return 0;
985
986 err_dealloc_pd:
987 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
988 err_uar:
989 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
990 err_roce:
991 mlx5_nic_vport_disable_roce(fdev->mdev);
992 out:
993 return err;
994 }
995
mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device * fdev)996 void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev)
997 {
998 mlx5_core_destroy_mkey(fdev->mdev, &fdev->conn_res.mkey);
999 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
1000 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
1001 mlx5_nic_vport_disable_roce(fdev->mdev);
1002 }
1003
