1 /*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/pci.h>
35 #include <linux/platform_device.h>
36 #include <rdma/ib_addr.h>
37 #include <rdma/ib_umem.h>
38 #include <rdma/uverbs_ioctl.h>
39 #include "hns_roce_common.h"
40 #include "hns_roce_device.h"
41 #include "hns_roce_hem.h"
42 #include <rdma/hns-abi.h>
43
44 #define SQP_NUM (2 * HNS_ROCE_MAX_PORTS)
45
hns_roce_qp_event(struct hns_roce_dev * hr_dev,u32 qpn,int event_type)46 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
47 {
48 struct device *dev = hr_dev->dev;
49 struct hns_roce_qp *qp;
50
51 xa_lock(&hr_dev->qp_table_xa);
52 qp = __hns_roce_qp_lookup(hr_dev, qpn);
53 if (qp)
54 atomic_inc(&qp->refcount);
55 xa_unlock(&hr_dev->qp_table_xa);
56
57 if (!qp) {
58 dev_warn(dev, "Async event for bogus QP %08x\n", qpn);
59 return;
60 }
61
62 qp->event(qp, (enum hns_roce_event)event_type);
63
64 if (atomic_dec_and_test(&qp->refcount))
65 complete(&qp->free);
66 }
67
hns_roce_ib_qp_event(struct hns_roce_qp * hr_qp,enum hns_roce_event type)68 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
69 enum hns_roce_event type)
70 {
71 struct ib_event event;
72 struct ib_qp *ibqp = &hr_qp->ibqp;
73
74 if (ibqp->event_handler) {
75 event.device = ibqp->device;
76 event.element.qp = ibqp;
77 switch (type) {
78 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
79 event.event = IB_EVENT_PATH_MIG;
80 break;
81 case HNS_ROCE_EVENT_TYPE_COMM_EST:
82 event.event = IB_EVENT_COMM_EST;
83 break;
84 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
85 event.event = IB_EVENT_SQ_DRAINED;
86 break;
87 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
88 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
89 break;
90 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
91 event.event = IB_EVENT_QP_FATAL;
92 break;
93 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
94 event.event = IB_EVENT_PATH_MIG_ERR;
95 break;
96 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
97 event.event = IB_EVENT_QP_REQ_ERR;
98 break;
99 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
100 event.event = IB_EVENT_QP_ACCESS_ERR;
101 break;
102 default:
103 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n",
104 type, hr_qp->qpn);
105 return;
106 }
107 ibqp->event_handler(&event, ibqp->qp_context);
108 }
109 }
110
hns_roce_reserve_range_qp(struct hns_roce_dev * hr_dev,int cnt,int align,unsigned long * base)111 static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt,
112 int align, unsigned long *base)
113 {
114 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
115
116 return hns_roce_bitmap_alloc_range(&qp_table->bitmap, cnt, align,
117 base) ?
118 -ENOMEM :
119 0;
120 }
121
to_hns_roce_state(enum ib_qp_state state)122 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state)
123 {
124 switch (state) {
125 case IB_QPS_RESET:
126 return HNS_ROCE_QP_STATE_RST;
127 case IB_QPS_INIT:
128 return HNS_ROCE_QP_STATE_INIT;
129 case IB_QPS_RTR:
130 return HNS_ROCE_QP_STATE_RTR;
131 case IB_QPS_RTS:
132 return HNS_ROCE_QP_STATE_RTS;
133 case IB_QPS_SQD:
134 return HNS_ROCE_QP_STATE_SQD;
135 case IB_QPS_ERR:
136 return HNS_ROCE_QP_STATE_ERR;
137 default:
138 return HNS_ROCE_QP_NUM_STATE;
139 }
140 }
141
hns_roce_gsi_qp_alloc(struct hns_roce_dev * hr_dev,unsigned long qpn,struct hns_roce_qp * hr_qp)142 static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
143 struct hns_roce_qp *hr_qp)
144 {
145 struct xarray *xa = &hr_dev->qp_table_xa;
146 int ret;
147
148 if (!qpn)
149 return -EINVAL;
150
151 hr_qp->qpn = qpn;
152 atomic_set(&hr_qp->refcount, 1);
153 init_completion(&hr_qp->free);
154
155 ret = xa_err(xa_store_irq(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1),
156 hr_qp, GFP_KERNEL));
157 if (ret)
158 dev_err(hr_dev->dev, "QPC xa_store failed\n");
159
160 return ret;
161 }
162
hns_roce_qp_alloc(struct hns_roce_dev * hr_dev,unsigned long qpn,struct hns_roce_qp * hr_qp)163 static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
164 struct hns_roce_qp *hr_qp)
165 {
166 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
167 struct device *dev = hr_dev->dev;
168 int ret;
169
170 if (!qpn)
171 return -EINVAL;
172
173 hr_qp->qpn = qpn;
174
175 /* Alloc memory for QPC */
176 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
177 if (ret) {
178 dev_err(dev, "QPC table get failed\n");
179 goto err_out;
180 }
181
182 /* Alloc memory for IRRL */
183 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
184 if (ret) {
185 dev_err(dev, "IRRL table get failed\n");
186 goto err_put_qp;
187 }
188
189 if (hr_dev->caps.trrl_entry_sz) {
190 /* Alloc memory for TRRL */
191 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table,
192 hr_qp->qpn);
193 if (ret) {
194 dev_err(dev, "TRRL table get failed\n");
195 goto err_put_irrl;
196 }
197 }
198
199 if (hr_dev->caps.sccc_entry_sz) {
200 /* Alloc memory for SCC CTX */
201 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
202 hr_qp->qpn);
203 if (ret) {
204 dev_err(dev, "SCC CTX table get failed\n");
205 goto err_put_trrl;
206 }
207 }
208
209 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);
210 if (ret)
211 goto err_put_sccc;
212
213 return 0;
214
215 err_put_sccc:
216 if (hr_dev->caps.sccc_entry_sz)
217 hns_roce_table_put(hr_dev, &qp_table->sccc_table,
218 hr_qp->qpn);
219
220 err_put_trrl:
221 if (hr_dev->caps.trrl_entry_sz)
222 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
223
224 err_put_irrl:
225 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
226
227 err_put_qp:
228 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
229
230 err_out:
231 return ret;
232 }
233
hns_roce_qp_remove(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp)234 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
235 {
236 struct xarray *xa = &hr_dev->qp_table_xa;
237 unsigned long flags;
238
239 xa_lock_irqsave(xa, flags);
240 __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1));
241 xa_unlock_irqrestore(xa, flags);
242 }
243
hns_roce_qp_free(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp)244 void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
245 {
246 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
247
248 if (atomic_dec_and_test(&hr_qp->refcount))
249 complete(&hr_qp->free);
250 wait_for_completion(&hr_qp->free);
251
252 if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) {
253 if (hr_dev->caps.trrl_entry_sz)
254 hns_roce_table_put(hr_dev, &qp_table->trrl_table,
255 hr_qp->qpn);
256 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
257 }
258 }
259
hns_roce_release_range_qp(struct hns_roce_dev * hr_dev,int base_qpn,int cnt)260 void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
261 int cnt)
262 {
263 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
264
265 if (base_qpn < hr_dev->caps.reserved_qps)
266 return;
267
268 hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR);
269 }
270
hns_roce_set_rq_size(struct hns_roce_dev * hr_dev,struct ib_qp_cap * cap,bool is_user,int has_rq,struct hns_roce_qp * hr_qp)271 static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,
272 struct ib_qp_cap *cap, bool is_user, int has_rq,
273 struct hns_roce_qp *hr_qp)
274 {
275 struct device *dev = hr_dev->dev;
276 u32 max_cnt;
277
278 /* Check the validity of QP support capacity */
279 if (cap->max_recv_wr > hr_dev->caps.max_wqes ||
280 cap->max_recv_sge > hr_dev->caps.max_rq_sg) {
281 dev_err(dev, "RQ WR or sge error!max_recv_wr=%d max_recv_sge=%d\n",
282 cap->max_recv_wr, cap->max_recv_sge);
283 return -EINVAL;
284 }
285
286 /* If srq exist, set zero for relative number of rq */
287 if (!has_rq) {
288 hr_qp->rq.wqe_cnt = 0;
289 hr_qp->rq.max_gs = 0;
290 cap->max_recv_wr = 0;
291 cap->max_recv_sge = 0;
292 } else {
293 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) {
294 dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n");
295 return -EINVAL;
296 }
297
298 if (hr_dev->caps.min_wqes)
299 max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes);
300 else
301 max_cnt = cap->max_recv_wr;
302
303 hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt);
304
305 if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) {
306 dev_err(dev, "while setting rq size, rq.wqe_cnt too large\n");
307 return -EINVAL;
308 }
309
310 max_cnt = max(1U, cap->max_recv_sge);
311 hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt);
312 if (hr_dev->caps.max_rq_sg <= 2)
313 hr_qp->rq.wqe_shift =
314 ilog2(hr_dev->caps.max_rq_desc_sz);
315 else
316 hr_qp->rq.wqe_shift =
317 ilog2(hr_dev->caps.max_rq_desc_sz
318 * hr_qp->rq.max_gs);
319 }
320
321 cap->max_recv_wr = hr_qp->rq.max_post = hr_qp->rq.wqe_cnt;
322 cap->max_recv_sge = hr_qp->rq.max_gs;
323
324 return 0;
325 }
326
check_sq_size_with_integrity(struct hns_roce_dev * hr_dev,struct ib_qp_cap * cap,struct hns_roce_ib_create_qp * ucmd)327 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
328 struct ib_qp_cap *cap,
329 struct hns_roce_ib_create_qp *ucmd)
330 {
331 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
332 u8 max_sq_stride = ilog2(roundup_sq_stride);
333
334 /* Sanity check SQ size before proceeding */
335 if ((u32)(1 << ucmd->log_sq_bb_count) > hr_dev->caps.max_wqes ||
336 ucmd->log_sq_stride > max_sq_stride ||
337 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
338 ibdev_err(&hr_dev->ib_dev, "check SQ size error!\n");
339 return -EINVAL;
340 }
341
342 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
343 ibdev_err(&hr_dev->ib_dev, "SQ sge error! max_send_sge=%d\n",
344 cap->max_send_sge);
345 return -EINVAL;
346 }
347
348 return 0;
349 }
350
hns_roce_set_user_sq_size(struct hns_roce_dev * hr_dev,struct ib_qp_cap * cap,struct hns_roce_qp * hr_qp,struct hns_roce_ib_create_qp * ucmd)351 static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev,
352 struct ib_qp_cap *cap,
353 struct hns_roce_qp *hr_qp,
354 struct hns_roce_ib_create_qp *ucmd)
355 {
356 u32 ex_sge_num;
357 u32 page_size;
358 u32 max_cnt;
359 int ret;
360
361 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
362 if (ret) {
363 ibdev_err(&hr_dev->ib_dev, "Sanity check sq size failed\n");
364 return ret;
365 }
366
367 hr_qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
368 hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
369
370 max_cnt = max(1U, cap->max_send_sge);
371 if (hr_dev->caps.max_sq_sg <= 2)
372 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt);
373 else
374 hr_qp->sq.max_gs = max_cnt;
375
376 if (hr_qp->sq.max_gs > 2)
377 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
378 (hr_qp->sq.max_gs - 2));
379
380 if ((hr_qp->sq.max_gs > 2) && (hr_dev->pci_dev->revision == 0x20)) {
381 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) {
382 dev_err(hr_dev->dev,
383 "The extended sge cnt error! sge_cnt=%d\n",
384 hr_qp->sge.sge_cnt);
385 return -EINVAL;
386 }
387 }
388
389 hr_qp->sge.sge_shift = 4;
390 ex_sge_num = hr_qp->sge.sge_cnt;
391
392 /* Get buf size, SQ and RQ are aligned to page_szie */
393 if (hr_dev->caps.max_sq_sg <= 2) {
394 hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt <<
395 hr_qp->rq.wqe_shift), PAGE_SIZE) +
396 HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
397 hr_qp->sq.wqe_shift), PAGE_SIZE);
398
399 hr_qp->sq.offset = 0;
400 hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
401 hr_qp->sq.wqe_shift), PAGE_SIZE);
402 } else {
403 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
404 hr_qp->sge.sge_cnt = ex_sge_num ?
405 max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0;
406 hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt <<
407 hr_qp->rq.wqe_shift), page_size) +
408 HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt <<
409 hr_qp->sge.sge_shift), page_size) +
410 HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
411 hr_qp->sq.wqe_shift), page_size);
412
413 hr_qp->sq.offset = 0;
414 if (ex_sge_num) {
415 hr_qp->sge.offset = HNS_ROCE_ALOGN_UP(
416 (hr_qp->sq.wqe_cnt <<
417 hr_qp->sq.wqe_shift),
418 page_size);
419 hr_qp->rq.offset = hr_qp->sge.offset +
420 HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt <<
421 hr_qp->sge.sge_shift),
422 page_size);
423 } else {
424 hr_qp->rq.offset = HNS_ROCE_ALOGN_UP(
425 (hr_qp->sq.wqe_cnt <<
426 hr_qp->sq.wqe_shift),
427 page_size);
428 }
429 }
430
431 return 0;
432 }
433
split_wqe_buf_region(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp,struct hns_roce_buf_region * regions,int region_max,int page_shift)434 static int split_wqe_buf_region(struct hns_roce_dev *hr_dev,
435 struct hns_roce_qp *hr_qp,
436 struct hns_roce_buf_region *regions,
437 int region_max, int page_shift)
438 {
439 int page_size = 1 << page_shift;
440 bool is_extend_sge;
441 int region_cnt = 0;
442 int buf_size;
443 int buf_cnt;
444
445 if (hr_qp->buff_size < 1 || region_max < 1)
446 return region_cnt;
447
448 if (hr_qp->sge.sge_cnt > 0)
449 is_extend_sge = true;
450 else
451 is_extend_sge = false;
452
453 /* sq region */
454 if (is_extend_sge)
455 buf_size = hr_qp->sge.offset - hr_qp->sq.offset;
456 else
457 buf_size = hr_qp->rq.offset - hr_qp->sq.offset;
458
459 if (buf_size > 0 && region_cnt < region_max) {
460 buf_cnt = DIV_ROUND_UP(buf_size, page_size);
461 hns_roce_init_buf_region(®ions[region_cnt],
462 hr_dev->caps.wqe_sq_hop_num,
463 hr_qp->sq.offset / page_size,
464 buf_cnt);
465 region_cnt++;
466 }
467
468 /* sge region */
469 if (is_extend_sge) {
470 buf_size = hr_qp->rq.offset - hr_qp->sge.offset;
471 if (buf_size > 0 && region_cnt < region_max) {
472 buf_cnt = DIV_ROUND_UP(buf_size, page_size);
473 hns_roce_init_buf_region(®ions[region_cnt],
474 hr_dev->caps.wqe_sge_hop_num,
475 hr_qp->sge.offset / page_size,
476 buf_cnt);
477 region_cnt++;
478 }
479 }
480
481 /* rq region */
482 buf_size = hr_qp->buff_size - hr_qp->rq.offset;
483 if (buf_size > 0) {
484 buf_cnt = DIV_ROUND_UP(buf_size, page_size);
485 hns_roce_init_buf_region(®ions[region_cnt],
486 hr_dev->caps.wqe_rq_hop_num,
487 hr_qp->rq.offset / page_size,
488 buf_cnt);
489 region_cnt++;
490 }
491
492 return region_cnt;
493 }
494
calc_wqe_bt_page_shift(struct hns_roce_dev * hr_dev,struct hns_roce_buf_region * regions,int region_cnt)495 static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev,
496 struct hns_roce_buf_region *regions,
497 int region_cnt)
498 {
499 int bt_pg_shift;
500 int ba_num;
501 int ret;
502
503 bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz;
504
505 /* all root ba entries must in one bt page */
506 do {
507 ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN;
508 ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt,
509 ba_num);
510 if (ret <= ba_num)
511 break;
512
513 bt_pg_shift++;
514 } while (ret > ba_num);
515
516 return bt_pg_shift - PAGE_SHIFT;
517 }
518
set_extend_sge_param(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp)519 static int set_extend_sge_param(struct hns_roce_dev *hr_dev,
520 struct hns_roce_qp *hr_qp)
521 {
522 struct device *dev = hr_dev->dev;
523
524 if (hr_qp->sq.max_gs > 2) {
525 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
526 (hr_qp->sq.max_gs - 2));
527 hr_qp->sge.sge_shift = 4;
528 }
529
530 /* ud sqwqe's sge use extend sge */
531 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->ibqp.qp_type == IB_QPT_GSI) {
532 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
533 hr_qp->sq.max_gs);
534 hr_qp->sge.sge_shift = 4;
535 }
536
537 if ((hr_qp->sq.max_gs > 2) && hr_dev->pci_dev->revision == 0x20) {
538 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) {
539 dev_err(dev, "The extended sge cnt error! sge_cnt=%d\n",
540 hr_qp->sge.sge_cnt);
541 return -EINVAL;
542 }
543 }
544
545 return 0;
546 }
547
hns_roce_set_kernel_sq_size(struct hns_roce_dev * hr_dev,struct ib_qp_cap * cap,struct hns_roce_qp * hr_qp)548 static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev,
549 struct ib_qp_cap *cap,
550 struct hns_roce_qp *hr_qp)
551 {
552 struct device *dev = hr_dev->dev;
553 u32 page_size;
554 u32 max_cnt;
555 int size;
556 int ret;
557
558 if (cap->max_send_wr > hr_dev->caps.max_wqes ||
559 cap->max_send_sge > hr_dev->caps.max_sq_sg ||
560 cap->max_inline_data > hr_dev->caps.max_sq_inline) {
561 dev_err(dev, "SQ WR or sge or inline data error!\n");
562 return -EINVAL;
563 }
564
565 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
566
567 if (hr_dev->caps.min_wqes)
568 max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes);
569 else
570 max_cnt = cap->max_send_wr;
571
572 hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt);
573 if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) {
574 dev_err(dev, "while setting kernel sq size, sq.wqe_cnt too large\n");
575 return -EINVAL;
576 }
577
578 /* Get data_seg numbers */
579 max_cnt = max(1U, cap->max_send_sge);
580 if (hr_dev->caps.max_sq_sg <= 2)
581 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt);
582 else
583 hr_qp->sq.max_gs = max_cnt;
584
585 ret = set_extend_sge_param(hr_dev, hr_qp);
586 if (ret) {
587 dev_err(dev, "set extend sge parameters fail\n");
588 return ret;
589 }
590
591 /* Get buf size, SQ and RQ are aligned to PAGE_SIZE */
592 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
593 hr_qp->sq.offset = 0;
594 size = HNS_ROCE_ALOGN_UP(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift,
595 page_size);
596
597 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) {
598 hr_qp->sge.sge_cnt = max(page_size/(1 << hr_qp->sge.sge_shift),
599 (u32)hr_qp->sge.sge_cnt);
600 hr_qp->sge.offset = size;
601 size += HNS_ROCE_ALOGN_UP(hr_qp->sge.sge_cnt <<
602 hr_qp->sge.sge_shift, page_size);
603 }
604
605 hr_qp->rq.offset = size;
606 size += HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift),
607 page_size);
608 hr_qp->buff_size = size;
609
610 /* Get wr and sge number which send */
611 cap->max_send_wr = hr_qp->sq.max_post = hr_qp->sq.wqe_cnt;
612 cap->max_send_sge = hr_qp->sq.max_gs;
613
614 /* We don't support inline sends for kernel QPs (yet) */
615 cap->max_inline_data = 0;
616
617 return 0;
618 }
619
hns_roce_qp_has_sq(struct ib_qp_init_attr * attr)620 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr)
621 {
622 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr)
623 return 0;
624
625 return 1;
626 }
627
hns_roce_qp_has_rq(struct ib_qp_init_attr * attr)628 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
629 {
630 if (attr->qp_type == IB_QPT_XRC_INI ||
631 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
632 !attr->cap.max_recv_wr)
633 return 0;
634
635 return 1;
636 }
637
alloc_rq_inline_buf(struct hns_roce_qp * hr_qp,struct ib_qp_init_attr * init_attr)638 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
639 struct ib_qp_init_attr *init_attr)
640 {
641 u32 max_recv_sge = init_attr->cap.max_recv_sge;
642 struct hns_roce_rinl_wqe *wqe_list;
643 u32 wqe_cnt = hr_qp->rq.wqe_cnt;
644 int i;
645
646 /* allocate recv inline buf */
647 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe),
648 GFP_KERNEL);
649
650 if (!wqe_list)
651 goto err;
652
653 /* Allocate a continuous buffer for all inline sge we need */
654 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge *
655 sizeof(struct hns_roce_rinl_sge)),
656 GFP_KERNEL);
657 if (!wqe_list[0].sg_list)
658 goto err_wqe_list;
659
660 /* Assign buffers of sg_list to each inline wqe */
661 for (i = 1; i < wqe_cnt; i++)
662 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
663
664 hr_qp->rq_inl_buf.wqe_list = wqe_list;
665 hr_qp->rq_inl_buf.wqe_cnt = wqe_cnt;
666
667 return 0;
668
669 err_wqe_list:
670 kfree(wqe_list);
671
672 err:
673 return -ENOMEM;
674 }
675
free_rq_inline_buf(struct hns_roce_qp * hr_qp)676 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
677 {
678 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
679 kfree(hr_qp->rq_inl_buf.wqe_list);
680 }
681
hns_roce_create_qp_common(struct hns_roce_dev * hr_dev,struct ib_pd * ib_pd,struct ib_qp_init_attr * init_attr,struct ib_udata * udata,unsigned long sqpn,struct hns_roce_qp * hr_qp)682 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
683 struct ib_pd *ib_pd,
684 struct ib_qp_init_attr *init_attr,
685 struct ib_udata *udata, unsigned long sqpn,
686 struct hns_roce_qp *hr_qp)
687 {
688 dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { NULL };
689 struct device *dev = hr_dev->dev;
690 struct hns_roce_ib_create_qp ucmd;
691 struct hns_roce_ib_create_qp_resp resp = {};
692 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
693 udata, struct hns_roce_ucontext, ibucontext);
694 struct hns_roce_buf_region *r;
695 unsigned long qpn = 0;
696 u32 page_shift;
697 int buf_count;
698 int ret;
699 int i;
700
701 mutex_init(&hr_qp->mutex);
702 spin_lock_init(&hr_qp->sq.lock);
703 spin_lock_init(&hr_qp->rq.lock);
704
705 hr_qp->state = IB_QPS_RESET;
706
707 hr_qp->ibqp.qp_type = init_attr->qp_type;
708
709 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
710 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
711 else
712 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
713
714 ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, udata,
715 hns_roce_qp_has_rq(init_attr), hr_qp);
716 if (ret) {
717 dev_err(dev, "hns_roce_set_rq_size failed\n");
718 goto err_out;
719 }
720
721 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
722 hns_roce_qp_has_rq(init_attr)) {
723 ret = alloc_rq_inline_buf(hr_qp, init_attr);
724 if (ret) {
725 dev_err(dev, "allocate receive inline buffer failed\n");
726 goto err_out;
727 }
728 }
729
730 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
731 if (udata) {
732 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
733 dev_err(dev, "ib_copy_from_udata error for create qp\n");
734 ret = -EFAULT;
735 goto err_alloc_rq_inline_buf;
736 }
737
738 ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp,
739 &ucmd);
740 if (ret) {
741 dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n");
742 goto err_alloc_rq_inline_buf;
743 }
744
745 hr_qp->umem = ib_umem_get(udata, ucmd.buf_addr,
746 hr_qp->buff_size, 0, 0);
747 if (IS_ERR(hr_qp->umem)) {
748 dev_err(dev, "ib_umem_get error for create qp\n");
749 ret = PTR_ERR(hr_qp->umem);
750 goto err_alloc_rq_inline_buf;
751 }
752 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp,
753 hr_qp->regions, ARRAY_SIZE(hr_qp->regions),
754 page_shift);
755 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list,
756 hr_qp->region_cnt);
757 if (ret) {
758 dev_err(dev, "alloc buf_list error for create qp\n");
759 goto err_alloc_list;
760 }
761
762 for (i = 0; i < hr_qp->region_cnt; i++) {
763 r = &hr_qp->regions[i];
764 buf_count = hns_roce_get_umem_bufs(hr_dev,
765 buf_list[i], r->count, r->offset,
766 hr_qp->umem, page_shift);
767 if (buf_count != r->count) {
768 dev_err(dev,
769 "get umem buf err, expect %d,ret %d.\n",
770 r->count, buf_count);
771 ret = -ENOBUFS;
772 goto err_get_bufs;
773 }
774 }
775
776 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &&
777 (udata->inlen >= sizeof(ucmd)) &&
778 (udata->outlen >= sizeof(resp)) &&
779 hns_roce_qp_has_sq(init_attr)) {
780 ret = hns_roce_db_map_user(uctx, udata, ucmd.sdb_addr,
781 &hr_qp->sdb);
782 if (ret) {
783 dev_err(dev, "sq record doorbell map failed!\n");
784 goto err_get_bufs;
785 }
786
787 /* indicate kernel supports sq record db */
788 resp.cap_flags |= HNS_ROCE_SUPPORT_SQ_RECORD_DB;
789 hr_qp->sdb_en = 1;
790 }
791
792 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
793 (udata->outlen >= sizeof(resp)) &&
794 hns_roce_qp_has_rq(init_attr)) {
795 ret = hns_roce_db_map_user(uctx, udata, ucmd.db_addr,
796 &hr_qp->rdb);
797 if (ret) {
798 dev_err(dev, "rq record doorbell map failed!\n");
799 goto err_sq_dbmap;
800 }
801
802 /* indicate kernel supports rq record db */
803 resp.cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB;
804 hr_qp->rdb_en = 1;
805 }
806 } else {
807 if (init_attr->create_flags &
808 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
809 dev_err(dev, "init_attr->create_flags error!\n");
810 ret = -EINVAL;
811 goto err_alloc_rq_inline_buf;
812 }
813
814 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) {
815 dev_err(dev, "init_attr->create_flags error!\n");
816 ret = -EINVAL;
817 goto err_alloc_rq_inline_buf;
818 }
819
820 /* Set SQ size */
821 ret = hns_roce_set_kernel_sq_size(hr_dev, &init_attr->cap,
822 hr_qp);
823 if (ret) {
824 dev_err(dev, "hns_roce_set_kernel_sq_size error!\n");
825 goto err_alloc_rq_inline_buf;
826 }
827
828 /* QP doorbell register address */
829 hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset +
830 DB_REG_OFFSET * hr_dev->priv_uar.index;
831 hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset +
832 DB_REG_OFFSET * hr_dev->priv_uar.index;
833
834 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
835 hns_roce_qp_has_rq(init_attr)) {
836 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
837 if (ret) {
838 dev_err(dev, "rq record doorbell alloc failed!\n");
839 goto err_alloc_rq_inline_buf;
840 }
841 *hr_qp->rdb.db_record = 0;
842 hr_qp->rdb_en = 1;
843 }
844
845 /* Allocate QP buf */
846 if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size,
847 (1 << page_shift) * 2,
848 &hr_qp->hr_buf, page_shift)) {
849 dev_err(dev, "hns_roce_buf_alloc error!\n");
850 ret = -ENOMEM;
851 goto err_db;
852 }
853 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp,
854 hr_qp->regions, ARRAY_SIZE(hr_qp->regions),
855 page_shift);
856 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list,
857 hr_qp->region_cnt);
858 if (ret) {
859 dev_err(dev, "alloc buf_list error for create qp!\n");
860 goto err_alloc_list;
861 }
862
863 for (i = 0; i < hr_qp->region_cnt; i++) {
864 r = &hr_qp->regions[i];
865 buf_count = hns_roce_get_kmem_bufs(hr_dev,
866 buf_list[i], r->count, r->offset,
867 &hr_qp->hr_buf);
868 if (buf_count != r->count) {
869 dev_err(dev,
870 "get kmem buf err, expect %d,ret %d.\n",
871 r->count, buf_count);
872 ret = -ENOBUFS;
873 goto err_get_bufs;
874 }
875 }
876
877 hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64),
878 GFP_KERNEL);
879 if (ZERO_OR_NULL_PTR(hr_qp->sq.wrid)) {
880 ret = -ENOMEM;
881 goto err_get_bufs;
882 }
883
884 if (hr_qp->rq.wqe_cnt) {
885 hr_qp->rq.wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64),
886 GFP_KERNEL);
887 if (ZERO_OR_NULL_PTR(hr_qp->rq.wrid)) {
888 ret = -ENOMEM;
889 goto err_sq_wrid;
890 }
891 }
892 }
893
894 if (sqpn) {
895 qpn = sqpn;
896 } else {
897 /* Get QPN */
898 ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn);
899 if (ret) {
900 dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n");
901 goto err_wrid;
902 }
903 }
904
905 hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions,
906 hr_qp->region_cnt);
907 hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift,
908 page_shift);
909 ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list,
910 hr_qp->regions, hr_qp->region_cnt);
911 if (ret) {
912 dev_err(dev, "mtr attach error for create qp\n");
913 goto err_mtr;
914 }
915
916 if (init_attr->qp_type == IB_QPT_GSI &&
917 hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
918 /* In v1 engine, GSI QP context in RoCE engine's register */
919 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);
920 if (ret) {
921 dev_err(dev, "hns_roce_qp_alloc failed!\n");
922 goto err_qpn;
923 }
924 } else {
925 ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp);
926 if (ret) {
927 dev_err(dev, "hns_roce_qp_alloc failed!\n");
928 goto err_qpn;
929 }
930 }
931
932 if (sqpn)
933 hr_qp->doorbell_qpn = 1;
934 else
935 hr_qp->doorbell_qpn = (u32)hr_qp->qpn;
936
937 if (udata) {
938 ret = ib_copy_to_udata(udata, &resp,
939 min(udata->outlen, sizeof(resp)));
940 if (ret)
941 goto err_qp;
942 }
943
944 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
945 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp);
946 if (ret)
947 goto err_qp;
948 }
949
950 hr_qp->event = hns_roce_ib_qp_event;
951 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt);
952
953 return 0;
954
955 err_qp:
956 if (init_attr->qp_type == IB_QPT_GSI &&
957 hr_dev->hw_rev == HNS_ROCE_HW_VER1)
958 hns_roce_qp_remove(hr_dev, hr_qp);
959 else
960 hns_roce_qp_free(hr_dev, hr_qp);
961
962 err_qpn:
963 if (!sqpn)
964 hns_roce_release_range_qp(hr_dev, qpn, 1);
965
966 err_mtr:
967 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
968
969 err_wrid:
970 if (udata) {
971 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
972 (udata->outlen >= sizeof(resp)) &&
973 hns_roce_qp_has_rq(init_attr))
974 hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
975 } else {
976 if (hr_qp->rq.wqe_cnt)
977 kfree(hr_qp->rq.wrid);
978 }
979
980 err_sq_dbmap:
981 if (udata)
982 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &&
983 (udata->inlen >= sizeof(ucmd)) &&
984 (udata->outlen >= sizeof(resp)) &&
985 hns_roce_qp_has_sq(init_attr))
986 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
987
988 err_sq_wrid:
989 if (!udata)
990 kfree(hr_qp->sq.wrid);
991
992 err_get_bufs:
993 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt);
994
995 err_alloc_list:
996 if (!hr_qp->umem)
997 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
998 ib_umem_release(hr_qp->umem);
999
1000 err_db:
1001 if (!udata && hns_roce_qp_has_rq(init_attr) &&
1002 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB))
1003 hns_roce_free_db(hr_dev, &hr_qp->rdb);
1004
1005 err_alloc_rq_inline_buf:
1006 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
1007 hns_roce_qp_has_rq(init_attr))
1008 free_rq_inline_buf(hr_qp);
1009
1010 err_out:
1011 return ret;
1012 }
1013
hns_roce_create_qp(struct ib_pd * pd,struct ib_qp_init_attr * init_attr,struct ib_udata * udata)1014 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
1015 struct ib_qp_init_attr *init_attr,
1016 struct ib_udata *udata)
1017 {
1018 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
1019 struct ib_device *ibdev = &hr_dev->ib_dev;
1020 struct hns_roce_sqp *hr_sqp;
1021 struct hns_roce_qp *hr_qp;
1022 int ret;
1023
1024 switch (init_attr->qp_type) {
1025 case IB_QPT_RC: {
1026 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
1027 if (!hr_qp)
1028 return ERR_PTR(-ENOMEM);
1029
1030 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0,
1031 hr_qp);
1032 if (ret) {
1033 ibdev_err(ibdev, "Create RC QP 0x%06lx failed(%d)\n",
1034 hr_qp->qpn, ret);
1035 kfree(hr_qp);
1036 return ERR_PTR(ret);
1037 }
1038
1039 hr_qp->ibqp.qp_num = hr_qp->qpn;
1040
1041 break;
1042 }
1043 case IB_QPT_GSI: {
1044 /* Userspace is not allowed to create special QPs: */
1045 if (udata) {
1046 ibdev_err(ibdev, "not support usr space GSI\n");
1047 return ERR_PTR(-EINVAL);
1048 }
1049
1050 hr_sqp = kzalloc(sizeof(*hr_sqp), GFP_KERNEL);
1051 if (!hr_sqp)
1052 return ERR_PTR(-ENOMEM);
1053
1054 hr_qp = &hr_sqp->hr_qp;
1055 hr_qp->port = init_attr->port_num - 1;
1056 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
1057
1058 /* when hw version is v1, the sqpn is allocated */
1059 if (hr_dev->caps.max_sq_sg <= 2)
1060 hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS +
1061 hr_dev->iboe.phy_port[hr_qp->port];
1062 else
1063 hr_qp->ibqp.qp_num = 1;
1064
1065 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata,
1066 hr_qp->ibqp.qp_num, hr_qp);
1067 if (ret) {
1068 ibdev_err(ibdev, "Create GSI QP failed!\n");
1069 kfree(hr_sqp);
1070 return ERR_PTR(ret);
1071 }
1072
1073 break;
1074 }
1075 default:{
1076 ibdev_err(ibdev, "not support QP type %d\n",
1077 init_attr->qp_type);
1078 return ERR_PTR(-EINVAL);
1079 }
1080 }
1081
1082 return &hr_qp->ibqp;
1083 }
1084
to_hr_qp_type(int qp_type)1085 int to_hr_qp_type(int qp_type)
1086 {
1087 int transport_type;
1088
1089 if (qp_type == IB_QPT_RC)
1090 transport_type = SERV_TYPE_RC;
1091 else if (qp_type == IB_QPT_UC)
1092 transport_type = SERV_TYPE_UC;
1093 else if (qp_type == IB_QPT_UD)
1094 transport_type = SERV_TYPE_UD;
1095 else if (qp_type == IB_QPT_GSI)
1096 transport_type = SERV_TYPE_UD;
1097 else
1098 transport_type = -1;
1099
1100 return transport_type;
1101 }
1102
check_mtu_validate(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp,struct ib_qp_attr * attr,int attr_mask)1103 static int check_mtu_validate(struct hns_roce_dev *hr_dev,
1104 struct hns_roce_qp *hr_qp,
1105 struct ib_qp_attr *attr, int attr_mask)
1106 {
1107 enum ib_mtu active_mtu;
1108 int p;
1109
1110 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1111 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu);
1112
1113 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 &&
1114 attr->path_mtu > hr_dev->caps.max_mtu) ||
1115 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) {
1116 ibdev_err(&hr_dev->ib_dev,
1117 "attr path_mtu(%d)invalid while modify qp",
1118 attr->path_mtu);
1119 return -EINVAL;
1120 }
1121
1122 return 0;
1123 }
1124
hns_roce_check_qp_attr(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask)1125 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1126 int attr_mask)
1127 {
1128 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1129 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1130 int p;
1131
1132 if ((attr_mask & IB_QP_PORT) &&
1133 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1134 ibdev_err(&hr_dev->ib_dev,
1135 "attr port_num invalid.attr->port_num=%d\n",
1136 attr->port_num);
1137 return -EINVAL;
1138 }
1139
1140 if (attr_mask & IB_QP_PKEY_INDEX) {
1141 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1142 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
1143 ibdev_err(&hr_dev->ib_dev,
1144 "attr pkey_index invalid.attr->pkey_index=%d\n",
1145 attr->pkey_index);
1146 return -EINVAL;
1147 }
1148 }
1149
1150 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1151 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
1152 ibdev_err(&hr_dev->ib_dev,
1153 "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n",
1154 attr->max_rd_atomic);
1155 return -EINVAL;
1156 }
1157
1158 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1159 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
1160 ibdev_err(&hr_dev->ib_dev,
1161 "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n",
1162 attr->max_dest_rd_atomic);
1163 return -EINVAL;
1164 }
1165
1166 if (attr_mask & IB_QP_PATH_MTU)
1167 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);
1168
1169 return 0;
1170 }
1171
hns_roce_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1172 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1173 int attr_mask, struct ib_udata *udata)
1174 {
1175 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1176 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1177 enum ib_qp_state cur_state, new_state;
1178 int ret = -EINVAL;
1179
1180 mutex_lock(&hr_qp->mutex);
1181
1182 cur_state = attr_mask & IB_QP_CUR_STATE ?
1183 attr->cur_qp_state : (enum ib_qp_state)hr_qp->state;
1184 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1185
1186 if (ibqp->uobject &&
1187 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
1188 if (hr_qp->sdb_en == 1) {
1189 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
1190
1191 if (hr_qp->rdb_en == 1)
1192 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
1193 } else {
1194 ibdev_warn(&hr_dev->ib_dev,
1195 "flush cqe is not supported in userspace!\n");
1196 goto out;
1197 }
1198 }
1199
1200 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1201 attr_mask)) {
1202 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n");
1203 goto out;
1204 }
1205
1206 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);
1207 if (ret)
1208 goto out;
1209
1210 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1211 if (hr_dev->caps.min_wqes) {
1212 ret = -EPERM;
1213 ibdev_err(&hr_dev->ib_dev,
1214 "cur_state=%d new_state=%d\n", cur_state,
1215 new_state);
1216 } else {
1217 ret = 0;
1218 }
1219
1220 goto out;
1221 }
1222
1223 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1224 new_state);
1225
1226 out:
1227 mutex_unlock(&hr_qp->mutex);
1228
1229 return ret;
1230 }
1231
hns_roce_lock_cqs(struct hns_roce_cq * send_cq,struct hns_roce_cq * recv_cq)1232 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
1233 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1234 {
1235 if (send_cq == recv_cq) {
1236 spin_lock_irq(&send_cq->lock);
1237 __acquire(&recv_cq->lock);
1238 } else if (send_cq->cqn < recv_cq->cqn) {
1239 spin_lock_irq(&send_cq->lock);
1240 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1241 } else {
1242 spin_lock_irq(&recv_cq->lock);
1243 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1244 }
1245 }
1246
hns_roce_unlock_cqs(struct hns_roce_cq * send_cq,struct hns_roce_cq * recv_cq)1247 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1248 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
1249 __releases(&recv_cq->lock)
1250 {
1251 if (send_cq == recv_cq) {
1252 __release(&recv_cq->lock);
1253 spin_unlock_irq(&send_cq->lock);
1254 } else if (send_cq->cqn < recv_cq->cqn) {
1255 spin_unlock(&recv_cq->lock);
1256 spin_unlock_irq(&send_cq->lock);
1257 } else {
1258 spin_unlock(&send_cq->lock);
1259 spin_unlock_irq(&recv_cq->lock);
1260 }
1261 }
1262
get_wqe(struct hns_roce_qp * hr_qp,int offset)1263 static void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
1264 {
1265
1266 return hns_roce_buf_offset(&hr_qp->hr_buf, offset);
1267 }
1268
get_recv_wqe(struct hns_roce_qp * hr_qp,int n)1269 void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n)
1270 {
1271 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
1272 }
1273
get_send_wqe(struct hns_roce_qp * hr_qp,int n)1274 void *get_send_wqe(struct hns_roce_qp *hr_qp, int n)
1275 {
1276 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
1277 }
1278
get_send_extend_sge(struct hns_roce_qp * hr_qp,int n)1279 void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n)
1280 {
1281 return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset +
1282 (n << hr_qp->sge.sge_shift));
1283 }
1284
hns_roce_wq_overflow(struct hns_roce_wq * hr_wq,int nreq,struct ib_cq * ib_cq)1285 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
1286 struct ib_cq *ib_cq)
1287 {
1288 struct hns_roce_cq *hr_cq;
1289 u32 cur;
1290
1291 cur = hr_wq->head - hr_wq->tail;
1292 if (likely(cur + nreq < hr_wq->max_post))
1293 return false;
1294
1295 hr_cq = to_hr_cq(ib_cq);
1296 spin_lock(&hr_cq->lock);
1297 cur = hr_wq->head - hr_wq->tail;
1298 spin_unlock(&hr_cq->lock);
1299
1300 return cur + nreq >= hr_wq->max_post;
1301 }
1302
hns_roce_init_qp_table(struct hns_roce_dev * hr_dev)1303 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
1304 {
1305 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1306 int reserved_from_top = 0;
1307 int reserved_from_bot;
1308 int ret;
1309
1310 mutex_init(&qp_table->scc_mutex);
1311 xa_init(&hr_dev->qp_table_xa);
1312
1313 reserved_from_bot = hr_dev->caps.reserved_qps;
1314
1315 ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps,
1316 hr_dev->caps.num_qps - 1, reserved_from_bot,
1317 reserved_from_top);
1318 if (ret) {
1319 dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n",
1320 ret);
1321 return ret;
1322 }
1323
1324 return 0;
1325 }
1326
hns_roce_cleanup_qp_table(struct hns_roce_dev * hr_dev)1327 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
1328 {
1329 hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap);
1330 }
1331
