1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #include <linux/skbuff.h>
8
9 #include "rxe.h"
10 #include "rxe_loc.h"
11 #include "rxe_queue.h"
12
13 enum resp_states {
14 RESPST_NONE,
15 RESPST_GET_REQ,
16 RESPST_CHK_PSN,
17 RESPST_CHK_OP_SEQ,
18 RESPST_CHK_OP_VALID,
19 RESPST_CHK_RESOURCE,
20 RESPST_CHK_LENGTH,
21 RESPST_CHK_RKEY,
22 RESPST_EXECUTE,
23 RESPST_READ_REPLY,
24 RESPST_ATOMIC_REPLY,
25 RESPST_COMPLETE,
26 RESPST_ACKNOWLEDGE,
27 RESPST_CLEANUP,
28 RESPST_DUPLICATE_REQUEST,
29 RESPST_ERR_MALFORMED_WQE,
30 RESPST_ERR_UNSUPPORTED_OPCODE,
31 RESPST_ERR_MISALIGNED_ATOMIC,
32 RESPST_ERR_PSN_OUT_OF_SEQ,
33 RESPST_ERR_MISSING_OPCODE_FIRST,
34 RESPST_ERR_MISSING_OPCODE_LAST_C,
35 RESPST_ERR_MISSING_OPCODE_LAST_D1E,
36 RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
37 RESPST_ERR_RNR,
38 RESPST_ERR_RKEY_VIOLATION,
39 RESPST_ERR_INVALIDATE_RKEY,
40 RESPST_ERR_LENGTH,
41 RESPST_ERR_CQ_OVERFLOW,
42 RESPST_ERROR,
43 RESPST_RESET,
44 RESPST_DONE,
45 RESPST_EXIT,
46 };
47
48 static char *resp_state_name[] = {
49 [RESPST_NONE] = "NONE",
50 [RESPST_GET_REQ] = "GET_REQ",
51 [RESPST_CHK_PSN] = "CHK_PSN",
52 [RESPST_CHK_OP_SEQ] = "CHK_OP_SEQ",
53 [RESPST_CHK_OP_VALID] = "CHK_OP_VALID",
54 [RESPST_CHK_RESOURCE] = "CHK_RESOURCE",
55 [RESPST_CHK_LENGTH] = "CHK_LENGTH",
56 [RESPST_CHK_RKEY] = "CHK_RKEY",
57 [RESPST_EXECUTE] = "EXECUTE",
58 [RESPST_READ_REPLY] = "READ_REPLY",
59 [RESPST_ATOMIC_REPLY] = "ATOMIC_REPLY",
60 [RESPST_COMPLETE] = "COMPLETE",
61 [RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE",
62 [RESPST_CLEANUP] = "CLEANUP",
63 [RESPST_DUPLICATE_REQUEST] = "DUPLICATE_REQUEST",
64 [RESPST_ERR_MALFORMED_WQE] = "ERR_MALFORMED_WQE",
65 [RESPST_ERR_UNSUPPORTED_OPCODE] = "ERR_UNSUPPORTED_OPCODE",
66 [RESPST_ERR_MISALIGNED_ATOMIC] = "ERR_MISALIGNED_ATOMIC",
67 [RESPST_ERR_PSN_OUT_OF_SEQ] = "ERR_PSN_OUT_OF_SEQ",
68 [RESPST_ERR_MISSING_OPCODE_FIRST] = "ERR_MISSING_OPCODE_FIRST",
69 [RESPST_ERR_MISSING_OPCODE_LAST_C] = "ERR_MISSING_OPCODE_LAST_C",
70 [RESPST_ERR_MISSING_OPCODE_LAST_D1E] = "ERR_MISSING_OPCODE_LAST_D1E",
71 [RESPST_ERR_TOO_MANY_RDMA_ATM_REQ] = "ERR_TOO_MANY_RDMA_ATM_REQ",
72 [RESPST_ERR_RNR] = "ERR_RNR",
73 [RESPST_ERR_RKEY_VIOLATION] = "ERR_RKEY_VIOLATION",
74 [RESPST_ERR_INVALIDATE_RKEY] = "ERR_INVALIDATE_RKEY_VIOLATION",
75 [RESPST_ERR_LENGTH] = "ERR_LENGTH",
76 [RESPST_ERR_CQ_OVERFLOW] = "ERR_CQ_OVERFLOW",
77 [RESPST_ERROR] = "ERROR",
78 [RESPST_RESET] = "RESET",
79 [RESPST_DONE] = "DONE",
80 [RESPST_EXIT] = "EXIT",
81 };
82
83 /* rxe_recv calls here to add a request packet to the input queue */
rxe_resp_queue_pkt(struct rxe_qp * qp,struct sk_buff * skb)84 void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
85 {
86 int must_sched;
87 struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
88
89 skb_queue_tail(&qp->req_pkts, skb);
90
91 must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) ||
92 (skb_queue_len(&qp->req_pkts) > 1);
93
94 rxe_run_task(&qp->resp.task, must_sched);
95 }
96
get_req(struct rxe_qp * qp,struct rxe_pkt_info ** pkt_p)97 static inline enum resp_states get_req(struct rxe_qp *qp,
98 struct rxe_pkt_info **pkt_p)
99 {
100 struct sk_buff *skb;
101
102 if (qp->resp.state == QP_STATE_ERROR) {
103 while ((skb = skb_dequeue(&qp->req_pkts))) {
104 rxe_put(qp);
105 kfree_skb(skb);
106 ib_device_put(qp->ibqp.device);
107 }
108
109 /* go drain recv wr queue */
110 return RESPST_CHK_RESOURCE;
111 }
112
113 skb = skb_peek(&qp->req_pkts);
114 if (!skb)
115 return RESPST_EXIT;
116
117 *pkt_p = SKB_TO_PKT(skb);
118
119 return (qp->resp.res) ? RESPST_READ_REPLY : RESPST_CHK_PSN;
120 }
121
check_psn(struct rxe_qp * qp,struct rxe_pkt_info * pkt)122 static enum resp_states check_psn(struct rxe_qp *qp,
123 struct rxe_pkt_info *pkt)
124 {
125 int diff = psn_compare(pkt->psn, qp->resp.psn);
126 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
127
128 switch (qp_type(qp)) {
129 case IB_QPT_RC:
130 if (diff > 0) {
131 if (qp->resp.sent_psn_nak)
132 return RESPST_CLEANUP;
133
134 qp->resp.sent_psn_nak = 1;
135 rxe_counter_inc(rxe, RXE_CNT_OUT_OF_SEQ_REQ);
136 return RESPST_ERR_PSN_OUT_OF_SEQ;
137
138 } else if (diff < 0) {
139 rxe_counter_inc(rxe, RXE_CNT_DUP_REQ);
140 return RESPST_DUPLICATE_REQUEST;
141 }
142
143 if (qp->resp.sent_psn_nak)
144 qp->resp.sent_psn_nak = 0;
145
146 break;
147
148 case IB_QPT_UC:
149 if (qp->resp.drop_msg || diff != 0) {
150 if (pkt->mask & RXE_START_MASK) {
151 qp->resp.drop_msg = 0;
152 return RESPST_CHK_OP_SEQ;
153 }
154
155 qp->resp.drop_msg = 1;
156 return RESPST_CLEANUP;
157 }
158 break;
159 default:
160 break;
161 }
162
163 return RESPST_CHK_OP_SEQ;
164 }
165
check_op_seq(struct rxe_qp * qp,struct rxe_pkt_info * pkt)166 static enum resp_states check_op_seq(struct rxe_qp *qp,
167 struct rxe_pkt_info *pkt)
168 {
169 switch (qp_type(qp)) {
170 case IB_QPT_RC:
171 switch (qp->resp.opcode) {
172 case IB_OPCODE_RC_SEND_FIRST:
173 case IB_OPCODE_RC_SEND_MIDDLE:
174 switch (pkt->opcode) {
175 case IB_OPCODE_RC_SEND_MIDDLE:
176 case IB_OPCODE_RC_SEND_LAST:
177 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:
178 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:
179 return RESPST_CHK_OP_VALID;
180 default:
181 return RESPST_ERR_MISSING_OPCODE_LAST_C;
182 }
183
184 case IB_OPCODE_RC_RDMA_WRITE_FIRST:
185 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
186 switch (pkt->opcode) {
187 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
188 case IB_OPCODE_RC_RDMA_WRITE_LAST:
189 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
190 return RESPST_CHK_OP_VALID;
191 default:
192 return RESPST_ERR_MISSING_OPCODE_LAST_C;
193 }
194
195 default:
196 switch (pkt->opcode) {
197 case IB_OPCODE_RC_SEND_MIDDLE:
198 case IB_OPCODE_RC_SEND_LAST:
199 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:
200 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:
201 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
202 case IB_OPCODE_RC_RDMA_WRITE_LAST:
203 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
204 return RESPST_ERR_MISSING_OPCODE_FIRST;
205 default:
206 return RESPST_CHK_OP_VALID;
207 }
208 }
209 break;
210
211 case IB_QPT_UC:
212 switch (qp->resp.opcode) {
213 case IB_OPCODE_UC_SEND_FIRST:
214 case IB_OPCODE_UC_SEND_MIDDLE:
215 switch (pkt->opcode) {
216 case IB_OPCODE_UC_SEND_MIDDLE:
217 case IB_OPCODE_UC_SEND_LAST:
218 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:
219 return RESPST_CHK_OP_VALID;
220 default:
221 return RESPST_ERR_MISSING_OPCODE_LAST_D1E;
222 }
223
224 case IB_OPCODE_UC_RDMA_WRITE_FIRST:
225 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
226 switch (pkt->opcode) {
227 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
228 case IB_OPCODE_UC_RDMA_WRITE_LAST:
229 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
230 return RESPST_CHK_OP_VALID;
231 default:
232 return RESPST_ERR_MISSING_OPCODE_LAST_D1E;
233 }
234
235 default:
236 switch (pkt->opcode) {
237 case IB_OPCODE_UC_SEND_MIDDLE:
238 case IB_OPCODE_UC_SEND_LAST:
239 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:
240 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
241 case IB_OPCODE_UC_RDMA_WRITE_LAST:
242 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
243 qp->resp.drop_msg = 1;
244 return RESPST_CLEANUP;
245 default:
246 return RESPST_CHK_OP_VALID;
247 }
248 }
249 break;
250
251 default:
252 return RESPST_CHK_OP_VALID;
253 }
254 }
255
check_op_valid(struct rxe_qp * qp,struct rxe_pkt_info * pkt)256 static enum resp_states check_op_valid(struct rxe_qp *qp,
257 struct rxe_pkt_info *pkt)
258 {
259 switch (qp_type(qp)) {
260 case IB_QPT_RC:
261 if (((pkt->mask & RXE_READ_MASK) &&
262 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) ||
263 ((pkt->mask & RXE_WRITE_MASK) &&
264 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) ||
265 ((pkt->mask & RXE_ATOMIC_MASK) &&
266 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) {
267 return RESPST_ERR_UNSUPPORTED_OPCODE;
268 }
269
270 break;
271
272 case IB_QPT_UC:
273 if ((pkt->mask & RXE_WRITE_MASK) &&
274 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) {
275 qp->resp.drop_msg = 1;
276 return RESPST_CLEANUP;
277 }
278
279 break;
280
281 case IB_QPT_UD:
282 case IB_QPT_GSI:
283 break;
284
285 default:
286 WARN_ON_ONCE(1);
287 break;
288 }
289
290 return RESPST_CHK_RESOURCE;
291 }
292
get_srq_wqe(struct rxe_qp * qp)293 static enum resp_states get_srq_wqe(struct rxe_qp *qp)
294 {
295 struct rxe_srq *srq = qp->srq;
296 struct rxe_queue *q = srq->rq.queue;
297 struct rxe_recv_wqe *wqe;
298 struct ib_event ev;
299 unsigned int count;
300 size_t size;
301 unsigned long flags;
302
303 if (srq->error)
304 return RESPST_ERR_RNR;
305
306 spin_lock_irqsave(&srq->rq.consumer_lock, flags);
307
308 wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT);
309 if (!wqe) {
310 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
311 return RESPST_ERR_RNR;
312 }
313
314 /* don't trust user space data */
315 if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) {
316 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
317 pr_warn("%s: invalid num_sge in SRQ entry\n", __func__);
318 return RESPST_ERR_MALFORMED_WQE;
319 }
320 size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);
321 memcpy(&qp->resp.srq_wqe, wqe, size);
322
323 qp->resp.wqe = &qp->resp.srq_wqe.wqe;
324 queue_advance_consumer(q, QUEUE_TYPE_FROM_CLIENT);
325 count = queue_count(q, QUEUE_TYPE_FROM_CLIENT);
326
327 if (srq->limit && srq->ibsrq.event_handler && (count < srq->limit)) {
328 srq->limit = 0;
329 goto event;
330 }
331
332 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
333 return RESPST_CHK_LENGTH;
334
335 event:
336 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
337 ev.device = qp->ibqp.device;
338 ev.element.srq = qp->ibqp.srq;
339 ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
340 srq->ibsrq.event_handler(&ev, srq->ibsrq.srq_context);
341 return RESPST_CHK_LENGTH;
342 }
343
check_resource(struct rxe_qp * qp,struct rxe_pkt_info * pkt)344 static enum resp_states check_resource(struct rxe_qp *qp,
345 struct rxe_pkt_info *pkt)
346 {
347 struct rxe_srq *srq = qp->srq;
348
349 if (qp->resp.state == QP_STATE_ERROR) {
350 if (qp->resp.wqe) {
351 qp->resp.status = IB_WC_WR_FLUSH_ERR;
352 return RESPST_COMPLETE;
353 } else if (!srq) {
354 qp->resp.wqe = queue_head(qp->rq.queue,
355 QUEUE_TYPE_FROM_CLIENT);
356 if (qp->resp.wqe) {
357 qp->resp.status = IB_WC_WR_FLUSH_ERR;
358 return RESPST_COMPLETE;
359 } else {
360 return RESPST_EXIT;
361 }
362 } else {
363 return RESPST_EXIT;
364 }
365 }
366
367 if (pkt->mask & RXE_READ_OR_ATOMIC_MASK) {
368 /* it is the requesters job to not send
369 * too many read/atomic ops, we just
370 * recycle the responder resource queue
371 */
372 if (likely(qp->attr.max_dest_rd_atomic > 0))
373 return RESPST_CHK_LENGTH;
374 else
375 return RESPST_ERR_TOO_MANY_RDMA_ATM_REQ;
376 }
377
378 if (pkt->mask & RXE_RWR_MASK) {
379 if (srq)
380 return get_srq_wqe(qp);
381
382 qp->resp.wqe = queue_head(qp->rq.queue,
383 QUEUE_TYPE_FROM_CLIENT);
384 return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR;
385 }
386
387 return RESPST_CHK_LENGTH;
388 }
389
check_length(struct rxe_qp * qp,struct rxe_pkt_info * pkt)390 static enum resp_states check_length(struct rxe_qp *qp,
391 struct rxe_pkt_info *pkt)
392 {
393 switch (qp_type(qp)) {
394 case IB_QPT_RC:
395 return RESPST_CHK_RKEY;
396
397 case IB_QPT_UC:
398 return RESPST_CHK_RKEY;
399
400 default:
401 return RESPST_CHK_RKEY;
402 }
403 }
404
check_rkey(struct rxe_qp * qp,struct rxe_pkt_info * pkt)405 static enum resp_states check_rkey(struct rxe_qp *qp,
406 struct rxe_pkt_info *pkt)
407 {
408 struct rxe_mr *mr = NULL;
409 struct rxe_mw *mw = NULL;
410 u64 va;
411 u32 rkey;
412 u32 resid;
413 u32 pktlen;
414 int mtu = qp->mtu;
415 enum resp_states state;
416 int access;
417
418 if (pkt->mask & RXE_READ_OR_WRITE_MASK) {
419 if (pkt->mask & RXE_RETH_MASK) {
420 qp->resp.va = reth_va(pkt);
421 qp->resp.offset = 0;
422 qp->resp.rkey = reth_rkey(pkt);
423 qp->resp.resid = reth_len(pkt);
424 qp->resp.length = reth_len(pkt);
425 }
426 access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ
427 : IB_ACCESS_REMOTE_WRITE;
428 } else if (pkt->mask & RXE_ATOMIC_MASK) {
429 qp->resp.va = atmeth_va(pkt);
430 qp->resp.offset = 0;
431 qp->resp.rkey = atmeth_rkey(pkt);
432 qp->resp.resid = sizeof(u64);
433 access = IB_ACCESS_REMOTE_ATOMIC;
434 } else {
435 return RESPST_EXECUTE;
436 }
437
438 /* A zero-byte op is not required to set an addr or rkey. */
439 if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&
440 (pkt->mask & RXE_RETH_MASK) &&
441 reth_len(pkt) == 0) {
442 return RESPST_EXECUTE;
443 }
444
445 va = qp->resp.va;
446 rkey = qp->resp.rkey;
447 resid = qp->resp.resid;
448 pktlen = payload_size(pkt);
449
450 if (rkey_is_mw(rkey)) {
451 mw = rxe_lookup_mw(qp, access, rkey);
452 if (!mw) {
453 pr_debug("%s: no MW matches rkey %#x\n",
454 __func__, rkey);
455 state = RESPST_ERR_RKEY_VIOLATION;
456 goto err;
457 }
458
459 mr = mw->mr;
460 if (!mr) {
461 pr_err("%s: MW doesn't have an MR\n", __func__);
462 state = RESPST_ERR_RKEY_VIOLATION;
463 goto err;
464 }
465
466 if (mw->access & IB_ZERO_BASED)
467 qp->resp.offset = mw->addr;
468
469 rxe_put(mw);
470 rxe_get(mr);
471 } else {
472 mr = lookup_mr(qp->pd, access, rkey, RXE_LOOKUP_REMOTE);
473 if (!mr) {
474 pr_debug("%s: no MR matches rkey %#x\n",
475 __func__, rkey);
476 state = RESPST_ERR_RKEY_VIOLATION;
477 goto err;
478 }
479 }
480
481 if (mr_check_range(mr, va + qp->resp.offset, resid)) {
482 state = RESPST_ERR_RKEY_VIOLATION;
483 goto err;
484 }
485
486 if (pkt->mask & RXE_WRITE_MASK) {
487 if (resid > mtu) {
488 if (pktlen != mtu || bth_pad(pkt)) {
489 state = RESPST_ERR_LENGTH;
490 goto err;
491 }
492 } else {
493 if (pktlen != resid) {
494 state = RESPST_ERR_LENGTH;
495 goto err;
496 }
497 if ((bth_pad(pkt) != (0x3 & (-resid)))) {
498 /* This case may not be exactly that
499 * but nothing else fits.
500 */
501 state = RESPST_ERR_LENGTH;
502 goto err;
503 }
504 }
505 }
506
507 WARN_ON_ONCE(qp->resp.mr);
508
509 qp->resp.mr = mr;
510 return RESPST_EXECUTE;
511
512 err:
513 if (mr)
514 rxe_put(mr);
515 if (mw)
516 rxe_put(mw);
517
518 return state;
519 }
520
send_data_in(struct rxe_qp * qp,void * data_addr,int data_len)521 static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,
522 int data_len)
523 {
524 int err;
525
526 err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
527 data_addr, data_len, RXE_TO_MR_OBJ);
528 if (unlikely(err))
529 return (err == -ENOSPC) ? RESPST_ERR_LENGTH
530 : RESPST_ERR_MALFORMED_WQE;
531
532 return RESPST_NONE;
533 }
534
write_data_in(struct rxe_qp * qp,struct rxe_pkt_info * pkt)535 static enum resp_states write_data_in(struct rxe_qp *qp,
536 struct rxe_pkt_info *pkt)
537 {
538 enum resp_states rc = RESPST_NONE;
539 int err;
540 int data_len = payload_size(pkt);
541
542 err = rxe_mr_copy(qp->resp.mr, qp->resp.va + qp->resp.offset,
543 payload_addr(pkt), data_len, RXE_TO_MR_OBJ);
544 if (err) {
545 rc = RESPST_ERR_RKEY_VIOLATION;
546 goto out;
547 }
548
549 qp->resp.va += data_len;
550 qp->resp.resid -= data_len;
551
552 out:
553 return rc;
554 }
555
rxe_prepare_res(struct rxe_qp * qp,struct rxe_pkt_info * pkt,int type)556 static struct resp_res *rxe_prepare_res(struct rxe_qp *qp,
557 struct rxe_pkt_info *pkt,
558 int type)
559 {
560 struct resp_res *res;
561 u32 pkts;
562
563 res = &qp->resp.resources[qp->resp.res_head];
564 rxe_advance_resp_resource(qp);
565 free_rd_atomic_resource(res);
566
567 res->type = type;
568 res->replay = 0;
569
570 switch (type) {
571 case RXE_READ_MASK:
572 res->read.va = qp->resp.va + qp->resp.offset;
573 res->read.va_org = qp->resp.va + qp->resp.offset;
574 res->read.resid = qp->resp.resid;
575 res->read.length = qp->resp.resid;
576 res->read.rkey = qp->resp.rkey;
577
578 pkts = max_t(u32, (reth_len(pkt) + qp->mtu - 1)/qp->mtu, 1);
579 res->first_psn = pkt->psn;
580 res->cur_psn = pkt->psn;
581 res->last_psn = (pkt->psn + pkts - 1) & BTH_PSN_MASK;
582
583 res->state = rdatm_res_state_new;
584 break;
585 case RXE_ATOMIC_MASK:
586 res->first_psn = pkt->psn;
587 res->last_psn = pkt->psn;
588 res->cur_psn = pkt->psn;
589 break;
590 }
591
592 return res;
593 }
594
595 /* Guarantee atomicity of atomic operations at the machine level. */
596 static DEFINE_SPINLOCK(atomic_ops_lock);
597
atomic_reply(struct rxe_qp * qp,struct rxe_pkt_info * pkt)598 static enum resp_states atomic_reply(struct rxe_qp *qp,
599 struct rxe_pkt_info *pkt)
600 {
601 u64 *vaddr;
602 enum resp_states ret;
603 struct rxe_mr *mr = qp->resp.mr;
604 struct resp_res *res = qp->resp.res;
605 u64 value;
606
607 if (!res) {
608 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_MASK);
609 qp->resp.res = res;
610 }
611
612 if (!res->replay) {
613 if (mr->state != RXE_MR_STATE_VALID) {
614 ret = RESPST_ERR_RKEY_VIOLATION;
615 goto out;
616 }
617
618 vaddr = iova_to_vaddr(mr, qp->resp.va + qp->resp.offset,
619 sizeof(u64));
620
621 /* check vaddr is 8 bytes aligned. */
622 if (!vaddr || (uintptr_t)vaddr & 7) {
623 ret = RESPST_ERR_MISALIGNED_ATOMIC;
624 goto out;
625 }
626
627 spin_lock_bh(&atomic_ops_lock);
628 res->atomic.orig_val = value = *vaddr;
629
630 if (pkt->opcode == IB_OPCODE_RC_COMPARE_SWAP) {
631 if (value == atmeth_comp(pkt))
632 value = atmeth_swap_add(pkt);
633 } else {
634 value += atmeth_swap_add(pkt);
635 }
636
637 *vaddr = value;
638 spin_unlock_bh(&atomic_ops_lock);
639
640 qp->resp.msn++;
641
642 /* next expected psn, read handles this separately */
643 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
644 qp->resp.ack_psn = qp->resp.psn;
645
646 qp->resp.opcode = pkt->opcode;
647 qp->resp.status = IB_WC_SUCCESS;
648 }
649
650 ret = RESPST_ACKNOWLEDGE;
651 out:
652 return ret;
653 }
654
prepare_ack_packet(struct rxe_qp * qp,struct rxe_pkt_info * ack,int opcode,int payload,u32 psn,u8 syndrome)655 static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
656 struct rxe_pkt_info *ack,
657 int opcode,
658 int payload,
659 u32 psn,
660 u8 syndrome)
661 {
662 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
663 struct sk_buff *skb;
664 int paylen;
665 int pad;
666 int err;
667
668 /*
669 * allocate packet
670 */
671 pad = (-payload) & 0x3;
672 paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
673
674 skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack);
675 if (!skb)
676 return NULL;
677
678 ack->qp = qp;
679 ack->opcode = opcode;
680 ack->mask = rxe_opcode[opcode].mask;
681 ack->paylen = paylen;
682 ack->psn = psn;
683
684 bth_init(ack, opcode, 0, 0, pad, IB_DEFAULT_PKEY_FULL,
685 qp->attr.dest_qp_num, 0, psn);
686
687 if (ack->mask & RXE_AETH_MASK) {
688 aeth_set_syn(ack, syndrome);
689 aeth_set_msn(ack, qp->resp.msn);
690 }
691
692 if (ack->mask & RXE_ATMACK_MASK)
693 atmack_set_orig(ack, qp->resp.res->atomic.orig_val);
694
695 err = rxe_prepare(&qp->pri_av, ack, skb);
696 if (err) {
697 kfree_skb(skb);
698 return NULL;
699 }
700
701 return skb;
702 }
703
704 /**
705 * rxe_recheck_mr - revalidate MR from rkey and get a reference
706 * @qp: the qp
707 * @rkey: the rkey
708 *
709 * This code allows the MR to be invalidated or deregistered or
710 * the MW if one was used to be invalidated or deallocated.
711 * It is assumed that the access permissions if originally good
712 * are OK and the mappings to be unchanged.
713 *
714 * TODO: If someone reregisters an MR to change its size or
715 * access permissions during the processing of an RDMA read
716 * we should kill the responder resource and complete the
717 * operation with an error.
718 *
719 * Return: mr on success else NULL
720 */
rxe_recheck_mr(struct rxe_qp * qp,u32 rkey)721 static struct rxe_mr *rxe_recheck_mr(struct rxe_qp *qp, u32 rkey)
722 {
723 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
724 struct rxe_mr *mr;
725 struct rxe_mw *mw;
726
727 if (rkey_is_mw(rkey)) {
728 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8);
729 if (!mw)
730 return NULL;
731
732 mr = mw->mr;
733 if (mw->rkey != rkey || mw->state != RXE_MW_STATE_VALID ||
734 !mr || mr->state != RXE_MR_STATE_VALID) {
735 rxe_put(mw);
736 return NULL;
737 }
738
739 rxe_get(mr);
740 rxe_put(mw);
741
742 return mr;
743 }
744
745 mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
746 if (!mr)
747 return NULL;
748
749 if (mr->rkey != rkey || mr->state != RXE_MR_STATE_VALID) {
750 rxe_put(mr);
751 return NULL;
752 }
753
754 return mr;
755 }
756
757 /* RDMA read response. If res is not NULL, then we have a current RDMA request
758 * being processed or replayed.
759 */
read_reply(struct rxe_qp * qp,struct rxe_pkt_info * req_pkt)760 static enum resp_states read_reply(struct rxe_qp *qp,
761 struct rxe_pkt_info *req_pkt)
762 {
763 struct rxe_pkt_info ack_pkt;
764 struct sk_buff *skb;
765 int mtu = qp->mtu;
766 enum resp_states state;
767 int payload;
768 int opcode;
769 int err;
770 struct resp_res *res = qp->resp.res;
771 struct rxe_mr *mr;
772
773 if (!res) {
774 res = rxe_prepare_res(qp, req_pkt, RXE_READ_MASK);
775 qp->resp.res = res;
776 }
777
778 if (res->state == rdatm_res_state_new) {
779 if (!res->replay) {
780 mr = qp->resp.mr;
781 qp->resp.mr = NULL;
782 } else {
783 mr = rxe_recheck_mr(qp, res->read.rkey);
784 if (!mr)
785 return RESPST_ERR_RKEY_VIOLATION;
786 }
787
788 if (res->read.resid <= mtu)
789 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY;
790 else
791 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
792 } else {
793 mr = rxe_recheck_mr(qp, res->read.rkey);
794 if (!mr)
795 return RESPST_ERR_RKEY_VIOLATION;
796
797 if (res->read.resid > mtu)
798 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
799 else
800 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
801 }
802
803 res->state = rdatm_res_state_next;
804
805 payload = min_t(int, res->read.resid, mtu);
806
807 skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,
808 res->cur_psn, AETH_ACK_UNLIMITED);
809 if (!skb) {
810 rxe_put(mr);
811 return RESPST_ERR_RNR;
812 }
813
814 rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
815 payload, RXE_FROM_MR_OBJ);
816 if (mr)
817 rxe_put(mr);
818
819 if (bth_pad(&ack_pkt)) {
820 u8 *pad = payload_addr(&ack_pkt) + payload;
821
822 memset(pad, 0, bth_pad(&ack_pkt));
823 }
824
825 err = rxe_xmit_packet(qp, &ack_pkt, skb);
826 if (err)
827 return RESPST_ERR_RNR;
828
829 res->read.va += payload;
830 res->read.resid -= payload;
831 res->cur_psn = (res->cur_psn + 1) & BTH_PSN_MASK;
832
833 if (res->read.resid > 0) {
834 state = RESPST_DONE;
835 } else {
836 qp->resp.res = NULL;
837 if (!res->replay)
838 qp->resp.opcode = -1;
839 if (psn_compare(res->cur_psn, qp->resp.psn) >= 0)
840 qp->resp.psn = res->cur_psn;
841 state = RESPST_CLEANUP;
842 }
843
844 return state;
845 }
846
invalidate_rkey(struct rxe_qp * qp,u32 rkey)847 static int invalidate_rkey(struct rxe_qp *qp, u32 rkey)
848 {
849 if (rkey_is_mw(rkey))
850 return rxe_invalidate_mw(qp, rkey);
851 else
852 return rxe_invalidate_mr(qp, rkey);
853 }
854
855 /* Executes a new request. A retried request never reach that function (send
856 * and writes are discarded, and reads and atomics are retried elsewhere.
857 */
execute(struct rxe_qp * qp,struct rxe_pkt_info * pkt)858 static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
859 {
860 enum resp_states err;
861 struct sk_buff *skb = PKT_TO_SKB(pkt);
862 union rdma_network_hdr hdr;
863
864 if (pkt->mask & RXE_SEND_MASK) {
865 if (qp_type(qp) == IB_QPT_UD ||
866 qp_type(qp) == IB_QPT_GSI) {
867 if (skb->protocol == htons(ETH_P_IP)) {
868 memset(&hdr.reserved, 0,
869 sizeof(hdr.reserved));
870 memcpy(&hdr.roce4grh, ip_hdr(skb),
871 sizeof(hdr.roce4grh));
872 err = send_data_in(qp, &hdr, sizeof(hdr));
873 } else {
874 err = send_data_in(qp, ipv6_hdr(skb),
875 sizeof(hdr));
876 }
877 if (err)
878 return err;
879 }
880 err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));
881 if (err)
882 return err;
883 } else if (pkt->mask & RXE_WRITE_MASK) {
884 err = write_data_in(qp, pkt);
885 if (err)
886 return err;
887 } else if (pkt->mask & RXE_READ_MASK) {
888 /* For RDMA Read we can increment the msn now. See C9-148. */
889 qp->resp.msn++;
890 return RESPST_READ_REPLY;
891 } else if (pkt->mask & RXE_ATOMIC_MASK) {
892 return RESPST_ATOMIC_REPLY;
893 } else {
894 /* Unreachable */
895 WARN_ON_ONCE(1);
896 }
897
898 if (pkt->mask & RXE_IETH_MASK) {
899 u32 rkey = ieth_rkey(pkt);
900
901 err = invalidate_rkey(qp, rkey);
902 if (err)
903 return RESPST_ERR_INVALIDATE_RKEY;
904 }
905
906 if (pkt->mask & RXE_END_MASK)
907 /* We successfully processed this new request. */
908 qp->resp.msn++;
909
910 /* next expected psn, read handles this separately */
911 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
912 qp->resp.ack_psn = qp->resp.psn;
913
914 qp->resp.opcode = pkt->opcode;
915 qp->resp.status = IB_WC_SUCCESS;
916
917 if (pkt->mask & RXE_COMP_MASK)
918 return RESPST_COMPLETE;
919 else if (qp_type(qp) == IB_QPT_RC)
920 return RESPST_ACKNOWLEDGE;
921 else
922 return RESPST_CLEANUP;
923 }
924
do_complete(struct rxe_qp * qp,struct rxe_pkt_info * pkt)925 static enum resp_states do_complete(struct rxe_qp *qp,
926 struct rxe_pkt_info *pkt)
927 {
928 struct rxe_cqe cqe;
929 struct ib_wc *wc = &cqe.ibwc;
930 struct ib_uverbs_wc *uwc = &cqe.uibwc;
931 struct rxe_recv_wqe *wqe = qp->resp.wqe;
932 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
933
934 if (!wqe)
935 goto finish;
936
937 memset(&cqe, 0, sizeof(cqe));
938
939 if (qp->rcq->is_user) {
940 uwc->status = qp->resp.status;
941 uwc->qp_num = qp->ibqp.qp_num;
942 uwc->wr_id = wqe->wr_id;
943 } else {
944 wc->status = qp->resp.status;
945 wc->qp = &qp->ibqp;
946 wc->wr_id = wqe->wr_id;
947 }
948
949 if (wc->status == IB_WC_SUCCESS) {
950 rxe_counter_inc(rxe, RXE_CNT_RDMA_RECV);
951 wc->opcode = (pkt->mask & RXE_IMMDT_MASK &&
952 pkt->mask & RXE_WRITE_MASK) ?
953 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
954 wc->byte_len = (pkt->mask & RXE_IMMDT_MASK &&
955 pkt->mask & RXE_WRITE_MASK) ?
956 qp->resp.length : wqe->dma.length - wqe->dma.resid;
957
958 /* fields after byte_len are different between kernel and user
959 * space
960 */
961 if (qp->rcq->is_user) {
962 uwc->wc_flags = IB_WC_GRH;
963
964 if (pkt->mask & RXE_IMMDT_MASK) {
965 uwc->wc_flags |= IB_WC_WITH_IMM;
966 uwc->ex.imm_data = immdt_imm(pkt);
967 }
968
969 if (pkt->mask & RXE_IETH_MASK) {
970 uwc->wc_flags |= IB_WC_WITH_INVALIDATE;
971 uwc->ex.invalidate_rkey = ieth_rkey(pkt);
972 }
973
974 if (pkt->mask & RXE_DETH_MASK)
975 uwc->src_qp = deth_sqp(pkt);
976
977 uwc->port_num = qp->attr.port_num;
978 } else {
979 struct sk_buff *skb = PKT_TO_SKB(pkt);
980
981 wc->wc_flags = IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE;
982 if (skb->protocol == htons(ETH_P_IP))
983 wc->network_hdr_type = RDMA_NETWORK_IPV4;
984 else
985 wc->network_hdr_type = RDMA_NETWORK_IPV6;
986
987 if (is_vlan_dev(skb->dev)) {
988 wc->wc_flags |= IB_WC_WITH_VLAN;
989 wc->vlan_id = vlan_dev_vlan_id(skb->dev);
990 }
991
992 if (pkt->mask & RXE_IMMDT_MASK) {
993 wc->wc_flags |= IB_WC_WITH_IMM;
994 wc->ex.imm_data = immdt_imm(pkt);
995 }
996
997 if (pkt->mask & RXE_IETH_MASK) {
998 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
999 wc->ex.invalidate_rkey = ieth_rkey(pkt);
1000 }
1001
1002 if (pkt->mask & RXE_DETH_MASK)
1003 wc->src_qp = deth_sqp(pkt);
1004
1005 wc->port_num = qp->attr.port_num;
1006 }
1007 }
1008
1009 /* have copy for srq and reference for !srq */
1010 if (!qp->srq)
1011 queue_advance_consumer(qp->rq.queue, QUEUE_TYPE_FROM_CLIENT);
1012
1013 qp->resp.wqe = NULL;
1014
1015 if (rxe_cq_post(qp->rcq, &cqe, pkt ? bth_se(pkt) : 1))
1016 return RESPST_ERR_CQ_OVERFLOW;
1017
1018 finish:
1019 if (unlikely(qp->resp.state == QP_STATE_ERROR))
1020 return RESPST_CHK_RESOURCE;
1021 if (unlikely(!pkt))
1022 return RESPST_DONE;
1023 if (qp_type(qp) == IB_QPT_RC)
1024 return RESPST_ACKNOWLEDGE;
1025 else
1026 return RESPST_CLEANUP;
1027 }
1028
1029
send_common_ack(struct rxe_qp * qp,u8 syndrome,u32 psn,int opcode,const char * msg)1030 static int send_common_ack(struct rxe_qp *qp, u8 syndrome, u32 psn,
1031 int opcode, const char *msg)
1032 {
1033 int err;
1034 struct rxe_pkt_info ack_pkt;
1035 struct sk_buff *skb;
1036
1037 skb = prepare_ack_packet(qp, &ack_pkt, opcode, 0, psn, syndrome);
1038 if (!skb)
1039 return -ENOMEM;
1040
1041 err = rxe_xmit_packet(qp, &ack_pkt, skb);
1042 if (err)
1043 pr_err_ratelimited("Failed sending %s\n", msg);
1044
1045 return err;
1046 }
1047
send_ack(struct rxe_qp * qp,u8 syndrome,u32 psn)1048 static int send_ack(struct rxe_qp *qp, u8 syndrome, u32 psn)
1049 {
1050 return send_common_ack(qp, syndrome, psn,
1051 IB_OPCODE_RC_ACKNOWLEDGE, "ACK");
1052 }
1053
send_atomic_ack(struct rxe_qp * qp,u8 syndrome,u32 psn)1054 static int send_atomic_ack(struct rxe_qp *qp, u8 syndrome, u32 psn)
1055 {
1056 int ret = send_common_ack(qp, syndrome, psn,
1057 IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE, "ATOMIC ACK");
1058
1059 /* have to clear this since it is used to trigger
1060 * long read replies
1061 */
1062 qp->resp.res = NULL;
1063 return ret;
1064 }
1065
acknowledge(struct rxe_qp * qp,struct rxe_pkt_info * pkt)1066 static enum resp_states acknowledge(struct rxe_qp *qp,
1067 struct rxe_pkt_info *pkt)
1068 {
1069 if (qp_type(qp) != IB_QPT_RC)
1070 return RESPST_CLEANUP;
1071
1072 if (qp->resp.aeth_syndrome != AETH_ACK_UNLIMITED)
1073 send_ack(qp, qp->resp.aeth_syndrome, pkt->psn);
1074 else if (pkt->mask & RXE_ATOMIC_MASK)
1075 send_atomic_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1076 else if (bth_ack(pkt))
1077 send_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1078
1079 return RESPST_CLEANUP;
1080 }
1081
cleanup(struct rxe_qp * qp,struct rxe_pkt_info * pkt)1082 static enum resp_states cleanup(struct rxe_qp *qp,
1083 struct rxe_pkt_info *pkt)
1084 {
1085 struct sk_buff *skb;
1086
1087 if (pkt) {
1088 skb = skb_dequeue(&qp->req_pkts);
1089 rxe_put(qp);
1090 kfree_skb(skb);
1091 ib_device_put(qp->ibqp.device);
1092 }
1093
1094 if (qp->resp.mr) {
1095 rxe_put(qp->resp.mr);
1096 qp->resp.mr = NULL;
1097 }
1098
1099 return RESPST_DONE;
1100 }
1101
find_resource(struct rxe_qp * qp,u32 psn)1102 static struct resp_res *find_resource(struct rxe_qp *qp, u32 psn)
1103 {
1104 int i;
1105
1106 for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
1107 struct resp_res *res = &qp->resp.resources[i];
1108
1109 if (res->type == 0)
1110 continue;
1111
1112 if (psn_compare(psn, res->first_psn) >= 0 &&
1113 psn_compare(psn, res->last_psn) <= 0) {
1114 return res;
1115 }
1116 }
1117
1118 return NULL;
1119 }
1120
duplicate_request(struct rxe_qp * qp,struct rxe_pkt_info * pkt)1121 static enum resp_states duplicate_request(struct rxe_qp *qp,
1122 struct rxe_pkt_info *pkt)
1123 {
1124 enum resp_states rc;
1125 u32 prev_psn = (qp->resp.ack_psn - 1) & BTH_PSN_MASK;
1126
1127 if (pkt->mask & RXE_SEND_MASK ||
1128 pkt->mask & RXE_WRITE_MASK) {
1129 /* SEND. Ack again and cleanup. C9-105. */
1130 send_ack(qp, AETH_ACK_UNLIMITED, prev_psn);
1131 return RESPST_CLEANUP;
1132 } else if (pkt->mask & RXE_READ_MASK) {
1133 struct resp_res *res;
1134
1135 res = find_resource(qp, pkt->psn);
1136 if (!res) {
1137 /* Resource not found. Class D error. Drop the
1138 * request.
1139 */
1140 rc = RESPST_CLEANUP;
1141 goto out;
1142 } else {
1143 /* Ensure this new request is the same as the previous
1144 * one or a subset of it.
1145 */
1146 u64 iova = reth_va(pkt);
1147 u32 resid = reth_len(pkt);
1148
1149 if (iova < res->read.va_org ||
1150 resid > res->read.length ||
1151 (iova + resid) > (res->read.va_org +
1152 res->read.length)) {
1153 rc = RESPST_CLEANUP;
1154 goto out;
1155 }
1156
1157 if (reth_rkey(pkt) != res->read.rkey) {
1158 rc = RESPST_CLEANUP;
1159 goto out;
1160 }
1161
1162 res->cur_psn = pkt->psn;
1163 res->state = (pkt->psn == res->first_psn) ?
1164 rdatm_res_state_new :
1165 rdatm_res_state_replay;
1166 res->replay = 1;
1167
1168 /* Reset the resource, except length. */
1169 res->read.va_org = iova;
1170 res->read.va = iova;
1171 res->read.resid = resid;
1172
1173 /* Replay the RDMA read reply. */
1174 qp->resp.res = res;
1175 rc = RESPST_READ_REPLY;
1176 goto out;
1177 }
1178 } else {
1179 struct resp_res *res;
1180
1181 /* Find the operation in our list of responder resources. */
1182 res = find_resource(qp, pkt->psn);
1183 if (res) {
1184 res->replay = 1;
1185 res->cur_psn = pkt->psn;
1186 qp->resp.res = res;
1187 rc = RESPST_ATOMIC_REPLY;
1188 goto out;
1189 }
1190
1191 /* Resource not found. Class D error. Drop the request. */
1192 rc = RESPST_CLEANUP;
1193 goto out;
1194 }
1195 out:
1196 return rc;
1197 }
1198
1199 /* Process a class A or C. Both are treated the same in this implementation. */
do_class_ac_error(struct rxe_qp * qp,u8 syndrome,enum ib_wc_status status)1200 static void do_class_ac_error(struct rxe_qp *qp, u8 syndrome,
1201 enum ib_wc_status status)
1202 {
1203 qp->resp.aeth_syndrome = syndrome;
1204 qp->resp.status = status;
1205
1206 /* indicate that we should go through the ERROR state */
1207 qp->resp.goto_error = 1;
1208 }
1209
do_class_d1e_error(struct rxe_qp * qp)1210 static enum resp_states do_class_d1e_error(struct rxe_qp *qp)
1211 {
1212 /* UC */
1213 if (qp->srq) {
1214 /* Class E */
1215 qp->resp.drop_msg = 1;
1216 if (qp->resp.wqe) {
1217 qp->resp.status = IB_WC_REM_INV_REQ_ERR;
1218 return RESPST_COMPLETE;
1219 } else {
1220 return RESPST_CLEANUP;
1221 }
1222 } else {
1223 /* Class D1. This packet may be the start of a
1224 * new message and could be valid. The previous
1225 * message is invalid and ignored. reset the
1226 * recv wr to its original state
1227 */
1228 if (qp->resp.wqe) {
1229 qp->resp.wqe->dma.resid = qp->resp.wqe->dma.length;
1230 qp->resp.wqe->dma.cur_sge = 0;
1231 qp->resp.wqe->dma.sge_offset = 0;
1232 qp->resp.opcode = -1;
1233 }
1234
1235 if (qp->resp.mr) {
1236 rxe_put(qp->resp.mr);
1237 qp->resp.mr = NULL;
1238 }
1239
1240 return RESPST_CLEANUP;
1241 }
1242 }
1243
rxe_drain_req_pkts(struct rxe_qp * qp,bool notify)1244 static void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)
1245 {
1246 struct sk_buff *skb;
1247 struct rxe_queue *q = qp->rq.queue;
1248
1249 while ((skb = skb_dequeue(&qp->req_pkts))) {
1250 rxe_put(qp);
1251 kfree_skb(skb);
1252 ib_device_put(qp->ibqp.device);
1253 }
1254
1255 if (notify)
1256 return;
1257
1258 while (!qp->srq && q && queue_head(q, q->type))
1259 queue_advance_consumer(q, q->type);
1260 }
1261
rxe_responder(void * arg)1262 int rxe_responder(void *arg)
1263 {
1264 struct rxe_qp *qp = (struct rxe_qp *)arg;
1265 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
1266 enum resp_states state;
1267 struct rxe_pkt_info *pkt = NULL;
1268 int ret;
1269
1270 if (!rxe_get(qp))
1271 return -EAGAIN;
1272
1273 qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;
1274
1275 if (!qp->valid)
1276 goto exit;
1277
1278 switch (qp->resp.state) {
1279 case QP_STATE_RESET:
1280 state = RESPST_RESET;
1281 break;
1282
1283 default:
1284 state = RESPST_GET_REQ;
1285 break;
1286 }
1287
1288 while (1) {
1289 pr_debug("qp#%d state = %s\n", qp_num(qp),
1290 resp_state_name[state]);
1291 switch (state) {
1292 case RESPST_GET_REQ:
1293 state = get_req(qp, &pkt);
1294 break;
1295 case RESPST_CHK_PSN:
1296 state = check_psn(qp, pkt);
1297 break;
1298 case RESPST_CHK_OP_SEQ:
1299 state = check_op_seq(qp, pkt);
1300 break;
1301 case RESPST_CHK_OP_VALID:
1302 state = check_op_valid(qp, pkt);
1303 break;
1304 case RESPST_CHK_RESOURCE:
1305 state = check_resource(qp, pkt);
1306 break;
1307 case RESPST_CHK_LENGTH:
1308 state = check_length(qp, pkt);
1309 break;
1310 case RESPST_CHK_RKEY:
1311 state = check_rkey(qp, pkt);
1312 break;
1313 case RESPST_EXECUTE:
1314 state = execute(qp, pkt);
1315 break;
1316 case RESPST_COMPLETE:
1317 state = do_complete(qp, pkt);
1318 break;
1319 case RESPST_READ_REPLY:
1320 state = read_reply(qp, pkt);
1321 break;
1322 case RESPST_ATOMIC_REPLY:
1323 state = atomic_reply(qp, pkt);
1324 break;
1325 case RESPST_ACKNOWLEDGE:
1326 state = acknowledge(qp, pkt);
1327 break;
1328 case RESPST_CLEANUP:
1329 state = cleanup(qp, pkt);
1330 break;
1331 case RESPST_DUPLICATE_REQUEST:
1332 state = duplicate_request(qp, pkt);
1333 break;
1334 case RESPST_ERR_PSN_OUT_OF_SEQ:
1335 /* RC only - Class B. Drop packet. */
1336 send_ack(qp, AETH_NAK_PSN_SEQ_ERROR, qp->resp.psn);
1337 state = RESPST_CLEANUP;
1338 break;
1339
1340 case RESPST_ERR_TOO_MANY_RDMA_ATM_REQ:
1341 case RESPST_ERR_MISSING_OPCODE_FIRST:
1342 case RESPST_ERR_MISSING_OPCODE_LAST_C:
1343 case RESPST_ERR_UNSUPPORTED_OPCODE:
1344 case RESPST_ERR_MISALIGNED_ATOMIC:
1345 /* RC Only - Class C. */
1346 do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
1347 IB_WC_REM_INV_REQ_ERR);
1348 state = RESPST_COMPLETE;
1349 break;
1350
1351 case RESPST_ERR_MISSING_OPCODE_LAST_D1E:
1352 state = do_class_d1e_error(qp);
1353 break;
1354 case RESPST_ERR_RNR:
1355 if (qp_type(qp) == IB_QPT_RC) {
1356 rxe_counter_inc(rxe, RXE_CNT_SND_RNR);
1357 /* RC - class B */
1358 send_ack(qp, AETH_RNR_NAK |
1359 (~AETH_TYPE_MASK &
1360 qp->attr.min_rnr_timer),
1361 pkt->psn);
1362 } else {
1363 /* UD/UC - class D */
1364 qp->resp.drop_msg = 1;
1365 }
1366 state = RESPST_CLEANUP;
1367 break;
1368
1369 case RESPST_ERR_RKEY_VIOLATION:
1370 if (qp_type(qp) == IB_QPT_RC) {
1371 /* Class C */
1372 do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR,
1373 IB_WC_REM_ACCESS_ERR);
1374 state = RESPST_COMPLETE;
1375 } else {
1376 qp->resp.drop_msg = 1;
1377 if (qp->srq) {
1378 /* UC/SRQ Class D */
1379 qp->resp.status = IB_WC_REM_ACCESS_ERR;
1380 state = RESPST_COMPLETE;
1381 } else {
1382 /* UC/non-SRQ Class E. */
1383 state = RESPST_CLEANUP;
1384 }
1385 }
1386 break;
1387
1388 case RESPST_ERR_INVALIDATE_RKEY:
1389 /* RC - Class J. */
1390 qp->resp.goto_error = 1;
1391 qp->resp.status = IB_WC_REM_INV_REQ_ERR;
1392 state = RESPST_COMPLETE;
1393 break;
1394
1395 case RESPST_ERR_LENGTH:
1396 if (qp_type(qp) == IB_QPT_RC) {
1397 /* Class C */
1398 do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
1399 IB_WC_REM_INV_REQ_ERR);
1400 state = RESPST_COMPLETE;
1401 } else if (qp->srq) {
1402 /* UC/UD - class E */
1403 qp->resp.status = IB_WC_REM_INV_REQ_ERR;
1404 state = RESPST_COMPLETE;
1405 } else {
1406 /* UC/UD - class D */
1407 qp->resp.drop_msg = 1;
1408 state = RESPST_CLEANUP;
1409 }
1410 break;
1411
1412 case RESPST_ERR_MALFORMED_WQE:
1413 /* All, Class A. */
1414 do_class_ac_error(qp, AETH_NAK_REM_OP_ERR,
1415 IB_WC_LOC_QP_OP_ERR);
1416 state = RESPST_COMPLETE;
1417 break;
1418
1419 case RESPST_ERR_CQ_OVERFLOW:
1420 /* All - Class G */
1421 state = RESPST_ERROR;
1422 break;
1423
1424 case RESPST_DONE:
1425 if (qp->resp.goto_error) {
1426 state = RESPST_ERROR;
1427 break;
1428 }
1429
1430 goto done;
1431
1432 case RESPST_EXIT:
1433 if (qp->resp.goto_error) {
1434 state = RESPST_ERROR;
1435 break;
1436 }
1437
1438 goto exit;
1439
1440 case RESPST_RESET:
1441 rxe_drain_req_pkts(qp, false);
1442 qp->resp.wqe = NULL;
1443 goto exit;
1444
1445 case RESPST_ERROR:
1446 qp->resp.goto_error = 0;
1447 pr_debug("qp#%d moved to error state\n", qp_num(qp));
1448 rxe_qp_error(qp);
1449 goto exit;
1450
1451 default:
1452 WARN_ON_ONCE(1);
1453 }
1454 }
1455
1456 /* A non-zero return value will cause rxe_do_task to
1457 * exit its loop and end the tasklet. A zero return
1458 * will continue looping and return to rxe_responder
1459 */
1460 done:
1461 ret = 0;
1462 goto out;
1463 exit:
1464 ret = -EAGAIN;
1465 out:
1466 rxe_put(qp);
1467 return ret;
1468 }
1469
