1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Work Requests exploiting Infiniband API
6 *
7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
8 * are submitted to either RC SQ or RC RQ respectively
9 * (reliably connected send/receive queue)
10 * and become work queue entries (WQEs).
11 * While an SQ WR/WQE is pending, we track it until transmission completion.
12 * Through a send or receive completion queue (CQ) respectively,
13 * we get completion queue entries (CQEs) [aka work completions (WCs)].
14 * Since the CQ callback is called from IRQ context, we split work by using
15 * bottom halves implemented by tasklets.
16 *
17 * SMC uses this to exchange LLC (link layer control)
18 * and CDC (connection data control) messages.
19 *
20 * Copyright IBM Corp. 2016
21 *
22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com>
23 */
24
25 #include <linux/atomic.h>
26 #include <linux/hashtable.h>
27 #include <linux/wait.h>
28 #include <rdma/ib_verbs.h>
29 #include <asm/div64.h>
30
31 #include "smc.h"
32 #include "smc_wr.h"
33
34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */
35
36 #define SMC_WR_RX_HASH_BITS 4
37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
39
40 struct smc_wr_tx_pend { /* control data for a pending send request */
41 u64 wr_id; /* work request id sent */
42 smc_wr_tx_handler handler;
43 enum ib_wc_status wc_status; /* CQE status */
44 struct smc_link *link;
45 u32 idx;
46 struct smc_wr_tx_pend_priv priv;
47 };
48
49 /******************************** send queue *********************************/
50
51 /*------------------------------- completion --------------------------------*/
52
smc_wr_tx_find_pending_index(struct smc_link * link,u64 wr_id)53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
54 {
55 u32 i;
56
57 for (i = 0; i < link->wr_tx_cnt; i++) {
58 if (link->wr_tx_pends[i].wr_id == wr_id)
59 return i;
60 }
61 return link->wr_tx_cnt;
62 }
63
smc_wr_tx_process_cqe(struct ib_wc * wc)64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
65 {
66 struct smc_wr_tx_pend pnd_snd;
67 struct smc_link *link;
68 u32 pnd_snd_idx;
69 int i;
70
71 link = wc->qp->qp_context;
72
73 if (wc->opcode == IB_WC_REG_MR) {
74 if (wc->status)
75 link->wr_reg_state = FAILED;
76 else
77 link->wr_reg_state = CONFIRMED;
78 wake_up(&link->wr_reg_wait);
79 return;
80 }
81
82 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
83 if (pnd_snd_idx == link->wr_tx_cnt)
84 return;
85 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
86 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
87 /* clear the full struct smc_wr_tx_pend including .priv */
88 memset(&link->wr_tx_pends[pnd_snd_idx], 0,
89 sizeof(link->wr_tx_pends[pnd_snd_idx]));
90 memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
91 sizeof(link->wr_tx_bufs[pnd_snd_idx]));
92 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
93 return;
94 if (wc->status) {
95 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
96 /* clear full struct smc_wr_tx_pend including .priv */
97 memset(&link->wr_tx_pends[i], 0,
98 sizeof(link->wr_tx_pends[i]));
99 memset(&link->wr_tx_bufs[i], 0,
100 sizeof(link->wr_tx_bufs[i]));
101 clear_bit(i, link->wr_tx_mask);
102 }
103 /* terminate connections of this link group abnormally */
104 smc_lgr_terminate(smc_get_lgr(link));
105 }
106 if (pnd_snd.handler)
107 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
108 wake_up(&link->wr_tx_wait);
109 }
110
smc_wr_tx_tasklet_fn(unsigned long data)111 static void smc_wr_tx_tasklet_fn(unsigned long data)
112 {
113 struct smc_ib_device *dev = (struct smc_ib_device *)data;
114 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
115 int i = 0, rc;
116 int polled = 0;
117
118 again:
119 polled++;
120 do {
121 memset(&wc, 0, sizeof(wc));
122 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
123 if (polled == 1) {
124 ib_req_notify_cq(dev->roce_cq_send,
125 IB_CQ_NEXT_COMP |
126 IB_CQ_REPORT_MISSED_EVENTS);
127 }
128 if (!rc)
129 break;
130 for (i = 0; i < rc; i++)
131 smc_wr_tx_process_cqe(&wc[i]);
132 } while (rc > 0);
133 if (polled == 1)
134 goto again;
135 }
136
smc_wr_tx_cq_handler(struct ib_cq * ib_cq,void * cq_context)137 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
138 {
139 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
140
141 tasklet_schedule(&dev->send_tasklet);
142 }
143
144 /*---------------------------- request submission ---------------------------*/
145
smc_wr_tx_get_free_slot_index(struct smc_link * link,u32 * idx)146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
147 {
148 *idx = link->wr_tx_cnt;
149 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
150 if (!test_and_set_bit(*idx, link->wr_tx_mask))
151 return 0;
152 }
153 *idx = link->wr_tx_cnt;
154 return -EBUSY;
155 }
156
157 /**
158 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
159 * and sets info for pending transmit tracking
160 * @link: Pointer to smc_link used to later send the message.
161 * @handler: Send completion handler function pointer.
162 * @wr_buf: Out value returns pointer to message buffer.
163 * @wr_pend_priv: Out value returns pointer serving as handler context.
164 *
165 * Return: 0 on success, or -errno on error.
166 */
smc_wr_tx_get_free_slot(struct smc_link * link,smc_wr_tx_handler handler,struct smc_wr_buf ** wr_buf,struct smc_wr_tx_pend_priv ** wr_pend_priv)167 int smc_wr_tx_get_free_slot(struct smc_link *link,
168 smc_wr_tx_handler handler,
169 struct smc_wr_buf **wr_buf,
170 struct smc_wr_tx_pend_priv **wr_pend_priv)
171 {
172 struct smc_wr_tx_pend *wr_pend;
173 u32 idx = link->wr_tx_cnt;
174 struct ib_send_wr *wr_ib;
175 u64 wr_id;
176 int rc;
177
178 *wr_buf = NULL;
179 *wr_pend_priv = NULL;
180 if (in_softirq()) {
181 rc = smc_wr_tx_get_free_slot_index(link, &idx);
182 if (rc)
183 return rc;
184 } else {
185 rc = wait_event_timeout(
186 link->wr_tx_wait,
187 link->state == SMC_LNK_INACTIVE ||
188 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
189 SMC_WR_TX_WAIT_FREE_SLOT_TIME);
190 if (!rc) {
191 /* timeout - terminate connections */
192 smc_lgr_terminate(smc_get_lgr(link));
193 return -EPIPE;
194 }
195 if (idx == link->wr_tx_cnt)
196 return -EPIPE;
197 }
198 wr_id = smc_wr_tx_get_next_wr_id(link);
199 wr_pend = &link->wr_tx_pends[idx];
200 wr_pend->wr_id = wr_id;
201 wr_pend->handler = handler;
202 wr_pend->link = link;
203 wr_pend->idx = idx;
204 wr_ib = &link->wr_tx_ibs[idx];
205 wr_ib->wr_id = wr_id;
206 *wr_buf = &link->wr_tx_bufs[idx];
207 *wr_pend_priv = &wr_pend->priv;
208 return 0;
209 }
210
smc_wr_tx_put_slot(struct smc_link * link,struct smc_wr_tx_pend_priv * wr_pend_priv)211 int smc_wr_tx_put_slot(struct smc_link *link,
212 struct smc_wr_tx_pend_priv *wr_pend_priv)
213 {
214 struct smc_wr_tx_pend *pend;
215
216 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
217 if (pend->idx < link->wr_tx_cnt) {
218 /* clear the full struct smc_wr_tx_pend including .priv */
219 memset(&link->wr_tx_pends[pend->idx], 0,
220 sizeof(link->wr_tx_pends[pend->idx]));
221 memset(&link->wr_tx_bufs[pend->idx], 0,
222 sizeof(link->wr_tx_bufs[pend->idx]));
223 test_and_clear_bit(pend->idx, link->wr_tx_mask);
224 return 1;
225 }
226
227 return 0;
228 }
229
230 /* Send prepared WR slot via ib_post_send.
231 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
232 */
smc_wr_tx_send(struct smc_link * link,struct smc_wr_tx_pend_priv * priv)233 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
234 {
235 struct smc_wr_tx_pend *pend;
236 int rc;
237
238 ib_req_notify_cq(link->smcibdev->roce_cq_send,
239 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
240 pend = container_of(priv, struct smc_wr_tx_pend, priv);
241 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
242 if (rc) {
243 smc_wr_tx_put_slot(link, priv);
244 smc_lgr_terminate(smc_get_lgr(link));
245 }
246 return rc;
247 }
248
249 /* Register a memory region and wait for result. */
smc_wr_reg_send(struct smc_link * link,struct ib_mr * mr)250 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
251 {
252 int rc;
253
254 ib_req_notify_cq(link->smcibdev->roce_cq_send,
255 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
256 link->wr_reg_state = POSTED;
257 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
258 link->wr_reg.mr = mr;
259 link->wr_reg.key = mr->rkey;
260 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
261 if (rc)
262 return rc;
263
264 rc = wait_event_interruptible_timeout(link->wr_reg_wait,
265 (link->wr_reg_state != POSTED),
266 SMC_WR_REG_MR_WAIT_TIME);
267 if (!rc) {
268 /* timeout - terminate connections */
269 smc_lgr_terminate(smc_get_lgr(link));
270 return -EPIPE;
271 }
272 if (rc == -ERESTARTSYS)
273 return -EINTR;
274 switch (link->wr_reg_state) {
275 case CONFIRMED:
276 rc = 0;
277 break;
278 case FAILED:
279 rc = -EIO;
280 break;
281 case POSTED:
282 rc = -EPIPE;
283 break;
284 }
285 return rc;
286 }
287
smc_wr_tx_dismiss_slots(struct smc_link * link,u8 wr_tx_hdr_type,smc_wr_tx_filter filter,smc_wr_tx_dismisser dismisser,unsigned long data)288 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
289 smc_wr_tx_filter filter,
290 smc_wr_tx_dismisser dismisser,
291 unsigned long data)
292 {
293 struct smc_wr_tx_pend_priv *tx_pend;
294 struct smc_wr_rx_hdr *wr_tx;
295 int i;
296
297 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
298 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
299 if (wr_tx->type != wr_tx_hdr_type)
300 continue;
301 tx_pend = &link->wr_tx_pends[i].priv;
302 if (filter(tx_pend, data))
303 dismisser(tx_pend);
304 }
305 }
306
307 /****************************** receive queue ********************************/
308
smc_wr_rx_register_handler(struct smc_wr_rx_handler * handler)309 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
310 {
311 struct smc_wr_rx_handler *h_iter;
312 int rc = 0;
313
314 spin_lock(&smc_wr_rx_hash_lock);
315 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
316 if (h_iter->type == handler->type) {
317 rc = -EEXIST;
318 goto out_unlock;
319 }
320 }
321 hash_add(smc_wr_rx_hash, &handler->list, handler->type);
322 out_unlock:
323 spin_unlock(&smc_wr_rx_hash_lock);
324 return rc;
325 }
326
327 /* Demultiplex a received work request based on the message type to its handler.
328 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
329 * and not being modified any more afterwards so we don't need to lock it.
330 */
smc_wr_rx_demultiplex(struct ib_wc * wc)331 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
332 {
333 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
334 struct smc_wr_rx_handler *handler;
335 struct smc_wr_rx_hdr *wr_rx;
336 u64 temp_wr_id;
337 u32 index;
338
339 if (wc->byte_len < sizeof(*wr_rx))
340 return; /* short message */
341 temp_wr_id = wc->wr_id;
342 index = do_div(temp_wr_id, link->wr_rx_cnt);
343 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
344 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
345 if (handler->type == wr_rx->type)
346 handler->handler(wc, wr_rx);
347 }
348 }
349
smc_wr_rx_process_cqes(struct ib_wc wc[],int num)350 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
351 {
352 struct smc_link *link;
353 int i;
354
355 for (i = 0; i < num; i++) {
356 link = wc[i].qp->qp_context;
357 if (wc[i].status == IB_WC_SUCCESS) {
358 link->wr_rx_tstamp = jiffies;
359 smc_wr_rx_demultiplex(&wc[i]);
360 smc_wr_rx_post(link); /* refill WR RX */
361 } else {
362 /* handle status errors */
363 switch (wc[i].status) {
364 case IB_WC_RETRY_EXC_ERR:
365 case IB_WC_RNR_RETRY_EXC_ERR:
366 case IB_WC_WR_FLUSH_ERR:
367 /* terminate connections of this link group
368 * abnormally
369 */
370 smc_lgr_terminate(smc_get_lgr(link));
371 break;
372 default:
373 smc_wr_rx_post(link); /* refill WR RX */
374 break;
375 }
376 }
377 }
378 }
379
smc_wr_rx_tasklet_fn(unsigned long data)380 static void smc_wr_rx_tasklet_fn(unsigned long data)
381 {
382 struct smc_ib_device *dev = (struct smc_ib_device *)data;
383 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
384 int polled = 0;
385 int rc;
386
387 again:
388 polled++;
389 do {
390 memset(&wc, 0, sizeof(wc));
391 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
392 if (polled == 1) {
393 ib_req_notify_cq(dev->roce_cq_recv,
394 IB_CQ_SOLICITED_MASK
395 | IB_CQ_REPORT_MISSED_EVENTS);
396 }
397 if (!rc)
398 break;
399 smc_wr_rx_process_cqes(&wc[0], rc);
400 } while (rc > 0);
401 if (polled == 1)
402 goto again;
403 }
404
smc_wr_rx_cq_handler(struct ib_cq * ib_cq,void * cq_context)405 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
406 {
407 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
408
409 tasklet_schedule(&dev->recv_tasklet);
410 }
411
smc_wr_rx_post_init(struct smc_link * link)412 int smc_wr_rx_post_init(struct smc_link *link)
413 {
414 u32 i;
415 int rc = 0;
416
417 for (i = 0; i < link->wr_rx_cnt; i++)
418 rc = smc_wr_rx_post(link);
419 return rc;
420 }
421
422 /***************************** init, exit, misc ******************************/
423
smc_wr_remember_qp_attr(struct smc_link * lnk)424 void smc_wr_remember_qp_attr(struct smc_link *lnk)
425 {
426 struct ib_qp_attr *attr = &lnk->qp_attr;
427 struct ib_qp_init_attr init_attr;
428
429 memset(attr, 0, sizeof(*attr));
430 memset(&init_attr, 0, sizeof(init_attr));
431 ib_query_qp(lnk->roce_qp, attr,
432 IB_QP_STATE |
433 IB_QP_CUR_STATE |
434 IB_QP_PKEY_INDEX |
435 IB_QP_PORT |
436 IB_QP_QKEY |
437 IB_QP_AV |
438 IB_QP_PATH_MTU |
439 IB_QP_TIMEOUT |
440 IB_QP_RETRY_CNT |
441 IB_QP_RNR_RETRY |
442 IB_QP_RQ_PSN |
443 IB_QP_ALT_PATH |
444 IB_QP_MIN_RNR_TIMER |
445 IB_QP_SQ_PSN |
446 IB_QP_PATH_MIG_STATE |
447 IB_QP_CAP |
448 IB_QP_DEST_QPN,
449 &init_attr);
450
451 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
452 lnk->qp_attr.cap.max_send_wr);
453 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
454 lnk->qp_attr.cap.max_recv_wr);
455 }
456
smc_wr_init_sge(struct smc_link * lnk)457 static void smc_wr_init_sge(struct smc_link *lnk)
458 {
459 u32 i;
460
461 for (i = 0; i < lnk->wr_tx_cnt; i++) {
462 lnk->wr_tx_sges[i].addr =
463 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
464 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
465 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
466 lnk->wr_tx_ibs[i].next = NULL;
467 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
468 lnk->wr_tx_ibs[i].num_sge = 1;
469 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
470 lnk->wr_tx_ibs[i].send_flags =
471 IB_SEND_SIGNALED | IB_SEND_SOLICITED;
472 }
473 for (i = 0; i < lnk->wr_rx_cnt; i++) {
474 lnk->wr_rx_sges[i].addr =
475 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
476 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
477 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
478 lnk->wr_rx_ibs[i].next = NULL;
479 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
480 lnk->wr_rx_ibs[i].num_sge = 1;
481 }
482 lnk->wr_reg.wr.next = NULL;
483 lnk->wr_reg.wr.num_sge = 0;
484 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
485 lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
486 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
487 }
488
smc_wr_free_link(struct smc_link * lnk)489 void smc_wr_free_link(struct smc_link *lnk)
490 {
491 struct ib_device *ibdev;
492
493 memset(lnk->wr_tx_mask, 0,
494 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
495
496 if (!lnk->smcibdev)
497 return;
498 ibdev = lnk->smcibdev->ibdev;
499
500 if (lnk->wr_rx_dma_addr) {
501 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
502 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
503 DMA_FROM_DEVICE);
504 lnk->wr_rx_dma_addr = 0;
505 }
506 if (lnk->wr_tx_dma_addr) {
507 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
508 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
509 DMA_TO_DEVICE);
510 lnk->wr_tx_dma_addr = 0;
511 }
512 }
513
smc_wr_free_link_mem(struct smc_link * lnk)514 void smc_wr_free_link_mem(struct smc_link *lnk)
515 {
516 kfree(lnk->wr_tx_pends);
517 lnk->wr_tx_pends = NULL;
518 kfree(lnk->wr_tx_mask);
519 lnk->wr_tx_mask = NULL;
520 kfree(lnk->wr_tx_sges);
521 lnk->wr_tx_sges = NULL;
522 kfree(lnk->wr_rx_sges);
523 lnk->wr_rx_sges = NULL;
524 kfree(lnk->wr_rx_ibs);
525 lnk->wr_rx_ibs = NULL;
526 kfree(lnk->wr_tx_ibs);
527 lnk->wr_tx_ibs = NULL;
528 kfree(lnk->wr_tx_bufs);
529 lnk->wr_tx_bufs = NULL;
530 kfree(lnk->wr_rx_bufs);
531 lnk->wr_rx_bufs = NULL;
532 }
533
smc_wr_alloc_link_mem(struct smc_link * link)534 int smc_wr_alloc_link_mem(struct smc_link *link)
535 {
536 /* allocate link related memory */
537 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
538 if (!link->wr_tx_bufs)
539 goto no_mem;
540 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
541 GFP_KERNEL);
542 if (!link->wr_rx_bufs)
543 goto no_mem_wr_tx_bufs;
544 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
545 GFP_KERNEL);
546 if (!link->wr_tx_ibs)
547 goto no_mem_wr_rx_bufs;
548 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
549 sizeof(link->wr_rx_ibs[0]),
550 GFP_KERNEL);
551 if (!link->wr_rx_ibs)
552 goto no_mem_wr_tx_ibs;
553 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
554 GFP_KERNEL);
555 if (!link->wr_tx_sges)
556 goto no_mem_wr_rx_ibs;
557 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
558 sizeof(link->wr_rx_sges[0]),
559 GFP_KERNEL);
560 if (!link->wr_rx_sges)
561 goto no_mem_wr_tx_sges;
562 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
563 sizeof(*link->wr_tx_mask),
564 GFP_KERNEL);
565 if (!link->wr_tx_mask)
566 goto no_mem_wr_rx_sges;
567 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
568 sizeof(link->wr_tx_pends[0]),
569 GFP_KERNEL);
570 if (!link->wr_tx_pends)
571 goto no_mem_wr_tx_mask;
572 return 0;
573
574 no_mem_wr_tx_mask:
575 kfree(link->wr_tx_mask);
576 no_mem_wr_rx_sges:
577 kfree(link->wr_rx_sges);
578 no_mem_wr_tx_sges:
579 kfree(link->wr_tx_sges);
580 no_mem_wr_rx_ibs:
581 kfree(link->wr_rx_ibs);
582 no_mem_wr_tx_ibs:
583 kfree(link->wr_tx_ibs);
584 no_mem_wr_rx_bufs:
585 kfree(link->wr_rx_bufs);
586 no_mem_wr_tx_bufs:
587 kfree(link->wr_tx_bufs);
588 no_mem:
589 return -ENOMEM;
590 }
591
smc_wr_remove_dev(struct smc_ib_device * smcibdev)592 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
593 {
594 tasklet_kill(&smcibdev->recv_tasklet);
595 tasklet_kill(&smcibdev->send_tasklet);
596 }
597
smc_wr_add_dev(struct smc_ib_device * smcibdev)598 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
599 {
600 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
601 (unsigned long)smcibdev);
602 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
603 (unsigned long)smcibdev);
604 }
605
smc_wr_create_link(struct smc_link * lnk)606 int smc_wr_create_link(struct smc_link *lnk)
607 {
608 struct ib_device *ibdev = lnk->smcibdev->ibdev;
609 int rc = 0;
610
611 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
612 lnk->wr_rx_id = 0;
613 lnk->wr_rx_dma_addr = ib_dma_map_single(
614 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
615 DMA_FROM_DEVICE);
616 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
617 lnk->wr_rx_dma_addr = 0;
618 rc = -EIO;
619 goto out;
620 }
621 lnk->wr_tx_dma_addr = ib_dma_map_single(
622 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
623 DMA_TO_DEVICE);
624 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
625 rc = -EIO;
626 goto dma_unmap;
627 }
628 smc_wr_init_sge(lnk);
629 memset(lnk->wr_tx_mask, 0,
630 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
631 init_waitqueue_head(&lnk->wr_tx_wait);
632 init_waitqueue_head(&lnk->wr_reg_wait);
633 return rc;
634
635 dma_unmap:
636 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
637 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
638 DMA_FROM_DEVICE);
639 lnk->wr_rx_dma_addr = 0;
640 out:
641 return rc;
642 }
643
