Lines Matching refs:link
44 struct smc_link *link; member
55 static inline bool smc_wr_is_tx_pend(struct smc_link *link) in smc_wr_is_tx_pend() argument
57 if (find_first_bit(link->wr_tx_mask, link->wr_tx_cnt) != in smc_wr_is_tx_pend()
58 link->wr_tx_cnt) { in smc_wr_is_tx_pend()
65 int smc_wr_tx_wait_no_pending_sends(struct smc_link *link) in smc_wr_tx_wait_no_pending_sends() argument
67 if (wait_event_timeout(link->wr_tx_wait, !smc_wr_is_tx_pend(link), in smc_wr_tx_wait_no_pending_sends()
74 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) in smc_wr_tx_find_pending_index() argument
78 for (i = 0; i < link->wr_tx_cnt; i++) { in smc_wr_tx_find_pending_index()
79 if (link->wr_tx_pends[i].wr_id == wr_id) in smc_wr_tx_find_pending_index()
82 return link->wr_tx_cnt; in smc_wr_tx_find_pending_index()
88 struct smc_link *link; in smc_wr_tx_process_cqe() local
92 link = wc->qp->qp_context; in smc_wr_tx_process_cqe()
96 link->wr_reg_state = FAILED; in smc_wr_tx_process_cqe()
98 link->wr_reg_state = CONFIRMED; in smc_wr_tx_process_cqe()
99 smc_wr_wakeup_reg_wait(link); in smc_wr_tx_process_cqe()
103 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); in smc_wr_tx_process_cqe()
104 if (pnd_snd_idx == link->wr_tx_cnt) in smc_wr_tx_process_cqe()
106 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; in smc_wr_tx_process_cqe()
107 if (link->wr_tx_pends[pnd_snd_idx].compl_requested) in smc_wr_tx_process_cqe()
108 complete(&link->wr_tx_compl[pnd_snd_idx]); in smc_wr_tx_process_cqe()
109 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); in smc_wr_tx_process_cqe()
111 memset(&link->wr_tx_pends[pnd_snd_idx], 0, in smc_wr_tx_process_cqe()
112 sizeof(link->wr_tx_pends[pnd_snd_idx])); in smc_wr_tx_process_cqe()
113 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, in smc_wr_tx_process_cqe()
114 sizeof(link->wr_tx_bufs[pnd_snd_idx])); in smc_wr_tx_process_cqe()
115 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) in smc_wr_tx_process_cqe()
118 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { in smc_wr_tx_process_cqe()
120 memset(&link->wr_tx_pends[i], 0, in smc_wr_tx_process_cqe()
121 sizeof(link->wr_tx_pends[i])); in smc_wr_tx_process_cqe()
122 memset(&link->wr_tx_bufs[i], 0, in smc_wr_tx_process_cqe()
123 sizeof(link->wr_tx_bufs[i])); in smc_wr_tx_process_cqe()
124 clear_bit(i, link->wr_tx_mask); in smc_wr_tx_process_cqe()
127 smcr_link_down_cond_sched(link); in smc_wr_tx_process_cqe()
130 pnd_snd.handler(&pnd_snd.priv, link, wc->status); in smc_wr_tx_process_cqe()
131 wake_up(&link->wr_tx_wait); in smc_wr_tx_process_cqe()
169 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) in smc_wr_tx_get_free_slot_index() argument
171 *idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot_index()
172 if (!smc_link_usable(link)) in smc_wr_tx_get_free_slot_index()
174 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { in smc_wr_tx_get_free_slot_index()
175 if (!test_and_set_bit(*idx, link->wr_tx_mask)) in smc_wr_tx_get_free_slot_index()
178 *idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot_index()
193 int smc_wr_tx_get_free_slot(struct smc_link *link, in smc_wr_tx_get_free_slot() argument
199 struct smc_link_group *lgr = smc_get_lgr(link); in smc_wr_tx_get_free_slot()
201 u32 idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot()
209 rc = smc_wr_tx_get_free_slot_index(link, &idx); in smc_wr_tx_get_free_slot()
214 link->wr_tx_wait, in smc_wr_tx_get_free_slot()
215 !smc_link_usable(link) || in smc_wr_tx_get_free_slot()
217 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), in smc_wr_tx_get_free_slot()
221 smcr_link_down_cond_sched(link); in smc_wr_tx_get_free_slot()
224 if (idx == link->wr_tx_cnt) in smc_wr_tx_get_free_slot()
227 wr_id = smc_wr_tx_get_next_wr_id(link); in smc_wr_tx_get_free_slot()
228 wr_pend = &link->wr_tx_pends[idx]; in smc_wr_tx_get_free_slot()
231 wr_pend->link = link; in smc_wr_tx_get_free_slot()
233 wr_ib = &link->wr_tx_ibs[idx]; in smc_wr_tx_get_free_slot()
235 *wr_buf = &link->wr_tx_bufs[idx]; in smc_wr_tx_get_free_slot()
237 *wr_rdma_buf = &link->wr_tx_rdmas[idx]; in smc_wr_tx_get_free_slot()
242 int smc_wr_tx_put_slot(struct smc_link *link, in smc_wr_tx_put_slot() argument
248 if (pend->idx < link->wr_tx_cnt) { in smc_wr_tx_put_slot()
252 memset(&link->wr_tx_pends[idx], 0, in smc_wr_tx_put_slot()
253 sizeof(link->wr_tx_pends[idx])); in smc_wr_tx_put_slot()
254 memset(&link->wr_tx_bufs[idx], 0, in smc_wr_tx_put_slot()
255 sizeof(link->wr_tx_bufs[idx])); in smc_wr_tx_put_slot()
256 test_and_clear_bit(idx, link->wr_tx_mask); in smc_wr_tx_put_slot()
257 wake_up(&link->wr_tx_wait); in smc_wr_tx_put_slot()
267 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) in smc_wr_tx_send() argument
272 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_tx_send()
275 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL); in smc_wr_tx_send()
277 smc_wr_tx_put_slot(link, priv); in smc_wr_tx_send()
278 smcr_link_down_cond_sched(link); in smc_wr_tx_send()
287 int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, in smc_wr_tx_send_wait() argument
295 init_completion(&link->wr_tx_compl[pend->idx]); in smc_wr_tx_send_wait()
297 rc = smc_wr_tx_send(link, priv); in smc_wr_tx_send_wait()
302 &link->wr_tx_compl[pend->idx], timeout); in smc_wr_tx_send_wait()
311 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) in smc_wr_reg_send() argument
315 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_reg_send()
317 link->wr_reg_state = POSTED; in smc_wr_reg_send()
318 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; in smc_wr_reg_send()
319 link->wr_reg.mr = mr; in smc_wr_reg_send()
320 link->wr_reg.key = mr->rkey; in smc_wr_reg_send()
321 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL); in smc_wr_reg_send()
325 atomic_inc(&link->wr_reg_refcnt); in smc_wr_reg_send()
326 rc = wait_event_interruptible_timeout(link->wr_reg_wait, in smc_wr_reg_send()
327 (link->wr_reg_state != POSTED), in smc_wr_reg_send()
329 if (atomic_dec_and_test(&link->wr_reg_refcnt)) in smc_wr_reg_send()
330 wake_up_all(&link->wr_reg_wait); in smc_wr_reg_send()
333 smcr_link_down_cond_sched(link); in smc_wr_reg_send()
338 switch (link->wr_reg_state) { in smc_wr_reg_send()
352 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type, in smc_wr_tx_dismiss_slots() argument
361 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { in smc_wr_tx_dismiss_slots()
362 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i]; in smc_wr_tx_dismiss_slots()
365 tx_pend = &link->wr_tx_pends[i].priv; in smc_wr_tx_dismiss_slots()
397 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; in smc_wr_rx_demultiplex() local
406 index = do_div(temp_wr_id, link->wr_rx_cnt); in smc_wr_rx_demultiplex()
407 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; in smc_wr_rx_demultiplex()
416 struct smc_link *link; in smc_wr_rx_process_cqes() local
420 link = wc[i].qp->qp_context; in smc_wr_rx_process_cqes()
422 link->wr_rx_tstamp = jiffies; in smc_wr_rx_process_cqes()
424 smc_wr_rx_post(link); /* refill WR RX */ in smc_wr_rx_process_cqes()
431 smcr_link_down_cond_sched(link); in smc_wr_rx_process_cqes()
434 smc_wr_rx_post(link); /* refill WR RX */ in smc_wr_rx_process_cqes()
473 int smc_wr_rx_post_init(struct smc_link *link) in smc_wr_rx_post_init() argument
478 for (i = 0; i < link->wr_rx_cnt; i++) in smc_wr_rx_post_init()
479 rc = smc_wr_rx_post(link); in smc_wr_rx_post_init()
622 int smc_wr_alloc_link_mem(struct smc_link *link) in smc_wr_alloc_link_mem() argument
625 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); in smc_wr_alloc_link_mem()
626 if (!link->wr_tx_bufs) in smc_wr_alloc_link_mem()
628 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, in smc_wr_alloc_link_mem()
630 if (!link->wr_rx_bufs) in smc_wr_alloc_link_mem()
632 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), in smc_wr_alloc_link_mem()
634 if (!link->wr_tx_ibs) in smc_wr_alloc_link_mem()
636 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, in smc_wr_alloc_link_mem()
637 sizeof(link->wr_rx_ibs[0]), in smc_wr_alloc_link_mem()
639 if (!link->wr_rx_ibs) in smc_wr_alloc_link_mem()
641 link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
642 sizeof(link->wr_tx_rdmas[0]), in smc_wr_alloc_link_mem()
644 if (!link->wr_tx_rdmas) in smc_wr_alloc_link_mem()
646 link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
647 sizeof(link->wr_tx_rdma_sges[0]), in smc_wr_alloc_link_mem()
649 if (!link->wr_tx_rdma_sges) in smc_wr_alloc_link_mem()
651 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), in smc_wr_alloc_link_mem()
653 if (!link->wr_tx_sges) in smc_wr_alloc_link_mem()
655 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, in smc_wr_alloc_link_mem()
656 sizeof(link->wr_rx_sges[0]), in smc_wr_alloc_link_mem()
658 if (!link->wr_rx_sges) in smc_wr_alloc_link_mem()
660 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT), in smc_wr_alloc_link_mem()
661 sizeof(*link->wr_tx_mask), in smc_wr_alloc_link_mem()
663 if (!link->wr_tx_mask) in smc_wr_alloc_link_mem()
665 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
666 sizeof(link->wr_tx_pends[0]), in smc_wr_alloc_link_mem()
668 if (!link->wr_tx_pends) in smc_wr_alloc_link_mem()
670 link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
671 sizeof(link->wr_tx_compl[0]), in smc_wr_alloc_link_mem()
673 if (!link->wr_tx_compl) in smc_wr_alloc_link_mem()
678 kfree(link->wr_tx_pends); in smc_wr_alloc_link_mem()
680 kfree(link->wr_tx_mask); in smc_wr_alloc_link_mem()
682 kfree(link->wr_rx_sges); in smc_wr_alloc_link_mem()
684 kfree(link->wr_tx_sges); in smc_wr_alloc_link_mem()
686 kfree(link->wr_tx_rdma_sges); in smc_wr_alloc_link_mem()
688 kfree(link->wr_tx_rdmas); in smc_wr_alloc_link_mem()
690 kfree(link->wr_rx_ibs); in smc_wr_alloc_link_mem()
692 kfree(link->wr_tx_ibs); in smc_wr_alloc_link_mem()
694 kfree(link->wr_rx_bufs); in smc_wr_alloc_link_mem()
696 kfree(link->wr_tx_bufs); in smc_wr_alloc_link_mem()