17  * SMC uses this to exchange LLC (link layer control)
44 	struct smc_link		*link;  member
53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)  in smc_wr_tx_find_pending_index()  argument
57 	for (i = 0; i < link->wr_tx_cnt; i++) {  in smc_wr_tx_find_pending_index()
58 		if (link->wr_tx_pends[i].wr_id == wr_id)  in smc_wr_tx_find_pending_index()
61 	return link->wr_tx_cnt;  in smc_wr_tx_find_pending_index()
67 	struct smc_link *link;  in smc_wr_tx_process_cqe()  local
71 	link = wc->qp->qp_context;  in smc_wr_tx_process_cqe()
75 			link->wr_reg_state = FAILED;  in smc_wr_tx_process_cqe()
77 			link->wr_reg_state = CONFIRMED;  in smc_wr_tx_process_cqe()
78 		wake_up(&link->wr_reg_wait);  in smc_wr_tx_process_cqe()
82 	pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);  in smc_wr_tx_process_cqe()
83 	if (pnd_snd_idx == link->wr_tx_cnt)  in smc_wr_tx_process_cqe()
85 	link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;  in smc_wr_tx_process_cqe()
86 	memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));  in smc_wr_tx_process_cqe()
88 	memset(&link->wr_tx_pends[pnd_snd_idx], 0,  in smc_wr_tx_process_cqe()
89 	       sizeof(link->wr_tx_pends[pnd_snd_idx]));  in smc_wr_tx_process_cqe()
90 	memset(&link->wr_tx_bufs[pnd_snd_idx], 0,  in smc_wr_tx_process_cqe()
91 	       sizeof(link->wr_tx_bufs[pnd_snd_idx]));  in smc_wr_tx_process_cqe()
92 	if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))  in smc_wr_tx_process_cqe()
95 		for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {  in smc_wr_tx_process_cqe()
97 			memset(&link->wr_tx_pends[i], 0,  in smc_wr_tx_process_cqe()
98 			       sizeof(link->wr_tx_pends[i]));  in smc_wr_tx_process_cqe()
99 			memset(&link->wr_tx_bufs[i], 0,  in smc_wr_tx_process_cqe()
100 			       sizeof(link->wr_tx_bufs[i]));  in smc_wr_tx_process_cqe()
101 			clear_bit(i, link->wr_tx_mask);  in smc_wr_tx_process_cqe()
103 		/* terminate connections of this link group abnormally */  in smc_wr_tx_process_cqe()
104 		smc_lgr_terminate(smc_get_lgr(link));  in smc_wr_tx_process_cqe()
107 		pnd_snd.handler(&pnd_snd.priv, link, wc->status);  in smc_wr_tx_process_cqe()
108 	wake_up(&link->wr_tx_wait);  in smc_wr_tx_process_cqe()
146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)  in smc_wr_tx_get_free_slot_index()  argument
148 	*idx = link->wr_tx_cnt;  in smc_wr_tx_get_free_slot_index()
149 	for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {  in smc_wr_tx_get_free_slot_index()
150 		if (!test_and_set_bit(*idx, link->wr_tx_mask))  in smc_wr_tx_get_free_slot_index()
153 	*idx = link->wr_tx_cnt;  in smc_wr_tx_get_free_slot_index()
160  * @link:		Pointer to smc_link used to later send the message.
168 int smc_wr_tx_get_free_slot(struct smc_link *link,  in smc_wr_tx_get_free_slot()  argument
175 	u32 idx = link->wr_tx_cnt;  in smc_wr_tx_get_free_slot()
183 		rc = smc_wr_tx_get_free_slot_index(link, &idx);  in smc_wr_tx_get_free_slot()
188 			link->wr_tx_wait,  in smc_wr_tx_get_free_slot()
189 			link->state == SMC_LNK_INACTIVE ||  in smc_wr_tx_get_free_slot()
190 			(smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),  in smc_wr_tx_get_free_slot()
194 			smc_lgr_terminate(smc_get_lgr(link));  in smc_wr_tx_get_free_slot()
197 		if (idx == link->wr_tx_cnt)  in smc_wr_tx_get_free_slot()
200 	wr_id = smc_wr_tx_get_next_wr_id(link);  in smc_wr_tx_get_free_slot()
201 	wr_pend = &link->wr_tx_pends[idx];  in smc_wr_tx_get_free_slot()
204 	wr_pend->link = link;  in smc_wr_tx_get_free_slot()
206 	wr_ib = &link->wr_tx_ibs[idx];  in smc_wr_tx_get_free_slot()
208 	*wr_buf = &link->wr_tx_bufs[idx];  in smc_wr_tx_get_free_slot()
210 		*wr_rdma_buf = &link->wr_tx_rdmas[idx];  in smc_wr_tx_get_free_slot()
215 int smc_wr_tx_put_slot(struct smc_link *link,  in smc_wr_tx_put_slot()  argument
221 	if (pend->idx < link->wr_tx_cnt) {  in smc_wr_tx_put_slot()
225 		memset(&link->wr_tx_pends[idx], 0,  in smc_wr_tx_put_slot()
226 		       sizeof(link->wr_tx_pends[idx]));  in smc_wr_tx_put_slot()
227 		memset(&link->wr_tx_bufs[idx], 0,  in smc_wr_tx_put_slot()
228 		       sizeof(link->wr_tx_bufs[idx]));  in smc_wr_tx_put_slot()
229 		test_and_clear_bit(idx, link->wr_tx_mask);  in smc_wr_tx_put_slot()
239 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)  in smc_wr_tx_send()  argument
244 	ib_req_notify_cq(link->smcibdev->roce_cq_send,  in smc_wr_tx_send()
247 	rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);  in smc_wr_tx_send()
249 		smc_wr_tx_put_slot(link, priv);  in smc_wr_tx_send()
250 		smc_lgr_terminate(smc_get_lgr(link));  in smc_wr_tx_send()
256 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)  in smc_wr_reg_send()  argument
260 	ib_req_notify_cq(link->smcibdev->roce_cq_send,  in smc_wr_reg_send()
262 	link->wr_reg_state = POSTED;  in smc_wr_reg_send()
263 	link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;  in smc_wr_reg_send()
264 	link->wr_reg.mr = mr;  in smc_wr_reg_send()
265 	link->wr_reg.key = mr->rkey;  in smc_wr_reg_send()
266 	rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);  in smc_wr_reg_send()
270 	rc = wait_event_interruptible_timeout(link->wr_reg_wait,  in smc_wr_reg_send()
271 					      (link->wr_reg_state != POSTED),  in smc_wr_reg_send()
275 		smc_lgr_terminate(smc_get_lgr(link));  in smc_wr_reg_send()
280 	switch (link->wr_reg_state) {  in smc_wr_reg_send()
294 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,  in smc_wr_tx_dismiss_slots()  argument
303 	for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {  in smc_wr_tx_dismiss_slots()
304 		wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];  in smc_wr_tx_dismiss_slots()
307 		tx_pend = &link->wr_tx_pends[i].priv;  in smc_wr_tx_dismiss_slots()
339 	struct smc_link *link = (struct smc_link *)wc->qp->qp_context;  in smc_wr_rx_demultiplex()  local
348 	index = do_div(temp_wr_id, link->wr_rx_cnt);  in smc_wr_rx_demultiplex()
349 	wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];  in smc_wr_rx_demultiplex()
358 	struct smc_link *link;  in smc_wr_rx_process_cqes()  local
362 		link = wc[i].qp->qp_context;  in smc_wr_rx_process_cqes()
364 			link->wr_rx_tstamp = jiffies;  in smc_wr_rx_process_cqes()
366 			smc_wr_rx_post(link); /* refill WR RX */  in smc_wr_rx_process_cqes()
373 				/* terminate connections of this link group  in smc_wr_rx_process_cqes()
376 				smc_lgr_terminate(smc_get_lgr(link));  in smc_wr_rx_process_cqes()
379 				smc_wr_rx_post(link); /* refill WR RX */  in smc_wr_rx_process_cqes()
418 int smc_wr_rx_post_init(struct smc_link *link)  in smc_wr_rx_post_init()  argument
423 	for (i = 0; i < link->wr_rx_cnt; i++)  in smc_wr_rx_post_init()
424 		rc = smc_wr_rx_post(link);  in smc_wr_rx_post_init()
558 int smc_wr_alloc_link_mem(struct smc_link *link)  in smc_wr_alloc_link_mem()  argument
560 	/* allocate link related memory */  in smc_wr_alloc_link_mem()
561 	link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);  in smc_wr_alloc_link_mem()
562 	if (!link->wr_tx_bufs)  in smc_wr_alloc_link_mem()
564 	link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,  in smc_wr_alloc_link_mem()
566 	if (!link->wr_rx_bufs)  in smc_wr_alloc_link_mem()
568 	link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),  in smc_wr_alloc_link_mem()
570 	if (!link->wr_tx_ibs)  in smc_wr_alloc_link_mem()
572 	link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,  in smc_wr_alloc_link_mem()
573 				  sizeof(link->wr_rx_ibs[0]),  in smc_wr_alloc_link_mem()
575 	if (!link->wr_rx_ibs)  in smc_wr_alloc_link_mem()
577 	link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,  in smc_wr_alloc_link_mem()
578 				    sizeof(link->wr_tx_rdmas[0]),  in smc_wr_alloc_link_mem()
580 	if (!link->wr_tx_rdmas)  in smc_wr_alloc_link_mem()
582 	link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,  in smc_wr_alloc_link_mem()
583 					sizeof(link->wr_tx_rdma_sges[0]),  in smc_wr_alloc_link_mem()
585 	if (!link->wr_tx_rdma_sges)  in smc_wr_alloc_link_mem()
587 	link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),  in smc_wr_alloc_link_mem()
589 	if (!link->wr_tx_sges)  in smc_wr_alloc_link_mem()
591 	link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,  in smc_wr_alloc_link_mem()
592 				   sizeof(link->wr_rx_sges[0]),  in smc_wr_alloc_link_mem()
594 	if (!link->wr_rx_sges)  in smc_wr_alloc_link_mem()
596 	link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),  in smc_wr_alloc_link_mem()
597 				   sizeof(*link->wr_tx_mask),  in smc_wr_alloc_link_mem()
599 	if (!link->wr_tx_mask)  in smc_wr_alloc_link_mem()
601 	link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,  in smc_wr_alloc_link_mem()
602 				    sizeof(link->wr_tx_pends[0]),  in smc_wr_alloc_link_mem()
604 	if (!link->wr_tx_pends)  in smc_wr_alloc_link_mem()
609 	kfree(link->wr_tx_mask);  in smc_wr_alloc_link_mem()
611 	kfree(link->wr_rx_sges);  in smc_wr_alloc_link_mem()
613 	kfree(link->wr_tx_sges);  in smc_wr_alloc_link_mem()
615 	kfree(link->wr_tx_rdma_sges);  in smc_wr_alloc_link_mem()
617 	kfree(link->wr_tx_rdmas);  in smc_wr_alloc_link_mem()
619 	kfree(link->wr_rx_ibs);  in smc_wr_alloc_link_mem()
621 	kfree(link->wr_tx_ibs);  in smc_wr_alloc_link_mem()
623 	kfree(link->wr_rx_bufs);  in smc_wr_alloc_link_mem()
625 	kfree(link->wr_tx_bufs);  in smc_wr_alloc_link_mem()