Lines Matching +full:long +full:- +full:term
1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
4 /* Copyright (c) 2008-2019, IBM Corporation */
29 * per-RDMAP message basis. Please keep order of initializer. All MPA len
35 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
43 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
50 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
58 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
65 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
72 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
79 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
86 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
97 read_lock(&sk->sk_callback_lock); in siw_qp_llp_data_ready()
99 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) in siw_qp_llp_data_ready()
104 if (likely(!qp->rx_stream.rx_suspend && in siw_qp_llp_data_ready()
105 down_read_trylock(&qp->state_lock))) { in siw_qp_llp_data_ready()
108 if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) in siw_qp_llp_data_ready()
117 up_read(&qp->state_lock); in siw_qp_llp_data_ready()
120 qp->rx_stream.rx_suspend); in siw_qp_llp_data_ready()
123 read_unlock(&sk->sk_callback_lock); in siw_qp_llp_data_ready()
129 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
131 down_write(&qp->state_lock); in siw_qp_llp_close()
133 qp->rx_stream.rx_suspend = 1; in siw_qp_llp_close()
134 qp->tx_ctx.tx_suspend = 1; in siw_qp_llp_close()
135 qp->attrs.sk = NULL; in siw_qp_llp_close()
137 switch (qp->attrs.state) { in siw_qp_llp_close()
142 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
151 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) in siw_qp_llp_close()
152 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
154 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_llp_close()
159 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
168 if (qp->cep) { in siw_qp_llp_close()
169 siw_cep_put(qp->cep); in siw_qp_llp_close()
170 qp->cep = NULL; in siw_qp_llp_close()
173 up_write(&qp->state_lock); in siw_qp_llp_close()
176 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
187 read_lock(&sk->sk_callback_lock); in siw_qp_llp_write_space()
191 cep->sk_write_space(sk); in siw_qp_llp_write_space()
193 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) in siw_qp_llp_write_space()
194 (void)siw_sq_start(cep->qp); in siw_qp_llp_write_space()
197 read_unlock(&sk->sk_callback_lock); in siw_qp_llp_write_space()
204 qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe)); in siw_qp_readq_init()
205 if (!qp->irq) { in siw_qp_readq_init()
206 qp->attrs.irq_size = 0; in siw_qp_readq_init()
207 return -ENOMEM; in siw_qp_readq_init()
212 qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe)); in siw_qp_readq_init()
213 if (!qp->orq) { in siw_qp_readq_init()
214 qp->attrs.orq_size = 0; in siw_qp_readq_init()
215 qp->attrs.irq_size = 0; in siw_qp_readq_init()
216 vfree(qp->irq); in siw_qp_readq_init()
217 return -ENOMEM; in siw_qp_readq_init()
220 qp->attrs.irq_size = irq_size; in siw_qp_readq_init()
221 qp->attrs.orq_size = orq_size; in siw_qp_readq_init()
228 struct siw_rx_stream *c_rx = &qp->rx_stream; in siw_qp_enable_crc()
229 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_qp_enable_crc()
233 return -ENOENT; in siw_qp_enable_crc()
238 c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); in siw_qp_enable_crc()
239 c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); in siw_qp_enable_crc()
240 if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) { in siw_qp_enable_crc()
241 kfree(c_tx->mpa_crc_hd); in siw_qp_enable_crc()
242 kfree(c_rx->mpa_crc_hd); in siw_qp_enable_crc()
243 c_tx->mpa_crc_hd = NULL; in siw_qp_enable_crc()
244 c_rx->mpa_crc_hd = NULL; in siw_qp_enable_crc()
245 return -ENOMEM; in siw_qp_enable_crc()
247 c_tx->mpa_crc_hd->tfm = siw_crypto_shash; in siw_qp_enable_crc()
248 c_rx->mpa_crc_hd->tfm = siw_crypto_shash; in siw_qp_enable_crc()
263 unsigned long flags; in siw_qp_mpa_rts()
266 spin_lock_irqsave(&qp->sq_lock, flags); in siw_qp_mpa_rts()
268 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { in siw_qp_mpa_rts()
269 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
270 return -EIO; in siw_qp_mpa_rts()
272 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_qp_mpa_rts()
274 wqe->wr_status = SIW_WR_QUEUED; in siw_qp_mpa_rts()
275 wqe->sqe.flags = 0; in siw_qp_mpa_rts()
276 wqe->sqe.num_sge = 1; in siw_qp_mpa_rts()
277 wqe->sqe.sge[0].length = 0; in siw_qp_mpa_rts()
278 wqe->sqe.sge[0].laddr = 0; in siw_qp_mpa_rts()
279 wqe->sqe.sge[0].lkey = 0; in siw_qp_mpa_rts()
284 wqe->sqe.rkey = 1; in siw_qp_mpa_rts()
285 wqe->sqe.raddr = 0; in siw_qp_mpa_rts()
286 wqe->processed = 0; in siw_qp_mpa_rts()
289 wqe->sqe.opcode = SIW_OP_WRITE; in siw_qp_mpa_rts()
293 wqe->sqe.opcode = SIW_OP_READ; in siw_qp_mpa_rts()
295 spin_lock(&qp->orq_lock); in siw_qp_mpa_rts()
297 if (qp->attrs.orq_size) in siw_qp_mpa_rts()
300 siw_read_to_orq(rreq, &wqe->sqe); in siw_qp_mpa_rts()
301 qp->orq_put++; in siw_qp_mpa_rts()
303 rv = -EIO; in siw_qp_mpa_rts()
305 spin_unlock(&qp->orq_lock); in siw_qp_mpa_rts()
307 rv = -EINVAL; in siw_qp_mpa_rts()
310 wqe->wr_status = SIW_WR_IDLE; in siw_qp_mpa_rts()
312 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
367 if (!qp->term_info.valid) { in siw_init_terminate()
368 memset(&qp->term_info, 0, sizeof(qp->term_info)); in siw_init_terminate()
369 qp->term_info.layer = layer; in siw_init_terminate()
370 qp->term_info.etype = etype; in siw_init_terminate()
371 qp->term_info.ecode = ecode; in siw_init_terminate()
372 qp->term_info.in_tx = in_tx; in siw_init_terminate()
373 qp->term_info.valid = 1; in siw_init_terminate()
375 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", in siw_init_terminate()
381 * Sending TERMINATE messages is best effort - such messages
383 * not have another outbound message in-progress, i.e. the
391 struct iwarp_terminate *term = NULL; in siw_send_terminate() local
393 struct socket *s = qp->attrs.sk; in siw_send_terminate()
394 struct siw_rx_stream *srx = &qp->rx_stream; in siw_send_terminate()
395 union iwarp_hdr *rx_hdr = &srx->hdr; in siw_send_terminate()
399 if (!qp->term_info.valid) in siw_send_terminate()
402 qp->term_info.valid = 0; in siw_send_terminate()
404 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { in siw_send_terminate()
409 if (!s && qp->cep) in siw_send_terminate()
411 s = qp->cep->sock; in siw_send_terminate()
418 term = kzalloc(sizeof(*term), GFP_KERNEL); in siw_send_terminate()
419 if (!term) in siw_send_terminate()
422 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE); in siw_send_terminate()
423 term->ddp_mo = 0; in siw_send_terminate()
424 term->ddp_msn = cpu_to_be32(1); in siw_send_terminate()
426 iov[0].iov_base = term; in siw_send_terminate()
427 iov[0].iov_len = sizeof(*term); in siw_send_terminate()
429 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || in siw_send_terminate()
430 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && in siw_send_terminate()
431 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { in siw_send_terminate()
434 kfree(term); in siw_send_terminate()
438 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl, in siw_send_terminate()
441 __rdmap_term_set_layer(term, qp->term_info.layer); in siw_send_terminate()
442 __rdmap_term_set_etype(term, qp->term_info.etype); in siw_send_terminate()
443 __rdmap_term_set_ecode(term, qp->term_info.ecode); in siw_send_terminate()
445 switch (qp->term_info.layer) { in siw_send_terminate()
447 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) in siw_send_terminate()
451 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { in siw_send_terminate()
456 term->flag_m = 1; in siw_send_terminate()
457 term->flag_d = 1; in siw_send_terminate()
458 term->flag_r = 1; in siw_send_terminate()
460 if (qp->term_info.in_tx) { in siw_send_terminate()
471 memcpy(&rreq->ctrl, in siw_send_terminate()
475 rreq->rsvd = 0; in siw_send_terminate()
476 rreq->ddp_qn = in siw_send_terminate()
480 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); in siw_send_terminate()
482 rreq->ddp_mo = htonl(wqe->processed); in siw_send_terminate()
483 rreq->sink_stag = htonl(wqe->sqe.rkey); in siw_send_terminate()
484 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_send_terminate()
485 rreq->read_size = htonl(wqe->sqe.sge[0].length); in siw_send_terminate()
486 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); in siw_send_terminate()
487 rreq->source_to = in siw_send_terminate()
488 cpu_to_be64(wqe->sqe.sge[0].laddr); in siw_send_terminate()
500 if (__rdmap_get_opcode(&rx_hdr->ctrl) == in siw_send_terminate()
512 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || in siw_send_terminate()
513 (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) in siw_send_terminate()
519 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) in siw_send_terminate()
525 term->flag_m = 1; in siw_send_terminate()
526 term->flag_d = 1; in siw_send_terminate()
528 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len); in siw_send_terminate()
540 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && in siw_send_terminate()
541 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || in siw_send_terminate()
542 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && in siw_send_terminate()
543 (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) in siw_send_terminate()
548 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) in siw_send_terminate()
553 term->flag_m = 1; in siw_send_terminate()
554 term->flag_d = 1; in siw_send_terminate()
560 if (term->flag_m || term->flag_d || term->flag_r) { in siw_send_terminate()
563 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE; in siw_send_terminate()
568 len_terminate = sizeof(*term) + MPA_CRC_SIZE; in siw_send_terminate()
573 if (term->flag_m) { in siw_send_terminate()
574 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len); in siw_send_terminate()
575 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl); in siw_send_terminate()
577 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE; in siw_send_terminate()
578 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len); in siw_send_terminate()
581 term->ctrl.mpa_len = in siw_send_terminate()
582 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE)); in siw_send_terminate()
583 if (qp->tx_ctx.mpa_crc_hd) { in siw_send_terminate()
584 crypto_shash_init(qp->tx_ctx.mpa_crc_hd); in siw_send_terminate()
585 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, in siw_send_terminate()
591 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, in siw_send_terminate()
596 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); in siw_send_terminate()
600 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", in siw_send_terminate()
602 __rdmap_term_layer(term), __rdmap_term_etype(term), in siw_send_terminate()
603 __rdmap_term_ecode(term), rv); in siw_send_terminate()
605 kfree(term); in siw_send_terminate()
617 if (attrs->flags & SIW_RDMA_BIND_ENABLED) in siw_qp_modify_nonstate()
618 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
620 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
622 if (attrs->flags & SIW_RDMA_WRITE_ENABLED) in siw_qp_modify_nonstate()
623 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
625 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
627 if (attrs->flags & SIW_RDMA_READ_ENABLED) in siw_qp_modify_nonstate()
628 qp->attrs.flags |= SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
630 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
640 switch (attrs->state) { in siw_qp_nextstate_from_idle()
642 if (attrs->flags & SIW_MPA_CRC) { in siw_qp_nextstate_from_idle()
649 rv = -EINVAL; in siw_qp_nextstate_from_idle()
654 rv = -EINVAL; in siw_qp_nextstate_from_idle()
660 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; in siw_qp_nextstate_from_idle()
661 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; in siw_qp_nextstate_from_idle()
662 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; in siw_qp_nextstate_from_idle()
667 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; in siw_qp_nextstate_from_idle()
668 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; in siw_qp_nextstate_from_idle()
669 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; in siw_qp_nextstate_from_idle()
675 rv = siw_qp_readq_init(qp, attrs->irq_size, in siw_qp_nextstate_from_idle()
676 attrs->orq_size); in siw_qp_nextstate_from_idle()
680 qp->attrs.sk = attrs->sk; in siw_qp_nextstate_from_idle()
681 qp->attrs.state = SIW_QP_STATE_RTS; in siw_qp_nextstate_from_idle()
684 attrs->flags & SIW_MPA_CRC ? "y" : "n", in siw_qp_nextstate_from_idle()
685 qp->attrs.orq_size, qp->attrs.irq_size); in siw_qp_nextstate_from_idle()
690 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_idle()
691 if (qp->cep) { in siw_qp_nextstate_from_idle()
692 siw_cep_put(qp->cep); in siw_qp_nextstate_from_idle()
693 qp->cep = NULL; in siw_qp_nextstate_from_idle()
708 switch (attrs->state) { in siw_qp_nextstate_from_rts()
718 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { in siw_qp_nextstate_from_rts()
719 qp->attrs.state = SIW_QP_STATE_CLOSING; in siw_qp_nextstate_from_rts()
721 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
730 qp->attrs.state = SIW_QP_STATE_TERMINATE; in siw_qp_nextstate_from_rts()
748 * Esp., how to handle the non-empty IRQ case? in siw_qp_nextstate_from_rts()
754 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
767 switch (attrs->state) { in siw_qp_nextstate_from_term()
770 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_term()
772 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_term()
786 switch (attrs->state) { in siw_qp_nextstate_from_close()
788 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); in siw_qp_nextstate_from_close()
789 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_nextstate_from_close()
804 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_close()
806 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_close()
814 siw_qp_state_to_string[qp->attrs.state], in siw_qp_nextstate_from_close()
815 siw_qp_state_to_string[attrs->state]); in siw_qp_nextstate_from_close()
817 rv = -ECONNABORTED; in siw_qp_nextstate_from_close()
823 * Caller must hold qp->state_lock
834 siw_qp_state_to_string[qp->attrs.state], in siw_qp_modify()
835 siw_qp_state_to_string[attrs->state]); in siw_qp_modify()
843 switch (qp->attrs.state) { in siw_qp_modify()
871 rreq->id = sqe->id; in siw_read_to_orq()
872 rreq->opcode = sqe->opcode; in siw_read_to_orq()
873 rreq->sge[0].laddr = sqe->sge[0].laddr; in siw_read_to_orq()
874 rreq->sge[0].length = sqe->sge[0].length; in siw_read_to_orq()
875 rreq->sge[0].lkey = sqe->sge[0].lkey; in siw_read_to_orq()
876 rreq->sge[1].lkey = sqe->sge[1].lkey; in siw_read_to_orq()
877 rreq->flags = sqe->flags | SIW_WQE_VALID; in siw_read_to_orq()
878 rreq->num_sge = 1; in siw_read_to_orq()
891 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_activate_tx_from_sq()
892 wqe->wr_status = SIW_WR_QUEUED; in siw_activate_tx_from_sq()
895 memcpy(&wqe->sqe, sqe, sizeof(*sqe)); in siw_activate_tx_from_sq()
897 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { in siw_activate_tx_from_sq()
898 rv = -EINVAL; in siw_activate_tx_from_sq()
901 if (wqe->sqe.flags & SIW_WQE_INLINE) { in siw_activate_tx_from_sq()
902 if (wqe->sqe.opcode != SIW_OP_SEND && in siw_activate_tx_from_sq()
903 wqe->sqe.opcode != SIW_OP_WRITE) { in siw_activate_tx_from_sq()
904 rv = -EINVAL; in siw_activate_tx_from_sq()
907 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { in siw_activate_tx_from_sq()
908 rv = -EINVAL; in siw_activate_tx_from_sq()
911 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; in siw_activate_tx_from_sq()
912 wqe->sqe.sge[0].lkey = 0; in siw_activate_tx_from_sq()
913 wqe->sqe.num_sge = 1; in siw_activate_tx_from_sq()
915 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { in siw_activate_tx_from_sq()
917 if (unlikely(wqe->sqe.opcode == SIW_OP_READ || in siw_activate_tx_from_sq()
918 wqe->sqe.opcode == in siw_activate_tx_from_sq()
921 rv = -EINVAL; in siw_activate_tx_from_sq()
924 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
926 if (qp->attrs.orq_size && !siw_orq_empty(qp)) { in siw_activate_tx_from_sq()
927 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
930 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
932 } else if (wqe->sqe.opcode == SIW_OP_READ || in siw_activate_tx_from_sq()
933 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { in siw_activate_tx_from_sq()
936 if (unlikely(!qp->attrs.orq_size)) { in siw_activate_tx_from_sq()
938 rv = -EINVAL; in siw_activate_tx_from_sq()
941 wqe->sqe.num_sge = 1; in siw_activate_tx_from_sq()
943 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
951 siw_read_to_orq(rreq, &wqe->sqe); in siw_activate_tx_from_sq()
952 qp->orq_put++; in siw_activate_tx_from_sq()
954 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
957 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
960 /* Clear SQE, can be re-used by application */ in siw_activate_tx_from_sq()
961 smp_store_mb(sqe->flags, 0); in siw_activate_tx_from_sq()
962 qp->sq_get++; in siw_activate_tx_from_sq()
966 wqe->wr_status = SIW_WR_IDLE; in siw_activate_tx_from_sq()
974 * the active IRQ will not be served after qp->irq_burst, if the
982 if (!qp->attrs.irq_size) in siw_activate_tx()
985 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; in siw_activate_tx()
987 if (!(irqe->flags & SIW_WQE_VALID)) in siw_activate_tx()
994 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { in siw_activate_tx()
995 qp->irq_burst = 0; in siw_activate_tx()
998 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_activate_tx()
999 wqe->wr_status = SIW_WR_QUEUED; in siw_activate_tx()
1002 wqe->sqe.opcode = SIW_OP_READ_RESPONSE; in siw_activate_tx()
1003 wqe->sqe.flags = 0; in siw_activate_tx()
1004 if (irqe->num_sge) { in siw_activate_tx()
1005 wqe->sqe.num_sge = 1; in siw_activate_tx()
1006 wqe->sqe.sge[0].length = irqe->sge[0].length; in siw_activate_tx()
1007 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; in siw_activate_tx()
1008 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; in siw_activate_tx()
1010 wqe->sqe.num_sge = 0; in siw_activate_tx()
1016 wqe->sqe.sge[1].length = irqe->sge[1].length; in siw_activate_tx()
1018 wqe->sqe.rkey = irqe->rkey; in siw_activate_tx()
1019 wqe->sqe.raddr = irqe->raddr; in siw_activate_tx()
1021 wqe->processed = 0; in siw_activate_tx()
1022 qp->irq_get++; in siw_activate_tx()
1025 smp_store_mb(irqe->flags, 0); in siw_activate_tx()
1038 if (!cq->base_cq.comp_handler) in siw_cq_notify_now()
1042 cq_notify = READ_ONCE(cq->notify->flags); in siw_cq_notify_now()
1048 * CQ notification is one-shot: Since the in siw_cq_notify_now()
1050 * the CQ gets dis-aremd and must be re-aremd in siw_cq_notify_now()
1053 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT); in siw_cq_notify_now()
1063 struct siw_cq *cq = qp->scq; in siw_sqe_complete()
1067 u32 sqe_flags = sqe->flags; in siw_sqe_complete()
1070 unsigned long flags; in siw_sqe_complete()
1072 spin_lock_irqsave(&cq->lock, flags); in siw_sqe_complete()
1074 idx = cq->cq_put % cq->num_cqe; in siw_sqe_complete()
1075 cqe = &cq->queue[idx]; in siw_sqe_complete()
1077 if (!READ_ONCE(cqe->flags)) { in siw_sqe_complete()
1080 cqe->id = sqe->id; in siw_sqe_complete()
1081 cqe->opcode = sqe->opcode; in siw_sqe_complete()
1082 cqe->status = status; in siw_sqe_complete()
1083 cqe->imm_data = 0; in siw_sqe_complete()
1084 cqe->bytes = bytes; in siw_sqe_complete()
1086 if (rdma_is_kernel_res(&cq->base_cq.res)) in siw_sqe_complete()
1087 cqe->base_qp = &qp->base_qp; in siw_sqe_complete()
1089 cqe->qp_id = qp_id(qp); in siw_sqe_complete()
1092 WRITE_ONCE(cqe->flags, SIW_WQE_VALID); in siw_sqe_complete()
1094 smp_store_mb(sqe->flags, 0); in siw_sqe_complete()
1096 cq->cq_put++; in siw_sqe_complete()
1099 spin_unlock_irqrestore(&cq->lock, flags); in siw_sqe_complete()
1103 cq->base_cq.comp_handler(&cq->base_cq, in siw_sqe_complete()
1104 cq->base_cq.cq_context); in siw_sqe_complete()
1107 spin_unlock_irqrestore(&cq->lock, flags); in siw_sqe_complete()
1108 rv = -ENOMEM; in siw_sqe_complete()
1113 smp_store_mb(sqe->flags, 0); in siw_sqe_complete()
1121 struct siw_cq *cq = qp->rcq; in siw_rqe_complete()
1127 unsigned long flags; in siw_rqe_complete()
1129 spin_lock_irqsave(&cq->lock, flags); in siw_rqe_complete()
1131 idx = cq->cq_put % cq->num_cqe; in siw_rqe_complete()
1132 cqe = &cq->queue[idx]; in siw_rqe_complete()
1134 if (!READ_ONCE(cqe->flags)) { in siw_rqe_complete()
1138 cqe->id = rqe->id; in siw_rqe_complete()
1139 cqe->opcode = SIW_OP_RECEIVE; in siw_rqe_complete()
1140 cqe->status = status; in siw_rqe_complete()
1141 cqe->imm_data = 0; in siw_rqe_complete()
1142 cqe->bytes = bytes; in siw_rqe_complete()
1144 if (rdma_is_kernel_res(&cq->base_cq.res)) { in siw_rqe_complete()
1145 cqe->base_qp = &qp->base_qp; in siw_rqe_complete()
1148 cqe->inval_stag = inval_stag; in siw_rqe_complete()
1151 cqe->qp_id = qp_id(qp); in siw_rqe_complete()
1154 WRITE_ONCE(cqe->flags, cqe_flags); in siw_rqe_complete()
1156 smp_store_mb(rqe->flags, 0); in siw_rqe_complete()
1158 cq->cq_put++; in siw_rqe_complete()
1161 spin_unlock_irqrestore(&cq->lock, flags); in siw_rqe_complete()
1165 cq->base_cq.comp_handler(&cq->base_cq, in siw_rqe_complete()
1166 cq->base_cq.cq_context); in siw_rqe_complete()
1169 spin_unlock_irqrestore(&cq->lock, flags); in siw_rqe_complete()
1170 rv = -ENOMEM; in siw_rqe_complete()
1175 smp_store_mb(rqe->flags, 0); in siw_rqe_complete()
1197 while (qp->attrs.orq_size) { in siw_sq_flush()
1198 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; in siw_sq_flush()
1199 if (!READ_ONCE(sqe->flags)) in siw_sq_flush()
1205 WRITE_ONCE(sqe->flags, 0); in siw_sq_flush()
1206 qp->orq_get++; in siw_sq_flush()
1209 * Flush an in-progress WQE if present in siw_sq_flush()
1211 if (wqe->wr_status != SIW_WR_IDLE) { in siw_sq_flush()
1213 tx_type(wqe), wqe->wr_status); in siw_sq_flush()
1220 wqe->wr_status == SIW_WR_QUEUED)) in siw_sq_flush()
1222 * An in-progress Read Request is already in in siw_sq_flush()
1225 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, in siw_sq_flush()
1228 wqe->wr_status = SIW_WR_IDLE; in siw_sq_flush()
1233 while (qp->attrs.sq_size) { in siw_sq_flush()
1234 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; in siw_sq_flush()
1235 if (!READ_ONCE(sqe->flags)) in siw_sq_flush()
1246 WRITE_ONCE(sqe->flags, 0); in siw_sq_flush()
1247 qp->sq_get++; in siw_sq_flush()
1266 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; in siw_rq_flush()
1269 * Flush an in-progress untagged operation if present in siw_rq_flush()
1271 if (wqe->wr_status != SIW_WR_IDLE) { in siw_rq_flush()
1273 rx_type(wqe), wqe->wr_status); in siw_rq_flush()
1278 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, in siw_rq_flush()
1283 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); in siw_rq_flush()
1285 wqe->wr_status = SIW_WR_IDLE; in siw_rq_flush()
1287 wqe = &qp->rx_tagged.wqe_active; in siw_rq_flush()
1289 if (wqe->wr_status != SIW_WR_IDLE) { in siw_rq_flush()
1291 wqe->wr_status = SIW_WR_IDLE; in siw_rq_flush()
1296 while (qp->attrs.rq_size) { in siw_rq_flush()
1298 &qp->recvq[qp->rq_get % qp->attrs.rq_size]; in siw_rq_flush()
1300 if (!READ_ONCE(rqe->flags)) in siw_rq_flush()
1306 WRITE_ONCE(rqe->flags, 0); in siw_rq_flush()
1307 qp->rq_get++; in siw_rq_flush()
1313 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, in siw_qp_add()
1317 kref_init(&qp->ref); in siw_qp_add()
1318 qp->sdev = sdev; in siw_qp_add()
1327 struct siw_device *sdev = qp->sdev; in siw_free_qp()
1328 unsigned long flags; in siw_free_qp()
1330 if (qp->cep) in siw_free_qp()
1331 siw_cep_put(qp->cep); in siw_free_qp()
1333 found = xa_erase(&sdev->qp_xa, qp_id(qp)); in siw_free_qp()
1335 spin_lock_irqsave(&sdev->lock, flags); in siw_free_qp()
1336 list_del(&qp->devq); in siw_free_qp()
1337 spin_unlock_irqrestore(&sdev->lock, flags); in siw_free_qp()
1339 vfree(qp->sendq); in siw_free_qp()
1340 vfree(qp->recvq); in siw_free_qp()
1341 vfree(qp->irq); in siw_free_qp()
1342 vfree(qp->orq); in siw_free_qp()
1344 siw_put_tx_cpu(qp->tx_cpu); in siw_free_qp()
1346 atomic_dec(&sdev->num_qp); in siw_free_qp()