Lines Matching refs:c_tx
40 static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr) in siw_try_1seg() argument
42 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_try_1seg()
61 } else if (c_tx->in_syscall) { in siw_try_1seg()
119 static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx) in siw_qp_prepare_tx() argument
121 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_qp_prepare_tx()
128 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
132 c_tx->pkt.rreq.rsvd = 0; in siw_qp_prepare_tx()
133 c_tx->pkt.rreq.ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ); in siw_qp_prepare_tx()
134 c_tx->pkt.rreq.ddp_msn = in siw_qp_prepare_tx()
135 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]); in siw_qp_prepare_tx()
136 c_tx->pkt.rreq.ddp_mo = 0; in siw_qp_prepare_tx()
137 c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey); in siw_qp_prepare_tx()
138 c_tx->pkt.rreq.sink_to = in siw_qp_prepare_tx()
140 c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey); in siw_qp_prepare_tx()
141 c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
142 c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length); in siw_qp_prepare_tx()
144 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rreq); in siw_qp_prepare_tx()
145 crc = (char *)&c_tx->pkt.rreq_pkt.crc; in siw_qp_prepare_tx()
150 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
154 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_SEND].ctrl, in siw_qp_prepare_tx()
157 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; in siw_qp_prepare_tx()
158 c_tx->pkt.send.ddp_msn = in siw_qp_prepare_tx()
159 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); in siw_qp_prepare_tx()
160 c_tx->pkt.send.ddp_mo = 0; in siw_qp_prepare_tx()
162 c_tx->pkt.send_inv.inval_stag = 0; in siw_qp_prepare_tx()
164 c_tx->ctrl_len = sizeof(struct iwarp_send); in siw_qp_prepare_tx()
166 crc = (char *)&c_tx->pkt.send_pkt.crc; in siw_qp_prepare_tx()
167 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
172 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
176 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
180 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; in siw_qp_prepare_tx()
181 c_tx->pkt.send.ddp_msn = in siw_qp_prepare_tx()
182 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); in siw_qp_prepare_tx()
183 c_tx->pkt.send.ddp_mo = 0; in siw_qp_prepare_tx()
185 c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey); in siw_qp_prepare_tx()
187 c_tx->ctrl_len = sizeof(struct iwarp_send_inv); in siw_qp_prepare_tx()
189 crc = (char *)&c_tx->pkt.send_pkt.crc; in siw_qp_prepare_tx()
190 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
194 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_RDMA_WRITE].ctrl, in siw_qp_prepare_tx()
197 c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey); in siw_qp_prepare_tx()
198 c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
199 c_tx->ctrl_len = sizeof(struct iwarp_rdma_write); in siw_qp_prepare_tx()
201 crc = (char *)&c_tx->pkt.write_pkt.crc; in siw_qp_prepare_tx()
202 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
206 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
211 c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey); in siw_qp_prepare_tx()
212 c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
214 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rresp); in siw_qp_prepare_tx()
216 crc = (char *)&c_tx->pkt.write_pkt.crc; in siw_qp_prepare_tx()
217 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
221 siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe)); in siw_qp_prepare_tx()
227 c_tx->ctrl_sent = 0; in siw_qp_prepare_tx()
233 c_tx->pkt.ctrl.mpa_len = in siw_qp_prepare_tx()
234 htons(c_tx->ctrl_len + data - MPA_HDR_SIZE); in siw_qp_prepare_tx()
240 c_tx->ctrl_len += data; in siw_qp_prepare_tx()
242 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) in siw_qp_prepare_tx()
243 c_tx->pkt.c_untagged.ddp_mo = 0; in siw_qp_prepare_tx()
245 c_tx->pkt.c_tagged.ddp_to = in siw_qp_prepare_tx()
253 if (c_tx->mpa_crc_hd) { in siw_qp_prepare_tx()
254 crypto_shash_init(c_tx->mpa_crc_hd); in siw_qp_prepare_tx()
255 if (crypto_shash_update(c_tx->mpa_crc_hd, in siw_qp_prepare_tx()
256 (u8 *)&c_tx->pkt, in siw_qp_prepare_tx()
257 c_tx->ctrl_len)) in siw_qp_prepare_tx()
259 crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)crc); in siw_qp_prepare_tx()
261 c_tx->ctrl_len += MPA_CRC_SIZE; in siw_qp_prepare_tx()
265 c_tx->ctrl_len += MPA_CRC_SIZE; in siw_qp_prepare_tx()
266 c_tx->sge_idx = 0; in siw_qp_prepare_tx()
267 c_tx->sge_off = 0; in siw_qp_prepare_tx()
268 c_tx->pbl_idx = 0; in siw_qp_prepare_tx()
279 if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH && in siw_qp_prepare_tx()
281 c_tx->use_sendpage = 1; in siw_qp_prepare_tx()
283 c_tx->use_sendpage = 0; in siw_qp_prepare_tx()
293 static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s, in siw_tx_ctrl() argument
298 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent, in siw_tx_ctrl()
299 .iov_len = c_tx->ctrl_len - c_tx->ctrl_sent }; in siw_tx_ctrl()
302 c_tx->ctrl_len - c_tx->ctrl_sent); in siw_tx_ctrl()
305 c_tx->ctrl_sent += rv; in siw_tx_ctrl()
307 if (c_tx->ctrl_sent == c_tx->ctrl_len) in siw_tx_ctrl()
431 static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s) in siw_tx_hdt() argument
433 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_tx_hdt()
434 struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx]; in siw_tx_hdt()
439 int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv; in siw_tx_hdt()
440 unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0, in siw_tx_hdt()
441 sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx, in siw_tx_hdt()
442 pbl_idx = c_tx->pbl_idx; in siw_tx_hdt()
445 if (c_tx->state == SIW_SEND_HDR) { in siw_tx_hdt()
446 if (c_tx->use_sendpage) { in siw_tx_hdt()
447 rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE); in siw_tx_hdt()
451 c_tx->state = SIW_SEND_DATA; in siw_tx_hdt()
454 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent; in siw_tx_hdt()
456 c_tx->ctrl_len - c_tx->ctrl_sent; in siw_tx_hdt()
474 if (is_kva && !c_tx->use_sendpage) { in siw_tx_hdt()
484 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
509 wqe->processed -= c_tx->bytes_unsent; in siw_tx_hdt()
515 if (!c_tx->use_sendpage) { in siw_tx_hdt()
525 c_tx->mpa_crc_hd, in siw_tx_hdt()
530 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
541 c_tx->mpa_crc_hd, in siw_tx_hdt()
552 siw_dbg_qp(tx_qp(c_tx), "to many fragments\n"); in siw_tx_hdt()
554 wqe->processed -= c_tx->bytes_unsent; in siw_tx_hdt()
569 if (likely(c_tx->state != SIW_SEND_TRAILER)) { in siw_tx_hdt()
570 iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad]; in siw_tx_hdt()
571 iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad); in siw_tx_hdt()
573 iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent]; in siw_tx_hdt()
574 iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent; in siw_tx_hdt()
577 if (c_tx->pad) { in siw_tx_hdt()
578 *(u32 *)c_tx->trailer.pad = 0; in siw_tx_hdt()
580 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
581 (u8 *)&c_tx->trailer.crc - c_tx->pad, in siw_tx_hdt()
582 c_tx->pad); in siw_tx_hdt()
584 if (!c_tx->mpa_crc_hd) in siw_tx_hdt()
585 c_tx->trailer.crc = 0; in siw_tx_hdt()
587 crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)&c_tx->trailer.crc); in siw_tx_hdt()
589 data_len = c_tx->bytes_unsent; in siw_tx_hdt()
591 if (c_tx->use_sendpage) { in siw_tx_hdt()
592 rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx], in siw_tx_hdt()
593 c_tx->sge_off, data_len); in siw_tx_hdt()
610 c_tx->ctrl_sent += rv; in siw_tx_hdt()
621 c_tx->sge_idx = sge_idx; in siw_tx_hdt()
622 c_tx->sge_off = sge_off; in siw_tx_hdt()
623 c_tx->pbl_idx = pbl_idx; in siw_tx_hdt()
630 c_tx->state = SIW_SEND_TRAILER; in siw_tx_hdt()
631 c_tx->ctrl_len = MAX_TRAILER; in siw_tx_hdt()
632 c_tx->ctrl_sent = rv + 4 - c_tx->pad; in siw_tx_hdt()
633 c_tx->bytes_unsent = 0; in siw_tx_hdt()
639 c_tx->state = SIW_SEND_DATA; in siw_tx_hdt()
649 c_tx->bytes_unsent -= rv; in siw_tx_hdt()
650 sge = &wqe->sqe.sge[c_tx->sge_idx]; in siw_tx_hdt()
651 sge_unsent = sge->length - c_tx->sge_off; in siw_tx_hdt()
655 c_tx->sge_idx++; in siw_tx_hdt()
656 c_tx->sge_off = 0; in siw_tx_hdt()
660 c_tx->sge_off += rv; in siw_tx_hdt()
665 c_tx->do_crc = 0; in siw_tx_hdt()
670 static void siw_update_tcpseg(struct siw_iwarp_tx *c_tx, in siw_update_tcpseg() argument
676 if (c_tx->gso_seg_limit == 0) in siw_update_tcpseg()
677 c_tx->tcp_seglen = tp->mss_cache * tp->gso_segs; in siw_update_tcpseg()
679 c_tx->tcp_seglen = in siw_update_tcpseg()
681 min_t(u16, c_tx->gso_seg_limit, tp->gso_segs); in siw_update_tcpseg()
683 c_tx->tcp_seglen = tp->mss_cache; in siw_update_tcpseg()
686 c_tx->tcp_seglen &= 0xfffffff8; in siw_update_tcpseg()
705 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_prepare_fpdu() local
708 c_tx->ctrl_len = in siw_prepare_fpdu()
709 iwarp_pktinfo[__rdmap_get_opcode(&c_tx->pkt.ctrl)].hdr_len; in siw_prepare_fpdu()
710 c_tx->ctrl_sent = 0; in siw_prepare_fpdu()
715 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) in siw_prepare_fpdu()
717 c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed); in siw_prepare_fpdu()
719 c_tx->pkt.c_tagged.ddp_to = in siw_prepare_fpdu()
723 if (data_len + c_tx->ctrl_len + MPA_CRC_SIZE > c_tx->tcp_seglen) { in siw_prepare_fpdu()
725 data_len = c_tx->tcp_seglen - (c_tx->ctrl_len + MPA_CRC_SIZE); in siw_prepare_fpdu()
726 c_tx->pkt.ctrl.ddp_rdmap_ctrl &= ~DDP_FLAG_LAST; in siw_prepare_fpdu()
727 c_tx->pad = 0; in siw_prepare_fpdu()
729 c_tx->pkt.ctrl.ddp_rdmap_ctrl |= DDP_FLAG_LAST; in siw_prepare_fpdu()
730 c_tx->pad = -data_len & 0x3; in siw_prepare_fpdu()
732 c_tx->bytes_unsent = data_len; in siw_prepare_fpdu()
734 c_tx->pkt.ctrl.mpa_len = in siw_prepare_fpdu()
735 htons(c_tx->ctrl_len + data_len - MPA_HDR_SIZE); in siw_prepare_fpdu()
740 if (c_tx->mpa_crc_hd) { in siw_prepare_fpdu()
741 crypto_shash_init(c_tx->mpa_crc_hd); in siw_prepare_fpdu()
742 crypto_shash_update(c_tx->mpa_crc_hd, (u8 *)&c_tx->pkt, in siw_prepare_fpdu()
743 c_tx->ctrl_len); in siw_prepare_fpdu()
744 c_tx->do_crc = 1; in siw_prepare_fpdu()
793 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_qp_sq_proc_tx() local
842 siw_update_tcpseg(c_tx, s); in siw_qp_sq_proc_tx()
844 rv = siw_qp_prepare_tx(c_tx); in siw_qp_sq_proc_tx()
846 c_tx->state = SIW_SEND_HDR; in siw_qp_sq_proc_tx()
849 c_tx->state = SIW_SEND_SHORT_FPDU; in siw_qp_sq_proc_tx()
864 if (c_tx->state == SIW_SEND_SHORT_FPDU) { in siw_qp_sq_proc_tx()
878 rv = siw_tx_ctrl(c_tx, s, msg_flags); in siw_qp_sq_proc_tx()
887 rv = siw_tx_hdt(c_tx, s); in siw_qp_sq_proc_tx()
894 if (unlikely(c_tx->tx_suspend)) { in siw_qp_sq_proc_tx()
903 if (c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_LAST) { in siw_qp_sq_proc_tx()
907 c_tx->state = SIW_SEND_HDR; in siw_qp_sq_proc_tx()
909 siw_update_tcpseg(c_tx, s); in siw_qp_sq_proc_tx()