Lines Matching refs:c_tx
40 static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr) in siw_try_1seg() argument
42 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_try_1seg()
61 } else if (c_tx->in_syscall) { in siw_try_1seg()
119 static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx) in siw_qp_prepare_tx() argument
121 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_qp_prepare_tx()
128 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
132 c_tx->pkt.rreq.rsvd = 0; in siw_qp_prepare_tx()
133 c_tx->pkt.rreq.ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ); in siw_qp_prepare_tx()
134 c_tx->pkt.rreq.ddp_msn = in siw_qp_prepare_tx()
135 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]); in siw_qp_prepare_tx()
136 c_tx->pkt.rreq.ddp_mo = 0; in siw_qp_prepare_tx()
137 c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey); in siw_qp_prepare_tx()
138 c_tx->pkt.rreq.sink_to = in siw_qp_prepare_tx()
140 c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey); in siw_qp_prepare_tx()
141 c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
142 c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length); in siw_qp_prepare_tx()
144 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rreq); in siw_qp_prepare_tx()
145 crc = (char *)&c_tx->pkt.rreq_pkt.crc; in siw_qp_prepare_tx()
150 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
154 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_SEND].ctrl, in siw_qp_prepare_tx()
157 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; in siw_qp_prepare_tx()
158 c_tx->pkt.send.ddp_msn = in siw_qp_prepare_tx()
159 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); in siw_qp_prepare_tx()
160 c_tx->pkt.send.ddp_mo = 0; in siw_qp_prepare_tx()
162 c_tx->pkt.send_inv.inval_stag = 0; in siw_qp_prepare_tx()
164 c_tx->ctrl_len = sizeof(struct iwarp_send); in siw_qp_prepare_tx()
166 crc = (char *)&c_tx->pkt.send_pkt.crc; in siw_qp_prepare_tx()
167 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
172 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
176 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
180 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; in siw_qp_prepare_tx()
181 c_tx->pkt.send.ddp_msn = in siw_qp_prepare_tx()
182 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); in siw_qp_prepare_tx()
183 c_tx->pkt.send.ddp_mo = 0; in siw_qp_prepare_tx()
185 c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey); in siw_qp_prepare_tx()
187 c_tx->ctrl_len = sizeof(struct iwarp_send_inv); in siw_qp_prepare_tx()
189 crc = (char *)&c_tx->pkt.send_pkt.crc; in siw_qp_prepare_tx()
190 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
194 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_RDMA_WRITE].ctrl, in siw_qp_prepare_tx()
197 c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey); in siw_qp_prepare_tx()
198 c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
199 c_tx->ctrl_len = sizeof(struct iwarp_rdma_write); in siw_qp_prepare_tx()
201 crc = (char *)&c_tx->pkt.write_pkt.crc; in siw_qp_prepare_tx()
202 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
206 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
211 c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey); in siw_qp_prepare_tx()
212 c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
214 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rresp); in siw_qp_prepare_tx()
216 crc = (char *)&c_tx->pkt.write_pkt.crc; in siw_qp_prepare_tx()
217 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
221 siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe)); in siw_qp_prepare_tx()
227 c_tx->ctrl_sent = 0; in siw_qp_prepare_tx()
233 c_tx->pkt.ctrl.mpa_len = in siw_qp_prepare_tx()
234 htons(c_tx->ctrl_len + data - MPA_HDR_SIZE); in siw_qp_prepare_tx()
240 c_tx->ctrl_len += data; in siw_qp_prepare_tx()
242 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) in siw_qp_prepare_tx()
243 c_tx->pkt.c_untagged.ddp_mo = 0; in siw_qp_prepare_tx()
245 c_tx->pkt.c_tagged.ddp_to = in siw_qp_prepare_tx()
253 if (c_tx->mpa_crc_hd) { in siw_qp_prepare_tx()
254 crypto_shash_init(c_tx->mpa_crc_hd); in siw_qp_prepare_tx()
255 if (crypto_shash_update(c_tx->mpa_crc_hd, in siw_qp_prepare_tx()
256 (u8 *)&c_tx->pkt, in siw_qp_prepare_tx()
257 c_tx->ctrl_len)) in siw_qp_prepare_tx()
259 crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)crc); in siw_qp_prepare_tx()
261 c_tx->ctrl_len += MPA_CRC_SIZE; in siw_qp_prepare_tx()
265 c_tx->ctrl_len += MPA_CRC_SIZE; in siw_qp_prepare_tx()
266 c_tx->sge_idx = 0; in siw_qp_prepare_tx()
267 c_tx->sge_off = 0; in siw_qp_prepare_tx()
268 c_tx->pbl_idx = 0; in siw_qp_prepare_tx()
279 if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH && in siw_qp_prepare_tx()
281 c_tx->use_sendpage = 1; in siw_qp_prepare_tx()
283 c_tx->use_sendpage = 0; in siw_qp_prepare_tx()
293 static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s, in siw_tx_ctrl() argument
298 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent, in siw_tx_ctrl()
299 .iov_len = c_tx->ctrl_len - c_tx->ctrl_sent }; in siw_tx_ctrl()
302 c_tx->ctrl_len - c_tx->ctrl_sent); in siw_tx_ctrl()
305 c_tx->ctrl_sent += rv; in siw_tx_ctrl()
307 if (c_tx->ctrl_sent == c_tx->ctrl_len) in siw_tx_ctrl()
424 static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s) in siw_tx_hdt() argument
426 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_tx_hdt()
427 struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx]; in siw_tx_hdt()
432 int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv; in siw_tx_hdt()
433 unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0, in siw_tx_hdt()
434 sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx, in siw_tx_hdt()
435 pbl_idx = c_tx->pbl_idx; in siw_tx_hdt()
438 if (c_tx->state == SIW_SEND_HDR) { in siw_tx_hdt()
439 if (c_tx->use_sendpage) { in siw_tx_hdt()
440 rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE); in siw_tx_hdt()
444 c_tx->state = SIW_SEND_DATA; in siw_tx_hdt()
447 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent; in siw_tx_hdt()
449 c_tx->ctrl_len - c_tx->ctrl_sent; in siw_tx_hdt()
467 if (is_kva && !c_tx->use_sendpage) { in siw_tx_hdt()
477 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
501 wqe->processed -= c_tx->bytes_unsent; in siw_tx_hdt()
507 if (!c_tx->use_sendpage) { in siw_tx_hdt()
516 c_tx->mpa_crc_hd, in siw_tx_hdt()
520 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
531 c_tx->mpa_crc_hd, in siw_tx_hdt()
542 siw_dbg_qp(tx_qp(c_tx), "to many fragments\n"); in siw_tx_hdt()
544 wqe->processed -= c_tx->bytes_unsent; in siw_tx_hdt()
559 if (likely(c_tx->state != SIW_SEND_TRAILER)) { in siw_tx_hdt()
560 iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad]; in siw_tx_hdt()
561 iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad); in siw_tx_hdt()
563 iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent]; in siw_tx_hdt()
564 iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent; in siw_tx_hdt()
567 if (c_tx->pad) { in siw_tx_hdt()
568 *(u32 *)c_tx->trailer.pad = 0; in siw_tx_hdt()
570 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
571 (u8 *)&c_tx->trailer.crc - c_tx->pad, in siw_tx_hdt()
572 c_tx->pad); in siw_tx_hdt()
574 if (!c_tx->mpa_crc_hd) in siw_tx_hdt()
575 c_tx->trailer.crc = 0; in siw_tx_hdt()
577 crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)&c_tx->trailer.crc); in siw_tx_hdt()
579 data_len = c_tx->bytes_unsent; in siw_tx_hdt()
581 if (c_tx->use_sendpage) { in siw_tx_hdt()
582 rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx], in siw_tx_hdt()
583 c_tx->sge_off, data_len); in siw_tx_hdt()
600 c_tx->ctrl_sent += rv; in siw_tx_hdt()
611 c_tx->sge_idx = sge_idx; in siw_tx_hdt()
612 c_tx->sge_off = sge_off; in siw_tx_hdt()
613 c_tx->pbl_idx = pbl_idx; in siw_tx_hdt()
620 c_tx->state = SIW_SEND_TRAILER; in siw_tx_hdt()
621 c_tx->ctrl_len = MAX_TRAILER; in siw_tx_hdt()
622 c_tx->ctrl_sent = rv + 4 - c_tx->pad; in siw_tx_hdt()
623 c_tx->bytes_unsent = 0; in siw_tx_hdt()
629 c_tx->state = SIW_SEND_DATA; in siw_tx_hdt()
639 c_tx->bytes_unsent -= rv; in siw_tx_hdt()
640 sge = &wqe->sqe.sge[c_tx->sge_idx]; in siw_tx_hdt()
641 sge_unsent = sge->length - c_tx->sge_off; in siw_tx_hdt()
645 c_tx->sge_idx++; in siw_tx_hdt()
646 c_tx->sge_off = 0; in siw_tx_hdt()
650 c_tx->sge_off += rv; in siw_tx_hdt()
655 c_tx->do_crc = 0; in siw_tx_hdt()
660 static void siw_update_tcpseg(struct siw_iwarp_tx *c_tx, in siw_update_tcpseg() argument
666 if (c_tx->gso_seg_limit == 0) in siw_update_tcpseg()
667 c_tx->tcp_seglen = tp->mss_cache * tp->gso_segs; in siw_update_tcpseg()
669 c_tx->tcp_seglen = in siw_update_tcpseg()
671 min_t(u16, c_tx->gso_seg_limit, tp->gso_segs); in siw_update_tcpseg()
673 c_tx->tcp_seglen = tp->mss_cache; in siw_update_tcpseg()
676 c_tx->tcp_seglen &= 0xfffffff8; in siw_update_tcpseg()
695 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_prepare_fpdu() local
698 c_tx->ctrl_len = in siw_prepare_fpdu()
699 iwarp_pktinfo[__rdmap_get_opcode(&c_tx->pkt.ctrl)].hdr_len; in siw_prepare_fpdu()
700 c_tx->ctrl_sent = 0; in siw_prepare_fpdu()
705 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) in siw_prepare_fpdu()
707 c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed); in siw_prepare_fpdu()
709 c_tx->pkt.c_tagged.ddp_to = in siw_prepare_fpdu()
713 if (data_len + c_tx->ctrl_len + MPA_CRC_SIZE > c_tx->tcp_seglen) { in siw_prepare_fpdu()
715 data_len = c_tx->tcp_seglen - (c_tx->ctrl_len + MPA_CRC_SIZE); in siw_prepare_fpdu()
716 c_tx->pkt.ctrl.ddp_rdmap_ctrl &= ~DDP_FLAG_LAST; in siw_prepare_fpdu()
717 c_tx->pad = 0; in siw_prepare_fpdu()
719 c_tx->pkt.ctrl.ddp_rdmap_ctrl |= DDP_FLAG_LAST; in siw_prepare_fpdu()
720 c_tx->pad = -data_len & 0x3; in siw_prepare_fpdu()
722 c_tx->bytes_unsent = data_len; in siw_prepare_fpdu()
724 c_tx->pkt.ctrl.mpa_len = in siw_prepare_fpdu()
725 htons(c_tx->ctrl_len + data_len - MPA_HDR_SIZE); in siw_prepare_fpdu()
730 if (c_tx->mpa_crc_hd) { in siw_prepare_fpdu()
731 crypto_shash_init(c_tx->mpa_crc_hd); in siw_prepare_fpdu()
732 crypto_shash_update(c_tx->mpa_crc_hd, (u8 *)&c_tx->pkt, in siw_prepare_fpdu()
733 c_tx->ctrl_len); in siw_prepare_fpdu()
734 c_tx->do_crc = 1; in siw_prepare_fpdu()
783 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_qp_sq_proc_tx() local
832 siw_update_tcpseg(c_tx, s); in siw_qp_sq_proc_tx()
834 rv = siw_qp_prepare_tx(c_tx); in siw_qp_sq_proc_tx()
836 c_tx->state = SIW_SEND_HDR; in siw_qp_sq_proc_tx()
839 c_tx->state = SIW_SEND_SHORT_FPDU; in siw_qp_sq_proc_tx()
854 if (c_tx->state == SIW_SEND_SHORT_FPDU) { in siw_qp_sq_proc_tx()
868 rv = siw_tx_ctrl(c_tx, s, msg_flags); in siw_qp_sq_proc_tx()
877 rv = siw_tx_hdt(c_tx, s); in siw_qp_sq_proc_tx()
884 if (unlikely(c_tx->tx_suspend)) { in siw_qp_sq_proc_tx()
893 if (c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_LAST) { in siw_qp_sq_proc_tx()
897 c_tx->state = SIW_SEND_HDR; in siw_qp_sq_proc_tx()
899 siw_update_tcpseg(c_tx, s); in siw_qp_sq_proc_tx()