Lines Matching refs:c_tx
40 static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr) in siw_try_1seg() argument
42 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_try_1seg()
61 } else if (c_tx->in_syscall) { in siw_try_1seg()
119 static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx) in siw_qp_prepare_tx() argument
121 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_qp_prepare_tx()
128 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
132 c_tx->pkt.rreq.rsvd = 0; in siw_qp_prepare_tx()
133 c_tx->pkt.rreq.ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ); in siw_qp_prepare_tx()
134 c_tx->pkt.rreq.ddp_msn = in siw_qp_prepare_tx()
135 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]); in siw_qp_prepare_tx()
136 c_tx->pkt.rreq.ddp_mo = 0; in siw_qp_prepare_tx()
137 c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey); in siw_qp_prepare_tx()
138 c_tx->pkt.rreq.sink_to = in siw_qp_prepare_tx()
140 c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey); in siw_qp_prepare_tx()
141 c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
142 c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length); in siw_qp_prepare_tx()
144 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rreq); in siw_qp_prepare_tx()
145 crc = (char *)&c_tx->pkt.rreq_pkt.crc; in siw_qp_prepare_tx()
150 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
154 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_SEND].ctrl, in siw_qp_prepare_tx()
157 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; in siw_qp_prepare_tx()
158 c_tx->pkt.send.ddp_msn = in siw_qp_prepare_tx()
159 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); in siw_qp_prepare_tx()
160 c_tx->pkt.send.ddp_mo = 0; in siw_qp_prepare_tx()
162 c_tx->pkt.send_inv.inval_stag = 0; in siw_qp_prepare_tx()
164 c_tx->ctrl_len = sizeof(struct iwarp_send); in siw_qp_prepare_tx()
166 crc = (char *)&c_tx->pkt.send_pkt.crc; in siw_qp_prepare_tx()
167 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
172 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
176 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
180 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; in siw_qp_prepare_tx()
181 c_tx->pkt.send.ddp_msn = in siw_qp_prepare_tx()
182 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); in siw_qp_prepare_tx()
183 c_tx->pkt.send.ddp_mo = 0; in siw_qp_prepare_tx()
185 c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey); in siw_qp_prepare_tx()
187 c_tx->ctrl_len = sizeof(struct iwarp_send_inv); in siw_qp_prepare_tx()
189 crc = (char *)&c_tx->pkt.send_pkt.crc; in siw_qp_prepare_tx()
190 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
194 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_RDMA_WRITE].ctrl, in siw_qp_prepare_tx()
197 c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey); in siw_qp_prepare_tx()
198 c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
199 c_tx->ctrl_len = sizeof(struct iwarp_rdma_write); in siw_qp_prepare_tx()
201 crc = (char *)&c_tx->pkt.write_pkt.crc; in siw_qp_prepare_tx()
202 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
206 memcpy(&c_tx->pkt.ctrl, in siw_qp_prepare_tx()
211 c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey); in siw_qp_prepare_tx()
212 c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_qp_prepare_tx()
214 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rresp); in siw_qp_prepare_tx()
216 crc = (char *)&c_tx->pkt.write_pkt.crc; in siw_qp_prepare_tx()
217 data = siw_try_1seg(c_tx, crc); in siw_qp_prepare_tx()
221 siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe)); in siw_qp_prepare_tx()
227 c_tx->ctrl_sent = 0; in siw_qp_prepare_tx()
233 c_tx->pkt.ctrl.mpa_len = in siw_qp_prepare_tx()
234 htons(c_tx->ctrl_len + data - MPA_HDR_SIZE); in siw_qp_prepare_tx()
240 c_tx->ctrl_len += data; in siw_qp_prepare_tx()
242 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) in siw_qp_prepare_tx()
243 c_tx->pkt.c_untagged.ddp_mo = 0; in siw_qp_prepare_tx()
245 c_tx->pkt.c_tagged.ddp_to = in siw_qp_prepare_tx()
253 if (c_tx->mpa_crc_hd) { in siw_qp_prepare_tx()
254 crypto_shash_init(c_tx->mpa_crc_hd); in siw_qp_prepare_tx()
255 if (crypto_shash_update(c_tx->mpa_crc_hd, in siw_qp_prepare_tx()
256 (u8 *)&c_tx->pkt, in siw_qp_prepare_tx()
257 c_tx->ctrl_len)) in siw_qp_prepare_tx()
259 crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)crc); in siw_qp_prepare_tx()
261 c_tx->ctrl_len += MPA_CRC_SIZE; in siw_qp_prepare_tx()
265 c_tx->ctrl_len += MPA_CRC_SIZE; in siw_qp_prepare_tx()
266 c_tx->sge_idx = 0; in siw_qp_prepare_tx()
267 c_tx->sge_off = 0; in siw_qp_prepare_tx()
268 c_tx->pbl_idx = 0; in siw_qp_prepare_tx()
279 if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH && in siw_qp_prepare_tx()
281 c_tx->use_sendpage = 1; in siw_qp_prepare_tx()
283 c_tx->use_sendpage = 0; in siw_qp_prepare_tx()
293 static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s, in siw_tx_ctrl() argument
298 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent, in siw_tx_ctrl()
299 .iov_len = c_tx->ctrl_len - c_tx->ctrl_sent }; in siw_tx_ctrl()
302 c_tx->ctrl_len - c_tx->ctrl_sent); in siw_tx_ctrl()
305 c_tx->ctrl_sent += rv; in siw_tx_ctrl()
307 if (c_tx->ctrl_sent == c_tx->ctrl_len) in siw_tx_ctrl()
431 static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s) in siw_tx_hdt() argument
433 struct siw_wqe *wqe = &c_tx->wqe_active; in siw_tx_hdt()
434 struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx]; in siw_tx_hdt()
439 int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv; in siw_tx_hdt()
440 unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0, in siw_tx_hdt()
441 sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx, in siw_tx_hdt()
442 pbl_idx = c_tx->pbl_idx; in siw_tx_hdt()
445 if (c_tx->state == SIW_SEND_HDR) { in siw_tx_hdt()
446 if (c_tx->use_sendpage) { in siw_tx_hdt()
447 rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE); in siw_tx_hdt()
451 c_tx->state = SIW_SEND_DATA; in siw_tx_hdt()
454 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent; in siw_tx_hdt()
456 c_tx->ctrl_len - c_tx->ctrl_sent; in siw_tx_hdt()
474 if (is_kva && !c_tx->use_sendpage) { in siw_tx_hdt()
484 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
509 wqe->processed -= c_tx->bytes_unsent; in siw_tx_hdt()
515 if (!c_tx->use_sendpage) { in siw_tx_hdt()
525 c_tx->mpa_crc_hd, in siw_tx_hdt()
530 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
551 c_tx->mpa_crc_hd, in siw_tx_hdt()
562 siw_dbg_qp(tx_qp(c_tx), "to many fragments\n"); in siw_tx_hdt()
564 wqe->processed -= c_tx->bytes_unsent; in siw_tx_hdt()
579 if (likely(c_tx->state != SIW_SEND_TRAILER)) { in siw_tx_hdt()
580 iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad]; in siw_tx_hdt()
581 iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad); in siw_tx_hdt()
583 iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent]; in siw_tx_hdt()
584 iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent; in siw_tx_hdt()
587 if (c_tx->pad) { in siw_tx_hdt()
588 *(u32 *)c_tx->trailer.pad = 0; in siw_tx_hdt()
590 crypto_shash_update(c_tx->mpa_crc_hd, in siw_tx_hdt()
591 (u8 *)&c_tx->trailer.crc - c_tx->pad, in siw_tx_hdt()
592 c_tx->pad); in siw_tx_hdt()
594 if (!c_tx->mpa_crc_hd) in siw_tx_hdt()
595 c_tx->trailer.crc = 0; in siw_tx_hdt()
597 crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)&c_tx->trailer.crc); in siw_tx_hdt()
599 data_len = c_tx->bytes_unsent; in siw_tx_hdt()
601 if (c_tx->use_sendpage) { in siw_tx_hdt()
602 rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx], in siw_tx_hdt()
603 c_tx->sge_off, data_len); in siw_tx_hdt()
620 c_tx->ctrl_sent += rv; in siw_tx_hdt()
631 c_tx->sge_idx = sge_idx; in siw_tx_hdt()
632 c_tx->sge_off = sge_off; in siw_tx_hdt()
633 c_tx->pbl_idx = pbl_idx; in siw_tx_hdt()
640 c_tx->state = SIW_SEND_TRAILER; in siw_tx_hdt()
641 c_tx->ctrl_len = MAX_TRAILER; in siw_tx_hdt()
642 c_tx->ctrl_sent = rv + 4 - c_tx->pad; in siw_tx_hdt()
643 c_tx->bytes_unsent = 0; in siw_tx_hdt()
649 c_tx->state = SIW_SEND_DATA; in siw_tx_hdt()
659 c_tx->bytes_unsent -= rv; in siw_tx_hdt()
660 sge = &wqe->sqe.sge[c_tx->sge_idx]; in siw_tx_hdt()
661 sge_unsent = sge->length - c_tx->sge_off; in siw_tx_hdt()
665 c_tx->sge_idx++; in siw_tx_hdt()
666 c_tx->sge_off = 0; in siw_tx_hdt()
670 c_tx->sge_off += rv; in siw_tx_hdt()
675 c_tx->do_crc = 0; in siw_tx_hdt()
680 static void siw_update_tcpseg(struct siw_iwarp_tx *c_tx, in siw_update_tcpseg() argument
686 if (c_tx->gso_seg_limit == 0) in siw_update_tcpseg()
687 c_tx->tcp_seglen = tp->mss_cache * tp->gso_segs; in siw_update_tcpseg()
689 c_tx->tcp_seglen = in siw_update_tcpseg()
691 min_t(u16, c_tx->gso_seg_limit, tp->gso_segs); in siw_update_tcpseg()
693 c_tx->tcp_seglen = tp->mss_cache; in siw_update_tcpseg()
696 c_tx->tcp_seglen &= 0xfffffff8; in siw_update_tcpseg()
715 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_prepare_fpdu() local
718 c_tx->ctrl_len = in siw_prepare_fpdu()
719 iwarp_pktinfo[__rdmap_get_opcode(&c_tx->pkt.ctrl)].hdr_len; in siw_prepare_fpdu()
720 c_tx->ctrl_sent = 0; in siw_prepare_fpdu()
725 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) in siw_prepare_fpdu()
727 c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed); in siw_prepare_fpdu()
729 c_tx->pkt.c_tagged.ddp_to = in siw_prepare_fpdu()
733 if (data_len + c_tx->ctrl_len + MPA_CRC_SIZE > c_tx->tcp_seglen) { in siw_prepare_fpdu()
735 data_len = c_tx->tcp_seglen - (c_tx->ctrl_len + MPA_CRC_SIZE); in siw_prepare_fpdu()
736 c_tx->pkt.ctrl.ddp_rdmap_ctrl &= ~DDP_FLAG_LAST; in siw_prepare_fpdu()
737 c_tx->pad = 0; in siw_prepare_fpdu()
739 c_tx->pkt.ctrl.ddp_rdmap_ctrl |= DDP_FLAG_LAST; in siw_prepare_fpdu()
740 c_tx->pad = -data_len & 0x3; in siw_prepare_fpdu()
742 c_tx->bytes_unsent = data_len; in siw_prepare_fpdu()
744 c_tx->pkt.ctrl.mpa_len = in siw_prepare_fpdu()
745 htons(c_tx->ctrl_len + data_len - MPA_HDR_SIZE); in siw_prepare_fpdu()
750 if (c_tx->mpa_crc_hd) { in siw_prepare_fpdu()
751 crypto_shash_init(c_tx->mpa_crc_hd); in siw_prepare_fpdu()
752 crypto_shash_update(c_tx->mpa_crc_hd, (u8 *)&c_tx->pkt, in siw_prepare_fpdu()
753 c_tx->ctrl_len); in siw_prepare_fpdu()
754 c_tx->do_crc = 1; in siw_prepare_fpdu()
803 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_qp_sq_proc_tx() local
852 siw_update_tcpseg(c_tx, s); in siw_qp_sq_proc_tx()
854 rv = siw_qp_prepare_tx(c_tx); in siw_qp_sq_proc_tx()
856 c_tx->state = SIW_SEND_HDR; in siw_qp_sq_proc_tx()
859 c_tx->state = SIW_SEND_SHORT_FPDU; in siw_qp_sq_proc_tx()
874 if (c_tx->state == SIW_SEND_SHORT_FPDU) { in siw_qp_sq_proc_tx()
888 rv = siw_tx_ctrl(c_tx, s, msg_flags); in siw_qp_sq_proc_tx()
897 rv = siw_tx_hdt(c_tx, s); in siw_qp_sq_proc_tx()
904 if (unlikely(c_tx->tx_suspend)) { in siw_qp_sq_proc_tx()
913 if (c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_LAST) { in siw_qp_sq_proc_tx()
917 c_tx->state = SIW_SEND_HDR; in siw_qp_sq_proc_tx()
919 siw_update_tcpseg(c_tx, s); in siw_qp_sq_proc_tx()