1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
3  * Copyright (c) 2015-2017  QLogic Corporation
4  * Copyright (c) 2019-2020 Marvell International Ltd.
69 	struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;  in qed_iwarp_init_devinfo()
71 	dev->max_inline = IWARP_REQ_MAX_INLINE_DATA_SIZE;  in qed_iwarp_init_devinfo()
72 	dev->max_qp = min_t(u32,  in qed_iwarp_init_devinfo()
74 			    p_hwfn->p_rdma_info->num_qps) -  in qed_iwarp_init_devinfo()
77 	dev->max_cq = dev->max_qp;  in qed_iwarp_init_devinfo()
79 	dev->max_qp_resp_rd_atomic_resc = QED_IWARP_IRD_DEFAULT;  in qed_iwarp_init_devinfo()
80 	dev->max_qp_req_rd_atomic_resc = QED_IWARP_ORD_DEFAULT;  in qed_iwarp_init_devinfo()
85 	p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_TCP;  in qed_iwarp_init_hw()
86 	qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 1);  in qed_iwarp_init_hw()
87 	p_hwfn->b_rdma_enabled_in_prs = true;  in qed_iwarp_init_hw()
91  * syn processing and replacing a pre-allocated ep in the list. The second
96 	cid -= qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto);  in qed_iwarp_cid_cleaned()
98 	spin_lock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_cid_cleaned()
101 		qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map,  in qed_iwarp_cid_cleaned()
104 		qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid);  in qed_iwarp_cid_cleaned()
106 	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_cid_cleaned()
113 	p_ramrod->iwarp.ll2_ooo_q_index =  in qed_iwarp_init_fw_ramrod()
115 	    p_hwfn->p_rdma_info->iwarp.ll2_ooo_handle;  in qed_iwarp_init_fw_ramrod()
117 	p_ramrod->tcp.max_fin_rt = QED_IWARP_MAX_FIN_RT_DEFAULT;  in qed_iwarp_init_fw_ramrod()
124 	int rc;  in qed_iwarp_alloc_cid()  local
126 	spin_lock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_alloc_cid()
127 	rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid);  in qed_iwarp_alloc_cid()
128 	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_alloc_cid()
129 	if (rc) {  in qed_iwarp_alloc_cid()
131 		return rc;  in qed_iwarp_alloc_cid()
133 	*cid += qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto);  in qed_iwarp_alloc_cid()
135 	rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *cid);  in qed_iwarp_alloc_cid()
136 	if (rc)  in qed_iwarp_alloc_cid()
139 	return rc;  in qed_iwarp_alloc_cid()
144 	cid -= qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto);  in qed_iwarp_set_tcp_cid()
146 	spin_lock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_set_tcp_cid()
147 	qed_bmap_set_id(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, cid);  in qed_iwarp_set_tcp_cid()
148 	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_set_tcp_cid()
158 	int rc;  in qed_iwarp_alloc_tcp_cid()  local
160 	spin_lock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_alloc_tcp_cid()
162 	rc = qed_rdma_bmap_alloc_id(p_hwfn,  in qed_iwarp_alloc_tcp_cid()
163 				    &p_hwfn->p_rdma_info->tcp_cid_map, cid);  in qed_iwarp_alloc_tcp_cid()
165 	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);  in qed_iwarp_alloc_tcp_cid()
167 	if (rc) {  in qed_iwarp_alloc_tcp_cid()
169 			   "can't allocate iwarp tcp cid max-count=%d\n",  in qed_iwarp_alloc_tcp_cid()
170 			   p_hwfn->p_rdma_info->tcp_cid_map.max_count);  in qed_iwarp_alloc_tcp_cid()
173 		return rc;  in qed_iwarp_alloc_tcp_cid()
177 					    p_hwfn->p_rdma_info->proto);  in qed_iwarp_alloc_tcp_cid()
190 	int rc;  in qed_iwarp_create_qp()  local
192 	qp->shared_queue = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,  in qed_iwarp_create_qp()
194 					      &qp->shared_queue_phys_addr,  in qed_iwarp_create_qp()
196 	if (!qp->shared_queue)  in qed_iwarp_create_qp()
197 		return -ENOMEM;  in qed_iwarp_create_qp()
199 	out_params->sq_pbl_virt = (u8 *)qp->shared_queue +  in qed_iwarp_create_qp()
201 	out_params->sq_pbl_phys = qp->shared_queue_phys_addr +  in qed_iwarp_create_qp()
203 	out_params->rq_pbl_virt = (u8 *)qp->shared_queue +  in qed_iwarp_create_qp()
205 	out_params->rq_pbl_phys = qp->shared_queue_phys_addr +  in qed_iwarp_create_qp()
208 	rc = qed_iwarp_alloc_cid(p_hwfn, &cid);  in qed_iwarp_create_qp()
209 	if (rc)  in qed_iwarp_create_qp()
212 	qp->icid = (u16)cid;  in qed_iwarp_create_qp()
215 	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;  in qed_iwarp_create_qp()
216 	init_data.cid = qp->icid;  in qed_iwarp_create_qp()
219 	rc = qed_sp_init_request(p_hwfn, &p_ent,  in qed_iwarp_create_qp()
222 	if (rc)  in qed_iwarp_create_qp()
225 	p_ramrod = &p_ent->ramrod.iwarp_create_qp;  in qed_iwarp_create_qp()
227 	SET_FIELD(p_ramrod->flags,  in qed_iwarp_create_qp()
229 		  qp->fmr_and_reserved_lkey);  in qed_iwarp_create_qp()
231 	SET_FIELD(p_ramrod->flags,  in qed_iwarp_create_qp()
232 		  IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP, qp->signal_all);  in qed_iwarp_create_qp()
234 	SET_FIELD(p_ramrod->flags,  in qed_iwarp_create_qp()
236 		  qp->incoming_rdma_read_en);  in qed_iwarp_create_qp()
238 	SET_FIELD(p_ramrod->flags,  in qed_iwarp_create_qp()
240 		  qp->incoming_rdma_write_en);  in qed_iwarp_create_qp()
242 	SET_FIELD(p_ramrod->flags,  in qed_iwarp_create_qp()
244 		  qp->incoming_atomic_en);  in qed_iwarp_create_qp()
246 	SET_FIELD(p_ramrod->flags,  in qed_iwarp_create_qp()
247 		  IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG, qp->use_srq);  in qed_iwarp_create_qp()
249 	p_ramrod->pd = cpu_to_le16(qp->pd);  in qed_iwarp_create_qp()
250 	p_ramrod->sq_num_pages = cpu_to_le16(qp->sq_num_pages);  in qed_iwarp_create_qp()
251 	p_ramrod->rq_num_pages = cpu_to_le16(qp->rq_num_pages);  in qed_iwarp_create_qp()
253 	p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id);  in qed_iwarp_create_qp()
254 	p_ramrod->srq_id.opaque_fid = cpu_to_le16(p_hwfn->hw_info.opaque_fid);  in qed_iwarp_create_qp()
255 	p_ramrod->qp_handle_for_cqe.hi = qp->qp_handle.hi;  in qed_iwarp_create_qp()
256 	p_ramrod->qp_handle_for_cqe.lo = qp->qp_handle.lo;  in qed_iwarp_create_qp()
258 	p_ramrod->cq_cid_for_sq =  in qed_iwarp_create_qp()
259 	    cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->sq_cq_id);  in qed_iwarp_create_qp()
260 	p_ramrod->cq_cid_for_rq =  in qed_iwarp_create_qp()
261 	    cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->rq_cq_id);  in qed_iwarp_create_qp()
263 	p_ramrod->dpi = cpu_to_le16(qp->dpi);  in qed_iwarp_create_qp()
266 	p_ramrod->physical_q0 = cpu_to_le16(physical_queue);  in qed_iwarp_create_qp()
268 	p_ramrod->physical_q1 = cpu_to_le16(physical_queue);  in qed_iwarp_create_qp()
270 	rc = qed_spq_post(p_hwfn, p_ent, NULL);  in qed_iwarp_create_qp()
271 	if (rc)  in qed_iwarp_create_qp()
274 	return rc;  in qed_iwarp_create_qp()
279 	dma_free_coherent(&p_hwfn->cdev->pdev->dev,  in qed_iwarp_create_qp()
281 			  qp->shared_queue, qp->shared_queue_phys_addr);  in qed_iwarp_create_qp()
283 	return rc;  in qed_iwarp_create_qp()
292 	int rc;  in qed_iwarp_modify_fw()  local
296 	init_data.cid = qp->icid;  in qed_iwarp_modify_fw()
297 	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;  in qed_iwarp_modify_fw()
300 	rc = qed_sp_init_request(p_hwfn, &p_ent,  in qed_iwarp_modify_fw()
302 				 p_hwfn->p_rdma_info->proto, &init_data);  in qed_iwarp_modify_fw()
303 	if (rc)  in qed_iwarp_modify_fw()
304 		return rc;  in qed_iwarp_modify_fw()
306 	p_ramrod = &p_ent->ramrod.iwarp_modify_qp;  in qed_iwarp_modify_fw()
308 	flags = le16_to_cpu(p_ramrod->flags);  in qed_iwarp_modify_fw()
310 	p_ramrod->flags = cpu_to_le16(flags);  in qed_iwarp_modify_fw()
312 	if (qp->iwarp_state == QED_IWARP_QP_STATE_CLOSING)  in qed_iwarp_modify_fw()
317 	p_ramrod->transition_to_state = cpu_to_le16(trans_to_state);  in qed_iwarp_modify_fw()
319 	rc = qed_spq_post(p_hwfn, p_ent, NULL);  in qed_iwarp_modify_fw()
321 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x)rc=%d\n", qp->icid, rc);  in qed_iwarp_modify_fw()
323 	return rc;  in qed_iwarp_modify_fw()
380 	int rc = 0;  in qed_iwarp_modify_qp()  local
382 	/* modify QP can be called from upper-layer or as a result of async  in qed_iwarp_modify_qp()
385 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock);  in qed_iwarp_modify_qp()
386 	prev_iw_state = qp->iwarp_state;  in qed_iwarp_modify_qp()
389 		spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock);  in qed_iwarp_modify_qp()
397 			qp->iwarp_state = QED_IWARP_QP_STATE_RTS;  in qed_iwarp_modify_qp()
400 			qp->iwarp_state = QED_IWARP_QP_STATE_ERROR;  in qed_iwarp_modify_qp()
414 			qp->iwarp_state = QED_IWARP_QP_STATE_CLOSING;  in qed_iwarp_modify_qp()
419 			qp->iwarp_state = QED_IWARP_QP_STATE_ERROR;  in qed_iwarp_modify_qp()
429 			qp->iwarp_state = new_state;  in qed_iwarp_modify_qp()
435 			rc = -EINVAL;  in qed_iwarp_modify_qp()
440 		qp->iwarp_state = new_state;  in qed_iwarp_modify_qp()
446 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) %s --> %s%s\n",  in qed_iwarp_modify_qp()
447 		   qp->icid,  in qed_iwarp_modify_qp()
449 		   iwarp_state_names[qp->iwarp_state],  in qed_iwarp_modify_qp()
452 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock);  in qed_iwarp_modify_qp()
455 		rc = qed_iwarp_modify_fw(p_hwfn, qp);  in qed_iwarp_modify_qp()
457 	return rc;  in qed_iwarp_modify_qp()
464 	int rc;  in qed_iwarp_fw_destroy()  local
468 	init_data.cid = qp->icid;  in qed_iwarp_fw_destroy()
469 	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;  in qed_iwarp_fw_destroy()
472 	rc = qed_sp_init_request(p_hwfn, &p_ent,  in qed_iwarp_fw_destroy()
474 				 p_hwfn->p_rdma_info->proto, &init_data);  in qed_iwarp_fw_destroy()
475 	if (rc)  in qed_iwarp_fw_destroy()
476 		return rc;  in qed_iwarp_fw_destroy()
478 	rc = qed_spq_post(p_hwfn, p_ent, NULL);  in qed_iwarp_fw_destroy()
480 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) rc = %d\n", qp->icid, rc);  in qed_iwarp_fw_destroy()
482 	return rc;  in qed_iwarp_fw_destroy()
489 	dma_free_coherent(&p_hwfn->cdev->pdev->dev,  in qed_iwarp_destroy_ep()
490 			  sizeof(*ep->ep_buffer_virt),  in qed_iwarp_destroy_ep()
491 			  ep->ep_buffer_virt, ep->ep_buffer_phys);  in qed_iwarp_destroy_ep()
494 		spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_destroy_ep()
495 		list_del(&ep->list_entry);  in qed_iwarp_destroy_ep()
496 		spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_destroy_ep()
499 	if (ep->qp)  in qed_iwarp_destroy_ep()
500 		ep->qp->ep = NULL;  in qed_iwarp_destroy_ep()
507 	struct qed_iwarp_ep *ep = qp->ep;  in qed_iwarp_destroy_qp()
509 	int rc = 0;  in qed_iwarp_destroy_qp()  local
511 	if (qp->iwarp_state != QED_IWARP_QP_STATE_ERROR) {  in qed_iwarp_destroy_qp()
512 		rc = qed_iwarp_modify_qp(p_hwfn, qp,  in qed_iwarp_destroy_qp()
514 		if (rc)  in qed_iwarp_destroy_qp()
515 			return rc;  in qed_iwarp_destroy_qp()
520 		while (READ_ONCE(ep->state) != QED_IWARP_EP_CLOSED &&  in qed_iwarp_destroy_qp()
524 		if (ep->state != QED_IWARP_EP_CLOSED)  in qed_iwarp_destroy_qp()
525 			DP_NOTICE(p_hwfn, "ep state close timeout state=%x\n",  in qed_iwarp_destroy_qp()
526 				  ep->state);  in qed_iwarp_destroy_qp()
531 	rc = qed_iwarp_fw_destroy(p_hwfn, qp);  in qed_iwarp_destroy_qp()
533 	if (qp->shared_queue)  in qed_iwarp_destroy_qp()
534 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,  in qed_iwarp_destroy_qp()
536 				  qp->shared_queue, qp->shared_queue_phys_addr);  in qed_iwarp_destroy_qp()
538 	return rc;  in qed_iwarp_destroy_qp()
545 	int rc;  in qed_iwarp_create_ep()  local
549 		return -ENOMEM;  in qed_iwarp_create_ep()
551 	ep->state = QED_IWARP_EP_INIT;  in qed_iwarp_create_ep()
553 	ep->ep_buffer_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,  in qed_iwarp_create_ep()
554 						sizeof(*ep->ep_buffer_virt),  in qed_iwarp_create_ep()
555 						&ep->ep_buffer_phys,  in qed_iwarp_create_ep()
557 	if (!ep->ep_buffer_virt) {  in qed_iwarp_create_ep()
558 		rc = -ENOMEM;  in qed_iwarp_create_ep()
562 	ep->sig = QED_EP_SIG;  in qed_iwarp_create_ep()
570 	return rc;  in qed_iwarp_create_ep()
577 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "local_mac=%x %x %x, remote_mac=%x %x %x\n",  in qed_iwarp_print_tcp_ramrod()
578 		   p_tcp_ramrod->tcp.local_mac_addr_lo,  in qed_iwarp_print_tcp_ramrod()
579 		   p_tcp_ramrod->tcp.local_mac_addr_mid,  in qed_iwarp_print_tcp_ramrod()
580 		   p_tcp_ramrod->tcp.local_mac_addr_hi,  in qed_iwarp_print_tcp_ramrod()
581 		   p_tcp_ramrod->tcp.remote_mac_addr_lo,  in qed_iwarp_print_tcp_ramrod()
582 		   p_tcp_ramrod->tcp.remote_mac_addr_mid,  in qed_iwarp_print_tcp_ramrod()
583 		   p_tcp_ramrod->tcp.remote_mac_addr_hi);  in qed_iwarp_print_tcp_ramrod()
585 	if (p_tcp_ramrod->tcp.ip_version == TCP_IPV4) {  in qed_iwarp_print_tcp_ramrod()
587 			   "local_ip=%pI4h:%x, remote_ip=%pI4h:%x, vlan=%x\n",  in qed_iwarp_print_tcp_ramrod()
588 			   p_tcp_ramrod->tcp.local_ip,  in qed_iwarp_print_tcp_ramrod()
589 			   p_tcp_ramrod->tcp.local_port,  in qed_iwarp_print_tcp_ramrod()
590 			   p_tcp_ramrod->tcp.remote_ip,  in qed_iwarp_print_tcp_ramrod()
591 			   p_tcp_ramrod->tcp.remote_port,  in qed_iwarp_print_tcp_ramrod()
592 			   p_tcp_ramrod->tcp.vlan_id);  in qed_iwarp_print_tcp_ramrod()
595 			   "local_ip=%pI6:%x, remote_ip=%pI6:%x, vlan=%x\n",  in qed_iwarp_print_tcp_ramrod()
596 			   p_tcp_ramrod->tcp.local_ip,  in qed_iwarp_print_tcp_ramrod()
597 			   p_tcp_ramrod->tcp.local_port,  in qed_iwarp_print_tcp_ramrod()
598 			   p_tcp_ramrod->tcp.remote_ip,  in qed_iwarp_print_tcp_ramrod()
599 			   p_tcp_ramrod->tcp.remote_port,  in qed_iwarp_print_tcp_ramrod()
600 			   p_tcp_ramrod->tcp.vlan_id);  in qed_iwarp_print_tcp_ramrod()
604 		   "flow_label=%x, ttl=%x, tos_or_tc=%x, mss=%x, rcv_wnd_scale=%x, connect_mode=%x, flags=%x\n",  in qed_iwarp_print_tcp_ramrod()
605 		   p_tcp_ramrod->tcp.flow_label,  in qed_iwarp_print_tcp_ramrod()
606 		   p_tcp_ramrod->tcp.ttl,  in qed_iwarp_print_tcp_ramrod()
607 		   p_tcp_ramrod->tcp.tos_or_tc,  in qed_iwarp_print_tcp_ramrod()
608 		   p_tcp_ramrod->tcp.mss,  in qed_iwarp_print_tcp_ramrod()
609 		   p_tcp_ramrod->tcp.rcv_wnd_scale,  in qed_iwarp_print_tcp_ramrod()
610 		   p_tcp_ramrod->tcp.connect_mode,  in qed_iwarp_print_tcp_ramrod()
611 		   p_tcp_ramrod->tcp.flags);  in qed_iwarp_print_tcp_ramrod()
613 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "syn_ip_payload_length=%x, lo=%x, hi=%x\n",  in qed_iwarp_print_tcp_ramrod()
614 		   p_tcp_ramrod->tcp.syn_ip_payload_length,  in qed_iwarp_print_tcp_ramrod()
615 		   p_tcp_ramrod->tcp.syn_phy_addr_lo,  in qed_iwarp_print_tcp_ramrod()
616 		   p_tcp_ramrod->tcp.syn_phy_addr_hi);  in qed_iwarp_print_tcp_ramrod()
622 	struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_tcp_offload()
632 	int rc;  in qed_iwarp_tcp_offload()  local
636 	init_data.cid = ep->tcp_cid;  in qed_iwarp_tcp_offload()
637 	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;  in qed_iwarp_tcp_offload()
638 	if (ep->connect_mode == TCP_CONNECT_PASSIVE)  in qed_iwarp_tcp_offload()
643 	rc = qed_sp_init_request(p_hwfn, &p_ent,  in qed_iwarp_tcp_offload()
646 	if (rc)  in qed_iwarp_tcp_offload()
647 		return rc;  in qed_iwarp_tcp_offload()
649 	p_tcp_ramrod = &p_ent->ramrod.iwarp_tcp_offload;  in qed_iwarp_tcp_offload()
651 	in_pdata_phys = ep->ep_buffer_phys +  in qed_iwarp_tcp_offload()
653 	DMA_REGPAIR_LE(p_tcp_ramrod->iwarp.incoming_ulp_buffer.addr,  in qed_iwarp_tcp_offload()
656 	p_tcp_ramrod->iwarp.incoming_ulp_buffer.len =  in qed_iwarp_tcp_offload()
657 	    cpu_to_le16(sizeof(ep->ep_buffer_virt->in_pdata));  in qed_iwarp_tcp_offload()
659 	async_output_phys = ep->ep_buffer_phys +  in qed_iwarp_tcp_offload()
661 	DMA_REGPAIR_LE(p_tcp_ramrod->iwarp.async_eqe_output_buf,  in qed_iwarp_tcp_offload()
664 	p_tcp_ramrod->iwarp.handle_for_async.hi = cpu_to_le32(PTR_HI(ep));  in qed_iwarp_tcp_offload()
665 	p_tcp_ramrod->iwarp.handle_for_async.lo = cpu_to_le32(PTR_LO(ep));  in qed_iwarp_tcp_offload()
668 	p_tcp_ramrod->iwarp.physical_q0 = cpu_to_le16(physical_q);  in qed_iwarp_tcp_offload()
670 	p_tcp_ramrod->iwarp.physical_q1 = cpu_to_le16(physical_q);  in qed_iwarp_tcp_offload()
671 	p_tcp_ramrod->iwarp.mpa_mode = iwarp_info->mpa_rev;  in qed_iwarp_tcp_offload()
673 	tcp = &p_tcp_ramrod->tcp;  in qed_iwarp_tcp_offload()
674 	qed_set_fw_mac_addr(&tcp->remote_mac_addr_hi,  in qed_iwarp_tcp_offload()
675 			    &tcp->remote_mac_addr_mid,  in qed_iwarp_tcp_offload()
676 			    &tcp->remote_mac_addr_lo, ep->remote_mac_addr);  in qed_iwarp_tcp_offload()
677 	qed_set_fw_mac_addr(&tcp->local_mac_addr_hi, &tcp->local_mac_addr_mid,  in qed_iwarp_tcp_offload()
678 			    &tcp->local_mac_addr_lo, ep->local_mac_addr);  in qed_iwarp_tcp_offload()
680 	tcp->vlan_id = cpu_to_le16(ep->cm_info.vlan);  in qed_iwarp_tcp_offload()
682 	tcp_flags = p_hwfn->p_rdma_info->iwarp.tcp_flags;  in qed_iwarp_tcp_offload()
690 	tcp->flags = cpu_to_le16(flags);  in qed_iwarp_tcp_offload()
691 	tcp->ip_version = ep->cm_info.ip_version;  in qed_iwarp_tcp_offload()
694 		tcp->remote_ip[i] = cpu_to_le32(ep->cm_info.remote_ip[i]);  in qed_iwarp_tcp_offload()
695 		tcp->local_ip[i] = cpu_to_le32(ep->cm_info.local_ip[i]);  in qed_iwarp_tcp_offload()
698 	tcp->remote_port = cpu_to_le16(ep->cm_info.remote_port);  in qed_iwarp_tcp_offload()
699 	tcp->local_port = cpu_to_le16(ep->cm_info.local_port);  in qed_iwarp_tcp_offload()
700 	tcp->mss = cpu_to_le16(ep->mss);  in qed_iwarp_tcp_offload()
701 	tcp->flow_label = 0;  in qed_iwarp_tcp_offload()
702 	tcp->ttl = 0x40;  in qed_iwarp_tcp_offload()
703 	tcp->tos_or_tc = 0;  in qed_iwarp_tcp_offload()
705 	tcp->max_rt_time = QED_IWARP_DEF_MAX_RT_TIME;  in qed_iwarp_tcp_offload()
706 	tcp->cwnd = cpu_to_le32(QED_IWARP_DEF_CWND_FACTOR * ep->mss);  in qed_iwarp_tcp_offload()
707 	tcp->ka_max_probe_cnt = QED_IWARP_DEF_KA_MAX_PROBE_CNT;  in qed_iwarp_tcp_offload()
708 	tcp->ka_timeout = cpu_to_le32(QED_IWARP_DEF_KA_TIMEOUT);  in qed_iwarp_tcp_offload()
709 	tcp->ka_interval = cpu_to_le32(QED_IWARP_DEF_KA_INTERVAL);  in qed_iwarp_tcp_offload()
711 	tcp->rcv_wnd_scale = (u8)p_hwfn->p_rdma_info->iwarp.rcv_wnd_scale;  in qed_iwarp_tcp_offload()
712 	tcp->connect_mode = ep->connect_mode;  in qed_iwarp_tcp_offload()
714 	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {  in qed_iwarp_tcp_offload()
715 		tcp->syn_ip_payload_length =  in qed_iwarp_tcp_offload()
716 			cpu_to_le16(ep->syn_ip_payload_length);  in qed_iwarp_tcp_offload()
717 		tcp->syn_phy_addr_hi = DMA_HI_LE(ep->syn_phy_addr);  in qed_iwarp_tcp_offload()
718 		tcp->syn_phy_addr_lo = DMA_LO_LE(ep->syn_phy_addr);  in qed_iwarp_tcp_offload()
723 	rc = qed_spq_post(p_hwfn, p_ent, NULL);  in qed_iwarp_tcp_offload()
726 		   "EP(0x%x) Offload completed rc=%d\n", ep->tcp_cid, rc);  in qed_iwarp_tcp_offload()
728 	return rc;  in qed_iwarp_tcp_offload()
734 	struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_mpa_received()
743 	async_data = &ep->ep_buffer_virt->async_output;  in qed_iwarp_mpa_received()
745 	mpa_rev = async_data->mpa_request.mpa_handshake_mode;  in qed_iwarp_mpa_received()
747 		   "private_data_len=%x handshake_mode=%x private_data=(%x)\n",  in qed_iwarp_mpa_received()
748 		   async_data->mpa_request.ulp_data_len,  in qed_iwarp_mpa_received()
749 		   mpa_rev, *((u32 *)(ep->ep_buffer_virt->in_pdata)));  in qed_iwarp_mpa_received()
753 		mpa_v2 = (struct mpa_v2_hdr *)ep->ep_buffer_virt->in_pdata;  in qed_iwarp_mpa_received()
756 		mpa_ord = ntohs(mpa_v2->ord);  in qed_iwarp_mpa_received()
757 		mpa_ird = ntohs(mpa_v2->ird);  in qed_iwarp_mpa_received()
762 		ep->cm_info.ord = (u8)min_t(u16,  in qed_iwarp_mpa_received()
766 		ep->cm_info.ird = (u8)min_t(u16,  in qed_iwarp_mpa_received()
771 		ep->rtr_type = MPA_RTR_TYPE_NONE;  in qed_iwarp_mpa_received()
774 				ep->rtr_type |= MPA_RTR_TYPE_ZERO_WRITE;  in qed_iwarp_mpa_received()
777 				ep->rtr_type |= MPA_RTR_TYPE_ZERO_READ;  in qed_iwarp_mpa_received()
780 				ep->rtr_type |= MPA_RTR_TYPE_ZERO_SEND;  in qed_iwarp_mpa_received()
782 			ep->rtr_type &= iwarp_info->rtr_type;  in qed_iwarp_mpa_received()
785 			if (ep->rtr_type == MPA_RTR_TYPE_NONE)  in qed_iwarp_mpa_received()
786 				ep->rtr_type = iwarp_info->rtr_type;  in qed_iwarp_mpa_received()
789 		ep->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED;  in qed_iwarp_mpa_received()
791 		ep->cm_info.ord = QED_IWARP_ORD_DEFAULT;  in qed_iwarp_mpa_received()
792 		ep->cm_info.ird = QED_IWARP_IRD_DEFAULT;  in qed_iwarp_mpa_received()
793 		ep->mpa_rev = MPA_NEGOTIATION_TYPE_BASIC;  in qed_iwarp_mpa_received()
797 		   "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x rtr:0x%x ulp_data_len = %x mpa_hdr_size = %x\n",  in qed_iwarp_mpa_received()
798 		   mpa_rev, ep->cm_info.ord, ep->cm_info.ird, ep->rtr_type,  in qed_iwarp_mpa_received()
799 		   async_data->mpa_request.ulp_data_len, mpa_hdr_size);  in qed_iwarp_mpa_received()
802 	ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_hdr_size;  in qed_iwarp_mpa_received()
804 	ulp_data_len = le16_to_cpu(async_data->mpa_request.ulp_data_len);  in qed_iwarp_mpa_received()
805 	ep->cm_info.private_data_len = ulp_data_len - mpa_hdr_size;  in qed_iwarp_mpa_received()
808 	params.cm_info = &ep->cm_info;  in qed_iwarp_mpa_received()
812 	ep->state = QED_IWARP_EP_MPA_REQ_RCVD;  in qed_iwarp_mpa_received()
813 	ep->event_cb(ep->cb_context, &params);  in qed_iwarp_mpa_received()
830 	int rc;  in qed_iwarp_mpa_offload()  local
833 		return -EINVAL;  in qed_iwarp_mpa_offload()
835 	qp = ep->qp;  in qed_iwarp_mpa_offload()
839 	init_data.cid = reject ? ep->tcp_cid : qp->icid;  in qed_iwarp_mpa_offload()
840 	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;  in qed_iwarp_mpa_offload()
842 	if (ep->connect_mode == TCP_CONNECT_ACTIVE)  in qed_iwarp_mpa_offload()
847 	rc = qed_sp_init_request(p_hwfn, &p_ent,  in qed_iwarp_mpa_offload()
850 	if (rc)  in qed_iwarp_mpa_offload()
851 		return rc;  in qed_iwarp_mpa_offload()
853 	p_mpa_ramrod = &p_ent->ramrod.iwarp_mpa_offload;  in qed_iwarp_mpa_offload()
854 	common = &p_mpa_ramrod->common;  in qed_iwarp_mpa_offload()
856 	out_pdata_phys = ep->ep_buffer_phys +  in qed_iwarp_mpa_offload()
858 	DMA_REGPAIR_LE(common->outgoing_ulp_buffer.addr, out_pdata_phys);  in qed_iwarp_mpa_offload()
860 	val = ep->cm_info.private_data_len;  in qed_iwarp_mpa_offload()
861 	common->outgoing_ulp_buffer.len = cpu_to_le16(val);  in qed_iwarp_mpa_offload()
862 	common->crc_needed = p_hwfn->p_rdma_info->iwarp.crc_needed;  in qed_iwarp_mpa_offload()
864 	common->out_rq.ord = cpu_to_le32(ep->cm_info.ord);  in qed_iwarp_mpa_offload()
865 	common->out_rq.ird = cpu_to_le32(ep->cm_info.ird);  in qed_iwarp_mpa_offload()
867 	val = p_hwfn->hw_info.opaque_fid << 16 | ep->tcp_cid;  in qed_iwarp_mpa_offload()
868 	p_mpa_ramrod->tcp_cid = cpu_to_le32(val);  in qed_iwarp_mpa_offload()
870 	in_pdata_phys = ep->ep_buffer_phys +  in qed_iwarp_mpa_offload()
872 	p_mpa_ramrod->tcp_connect_side = ep->connect_mode;  in qed_iwarp_mpa_offload()
873 	DMA_REGPAIR_LE(p_mpa_ramrod->incoming_ulp_buffer.addr,  in qed_iwarp_mpa_offload()
875 	p_mpa_ramrod->incoming_ulp_buffer.len =  in qed_iwarp_mpa_offload()
876 	    cpu_to_le16(sizeof(ep->ep_buffer_virt->in_pdata));  in qed_iwarp_mpa_offload()
877 	async_output_phys = ep->ep_buffer_phys +  in qed_iwarp_mpa_offload()
879 	DMA_REGPAIR_LE(p_mpa_ramrod->async_eqe_output_buf,  in qed_iwarp_mpa_offload()
881 	p_mpa_ramrod->handle_for_async.hi = cpu_to_le32(PTR_HI(ep));  in qed_iwarp_mpa_offload()
882 	p_mpa_ramrod->handle_for_async.lo = cpu_to_le32(PTR_LO(ep));  in qed_iwarp_mpa_offload()
885 		DMA_REGPAIR_LE(p_mpa_ramrod->shared_queue_addr,  in qed_iwarp_mpa_offload()
886 			       qp->shared_queue_phys_addr);  in qed_iwarp_mpa_offload()
887 		p_mpa_ramrod->stats_counter_id =  in qed_iwarp_mpa_offload()
888 		    RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + qp->stats_queue;  in qed_iwarp_mpa_offload()
890 		common->reject = 1;  in qed_iwarp_mpa_offload()
893 	iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_mpa_offload()
894 	p_mpa_ramrod->rcv_wnd = cpu_to_le16(iwarp_info->rcv_wnd_size);  in qed_iwarp_mpa_offload()
895 	p_mpa_ramrod->mode = ep->mpa_rev;  in qed_iwarp_mpa_offload()
896 	SET_FIELD(p_mpa_ramrod->rtr_pref,  in qed_iwarp_mpa_offload()
897 		  IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED, ep->rtr_type);  in qed_iwarp_mpa_offload()
899 	ep->state = QED_IWARP_EP_MPA_OFFLOADED;  in qed_iwarp_mpa_offload()
900 	rc = qed_spq_post(p_hwfn, p_ent, NULL);  in qed_iwarp_mpa_offload()
902 		ep->cid = qp->icid;	/* Now they're migrated. */  in qed_iwarp_mpa_offload()
906 		   "QP(0x%x) EP(0x%x) MPA Offload rc = %d IRD=0x%x ORD=0x%x rtr_type=%d mpa_rev=%d reject=%d\n",  in qed_iwarp_mpa_offload()
907 		   reject ? 0xffff : qp->icid,  in qed_iwarp_mpa_offload()
908 		   ep->tcp_cid,  in qed_iwarp_mpa_offload()
909 		   rc,  in qed_iwarp_mpa_offload()
910 		   ep->cm_info.ird,  in qed_iwarp_mpa_offload()
911 		   ep->cm_info.ord, ep->rtr_type, ep->mpa_rev, reject);  in qed_iwarp_mpa_offload()
912 	return rc;  in qed_iwarp_mpa_offload()
918 	ep->state = QED_IWARP_EP_INIT;  in qed_iwarp_return_ep()
919 	if (ep->qp)  in qed_iwarp_return_ep()
920 		ep->qp->ep = NULL;  in qed_iwarp_return_ep()
921 	ep->qp = NULL;  in qed_iwarp_return_ep()
922 	memset(&ep->cm_info, 0, sizeof(ep->cm_info));  in qed_iwarp_return_ep()
924 	if (ep->tcp_cid == QED_IWARP_INVALID_TCP_CID) {  in qed_iwarp_return_ep()
928 		qed_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid);  in qed_iwarp_return_ep()
930 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_return_ep()
932 	list_move_tail(&ep->list_entry,  in qed_iwarp_return_ep()
933 		       &p_hwfn->p_rdma_info->iwarp.ep_free_list);  in qed_iwarp_return_ep()
935 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_return_ep()
947 	if (MPA_REV2(p_hwfn->p_rdma_info->iwarp.mpa_rev)) {  in qed_iwarp_parse_private_data()
949 			(struct mpa_v2_hdr *)(ep->ep_buffer_virt->in_pdata);  in qed_iwarp_parse_private_data()
951 		mpa_ird = ntohs(mpa_v2_params->ird);  in qed_iwarp_parse_private_data()
952 		mpa_ord = ntohs(mpa_v2_params->ord);  in qed_iwarp_parse_private_data()
954 		ep->cm_info.ird = (u8)(mpa_ord & MPA_V2_IRD_ORD_MASK);  in qed_iwarp_parse_private_data()
955 		ep->cm_info.ord = (u8)(mpa_ird & MPA_V2_IRD_ORD_MASK);  in qed_iwarp_parse_private_data()
958 	async_data = &ep->ep_buffer_virt->async_output;  in qed_iwarp_parse_private_data()
959 	ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_data_size;  in qed_iwarp_parse_private_data()
961 	ulp_data_len = le16_to_cpu(async_data->mpa_response.ulp_data_len);  in qed_iwarp_parse_private_data()
962 	ep->cm_info.private_data_len = ulp_data_len - mpa_data_size;  in qed_iwarp_parse_private_data()
970 	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {  in qed_iwarp_mpa_reply_arrived()
981 		   "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n",  in qed_iwarp_mpa_reply_arrived()
982 		   ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird);  in qed_iwarp_mpa_reply_arrived()
984 	params.cm_info = &ep->cm_info;  in qed_iwarp_mpa_reply_arrived()
988 	ep->mpa_reply_processed = true;  in qed_iwarp_mpa_reply_arrived()
990 	ep->event_cb(ep->cb_context, &params);  in qed_iwarp_mpa_reply_arrived()
994 	((ep)->connect_mode == TCP_CONNECT_PASSIVE) ? "Passive" : "Active"
1005 	if (ep->connect_mode == TCP_CONNECT_ACTIVE)  in qed_iwarp_mpa_complete()
1010 	if (ep->connect_mode == TCP_CONNECT_ACTIVE && !ep->mpa_reply_processed)  in qed_iwarp_mpa_complete()
1014 		   "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n",  in qed_iwarp_mpa_complete()
1015 		   ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird);  in qed_iwarp_mpa_complete()
1017 	params.cm_info = &ep->cm_info;  in qed_iwarp_mpa_complete()
1023 		ep->qp->max_rd_atomic_req = ep->cm_info.ord;  in qed_iwarp_mpa_complete()
1024 		ep->qp->max_rd_atomic_resp = ep->cm_info.ird;  in qed_iwarp_mpa_complete()
1025 		qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_RTS, 1);  in qed_iwarp_mpa_complete()
1026 		ep->state = QED_IWARP_EP_ESTABLISHED;  in qed_iwarp_mpa_complete()
1030 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA timeout\n",  in qed_iwarp_mpa_complete()
1031 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1032 		params.status = -EBUSY;  in qed_iwarp_mpa_complete()
1035 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA Reject\n",  in qed_iwarp_mpa_complete()
1036 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1037 		params.status = -ECONNREFUSED;  in qed_iwarp_mpa_complete()
1040 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA reset(tcp cid: 0x%x)\n",  in qed_iwarp_mpa_complete()
1041 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid,  in qed_iwarp_mpa_complete()
1042 			  ep->tcp_cid);  in qed_iwarp_mpa_complete()
1043 		params.status = -ECONNRESET;  in qed_iwarp_mpa_complete()
1046 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA received FIN\n",  in qed_iwarp_mpa_complete()
1047 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1048 		params.status = -ECONNREFUSED;  in qed_iwarp_mpa_complete()
1051 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA insufficient ird\n",  in qed_iwarp_mpa_complete()
1052 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1053 		params.status = -ECONNREFUSED;  in qed_iwarp_mpa_complete()
1056 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA RTR MISMATCH\n",  in qed_iwarp_mpa_complete()
1057 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1058 		params.status = -ECONNREFUSED;  in qed_iwarp_mpa_complete()
1061 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA Invalid Packet\n",  in qed_iwarp_mpa_complete()
1062 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1063 		params.status = -ECONNREFUSED;  in qed_iwarp_mpa_complete()
1066 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA Local Error\n",  in qed_iwarp_mpa_complete()
1067 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1068 		params.status = -ECONNREFUSED;  in qed_iwarp_mpa_complete()
1071 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA TERMINATE\n",  in qed_iwarp_mpa_complete()
1072 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);  in qed_iwarp_mpa_complete()
1073 		params.status = -ECONNREFUSED;  in qed_iwarp_mpa_complete()
1076 		params.status = -ECONNRESET;  in qed_iwarp_mpa_complete()
1082 		smp_store_release(&ep->state, QED_IWARP_EP_CLOSED);  in qed_iwarp_mpa_complete()
1084 	ep->event_cb(ep->cb_context, &params);  in qed_iwarp_mpa_complete()
1092 		ep->tcp_cid = QED_IWARP_INVALID_TCP_CID;  in qed_iwarp_mpa_complete()
1093 		if ((ep->connect_mode == TCP_CONNECT_PASSIVE) &&  in qed_iwarp_mpa_complete()
1094 		    (!ep->qp)) {	/* Rejected */  in qed_iwarp_mpa_complete()
1097 			spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_mpa_complete()
1098 			list_del(&ep->list_entry);  in qed_iwarp_mpa_complete()
1099 			spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_mpa_complete()
1112 	if (MPA_REV2(ep->mpa_rev)) {  in qed_iwarp_mpa_v2_set_private()
1114 		    (struct mpa_v2_hdr *)ep->ep_buffer_virt->out_pdata;  in qed_iwarp_mpa_v2_set_private()
1117 		mpa_ird = (u16)ep->cm_info.ird;  in qed_iwarp_mpa_v2_set_private()
1118 		mpa_ord = (u16)ep->cm_info.ord;  in qed_iwarp_mpa_v2_set_private()
1120 		if (ep->rtr_type != MPA_RTR_TYPE_NONE) {  in qed_iwarp_mpa_v2_set_private()
1123 			if (ep->rtr_type & MPA_RTR_TYPE_ZERO_SEND)  in qed_iwarp_mpa_v2_set_private()
1126 			if (ep->rtr_type & MPA_RTR_TYPE_ZERO_WRITE)  in qed_iwarp_mpa_v2_set_private()
1129 			if (ep->rtr_type & MPA_RTR_TYPE_ZERO_READ)  in qed_iwarp_mpa_v2_set_private()
1133 		mpa_v2_params->ird = htons(mpa_ird);  in qed_iwarp_mpa_v2_set_private()
1134 		mpa_v2_params->ord = htons(mpa_ord);  in qed_iwarp_mpa_v2_set_private()
1138 …"MPA_NEGOTIATE Header: [%x ord:%x ird] %x ord:%x ird:%x peer2peer:%x rtr_send:%x rtr_write:%x rtr_…  in qed_iwarp_mpa_v2_set_private()
1139 			   mpa_v2_params->ird,  in qed_iwarp_mpa_v2_set_private()
1140 			   mpa_v2_params->ord,  in qed_iwarp_mpa_v2_set_private()
1160 	int rc;  in qed_iwarp_connect()  local
1162 	if ((iparams->cm_info.ord > QED_IWARP_ORD_DEFAULT) ||  in qed_iwarp_connect()
1163 	    (iparams->cm_info.ird > QED_IWARP_IRD_DEFAULT)) {  in qed_iwarp_connect()
1165 			  "QP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n",  in qed_iwarp_connect()
1166 			  iparams->qp->icid, iparams->cm_info.ord,  in qed_iwarp_connect()
1167 			  iparams->cm_info.ird);  in qed_iwarp_connect()
1169 		return -EINVAL;  in qed_iwarp_connect()
1172 	iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_connect()
1175 	rc = qed_iwarp_alloc_cid(p_hwfn, &cid);  in qed_iwarp_connect()
1176 	if (rc)  in qed_iwarp_connect()
1177 		return rc;  in qed_iwarp_connect()
1179 	rc = qed_iwarp_create_ep(p_hwfn, &ep);  in qed_iwarp_connect()
1180 	if (rc)  in qed_iwarp_connect()
1183 	ep->tcp_cid = cid;  in qed_iwarp_connect()
1185 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_connect()
1186 	list_add_tail(&ep->list_entry, &p_hwfn->p_rdma_info->iwarp.ep_list);  in qed_iwarp_connect()
1187 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_connect()
1189 	ep->qp = iparams->qp;  in qed_iwarp_connect()
1190 	ep->qp->ep = ep;  in qed_iwarp_connect()
1191 	ether_addr_copy(ep->remote_mac_addr, iparams->remote_mac_addr);  in qed_iwarp_connect()
1192 	ether_addr_copy(ep->local_mac_addr, iparams->local_mac_addr);  in qed_iwarp_connect()
1193 	memcpy(&ep->cm_info, &iparams->cm_info, sizeof(ep->cm_info));  in qed_iwarp_connect()
1195 	ep->cm_info.ord = iparams->cm_info.ord;  in qed_iwarp_connect()
1196 	ep->cm_info.ird = iparams->cm_info.ird;  in qed_iwarp_connect()
1198 	ep->rtr_type = iwarp_info->rtr_type;  in qed_iwarp_connect()
1199 	if (!iwarp_info->peer2peer)  in qed_iwarp_connect()
1200 		ep->rtr_type = MPA_RTR_TYPE_NONE;  in qed_iwarp_connect()
1202 	if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) && (ep->cm_info.ord == 0))  in qed_iwarp_connect()
1203 		ep->cm_info.ord = 1;  in qed_iwarp_connect()
1205 	ep->mpa_rev = iwarp_info->mpa_rev;  in qed_iwarp_connect()
1209 	ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;  in qed_iwarp_connect()
1210 	ep->cm_info.private_data_len = iparams->cm_info.private_data_len +  in qed_iwarp_connect()
1213 	memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,  in qed_iwarp_connect()
1214 	       iparams->cm_info.private_data,  in qed_iwarp_connect()
1215 	       iparams->cm_info.private_data_len);  in qed_iwarp_connect()
1217 	ep->mss = iparams->mss;  in qed_iwarp_connect()
1218 	ep->mss = min_t(u16, QED_IWARP_MAX_FW_MSS, ep->mss);  in qed_iwarp_connect()
1220 	ep->event_cb = iparams->event_cb;  in qed_iwarp_connect()
1221 	ep->cb_context = iparams->cb_context;  in qed_iwarp_connect()
1222 	ep->connect_mode = TCP_CONNECT_ACTIVE;  in qed_iwarp_connect()
1224 	oparams->ep_context = ep;  in qed_iwarp_connect()
1226 	rc = qed_iwarp_tcp_offload(p_hwfn, ep);  in qed_iwarp_connect()
1228 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x) rc = %d\n",  in qed_iwarp_connect()
1229 		   iparams->qp->icid, ep->tcp_cid, rc);  in qed_iwarp_connect()
1231 	if (rc) {  in qed_iwarp_connect()
1236 	return rc;  in qed_iwarp_connect()
1240 	return rc;  in qed_iwarp_connect()
1246 	int rc;  in qed_iwarp_get_free_ep()  local
1248 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_get_free_ep()
1250 	if (list_empty(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) {  in qed_iwarp_get_free_ep()
1255 	ep = list_first_entry(&p_hwfn->p_rdma_info->iwarp.ep_free_list,  in qed_iwarp_get_free_ep()
1261 	if (ep->tcp_cid == QED_IWARP_INVALID_TCP_CID) {  in qed_iwarp_get_free_ep()
1262 		rc = qed_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid);  in qed_iwarp_get_free_ep()
1268 		if (rc) {  in qed_iwarp_get_free_ep()
1269 			ep->tcp_cid = QED_IWARP_INVALID_TCP_CID;  in qed_iwarp_get_free_ep()
1275 	list_del(&ep->list_entry);  in qed_iwarp_get_free_ep()
1278 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_get_free_ep()
1296 	weight = bitmap_weight(bmap->bitmap, bmap->max_count);  in qed_iwarp_wait_cid_map_cleared()
1301 		 * immediately reset without receiving a per-cid indication  in qed_iwarp_wait_cid_map_cleared()
1305 		if (p_hwfn->cdev->recov_in_prog)  in qed_iwarp_wait_cid_map_cleared()
1310 		weight = bitmap_weight(bmap->bitmap, bmap->max_count);  in qed_iwarp_wait_cid_map_cleared()
1322 				  bmap->name, weight);  in qed_iwarp_wait_cid_map_cleared()
1323 			return -EBUSY;  in qed_iwarp_wait_cid_map_cleared()
1331 	int rc;  in qed_iwarp_wait_for_all_cids()  local
1334 	rc = qed_iwarp_wait_cid_map_cleared(p_hwfn,  in qed_iwarp_wait_for_all_cids()
1335 					    &p_hwfn->p_rdma_info->tcp_cid_map);  in qed_iwarp_wait_for_all_cids()
1336 	if (rc)  in qed_iwarp_wait_for_all_cids()
1337 		return rc;  in qed_iwarp_wait_for_all_cids()
1341 		qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, i);  in qed_iwarp_wait_for_all_cids()
1345 					      &p_hwfn->p_rdma_info->cid_map);  in qed_iwarp_wait_for_all_cids()
1352 	while (!list_empty(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) {  in qed_iwarp_free_prealloc_ep()
1353 		spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_free_prealloc_ep()
1355 		ep = list_first_entry(&p_hwfn->p_rdma_info->iwarp.ep_free_list,  in qed_iwarp_free_prealloc_ep()
1359 			spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_free_prealloc_ep()
1362 		list_del(&ep->list_entry);  in qed_iwarp_free_prealloc_ep()
1364 		spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_free_prealloc_ep()
1366 		if (ep->tcp_cid != QED_IWARP_INVALID_TCP_CID)  in qed_iwarp_free_prealloc_ep()
1367 			qed_iwarp_cid_cleaned(p_hwfn, ep->tcp_cid);  in qed_iwarp_free_prealloc_ep()
1376 	int rc = 0;  in qed_iwarp_prealloc_ep()  local
1383 		rc = qed_iwarp_create_ep(p_hwfn, &ep);  in qed_iwarp_prealloc_ep()
1384 		if (rc)  in qed_iwarp_prealloc_ep()
1385 			return rc;  in qed_iwarp_prealloc_ep()
1391 			rc = qed_iwarp_alloc_cid(p_hwfn, &cid);  in qed_iwarp_prealloc_ep()
1392 			if (rc)  in qed_iwarp_prealloc_ep()
1403 		ep->tcp_cid = cid;  in qed_iwarp_prealloc_ep()
1405 		spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_prealloc_ep()
1406 		list_add_tail(&ep->list_entry,  in qed_iwarp_prealloc_ep()
1407 			      &p_hwfn->p_rdma_info->iwarp.ep_free_list);  in qed_iwarp_prealloc_ep()
1408 		spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_prealloc_ep()
1411 	return rc;  in qed_iwarp_prealloc_ep()
1416 	return rc;  in qed_iwarp_prealloc_ep()
1421 	int rc;  in qed_iwarp_alloc()  local
1425 	 * pre-acquired and doesn't require dynamic allocation of ilt  in qed_iwarp_alloc()
1427 	rc = qed_rdma_bmap_alloc(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map,  in qed_iwarp_alloc()
1429 	if (rc) {  in qed_iwarp_alloc()
1431 			   "Failed to allocate tcp cid, rc = %d\n", rc);  in qed_iwarp_alloc()
1432 		return rc;  in qed_iwarp_alloc()
1435 	INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.ep_free_list);  in qed_iwarp_alloc()
1436 	spin_lock_init(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_alloc()
1438 	rc = qed_iwarp_prealloc_ep(p_hwfn, true);  in qed_iwarp_alloc()
1439 	if (rc)  in qed_iwarp_alloc()
1440 		return rc;  in qed_iwarp_alloc()
1447 	struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_resc_free()
1450 	qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, 1);  in qed_iwarp_resc_free()
1451 	kfree(iwarp_info->mpa_bufs);  in qed_iwarp_resc_free()
1452 	kfree(iwarp_info->partial_fpdus);  in qed_iwarp_resc_free()
1453 	kfree(iwarp_info->mpa_intermediate_buf);  in qed_iwarp_resc_free()
1461 	int rc;  in qed_iwarp_accept()  local
1463 	ep = iparams->ep_context;  in qed_iwarp_accept()
1466 		return -EINVAL;  in qed_iwarp_accept()
1469 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x)\n",  in qed_iwarp_accept()
1470 		   iparams->qp->icid, ep->tcp_cid);  in qed_iwarp_accept()
1472 	if ((iparams->ord > QED_IWARP_ORD_DEFAULT) ||  in qed_iwarp_accept()
1473 	    (iparams->ird > QED_IWARP_IRD_DEFAULT)) {  in qed_iwarp_accept()
1476 			   "QP(0x%x) EP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n",  in qed_iwarp_accept()
1477 			   iparams->qp->icid,  in qed_iwarp_accept()
1478 			   ep->tcp_cid, iparams->ord, iparams->ord);  in qed_iwarp_accept()
1479 		return -EINVAL;  in qed_iwarp_accept()
1484 	ep->cb_context = iparams->cb_context;  in qed_iwarp_accept()
1485 	ep->qp = iparams->qp;  in qed_iwarp_accept()
1486 	ep->qp->ep = ep;  in qed_iwarp_accept()
1488 	if (ep->mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) {  in qed_iwarp_accept()
1492 		if (iparams->ord > ep->cm_info.ird)  in qed_iwarp_accept()
1493 			iparams->ord = ep->cm_info.ird;  in qed_iwarp_accept()
1495 		if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) &&  in qed_iwarp_accept()
1496 		    (iparams->ird == 0))  in qed_iwarp_accept()
1497 			iparams->ird = 1;  in qed_iwarp_accept()
1501 	ep->cm_info.ord = iparams->ord;  in qed_iwarp_accept()
1502 	ep->cm_info.ird = iparams->ird;  in qed_iwarp_accept()
1506 	ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;  in qed_iwarp_accept()
1507 	ep->cm_info.private_data_len = iparams->private_data_len +  in qed_iwarp_accept()
1510 	memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,  in qed_iwarp_accept()
1511 	       iparams->private_data, iparams->private_data_len);  in qed_iwarp_accept()
1513 	rc = qed_iwarp_mpa_offload(p_hwfn, ep);  in qed_iwarp_accept()
1514 	if (rc)  in qed_iwarp_accept()
1516 				    iparams->qp, QED_IWARP_QP_STATE_ERROR, 1);  in qed_iwarp_accept()
1518 	return rc;  in qed_iwarp_accept()
1527 	ep = iparams->ep_context;  in qed_iwarp_reject()
1530 		return -EINVAL;  in qed_iwarp_reject()
1533 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "EP(0x%x)\n", ep->tcp_cid);  in qed_iwarp_reject()
1535 	ep->cb_context = iparams->cb_context;  in qed_iwarp_reject()
1536 	ep->qp = NULL;  in qed_iwarp_reject()
1540 	ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;  in qed_iwarp_reject()
1541 	ep->cm_info.private_data_len = iparams->private_data_len +  in qed_iwarp_reject()
1544 	memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,  in qed_iwarp_reject()
1545 	       iparams->private_data, iparams->private_data_len);  in qed_iwarp_reject()
1555 		   cm_info->ip_version);  in qed_iwarp_print_cm_info()
1557 	if (cm_info->ip_version == QED_TCP_IPV4)  in qed_iwarp_print_cm_info()
1559 			   "remote_ip %pI4h:%x, local_ip %pI4h:%x vlan=%x\n",  in qed_iwarp_print_cm_info()
1560 			   cm_info->remote_ip, cm_info->remote_port,  in qed_iwarp_print_cm_info()
1561 			   cm_info->local_ip, cm_info->local_port,  in qed_iwarp_print_cm_info()
1562 			   cm_info->vlan);  in qed_iwarp_print_cm_info()
1565 			   "remote_ip %pI6:%x, local_ip %pI6:%x vlan=%x\n",  in qed_iwarp_print_cm_info()
1566 			   cm_info->remote_ip, cm_info->remote_port,  in qed_iwarp_print_cm_info()
1567 			   cm_info->local_ip, cm_info->local_port,  in qed_iwarp_print_cm_info()
1568 			   cm_info->vlan);  in qed_iwarp_print_cm_info()
1571 		   "private_data_len = %x ord = %d, ird = %d\n",  in qed_iwarp_print_cm_info()
1572 		   cm_info->private_data_len, cm_info->ord, cm_info->ird);  in qed_iwarp_print_cm_info()
1579 	int rc;  in qed_iwarp_ll2_post_rx()  local
1581 	rc = qed_ll2_post_rx_buffer(p_hwfn, handle, buf->data_phys_addr,  in qed_iwarp_ll2_post_rx()
1582 				    (u16)buf->buff_size, buf, 1);  in qed_iwarp_ll2_post_rx()
1583 	if (rc) {  in qed_iwarp_ll2_post_rx()
1585 			  "Failed to repost rx buffer to ll2 rc = %d, handle=%d\n",  in qed_iwarp_ll2_post_rx()
1586 			  rc, handle);  in qed_iwarp_ll2_post_rx()
1587 		dma_free_coherent(&p_hwfn->cdev->pdev->dev, buf->buff_size,  in qed_iwarp_ll2_post_rx()
1588 				  buf->data, buf->data_phys_addr);  in qed_iwarp_ll2_post_rx()
1592 	return rc;  in qed_iwarp_ll2_post_rx()
1602 			    &p_hwfn->p_rdma_info->iwarp.ep_list,  in qed_iwarp_ep_exists()
1604 		if ((ep->cm_info.local_port == cm_info->local_port) &&  in qed_iwarp_ep_exists()
1605 		    (ep->cm_info.remote_port == cm_info->remote_port) &&  in qed_iwarp_ep_exists()
1606 		    (ep->cm_info.vlan == cm_info->vlan) &&  in qed_iwarp_ep_exists()
1607 		    !memcmp(&ep->cm_info.local_ip, cm_info->local_ip,  in qed_iwarp_ep_exists()
1608 			    sizeof(cm_info->local_ip)) &&  in qed_iwarp_ep_exists()
1609 		    !memcmp(&ep->cm_info.remote_ip, cm_info->remote_ip,  in qed_iwarp_ep_exists()
1610 			    sizeof(cm_info->remote_ip))) {  in qed_iwarp_ep_exists()
1618 			  "SYN received on active connection - dropping\n");  in qed_iwarp_ep_exists()
1636 			    &p_hwfn->p_rdma_info->iwarp.listen_list,  in qed_iwarp_get_listener()
1638 		if (listener->port == cm_info->local_port) {  in qed_iwarp_get_listener()
1639 			if (!memcmp(listener->ip_addr,  in qed_iwarp_get_listener()
1645 			if (!memcmp(listener->ip_addr,  in qed_iwarp_get_listener()
1646 				    cm_info->local_ip,  in qed_iwarp_get_listener()
1647 				    sizeof(cm_info->local_ip)) &&  in qed_iwarp_get_listener()
1648 			    (listener->vlan == cm_info->vlan)) {  in qed_iwarp_get_listener()
1685 	eth_type = ntohs(ethh->h_proto);  in qed_iwarp_parse_rx_pkt()
1689 		cm_info->vlan = ntohs(vethh->h_vlan_TCI) & VLAN_VID_MASK;  in qed_iwarp_parse_rx_pkt()
1690 		eth_type = ntohs(vethh->h_vlan_encapsulated_proto);  in qed_iwarp_parse_rx_pkt()
1695 	if (!ether_addr_equal(ethh->h_dest,  in qed_iwarp_parse_rx_pkt()
1696 			      p_hwfn->p_rdma_info->iwarp.mac_addr)) {  in qed_iwarp_parse_rx_pkt()
1700 			   ethh->h_dest, p_hwfn->p_rdma_info->iwarp.mac_addr);  in qed_iwarp_parse_rx_pkt()
1701 		return -EINVAL;  in qed_iwarp_parse_rx_pkt()
1704 	ether_addr_copy(remote_mac_addr, ethh->h_source);  in qed_iwarp_parse_rx_pkt()
1705 	ether_addr_copy(local_mac_addr, ethh->h_dest);  in qed_iwarp_parse_rx_pkt()
1708 		   eth_type, ethh->h_source);  in qed_iwarp_parse_rx_pkt()
1711 		   eth_hlen, ethh->h_dest);  in qed_iwarp_parse_rx_pkt()
1716 		if (iph->protocol != IPPROTO_TCP) {  in qed_iwarp_parse_rx_pkt()
1718 				  "Unexpected ip protocol on ll2 %x\n",  in qed_iwarp_parse_rx_pkt()
1719 				  iph->protocol);  in qed_iwarp_parse_rx_pkt()
1720 			return -EINVAL;  in qed_iwarp_parse_rx_pkt()
1723 		cm_info->local_ip[0] = ntohl(iph->daddr);  in qed_iwarp_parse_rx_pkt()
1724 		cm_info->remote_ip[0] = ntohl(iph->saddr);  in qed_iwarp_parse_rx_pkt()
1725 		cm_info->ip_version = QED_TCP_IPV4;  in qed_iwarp_parse_rx_pkt()
1727 		ip_hlen = (iph->ihl) * sizeof(u32);  in qed_iwarp_parse_rx_pkt()
1728 		*payload_len = ntohs(iph->tot_len) - ip_hlen;  in qed_iwarp_parse_rx_pkt()
1732 		if (ip6h->nexthdr != IPPROTO_TCP) {  in qed_iwarp_parse_rx_pkt()
1734 				  "Unexpected ip protocol on ll2 %x\n",  in qed_iwarp_parse_rx_pkt()
1735 				  iph->protocol);  in qed_iwarp_parse_rx_pkt()
1736 			return -EINVAL;  in qed_iwarp_parse_rx_pkt()
1740 			cm_info->local_ip[i] =  in qed_iwarp_parse_rx_pkt()
1741 			    ntohl(ip6h->daddr.in6_u.u6_addr32[i]);  in qed_iwarp_parse_rx_pkt()
1742 			cm_info->remote_ip[i] =  in qed_iwarp_parse_rx_pkt()
1743 			    ntohl(ip6h->saddr.in6_u.u6_addr32[i]);  in qed_iwarp_parse_rx_pkt()
1745 		cm_info->ip_version = QED_TCP_IPV6;  in qed_iwarp_parse_rx_pkt()
1748 		*payload_len = ntohs(ip6h->payload_len);  in qed_iwarp_parse_rx_pkt()
1750 		DP_NOTICE(p_hwfn, "Unexpected ethertype on ll2 %x\n", eth_type);  in qed_iwarp_parse_rx_pkt()
1751 		return -EINVAL;  in qed_iwarp_parse_rx_pkt()
1756 	if (!tcph->syn) {  in qed_iwarp_parse_rx_pkt()
1758 			  "Only SYN type packet expected on this ll2 conn, iph->ihl=%d source=%d dest=%d\n",  in qed_iwarp_parse_rx_pkt()
1759 			  iph->ihl, tcph->source, tcph->dest);  in qed_iwarp_parse_rx_pkt()
1760 		return -EINVAL;  in qed_iwarp_parse_rx_pkt()
1763 	cm_info->local_port = ntohs(tcph->dest);  in qed_iwarp_parse_rx_pkt()
1764 	cm_info->remote_port = ntohs(tcph->source);  in qed_iwarp_parse_rx_pkt()
1776 	struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_get_curr_fpdu()
1780 	idx = cid - qed_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_IWARP);  in qed_iwarp_get_curr_fpdu()
1781 	if (idx >= iwarp_info->max_num_partial_fpdus) {  in qed_iwarp_get_curr_fpdu()
1782 		DP_ERR(p_hwfn, "Invalid cid %x max_num_partial_fpdus=%x\n", cid,  in qed_iwarp_get_curr_fpdu()
1783 		       iwarp_info->max_num_partial_fpdus);  in qed_iwarp_get_curr_fpdu()
1787 	partial_fpdu = &iwarp_info->partial_fpdus[idx];  in qed_iwarp_get_curr_fpdu()
1831 	if (fpdu->incomplete_bytes) {  in qed_iwarp_mpa_classify()
1840 		fpdu->fpdu_length = *mpa_data << BITS_PER_BYTE;  in qed_iwarp_mpa_classify()
1846 	fpdu->fpdu_length = QED_IWARP_FPDU_LEN_WITH_PAD(mpa_len);  in qed_iwarp_mpa_classify()
1848 	if (fpdu->fpdu_length <= tcp_payload_len)  in qed_iwarp_mpa_classify()
1855 		   "MPA_ALIGN: %s: fpdu_length=0x%x tcp_payload_len:0x%x\n",  in qed_iwarp_mpa_classify()
1856 		   pkt_type_str[pkt_type], fpdu->fpdu_length, tcp_payload_len);  in qed_iwarp_mpa_classify()
1867 	u16 first_mpa_offset = le16_to_cpu(pkt_data->first_mpa_offset);  in qed_iwarp_init_fpdu()
1869 	fpdu->mpa_buf = buf;  in qed_iwarp_init_fpdu()
1870 	fpdu->pkt_hdr = buf->data_phys_addr + placement_offset;  in qed_iwarp_init_fpdu()
1871 	fpdu->pkt_hdr_size = pkt_data->tcp_payload_offset;  in qed_iwarp_init_fpdu()
1872 	fpdu->mpa_frag = buf->data_phys_addr + first_mpa_offset;  in qed_iwarp_init_fpdu()
1873 	fpdu->mpa_frag_virt = (u8 *)(buf->data) + first_mpa_offset;  in qed_iwarp_init_fpdu()
1876 		fpdu->incomplete_bytes = QED_IWARP_INVALID_FPDU_LENGTH;  in qed_iwarp_init_fpdu()
1877 	else if (tcp_payload_size < fpdu->fpdu_length)  in qed_iwarp_init_fpdu()
1878 		fpdu->incomplete_bytes = fpdu->fpdu_length - tcp_payload_size;  in qed_iwarp_init_fpdu()
1880 		fpdu->incomplete_bytes = 0;	/* complete fpdu */  in qed_iwarp_init_fpdu()
1882 	fpdu->mpa_frag_len = fpdu->fpdu_length - fpdu->incomplete_bytes;  in qed_iwarp_init_fpdu()
1891 	u16 first_mpa_offset = le16_to_cpu(pkt_data->first_mpa_offset);  in qed_iwarp_cp_pkt()
1892 	u8 *tmp_buf = p_hwfn->p_rdma_info->iwarp.mpa_intermediate_buf;  in qed_iwarp_cp_pkt()
1893 	int rc;  in qed_iwarp_cp_pkt()  local
1901 	if ((fpdu->mpa_frag_len + tcp_payload_size) > (u16)buf->buff_size) {  in qed_iwarp_cp_pkt()
1904 		       buf->buff_size, fpdu->mpa_frag_len,  in qed_iwarp_cp_pkt()
1905 		       tcp_payload_size, fpdu->incomplete_bytes);  in qed_iwarp_cp_pkt()
1906 		return -EINVAL;  in qed_iwarp_cp_pkt()
1911 		   fpdu->mpa_frag_virt, fpdu->mpa_frag_len,  in qed_iwarp_cp_pkt()
1912 		   (u8 *)(buf->data) + first_mpa_offset, tcp_payload_size);  in qed_iwarp_cp_pkt()
1914 	memcpy(tmp_buf, fpdu->mpa_frag_virt, fpdu->mpa_frag_len);  in qed_iwarp_cp_pkt()
1915 	memcpy(tmp_buf + fpdu->mpa_frag_len,  in qed_iwarp_cp_pkt()
1916 	       (u8 *)(buf->data) + first_mpa_offset, tcp_payload_size);  in qed_iwarp_cp_pkt()
1918 	rc = qed_iwarp_recycle_pkt(p_hwfn, fpdu, fpdu->mpa_buf);  in qed_iwarp_cp_pkt()
1919 	if (rc)  in qed_iwarp_cp_pkt()
1920 		return rc;  in qed_iwarp_cp_pkt()
1925 	memcpy((u8 *)(buf->data), tmp_buf,  in qed_iwarp_cp_pkt()
1926 	       fpdu->mpa_frag_len + tcp_payload_size);  in qed_iwarp_cp_pkt()
1928 	fpdu->mpa_buf = buf;  in qed_iwarp_cp_pkt()
1929 	/* fpdu->pkt_hdr remains as is */  in qed_iwarp_cp_pkt()
1930 	/* fpdu->mpa_frag is overridden with new buf */  in qed_iwarp_cp_pkt()
1931 	fpdu->mpa_frag = buf->data_phys_addr;  in qed_iwarp_cp_pkt()
1932 	fpdu->mpa_frag_virt = buf->data;  in qed_iwarp_cp_pkt()
1933 	fpdu->mpa_frag_len += tcp_payload_size;  in qed_iwarp_cp_pkt()
1935 	fpdu->incomplete_bytes -= tcp_payload_size;  in qed_iwarp_cp_pkt()
1940 		   buf->buff_size, fpdu->mpa_frag_len, tcp_payload_size,  in qed_iwarp_cp_pkt()
1941 		   fpdu->incomplete_bytes);  in qed_iwarp_cp_pkt()
1953 	if (fpdu->incomplete_bytes == QED_IWARP_INVALID_FPDU_LENGTH) {  in qed_iwarp_update_fpdu_length()
1955 		mpa_len = fpdu->fpdu_length | *mpa_data;  in qed_iwarp_update_fpdu_length()
1956 		fpdu->fpdu_length = QED_IWARP_FPDU_LEN_WITH_PAD(mpa_len);  in qed_iwarp_update_fpdu_length()
1958 		fpdu->mpa_frag_len = 1;  in qed_iwarp_update_fpdu_length()
1959 		fpdu->incomplete_bytes = fpdu->fpdu_length - 1;  in qed_iwarp_update_fpdu_length()
1962 			   "MPA_ALIGN: Partial header mpa_len=%x fpdu_length=%x incomplete_bytes=%x\n",  in qed_iwarp_update_fpdu_length()
1963 			   mpa_len, fpdu->fpdu_length, fpdu->incomplete_bytes);  in qed_iwarp_update_fpdu_length()
1968 	(GET_FIELD((_curr_pkt)->flags,	   \
1976  * segments. In this case the driver needs to re-post a rx buffer instead of
1988 	int rc;  in qed_iwarp_recycle_pkt()  local
1993 	tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2;  in qed_iwarp_recycle_pkt()
1994 	tx_pkt.first_frag = fpdu->pkt_hdr;  in qed_iwarp_recycle_pkt()
1995 	tx_pkt.first_frag_len = fpdu->pkt_hdr_size;  in qed_iwarp_recycle_pkt()
1996 	buf->piggy_buf = NULL;  in qed_iwarp_recycle_pkt()
1999 	ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle;  in qed_iwarp_recycle_pkt()
2001 	rc = qed_ll2_prepare_tx_packet(p_hwfn, ll2_handle, &tx_pkt, true);  in qed_iwarp_recycle_pkt()
2002 	if (rc)  in qed_iwarp_recycle_pkt()
2004 			   "Can't drop packet rc=%d\n", rc);  in qed_iwarp_recycle_pkt()
2008 		   "MPA_ALIGN: send drop tx packet [%lx, 0x%x], buf=%p, rc=%d\n",  in qed_iwarp_recycle_pkt()
2010 		   tx_pkt.first_frag_len, buf, rc);  in qed_iwarp_recycle_pkt()
2012 	return rc;  in qed_iwarp_recycle_pkt()
2020 	int rc;  in qed_iwarp_win_right_edge()  local
2025 	tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2;  in qed_iwarp_win_right_edge()
2027 	tx_pkt.first_frag = fpdu->pkt_hdr;  in qed_iwarp_win_right_edge()
2028 	tx_pkt.first_frag_len = fpdu->pkt_hdr_size;  in qed_iwarp_win_right_edge()
2035 	ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle;  in qed_iwarp_win_right_edge()
2037 	rc = qed_ll2_prepare_tx_packet(p_hwfn, ll2_handle, &tx_pkt, true);  in qed_iwarp_win_right_edge()
2038 	if (rc)  in qed_iwarp_win_right_edge()
2040 			   "Can't send right edge rc=%d\n", rc);  in qed_iwarp_win_right_edge()
2043 		   "MPA_ALIGN: Sent right edge FPDU num_bds=%d [%lx, 0x%x], rc=%d\n",  in qed_iwarp_win_right_edge()
2046 		   tx_pkt.first_frag_len, rc);  in qed_iwarp_win_right_edge()
2048 	return rc;  in qed_iwarp_win_right_edge()
2061 	int rc;  in qed_iwarp_send_fpdu()  local
2073 	tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2; /* offset in words */  in qed_iwarp_send_fpdu()
2077 	    tcp_payload_size <= fpdu->fpdu_length)  in qed_iwarp_send_fpdu()
2078 		tx_pkt.cookie = fpdu->mpa_buf;  in qed_iwarp_send_fpdu()
2080 	tx_pkt.first_frag = fpdu->pkt_hdr;  in qed_iwarp_send_fpdu()
2081 	tx_pkt.first_frag_len = fpdu->pkt_hdr_size;  in qed_iwarp_send_fpdu()
2091 	if (tcp_payload_size == fpdu->incomplete_bytes)  in qed_iwarp_send_fpdu()
2092 		fpdu->mpa_buf->piggy_buf = buf;  in qed_iwarp_send_fpdu()
2094 	ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle;  in qed_iwarp_send_fpdu()
2097 	rc = qed_ll2_prepare_tx_packet(p_hwfn, ll2_handle, &tx_pkt, true);  in qed_iwarp_send_fpdu()
2098 	if (rc)  in qed_iwarp_send_fpdu()
2102 	rc = qed_ll2_set_fragment_of_tx_packet(p_hwfn, ll2_handle,  in qed_iwarp_send_fpdu()
2103 					       fpdu->mpa_frag,  in qed_iwarp_send_fpdu()
2104 					       fpdu->mpa_frag_len);  in qed_iwarp_send_fpdu()
2105 	if (rc)  in qed_iwarp_send_fpdu()
2108 	if (!fpdu->incomplete_bytes)  in qed_iwarp_send_fpdu()
2111 	first_mpa_offset = le16_to_cpu(curr_pkt->first_mpa_offset);  in qed_iwarp_send_fpdu()
2114 	rc = qed_ll2_set_fragment_of_tx_packet(p_hwfn,  in qed_iwarp_send_fpdu()
2116 					       buf->data_phys_addr +  in qed_iwarp_send_fpdu()
2118 					       fpdu->incomplete_bytes);  in qed_iwarp_send_fpdu()
2122 …"MPA_ALIGN: Sent FPDU num_bds=%d first_frag_len=%x, mpa_frag_len=0x%x, incomplete_bytes:0x%x rc=%d…  in qed_iwarp_send_fpdu()
2125 		   fpdu->mpa_frag_len,  in qed_iwarp_send_fpdu()
2126 		   fpdu->incomplete_bytes, rc);  in qed_iwarp_send_fpdu()
2128 	return rc;  in qed_iwarp_send_fpdu()
2142 	le16_add_cpu(&curr_pkt->first_mpa_offset,  in qed_iwarp_mpa_get_data()
2143 		     curr_pkt->tcp_payload_offset);  in qed_iwarp_mpa_get_data()
2154 	struct unaligned_opaque_data *curr_pkt = &mpa_buf->data;  in qed_iwarp_process_mpa_pkt()
2155 	struct qed_iwarp_ll2_buff *buf = mpa_buf->ll2_buf;  in qed_iwarp_process_mpa_pkt()
2159 	int rc = -EINVAL;  in qed_iwarp_process_mpa_pkt()  local
2162 	cid = le32_to_cpu(curr_pkt->cid);  in qed_iwarp_process_mpa_pkt()
2166 		DP_ERR(p_hwfn, "Invalid cid, drop and post back to rx cid=%x\n",  in qed_iwarp_process_mpa_pkt()
2172 		first_mpa_offset = le16_to_cpu(curr_pkt->first_mpa_offset);  in qed_iwarp_process_mpa_pkt()
2173 		mpa_data = ((u8 *)(buf->data) + first_mpa_offset);  in qed_iwarp_process_mpa_pkt()
2176 						  mpa_buf->tcp_payload_len,  in qed_iwarp_process_mpa_pkt()
2183 					    mpa_buf->tcp_payload_len,  in qed_iwarp_process_mpa_pkt()
2184 					    mpa_buf->placement_offset);  in qed_iwarp_process_mpa_pkt()
2187 				mpa_buf->tcp_payload_len = 0;  in qed_iwarp_process_mpa_pkt()
2191 			rc = qed_iwarp_win_right_edge(p_hwfn, fpdu);  in qed_iwarp_process_mpa_pkt()
2193 			if (rc) {  in qed_iwarp_process_mpa_pkt()
2195 					   "Can't send FPDU:reset rc=%d\n", rc);  in qed_iwarp_process_mpa_pkt()
2200 			mpa_buf->tcp_payload_len = 0;  in qed_iwarp_process_mpa_pkt()
2205 					    mpa_buf->tcp_payload_len,  in qed_iwarp_process_mpa_pkt()
2206 					    mpa_buf->placement_offset);  in qed_iwarp_process_mpa_pkt()
2208 			rc = qed_iwarp_send_fpdu(p_hwfn, fpdu, curr_pkt, buf,  in qed_iwarp_process_mpa_pkt()
2209 						 mpa_buf->tcp_payload_len,  in qed_iwarp_process_mpa_pkt()
2211 			if (rc) {  in qed_iwarp_process_mpa_pkt()
2213 					   "Can't send FPDU:reset rc=%d\n", rc);  in qed_iwarp_process_mpa_pkt()
2218 			mpa_buf->tcp_payload_len -= fpdu->fpdu_length;  in qed_iwarp_process_mpa_pkt()
2219 			le16_add_cpu(&curr_pkt->first_mpa_offset,  in qed_iwarp_process_mpa_pkt()
2220 				     fpdu->fpdu_length);  in qed_iwarp_process_mpa_pkt()
2224 			if (mpa_buf->tcp_payload_len < fpdu->incomplete_bytes) {  in qed_iwarp_process_mpa_pkt()
2229 					rc = qed_iwarp_win_right_edge(p_hwfn,  in qed_iwarp_process_mpa_pkt()
2231 					/* packet will be re-processed later */  in qed_iwarp_process_mpa_pkt()
2232 					if (rc)  in qed_iwarp_process_mpa_pkt()
2233 						return rc;  in qed_iwarp_process_mpa_pkt()
2236 				rc = qed_iwarp_cp_pkt(p_hwfn, fpdu, curr_pkt,  in qed_iwarp_process_mpa_pkt()
2238 						      mpa_buf->tcp_payload_len);  in qed_iwarp_process_mpa_pkt()
2239 				if (rc) /* packet will be re-processed later */  in qed_iwarp_process_mpa_pkt()
2240 					return rc;  in qed_iwarp_process_mpa_pkt()
2242 				mpa_buf->tcp_payload_len = 0;  in qed_iwarp_process_mpa_pkt()
2246 			rc = qed_iwarp_send_fpdu(p_hwfn, fpdu, curr_pkt, buf,  in qed_iwarp_process_mpa_pkt()
2247 						 mpa_buf->tcp_payload_len,  in qed_iwarp_process_mpa_pkt()
2249 			if (rc) {  in qed_iwarp_process_mpa_pkt()
2251 					   "Can't send FPDU:delay rc=%d\n", rc);  in qed_iwarp_process_mpa_pkt()
2252 				/* don't reset fpdu -> we need it for next  in qed_iwarp_process_mpa_pkt()
2258 			mpa_buf->tcp_payload_len -= fpdu->incomplete_bytes;  in qed_iwarp_process_mpa_pkt()
2259 			le16_add_cpu(&curr_pkt->first_mpa_offset,  in qed_iwarp_process_mpa_pkt()
2260 				     fpdu->incomplete_bytes);  in qed_iwarp_process_mpa_pkt()
2262 			/* The framed PDU was sent - no more incomplete bytes */  in qed_iwarp_process_mpa_pkt()
2263 			fpdu->incomplete_bytes = 0;  in qed_iwarp_process_mpa_pkt()
2266 	} while (mpa_buf->tcp_payload_len && !rc);  in qed_iwarp_process_mpa_pkt()
2268 	return rc;  in qed_iwarp_process_mpa_pkt()
2273 			      p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle);  in qed_iwarp_process_mpa_pkt()
2274 	return rc;  in qed_iwarp_process_mpa_pkt()
2279 	struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_process_pending_pkts()
2281 	int rc;  in qed_iwarp_process_pending_pkts()  local
2283 	while (!list_empty(&iwarp_info->mpa_buf_pending_list)) {  in qed_iwarp_process_pending_pkts()
2284 		mpa_buf = list_first_entry(&iwarp_info->mpa_buf_pending_list,  in qed_iwarp_process_pending_pkts()
2288 		rc = qed_iwarp_process_mpa_pkt(p_hwfn, mpa_buf);  in qed_iwarp_process_pending_pkts()
2293 		if (rc == -EBUSY)  in qed_iwarp_process_pending_pkts()
2296 		list_move_tail(&mpa_buf->list_entry,  in qed_iwarp_process_pending_pkts()
2297 			       &iwarp_info->mpa_buf_list);  in qed_iwarp_process_pending_pkts()
2299 		if (rc) {	/* different error, don't continue */  in qed_iwarp_process_pending_pkts()
2300 			DP_NOTICE(p_hwfn, "process pkts failed rc=%d\n", rc);  in qed_iwarp_process_pending_pkts()
2314 	iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_ll2_comp_mpa_pkt()
2315 	mpa_buf = list_first_entry(&iwarp_info->mpa_buf_list,  in qed_iwarp_ll2_comp_mpa_pkt()
2322 	list_del(&mpa_buf->list_entry);  in qed_iwarp_ll2_comp_mpa_pkt()
2323 	qed_iwarp_mpa_get_data(p_hwfn, &mpa_buf->data,  in qed_iwarp_ll2_comp_mpa_pkt()
2324 			       data->opaque_data_0, data->opaque_data_1);  in qed_iwarp_ll2_comp_mpa_pkt()
2326 	first_mpa_offset = le16_to_cpu(mpa_buf->data.first_mpa_offset);  in qed_iwarp_ll2_comp_mpa_pkt()
2330 …"LL2 MPA CompRx payload_len:0x%x\tfirst_mpa_offset:0x%x\ttcp_payload_offset:0x%x\tflags:0x%x\tcid:…  in qed_iwarp_ll2_comp_mpa_pkt()
2331 		   data->length.packet_length, first_mpa_offset,  in qed_iwarp_ll2_comp_mpa_pkt()
2332 		   mpa_buf->data.tcp_payload_offset, mpa_buf->data.flags,  in qed_iwarp_ll2_comp_mpa_pkt()
2333 		   mpa_buf->data.cid);  in qed_iwarp_ll2_comp_mpa_pkt()
2335 	mpa_buf->ll2_buf = data->cookie;  in qed_iwarp_ll2_comp_mpa_pkt()
2336 	mpa_buf->tcp_payload_len = data->length.packet_length -  in qed_iwarp_ll2_comp_mpa_pkt()
2339 	first_mpa_offset += data->u.placement_offset;  in qed_iwarp_ll2_comp_mpa_pkt()
2340 	mpa_buf->data.first_mpa_offset = cpu_to_le16(first_mpa_offset);  in qed_iwarp_ll2_comp_mpa_pkt()
2341 	mpa_buf->placement_offset = data->u.placement_offset;  in qed_iwarp_ll2_comp_mpa_pkt()
2343 	list_add_tail(&mpa_buf->list_entry, &iwarp_info->mpa_buf_pending_list);  in qed_iwarp_ll2_comp_mpa_pkt()
2348 	qed_iwarp_ll2_post_rx(p_hwfn, data->cookie,  in qed_iwarp_ll2_comp_mpa_pkt()
2349 			      iwarp_info->ll2_mpa_handle);  in qed_iwarp_ll2_comp_mpa_pkt()
2355 	struct qed_iwarp_ll2_buff *buf = data->cookie;  in qed_iwarp_ll2_comp_syn_pkt()
2367 	int rc;  in qed_iwarp_ll2_comp_syn_pkt()  local
2370 	ll2_syn_handle = p_hwfn->p_rdma_info->iwarp.ll2_syn_handle;  in qed_iwarp_ll2_comp_syn_pkt()
2373 	if (data->err_flags) {  in qed_iwarp_ll2_comp_syn_pkt()
2374 		DP_NOTICE(p_hwfn, "Error received on SYN packet: 0x%x\n",  in qed_iwarp_ll2_comp_syn_pkt()
2375 			  data->err_flags);  in qed_iwarp_ll2_comp_syn_pkt()
2379 	if (GET_FIELD(data->parse_flags,  in qed_iwarp_ll2_comp_syn_pkt()
2381 	    GET_FIELD(data->parse_flags, PARSING_AND_ERR_FLAGS_L4CHKSMERROR)) {  in qed_iwarp_ll2_comp_syn_pkt()
2386 	rc = qed_iwarp_parse_rx_pkt(p_hwfn, &cm_info, (u8 *)(buf->data) +  in qed_iwarp_ll2_comp_syn_pkt()
2387 				    data->u.placement_offset, remote_mac_addr,  in qed_iwarp_ll2_comp_syn_pkt()
2390 	if (rc)  in qed_iwarp_ll2_comp_syn_pkt()
2393 	/* Check if there is a listener for this 4-tuple+vlan */  in qed_iwarp_ll2_comp_syn_pkt()
2399 			   data->parse_flags, data->length.packet_length);  in qed_iwarp_ll2_comp_syn_pkt()
2403 		tx_pkt.l4_hdr_offset_w = (data->length.packet_length) >> 2;  in qed_iwarp_ll2_comp_syn_pkt()
2405 		tx_pkt.first_frag = buf->data_phys_addr +  in qed_iwarp_ll2_comp_syn_pkt()
2406 				    data->u.placement_offset;  in qed_iwarp_ll2_comp_syn_pkt()
2407 		tx_pkt.first_frag_len = data->length.packet_length;  in qed_iwarp_ll2_comp_syn_pkt()
2410 		rc = qed_ll2_prepare_tx_packet(p_hwfn, ll2_syn_handle,  in qed_iwarp_ll2_comp_syn_pkt()
2413 		if (rc) {  in qed_iwarp_ll2_comp_syn_pkt()
2415 				  "Can't post SYN back to chip rc=%d\n", rc);  in qed_iwarp_ll2_comp_syn_pkt()
2432 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_ll2_comp_syn_pkt()
2433 	list_add_tail(&ep->list_entry, &p_hwfn->p_rdma_info->iwarp.ep_list);  in qed_iwarp_ll2_comp_syn_pkt()
2434 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_ll2_comp_syn_pkt()
2436 	ether_addr_copy(ep->remote_mac_addr, remote_mac_addr);  in qed_iwarp_ll2_comp_syn_pkt()
2437 	ether_addr_copy(ep->local_mac_addr, local_mac_addr);  in qed_iwarp_ll2_comp_syn_pkt()
2439 	memcpy(&ep->cm_info, &cm_info, sizeof(ep->cm_info));  in qed_iwarp_ll2_comp_syn_pkt()
2442 	ep->mss = p_hwfn->p_rdma_info->iwarp.max_mtu - hdr_size;  in qed_iwarp_ll2_comp_syn_pkt()
2443 	ep->mss = min_t(u16, QED_IWARP_MAX_FW_MSS, ep->mss);  in qed_iwarp_ll2_comp_syn_pkt()
2445 	ep->event_cb = listener->event_cb;  in qed_iwarp_ll2_comp_syn_pkt()
2446 	ep->cb_context = listener->cb_context;  in qed_iwarp_ll2_comp_syn_pkt()
2447 	ep->connect_mode = TCP_CONNECT_PASSIVE;  in qed_iwarp_ll2_comp_syn_pkt()
2449 	ep->syn = buf;  in qed_iwarp_ll2_comp_syn_pkt()
2450 	ep->syn_ip_payload_length = (u16)payload_len;  in qed_iwarp_ll2_comp_syn_pkt()
2451 	ep->syn_phy_addr = buf->data_phys_addr + data->u.placement_offset +  in qed_iwarp_ll2_comp_syn_pkt()
2454 	rc = qed_iwarp_tcp_offload(p_hwfn, ep);  in qed_iwarp_ll2_comp_syn_pkt()
2455 	if (rc) {  in qed_iwarp_ll2_comp_syn_pkt()
2472 	dma_free_coherent(&p_hwfn->cdev->pdev->dev, buffer->buff_size,  in qed_iwarp_ll2_rel_rx_pkt()
2473 			  buffer->data, buffer->data_phys_addr);  in qed_iwarp_ll2_rel_rx_pkt()
2489 	piggy = buffer->piggy_buf;  in qed_iwarp_ll2_comp_tx_pkt()
2491 		buffer->piggy_buf = NULL;  in qed_iwarp_ll2_comp_tx_pkt()
2497 	if (connection_handle == p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle)  in qed_iwarp_ll2_comp_tx_pkt()
2513 	if (buffer->piggy_buf) {  in qed_iwarp_ll2_rel_tx_pkt()
2514 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,  in qed_iwarp_ll2_rel_tx_pkt()
2515 				  buffer->piggy_buf->buff_size,  in qed_iwarp_ll2_rel_tx_pkt()
2516 				  buffer->piggy_buf->data,  in qed_iwarp_ll2_rel_tx_pkt()
2517 				  buffer->piggy_buf->data_phys_addr);  in qed_iwarp_ll2_rel_tx_pkt()
2519 		kfree(buffer->piggy_buf);  in qed_iwarp_ll2_rel_tx_pkt()
2522 	dma_free_coherent(&p_hwfn->cdev->pdev->dev, buffer->buff_size,  in qed_iwarp_ll2_rel_tx_pkt()
2523 			  buffer->data, buffer->data_phys_addr);  in qed_iwarp_ll2_rel_tx_pkt()
2546 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "(0x%x) Flush fpdu\n", cid);  in qed_iwarp_ll2_slowpath()
2555 	struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_ll2_stop()
2556 	int rc = 0;  in qed_iwarp_ll2_stop()  local
2558 	if (iwarp_info->ll2_syn_handle != QED_IWARP_HANDLE_INVAL) {  in qed_iwarp_ll2_stop()
2559 		rc = qed_ll2_terminate_connection(p_hwfn,  in qed_iwarp_ll2_stop()
2560 						  iwarp_info->ll2_syn_handle);  in qed_iwarp_ll2_stop()
2561 		if (rc)  in qed_iwarp_ll2_stop()
2564 		qed_ll2_release_connection(p_hwfn, iwarp_info->ll2_syn_handle);  in qed_iwarp_ll2_stop()
2565 		iwarp_info->ll2_syn_handle = QED_IWARP_HANDLE_INVAL;  in qed_iwarp_ll2_stop()
2568 	if (iwarp_info->ll2_ooo_handle != QED_IWARP_HANDLE_INVAL) {  in qed_iwarp_ll2_stop()
2569 		rc = qed_ll2_terminate_connection(p_hwfn,  in qed_iwarp_ll2_stop()
2570 						  iwarp_info->ll2_ooo_handle);  in qed_iwarp_ll2_stop()
2571 		if (rc)  in qed_iwarp_ll2_stop()
2574 		qed_ll2_release_connection(p_hwfn, iwarp_info->ll2_ooo_handle);  in qed_iwarp_ll2_stop()
2575 		iwarp_info->ll2_ooo_handle = QED_IWARP_HANDLE_INVAL;  in qed_iwarp_ll2_stop()
2578 	if (iwarp_info->ll2_mpa_handle != QED_IWARP_HANDLE_INVAL) {  in qed_iwarp_ll2_stop()
2579 		rc = qed_ll2_terminate_connection(p_hwfn,  in qed_iwarp_ll2_stop()
2580 						  iwarp_info->ll2_mpa_handle);  in qed_iwarp_ll2_stop()
2581 		if (rc)  in qed_iwarp_ll2_stop()
2584 		qed_ll2_release_connection(p_hwfn, iwarp_info->ll2_mpa_handle);  in qed_iwarp_ll2_stop()
2585 		iwarp_info->ll2_mpa_handle = QED_IWARP_HANDLE_INVAL;  in qed_iwarp_ll2_stop()
2588 	qed_llh_remove_mac_filter(p_hwfn->cdev, 0,  in qed_iwarp_ll2_stop()
2589 				  p_hwfn->p_rdma_info->iwarp.mac_addr);  in qed_iwarp_ll2_stop()
2591 	return rc;  in qed_iwarp_ll2_stop()
2599 	int rc = 0;  in qed_iwarp_ll2_alloc_buffers()  local
2605 			rc = -ENOMEM;  in qed_iwarp_ll2_alloc_buffers()
2609 		buffer->data = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,  in qed_iwarp_ll2_alloc_buffers()
2611 						  &buffer->data_phys_addr,  in qed_iwarp_ll2_alloc_buffers()
2613 		if (!buffer->data) {  in qed_iwarp_ll2_alloc_buffers()
2615 			rc = -ENOMEM;  in qed_iwarp_ll2_alloc_buffers()
2619 		buffer->buff_size = buff_size;  in qed_iwarp_ll2_alloc_buffers()
2620 		rc = qed_iwarp_ll2_post_rx(p_hwfn, buffer, ll2_handle);  in qed_iwarp_ll2_alloc_buffers()
2621 		if (rc)  in qed_iwarp_ll2_alloc_buffers()
2625 	return rc;  in qed_iwarp_ll2_alloc_buffers()
2642 	int rc = 0;  in qed_iwarp_ll2_start()  local
2645 	iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_ll2_start()
2646 	iwarp_info->ll2_syn_handle = QED_IWARP_HANDLE_INVAL;  in qed_iwarp_ll2_start()
2647 	iwarp_info->ll2_ooo_handle = QED_IWARP_HANDLE_INVAL;  in qed_iwarp_ll2_start()
2648 	iwarp_info->ll2_mpa_handle = QED_IWARP_HANDLE_INVAL;  in qed_iwarp_ll2_start()
2650 	iwarp_info->max_mtu = params->max_mtu;  in qed_iwarp_ll2_start()
2652 	ether_addr_copy(p_hwfn->p_rdma_info->iwarp.mac_addr, params->mac_addr);  in qed_iwarp_ll2_start()
2654 	rc = qed_llh_add_mac_filter(p_hwfn->cdev, 0, params->mac_addr);  in qed_iwarp_ll2_start()
2655 	if (rc)  in qed_iwarp_ll2_start()
2656 		return rc;  in qed_iwarp_ll2_start()
2670 	data.input.mtu = params->max_mtu;  in qed_iwarp_ll2_start()
2676 	data.p_connection_handle = &iwarp_info->ll2_syn_handle;  in qed_iwarp_ll2_start()
2679 	rc = qed_ll2_acquire_connection(p_hwfn, &data);  in qed_iwarp_ll2_start()
2680 	if (rc) {  in qed_iwarp_ll2_start()
2682 		qed_llh_remove_mac_filter(p_hwfn->cdev, 0, params->mac_addr);  in qed_iwarp_ll2_start()
2683 		return rc;  in qed_iwarp_ll2_start()
2686 	rc = qed_ll2_establish_connection(p_hwfn, iwarp_info->ll2_syn_handle);  in qed_iwarp_ll2_start()
2687 	if (rc) {  in qed_iwarp_ll2_start()
2692 	buff_size = QED_IWARP_MAX_BUF_SIZE(params->max_mtu);  in qed_iwarp_ll2_start()
2693 	rc = qed_iwarp_ll2_alloc_buffers(p_hwfn,  in qed_iwarp_ll2_start()
2696 					 iwarp_info->ll2_syn_handle);  in qed_iwarp_ll2_start()
2697 	if (rc)  in qed_iwarp_ll2_start()
2704 	data.input.mtu = params->max_mtu;  in qed_iwarp_ll2_start()
2707 		     iwarp_info->max_mtu;  in qed_iwarp_ll2_start()
2715 	data.p_connection_handle = &iwarp_info->ll2_ooo_handle;  in qed_iwarp_ll2_start()
2717 	rc = qed_ll2_acquire_connection(p_hwfn, &data);  in qed_iwarp_ll2_start()
2718 	if (rc)  in qed_iwarp_ll2_start()
2721 	rc = qed_ll2_establish_connection(p_hwfn, iwarp_info->ll2_ooo_handle);  in qed_iwarp_ll2_start()
2722 	if (rc)  in qed_iwarp_ll2_start()
2731 	data.input.mtu = params->max_mtu;  in qed_iwarp_ll2_start()
2741 	data.p_connection_handle = &iwarp_info->ll2_mpa_handle;  in qed_iwarp_ll2_start()
2745 	rc = qed_ll2_acquire_connection(p_hwfn, &data);  in qed_iwarp_ll2_start()
2746 	if (rc)  in qed_iwarp_ll2_start()
2749 	rc = qed_ll2_establish_connection(p_hwfn, iwarp_info->ll2_mpa_handle);  in qed_iwarp_ll2_start()
2750 	if (rc)  in qed_iwarp_ll2_start()
2753 	rc = qed_iwarp_ll2_alloc_buffers(p_hwfn,  in qed_iwarp_ll2_start()
2756 					 iwarp_info->ll2_mpa_handle);  in qed_iwarp_ll2_start()
2757 	if (rc)  in qed_iwarp_ll2_start()
2760 	iwarp_info->partial_fpdus = kcalloc((u16)p_hwfn->p_rdma_info->num_qps,  in qed_iwarp_ll2_start()
2761 					    sizeof(*iwarp_info->partial_fpdus),  in qed_iwarp_ll2_start()
2763 	if (!iwarp_info->partial_fpdus) {  in qed_iwarp_ll2_start()
2764 		rc = -ENOMEM;  in qed_iwarp_ll2_start()
2768 	iwarp_info->max_num_partial_fpdus = (u16)p_hwfn->p_rdma_info->num_qps;  in qed_iwarp_ll2_start()
2770 	iwarp_info->mpa_intermediate_buf = kzalloc(buff_size, GFP_KERNEL);  in qed_iwarp_ll2_start()
2771 	if (!iwarp_info->mpa_intermediate_buf) {  in qed_iwarp_ll2_start()
2772 		rc = -ENOMEM;  in qed_iwarp_ll2_start()
2781 	iwarp_info->mpa_bufs = kcalloc(data.input.rx_num_desc,  in qed_iwarp_ll2_start()
2782 				       sizeof(*iwarp_info->mpa_bufs),  in qed_iwarp_ll2_start()
2784 	if (!iwarp_info->mpa_bufs) {  in qed_iwarp_ll2_start()
2785 		rc = -ENOMEM;  in qed_iwarp_ll2_start()
2789 	INIT_LIST_HEAD(&iwarp_info->mpa_buf_pending_list);  in qed_iwarp_ll2_start()
2790 	INIT_LIST_HEAD(&iwarp_info->mpa_buf_list);  in qed_iwarp_ll2_start()
2792 		list_add_tail(&iwarp_info->mpa_bufs[i].list_entry,  in qed_iwarp_ll2_start()
2793 			      &iwarp_info->mpa_buf_list);  in qed_iwarp_ll2_start()
2794 	return rc;  in qed_iwarp_ll2_start()
2798 	return rc;  in qed_iwarp_ll2_start()
2812 	struct qed_dev *cdev = p_hwfn->cdev;  in qed_iwarp_setup()
2817 	iwarp_info = &p_hwfn->p_rdma_info->iwarp;  in qed_iwarp_setup()
2819 	iwarp_info->tcp_flags = QED_IWARP_TS_EN;  in qed_iwarp_setup()
2827 	iwarp_info->rcv_wnd_scale = ilog2(rcv_wnd_size) -  in qed_iwarp_setup()
2829 	iwarp_info->rcv_wnd_size = rcv_wnd_size >> iwarp_info->rcv_wnd_scale;  in qed_iwarp_setup()
2830 	iwarp_info->crc_needed = QED_IWARP_PARAM_CRC_NEEDED;  in qed_iwarp_setup()
2831 	iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED;  in qed_iwarp_setup()
2833 	iwarp_info->peer2peer = QED_IWARP_PARAM_P2P;  in qed_iwarp_setup()
2835 	iwarp_info->rtr_type =  MPA_RTR_TYPE_ZERO_SEND |  in qed_iwarp_setup()
2839 	spin_lock_init(&p_hwfn->p_rdma_info->iwarp.qp_lock);  in qed_iwarp_setup()
2840 	INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.ep_list);  in qed_iwarp_setup()
2841 	INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.listen_list);  in qed_iwarp_setup()
2852 	int rc;  in qed_iwarp_stop()  local
2855 	rc = qed_iwarp_wait_for_all_cids(p_hwfn);  in qed_iwarp_stop()
2856 	if (rc)  in qed_iwarp_stop()
2857 		return rc;  in qed_iwarp_stop()
2868 	qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_ERROR, true);  in qed_iwarp_qp_in_error()
2872 	params.cm_info = &ep->cm_info;  in qed_iwarp_qp_in_error()
2874 			 0 : -ECONNRESET;  in qed_iwarp_qp_in_error()
2877 	smp_store_release(&ep->state, QED_IWARP_EP_CLOSED);  in qed_iwarp_qp_in_error()
2879 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_qp_in_error()
2880 	list_del(&ep->list_entry);  in qed_iwarp_qp_in_error()
2881 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_qp_in_error()
2883 	ep->event_cb(ep->cb_context, &params);  in qed_iwarp_qp_in_error()
2893 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "EP(0x%x) fw_ret_code=%d\n",  in qed_iwarp_exception_received()
2894 		   ep->cid, fw_ret_code);  in qed_iwarp_exception_received()
2903 		params.status = -ECONNRESET;  in qed_iwarp_exception_received()
2952 		params.cm_info = &ep->cm_info;  in qed_iwarp_exception_received()
2953 		ep->event_cb(ep->cb_context, &params);  in qed_iwarp_exception_received()
2966 	params.cm_info = &ep->cm_info;  in qed_iwarp_tcp_connect_unsuccessful()
2968 	smp_store_release(&ep->state, QED_IWARP_EP_CLOSED);  in qed_iwarp_tcp_connect_unsuccessful()
2973 			   "%s(0x%x) TCP connect got invalid packet\n",  in qed_iwarp_tcp_connect_unsuccessful()
2974 			   QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);  in qed_iwarp_tcp_connect_unsuccessful()
2975 		params.status = -ECONNRESET;  in qed_iwarp_tcp_connect_unsuccessful()
2979 			   "%s(0x%x) TCP Connection Reset\n",  in qed_iwarp_tcp_connect_unsuccessful()
2980 			   QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);  in qed_iwarp_tcp_connect_unsuccessful()
2981 		params.status = -ECONNRESET;  in qed_iwarp_tcp_connect_unsuccessful()
2984 		DP_NOTICE(p_hwfn, "%s(0x%x) TCP timeout\n",  in qed_iwarp_tcp_connect_unsuccessful()
2985 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);  in qed_iwarp_tcp_connect_unsuccessful()
2986 		params.status = -EBUSY;  in qed_iwarp_tcp_connect_unsuccessful()
2989 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA not supported VER\n",  in qed_iwarp_tcp_connect_unsuccessful()
2990 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);  in qed_iwarp_tcp_connect_unsuccessful()
2991 		params.status = -ECONNREFUSED;  in qed_iwarp_tcp_connect_unsuccessful()
2994 		DP_NOTICE(p_hwfn, "%s(0x%x) MPA Invalid Packet\n",  in qed_iwarp_tcp_connect_unsuccessful()
2995 			  QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);  in qed_iwarp_tcp_connect_unsuccessful()
2996 		params.status = -ECONNRESET;  in qed_iwarp_tcp_connect_unsuccessful()
3000 		       "%s(0x%x) Unexpected return code tcp connect: %d\n",  in qed_iwarp_tcp_connect_unsuccessful()
3002 		       ep->tcp_cid, fw_return_code);  in qed_iwarp_tcp_connect_unsuccessful()
3003 		params.status = -ECONNRESET;  in qed_iwarp_tcp_connect_unsuccessful()
3007 	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {  in qed_iwarp_tcp_connect_unsuccessful()
3008 		ep->tcp_cid = QED_IWARP_INVALID_TCP_CID;  in qed_iwarp_tcp_connect_unsuccessful()
3011 		ep->event_cb(ep->cb_context, &params);  in qed_iwarp_tcp_connect_unsuccessful()
3012 		spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_tcp_connect_unsuccessful()
3013 		list_del(&ep->list_entry);  in qed_iwarp_tcp_connect_unsuccessful()
3014 		spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_tcp_connect_unsuccessful()
3022 	u8 ll2_syn_handle = p_hwfn->p_rdma_info->iwarp.ll2_syn_handle;  in qed_iwarp_connect_complete()
3024 	if (ep->connect_mode == TCP_CONNECT_PASSIVE) {  in qed_iwarp_connect_complete()
3026 		qed_iwarp_ll2_post_rx(p_hwfn, ep->syn, ll2_syn_handle);  in qed_iwarp_connect_complete()
3028 		ep->syn = NULL;  in qed_iwarp_connect_complete()
3030 		/* If connect failed - upper layer doesn't know about it */  in qed_iwarp_connect_complete()
3048 	if (!ep || (ep->sig != QED_EP_SIG)) {  in qed_iwarp_check_ep_ok()
3060 	struct qed_rdma_events events = p_hwfn->p_rdma_info->events;  in qed_iwarp_async_event()
3061 	struct regpair *fw_handle = &data->rdma_data.async_handle;  in qed_iwarp_async_event()
3067 	ep = (struct qed_iwarp_ep *)(uintptr_t)HILO_64(fw_handle->hi,  in qed_iwarp_async_event()
3068 						       fw_handle->lo);  in qed_iwarp_async_event()
3072 		/* Async completion after TCP 3-way handshake */  in qed_iwarp_async_event()
3074 			return -EINVAL;  in qed_iwarp_async_event()
3077 			   "EP(0x%x) IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE fw_ret_code=%d\n",  in qed_iwarp_async_event()
3078 			   ep->tcp_cid, fw_return_code);  in qed_iwarp_async_event()
3083 			return -EINVAL;  in qed_iwarp_async_event()
3086 			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED fw_ret_code=%d\n",  in qed_iwarp_async_event()
3087 			   ep->cid, fw_return_code);  in qed_iwarp_async_event()
3093 			return -EINVAL;  in qed_iwarp_async_event()
3096 			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE fw_ret_code=%d\n",  in qed_iwarp_async_event()
3097 			   ep->cid, fw_return_code);  in qed_iwarp_async_event()
3103 			return -EINVAL;  in qed_iwarp_async_event()
3106 			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_MPA_REPLY_ARRIVED fw_ret_code=%d\n",  in qed_iwarp_async_event()
3107 			   ep->cid, fw_return_code);  in qed_iwarp_async_event()
3112 			return -EINVAL;  in qed_iwarp_async_event()
3115 			   "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE fw_ret_code=%d\n",  in qed_iwarp_async_event()
3116 			   ep->cid, fw_return_code);  in qed_iwarp_async_event()
3120 		cid = (u16)le32_to_cpu(fw_handle->lo);  in qed_iwarp_async_event()
3122 			   "(0x%x)IWARP_EVENT_TYPE_ASYNC_CID_CLEANED\n", cid);  in qed_iwarp_async_event()
3128 		srq_offset = p_hwfn->p_rdma_info->srq_id_offset;  in qed_iwarp_async_event()
3130 		srq_id = ((u16)le32_to_cpu(fw_handle->lo)) - srq_offset;  in qed_iwarp_async_event()
3137 		srq_offset = p_hwfn->p_rdma_info->srq_id_offset;  in qed_iwarp_async_event()
3139 		srq_id = ((u16)le32_to_cpu(fw_handle->lo)) - srq_offset;  in qed_iwarp_async_event()
3147 		p_hwfn->p_rdma_info->events.affiliated_event(  in qed_iwarp_async_event()
3148 			p_hwfn->p_rdma_info->events.context,  in qed_iwarp_async_event()
3155 		return -EINVAL;  in qed_iwarp_async_event()
3170 		return -ENOMEM;  in qed_iwarp_create_listen()
3172 	listener->ip_version = iparams->ip_version;  in qed_iwarp_create_listen()
3173 	memcpy(listener->ip_addr, iparams->ip_addr, sizeof(listener->ip_addr));  in qed_iwarp_create_listen()
3174 	listener->port = iparams->port;  in qed_iwarp_create_listen()
3175 	listener->vlan = iparams->vlan;  in qed_iwarp_create_listen()
3177 	listener->event_cb = iparams->event_cb;  in qed_iwarp_create_listen()
3178 	listener->cb_context = iparams->cb_context;  in qed_iwarp_create_listen()
3179 	listener->max_backlog = iparams->max_backlog;  in qed_iwarp_create_listen()
3180 	oparams->handle = listener;  in qed_iwarp_create_listen()
3182 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_create_listen()
3183 	list_add_tail(&listener->list_entry,  in qed_iwarp_create_listen()
3184 		      &p_hwfn->p_rdma_info->iwarp.listen_list);  in qed_iwarp_create_listen()
3185 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_create_listen()
3189 		   "callback=%p handle=%p ip=%x:%x:%x:%x port=0x%x vlan=0x%x\n",  in qed_iwarp_create_listen()
3190 		   listener->event_cb,  in qed_iwarp_create_listen()
3192 		   listener->ip_addr[0],  in qed_iwarp_create_listen()
3193 		   listener->ip_addr[1],  in qed_iwarp_create_listen()
3194 		   listener->ip_addr[2],  in qed_iwarp_create_listen()
3195 		   listener->ip_addr[3], listener->port, listener->vlan);  in qed_iwarp_create_listen()
3207 	spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_destroy_listen()
3208 	list_del(&listener->list_entry);  in qed_iwarp_destroy_listen()
3209 	spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);  in qed_iwarp_destroy_listen()
3223 	int rc;  in qed_iwarp_send_rtr()  local
3225 	ep = iparams->ep_context;  in qed_iwarp_send_rtr()
3228 		return -EINVAL;  in qed_iwarp_send_rtr()
3231 	qp = ep->qp;  in qed_iwarp_send_rtr()
3233 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x)\n",  in qed_iwarp_send_rtr()
3234 		   qp->icid, ep->tcp_cid);  in qed_iwarp_send_rtr()
3237 	init_data.cid = qp->icid;  in qed_iwarp_send_rtr()
3238 	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;  in qed_iwarp_send_rtr()
3241 	rc = qed_sp_init_request(p_hwfn, &p_ent,  in qed_iwarp_send_rtr()
3245 	if (rc)  in qed_iwarp_send_rtr()
3246 		return rc;  in qed_iwarp_send_rtr()
3248 	rc = qed_spq_post(p_hwfn, p_ent, NULL);  in qed_iwarp_send_rtr()
3250 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = 0x%x\n", rc);  in qed_iwarp_send_rtr()
3252 	return rc;  in qed_iwarp_send_rtr()
3259 	out_params->state = qed_iwarp2roce_state(qp->iwarp_state);  in qed_iwarp_query_qp()