Lines Matching refs:packet
213 struct hfi1_packet *packet) in rcv_hdrerr() argument
215 struct ib_header *rhdr = packet->hdr; in rcv_hdrerr()
216 u32 rte = rhf_rcv_type_err(packet->rhf); in rcv_hdrerr()
223 if ((packet->rhf & RHF_DC_ERR) && in rcv_hdrerr()
227 if (packet->rhf & RHF_ICRC_ERR) in rcv_hdrerr()
230 if (packet->etype == RHF_RCV_TYPE_BYPASS) { in rcv_hdrerr()
237 packet->ohdr = &rhdr->u.oth; in rcv_hdrerr()
239 packet->ohdr = &rhdr->u.l.oth; in rcv_hdrerr()
240 packet->grh = &rhdr->u.l.grh; in rcv_hdrerr()
246 if (packet->rhf & RHF_TID_ERR) { in rcv_hdrerr()
248 u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ in rcv_hdrerr()
257 if (packet->grh) { in rcv_hdrerr()
259 struct ib_grh *grh = packet->grh; in rcv_hdrerr()
269 qp_num = ib_bth_get_qpn(packet->ohdr); in rcv_hdrerr()
295 hfi1_rc_hdrerr(rcd, packet, qp); in rcv_hdrerr()
314 if (rhf_use_egr_bfr(packet->rhf)) in rcv_hdrerr()
315 ebuf = packet->ebuf; in rcv_hdrerr()
320 opcode = ib_bth_get_opcode(packet->ohdr); in rcv_hdrerr()
331 sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf); in rcv_hdrerr()
334 lqpn = ib_bth_get_qpn(packet->ohdr); in rcv_hdrerr()
362 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK; in rcv_hdrerr()
374 struct hfi1_packet *packet) in init_packet() argument
376 packet->rsize = get_hdrqentsize(rcd); /* words */ in init_packet()
377 packet->maxcnt = get_hdrq_cnt(rcd) * packet->rsize; /* words */ in init_packet()
378 packet->rcd = rcd; in init_packet()
379 packet->updegr = 0; in init_packet()
380 packet->etail = -1; in init_packet()
381 packet->rhf_addr = get_rhf_addr(rcd); in init_packet()
382 packet->rhf = rhf_to_cpu(packet->rhf_addr); in init_packet()
383 packet->rhqoff = hfi1_rcd_head(rcd); in init_packet()
384 packet->numpkt = 0; in init_packet()
507 struct hfi1_packet *packet) in init_ps_mdata() argument
509 struct hfi1_ctxtdata *rcd = packet->rcd; in init_ps_mdata()
512 mdata->rsize = packet->rsize; in init_ps_mdata()
513 mdata->maxcnt = packet->maxcnt; in init_ps_mdata()
514 mdata->ps_head = packet->rhqoff; in init_ps_mdata()
570 #define prescan_rxq(rcd, packet) \ argument
573 __prescan_rxq(packet); \
575 static void __prescan_rxq(struct hfi1_packet *packet) in __prescan_rxq() argument
577 struct hfi1_ctxtdata *rcd = packet->rcd; in __prescan_rxq()
580 init_ps_mdata(&mdata, packet); in __prescan_rxq()
585 packet->rcd->rhf_offset; in __prescan_rxq()
602 packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr); in __prescan_rxq()
603 hdr = packet->hdr; in __prescan_rxq()
607 packet->ohdr = &hdr->u.oth; in __prescan_rxq()
608 packet->grh = NULL; in __prescan_rxq()
610 packet->ohdr = &hdr->u.l.oth; in __prescan_rxq()
611 packet->grh = &hdr->u.l.grh; in __prescan_rxq()
616 if (!hfi1_may_ecn(packet)) in __prescan_rxq()
619 bth1 = be32_to_cpu(packet->ohdr->bth[1]); in __prescan_rxq()
629 hfi1_process_ecn_slowpath(qp, packet, true); in __prescan_rxq()
634 packet->ohdr->bth[1] = cpu_to_be32(bth1); in __prescan_rxq()
640 static void process_rcv_qp_work(struct hfi1_packet *packet) in process_rcv_qp_work() argument
643 struct hfi1_ctxtdata *rcd = packet->rcd; in process_rcv_qp_work()
653 packet->qp = qp; in process_rcv_qp_work()
654 hfi1_send_rc_ack(packet, 0); in process_rcv_qp_work()
670 static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) in max_packet_exceeded() argument
673 if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) in max_packet_exceeded()
675 process_rcv_qp_work(packet); in max_packet_exceeded()
679 this_cpu_inc(*packet->rcd->dd->rcv_limit); in max_packet_exceeded()
684 static inline int check_max_packet(struct hfi1_packet *packet, int thread) in check_max_packet() argument
688 if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) in check_max_packet()
689 ret = max_packet_exceeded(packet, thread); in check_max_packet()
693 static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) in skip_rcv_packet() argument
697 packet->rcd->dd->ctx0_seq_drop++; in skip_rcv_packet()
699 packet->rhqoff += packet->rsize; in skip_rcv_packet()
700 if (packet->rhqoff >= packet->maxcnt) in skip_rcv_packet()
701 packet->rhqoff = 0; in skip_rcv_packet()
703 packet->numpkt++; in skip_rcv_packet()
704 ret = check_max_packet(packet, thread); in skip_rcv_packet()
706 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + in skip_rcv_packet()
707 packet->rcd->rhf_offset; in skip_rcv_packet()
708 packet->rhf = rhf_to_cpu(packet->rhf_addr); in skip_rcv_packet()
713 static void process_rcv_packet_napi(struct hfi1_packet *packet) in process_rcv_packet_napi() argument
715 packet->etype = rhf_rcv_type(packet->rhf); in process_rcv_packet_napi()
718 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ in process_rcv_packet_napi()
720 packet->etail = rhf_egr_index(packet->rhf); in process_rcv_packet_napi()
721 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, in process_rcv_packet_napi()
722 &packet->updegr); in process_rcv_packet_napi()
728 prefetch_range(packet->ebuf, in process_rcv_packet_napi()
729 packet->tlen - ((packet->rcd->rcvhdrqentsize - in process_rcv_packet_napi()
730 (rhf_hdrq_offset(packet->rhf) in process_rcv_packet_napi()
733 packet->rcd->rhf_rcv_function_map[packet->etype](packet); in process_rcv_packet_napi()
734 packet->numpkt++; in process_rcv_packet_napi()
737 packet->rhqoff += packet->rsize; in process_rcv_packet_napi()
738 if (packet->rhqoff >= packet->maxcnt) in process_rcv_packet_napi()
739 packet->rhqoff = 0; in process_rcv_packet_napi()
741 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + in process_rcv_packet_napi()
742 packet->rcd->rhf_offset; in process_rcv_packet_napi()
743 packet->rhf = rhf_to_cpu(packet->rhf_addr); in process_rcv_packet_napi()
746 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) in process_rcv_packet() argument
750 packet->etype = rhf_rcv_type(packet->rhf); in process_rcv_packet()
753 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ in process_rcv_packet()
755 packet->ebuf = NULL; in process_rcv_packet()
756 if (rhf_use_egr_bfr(packet->rhf)) { in process_rcv_packet()
757 packet->etail = rhf_egr_index(packet->rhf); in process_rcv_packet()
758 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, in process_rcv_packet()
759 &packet->updegr); in process_rcv_packet()
765 prefetch_range(packet->ebuf, in process_rcv_packet()
766 packet->tlen - ((get_hdrqentsize(packet->rcd) - in process_rcv_packet()
767 (rhf_hdrq_offset(packet->rhf) in process_rcv_packet()
779 packet->rcd->rhf_rcv_function_map[packet->etype](packet); in process_rcv_packet()
780 packet->numpkt++; in process_rcv_packet()
783 packet->rhqoff += packet->rsize; in process_rcv_packet()
784 if (packet->rhqoff >= packet->maxcnt) in process_rcv_packet()
785 packet->rhqoff = 0; in process_rcv_packet()
787 ret = check_max_packet(packet, thread); in process_rcv_packet()
789 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + in process_rcv_packet()
790 packet->rcd->rhf_offset; in process_rcv_packet()
791 packet->rhf = rhf_to_cpu(packet->rhf_addr); in process_rcv_packet()
796 static inline void process_rcv_update(int last, struct hfi1_packet *packet) in process_rcv_update() argument
804 if (!last && !(packet->numpkt & 0xf)) { in process_rcv_update()
805 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr, in process_rcv_update()
806 packet->etail, 0, 0); in process_rcv_update()
807 packet->updegr = 0; in process_rcv_update()
809 packet->grh = NULL; in process_rcv_update()
812 static inline void finish_packet(struct hfi1_packet *packet) in finish_packet() argument
820 update_usrhead(packet->rcd, hfi1_rcd_head(packet->rcd), packet->updegr, in finish_packet()
821 packet->etail, rcv_intr_dynamic, packet->numpkt); in finish_packet()
835 struct hfi1_packet packet; in handle_receive_interrupt_napi_fp() local
837 init_packet(rcd, &packet); in handle_receive_interrupt_napi_fp()
838 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_fp()
841 while (packet.numpkt < budget) { in handle_receive_interrupt_napi_fp()
842 process_rcv_packet_napi(&packet); in handle_receive_interrupt_napi_fp()
843 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_fp()
846 process_rcv_update(0, &packet); in handle_receive_interrupt_napi_fp()
848 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_napi_fp()
850 finish_packet(&packet); in handle_receive_interrupt_napi_fp()
851 return packet.numpkt; in handle_receive_interrupt_napi_fp()
860 struct hfi1_packet packet; in handle_receive_interrupt_nodma_rtail() local
862 init_packet(rcd, &packet); in handle_receive_interrupt_nodma_rtail()
863 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) { in handle_receive_interrupt_nodma_rtail()
868 prescan_rxq(rcd, &packet); in handle_receive_interrupt_nodma_rtail()
871 last = process_rcv_packet(&packet, thread); in handle_receive_interrupt_nodma_rtail()
872 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_nodma_rtail()
874 process_rcv_update(last, &packet); in handle_receive_interrupt_nodma_rtail()
876 process_rcv_qp_work(&packet); in handle_receive_interrupt_nodma_rtail()
877 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_nodma_rtail()
879 finish_packet(&packet); in handle_receive_interrupt_nodma_rtail()
887 struct hfi1_packet packet; in handle_receive_interrupt_dma_rtail() local
889 init_packet(rcd, &packet); in handle_receive_interrupt_dma_rtail()
891 if (packet.rhqoff == hdrqtail) { in handle_receive_interrupt_dma_rtail()
897 prescan_rxq(rcd, &packet); in handle_receive_interrupt_dma_rtail()
900 last = process_rcv_packet(&packet, thread); in handle_receive_interrupt_dma_rtail()
901 if (packet.rhqoff == hdrqtail) in handle_receive_interrupt_dma_rtail()
903 process_rcv_update(last, &packet); in handle_receive_interrupt_dma_rtail()
905 process_rcv_qp_work(&packet); in handle_receive_interrupt_dma_rtail()
906 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_dma_rtail()
908 finish_packet(&packet); in handle_receive_interrupt_dma_rtail()
953 static bool __set_armed_to_active(struct hfi1_packet *packet) in __set_armed_to_active() argument
955 u8 etype = rhf_rcv_type(packet->rhf); in __set_armed_to_active()
959 struct ib_header *hdr = hfi1_get_msgheader(packet->rcd, in __set_armed_to_active()
960 packet->rhf_addr); in __set_armed_to_active()
961 sc = hfi1_9B_get_sc5(hdr, packet->rhf); in __set_armed_to_active()
964 packet->rcd, in __set_armed_to_active()
965 packet->rhf_addr); in __set_armed_to_active()
969 int hwstate = driver_lstate(packet->rcd->ppd); in __set_armed_to_active()
971 &packet->rcd->ppd->linkstate_active_work; in __set_armed_to_active()
974 dd_dev_info(packet->rcd->dd, in __set_armed_to_active()
980 queue_work(packet->rcd->ppd->link_wq, lsaw); in __set_armed_to_active()
992 static bool set_armed_to_active(struct hfi1_packet *packet) in set_armed_to_active() argument
994 if (likely(packet->rcd->ppd->host_link_state != HLS_UP_ARMED)) in set_armed_to_active()
996 return __set_armed_to_active(packet); in set_armed_to_active()
1011 struct hfi1_packet packet; in handle_receive_interrupt() local
1017 init_packet(rcd, &packet); in handle_receive_interrupt()
1020 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) { in handle_receive_interrupt()
1027 if (packet.rhqoff == hdrqtail) { in handle_receive_interrupt()
1038 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt()
1042 prescan_rxq(rcd, &packet); in handle_receive_interrupt()
1047 packet.rhqoff += packet.rsize; in handle_receive_interrupt()
1048 packet.rhf_addr = (__le32 *)rcd->rcvhdrq + in handle_receive_interrupt()
1049 packet.rhqoff + in handle_receive_interrupt()
1051 packet.rhf = rhf_to_cpu(packet.rhf_addr); in handle_receive_interrupt()
1054 last = skip_rcv_packet(&packet, thread); in handle_receive_interrupt()
1057 if (set_armed_to_active(&packet)) in handle_receive_interrupt()
1059 last = process_rcv_packet(&packet, thread); in handle_receive_interrupt()
1063 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt()
1066 if (packet.rhqoff == hdrqtail) in handle_receive_interrupt()
1076 rhf_rcv_seq(packet.rhf)); in handle_receive_interrupt()
1086 process_rcv_update(last, &packet); in handle_receive_interrupt()
1089 process_rcv_qp_work(&packet); in handle_receive_interrupt()
1090 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt()
1097 finish_packet(&packet); in handle_receive_interrupt()
1115 struct hfi1_packet packet; in handle_receive_interrupt_napi_sp() local
1117 init_packet(rcd, &packet); in handle_receive_interrupt_napi_sp()
1118 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_sp()
1121 while (last != RCV_PKT_DONE && packet.numpkt < budget) { in handle_receive_interrupt_napi_sp()
1124 packet.rhqoff += packet.rsize; in handle_receive_interrupt_napi_sp()
1125 packet.rhf_addr = (__le32 *)rcd->rcvhdrq + in handle_receive_interrupt_napi_sp()
1126 packet.rhqoff + in handle_receive_interrupt_napi_sp()
1128 packet.rhf = rhf_to_cpu(packet.rhf_addr); in handle_receive_interrupt_napi_sp()
1131 if (set_armed_to_active(&packet)) in handle_receive_interrupt_napi_sp()
1133 process_rcv_packet_napi(&packet); in handle_receive_interrupt_napi_sp()
1136 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_sp()
1144 process_rcv_update(last, &packet); in handle_receive_interrupt_napi_sp()
1147 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_napi_sp()
1154 finish_packet(&packet); in handle_receive_interrupt_napi_sp()
1155 return packet.numpkt; in handle_receive_interrupt_napi_sp()
1439 static inline void hfi1_setup_ib_header(struct hfi1_packet *packet) in hfi1_setup_ib_header() argument
1441 packet->hdr = (struct hfi1_ib_message_header *) in hfi1_setup_ib_header()
1442 hfi1_get_msgheader(packet->rcd, in hfi1_setup_ib_header()
1443 packet->rhf_addr); in hfi1_setup_ib_header()
1444 packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; in hfi1_setup_ib_header()
1447 static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet) in hfi1_bypass_ingress_pkt_check() argument
1449 struct hfi1_pportdata *ppd = packet->rcd->ppd; in hfi1_bypass_ingress_pkt_check()
1452 if ((!packet->slid) || (!packet->dlid)) in hfi1_bypass_ingress_pkt_check()
1456 if ((!(hfi1_is_16B_mcast(packet->dlid))) && in hfi1_bypass_ingress_pkt_check()
1457 (packet->dlid != in hfi1_bypass_ingress_pkt_check()
1459 if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid) in hfi1_bypass_ingress_pkt_check()
1464 if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF)) in hfi1_bypass_ingress_pkt_check()
1468 if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), in hfi1_bypass_ingress_pkt_check()
1470 (packet->sc != 0xF)) in hfi1_bypass_ingress_pkt_check()
1476 static int hfi1_setup_9B_packet(struct hfi1_packet *packet) in hfi1_setup_9B_packet() argument
1478 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); in hfi1_setup_9B_packet()
1482 hfi1_setup_ib_header(packet); in hfi1_setup_9B_packet()
1483 hdr = packet->hdr; in hfi1_setup_9B_packet()
1487 packet->ohdr = &hdr->u.oth; in hfi1_setup_9B_packet()
1488 packet->grh = NULL; in hfi1_setup_9B_packet()
1492 packet->ohdr = &hdr->u.l.oth; in hfi1_setup_9B_packet()
1493 packet->grh = &hdr->u.l.grh; in hfi1_setup_9B_packet()
1494 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) in hfi1_setup_9B_packet()
1496 vtf = be32_to_cpu(packet->grh->version_tclass_flow); in hfi1_setup_9B_packet()
1504 packet->payload = packet->ebuf; in hfi1_setup_9B_packet()
1505 packet->opcode = ib_bth_get_opcode(packet->ohdr); in hfi1_setup_9B_packet()
1506 packet->slid = ib_get_slid(hdr); in hfi1_setup_9B_packet()
1507 packet->dlid = ib_get_dlid(hdr); in hfi1_setup_9B_packet()
1508 if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) && in hfi1_setup_9B_packet()
1509 (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE)))) in hfi1_setup_9B_packet()
1510 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - in hfi1_setup_9B_packet()
1512 packet->sl = ib_get_sl(hdr); in hfi1_setup_9B_packet()
1513 packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf); in hfi1_setup_9B_packet()
1514 packet->pad = ib_bth_get_pad(packet->ohdr); in hfi1_setup_9B_packet()
1515 packet->extra_byte = 0; in hfi1_setup_9B_packet()
1516 packet->pkey = ib_bth_get_pkey(packet->ohdr); in hfi1_setup_9B_packet()
1517 packet->migrated = ib_bth_is_migration(packet->ohdr); in hfi1_setup_9B_packet()
1525 static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) in hfi1_setup_bypass_packet() argument
1537 struct hfi1_ctxtdata *rcd = packet->rcd; in hfi1_setup_bypass_packet()
1542 packet->hdr = (struct hfi1_16b_header *) in hfi1_setup_bypass_packet()
1543 hfi1_get_16B_header(packet->rcd, in hfi1_setup_bypass_packet()
1544 packet->rhf_addr); in hfi1_setup_bypass_packet()
1545 l4 = hfi1_16B_get_l4(packet->hdr); in hfi1_setup_bypass_packet()
1547 packet->ohdr = packet->ebuf; in hfi1_setup_bypass_packet()
1548 packet->grh = NULL; in hfi1_setup_bypass_packet()
1549 packet->opcode = ib_bth_get_opcode(packet->ohdr); in hfi1_setup_bypass_packet()
1550 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); in hfi1_setup_bypass_packet()
1552 packet->hlen = hdr_len_by_opcode[packet->opcode] + in hfi1_setup_bypass_packet()
1554 packet->migrated = opa_bth_is_migration(packet->ohdr); in hfi1_setup_bypass_packet()
1559 packet->ohdr = packet->ebuf + grh_len; in hfi1_setup_bypass_packet()
1560 packet->grh = packet->ebuf; in hfi1_setup_bypass_packet()
1561 packet->opcode = ib_bth_get_opcode(packet->ohdr); in hfi1_setup_bypass_packet()
1562 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); in hfi1_setup_bypass_packet()
1564 packet->hlen = hdr_len_by_opcode[packet->opcode] + in hfi1_setup_bypass_packet()
1566 packet->migrated = opa_bth_is_migration(packet->ohdr); in hfi1_setup_bypass_packet()
1568 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) in hfi1_setup_bypass_packet()
1570 vtf = be32_to_cpu(packet->grh->version_tclass_flow); in hfi1_setup_bypass_packet()
1574 packet->mgmt = packet->ebuf; in hfi1_setup_bypass_packet()
1575 packet->ohdr = NULL; in hfi1_setup_bypass_packet()
1576 packet->grh = NULL; in hfi1_setup_bypass_packet()
1577 packet->opcode = IB_OPCODE_UD_SEND_ONLY; in hfi1_setup_bypass_packet()
1578 packet->pad = OPA_16B_L4_FM_PAD; in hfi1_setup_bypass_packet()
1579 packet->hlen = OPA_16B_L4_FM_HLEN; in hfi1_setup_bypass_packet()
1580 packet->migrated = false; in hfi1_setup_bypass_packet()
1586 packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES; in hfi1_setup_bypass_packet()
1587 packet->slid = hfi1_16B_get_slid(packet->hdr); in hfi1_setup_bypass_packet()
1588 packet->dlid = hfi1_16B_get_dlid(packet->hdr); in hfi1_setup_bypass_packet()
1589 if (unlikely(hfi1_is_16B_mcast(packet->dlid))) in hfi1_setup_bypass_packet()
1590 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - in hfi1_setup_bypass_packet()
1593 packet->sc = hfi1_16B_get_sc(packet->hdr); in hfi1_setup_bypass_packet()
1594 packet->sl = ibp->sc_to_sl[packet->sc]; in hfi1_setup_bypass_packet()
1595 packet->extra_byte = SIZE_OF_LT; in hfi1_setup_bypass_packet()
1596 packet->pkey = hfi1_16B_get_pkey(packet->hdr); in hfi1_setup_bypass_packet()
1598 if (hfi1_bypass_ingress_pkt_check(packet)) in hfi1_setup_bypass_packet()
1608 static void show_eflags_errs(struct hfi1_packet *packet) in show_eflags_errs() argument
1610 struct hfi1_ctxtdata *rcd = packet->rcd; in show_eflags_errs()
1611 u32 rte = rhf_rcv_type_err(packet->rhf); in show_eflags_errs()
1615 rcd->ctxt, packet->rhf, in show_eflags_errs()
1616 packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", in show_eflags_errs()
1617 packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", in show_eflags_errs()
1618 packet->rhf & RHF_DC_ERR ? "dc " : "", in show_eflags_errs()
1619 packet->rhf & RHF_TID_ERR ? "tid " : "", in show_eflags_errs()
1620 packet->rhf & RHF_LEN_ERR ? "len " : "", in show_eflags_errs()
1621 packet->rhf & RHF_ECC_ERR ? "ecc " : "", in show_eflags_errs()
1622 packet->rhf & RHF_ICRC_ERR ? "icrc " : "", in show_eflags_errs()
1626 void handle_eflags(struct hfi1_packet *packet) in handle_eflags() argument
1628 struct hfi1_ctxtdata *rcd = packet->rcd; in handle_eflags()
1630 rcv_hdrerr(rcd, rcd->ppd, packet); in handle_eflags()
1631 if (rhf_err_flags(packet->rhf)) in handle_eflags()
1632 show_eflags_errs(packet); in handle_eflags()
1635 static void hfi1_ipoib_ib_rcv(struct hfi1_packet *packet) in hfi1_ipoib_ib_rcv() argument
1639 struct hfi1_ctxtdata *rcd = packet->rcd; in hfi1_ipoib_ib_rcv()
1648 trace_hfi1_rcvhdr(packet); in hfi1_ipoib_ib_rcv()
1650 hfi1_setup_ib_header(packet); in hfi1_ipoib_ib_rcv()
1652 packet->ohdr = &((struct ib_header *)packet->hdr)->u.oth; in hfi1_ipoib_ib_rcv()
1653 packet->grh = NULL; in hfi1_ipoib_ib_rcv()
1655 if (unlikely(rhf_err_flags(packet->rhf))) { in hfi1_ipoib_ib_rcv()
1656 handle_eflags(packet); in hfi1_ipoib_ib_rcv()
1660 qpnum = ib_bth_get_qpn(packet->ohdr); in hfi1_ipoib_ib_rcv()
1665 trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf))); in hfi1_ipoib_ib_rcv()
1669 do_work = hfi1_may_ecn(packet); in hfi1_ipoib_ib_rcv()
1671 do_cnp = (packet->opcode != IB_OPCODE_CNP); in hfi1_ipoib_ib_rcv()
1673 packet, do_cnp); in hfi1_ipoib_ib_rcv()
1682 tlen = packet->tlen; in hfi1_ipoib_ib_rcv()
1683 extra_bytes = ib_bth_get_pad(packet->ohdr) + (SIZE_OF_CRC << 2) + in hfi1_ipoib_ib_rcv()
1684 packet->hlen; in hfi1_ipoib_ib_rcv()
1690 skb = hfi1_ipoib_prepare_skb(rxq, tlen, packet->ebuf); in hfi1_ipoib_ib_rcv()
1705 ibp = rcd_to_iport(packet->rcd); in hfi1_ipoib_ib_rcv()
1713 static void process_receive_ib(struct hfi1_packet *packet) in process_receive_ib() argument
1715 if (hfi1_setup_9B_packet(packet)) in process_receive_ib()
1718 if (unlikely(hfi1_dbg_should_fault_rx(packet))) in process_receive_ib()
1721 trace_hfi1_rcvhdr(packet); in process_receive_ib()
1723 if (unlikely(rhf_err_flags(packet->rhf))) { in process_receive_ib()
1724 handle_eflags(packet); in process_receive_ib()
1728 hfi1_ib_rcv(packet); in process_receive_ib()
1731 static void process_receive_bypass(struct hfi1_packet *packet) in process_receive_bypass() argument
1733 struct hfi1_devdata *dd = packet->rcd->dd; in process_receive_bypass()
1735 if (hfi1_setup_bypass_packet(packet)) in process_receive_bypass()
1738 trace_hfi1_rcvhdr(packet); in process_receive_bypass()
1740 if (unlikely(rhf_err_flags(packet->rhf))) { in process_receive_bypass()
1741 handle_eflags(packet); in process_receive_bypass()
1745 if (hfi1_16B_get_l2(packet->hdr) == 0x2) { in process_receive_bypass()
1746 hfi1_16B_rcv(packet); in process_receive_bypass()
1753 u64 *flits = packet->ebuf; in process_receive_bypass()
1755 if (flits && !(packet->rhf & RHF_LEN_ERR)) { in process_receive_bypass()
1758 packet->tlen > sizeof(flits[0]) ? in process_receive_bypass()
1767 static void process_receive_error(struct hfi1_packet *packet) in process_receive_error() argument
1771 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && in process_receive_error()
1772 (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR || in process_receive_error()
1773 packet->rhf & RHF_DC_ERR))) in process_receive_error()
1776 hfi1_setup_ib_header(packet); in process_receive_error()
1777 handle_eflags(packet); in process_receive_error()
1779 if (unlikely(rhf_err_flags(packet->rhf))) in process_receive_error()
1780 dd_dev_err(packet->rcd->dd, in process_receive_error()
1784 static void kdeth_process_expected(struct hfi1_packet *packet) in kdeth_process_expected() argument
1786 hfi1_setup_9B_packet(packet); in kdeth_process_expected()
1787 if (unlikely(hfi1_dbg_should_fault_rx(packet))) in kdeth_process_expected()
1790 if (unlikely(rhf_err_flags(packet->rhf))) { in kdeth_process_expected()
1791 struct hfi1_ctxtdata *rcd = packet->rcd; in kdeth_process_expected()
1793 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) in kdeth_process_expected()
1797 hfi1_kdeth_expected_rcv(packet); in kdeth_process_expected()
1800 static void kdeth_process_eager(struct hfi1_packet *packet) in kdeth_process_eager() argument
1802 hfi1_setup_9B_packet(packet); in kdeth_process_eager()
1803 if (unlikely(hfi1_dbg_should_fault_rx(packet))) in kdeth_process_eager()
1806 trace_hfi1_rcvhdr(packet); in kdeth_process_eager()
1807 if (unlikely(rhf_err_flags(packet->rhf))) { in kdeth_process_eager()
1808 struct hfi1_ctxtdata *rcd = packet->rcd; in kdeth_process_eager()
1810 show_eflags_errs(packet); in kdeth_process_eager()
1811 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) in kdeth_process_eager()
1815 hfi1_kdeth_eager_rcv(packet); in kdeth_process_eager()
1818 static void process_receive_invalid(struct hfi1_packet *packet) in process_receive_invalid() argument
1820 dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", in process_receive_invalid()
1821 rhf_rcv_type(packet->rhf)); in process_receive_invalid()
1828 struct hfi1_packet packet; in seqfile_dump_rcd() local
1843 init_packet(rcd, &packet); in seqfile_dump_rcd()
1844 init_ps_mdata(&mdata, &packet); in seqfile_dump_rcd()
1865 packet.hdr = hfi1_get_msgheader(rcd, rhf_addr); in seqfile_dump_rcd()
1866 hdr = packet.hdr; in seqfile_dump_rcd()
1871 packet.ohdr = &hdr->u.oth; in seqfile_dump_rcd()
1873 packet.ohdr = &hdr->u.l.oth; in seqfile_dump_rcd()
1877 opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24); in seqfile_dump_rcd()
1878 qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK; in seqfile_dump_rcd()
1879 psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2])); in seqfile_dump_rcd()