1 /*
2 * Copyright(c) 2015-2018 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48 #include <linux/spinlock.h>
49 #include <linux/pci.h>
50 #include <linux/io.h>
51 #include <linux/delay.h>
52 #include <linux/netdevice.h>
53 #include <linux/vmalloc.h>
54 #include <linux/module.h>
55 #include <linux/prefetch.h>
56 #include <rdma/ib_verbs.h>
57
58 #include "hfi.h"
59 #include "trace.h"
60 #include "qp.h"
61 #include "sdma.h"
62 #include "debugfs.h"
63 #include "vnic.h"
64 #include "fault.h"
65
66 #undef pr_fmt
67 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
68
69 /*
70 * The size has to be longer than this string, so we can append
71 * board/chip information to it in the initialization code.
72 */
73 const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";
74
75 DEFINE_MUTEX(hfi1_mutex); /* general driver use */
76
77 unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
78 module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO);
79 MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify(
80 HFI1_DEFAULT_MAX_MTU));
81
82 unsigned int hfi1_cu = 1;
83 module_param_named(cu, hfi1_cu, uint, S_IRUGO);
84 MODULE_PARM_DESC(cu, "Credit return units");
85
86 unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
87 static int hfi1_caps_set(const char *val, const struct kernel_param *kp);
88 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp);
89 static const struct kernel_param_ops cap_ops = {
90 .set = hfi1_caps_set,
91 .get = hfi1_caps_get
92 };
93 module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO);
94 MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");
95
96 MODULE_LICENSE("Dual BSD/GPL");
97 MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");
98
99 /*
100 * MAX_PKT_RCV is the max # if packets processed per receive interrupt.
101 */
102 #define MAX_PKT_RECV 64
103 /*
104 * MAX_PKT_THREAD_RCV is the max # of packets processed before
105 * the qp_wait_list queue is flushed.
106 */
107 #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4)
108 #define EGR_HEAD_UPDATE_THRESHOLD 16
109
110 struct hfi1_ib_stats hfi1_stats;
111
hfi1_caps_set(const char * val,const struct kernel_param * kp)112 static int hfi1_caps_set(const char *val, const struct kernel_param *kp)
113 {
114 int ret = 0;
115 unsigned long *cap_mask_ptr = (unsigned long *)kp->arg,
116 cap_mask = *cap_mask_ptr, value, diff,
117 write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) |
118 HFI1_CAP_WRITABLE_MASK);
119
120 ret = kstrtoul(val, 0, &value);
121 if (ret) {
122 pr_warn("Invalid module parameter value for 'cap_mask'\n");
123 goto done;
124 }
125 /* Get the changed bits (except the locked bit) */
126 diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK);
127
128 /* Remove any bits that are not allowed to change after driver load */
129 if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) {
130 pr_warn("Ignoring non-writable capability bits %#lx\n",
131 diff & ~write_mask);
132 diff &= write_mask;
133 }
134
135 /* Mask off any reserved bits */
136 diff &= ~HFI1_CAP_RESERVED_MASK;
137 /* Clear any previously set and changing bits */
138 cap_mask &= ~diff;
139 /* Update the bits with the new capability */
140 cap_mask |= (value & diff);
141 /* Check for any kernel/user restrictions */
142 diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^
143 ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT);
144 cap_mask &= ~diff;
145 /* Set the bitmask to the final set */
146 *cap_mask_ptr = cap_mask;
147 done:
148 return ret;
149 }
150
hfi1_caps_get(char * buffer,const struct kernel_param * kp)151 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
152 {
153 unsigned long cap_mask = *(unsigned long *)kp->arg;
154
155 cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
156 cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
157
158 return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
159 }
160
get_pci_dev(struct rvt_dev_info * rdi)161 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
162 {
163 struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
164 struct hfi1_devdata *dd = container_of(ibdev,
165 struct hfi1_devdata, verbs_dev);
166 return dd->pcidev;
167 }
168
169 /*
170 * Return count of units with at least one port ACTIVE.
171 */
hfi1_count_active_units(void)172 int hfi1_count_active_units(void)
173 {
174 struct hfi1_devdata *dd;
175 struct hfi1_pportdata *ppd;
176 unsigned long index, flags;
177 int pidx, nunits_active = 0;
178
179 xa_lock_irqsave(&hfi1_dev_table, flags);
180 xa_for_each(&hfi1_dev_table, index, dd) {
181 if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1)
182 continue;
183 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
184 ppd = dd->pport + pidx;
185 if (ppd->lid && ppd->linkup) {
186 nunits_active++;
187 break;
188 }
189 }
190 }
191 xa_unlock_irqrestore(&hfi1_dev_table, flags);
192 return nunits_active;
193 }
194
195 /*
196 * Get address of eager buffer from it's index (allocated in chunks, not
197 * contiguous).
198 */
get_egrbuf(const struct hfi1_ctxtdata * rcd,u64 rhf,u8 * update)199 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
200 u8 *update)
201 {
202 u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf);
203
204 *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;
205 return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +
206 (offset * RCV_BUF_BLOCK_SIZE));
207 }
208
hfi1_get_header(struct hfi1_ctxtdata * rcd,__le32 * rhf_addr)209 static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd,
210 __le32 *rhf_addr)
211 {
212 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
213
214 return (void *)(rhf_addr - rcd->rhf_offset + offset);
215 }
216
hfi1_get_msgheader(struct hfi1_ctxtdata * rcd,__le32 * rhf_addr)217 static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd,
218 __le32 *rhf_addr)
219 {
220 return (struct ib_header *)hfi1_get_header(rcd, rhf_addr);
221 }
222
223 static inline struct hfi1_16b_header
hfi1_get_16B_header(struct hfi1_ctxtdata * rcd,__le32 * rhf_addr)224 *hfi1_get_16B_header(struct hfi1_ctxtdata *rcd,
225 __le32 *rhf_addr)
226 {
227 return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr);
228 }
229
230 /*
231 * Validate and encode the a given RcvArray Buffer size.
232 * The function will check whether the given size falls within
233 * allowed size ranges for the respective type and, optionally,
234 * return the proper encoding.
235 */
hfi1_rcvbuf_validate(u32 size,u8 type,u16 * encoded)236 int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
237 {
238 if (unlikely(!PAGE_ALIGNED(size)))
239 return 0;
240 if (unlikely(size < MIN_EAGER_BUFFER))
241 return 0;
242 if (size >
243 (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER))
244 return 0;
245 if (encoded)
246 *encoded = ilog2(size / PAGE_SIZE) + 1;
247 return 1;
248 }
249
rcv_hdrerr(struct hfi1_ctxtdata * rcd,struct hfi1_pportdata * ppd,struct hfi1_packet * packet)250 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
251 struct hfi1_packet *packet)
252 {
253 struct ib_header *rhdr = packet->hdr;
254 u32 rte = rhf_rcv_type_err(packet->rhf);
255 u32 mlid_base;
256 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
257 struct hfi1_devdata *dd = ppd->dd;
258 struct hfi1_ibdev *verbs_dev = &dd->verbs_dev;
259 struct rvt_dev_info *rdi = &verbs_dev->rdi;
260
261 if ((packet->rhf & RHF_DC_ERR) &&
262 hfi1_dbg_fault_suppress_err(verbs_dev))
263 return;
264
265 if (packet->rhf & RHF_ICRC_ERR)
266 return;
267
268 if (packet->etype == RHF_RCV_TYPE_BYPASS) {
269 goto drop;
270 } else {
271 u8 lnh = ib_get_lnh(rhdr);
272
273 mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE);
274 if (lnh == HFI1_LRH_BTH) {
275 packet->ohdr = &rhdr->u.oth;
276 } else if (lnh == HFI1_LRH_GRH) {
277 packet->ohdr = &rhdr->u.l.oth;
278 packet->grh = &rhdr->u.l.grh;
279 } else {
280 goto drop;
281 }
282 }
283
284 if (packet->rhf & RHF_TID_ERR) {
285 /* For TIDERR and RC QPs preemptively schedule a NAK */
286 u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
287 u32 dlid = ib_get_dlid(rhdr);
288 u32 qp_num;
289
290 /* Sanity check packet */
291 if (tlen < 24)
292 goto drop;
293
294 /* Check for GRH */
295 if (packet->grh) {
296 u32 vtf;
297 struct ib_grh *grh = packet->grh;
298
299 if (grh->next_hdr != IB_GRH_NEXT_HDR)
300 goto drop;
301 vtf = be32_to_cpu(grh->version_tclass_flow);
302 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
303 goto drop;
304 }
305
306 /* Get the destination QP number. */
307 qp_num = ib_bth_get_qpn(packet->ohdr);
308 if (dlid < mlid_base) {
309 struct rvt_qp *qp;
310 unsigned long flags;
311
312 rcu_read_lock();
313 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
314 if (!qp) {
315 rcu_read_unlock();
316 goto drop;
317 }
318
319 /*
320 * Handle only RC QPs - for other QP types drop error
321 * packet.
322 */
323 spin_lock_irqsave(&qp->r_lock, flags);
324
325 /* Check for valid receive state. */
326 if (!(ib_rvt_state_ops[qp->state] &
327 RVT_PROCESS_RECV_OK)) {
328 ibp->rvp.n_pkt_drops++;
329 }
330
331 switch (qp->ibqp.qp_type) {
332 case IB_QPT_RC:
333 hfi1_rc_hdrerr(rcd, packet, qp);
334 break;
335 default:
336 /* For now don't handle any other QP types */
337 break;
338 }
339
340 spin_unlock_irqrestore(&qp->r_lock, flags);
341 rcu_read_unlock();
342 } /* Unicast QP */
343 } /* Valid packet with TIDErr */
344
345 /* handle "RcvTypeErr" flags */
346 switch (rte) {
347 case RHF_RTE_ERROR_OP_CODE_ERR:
348 {
349 void *ebuf = NULL;
350 u8 opcode;
351
352 if (rhf_use_egr_bfr(packet->rhf))
353 ebuf = packet->ebuf;
354
355 if (!ebuf)
356 goto drop; /* this should never happen */
357
358 opcode = ib_bth_get_opcode(packet->ohdr);
359 if (opcode == IB_OPCODE_CNP) {
360 /*
361 * Only in pre-B0 h/w is the CNP_OPCODE handled
362 * via this code path.
363 */
364 struct rvt_qp *qp = NULL;
365 u32 lqpn, rqpn;
366 u16 rlid;
367 u8 svc_type, sl, sc5;
368
369 sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf);
370 sl = ibp->sc_to_sl[sc5];
371
372 lqpn = ib_bth_get_qpn(packet->ohdr);
373 rcu_read_lock();
374 qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
375 if (!qp) {
376 rcu_read_unlock();
377 goto drop;
378 }
379
380 switch (qp->ibqp.qp_type) {
381 case IB_QPT_UD:
382 rlid = 0;
383 rqpn = 0;
384 svc_type = IB_CC_SVCTYPE_UD;
385 break;
386 case IB_QPT_UC:
387 rlid = ib_get_slid(rhdr);
388 rqpn = qp->remote_qpn;
389 svc_type = IB_CC_SVCTYPE_UC;
390 break;
391 default:
392 rcu_read_unlock();
393 goto drop;
394 }
395
396 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
397 rcu_read_unlock();
398 }
399
400 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK;
401 break;
402 }
403 default:
404 break;
405 }
406
407 drop:
408 return;
409 }
410
init_packet(struct hfi1_ctxtdata * rcd,struct hfi1_packet * packet)411 static inline void init_packet(struct hfi1_ctxtdata *rcd,
412 struct hfi1_packet *packet)
413 {
414 packet->rsize = rcd->rcvhdrqentsize; /* words */
415 packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */
416 packet->rcd = rcd;
417 packet->updegr = 0;
418 packet->etail = -1;
419 packet->rhf_addr = get_rhf_addr(rcd);
420 packet->rhf = rhf_to_cpu(packet->rhf_addr);
421 packet->rhqoff = rcd->head;
422 packet->numpkt = 0;
423 }
424
425 /* We support only two types - 9B and 16B for now */
426 static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = {
427 [HFI1_PKT_TYPE_9B] = &return_cnp,
428 [HFI1_PKT_TYPE_16B] = &return_cnp_16B
429 };
430
431 /**
432 * hfi1_process_ecn_slowpath - Process FECN or BECN bits
433 * @qp: The packet's destination QP
434 * @pkt: The packet itself.
435 * @prescan: Is the caller the RXQ prescan
436 *
437 * Process the packet's FECN or BECN bits. By now, the packet
438 * has already been evaluated whether processing of those bit should
439 * be done.
440 * The significance of the @prescan argument is that if the caller
441 * is the RXQ prescan, a CNP will be send out instead of waiting for the
442 * normal packet processing to send an ACK with BECN set (or a CNP).
443 */
hfi1_process_ecn_slowpath(struct rvt_qp * qp,struct hfi1_packet * pkt,bool prescan)444 bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
445 bool prescan)
446 {
447 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
448 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
449 struct ib_other_headers *ohdr = pkt->ohdr;
450 struct ib_grh *grh = pkt->grh;
451 u32 rqpn = 0;
452 u16 pkey;
453 u32 rlid, slid, dlid = 0;
454 u8 hdr_type, sc, svc_type, opcode;
455 bool is_mcast = false, ignore_fecn = false, do_cnp = false,
456 fecn, becn;
457
458 /* can be called from prescan */
459 if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
460 pkey = hfi1_16B_get_pkey(pkt->hdr);
461 sc = hfi1_16B_get_sc(pkt->hdr);
462 dlid = hfi1_16B_get_dlid(pkt->hdr);
463 slid = hfi1_16B_get_slid(pkt->hdr);
464 is_mcast = hfi1_is_16B_mcast(dlid);
465 opcode = ib_bth_get_opcode(ohdr);
466 hdr_type = HFI1_PKT_TYPE_16B;
467 fecn = hfi1_16B_get_fecn(pkt->hdr);
468 becn = hfi1_16B_get_becn(pkt->hdr);
469 } else {
470 pkey = ib_bth_get_pkey(ohdr);
471 sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf);
472 dlid = qp->ibqp.qp_type != IB_QPT_UD ? ib_get_dlid(pkt->hdr) :
473 ppd->lid;
474 slid = ib_get_slid(pkt->hdr);
475 is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
476 (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
477 opcode = ib_bth_get_opcode(ohdr);
478 hdr_type = HFI1_PKT_TYPE_9B;
479 fecn = ib_bth_get_fecn(ohdr);
480 becn = ib_bth_get_becn(ohdr);
481 }
482
483 switch (qp->ibqp.qp_type) {
484 case IB_QPT_UD:
485 rlid = slid;
486 rqpn = ib_get_sqpn(pkt->ohdr);
487 svc_type = IB_CC_SVCTYPE_UD;
488 break;
489 case IB_QPT_SMI:
490 case IB_QPT_GSI:
491 rlid = slid;
492 rqpn = ib_get_sqpn(pkt->ohdr);
493 svc_type = IB_CC_SVCTYPE_UD;
494 break;
495 case IB_QPT_UC:
496 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
497 rqpn = qp->remote_qpn;
498 svc_type = IB_CC_SVCTYPE_UC;
499 break;
500 case IB_QPT_RC:
501 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
502 rqpn = qp->remote_qpn;
503 svc_type = IB_CC_SVCTYPE_RC;
504 break;
505 default:
506 return false;
507 }
508
509 ignore_fecn = is_mcast || (opcode == IB_OPCODE_CNP) ||
510 (opcode == IB_OPCODE_RC_ACKNOWLEDGE);
511 /*
512 * ACKNOWLEDGE packets do not get a CNP but this will be
513 * guarded by ignore_fecn above.
514 */
515 do_cnp = prescan ||
516 (opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST &&
517 opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE) ||
518 opcode == TID_OP(READ_RESP) ||
519 opcode == TID_OP(ACK);
520
521 /* Call appropriate CNP handler */
522 if (!ignore_fecn && do_cnp && fecn)
523 hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey,
524 dlid, rlid, sc, grh);
525
526 if (becn) {
527 u32 lqpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
528 u8 sl = ibp->sc_to_sl[sc];
529
530 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
531 }
532 return !ignore_fecn && fecn;
533 }
534
535 struct ps_mdata {
536 struct hfi1_ctxtdata *rcd;
537 u32 rsize;
538 u32 maxcnt;
539 u32 ps_head;
540 u32 ps_tail;
541 u32 ps_seq;
542 };
543
init_ps_mdata(struct ps_mdata * mdata,struct hfi1_packet * packet)544 static inline void init_ps_mdata(struct ps_mdata *mdata,
545 struct hfi1_packet *packet)
546 {
547 struct hfi1_ctxtdata *rcd = packet->rcd;
548
549 mdata->rcd = rcd;
550 mdata->rsize = packet->rsize;
551 mdata->maxcnt = packet->maxcnt;
552 mdata->ps_head = packet->rhqoff;
553
554 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
555 mdata->ps_tail = get_rcvhdrtail(rcd);
556 if (rcd->ctxt == HFI1_CTRL_CTXT)
557 mdata->ps_seq = rcd->seq_cnt;
558 else
559 mdata->ps_seq = 0; /* not used with DMA_RTAIL */
560 } else {
561 mdata->ps_tail = 0; /* used only with DMA_RTAIL*/
562 mdata->ps_seq = rcd->seq_cnt;
563 }
564 }
565
ps_done(struct ps_mdata * mdata,u64 rhf,struct hfi1_ctxtdata * rcd)566 static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
567 struct hfi1_ctxtdata *rcd)
568 {
569 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
570 return mdata->ps_head == mdata->ps_tail;
571 return mdata->ps_seq != rhf_rcv_seq(rhf);
572 }
573
ps_skip(struct ps_mdata * mdata,u64 rhf,struct hfi1_ctxtdata * rcd)574 static inline int ps_skip(struct ps_mdata *mdata, u64 rhf,
575 struct hfi1_ctxtdata *rcd)
576 {
577 /*
578 * Control context can potentially receive an invalid rhf.
579 * Drop such packets.
580 */
581 if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))
582 return mdata->ps_seq != rhf_rcv_seq(rhf);
583
584 return 0;
585 }
586
update_ps_mdata(struct ps_mdata * mdata,struct hfi1_ctxtdata * rcd)587 static inline void update_ps_mdata(struct ps_mdata *mdata,
588 struct hfi1_ctxtdata *rcd)
589 {
590 mdata->ps_head += mdata->rsize;
591 if (mdata->ps_head >= mdata->maxcnt)
592 mdata->ps_head = 0;
593
594 /* Control context must do seq counting */
595 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
596 (rcd->ctxt == HFI1_CTRL_CTXT)) {
597 if (++mdata->ps_seq > 13)
598 mdata->ps_seq = 1;
599 }
600 }
601
602 /*
603 * prescan_rxq - search through the receive queue looking for packets
604 * containing Excplicit Congestion Notifications (FECNs, or BECNs).
605 * When an ECN is found, process the Congestion Notification, and toggle
606 * it off.
607 * This is declared as a macro to allow quick checking of the port to avoid
608 * the overhead of a function call if not enabled.
609 */
610 #define prescan_rxq(rcd, packet) \
611 do { \
612 if (rcd->ppd->cc_prescan) \
613 __prescan_rxq(packet); \
614 } while (0)
__prescan_rxq(struct hfi1_packet * packet)615 static void __prescan_rxq(struct hfi1_packet *packet)
616 {
617 struct hfi1_ctxtdata *rcd = packet->rcd;
618 struct ps_mdata mdata;
619
620 init_ps_mdata(&mdata, packet);
621
622 while (1) {
623 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
624 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
625 packet->rcd->rhf_offset;
626 struct rvt_qp *qp;
627 struct ib_header *hdr;
628 struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi;
629 u64 rhf = rhf_to_cpu(rhf_addr);
630 u32 etype = rhf_rcv_type(rhf), qpn, bth1;
631 u8 lnh;
632
633 if (ps_done(&mdata, rhf, rcd))
634 break;
635
636 if (ps_skip(&mdata, rhf, rcd))
637 goto next;
638
639 if (etype != RHF_RCV_TYPE_IB)
640 goto next;
641
642 packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr);
643 hdr = packet->hdr;
644 lnh = ib_get_lnh(hdr);
645
646 if (lnh == HFI1_LRH_BTH) {
647 packet->ohdr = &hdr->u.oth;
648 packet->grh = NULL;
649 } else if (lnh == HFI1_LRH_GRH) {
650 packet->ohdr = &hdr->u.l.oth;
651 packet->grh = &hdr->u.l.grh;
652 } else {
653 goto next; /* just in case */
654 }
655
656 if (!hfi1_may_ecn(packet))
657 goto next;
658
659 bth1 = be32_to_cpu(packet->ohdr->bth[1]);
660 qpn = bth1 & RVT_QPN_MASK;
661 rcu_read_lock();
662 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn);
663
664 if (!qp) {
665 rcu_read_unlock();
666 goto next;
667 }
668
669 hfi1_process_ecn_slowpath(qp, packet, true);
670 rcu_read_unlock();
671
672 /* turn off BECN, FECN */
673 bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK);
674 packet->ohdr->bth[1] = cpu_to_be32(bth1);
675 next:
676 update_ps_mdata(&mdata, rcd);
677 }
678 }
679
process_rcv_qp_work(struct hfi1_packet * packet)680 static void process_rcv_qp_work(struct hfi1_packet *packet)
681 {
682 struct rvt_qp *qp, *nqp;
683 struct hfi1_ctxtdata *rcd = packet->rcd;
684
685 /*
686 * Iterate over all QPs waiting to respond.
687 * The list won't change since the IRQ is only run on one CPU.
688 */
689 list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
690 list_del_init(&qp->rspwait);
691 if (qp->r_flags & RVT_R_RSP_NAK) {
692 qp->r_flags &= ~RVT_R_RSP_NAK;
693 packet->qp = qp;
694 hfi1_send_rc_ack(packet, 0);
695 }
696 if (qp->r_flags & RVT_R_RSP_SEND) {
697 unsigned long flags;
698
699 qp->r_flags &= ~RVT_R_RSP_SEND;
700 spin_lock_irqsave(&qp->s_lock, flags);
701 if (ib_rvt_state_ops[qp->state] &
702 RVT_PROCESS_OR_FLUSH_SEND)
703 hfi1_schedule_send(qp);
704 spin_unlock_irqrestore(&qp->s_lock, flags);
705 }
706 rvt_put_qp(qp);
707 }
708 }
709
max_packet_exceeded(struct hfi1_packet * packet,int thread)710 static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread)
711 {
712 if (thread) {
713 if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0)
714 /* allow defered processing */
715 process_rcv_qp_work(packet);
716 cond_resched();
717 return RCV_PKT_OK;
718 } else {
719 this_cpu_inc(*packet->rcd->dd->rcv_limit);
720 return RCV_PKT_LIMIT;
721 }
722 }
723
check_max_packet(struct hfi1_packet * packet,int thread)724 static inline int check_max_packet(struct hfi1_packet *packet, int thread)
725 {
726 int ret = RCV_PKT_OK;
727
728 if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0))
729 ret = max_packet_exceeded(packet, thread);
730 return ret;
731 }
732
skip_rcv_packet(struct hfi1_packet * packet,int thread)733 static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
734 {
735 int ret;
736
737 /* Set up for the next packet */
738 packet->rhqoff += packet->rsize;
739 if (packet->rhqoff >= packet->maxcnt)
740 packet->rhqoff = 0;
741
742 packet->numpkt++;
743 ret = check_max_packet(packet, thread);
744
745 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
746 packet->rcd->rhf_offset;
747 packet->rhf = rhf_to_cpu(packet->rhf_addr);
748
749 return ret;
750 }
751
process_rcv_packet(struct hfi1_packet * packet,int thread)752 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
753 {
754 int ret;
755
756 packet->etype = rhf_rcv_type(packet->rhf);
757
758 /* total length */
759 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
760 /* retrieve eager buffer details */
761 packet->ebuf = NULL;
762 if (rhf_use_egr_bfr(packet->rhf)) {
763 packet->etail = rhf_egr_index(packet->rhf);
764 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
765 &packet->updegr);
766 /*
767 * Prefetch the contents of the eager buffer. It is
768 * OK to send a negative length to prefetch_range().
769 * The +2 is the size of the RHF.
770 */
771 prefetch_range(packet->ebuf,
772 packet->tlen - ((packet->rcd->rcvhdrqentsize -
773 (rhf_hdrq_offset(packet->rhf)
774 + 2)) * 4));
775 }
776
777 /*
778 * Call a type specific handler for the packet. We
779 * should be able to trust that etype won't be beyond
780 * the range of valid indexes. If so something is really
781 * wrong and we can probably just let things come
782 * crashing down. There is no need to eat another
783 * comparison in this performance critical code.
784 */
785 packet->rcd->rhf_rcv_function_map[packet->etype](packet);
786 packet->numpkt++;
787
788 /* Set up for the next packet */
789 packet->rhqoff += packet->rsize;
790 if (packet->rhqoff >= packet->maxcnt)
791 packet->rhqoff = 0;
792
793 ret = check_max_packet(packet, thread);
794
795 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
796 packet->rcd->rhf_offset;
797 packet->rhf = rhf_to_cpu(packet->rhf_addr);
798
799 return ret;
800 }
801
process_rcv_update(int last,struct hfi1_packet * packet)802 static inline void process_rcv_update(int last, struct hfi1_packet *packet)
803 {
804 /*
805 * Update head regs etc., every 16 packets, if not last pkt,
806 * to help prevent rcvhdrq overflows, when many packets
807 * are processed and queue is nearly full.
808 * Don't request an interrupt for intermediate updates.
809 */
810 if (!last && !(packet->numpkt & 0xf)) {
811 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,
812 packet->etail, 0, 0);
813 packet->updegr = 0;
814 }
815 packet->grh = NULL;
816 }
817
finish_packet(struct hfi1_packet * packet)818 static inline void finish_packet(struct hfi1_packet *packet)
819 {
820 /*
821 * Nothing we need to free for the packet.
822 *
823 * The only thing we need to do is a final update and call for an
824 * interrupt
825 */
826 update_usrhead(packet->rcd, packet->rcd->head, packet->updegr,
827 packet->etail, rcv_intr_dynamic, packet->numpkt);
828 }
829
830 /*
831 * Handle receive interrupts when using the no dma rtail option.
832 */
handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata * rcd,int thread)833 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
834 {
835 u32 seq;
836 int last = RCV_PKT_OK;
837 struct hfi1_packet packet;
838
839 init_packet(rcd, &packet);
840 seq = rhf_rcv_seq(packet.rhf);
841 if (seq != rcd->seq_cnt) {
842 last = RCV_PKT_DONE;
843 goto bail;
844 }
845
846 prescan_rxq(rcd, &packet);
847
848 while (last == RCV_PKT_OK) {
849 last = process_rcv_packet(&packet, thread);
850 seq = rhf_rcv_seq(packet.rhf);
851 if (++rcd->seq_cnt > 13)
852 rcd->seq_cnt = 1;
853 if (seq != rcd->seq_cnt)
854 last = RCV_PKT_DONE;
855 process_rcv_update(last, &packet);
856 }
857 process_rcv_qp_work(&packet);
858 rcd->head = packet.rhqoff;
859 bail:
860 finish_packet(&packet);
861 return last;
862 }
863
handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata * rcd,int thread)864 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
865 {
866 u32 hdrqtail;
867 int last = RCV_PKT_OK;
868 struct hfi1_packet packet;
869
870 init_packet(rcd, &packet);
871 hdrqtail = get_rcvhdrtail(rcd);
872 if (packet.rhqoff == hdrqtail) {
873 last = RCV_PKT_DONE;
874 goto bail;
875 }
876 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */
877
878 prescan_rxq(rcd, &packet);
879
880 while (last == RCV_PKT_OK) {
881 last = process_rcv_packet(&packet, thread);
882 if (packet.rhqoff == hdrqtail)
883 last = RCV_PKT_DONE;
884 process_rcv_update(last, &packet);
885 }
886 process_rcv_qp_work(&packet);
887 rcd->head = packet.rhqoff;
888 bail:
889 finish_packet(&packet);
890 return last;
891 }
892
set_nodma_rtail(struct hfi1_devdata * dd,u16 ctxt)893 static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt)
894 {
895 struct hfi1_ctxtdata *rcd;
896 u16 i;
897
898 /*
899 * For dynamically allocated kernel contexts (like vnic) switch
900 * interrupt handler only for that context. Otherwise, switch
901 * interrupt handler for all statically allocated kernel contexts.
902 */
903 if (ctxt >= dd->first_dyn_alloc_ctxt) {
904 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
905 if (rcd) {
906 rcd->do_interrupt =
907 &handle_receive_interrupt_nodma_rtail;
908 hfi1_rcd_put(rcd);
909 }
910 return;
911 }
912
913 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
914 rcd = hfi1_rcd_get_by_index(dd, i);
915 if (rcd)
916 rcd->do_interrupt =
917 &handle_receive_interrupt_nodma_rtail;
918 hfi1_rcd_put(rcd);
919 }
920 }
921
set_dma_rtail(struct hfi1_devdata * dd,u16 ctxt)922 static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt)
923 {
924 struct hfi1_ctxtdata *rcd;
925 u16 i;
926
927 /*
928 * For dynamically allocated kernel contexts (like vnic) switch
929 * interrupt handler only for that context. Otherwise, switch
930 * interrupt handler for all statically allocated kernel contexts.
931 */
932 if (ctxt >= dd->first_dyn_alloc_ctxt) {
933 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
934 if (rcd) {
935 rcd->do_interrupt =
936 &handle_receive_interrupt_dma_rtail;
937 hfi1_rcd_put(rcd);
938 }
939 return;
940 }
941
942 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
943 rcd = hfi1_rcd_get_by_index(dd, i);
944 if (rcd)
945 rcd->do_interrupt =
946 &handle_receive_interrupt_dma_rtail;
947 hfi1_rcd_put(rcd);
948 }
949 }
950
set_all_slowpath(struct hfi1_devdata * dd)951 void set_all_slowpath(struct hfi1_devdata *dd)
952 {
953 struct hfi1_ctxtdata *rcd;
954 u16 i;
955
956 /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */
957 for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
958 rcd = hfi1_rcd_get_by_index(dd, i);
959 if (!rcd)
960 continue;
961 if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)
962 rcd->do_interrupt = &handle_receive_interrupt;
963
964 hfi1_rcd_put(rcd);
965 }
966 }
967
set_armed_to_active(struct hfi1_ctxtdata * rcd,struct hfi1_packet * packet,struct hfi1_devdata * dd)968 static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
969 struct hfi1_packet *packet,
970 struct hfi1_devdata *dd)
971 {
972 struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
973 u8 etype = rhf_rcv_type(packet->rhf);
974 u8 sc = SC15_PACKET;
975
976 if (etype == RHF_RCV_TYPE_IB) {
977 struct ib_header *hdr = hfi1_get_msgheader(packet->rcd,
978 packet->rhf_addr);
979 sc = hfi1_9B_get_sc5(hdr, packet->rhf);
980 } else if (etype == RHF_RCV_TYPE_BYPASS) {
981 struct hfi1_16b_header *hdr = hfi1_get_16B_header(
982 packet->rcd,
983 packet->rhf_addr);
984 sc = hfi1_16B_get_sc(hdr);
985 }
986 if (sc != SC15_PACKET) {
987 int hwstate = driver_lstate(rcd->ppd);
988
989 if (hwstate != IB_PORT_ACTIVE) {
990 dd_dev_info(dd,
991 "Unexpected link state %s\n",
992 opa_lstate_name(hwstate));
993 return 0;
994 }
995
996 queue_work(rcd->ppd->link_wq, lsaw);
997 return 1;
998 }
999 return 0;
1000 }
1001
1002 /*
1003 * handle_receive_interrupt - receive a packet
1004 * @rcd: the context
1005 *
1006 * Called from interrupt handler for errors or receive interrupt.
1007 * This is the slow path interrupt handler.
1008 */
handle_receive_interrupt(struct hfi1_ctxtdata * rcd,int thread)1009 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
1010 {
1011 struct hfi1_devdata *dd = rcd->dd;
1012 u32 hdrqtail;
1013 int needset, last = RCV_PKT_OK;
1014 struct hfi1_packet packet;
1015 int skip_pkt = 0;
1016
1017 /* Control context will always use the slow path interrupt handler */
1018 needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
1019
1020 init_packet(rcd, &packet);
1021
1022 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1023 u32 seq = rhf_rcv_seq(packet.rhf);
1024
1025 if (seq != rcd->seq_cnt) {
1026 last = RCV_PKT_DONE;
1027 goto bail;
1028 }
1029 hdrqtail = 0;
1030 } else {
1031 hdrqtail = get_rcvhdrtail(rcd);
1032 if (packet.rhqoff == hdrqtail) {
1033 last = RCV_PKT_DONE;
1034 goto bail;
1035 }
1036 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */
1037
1038 /*
1039 * Control context can potentially receive an invalid
1040 * rhf. Drop such packets.
1041 */
1042 if (rcd->ctxt == HFI1_CTRL_CTXT) {
1043 u32 seq = rhf_rcv_seq(packet.rhf);
1044
1045 if (seq != rcd->seq_cnt)
1046 skip_pkt = 1;
1047 }
1048 }
1049
1050 prescan_rxq(rcd, &packet);
1051
1052 while (last == RCV_PKT_OK) {
1053 if (unlikely(dd->do_drop &&
1054 atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
1055 DROP_PACKET_ON)) {
1056 dd->do_drop = 0;
1057
1058 /* On to the next packet */
1059 packet.rhqoff += packet.rsize;
1060 packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
1061 packet.rhqoff +
1062 rcd->rhf_offset;
1063 packet.rhf = rhf_to_cpu(packet.rhf_addr);
1064
1065 } else if (skip_pkt) {
1066 last = skip_rcv_packet(&packet, thread);
1067 skip_pkt = 0;
1068 } else {
1069 /* Auto activate link on non-SC15 packet receive */
1070 if (unlikely(rcd->ppd->host_link_state ==
1071 HLS_UP_ARMED) &&
1072 set_armed_to_active(rcd, &packet, dd))
1073 goto bail;
1074 last = process_rcv_packet(&packet, thread);
1075 }
1076
1077 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1078 u32 seq = rhf_rcv_seq(packet.rhf);
1079
1080 if (++rcd->seq_cnt > 13)
1081 rcd->seq_cnt = 1;
1082 if (seq != rcd->seq_cnt)
1083 last = RCV_PKT_DONE;
1084 if (needset) {
1085 dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");
1086 set_nodma_rtail(dd, rcd->ctxt);
1087 needset = 0;
1088 }
1089 } else {
1090 if (packet.rhqoff == hdrqtail)
1091 last = RCV_PKT_DONE;
1092 /*
1093 * Control context can potentially receive an invalid
1094 * rhf. Drop such packets.
1095 */
1096 if (rcd->ctxt == HFI1_CTRL_CTXT) {
1097 u32 seq = rhf_rcv_seq(packet.rhf);
1098
1099 if (++rcd->seq_cnt > 13)
1100 rcd->seq_cnt = 1;
1101 if (!last && (seq != rcd->seq_cnt))
1102 skip_pkt = 1;
1103 }
1104
1105 if (needset) {
1106 dd_dev_info(dd,
1107 "Switching to DMA_RTAIL\n");
1108 set_dma_rtail(dd, rcd->ctxt);
1109 needset = 0;
1110 }
1111 }
1112
1113 process_rcv_update(last, &packet);
1114 }
1115
1116 process_rcv_qp_work(&packet);
1117 rcd->head = packet.rhqoff;
1118
1119 bail:
1120 /*
1121 * Always write head at end, and setup rcv interrupt, even
1122 * if no packets were processed.
1123 */
1124 finish_packet(&packet);
1125 return last;
1126 }
1127
1128 /*
1129 * We may discover in the interrupt that the hardware link state has
1130 * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),
1131 * and we need to update the driver's notion of the link state. We cannot
1132 * run set_link_state from interrupt context, so we queue this function on
1133 * a workqueue.
1134 *
1135 * We delay the regular interrupt processing until after the state changes
1136 * so that the link will be in the correct state by the time any application
1137 * we wake up attempts to send a reply to any message it received.
1138 * (Subsequent receive interrupts may possibly force the wakeup before we
1139 * update the link state.)
1140 *
1141 * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes
1142 * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues,
1143 * so we're safe from use-after-free of the rcd.
1144 */
receive_interrupt_work(struct work_struct * work)1145 void receive_interrupt_work(struct work_struct *work)
1146 {
1147 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
1148 linkstate_active_work);
1149 struct hfi1_devdata *dd = ppd->dd;
1150 struct hfi1_ctxtdata *rcd;
1151 u16 i;
1152
1153 /* Received non-SC15 packet implies neighbor_normal */
1154 ppd->neighbor_normal = 1;
1155 set_link_state(ppd, HLS_UP_ACTIVE);
1156
1157 /*
1158 * Interrupt all statically allocated kernel contexts that could
1159 * have had an interrupt during auto activation.
1160 */
1161 for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) {
1162 rcd = hfi1_rcd_get_by_index(dd, i);
1163 if (rcd)
1164 force_recv_intr(rcd);
1165 hfi1_rcd_put(rcd);
1166 }
1167 }
1168
1169 /*
1170 * Convert a given MTU size to the on-wire MAD packet enumeration.
1171 * Return -1 if the size is invalid.
1172 */
mtu_to_enum(u32 mtu,int default_if_bad)1173 int mtu_to_enum(u32 mtu, int default_if_bad)
1174 {
1175 switch (mtu) {
1176 case 0: return OPA_MTU_0;
1177 case 256: return OPA_MTU_256;
1178 case 512: return OPA_MTU_512;
1179 case 1024: return OPA_MTU_1024;
1180 case 2048: return OPA_MTU_2048;
1181 case 4096: return OPA_MTU_4096;
1182 case 8192: return OPA_MTU_8192;
1183 case 10240: return OPA_MTU_10240;
1184 }
1185 return default_if_bad;
1186 }
1187
enum_to_mtu(int mtu)1188 u16 enum_to_mtu(int mtu)
1189 {
1190 switch (mtu) {
1191 case OPA_MTU_0: return 0;
1192 case OPA_MTU_256: return 256;
1193 case OPA_MTU_512: return 512;
1194 case OPA_MTU_1024: return 1024;
1195 case OPA_MTU_2048: return 2048;
1196 case OPA_MTU_4096: return 4096;
1197 case OPA_MTU_8192: return 8192;
1198 case OPA_MTU_10240: return 10240;
1199 default: return 0xffff;
1200 }
1201 }
1202
1203 /*
1204 * set_mtu - set the MTU
1205 * @ppd: the per port data
1206 *
1207 * We can handle "any" incoming size, the issue here is whether we
1208 * need to restrict our outgoing size. We do not deal with what happens
1209 * to programs that are already running when the size changes.
1210 */
set_mtu(struct hfi1_pportdata * ppd)1211 int set_mtu(struct hfi1_pportdata *ppd)
1212 {
1213 struct hfi1_devdata *dd = ppd->dd;
1214 int i, drain, ret = 0, is_up = 0;
1215
1216 ppd->ibmtu = 0;
1217 for (i = 0; i < ppd->vls_supported; i++)
1218 if (ppd->ibmtu < dd->vld[i].mtu)
1219 ppd->ibmtu = dd->vld[i].mtu;
1220 ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd);
1221
1222 mutex_lock(&ppd->hls_lock);
1223 if (ppd->host_link_state == HLS_UP_INIT ||
1224 ppd->host_link_state == HLS_UP_ARMED ||
1225 ppd->host_link_state == HLS_UP_ACTIVE)
1226 is_up = 1;
1227
1228 drain = !is_ax(dd) && is_up;
1229
1230 if (drain)
1231 /*
1232 * MTU is specified per-VL. To ensure that no packet gets
1233 * stuck (due, e.g., to the MTU for the packet's VL being
1234 * reduced), empty the per-VL FIFOs before adjusting MTU.
1235 */
1236 ret = stop_drain_data_vls(dd);
1237
1238 if (ret) {
1239 dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n",
1240 __func__);
1241 goto err;
1242 }
1243
1244 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0);
1245
1246 if (drain)
1247 open_fill_data_vls(dd); /* reopen all VLs */
1248
1249 err:
1250 mutex_unlock(&ppd->hls_lock);
1251
1252 return ret;
1253 }
1254
hfi1_set_lid(struct hfi1_pportdata * ppd,u32 lid,u8 lmc)1255 int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc)
1256 {
1257 struct hfi1_devdata *dd = ppd->dd;
1258
1259 ppd->lid = lid;
1260 ppd->lmc = lmc;
1261 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0);
1262
1263 dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid);
1264
1265 return 0;
1266 }
1267
shutdown_led_override(struct hfi1_pportdata * ppd)1268 void shutdown_led_override(struct hfi1_pportdata *ppd)
1269 {
1270 struct hfi1_devdata *dd = ppd->dd;
1271
1272 /*
1273 * This pairs with the memory barrier in hfi1_start_led_override to
1274 * ensure that we read the correct state of LED beaconing represented
1275 * by led_override_timer_active
1276 */
1277 smp_rmb();
1278 if (atomic_read(&ppd->led_override_timer_active)) {
1279 del_timer_sync(&ppd->led_override_timer);
1280 atomic_set(&ppd->led_override_timer_active, 0);
1281 /* Ensure the atomic_set is visible to all CPUs */
1282 smp_wmb();
1283 }
1284
1285 /* Hand control of the LED to the DC for normal operation */
1286 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
1287 }
1288
run_led_override(struct timer_list * t)1289 static void run_led_override(struct timer_list *t)
1290 {
1291 struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer);
1292 struct hfi1_devdata *dd = ppd->dd;
1293 unsigned long timeout;
1294 int phase_idx;
1295
1296 if (!(dd->flags & HFI1_INITTED))
1297 return;
1298
1299 phase_idx = ppd->led_override_phase & 1;
1300
1301 setextled(dd, phase_idx);
1302
1303 timeout = ppd->led_override_vals[phase_idx];
1304
1305 /* Set up for next phase */
1306 ppd->led_override_phase = !ppd->led_override_phase;
1307
1308 mod_timer(&ppd->led_override_timer, jiffies + timeout);
1309 }
1310
1311 /*
1312 * To have the LED blink in a particular pattern, provide timeon and timeoff
1313 * in milliseconds.
1314 * To turn off custom blinking and return to normal operation, use
1315 * shutdown_led_override()
1316 */
hfi1_start_led_override(struct hfi1_pportdata * ppd,unsigned int timeon,unsigned int timeoff)1317 void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
1318 unsigned int timeoff)
1319 {
1320 if (!(ppd->dd->flags & HFI1_INITTED))
1321 return;
1322
1323 /* Convert to jiffies for direct use in timer */
1324 ppd->led_override_vals[0] = msecs_to_jiffies(timeoff);
1325 ppd->led_override_vals[1] = msecs_to_jiffies(timeon);
1326
1327 /* Arbitrarily start from LED on phase */
1328 ppd->led_override_phase = 1;
1329
1330 /*
1331 * If the timer has not already been started, do so. Use a "quick"
1332 * timeout so the handler will be called soon to look at our request.
1333 */
1334 if (!timer_pending(&ppd->led_override_timer)) {
1335 timer_setup(&ppd->led_override_timer, run_led_override, 0);
1336 ppd->led_override_timer.expires = jiffies + 1;
1337 add_timer(&ppd->led_override_timer);
1338 atomic_set(&ppd->led_override_timer_active, 1);
1339 /* Ensure the atomic_set is visible to all CPUs */
1340 smp_wmb();
1341 }
1342 }
1343
1344 /**
1345 * hfi1_reset_device - reset the chip if possible
1346 * @unit: the device to reset
1347 *
1348 * Whether or not reset is successful, we attempt to re-initialize the chip
1349 * (that is, much like a driver unload/reload). We clear the INITTED flag
1350 * so that the various entry points will fail until we reinitialize. For
1351 * now, we only allow this if no user contexts are open that use chip resources
1352 */
hfi1_reset_device(int unit)1353 int hfi1_reset_device(int unit)
1354 {
1355 int ret;
1356 struct hfi1_devdata *dd = hfi1_lookup(unit);
1357 struct hfi1_pportdata *ppd;
1358 int pidx;
1359
1360 if (!dd) {
1361 ret = -ENODEV;
1362 goto bail;
1363 }
1364
1365 dd_dev_info(dd, "Reset on unit %u requested\n", unit);
1366
1367 if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) {
1368 dd_dev_info(dd,
1369 "Invalid unit number %u or not initialized or not present\n",
1370 unit);
1371 ret = -ENXIO;
1372 goto bail;
1373 }
1374
1375 /* If there are any user/vnic contexts, we cannot reset */
1376 mutex_lock(&hfi1_mutex);
1377 if (dd->rcd)
1378 if (hfi1_stats.sps_ctxts) {
1379 mutex_unlock(&hfi1_mutex);
1380 ret = -EBUSY;
1381 goto bail;
1382 }
1383 mutex_unlock(&hfi1_mutex);
1384
1385 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1386 ppd = dd->pport + pidx;
1387
1388 shutdown_led_override(ppd);
1389 }
1390 if (dd->flags & HFI1_HAS_SEND_DMA)
1391 sdma_exit(dd);
1392
1393 hfi1_reset_cpu_counters(dd);
1394
1395 ret = hfi1_init(dd, 1);
1396
1397 if (ret)
1398 dd_dev_err(dd,
1399 "Reinitialize unit %u after reset failed with %d\n",
1400 unit, ret);
1401 else
1402 dd_dev_info(dd, "Reinitialized unit %u after resetting\n",
1403 unit);
1404
1405 bail:
1406 return ret;
1407 }
1408
hfi1_setup_ib_header(struct hfi1_packet * packet)1409 static inline void hfi1_setup_ib_header(struct hfi1_packet *packet)
1410 {
1411 packet->hdr = (struct hfi1_ib_message_header *)
1412 hfi1_get_msgheader(packet->rcd,
1413 packet->rhf_addr);
1414 packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
1415 }
1416
hfi1_bypass_ingress_pkt_check(struct hfi1_packet * packet)1417 static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet)
1418 {
1419 struct hfi1_pportdata *ppd = packet->rcd->ppd;
1420
1421 /* slid and dlid cannot be 0 */
1422 if ((!packet->slid) || (!packet->dlid))
1423 return -EINVAL;
1424
1425 /* Compare port lid with incoming packet dlid */
1426 if ((!(hfi1_is_16B_mcast(packet->dlid))) &&
1427 (packet->dlid !=
1428 opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) {
1429 if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid)
1430 return -EINVAL;
1431 }
1432
1433 /* No multicast packets with SC15 */
1434 if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF))
1435 return -EINVAL;
1436
1437 /* Packets with permissive DLID always on SC15 */
1438 if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE),
1439 16B)) &&
1440 (packet->sc != 0xF))
1441 return -EINVAL;
1442
1443 return 0;
1444 }
1445
hfi1_setup_9B_packet(struct hfi1_packet * packet)1446 static int hfi1_setup_9B_packet(struct hfi1_packet *packet)
1447 {
1448 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
1449 struct ib_header *hdr;
1450 u8 lnh;
1451
1452 hfi1_setup_ib_header(packet);
1453 hdr = packet->hdr;
1454
1455 lnh = ib_get_lnh(hdr);
1456 if (lnh == HFI1_LRH_BTH) {
1457 packet->ohdr = &hdr->u.oth;
1458 packet->grh = NULL;
1459 } else if (lnh == HFI1_LRH_GRH) {
1460 u32 vtf;
1461
1462 packet->ohdr = &hdr->u.l.oth;
1463 packet->grh = &hdr->u.l.grh;
1464 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
1465 goto drop;
1466 vtf = be32_to_cpu(packet->grh->version_tclass_flow);
1467 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
1468 goto drop;
1469 } else {
1470 goto drop;
1471 }
1472
1473 /* Query commonly used fields from packet header */
1474 packet->payload = packet->ebuf;
1475 packet->opcode = ib_bth_get_opcode(packet->ohdr);
1476 packet->slid = ib_get_slid(hdr);
1477 packet->dlid = ib_get_dlid(hdr);
1478 if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
1479 (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE))))
1480 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
1481 be16_to_cpu(IB_MULTICAST_LID_BASE);
1482 packet->sl = ib_get_sl(hdr);
1483 packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf);
1484 packet->pad = ib_bth_get_pad(packet->ohdr);
1485 packet->extra_byte = 0;
1486 packet->pkey = ib_bth_get_pkey(packet->ohdr);
1487 packet->migrated = ib_bth_is_migration(packet->ohdr);
1488
1489 return 0;
1490 drop:
1491 ibp->rvp.n_pkt_drops++;
1492 return -EINVAL;
1493 }
1494
hfi1_setup_bypass_packet(struct hfi1_packet * packet)1495 static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
1496 {
1497 /*
1498 * Bypass packets have a different header/payload split
1499 * compared to an IB packet.
1500 * Current split is set such that 16 bytes of the actual
1501 * header is in the header buffer and the remining is in
1502 * the eager buffer. We chose 16 since hfi1 driver only
1503 * supports 16B bypass packets and we will be able to
1504 * receive the entire LRH with such a split.
1505 */
1506
1507 struct hfi1_ctxtdata *rcd = packet->rcd;
1508 struct hfi1_pportdata *ppd = rcd->ppd;
1509 struct hfi1_ibport *ibp = &ppd->ibport_data;
1510 u8 l4;
1511
1512 packet->hdr = (struct hfi1_16b_header *)
1513 hfi1_get_16B_header(packet->rcd,
1514 packet->rhf_addr);
1515 l4 = hfi1_16B_get_l4(packet->hdr);
1516 if (l4 == OPA_16B_L4_IB_LOCAL) {
1517 packet->ohdr = packet->ebuf;
1518 packet->grh = NULL;
1519 packet->opcode = ib_bth_get_opcode(packet->ohdr);
1520 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
1521 /* hdr_len_by_opcode already has an IB LRH factored in */
1522 packet->hlen = hdr_len_by_opcode[packet->opcode] +
1523 (LRH_16B_BYTES - LRH_9B_BYTES);
1524 packet->migrated = opa_bth_is_migration(packet->ohdr);
1525 } else if (l4 == OPA_16B_L4_IB_GLOBAL) {
1526 u32 vtf;
1527 u8 grh_len = sizeof(struct ib_grh);
1528
1529 packet->ohdr = packet->ebuf + grh_len;
1530 packet->grh = packet->ebuf;
1531 packet->opcode = ib_bth_get_opcode(packet->ohdr);
1532 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
1533 /* hdr_len_by_opcode already has an IB LRH factored in */
1534 packet->hlen = hdr_len_by_opcode[packet->opcode] +
1535 (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len;
1536 packet->migrated = opa_bth_is_migration(packet->ohdr);
1537
1538 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
1539 goto drop;
1540 vtf = be32_to_cpu(packet->grh->version_tclass_flow);
1541 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
1542 goto drop;
1543 } else if (l4 == OPA_16B_L4_FM) {
1544 packet->mgmt = packet->ebuf;
1545 packet->ohdr = NULL;
1546 packet->grh = NULL;
1547 packet->opcode = IB_OPCODE_UD_SEND_ONLY;
1548 packet->pad = OPA_16B_L4_FM_PAD;
1549 packet->hlen = OPA_16B_L4_FM_HLEN;
1550 packet->migrated = false;
1551 } else {
1552 goto drop;
1553 }
1554
1555 /* Query commonly used fields from packet header */
1556 packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES;
1557 packet->slid = hfi1_16B_get_slid(packet->hdr);
1558 packet->dlid = hfi1_16B_get_dlid(packet->hdr);
1559 if (unlikely(hfi1_is_16B_mcast(packet->dlid)))
1560 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
1561 opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR),
1562 16B);
1563 packet->sc = hfi1_16B_get_sc(packet->hdr);
1564 packet->sl = ibp->sc_to_sl[packet->sc];
1565 packet->extra_byte = SIZE_OF_LT;
1566 packet->pkey = hfi1_16B_get_pkey(packet->hdr);
1567
1568 if (hfi1_bypass_ingress_pkt_check(packet))
1569 goto drop;
1570
1571 return 0;
1572 drop:
1573 hfi1_cdbg(PKT, "%s: packet dropped\n", __func__);
1574 ibp->rvp.n_pkt_drops++;
1575 return -EINVAL;
1576 }
1577
show_eflags_errs(struct hfi1_packet * packet)1578 static void show_eflags_errs(struct hfi1_packet *packet)
1579 {
1580 struct hfi1_ctxtdata *rcd = packet->rcd;
1581 u32 rte = rhf_rcv_type_err(packet->rhf);
1582
1583 dd_dev_err(rcd->dd,
1584 "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s] rte 0x%x\n",
1585 rcd->ctxt, packet->rhf,
1586 packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
1587 packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
1588 packet->rhf & RHF_DC_ERR ? "dc " : "",
1589 packet->rhf & RHF_TID_ERR ? "tid " : "",
1590 packet->rhf & RHF_LEN_ERR ? "len " : "",
1591 packet->rhf & RHF_ECC_ERR ? "ecc " : "",
1592 packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
1593 rte);
1594 }
1595
handle_eflags(struct hfi1_packet * packet)1596 void handle_eflags(struct hfi1_packet *packet)
1597 {
1598 struct hfi1_ctxtdata *rcd = packet->rcd;
1599
1600 rcv_hdrerr(rcd, rcd->ppd, packet);
1601 if (rhf_err_flags(packet->rhf))
1602 show_eflags_errs(packet);
1603 }
1604
1605 /*
1606 * The following functions are called by the interrupt handler. They are type
1607 * specific handlers for each packet type.
1608 */
process_receive_ib(struct hfi1_packet * packet)1609 static int process_receive_ib(struct hfi1_packet *packet)
1610 {
1611 if (hfi1_setup_9B_packet(packet))
1612 return RHF_RCV_CONTINUE;
1613
1614 if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1615 return RHF_RCV_CONTINUE;
1616
1617 trace_hfi1_rcvhdr(packet);
1618
1619 if (unlikely(rhf_err_flags(packet->rhf))) {
1620 handle_eflags(packet);
1621 return RHF_RCV_CONTINUE;
1622 }
1623
1624 hfi1_ib_rcv(packet);
1625 return RHF_RCV_CONTINUE;
1626 }
1627
hfi1_is_vnic_packet(struct hfi1_packet * packet)1628 static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet)
1629 {
1630 /* Packet received in VNIC context via RSM */
1631 if (packet->rcd->is_vnic)
1632 return true;
1633
1634 if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) &&
1635 (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR))
1636 return true;
1637
1638 return false;
1639 }
1640
process_receive_bypass(struct hfi1_packet * packet)1641 static int process_receive_bypass(struct hfi1_packet *packet)
1642 {
1643 struct hfi1_devdata *dd = packet->rcd->dd;
1644
1645 if (hfi1_is_vnic_packet(packet)) {
1646 hfi1_vnic_bypass_rcv(packet);
1647 return RHF_RCV_CONTINUE;
1648 }
1649
1650 if (hfi1_setup_bypass_packet(packet))
1651 return RHF_RCV_CONTINUE;
1652
1653 trace_hfi1_rcvhdr(packet);
1654
1655 if (unlikely(rhf_err_flags(packet->rhf))) {
1656 handle_eflags(packet);
1657 return RHF_RCV_CONTINUE;
1658 }
1659
1660 if (hfi1_16B_get_l2(packet->hdr) == 0x2) {
1661 hfi1_16B_rcv(packet);
1662 } else {
1663 dd_dev_err(dd,
1664 "Bypass packets other than 16B are not supported in normal operation. Dropping\n");
1665 incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
1666 if (!(dd->err_info_rcvport.status_and_code &
1667 OPA_EI_STATUS_SMASK)) {
1668 u64 *flits = packet->ebuf;
1669
1670 if (flits && !(packet->rhf & RHF_LEN_ERR)) {
1671 dd->err_info_rcvport.packet_flit1 = flits[0];
1672 dd->err_info_rcvport.packet_flit2 =
1673 packet->tlen > sizeof(flits[0]) ?
1674 flits[1] : 0;
1675 }
1676 dd->err_info_rcvport.status_and_code |=
1677 (OPA_EI_STATUS_SMASK | BAD_L2_ERR);
1678 }
1679 }
1680 return RHF_RCV_CONTINUE;
1681 }
1682
process_receive_error(struct hfi1_packet * packet)1683 static int process_receive_error(struct hfi1_packet *packet)
1684 {
1685 /* KHdrHCRCErr -- KDETH packet with a bad HCRC */
1686 if (unlikely(
1687 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
1688 (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR ||
1689 packet->rhf & RHF_DC_ERR)))
1690 return RHF_RCV_CONTINUE;
1691
1692 hfi1_setup_ib_header(packet);
1693 handle_eflags(packet);
1694
1695 if (unlikely(rhf_err_flags(packet->rhf)))
1696 dd_dev_err(packet->rcd->dd,
1697 "Unhandled error packet received. Dropping.\n");
1698
1699 return RHF_RCV_CONTINUE;
1700 }
1701
kdeth_process_expected(struct hfi1_packet * packet)1702 static int kdeth_process_expected(struct hfi1_packet *packet)
1703 {
1704 hfi1_setup_9B_packet(packet);
1705 if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1706 return RHF_RCV_CONTINUE;
1707
1708 if (unlikely(rhf_err_flags(packet->rhf))) {
1709 struct hfi1_ctxtdata *rcd = packet->rcd;
1710
1711 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
1712 return RHF_RCV_CONTINUE;
1713 }
1714
1715 hfi1_kdeth_expected_rcv(packet);
1716 return RHF_RCV_CONTINUE;
1717 }
1718
kdeth_process_eager(struct hfi1_packet * packet)1719 static int kdeth_process_eager(struct hfi1_packet *packet)
1720 {
1721 hfi1_setup_9B_packet(packet);
1722 if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1723 return RHF_RCV_CONTINUE;
1724
1725 trace_hfi1_rcvhdr(packet);
1726 if (unlikely(rhf_err_flags(packet->rhf))) {
1727 struct hfi1_ctxtdata *rcd = packet->rcd;
1728
1729 show_eflags_errs(packet);
1730 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
1731 return RHF_RCV_CONTINUE;
1732 }
1733
1734 hfi1_kdeth_eager_rcv(packet);
1735 return RHF_RCV_CONTINUE;
1736 }
1737
process_receive_invalid(struct hfi1_packet * packet)1738 static int process_receive_invalid(struct hfi1_packet *packet)
1739 {
1740 dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1741 rhf_rcv_type(packet->rhf));
1742 return RHF_RCV_CONTINUE;
1743 }
1744
seqfile_dump_rcd(struct seq_file * s,struct hfi1_ctxtdata * rcd)1745 void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
1746 {
1747 struct hfi1_packet packet;
1748 struct ps_mdata mdata;
1749
1750 seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n",
1751 rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize,
1752 HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
1753 "dma_rtail" : "nodma_rtail",
1754 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &
1755 RCV_HDR_HEAD_HEAD_MASK,
1756 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL));
1757
1758 init_packet(rcd, &packet);
1759 init_ps_mdata(&mdata, &packet);
1760
1761 while (1) {
1762 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
1763 rcd->rhf_offset;
1764 struct ib_header *hdr;
1765 u64 rhf = rhf_to_cpu(rhf_addr);
1766 u32 etype = rhf_rcv_type(rhf), qpn;
1767 u8 opcode;
1768 u32 psn;
1769 u8 lnh;
1770
1771 if (ps_done(&mdata, rhf, rcd))
1772 break;
1773
1774 if (ps_skip(&mdata, rhf, rcd))
1775 goto next;
1776
1777 if (etype > RHF_RCV_TYPE_IB)
1778 goto next;
1779
1780 packet.hdr = hfi1_get_msgheader(rcd, rhf_addr);
1781 hdr = packet.hdr;
1782
1783 lnh = be16_to_cpu(hdr->lrh[0]) & 3;
1784
1785 if (lnh == HFI1_LRH_BTH)
1786 packet.ohdr = &hdr->u.oth;
1787 else if (lnh == HFI1_LRH_GRH)
1788 packet.ohdr = &hdr->u.l.oth;
1789 else
1790 goto next; /* just in case */
1791
1792 opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24);
1793 qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK;
1794 psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2]));
1795
1796 seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n",
1797 mdata.ps_head, opcode, qpn, psn);
1798 next:
1799 update_ps_mdata(&mdata, rcd);
1800 }
1801 }
1802
1803 const rhf_rcv_function_ptr normal_rhf_rcv_functions[] = {
1804 [RHF_RCV_TYPE_EXPECTED] = kdeth_process_expected,
1805 [RHF_RCV_TYPE_EAGER] = kdeth_process_eager,
1806 [RHF_RCV_TYPE_IB] = process_receive_ib,
1807 [RHF_RCV_TYPE_ERROR] = process_receive_error,
1808 [RHF_RCV_TYPE_BYPASS] = process_receive_bypass,
1809 [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
1810 [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
1811 [RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
1812 };
1813
