1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2 /*
3 * Copyright(c) 2015 - 2020 Intel Corporation.
4 */
5
6 /*
7 * This file contains all of the code that is specific to the HFI chip
8 */
9
10 #include <linux/pci.h>
11 #include <linux/delay.h>
12 #include <linux/interrupt.h>
13 #include <linux/module.h>
14
15 #include "hfi.h"
16 #include "trace.h"
17 #include "mad.h"
18 #include "pio.h"
19 #include "sdma.h"
20 #include "eprom.h"
21 #include "efivar.h"
22 #include "platform.h"
23 #include "aspm.h"
24 #include "affinity.h"
25 #include "debugfs.h"
26 #include "fault.h"
27 #include "netdev.h"
28
29 uint num_vls = HFI1_MAX_VLS_SUPPORTED;
30 module_param(num_vls, uint, S_IRUGO);
31 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
32
33 /*
34 * Default time to aggregate two 10K packets from the idle state
35 * (timer not running). The timer starts at the end of the first packet,
36 * so only the time for one 10K packet and header plus a bit extra is needed.
37 * 10 * 1024 + 64 header byte = 10304 byte
38 * 10304 byte / 12.5 GB/s = 824.32ns
39 */
40 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
41 module_param(rcv_intr_timeout, uint, S_IRUGO);
42 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
43
44 uint rcv_intr_count = 16; /* same as qib */
45 module_param(rcv_intr_count, uint, S_IRUGO);
46 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
47
48 ushort link_crc_mask = SUPPORTED_CRCS;
49 module_param(link_crc_mask, ushort, S_IRUGO);
50 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
51
52 uint loopback;
53 module_param_named(loopback, loopback, uint, S_IRUGO);
54 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
55
56 /* Other driver tunables */
57 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
58 static ushort crc_14b_sideband = 1;
59 static uint use_flr = 1;
60 uint quick_linkup; /* skip LNI */
61
62 struct flag_table {
63 u64 flag; /* the flag */
64 char *str; /* description string */
65 u16 extra; /* extra information */
66 u16 unused0;
67 u32 unused1;
68 };
69
70 /* str must be a string constant */
71 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
72 #define FLAG_ENTRY0(str, flag) {flag, str, 0}
73
74 /* Send Error Consequences */
75 #define SEC_WRITE_DROPPED 0x1
76 #define SEC_PACKET_DROPPED 0x2
77 #define SEC_SC_HALTED 0x4 /* per-context only */
78 #define SEC_SPC_FREEZE 0x8 /* per-HFI only */
79
80 #define DEFAULT_KRCVQS 2
81 #define MIN_KERNEL_KCTXTS 2
82 #define FIRST_KERNEL_KCTXT 1
83
84 /*
85 * RSM instance allocation
86 * 0 - User Fecn Handling
87 * 1 - Vnic
88 * 2 - AIP
89 * 3 - Verbs
90 */
91 #define RSM_INS_FECN 0
92 #define RSM_INS_VNIC 1
93 #define RSM_INS_AIP 2
94 #define RSM_INS_VERBS 3
95
96 /* Bit offset into the GUID which carries HFI id information */
97 #define GUID_HFI_INDEX_SHIFT 39
98
99 /* extract the emulation revision */
100 #define emulator_rev(dd) ((dd)->irev >> 8)
101 /* parallel and serial emulation versions are 3 and 4 respectively */
102 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
103 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
104
105 /* RSM fields for Verbs */
106 /* packet type */
107 #define IB_PACKET_TYPE 2ull
108 #define QW_SHIFT 6ull
109 /* QPN[7..1] */
110 #define QPN_WIDTH 7ull
111
112 /* LRH.BTH: QW 0, OFFSET 48 - for match */
113 #define LRH_BTH_QW 0ull
114 #define LRH_BTH_BIT_OFFSET 48ull
115 #define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off))
116 #define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
117 #define LRH_BTH_SELECT
118 #define LRH_BTH_MASK 3ull
119 #define LRH_BTH_VALUE 2ull
120
121 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
122 #define LRH_SC_QW 0ull
123 #define LRH_SC_BIT_OFFSET 56ull
124 #define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off))
125 #define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
126 #define LRH_SC_MASK 128ull
127 #define LRH_SC_VALUE 0ull
128
129 /* SC[n..0] QW 0, OFFSET 60 - for select */
130 #define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull))
131
132 /* QPN[m+n:1] QW 1, OFFSET 1 */
133 #define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull))
134
135 /* RSM fields for AIP */
136 /* LRH.BTH above is reused for this rule */
137
138 /* BTH.DESTQP: QW 1, OFFSET 16 for match */
139 #define BTH_DESTQP_QW 1ull
140 #define BTH_DESTQP_BIT_OFFSET 16ull
141 #define BTH_DESTQP_OFFSET(off) ((BTH_DESTQP_QW << QW_SHIFT) | (off))
142 #define BTH_DESTQP_MATCH_OFFSET BTH_DESTQP_OFFSET(BTH_DESTQP_BIT_OFFSET)
143 #define BTH_DESTQP_MASK 0xFFull
144 #define BTH_DESTQP_VALUE 0x81ull
145
146 /* DETH.SQPN: QW 1 Offset 56 for select */
147 /* We use 8 most significant Soure QPN bits as entropy fpr AIP */
148 #define DETH_AIP_SQPN_QW 3ull
149 #define DETH_AIP_SQPN_BIT_OFFSET 56ull
150 #define DETH_AIP_SQPN_OFFSET(off) ((DETH_AIP_SQPN_QW << QW_SHIFT) | (off))
151 #define DETH_AIP_SQPN_SELECT_OFFSET \
152 DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET)
153
154 /* RSM fields for Vnic */
155 /* L2_TYPE: QW 0, OFFSET 61 - for match */
156 #define L2_TYPE_QW 0ull
157 #define L2_TYPE_BIT_OFFSET 61ull
158 #define L2_TYPE_OFFSET(off) ((L2_TYPE_QW << QW_SHIFT) | (off))
159 #define L2_TYPE_MATCH_OFFSET L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET)
160 #define L2_TYPE_MASK 3ull
161 #define L2_16B_VALUE 2ull
162
163 /* L4_TYPE QW 1, OFFSET 0 - for match */
164 #define L4_TYPE_QW 1ull
165 #define L4_TYPE_BIT_OFFSET 0ull
166 #define L4_TYPE_OFFSET(off) ((L4_TYPE_QW << QW_SHIFT) | (off))
167 #define L4_TYPE_MATCH_OFFSET L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET)
168 #define L4_16B_TYPE_MASK 0xFFull
169 #define L4_16B_ETH_VALUE 0x78ull
170
171 /* 16B VESWID - for select */
172 #define L4_16B_HDR_VESWID_OFFSET ((2 << QW_SHIFT) | (16ull))
173 /* 16B ENTROPY - for select */
174 #define L2_16B_ENTROPY_OFFSET ((1 << QW_SHIFT) | (32ull))
175
176 /* defines to build power on SC2VL table */
177 #define SC2VL_VAL( \
178 num, \
179 sc0, sc0val, \
180 sc1, sc1val, \
181 sc2, sc2val, \
182 sc3, sc3val, \
183 sc4, sc4val, \
184 sc5, sc5val, \
185 sc6, sc6val, \
186 sc7, sc7val) \
187 ( \
188 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
189 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
190 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
191 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
192 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
193 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
194 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
195 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \
196 )
197
198 #define DC_SC_VL_VAL( \
199 range, \
200 e0, e0val, \
201 e1, e1val, \
202 e2, e2val, \
203 e3, e3val, \
204 e4, e4val, \
205 e5, e5val, \
206 e6, e6val, \
207 e7, e7val, \
208 e8, e8val, \
209 e9, e9val, \
210 e10, e10val, \
211 e11, e11val, \
212 e12, e12val, \
213 e13, e13val, \
214 e14, e14val, \
215 e15, e15val) \
216 ( \
217 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
218 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
219 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
220 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
221 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
222 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
223 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
224 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
225 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
226 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
227 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
228 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
229 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
230 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
231 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
232 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
233 )
234
235 /* all CceStatus sub-block freeze bits */
236 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
237 | CCE_STATUS_RXE_FROZE_SMASK \
238 | CCE_STATUS_TXE_FROZE_SMASK \
239 | CCE_STATUS_TXE_PIO_FROZE_SMASK)
240 /* all CceStatus sub-block TXE pause bits */
241 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
242 | CCE_STATUS_TXE_PAUSED_SMASK \
243 | CCE_STATUS_SDMA_PAUSED_SMASK)
244 /* all CceStatus sub-block RXE pause bits */
245 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
246
247 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
248 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF
249
250 /*
251 * CCE Error flags.
252 */
253 static struct flag_table cce_err_status_flags[] = {
254 /* 0*/ FLAG_ENTRY0("CceCsrParityErr",
255 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
256 /* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr",
257 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
258 /* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr",
259 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
260 /* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
261 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
262 /* 4*/ FLAG_ENTRY0("CceTrgtAccessErr",
263 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
264 /* 5*/ FLAG_ENTRY0("CceRspdDataParityErr",
265 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
266 /* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
267 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
268 /* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr",
269 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
270 /* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
271 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
272 /* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
273 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
274 /*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
275 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
276 /*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
277 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
278 /*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
279 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
280 /*13*/ FLAG_ENTRY0("PcicRetryMemCorErr",
281 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
282 /*14*/ FLAG_ENTRY0("PcicRetryMemCorErr",
283 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
284 /*15*/ FLAG_ENTRY0("PcicPostHdQCorErr",
285 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
286 /*16*/ FLAG_ENTRY0("PcicPostHdQCorErr",
287 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
288 /*17*/ FLAG_ENTRY0("PcicPostHdQCorErr",
289 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
290 /*18*/ FLAG_ENTRY0("PcicCplDatQCorErr",
291 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
292 /*19*/ FLAG_ENTRY0("PcicNPostHQParityErr",
293 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
294 /*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr",
295 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
296 /*21*/ FLAG_ENTRY0("PcicRetryMemUncErr",
297 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
298 /*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr",
299 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
300 /*23*/ FLAG_ENTRY0("PcicPostHdQUncErr",
301 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
302 /*24*/ FLAG_ENTRY0("PcicPostDatQUncErr",
303 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
304 /*25*/ FLAG_ENTRY0("PcicCplHdQUncErr",
305 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
306 /*26*/ FLAG_ENTRY0("PcicCplDatQUncErr",
307 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
308 /*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr",
309 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
310 /*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr",
311 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
312 /*29*/ FLAG_ENTRY0("PcicReceiveParityErr",
313 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
314 /*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr",
315 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
316 /*31*/ FLAG_ENTRY0("LATriggered",
317 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
318 /*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr",
319 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
320 /*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr",
321 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
322 /*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
323 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
324 /*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
325 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
326 /*36*/ FLAG_ENTRY0("CceMsixTableCorErr",
327 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
328 /*37*/ FLAG_ENTRY0("CceMsixTableUncErr",
329 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
330 /*38*/ FLAG_ENTRY0("CceIntMapCorErr",
331 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
332 /*39*/ FLAG_ENTRY0("CceIntMapUncErr",
333 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
334 /*40*/ FLAG_ENTRY0("CceMsixCsrParityErr",
335 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
336 /*41-63 reserved*/
337 };
338
339 /*
340 * Misc Error flags
341 */
342 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
343 static struct flag_table misc_err_status_flags[] = {
344 /* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
345 /* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
346 /* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
347 /* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
348 /* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
349 /* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
350 /* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
351 /* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
352 /* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
353 /* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
354 /*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
355 /*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
356 /*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
357 };
358
359 /*
360 * TXE PIO Error flags and consequences
361 */
362 static struct flag_table pio_err_status_flags[] = {
363 /* 0*/ FLAG_ENTRY("PioWriteBadCtxt",
364 SEC_WRITE_DROPPED,
365 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
366 /* 1*/ FLAG_ENTRY("PioWriteAddrParity",
367 SEC_SPC_FREEZE,
368 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
369 /* 2*/ FLAG_ENTRY("PioCsrParity",
370 SEC_SPC_FREEZE,
371 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
372 /* 3*/ FLAG_ENTRY("PioSbMemFifo0",
373 SEC_SPC_FREEZE,
374 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
375 /* 4*/ FLAG_ENTRY("PioSbMemFifo1",
376 SEC_SPC_FREEZE,
377 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
378 /* 5*/ FLAG_ENTRY("PioPccFifoParity",
379 SEC_SPC_FREEZE,
380 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
381 /* 6*/ FLAG_ENTRY("PioPecFifoParity",
382 SEC_SPC_FREEZE,
383 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
384 /* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity",
385 SEC_SPC_FREEZE,
386 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
387 /* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
388 SEC_SPC_FREEZE,
389 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
390 /* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr",
391 SEC_SPC_FREEZE,
392 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
393 /*10*/ FLAG_ENTRY("PioSmPktResetParity",
394 SEC_SPC_FREEZE,
395 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
396 /*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc",
397 SEC_SPC_FREEZE,
398 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
399 /*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc",
400 SEC_SPC_FREEZE,
401 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
402 /*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor",
403 0,
404 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
405 /*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor",
406 0,
407 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
408 /*15*/ FLAG_ENTRY("PioCreditRetFifoParity",
409 SEC_SPC_FREEZE,
410 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
411 /*16*/ FLAG_ENTRY("PioPpmcPblFifo",
412 SEC_SPC_FREEZE,
413 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
414 /*17*/ FLAG_ENTRY("PioInitSmIn",
415 0,
416 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
417 /*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm",
418 SEC_SPC_FREEZE,
419 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
420 /*19*/ FLAG_ENTRY("PioHostAddrMemUnc",
421 SEC_SPC_FREEZE,
422 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
423 /*20*/ FLAG_ENTRY("PioHostAddrMemCor",
424 0,
425 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
426 /*21*/ FLAG_ENTRY("PioWriteDataParity",
427 SEC_SPC_FREEZE,
428 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
429 /*22*/ FLAG_ENTRY("PioStateMachine",
430 SEC_SPC_FREEZE,
431 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
432 /*23*/ FLAG_ENTRY("PioWriteQwValidParity",
433 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
434 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
435 /*24*/ FLAG_ENTRY("PioBlockQwCountParity",
436 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
437 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
438 /*25*/ FLAG_ENTRY("PioVlfVlLenParity",
439 SEC_SPC_FREEZE,
440 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
441 /*26*/ FLAG_ENTRY("PioVlfSopParity",
442 SEC_SPC_FREEZE,
443 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
444 /*27*/ FLAG_ENTRY("PioVlFifoParity",
445 SEC_SPC_FREEZE,
446 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
447 /*28*/ FLAG_ENTRY("PioPpmcBqcMemParity",
448 SEC_SPC_FREEZE,
449 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
450 /*29*/ FLAG_ENTRY("PioPpmcSopLen",
451 SEC_SPC_FREEZE,
452 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
453 /*30-31 reserved*/
454 /*32*/ FLAG_ENTRY("PioCurrentFreeCntParity",
455 SEC_SPC_FREEZE,
456 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
457 /*33*/ FLAG_ENTRY("PioLastReturnedCntParity",
458 SEC_SPC_FREEZE,
459 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
460 /*34*/ FLAG_ENTRY("PioPccSopHeadParity",
461 SEC_SPC_FREEZE,
462 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
463 /*35*/ FLAG_ENTRY("PioPecSopHeadParityErr",
464 SEC_SPC_FREEZE,
465 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
466 /*36-63 reserved*/
467 };
468
469 /* TXE PIO errors that cause an SPC freeze */
470 #define ALL_PIO_FREEZE_ERR \
471 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
472 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
473 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
474 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
475 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
476 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
477 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
478 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
479 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
480 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
481 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
482 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
483 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
484 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
485 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
486 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
487 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
488 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
489 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
490 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
491 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
492 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
493 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
494 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
495 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
496 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
497 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
498 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
499 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
500
501 /*
502 * TXE SDMA Error flags
503 */
504 static struct flag_table sdma_err_status_flags[] = {
505 /* 0*/ FLAG_ENTRY0("SDmaRpyTagErr",
506 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
507 /* 1*/ FLAG_ENTRY0("SDmaCsrParityErr",
508 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
509 /* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
510 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
511 /* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
512 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
513 /*04-63 reserved*/
514 };
515
516 /* TXE SDMA errors that cause an SPC freeze */
517 #define ALL_SDMA_FREEZE_ERR \
518 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
519 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
520 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
521
522 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
523 #define PORT_DISCARD_EGRESS_ERRS \
524 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
525 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
526 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
527
528 /*
529 * TXE Egress Error flags
530 */
531 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
532 static struct flag_table egress_err_status_flags[] = {
533 /* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
534 /* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
535 /* 2 reserved */
536 /* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
537 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
538 /* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
539 /* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
540 /* 6 reserved */
541 /* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr",
542 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
543 /* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
544 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
545 /* 9-10 reserved */
546 /*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
547 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
548 /*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
549 /*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
550 /*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
551 /*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
552 /*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
553 SEES(TX_SDMA0_DISALLOWED_PACKET)),
554 /*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
555 SEES(TX_SDMA1_DISALLOWED_PACKET)),
556 /*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
557 SEES(TX_SDMA2_DISALLOWED_PACKET)),
558 /*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
559 SEES(TX_SDMA3_DISALLOWED_PACKET)),
560 /*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
561 SEES(TX_SDMA4_DISALLOWED_PACKET)),
562 /*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
563 SEES(TX_SDMA5_DISALLOWED_PACKET)),
564 /*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
565 SEES(TX_SDMA6_DISALLOWED_PACKET)),
566 /*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
567 SEES(TX_SDMA7_DISALLOWED_PACKET)),
568 /*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
569 SEES(TX_SDMA8_DISALLOWED_PACKET)),
570 /*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
571 SEES(TX_SDMA9_DISALLOWED_PACKET)),
572 /*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
573 SEES(TX_SDMA10_DISALLOWED_PACKET)),
574 /*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
575 SEES(TX_SDMA11_DISALLOWED_PACKET)),
576 /*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
577 SEES(TX_SDMA12_DISALLOWED_PACKET)),
578 /*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
579 SEES(TX_SDMA13_DISALLOWED_PACKET)),
580 /*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
581 SEES(TX_SDMA14_DISALLOWED_PACKET)),
582 /*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
583 SEES(TX_SDMA15_DISALLOWED_PACKET)),
584 /*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
585 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
586 /*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
587 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
588 /*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
589 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
590 /*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
591 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
592 /*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
593 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
594 /*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
595 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
596 /*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
597 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
598 /*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
599 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
600 /*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
601 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
602 /*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
603 /*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
604 /*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
605 /*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
606 /*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
607 /*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
608 /*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
609 /*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
610 /*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
611 /*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
612 /*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
613 /*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
614 /*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
615 /*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
616 /*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
617 /*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
618 /*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
619 /*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
620 /*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
621 /*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
622 /*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
623 /*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
624 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
625 /*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
626 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
627 };
628
629 /*
630 * TXE Egress Error Info flags
631 */
632 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
633 static struct flag_table egress_err_info_flags[] = {
634 /* 0*/ FLAG_ENTRY0("Reserved", 0ull),
635 /* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)),
636 /* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
637 /* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
638 /* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
639 /* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
640 /* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
641 /* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
642 /* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)),
643 /* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
644 /*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)),
645 /*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
646 /*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
647 /*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
648 /*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
649 /*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
650 /*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
651 /*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
652 /*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
653 /*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
654 /*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
655 /*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
656 };
657
658 /* TXE Egress errors that cause an SPC freeze */
659 #define ALL_TXE_EGRESS_FREEZE_ERR \
660 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
661 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
662 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
663 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
664 | SEES(TX_LAUNCH_CSR_PARITY) \
665 | SEES(TX_SBRD_CTL_CSR_PARITY) \
666 | SEES(TX_CONFIG_PARITY) \
667 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
668 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
669 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
670 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
671 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
672 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
673 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
674 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
675 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
676 | SEES(TX_CREDIT_RETURN_PARITY))
677
678 /*
679 * TXE Send error flags
680 */
681 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
682 static struct flag_table send_err_status_flags[] = {
683 /* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
684 /* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
685 /* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
686 };
687
688 /*
689 * TXE Send Context Error flags and consequences
690 */
691 static struct flag_table sc_err_status_flags[] = {
692 /* 0*/ FLAG_ENTRY("InconsistentSop",
693 SEC_PACKET_DROPPED | SEC_SC_HALTED,
694 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
695 /* 1*/ FLAG_ENTRY("DisallowedPacket",
696 SEC_PACKET_DROPPED | SEC_SC_HALTED,
697 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
698 /* 2*/ FLAG_ENTRY("WriteCrossesBoundary",
699 SEC_WRITE_DROPPED | SEC_SC_HALTED,
700 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
701 /* 3*/ FLAG_ENTRY("WriteOverflow",
702 SEC_WRITE_DROPPED | SEC_SC_HALTED,
703 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
704 /* 4*/ FLAG_ENTRY("WriteOutOfBounds",
705 SEC_WRITE_DROPPED | SEC_SC_HALTED,
706 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
707 /* 5-63 reserved*/
708 };
709
710 /*
711 * RXE Receive Error flags
712 */
713 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
714 static struct flag_table rxe_err_status_flags[] = {
715 /* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
716 /* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
717 /* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
718 /* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
719 /* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
720 /* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
721 /* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
722 /* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
723 /* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
724 /* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
725 /*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
726 /*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
727 /*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
728 /*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
729 /*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
730 /*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
731 /*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
732 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
733 /*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
734 /*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
735 /*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr",
736 RXES(RBUF_BLOCK_LIST_READ_UNC)),
737 /*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr",
738 RXES(RBUF_BLOCK_LIST_READ_COR)),
739 /*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
740 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
741 /*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
742 RXES(RBUF_CSR_QENT_CNT_PARITY)),
743 /*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
744 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
745 /*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
746 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
747 /*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
748 /*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
749 /*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
750 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
751 /*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
752 /*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
753 /*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
754 /*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
755 /*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
756 /*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
757 /*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
758 /*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
759 RXES(RBUF_FL_INITDONE_PARITY)),
760 /*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
761 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
762 /*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
763 /*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
764 /*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
765 /*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
766 RXES(LOOKUP_DES_PART1_UNC_COR)),
767 /*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr",
768 RXES(LOOKUP_DES_PART2_PARITY)),
769 /*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
770 /*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
771 /*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
772 /*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
773 /*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
774 /*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
775 /*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
776 /*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
777 /*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
778 /*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
779 /*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
780 /*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
781 /*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
782 /*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
783 /*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
784 /*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
785 /*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
786 /*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
787 /*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
788 /*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
789 /*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
790 /*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
791 };
792
793 /* RXE errors that will trigger an SPC freeze */
794 #define ALL_RXE_FREEZE_ERR \
795 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
796 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
797 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
798 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
799 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
800 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
801 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
802 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
803 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
804 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
805 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
806 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
807 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
808 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
809 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
810 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
811 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
812 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
813 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
814 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
815 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
816 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
817 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
818 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
819 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
820 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
821 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
822 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
823 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
824 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
825 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
826 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
827 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
828 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
829 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
830 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
831 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
832 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
833 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
834 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
835 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
836 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
837 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
838 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
839
840 #define RXE_FREEZE_ABORT_MASK \
841 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
842 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
843 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
844
845 /*
846 * DCC Error Flags
847 */
848 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK
849 static struct flag_table dcc_err_flags[] = {
850 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
851 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
852 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
853 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
854 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
855 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
856 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
857 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
858 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
859 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
860 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
861 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
862 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
863 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
864 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
865 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
866 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
867 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
868 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
869 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
870 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
871 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
872 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
873 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
874 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
875 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
876 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
877 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
878 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
879 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
880 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
881 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
882 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
883 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
884 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
885 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
886 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
887 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
888 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
889 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
890 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
891 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
892 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
893 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
894 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
895 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
896 };
897
898 /*
899 * LCB error flags
900 */
901 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
902 static struct flag_table lcb_err_flags[] = {
903 /* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
904 /* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
905 /* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
906 /* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
907 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
908 /* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
909 /* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
910 /* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
911 /* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
912 /* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
913 /* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
914 /*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
915 /*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
916 /*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
917 /*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
918 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
919 /*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
920 /*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
921 /*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
922 /*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
923 /*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
924 /*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
925 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
926 /*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
927 /*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
928 /*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
929 /*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
930 /*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
931 /*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
932 /*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
933 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
934 /*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
935 /*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
936 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
937 /*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
938 LCBE(REDUNDANT_FLIT_PARITY_ERR))
939 };
940
941 /*
942 * DC8051 Error Flags
943 */
944 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
945 static struct flag_table dc8051_err_flags[] = {
946 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
947 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
948 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
949 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
950 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
951 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
952 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
953 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
954 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
955 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
956 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
957 };
958
959 /*
960 * DC8051 Information Error flags
961 *
962 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
963 */
964 static struct flag_table dc8051_info_err_flags[] = {
965 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED),
966 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME),
967 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET),
968 FLAG_ENTRY0("Serdes internal loopback failure",
969 FAILED_SERDES_INTERNAL_LOOPBACK),
970 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT),
971 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING),
972 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE),
973 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM),
974 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ),
975 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
976 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
977 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT),
978 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT),
979 FLAG_ENTRY0("External Device Request Timeout",
980 EXTERNAL_DEVICE_REQ_TIMEOUT),
981 };
982
983 /*
984 * DC8051 Information Host Information flags
985 *
986 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
987 */
988 static struct flag_table dc8051_info_host_msg_flags[] = {
989 FLAG_ENTRY0("Host request done", 0x0001),
990 FLAG_ENTRY0("BC PWR_MGM message", 0x0002),
991 FLAG_ENTRY0("BC SMA message", 0x0004),
992 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
993 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
994 FLAG_ENTRY0("External device config request", 0x0020),
995 FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
996 FLAG_ENTRY0("LinkUp achieved", 0x0080),
997 FLAG_ENTRY0("Link going down", 0x0100),
998 FLAG_ENTRY0("Link width downgraded", 0x0200),
999 };
1000
1001 static u32 encoded_size(u32 size);
1002 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
1003 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
1004 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
1005 u8 *continuous);
1006 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
1007 u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
1008 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
1009 u8 *remote_tx_rate, u16 *link_widths);
1010 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
1011 u8 *flag_bits, u16 *link_widths);
1012 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
1013 u8 *device_rev);
1014 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1015 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1016 u8 *tx_polarity_inversion,
1017 u8 *rx_polarity_inversion, u8 *max_rate);
1018 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1019 unsigned int context, u64 err_status);
1020 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1021 static void handle_dcc_err(struct hfi1_devdata *dd,
1022 unsigned int context, u64 err_status);
1023 static void handle_lcb_err(struct hfi1_devdata *dd,
1024 unsigned int context, u64 err_status);
1025 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1026 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1027 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1028 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1029 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1030 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1031 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1032 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1033 static void set_partition_keys(struct hfi1_pportdata *ppd);
1034 static const char *link_state_name(u32 state);
1035 static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1036 u32 state);
1037 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1038 u64 *out_data);
1039 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1040 static int thermal_init(struct hfi1_devdata *dd);
1041
1042 static void update_statusp(struct hfi1_pportdata *ppd, u32 state);
1043 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
1044 int msecs);
1045 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1046 int msecs);
1047 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state);
1048 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state);
1049 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1050 int msecs);
1051 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd,
1052 int msecs);
1053 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1054 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
1055 static void handle_temp_err(struct hfi1_devdata *dd);
1056 static void dc_shutdown(struct hfi1_devdata *dd);
1057 static void dc_start(struct hfi1_devdata *dd);
1058 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
1059 unsigned int *np);
1060 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
1061 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
1062 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index);
1063 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width);
1064
1065 /*
1066 * Error interrupt table entry. This is used as input to the interrupt
1067 * "clear down" routine used for all second tier error interrupt register.
1068 * Second tier interrupt registers have a single bit representing them
1069 * in the top-level CceIntStatus.
1070 */
1071 struct err_reg_info {
1072 u32 status; /* status CSR offset */
1073 u32 clear; /* clear CSR offset */
1074 u32 mask; /* mask CSR offset */
1075 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1076 const char *desc;
1077 };
1078
1079 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END + 1 - IS_GENERAL_ERR_START)
1080 #define NUM_DC_ERRS (IS_DC_END + 1 - IS_DC_START)
1081 #define NUM_VARIOUS (IS_VARIOUS_END + 1 - IS_VARIOUS_START)
1082
1083 /*
1084 * Helpers for building HFI and DC error interrupt table entries. Different
1085 * helpers are needed because of inconsistent register names.
1086 */
1087 #define EE(reg, handler, desc) \
1088 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1089 handler, desc }
1090 #define DC_EE1(reg, handler, desc) \
1091 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1092 #define DC_EE2(reg, handler, desc) \
1093 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1094
1095 /*
1096 * Table of the "misc" grouping of error interrupts. Each entry refers to
1097 * another register containing more information.
1098 */
1099 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1100 /* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"),
1101 /* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"),
1102 /* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"),
1103 /* 3*/ { 0, 0, 0, NULL }, /* reserved */
1104 /* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"),
1105 /* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"),
1106 /* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1107 /* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr")
1108 /* the rest are reserved */
1109 };
1110
1111 /*
1112 * Index into the Various section of the interrupt sources
1113 * corresponding to the Critical Temperature interrupt.
1114 */
1115 #define TCRIT_INT_SOURCE 4
1116
1117 /*
1118 * SDMA error interrupt entry - refers to another register containing more
1119 * information.
1120 */
1121 static const struct err_reg_info sdma_eng_err =
1122 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1123
1124 static const struct err_reg_info various_err[NUM_VARIOUS] = {
1125 /* 0*/ { 0, 0, 0, NULL }, /* PbcInt */
1126 /* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */
1127 /* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"),
1128 /* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"),
1129 /* 4*/ { 0, 0, 0, NULL }, /* TCritInt */
1130 /* rest are reserved */
1131 };
1132
1133 /*
1134 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1135 * register can not be derived from the MTU value because 10K is not
1136 * a power of 2. Therefore, we need a constant. Everything else can
1137 * be calculated.
1138 */
1139 #define DCC_CFG_PORT_MTU_CAP_10240 7
1140
1141 /*
1142 * Table of the DC grouping of error interrupts. Each entry refers to
1143 * another register containing more information.
1144 */
1145 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1146 /* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"),
1147 /* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"),
1148 /* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"),
1149 /* 3*/ /* dc_lbm_int - special, see is_dc_int() */
1150 /* the rest are reserved */
1151 };
1152
1153 struct cntr_entry {
1154 /*
1155 * counter name
1156 */
1157 char *name;
1158
1159 /*
1160 * csr to read for name (if applicable)
1161 */
1162 u64 csr;
1163
1164 /*
1165 * offset into dd or ppd to store the counter's value
1166 */
1167 int offset;
1168
1169 /*
1170 * flags
1171 */
1172 u8 flags;
1173
1174 /*
1175 * accessor for stat element, context either dd or ppd
1176 */
1177 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1178 int mode, u64 data);
1179 };
1180
1181 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1182 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1183
1184 #define CNTR_ELEM(name, csr, offset, flags, accessor) \
1185 { \
1186 name, \
1187 csr, \
1188 offset, \
1189 flags, \
1190 accessor \
1191 }
1192
1193 /* 32bit RXE */
1194 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1195 CNTR_ELEM(#name, \
1196 (counter * 8 + RCV_COUNTER_ARRAY32), \
1197 0, flags | CNTR_32BIT, \
1198 port_access_u32_csr)
1199
1200 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1201 CNTR_ELEM(#name, \
1202 (counter * 8 + RCV_COUNTER_ARRAY32), \
1203 0, flags | CNTR_32BIT, \
1204 dev_access_u32_csr)
1205
1206 /* 64bit RXE */
1207 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1208 CNTR_ELEM(#name, \
1209 (counter * 8 + RCV_COUNTER_ARRAY64), \
1210 0, flags, \
1211 port_access_u64_csr)
1212
1213 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1214 CNTR_ELEM(#name, \
1215 (counter * 8 + RCV_COUNTER_ARRAY64), \
1216 0, flags, \
1217 dev_access_u64_csr)
1218
1219 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1220 #define OVR_ELM(ctx) \
1221 CNTR_ELEM("RcvHdrOvr" #ctx, \
1222 (RCV_HDR_OVFL_CNT + ctx * 0x100), \
1223 0, CNTR_NORMAL, port_access_u64_csr)
1224
1225 /* 32bit TXE */
1226 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1227 CNTR_ELEM(#name, \
1228 (counter * 8 + SEND_COUNTER_ARRAY32), \
1229 0, flags | CNTR_32BIT, \
1230 port_access_u32_csr)
1231
1232 /* 64bit TXE */
1233 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1234 CNTR_ELEM(#name, \
1235 (counter * 8 + SEND_COUNTER_ARRAY64), \
1236 0, flags, \
1237 port_access_u64_csr)
1238
1239 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1240 CNTR_ELEM(#name,\
1241 counter * 8 + SEND_COUNTER_ARRAY64, \
1242 0, \
1243 flags, \
1244 dev_access_u64_csr)
1245
1246 /* CCE */
1247 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1248 CNTR_ELEM(#name, \
1249 (counter * 8 + CCE_COUNTER_ARRAY32), \
1250 0, flags | CNTR_32BIT, \
1251 dev_access_u32_csr)
1252
1253 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1254 CNTR_ELEM(#name, \
1255 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1256 0, flags | CNTR_32BIT, \
1257 dev_access_u32_csr)
1258
1259 /* DC */
1260 #define DC_PERF_CNTR(name, counter, flags) \
1261 CNTR_ELEM(#name, \
1262 counter, \
1263 0, \
1264 flags, \
1265 dev_access_u64_csr)
1266
1267 #define DC_PERF_CNTR_LCB(name, counter, flags) \
1268 CNTR_ELEM(#name, \
1269 counter, \
1270 0, \
1271 flags, \
1272 dc_access_lcb_cntr)
1273
1274 /* ibp counters */
1275 #define SW_IBP_CNTR(name, cntr) \
1276 CNTR_ELEM(#name, \
1277 0, \
1278 0, \
1279 CNTR_SYNTH, \
1280 access_ibp_##cntr)
1281
1282 /**
1283 * hfi1_addr_from_offset - return addr for readq/writeq
1284 * @dd: the dd device
1285 * @offset: the offset of the CSR within bar0
1286 *
1287 * This routine selects the appropriate base address
1288 * based on the indicated offset.
1289 */
hfi1_addr_from_offset(const struct hfi1_devdata * dd,u32 offset)1290 static inline void __iomem *hfi1_addr_from_offset(
1291 const struct hfi1_devdata *dd,
1292 u32 offset)
1293 {
1294 if (offset >= dd->base2_start)
1295 return dd->kregbase2 + (offset - dd->base2_start);
1296 return dd->kregbase1 + offset;
1297 }
1298
1299 /**
1300 * read_csr - read CSR at the indicated offset
1301 * @dd: the dd device
1302 * @offset: the offset of the CSR within bar0
1303 *
1304 * Return: the value read or all FF's if there
1305 * is no mapping
1306 */
read_csr(const struct hfi1_devdata * dd,u32 offset)1307 u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1308 {
1309 if (dd->flags & HFI1_PRESENT)
1310 return readq(hfi1_addr_from_offset(dd, offset));
1311 return -1;
1312 }
1313
1314 /**
1315 * write_csr - write CSR at the indicated offset
1316 * @dd: the dd device
1317 * @offset: the offset of the CSR within bar0
1318 * @value: value to write
1319 */
write_csr(const struct hfi1_devdata * dd,u32 offset,u64 value)1320 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1321 {
1322 if (dd->flags & HFI1_PRESENT) {
1323 void __iomem *base = hfi1_addr_from_offset(dd, offset);
1324
1325 /* avoid write to RcvArray */
1326 if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start))
1327 return;
1328 writeq(value, base);
1329 }
1330 }
1331
1332 /**
1333 * get_csr_addr - return te iomem address for offset
1334 * @dd: the dd device
1335 * @offset: the offset of the CSR within bar0
1336 *
1337 * Return: The iomem address to use in subsequent
1338 * writeq/readq operations.
1339 */
get_csr_addr(const struct hfi1_devdata * dd,u32 offset)1340 void __iomem *get_csr_addr(
1341 const struct hfi1_devdata *dd,
1342 u32 offset)
1343 {
1344 if (dd->flags & HFI1_PRESENT)
1345 return hfi1_addr_from_offset(dd, offset);
1346 return NULL;
1347 }
1348
read_write_csr(const struct hfi1_devdata * dd,u32 csr,int mode,u64 value)1349 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1350 int mode, u64 value)
1351 {
1352 u64 ret;
1353
1354 if (mode == CNTR_MODE_R) {
1355 ret = read_csr(dd, csr);
1356 } else if (mode == CNTR_MODE_W) {
1357 write_csr(dd, csr, value);
1358 ret = value;
1359 } else {
1360 dd_dev_err(dd, "Invalid cntr register access mode");
1361 return 0;
1362 }
1363
1364 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1365 return ret;
1366 }
1367
1368 /* Dev Access */
dev_access_u32_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1369 static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1370 void *context, int vl, int mode, u64 data)
1371 {
1372 struct hfi1_devdata *dd = context;
1373 u64 csr = entry->csr;
1374
1375 if (entry->flags & CNTR_SDMA) {
1376 if (vl == CNTR_INVALID_VL)
1377 return 0;
1378 csr += 0x100 * vl;
1379 } else {
1380 if (vl != CNTR_INVALID_VL)
1381 return 0;
1382 }
1383 return read_write_csr(dd, csr, mode, data);
1384 }
1385
access_sde_err_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1386 static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1387 void *context, int idx, int mode, u64 data)
1388 {
1389 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1390
1391 if (dd->per_sdma && idx < dd->num_sdma)
1392 return dd->per_sdma[idx].err_cnt;
1393 return 0;
1394 }
1395
access_sde_int_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1396 static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1397 void *context, int idx, int mode, u64 data)
1398 {
1399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1400
1401 if (dd->per_sdma && idx < dd->num_sdma)
1402 return dd->per_sdma[idx].sdma_int_cnt;
1403 return 0;
1404 }
1405
access_sde_idle_int_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1406 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1407 void *context, int idx, int mode, u64 data)
1408 {
1409 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1410
1411 if (dd->per_sdma && idx < dd->num_sdma)
1412 return dd->per_sdma[idx].idle_int_cnt;
1413 return 0;
1414 }
1415
access_sde_progress_int_cnt(const struct cntr_entry * entry,void * context,int idx,int mode,u64 data)1416 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1417 void *context, int idx, int mode,
1418 u64 data)
1419 {
1420 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1421
1422 if (dd->per_sdma && idx < dd->num_sdma)
1423 return dd->per_sdma[idx].progress_int_cnt;
1424 return 0;
1425 }
1426
dev_access_u64_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1427 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1428 int vl, int mode, u64 data)
1429 {
1430 struct hfi1_devdata *dd = context;
1431
1432 u64 val = 0;
1433 u64 csr = entry->csr;
1434
1435 if (entry->flags & CNTR_VL) {
1436 if (vl == CNTR_INVALID_VL)
1437 return 0;
1438 csr += 8 * vl;
1439 } else {
1440 if (vl != CNTR_INVALID_VL)
1441 return 0;
1442 }
1443
1444 val = read_write_csr(dd, csr, mode, data);
1445 return val;
1446 }
1447
dc_access_lcb_cntr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1448 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1449 int vl, int mode, u64 data)
1450 {
1451 struct hfi1_devdata *dd = context;
1452 u32 csr = entry->csr;
1453 int ret = 0;
1454
1455 if (vl != CNTR_INVALID_VL)
1456 return 0;
1457 if (mode == CNTR_MODE_R)
1458 ret = read_lcb_csr(dd, csr, &data);
1459 else if (mode == CNTR_MODE_W)
1460 ret = write_lcb_csr(dd, csr, data);
1461
1462 if (ret) {
1463 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1464 return 0;
1465 }
1466
1467 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1468 return data;
1469 }
1470
1471 /* Port Access */
port_access_u32_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1472 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1473 int vl, int mode, u64 data)
1474 {
1475 struct hfi1_pportdata *ppd = context;
1476
1477 if (vl != CNTR_INVALID_VL)
1478 return 0;
1479 return read_write_csr(ppd->dd, entry->csr, mode, data);
1480 }
1481
port_access_u64_csr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1482 static u64 port_access_u64_csr(const struct cntr_entry *entry,
1483 void *context, int vl, int mode, u64 data)
1484 {
1485 struct hfi1_pportdata *ppd = context;
1486 u64 val;
1487 u64 csr = entry->csr;
1488
1489 if (entry->flags & CNTR_VL) {
1490 if (vl == CNTR_INVALID_VL)
1491 return 0;
1492 csr += 8 * vl;
1493 } else {
1494 if (vl != CNTR_INVALID_VL)
1495 return 0;
1496 }
1497 val = read_write_csr(ppd->dd, csr, mode, data);
1498 return val;
1499 }
1500
1501 /* Software defined */
read_write_sw(struct hfi1_devdata * dd,u64 * cntr,int mode,u64 data)1502 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1503 u64 data)
1504 {
1505 u64 ret;
1506
1507 if (mode == CNTR_MODE_R) {
1508 ret = *cntr;
1509 } else if (mode == CNTR_MODE_W) {
1510 *cntr = data;
1511 ret = data;
1512 } else {
1513 dd_dev_err(dd, "Invalid cntr sw access mode");
1514 return 0;
1515 }
1516
1517 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1518
1519 return ret;
1520 }
1521
access_sw_link_dn_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1522 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1523 int vl, int mode, u64 data)
1524 {
1525 struct hfi1_pportdata *ppd = context;
1526
1527 if (vl != CNTR_INVALID_VL)
1528 return 0;
1529 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1530 }
1531
access_sw_link_up_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1532 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1533 int vl, int mode, u64 data)
1534 {
1535 struct hfi1_pportdata *ppd = context;
1536
1537 if (vl != CNTR_INVALID_VL)
1538 return 0;
1539 return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1540 }
1541
access_sw_unknown_frame_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1542 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1543 void *context, int vl, int mode,
1544 u64 data)
1545 {
1546 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1547
1548 if (vl != CNTR_INVALID_VL)
1549 return 0;
1550 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1551 }
1552
access_sw_xmit_discards(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1553 static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1554 void *context, int vl, int mode, u64 data)
1555 {
1556 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1557 u64 zero = 0;
1558 u64 *counter;
1559
1560 if (vl == CNTR_INVALID_VL)
1561 counter = &ppd->port_xmit_discards;
1562 else if (vl >= 0 && vl < C_VL_COUNT)
1563 counter = &ppd->port_xmit_discards_vl[vl];
1564 else
1565 counter = &zero;
1566
1567 return read_write_sw(ppd->dd, counter, mode, data);
1568 }
1569
access_xmit_constraint_errs(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1570 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1571 void *context, int vl, int mode,
1572 u64 data)
1573 {
1574 struct hfi1_pportdata *ppd = context;
1575
1576 if (vl != CNTR_INVALID_VL)
1577 return 0;
1578
1579 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1580 mode, data);
1581 }
1582
access_rcv_constraint_errs(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1583 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1584 void *context, int vl, int mode, u64 data)
1585 {
1586 struct hfi1_pportdata *ppd = context;
1587
1588 if (vl != CNTR_INVALID_VL)
1589 return 0;
1590
1591 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1592 mode, data);
1593 }
1594
get_all_cpu_total(u64 __percpu * cntr)1595 u64 get_all_cpu_total(u64 __percpu *cntr)
1596 {
1597 int cpu;
1598 u64 counter = 0;
1599
1600 for_each_possible_cpu(cpu)
1601 counter += *per_cpu_ptr(cntr, cpu);
1602 return counter;
1603 }
1604
read_write_cpu(struct hfi1_devdata * dd,u64 * z_val,u64 __percpu * cntr,int vl,int mode,u64 data)1605 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1606 u64 __percpu *cntr,
1607 int vl, int mode, u64 data)
1608 {
1609 u64 ret = 0;
1610
1611 if (vl != CNTR_INVALID_VL)
1612 return 0;
1613
1614 if (mode == CNTR_MODE_R) {
1615 ret = get_all_cpu_total(cntr) - *z_val;
1616 } else if (mode == CNTR_MODE_W) {
1617 /* A write can only zero the counter */
1618 if (data == 0)
1619 *z_val = get_all_cpu_total(cntr);
1620 else
1621 dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1622 } else {
1623 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1624 return 0;
1625 }
1626
1627 return ret;
1628 }
1629
access_sw_cpu_intr(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1630 static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1631 void *context, int vl, int mode, u64 data)
1632 {
1633 struct hfi1_devdata *dd = context;
1634
1635 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1636 mode, data);
1637 }
1638
access_sw_cpu_rcv_limit(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1639 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1640 void *context, int vl, int mode, u64 data)
1641 {
1642 struct hfi1_devdata *dd = context;
1643
1644 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1645 mode, data);
1646 }
1647
access_sw_pio_wait(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1648 static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1649 void *context, int vl, int mode, u64 data)
1650 {
1651 struct hfi1_devdata *dd = context;
1652
1653 return dd->verbs_dev.n_piowait;
1654 }
1655
access_sw_pio_drain(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1656 static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1657 void *context, int vl, int mode, u64 data)
1658 {
1659 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1660
1661 return dd->verbs_dev.n_piodrain;
1662 }
1663
access_sw_ctx0_seq_drop(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1664 static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry,
1665 void *context, int vl, int mode, u64 data)
1666 {
1667 struct hfi1_devdata *dd = context;
1668
1669 return dd->ctx0_seq_drop;
1670 }
1671
access_sw_vtx_wait(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1672 static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1673 void *context, int vl, int mode, u64 data)
1674 {
1675 struct hfi1_devdata *dd = context;
1676
1677 return dd->verbs_dev.n_txwait;
1678 }
1679
access_sw_kmem_wait(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1680 static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1681 void *context, int vl, int mode, u64 data)
1682 {
1683 struct hfi1_devdata *dd = context;
1684
1685 return dd->verbs_dev.n_kmem_wait;
1686 }
1687
access_sw_send_schedule(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1688 static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1689 void *context, int vl, int mode, u64 data)
1690 {
1691 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1692
1693 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1694 mode, data);
1695 }
1696
1697 /* Software counters for the error status bits within MISC_ERR_STATUS */
access_misc_pll_lock_fail_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1698 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1699 void *context, int vl, int mode,
1700 u64 data)
1701 {
1702 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1703
1704 return dd->misc_err_status_cnt[12];
1705 }
1706
access_misc_mbist_fail_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1707 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1708 void *context, int vl, int mode,
1709 u64 data)
1710 {
1711 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1712
1713 return dd->misc_err_status_cnt[11];
1714 }
1715
access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1716 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1717 void *context, int vl, int mode,
1718 u64 data)
1719 {
1720 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1721
1722 return dd->misc_err_status_cnt[10];
1723 }
1724
access_misc_efuse_done_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1725 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1726 void *context, int vl,
1727 int mode, u64 data)
1728 {
1729 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1730
1731 return dd->misc_err_status_cnt[9];
1732 }
1733
access_misc_efuse_write_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1734 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1735 void *context, int vl, int mode,
1736 u64 data)
1737 {
1738 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1739
1740 return dd->misc_err_status_cnt[8];
1741 }
1742
access_misc_efuse_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1743 static u64 access_misc_efuse_read_bad_addr_err_cnt(
1744 const struct cntr_entry *entry,
1745 void *context, int vl, int mode, u64 data)
1746 {
1747 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1748
1749 return dd->misc_err_status_cnt[7];
1750 }
1751
access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1752 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1753 void *context, int vl,
1754 int mode, u64 data)
1755 {
1756 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1757
1758 return dd->misc_err_status_cnt[6];
1759 }
1760
access_misc_fw_auth_failed_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1761 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1762 void *context, int vl, int mode,
1763 u64 data)
1764 {
1765 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1766
1767 return dd->misc_err_status_cnt[5];
1768 }
1769
access_misc_key_mismatch_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1770 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1771 void *context, int vl, int mode,
1772 u64 data)
1773 {
1774 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1775
1776 return dd->misc_err_status_cnt[4];
1777 }
1778
access_misc_sbus_write_failed_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1779 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1780 void *context, int vl,
1781 int mode, u64 data)
1782 {
1783 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1784
1785 return dd->misc_err_status_cnt[3];
1786 }
1787
access_misc_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1788 static u64 access_misc_csr_write_bad_addr_err_cnt(
1789 const struct cntr_entry *entry,
1790 void *context, int vl, int mode, u64 data)
1791 {
1792 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1793
1794 return dd->misc_err_status_cnt[2];
1795 }
1796
access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1797 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1798 void *context, int vl,
1799 int mode, u64 data)
1800 {
1801 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1802
1803 return dd->misc_err_status_cnt[1];
1804 }
1805
access_misc_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1806 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1807 void *context, int vl, int mode,
1808 u64 data)
1809 {
1810 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1811
1812 return dd->misc_err_status_cnt[0];
1813 }
1814
1815 /*
1816 * Software counter for the aggregate of
1817 * individual CceErrStatus counters
1818 */
access_sw_cce_err_status_aggregated_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1819 static u64 access_sw_cce_err_status_aggregated_cnt(
1820 const struct cntr_entry *entry,
1821 void *context, int vl, int mode, u64 data)
1822 {
1823 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1824
1825 return dd->sw_cce_err_status_aggregate;
1826 }
1827
1828 /*
1829 * Software counters corresponding to each of the
1830 * error status bits within CceErrStatus
1831 */
access_cce_msix_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1832 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1833 void *context, int vl, int mode,
1834 u64 data)
1835 {
1836 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1837
1838 return dd->cce_err_status_cnt[40];
1839 }
1840
access_cce_int_map_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1841 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1842 void *context, int vl, int mode,
1843 u64 data)
1844 {
1845 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1846
1847 return dd->cce_err_status_cnt[39];
1848 }
1849
access_cce_int_map_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1850 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1851 void *context, int vl, int mode,
1852 u64 data)
1853 {
1854 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1855
1856 return dd->cce_err_status_cnt[38];
1857 }
1858
access_cce_msix_table_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1859 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1860 void *context, int vl, int mode,
1861 u64 data)
1862 {
1863 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1864
1865 return dd->cce_err_status_cnt[37];
1866 }
1867
access_cce_msix_table_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1868 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1869 void *context, int vl, int mode,
1870 u64 data)
1871 {
1872 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1873
1874 return dd->cce_err_status_cnt[36];
1875 }
1876
access_cce_rxdma_conv_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1877 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1878 const struct cntr_entry *entry,
1879 void *context, int vl, int mode, u64 data)
1880 {
1881 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1882
1883 return dd->cce_err_status_cnt[35];
1884 }
1885
access_cce_rcpl_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1886 static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1887 const struct cntr_entry *entry,
1888 void *context, int vl, int mode, u64 data)
1889 {
1890 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1891
1892 return dd->cce_err_status_cnt[34];
1893 }
1894
access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1895 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1896 void *context, int vl,
1897 int mode, u64 data)
1898 {
1899 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1900
1901 return dd->cce_err_status_cnt[33];
1902 }
1903
access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1904 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1905 void *context, int vl, int mode,
1906 u64 data)
1907 {
1908 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1909
1910 return dd->cce_err_status_cnt[32];
1911 }
1912
access_la_triggered_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1913 static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1914 void *context, int vl, int mode, u64 data)
1915 {
1916 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1917
1918 return dd->cce_err_status_cnt[31];
1919 }
1920
access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1921 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1922 void *context, int vl, int mode,
1923 u64 data)
1924 {
1925 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1926
1927 return dd->cce_err_status_cnt[30];
1928 }
1929
access_pcic_receive_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1930 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1931 void *context, int vl, int mode,
1932 u64 data)
1933 {
1934 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1935
1936 return dd->cce_err_status_cnt[29];
1937 }
1938
access_pcic_transmit_back_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1939 static u64 access_pcic_transmit_back_parity_err_cnt(
1940 const struct cntr_entry *entry,
1941 void *context, int vl, int mode, u64 data)
1942 {
1943 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1944
1945 return dd->cce_err_status_cnt[28];
1946 }
1947
access_pcic_transmit_front_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1948 static u64 access_pcic_transmit_front_parity_err_cnt(
1949 const struct cntr_entry *entry,
1950 void *context, int vl, int mode, u64 data)
1951 {
1952 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1953
1954 return dd->cce_err_status_cnt[27];
1955 }
1956
access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1957 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1958 void *context, int vl, int mode,
1959 u64 data)
1960 {
1961 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1962
1963 return dd->cce_err_status_cnt[26];
1964 }
1965
access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1966 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1967 void *context, int vl, int mode,
1968 u64 data)
1969 {
1970 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1971
1972 return dd->cce_err_status_cnt[25];
1973 }
1974
access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1975 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1976 void *context, int vl, int mode,
1977 u64 data)
1978 {
1979 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1980
1981 return dd->cce_err_status_cnt[24];
1982 }
1983
access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1984 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1985 void *context, int vl, int mode,
1986 u64 data)
1987 {
1988 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1989
1990 return dd->cce_err_status_cnt[23];
1991 }
1992
access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)1993 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
1994 void *context, int vl,
1995 int mode, u64 data)
1996 {
1997 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1998
1999 return dd->cce_err_status_cnt[22];
2000 }
2001
access_pcic_retry_mem_unc_err(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2002 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
2003 void *context, int vl, int mode,
2004 u64 data)
2005 {
2006 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2007
2008 return dd->cce_err_status_cnt[21];
2009 }
2010
access_pcic_n_post_dat_q_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2011 static u64 access_pcic_n_post_dat_q_parity_err_cnt(
2012 const struct cntr_entry *entry,
2013 void *context, int vl, int mode, u64 data)
2014 {
2015 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2016
2017 return dd->cce_err_status_cnt[20];
2018 }
2019
access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2020 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
2021 void *context, int vl,
2022 int mode, u64 data)
2023 {
2024 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2025
2026 return dd->cce_err_status_cnt[19];
2027 }
2028
access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2029 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2030 void *context, int vl, int mode,
2031 u64 data)
2032 {
2033 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2034
2035 return dd->cce_err_status_cnt[18];
2036 }
2037
access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2038 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2039 void *context, int vl, int mode,
2040 u64 data)
2041 {
2042 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2043
2044 return dd->cce_err_status_cnt[17];
2045 }
2046
access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2047 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2048 void *context, int vl, int mode,
2049 u64 data)
2050 {
2051 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2052
2053 return dd->cce_err_status_cnt[16];
2054 }
2055
access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2056 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2057 void *context, int vl, int mode,
2058 u64 data)
2059 {
2060 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2061
2062 return dd->cce_err_status_cnt[15];
2063 }
2064
access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2065 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
2066 void *context, int vl,
2067 int mode, u64 data)
2068 {
2069 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2070
2071 return dd->cce_err_status_cnt[14];
2072 }
2073
access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2074 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
2075 void *context, int vl, int mode,
2076 u64 data)
2077 {
2078 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2079
2080 return dd->cce_err_status_cnt[13];
2081 }
2082
access_cce_cli1_async_fifo_dbg_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2083 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2084 const struct cntr_entry *entry,
2085 void *context, int vl, int mode, u64 data)
2086 {
2087 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2088
2089 return dd->cce_err_status_cnt[12];
2090 }
2091
access_cce_cli1_async_fifo_rxdma_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2092 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2093 const struct cntr_entry *entry,
2094 void *context, int vl, int mode, u64 data)
2095 {
2096 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2097
2098 return dd->cce_err_status_cnt[11];
2099 }
2100
access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2101 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2102 const struct cntr_entry *entry,
2103 void *context, int vl, int mode, u64 data)
2104 {
2105 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2106
2107 return dd->cce_err_status_cnt[10];
2108 }
2109
access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2110 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2111 const struct cntr_entry *entry,
2112 void *context, int vl, int mode, u64 data)
2113 {
2114 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2115
2116 return dd->cce_err_status_cnt[9];
2117 }
2118
access_cce_cli2_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2119 static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2120 const struct cntr_entry *entry,
2121 void *context, int vl, int mode, u64 data)
2122 {
2123 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2124
2125 return dd->cce_err_status_cnt[8];
2126 }
2127
access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2128 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2129 void *context, int vl,
2130 int mode, u64 data)
2131 {
2132 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2133
2134 return dd->cce_err_status_cnt[7];
2135 }
2136
access_cce_cli0_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2137 static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2138 const struct cntr_entry *entry,
2139 void *context, int vl, int mode, u64 data)
2140 {
2141 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2142
2143 return dd->cce_err_status_cnt[6];
2144 }
2145
access_cce_rspd_data_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2146 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2147 void *context, int vl, int mode,
2148 u64 data)
2149 {
2150 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2151
2152 return dd->cce_err_status_cnt[5];
2153 }
2154
access_cce_trgt_access_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2155 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2156 void *context, int vl, int mode,
2157 u64 data)
2158 {
2159 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2160
2161 return dd->cce_err_status_cnt[4];
2162 }
2163
access_cce_trgt_async_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2164 static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2165 const struct cntr_entry *entry,
2166 void *context, int vl, int mode, u64 data)
2167 {
2168 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2169
2170 return dd->cce_err_status_cnt[3];
2171 }
2172
access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2173 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2174 void *context, int vl,
2175 int mode, u64 data)
2176 {
2177 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2178
2179 return dd->cce_err_status_cnt[2];
2180 }
2181
access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2182 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2183 void *context, int vl,
2184 int mode, u64 data)
2185 {
2186 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2187
2188 return dd->cce_err_status_cnt[1];
2189 }
2190
access_ccs_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2191 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2192 void *context, int vl, int mode,
2193 u64 data)
2194 {
2195 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2196
2197 return dd->cce_err_status_cnt[0];
2198 }
2199
2200 /*
2201 * Software counters corresponding to each of the
2202 * error status bits within RcvErrStatus
2203 */
access_rx_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2204 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2205 void *context, int vl, int mode,
2206 u64 data)
2207 {
2208 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2209
2210 return dd->rcv_err_status_cnt[63];
2211 }
2212
access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2213 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2214 void *context, int vl,
2215 int mode, u64 data)
2216 {
2217 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2218
2219 return dd->rcv_err_status_cnt[62];
2220 }
2221
access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2222 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2223 void *context, int vl, int mode,
2224 u64 data)
2225 {
2226 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2227
2228 return dd->rcv_err_status_cnt[61];
2229 }
2230
access_rx_dma_csr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2231 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2232 void *context, int vl, int mode,
2233 u64 data)
2234 {
2235 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2236
2237 return dd->rcv_err_status_cnt[60];
2238 }
2239
access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2240 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2241 void *context, int vl,
2242 int mode, u64 data)
2243 {
2244 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2245
2246 return dd->rcv_err_status_cnt[59];
2247 }
2248
access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2249 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2250 void *context, int vl,
2251 int mode, u64 data)
2252 {
2253 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2254
2255 return dd->rcv_err_status_cnt[58];
2256 }
2257
access_rx_dma_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2258 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2259 void *context, int vl, int mode,
2260 u64 data)
2261 {
2262 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2263
2264 return dd->rcv_err_status_cnt[57];
2265 }
2266
access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2267 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2268 void *context, int vl, int mode,
2269 u64 data)
2270 {
2271 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2272
2273 return dd->rcv_err_status_cnt[56];
2274 }
2275
access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2276 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2277 void *context, int vl, int mode,
2278 u64 data)
2279 {
2280 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2281
2282 return dd->rcv_err_status_cnt[55];
2283 }
2284
access_rx_dma_data_fifo_rd_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2285 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2286 const struct cntr_entry *entry,
2287 void *context, int vl, int mode, u64 data)
2288 {
2289 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2290
2291 return dd->rcv_err_status_cnt[54];
2292 }
2293
access_rx_dma_data_fifo_rd_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2294 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2295 const struct cntr_entry *entry,
2296 void *context, int vl, int mode, u64 data)
2297 {
2298 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2299
2300 return dd->rcv_err_status_cnt[53];
2301 }
2302
access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2303 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2304 void *context, int vl,
2305 int mode, u64 data)
2306 {
2307 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2308
2309 return dd->rcv_err_status_cnt[52];
2310 }
2311
access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2312 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2313 void *context, int vl,
2314 int mode, u64 data)
2315 {
2316 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2317
2318 return dd->rcv_err_status_cnt[51];
2319 }
2320
access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2321 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2322 void *context, int vl,
2323 int mode, u64 data)
2324 {
2325 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2326
2327 return dd->rcv_err_status_cnt[50];
2328 }
2329
access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2330 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2331 void *context, int vl,
2332 int mode, u64 data)
2333 {
2334 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2335
2336 return dd->rcv_err_status_cnt[49];
2337 }
2338
access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2339 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2340 void *context, int vl,
2341 int mode, u64 data)
2342 {
2343 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2344
2345 return dd->rcv_err_status_cnt[48];
2346 }
2347
access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2348 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2349 void *context, int vl,
2350 int mode, u64 data)
2351 {
2352 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2353
2354 return dd->rcv_err_status_cnt[47];
2355 }
2356
access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2357 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2358 void *context, int vl, int mode,
2359 u64 data)
2360 {
2361 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2362
2363 return dd->rcv_err_status_cnt[46];
2364 }
2365
access_rx_hq_intr_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2366 static u64 access_rx_hq_intr_csr_parity_err_cnt(
2367 const struct cntr_entry *entry,
2368 void *context, int vl, int mode, u64 data)
2369 {
2370 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2371
2372 return dd->rcv_err_status_cnt[45];
2373 }
2374
access_rx_lookup_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2375 static u64 access_rx_lookup_csr_parity_err_cnt(
2376 const struct cntr_entry *entry,
2377 void *context, int vl, int mode, u64 data)
2378 {
2379 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2380
2381 return dd->rcv_err_status_cnt[44];
2382 }
2383
access_rx_lookup_rcv_array_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2384 static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2385 const struct cntr_entry *entry,
2386 void *context, int vl, int mode, u64 data)
2387 {
2388 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2389
2390 return dd->rcv_err_status_cnt[43];
2391 }
2392
access_rx_lookup_rcv_array_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2393 static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2394 const struct cntr_entry *entry,
2395 void *context, int vl, int mode, u64 data)
2396 {
2397 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2398
2399 return dd->rcv_err_status_cnt[42];
2400 }
2401
access_rx_lookup_des_part2_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2402 static u64 access_rx_lookup_des_part2_parity_err_cnt(
2403 const struct cntr_entry *entry,
2404 void *context, int vl, int mode, u64 data)
2405 {
2406 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2407
2408 return dd->rcv_err_status_cnt[41];
2409 }
2410
access_rx_lookup_des_part1_unc_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2411 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2412 const struct cntr_entry *entry,
2413 void *context, int vl, int mode, u64 data)
2414 {
2415 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2416
2417 return dd->rcv_err_status_cnt[40];
2418 }
2419
access_rx_lookup_des_part1_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2420 static u64 access_rx_lookup_des_part1_unc_err_cnt(
2421 const struct cntr_entry *entry,
2422 void *context, int vl, int mode, u64 data)
2423 {
2424 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2425
2426 return dd->rcv_err_status_cnt[39];
2427 }
2428
access_rx_rbuf_next_free_buf_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2429 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2430 const struct cntr_entry *entry,
2431 void *context, int vl, int mode, u64 data)
2432 {
2433 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2434
2435 return dd->rcv_err_status_cnt[38];
2436 }
2437
access_rx_rbuf_next_free_buf_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2438 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2439 const struct cntr_entry *entry,
2440 void *context, int vl, int mode, u64 data)
2441 {
2442 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2443
2444 return dd->rcv_err_status_cnt[37];
2445 }
2446
access_rbuf_fl_init_wr_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2447 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2448 const struct cntr_entry *entry,
2449 void *context, int vl, int mode, u64 data)
2450 {
2451 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2452
2453 return dd->rcv_err_status_cnt[36];
2454 }
2455
access_rx_rbuf_fl_initdone_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2456 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2457 const struct cntr_entry *entry,
2458 void *context, int vl, int mode, u64 data)
2459 {
2460 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2461
2462 return dd->rcv_err_status_cnt[35];
2463 }
2464
access_rx_rbuf_fl_write_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2465 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2466 const struct cntr_entry *entry,
2467 void *context, int vl, int mode, u64 data)
2468 {
2469 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2470
2471 return dd->rcv_err_status_cnt[34];
2472 }
2473
access_rx_rbuf_fl_rd_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2474 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2475 const struct cntr_entry *entry,
2476 void *context, int vl, int mode, u64 data)
2477 {
2478 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2479
2480 return dd->rcv_err_status_cnt[33];
2481 }
2482
access_rx_rbuf_empty_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2483 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2484 void *context, int vl, int mode,
2485 u64 data)
2486 {
2487 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2488
2489 return dd->rcv_err_status_cnt[32];
2490 }
2491
access_rx_rbuf_full_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2492 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2493 void *context, int vl, int mode,
2494 u64 data)
2495 {
2496 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2497
2498 return dd->rcv_err_status_cnt[31];
2499 }
2500
access_rbuf_bad_lookup_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2501 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2502 void *context, int vl, int mode,
2503 u64 data)
2504 {
2505 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2506
2507 return dd->rcv_err_status_cnt[30];
2508 }
2509
access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2510 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2511 void *context, int vl, int mode,
2512 u64 data)
2513 {
2514 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2515
2516 return dd->rcv_err_status_cnt[29];
2517 }
2518
access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2519 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2520 void *context, int vl,
2521 int mode, u64 data)
2522 {
2523 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2524
2525 return dd->rcv_err_status_cnt[28];
2526 }
2527
access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2528 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2529 const struct cntr_entry *entry,
2530 void *context, int vl, int mode, u64 data)
2531 {
2532 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2533
2534 return dd->rcv_err_status_cnt[27];
2535 }
2536
access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2537 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2538 const struct cntr_entry *entry,
2539 void *context, int vl, int mode, u64 data)
2540 {
2541 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2542
2543 return dd->rcv_err_status_cnt[26];
2544 }
2545
access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2546 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2547 const struct cntr_entry *entry,
2548 void *context, int vl, int mode, u64 data)
2549 {
2550 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2551
2552 return dd->rcv_err_status_cnt[25];
2553 }
2554
access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2555 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2556 const struct cntr_entry *entry,
2557 void *context, int vl, int mode, u64 data)
2558 {
2559 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2560
2561 return dd->rcv_err_status_cnt[24];
2562 }
2563
access_rx_rbuf_csr_q_next_buf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2564 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2565 const struct cntr_entry *entry,
2566 void *context, int vl, int mode, u64 data)
2567 {
2568 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2569
2570 return dd->rcv_err_status_cnt[23];
2571 }
2572
access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2573 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2574 const struct cntr_entry *entry,
2575 void *context, int vl, int mode, u64 data)
2576 {
2577 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2578
2579 return dd->rcv_err_status_cnt[22];
2580 }
2581
access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2582 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2583 const struct cntr_entry *entry,
2584 void *context, int vl, int mode, u64 data)
2585 {
2586 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2587
2588 return dd->rcv_err_status_cnt[21];
2589 }
2590
access_rx_rbuf_block_list_read_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2591 static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2592 const struct cntr_entry *entry,
2593 void *context, int vl, int mode, u64 data)
2594 {
2595 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2596
2597 return dd->rcv_err_status_cnt[20];
2598 }
2599
access_rx_rbuf_block_list_read_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2600 static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2601 const struct cntr_entry *entry,
2602 void *context, int vl, int mode, u64 data)
2603 {
2604 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2605
2606 return dd->rcv_err_status_cnt[19];
2607 }
2608
access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2609 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2610 void *context, int vl,
2611 int mode, u64 data)
2612 {
2613 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2614
2615 return dd->rcv_err_status_cnt[18];
2616 }
2617
access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2618 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2619 void *context, int vl,
2620 int mode, u64 data)
2621 {
2622 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2623
2624 return dd->rcv_err_status_cnt[17];
2625 }
2626
access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2627 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2628 const struct cntr_entry *entry,
2629 void *context, int vl, int mode, u64 data)
2630 {
2631 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2632
2633 return dd->rcv_err_status_cnt[16];
2634 }
2635
access_rx_rbuf_lookup_des_reg_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2636 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2637 const struct cntr_entry *entry,
2638 void *context, int vl, int mode, u64 data)
2639 {
2640 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2641
2642 return dd->rcv_err_status_cnt[15];
2643 }
2644
access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2645 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2646 void *context, int vl,
2647 int mode, u64 data)
2648 {
2649 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2650
2651 return dd->rcv_err_status_cnt[14];
2652 }
2653
access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2654 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2655 void *context, int vl,
2656 int mode, u64 data)
2657 {
2658 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2659
2660 return dd->rcv_err_status_cnt[13];
2661 }
2662
access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2663 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2664 void *context, int vl, int mode,
2665 u64 data)
2666 {
2667 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2668
2669 return dd->rcv_err_status_cnt[12];
2670 }
2671
access_rx_dma_flag_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2672 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2673 void *context, int vl, int mode,
2674 u64 data)
2675 {
2676 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2677
2678 return dd->rcv_err_status_cnt[11];
2679 }
2680
access_rx_dma_flag_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2681 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2682 void *context, int vl, int mode,
2683 u64 data)
2684 {
2685 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2686
2687 return dd->rcv_err_status_cnt[10];
2688 }
2689
access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2690 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2691 void *context, int vl, int mode,
2692 u64 data)
2693 {
2694 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2695
2696 return dd->rcv_err_status_cnt[9];
2697 }
2698
access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2699 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2700 void *context, int vl, int mode,
2701 u64 data)
2702 {
2703 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2704
2705 return dd->rcv_err_status_cnt[8];
2706 }
2707
access_rx_rcv_qp_map_table_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2708 static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2709 const struct cntr_entry *entry,
2710 void *context, int vl, int mode, u64 data)
2711 {
2712 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2713
2714 return dd->rcv_err_status_cnt[7];
2715 }
2716
access_rx_rcv_qp_map_table_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2717 static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2718 const struct cntr_entry *entry,
2719 void *context, int vl, int mode, u64 data)
2720 {
2721 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2722
2723 return dd->rcv_err_status_cnt[6];
2724 }
2725
access_rx_rcv_data_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2726 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2727 void *context, int vl, int mode,
2728 u64 data)
2729 {
2730 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2731
2732 return dd->rcv_err_status_cnt[5];
2733 }
2734
access_rx_rcv_data_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2735 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2736 void *context, int vl, int mode,
2737 u64 data)
2738 {
2739 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2740
2741 return dd->rcv_err_status_cnt[4];
2742 }
2743
access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2744 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2745 void *context, int vl, int mode,
2746 u64 data)
2747 {
2748 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2749
2750 return dd->rcv_err_status_cnt[3];
2751 }
2752
access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2753 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2754 void *context, int vl, int mode,
2755 u64 data)
2756 {
2757 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2758
2759 return dd->rcv_err_status_cnt[2];
2760 }
2761
access_rx_dc_intf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2762 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2763 void *context, int vl, int mode,
2764 u64 data)
2765 {
2766 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2767
2768 return dd->rcv_err_status_cnt[1];
2769 }
2770
access_rx_dma_csr_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2771 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2772 void *context, int vl, int mode,
2773 u64 data)
2774 {
2775 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2776
2777 return dd->rcv_err_status_cnt[0];
2778 }
2779
2780 /*
2781 * Software counters corresponding to each of the
2782 * error status bits within SendPioErrStatus
2783 */
access_pio_pec_sop_head_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2784 static u64 access_pio_pec_sop_head_parity_err_cnt(
2785 const struct cntr_entry *entry,
2786 void *context, int vl, int mode, u64 data)
2787 {
2788 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2789
2790 return dd->send_pio_err_status_cnt[35];
2791 }
2792
access_pio_pcc_sop_head_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2793 static u64 access_pio_pcc_sop_head_parity_err_cnt(
2794 const struct cntr_entry *entry,
2795 void *context, int vl, int mode, u64 data)
2796 {
2797 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2798
2799 return dd->send_pio_err_status_cnt[34];
2800 }
2801
access_pio_last_returned_cnt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2802 static u64 access_pio_last_returned_cnt_parity_err_cnt(
2803 const struct cntr_entry *entry,
2804 void *context, int vl, int mode, u64 data)
2805 {
2806 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2807
2808 return dd->send_pio_err_status_cnt[33];
2809 }
2810
access_pio_current_free_cnt_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2811 static u64 access_pio_current_free_cnt_parity_err_cnt(
2812 const struct cntr_entry *entry,
2813 void *context, int vl, int mode, u64 data)
2814 {
2815 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2816
2817 return dd->send_pio_err_status_cnt[32];
2818 }
2819
access_pio_reserved_31_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2820 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2821 void *context, int vl, int mode,
2822 u64 data)
2823 {
2824 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2825
2826 return dd->send_pio_err_status_cnt[31];
2827 }
2828
access_pio_reserved_30_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2829 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2830 void *context, int vl, int mode,
2831 u64 data)
2832 {
2833 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2834
2835 return dd->send_pio_err_status_cnt[30];
2836 }
2837
access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2838 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2839 void *context, int vl, int mode,
2840 u64 data)
2841 {
2842 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2843
2844 return dd->send_pio_err_status_cnt[29];
2845 }
2846
access_pio_ppmc_bqc_mem_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2847 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2848 const struct cntr_entry *entry,
2849 void *context, int vl, int mode, u64 data)
2850 {
2851 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2852
2853 return dd->send_pio_err_status_cnt[28];
2854 }
2855
access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2856 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2857 void *context, int vl, int mode,
2858 u64 data)
2859 {
2860 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2861
2862 return dd->send_pio_err_status_cnt[27];
2863 }
2864
access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2865 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2866 void *context, int vl, int mode,
2867 u64 data)
2868 {
2869 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2870
2871 return dd->send_pio_err_status_cnt[26];
2872 }
2873
access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2874 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2875 void *context, int vl,
2876 int mode, u64 data)
2877 {
2878 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2879
2880 return dd->send_pio_err_status_cnt[25];
2881 }
2882
access_pio_block_qw_count_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2883 static u64 access_pio_block_qw_count_parity_err_cnt(
2884 const struct cntr_entry *entry,
2885 void *context, int vl, int mode, u64 data)
2886 {
2887 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2888
2889 return dd->send_pio_err_status_cnt[24];
2890 }
2891
access_pio_write_qw_valid_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2892 static u64 access_pio_write_qw_valid_parity_err_cnt(
2893 const struct cntr_entry *entry,
2894 void *context, int vl, int mode, u64 data)
2895 {
2896 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2897
2898 return dd->send_pio_err_status_cnt[23];
2899 }
2900
access_pio_state_machine_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2901 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2902 void *context, int vl, int mode,
2903 u64 data)
2904 {
2905 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2906
2907 return dd->send_pio_err_status_cnt[22];
2908 }
2909
access_pio_write_data_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2910 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2911 void *context, int vl,
2912 int mode, u64 data)
2913 {
2914 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2915
2916 return dd->send_pio_err_status_cnt[21];
2917 }
2918
access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2919 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2920 void *context, int vl,
2921 int mode, u64 data)
2922 {
2923 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2924
2925 return dd->send_pio_err_status_cnt[20];
2926 }
2927
access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2928 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2929 void *context, int vl,
2930 int mode, u64 data)
2931 {
2932 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2933
2934 return dd->send_pio_err_status_cnt[19];
2935 }
2936
access_pio_pkt_evict_sm_or_arb_sm_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2937 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2938 const struct cntr_entry *entry,
2939 void *context, int vl, int mode, u64 data)
2940 {
2941 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2942
2943 return dd->send_pio_err_status_cnt[18];
2944 }
2945
access_pio_init_sm_in_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2946 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2947 void *context, int vl, int mode,
2948 u64 data)
2949 {
2950 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2951
2952 return dd->send_pio_err_status_cnt[17];
2953 }
2954
access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2955 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2956 void *context, int vl, int mode,
2957 u64 data)
2958 {
2959 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2960
2961 return dd->send_pio_err_status_cnt[16];
2962 }
2963
access_pio_credit_ret_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2964 static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2965 const struct cntr_entry *entry,
2966 void *context, int vl, int mode, u64 data)
2967 {
2968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2969
2970 return dd->send_pio_err_status_cnt[15];
2971 }
2972
access_pio_v1_len_mem_bank1_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2973 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2974 const struct cntr_entry *entry,
2975 void *context, int vl, int mode, u64 data)
2976 {
2977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2978
2979 return dd->send_pio_err_status_cnt[14];
2980 }
2981
access_pio_v1_len_mem_bank0_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2982 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2983 const struct cntr_entry *entry,
2984 void *context, int vl, int mode, u64 data)
2985 {
2986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2987
2988 return dd->send_pio_err_status_cnt[13];
2989 }
2990
access_pio_v1_len_mem_bank1_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)2991 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
2992 const struct cntr_entry *entry,
2993 void *context, int vl, int mode, u64 data)
2994 {
2995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2996
2997 return dd->send_pio_err_status_cnt[12];
2998 }
2999
access_pio_v1_len_mem_bank0_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3000 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
3001 const struct cntr_entry *entry,
3002 void *context, int vl, int mode, u64 data)
3003 {
3004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3005
3006 return dd->send_pio_err_status_cnt[11];
3007 }
3008
access_pio_sm_pkt_reset_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3009 static u64 access_pio_sm_pkt_reset_parity_err_cnt(
3010 const struct cntr_entry *entry,
3011 void *context, int vl, int mode, u64 data)
3012 {
3013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3014
3015 return dd->send_pio_err_status_cnt[10];
3016 }
3017
access_pio_pkt_evict_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3018 static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
3019 const struct cntr_entry *entry,
3020 void *context, int vl, int mode, u64 data)
3021 {
3022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3023
3024 return dd->send_pio_err_status_cnt[9];
3025 }
3026
access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3027 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
3028 const struct cntr_entry *entry,
3029 void *context, int vl, int mode, u64 data)
3030 {
3031 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3032
3033 return dd->send_pio_err_status_cnt[8];
3034 }
3035
access_pio_sbrdctl_crrel_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3036 static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
3037 const struct cntr_entry *entry,
3038 void *context, int vl, int mode, u64 data)
3039 {
3040 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3041
3042 return dd->send_pio_err_status_cnt[7];
3043 }
3044
access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3045 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
3046 void *context, int vl, int mode,
3047 u64 data)
3048 {
3049 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3050
3051 return dd->send_pio_err_status_cnt[6];
3052 }
3053
access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3054 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
3055 void *context, int vl, int mode,
3056 u64 data)
3057 {
3058 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3059
3060 return dd->send_pio_err_status_cnt[5];
3061 }
3062
access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3063 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
3064 void *context, int vl, int mode,
3065 u64 data)
3066 {
3067 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3068
3069 return dd->send_pio_err_status_cnt[4];
3070 }
3071
access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3072 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
3073 void *context, int vl, int mode,
3074 u64 data)
3075 {
3076 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3077
3078 return dd->send_pio_err_status_cnt[3];
3079 }
3080
access_pio_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3081 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3082 void *context, int vl, int mode,
3083 u64 data)
3084 {
3085 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3086
3087 return dd->send_pio_err_status_cnt[2];
3088 }
3089
access_pio_write_addr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3090 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3091 void *context, int vl,
3092 int mode, u64 data)
3093 {
3094 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3095
3096 return dd->send_pio_err_status_cnt[1];
3097 }
3098
access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3099 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3100 void *context, int vl, int mode,
3101 u64 data)
3102 {
3103 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3104
3105 return dd->send_pio_err_status_cnt[0];
3106 }
3107
3108 /*
3109 * Software counters corresponding to each of the
3110 * error status bits within SendDmaErrStatus
3111 */
access_sdma_pcie_req_tracking_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3112 static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3113 const struct cntr_entry *entry,
3114 void *context, int vl, int mode, u64 data)
3115 {
3116 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3117
3118 return dd->send_dma_err_status_cnt[3];
3119 }
3120
access_sdma_pcie_req_tracking_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3121 static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3122 const struct cntr_entry *entry,
3123 void *context, int vl, int mode, u64 data)
3124 {
3125 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3126
3127 return dd->send_dma_err_status_cnt[2];
3128 }
3129
access_sdma_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3130 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3131 void *context, int vl, int mode,
3132 u64 data)
3133 {
3134 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3135
3136 return dd->send_dma_err_status_cnt[1];
3137 }
3138
access_sdma_rpy_tag_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3139 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3140 void *context, int vl, int mode,
3141 u64 data)
3142 {
3143 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3144
3145 return dd->send_dma_err_status_cnt[0];
3146 }
3147
3148 /*
3149 * Software counters corresponding to each of the
3150 * error status bits within SendEgressErrStatus
3151 */
access_tx_read_pio_memory_csr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3152 static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3153 const struct cntr_entry *entry,
3154 void *context, int vl, int mode, u64 data)
3155 {
3156 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3157
3158 return dd->send_egress_err_status_cnt[63];
3159 }
3160
access_tx_read_sdma_memory_csr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3161 static u64 access_tx_read_sdma_memory_csr_err_cnt(
3162 const struct cntr_entry *entry,
3163 void *context, int vl, int mode, u64 data)
3164 {
3165 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3166
3167 return dd->send_egress_err_status_cnt[62];
3168 }
3169
access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3170 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3171 void *context, int vl, int mode,
3172 u64 data)
3173 {
3174 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3175
3176 return dd->send_egress_err_status_cnt[61];
3177 }
3178
access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3179 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3180 void *context, int vl,
3181 int mode, u64 data)
3182 {
3183 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3184
3185 return dd->send_egress_err_status_cnt[60];
3186 }
3187
access_tx_read_sdma_memory_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3188 static u64 access_tx_read_sdma_memory_cor_err_cnt(
3189 const struct cntr_entry *entry,
3190 void *context, int vl, int mode, u64 data)
3191 {
3192 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3193
3194 return dd->send_egress_err_status_cnt[59];
3195 }
3196
access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3197 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3198 void *context, int vl, int mode,
3199 u64 data)
3200 {
3201 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3202
3203 return dd->send_egress_err_status_cnt[58];
3204 }
3205
access_tx_credit_overrun_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3206 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3207 void *context, int vl, int mode,
3208 u64 data)
3209 {
3210 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3211
3212 return dd->send_egress_err_status_cnt[57];
3213 }
3214
access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3215 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3216 void *context, int vl, int mode,
3217 u64 data)
3218 {
3219 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3220
3221 return dd->send_egress_err_status_cnt[56];
3222 }
3223
access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3224 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3225 void *context, int vl, int mode,
3226 u64 data)
3227 {
3228 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3229
3230 return dd->send_egress_err_status_cnt[55];
3231 }
3232
access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3233 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3234 void *context, int vl, int mode,
3235 u64 data)
3236 {
3237 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3238
3239 return dd->send_egress_err_status_cnt[54];
3240 }
3241
access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3242 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3243 void *context, int vl, int mode,
3244 u64 data)
3245 {
3246 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3247
3248 return dd->send_egress_err_status_cnt[53];
3249 }
3250
access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3251 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3252 void *context, int vl, int mode,
3253 u64 data)
3254 {
3255 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3256
3257 return dd->send_egress_err_status_cnt[52];
3258 }
3259
access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3260 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3261 void *context, int vl, int mode,
3262 u64 data)
3263 {
3264 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3265
3266 return dd->send_egress_err_status_cnt[51];
3267 }
3268
access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3269 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3270 void *context, int vl, int mode,
3271 u64 data)
3272 {
3273 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3274
3275 return dd->send_egress_err_status_cnt[50];
3276 }
3277
access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3278 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3279 void *context, int vl, int mode,
3280 u64 data)
3281 {
3282 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3283
3284 return dd->send_egress_err_status_cnt[49];
3285 }
3286
access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3287 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3288 void *context, int vl, int mode,
3289 u64 data)
3290 {
3291 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3292
3293 return dd->send_egress_err_status_cnt[48];
3294 }
3295
access_tx_credit_return_vl_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3296 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3297 void *context, int vl, int mode,
3298 u64 data)
3299 {
3300 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3301
3302 return dd->send_egress_err_status_cnt[47];
3303 }
3304
access_tx_hcrc_insertion_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3305 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3306 void *context, int vl, int mode,
3307 u64 data)
3308 {
3309 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3310
3311 return dd->send_egress_err_status_cnt[46];
3312 }
3313
access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3314 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3315 void *context, int vl, int mode,
3316 u64 data)
3317 {
3318 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3319
3320 return dd->send_egress_err_status_cnt[45];
3321 }
3322
access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3323 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3324 void *context, int vl,
3325 int mode, u64 data)
3326 {
3327 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3328
3329 return dd->send_egress_err_status_cnt[44];
3330 }
3331
access_tx_read_sdma_memory_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3332 static u64 access_tx_read_sdma_memory_unc_err_cnt(
3333 const struct cntr_entry *entry,
3334 void *context, int vl, int mode, u64 data)
3335 {
3336 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3337
3338 return dd->send_egress_err_status_cnt[43];
3339 }
3340
access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3341 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3342 void *context, int vl, int mode,
3343 u64 data)
3344 {
3345 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3346
3347 return dd->send_egress_err_status_cnt[42];
3348 }
3349
access_tx_credit_return_partiy_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3350 static u64 access_tx_credit_return_partiy_err_cnt(
3351 const struct cntr_entry *entry,
3352 void *context, int vl, int mode, u64 data)
3353 {
3354 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3355
3356 return dd->send_egress_err_status_cnt[41];
3357 }
3358
access_tx_launch_fifo8_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3359 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3360 const struct cntr_entry *entry,
3361 void *context, int vl, int mode, u64 data)
3362 {
3363 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3364
3365 return dd->send_egress_err_status_cnt[40];
3366 }
3367
access_tx_launch_fifo7_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3368 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3369 const struct cntr_entry *entry,
3370 void *context, int vl, int mode, u64 data)
3371 {
3372 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3373
3374 return dd->send_egress_err_status_cnt[39];
3375 }
3376
access_tx_launch_fifo6_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3377 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3378 const struct cntr_entry *entry,
3379 void *context, int vl, int mode, u64 data)
3380 {
3381 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3382
3383 return dd->send_egress_err_status_cnt[38];
3384 }
3385
access_tx_launch_fifo5_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3386 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3387 const struct cntr_entry *entry,
3388 void *context, int vl, int mode, u64 data)
3389 {
3390 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3391
3392 return dd->send_egress_err_status_cnt[37];
3393 }
3394
access_tx_launch_fifo4_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3395 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3396 const struct cntr_entry *entry,
3397 void *context, int vl, int mode, u64 data)
3398 {
3399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3400
3401 return dd->send_egress_err_status_cnt[36];
3402 }
3403
access_tx_launch_fifo3_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3404 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3405 const struct cntr_entry *entry,
3406 void *context, int vl, int mode, u64 data)
3407 {
3408 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3409
3410 return dd->send_egress_err_status_cnt[35];
3411 }
3412
access_tx_launch_fifo2_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3413 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3414 const struct cntr_entry *entry,
3415 void *context, int vl, int mode, u64 data)
3416 {
3417 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3418
3419 return dd->send_egress_err_status_cnt[34];
3420 }
3421
access_tx_launch_fifo1_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3422 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3423 const struct cntr_entry *entry,
3424 void *context, int vl, int mode, u64 data)
3425 {
3426 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3427
3428 return dd->send_egress_err_status_cnt[33];
3429 }
3430
access_tx_launch_fifo0_unc_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3431 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3432 const struct cntr_entry *entry,
3433 void *context, int vl, int mode, u64 data)
3434 {
3435 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3436
3437 return dd->send_egress_err_status_cnt[32];
3438 }
3439
access_tx_sdma15_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3440 static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3441 const struct cntr_entry *entry,
3442 void *context, int vl, int mode, u64 data)
3443 {
3444 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3445
3446 return dd->send_egress_err_status_cnt[31];
3447 }
3448
access_tx_sdma14_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3449 static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3450 const struct cntr_entry *entry,
3451 void *context, int vl, int mode, u64 data)
3452 {
3453 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3454
3455 return dd->send_egress_err_status_cnt[30];
3456 }
3457
access_tx_sdma13_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3458 static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3459 const struct cntr_entry *entry,
3460 void *context, int vl, int mode, u64 data)
3461 {
3462 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3463
3464 return dd->send_egress_err_status_cnt[29];
3465 }
3466
access_tx_sdma12_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3467 static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3468 const struct cntr_entry *entry,
3469 void *context, int vl, int mode, u64 data)
3470 {
3471 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3472
3473 return dd->send_egress_err_status_cnt[28];
3474 }
3475
access_tx_sdma11_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3476 static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3477 const struct cntr_entry *entry,
3478 void *context, int vl, int mode, u64 data)
3479 {
3480 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3481
3482 return dd->send_egress_err_status_cnt[27];
3483 }
3484
access_tx_sdma10_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3485 static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3486 const struct cntr_entry *entry,
3487 void *context, int vl, int mode, u64 data)
3488 {
3489 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3490
3491 return dd->send_egress_err_status_cnt[26];
3492 }
3493
access_tx_sdma9_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3494 static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3495 const struct cntr_entry *entry,
3496 void *context, int vl, int mode, u64 data)
3497 {
3498 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3499
3500 return dd->send_egress_err_status_cnt[25];
3501 }
3502
access_tx_sdma8_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3503 static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3504 const struct cntr_entry *entry,
3505 void *context, int vl, int mode, u64 data)
3506 {
3507 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3508
3509 return dd->send_egress_err_status_cnt[24];
3510 }
3511
access_tx_sdma7_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3512 static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3513 const struct cntr_entry *entry,
3514 void *context, int vl, int mode, u64 data)
3515 {
3516 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3517
3518 return dd->send_egress_err_status_cnt[23];
3519 }
3520
access_tx_sdma6_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3521 static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3522 const struct cntr_entry *entry,
3523 void *context, int vl, int mode, u64 data)
3524 {
3525 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3526
3527 return dd->send_egress_err_status_cnt[22];
3528 }
3529
access_tx_sdma5_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3530 static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3531 const struct cntr_entry *entry,
3532 void *context, int vl, int mode, u64 data)
3533 {
3534 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3535
3536 return dd->send_egress_err_status_cnt[21];
3537 }
3538
access_tx_sdma4_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3539 static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3540 const struct cntr_entry *entry,
3541 void *context, int vl, int mode, u64 data)
3542 {
3543 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3544
3545 return dd->send_egress_err_status_cnt[20];
3546 }
3547
access_tx_sdma3_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3548 static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3549 const struct cntr_entry *entry,
3550 void *context, int vl, int mode, u64 data)
3551 {
3552 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3553
3554 return dd->send_egress_err_status_cnt[19];
3555 }
3556
access_tx_sdma2_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3557 static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3558 const struct cntr_entry *entry,
3559 void *context, int vl, int mode, u64 data)
3560 {
3561 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3562
3563 return dd->send_egress_err_status_cnt[18];
3564 }
3565
access_tx_sdma1_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3566 static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3567 const struct cntr_entry *entry,
3568 void *context, int vl, int mode, u64 data)
3569 {
3570 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3571
3572 return dd->send_egress_err_status_cnt[17];
3573 }
3574
access_tx_sdma0_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3575 static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3576 const struct cntr_entry *entry,
3577 void *context, int vl, int mode, u64 data)
3578 {
3579 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3580
3581 return dd->send_egress_err_status_cnt[16];
3582 }
3583
access_tx_config_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3584 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3585 void *context, int vl, int mode,
3586 u64 data)
3587 {
3588 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3589
3590 return dd->send_egress_err_status_cnt[15];
3591 }
3592
access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3593 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3594 void *context, int vl,
3595 int mode, u64 data)
3596 {
3597 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3598
3599 return dd->send_egress_err_status_cnt[14];
3600 }
3601
access_tx_launch_csr_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3602 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3603 void *context, int vl, int mode,
3604 u64 data)
3605 {
3606 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3607
3608 return dd->send_egress_err_status_cnt[13];
3609 }
3610
access_tx_illegal_vl_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3611 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3612 void *context, int vl, int mode,
3613 u64 data)
3614 {
3615 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3616
3617 return dd->send_egress_err_status_cnt[12];
3618 }
3619
access_tx_sbrd_ctl_state_machine_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3620 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3621 const struct cntr_entry *entry,
3622 void *context, int vl, int mode, u64 data)
3623 {
3624 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3625
3626 return dd->send_egress_err_status_cnt[11];
3627 }
3628
access_egress_reserved_10_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3629 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3630 void *context, int vl, int mode,
3631 u64 data)
3632 {
3633 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3634
3635 return dd->send_egress_err_status_cnt[10];
3636 }
3637
access_egress_reserved_9_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3638 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3639 void *context, int vl, int mode,
3640 u64 data)
3641 {
3642 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3643
3644 return dd->send_egress_err_status_cnt[9];
3645 }
3646
access_tx_sdma_launch_intf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3647 static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3648 const struct cntr_entry *entry,
3649 void *context, int vl, int mode, u64 data)
3650 {
3651 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3652
3653 return dd->send_egress_err_status_cnt[8];
3654 }
3655
access_tx_pio_launch_intf_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3656 static u64 access_tx_pio_launch_intf_parity_err_cnt(
3657 const struct cntr_entry *entry,
3658 void *context, int vl, int mode, u64 data)
3659 {
3660 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3661
3662 return dd->send_egress_err_status_cnt[7];
3663 }
3664
access_egress_reserved_6_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3665 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3666 void *context, int vl, int mode,
3667 u64 data)
3668 {
3669 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3670
3671 return dd->send_egress_err_status_cnt[6];
3672 }
3673
access_tx_incorrect_link_state_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3674 static u64 access_tx_incorrect_link_state_err_cnt(
3675 const struct cntr_entry *entry,
3676 void *context, int vl, int mode, u64 data)
3677 {
3678 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3679
3680 return dd->send_egress_err_status_cnt[5];
3681 }
3682
access_tx_linkdown_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3683 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3684 void *context, int vl, int mode,
3685 u64 data)
3686 {
3687 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3688
3689 return dd->send_egress_err_status_cnt[4];
3690 }
3691
access_tx_egress_fifi_underrun_or_parity_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3692 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3693 const struct cntr_entry *entry,
3694 void *context, int vl, int mode, u64 data)
3695 {
3696 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3697
3698 return dd->send_egress_err_status_cnt[3];
3699 }
3700
access_egress_reserved_2_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3701 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3702 void *context, int vl, int mode,
3703 u64 data)
3704 {
3705 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3706
3707 return dd->send_egress_err_status_cnt[2];
3708 }
3709
access_tx_pkt_integrity_mem_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3710 static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3711 const struct cntr_entry *entry,
3712 void *context, int vl, int mode, u64 data)
3713 {
3714 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3715
3716 return dd->send_egress_err_status_cnt[1];
3717 }
3718
access_tx_pkt_integrity_mem_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3719 static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3720 const struct cntr_entry *entry,
3721 void *context, int vl, int mode, u64 data)
3722 {
3723 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3724
3725 return dd->send_egress_err_status_cnt[0];
3726 }
3727
3728 /*
3729 * Software counters corresponding to each of the
3730 * error status bits within SendErrStatus
3731 */
access_send_csr_write_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3732 static u64 access_send_csr_write_bad_addr_err_cnt(
3733 const struct cntr_entry *entry,
3734 void *context, int vl, int mode, u64 data)
3735 {
3736 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3737
3738 return dd->send_err_status_cnt[2];
3739 }
3740
access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3741 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3742 void *context, int vl,
3743 int mode, u64 data)
3744 {
3745 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3746
3747 return dd->send_err_status_cnt[1];
3748 }
3749
access_send_csr_parity_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3750 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3751 void *context, int vl, int mode,
3752 u64 data)
3753 {
3754 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3755
3756 return dd->send_err_status_cnt[0];
3757 }
3758
3759 /*
3760 * Software counters corresponding to each of the
3761 * error status bits within SendCtxtErrStatus
3762 */
access_pio_write_out_of_bounds_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3763 static u64 access_pio_write_out_of_bounds_err_cnt(
3764 const struct cntr_entry *entry,
3765 void *context, int vl, int mode, u64 data)
3766 {
3767 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3768
3769 return dd->sw_ctxt_err_status_cnt[4];
3770 }
3771
access_pio_write_overflow_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3772 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3773 void *context, int vl, int mode,
3774 u64 data)
3775 {
3776 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3777
3778 return dd->sw_ctxt_err_status_cnt[3];
3779 }
3780
access_pio_write_crosses_boundary_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3781 static u64 access_pio_write_crosses_boundary_err_cnt(
3782 const struct cntr_entry *entry,
3783 void *context, int vl, int mode, u64 data)
3784 {
3785 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3786
3787 return dd->sw_ctxt_err_status_cnt[2];
3788 }
3789
access_pio_disallowed_packet_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3790 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3791 void *context, int vl,
3792 int mode, u64 data)
3793 {
3794 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3795
3796 return dd->sw_ctxt_err_status_cnt[1];
3797 }
3798
access_pio_inconsistent_sop_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3799 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3800 void *context, int vl, int mode,
3801 u64 data)
3802 {
3803 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3804
3805 return dd->sw_ctxt_err_status_cnt[0];
3806 }
3807
3808 /*
3809 * Software counters corresponding to each of the
3810 * error status bits within SendDmaEngErrStatus
3811 */
access_sdma_header_request_fifo_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3812 static u64 access_sdma_header_request_fifo_cor_err_cnt(
3813 const struct cntr_entry *entry,
3814 void *context, int vl, int mode, u64 data)
3815 {
3816 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3817
3818 return dd->sw_send_dma_eng_err_status_cnt[23];
3819 }
3820
access_sdma_header_storage_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3821 static u64 access_sdma_header_storage_cor_err_cnt(
3822 const struct cntr_entry *entry,
3823 void *context, int vl, int mode, u64 data)
3824 {
3825 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3826
3827 return dd->sw_send_dma_eng_err_status_cnt[22];
3828 }
3829
access_sdma_packet_tracking_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3830 static u64 access_sdma_packet_tracking_cor_err_cnt(
3831 const struct cntr_entry *entry,
3832 void *context, int vl, int mode, u64 data)
3833 {
3834 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3835
3836 return dd->sw_send_dma_eng_err_status_cnt[21];
3837 }
3838
access_sdma_assembly_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3839 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3840 void *context, int vl, int mode,
3841 u64 data)
3842 {
3843 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3844
3845 return dd->sw_send_dma_eng_err_status_cnt[20];
3846 }
3847
access_sdma_desc_table_cor_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3848 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3849 void *context, int vl, int mode,
3850 u64 data)
3851 {
3852 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3853
3854 return dd->sw_send_dma_eng_err_status_cnt[19];
3855 }
3856
access_sdma_header_request_fifo_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3857 static u64 access_sdma_header_request_fifo_unc_err_cnt(
3858 const struct cntr_entry *entry,
3859 void *context, int vl, int mode, u64 data)
3860 {
3861 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3862
3863 return dd->sw_send_dma_eng_err_status_cnt[18];
3864 }
3865
access_sdma_header_storage_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3866 static u64 access_sdma_header_storage_unc_err_cnt(
3867 const struct cntr_entry *entry,
3868 void *context, int vl, int mode, u64 data)
3869 {
3870 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3871
3872 return dd->sw_send_dma_eng_err_status_cnt[17];
3873 }
3874
access_sdma_packet_tracking_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3875 static u64 access_sdma_packet_tracking_unc_err_cnt(
3876 const struct cntr_entry *entry,
3877 void *context, int vl, int mode, u64 data)
3878 {
3879 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3880
3881 return dd->sw_send_dma_eng_err_status_cnt[16];
3882 }
3883
access_sdma_assembly_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3884 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3885 void *context, int vl, int mode,
3886 u64 data)
3887 {
3888 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3889
3890 return dd->sw_send_dma_eng_err_status_cnt[15];
3891 }
3892
access_sdma_desc_table_unc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3893 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3894 void *context, int vl, int mode,
3895 u64 data)
3896 {
3897 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3898
3899 return dd->sw_send_dma_eng_err_status_cnt[14];
3900 }
3901
access_sdma_timeout_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3902 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3903 void *context, int vl, int mode,
3904 u64 data)
3905 {
3906 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3907
3908 return dd->sw_send_dma_eng_err_status_cnt[13];
3909 }
3910
access_sdma_header_length_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3911 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3912 void *context, int vl, int mode,
3913 u64 data)
3914 {
3915 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3916
3917 return dd->sw_send_dma_eng_err_status_cnt[12];
3918 }
3919
access_sdma_header_address_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3920 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3921 void *context, int vl, int mode,
3922 u64 data)
3923 {
3924 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3925
3926 return dd->sw_send_dma_eng_err_status_cnt[11];
3927 }
3928
access_sdma_header_select_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3929 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3930 void *context, int vl, int mode,
3931 u64 data)
3932 {
3933 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3934
3935 return dd->sw_send_dma_eng_err_status_cnt[10];
3936 }
3937
access_sdma_reserved_9_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3938 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3939 void *context, int vl, int mode,
3940 u64 data)
3941 {
3942 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3943
3944 return dd->sw_send_dma_eng_err_status_cnt[9];
3945 }
3946
access_sdma_packet_desc_overflow_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3947 static u64 access_sdma_packet_desc_overflow_err_cnt(
3948 const struct cntr_entry *entry,
3949 void *context, int vl, int mode, u64 data)
3950 {
3951 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3952
3953 return dd->sw_send_dma_eng_err_status_cnt[8];
3954 }
3955
access_sdma_length_mismatch_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3956 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3957 void *context, int vl,
3958 int mode, u64 data)
3959 {
3960 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3961
3962 return dd->sw_send_dma_eng_err_status_cnt[7];
3963 }
3964
access_sdma_halt_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3965 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3966 void *context, int vl, int mode, u64 data)
3967 {
3968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3969
3970 return dd->sw_send_dma_eng_err_status_cnt[6];
3971 }
3972
access_sdma_mem_read_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3973 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3974 void *context, int vl, int mode,
3975 u64 data)
3976 {
3977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3978
3979 return dd->sw_send_dma_eng_err_status_cnt[5];
3980 }
3981
access_sdma_first_desc_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3982 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3983 void *context, int vl, int mode,
3984 u64 data)
3985 {
3986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3987
3988 return dd->sw_send_dma_eng_err_status_cnt[4];
3989 }
3990
access_sdma_tail_out_of_bounds_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)3991 static u64 access_sdma_tail_out_of_bounds_err_cnt(
3992 const struct cntr_entry *entry,
3993 void *context, int vl, int mode, u64 data)
3994 {
3995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3996
3997 return dd->sw_send_dma_eng_err_status_cnt[3];
3998 }
3999
access_sdma_too_long_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4000 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
4001 void *context, int vl, int mode,
4002 u64 data)
4003 {
4004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4005
4006 return dd->sw_send_dma_eng_err_status_cnt[2];
4007 }
4008
access_sdma_gen_mismatch_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4009 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
4010 void *context, int vl, int mode,
4011 u64 data)
4012 {
4013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4014
4015 return dd->sw_send_dma_eng_err_status_cnt[1];
4016 }
4017
access_sdma_wrong_dw_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4018 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
4019 void *context, int vl, int mode,
4020 u64 data)
4021 {
4022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4023
4024 return dd->sw_send_dma_eng_err_status_cnt[0];
4025 }
4026
access_dc_rcv_err_cnt(const struct cntr_entry * entry,void * context,int vl,int mode,u64 data)4027 static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry,
4028 void *context, int vl, int mode,
4029 u64 data)
4030 {
4031 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4032
4033 u64 val = 0;
4034 u64 csr = entry->csr;
4035
4036 val = read_write_csr(dd, csr, mode, data);
4037 if (mode == CNTR_MODE_R) {
4038 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ?
4039 CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors;
4040 } else if (mode == CNTR_MODE_W) {
4041 dd->sw_rcv_bypass_packet_errors = 0;
4042 } else {
4043 dd_dev_err(dd, "Invalid cntr register access mode");
4044 return 0;
4045 }
4046 return val;
4047 }
4048
4049 #define def_access_sw_cpu(cntr) \
4050 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
4051 void *context, int vl, int mode, u64 data) \
4052 { \
4053 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4054 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \
4055 ppd->ibport_data.rvp.cntr, vl, \
4056 mode, data); \
4057 }
4058
4059 def_access_sw_cpu(rc_acks);
4060 def_access_sw_cpu(rc_qacks);
4061 def_access_sw_cpu(rc_delayed_comp);
4062
4063 #define def_access_ibp_counter(cntr) \
4064 static u64 access_ibp_##cntr(const struct cntr_entry *entry, \
4065 void *context, int vl, int mode, u64 data) \
4066 { \
4067 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4068 \
4069 if (vl != CNTR_INVALID_VL) \
4070 return 0; \
4071 \
4072 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \
4073 mode, data); \
4074 }
4075
4076 def_access_ibp_counter(loop_pkts);
4077 def_access_ibp_counter(rc_resends);
4078 def_access_ibp_counter(rnr_naks);
4079 def_access_ibp_counter(other_naks);
4080 def_access_ibp_counter(rc_timeouts);
4081 def_access_ibp_counter(pkt_drops);
4082 def_access_ibp_counter(dmawait);
4083 def_access_ibp_counter(rc_seqnak);
4084 def_access_ibp_counter(rc_dupreq);
4085 def_access_ibp_counter(rdma_seq);
4086 def_access_ibp_counter(unaligned);
4087 def_access_ibp_counter(seq_naks);
4088 def_access_ibp_counter(rc_crwaits);
4089
4090 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
4091 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
4092 [C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH),
4093 [C_RX_SHORT_ERR] = RXE32_DEV_CNTR_ELEM(RxShrErr, RCV_SHORT_ERR_CNT, CNTR_SYNTH),
4094 [C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH),
4095 [C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH),
4096 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
4097 CNTR_NORMAL),
4098 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
4099 CNTR_NORMAL),
4100 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
4101 RCV_TID_FLOW_GEN_MISMATCH_CNT,
4102 CNTR_NORMAL),
4103 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
4104 CNTR_NORMAL),
4105 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
4106 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
4107 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4108 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4109 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4110 CNTR_NORMAL),
4111 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4112 CNTR_NORMAL),
4113 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4114 CNTR_NORMAL),
4115 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4116 CNTR_NORMAL),
4117 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4118 CNTR_NORMAL),
4119 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4120 CNTR_NORMAL),
4121 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4122 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4123 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4124 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4125 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4126 CNTR_SYNTH),
4127 [C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH,
4128 access_dc_rcv_err_cnt),
4129 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4130 CNTR_SYNTH),
4131 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4132 CNTR_SYNTH),
4133 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4134 CNTR_SYNTH),
4135 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4136 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4137 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4138 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4139 CNTR_SYNTH),
4140 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4141 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4142 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4143 CNTR_SYNTH),
4144 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4145 CNTR_SYNTH),
4146 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4147 CNTR_SYNTH),
4148 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4149 CNTR_SYNTH),
4150 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4151 CNTR_SYNTH),
4152 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4153 CNTR_SYNTH),
4154 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4155 CNTR_SYNTH),
4156 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4157 CNTR_SYNTH | CNTR_VL),
4158 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4159 CNTR_SYNTH | CNTR_VL),
4160 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4161 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4162 CNTR_SYNTH | CNTR_VL),
4163 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4164 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4165 CNTR_SYNTH | CNTR_VL),
4166 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4167 CNTR_SYNTH),
4168 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4169 CNTR_SYNTH | CNTR_VL),
4170 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4171 CNTR_SYNTH),
4172 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4173 CNTR_SYNTH | CNTR_VL),
4174 [C_DC_TOTAL_CRC] =
4175 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4176 CNTR_SYNTH),
4177 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4178 CNTR_SYNTH),
4179 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4180 CNTR_SYNTH),
4181 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4182 CNTR_SYNTH),
4183 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4184 CNTR_SYNTH),
4185 [C_DC_CRC_MULT_LN] =
4186 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4187 CNTR_SYNTH),
4188 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4189 CNTR_SYNTH),
4190 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4191 CNTR_SYNTH),
4192 [C_DC_SEQ_CRC_CNT] =
4193 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4194 CNTR_SYNTH),
4195 [C_DC_ESC0_ONLY_CNT] =
4196 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4197 CNTR_SYNTH),
4198 [C_DC_ESC0_PLUS1_CNT] =
4199 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4200 CNTR_SYNTH),
4201 [C_DC_ESC0_PLUS2_CNT] =
4202 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4203 CNTR_SYNTH),
4204 [C_DC_REINIT_FROM_PEER_CNT] =
4205 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4206 CNTR_SYNTH),
4207 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4208 CNTR_SYNTH),
4209 [C_DC_MISC_FLG_CNT] =
4210 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4211 CNTR_SYNTH),
4212 [C_DC_PRF_GOOD_LTP_CNT] =
4213 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4214 [C_DC_PRF_ACCEPTED_LTP_CNT] =
4215 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4216 CNTR_SYNTH),
4217 [C_DC_PRF_RX_FLIT_CNT] =
4218 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4219 [C_DC_PRF_TX_FLIT_CNT] =
4220 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4221 [C_DC_PRF_CLK_CNTR] =
4222 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4223 [C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4224 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4225 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4226 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4227 CNTR_SYNTH),
4228 [C_DC_PG_STS_TX_SBE_CNT] =
4229 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4230 [C_DC_PG_STS_TX_MBE_CNT] =
4231 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4232 CNTR_SYNTH),
4233 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4234 access_sw_cpu_intr),
4235 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4236 access_sw_cpu_rcv_limit),
4237 [C_SW_CTX0_SEQ_DROP] = CNTR_ELEM("SeqDrop0", 0, 0, CNTR_NORMAL,
4238 access_sw_ctx0_seq_drop),
4239 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4240 access_sw_vtx_wait),
4241 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4242 access_sw_pio_wait),
4243 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4244 access_sw_pio_drain),
4245 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4246 access_sw_kmem_wait),
4247 [C_SW_TID_WAIT] = CNTR_ELEM("TidWait", 0, 0, CNTR_NORMAL,
4248 hfi1_access_sw_tid_wait),
4249 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4250 access_sw_send_schedule),
4251 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4252 SEND_DMA_DESC_FETCHED_CNT, 0,
4253 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4254 dev_access_u32_csr),
4255 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4256 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4257 access_sde_int_cnt),
4258 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4259 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4260 access_sde_err_cnt),
4261 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4262 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4263 access_sde_idle_int_cnt),
4264 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4265 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4266 access_sde_progress_int_cnt),
4267 /* MISC_ERR_STATUS */
4268 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4269 CNTR_NORMAL,
4270 access_misc_pll_lock_fail_err_cnt),
4271 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4272 CNTR_NORMAL,
4273 access_misc_mbist_fail_err_cnt),
4274 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4275 CNTR_NORMAL,
4276 access_misc_invalid_eep_cmd_err_cnt),
4277 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4278 CNTR_NORMAL,
4279 access_misc_efuse_done_parity_err_cnt),
4280 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4281 CNTR_NORMAL,
4282 access_misc_efuse_write_err_cnt),
4283 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4284 0, CNTR_NORMAL,
4285 access_misc_efuse_read_bad_addr_err_cnt),
4286 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4287 CNTR_NORMAL,
4288 access_misc_efuse_csr_parity_err_cnt),
4289 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4290 CNTR_NORMAL,
4291 access_misc_fw_auth_failed_err_cnt),
4292 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4293 CNTR_NORMAL,
4294 access_misc_key_mismatch_err_cnt),
4295 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4296 CNTR_NORMAL,
4297 access_misc_sbus_write_failed_err_cnt),
4298 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4299 CNTR_NORMAL,
4300 access_misc_csr_write_bad_addr_err_cnt),
4301 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4302 CNTR_NORMAL,
4303 access_misc_csr_read_bad_addr_err_cnt),
4304 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4305 CNTR_NORMAL,
4306 access_misc_csr_parity_err_cnt),
4307 /* CceErrStatus */
4308 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4309 CNTR_NORMAL,
4310 access_sw_cce_err_status_aggregated_cnt),
4311 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4312 CNTR_NORMAL,
4313 access_cce_msix_csr_parity_err_cnt),
4314 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4315 CNTR_NORMAL,
4316 access_cce_int_map_unc_err_cnt),
4317 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4318 CNTR_NORMAL,
4319 access_cce_int_map_cor_err_cnt),
4320 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4321 CNTR_NORMAL,
4322 access_cce_msix_table_unc_err_cnt),
4323 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4324 CNTR_NORMAL,
4325 access_cce_msix_table_cor_err_cnt),
4326 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4327 0, CNTR_NORMAL,
4328 access_cce_rxdma_conv_fifo_parity_err_cnt),
4329 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4330 0, CNTR_NORMAL,
4331 access_cce_rcpl_async_fifo_parity_err_cnt),
4332 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4333 CNTR_NORMAL,
4334 access_cce_seg_write_bad_addr_err_cnt),
4335 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4336 CNTR_NORMAL,
4337 access_cce_seg_read_bad_addr_err_cnt),
4338 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4339 CNTR_NORMAL,
4340 access_la_triggered_cnt),
4341 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4342 CNTR_NORMAL,
4343 access_cce_trgt_cpl_timeout_err_cnt),
4344 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4345 CNTR_NORMAL,
4346 access_pcic_receive_parity_err_cnt),
4347 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4348 CNTR_NORMAL,
4349 access_pcic_transmit_back_parity_err_cnt),
4350 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4351 0, CNTR_NORMAL,
4352 access_pcic_transmit_front_parity_err_cnt),
4353 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4354 CNTR_NORMAL,
4355 access_pcic_cpl_dat_q_unc_err_cnt),
4356 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4357 CNTR_NORMAL,
4358 access_pcic_cpl_hd_q_unc_err_cnt),
4359 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4360 CNTR_NORMAL,
4361 access_pcic_post_dat_q_unc_err_cnt),
4362 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4363 CNTR_NORMAL,
4364 access_pcic_post_hd_q_unc_err_cnt),
4365 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4366 CNTR_NORMAL,
4367 access_pcic_retry_sot_mem_unc_err_cnt),
4368 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4369 CNTR_NORMAL,
4370 access_pcic_retry_mem_unc_err),
4371 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4372 CNTR_NORMAL,
4373 access_pcic_n_post_dat_q_parity_err_cnt),
4374 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4375 CNTR_NORMAL,
4376 access_pcic_n_post_h_q_parity_err_cnt),
4377 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4378 CNTR_NORMAL,
4379 access_pcic_cpl_dat_q_cor_err_cnt),
4380 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4381 CNTR_NORMAL,
4382 access_pcic_cpl_hd_q_cor_err_cnt),
4383 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4384 CNTR_NORMAL,
4385 access_pcic_post_dat_q_cor_err_cnt),
4386 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4387 CNTR_NORMAL,
4388 access_pcic_post_hd_q_cor_err_cnt),
4389 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4390 CNTR_NORMAL,
4391 access_pcic_retry_sot_mem_cor_err_cnt),
4392 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4393 CNTR_NORMAL,
4394 access_pcic_retry_mem_cor_err_cnt),
4395 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4396 "CceCli1AsyncFifoDbgParityError", 0, 0,
4397 CNTR_NORMAL,
4398 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4399 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4400 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4401 CNTR_NORMAL,
4402 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4403 ),
4404 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4405 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4406 CNTR_NORMAL,
4407 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4408 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4409 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4410 CNTR_NORMAL,
4411 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4412 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4413 0, CNTR_NORMAL,
4414 access_cce_cli2_async_fifo_parity_err_cnt),
4415 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4416 CNTR_NORMAL,
4417 access_cce_csr_cfg_bus_parity_err_cnt),
4418 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4419 0, CNTR_NORMAL,
4420 access_cce_cli0_async_fifo_parity_err_cnt),
4421 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4422 CNTR_NORMAL,
4423 access_cce_rspd_data_parity_err_cnt),
4424 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4425 CNTR_NORMAL,
4426 access_cce_trgt_access_err_cnt),
4427 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4428 0, CNTR_NORMAL,
4429 access_cce_trgt_async_fifo_parity_err_cnt),
4430 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4431 CNTR_NORMAL,
4432 access_cce_csr_write_bad_addr_err_cnt),
4433 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4434 CNTR_NORMAL,
4435 access_cce_csr_read_bad_addr_err_cnt),
4436 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4437 CNTR_NORMAL,
4438 access_ccs_csr_parity_err_cnt),
4439
4440 /* RcvErrStatus */
4441 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4442 CNTR_NORMAL,
4443 access_rx_csr_parity_err_cnt),
4444 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4445 CNTR_NORMAL,
4446 access_rx_csr_write_bad_addr_err_cnt),
4447 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4448 CNTR_NORMAL,
4449 access_rx_csr_read_bad_addr_err_cnt),
4450 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4451 CNTR_NORMAL,
4452 access_rx_dma_csr_unc_err_cnt),
4453 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4454 CNTR_NORMAL,
4455 access_rx_dma_dq_fsm_encoding_err_cnt),
4456 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4457 CNTR_NORMAL,
4458 access_rx_dma_eq_fsm_encoding_err_cnt),
4459 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4460 CNTR_NORMAL,
4461 access_rx_dma_csr_parity_err_cnt),
4462 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4463 CNTR_NORMAL,
4464 access_rx_rbuf_data_cor_err_cnt),
4465 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4466 CNTR_NORMAL,
4467 access_rx_rbuf_data_unc_err_cnt),
4468 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4469 CNTR_NORMAL,
4470 access_rx_dma_data_fifo_rd_cor_err_cnt),
4471 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4472 CNTR_NORMAL,
4473 access_rx_dma_data_fifo_rd_unc_err_cnt),
4474 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4475 CNTR_NORMAL,
4476 access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4477 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4478 CNTR_NORMAL,
4479 access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4480 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4481 CNTR_NORMAL,
4482 access_rx_rbuf_desc_part2_cor_err_cnt),
4483 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4484 CNTR_NORMAL,
4485 access_rx_rbuf_desc_part2_unc_err_cnt),
4486 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4487 CNTR_NORMAL,
4488 access_rx_rbuf_desc_part1_cor_err_cnt),
4489 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4490 CNTR_NORMAL,
4491 access_rx_rbuf_desc_part1_unc_err_cnt),
4492 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4493 CNTR_NORMAL,
4494 access_rx_hq_intr_fsm_err_cnt),
4495 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4496 CNTR_NORMAL,
4497 access_rx_hq_intr_csr_parity_err_cnt),
4498 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4499 CNTR_NORMAL,
4500 access_rx_lookup_csr_parity_err_cnt),
4501 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4502 CNTR_NORMAL,
4503 access_rx_lookup_rcv_array_cor_err_cnt),
4504 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4505 CNTR_NORMAL,
4506 access_rx_lookup_rcv_array_unc_err_cnt),
4507 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4508 0, CNTR_NORMAL,
4509 access_rx_lookup_des_part2_parity_err_cnt),
4510 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4511 0, CNTR_NORMAL,
4512 access_rx_lookup_des_part1_unc_cor_err_cnt),
4513 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4514 CNTR_NORMAL,
4515 access_rx_lookup_des_part1_unc_err_cnt),
4516 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4517 CNTR_NORMAL,
4518 access_rx_rbuf_next_free_buf_cor_err_cnt),
4519 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4520 CNTR_NORMAL,
4521 access_rx_rbuf_next_free_buf_unc_err_cnt),
4522 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4523 "RxRbufFlInitWrAddrParityErr", 0, 0,
4524 CNTR_NORMAL,
4525 access_rbuf_fl_init_wr_addr_parity_err_cnt),
4526 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4527 0, CNTR_NORMAL,
4528 access_rx_rbuf_fl_initdone_parity_err_cnt),
4529 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4530 0, CNTR_NORMAL,
4531 access_rx_rbuf_fl_write_addr_parity_err_cnt),
4532 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4533 CNTR_NORMAL,
4534 access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4535 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4536 CNTR_NORMAL,
4537 access_rx_rbuf_empty_err_cnt),
4538 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4539 CNTR_NORMAL,
4540 access_rx_rbuf_full_err_cnt),
4541 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4542 CNTR_NORMAL,
4543 access_rbuf_bad_lookup_err_cnt),
4544 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4545 CNTR_NORMAL,
4546 access_rbuf_ctx_id_parity_err_cnt),
4547 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4548 CNTR_NORMAL,
4549 access_rbuf_csr_qeopdw_parity_err_cnt),
4550 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4551 "RxRbufCsrQNumOfPktParityErr", 0, 0,
4552 CNTR_NORMAL,
4553 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4554 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4555 "RxRbufCsrQTlPtrParityErr", 0, 0,
4556 CNTR_NORMAL,
4557 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4558 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4559 0, CNTR_NORMAL,
4560 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4561 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4562 0, CNTR_NORMAL,
4563 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4564 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4565 0, 0, CNTR_NORMAL,
4566 access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4567 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4568 0, CNTR_NORMAL,
4569 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4570 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4571 "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4572 CNTR_NORMAL,
4573 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4574 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4575 0, CNTR_NORMAL,
4576 access_rx_rbuf_block_list_read_cor_err_cnt),
4577 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4578 0, CNTR_NORMAL,
4579 access_rx_rbuf_block_list_read_unc_err_cnt),
4580 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4581 CNTR_NORMAL,
4582 access_rx_rbuf_lookup_des_cor_err_cnt),
4583 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4584 CNTR_NORMAL,
4585 access_rx_rbuf_lookup_des_unc_err_cnt),
4586 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4587 "RxRbufLookupDesRegUncCorErr", 0, 0,
4588 CNTR_NORMAL,
4589 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4590 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4591 CNTR_NORMAL,
4592 access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4593 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4594 CNTR_NORMAL,
4595 access_rx_rbuf_free_list_cor_err_cnt),
4596 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4597 CNTR_NORMAL,
4598 access_rx_rbuf_free_list_unc_err_cnt),
4599 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4600 CNTR_NORMAL,
4601 access_rx_rcv_fsm_encoding_err_cnt),
4602 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4603 CNTR_NORMAL,
4604 access_rx_dma_flag_cor_err_cnt),
4605 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4606 CNTR_NORMAL,
4607 access_rx_dma_flag_unc_err_cnt),
4608 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4609 CNTR_NORMAL,
4610 access_rx_dc_sop_eop_parity_err_cnt),
4611 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4612 CNTR_NORMAL,
4613 access_rx_rcv_csr_parity_err_cnt),
4614 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4615 CNTR_NORMAL,
4616 access_rx_rcv_qp_map_table_cor_err_cnt),
4617 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4618 CNTR_NORMAL,
4619 access_rx_rcv_qp_map_table_unc_err_cnt),
4620 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4621 CNTR_NORMAL,
4622 access_rx_rcv_data_cor_err_cnt),
4623 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4624 CNTR_NORMAL,
4625 access_rx_rcv_data_unc_err_cnt),
4626 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4627 CNTR_NORMAL,
4628 access_rx_rcv_hdr_cor_err_cnt),
4629 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4630 CNTR_NORMAL,
4631 access_rx_rcv_hdr_unc_err_cnt),
4632 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4633 CNTR_NORMAL,
4634 access_rx_dc_intf_parity_err_cnt),
4635 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4636 CNTR_NORMAL,
4637 access_rx_dma_csr_cor_err_cnt),
4638 /* SendPioErrStatus */
4639 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4640 CNTR_NORMAL,
4641 access_pio_pec_sop_head_parity_err_cnt),
4642 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4643 CNTR_NORMAL,
4644 access_pio_pcc_sop_head_parity_err_cnt),
4645 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4646 0, 0, CNTR_NORMAL,
4647 access_pio_last_returned_cnt_parity_err_cnt),
4648 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4649 0, CNTR_NORMAL,
4650 access_pio_current_free_cnt_parity_err_cnt),
4651 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4652 CNTR_NORMAL,
4653 access_pio_reserved_31_err_cnt),
4654 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4655 CNTR_NORMAL,
4656 access_pio_reserved_30_err_cnt),
4657 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4658 CNTR_NORMAL,
4659 access_pio_ppmc_sop_len_err_cnt),
4660 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4661 CNTR_NORMAL,
4662 access_pio_ppmc_bqc_mem_parity_err_cnt),
4663 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4664 CNTR_NORMAL,
4665 access_pio_vl_fifo_parity_err_cnt),
4666 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4667 CNTR_NORMAL,
4668 access_pio_vlf_sop_parity_err_cnt),
4669 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4670 CNTR_NORMAL,
4671 access_pio_vlf_v1_len_parity_err_cnt),
4672 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4673 CNTR_NORMAL,
4674 access_pio_block_qw_count_parity_err_cnt),
4675 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4676 CNTR_NORMAL,
4677 access_pio_write_qw_valid_parity_err_cnt),
4678 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4679 CNTR_NORMAL,
4680 access_pio_state_machine_err_cnt),
4681 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4682 CNTR_NORMAL,
4683 access_pio_write_data_parity_err_cnt),
4684 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4685 CNTR_NORMAL,
4686 access_pio_host_addr_mem_cor_err_cnt),
4687 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4688 CNTR_NORMAL,
4689 access_pio_host_addr_mem_unc_err_cnt),
4690 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4691 CNTR_NORMAL,
4692 access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4693 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4694 CNTR_NORMAL,
4695 access_pio_init_sm_in_err_cnt),
4696 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4697 CNTR_NORMAL,
4698 access_pio_ppmc_pbl_fifo_err_cnt),
4699 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4700 0, CNTR_NORMAL,
4701 access_pio_credit_ret_fifo_parity_err_cnt),
4702 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4703 CNTR_NORMAL,
4704 access_pio_v1_len_mem_bank1_cor_err_cnt),
4705 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4706 CNTR_NORMAL,
4707 access_pio_v1_len_mem_bank0_cor_err_cnt),
4708 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4709 CNTR_NORMAL,
4710 access_pio_v1_len_mem_bank1_unc_err_cnt),
4711 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4712 CNTR_NORMAL,
4713 access_pio_v1_len_mem_bank0_unc_err_cnt),
4714 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4715 CNTR_NORMAL,
4716 access_pio_sm_pkt_reset_parity_err_cnt),
4717 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4718 CNTR_NORMAL,
4719 access_pio_pkt_evict_fifo_parity_err_cnt),
4720 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4721 "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4722 CNTR_NORMAL,
4723 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4724 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4725 CNTR_NORMAL,
4726 access_pio_sbrdctl_crrel_parity_err_cnt),
4727 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4728 CNTR_NORMAL,
4729 access_pio_pec_fifo_parity_err_cnt),
4730 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4731 CNTR_NORMAL,
4732 access_pio_pcc_fifo_parity_err_cnt),
4733 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4734 CNTR_NORMAL,
4735 access_pio_sb_mem_fifo1_err_cnt),
4736 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4737 CNTR_NORMAL,
4738 access_pio_sb_mem_fifo0_err_cnt),
4739 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4740 CNTR_NORMAL,
4741 access_pio_csr_parity_err_cnt),
4742 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4743 CNTR_NORMAL,
4744 access_pio_write_addr_parity_err_cnt),
4745 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4746 CNTR_NORMAL,
4747 access_pio_write_bad_ctxt_err_cnt),
4748 /* SendDmaErrStatus */
4749 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4750 0, CNTR_NORMAL,
4751 access_sdma_pcie_req_tracking_cor_err_cnt),
4752 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4753 0, CNTR_NORMAL,
4754 access_sdma_pcie_req_tracking_unc_err_cnt),
4755 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4756 CNTR_NORMAL,
4757 access_sdma_csr_parity_err_cnt),
4758 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4759 CNTR_NORMAL,
4760 access_sdma_rpy_tag_err_cnt),
4761 /* SendEgressErrStatus */
4762 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4763 CNTR_NORMAL,
4764 access_tx_read_pio_memory_csr_unc_err_cnt),
4765 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4766 0, CNTR_NORMAL,
4767 access_tx_read_sdma_memory_csr_err_cnt),
4768 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4769 CNTR_NORMAL,
4770 access_tx_egress_fifo_cor_err_cnt),
4771 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4772 CNTR_NORMAL,
4773 access_tx_read_pio_memory_cor_err_cnt),
4774 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4775 CNTR_NORMAL,
4776 access_tx_read_sdma_memory_cor_err_cnt),
4777 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4778 CNTR_NORMAL,
4779 access_tx_sb_hdr_cor_err_cnt),
4780 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4781 CNTR_NORMAL,
4782 access_tx_credit_overrun_err_cnt),
4783 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4784 CNTR_NORMAL,
4785 access_tx_launch_fifo8_cor_err_cnt),
4786 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4787 CNTR_NORMAL,
4788 access_tx_launch_fifo7_cor_err_cnt),
4789 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4790 CNTR_NORMAL,
4791 access_tx_launch_fifo6_cor_err_cnt),
4792 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4793 CNTR_NORMAL,
4794 access_tx_launch_fifo5_cor_err_cnt),
4795 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4796 CNTR_NORMAL,
4797 access_tx_launch_fifo4_cor_err_cnt),
4798 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4799 CNTR_NORMAL,
4800 access_tx_launch_fifo3_cor_err_cnt),
4801 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4802 CNTR_NORMAL,
4803 access_tx_launch_fifo2_cor_err_cnt),
4804 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4805 CNTR_NORMAL,
4806 access_tx_launch_fifo1_cor_err_cnt),
4807 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4808 CNTR_NORMAL,
4809 access_tx_launch_fifo0_cor_err_cnt),
4810 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4811 CNTR_NORMAL,
4812 access_tx_credit_return_vl_err_cnt),
4813 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4814 CNTR_NORMAL,
4815 access_tx_hcrc_insertion_err_cnt),
4816 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4817 CNTR_NORMAL,
4818 access_tx_egress_fifo_unc_err_cnt),
4819 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4820 CNTR_NORMAL,
4821 access_tx_read_pio_memory_unc_err_cnt),
4822 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4823 CNTR_NORMAL,
4824 access_tx_read_sdma_memory_unc_err_cnt),
4825 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4826 CNTR_NORMAL,
4827 access_tx_sb_hdr_unc_err_cnt),
4828 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4829 CNTR_NORMAL,
4830 access_tx_credit_return_partiy_err_cnt),
4831 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4832 0, 0, CNTR_NORMAL,
4833 access_tx_launch_fifo8_unc_or_parity_err_cnt),
4834 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4835 0, 0, CNTR_NORMAL,
4836 access_tx_launch_fifo7_unc_or_parity_err_cnt),
4837 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4838 0, 0, CNTR_NORMAL,
4839 access_tx_launch_fifo6_unc_or_parity_err_cnt),
4840 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4841 0, 0, CNTR_NORMAL,
4842 access_tx_launch_fifo5_unc_or_parity_err_cnt),
4843 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4844 0, 0, CNTR_NORMAL,
4845 access_tx_launch_fifo4_unc_or_parity_err_cnt),
4846 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4847 0, 0, CNTR_NORMAL,
4848 access_tx_launch_fifo3_unc_or_parity_err_cnt),
4849 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4850 0, 0, CNTR_NORMAL,
4851 access_tx_launch_fifo2_unc_or_parity_err_cnt),
4852 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4853 0, 0, CNTR_NORMAL,
4854 access_tx_launch_fifo1_unc_or_parity_err_cnt),
4855 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4856 0, 0, CNTR_NORMAL,
4857 access_tx_launch_fifo0_unc_or_parity_err_cnt),
4858 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4859 0, 0, CNTR_NORMAL,
4860 access_tx_sdma15_disallowed_packet_err_cnt),
4861 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4862 0, 0, CNTR_NORMAL,
4863 access_tx_sdma14_disallowed_packet_err_cnt),
4864 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4865 0, 0, CNTR_NORMAL,
4866 access_tx_sdma13_disallowed_packet_err_cnt),
4867 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4868 0, 0, CNTR_NORMAL,
4869 access_tx_sdma12_disallowed_packet_err_cnt),
4870 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4871 0, 0, CNTR_NORMAL,
4872 access_tx_sdma11_disallowed_packet_err_cnt),
4873 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4874 0, 0, CNTR_NORMAL,
4875 access_tx_sdma10_disallowed_packet_err_cnt),
4876 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4877 0, 0, CNTR_NORMAL,
4878 access_tx_sdma9_disallowed_packet_err_cnt),
4879 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4880 0, 0, CNTR_NORMAL,
4881 access_tx_sdma8_disallowed_packet_err_cnt),
4882 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4883 0, 0, CNTR_NORMAL,
4884 access_tx_sdma7_disallowed_packet_err_cnt),
4885 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4886 0, 0, CNTR_NORMAL,
4887 access_tx_sdma6_disallowed_packet_err_cnt),
4888 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4889 0, 0, CNTR_NORMAL,
4890 access_tx_sdma5_disallowed_packet_err_cnt),
4891 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4892 0, 0, CNTR_NORMAL,
4893 access_tx_sdma4_disallowed_packet_err_cnt),
4894 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4895 0, 0, CNTR_NORMAL,
4896 access_tx_sdma3_disallowed_packet_err_cnt),
4897 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4898 0, 0, CNTR_NORMAL,
4899 access_tx_sdma2_disallowed_packet_err_cnt),
4900 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4901 0, 0, CNTR_NORMAL,
4902 access_tx_sdma1_disallowed_packet_err_cnt),
4903 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4904 0, 0, CNTR_NORMAL,
4905 access_tx_sdma0_disallowed_packet_err_cnt),
4906 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4907 CNTR_NORMAL,
4908 access_tx_config_parity_err_cnt),
4909 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4910 CNTR_NORMAL,
4911 access_tx_sbrd_ctl_csr_parity_err_cnt),
4912 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4913 CNTR_NORMAL,
4914 access_tx_launch_csr_parity_err_cnt),
4915 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4916 CNTR_NORMAL,
4917 access_tx_illegal_vl_err_cnt),
4918 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4919 "TxSbrdCtlStateMachineParityErr", 0, 0,
4920 CNTR_NORMAL,
4921 access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4922 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4923 CNTR_NORMAL,
4924 access_egress_reserved_10_err_cnt),
4925 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4926 CNTR_NORMAL,
4927 access_egress_reserved_9_err_cnt),
4928 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4929 0, 0, CNTR_NORMAL,
4930 access_tx_sdma_launch_intf_parity_err_cnt),
4931 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4932 CNTR_NORMAL,
4933 access_tx_pio_launch_intf_parity_err_cnt),
4934 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4935 CNTR_NORMAL,
4936 access_egress_reserved_6_err_cnt),
4937 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4938 CNTR_NORMAL,
4939 access_tx_incorrect_link_state_err_cnt),
4940 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4941 CNTR_NORMAL,
4942 access_tx_linkdown_err_cnt),
4943 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4944 "EgressFifoUnderrunOrParityErr", 0, 0,
4945 CNTR_NORMAL,
4946 access_tx_egress_fifi_underrun_or_parity_err_cnt),
4947 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4948 CNTR_NORMAL,
4949 access_egress_reserved_2_err_cnt),
4950 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4951 CNTR_NORMAL,
4952 access_tx_pkt_integrity_mem_unc_err_cnt),
4953 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4954 CNTR_NORMAL,
4955 access_tx_pkt_integrity_mem_cor_err_cnt),
4956 /* SendErrStatus */
4957 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4958 CNTR_NORMAL,
4959 access_send_csr_write_bad_addr_err_cnt),
4960 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4961 CNTR_NORMAL,
4962 access_send_csr_read_bad_addr_err_cnt),
4963 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4964 CNTR_NORMAL,
4965 access_send_csr_parity_cnt),
4966 /* SendCtxtErrStatus */
4967 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4968 CNTR_NORMAL,
4969 access_pio_write_out_of_bounds_err_cnt),
4970 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4971 CNTR_NORMAL,
4972 access_pio_write_overflow_err_cnt),
4973 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4974 0, 0, CNTR_NORMAL,
4975 access_pio_write_crosses_boundary_err_cnt),
4976 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4977 CNTR_NORMAL,
4978 access_pio_disallowed_packet_err_cnt),
4979 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4980 CNTR_NORMAL,
4981 access_pio_inconsistent_sop_err_cnt),
4982 /* SendDmaEngErrStatus */
4983 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4984 0, 0, CNTR_NORMAL,
4985 access_sdma_header_request_fifo_cor_err_cnt),
4986 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4987 CNTR_NORMAL,
4988 access_sdma_header_storage_cor_err_cnt),
4989 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
4990 CNTR_NORMAL,
4991 access_sdma_packet_tracking_cor_err_cnt),
4992 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
4993 CNTR_NORMAL,
4994 access_sdma_assembly_cor_err_cnt),
4995 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
4996 CNTR_NORMAL,
4997 access_sdma_desc_table_cor_err_cnt),
4998 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
4999 0, 0, CNTR_NORMAL,
5000 access_sdma_header_request_fifo_unc_err_cnt),
5001 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
5002 CNTR_NORMAL,
5003 access_sdma_header_storage_unc_err_cnt),
5004 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
5005 CNTR_NORMAL,
5006 access_sdma_packet_tracking_unc_err_cnt),
5007 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
5008 CNTR_NORMAL,
5009 access_sdma_assembly_unc_err_cnt),
5010 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
5011 CNTR_NORMAL,
5012 access_sdma_desc_table_unc_err_cnt),
5013 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
5014 CNTR_NORMAL,
5015 access_sdma_timeout_err_cnt),
5016 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
5017 CNTR_NORMAL,
5018 access_sdma_header_length_err_cnt),
5019 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
5020 CNTR_NORMAL,
5021 access_sdma_header_address_err_cnt),
5022 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
5023 CNTR_NORMAL,
5024 access_sdma_header_select_err_cnt),
5025 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
5026 CNTR_NORMAL,
5027 access_sdma_reserved_9_err_cnt),
5028 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
5029 CNTR_NORMAL,
5030 access_sdma_packet_desc_overflow_err_cnt),
5031 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
5032 CNTR_NORMAL,
5033 access_sdma_length_mismatch_err_cnt),
5034 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
5035 CNTR_NORMAL,
5036 access_sdma_halt_err_cnt),
5037 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
5038 CNTR_NORMAL,
5039 access_sdma_mem_read_err_cnt),
5040 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
5041 CNTR_NORMAL,
5042 access_sdma_first_desc_err_cnt),
5043 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
5044 CNTR_NORMAL,
5045 access_sdma_tail_out_of_bounds_err_cnt),
5046 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
5047 CNTR_NORMAL,
5048 access_sdma_too_long_err_cnt),
5049 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
5050 CNTR_NORMAL,
5051 access_sdma_gen_mismatch_err_cnt),
5052 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
5053 CNTR_NORMAL,
5054 access_sdma_wrong_dw_err_cnt),
5055 };
5056
5057 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
5058 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
5059 CNTR_NORMAL),
5060 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
5061 CNTR_NORMAL),
5062 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
5063 CNTR_NORMAL),
5064 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
5065 CNTR_NORMAL),
5066 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
5067 CNTR_NORMAL),
5068 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
5069 CNTR_NORMAL),
5070 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
5071 CNTR_NORMAL),
5072 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
5073 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
5074 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
5075 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
5076 CNTR_SYNTH | CNTR_VL),
5077 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
5078 CNTR_SYNTH | CNTR_VL),
5079 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
5080 CNTR_SYNTH | CNTR_VL),
5081 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
5082 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
5083 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5084 access_sw_link_dn_cnt),
5085 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5086 access_sw_link_up_cnt),
5087 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
5088 access_sw_unknown_frame_cnt),
5089 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5090 access_sw_xmit_discards),
5091 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
5092 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
5093 access_sw_xmit_discards),
5094 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
5095 access_xmit_constraint_errs),
5096 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
5097 access_rcv_constraint_errs),
5098 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
5099 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
5100 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
5101 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
5102 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
5103 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
5104 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
5105 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
5106 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
5107 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
5108 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
5109 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
5110 [C_SW_IBP_RC_CRWAITS] = SW_IBP_CNTR(RcCrWait, rc_crwaits),
5111 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
5112 access_sw_cpu_rc_acks),
5113 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
5114 access_sw_cpu_rc_qacks),
5115 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
5116 access_sw_cpu_rc_delayed_comp),
5117 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5118 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5119 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5120 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5121 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5122 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5123 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5124 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5125 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5126 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5127 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5128 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5129 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5130 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5131 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5132 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5133 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5134 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5135 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5136 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5137 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5138 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5139 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5140 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5141 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5142 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5143 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5144 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5145 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5146 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5147 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5148 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5149 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5150 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5151 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5152 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5153 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5154 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5155 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5156 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5157 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5158 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5159 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5160 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5161 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5162 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5163 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5164 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5165 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5166 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5167 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5168 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5169 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5170 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5171 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5172 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5173 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5174 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5175 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5176 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5177 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5178 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5179 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5180 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5181 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5182 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5183 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5184 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5185 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5186 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5187 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5188 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5189 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5190 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5191 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5192 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5193 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5194 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5195 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5196 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5197 };
5198
5199 /* ======================================================================== */
5200
5201 /* return true if this is chip revision revision a */
is_ax(struct hfi1_devdata * dd)5202 int is_ax(struct hfi1_devdata *dd)
5203 {
5204 u8 chip_rev_minor =
5205 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5206 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5207 return (chip_rev_minor & 0xf0) == 0;
5208 }
5209
5210 /* return true if this is chip revision revision b */
is_bx(struct hfi1_devdata * dd)5211 int is_bx(struct hfi1_devdata *dd)
5212 {
5213 u8 chip_rev_minor =
5214 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5215 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5216 return (chip_rev_minor & 0xF0) == 0x10;
5217 }
5218
5219 /* return true is kernel urg disabled for rcd */
is_urg_masked(struct hfi1_ctxtdata * rcd)5220 bool is_urg_masked(struct hfi1_ctxtdata *rcd)
5221 {
5222 u64 mask;
5223 u32 is = IS_RCVURGENT_START + rcd->ctxt;
5224 u8 bit = is % 64;
5225
5226 mask = read_csr(rcd->dd, CCE_INT_MASK + (8 * (is / 64)));
5227 return !(mask & BIT_ULL(bit));
5228 }
5229
5230 /*
5231 * Append string s to buffer buf. Arguments curp and len are the current
5232 * position and remaining length, respectively.
5233 *
5234 * return 0 on success, 1 on out of room
5235 */
append_str(char * buf,char ** curp,int * lenp,const char * s)5236 static int append_str(char *buf, char **curp, int *lenp, const char *s)
5237 {
5238 char *p = *curp;
5239 int len = *lenp;
5240 int result = 0; /* success */
5241 char c;
5242
5243 /* add a comma, if first in the buffer */
5244 if (p != buf) {
5245 if (len == 0) {
5246 result = 1; /* out of room */
5247 goto done;
5248 }
5249 *p++ = ',';
5250 len--;
5251 }
5252
5253 /* copy the string */
5254 while ((c = *s++) != 0) {
5255 if (len == 0) {
5256 result = 1; /* out of room */
5257 goto done;
5258 }
5259 *p++ = c;
5260 len--;
5261 }
5262
5263 done:
5264 /* write return values */
5265 *curp = p;
5266 *lenp = len;
5267
5268 return result;
5269 }
5270
5271 /*
5272 * Using the given flag table, print a comma separated string into
5273 * the buffer. End in '*' if the buffer is too short.
5274 */
flag_string(char * buf,int buf_len,u64 flags,struct flag_table * table,int table_size)5275 static char *flag_string(char *buf, int buf_len, u64 flags,
5276 struct flag_table *table, int table_size)
5277 {
5278 char extra[32];
5279 char *p = buf;
5280 int len = buf_len;
5281 int no_room = 0;
5282 int i;
5283
5284 /* make sure there is at least 2 so we can form "*" */
5285 if (len < 2)
5286 return "";
5287
5288 len--; /* leave room for a nul */
5289 for (i = 0; i < table_size; i++) {
5290 if (flags & table[i].flag) {
5291 no_room = append_str(buf, &p, &len, table[i].str);
5292 if (no_room)
5293 break;
5294 flags &= ~table[i].flag;
5295 }
5296 }
5297
5298 /* any undocumented bits left? */
5299 if (!no_room && flags) {
5300 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5301 no_room = append_str(buf, &p, &len, extra);
5302 }
5303
5304 /* add * if ran out of room */
5305 if (no_room) {
5306 /* may need to back up to add space for a '*' */
5307 if (len == 0)
5308 --p;
5309 *p++ = '*';
5310 }
5311
5312 /* add final nul - space already allocated above */
5313 *p = 0;
5314 return buf;
5315 }
5316
5317 /* first 8 CCE error interrupt source names */
5318 static const char * const cce_misc_names[] = {
5319 "CceErrInt", /* 0 */
5320 "RxeErrInt", /* 1 */
5321 "MiscErrInt", /* 2 */
5322 "Reserved3", /* 3 */
5323 "PioErrInt", /* 4 */
5324 "SDmaErrInt", /* 5 */
5325 "EgressErrInt", /* 6 */
5326 "TxeErrInt" /* 7 */
5327 };
5328
5329 /*
5330 * Return the miscellaneous error interrupt name.
5331 */
is_misc_err_name(char * buf,size_t bsize,unsigned int source)5332 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5333 {
5334 if (source < ARRAY_SIZE(cce_misc_names))
5335 strncpy(buf, cce_misc_names[source], bsize);
5336 else
5337 snprintf(buf, bsize, "Reserved%u",
5338 source + IS_GENERAL_ERR_START);
5339
5340 return buf;
5341 }
5342
5343 /*
5344 * Return the SDMA engine error interrupt name.
5345 */
is_sdma_eng_err_name(char * buf,size_t bsize,unsigned int source)5346 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5347 {
5348 snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5349 return buf;
5350 }
5351
5352 /*
5353 * Return the send context error interrupt name.
5354 */
is_sendctxt_err_name(char * buf,size_t bsize,unsigned int source)5355 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5356 {
5357 snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5358 return buf;
5359 }
5360
5361 static const char * const various_names[] = {
5362 "PbcInt",
5363 "GpioAssertInt",
5364 "Qsfp1Int",
5365 "Qsfp2Int",
5366 "TCritInt"
5367 };
5368
5369 /*
5370 * Return the various interrupt name.
5371 */
is_various_name(char * buf,size_t bsize,unsigned int source)5372 static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5373 {
5374 if (source < ARRAY_SIZE(various_names))
5375 strncpy(buf, various_names[source], bsize);
5376 else
5377 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
5378 return buf;
5379 }
5380
5381 /*
5382 * Return the DC interrupt name.
5383 */
is_dc_name(char * buf,size_t bsize,unsigned int source)5384 static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5385 {
5386 static const char * const dc_int_names[] = {
5387 "common",
5388 "lcb",
5389 "8051",
5390 "lbm" /* local block merge */
5391 };
5392
5393 if (source < ARRAY_SIZE(dc_int_names))
5394 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5395 else
5396 snprintf(buf, bsize, "DCInt%u", source);
5397 return buf;
5398 }
5399
5400 static const char * const sdma_int_names[] = {
5401 "SDmaInt",
5402 "SdmaIdleInt",
5403 "SdmaProgressInt",
5404 };
5405
5406 /*
5407 * Return the SDMA engine interrupt name.
5408 */
is_sdma_eng_name(char * buf,size_t bsize,unsigned int source)5409 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5410 {
5411 /* what interrupt */
5412 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
5413 /* which engine */
5414 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5415
5416 if (likely(what < 3))
5417 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5418 else
5419 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5420 return buf;
5421 }
5422
5423 /*
5424 * Return the receive available interrupt name.
5425 */
is_rcv_avail_name(char * buf,size_t bsize,unsigned int source)5426 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5427 {
5428 snprintf(buf, bsize, "RcvAvailInt%u", source);
5429 return buf;
5430 }
5431
5432 /*
5433 * Return the receive urgent interrupt name.
5434 */
is_rcv_urgent_name(char * buf,size_t bsize,unsigned int source)5435 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5436 {
5437 snprintf(buf, bsize, "RcvUrgentInt%u", source);
5438 return buf;
5439 }
5440
5441 /*
5442 * Return the send credit interrupt name.
5443 */
is_send_credit_name(char * buf,size_t bsize,unsigned int source)5444 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5445 {
5446 snprintf(buf, bsize, "SendCreditInt%u", source);
5447 return buf;
5448 }
5449
5450 /*
5451 * Return the reserved interrupt name.
5452 */
is_reserved_name(char * buf,size_t bsize,unsigned int source)5453 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5454 {
5455 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5456 return buf;
5457 }
5458
cce_err_status_string(char * buf,int buf_len,u64 flags)5459 static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5460 {
5461 return flag_string(buf, buf_len, flags,
5462 cce_err_status_flags,
5463 ARRAY_SIZE(cce_err_status_flags));
5464 }
5465
rxe_err_status_string(char * buf,int buf_len,u64 flags)5466 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5467 {
5468 return flag_string(buf, buf_len, flags,
5469 rxe_err_status_flags,
5470 ARRAY_SIZE(rxe_err_status_flags));
5471 }
5472
misc_err_status_string(char * buf,int buf_len,u64 flags)5473 static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5474 {
5475 return flag_string(buf, buf_len, flags, misc_err_status_flags,
5476 ARRAY_SIZE(misc_err_status_flags));
5477 }
5478
pio_err_status_string(char * buf,int buf_len,u64 flags)5479 static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5480 {
5481 return flag_string(buf, buf_len, flags,
5482 pio_err_status_flags,
5483 ARRAY_SIZE(pio_err_status_flags));
5484 }
5485
sdma_err_status_string(char * buf,int buf_len,u64 flags)5486 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5487 {
5488 return flag_string(buf, buf_len, flags,
5489 sdma_err_status_flags,
5490 ARRAY_SIZE(sdma_err_status_flags));
5491 }
5492
egress_err_status_string(char * buf,int buf_len,u64 flags)5493 static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5494 {
5495 return flag_string(buf, buf_len, flags,
5496 egress_err_status_flags,
5497 ARRAY_SIZE(egress_err_status_flags));
5498 }
5499
egress_err_info_string(char * buf,int buf_len,u64 flags)5500 static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5501 {
5502 return flag_string(buf, buf_len, flags,
5503 egress_err_info_flags,
5504 ARRAY_SIZE(egress_err_info_flags));
5505 }
5506
send_err_status_string(char * buf,int buf_len,u64 flags)5507 static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5508 {
5509 return flag_string(buf, buf_len, flags,
5510 send_err_status_flags,
5511 ARRAY_SIZE(send_err_status_flags));
5512 }
5513
handle_cce_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5514 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5515 {
5516 char buf[96];
5517 int i = 0;
5518
5519 /*
5520 * For most these errors, there is nothing that can be done except
5521 * report or record it.
5522 */
5523 dd_dev_info(dd, "CCE Error: %s\n",
5524 cce_err_status_string(buf, sizeof(buf), reg));
5525
5526 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5527 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
5528 /* this error requires a manual drop into SPC freeze mode */
5529 /* then a fix up */
5530 start_freeze_handling(dd->pport, FREEZE_SELF);
5531 }
5532
5533 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5534 if (reg & (1ull << i)) {
5535 incr_cntr64(&dd->cce_err_status_cnt[i]);
5536 /* maintain a counter over all cce_err_status errors */
5537 incr_cntr64(&dd->sw_cce_err_status_aggregate);
5538 }
5539 }
5540 }
5541
5542 /*
5543 * Check counters for receive errors that do not have an interrupt
5544 * associated with them.
5545 */
5546 #define RCVERR_CHECK_TIME 10
update_rcverr_timer(struct timer_list * t)5547 static void update_rcverr_timer(struct timer_list *t)
5548 {
5549 struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer);
5550 struct hfi1_pportdata *ppd = dd->pport;
5551 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5552
5553 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
5554 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
5555 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
5556 set_link_down_reason(
5557 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5558 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
5559 queue_work(ppd->link_wq, &ppd->link_bounce_work);
5560 }
5561 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
5562
5563 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5564 }
5565
init_rcverr(struct hfi1_devdata * dd)5566 static int init_rcverr(struct hfi1_devdata *dd)
5567 {
5568 timer_setup(&dd->rcverr_timer, update_rcverr_timer, 0);
5569 /* Assume the hardware counter has been reset */
5570 dd->rcv_ovfl_cnt = 0;
5571 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5572 }
5573
free_rcverr(struct hfi1_devdata * dd)5574 static void free_rcverr(struct hfi1_devdata *dd)
5575 {
5576 if (dd->rcverr_timer.function)
5577 del_timer_sync(&dd->rcverr_timer);
5578 }
5579
handle_rxe_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5580 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5581 {
5582 char buf[96];
5583 int i = 0;
5584
5585 dd_dev_info(dd, "Receive Error: %s\n",
5586 rxe_err_status_string(buf, sizeof(buf), reg));
5587
5588 if (reg & ALL_RXE_FREEZE_ERR) {
5589 int flags = 0;
5590
5591 /*
5592 * Freeze mode recovery is disabled for the errors
5593 * in RXE_FREEZE_ABORT_MASK
5594 */
5595 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
5596 flags = FREEZE_ABORT;
5597
5598 start_freeze_handling(dd->pport, flags);
5599 }
5600
5601 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5602 if (reg & (1ull << i))
5603 incr_cntr64(&dd->rcv_err_status_cnt[i]);
5604 }
5605 }
5606
handle_misc_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5607 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5608 {
5609 char buf[96];
5610 int i = 0;
5611
5612 dd_dev_info(dd, "Misc Error: %s",
5613 misc_err_status_string(buf, sizeof(buf), reg));
5614 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5615 if (reg & (1ull << i))
5616 incr_cntr64(&dd->misc_err_status_cnt[i]);
5617 }
5618 }
5619
handle_pio_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5620 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5621 {
5622 char buf[96];
5623 int i = 0;
5624
5625 dd_dev_info(dd, "PIO Error: %s\n",
5626 pio_err_status_string(buf, sizeof(buf), reg));
5627
5628 if (reg & ALL_PIO_FREEZE_ERR)
5629 start_freeze_handling(dd->pport, 0);
5630
5631 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5632 if (reg & (1ull << i))
5633 incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5634 }
5635 }
5636
handle_sdma_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5637 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5638 {
5639 char buf[96];
5640 int i = 0;
5641
5642 dd_dev_info(dd, "SDMA Error: %s\n",
5643 sdma_err_status_string(buf, sizeof(buf), reg));
5644
5645 if (reg & ALL_SDMA_FREEZE_ERR)
5646 start_freeze_handling(dd->pport, 0);
5647
5648 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5649 if (reg & (1ull << i))
5650 incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5651 }
5652 }
5653
__count_port_discards(struct hfi1_pportdata * ppd)5654 static inline void __count_port_discards(struct hfi1_pportdata *ppd)
5655 {
5656 incr_cntr64(&ppd->port_xmit_discards);
5657 }
5658
count_port_inactive(struct hfi1_devdata * dd)5659 static void count_port_inactive(struct hfi1_devdata *dd)
5660 {
5661 __count_port_discards(dd->pport);
5662 }
5663
5664 /*
5665 * We have had a "disallowed packet" error during egress. Determine the
5666 * integrity check which failed, and update relevant error counter, etc.
5667 *
5668 * Note that the SEND_EGRESS_ERR_INFO register has only a single
5669 * bit of state per integrity check, and so we can miss the reason for an
5670 * egress error if more than one packet fails the same integrity check
5671 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
5672 */
handle_send_egress_err_info(struct hfi1_devdata * dd,int vl)5673 static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5674 int vl)
5675 {
5676 struct hfi1_pportdata *ppd = dd->pport;
5677 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
5678 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5679 char buf[96];
5680
5681 /* clear down all observed info as quickly as possible after read */
5682 write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5683
5684 dd_dev_info(dd,
5685 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5686 info, egress_err_info_string(buf, sizeof(buf), info), src);
5687
5688 /* Eventually add other counters for each bit */
5689 if (info & PORT_DISCARD_EGRESS_ERRS) {
5690 int weight, i;
5691
5692 /*
5693 * Count all applicable bits as individual errors and
5694 * attribute them to the packet that triggered this handler.
5695 * This may not be completely accurate due to limitations
5696 * on the available hardware error information. There is
5697 * a single information register and any number of error
5698 * packets may have occurred and contributed to it before
5699 * this routine is called. This means that:
5700 * a) If multiple packets with the same error occur before
5701 * this routine is called, earlier packets are missed.
5702 * There is only a single bit for each error type.
5703 * b) Errors may not be attributed to the correct VL.
5704 * The driver is attributing all bits in the info register
5705 * to the packet that triggered this call, but bits
5706 * could be an accumulation of different packets with
5707 * different VLs.
5708 * c) A single error packet may have multiple counts attached
5709 * to it. There is no way for the driver to know if
5710 * multiple bits set in the info register are due to a
5711 * single packet or multiple packets. The driver assumes
5712 * multiple packets.
5713 */
5714 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS);
5715 for (i = 0; i < weight; i++) {
5716 __count_port_discards(ppd);
5717 if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5718 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5719 else if (vl == 15)
5720 incr_cntr64(&ppd->port_xmit_discards_vl
5721 [C_VL_15]);
5722 }
5723 }
5724 }
5725
5726 /*
5727 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5728 * register. Does it represent a 'port inactive' error?
5729 */
port_inactive_err(u64 posn)5730 static inline int port_inactive_err(u64 posn)
5731 {
5732 return (posn >= SEES(TX_LINKDOWN) &&
5733 posn <= SEES(TX_INCORRECT_LINK_STATE));
5734 }
5735
5736 /*
5737 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5738 * register. Does it represent a 'disallowed packet' error?
5739 */
disallowed_pkt_err(int posn)5740 static inline int disallowed_pkt_err(int posn)
5741 {
5742 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5743 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5744 }
5745
5746 /*
5747 * Input value is a bit position of one of the SDMA engine disallowed
5748 * packet errors. Return which engine. Use of this must be guarded by
5749 * disallowed_pkt_err().
5750 */
disallowed_pkt_engine(int posn)5751 static inline int disallowed_pkt_engine(int posn)
5752 {
5753 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5754 }
5755
5756 /*
5757 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot
5758 * be done.
5759 */
engine_to_vl(struct hfi1_devdata * dd,int engine)5760 static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5761 {
5762 struct sdma_vl_map *m;
5763 int vl;
5764
5765 /* range check */
5766 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5767 return -1;
5768
5769 rcu_read_lock();
5770 m = rcu_dereference(dd->sdma_map);
5771 vl = m->engine_to_vl[engine];
5772 rcu_read_unlock();
5773
5774 return vl;
5775 }
5776
5777 /*
5778 * Translate the send context (sofware index) into a VL. Return -1 if the
5779 * translation cannot be done.
5780 */
sc_to_vl(struct hfi1_devdata * dd,int sw_index)5781 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5782 {
5783 struct send_context_info *sci;
5784 struct send_context *sc;
5785 int i;
5786
5787 sci = &dd->send_contexts[sw_index];
5788
5789 /* there is no information for user (PSM) and ack contexts */
5790 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15))
5791 return -1;
5792
5793 sc = sci->sc;
5794 if (!sc)
5795 return -1;
5796 if (dd->vld[15].sc == sc)
5797 return 15;
5798 for (i = 0; i < num_vls; i++)
5799 if (dd->vld[i].sc == sc)
5800 return i;
5801
5802 return -1;
5803 }
5804
handle_egress_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5805 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5806 {
5807 u64 reg_copy = reg, handled = 0;
5808 char buf[96];
5809 int i = 0;
5810
5811 if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5812 start_freeze_handling(dd->pport, 0);
5813 else if (is_ax(dd) &&
5814 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5815 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
5816 start_freeze_handling(dd->pport, 0);
5817
5818 while (reg_copy) {
5819 int posn = fls64(reg_copy);
5820 /* fls64() returns a 1-based offset, we want it zero based */
5821 int shift = posn - 1;
5822 u64 mask = 1ULL << shift;
5823
5824 if (port_inactive_err(shift)) {
5825 count_port_inactive(dd);
5826 handled |= mask;
5827 } else if (disallowed_pkt_err(shift)) {
5828 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5829
5830 handle_send_egress_err_info(dd, vl);
5831 handled |= mask;
5832 }
5833 reg_copy &= ~mask;
5834 }
5835
5836 reg &= ~handled;
5837
5838 if (reg)
5839 dd_dev_info(dd, "Egress Error: %s\n",
5840 egress_err_status_string(buf, sizeof(buf), reg));
5841
5842 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5843 if (reg & (1ull << i))
5844 incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5845 }
5846 }
5847
handle_txe_err(struct hfi1_devdata * dd,u32 unused,u64 reg)5848 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5849 {
5850 char buf[96];
5851 int i = 0;
5852
5853 dd_dev_info(dd, "Send Error: %s\n",
5854 send_err_status_string(buf, sizeof(buf), reg));
5855
5856 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5857 if (reg & (1ull << i))
5858 incr_cntr64(&dd->send_err_status_cnt[i]);
5859 }
5860 }
5861
5862 /*
5863 * The maximum number of times the error clear down will loop before
5864 * blocking a repeating error. This value is arbitrary.
5865 */
5866 #define MAX_CLEAR_COUNT 20
5867
5868 /*
5869 * Clear and handle an error register. All error interrupts are funneled
5870 * through here to have a central location to correctly handle single-
5871 * or multi-shot errors.
5872 *
5873 * For non per-context registers, call this routine with a context value
5874 * of 0 so the per-context offset is zero.
5875 *
5876 * If the handler loops too many times, assume that something is wrong
5877 * and can't be fixed, so mask the error bits.
5878 */
interrupt_clear_down(struct hfi1_devdata * dd,u32 context,const struct err_reg_info * eri)5879 static void interrupt_clear_down(struct hfi1_devdata *dd,
5880 u32 context,
5881 const struct err_reg_info *eri)
5882 {
5883 u64 reg;
5884 u32 count;
5885
5886 /* read in a loop until no more errors are seen */
5887 count = 0;
5888 while (1) {
5889 reg = read_kctxt_csr(dd, context, eri->status);
5890 if (reg == 0)
5891 break;
5892 write_kctxt_csr(dd, context, eri->clear, reg);
5893 if (likely(eri->handler))
5894 eri->handler(dd, context, reg);
5895 count++;
5896 if (count > MAX_CLEAR_COUNT) {
5897 u64 mask;
5898
5899 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
5900 eri->desc, reg);
5901 /*
5902 * Read-modify-write so any other masked bits
5903 * remain masked.
5904 */
5905 mask = read_kctxt_csr(dd, context, eri->mask);
5906 mask &= ~reg;
5907 write_kctxt_csr(dd, context, eri->mask, mask);
5908 break;
5909 }
5910 }
5911 }
5912
5913 /*
5914 * CCE block "misc" interrupt. Source is < 16.
5915 */
is_misc_err_int(struct hfi1_devdata * dd,unsigned int source)5916 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5917 {
5918 const struct err_reg_info *eri = &misc_errs[source];
5919
5920 if (eri->handler) {
5921 interrupt_clear_down(dd, 0, eri);
5922 } else {
5923 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
5924 source);
5925 }
5926 }
5927
send_context_err_status_string(char * buf,int buf_len,u64 flags)5928 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5929 {
5930 return flag_string(buf, buf_len, flags,
5931 sc_err_status_flags,
5932 ARRAY_SIZE(sc_err_status_flags));
5933 }
5934
5935 /*
5936 * Send context error interrupt. Source (hw_context) is < 160.
5937 *
5938 * All send context errors cause the send context to halt. The normal
5939 * clear-down mechanism cannot be used because we cannot clear the
5940 * error bits until several other long-running items are done first.
5941 * This is OK because with the context halted, nothing else is going
5942 * to happen on it anyway.
5943 */
is_sendctxt_err_int(struct hfi1_devdata * dd,unsigned int hw_context)5944 static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5945 unsigned int hw_context)
5946 {
5947 struct send_context_info *sci;
5948 struct send_context *sc;
5949 char flags[96];
5950 u64 status;
5951 u32 sw_index;
5952 int i = 0;
5953 unsigned long irq_flags;
5954
5955 sw_index = dd->hw_to_sw[hw_context];
5956 if (sw_index >= dd->num_send_contexts) {
5957 dd_dev_err(dd,
5958 "out of range sw index %u for send context %u\n",
5959 sw_index, hw_context);
5960 return;
5961 }
5962 sci = &dd->send_contexts[sw_index];
5963 spin_lock_irqsave(&dd->sc_lock, irq_flags);
5964 sc = sci->sc;
5965 if (!sc) {
5966 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
5967 sw_index, hw_context);
5968 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5969 return;
5970 }
5971
5972 /* tell the software that a halt has begun */
5973 sc_stop(sc, SCF_HALTED);
5974
5975 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5976
5977 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
5978 send_context_err_status_string(flags, sizeof(flags),
5979 status));
5980
5981 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
5982 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
5983
5984 /*
5985 * Automatically restart halted kernel contexts out of interrupt
5986 * context. User contexts must ask the driver to restart the context.
5987 */
5988 if (sc->type != SC_USER)
5989 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
5990 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5991
5992 /*
5993 * Update the counters for the corresponding status bits.
5994 * Note that these particular counters are aggregated over all
5995 * 160 contexts.
5996 */
5997 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
5998 if (status & (1ull << i))
5999 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
6000 }
6001 }
6002
handle_sdma_eng_err(struct hfi1_devdata * dd,unsigned int source,u64 status)6003 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
6004 unsigned int source, u64 status)
6005 {
6006 struct sdma_engine *sde;
6007 int i = 0;
6008
6009 sde = &dd->per_sdma[source];
6010 #ifdef CONFIG_SDMA_VERBOSITY
6011 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6012 slashstrip(__FILE__), __LINE__, __func__);
6013 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
6014 sde->this_idx, source, (unsigned long long)status);
6015 #endif
6016 sde->err_cnt++;
6017 sdma_engine_error(sde, status);
6018
6019 /*
6020 * Update the counters for the corresponding status bits.
6021 * Note that these particular counters are aggregated over
6022 * all 16 DMA engines.
6023 */
6024 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
6025 if (status & (1ull << i))
6026 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
6027 }
6028 }
6029
6030 /*
6031 * CCE block SDMA error interrupt. Source is < 16.
6032 */
is_sdma_eng_err_int(struct hfi1_devdata * dd,unsigned int source)6033 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
6034 {
6035 #ifdef CONFIG_SDMA_VERBOSITY
6036 struct sdma_engine *sde = &dd->per_sdma[source];
6037
6038 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6039 slashstrip(__FILE__), __LINE__, __func__);
6040 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
6041 source);
6042 sdma_dumpstate(sde);
6043 #endif
6044 interrupt_clear_down(dd, source, &sdma_eng_err);
6045 }
6046
6047 /*
6048 * CCE block "various" interrupt. Source is < 8.
6049 */
is_various_int(struct hfi1_devdata * dd,unsigned int source)6050 static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
6051 {
6052 const struct err_reg_info *eri = &various_err[source];
6053
6054 /*
6055 * TCritInt cannot go through interrupt_clear_down()
6056 * because it is not a second tier interrupt. The handler
6057 * should be called directly.
6058 */
6059 if (source == TCRIT_INT_SOURCE)
6060 handle_temp_err(dd);
6061 else if (eri->handler)
6062 interrupt_clear_down(dd, 0, eri);
6063 else
6064 dd_dev_info(dd,
6065 "%s: Unimplemented/reserved interrupt %d\n",
6066 __func__, source);
6067 }
6068
handle_qsfp_int(struct hfi1_devdata * dd,u32 src_ctx,u64 reg)6069 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
6070 {
6071 /* src_ctx is always zero */
6072 struct hfi1_pportdata *ppd = dd->pport;
6073 unsigned long flags;
6074 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
6075
6076 if (reg & QSFP_HFI0_MODPRST_N) {
6077 if (!qsfp_mod_present(ppd)) {
6078 dd_dev_info(dd, "%s: QSFP module removed\n",
6079 __func__);
6080
6081 ppd->driver_link_ready = 0;
6082 /*
6083 * Cable removed, reset all our information about the
6084 * cache and cable capabilities
6085 */
6086
6087 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6088 /*
6089 * We don't set cache_refresh_required here as we expect
6090 * an interrupt when a cable is inserted
6091 */
6092 ppd->qsfp_info.cache_valid = 0;
6093 ppd->qsfp_info.reset_needed = 0;
6094 ppd->qsfp_info.limiting_active = 0;
6095 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6096 flags);
6097 /* Invert the ModPresent pin now to detect plug-in */
6098 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6099 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6100
6101 if ((ppd->offline_disabled_reason >
6102 HFI1_ODR_MASK(
6103 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
6104 (ppd->offline_disabled_reason ==
6105 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
6106 ppd->offline_disabled_reason =
6107 HFI1_ODR_MASK(
6108 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
6109
6110 if (ppd->host_link_state == HLS_DN_POLL) {
6111 /*
6112 * The link is still in POLL. This means
6113 * that the normal link down processing
6114 * will not happen. We have to do it here
6115 * before turning the DC off.
6116 */
6117 queue_work(ppd->link_wq, &ppd->link_down_work);
6118 }
6119 } else {
6120 dd_dev_info(dd, "%s: QSFP module inserted\n",
6121 __func__);
6122
6123 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6124 ppd->qsfp_info.cache_valid = 0;
6125 ppd->qsfp_info.cache_refresh_required = 1;
6126 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6127 flags);
6128
6129 /*
6130 * Stop inversion of ModPresent pin to detect
6131 * removal of the cable
6132 */
6133 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
6134 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6135 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6136
6137 ppd->offline_disabled_reason =
6138 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
6139 }
6140 }
6141
6142 if (reg & QSFP_HFI0_INT_N) {
6143 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n",
6144 __func__);
6145 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6146 ppd->qsfp_info.check_interrupt_flags = 1;
6147 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6148 }
6149
6150 /* Schedule the QSFP work only if there is a cable attached. */
6151 if (qsfp_mod_present(ppd))
6152 queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work);
6153 }
6154
request_host_lcb_access(struct hfi1_devdata * dd)6155 static int request_host_lcb_access(struct hfi1_devdata *dd)
6156 {
6157 int ret;
6158
6159 ret = do_8051_command(dd, HCMD_MISC,
6160 (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6161 LOAD_DATA_FIELD_ID_SHIFT, NULL);
6162 if (ret != HCMD_SUCCESS) {
6163 dd_dev_err(dd, "%s: command failed with error %d\n",
6164 __func__, ret);
6165 }
6166 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6167 }
6168
request_8051_lcb_access(struct hfi1_devdata * dd)6169 static int request_8051_lcb_access(struct hfi1_devdata *dd)
6170 {
6171 int ret;
6172
6173 ret = do_8051_command(dd, HCMD_MISC,
6174 (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6175 LOAD_DATA_FIELD_ID_SHIFT, NULL);
6176 if (ret != HCMD_SUCCESS) {
6177 dd_dev_err(dd, "%s: command failed with error %d\n",
6178 __func__, ret);
6179 }
6180 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6181 }
6182
6183 /*
6184 * Set the LCB selector - allow host access. The DCC selector always
6185 * points to the host.
6186 */
set_host_lcb_access(struct hfi1_devdata * dd)6187 static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6188 {
6189 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6190 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6191 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
6192 }
6193
6194 /*
6195 * Clear the LCB selector - allow 8051 access. The DCC selector always
6196 * points to the host.
6197 */
set_8051_lcb_access(struct hfi1_devdata * dd)6198 static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6199 {
6200 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6201 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
6202 }
6203
6204 /*
6205 * Acquire LCB access from the 8051. If the host already has access,
6206 * just increment a counter. Otherwise, inform the 8051 that the
6207 * host is taking access.
6208 *
6209 * Returns:
6210 * 0 on success
6211 * -EBUSY if the 8051 has control and cannot be disturbed
6212 * -errno if unable to acquire access from the 8051
6213 */
acquire_lcb_access(struct hfi1_devdata * dd,int sleep_ok)6214 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6215 {
6216 struct hfi1_pportdata *ppd = dd->pport;
6217 int ret = 0;
6218
6219 /*
6220 * Use the host link state lock so the operation of this routine
6221 * { link state check, selector change, count increment } can occur
6222 * as a unit against a link state change. Otherwise there is a
6223 * race between the state change and the count increment.
6224 */
6225 if (sleep_ok) {
6226 mutex_lock(&ppd->hls_lock);
6227 } else {
6228 while (!mutex_trylock(&ppd->hls_lock))
6229 udelay(1);
6230 }
6231
6232 /* this access is valid only when the link is up */
6233 if (ppd->host_link_state & HLS_DOWN) {
6234 dd_dev_info(dd, "%s: link state %s not up\n",
6235 __func__, link_state_name(ppd->host_link_state));
6236 ret = -EBUSY;
6237 goto done;
6238 }
6239
6240 if (dd->lcb_access_count == 0) {
6241 ret = request_host_lcb_access(dd);
6242 if (ret) {
6243 dd_dev_err(dd,
6244 "%s: unable to acquire LCB access, err %d\n",
6245 __func__, ret);
6246 goto done;
6247 }
6248 set_host_lcb_access(dd);
6249 }
6250 dd->lcb_access_count++;
6251 done:
6252 mutex_unlock(&ppd->hls_lock);
6253 return ret;
6254 }
6255
6256 /*
6257 * Release LCB access by decrementing the use count. If the count is moving
6258 * from 1 to 0, inform 8051 that it has control back.
6259 *
6260 * Returns:
6261 * 0 on success
6262 * -errno if unable to release access to the 8051
6263 */
release_lcb_access(struct hfi1_devdata * dd,int sleep_ok)6264 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6265 {
6266 int ret = 0;
6267
6268 /*
6269 * Use the host link state lock because the acquire needed it.
6270 * Here, we only need to keep { selector change, count decrement }
6271 * as a unit.
6272 */
6273 if (sleep_ok) {
6274 mutex_lock(&dd->pport->hls_lock);
6275 } else {
6276 while (!mutex_trylock(&dd->pport->hls_lock))
6277 udelay(1);
6278 }
6279
6280 if (dd->lcb_access_count == 0) {
6281 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n",
6282 __func__);
6283 goto done;
6284 }
6285
6286 if (dd->lcb_access_count == 1) {
6287 set_8051_lcb_access(dd);
6288 ret = request_8051_lcb_access(dd);
6289 if (ret) {
6290 dd_dev_err(dd,
6291 "%s: unable to release LCB access, err %d\n",
6292 __func__, ret);
6293 /* restore host access if the grant didn't work */
6294 set_host_lcb_access(dd);
6295 goto done;
6296 }
6297 }
6298 dd->lcb_access_count--;
6299 done:
6300 mutex_unlock(&dd->pport->hls_lock);
6301 return ret;
6302 }
6303
6304 /*
6305 * Initialize LCB access variables and state. Called during driver load,
6306 * after most of the initialization is finished.
6307 *
6308 * The DC default is LCB access on for the host. The driver defaults to
6309 * leaving access to the 8051. Assign access now - this constrains the call
6310 * to this routine to be after all LCB set-up is done. In particular, after
6311 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
6312 */
init_lcb_access(struct hfi1_devdata * dd)6313 static void init_lcb_access(struct hfi1_devdata *dd)
6314 {
6315 dd->lcb_access_count = 0;
6316 }
6317
6318 /*
6319 * Write a response back to a 8051 request.
6320 */
hreq_response(struct hfi1_devdata * dd,u8 return_code,u16 rsp_data)6321 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6322 {
6323 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
6324 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6325 (u64)return_code <<
6326 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6327 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
6328 }
6329
6330 /*
6331 * Handle host requests from the 8051.
6332 */
handle_8051_request(struct hfi1_pportdata * ppd)6333 static void handle_8051_request(struct hfi1_pportdata *ppd)
6334 {
6335 struct hfi1_devdata *dd = ppd->dd;
6336 u64 reg;
6337 u16 data = 0;
6338 u8 type;
6339
6340 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6341 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6342 return; /* no request */
6343
6344 /* zero out COMPLETED so the response is seen */
6345 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6346
6347 /* extract request details */
6348 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6349 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6350 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6351 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6352
6353 switch (type) {
6354 case HREQ_LOAD_CONFIG:
6355 case HREQ_SAVE_CONFIG:
6356 case HREQ_READ_CONFIG:
6357 case HREQ_SET_TX_EQ_ABS:
6358 case HREQ_SET_TX_EQ_REL:
6359 case HREQ_ENABLE:
6360 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
6361 type);
6362 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6363 break;
6364 case HREQ_LCB_RESET:
6365 /* Put the LCB, RX FPE and TX FPE into reset */
6366 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_INTO_RESET);
6367 /* Make sure the write completed */
6368 (void)read_csr(dd, DCC_CFG_RESET);
6369 /* Hold the reset long enough to take effect */
6370 udelay(1);
6371 /* Take the LCB, RX FPE and TX FPE out of reset */
6372 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6373 hreq_response(dd, HREQ_SUCCESS, 0);
6374
6375 break;
6376 case HREQ_CONFIG_DONE:
6377 hreq_response(dd, HREQ_SUCCESS, 0);
6378 break;
6379
6380 case HREQ_INTERFACE_TEST:
6381 hreq_response(dd, HREQ_SUCCESS, data);
6382 break;
6383 default:
6384 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6385 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6386 break;
6387 }
6388 }
6389
6390 /*
6391 * Set up allocation unit vaulue.
6392 */
set_up_vau(struct hfi1_devdata * dd,u8 vau)6393 void set_up_vau(struct hfi1_devdata *dd, u8 vau)
6394 {
6395 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6396
6397 /* do not modify other values in the register */
6398 reg &= ~SEND_CM_GLOBAL_CREDIT_AU_SMASK;
6399 reg |= (u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT;
6400 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6401 }
6402
6403 /*
6404 * Set up initial VL15 credits of the remote. Assumes the rest of
6405 * the CM credit registers are zero from a previous global or credit reset.
6406 * Shared limit for VL15 will always be 0.
6407 */
set_up_vl15(struct hfi1_devdata * dd,u16 vl15buf)6408 void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf)
6409 {
6410 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6411
6412 /* set initial values for total and shared credit limit */
6413 reg &= ~(SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK |
6414 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK);
6415
6416 /*
6417 * Set total limit to be equal to VL15 credits.
6418 * Leave shared limit at 0.
6419 */
6420 reg |= (u64)vl15buf << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
6421 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6422
6423 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6424 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6425 }
6426
6427 /*
6428 * Zero all credit details from the previous connection and
6429 * reset the CM manager's internal counters.
6430 */
reset_link_credits(struct hfi1_devdata * dd)6431 void reset_link_credits(struct hfi1_devdata *dd)
6432 {
6433 int i;
6434
6435 /* remove all previous VL credit limits */
6436 for (i = 0; i < TXE_NUM_DATA_VL; i++)
6437 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
6438 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6439 write_csr(dd, SEND_CM_GLOBAL_CREDIT, 0);
6440 /* reset the CM block */
6441 pio_send_control(dd, PSC_CM_RESET);
6442 /* reset cached value */
6443 dd->vl15buf_cached = 0;
6444 }
6445
6446 /* convert a vCU to a CU */
vcu_to_cu(u8 vcu)6447 static u32 vcu_to_cu(u8 vcu)
6448 {
6449 return 1 << vcu;
6450 }
6451
6452 /* convert a CU to a vCU */
cu_to_vcu(u32 cu)6453 static u8 cu_to_vcu(u32 cu)
6454 {
6455 return ilog2(cu);
6456 }
6457
6458 /* convert a vAU to an AU */
vau_to_au(u8 vau)6459 static u32 vau_to_au(u8 vau)
6460 {
6461 return 8 * (1 << vau);
6462 }
6463
set_linkup_defaults(struct hfi1_pportdata * ppd)6464 static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6465 {
6466 ppd->sm_trap_qp = 0x0;
6467 ppd->sa_qp = 0x1;
6468 }
6469
6470 /*
6471 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset.
6472 */
lcb_shutdown(struct hfi1_devdata * dd,int abort)6473 static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6474 {
6475 u64 reg;
6476
6477 /* clear lcb run: LCB_CFG_RUN.EN = 0 */
6478 write_csr(dd, DC_LCB_CFG_RUN, 0);
6479 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
6480 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
6481 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
6482 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
6483 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6484 reg = read_csr(dd, DCC_CFG_RESET);
6485 write_csr(dd, DCC_CFG_RESET, reg |
6486 DCC_CFG_RESET_RESET_LCB | DCC_CFG_RESET_RESET_RX_FPE);
6487 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
6488 if (!abort) {
6489 udelay(1); /* must hold for the longer of 16cclks or 20ns */
6490 write_csr(dd, DCC_CFG_RESET, reg);
6491 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6492 }
6493 }
6494
6495 /*
6496 * This routine should be called after the link has been transitioned to
6497 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
6498 * reset).
6499 *
6500 * The expectation is that the caller of this routine would have taken
6501 * care of properly transitioning the link into the correct state.
6502 * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6503 * before calling this function.
6504 */
_dc_shutdown(struct hfi1_devdata * dd)6505 static void _dc_shutdown(struct hfi1_devdata *dd)
6506 {
6507 lockdep_assert_held(&dd->dc8051_lock);
6508
6509 if (dd->dc_shutdown)
6510 return;
6511
6512 dd->dc_shutdown = 1;
6513 /* Shutdown the LCB */
6514 lcb_shutdown(dd, 1);
6515 /*
6516 * Going to OFFLINE would have causes the 8051 to put the
6517 * SerDes into reset already. Just need to shut down the 8051,
6518 * itself.
6519 */
6520 write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6521 }
6522
dc_shutdown(struct hfi1_devdata * dd)6523 static void dc_shutdown(struct hfi1_devdata *dd)
6524 {
6525 mutex_lock(&dd->dc8051_lock);
6526 _dc_shutdown(dd);
6527 mutex_unlock(&dd->dc8051_lock);
6528 }
6529
6530 /*
6531 * Calling this after the DC has been brought out of reset should not
6532 * do any damage.
6533 * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6534 * before calling this function.
6535 */
_dc_start(struct hfi1_devdata * dd)6536 static void _dc_start(struct hfi1_devdata *dd)
6537 {
6538 lockdep_assert_held(&dd->dc8051_lock);
6539
6540 if (!dd->dc_shutdown)
6541 return;
6542
6543 /* Take the 8051 out of reset */
6544 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6545 /* Wait until 8051 is ready */
6546 if (wait_fm_ready(dd, TIMEOUT_8051_START))
6547 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
6548 __func__);
6549
6550 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
6551 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6552 /* lcb_shutdown() with abort=1 does not restore these */
6553 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6554 dd->dc_shutdown = 0;
6555 }
6556
dc_start(struct hfi1_devdata * dd)6557 static void dc_start(struct hfi1_devdata *dd)
6558 {
6559 mutex_lock(&dd->dc8051_lock);
6560 _dc_start(dd);
6561 mutex_unlock(&dd->dc8051_lock);
6562 }
6563
6564 /*
6565 * These LCB adjustments are for the Aurora SerDes core in the FPGA.
6566 */
adjust_lcb_for_fpga_serdes(struct hfi1_devdata * dd)6567 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6568 {
6569 u64 rx_radr, tx_radr;
6570 u32 version;
6571
6572 if (dd->icode != ICODE_FPGA_EMULATION)
6573 return;
6574
6575 /*
6576 * These LCB defaults on emulator _s are good, nothing to do here:
6577 * LCB_CFG_TX_FIFOS_RADR
6578 * LCB_CFG_RX_FIFOS_RADR
6579 * LCB_CFG_LN_DCLK
6580 * LCB_CFG_IGNORE_LOST_RCLK
6581 */
6582 if (is_emulator_s(dd))
6583 return;
6584 /* else this is _p */
6585
6586 version = emulator_rev(dd);
6587 if (!is_ax(dd))
6588 version = 0x2d; /* all B0 use 0x2d or higher settings */
6589
6590 if (version <= 0x12) {
6591 /* release 0x12 and below */
6592
6593 /*
6594 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
6595 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
6596 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
6597 */
6598 rx_radr =
6599 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6600 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6601 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6602 /*
6603 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
6604 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
6605 */
6606 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6607 } else if (version <= 0x18) {
6608 /* release 0x13 up to 0x18 */
6609 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6610 rx_radr =
6611 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6612 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6613 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6614 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6615 } else if (version == 0x19) {
6616 /* release 0x19 */
6617 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
6618 rx_radr =
6619 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6620 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6621 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6622 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6623 } else if (version == 0x1a) {
6624 /* release 0x1a */
6625 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6626 rx_radr =
6627 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6628 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6629 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6630 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6631 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6632 } else {
6633 /* release 0x1b and higher */
6634 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
6635 rx_radr =
6636 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6637 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6638 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6639 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6640 }
6641
6642 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6643 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
6644 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
6645 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
6646 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6647 }
6648
6649 /*
6650 * Handle a SMA idle message
6651 *
6652 * This is a work-queue function outside of the interrupt.
6653 */
handle_sma_message(struct work_struct * work)6654 void handle_sma_message(struct work_struct *work)
6655 {
6656 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6657 sma_message_work);
6658 struct hfi1_devdata *dd = ppd->dd;
6659 u64 msg;
6660 int ret;
6661
6662 /*
6663 * msg is bytes 1-4 of the 40-bit idle message - the command code
6664 * is stripped off
6665 */
6666 ret = read_idle_sma(dd, &msg);
6667 if (ret)
6668 return;
6669 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6670 /*
6671 * React to the SMA message. Byte[1] (0 for us) is the command.
6672 */
6673 switch (msg & 0xff) {
6674 case SMA_IDLE_ARM:
6675 /*
6676 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6677 * State Transitions
6678 *
6679 * Only expected in INIT or ARMED, discard otherwise.
6680 */
6681 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6682 ppd->neighbor_normal = 1;
6683 break;
6684 case SMA_IDLE_ACTIVE:
6685 /*
6686 * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6687 * State Transitions
6688 *
6689 * Can activate the node. Discard otherwise.
6690 */
6691 if (ppd->host_link_state == HLS_UP_ARMED &&
6692 ppd->is_active_optimize_enabled) {
6693 ppd->neighbor_normal = 1;
6694 ret = set_link_state(ppd, HLS_UP_ACTIVE);
6695 if (ret)
6696 dd_dev_err(
6697 dd,
6698 "%s: received Active SMA idle message, couldn't set link to Active\n",
6699 __func__);
6700 }
6701 break;
6702 default:
6703 dd_dev_err(dd,
6704 "%s: received unexpected SMA idle message 0x%llx\n",
6705 __func__, msg);
6706 break;
6707 }
6708 }
6709
adjust_rcvctrl(struct hfi1_devdata * dd,u64 add,u64 clear)6710 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6711 {
6712 u64 rcvctrl;
6713 unsigned long flags;
6714
6715 spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6716 rcvctrl = read_csr(dd, RCV_CTRL);
6717 rcvctrl |= add;
6718 rcvctrl &= ~clear;
6719 write_csr(dd, RCV_CTRL, rcvctrl);
6720 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6721 }
6722
add_rcvctrl(struct hfi1_devdata * dd,u64 add)6723 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6724 {
6725 adjust_rcvctrl(dd, add, 0);
6726 }
6727
clear_rcvctrl(struct hfi1_devdata * dd,u64 clear)6728 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6729 {
6730 adjust_rcvctrl(dd, 0, clear);
6731 }
6732
6733 /*
6734 * Called from all interrupt handlers to start handling an SPC freeze.
6735 */
start_freeze_handling(struct hfi1_pportdata * ppd,int flags)6736 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6737 {
6738 struct hfi1_devdata *dd = ppd->dd;
6739 struct send_context *sc;
6740 int i;
6741 int sc_flags;
6742
6743 if (flags & FREEZE_SELF)
6744 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6745
6746 /* enter frozen mode */
6747 dd->flags |= HFI1_FROZEN;
6748
6749 /* notify all SDMA engines that they are going into a freeze */
6750 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6751
6752 sc_flags = SCF_FROZEN | SCF_HALTED | (flags & FREEZE_LINK_DOWN ?
6753 SCF_LINK_DOWN : 0);
6754 /* do halt pre-handling on all enabled send contexts */
6755 for (i = 0; i < dd->num_send_contexts; i++) {
6756 sc = dd->send_contexts[i].sc;
6757 if (sc && (sc->flags & SCF_ENABLED))
6758 sc_stop(sc, sc_flags);
6759 }
6760
6761 /* Send context are frozen. Notify user space */
6762 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6763
6764 if (flags & FREEZE_ABORT) {
6765 dd_dev_err(dd,
6766 "Aborted freeze recovery. Please REBOOT system\n");
6767 return;
6768 }
6769 /* queue non-interrupt handler */
6770 queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6771 }
6772
6773 /*
6774 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
6775 * depending on the "freeze" parameter.
6776 *
6777 * No need to return an error if it times out, our only option
6778 * is to proceed anyway.
6779 */
wait_for_freeze_status(struct hfi1_devdata * dd,int freeze)6780 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6781 {
6782 unsigned long timeout;
6783 u64 reg;
6784
6785 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6786 while (1) {
6787 reg = read_csr(dd, CCE_STATUS);
6788 if (freeze) {
6789 /* waiting until all indicators are set */
6790 if ((reg & ALL_FROZE) == ALL_FROZE)
6791 return; /* all done */
6792 } else {
6793 /* waiting until all indicators are clear */
6794 if ((reg & ALL_FROZE) == 0)
6795 return; /* all done */
6796 }
6797
6798 if (time_after(jiffies, timeout)) {
6799 dd_dev_err(dd,
6800 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6801 freeze ? "" : "un", reg & ALL_FROZE,
6802 freeze ? ALL_FROZE : 0ull);
6803 return;
6804 }
6805 usleep_range(80, 120);
6806 }
6807 }
6808
6809 /*
6810 * Do all freeze handling for the RXE block.
6811 */
rxe_freeze(struct hfi1_devdata * dd)6812 static void rxe_freeze(struct hfi1_devdata *dd)
6813 {
6814 int i;
6815 struct hfi1_ctxtdata *rcd;
6816
6817 /* disable port */
6818 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6819
6820 /* disable all receive contexts */
6821 for (i = 0; i < dd->num_rcv_contexts; i++) {
6822 rcd = hfi1_rcd_get_by_index(dd, i);
6823 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd);
6824 hfi1_rcd_put(rcd);
6825 }
6826 }
6827
6828 /*
6829 * Unfreeze handling for the RXE block - kernel contexts only.
6830 * This will also enable the port. User contexts will do unfreeze
6831 * handling on a per-context basis as they call into the driver.
6832 *
6833 */
rxe_kernel_unfreeze(struct hfi1_devdata * dd)6834 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6835 {
6836 u32 rcvmask;
6837 u16 i;
6838 struct hfi1_ctxtdata *rcd;
6839
6840 /* enable all kernel contexts */
6841 for (i = 0; i < dd->num_rcv_contexts; i++) {
6842 rcd = hfi1_rcd_get_by_index(dd, i);
6843
6844 /* Ensure all non-user contexts(including vnic) are enabled */
6845 if (!rcd ||
6846 (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) {
6847 hfi1_rcd_put(rcd);
6848 continue;
6849 }
6850 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6851 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
6852 rcvmask |= hfi1_rcvhdrtail_kvaddr(rcd) ?
6853 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6854 hfi1_rcvctrl(dd, rcvmask, rcd);
6855 hfi1_rcd_put(rcd);
6856 }
6857
6858 /* enable port */
6859 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6860 }
6861
6862 /*
6863 * Non-interrupt SPC freeze handling.
6864 *
6865 * This is a work-queue function outside of the triggering interrupt.
6866 */
handle_freeze(struct work_struct * work)6867 void handle_freeze(struct work_struct *work)
6868 {
6869 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6870 freeze_work);
6871 struct hfi1_devdata *dd = ppd->dd;
6872
6873 /* wait for freeze indicators on all affected blocks */
6874 wait_for_freeze_status(dd, 1);
6875
6876 /* SPC is now frozen */
6877
6878 /* do send PIO freeze steps */
6879 pio_freeze(dd);
6880
6881 /* do send DMA freeze steps */
6882 sdma_freeze(dd);
6883
6884 /* do send egress freeze steps - nothing to do */
6885
6886 /* do receive freeze steps */
6887 rxe_freeze(dd);
6888
6889 /*
6890 * Unfreeze the hardware - clear the freeze, wait for each
6891 * block's frozen bit to clear, then clear the frozen flag.
6892 */
6893 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6894 wait_for_freeze_status(dd, 0);
6895
6896 if (is_ax(dd)) {
6897 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6898 wait_for_freeze_status(dd, 1);
6899 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6900 wait_for_freeze_status(dd, 0);
6901 }
6902
6903 /* do send PIO unfreeze steps for kernel contexts */
6904 pio_kernel_unfreeze(dd);
6905
6906 /* do send DMA unfreeze steps */
6907 sdma_unfreeze(dd);
6908
6909 /* do send egress unfreeze steps - nothing to do */
6910
6911 /* do receive unfreeze steps for kernel contexts */
6912 rxe_kernel_unfreeze(dd);
6913
6914 /*
6915 * The unfreeze procedure touches global device registers when
6916 * it disables and re-enables RXE. Mark the device unfrozen
6917 * after all that is done so other parts of the driver waiting
6918 * for the device to unfreeze don't do things out of order.
6919 *
6920 * The above implies that the meaning of HFI1_FROZEN flag is
6921 * "Device has gone into freeze mode and freeze mode handling
6922 * is still in progress."
6923 *
6924 * The flag will be removed when freeze mode processing has
6925 * completed.
6926 */
6927 dd->flags &= ~HFI1_FROZEN;
6928 wake_up(&dd->event_queue);
6929
6930 /* no longer frozen */
6931 }
6932
6933 /**
6934 * update_xmit_counters - update PortXmitWait/PortVlXmitWait
6935 * counters.
6936 * @ppd: info of physical Hfi port
6937 * @link_width: new link width after link up or downgrade
6938 *
6939 * Update the PortXmitWait and PortVlXmitWait counters after
6940 * a link up or downgrade event to reflect a link width change.
6941 */
update_xmit_counters(struct hfi1_pportdata * ppd,u16 link_width)6942 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width)
6943 {
6944 int i;
6945 u16 tx_width;
6946 u16 link_speed;
6947
6948 tx_width = tx_link_width(link_width);
6949 link_speed = get_link_speed(ppd->link_speed_active);
6950
6951 /*
6952 * There are C_VL_COUNT number of PortVLXmitWait counters.
6953 * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
6954 */
6955 for (i = 0; i < C_VL_COUNT + 1; i++)
6956 get_xmit_wait_counters(ppd, tx_width, link_speed, i);
6957 }
6958
6959 /*
6960 * Handle a link up interrupt from the 8051.
6961 *
6962 * This is a work-queue function outside of the interrupt.
6963 */
handle_link_up(struct work_struct * work)6964 void handle_link_up(struct work_struct *work)
6965 {
6966 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6967 link_up_work);
6968 struct hfi1_devdata *dd = ppd->dd;
6969
6970 set_link_state(ppd, HLS_UP_INIT);
6971
6972 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
6973 read_ltp_rtt(dd);
6974 /*
6975 * OPA specifies that certain counters are cleared on a transition
6976 * to link up, so do that.
6977 */
6978 clear_linkup_counters(dd);
6979 /*
6980 * And (re)set link up default values.
6981 */
6982 set_linkup_defaults(ppd);
6983
6984 /*
6985 * Set VL15 credits. Use cached value from verify cap interrupt.
6986 * In case of quick linkup or simulator, vl15 value will be set by
6987 * handle_linkup_change. VerifyCap interrupt handler will not be
6988 * called in those scenarios.
6989 */
6990 if (!(quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR))
6991 set_up_vl15(dd, dd->vl15buf_cached);
6992
6993 /* enforce link speed enabled */
6994 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
6995 /* oops - current speed is not enabled, bounce */
6996 dd_dev_err(dd,
6997 "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
6998 ppd->link_speed_active, ppd->link_speed_enabled);
6999 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
7000 OPA_LINKDOWN_REASON_SPEED_POLICY);
7001 set_link_state(ppd, HLS_DN_OFFLINE);
7002 start_link(ppd);
7003 }
7004 }
7005
7006 /*
7007 * Several pieces of LNI information were cached for SMA in ppd.
7008 * Reset these on link down
7009 */
reset_neighbor_info(struct hfi1_pportdata * ppd)7010 static void reset_neighbor_info(struct hfi1_pportdata *ppd)
7011 {
7012 ppd->neighbor_guid = 0;
7013 ppd->neighbor_port_number = 0;
7014 ppd->neighbor_type = 0;
7015 ppd->neighbor_fm_security = 0;
7016 }
7017
7018 static const char * const link_down_reason_strs[] = {
7019 [OPA_LINKDOWN_REASON_NONE] = "None",
7020 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0",
7021 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
7022 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
7023 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
7024 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID",
7025 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID",
7026 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2",
7027 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC",
7028 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8",
7029 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail",
7030 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10",
7031 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error",
7032 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15",
7033 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker",
7034 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14",
7035 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15",
7036 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance",
7037 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance",
7038 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance",
7039 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack",
7040 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker",
7041 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt",
7042 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit",
7043 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit",
7044 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24",
7045 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25",
7046 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26",
7047 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27",
7048 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28",
7049 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29",
7050 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30",
7051 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] =
7052 "Excessive buffer overrun",
7053 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown",
7054 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot",
7055 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown",
7056 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce",
7057 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy",
7058 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy",
7059 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected",
7060 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] =
7061 "Local media not installed",
7062 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed",
7063 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config",
7064 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] =
7065 "End to end not installed",
7066 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy",
7067 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy",
7068 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy",
7069 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management",
7070 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled",
7071 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient"
7072 };
7073
7074 /* return the neighbor link down reason string */
link_down_reason_str(u8 reason)7075 static const char *link_down_reason_str(u8 reason)
7076 {
7077 const char *str = NULL;
7078
7079 if (reason < ARRAY_SIZE(link_down_reason_strs))
7080 str = link_down_reason_strs[reason];
7081 if (!str)
7082 str = "(invalid)";
7083
7084 return str;
7085 }
7086
7087 /*
7088 * Handle a link down interrupt from the 8051.
7089 *
7090 * This is a work-queue function outside of the interrupt.
7091 */
handle_link_down(struct work_struct * work)7092 void handle_link_down(struct work_struct *work)
7093 {
7094 u8 lcl_reason, neigh_reason = 0;
7095 u8 link_down_reason;
7096 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7097 link_down_work);
7098 int was_up;
7099 static const char ldr_str[] = "Link down reason: ";
7100
7101 if ((ppd->host_link_state &
7102 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
7103 ppd->port_type == PORT_TYPE_FIXED)
7104 ppd->offline_disabled_reason =
7105 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
7106
7107 /* Go offline first, then deal with reading/writing through 8051 */
7108 was_up = !!(ppd->host_link_state & HLS_UP);
7109 set_link_state(ppd, HLS_DN_OFFLINE);
7110 xchg(&ppd->is_link_down_queued, 0);
7111
7112 if (was_up) {
7113 lcl_reason = 0;
7114 /* link down reason is only valid if the link was up */
7115 read_link_down_reason(ppd->dd, &link_down_reason);
7116 switch (link_down_reason) {
7117 case LDR_LINK_TRANSFER_ACTIVE_LOW:
7118 /* the link went down, no idle message reason */
7119 dd_dev_info(ppd->dd, "%sUnexpected link down\n",
7120 ldr_str);
7121 break;
7122 case LDR_RECEIVED_LINKDOWN_IDLE_MSG:
7123 /*
7124 * The neighbor reason is only valid if an idle message
7125 * was received for it.
7126 */
7127 read_planned_down_reason_code(ppd->dd, &neigh_reason);
7128 dd_dev_info(ppd->dd,
7129 "%sNeighbor link down message %d, %s\n",
7130 ldr_str, neigh_reason,
7131 link_down_reason_str(neigh_reason));
7132 break;
7133 case LDR_RECEIVED_HOST_OFFLINE_REQ:
7134 dd_dev_info(ppd->dd,
7135 "%sHost requested link to go offline\n",
7136 ldr_str);
7137 break;
7138 default:
7139 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n",
7140 ldr_str, link_down_reason);
7141 break;
7142 }
7143
7144 /*
7145 * If no reason, assume peer-initiated but missed
7146 * LinkGoingDown idle flits.
7147 */
7148 if (neigh_reason == 0)
7149 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
7150 } else {
7151 /* went down while polling or going up */
7152 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT;
7153 }
7154
7155 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
7156
7157 /* inform the SMA when the link transitions from up to down */
7158 if (was_up && ppd->local_link_down_reason.sma == 0 &&
7159 ppd->neigh_link_down_reason.sma == 0) {
7160 ppd->local_link_down_reason.sma =
7161 ppd->local_link_down_reason.latest;
7162 ppd->neigh_link_down_reason.sma =
7163 ppd->neigh_link_down_reason.latest;
7164 }
7165
7166 reset_neighbor_info(ppd);
7167
7168 /* disable the port */
7169 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
7170
7171 /*
7172 * If there is no cable attached, turn the DC off. Otherwise,
7173 * start the link bring up.
7174 */
7175 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd))
7176 dc_shutdown(ppd->dd);
7177 else
7178 start_link(ppd);
7179 }
7180
handle_link_bounce(struct work_struct * work)7181 void handle_link_bounce(struct work_struct *work)
7182 {
7183 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7184 link_bounce_work);
7185
7186 /*
7187 * Only do something if the link is currently up.
7188 */
7189 if (ppd->host_link_state & HLS_UP) {
7190 set_link_state(ppd, HLS_DN_OFFLINE);
7191 start_link(ppd);
7192 } else {
7193 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
7194 __func__, link_state_name(ppd->host_link_state));
7195 }
7196 }
7197
7198 /*
7199 * Mask conversion: Capability exchange to Port LTP. The capability
7200 * exchange has an implicit 16b CRC that is mandatory.
7201 */
cap_to_port_ltp(int cap)7202 static int cap_to_port_ltp(int cap)
7203 {
7204 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
7205
7206 if (cap & CAP_CRC_14B)
7207 port_ltp |= PORT_LTP_CRC_MODE_14;
7208 if (cap & CAP_CRC_48B)
7209 port_ltp |= PORT_LTP_CRC_MODE_48;
7210 if (cap & CAP_CRC_12B_16B_PER_LANE)
7211 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
7212
7213 return port_ltp;
7214 }
7215
7216 /*
7217 * Convert an OPA Port LTP mask to capability mask
7218 */
port_ltp_to_cap(int port_ltp)7219 int port_ltp_to_cap(int port_ltp)
7220 {
7221 int cap_mask = 0;
7222
7223 if (port_ltp & PORT_LTP_CRC_MODE_14)
7224 cap_mask |= CAP_CRC_14B;
7225 if (port_ltp & PORT_LTP_CRC_MODE_48)
7226 cap_mask |= CAP_CRC_48B;
7227 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
7228 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
7229
7230 return cap_mask;
7231 }
7232
7233 /*
7234 * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
7235 */
lcb_to_port_ltp(int lcb_crc)7236 static int lcb_to_port_ltp(int lcb_crc)
7237 {
7238 int port_ltp = 0;
7239
7240 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
7241 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
7242 else if (lcb_crc == LCB_CRC_48B)
7243 port_ltp = PORT_LTP_CRC_MODE_48;
7244 else if (lcb_crc == LCB_CRC_14B)
7245 port_ltp = PORT_LTP_CRC_MODE_14;
7246 else
7247 port_ltp = PORT_LTP_CRC_MODE_16;
7248
7249 return port_ltp;
7250 }
7251
clear_full_mgmt_pkey(struct hfi1_pportdata * ppd)7252 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7253 {
7254 if (ppd->pkeys[2] != 0) {
7255 ppd->pkeys[2] = 0;
7256 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7257 hfi1_event_pkey_change(ppd->dd, ppd->port);
7258 }
7259 }
7260
7261 /*
7262 * Convert the given link width to the OPA link width bitmask.
7263 */
link_width_to_bits(struct hfi1_devdata * dd,u16 width)7264 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
7265 {
7266 switch (width) {
7267 case 0:
7268 /*
7269 * Simulator and quick linkup do not set the width.
7270 * Just set it to 4x without complaint.
7271 */
7272 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
7273 return OPA_LINK_WIDTH_4X;
7274 return 0; /* no lanes up */
7275 case 1: return OPA_LINK_WIDTH_1X;
7276 case 2: return OPA_LINK_WIDTH_2X;
7277 case 3: return OPA_LINK_WIDTH_3X;
7278 case 4: return OPA_LINK_WIDTH_4X;
7279 default:
7280 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
7281 __func__, width);
7282 return OPA_LINK_WIDTH_4X;
7283 }
7284 }
7285
7286 /*
7287 * Do a population count on the bottom nibble.
7288 */
7289 static const u8 bit_counts[16] = {
7290 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7291 };
7292
nibble_to_count(u8 nibble)7293 static inline u8 nibble_to_count(u8 nibble)
7294 {
7295 return bit_counts[nibble & 0xf];
7296 }
7297
7298 /*
7299 * Read the active lane information from the 8051 registers and return
7300 * their widths.
7301 *
7302 * Active lane information is found in these 8051 registers:
7303 * enable_lane_tx
7304 * enable_lane_rx
7305 */
get_link_widths(struct hfi1_devdata * dd,u16 * tx_width,u16 * rx_width)7306 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7307 u16 *rx_width)
7308 {
7309 u16 tx, rx;
7310 u8 enable_lane_rx;
7311 u8 enable_lane_tx;
7312 u8 tx_polarity_inversion;
7313 u8 rx_polarity_inversion;
7314 u8 max_rate;
7315
7316 /* read the active lanes */
7317 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
7318 &rx_polarity_inversion, &max_rate);
7319 read_local_lni(dd, &enable_lane_rx);
7320
7321 /* convert to counts */
7322 tx = nibble_to_count(enable_lane_tx);
7323 rx = nibble_to_count(enable_lane_rx);
7324
7325 /*
7326 * Set link_speed_active here, overriding what was set in
7327 * handle_verify_cap(). The ASIC 8051 firmware does not correctly
7328 * set the max_rate field in handle_verify_cap until v0.19.
7329 */
7330 if ((dd->icode == ICODE_RTL_SILICON) &&
7331 (dd->dc8051_ver < dc8051_ver(0, 19, 0))) {
7332 /* max_rate: 0 = 12.5G, 1 = 25G */
7333 switch (max_rate) {
7334 case 0:
7335 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7336 break;
7337 case 1:
7338 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7339 break;
7340 default:
7341 dd_dev_err(dd,
7342 "%s: unexpected max rate %d, using 25Gb\n",
7343 __func__, (int)max_rate);
7344 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7345 break;
7346 }
7347 }
7348
7349 dd_dev_info(dd,
7350 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7351 enable_lane_tx, tx, enable_lane_rx, rx);
7352 *tx_width = link_width_to_bits(dd, tx);
7353 *rx_width = link_width_to_bits(dd, rx);
7354 }
7355
7356 /*
7357 * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
7358 * Valid after the end of VerifyCap and during LinkUp. Does not change
7359 * after link up. I.e. look elsewhere for downgrade information.
7360 *
7361 * Bits are:
7362 * + bits [7:4] contain the number of active transmitters
7363 * + bits [3:0] contain the number of active receivers
7364 * These are numbers 1 through 4 and can be different values if the
7365 * link is asymmetric.
7366 *
7367 * verify_cap_local_fm_link_width[0] retains its original value.
7368 */
get_linkup_widths(struct hfi1_devdata * dd,u16 * tx_width,u16 * rx_width)7369 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7370 u16 *rx_width)
7371 {
7372 u16 widths, tx, rx;
7373 u8 misc_bits, local_flags;
7374 u16 active_tx, active_rx;
7375
7376 read_vc_local_link_mode(dd, &misc_bits, &local_flags, &widths);
7377 tx = widths >> 12;
7378 rx = (widths >> 8) & 0xf;
7379
7380 *tx_width = link_width_to_bits(dd, tx);
7381 *rx_width = link_width_to_bits(dd, rx);
7382
7383 /* print the active widths */
7384 get_link_widths(dd, &active_tx, &active_rx);
7385 }
7386
7387 /*
7388 * Set ppd->link_width_active and ppd->link_width_downgrade_active using
7389 * hardware information when the link first comes up.
7390 *
7391 * The link width is not available until after VerifyCap.AllFramesReceived
7392 * (the trigger for handle_verify_cap), so this is outside that routine
7393 * and should be called when the 8051 signals linkup.
7394 */
get_linkup_link_widths(struct hfi1_pportdata * ppd)7395 void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7396 {
7397 u16 tx_width, rx_width;
7398
7399 /* get end-of-LNI link widths */
7400 get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7401
7402 /* use tx_width as the link is supposed to be symmetric on link up */
7403 ppd->link_width_active = tx_width;
7404 /* link width downgrade active (LWD.A) starts out matching LW.A */
7405 ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7406 ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7407 /* per OPA spec, on link up LWD.E resets to LWD.S */
7408 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7409 /* cache the active egress rate (units {10^6 bits/sec]) */
7410 ppd->current_egress_rate = active_egress_rate(ppd);
7411 }
7412
7413 /*
7414 * Handle a verify capabilities interrupt from the 8051.
7415 *
7416 * This is a work-queue function outside of the interrupt.
7417 */
handle_verify_cap(struct work_struct * work)7418 void handle_verify_cap(struct work_struct *work)
7419 {
7420 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7421 link_vc_work);
7422 struct hfi1_devdata *dd = ppd->dd;
7423 u64 reg;
7424 u8 power_management;
7425 u8 continuous;
7426 u8 vcu;
7427 u8 vau;
7428 u8 z;
7429 u16 vl15buf;
7430 u16 link_widths;
7431 u16 crc_mask;
7432 u16 crc_val;
7433 u16 device_id;
7434 u16 active_tx, active_rx;
7435 u8 partner_supported_crc;
7436 u8 remote_tx_rate;
7437 u8 device_rev;
7438
7439 set_link_state(ppd, HLS_VERIFY_CAP);
7440
7441 lcb_shutdown(dd, 0);
7442 adjust_lcb_for_fpga_serdes(dd);
7443
7444 read_vc_remote_phy(dd, &power_management, &continuous);
7445 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7446 &partner_supported_crc);
7447 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7448 read_remote_device_id(dd, &device_id, &device_rev);
7449
7450 /* print the active widths */
7451 get_link_widths(dd, &active_tx, &active_rx);
7452 dd_dev_info(dd,
7453 "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7454 (int)power_management, (int)continuous);
7455 dd_dev_info(dd,
7456 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7457 (int)vau, (int)z, (int)vcu, (int)vl15buf,
7458 (int)partner_supported_crc);
7459 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
7460 (u32)remote_tx_rate, (u32)link_widths);
7461 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
7462 (u32)device_id, (u32)device_rev);
7463 /*
7464 * The peer vAU value just read is the peer receiver value. HFI does
7465 * not support a transmit vAU of 0 (AU == 8). We advertised that
7466 * with Z=1 in the fabric capabilities sent to the peer. The peer
7467 * will see our Z=1, and, if it advertised a vAU of 0, will move its
7468 * receive to vAU of 1 (AU == 16). Do the same here. We do not care
7469 * about the peer Z value - our sent vAU is 3 (hardwired) and is not
7470 * subject to the Z value exception.
7471 */
7472 if (vau == 0)
7473 vau = 1;
7474 set_up_vau(dd, vau);
7475
7476 /*
7477 * Set VL15 credits to 0 in global credit register. Cache remote VL15
7478 * credits value and wait for link-up interrupt ot set it.
7479 */
7480 set_up_vl15(dd, 0);
7481 dd->vl15buf_cached = vl15buf;
7482
7483 /* set up the LCB CRC mode */
7484 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7485
7486 /* order is important: use the lowest bit in common */
7487 if (crc_mask & CAP_CRC_14B)
7488 crc_val = LCB_CRC_14B;
7489 else if (crc_mask & CAP_CRC_48B)
7490 crc_val = LCB_CRC_48B;
7491 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7492 crc_val = LCB_CRC_12B_16B_PER_LANE;
7493 else
7494 crc_val = LCB_CRC_16B;
7495
7496 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7497 write_csr(dd, DC_LCB_CFG_CRC_MODE,
7498 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7499
7500 /* set (14b only) or clear sideband credit */
7501 reg = read_csr(dd, SEND_CM_CTRL);
7502 if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7503 write_csr(dd, SEND_CM_CTRL,
7504 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7505 } else {
7506 write_csr(dd, SEND_CM_CTRL,
7507 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7508 }
7509
7510 ppd->link_speed_active = 0; /* invalid value */
7511 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
7512 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
7513 switch (remote_tx_rate) {
7514 case 0:
7515 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7516 break;
7517 case 1:
7518 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7519 break;
7520 }
7521 } else {
7522 /* actual rate is highest bit of the ANDed rates */
7523 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7524
7525 if (rate & 2)
7526 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7527 else if (rate & 1)
7528 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7529 }
7530 if (ppd->link_speed_active == 0) {
7531 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
7532 __func__, (int)remote_tx_rate);
7533 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7534 }
7535
7536 /*
7537 * Cache the values of the supported, enabled, and active
7538 * LTP CRC modes to return in 'portinfo' queries. But the bit
7539 * flags that are returned in the portinfo query differ from
7540 * what's in the link_crc_mask, crc_sizes, and crc_val
7541 * variables. Convert these here.
7542 */
7543 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7544 /* supported crc modes */
7545 ppd->port_ltp_crc_mode |=
7546 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7547 /* enabled crc modes */
7548 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7549 /* active crc mode */
7550
7551 /* set up the remote credit return table */
7552 assign_remote_cm_au_table(dd, vcu);
7553
7554 /*
7555 * The LCB is reset on entry to handle_verify_cap(), so this must
7556 * be applied on every link up.
7557 *
7558 * Adjust LCB error kill enable to kill the link if
7559 * these RBUF errors are seen:
7560 * REPLAY_BUF_MBE_SMASK
7561 * FLIT_INPUT_BUF_MBE_SMASK
7562 */
7563 if (is_ax(dd)) { /* fixed in B0 */
7564 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7565 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7566 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7567 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7568 }
7569
7570 /* pull LCB fifos out of reset - all fifo clocks must be stable */
7571 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7572
7573 /* give 8051 access to the LCB CSRs */
7574 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
7575 set_8051_lcb_access(dd);
7576
7577 /* tell the 8051 to go to LinkUp */
7578 set_link_state(ppd, HLS_GOING_UP);
7579 }
7580
7581 /**
7582 * apply_link_downgrade_policy - Apply the link width downgrade enabled
7583 * policy against the current active link widths.
7584 * @ppd: info of physical Hfi port
7585 * @refresh_widths: True indicates link downgrade event
7586 * @return: True indicates a successful link downgrade. False indicates
7587 * link downgrade event failed and the link will bounce back to
7588 * default link width.
7589 *
7590 * Called when the enabled policy changes or the active link widths
7591 * change.
7592 * Refresh_widths indicates that a link downgrade occurred. The
7593 * link_downgraded variable is set by refresh_widths and
7594 * determines the success/failure of the policy application.
7595 */
apply_link_downgrade_policy(struct hfi1_pportdata * ppd,bool refresh_widths)7596 bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
7597 bool refresh_widths)
7598 {
7599 int do_bounce = 0;
7600 int tries;
7601 u16 lwde;
7602 u16 tx, rx;
7603 bool link_downgraded = refresh_widths;
7604
7605 /* use the hls lock to avoid a race with actual link up */
7606 tries = 0;
7607 retry:
7608 mutex_lock(&ppd->hls_lock);
7609 /* only apply if the link is up */
7610 if (ppd->host_link_state & HLS_DOWN) {
7611 /* still going up..wait and retry */
7612 if (ppd->host_link_state & HLS_GOING_UP) {
7613 if (++tries < 1000) {
7614 mutex_unlock(&ppd->hls_lock);
7615 usleep_range(100, 120); /* arbitrary */
7616 goto retry;
7617 }
7618 dd_dev_err(ppd->dd,
7619 "%s: giving up waiting for link state change\n",
7620 __func__);
7621 }
7622 goto done;
7623 }
7624
7625 lwde = ppd->link_width_downgrade_enabled;
7626
7627 if (refresh_widths) {
7628 get_link_widths(ppd->dd, &tx, &rx);
7629 ppd->link_width_downgrade_tx_active = tx;
7630 ppd->link_width_downgrade_rx_active = rx;
7631 }
7632
7633 if (ppd->link_width_downgrade_tx_active == 0 ||
7634 ppd->link_width_downgrade_rx_active == 0) {
7635 /* the 8051 reported a dead link as a downgrade */
7636 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
7637 link_downgraded = false;
7638 } else if (lwde == 0) {
7639 /* downgrade is disabled */
7640
7641 /* bounce if not at starting active width */
7642 if ((ppd->link_width_active !=
7643 ppd->link_width_downgrade_tx_active) ||
7644 (ppd->link_width_active !=
7645 ppd->link_width_downgrade_rx_active)) {
7646 dd_dev_err(ppd->dd,
7647 "Link downgrade is disabled and link has downgraded, downing link\n");
7648 dd_dev_err(ppd->dd,
7649 " original 0x%x, tx active 0x%x, rx active 0x%x\n",
7650 ppd->link_width_active,
7651 ppd->link_width_downgrade_tx_active,
7652 ppd->link_width_downgrade_rx_active);
7653 do_bounce = 1;
7654 link_downgraded = false;
7655 }
7656 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7657 (lwde & ppd->link_width_downgrade_rx_active) == 0) {
7658 /* Tx or Rx is outside the enabled policy */
7659 dd_dev_err(ppd->dd,
7660 "Link is outside of downgrade allowed, downing link\n");
7661 dd_dev_err(ppd->dd,
7662 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7663 lwde, ppd->link_width_downgrade_tx_active,
7664 ppd->link_width_downgrade_rx_active);
7665 do_bounce = 1;
7666 link_downgraded = false;
7667 }
7668
7669 done:
7670 mutex_unlock(&ppd->hls_lock);
7671
7672 if (do_bounce) {
7673 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
7674 OPA_LINKDOWN_REASON_WIDTH_POLICY);
7675 set_link_state(ppd, HLS_DN_OFFLINE);
7676 start_link(ppd);
7677 }
7678
7679 return link_downgraded;
7680 }
7681
7682 /*
7683 * Handle a link downgrade interrupt from the 8051.
7684 *
7685 * This is a work-queue function outside of the interrupt.
7686 */
handle_link_downgrade(struct work_struct * work)7687 void handle_link_downgrade(struct work_struct *work)
7688 {
7689 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7690 link_downgrade_work);
7691
7692 dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7693 if (apply_link_downgrade_policy(ppd, true))
7694 update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active);
7695 }
7696
dcc_err_string(char * buf,int buf_len,u64 flags)7697 static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7698 {
7699 return flag_string(buf, buf_len, flags, dcc_err_flags,
7700 ARRAY_SIZE(dcc_err_flags));
7701 }
7702
lcb_err_string(char * buf,int buf_len,u64 flags)7703 static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7704 {
7705 return flag_string(buf, buf_len, flags, lcb_err_flags,
7706 ARRAY_SIZE(lcb_err_flags));
7707 }
7708
dc8051_err_string(char * buf,int buf_len,u64 flags)7709 static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7710 {
7711 return flag_string(buf, buf_len, flags, dc8051_err_flags,
7712 ARRAY_SIZE(dc8051_err_flags));
7713 }
7714
dc8051_info_err_string(char * buf,int buf_len,u64 flags)7715 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7716 {
7717 return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7718 ARRAY_SIZE(dc8051_info_err_flags));
7719 }
7720
dc8051_info_host_msg_string(char * buf,int buf_len,u64 flags)7721 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7722 {
7723 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7724 ARRAY_SIZE(dc8051_info_host_msg_flags));
7725 }
7726
handle_8051_interrupt(struct hfi1_devdata * dd,u32 unused,u64 reg)7727 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7728 {
7729 struct hfi1_pportdata *ppd = dd->pport;
7730 u64 info, err, host_msg;
7731 int queue_link_down = 0;
7732 char buf[96];
7733
7734 /* look at the flags */
7735 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7736 /* 8051 information set by firmware */
7737 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
7738 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7739 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7740 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7741 host_msg = (info >>
7742 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7743 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7744
7745 /*
7746 * Handle error flags.
7747 */
7748 if (err & FAILED_LNI) {
7749 /*
7750 * LNI error indications are cleared by the 8051
7751 * only when starting polling. Only pay attention
7752 * to them when in the states that occur during
7753 * LNI.
7754 */
7755 if (ppd->host_link_state
7756 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7757 queue_link_down = 1;
7758 dd_dev_info(dd, "Link error: %s\n",
7759 dc8051_info_err_string(buf,
7760 sizeof(buf),
7761 err &
7762 FAILED_LNI));
7763 }
7764 err &= ~(u64)FAILED_LNI;
7765 }
7766 /* unknown frames can happen durning LNI, just count */
7767 if (err & UNKNOWN_FRAME) {
7768 ppd->unknown_frame_count++;
7769 err &= ~(u64)UNKNOWN_FRAME;
7770 }
7771 if (err) {
7772 /* report remaining errors, but do not do anything */
7773 dd_dev_err(dd, "8051 info error: %s\n",
7774 dc8051_info_err_string(buf, sizeof(buf),
7775 err));
7776 }
7777
7778 /*
7779 * Handle host message flags.
7780 */
7781 if (host_msg & HOST_REQ_DONE) {
7782 /*
7783 * Presently, the driver does a busy wait for
7784 * host requests to complete. This is only an
7785 * informational message.
7786 * NOTE: The 8051 clears the host message
7787 * information *on the next 8051 command*.
7788 * Therefore, when linkup is achieved,
7789 * this flag will still be set.
7790 */
7791 host_msg &= ~(u64)HOST_REQ_DONE;
7792 }
7793 if (host_msg & BC_SMA_MSG) {
7794 queue_work(ppd->link_wq, &ppd->sma_message_work);
7795 host_msg &= ~(u64)BC_SMA_MSG;
7796 }
7797 if (host_msg & LINKUP_ACHIEVED) {
7798 dd_dev_info(dd, "8051: Link up\n");
7799 queue_work(ppd->link_wq, &ppd->link_up_work);
7800 host_msg &= ~(u64)LINKUP_ACHIEVED;
7801 }
7802 if (host_msg & EXT_DEVICE_CFG_REQ) {
7803 handle_8051_request(ppd);
7804 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7805 }
7806 if (host_msg & VERIFY_CAP_FRAME) {
7807 queue_work(ppd->link_wq, &ppd->link_vc_work);
7808 host_msg &= ~(u64)VERIFY_CAP_FRAME;
7809 }
7810 if (host_msg & LINK_GOING_DOWN) {
7811 const char *extra = "";
7812 /* no downgrade action needed if going down */
7813 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7814 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7815 extra = " (ignoring downgrade)";
7816 }
7817 dd_dev_info(dd, "8051: Link down%s\n", extra);
7818 queue_link_down = 1;
7819 host_msg &= ~(u64)LINK_GOING_DOWN;
7820 }
7821 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7822 queue_work(ppd->link_wq, &ppd->link_downgrade_work);
7823 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7824 }
7825 if (host_msg) {
7826 /* report remaining messages, but do not do anything */
7827 dd_dev_info(dd, "8051 info host message: %s\n",
7828 dc8051_info_host_msg_string(buf,
7829 sizeof(buf),
7830 host_msg));
7831 }
7832
7833 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7834 }
7835 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7836 /*
7837 * Lost the 8051 heartbeat. If this happens, we
7838 * receive constant interrupts about it. Disable
7839 * the interrupt after the first.
7840 */
7841 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7842 write_csr(dd, DC_DC8051_ERR_EN,
7843 read_csr(dd, DC_DC8051_ERR_EN) &
7844 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
7845
7846 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7847 }
7848 if (reg) {
7849 /* report the error, but do not do anything */
7850 dd_dev_err(dd, "8051 error: %s\n",
7851 dc8051_err_string(buf, sizeof(buf), reg));
7852 }
7853
7854 if (queue_link_down) {
7855 /*
7856 * if the link is already going down or disabled, do not
7857 * queue another. If there's a link down entry already
7858 * queued, don't queue another one.
7859 */
7860 if ((ppd->host_link_state &
7861 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7862 ppd->link_enabled == 0) {
7863 dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n",
7864 __func__, ppd->host_link_state,
7865 ppd->link_enabled);
7866 } else {
7867 if (xchg(&ppd->is_link_down_queued, 1) == 1)
7868 dd_dev_info(dd,
7869 "%s: link down request already queued\n",
7870 __func__);
7871 else
7872 queue_work(ppd->link_wq, &ppd->link_down_work);
7873 }
7874 }
7875 }
7876
7877 static const char * const fm_config_txt[] = {
7878 [0] =
7879 "BadHeadDist: Distance violation between two head flits",
7880 [1] =
7881 "BadTailDist: Distance violation between two tail flits",
7882 [2] =
7883 "BadCtrlDist: Distance violation between two credit control flits",
7884 [3] =
7885 "BadCrdAck: Credits return for unsupported VL",
7886 [4] =
7887 "UnsupportedVLMarker: Received VL Marker",
7888 [5] =
7889 "BadPreempt: Exceeded the preemption nesting level",
7890 [6] =
7891 "BadControlFlit: Received unsupported control flit",
7892 /* no 7 */
7893 [8] =
7894 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7895 };
7896
7897 static const char * const port_rcv_txt[] = {
7898 [1] =
7899 "BadPktLen: Illegal PktLen",
7900 [2] =
7901 "PktLenTooLong: Packet longer than PktLen",
7902 [3] =
7903 "PktLenTooShort: Packet shorter than PktLen",
7904 [4] =
7905 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7906 [5] =
7907 "BadDLID: Illegal DLID (0, doesn't match HFI)",
7908 [6] =
7909 "BadL2: Illegal L2 opcode",
7910 [7] =
7911 "BadSC: Unsupported SC",
7912 [9] =
7913 "BadRC: Illegal RC",
7914 [11] =
7915 "PreemptError: Preempting with same VL",
7916 [12] =
7917 "PreemptVL15: Preempting a VL15 packet",
7918 };
7919
7920 #define OPA_LDR_FMCONFIG_OFFSET 16
7921 #define OPA_LDR_PORTRCV_OFFSET 0
handle_dcc_err(struct hfi1_devdata * dd,u32 unused,u64 reg)7922 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7923 {
7924 u64 info, hdr0, hdr1;
7925 const char *extra;
7926 char buf[96];
7927 struct hfi1_pportdata *ppd = dd->pport;
7928 u8 lcl_reason = 0;
7929 int do_bounce = 0;
7930
7931 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7932 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7933 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7934 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7935 /* set status bit */
7936 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7937 }
7938 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7939 }
7940
7941 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7942 struct hfi1_pportdata *ppd = dd->pport;
7943 /* this counter saturates at (2^32) - 1 */
7944 if (ppd->link_downed < (u32)UINT_MAX)
7945 ppd->link_downed++;
7946 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7947 }
7948
7949 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7950 u8 reason_valid = 1;
7951
7952 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7953 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7954 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7955 /* set status bit */
7956 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7957 }
7958 switch (info) {
7959 case 0:
7960 case 1:
7961 case 2:
7962 case 3:
7963 case 4:
7964 case 5:
7965 case 6:
7966 extra = fm_config_txt[info];
7967 break;
7968 case 8:
7969 extra = fm_config_txt[info];
7970 if (ppd->port_error_action &
7971 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7972 do_bounce = 1;
7973 /*
7974 * lcl_reason cannot be derived from info
7975 * for this error
7976 */
7977 lcl_reason =
7978 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7979 }
7980 break;
7981 default:
7982 reason_valid = 0;
7983 snprintf(buf, sizeof(buf), "reserved%lld", info);
7984 extra = buf;
7985 break;
7986 }
7987
7988 if (reason_valid && !do_bounce) {
7989 do_bounce = ppd->port_error_action &
7990 (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
7991 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
7992 }
7993
7994 /* just report this */
7995 dd_dev_info_ratelimited(dd, "DCC Error: fmconfig error: %s\n",
7996 extra);
7997 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
7998 }
7999
8000 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
8001 u8 reason_valid = 1;
8002
8003 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
8004 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
8005 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
8006 if (!(dd->err_info_rcvport.status_and_code &
8007 OPA_EI_STATUS_SMASK)) {
8008 dd->err_info_rcvport.status_and_code =
8009 info & OPA_EI_CODE_SMASK;
8010 /* set status bit */
8011 dd->err_info_rcvport.status_and_code |=
8012 OPA_EI_STATUS_SMASK;
8013 /*
8014 * save first 2 flits in the packet that caused
8015 * the error
8016 */
8017 dd->err_info_rcvport.packet_flit1 = hdr0;
8018 dd->err_info_rcvport.packet_flit2 = hdr1;
8019 }
8020 switch (info) {
8021 case 1:
8022 case 2:
8023 case 3:
8024 case 4:
8025 case 5:
8026 case 6:
8027 case 7:
8028 case 9:
8029 case 11:
8030 case 12:
8031 extra = port_rcv_txt[info];
8032 break;
8033 default:
8034 reason_valid = 0;
8035 snprintf(buf, sizeof(buf), "reserved%lld", info);
8036 extra = buf;
8037 break;
8038 }
8039
8040 if (reason_valid && !do_bounce) {
8041 do_bounce = ppd->port_error_action &
8042 (1 << (OPA_LDR_PORTRCV_OFFSET + info));
8043 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
8044 }
8045
8046 /* just report this */
8047 dd_dev_info_ratelimited(dd, "DCC Error: PortRcv error: %s\n"
8048 " hdr0 0x%llx, hdr1 0x%llx\n",
8049 extra, hdr0, hdr1);
8050
8051 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
8052 }
8053
8054 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
8055 /* informative only */
8056 dd_dev_info_ratelimited(dd, "8051 access to LCB blocked\n");
8057 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
8058 }
8059 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
8060 /* informative only */
8061 dd_dev_info_ratelimited(dd, "host access to LCB blocked\n");
8062 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
8063 }
8064
8065 if (unlikely(hfi1_dbg_fault_suppress_err(&dd->verbs_dev)))
8066 reg &= ~DCC_ERR_FLG_LATE_EBP_ERR_SMASK;
8067
8068 /* report any remaining errors */
8069 if (reg)
8070 dd_dev_info_ratelimited(dd, "DCC Error: %s\n",
8071 dcc_err_string(buf, sizeof(buf), reg));
8072
8073 if (lcl_reason == 0)
8074 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
8075
8076 if (do_bounce) {
8077 dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n",
8078 __func__);
8079 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
8080 queue_work(ppd->link_wq, &ppd->link_bounce_work);
8081 }
8082 }
8083
handle_lcb_err(struct hfi1_devdata * dd,u32 unused,u64 reg)8084 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
8085 {
8086 char buf[96];
8087
8088 dd_dev_info(dd, "LCB Error: %s\n",
8089 lcb_err_string(buf, sizeof(buf), reg));
8090 }
8091
8092 /*
8093 * CCE block DC interrupt. Source is < 8.
8094 */
is_dc_int(struct hfi1_devdata * dd,unsigned int source)8095 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
8096 {
8097 const struct err_reg_info *eri = &dc_errs[source];
8098
8099 if (eri->handler) {
8100 interrupt_clear_down(dd, 0, eri);
8101 } else if (source == 3 /* dc_lbm_int */) {
8102 /*
8103 * This indicates that a parity error has occurred on the
8104 * address/control lines presented to the LBM. The error
8105 * is a single pulse, there is no associated error flag,
8106 * and it is non-maskable. This is because if a parity
8107 * error occurs on the request the request is dropped.
8108 * This should never occur, but it is nice to know if it
8109 * ever does.
8110 */
8111 dd_dev_err(dd, "Parity error in DC LBM block\n");
8112 } else {
8113 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
8114 }
8115 }
8116
8117 /*
8118 * TX block send credit interrupt. Source is < 160.
8119 */
is_send_credit_int(struct hfi1_devdata * dd,unsigned int source)8120 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
8121 {
8122 sc_group_release_update(dd, source);
8123 }
8124
8125 /*
8126 * TX block SDMA interrupt. Source is < 48.
8127 *
8128 * SDMA interrupts are grouped by type:
8129 *
8130 * 0 - N-1 = SDma
8131 * N - 2N-1 = SDmaProgress
8132 * 2N - 3N-1 = SDmaIdle
8133 */
is_sdma_eng_int(struct hfi1_devdata * dd,unsigned int source)8134 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
8135 {
8136 /* what interrupt */
8137 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
8138 /* which engine */
8139 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
8140
8141 #ifdef CONFIG_SDMA_VERBOSITY
8142 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
8143 slashstrip(__FILE__), __LINE__, __func__);
8144 sdma_dumpstate(&dd->per_sdma[which]);
8145 #endif
8146
8147 if (likely(what < 3 && which < dd->num_sdma)) {
8148 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
8149 } else {
8150 /* should not happen */
8151 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
8152 }
8153 }
8154
8155 /**
8156 * is_rcv_avail_int() - User receive context available IRQ handler
8157 * @dd: valid dd
8158 * @source: logical IRQ source (offset from IS_RCVAVAIL_START)
8159 *
8160 * RX block receive available interrupt. Source is < 160.
8161 *
8162 * This is the general interrupt handler for user (PSM) receive contexts,
8163 * and can only be used for non-threaded IRQs.
8164 */
is_rcv_avail_int(struct hfi1_devdata * dd,unsigned int source)8165 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
8166 {
8167 struct hfi1_ctxtdata *rcd;
8168 char *err_detail;
8169
8170 if (likely(source < dd->num_rcv_contexts)) {
8171 rcd = hfi1_rcd_get_by_index(dd, source);
8172 if (rcd) {
8173 handle_user_interrupt(rcd);
8174 hfi1_rcd_put(rcd);
8175 return; /* OK */
8176 }
8177 /* received an interrupt, but no rcd */
8178 err_detail = "dataless";
8179 } else {
8180 /* received an interrupt, but are not using that context */
8181 err_detail = "out of range";
8182 }
8183 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
8184 err_detail, source);
8185 }
8186
8187 /**
8188 * is_rcv_urgent_int() - User receive context urgent IRQ handler
8189 * @dd: valid dd
8190 * @source: logical IRQ source (offset from IS_RCVURGENT_START)
8191 *
8192 * RX block receive urgent interrupt. Source is < 160.
8193 *
8194 * NOTE: kernel receive contexts specifically do NOT enable this IRQ.
8195 */
is_rcv_urgent_int(struct hfi1_devdata * dd,unsigned int source)8196 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
8197 {
8198 struct hfi1_ctxtdata *rcd;
8199 char *err_detail;
8200
8201 if (likely(source < dd->num_rcv_contexts)) {
8202 rcd = hfi1_rcd_get_by_index(dd, source);
8203 if (rcd) {
8204 handle_user_interrupt(rcd);
8205 hfi1_rcd_put(rcd);
8206 return; /* OK */
8207 }
8208 /* received an interrupt, but no rcd */
8209 err_detail = "dataless";
8210 } else {
8211 /* received an interrupt, but are not using that context */
8212 err_detail = "out of range";
8213 }
8214 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
8215 err_detail, source);
8216 }
8217
8218 /*
8219 * Reserved range interrupt. Should not be called in normal operation.
8220 */
is_reserved_int(struct hfi1_devdata * dd,unsigned int source)8221 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
8222 {
8223 char name[64];
8224
8225 dd_dev_err(dd, "unexpected %s interrupt\n",
8226 is_reserved_name(name, sizeof(name), source));
8227 }
8228
8229 static const struct is_table is_table[] = {
8230 /*
8231 * start end
8232 * name func interrupt func
8233 */
8234 { IS_GENERAL_ERR_START, IS_GENERAL_ERR_END,
8235 is_misc_err_name, is_misc_err_int },
8236 { IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END,
8237 is_sdma_eng_err_name, is_sdma_eng_err_int },
8238 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
8239 is_sendctxt_err_name, is_sendctxt_err_int },
8240 { IS_SDMA_START, IS_SDMA_IDLE_END,
8241 is_sdma_eng_name, is_sdma_eng_int },
8242 { IS_VARIOUS_START, IS_VARIOUS_END,
8243 is_various_name, is_various_int },
8244 { IS_DC_START, IS_DC_END,
8245 is_dc_name, is_dc_int },
8246 { IS_RCVAVAIL_START, IS_RCVAVAIL_END,
8247 is_rcv_avail_name, is_rcv_avail_int },
8248 { IS_RCVURGENT_START, IS_RCVURGENT_END,
8249 is_rcv_urgent_name, is_rcv_urgent_int },
8250 { IS_SENDCREDIT_START, IS_SENDCREDIT_END,
8251 is_send_credit_name, is_send_credit_int},
8252 { IS_RESERVED_START, IS_RESERVED_END,
8253 is_reserved_name, is_reserved_int},
8254 };
8255
8256 /*
8257 * Interrupt source interrupt - called when the given source has an interrupt.
8258 * Source is a bit index into an array of 64-bit integers.
8259 */
is_interrupt(struct hfi1_devdata * dd,unsigned int source)8260 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
8261 {
8262 const struct is_table *entry;
8263
8264 /* avoids a double compare by walking the table in-order */
8265 for (entry = &is_table[0]; entry->is_name; entry++) {
8266 if (source <= entry->end) {
8267 trace_hfi1_interrupt(dd, entry, source);
8268 entry->is_int(dd, source - entry->start);
8269 return;
8270 }
8271 }
8272 /* fell off the end */
8273 dd_dev_err(dd, "invalid interrupt source %u\n", source);
8274 }
8275
8276 /**
8277 * general_interrupt - General interrupt handler
8278 * @irq: MSIx IRQ vector
8279 * @data: hfi1 devdata
8280 *
8281 * This is able to correctly handle all non-threaded interrupts. Receive
8282 * context DATA IRQs are threaded and are not supported by this handler.
8283 *
8284 */
general_interrupt(int irq,void * data)8285 irqreturn_t general_interrupt(int irq, void *data)
8286 {
8287 struct hfi1_devdata *dd = data;
8288 u64 regs[CCE_NUM_INT_CSRS];
8289 u32 bit;
8290 int i;
8291 irqreturn_t handled = IRQ_NONE;
8292
8293 this_cpu_inc(*dd->int_counter);
8294
8295 /* phase 1: scan and clear all handled interrupts */
8296 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
8297 if (dd->gi_mask[i] == 0) {
8298 regs[i] = 0; /* used later */
8299 continue;
8300 }
8301 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
8302 dd->gi_mask[i];
8303 /* only clear if anything is set */
8304 if (regs[i])
8305 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
8306 }
8307
8308 /* phase 2: call the appropriate handler */
8309 for_each_set_bit(bit, (unsigned long *)®s[0],
8310 CCE_NUM_INT_CSRS * 64) {
8311 is_interrupt(dd, bit);
8312 handled = IRQ_HANDLED;
8313 }
8314
8315 return handled;
8316 }
8317
sdma_interrupt(int irq,void * data)8318 irqreturn_t sdma_interrupt(int irq, void *data)
8319 {
8320 struct sdma_engine *sde = data;
8321 struct hfi1_devdata *dd = sde->dd;
8322 u64 status;
8323
8324 #ifdef CONFIG_SDMA_VERBOSITY
8325 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8326 slashstrip(__FILE__), __LINE__, __func__);
8327 sdma_dumpstate(sde);
8328 #endif
8329
8330 this_cpu_inc(*dd->int_counter);
8331
8332 /* This read_csr is really bad in the hot path */
8333 status = read_csr(dd,
8334 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8335 & sde->imask;
8336 if (likely(status)) {
8337 /* clear the interrupt(s) */
8338 write_csr(dd,
8339 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8340 status);
8341
8342 /* handle the interrupt(s) */
8343 sdma_engine_interrupt(sde, status);
8344 } else {
8345 dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n",
8346 sde->this_idx);
8347 }
8348 return IRQ_HANDLED;
8349 }
8350
8351 /*
8352 * Clear the receive interrupt. Use a read of the interrupt clear CSR
8353 * to insure that the write completed. This does NOT guarantee that
8354 * queued DMA writes to memory from the chip are pushed.
8355 */
clear_recv_intr(struct hfi1_ctxtdata * rcd)8356 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8357 {
8358 struct hfi1_devdata *dd = rcd->dd;
8359 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8360
8361 write_csr(dd, addr, rcd->imask);
8362 /* force the above write on the chip and get a value back */
8363 (void)read_csr(dd, addr);
8364 }
8365
8366 /* force the receive interrupt */
force_recv_intr(struct hfi1_ctxtdata * rcd)8367 void force_recv_intr(struct hfi1_ctxtdata *rcd)
8368 {
8369 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8370 }
8371
8372 /*
8373 * Return non-zero if a packet is present.
8374 *
8375 * This routine is called when rechecking for packets after the RcvAvail
8376 * interrupt has been cleared down. First, do a quick check of memory for
8377 * a packet present. If not found, use an expensive CSR read of the context
8378 * tail to determine the actual tail. The CSR read is necessary because there
8379 * is no method to push pending DMAs to memory other than an interrupt and we
8380 * are trying to determine if we need to force an interrupt.
8381 */
check_packet_present(struct hfi1_ctxtdata * rcd)8382 static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8383 {
8384 u32 tail;
8385
8386 if (hfi1_packet_present(rcd))
8387 return 1;
8388
8389 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */
8390 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8391 return hfi1_rcd_head(rcd) != tail;
8392 }
8393
8394 /*
8395 * Common code for receive contexts interrupt handlers.
8396 * Update traces, increment kernel IRQ counter and
8397 * setup ASPM when needed.
8398 */
receive_interrupt_common(struct hfi1_ctxtdata * rcd)8399 static void receive_interrupt_common(struct hfi1_ctxtdata *rcd)
8400 {
8401 struct hfi1_devdata *dd = rcd->dd;
8402
8403 trace_hfi1_receive_interrupt(dd, rcd);
8404 this_cpu_inc(*dd->int_counter);
8405 aspm_ctx_disable(rcd);
8406 }
8407
8408 /*
8409 * __hfi1_rcd_eoi_intr() - Make HW issue receive interrupt
8410 * when there are packets present in the queue. When calling
8411 * with interrupts enabled please use hfi1_rcd_eoi_intr.
8412 *
8413 * @rcd: valid receive context
8414 */
__hfi1_rcd_eoi_intr(struct hfi1_ctxtdata * rcd)8415 static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)
8416 {
8417 clear_recv_intr(rcd);
8418 if (check_packet_present(rcd))
8419 force_recv_intr(rcd);
8420 }
8421
8422 /**
8423 * hfi1_rcd_eoi_intr() - End of Interrupt processing action
8424 *
8425 * @rcd: Ptr to hfi1_ctxtdata of receive context
8426 *
8427 * Hold IRQs so we can safely clear the interrupt and
8428 * recheck for a packet that may have arrived after the previous
8429 * check and the interrupt clear. If a packet arrived, force another
8430 * interrupt. This routine can be called at the end of receive packet
8431 * processing in interrupt service routines, interrupt service thread
8432 * and softirqs
8433 */
hfi1_rcd_eoi_intr(struct hfi1_ctxtdata * rcd)8434 static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)
8435 {
8436 unsigned long flags;
8437
8438 local_irq_save(flags);
8439 __hfi1_rcd_eoi_intr(rcd);
8440 local_irq_restore(flags);
8441 }
8442
8443 /**
8444 * hfi1_netdev_rx_napi - napi poll function to move eoi inline
8445 * @napi: pointer to napi object
8446 * @budget: netdev budget
8447 */
hfi1_netdev_rx_napi(struct napi_struct * napi,int budget)8448 int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget)
8449 {
8450 struct hfi1_netdev_rxq *rxq = container_of(napi,
8451 struct hfi1_netdev_rxq, napi);
8452 struct hfi1_ctxtdata *rcd = rxq->rcd;
8453 int work_done = 0;
8454
8455 work_done = rcd->do_interrupt(rcd, budget);
8456
8457 if (work_done < budget) {
8458 napi_complete_done(napi, work_done);
8459 hfi1_rcd_eoi_intr(rcd);
8460 }
8461
8462 return work_done;
8463 }
8464
8465 /* Receive packet napi handler for netdevs VNIC and AIP */
receive_context_interrupt_napi(int irq,void * data)8466 irqreturn_t receive_context_interrupt_napi(int irq, void *data)
8467 {
8468 struct hfi1_ctxtdata *rcd = data;
8469
8470 receive_interrupt_common(rcd);
8471
8472 if (likely(rcd->napi)) {
8473 if (likely(napi_schedule_prep(rcd->napi)))
8474 __napi_schedule_irqoff(rcd->napi);
8475 else
8476 __hfi1_rcd_eoi_intr(rcd);
8477 } else {
8478 WARN_ONCE(1, "Napi IRQ handler without napi set up ctxt=%d\n",
8479 rcd->ctxt);
8480 __hfi1_rcd_eoi_intr(rcd);
8481 }
8482
8483 return IRQ_HANDLED;
8484 }
8485
8486 /*
8487 * Receive packet IRQ handler. This routine expects to be on its own IRQ.
8488 * This routine will try to handle packets immediately (latency), but if
8489 * it finds too many, it will invoke the thread handler (bandwitdh). The
8490 * chip receive interrupt is *not* cleared down until this or the thread (if
8491 * invoked) is finished. The intent is to avoid extra interrupts while we
8492 * are processing packets anyway.
8493 */
receive_context_interrupt(int irq,void * data)8494 irqreturn_t receive_context_interrupt(int irq, void *data)
8495 {
8496 struct hfi1_ctxtdata *rcd = data;
8497 int disposition;
8498
8499 receive_interrupt_common(rcd);
8500
8501 /* receive interrupt remains blocked while processing packets */
8502 disposition = rcd->do_interrupt(rcd, 0);
8503
8504 /*
8505 * Too many packets were seen while processing packets in this
8506 * IRQ handler. Invoke the handler thread. The receive interrupt
8507 * remains blocked.
8508 */
8509 if (disposition == RCV_PKT_LIMIT)
8510 return IRQ_WAKE_THREAD;
8511
8512 __hfi1_rcd_eoi_intr(rcd);
8513 return IRQ_HANDLED;
8514 }
8515
8516 /*
8517 * Receive packet thread handler. This expects to be invoked with the
8518 * receive interrupt still blocked.
8519 */
receive_context_thread(int irq,void * data)8520 irqreturn_t receive_context_thread(int irq, void *data)
8521 {
8522 struct hfi1_ctxtdata *rcd = data;
8523
8524 /* receive interrupt is still blocked from the IRQ handler */
8525 (void)rcd->do_interrupt(rcd, 1);
8526
8527 hfi1_rcd_eoi_intr(rcd);
8528
8529 return IRQ_HANDLED;
8530 }
8531
8532 /* ========================================================================= */
8533
read_physical_state(struct hfi1_devdata * dd)8534 u32 read_physical_state(struct hfi1_devdata *dd)
8535 {
8536 u64 reg;
8537
8538 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8539 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8540 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8541 }
8542
read_logical_state(struct hfi1_devdata * dd)8543 u32 read_logical_state(struct hfi1_devdata *dd)
8544 {
8545 u64 reg;
8546
8547 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8548 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8549 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8550 }
8551
set_logical_state(struct hfi1_devdata * dd,u32 chip_lstate)8552 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8553 {
8554 u64 reg;
8555
8556 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8557 /* clear current state, set new state */
8558 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8559 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8560 write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8561 }
8562
8563 /*
8564 * Use the 8051 to read a LCB CSR.
8565 */
read_lcb_via_8051(struct hfi1_devdata * dd,u32 addr,u64 * data)8566 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8567 {
8568 u32 regno;
8569 int ret;
8570
8571 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8572 if (acquire_lcb_access(dd, 0) == 0) {
8573 *data = read_csr(dd, addr);
8574 release_lcb_access(dd, 0);
8575 return 0;
8576 }
8577 return -EBUSY;
8578 }
8579
8580 /* register is an index of LCB registers: (offset - base) / 8 */
8581 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8582 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8583 if (ret != HCMD_SUCCESS)
8584 return -EBUSY;
8585 return 0;
8586 }
8587
8588 /*
8589 * Provide a cache for some of the LCB registers in case the LCB is
8590 * unavailable.
8591 * (The LCB is unavailable in certain link states, for example.)
8592 */
8593 struct lcb_datum {
8594 u32 off;
8595 u64 val;
8596 };
8597
8598 static struct lcb_datum lcb_cache[] = {
8599 { DC_LCB_ERR_INFO_RX_REPLAY_CNT, 0},
8600 { DC_LCB_ERR_INFO_SEQ_CRC_CNT, 0 },
8601 { DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 0 },
8602 };
8603
update_lcb_cache(struct hfi1_devdata * dd)8604 static void update_lcb_cache(struct hfi1_devdata *dd)
8605 {
8606 int i;
8607 int ret;
8608 u64 val;
8609
8610 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8611 ret = read_lcb_csr(dd, lcb_cache[i].off, &val);
8612
8613 /* Update if we get good data */
8614 if (likely(ret != -EBUSY))
8615 lcb_cache[i].val = val;
8616 }
8617 }
8618
read_lcb_cache(u32 off,u64 * val)8619 static int read_lcb_cache(u32 off, u64 *val)
8620 {
8621 int i;
8622
8623 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8624 if (lcb_cache[i].off == off) {
8625 *val = lcb_cache[i].val;
8626 return 0;
8627 }
8628 }
8629
8630 pr_warn("%s bad offset 0x%x\n", __func__, off);
8631 return -1;
8632 }
8633
8634 /*
8635 * Read an LCB CSR. Access may not be in host control, so check.
8636 * Return 0 on success, -EBUSY on failure.
8637 */
read_lcb_csr(struct hfi1_devdata * dd,u32 addr,u64 * data)8638 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8639 {
8640 struct hfi1_pportdata *ppd = dd->pport;
8641
8642 /* if up, go through the 8051 for the value */
8643 if (ppd->host_link_state & HLS_UP)
8644 return read_lcb_via_8051(dd, addr, data);
8645 /* if going up or down, check the cache, otherwise, no access */
8646 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) {
8647 if (read_lcb_cache(addr, data))
8648 return -EBUSY;
8649 return 0;
8650 }
8651
8652 /* otherwise, host has access */
8653 *data = read_csr(dd, addr);
8654 return 0;
8655 }
8656
8657 /*
8658 * Use the 8051 to write a LCB CSR.
8659 */
write_lcb_via_8051(struct hfi1_devdata * dd,u32 addr,u64 data)8660 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8661 {
8662 u32 regno;
8663 int ret;
8664
8665 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8666 (dd->dc8051_ver < dc8051_ver(0, 20, 0))) {
8667 if (acquire_lcb_access(dd, 0) == 0) {
8668 write_csr(dd, addr, data);
8669 release_lcb_access(dd, 0);
8670 return 0;
8671 }
8672 return -EBUSY;
8673 }
8674
8675 /* register is an index of LCB registers: (offset - base) / 8 */
8676 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8677 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8678 if (ret != HCMD_SUCCESS)
8679 return -EBUSY;
8680 return 0;
8681 }
8682
8683 /*
8684 * Write an LCB CSR. Access may not be in host control, so check.
8685 * Return 0 on success, -EBUSY on failure.
8686 */
write_lcb_csr(struct hfi1_devdata * dd,u32 addr,u64 data)8687 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8688 {
8689 struct hfi1_pportdata *ppd = dd->pport;
8690
8691 /* if up, go through the 8051 for the value */
8692 if (ppd->host_link_state & HLS_UP)
8693 return write_lcb_via_8051(dd, addr, data);
8694 /* if going up or down, no access */
8695 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8696 return -EBUSY;
8697 /* otherwise, host has access */
8698 write_csr(dd, addr, data);
8699 return 0;
8700 }
8701
8702 /*
8703 * Returns:
8704 * < 0 = Linux error, not able to get access
8705 * > 0 = 8051 command RETURN_CODE
8706 */
do_8051_command(struct hfi1_devdata * dd,u32 type,u64 in_data,u64 * out_data)8707 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
8708 u64 *out_data)
8709 {
8710 u64 reg, completed;
8711 int return_code;
8712 unsigned long timeout;
8713
8714 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8715
8716 mutex_lock(&dd->dc8051_lock);
8717
8718 /* We can't send any commands to the 8051 if it's in reset */
8719 if (dd->dc_shutdown) {
8720 return_code = -ENODEV;
8721 goto fail;
8722 }
8723
8724 /*
8725 * If an 8051 host command timed out previously, then the 8051 is
8726 * stuck.
8727 *
8728 * On first timeout, attempt to reset and restart the entire DC
8729 * block (including 8051). (Is this too big of a hammer?)
8730 *
8731 * If the 8051 times out a second time, the reset did not bring it
8732 * back to healthy life. In that case, fail any subsequent commands.
8733 */
8734 if (dd->dc8051_timed_out) {
8735 if (dd->dc8051_timed_out > 1) {
8736 dd_dev_err(dd,
8737 "Previous 8051 host command timed out, skipping command %u\n",
8738 type);
8739 return_code = -ENXIO;
8740 goto fail;
8741 }
8742 _dc_shutdown(dd);
8743 _dc_start(dd);
8744 }
8745
8746 /*
8747 * If there is no timeout, then the 8051 command interface is
8748 * waiting for a command.
8749 */
8750
8751 /*
8752 * When writing a LCB CSR, out_data contains the full value to
8753 * to be written, while in_data contains the relative LCB
8754 * address in 7:0. Do the work here, rather than the caller,
8755 * of distrubting the write data to where it needs to go:
8756 *
8757 * Write data
8758 * 39:00 -> in_data[47:8]
8759 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
8760 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
8761 */
8762 if (type == HCMD_WRITE_LCB_CSR) {
8763 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8764 /* must preserve COMPLETED - it is tied to hardware */
8765 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0);
8766 reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK;
8767 reg |= ((((*out_data) >> 40) & 0xff) <<
8768 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8769 | ((((*out_data) >> 48) & 0xffff) <<
8770 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8771 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8772 }
8773
8774 /*
8775 * Do two writes: the first to stabilize the type and req_data, the
8776 * second to activate.
8777 */
8778 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8779 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8780 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8781 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8782 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8783 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8784 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8785
8786 /* wait for completion, alternate: interrupt */
8787 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8788 while (1) {
8789 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8790 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8791 if (completed)
8792 break;
8793 if (time_after(jiffies, timeout)) {
8794 dd->dc8051_timed_out++;
8795 dd_dev_err(dd, "8051 host command %u timeout\n", type);
8796 if (out_data)
8797 *out_data = 0;
8798 return_code = -ETIMEDOUT;
8799 goto fail;
8800 }
8801 udelay(2);
8802 }
8803
8804 if (out_data) {
8805 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8806 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8807 if (type == HCMD_READ_LCB_CSR) {
8808 /* top 16 bits are in a different register */
8809 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8810 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8811 << (48
8812 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8813 }
8814 }
8815 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8816 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8817 dd->dc8051_timed_out = 0;
8818 /*
8819 * Clear command for next user.
8820 */
8821 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8822
8823 fail:
8824 mutex_unlock(&dd->dc8051_lock);
8825 return return_code;
8826 }
8827
set_physical_link_state(struct hfi1_devdata * dd,u64 state)8828 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8829 {
8830 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8831 }
8832
load_8051_config(struct hfi1_devdata * dd,u8 field_id,u8 lane_id,u32 config_data)8833 int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8834 u8 lane_id, u32 config_data)
8835 {
8836 u64 data;
8837 int ret;
8838
8839 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8840 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8841 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8842 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8843 if (ret != HCMD_SUCCESS) {
8844 dd_dev_err(dd,
8845 "load 8051 config: field id %d, lane %d, err %d\n",
8846 (int)field_id, (int)lane_id, ret);
8847 }
8848 return ret;
8849 }
8850
8851 /*
8852 * Read the 8051 firmware "registers". Use the RAM directly. Always
8853 * set the result, even on error.
8854 * Return 0 on success, -errno on failure
8855 */
read_8051_config(struct hfi1_devdata * dd,u8 field_id,u8 lane_id,u32 * result)8856 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8857 u32 *result)
8858 {
8859 u64 big_data;
8860 u32 addr;
8861 int ret;
8862
8863 /* address start depends on the lane_id */
8864 if (lane_id < 4)
8865 addr = (4 * NUM_GENERAL_FIELDS)
8866 + (lane_id * 4 * NUM_LANE_FIELDS);
8867 else
8868 addr = 0;
8869 addr += field_id * 4;
8870
8871 /* read is in 8-byte chunks, hardware will truncate the address down */
8872 ret = read_8051_data(dd, addr, 8, &big_data);
8873
8874 if (ret == 0) {
8875 /* extract the 4 bytes we want */
8876 if (addr & 0x4)
8877 *result = (u32)(big_data >> 32);
8878 else
8879 *result = (u32)big_data;
8880 } else {
8881 *result = 0;
8882 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
8883 __func__, lane_id, field_id);
8884 }
8885
8886 return ret;
8887 }
8888
write_vc_local_phy(struct hfi1_devdata * dd,u8 power_management,u8 continuous)8889 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8890 u8 continuous)
8891 {
8892 u32 frame;
8893
8894 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8895 | power_management << POWER_MANAGEMENT_SHIFT;
8896 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8897 GENERAL_CONFIG, frame);
8898 }
8899
write_vc_local_fabric(struct hfi1_devdata * dd,u8 vau,u8 z,u8 vcu,u16 vl15buf,u8 crc_sizes)8900 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8901 u16 vl15buf, u8 crc_sizes)
8902 {
8903 u32 frame;
8904
8905 frame = (u32)vau << VAU_SHIFT
8906 | (u32)z << Z_SHIFT
8907 | (u32)vcu << VCU_SHIFT
8908 | (u32)vl15buf << VL15BUF_SHIFT
8909 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8910 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8911 GENERAL_CONFIG, frame);
8912 }
8913
read_vc_local_link_mode(struct hfi1_devdata * dd,u8 * misc_bits,u8 * flag_bits,u16 * link_widths)8914 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
8915 u8 *flag_bits, u16 *link_widths)
8916 {
8917 u32 frame;
8918
8919 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8920 &frame);
8921 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8922 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8923 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8924 }
8925
write_vc_local_link_mode(struct hfi1_devdata * dd,u8 misc_bits,u8 flag_bits,u16 link_widths)8926 static int write_vc_local_link_mode(struct hfi1_devdata *dd,
8927 u8 misc_bits,
8928 u8 flag_bits,
8929 u16 link_widths)
8930 {
8931 u32 frame;
8932
8933 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8934 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8935 | (u32)link_widths << LINK_WIDTH_SHIFT;
8936 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8937 frame);
8938 }
8939
write_local_device_id(struct hfi1_devdata * dd,u16 device_id,u8 device_rev)8940 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8941 u8 device_rev)
8942 {
8943 u32 frame;
8944
8945 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8946 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8947 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8948 }
8949
read_remote_device_id(struct hfi1_devdata * dd,u16 * device_id,u8 * device_rev)8950 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8951 u8 *device_rev)
8952 {
8953 u32 frame;
8954
8955 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8956 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8957 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8958 & REMOTE_DEVICE_REV_MASK;
8959 }
8960
write_host_interface_version(struct hfi1_devdata * dd,u8 version)8961 int write_host_interface_version(struct hfi1_devdata *dd, u8 version)
8962 {
8963 u32 frame;
8964 u32 mask;
8965
8966 mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT);
8967 read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame);
8968 /* Clear, then set field */
8969 frame &= ~mask;
8970 frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT);
8971 return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG,
8972 frame);
8973 }
8974
read_misc_status(struct hfi1_devdata * dd,u8 * ver_major,u8 * ver_minor,u8 * ver_patch)8975 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
8976 u8 *ver_patch)
8977 {
8978 u32 frame;
8979
8980 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8981 *ver_major = (frame >> STS_FM_VERSION_MAJOR_SHIFT) &
8982 STS_FM_VERSION_MAJOR_MASK;
8983 *ver_minor = (frame >> STS_FM_VERSION_MINOR_SHIFT) &
8984 STS_FM_VERSION_MINOR_MASK;
8985
8986 read_8051_config(dd, VERSION_PATCH, GENERAL_CONFIG, &frame);
8987 *ver_patch = (frame >> STS_FM_VERSION_PATCH_SHIFT) &
8988 STS_FM_VERSION_PATCH_MASK;
8989 }
8990
read_vc_remote_phy(struct hfi1_devdata * dd,u8 * power_management,u8 * continuous)8991 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8992 u8 *continuous)
8993 {
8994 u32 frame;
8995
8996 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8997 *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
8998 & POWER_MANAGEMENT_MASK;
8999 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
9000 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
9001 }
9002
read_vc_remote_fabric(struct hfi1_devdata * dd,u8 * vau,u8 * z,u8 * vcu,u16 * vl15buf,u8 * crc_sizes)9003 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
9004 u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
9005 {
9006 u32 frame;
9007
9008 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
9009 *vau = (frame >> VAU_SHIFT) & VAU_MASK;
9010 *z = (frame >> Z_SHIFT) & Z_MASK;
9011 *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
9012 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
9013 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
9014 }
9015
read_vc_remote_link_width(struct hfi1_devdata * dd,u8 * remote_tx_rate,u16 * link_widths)9016 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
9017 u8 *remote_tx_rate,
9018 u16 *link_widths)
9019 {
9020 u32 frame;
9021
9022 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
9023 &frame);
9024 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
9025 & REMOTE_TX_RATE_MASK;
9026 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
9027 }
9028
read_local_lni(struct hfi1_devdata * dd,u8 * enable_lane_rx)9029 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
9030 {
9031 u32 frame;
9032
9033 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
9034 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
9035 }
9036
read_last_local_state(struct hfi1_devdata * dd,u32 * lls)9037 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
9038 {
9039 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
9040 }
9041
read_last_remote_state(struct hfi1_devdata * dd,u32 * lrs)9042 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
9043 {
9044 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
9045 }
9046
hfi1_read_link_quality(struct hfi1_devdata * dd,u8 * link_quality)9047 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
9048 {
9049 u32 frame;
9050 int ret;
9051
9052 *link_quality = 0;
9053 if (dd->pport->host_link_state & HLS_UP) {
9054 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
9055 &frame);
9056 if (ret == 0)
9057 *link_quality = (frame >> LINK_QUALITY_SHIFT)
9058 & LINK_QUALITY_MASK;
9059 }
9060 }
9061
read_planned_down_reason_code(struct hfi1_devdata * dd,u8 * pdrrc)9062 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
9063 {
9064 u32 frame;
9065
9066 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
9067 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
9068 }
9069
read_link_down_reason(struct hfi1_devdata * dd,u8 * ldr)9070 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr)
9071 {
9072 u32 frame;
9073
9074 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame);
9075 *ldr = (frame & 0xff);
9076 }
9077
read_tx_settings(struct hfi1_devdata * dd,u8 * enable_lane_tx,u8 * tx_polarity_inversion,u8 * rx_polarity_inversion,u8 * max_rate)9078 static int read_tx_settings(struct hfi1_devdata *dd,
9079 u8 *enable_lane_tx,
9080 u8 *tx_polarity_inversion,
9081 u8 *rx_polarity_inversion,
9082 u8 *max_rate)
9083 {
9084 u32 frame;
9085 int ret;
9086
9087 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
9088 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
9089 & ENABLE_LANE_TX_MASK;
9090 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
9091 & TX_POLARITY_INVERSION_MASK;
9092 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
9093 & RX_POLARITY_INVERSION_MASK;
9094 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
9095 return ret;
9096 }
9097
write_tx_settings(struct hfi1_devdata * dd,u8 enable_lane_tx,u8 tx_polarity_inversion,u8 rx_polarity_inversion,u8 max_rate)9098 static int write_tx_settings(struct hfi1_devdata *dd,
9099 u8 enable_lane_tx,
9100 u8 tx_polarity_inversion,
9101 u8 rx_polarity_inversion,
9102 u8 max_rate)
9103 {
9104 u32 frame;
9105
9106 /* no need to mask, all variable sizes match field widths */
9107 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
9108 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
9109 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
9110 | max_rate << MAX_RATE_SHIFT;
9111 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
9112 }
9113
9114 /*
9115 * Read an idle LCB message.
9116 *
9117 * Returns 0 on success, -EINVAL on error
9118 */
read_idle_message(struct hfi1_devdata * dd,u64 type,u64 * data_out)9119 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
9120 {
9121 int ret;
9122
9123 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
9124 if (ret != HCMD_SUCCESS) {
9125 dd_dev_err(dd, "read idle message: type %d, err %d\n",
9126 (u32)type, ret);
9127 return -EINVAL;
9128 }
9129 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
9130 /* return only the payload as we already know the type */
9131 *data_out >>= IDLE_PAYLOAD_SHIFT;
9132 return 0;
9133 }
9134
9135 /*
9136 * Read an idle SMA message. To be done in response to a notification from
9137 * the 8051.
9138 *
9139 * Returns 0 on success, -EINVAL on error
9140 */
read_idle_sma(struct hfi1_devdata * dd,u64 * data)9141 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
9142 {
9143 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
9144 data);
9145 }
9146
9147 /*
9148 * Send an idle LCB message.
9149 *
9150 * Returns 0 on success, -EINVAL on error
9151 */
send_idle_message(struct hfi1_devdata * dd,u64 data)9152 static int send_idle_message(struct hfi1_devdata *dd, u64 data)
9153 {
9154 int ret;
9155
9156 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
9157 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
9158 if (ret != HCMD_SUCCESS) {
9159 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
9160 data, ret);
9161 return -EINVAL;
9162 }
9163 return 0;
9164 }
9165
9166 /*
9167 * Send an idle SMA message.
9168 *
9169 * Returns 0 on success, -EINVAL on error
9170 */
send_idle_sma(struct hfi1_devdata * dd,u64 message)9171 int send_idle_sma(struct hfi1_devdata *dd, u64 message)
9172 {
9173 u64 data;
9174
9175 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
9176 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
9177 return send_idle_message(dd, data);
9178 }
9179
9180 /*
9181 * Initialize the LCB then do a quick link up. This may or may not be
9182 * in loopback.
9183 *
9184 * return 0 on success, -errno on error
9185 */
do_quick_linkup(struct hfi1_devdata * dd)9186 static int do_quick_linkup(struct hfi1_devdata *dd)
9187 {
9188 int ret;
9189
9190 lcb_shutdown(dd, 0);
9191
9192 if (loopback) {
9193 /* LCB_CFG_LOOPBACK.VAL = 2 */
9194 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
9195 write_csr(dd, DC_LCB_CFG_LOOPBACK,
9196 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
9197 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
9198 }
9199
9200 /* start the LCBs */
9201 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
9202 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
9203
9204 /* simulator only loopback steps */
9205 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
9206 /* LCB_CFG_RUN.EN = 1 */
9207 write_csr(dd, DC_LCB_CFG_RUN,
9208 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
9209
9210 ret = wait_link_transfer_active(dd, 10);
9211 if (ret)
9212 return ret;
9213
9214 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
9215 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
9216 }
9217
9218 if (!loopback) {
9219 /*
9220 * When doing quick linkup and not in loopback, both
9221 * sides must be done with LCB set-up before either
9222 * starts the quick linkup. Put a delay here so that
9223 * both sides can be started and have a chance to be
9224 * done with LCB set up before resuming.
9225 */
9226 dd_dev_err(dd,
9227 "Pausing for peer to be finished with LCB set up\n");
9228 msleep(5000);
9229 dd_dev_err(dd, "Continuing with quick linkup\n");
9230 }
9231
9232 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
9233 set_8051_lcb_access(dd);
9234
9235 /*
9236 * State "quick" LinkUp request sets the physical link state to
9237 * LinkUp without a verify capability sequence.
9238 * This state is in simulator v37 and later.
9239 */
9240 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
9241 if (ret != HCMD_SUCCESS) {
9242 dd_dev_err(dd,
9243 "%s: set physical link state to quick LinkUp failed with return %d\n",
9244 __func__, ret);
9245
9246 set_host_lcb_access(dd);
9247 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
9248
9249 if (ret >= 0)
9250 ret = -EINVAL;
9251 return ret;
9252 }
9253
9254 return 0; /* success */
9255 }
9256
9257 /*
9258 * Do all special steps to set up loopback.
9259 */
init_loopback(struct hfi1_devdata * dd)9260 static int init_loopback(struct hfi1_devdata *dd)
9261 {
9262 dd_dev_info(dd, "Entering loopback mode\n");
9263
9264 /* all loopbacks should disable self GUID check */
9265 write_csr(dd, DC_DC8051_CFG_MODE,
9266 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
9267
9268 /*
9269 * The simulator has only one loopback option - LCB. Switch
9270 * to that option, which includes quick link up.
9271 *
9272 * Accept all valid loopback values.
9273 */
9274 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
9275 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
9276 loopback == LOOPBACK_CABLE)) {
9277 loopback = LOOPBACK_LCB;
9278 quick_linkup = 1;
9279 return 0;
9280 }
9281
9282 /*
9283 * SerDes loopback init sequence is handled in set_local_link_attributes
9284 */
9285 if (loopback == LOOPBACK_SERDES)
9286 return 0;
9287
9288 /* LCB loopback - handled at poll time */
9289 if (loopback == LOOPBACK_LCB) {
9290 quick_linkup = 1; /* LCB is always quick linkup */
9291
9292 /* not supported in emulation due to emulation RTL changes */
9293 if (dd->icode == ICODE_FPGA_EMULATION) {
9294 dd_dev_err(dd,
9295 "LCB loopback not supported in emulation\n");
9296 return -EINVAL;
9297 }
9298 return 0;
9299 }
9300
9301 /* external cable loopback requires no extra steps */
9302 if (loopback == LOOPBACK_CABLE)
9303 return 0;
9304
9305 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
9306 return -EINVAL;
9307 }
9308
9309 /*
9310 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
9311 * used in the Verify Capability link width attribute.
9312 */
opa_to_vc_link_widths(u16 opa_widths)9313 static u16 opa_to_vc_link_widths(u16 opa_widths)
9314 {
9315 int i;
9316 u16 result = 0;
9317
9318 static const struct link_bits {
9319 u16 from;
9320 u16 to;
9321 } opa_link_xlate[] = {
9322 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) },
9323 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) },
9324 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) },
9325 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) },
9326 };
9327
9328 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
9329 if (opa_widths & opa_link_xlate[i].from)
9330 result |= opa_link_xlate[i].to;
9331 }
9332 return result;
9333 }
9334
9335 /*
9336 * Set link attributes before moving to polling.
9337 */
set_local_link_attributes(struct hfi1_pportdata * ppd)9338 static int set_local_link_attributes(struct hfi1_pportdata *ppd)
9339 {
9340 struct hfi1_devdata *dd = ppd->dd;
9341 u8 enable_lane_tx;
9342 u8 tx_polarity_inversion;
9343 u8 rx_polarity_inversion;
9344 int ret;
9345 u32 misc_bits = 0;
9346 /* reset our fabric serdes to clear any lingering problems */
9347 fabric_serdes_reset(dd);
9348
9349 /* set the local tx rate - need to read-modify-write */
9350 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
9351 &rx_polarity_inversion, &ppd->local_tx_rate);
9352 if (ret)
9353 goto set_local_link_attributes_fail;
9354
9355 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
9356 /* set the tx rate to the fastest enabled */
9357 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9358 ppd->local_tx_rate = 1;
9359 else
9360 ppd->local_tx_rate = 0;
9361 } else {
9362 /* set the tx rate to all enabled */
9363 ppd->local_tx_rate = 0;
9364 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9365 ppd->local_tx_rate |= 2;
9366 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
9367 ppd->local_tx_rate |= 1;
9368 }
9369
9370 enable_lane_tx = 0xF; /* enable all four lanes */
9371 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
9372 rx_polarity_inversion, ppd->local_tx_rate);
9373 if (ret != HCMD_SUCCESS)
9374 goto set_local_link_attributes_fail;
9375
9376 ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
9377 if (ret != HCMD_SUCCESS) {
9378 dd_dev_err(dd,
9379 "Failed to set host interface version, return 0x%x\n",
9380 ret);
9381 goto set_local_link_attributes_fail;
9382 }
9383
9384 /*
9385 * DC supports continuous updates.
9386 */
9387 ret = write_vc_local_phy(dd,
9388 0 /* no power management */,
9389 1 /* continuous updates */);
9390 if (ret != HCMD_SUCCESS)
9391 goto set_local_link_attributes_fail;
9392
9393 /* z=1 in the next call: AU of 0 is not supported by the hardware */
9394 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9395 ppd->port_crc_mode_enabled);
9396 if (ret != HCMD_SUCCESS)
9397 goto set_local_link_attributes_fail;
9398
9399 /*
9400 * SerDes loopback init sequence requires
9401 * setting bit 0 of MISC_CONFIG_BITS
9402 */
9403 if (loopback == LOOPBACK_SERDES)
9404 misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT;
9405
9406 /*
9407 * An external device configuration request is used to reset the LCB
9408 * to retry to obtain operational lanes when the first attempt is
9409 * unsuccesful.
9410 */
9411 if (dd->dc8051_ver >= dc8051_ver(1, 25, 0))
9412 misc_bits |= 1 << EXT_CFG_LCB_RESET_SUPPORTED_SHIFT;
9413
9414 ret = write_vc_local_link_mode(dd, misc_bits, 0,
9415 opa_to_vc_link_widths(
9416 ppd->link_width_enabled));
9417 if (ret != HCMD_SUCCESS)
9418 goto set_local_link_attributes_fail;
9419
9420 /* let peer know who we are */
9421 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9422 if (ret == HCMD_SUCCESS)
9423 return 0;
9424
9425 set_local_link_attributes_fail:
9426 dd_dev_err(dd,
9427 "Failed to set local link attributes, return 0x%x\n",
9428 ret);
9429 return ret;
9430 }
9431
9432 /*
9433 * Call this to start the link.
9434 * Do not do anything if the link is disabled.
9435 * Returns 0 if link is disabled, moved to polling, or the driver is not ready.
9436 */
start_link(struct hfi1_pportdata * ppd)9437 int start_link(struct hfi1_pportdata *ppd)
9438 {
9439 /*
9440 * Tune the SerDes to a ballpark setting for optimal signal and bit
9441 * error rate. Needs to be done before starting the link.
9442 */
9443 tune_serdes(ppd);
9444
9445 if (!ppd->driver_link_ready) {
9446 dd_dev_info(ppd->dd,
9447 "%s: stopping link start because driver is not ready\n",
9448 __func__);
9449 return 0;
9450 }
9451
9452 /*
9453 * FULL_MGMT_P_KEY is cleared from the pkey table, so that the
9454 * pkey table can be configured properly if the HFI unit is connected
9455 * to switch port with MgmtAllowed=NO
9456 */
9457 clear_full_mgmt_pkey(ppd);
9458
9459 return set_link_state(ppd, HLS_DN_POLL);
9460 }
9461
wait_for_qsfp_init(struct hfi1_pportdata * ppd)9462 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9463 {
9464 struct hfi1_devdata *dd = ppd->dd;
9465 u64 mask;
9466 unsigned long timeout;
9467
9468 /*
9469 * Some QSFP cables have a quirk that asserts the IntN line as a side
9470 * effect of power up on plug-in. We ignore this false positive
9471 * interrupt until the module has finished powering up by waiting for
9472 * a minimum timeout of the module inrush initialization time of
9473 * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the
9474 * module have stabilized.
9475 */
9476 msleep(500);
9477
9478 /*
9479 * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1)
9480 */
9481 timeout = jiffies + msecs_to_jiffies(2000);
9482 while (1) {
9483 mask = read_csr(dd, dd->hfi1_id ?
9484 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9485 if (!(mask & QSFP_HFI0_INT_N))
9486 break;
9487 if (time_after(jiffies, timeout)) {
9488 dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9489 __func__);
9490 break;
9491 }
9492 udelay(2);
9493 }
9494 }
9495
set_qsfp_int_n(struct hfi1_pportdata * ppd,u8 enable)9496 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9497 {
9498 struct hfi1_devdata *dd = ppd->dd;
9499 u64 mask;
9500
9501 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9502 if (enable) {
9503 /*
9504 * Clear the status register to avoid an immediate interrupt
9505 * when we re-enable the IntN pin
9506 */
9507 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9508 QSFP_HFI0_INT_N);
9509 mask |= (u64)QSFP_HFI0_INT_N;
9510 } else {
9511 mask &= ~(u64)QSFP_HFI0_INT_N;
9512 }
9513 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9514 }
9515
reset_qsfp(struct hfi1_pportdata * ppd)9516 int reset_qsfp(struct hfi1_pportdata *ppd)
9517 {
9518 struct hfi1_devdata *dd = ppd->dd;
9519 u64 mask, qsfp_mask;
9520
9521 /* Disable INT_N from triggering QSFP interrupts */
9522 set_qsfp_int_n(ppd, 0);
9523
9524 /* Reset the QSFP */
9525 mask = (u64)QSFP_HFI0_RESET_N;
9526
9527 qsfp_mask = read_csr(dd,
9528 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
9529 qsfp_mask &= ~mask;
9530 write_csr(dd,
9531 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9532
9533 udelay(10);
9534
9535 qsfp_mask |= mask;
9536 write_csr(dd,
9537 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9538
9539 wait_for_qsfp_init(ppd);
9540
9541 /*
9542 * Allow INT_N to trigger the QSFP interrupt to watch
9543 * for alarms and warnings
9544 */
9545 set_qsfp_int_n(ppd, 1);
9546
9547 /*
9548 * After the reset, AOC transmitters are enabled by default. They need
9549 * to be turned off to complete the QSFP setup before they can be
9550 * enabled again.
9551 */
9552 return set_qsfp_tx(ppd, 0);
9553 }
9554
handle_qsfp_error_conditions(struct hfi1_pportdata * ppd,u8 * qsfp_interrupt_status)9555 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9556 u8 *qsfp_interrupt_status)
9557 {
9558 struct hfi1_devdata *dd = ppd->dd;
9559
9560 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
9561 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9562 dd_dev_err(dd, "%s: QSFP cable temperature too high\n",
9563 __func__);
9564
9565 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
9566 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9567 dd_dev_err(dd, "%s: QSFP cable temperature too low\n",
9568 __func__);
9569
9570 /*
9571 * The remaining alarms/warnings don't matter if the link is down.
9572 */
9573 if (ppd->host_link_state & HLS_DOWN)
9574 return 0;
9575
9576 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
9577 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9578 dd_dev_err(dd, "%s: QSFP supply voltage too high\n",
9579 __func__);
9580
9581 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
9582 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9583 dd_dev_err(dd, "%s: QSFP supply voltage too low\n",
9584 __func__);
9585
9586 /* Byte 2 is vendor specific */
9587
9588 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
9589 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9590 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n",
9591 __func__);
9592
9593 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
9594 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9595 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n",
9596 __func__);
9597
9598 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
9599 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9600 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n",
9601 __func__);
9602
9603 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
9604 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9605 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n",
9606 __func__);
9607
9608 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
9609 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9610 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n",
9611 __func__);
9612
9613 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
9614 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9615 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n",
9616 __func__);
9617
9618 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
9619 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9620 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n",
9621 __func__);
9622
9623 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
9624 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9625 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n",
9626 __func__);
9627
9628 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
9629 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9630 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n",
9631 __func__);
9632
9633 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
9634 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9635 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n",
9636 __func__);
9637
9638 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
9639 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9640 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n",
9641 __func__);
9642
9643 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
9644 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9645 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n",
9646 __func__);
9647
9648 /* Bytes 9-10 and 11-12 are reserved */
9649 /* Bytes 13-15 are vendor specific */
9650
9651 return 0;
9652 }
9653
9654 /* This routine will only be scheduled if the QSFP module present is asserted */
qsfp_event(struct work_struct * work)9655 void qsfp_event(struct work_struct *work)
9656 {
9657 struct qsfp_data *qd;
9658 struct hfi1_pportdata *ppd;
9659 struct hfi1_devdata *dd;
9660
9661 qd = container_of(work, struct qsfp_data, qsfp_work);
9662 ppd = qd->ppd;
9663 dd = ppd->dd;
9664
9665 /* Sanity check */
9666 if (!qsfp_mod_present(ppd))
9667 return;
9668
9669 if (ppd->host_link_state == HLS_DN_DISABLE) {
9670 dd_dev_info(ppd->dd,
9671 "%s: stopping link start because link is disabled\n",
9672 __func__);
9673 return;
9674 }
9675
9676 /*
9677 * Turn DC back on after cable has been re-inserted. Up until
9678 * now, the DC has been in reset to save power.
9679 */
9680 dc_start(dd);
9681
9682 if (qd->cache_refresh_required) {
9683 set_qsfp_int_n(ppd, 0);
9684
9685 wait_for_qsfp_init(ppd);
9686
9687 /*
9688 * Allow INT_N to trigger the QSFP interrupt to watch
9689 * for alarms and warnings
9690 */
9691 set_qsfp_int_n(ppd, 1);
9692
9693 start_link(ppd);
9694 }
9695
9696 if (qd->check_interrupt_flags) {
9697 u8 qsfp_interrupt_status[16] = {0,};
9698
9699 if (one_qsfp_read(ppd, dd->hfi1_id, 6,
9700 &qsfp_interrupt_status[0], 16) != 16) {
9701 dd_dev_info(dd,
9702 "%s: Failed to read status of QSFP module\n",
9703 __func__);
9704 } else {
9705 unsigned long flags;
9706
9707 handle_qsfp_error_conditions(
9708 ppd, qsfp_interrupt_status);
9709 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9710 ppd->qsfp_info.check_interrupt_flags = 0;
9711 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
9712 flags);
9713 }
9714 }
9715 }
9716
init_qsfp_int(struct hfi1_devdata * dd)9717 void init_qsfp_int(struct hfi1_devdata *dd)
9718 {
9719 struct hfi1_pportdata *ppd = dd->pport;
9720 u64 qsfp_mask;
9721
9722 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9723 /* Clear current status to avoid spurious interrupts */
9724 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9725 qsfp_mask);
9726 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9727 qsfp_mask);
9728
9729 set_qsfp_int_n(ppd, 0);
9730
9731 /* Handle active low nature of INT_N and MODPRST_N pins */
9732 if (qsfp_mod_present(ppd))
9733 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9734 write_csr(dd,
9735 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9736 qsfp_mask);
9737
9738 /* Enable the appropriate QSFP IRQ source */
9739 if (!dd->hfi1_id)
9740 set_intr_bits(dd, QSFP1_INT, QSFP1_INT, true);
9741 else
9742 set_intr_bits(dd, QSFP2_INT, QSFP2_INT, true);
9743 }
9744
9745 /*
9746 * Do a one-time initialize of the LCB block.
9747 */
init_lcb(struct hfi1_devdata * dd)9748 static void init_lcb(struct hfi1_devdata *dd)
9749 {
9750 /* simulator does not correctly handle LCB cclk loopback, skip */
9751 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9752 return;
9753
9754 /* the DC has been reset earlier in the driver load */
9755
9756 /* set LCB for cclk loopback on the port */
9757 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9758 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9759 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9760 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9761 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9762 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9763 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9764 }
9765
9766 /*
9767 * Perform a test read on the QSFP. Return 0 on success, -ERRNO
9768 * on error.
9769 */
test_qsfp_read(struct hfi1_pportdata * ppd)9770 static int test_qsfp_read(struct hfi1_pportdata *ppd)
9771 {
9772 int ret;
9773 u8 status;
9774
9775 /*
9776 * Report success if not a QSFP or, if it is a QSFP, but the cable is
9777 * not present
9778 */
9779 if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd))
9780 return 0;
9781
9782 /* read byte 2, the status byte */
9783 ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1);
9784 if (ret < 0)
9785 return ret;
9786 if (ret != 1)
9787 return -EIO;
9788
9789 return 0; /* success */
9790 }
9791
9792 /*
9793 * Values for QSFP retry.
9794 *
9795 * Give up after 10s (20 x 500ms). The overall timeout was empirically
9796 * arrived at from experience on a large cluster.
9797 */
9798 #define MAX_QSFP_RETRIES 20
9799 #define QSFP_RETRY_WAIT 500 /* msec */
9800
9801 /*
9802 * Try a QSFP read. If it fails, schedule a retry for later.
9803 * Called on first link activation after driver load.
9804 */
try_start_link(struct hfi1_pportdata * ppd)9805 static void try_start_link(struct hfi1_pportdata *ppd)
9806 {
9807 if (test_qsfp_read(ppd)) {
9808 /* read failed */
9809 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) {
9810 dd_dev_err(ppd->dd, "QSFP not responding, giving up\n");
9811 return;
9812 }
9813 dd_dev_info(ppd->dd,
9814 "QSFP not responding, waiting and retrying %d\n",
9815 (int)ppd->qsfp_retry_count);
9816 ppd->qsfp_retry_count++;
9817 queue_delayed_work(ppd->link_wq, &ppd->start_link_work,
9818 msecs_to_jiffies(QSFP_RETRY_WAIT));
9819 return;
9820 }
9821 ppd->qsfp_retry_count = 0;
9822
9823 start_link(ppd);
9824 }
9825
9826 /*
9827 * Workqueue function to start the link after a delay.
9828 */
handle_start_link(struct work_struct * work)9829 void handle_start_link(struct work_struct *work)
9830 {
9831 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
9832 start_link_work.work);
9833 try_start_link(ppd);
9834 }
9835
bringup_serdes(struct hfi1_pportdata * ppd)9836 int bringup_serdes(struct hfi1_pportdata *ppd)
9837 {
9838 struct hfi1_devdata *dd = ppd->dd;
9839 u64 guid;
9840 int ret;
9841
9842 if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9843 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9844
9845 guid = ppd->guids[HFI1_PORT_GUID_INDEX];
9846 if (!guid) {
9847 if (dd->base_guid)
9848 guid = dd->base_guid + ppd->port - 1;
9849 ppd->guids[HFI1_PORT_GUID_INDEX] = guid;
9850 }
9851
9852 /* Set linkinit_reason on power up per OPA spec */
9853 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9854
9855 /* one-time init of the LCB */
9856 init_lcb(dd);
9857
9858 if (loopback) {
9859 ret = init_loopback(dd);
9860 if (ret < 0)
9861 return ret;
9862 }
9863
9864 get_port_type(ppd);
9865 if (ppd->port_type == PORT_TYPE_QSFP) {
9866 set_qsfp_int_n(ppd, 0);
9867 wait_for_qsfp_init(ppd);
9868 set_qsfp_int_n(ppd, 1);
9869 }
9870
9871 try_start_link(ppd);
9872 return 0;
9873 }
9874
hfi1_quiet_serdes(struct hfi1_pportdata * ppd)9875 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9876 {
9877 struct hfi1_devdata *dd = ppd->dd;
9878
9879 /*
9880 * Shut down the link and keep it down. First turn off that the
9881 * driver wants to allow the link to be up (driver_link_ready).
9882 * Then make sure the link is not automatically restarted
9883 * (link_enabled). Cancel any pending restart. And finally
9884 * go offline.
9885 */
9886 ppd->driver_link_ready = 0;
9887 ppd->link_enabled = 0;
9888
9889 ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */
9890 flush_delayed_work(&ppd->start_link_work);
9891 cancel_delayed_work_sync(&ppd->start_link_work);
9892
9893 ppd->offline_disabled_reason =
9894 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_REBOOT);
9895 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_REBOOT, 0,
9896 OPA_LINKDOWN_REASON_REBOOT);
9897 set_link_state(ppd, HLS_DN_OFFLINE);
9898
9899 /* disable the port */
9900 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9901 cancel_work_sync(&ppd->freeze_work);
9902 }
9903
init_cpu_counters(struct hfi1_devdata * dd)9904 static inline int init_cpu_counters(struct hfi1_devdata *dd)
9905 {
9906 struct hfi1_pportdata *ppd;
9907 int i;
9908
9909 ppd = (struct hfi1_pportdata *)(dd + 1);
9910 for (i = 0; i < dd->num_pports; i++, ppd++) {
9911 ppd->ibport_data.rvp.rc_acks = NULL;
9912 ppd->ibport_data.rvp.rc_qacks = NULL;
9913 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9914 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9915 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9916 if (!ppd->ibport_data.rvp.rc_acks ||
9917 !ppd->ibport_data.rvp.rc_delayed_comp ||
9918 !ppd->ibport_data.rvp.rc_qacks)
9919 return -ENOMEM;
9920 }
9921
9922 return 0;
9923 }
9924
9925 /*
9926 * index is the index into the receive array
9927 */
hfi1_put_tid(struct hfi1_devdata * dd,u32 index,u32 type,unsigned long pa,u16 order)9928 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9929 u32 type, unsigned long pa, u16 order)
9930 {
9931 u64 reg;
9932
9933 if (!(dd->flags & HFI1_PRESENT))
9934 goto done;
9935
9936 if (type == PT_INVALID || type == PT_INVALID_FLUSH) {
9937 pa = 0;
9938 order = 0;
9939 } else if (type > PT_INVALID) {
9940 dd_dev_err(dd,
9941 "unexpected receive array type %u for index %u, not handled\n",
9942 type, index);
9943 goto done;
9944 }
9945 trace_hfi1_put_tid(dd, index, type, pa, order);
9946
9947 #define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */
9948 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9949 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9950 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9951 << RCV_ARRAY_RT_ADDR_SHIFT;
9952 trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg);
9953 writeq(reg, dd->rcvarray_wc + (index * 8));
9954
9955 if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3)
9956 /*
9957 * Eager entries are written and flushed
9958 *
9959 * Expected entries are flushed every 4 writes
9960 */
9961 flush_wc();
9962 done:
9963 return;
9964 }
9965
hfi1_clear_tids(struct hfi1_ctxtdata * rcd)9966 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9967 {
9968 struct hfi1_devdata *dd = rcd->dd;
9969 u32 i;
9970
9971 /* this could be optimized */
9972 for (i = rcd->eager_base; i < rcd->eager_base +
9973 rcd->egrbufs.alloced; i++)
9974 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9975
9976 for (i = rcd->expected_base;
9977 i < rcd->expected_base + rcd->expected_count; i++)
9978 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9979 }
9980
9981 static const char * const ib_cfg_name_strings[] = {
9982 "HFI1_IB_CFG_LIDLMC",
9983 "HFI1_IB_CFG_LWID_DG_ENB",
9984 "HFI1_IB_CFG_LWID_ENB",
9985 "HFI1_IB_CFG_LWID",
9986 "HFI1_IB_CFG_SPD_ENB",
9987 "HFI1_IB_CFG_SPD",
9988 "HFI1_IB_CFG_RXPOL_ENB",
9989 "HFI1_IB_CFG_LREV_ENB",
9990 "HFI1_IB_CFG_LINKLATENCY",
9991 "HFI1_IB_CFG_HRTBT",
9992 "HFI1_IB_CFG_OP_VLS",
9993 "HFI1_IB_CFG_VL_HIGH_CAP",
9994 "HFI1_IB_CFG_VL_LOW_CAP",
9995 "HFI1_IB_CFG_OVERRUN_THRESH",
9996 "HFI1_IB_CFG_PHYERR_THRESH",
9997 "HFI1_IB_CFG_LINKDEFAULT",
9998 "HFI1_IB_CFG_PKEYS",
9999 "HFI1_IB_CFG_MTU",
10000 "HFI1_IB_CFG_LSTATE",
10001 "HFI1_IB_CFG_VL_HIGH_LIMIT",
10002 "HFI1_IB_CFG_PMA_TICKS",
10003 "HFI1_IB_CFG_PORT"
10004 };
10005
ib_cfg_name(int which)10006 static const char *ib_cfg_name(int which)
10007 {
10008 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
10009 return "invalid";
10010 return ib_cfg_name_strings[which];
10011 }
10012
hfi1_get_ib_cfg(struct hfi1_pportdata * ppd,int which)10013 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
10014 {
10015 struct hfi1_devdata *dd = ppd->dd;
10016 int val = 0;
10017
10018 switch (which) {
10019 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
10020 val = ppd->link_width_enabled;
10021 break;
10022 case HFI1_IB_CFG_LWID: /* currently active Link-width */
10023 val = ppd->link_width_active;
10024 break;
10025 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
10026 val = ppd->link_speed_enabled;
10027 break;
10028 case HFI1_IB_CFG_SPD: /* current Link speed */
10029 val = ppd->link_speed_active;
10030 break;
10031
10032 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
10033 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
10034 case HFI1_IB_CFG_LINKLATENCY:
10035 goto unimplemented;
10036
10037 case HFI1_IB_CFG_OP_VLS:
10038 val = ppd->actual_vls_operational;
10039 break;
10040 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
10041 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
10042 break;
10043 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
10044 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
10045 break;
10046 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
10047 val = ppd->overrun_threshold;
10048 break;
10049 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
10050 val = ppd->phy_error_threshold;
10051 break;
10052 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10053 val = HLS_DEFAULT;
10054 break;
10055
10056 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
10057 case HFI1_IB_CFG_PMA_TICKS:
10058 default:
10059 unimplemented:
10060 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10061 dd_dev_info(
10062 dd,
10063 "%s: which %s: not implemented\n",
10064 __func__,
10065 ib_cfg_name(which));
10066 break;
10067 }
10068
10069 return val;
10070 }
10071
10072 /*
10073 * The largest MAD packet size.
10074 */
10075 #define MAX_MAD_PACKET 2048
10076
10077 /*
10078 * Return the maximum header bytes that can go on the _wire_
10079 * for this device. This count includes the ICRC which is
10080 * not part of the packet held in memory but it is appended
10081 * by the HW.
10082 * This is dependent on the device's receive header entry size.
10083 * HFI allows this to be set per-receive context, but the
10084 * driver presently enforces a global value.
10085 */
lrh_max_header_bytes(struct hfi1_devdata * dd)10086 u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
10087 {
10088 /*
10089 * The maximum non-payload (MTU) bytes in LRH.PktLen are
10090 * the Receive Header Entry Size minus the PBC (or RHF) size
10091 * plus one DW for the ICRC appended by HW.
10092 *
10093 * dd->rcd[0].rcvhdrqentsize is in DW.
10094 * We use rcd[0] as all context will have the same value. Also,
10095 * the first kernel context would have been allocated by now so
10096 * we are guaranteed a valid value.
10097 */
10098 return (get_hdrqentsize(dd->rcd[0]) - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
10099 }
10100
10101 /*
10102 * Set Send Length
10103 * @ppd: per port data
10104 *
10105 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck*
10106 * registers compare against LRH.PktLen, so use the max bytes included
10107 * in the LRH.
10108 *
10109 * This routine changes all VL values except VL15, which it maintains at
10110 * the same value.
10111 */
set_send_length(struct hfi1_pportdata * ppd)10112 static void set_send_length(struct hfi1_pportdata *ppd)
10113 {
10114 struct hfi1_devdata *dd = ppd->dd;
10115 u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
10116 u32 maxvlmtu = dd->vld[15].mtu;
10117 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
10118 & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
10119 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
10120 int i, j;
10121 u32 thres;
10122
10123 for (i = 0; i < ppd->vls_supported; i++) {
10124 if (dd->vld[i].mtu > maxvlmtu)
10125 maxvlmtu = dd->vld[i].mtu;
10126 if (i <= 3)
10127 len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
10128 & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
10129 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
10130 else
10131 len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
10132 & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
10133 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
10134 }
10135 write_csr(dd, SEND_LEN_CHECK0, len1);
10136 write_csr(dd, SEND_LEN_CHECK1, len2);
10137 /* adjust kernel credit return thresholds based on new MTUs */
10138 /* all kernel receive contexts have the same hdrqentsize */
10139 for (i = 0; i < ppd->vls_supported; i++) {
10140 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
10141 sc_mtu_to_threshold(dd->vld[i].sc,
10142 dd->vld[i].mtu,
10143 get_hdrqentsize(dd->rcd[0])));
10144 for (j = 0; j < INIT_SC_PER_VL; j++)
10145 sc_set_cr_threshold(
10146 pio_select_send_context_vl(dd, j, i),
10147 thres);
10148 }
10149 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50),
10150 sc_mtu_to_threshold(dd->vld[15].sc,
10151 dd->vld[15].mtu,
10152 dd->rcd[0]->rcvhdrqentsize));
10153 sc_set_cr_threshold(dd->vld[15].sc, thres);
10154
10155 /* Adjust maximum MTU for the port in DC */
10156 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
10157 (ilog2(maxvlmtu >> 8) + 1);
10158 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
10159 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
10160 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
10161 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
10162 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
10163 }
10164
set_lidlmc(struct hfi1_pportdata * ppd)10165 static void set_lidlmc(struct hfi1_pportdata *ppd)
10166 {
10167 int i;
10168 u64 sreg = 0;
10169 struct hfi1_devdata *dd = ppd->dd;
10170 u32 mask = ~((1U << ppd->lmc) - 1);
10171 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
10172 u32 lid;
10173
10174 /*
10175 * Program 0 in CSR if port lid is extended. This prevents
10176 * 9B packets being sent out for large lids.
10177 */
10178 lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;
10179 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
10180 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
10181 c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
10182 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
10183 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
10184 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
10185 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
10186
10187 /*
10188 * Iterate over all the send contexts and set their SLID check
10189 */
10190 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
10191 SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
10192 (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
10193 SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
10194
10195 for (i = 0; i < chip_send_contexts(dd); i++) {
10196 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
10197 i, (u32)sreg);
10198 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
10199 }
10200
10201 /* Now we have to do the same thing for the sdma engines */
10202 sdma_update_lmc(dd, mask, lid);
10203 }
10204
state_completed_string(u32 completed)10205 static const char *state_completed_string(u32 completed)
10206 {
10207 static const char * const state_completed[] = {
10208 "EstablishComm",
10209 "OptimizeEQ",
10210 "VerifyCap"
10211 };
10212
10213 if (completed < ARRAY_SIZE(state_completed))
10214 return state_completed[completed];
10215
10216 return "unknown";
10217 }
10218
10219 static const char all_lanes_dead_timeout_expired[] =
10220 "All lanes were inactive – was the interconnect media removed?";
10221 static const char tx_out_of_policy[] =
10222 "Passing lanes on local port do not meet the local link width policy";
10223 static const char no_state_complete[] =
10224 "State timeout occurred before link partner completed the state";
10225 static const char * const state_complete_reasons[] = {
10226 [0x00] = "Reason unknown",
10227 [0x01] = "Link was halted by driver, refer to LinkDownReason",
10228 [0x02] = "Link partner reported failure",
10229 [0x10] = "Unable to achieve frame sync on any lane",
10230 [0x11] =
10231 "Unable to find a common bit rate with the link partner",
10232 [0x12] =
10233 "Unable to achieve frame sync on sufficient lanes to meet the local link width policy",
10234 [0x13] =
10235 "Unable to identify preset equalization on sufficient lanes to meet the local link width policy",
10236 [0x14] = no_state_complete,
10237 [0x15] =
10238 "State timeout occurred before link partner identified equalization presets",
10239 [0x16] =
10240 "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy",
10241 [0x17] = tx_out_of_policy,
10242 [0x20] = all_lanes_dead_timeout_expired,
10243 [0x21] =
10244 "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy",
10245 [0x22] = no_state_complete,
10246 [0x23] =
10247 "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy",
10248 [0x24] = tx_out_of_policy,
10249 [0x30] = all_lanes_dead_timeout_expired,
10250 [0x31] =
10251 "State timeout occurred waiting for host to process received frames",
10252 [0x32] = no_state_complete,
10253 [0x33] =
10254 "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy",
10255 [0x34] = tx_out_of_policy,
10256 [0x35] = "Negotiated link width is mutually exclusive",
10257 [0x36] =
10258 "Timed out before receiving verifycap frames in VerifyCap.Exchange",
10259 [0x37] = "Unable to resolve secure data exchange",
10260 };
10261
state_complete_reason_code_string(struct hfi1_pportdata * ppd,u32 code)10262 static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd,
10263 u32 code)
10264 {
10265 const char *str = NULL;
10266
10267 if (code < ARRAY_SIZE(state_complete_reasons))
10268 str = state_complete_reasons[code];
10269
10270 if (str)
10271 return str;
10272 return "Reserved";
10273 }
10274
10275 /* describe the given last state complete frame */
decode_state_complete(struct hfi1_pportdata * ppd,u32 frame,const char * prefix)10276 static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame,
10277 const char *prefix)
10278 {
10279 struct hfi1_devdata *dd = ppd->dd;
10280 u32 success;
10281 u32 state;
10282 u32 reason;
10283 u32 lanes;
10284
10285 /*
10286 * Decode frame:
10287 * [ 0: 0] - success
10288 * [ 3: 1] - state
10289 * [ 7: 4] - next state timeout
10290 * [15: 8] - reason code
10291 * [31:16] - lanes
10292 */
10293 success = frame & 0x1;
10294 state = (frame >> 1) & 0x7;
10295 reason = (frame >> 8) & 0xff;
10296 lanes = (frame >> 16) & 0xffff;
10297
10298 dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n",
10299 prefix, frame);
10300 dd_dev_err(dd, " last reported state state: %s (0x%x)\n",
10301 state_completed_string(state), state);
10302 dd_dev_err(dd, " state successfully completed: %s\n",
10303 success ? "yes" : "no");
10304 dd_dev_err(dd, " fail reason 0x%x: %s\n",
10305 reason, state_complete_reason_code_string(ppd, reason));
10306 dd_dev_err(dd, " passing lane mask: 0x%x", lanes);
10307 }
10308
10309 /*
10310 * Read the last state complete frames and explain them. This routine
10311 * expects to be called if the link went down during link negotiation
10312 * and initialization (LNI). That is, anywhere between polling and link up.
10313 */
check_lni_states(struct hfi1_pportdata * ppd)10314 static void check_lni_states(struct hfi1_pportdata *ppd)
10315 {
10316 u32 last_local_state;
10317 u32 last_remote_state;
10318
10319 read_last_local_state(ppd->dd, &last_local_state);
10320 read_last_remote_state(ppd->dd, &last_remote_state);
10321
10322 /*
10323 * Don't report anything if there is nothing to report. A value of
10324 * 0 means the link was taken down while polling and there was no
10325 * training in-process.
10326 */
10327 if (last_local_state == 0 && last_remote_state == 0)
10328 return;
10329
10330 decode_state_complete(ppd, last_local_state, "transmitted");
10331 decode_state_complete(ppd, last_remote_state, "received");
10332 }
10333
10334 /* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */
wait_link_transfer_active(struct hfi1_devdata * dd,int wait_ms)10335 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms)
10336 {
10337 u64 reg;
10338 unsigned long timeout;
10339
10340 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
10341 timeout = jiffies + msecs_to_jiffies(wait_ms);
10342 while (1) {
10343 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
10344 if (reg)
10345 break;
10346 if (time_after(jiffies, timeout)) {
10347 dd_dev_err(dd,
10348 "timeout waiting for LINK_TRANSFER_ACTIVE\n");
10349 return -ETIMEDOUT;
10350 }
10351 udelay(2);
10352 }
10353 return 0;
10354 }
10355
10356 /* called when the logical link state is not down as it should be */
force_logical_link_state_down(struct hfi1_pportdata * ppd)10357 static void force_logical_link_state_down(struct hfi1_pportdata *ppd)
10358 {
10359 struct hfi1_devdata *dd = ppd->dd;
10360
10361 /*
10362 * Bring link up in LCB loopback
10363 */
10364 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10365 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
10366 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
10367
10368 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
10369 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0);
10370 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
10371 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2);
10372
10373 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
10374 (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET);
10375 udelay(3);
10376 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1);
10377 write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
10378
10379 wait_link_transfer_active(dd, 100);
10380
10381 /*
10382 * Bring the link down again.
10383 */
10384 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10385 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0);
10386 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0);
10387
10388 dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n");
10389 }
10390
10391 /*
10392 * Helper for set_link_state(). Do not call except from that routine.
10393 * Expects ppd->hls_mutex to be held.
10394 *
10395 * @rem_reason value to be sent to the neighbor
10396 *
10397 * LinkDownReasons only set if transition succeeds.
10398 */
goto_offline(struct hfi1_pportdata * ppd,u8 rem_reason)10399 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
10400 {
10401 struct hfi1_devdata *dd = ppd->dd;
10402 u32 previous_state;
10403 int offline_state_ret;
10404 int ret;
10405
10406 update_lcb_cache(dd);
10407
10408 previous_state = ppd->host_link_state;
10409 ppd->host_link_state = HLS_GOING_OFFLINE;
10410
10411 /* start offline transition */
10412 ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE);
10413
10414 if (ret != HCMD_SUCCESS) {
10415 dd_dev_err(dd,
10416 "Failed to transition to Offline link state, return %d\n",
10417 ret);
10418 return -EINVAL;
10419 }
10420 if (ppd->offline_disabled_reason ==
10421 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
10422 ppd->offline_disabled_reason =
10423 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
10424
10425 offline_state_ret = wait_phys_link_offline_substates(ppd, 10000);
10426 if (offline_state_ret < 0)
10427 return offline_state_ret;
10428
10429 /* Disabling AOC transmitters */
10430 if (ppd->port_type == PORT_TYPE_QSFP &&
10431 ppd->qsfp_info.limiting_active &&
10432 qsfp_mod_present(ppd)) {
10433 int ret;
10434
10435 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT);
10436 if (ret == 0) {
10437 set_qsfp_tx(ppd, 0);
10438 release_chip_resource(dd, qsfp_resource(dd));
10439 } else {
10440 /* not fatal, but should warn */
10441 dd_dev_err(dd,
10442 "Unable to acquire lock to turn off QSFP TX\n");
10443 }
10444 }
10445
10446 /*
10447 * Wait for the offline.Quiet transition if it hasn't happened yet. It
10448 * can take a while for the link to go down.
10449 */
10450 if (offline_state_ret != PLS_OFFLINE_QUIET) {
10451 ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 30000);
10452 if (ret < 0)
10453 return ret;
10454 }
10455
10456 /*
10457 * Now in charge of LCB - must be after the physical state is
10458 * offline.quiet and before host_link_state is changed.
10459 */
10460 set_host_lcb_access(dd);
10461 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
10462
10463 /* make sure the logical state is also down */
10464 ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
10465 if (ret)
10466 force_logical_link_state_down(ppd);
10467
10468 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
10469 update_statusp(ppd, IB_PORT_DOWN);
10470
10471 /*
10472 * The LNI has a mandatory wait time after the physical state
10473 * moves to Offline.Quiet. The wait time may be different
10474 * depending on how the link went down. The 8051 firmware
10475 * will observe the needed wait time and only move to ready
10476 * when that is completed. The largest of the quiet timeouts
10477 * is 6s, so wait that long and then at least 0.5s more for
10478 * other transitions, and another 0.5s for a buffer.
10479 */
10480 ret = wait_fm_ready(dd, 7000);
10481 if (ret) {
10482 dd_dev_err(dd,
10483 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
10484 /* state is really offline, so make it so */
10485 ppd->host_link_state = HLS_DN_OFFLINE;
10486 return ret;
10487 }
10488
10489 /*
10490 * The state is now offline and the 8051 is ready to accept host
10491 * requests.
10492 * - change our state
10493 * - notify others if we were previously in a linkup state
10494 */
10495 ppd->host_link_state = HLS_DN_OFFLINE;
10496 if (previous_state & HLS_UP) {
10497 /* went down while link was up */
10498 handle_linkup_change(dd, 0);
10499 } else if (previous_state
10500 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
10501 /* went down while attempting link up */
10502 check_lni_states(ppd);
10503
10504 /* The QSFP doesn't need to be reset on LNI failure */
10505 ppd->qsfp_info.reset_needed = 0;
10506 }
10507
10508 /* the active link width (downgrade) is 0 on link down */
10509 ppd->link_width_active = 0;
10510 ppd->link_width_downgrade_tx_active = 0;
10511 ppd->link_width_downgrade_rx_active = 0;
10512 ppd->current_egress_rate = 0;
10513 return 0;
10514 }
10515
10516 /* return the link state name */
link_state_name(u32 state)10517 static const char *link_state_name(u32 state)
10518 {
10519 const char *name;
10520 int n = ilog2(state);
10521 static const char * const names[] = {
10522 [__HLS_UP_INIT_BP] = "INIT",
10523 [__HLS_UP_ARMED_BP] = "ARMED",
10524 [__HLS_UP_ACTIVE_BP] = "ACTIVE",
10525 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF",
10526 [__HLS_DN_POLL_BP] = "POLL",
10527 [__HLS_DN_DISABLE_BP] = "DISABLE",
10528 [__HLS_DN_OFFLINE_BP] = "OFFLINE",
10529 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP",
10530 [__HLS_GOING_UP_BP] = "GOING_UP",
10531 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
10532 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
10533 };
10534
10535 name = n < ARRAY_SIZE(names) ? names[n] : NULL;
10536 return name ? name : "unknown";
10537 }
10538
10539 /* return the link state reason name */
link_state_reason_name(struct hfi1_pportdata * ppd,u32 state)10540 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
10541 {
10542 if (state == HLS_UP_INIT) {
10543 switch (ppd->linkinit_reason) {
10544 case OPA_LINKINIT_REASON_LINKUP:
10545 return "(LINKUP)";
10546 case OPA_LINKINIT_REASON_FLAPPING:
10547 return "(FLAPPING)";
10548 case OPA_LINKINIT_OUTSIDE_POLICY:
10549 return "(OUTSIDE_POLICY)";
10550 case OPA_LINKINIT_QUARANTINED:
10551 return "(QUARANTINED)";
10552 case OPA_LINKINIT_INSUFIC_CAPABILITY:
10553 return "(INSUFIC_CAPABILITY)";
10554 default:
10555 break;
10556 }
10557 }
10558 return "";
10559 }
10560
10561 /*
10562 * driver_pstate - convert the driver's notion of a port's
10563 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
10564 * Return -1 (converted to a u32) to indicate error.
10565 */
driver_pstate(struct hfi1_pportdata * ppd)10566 u32 driver_pstate(struct hfi1_pportdata *ppd)
10567 {
10568 switch (ppd->host_link_state) {
10569 case HLS_UP_INIT:
10570 case HLS_UP_ARMED:
10571 case HLS_UP_ACTIVE:
10572 return IB_PORTPHYSSTATE_LINKUP;
10573 case HLS_DN_POLL:
10574 return IB_PORTPHYSSTATE_POLLING;
10575 case HLS_DN_DISABLE:
10576 return IB_PORTPHYSSTATE_DISABLED;
10577 case HLS_DN_OFFLINE:
10578 return OPA_PORTPHYSSTATE_OFFLINE;
10579 case HLS_VERIFY_CAP:
10580 return IB_PORTPHYSSTATE_TRAINING;
10581 case HLS_GOING_UP:
10582 return IB_PORTPHYSSTATE_TRAINING;
10583 case HLS_GOING_OFFLINE:
10584 return OPA_PORTPHYSSTATE_OFFLINE;
10585 case HLS_LINK_COOLDOWN:
10586 return OPA_PORTPHYSSTATE_OFFLINE;
10587 case HLS_DN_DOWNDEF:
10588 default:
10589 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10590 ppd->host_link_state);
10591 return -1;
10592 }
10593 }
10594
10595 /*
10596 * driver_lstate - convert the driver's notion of a port's
10597 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
10598 * (converted to a u32) to indicate error.
10599 */
driver_lstate(struct hfi1_pportdata * ppd)10600 u32 driver_lstate(struct hfi1_pportdata *ppd)
10601 {
10602 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
10603 return IB_PORT_DOWN;
10604
10605 switch (ppd->host_link_state & HLS_UP) {
10606 case HLS_UP_INIT:
10607 return IB_PORT_INIT;
10608 case HLS_UP_ARMED:
10609 return IB_PORT_ARMED;
10610 case HLS_UP_ACTIVE:
10611 return IB_PORT_ACTIVE;
10612 default:
10613 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10614 ppd->host_link_state);
10615 return -1;
10616 }
10617 }
10618
set_link_down_reason(struct hfi1_pportdata * ppd,u8 lcl_reason,u8 neigh_reason,u8 rem_reason)10619 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
10620 u8 neigh_reason, u8 rem_reason)
10621 {
10622 if (ppd->local_link_down_reason.latest == 0 &&
10623 ppd->neigh_link_down_reason.latest == 0) {
10624 ppd->local_link_down_reason.latest = lcl_reason;
10625 ppd->neigh_link_down_reason.latest = neigh_reason;
10626 ppd->remote_link_down_reason = rem_reason;
10627 }
10628 }
10629
10630 /**
10631 * data_vls_operational() - Verify if data VL BCT credits and MTU
10632 * are both set.
10633 * @ppd: pointer to hfi1_pportdata structure
10634 *
10635 * Return: true - Ok, false -otherwise.
10636 */
data_vls_operational(struct hfi1_pportdata * ppd)10637 static inline bool data_vls_operational(struct hfi1_pportdata *ppd)
10638 {
10639 int i;
10640 u64 reg;
10641
10642 if (!ppd->actual_vls_operational)
10643 return false;
10644
10645 for (i = 0; i < ppd->vls_supported; i++) {
10646 reg = read_csr(ppd->dd, SEND_CM_CREDIT_VL + (8 * i));
10647 if ((reg && !ppd->dd->vld[i].mtu) ||
10648 (!reg && ppd->dd->vld[i].mtu))
10649 return false;
10650 }
10651
10652 return true;
10653 }
10654
10655 /*
10656 * Change the physical and/or logical link state.
10657 *
10658 * Do not call this routine while inside an interrupt. It contains
10659 * calls to routines that can take multiple seconds to finish.
10660 *
10661 * Returns 0 on success, -errno on failure.
10662 */
set_link_state(struct hfi1_pportdata * ppd,u32 state)10663 int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10664 {
10665 struct hfi1_devdata *dd = ppd->dd;
10666 struct ib_event event = {.device = NULL};
10667 int ret1, ret = 0;
10668 int orig_new_state, poll_bounce;
10669
10670 mutex_lock(&ppd->hls_lock);
10671
10672 orig_new_state = state;
10673 if (state == HLS_DN_DOWNDEF)
10674 state = HLS_DEFAULT;
10675
10676 /* interpret poll -> poll as a link bounce */
10677 poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10678 state == HLS_DN_POLL;
10679
10680 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
10681 link_state_name(ppd->host_link_state),
10682 link_state_name(orig_new_state),
10683 poll_bounce ? "(bounce) " : "",
10684 link_state_reason_name(ppd, state));
10685
10686 /*
10687 * If we're going to a (HLS_*) link state that implies the logical
10688 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
10689 * reset is_sm_config_started to 0.
10690 */
10691 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10692 ppd->is_sm_config_started = 0;
10693
10694 /*
10695 * Do nothing if the states match. Let a poll to poll link bounce
10696 * go through.
10697 */
10698 if (ppd->host_link_state == state && !poll_bounce)
10699 goto done;
10700
10701 switch (state) {
10702 case HLS_UP_INIT:
10703 if (ppd->host_link_state == HLS_DN_POLL &&
10704 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
10705 /*
10706 * Quick link up jumps from polling to here.
10707 *
10708 * Whether in normal or loopback mode, the
10709 * simulator jumps from polling to link up.
10710 * Accept that here.
10711 */
10712 /* OK */
10713 } else if (ppd->host_link_state != HLS_GOING_UP) {
10714 goto unexpected;
10715 }
10716
10717 /*
10718 * Wait for Link_Up physical state.
10719 * Physical and Logical states should already be
10720 * be transitioned to LinkUp and LinkInit respectively.
10721 */
10722 ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000);
10723 if (ret) {
10724 dd_dev_err(dd,
10725 "%s: physical state did not change to LINK-UP\n",
10726 __func__);
10727 break;
10728 }
10729
10730 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10731 if (ret) {
10732 dd_dev_err(dd,
10733 "%s: logical state did not change to INIT\n",
10734 __func__);
10735 break;
10736 }
10737
10738 /* clear old transient LINKINIT_REASON code */
10739 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10740 ppd->linkinit_reason =
10741 OPA_LINKINIT_REASON_LINKUP;
10742
10743 /* enable the port */
10744 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10745
10746 handle_linkup_change(dd, 1);
10747 pio_kernel_linkup(dd);
10748
10749 /*
10750 * After link up, a new link width will have been set.
10751 * Update the xmit counters with regards to the new
10752 * link width.
10753 */
10754 update_xmit_counters(ppd, ppd->link_width_active);
10755
10756 ppd->host_link_state = HLS_UP_INIT;
10757 update_statusp(ppd, IB_PORT_INIT);
10758 break;
10759 case HLS_UP_ARMED:
10760 if (ppd->host_link_state != HLS_UP_INIT)
10761 goto unexpected;
10762
10763 if (!data_vls_operational(ppd)) {
10764 dd_dev_err(dd,
10765 "%s: Invalid data VL credits or mtu\n",
10766 __func__);
10767 ret = -EINVAL;
10768 break;
10769 }
10770
10771 set_logical_state(dd, LSTATE_ARMED);
10772 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10773 if (ret) {
10774 dd_dev_err(dd,
10775 "%s: logical state did not change to ARMED\n",
10776 __func__);
10777 break;
10778 }
10779 ppd->host_link_state = HLS_UP_ARMED;
10780 update_statusp(ppd, IB_PORT_ARMED);
10781 /*
10782 * The simulator does not currently implement SMA messages,
10783 * so neighbor_normal is not set. Set it here when we first
10784 * move to Armed.
10785 */
10786 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10787 ppd->neighbor_normal = 1;
10788 break;
10789 case HLS_UP_ACTIVE:
10790 if (ppd->host_link_state != HLS_UP_ARMED)
10791 goto unexpected;
10792
10793 set_logical_state(dd, LSTATE_ACTIVE);
10794 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10795 if (ret) {
10796 dd_dev_err(dd,
10797 "%s: logical state did not change to ACTIVE\n",
10798 __func__);
10799 } else {
10800 /* tell all engines to go running */
10801 sdma_all_running(dd);
10802 ppd->host_link_state = HLS_UP_ACTIVE;
10803 update_statusp(ppd, IB_PORT_ACTIVE);
10804
10805 /* Signal the IB layer that the port has went active */
10806 event.device = &dd->verbs_dev.rdi.ibdev;
10807 event.element.port_num = ppd->port;
10808 event.event = IB_EVENT_PORT_ACTIVE;
10809 }
10810 break;
10811 case HLS_DN_POLL:
10812 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10813 ppd->host_link_state == HLS_DN_OFFLINE) &&
10814 dd->dc_shutdown)
10815 dc_start(dd);
10816 /* Hand LED control to the DC */
10817 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10818
10819 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10820 u8 tmp = ppd->link_enabled;
10821
10822 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10823 if (ret) {
10824 ppd->link_enabled = tmp;
10825 break;
10826 }
10827 ppd->remote_link_down_reason = 0;
10828
10829 if (ppd->driver_link_ready)
10830 ppd->link_enabled = 1;
10831 }
10832
10833 set_all_slowpath(ppd->dd);
10834 ret = set_local_link_attributes(ppd);
10835 if (ret)
10836 break;
10837
10838 ppd->port_error_action = 0;
10839
10840 if (quick_linkup) {
10841 /* quick linkup does not go into polling */
10842 ret = do_quick_linkup(dd);
10843 } else {
10844 ret1 = set_physical_link_state(dd, PLS_POLLING);
10845 if (!ret1)
10846 ret1 = wait_phys_link_out_of_offline(ppd,
10847 3000);
10848 if (ret1 != HCMD_SUCCESS) {
10849 dd_dev_err(dd,
10850 "Failed to transition to Polling link state, return 0x%x\n",
10851 ret1);
10852 ret = -EINVAL;
10853 }
10854 }
10855
10856 /*
10857 * Change the host link state after requesting DC8051 to
10858 * change its physical state so that we can ignore any
10859 * interrupt with stale LNI(XX) error, which will not be
10860 * cleared until DC8051 transitions to Polling state.
10861 */
10862 ppd->host_link_state = HLS_DN_POLL;
10863 ppd->offline_disabled_reason =
10864 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
10865 /*
10866 * If an error occurred above, go back to offline. The
10867 * caller may reschedule another attempt.
10868 */
10869 if (ret)
10870 goto_offline(ppd, 0);
10871 else
10872 log_physical_state(ppd, PLS_POLLING);
10873 break;
10874 case HLS_DN_DISABLE:
10875 /* link is disabled */
10876 ppd->link_enabled = 0;
10877
10878 /* allow any state to transition to disabled */
10879
10880 /* must transition to offline first */
10881 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10882 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10883 if (ret)
10884 break;
10885 ppd->remote_link_down_reason = 0;
10886 }
10887
10888 if (!dd->dc_shutdown) {
10889 ret1 = set_physical_link_state(dd, PLS_DISABLED);
10890 if (ret1 != HCMD_SUCCESS) {
10891 dd_dev_err(dd,
10892 "Failed to transition to Disabled link state, return 0x%x\n",
10893 ret1);
10894 ret = -EINVAL;
10895 break;
10896 }
10897 ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000);
10898 if (ret) {
10899 dd_dev_err(dd,
10900 "%s: physical state did not change to DISABLED\n",
10901 __func__);
10902 break;
10903 }
10904 dc_shutdown(dd);
10905 }
10906 ppd->host_link_state = HLS_DN_DISABLE;
10907 break;
10908 case HLS_DN_OFFLINE:
10909 if (ppd->host_link_state == HLS_DN_DISABLE)
10910 dc_start(dd);
10911
10912 /* allow any state to transition to offline */
10913 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10914 if (!ret)
10915 ppd->remote_link_down_reason = 0;
10916 break;
10917 case HLS_VERIFY_CAP:
10918 if (ppd->host_link_state != HLS_DN_POLL)
10919 goto unexpected;
10920 ppd->host_link_state = HLS_VERIFY_CAP;
10921 log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP);
10922 break;
10923 case HLS_GOING_UP:
10924 if (ppd->host_link_state != HLS_VERIFY_CAP)
10925 goto unexpected;
10926
10927 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10928 if (ret1 != HCMD_SUCCESS) {
10929 dd_dev_err(dd,
10930 "Failed to transition to link up state, return 0x%x\n",
10931 ret1);
10932 ret = -EINVAL;
10933 break;
10934 }
10935 ppd->host_link_state = HLS_GOING_UP;
10936 break;
10937
10938 case HLS_GOING_OFFLINE: /* transient within goto_offline() */
10939 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */
10940 default:
10941 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
10942 __func__, state);
10943 ret = -EINVAL;
10944 break;
10945 }
10946
10947 goto done;
10948
10949 unexpected:
10950 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
10951 __func__, link_state_name(ppd->host_link_state),
10952 link_state_name(state));
10953 ret = -EINVAL;
10954
10955 done:
10956 mutex_unlock(&ppd->hls_lock);
10957
10958 if (event.device)
10959 ib_dispatch_event(&event);
10960
10961 return ret;
10962 }
10963
hfi1_set_ib_cfg(struct hfi1_pportdata * ppd,int which,u32 val)10964 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10965 {
10966 u64 reg;
10967 int ret = 0;
10968
10969 switch (which) {
10970 case HFI1_IB_CFG_LIDLMC:
10971 set_lidlmc(ppd);
10972 break;
10973 case HFI1_IB_CFG_VL_HIGH_LIMIT:
10974 /*
10975 * The VL Arbitrator high limit is sent in units of 4k
10976 * bytes, while HFI stores it in units of 64 bytes.
10977 */
10978 val *= 4096 / 64;
10979 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10980 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10981 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10982 break;
10983 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10984 /* HFI only supports POLL as the default link down state */
10985 if (val != HLS_DN_POLL)
10986 ret = -EINVAL;
10987 break;
10988 case HFI1_IB_CFG_OP_VLS:
10989 if (ppd->vls_operational != val) {
10990 ppd->vls_operational = val;
10991 if (!ppd->port)
10992 ret = -EINVAL;
10993 }
10994 break;
10995 /*
10996 * For link width, link width downgrade, and speed enable, always AND
10997 * the setting with what is actually supported. This has two benefits.
10998 * First, enabled can't have unsupported values, no matter what the
10999 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean
11000 * "fill in with your supported value" have all the bits in the
11001 * field set, so simply ANDing with supported has the desired result.
11002 */
11003 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
11004 ppd->link_width_enabled = val & ppd->link_width_supported;
11005 break;
11006 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
11007 ppd->link_width_downgrade_enabled =
11008 val & ppd->link_width_downgrade_supported;
11009 break;
11010 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
11011 ppd->link_speed_enabled = val & ppd->link_speed_supported;
11012 break;
11013 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
11014 /*
11015 * HFI does not follow IB specs, save this value
11016 * so we can report it, if asked.
11017 */
11018 ppd->overrun_threshold = val;
11019 break;
11020 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
11021 /*
11022 * HFI does not follow IB specs, save this value
11023 * so we can report it, if asked.
11024 */
11025 ppd->phy_error_threshold = val;
11026 break;
11027
11028 case HFI1_IB_CFG_MTU:
11029 set_send_length(ppd);
11030 break;
11031
11032 case HFI1_IB_CFG_PKEYS:
11033 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
11034 set_partition_keys(ppd);
11035 break;
11036
11037 default:
11038 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
11039 dd_dev_info(ppd->dd,
11040 "%s: which %s, val 0x%x: not implemented\n",
11041 __func__, ib_cfg_name(which), val);
11042 break;
11043 }
11044 return ret;
11045 }
11046
11047 /* begin functions related to vl arbitration table caching */
init_vl_arb_caches(struct hfi1_pportdata * ppd)11048 static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
11049 {
11050 int i;
11051
11052 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
11053 VL_ARB_LOW_PRIO_TABLE_SIZE);
11054 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
11055 VL_ARB_HIGH_PRIO_TABLE_SIZE);
11056
11057 /*
11058 * Note that we always return values directly from the
11059 * 'vl_arb_cache' (and do no CSR reads) in response to a
11060 * 'Get(VLArbTable)'. This is obviously correct after a
11061 * 'Set(VLArbTable)', since the cache will then be up to
11062 * date. But it's also correct prior to any 'Set(VLArbTable)'
11063 * since then both the cache, and the relevant h/w registers
11064 * will be zeroed.
11065 */
11066
11067 for (i = 0; i < MAX_PRIO_TABLE; i++)
11068 spin_lock_init(&ppd->vl_arb_cache[i].lock);
11069 }
11070
11071 /*
11072 * vl_arb_lock_cache
11073 *
11074 * All other vl_arb_* functions should be called only after locking
11075 * the cache.
11076 */
11077 static inline struct vl_arb_cache *
vl_arb_lock_cache(struct hfi1_pportdata * ppd,int idx)11078 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
11079 {
11080 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
11081 return NULL;
11082 spin_lock(&ppd->vl_arb_cache[idx].lock);
11083 return &ppd->vl_arb_cache[idx];
11084 }
11085
vl_arb_unlock_cache(struct hfi1_pportdata * ppd,int idx)11086 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
11087 {
11088 spin_unlock(&ppd->vl_arb_cache[idx].lock);
11089 }
11090
vl_arb_get_cache(struct vl_arb_cache * cache,struct ib_vl_weight_elem * vl)11091 static void vl_arb_get_cache(struct vl_arb_cache *cache,
11092 struct ib_vl_weight_elem *vl)
11093 {
11094 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
11095 }
11096
vl_arb_set_cache(struct vl_arb_cache * cache,struct ib_vl_weight_elem * vl)11097 static void vl_arb_set_cache(struct vl_arb_cache *cache,
11098 struct ib_vl_weight_elem *vl)
11099 {
11100 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11101 }
11102
vl_arb_match_cache(struct vl_arb_cache * cache,struct ib_vl_weight_elem * vl)11103 static int vl_arb_match_cache(struct vl_arb_cache *cache,
11104 struct ib_vl_weight_elem *vl)
11105 {
11106 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11107 }
11108
11109 /* end functions related to vl arbitration table caching */
11110
set_vl_weights(struct hfi1_pportdata * ppd,u32 target,u32 size,struct ib_vl_weight_elem * vl)11111 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
11112 u32 size, struct ib_vl_weight_elem *vl)
11113 {
11114 struct hfi1_devdata *dd = ppd->dd;
11115 u64 reg;
11116 unsigned int i, is_up = 0;
11117 int drain, ret = 0;
11118
11119 mutex_lock(&ppd->hls_lock);
11120
11121 if (ppd->host_link_state & HLS_UP)
11122 is_up = 1;
11123
11124 drain = !is_ax(dd) && is_up;
11125
11126 if (drain)
11127 /*
11128 * Before adjusting VL arbitration weights, empty per-VL
11129 * FIFOs, otherwise a packet whose VL weight is being
11130 * set to 0 could get stuck in a FIFO with no chance to
11131 * egress.
11132 */
11133 ret = stop_drain_data_vls(dd);
11134
11135 if (ret) {
11136 dd_dev_err(
11137 dd,
11138 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
11139 __func__);
11140 goto err;
11141 }
11142
11143 for (i = 0; i < size; i++, vl++) {
11144 /*
11145 * NOTE: The low priority shift and mask are used here, but
11146 * they are the same for both the low and high registers.
11147 */
11148 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
11149 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
11150 | (((u64)vl->weight
11151 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
11152 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
11153 write_csr(dd, target + (i * 8), reg);
11154 }
11155 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
11156
11157 if (drain)
11158 open_fill_data_vls(dd); /* reopen all VLs */
11159
11160 err:
11161 mutex_unlock(&ppd->hls_lock);
11162
11163 return ret;
11164 }
11165
11166 /*
11167 * Read one credit merge VL register.
11168 */
read_one_cm_vl(struct hfi1_devdata * dd,u32 csr,struct vl_limit * vll)11169 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
11170 struct vl_limit *vll)
11171 {
11172 u64 reg = read_csr(dd, csr);
11173
11174 vll->dedicated = cpu_to_be16(
11175 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
11176 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
11177 vll->shared = cpu_to_be16(
11178 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
11179 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
11180 }
11181
11182 /*
11183 * Read the current credit merge limits.
11184 */
get_buffer_control(struct hfi1_devdata * dd,struct buffer_control * bc,u16 * overall_limit)11185 static int get_buffer_control(struct hfi1_devdata *dd,
11186 struct buffer_control *bc, u16 *overall_limit)
11187 {
11188 u64 reg;
11189 int i;
11190
11191 /* not all entries are filled in */
11192 memset(bc, 0, sizeof(*bc));
11193
11194 /* OPA and HFI have a 1-1 mapping */
11195 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11196 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
11197
11198 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
11199 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
11200
11201 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11202 bc->overall_shared_limit = cpu_to_be16(
11203 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
11204 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
11205 if (overall_limit)
11206 *overall_limit = (reg
11207 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
11208 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
11209 return sizeof(struct buffer_control);
11210 }
11211
get_sc2vlnt(struct hfi1_devdata * dd,struct sc2vlnt * dp)11212 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11213 {
11214 u64 reg;
11215 int i;
11216
11217 /* each register contains 16 SC->VLnt mappings, 4 bits each */
11218 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
11219 for (i = 0; i < sizeof(u64); i++) {
11220 u8 byte = *(((u8 *)®) + i);
11221
11222 dp->vlnt[2 * i] = byte & 0xf;
11223 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
11224 }
11225
11226 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
11227 for (i = 0; i < sizeof(u64); i++) {
11228 u8 byte = *(((u8 *)®) + i);
11229
11230 dp->vlnt[16 + (2 * i)] = byte & 0xf;
11231 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
11232 }
11233 return sizeof(struct sc2vlnt);
11234 }
11235
get_vlarb_preempt(struct hfi1_devdata * dd,u32 nelems,struct ib_vl_weight_elem * vl)11236 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
11237 struct ib_vl_weight_elem *vl)
11238 {
11239 unsigned int i;
11240
11241 for (i = 0; i < nelems; i++, vl++) {
11242 vl->vl = 0xf;
11243 vl->weight = 0;
11244 }
11245 }
11246
set_sc2vlnt(struct hfi1_devdata * dd,struct sc2vlnt * dp)11247 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11248 {
11249 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
11250 DC_SC_VL_VAL(15_0,
11251 0, dp->vlnt[0] & 0xf,
11252 1, dp->vlnt[1] & 0xf,
11253 2, dp->vlnt[2] & 0xf,
11254 3, dp->vlnt[3] & 0xf,
11255 4, dp->vlnt[4] & 0xf,
11256 5, dp->vlnt[5] & 0xf,
11257 6, dp->vlnt[6] & 0xf,
11258 7, dp->vlnt[7] & 0xf,
11259 8, dp->vlnt[8] & 0xf,
11260 9, dp->vlnt[9] & 0xf,
11261 10, dp->vlnt[10] & 0xf,
11262 11, dp->vlnt[11] & 0xf,
11263 12, dp->vlnt[12] & 0xf,
11264 13, dp->vlnt[13] & 0xf,
11265 14, dp->vlnt[14] & 0xf,
11266 15, dp->vlnt[15] & 0xf));
11267 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
11268 DC_SC_VL_VAL(31_16,
11269 16, dp->vlnt[16] & 0xf,
11270 17, dp->vlnt[17] & 0xf,
11271 18, dp->vlnt[18] & 0xf,
11272 19, dp->vlnt[19] & 0xf,
11273 20, dp->vlnt[20] & 0xf,
11274 21, dp->vlnt[21] & 0xf,
11275 22, dp->vlnt[22] & 0xf,
11276 23, dp->vlnt[23] & 0xf,
11277 24, dp->vlnt[24] & 0xf,
11278 25, dp->vlnt[25] & 0xf,
11279 26, dp->vlnt[26] & 0xf,
11280 27, dp->vlnt[27] & 0xf,
11281 28, dp->vlnt[28] & 0xf,
11282 29, dp->vlnt[29] & 0xf,
11283 30, dp->vlnt[30] & 0xf,
11284 31, dp->vlnt[31] & 0xf));
11285 }
11286
nonzero_msg(struct hfi1_devdata * dd,int idx,const char * what,u16 limit)11287 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
11288 u16 limit)
11289 {
11290 if (limit != 0)
11291 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
11292 what, (int)limit, idx);
11293 }
11294
11295 /* change only the shared limit portion of SendCmGLobalCredit */
set_global_shared(struct hfi1_devdata * dd,u16 limit)11296 static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
11297 {
11298 u64 reg;
11299
11300 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11301 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
11302 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
11303 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11304 }
11305
11306 /* change only the total credit limit portion of SendCmGLobalCredit */
set_global_limit(struct hfi1_devdata * dd,u16 limit)11307 static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
11308 {
11309 u64 reg;
11310
11311 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11312 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
11313 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
11314 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11315 }
11316
11317 /* set the given per-VL shared limit */
set_vl_shared(struct hfi1_devdata * dd,int vl,u16 limit)11318 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
11319 {
11320 u64 reg;
11321 u32 addr;
11322
11323 if (vl < TXE_NUM_DATA_VL)
11324 addr = SEND_CM_CREDIT_VL + (8 * vl);
11325 else
11326 addr = SEND_CM_CREDIT_VL15;
11327
11328 reg = read_csr(dd, addr);
11329 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
11330 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
11331 write_csr(dd, addr, reg);
11332 }
11333
11334 /* set the given per-VL dedicated limit */
set_vl_dedicated(struct hfi1_devdata * dd,int vl,u16 limit)11335 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
11336 {
11337 u64 reg;
11338 u32 addr;
11339
11340 if (vl < TXE_NUM_DATA_VL)
11341 addr = SEND_CM_CREDIT_VL + (8 * vl);
11342 else
11343 addr = SEND_CM_CREDIT_VL15;
11344
11345 reg = read_csr(dd, addr);
11346 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
11347 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
11348 write_csr(dd, addr, reg);
11349 }
11350
11351 /* spin until the given per-VL status mask bits clear */
wait_for_vl_status_clear(struct hfi1_devdata * dd,u64 mask,const char * which)11352 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
11353 const char *which)
11354 {
11355 unsigned long timeout;
11356 u64 reg;
11357
11358 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
11359 while (1) {
11360 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
11361
11362 if (reg == 0)
11363 return; /* success */
11364 if (time_after(jiffies, timeout))
11365 break; /* timed out */
11366 udelay(1);
11367 }
11368
11369 dd_dev_err(dd,
11370 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
11371 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
11372 /*
11373 * If this occurs, it is likely there was a credit loss on the link.
11374 * The only recovery from that is a link bounce.
11375 */
11376 dd_dev_err(dd,
11377 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n");
11378 }
11379
11380 /*
11381 * The number of credits on the VLs may be changed while everything
11382 * is "live", but the following algorithm must be followed due to
11383 * how the hardware is actually implemented. In particular,
11384 * Return_Credit_Status[] is the only correct status check.
11385 *
11386 * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
11387 * set Global_Shared_Credit_Limit = 0
11388 * use_all_vl = 1
11389 * mask0 = all VLs that are changing either dedicated or shared limits
11390 * set Shared_Limit[mask0] = 0
11391 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
11392 * if (changing any dedicated limit)
11393 * mask1 = all VLs that are lowering dedicated limits
11394 * lower Dedicated_Limit[mask1]
11395 * spin until Return_Credit_Status[mask1] == 0
11396 * raise Dedicated_Limits
11397 * raise Shared_Limits
11398 * raise Global_Shared_Credit_Limit
11399 *
11400 * lower = if the new limit is lower, set the limit to the new value
11401 * raise = if the new limit is higher than the current value (may be changed
11402 * earlier in the algorithm), set the new limit to the new value
11403 */
set_buffer_control(struct hfi1_pportdata * ppd,struct buffer_control * new_bc)11404 int set_buffer_control(struct hfi1_pportdata *ppd,
11405 struct buffer_control *new_bc)
11406 {
11407 struct hfi1_devdata *dd = ppd->dd;
11408 u64 changing_mask, ld_mask, stat_mask;
11409 int change_count;
11410 int i, use_all_mask;
11411 int this_shared_changing;
11412 int vl_count = 0, ret;
11413 /*
11414 * A0: add the variable any_shared_limit_changing below and in the
11415 * algorithm above. If removing A0 support, it can be removed.
11416 */
11417 int any_shared_limit_changing;
11418 struct buffer_control cur_bc;
11419 u8 changing[OPA_MAX_VLS];
11420 u8 lowering_dedicated[OPA_MAX_VLS];
11421 u16 cur_total;
11422 u32 new_total = 0;
11423 const u64 all_mask =
11424 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
11425 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
11426 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
11427 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
11428 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
11429 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
11430 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
11431 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
11432 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
11433
11434 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
11435 #define NUM_USABLE_VLS 16 /* look at VL15 and less */
11436
11437 /* find the new total credits, do sanity check on unused VLs */
11438 for (i = 0; i < OPA_MAX_VLS; i++) {
11439 if (valid_vl(i)) {
11440 new_total += be16_to_cpu(new_bc->vl[i].dedicated);
11441 continue;
11442 }
11443 nonzero_msg(dd, i, "dedicated",
11444 be16_to_cpu(new_bc->vl[i].dedicated));
11445 nonzero_msg(dd, i, "shared",
11446 be16_to_cpu(new_bc->vl[i].shared));
11447 new_bc->vl[i].dedicated = 0;
11448 new_bc->vl[i].shared = 0;
11449 }
11450 new_total += be16_to_cpu(new_bc->overall_shared_limit);
11451
11452 /* fetch the current values */
11453 get_buffer_control(dd, &cur_bc, &cur_total);
11454
11455 /*
11456 * Create the masks we will use.
11457 */
11458 memset(changing, 0, sizeof(changing));
11459 memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
11460 /*
11461 * NOTE: Assumes that the individual VL bits are adjacent and in
11462 * increasing order
11463 */
11464 stat_mask =
11465 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
11466 changing_mask = 0;
11467 ld_mask = 0;
11468 change_count = 0;
11469 any_shared_limit_changing = 0;
11470 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
11471 if (!valid_vl(i))
11472 continue;
11473 this_shared_changing = new_bc->vl[i].shared
11474 != cur_bc.vl[i].shared;
11475 if (this_shared_changing)
11476 any_shared_limit_changing = 1;
11477 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
11478 this_shared_changing) {
11479 changing[i] = 1;
11480 changing_mask |= stat_mask;
11481 change_count++;
11482 }
11483 if (be16_to_cpu(new_bc->vl[i].dedicated) <
11484 be16_to_cpu(cur_bc.vl[i].dedicated)) {
11485 lowering_dedicated[i] = 1;
11486 ld_mask |= stat_mask;
11487 }
11488 }
11489
11490 /* bracket the credit change with a total adjustment */
11491 if (new_total > cur_total)
11492 set_global_limit(dd, new_total);
11493
11494 /*
11495 * Start the credit change algorithm.
11496 */
11497 use_all_mask = 0;
11498 if ((be16_to_cpu(new_bc->overall_shared_limit) <
11499 be16_to_cpu(cur_bc.overall_shared_limit)) ||
11500 (is_ax(dd) && any_shared_limit_changing)) {
11501 set_global_shared(dd, 0);
11502 cur_bc.overall_shared_limit = 0;
11503 use_all_mask = 1;
11504 }
11505
11506 for (i = 0; i < NUM_USABLE_VLS; i++) {
11507 if (!valid_vl(i))
11508 continue;
11509
11510 if (changing[i]) {
11511 set_vl_shared(dd, i, 0);
11512 cur_bc.vl[i].shared = 0;
11513 }
11514 }
11515
11516 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
11517 "shared");
11518
11519 if (change_count > 0) {
11520 for (i = 0; i < NUM_USABLE_VLS; i++) {
11521 if (!valid_vl(i))
11522 continue;
11523
11524 if (lowering_dedicated[i]) {
11525 set_vl_dedicated(dd, i,
11526 be16_to_cpu(new_bc->
11527 vl[i].dedicated));
11528 cur_bc.vl[i].dedicated =
11529 new_bc->vl[i].dedicated;
11530 }
11531 }
11532
11533 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
11534
11535 /* now raise all dedicated that are going up */
11536 for (i = 0; i < NUM_USABLE_VLS; i++) {
11537 if (!valid_vl(i))
11538 continue;
11539
11540 if (be16_to_cpu(new_bc->vl[i].dedicated) >
11541 be16_to_cpu(cur_bc.vl[i].dedicated))
11542 set_vl_dedicated(dd, i,
11543 be16_to_cpu(new_bc->
11544 vl[i].dedicated));
11545 }
11546 }
11547
11548 /* next raise all shared that are going up */
11549 for (i = 0; i < NUM_USABLE_VLS; i++) {
11550 if (!valid_vl(i))
11551 continue;
11552
11553 if (be16_to_cpu(new_bc->vl[i].shared) >
11554 be16_to_cpu(cur_bc.vl[i].shared))
11555 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
11556 }
11557
11558 /* finally raise the global shared */
11559 if (be16_to_cpu(new_bc->overall_shared_limit) >
11560 be16_to_cpu(cur_bc.overall_shared_limit))
11561 set_global_shared(dd,
11562 be16_to_cpu(new_bc->overall_shared_limit));
11563
11564 /* bracket the credit change with a total adjustment */
11565 if (new_total < cur_total)
11566 set_global_limit(dd, new_total);
11567
11568 /*
11569 * Determine the actual number of operational VLS using the number of
11570 * dedicated and shared credits for each VL.
11571 */
11572 if (change_count > 0) {
11573 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11574 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
11575 be16_to_cpu(new_bc->vl[i].shared) > 0)
11576 vl_count++;
11577 ppd->actual_vls_operational = vl_count;
11578 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
11579 ppd->actual_vls_operational :
11580 ppd->vls_operational,
11581 NULL);
11582 if (ret == 0)
11583 ret = pio_map_init(dd, ppd->port - 1, vl_count ?
11584 ppd->actual_vls_operational :
11585 ppd->vls_operational, NULL);
11586 if (ret)
11587 return ret;
11588 }
11589 return 0;
11590 }
11591
11592 /*
11593 * Read the given fabric manager table. Return the size of the
11594 * table (in bytes) on success, and a negative error code on
11595 * failure.
11596 */
fm_get_table(struct hfi1_pportdata * ppd,int which,void * t)11597 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
11598
11599 {
11600 int size;
11601 struct vl_arb_cache *vlc;
11602
11603 switch (which) {
11604 case FM_TBL_VL_HIGH_ARB:
11605 size = 256;
11606 /*
11607 * OPA specifies 128 elements (of 2 bytes each), though
11608 * HFI supports only 16 elements in h/w.
11609 */
11610 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11611 vl_arb_get_cache(vlc, t);
11612 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11613 break;
11614 case FM_TBL_VL_LOW_ARB:
11615 size = 256;
11616 /*
11617 * OPA specifies 128 elements (of 2 bytes each), though
11618 * HFI supports only 16 elements in h/w.
11619 */
11620 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11621 vl_arb_get_cache(vlc, t);
11622 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11623 break;
11624 case FM_TBL_BUFFER_CONTROL:
11625 size = get_buffer_control(ppd->dd, t, NULL);
11626 break;
11627 case FM_TBL_SC2VLNT:
11628 size = get_sc2vlnt(ppd->dd, t);
11629 break;
11630 case FM_TBL_VL_PREEMPT_ELEMS:
11631 size = 256;
11632 /* OPA specifies 128 elements, of 2 bytes each */
11633 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
11634 break;
11635 case FM_TBL_VL_PREEMPT_MATRIX:
11636 size = 256;
11637 /*
11638 * OPA specifies that this is the same size as the VL
11639 * arbitration tables (i.e., 256 bytes).
11640 */
11641 break;
11642 default:
11643 return -EINVAL;
11644 }
11645 return size;
11646 }
11647
11648 /*
11649 * Write the given fabric manager table.
11650 */
fm_set_table(struct hfi1_pportdata * ppd,int which,void * t)11651 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
11652 {
11653 int ret = 0;
11654 struct vl_arb_cache *vlc;
11655
11656 switch (which) {
11657 case FM_TBL_VL_HIGH_ARB:
11658 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11659 if (vl_arb_match_cache(vlc, t)) {
11660 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11661 break;
11662 }
11663 vl_arb_set_cache(vlc, t);
11664 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11665 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
11666 VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
11667 break;
11668 case FM_TBL_VL_LOW_ARB:
11669 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11670 if (vl_arb_match_cache(vlc, t)) {
11671 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11672 break;
11673 }
11674 vl_arb_set_cache(vlc, t);
11675 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11676 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
11677 VL_ARB_LOW_PRIO_TABLE_SIZE, t);
11678 break;
11679 case FM_TBL_BUFFER_CONTROL:
11680 ret = set_buffer_control(ppd, t);
11681 break;
11682 case FM_TBL_SC2VLNT:
11683 set_sc2vlnt(ppd->dd, t);
11684 break;
11685 default:
11686 ret = -EINVAL;
11687 }
11688 return ret;
11689 }
11690
11691 /*
11692 * Disable all data VLs.
11693 *
11694 * Return 0 if disabled, non-zero if the VLs cannot be disabled.
11695 */
disable_data_vls(struct hfi1_devdata * dd)11696 static int disable_data_vls(struct hfi1_devdata *dd)
11697 {
11698 if (is_ax(dd))
11699 return 1;
11700
11701 pio_send_control(dd, PSC_DATA_VL_DISABLE);
11702
11703 return 0;
11704 }
11705
11706 /*
11707 * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
11708 * Just re-enables all data VLs (the "fill" part happens
11709 * automatically - the name was chosen for symmetry with
11710 * stop_drain_data_vls()).
11711 *
11712 * Return 0 if successful, non-zero if the VLs cannot be enabled.
11713 */
open_fill_data_vls(struct hfi1_devdata * dd)11714 int open_fill_data_vls(struct hfi1_devdata *dd)
11715 {
11716 if (is_ax(dd))
11717 return 1;
11718
11719 pio_send_control(dd, PSC_DATA_VL_ENABLE);
11720
11721 return 0;
11722 }
11723
11724 /*
11725 * drain_data_vls() - assumes that disable_data_vls() has been called,
11726 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
11727 * engines to drop to 0.
11728 */
drain_data_vls(struct hfi1_devdata * dd)11729 static void drain_data_vls(struct hfi1_devdata *dd)
11730 {
11731 sc_wait(dd);
11732 sdma_wait(dd);
11733 pause_for_credit_return(dd);
11734 }
11735
11736 /*
11737 * stop_drain_data_vls() - disable, then drain all per-VL fifos.
11738 *
11739 * Use open_fill_data_vls() to resume using data VLs. This pair is
11740 * meant to be used like this:
11741 *
11742 * stop_drain_data_vls(dd);
11743 * // do things with per-VL resources
11744 * open_fill_data_vls(dd);
11745 */
stop_drain_data_vls(struct hfi1_devdata * dd)11746 int stop_drain_data_vls(struct hfi1_devdata *dd)
11747 {
11748 int ret;
11749
11750 ret = disable_data_vls(dd);
11751 if (ret == 0)
11752 drain_data_vls(dd);
11753
11754 return ret;
11755 }
11756
11757 /*
11758 * Convert a nanosecond time to a cclock count. No matter how slow
11759 * the cclock, a non-zero ns will always have a non-zero result.
11760 */
ns_to_cclock(struct hfi1_devdata * dd,u32 ns)11761 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11762 {
11763 u32 cclocks;
11764
11765 if (dd->icode == ICODE_FPGA_EMULATION)
11766 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11767 else /* simulation pretends to be ASIC */
11768 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11769 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */
11770 cclocks = 1;
11771 return cclocks;
11772 }
11773
11774 /*
11775 * Convert a cclock count to nanoseconds. Not matter how slow
11776 * the cclock, a non-zero cclocks will always have a non-zero result.
11777 */
cclock_to_ns(struct hfi1_devdata * dd,u32 cclocks)11778 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11779 {
11780 u32 ns;
11781
11782 if (dd->icode == ICODE_FPGA_EMULATION)
11783 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11784 else /* simulation pretends to be ASIC */
11785 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11786 if (cclocks && !ns)
11787 ns = 1;
11788 return ns;
11789 }
11790
11791 /*
11792 * Dynamically adjust the receive interrupt timeout for a context based on
11793 * incoming packet rate.
11794 *
11795 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
11796 */
adjust_rcv_timeout(struct hfi1_ctxtdata * rcd,u32 npkts)11797 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11798 {
11799 struct hfi1_devdata *dd = rcd->dd;
11800 u32 timeout = rcd->rcvavail_timeout;
11801
11802 /*
11803 * This algorithm doubles or halves the timeout depending on whether
11804 * the number of packets received in this interrupt were less than or
11805 * greater equal the interrupt count.
11806 *
11807 * The calculations below do not allow a steady state to be achieved.
11808 * Only at the endpoints it is possible to have an unchanging
11809 * timeout.
11810 */
11811 if (npkts < rcv_intr_count) {
11812 /*
11813 * Not enough packets arrived before the timeout, adjust
11814 * timeout downward.
11815 */
11816 if (timeout < 2) /* already at minimum? */
11817 return;
11818 timeout >>= 1;
11819 } else {
11820 /*
11821 * More than enough packets arrived before the timeout, adjust
11822 * timeout upward.
11823 */
11824 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
11825 return;
11826 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11827 }
11828
11829 rcd->rcvavail_timeout = timeout;
11830 /*
11831 * timeout cannot be larger than rcv_intr_timeout_csr which has already
11832 * been verified to be in range
11833 */
11834 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
11835 (u64)timeout <<
11836 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11837 }
11838
update_usrhead(struct hfi1_ctxtdata * rcd,u32 hd,u32 updegr,u32 egrhd,u32 intr_adjust,u32 npkts)11839 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11840 u32 intr_adjust, u32 npkts)
11841 {
11842 struct hfi1_devdata *dd = rcd->dd;
11843 u64 reg;
11844 u32 ctxt = rcd->ctxt;
11845
11846 /*
11847 * Need to write timeout register before updating RcvHdrHead to ensure
11848 * that a new value is used when the HW decides to restart counting.
11849 */
11850 if (intr_adjust)
11851 adjust_rcv_timeout(rcd, npkts);
11852 if (updegr) {
11853 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11854 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11855 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11856 }
11857 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11858 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11859 << RCV_HDR_HEAD_HEAD_SHIFT);
11860 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11861 }
11862
hdrqempty(struct hfi1_ctxtdata * rcd)11863 u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11864 {
11865 u32 head, tail;
11866
11867 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11868 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11869
11870 if (hfi1_rcvhdrtail_kvaddr(rcd))
11871 tail = get_rcvhdrtail(rcd);
11872 else
11873 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11874
11875 return head == tail;
11876 }
11877
11878 /*
11879 * Context Control and Receive Array encoding for buffer size:
11880 * 0x0 invalid
11881 * 0x1 4 KB
11882 * 0x2 8 KB
11883 * 0x3 16 KB
11884 * 0x4 32 KB
11885 * 0x5 64 KB
11886 * 0x6 128 KB
11887 * 0x7 256 KB
11888 * 0x8 512 KB (Receive Array only)
11889 * 0x9 1 MB (Receive Array only)
11890 * 0xa 2 MB (Receive Array only)
11891 *
11892 * 0xB-0xF - reserved (Receive Array only)
11893 *
11894 *
11895 * This routine assumes that the value has already been sanity checked.
11896 */
encoded_size(u32 size)11897 static u32 encoded_size(u32 size)
11898 {
11899 switch (size) {
11900 case 4 * 1024: return 0x1;
11901 case 8 * 1024: return 0x2;
11902 case 16 * 1024: return 0x3;
11903 case 32 * 1024: return 0x4;
11904 case 64 * 1024: return 0x5;
11905 case 128 * 1024: return 0x6;
11906 case 256 * 1024: return 0x7;
11907 case 512 * 1024: return 0x8;
11908 case 1 * 1024 * 1024: return 0x9;
11909 case 2 * 1024 * 1024: return 0xa;
11910 }
11911 return 0x1; /* if invalid, go with the minimum size */
11912 }
11913
11914 /**
11915 * encode_rcv_header_entry_size - return chip specific encoding for size
11916 * @size: size in dwords
11917 *
11918 * Convert a receive header entry size that to the encoding used in the CSR.
11919 *
11920 * Return a zero if the given size is invalid, otherwise the encoding.
11921 */
encode_rcv_header_entry_size(u8 size)11922 u8 encode_rcv_header_entry_size(u8 size)
11923 {
11924 /* there are only 3 valid receive header entry sizes */
11925 if (size == 2)
11926 return 1;
11927 if (size == 16)
11928 return 2;
11929 if (size == 32)
11930 return 4;
11931 return 0; /* invalid */
11932 }
11933
11934 /**
11935 * hfi1_validate_rcvhdrcnt - validate hdrcnt
11936 * @dd: the device data
11937 * @thecnt: the header count
11938 */
hfi1_validate_rcvhdrcnt(struct hfi1_devdata * dd,uint thecnt)11939 int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt)
11940 {
11941 if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) {
11942 dd_dev_err(dd, "Receive header queue count too small\n");
11943 return -EINVAL;
11944 }
11945
11946 if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) {
11947 dd_dev_err(dd,
11948 "Receive header queue count cannot be greater than %u\n",
11949 HFI1_MAX_HDRQ_EGRBUF_CNT);
11950 return -EINVAL;
11951 }
11952
11953 if (thecnt % HDRQ_INCREMENT) {
11954 dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n",
11955 thecnt, HDRQ_INCREMENT);
11956 return -EINVAL;
11957 }
11958
11959 return 0;
11960 }
11961
11962 /**
11963 * set_hdrq_regs - set header queue registers for context
11964 * @dd: the device data
11965 * @ctxt: the context
11966 * @entsize: the dword entry size
11967 * @hdrcnt: the number of header entries
11968 */
set_hdrq_regs(struct hfi1_devdata * dd,u8 ctxt,u8 entsize,u16 hdrcnt)11969 void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt)
11970 {
11971 u64 reg;
11972
11973 reg = (((u64)hdrcnt >> HDRQ_SIZE_SHIFT) & RCV_HDR_CNT_CNT_MASK) <<
11974 RCV_HDR_CNT_CNT_SHIFT;
11975 write_kctxt_csr(dd, ctxt, RCV_HDR_CNT, reg);
11976 reg = ((u64)encode_rcv_header_entry_size(entsize) &
11977 RCV_HDR_ENT_SIZE_ENT_SIZE_MASK) <<
11978 RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT;
11979 write_kctxt_csr(dd, ctxt, RCV_HDR_ENT_SIZE, reg);
11980 reg = ((u64)DEFAULT_RCVHDRSIZE & RCV_HDR_SIZE_HDR_SIZE_MASK) <<
11981 RCV_HDR_SIZE_HDR_SIZE_SHIFT;
11982 write_kctxt_csr(dd, ctxt, RCV_HDR_SIZE, reg);
11983
11984 /*
11985 * Program dummy tail address for every receive context
11986 * before enabling any receive context
11987 */
11988 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11989 dd->rcvhdrtail_dummy_dma);
11990 }
11991
hfi1_rcvctrl(struct hfi1_devdata * dd,unsigned int op,struct hfi1_ctxtdata * rcd)11992 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
11993 struct hfi1_ctxtdata *rcd)
11994 {
11995 u64 rcvctrl, reg;
11996 int did_enable = 0;
11997 u16 ctxt;
11998
11999 if (!rcd)
12000 return;
12001
12002 ctxt = rcd->ctxt;
12003
12004 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
12005
12006 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
12007 /* if the context already enabled, don't do the extra steps */
12008 if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
12009 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
12010 /* reset the tail and hdr addresses, and sequence count */
12011 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
12012 rcd->rcvhdrq_dma);
12013 if (hfi1_rcvhdrtail_kvaddr(rcd))
12014 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12015 rcd->rcvhdrqtailaddr_dma);
12016 hfi1_set_seq_cnt(rcd, 1);
12017
12018 /* reset the cached receive header queue head value */
12019 hfi1_set_rcd_head(rcd, 0);
12020
12021 /*
12022 * Zero the receive header queue so we don't get false
12023 * positives when checking the sequence number. The
12024 * sequence numbers could land exactly on the same spot.
12025 * E.g. a rcd restart before the receive header wrapped.
12026 */
12027 memset(rcd->rcvhdrq, 0, rcvhdrq_size(rcd));
12028
12029 /* starting timeout */
12030 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
12031
12032 /* enable the context */
12033 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
12034
12035 /* clean the egr buffer size first */
12036 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
12037 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
12038 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
12039 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
12040
12041 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
12042 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
12043 did_enable = 1;
12044
12045 /* zero RcvEgrIndexHead */
12046 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
12047
12048 /* set eager count and base index */
12049 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
12050 & RCV_EGR_CTRL_EGR_CNT_MASK)
12051 << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
12052 (((rcd->eager_base >> RCV_SHIFT)
12053 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
12054 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
12055 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
12056
12057 /*
12058 * Set TID (expected) count and base index.
12059 * rcd->expected_count is set to individual RcvArray entries,
12060 * not pairs, and the CSR takes a pair-count in groups of
12061 * four, so divide by 8.
12062 */
12063 reg = (((rcd->expected_count >> RCV_SHIFT)
12064 & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
12065 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
12066 (((rcd->expected_base >> RCV_SHIFT)
12067 & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
12068 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
12069 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
12070 if (ctxt == HFI1_CTRL_CTXT)
12071 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
12072 }
12073 if (op & HFI1_RCVCTRL_CTXT_DIS) {
12074 write_csr(dd, RCV_VL15, 0);
12075 /*
12076 * When receive context is being disabled turn on tail
12077 * update with a dummy tail address and then disable
12078 * receive context.
12079 */
12080 if (dd->rcvhdrtail_dummy_dma) {
12081 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12082 dd->rcvhdrtail_dummy_dma);
12083 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */
12084 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
12085 }
12086
12087 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
12088 }
12089 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) {
12090 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
12091 IS_RCVAVAIL_START + rcd->ctxt, true);
12092 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
12093 }
12094 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) {
12095 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
12096 IS_RCVAVAIL_START + rcd->ctxt, false);
12097 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
12098 }
12099 if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && hfi1_rcvhdrtail_kvaddr(rcd))
12100 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
12101 if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
12102 /* See comment on RcvCtxtCtrl.TailUpd above */
12103 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
12104 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
12105 }
12106 if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
12107 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
12108 if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
12109 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
12110 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
12111 /*
12112 * In one-packet-per-eager mode, the size comes from
12113 * the RcvArray entry.
12114 */
12115 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
12116 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
12117 }
12118 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
12119 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
12120 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
12121 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
12122 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
12123 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
12124 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
12125 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
12126 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
12127 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
12128 if (op & HFI1_RCVCTRL_URGENT_ENB)
12129 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
12130 IS_RCVURGENT_START + rcd->ctxt, true);
12131 if (op & HFI1_RCVCTRL_URGENT_DIS)
12132 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
12133 IS_RCVURGENT_START + rcd->ctxt, false);
12134
12135 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
12136 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl);
12137
12138 /* work around sticky RcvCtxtStatus.BlockedRHQFull */
12139 if (did_enable &&
12140 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
12141 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
12142 if (reg != 0) {
12143 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
12144 ctxt, reg);
12145 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
12146 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
12147 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
12148 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
12149 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
12150 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
12151 ctxt, reg, reg == 0 ? "not" : "still");
12152 }
12153 }
12154
12155 if (did_enable) {
12156 /*
12157 * The interrupt timeout and count must be set after
12158 * the context is enabled to take effect.
12159 */
12160 /* set interrupt timeout */
12161 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
12162 (u64)rcd->rcvavail_timeout <<
12163 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
12164
12165 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
12166 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
12167 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
12168 }
12169
12170 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
12171 /*
12172 * If the context has been disabled and the Tail Update has
12173 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
12174 * so it doesn't contain an address that is invalid.
12175 */
12176 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12177 dd->rcvhdrtail_dummy_dma);
12178 }
12179
hfi1_read_cntrs(struct hfi1_devdata * dd,char ** namep,u64 ** cntrp)12180 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
12181 {
12182 int ret;
12183 u64 val = 0;
12184
12185 if (namep) {
12186 ret = dd->cntrnameslen;
12187 *namep = dd->cntrnames;
12188 } else {
12189 const struct cntr_entry *entry;
12190 int i, j;
12191
12192 ret = (dd->ndevcntrs) * sizeof(u64);
12193
12194 /* Get the start of the block of counters */
12195 *cntrp = dd->cntrs;
12196
12197 /*
12198 * Now go and fill in each counter in the block.
12199 */
12200 for (i = 0; i < DEV_CNTR_LAST; i++) {
12201 entry = &dev_cntrs[i];
12202 hfi1_cdbg(CNTR, "reading %s", entry->name);
12203 if (entry->flags & CNTR_DISABLED) {
12204 /* Nothing */
12205 hfi1_cdbg(CNTR, "\tDisabled\n");
12206 } else {
12207 if (entry->flags & CNTR_VL) {
12208 hfi1_cdbg(CNTR, "\tPer VL\n");
12209 for (j = 0; j < C_VL_COUNT; j++) {
12210 val = entry->rw_cntr(entry,
12211 dd, j,
12212 CNTR_MODE_R,
12213 0);
12214 hfi1_cdbg(
12215 CNTR,
12216 "\t\tRead 0x%llx for %d\n",
12217 val, j);
12218 dd->cntrs[entry->offset + j] =
12219 val;
12220 }
12221 } else if (entry->flags & CNTR_SDMA) {
12222 hfi1_cdbg(CNTR,
12223 "\t Per SDMA Engine\n");
12224 for (j = 0; j < chip_sdma_engines(dd);
12225 j++) {
12226 val =
12227 entry->rw_cntr(entry, dd, j,
12228 CNTR_MODE_R, 0);
12229 hfi1_cdbg(CNTR,
12230 "\t\tRead 0x%llx for %d\n",
12231 val, j);
12232 dd->cntrs[entry->offset + j] =
12233 val;
12234 }
12235 } else {
12236 val = entry->rw_cntr(entry, dd,
12237 CNTR_INVALID_VL,
12238 CNTR_MODE_R, 0);
12239 dd->cntrs[entry->offset] = val;
12240 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12241 }
12242 }
12243 }
12244 }
12245 return ret;
12246 }
12247
12248 /*
12249 * Used by sysfs to create files for hfi stats to read
12250 */
hfi1_read_portcntrs(struct hfi1_pportdata * ppd,char ** namep,u64 ** cntrp)12251 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
12252 {
12253 int ret;
12254 u64 val = 0;
12255
12256 if (namep) {
12257 ret = ppd->dd->portcntrnameslen;
12258 *namep = ppd->dd->portcntrnames;
12259 } else {
12260 const struct cntr_entry *entry;
12261 int i, j;
12262
12263 ret = ppd->dd->nportcntrs * sizeof(u64);
12264 *cntrp = ppd->cntrs;
12265
12266 for (i = 0; i < PORT_CNTR_LAST; i++) {
12267 entry = &port_cntrs[i];
12268 hfi1_cdbg(CNTR, "reading %s", entry->name);
12269 if (entry->flags & CNTR_DISABLED) {
12270 /* Nothing */
12271 hfi1_cdbg(CNTR, "\tDisabled\n");
12272 continue;
12273 }
12274
12275 if (entry->flags & CNTR_VL) {
12276 hfi1_cdbg(CNTR, "\tPer VL");
12277 for (j = 0; j < C_VL_COUNT; j++) {
12278 val = entry->rw_cntr(entry, ppd, j,
12279 CNTR_MODE_R,
12280 0);
12281 hfi1_cdbg(
12282 CNTR,
12283 "\t\tRead 0x%llx for %d",
12284 val, j);
12285 ppd->cntrs[entry->offset + j] = val;
12286 }
12287 } else {
12288 val = entry->rw_cntr(entry, ppd,
12289 CNTR_INVALID_VL,
12290 CNTR_MODE_R,
12291 0);
12292 ppd->cntrs[entry->offset] = val;
12293 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12294 }
12295 }
12296 }
12297 return ret;
12298 }
12299
free_cntrs(struct hfi1_devdata * dd)12300 static void free_cntrs(struct hfi1_devdata *dd)
12301 {
12302 struct hfi1_pportdata *ppd;
12303 int i;
12304
12305 if (dd->synth_stats_timer.function)
12306 del_timer_sync(&dd->synth_stats_timer);
12307 ppd = (struct hfi1_pportdata *)(dd + 1);
12308 for (i = 0; i < dd->num_pports; i++, ppd++) {
12309 kfree(ppd->cntrs);
12310 kfree(ppd->scntrs);
12311 free_percpu(ppd->ibport_data.rvp.rc_acks);
12312 free_percpu(ppd->ibport_data.rvp.rc_qacks);
12313 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
12314 ppd->cntrs = NULL;
12315 ppd->scntrs = NULL;
12316 ppd->ibport_data.rvp.rc_acks = NULL;
12317 ppd->ibport_data.rvp.rc_qacks = NULL;
12318 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
12319 }
12320 kfree(dd->portcntrnames);
12321 dd->portcntrnames = NULL;
12322 kfree(dd->cntrs);
12323 dd->cntrs = NULL;
12324 kfree(dd->scntrs);
12325 dd->scntrs = NULL;
12326 kfree(dd->cntrnames);
12327 dd->cntrnames = NULL;
12328 if (dd->update_cntr_wq) {
12329 destroy_workqueue(dd->update_cntr_wq);
12330 dd->update_cntr_wq = NULL;
12331 }
12332 }
12333
read_dev_port_cntr(struct hfi1_devdata * dd,struct cntr_entry * entry,u64 * psval,void * context,int vl)12334 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
12335 u64 *psval, void *context, int vl)
12336 {
12337 u64 val;
12338 u64 sval = *psval;
12339
12340 if (entry->flags & CNTR_DISABLED) {
12341 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12342 return 0;
12343 }
12344
12345 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12346
12347 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
12348
12349 /* If its a synthetic counter there is more work we need to do */
12350 if (entry->flags & CNTR_SYNTH) {
12351 if (sval == CNTR_MAX) {
12352 /* No need to read already saturated */
12353 return CNTR_MAX;
12354 }
12355
12356 if (entry->flags & CNTR_32BIT) {
12357 /* 32bit counters can wrap multiple times */
12358 u64 upper = sval >> 32;
12359 u64 lower = (sval << 32) >> 32;
12360
12361 if (lower > val) { /* hw wrapped */
12362 if (upper == CNTR_32BIT_MAX)
12363 val = CNTR_MAX;
12364 else
12365 upper++;
12366 }
12367
12368 if (val != CNTR_MAX)
12369 val = (upper << 32) | val;
12370
12371 } else {
12372 /* If we rolled we are saturated */
12373 if ((val < sval) || (val > CNTR_MAX))
12374 val = CNTR_MAX;
12375 }
12376 }
12377
12378 *psval = val;
12379
12380 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12381
12382 return val;
12383 }
12384
write_dev_port_cntr(struct hfi1_devdata * dd,struct cntr_entry * entry,u64 * psval,void * context,int vl,u64 data)12385 static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
12386 struct cntr_entry *entry,
12387 u64 *psval, void *context, int vl, u64 data)
12388 {
12389 u64 val;
12390
12391 if (entry->flags & CNTR_DISABLED) {
12392 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12393 return 0;
12394 }
12395
12396 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12397
12398 if (entry->flags & CNTR_SYNTH) {
12399 *psval = data;
12400 if (entry->flags & CNTR_32BIT) {
12401 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12402 (data << 32) >> 32);
12403 val = data; /* return the full 64bit value */
12404 } else {
12405 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12406 data);
12407 }
12408 } else {
12409 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
12410 }
12411
12412 *psval = val;
12413
12414 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12415
12416 return val;
12417 }
12418
read_dev_cntr(struct hfi1_devdata * dd,int index,int vl)12419 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
12420 {
12421 struct cntr_entry *entry;
12422 u64 *sval;
12423
12424 entry = &dev_cntrs[index];
12425 sval = dd->scntrs + entry->offset;
12426
12427 if (vl != CNTR_INVALID_VL)
12428 sval += vl;
12429
12430 return read_dev_port_cntr(dd, entry, sval, dd, vl);
12431 }
12432
write_dev_cntr(struct hfi1_devdata * dd,int index,int vl,u64 data)12433 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
12434 {
12435 struct cntr_entry *entry;
12436 u64 *sval;
12437
12438 entry = &dev_cntrs[index];
12439 sval = dd->scntrs + entry->offset;
12440
12441 if (vl != CNTR_INVALID_VL)
12442 sval += vl;
12443
12444 return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
12445 }
12446
read_port_cntr(struct hfi1_pportdata * ppd,int index,int vl)12447 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
12448 {
12449 struct cntr_entry *entry;
12450 u64 *sval;
12451
12452 entry = &port_cntrs[index];
12453 sval = ppd->scntrs + entry->offset;
12454
12455 if (vl != CNTR_INVALID_VL)
12456 sval += vl;
12457
12458 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12459 (index <= C_RCV_HDR_OVF_LAST)) {
12460 /* We do not want to bother for disabled contexts */
12461 return 0;
12462 }
12463
12464 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
12465 }
12466
write_port_cntr(struct hfi1_pportdata * ppd,int index,int vl,u64 data)12467 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
12468 {
12469 struct cntr_entry *entry;
12470 u64 *sval;
12471
12472 entry = &port_cntrs[index];
12473 sval = ppd->scntrs + entry->offset;
12474
12475 if (vl != CNTR_INVALID_VL)
12476 sval += vl;
12477
12478 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12479 (index <= C_RCV_HDR_OVF_LAST)) {
12480 /* We do not want to bother for disabled contexts */
12481 return 0;
12482 }
12483
12484 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
12485 }
12486
do_update_synth_timer(struct work_struct * work)12487 static void do_update_synth_timer(struct work_struct *work)
12488 {
12489 u64 cur_tx;
12490 u64 cur_rx;
12491 u64 total_flits;
12492 u8 update = 0;
12493 int i, j, vl;
12494 struct hfi1_pportdata *ppd;
12495 struct cntr_entry *entry;
12496 struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata,
12497 update_cntr_work);
12498
12499 /*
12500 * Rather than keep beating on the CSRs pick a minimal set that we can
12501 * check to watch for potential roll over. We can do this by looking at
12502 * the number of flits sent/recv. If the total flits exceeds 32bits then
12503 * we have to iterate all the counters and update.
12504 */
12505 entry = &dev_cntrs[C_DC_RCV_FLITS];
12506 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12507
12508 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12509 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12510
12511 hfi1_cdbg(
12512 CNTR,
12513 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
12514 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
12515
12516 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
12517 /*
12518 * May not be strictly necessary to update but it won't hurt and
12519 * simplifies the logic here.
12520 */
12521 update = 1;
12522 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
12523 dd->unit);
12524 } else {
12525 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
12526 hfi1_cdbg(CNTR,
12527 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
12528 total_flits, (u64)CNTR_32BIT_MAX);
12529 if (total_flits >= CNTR_32BIT_MAX) {
12530 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
12531 dd->unit);
12532 update = 1;
12533 }
12534 }
12535
12536 if (update) {
12537 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
12538 for (i = 0; i < DEV_CNTR_LAST; i++) {
12539 entry = &dev_cntrs[i];
12540 if (entry->flags & CNTR_VL) {
12541 for (vl = 0; vl < C_VL_COUNT; vl++)
12542 read_dev_cntr(dd, i, vl);
12543 } else {
12544 read_dev_cntr(dd, i, CNTR_INVALID_VL);
12545 }
12546 }
12547 ppd = (struct hfi1_pportdata *)(dd + 1);
12548 for (i = 0; i < dd->num_pports; i++, ppd++) {
12549 for (j = 0; j < PORT_CNTR_LAST; j++) {
12550 entry = &port_cntrs[j];
12551 if (entry->flags & CNTR_VL) {
12552 for (vl = 0; vl < C_VL_COUNT; vl++)
12553 read_port_cntr(ppd, j, vl);
12554 } else {
12555 read_port_cntr(ppd, j, CNTR_INVALID_VL);
12556 }
12557 }
12558 }
12559
12560 /*
12561 * We want the value in the register. The goal is to keep track
12562 * of the number of "ticks" not the counter value. In other
12563 * words if the register rolls we want to notice it and go ahead
12564 * and force an update.
12565 */
12566 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12567 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12568 CNTR_MODE_R, 0);
12569
12570 entry = &dev_cntrs[C_DC_RCV_FLITS];
12571 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12572 CNTR_MODE_R, 0);
12573
12574 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
12575 dd->unit, dd->last_tx, dd->last_rx);
12576
12577 } else {
12578 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
12579 }
12580 }
12581
update_synth_timer(struct timer_list * t)12582 static void update_synth_timer(struct timer_list *t)
12583 {
12584 struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer);
12585
12586 queue_work(dd->update_cntr_wq, &dd->update_cntr_work);
12587 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12588 }
12589
12590 #define C_MAX_NAME 16 /* 15 chars + one for /0 */
init_cntrs(struct hfi1_devdata * dd)12591 static int init_cntrs(struct hfi1_devdata *dd)
12592 {
12593 int i, rcv_ctxts, j;
12594 size_t sz;
12595 char *p;
12596 char name[C_MAX_NAME];
12597 struct hfi1_pportdata *ppd;
12598 const char *bit_type_32 = ",32";
12599 const int bit_type_32_sz = strlen(bit_type_32);
12600 u32 sdma_engines = chip_sdma_engines(dd);
12601
12602 /* set up the stats timer; the add_timer is done at the end */
12603 timer_setup(&dd->synth_stats_timer, update_synth_timer, 0);
12604
12605 /***********************/
12606 /* per device counters */
12607 /***********************/
12608
12609 /* size names and determine how many we have*/
12610 dd->ndevcntrs = 0;
12611 sz = 0;
12612
12613 for (i = 0; i < DEV_CNTR_LAST; i++) {
12614 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12615 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
12616 continue;
12617 }
12618
12619 if (dev_cntrs[i].flags & CNTR_VL) {
12620 dev_cntrs[i].offset = dd->ndevcntrs;
12621 for (j = 0; j < C_VL_COUNT; j++) {
12622 snprintf(name, C_MAX_NAME, "%s%d",
12623 dev_cntrs[i].name, vl_from_idx(j));
12624 sz += strlen(name);
12625 /* Add ",32" for 32-bit counters */
12626 if (dev_cntrs[i].flags & CNTR_32BIT)
12627 sz += bit_type_32_sz;
12628 sz++;
12629 dd->ndevcntrs++;
12630 }
12631 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12632 dev_cntrs[i].offset = dd->ndevcntrs;
12633 for (j = 0; j < sdma_engines; j++) {
12634 snprintf(name, C_MAX_NAME, "%s%d",
12635 dev_cntrs[i].name, j);
12636 sz += strlen(name);
12637 /* Add ",32" for 32-bit counters */
12638 if (dev_cntrs[i].flags & CNTR_32BIT)
12639 sz += bit_type_32_sz;
12640 sz++;
12641 dd->ndevcntrs++;
12642 }
12643 } else {
12644 /* +1 for newline. */
12645 sz += strlen(dev_cntrs[i].name) + 1;
12646 /* Add ",32" for 32-bit counters */
12647 if (dev_cntrs[i].flags & CNTR_32BIT)
12648 sz += bit_type_32_sz;
12649 dev_cntrs[i].offset = dd->ndevcntrs;
12650 dd->ndevcntrs++;
12651 }
12652 }
12653
12654 /* allocate space for the counter values */
12655 dd->cntrs = kcalloc(dd->ndevcntrs + num_driver_cntrs, sizeof(u64),
12656 GFP_KERNEL);
12657 if (!dd->cntrs)
12658 goto bail;
12659
12660 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
12661 if (!dd->scntrs)
12662 goto bail;
12663
12664 /* allocate space for the counter names */
12665 dd->cntrnameslen = sz;
12666 dd->cntrnames = kmalloc(sz, GFP_KERNEL);
12667 if (!dd->cntrnames)
12668 goto bail;
12669
12670 /* fill in the names */
12671 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
12672 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12673 /* Nothing */
12674 } else if (dev_cntrs[i].flags & CNTR_VL) {
12675 for (j = 0; j < C_VL_COUNT; j++) {
12676 snprintf(name, C_MAX_NAME, "%s%d",
12677 dev_cntrs[i].name,
12678 vl_from_idx(j));
12679 memcpy(p, name, strlen(name));
12680 p += strlen(name);
12681
12682 /* Counter is 32 bits */
12683 if (dev_cntrs[i].flags & CNTR_32BIT) {
12684 memcpy(p, bit_type_32, bit_type_32_sz);
12685 p += bit_type_32_sz;
12686 }
12687
12688 *p++ = '\n';
12689 }
12690 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12691 for (j = 0; j < sdma_engines; j++) {
12692 snprintf(name, C_MAX_NAME, "%s%d",
12693 dev_cntrs[i].name, j);
12694 memcpy(p, name, strlen(name));
12695 p += strlen(name);
12696
12697 /* Counter is 32 bits */
12698 if (dev_cntrs[i].flags & CNTR_32BIT) {
12699 memcpy(p, bit_type_32, bit_type_32_sz);
12700 p += bit_type_32_sz;
12701 }
12702
12703 *p++ = '\n';
12704 }
12705 } else {
12706 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
12707 p += strlen(dev_cntrs[i].name);
12708
12709 /* Counter is 32 bits */
12710 if (dev_cntrs[i].flags & CNTR_32BIT) {
12711 memcpy(p, bit_type_32, bit_type_32_sz);
12712 p += bit_type_32_sz;
12713 }
12714
12715 *p++ = '\n';
12716 }
12717 }
12718
12719 /*********************/
12720 /* per port counters */
12721 /*********************/
12722
12723 /*
12724 * Go through the counters for the overflows and disable the ones we
12725 * don't need. This varies based on platform so we need to do it
12726 * dynamically here.
12727 */
12728 rcv_ctxts = dd->num_rcv_contexts;
12729 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
12730 i <= C_RCV_HDR_OVF_LAST; i++) {
12731 port_cntrs[i].flags |= CNTR_DISABLED;
12732 }
12733
12734 /* size port counter names and determine how many we have*/
12735 sz = 0;
12736 dd->nportcntrs = 0;
12737 for (i = 0; i < PORT_CNTR_LAST; i++) {
12738 if (port_cntrs[i].flags & CNTR_DISABLED) {
12739 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
12740 continue;
12741 }
12742
12743 if (port_cntrs[i].flags & CNTR_VL) {
12744 port_cntrs[i].offset = dd->nportcntrs;
12745 for (j = 0; j < C_VL_COUNT; j++) {
12746 snprintf(name, C_MAX_NAME, "%s%d",
12747 port_cntrs[i].name, vl_from_idx(j));
12748 sz += strlen(name);
12749 /* Add ",32" for 32-bit counters */
12750 if (port_cntrs[i].flags & CNTR_32BIT)
12751 sz += bit_type_32_sz;
12752 sz++;
12753 dd->nportcntrs++;
12754 }
12755 } else {
12756 /* +1 for newline */
12757 sz += strlen(port_cntrs[i].name) + 1;
12758 /* Add ",32" for 32-bit counters */
12759 if (port_cntrs[i].flags & CNTR_32BIT)
12760 sz += bit_type_32_sz;
12761 port_cntrs[i].offset = dd->nportcntrs;
12762 dd->nportcntrs++;
12763 }
12764 }
12765
12766 /* allocate space for the counter names */
12767 dd->portcntrnameslen = sz;
12768 dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
12769 if (!dd->portcntrnames)
12770 goto bail;
12771
12772 /* fill in port cntr names */
12773 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
12774 if (port_cntrs[i].flags & CNTR_DISABLED)
12775 continue;
12776
12777 if (port_cntrs[i].flags & CNTR_VL) {
12778 for (j = 0; j < C_VL_COUNT; j++) {
12779 snprintf(name, C_MAX_NAME, "%s%d",
12780 port_cntrs[i].name, vl_from_idx(j));
12781 memcpy(p, name, strlen(name));
12782 p += strlen(name);
12783
12784 /* Counter is 32 bits */
12785 if (port_cntrs[i].flags & CNTR_32BIT) {
12786 memcpy(p, bit_type_32, bit_type_32_sz);
12787 p += bit_type_32_sz;
12788 }
12789
12790 *p++ = '\n';
12791 }
12792 } else {
12793 memcpy(p, port_cntrs[i].name,
12794 strlen(port_cntrs[i].name));
12795 p += strlen(port_cntrs[i].name);
12796
12797 /* Counter is 32 bits */
12798 if (port_cntrs[i].flags & CNTR_32BIT) {
12799 memcpy(p, bit_type_32, bit_type_32_sz);
12800 p += bit_type_32_sz;
12801 }
12802
12803 *p++ = '\n';
12804 }
12805 }
12806
12807 /* allocate per port storage for counter values */
12808 ppd = (struct hfi1_pportdata *)(dd + 1);
12809 for (i = 0; i < dd->num_pports; i++, ppd++) {
12810 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12811 if (!ppd->cntrs)
12812 goto bail;
12813
12814 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12815 if (!ppd->scntrs)
12816 goto bail;
12817 }
12818
12819 /* CPU counters need to be allocated and zeroed */
12820 if (init_cpu_counters(dd))
12821 goto bail;
12822
12823 dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d",
12824 WQ_MEM_RECLAIM, dd->unit);
12825 if (!dd->update_cntr_wq)
12826 goto bail;
12827
12828 INIT_WORK(&dd->update_cntr_work, do_update_synth_timer);
12829
12830 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12831 return 0;
12832 bail:
12833 free_cntrs(dd);
12834 return -ENOMEM;
12835 }
12836
chip_to_opa_lstate(struct hfi1_devdata * dd,u32 chip_lstate)12837 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12838 {
12839 switch (chip_lstate) {
12840 case LSTATE_DOWN:
12841 return IB_PORT_DOWN;
12842 case LSTATE_INIT:
12843 return IB_PORT_INIT;
12844 case LSTATE_ARMED:
12845 return IB_PORT_ARMED;
12846 case LSTATE_ACTIVE:
12847 return IB_PORT_ACTIVE;
12848 default:
12849 dd_dev_err(dd,
12850 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12851 chip_lstate);
12852 return IB_PORT_DOWN;
12853 }
12854 }
12855
chip_to_opa_pstate(struct hfi1_devdata * dd,u32 chip_pstate)12856 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12857 {
12858 /* look at the HFI meta-states only */
12859 switch (chip_pstate & 0xf0) {
12860 case PLS_DISABLED:
12861 return IB_PORTPHYSSTATE_DISABLED;
12862 case PLS_OFFLINE:
12863 return OPA_PORTPHYSSTATE_OFFLINE;
12864 case PLS_POLLING:
12865 return IB_PORTPHYSSTATE_POLLING;
12866 case PLS_CONFIGPHY:
12867 return IB_PORTPHYSSTATE_TRAINING;
12868 case PLS_LINKUP:
12869 return IB_PORTPHYSSTATE_LINKUP;
12870 case PLS_PHYTEST:
12871 return IB_PORTPHYSSTATE_PHY_TEST;
12872 default:
12873 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
12874 chip_pstate);
12875 return IB_PORTPHYSSTATE_DISABLED;
12876 }
12877 }
12878
12879 /* return the OPA port logical state name */
opa_lstate_name(u32 lstate)12880 const char *opa_lstate_name(u32 lstate)
12881 {
12882 static const char * const port_logical_names[] = {
12883 "PORT_NOP",
12884 "PORT_DOWN",
12885 "PORT_INIT",
12886 "PORT_ARMED",
12887 "PORT_ACTIVE",
12888 "PORT_ACTIVE_DEFER",
12889 };
12890 if (lstate < ARRAY_SIZE(port_logical_names))
12891 return port_logical_names[lstate];
12892 return "unknown";
12893 }
12894
12895 /* return the OPA port physical state name */
opa_pstate_name(u32 pstate)12896 const char *opa_pstate_name(u32 pstate)
12897 {
12898 static const char * const port_physical_names[] = {
12899 "PHYS_NOP",
12900 "reserved1",
12901 "PHYS_POLL",
12902 "PHYS_DISABLED",
12903 "PHYS_TRAINING",
12904 "PHYS_LINKUP",
12905 "PHYS_LINK_ERR_RECOVER",
12906 "PHYS_PHY_TEST",
12907 "reserved8",
12908 "PHYS_OFFLINE",
12909 "PHYS_GANGED",
12910 "PHYS_TEST",
12911 };
12912 if (pstate < ARRAY_SIZE(port_physical_names))
12913 return port_physical_names[pstate];
12914 return "unknown";
12915 }
12916
12917 /**
12918 * update_statusp - Update userspace status flag
12919 * @ppd: Port data structure
12920 * @state: port state information
12921 *
12922 * Actual port status is determined by the host_link_state value
12923 * in the ppd.
12924 *
12925 * host_link_state MUST be updated before updating the user space
12926 * statusp.
12927 */
update_statusp(struct hfi1_pportdata * ppd,u32 state)12928 static void update_statusp(struct hfi1_pportdata *ppd, u32 state)
12929 {
12930 /*
12931 * Set port status flags in the page mapped into userspace
12932 * memory. Do it here to ensure a reliable state - this is
12933 * the only function called by all state handling code.
12934 * Always set the flags due to the fact that the cache value
12935 * might have been changed explicitly outside of this
12936 * function.
12937 */
12938 if (ppd->statusp) {
12939 switch (state) {
12940 case IB_PORT_DOWN:
12941 case IB_PORT_INIT:
12942 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12943 HFI1_STATUS_IB_READY);
12944 break;
12945 case IB_PORT_ARMED:
12946 *ppd->statusp |= HFI1_STATUS_IB_CONF;
12947 break;
12948 case IB_PORT_ACTIVE:
12949 *ppd->statusp |= HFI1_STATUS_IB_READY;
12950 break;
12951 }
12952 }
12953 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
12954 opa_lstate_name(state), state);
12955 }
12956
12957 /**
12958 * wait_logical_linkstate - wait for an IB link state change to occur
12959 * @ppd: port device
12960 * @state: the state to wait for
12961 * @msecs: the number of milliseconds to wait
12962 *
12963 * Wait up to msecs milliseconds for IB link state change to occur.
12964 * For now, take the easy polling route.
12965 * Returns 0 if state reached, otherwise -ETIMEDOUT.
12966 */
wait_logical_linkstate(struct hfi1_pportdata * ppd,u32 state,int msecs)12967 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12968 int msecs)
12969 {
12970 unsigned long timeout;
12971 u32 new_state;
12972
12973 timeout = jiffies + msecs_to_jiffies(msecs);
12974 while (1) {
12975 new_state = chip_to_opa_lstate(ppd->dd,
12976 read_logical_state(ppd->dd));
12977 if (new_state == state)
12978 break;
12979 if (time_after(jiffies, timeout)) {
12980 dd_dev_err(ppd->dd,
12981 "timeout waiting for link state 0x%x\n",
12982 state);
12983 return -ETIMEDOUT;
12984 }
12985 msleep(20);
12986 }
12987
12988 return 0;
12989 }
12990
log_state_transition(struct hfi1_pportdata * ppd,u32 state)12991 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state)
12992 {
12993 u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state);
12994
12995 dd_dev_info(ppd->dd,
12996 "physical state changed to %s (0x%x), phy 0x%x\n",
12997 opa_pstate_name(ib_pstate), ib_pstate, state);
12998 }
12999
13000 /*
13001 * Read the physical hardware link state and check if it matches host
13002 * drivers anticipated state.
13003 */
log_physical_state(struct hfi1_pportdata * ppd,u32 state)13004 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state)
13005 {
13006 u32 read_state = read_physical_state(ppd->dd);
13007
13008 if (read_state == state) {
13009 log_state_transition(ppd, state);
13010 } else {
13011 dd_dev_err(ppd->dd,
13012 "anticipated phy link state 0x%x, read 0x%x\n",
13013 state, read_state);
13014 }
13015 }
13016
13017 /*
13018 * wait_physical_linkstate - wait for an physical link state change to occur
13019 * @ppd: port device
13020 * @state: the state to wait for
13021 * @msecs: the number of milliseconds to wait
13022 *
13023 * Wait up to msecs milliseconds for physical link state change to occur.
13024 * Returns 0 if state reached, otherwise -ETIMEDOUT.
13025 */
wait_physical_linkstate(struct hfi1_pportdata * ppd,u32 state,int msecs)13026 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
13027 int msecs)
13028 {
13029 u32 read_state;
13030 unsigned long timeout;
13031
13032 timeout = jiffies + msecs_to_jiffies(msecs);
13033 while (1) {
13034 read_state = read_physical_state(ppd->dd);
13035 if (read_state == state)
13036 break;
13037 if (time_after(jiffies, timeout)) {
13038 dd_dev_err(ppd->dd,
13039 "timeout waiting for phy link state 0x%x\n",
13040 state);
13041 return -ETIMEDOUT;
13042 }
13043 usleep_range(1950, 2050); /* sleep 2ms-ish */
13044 }
13045
13046 log_state_transition(ppd, state);
13047 return 0;
13048 }
13049
13050 /*
13051 * wait_phys_link_offline_quiet_substates - wait for any offline substate
13052 * @ppd: port device
13053 * @msecs: the number of milliseconds to wait
13054 *
13055 * Wait up to msecs milliseconds for any offline physical link
13056 * state change to occur.
13057 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT.
13058 */
wait_phys_link_offline_substates(struct hfi1_pportdata * ppd,int msecs)13059 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
13060 int msecs)
13061 {
13062 u32 read_state;
13063 unsigned long timeout;
13064
13065 timeout = jiffies + msecs_to_jiffies(msecs);
13066 while (1) {
13067 read_state = read_physical_state(ppd->dd);
13068 if ((read_state & 0xF0) == PLS_OFFLINE)
13069 break;
13070 if (time_after(jiffies, timeout)) {
13071 dd_dev_err(ppd->dd,
13072 "timeout waiting for phy link offline.quiet substates. Read state 0x%x, %dms\n",
13073 read_state, msecs);
13074 return -ETIMEDOUT;
13075 }
13076 usleep_range(1950, 2050); /* sleep 2ms-ish */
13077 }
13078
13079 log_state_transition(ppd, read_state);
13080 return read_state;
13081 }
13082
13083 /*
13084 * wait_phys_link_out_of_offline - wait for any out of offline state
13085 * @ppd: port device
13086 * @msecs: the number of milliseconds to wait
13087 *
13088 * Wait up to msecs milliseconds for any out of offline physical link
13089 * state change to occur.
13090 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT.
13091 */
wait_phys_link_out_of_offline(struct hfi1_pportdata * ppd,int msecs)13092 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd,
13093 int msecs)
13094 {
13095 u32 read_state;
13096 unsigned long timeout;
13097
13098 timeout = jiffies + msecs_to_jiffies(msecs);
13099 while (1) {
13100 read_state = read_physical_state(ppd->dd);
13101 if ((read_state & 0xF0) != PLS_OFFLINE)
13102 break;
13103 if (time_after(jiffies, timeout)) {
13104 dd_dev_err(ppd->dd,
13105 "timeout waiting for phy link out of offline. Read state 0x%x, %dms\n",
13106 read_state, msecs);
13107 return -ETIMEDOUT;
13108 }
13109 usleep_range(1950, 2050); /* sleep 2ms-ish */
13110 }
13111
13112 log_state_transition(ppd, read_state);
13113 return read_state;
13114 }
13115
13116 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
13117 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
13118
13119 #define SET_STATIC_RATE_CONTROL_SMASK(r) \
13120 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
13121
hfi1_init_ctxt(struct send_context * sc)13122 void hfi1_init_ctxt(struct send_context *sc)
13123 {
13124 if (sc) {
13125 struct hfi1_devdata *dd = sc->dd;
13126 u64 reg;
13127 u8 set = (sc->type == SC_USER ?
13128 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
13129 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
13130 reg = read_kctxt_csr(dd, sc->hw_context,
13131 SEND_CTXT_CHECK_ENABLE);
13132 if (set)
13133 CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
13134 else
13135 SET_STATIC_RATE_CONTROL_SMASK(reg);
13136 write_kctxt_csr(dd, sc->hw_context,
13137 SEND_CTXT_CHECK_ENABLE, reg);
13138 }
13139 }
13140
hfi1_tempsense_rd(struct hfi1_devdata * dd,struct hfi1_temp * temp)13141 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
13142 {
13143 int ret = 0;
13144 u64 reg;
13145
13146 if (dd->icode != ICODE_RTL_SILICON) {
13147 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
13148 dd_dev_info(dd, "%s: tempsense not supported by HW\n",
13149 __func__);
13150 return -EINVAL;
13151 }
13152 reg = read_csr(dd, ASIC_STS_THERM);
13153 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
13154 ASIC_STS_THERM_CURR_TEMP_MASK);
13155 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
13156 ASIC_STS_THERM_LO_TEMP_MASK);
13157 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
13158 ASIC_STS_THERM_HI_TEMP_MASK);
13159 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
13160 ASIC_STS_THERM_CRIT_TEMP_MASK);
13161 /* triggers is a 3-bit value - 1 bit per trigger. */
13162 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
13163
13164 return ret;
13165 }
13166
13167 /* ========================================================================= */
13168
13169 /**
13170 * read_mod_write() - Calculate the IRQ register index and set/clear the bits
13171 * @dd: valid devdata
13172 * @src: IRQ source to determine register index from
13173 * @bits: the bits to set or clear
13174 * @set: true == set the bits, false == clear the bits
13175 *
13176 */
read_mod_write(struct hfi1_devdata * dd,u16 src,u64 bits,bool set)13177 static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits,
13178 bool set)
13179 {
13180 u64 reg;
13181 u16 idx = src / BITS_PER_REGISTER;
13182
13183 spin_lock(&dd->irq_src_lock);
13184 reg = read_csr(dd, CCE_INT_MASK + (8 * idx));
13185 if (set)
13186 reg |= bits;
13187 else
13188 reg &= ~bits;
13189 write_csr(dd, CCE_INT_MASK + (8 * idx), reg);
13190 spin_unlock(&dd->irq_src_lock);
13191 }
13192
13193 /**
13194 * set_intr_bits() - Enable/disable a range (one or more) IRQ sources
13195 * @dd: valid devdata
13196 * @first: first IRQ source to set/clear
13197 * @last: last IRQ source (inclusive) to set/clear
13198 * @set: true == set the bits, false == clear the bits
13199 *
13200 * If first == last, set the exact source.
13201 */
set_intr_bits(struct hfi1_devdata * dd,u16 first,u16 last,bool set)13202 int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set)
13203 {
13204 u64 bits = 0;
13205 u64 bit;
13206 u16 src;
13207
13208 if (first > NUM_INTERRUPT_SOURCES || last > NUM_INTERRUPT_SOURCES)
13209 return -EINVAL;
13210
13211 if (last < first)
13212 return -ERANGE;
13213
13214 for (src = first; src <= last; src++) {
13215 bit = src % BITS_PER_REGISTER;
13216 /* wrapped to next register? */
13217 if (!bit && bits) {
13218 read_mod_write(dd, src - 1, bits, set);
13219 bits = 0;
13220 }
13221 bits |= BIT_ULL(bit);
13222 }
13223 read_mod_write(dd, last, bits, set);
13224
13225 return 0;
13226 }
13227
13228 /*
13229 * Clear all interrupt sources on the chip.
13230 */
clear_all_interrupts(struct hfi1_devdata * dd)13231 void clear_all_interrupts(struct hfi1_devdata *dd)
13232 {
13233 int i;
13234
13235 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13236 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
13237
13238 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
13239 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
13240 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
13241 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
13242 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
13243 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
13244 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
13245 for (i = 0; i < chip_send_contexts(dd); i++)
13246 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
13247 for (i = 0; i < chip_sdma_engines(dd); i++)
13248 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
13249
13250 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
13251 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
13252 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
13253 }
13254
13255 /*
13256 * Remap the interrupt source from the general handler to the given MSI-X
13257 * interrupt.
13258 */
remap_intr(struct hfi1_devdata * dd,int isrc,int msix_intr)13259 void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
13260 {
13261 u64 reg;
13262 int m, n;
13263
13264 /* clear from the handled mask of the general interrupt */
13265 m = isrc / 64;
13266 n = isrc % 64;
13267 if (likely(m < CCE_NUM_INT_CSRS)) {
13268 dd->gi_mask[m] &= ~((u64)1 << n);
13269 } else {
13270 dd_dev_err(dd, "remap interrupt err\n");
13271 return;
13272 }
13273
13274 /* direct the chip source to the given MSI-X interrupt */
13275 m = isrc / 8;
13276 n = isrc % 8;
13277 reg = read_csr(dd, CCE_INT_MAP + (8 * m));
13278 reg &= ~((u64)0xff << (8 * n));
13279 reg |= ((u64)msix_intr & 0xff) << (8 * n);
13280 write_csr(dd, CCE_INT_MAP + (8 * m), reg);
13281 }
13282
remap_sdma_interrupts(struct hfi1_devdata * dd,int engine,int msix_intr)13283 void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr)
13284 {
13285 /*
13286 * SDMA engine interrupt sources grouped by type, rather than
13287 * engine. Per-engine interrupts are as follows:
13288 * SDMA
13289 * SDMAProgress
13290 * SDMAIdle
13291 */
13292 remap_intr(dd, IS_SDMA_START + engine, msix_intr);
13293 remap_intr(dd, IS_SDMA_PROGRESS_START + engine, msix_intr);
13294 remap_intr(dd, IS_SDMA_IDLE_START + engine, msix_intr);
13295 }
13296
13297 /*
13298 * Set the general handler to accept all interrupts, remap all
13299 * chip interrupts back to MSI-X 0.
13300 */
reset_interrupts(struct hfi1_devdata * dd)13301 void reset_interrupts(struct hfi1_devdata *dd)
13302 {
13303 int i;
13304
13305 /* all interrupts handled by the general handler */
13306 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13307 dd->gi_mask[i] = ~(u64)0;
13308
13309 /* all chip interrupts map to MSI-X 0 */
13310 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13311 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13312 }
13313
13314 /**
13315 * set_up_interrupts() - Initialize the IRQ resources and state
13316 * @dd: valid devdata
13317 *
13318 */
set_up_interrupts(struct hfi1_devdata * dd)13319 static int set_up_interrupts(struct hfi1_devdata *dd)
13320 {
13321 int ret;
13322
13323 /* mask all interrupts */
13324 set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false);
13325
13326 /* clear all pending interrupts */
13327 clear_all_interrupts(dd);
13328
13329 /* reset general handler mask, chip MSI-X mappings */
13330 reset_interrupts(dd);
13331
13332 /* ask for MSI-X interrupts */
13333 ret = msix_initialize(dd);
13334 if (ret)
13335 return ret;
13336
13337 ret = msix_request_irqs(dd);
13338 if (ret)
13339 msix_clean_up_interrupts(dd);
13340
13341 return ret;
13342 }
13343
13344 /*
13345 * Set up context values in dd. Sets:
13346 *
13347 * num_rcv_contexts - number of contexts being used
13348 * n_krcv_queues - number of kernel contexts
13349 * first_dyn_alloc_ctxt - first dynamically allocated context
13350 * in array of contexts
13351 * freectxts - number of free user contexts
13352 * num_send_contexts - number of PIO send contexts being used
13353 * num_netdev_contexts - number of contexts reserved for netdev
13354 */
set_up_context_variables(struct hfi1_devdata * dd)13355 static int set_up_context_variables(struct hfi1_devdata *dd)
13356 {
13357 unsigned long num_kernel_contexts;
13358 u16 num_netdev_contexts;
13359 int ret;
13360 unsigned ngroups;
13361 int rmt_count;
13362 int user_rmt_reduced;
13363 u32 n_usr_ctxts;
13364 u32 send_contexts = chip_send_contexts(dd);
13365 u32 rcv_contexts = chip_rcv_contexts(dd);
13366
13367 /*
13368 * Kernel receive contexts:
13369 * - Context 0 - control context (VL15/multicast/error)
13370 * - Context 1 - first kernel context
13371 * - Context 2 - second kernel context
13372 * ...
13373 */
13374 if (n_krcvqs)
13375 /*
13376 * n_krcvqs is the sum of module parameter kernel receive
13377 * contexts, krcvqs[]. It does not include the control
13378 * context, so add that.
13379 */
13380 num_kernel_contexts = n_krcvqs + 1;
13381 else
13382 num_kernel_contexts = DEFAULT_KRCVQS + 1;
13383 /*
13384 * Every kernel receive context needs an ACK send context.
13385 * one send context is allocated for each VL{0-7} and VL15
13386 */
13387 if (num_kernel_contexts > (send_contexts - num_vls - 1)) {
13388 dd_dev_err(dd,
13389 "Reducing # kernel rcv contexts to: %d, from %lu\n",
13390 send_contexts - num_vls - 1,
13391 num_kernel_contexts);
13392 num_kernel_contexts = send_contexts - num_vls - 1;
13393 }
13394
13395 /*
13396 * User contexts:
13397 * - default to 1 user context per real (non-HT) CPU core if
13398 * num_user_contexts is negative
13399 */
13400 if (num_user_contexts < 0)
13401 n_usr_ctxts = cpumask_weight(&node_affinity.real_cpu_mask);
13402 else
13403 n_usr_ctxts = num_user_contexts;
13404 /*
13405 * Adjust the counts given a global max.
13406 */
13407 if (num_kernel_contexts + n_usr_ctxts > rcv_contexts) {
13408 dd_dev_err(dd,
13409 "Reducing # user receive contexts to: %u, from %u\n",
13410 (u32)(rcv_contexts - num_kernel_contexts),
13411 n_usr_ctxts);
13412 /* recalculate */
13413 n_usr_ctxts = rcv_contexts - num_kernel_contexts;
13414 }
13415
13416 num_netdev_contexts =
13417 hfi1_num_netdev_contexts(dd, rcv_contexts -
13418 (num_kernel_contexts + n_usr_ctxts),
13419 &node_affinity.real_cpu_mask);
13420 /*
13421 * The RMT entries are currently allocated as shown below:
13422 * 1. QOS (0 to 128 entries);
13423 * 2. FECN (num_kernel_context - 1 + num_user_contexts +
13424 * num_netdev_contexts);
13425 * 3. netdev (num_netdev_contexts).
13426 * It should be noted that FECN oversubscribe num_netdev_contexts
13427 * entries of RMT because both netdev and PSM could allocate any receive
13428 * context between dd->first_dyn_alloc_text and dd->num_rcv_contexts,
13429 * and PSM FECN must reserve an RMT entry for each possible PSM receive
13430 * context.
13431 */
13432 rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_netdev_contexts * 2);
13433 if (HFI1_CAP_IS_KSET(TID_RDMA))
13434 rmt_count += num_kernel_contexts - 1;
13435 if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) {
13436 user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count;
13437 dd_dev_err(dd,
13438 "RMT size is reducing the number of user receive contexts from %u to %d\n",
13439 n_usr_ctxts,
13440 user_rmt_reduced);
13441 /* recalculate */
13442 n_usr_ctxts = user_rmt_reduced;
13443 }
13444
13445 /* the first N are kernel contexts, the rest are user/netdev contexts */
13446 dd->num_rcv_contexts =
13447 num_kernel_contexts + n_usr_ctxts + num_netdev_contexts;
13448 dd->n_krcv_queues = num_kernel_contexts;
13449 dd->first_dyn_alloc_ctxt = num_kernel_contexts;
13450 dd->num_netdev_contexts = num_netdev_contexts;
13451 dd->num_user_contexts = n_usr_ctxts;
13452 dd->freectxts = n_usr_ctxts;
13453 dd_dev_info(dd,
13454 "rcv contexts: chip %d, used %d (kernel %d, netdev %u, user %u)\n",
13455 rcv_contexts,
13456 (int)dd->num_rcv_contexts,
13457 (int)dd->n_krcv_queues,
13458 dd->num_netdev_contexts,
13459 dd->num_user_contexts);
13460
13461 /*
13462 * Receive array allocation:
13463 * All RcvArray entries are divided into groups of 8. This
13464 * is required by the hardware and will speed up writes to
13465 * consecutive entries by using write-combining of the entire
13466 * cacheline.
13467 *
13468 * The number of groups are evenly divided among all contexts.
13469 * any left over groups will be given to the first N user
13470 * contexts.
13471 */
13472 dd->rcv_entries.group_size = RCV_INCREMENT;
13473 ngroups = chip_rcv_array_count(dd) / dd->rcv_entries.group_size;
13474 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
13475 dd->rcv_entries.nctxt_extra = ngroups -
13476 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
13477 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
13478 dd->rcv_entries.ngroups,
13479 dd->rcv_entries.nctxt_extra);
13480 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
13481 MAX_EAGER_ENTRIES * 2) {
13482 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
13483 dd->rcv_entries.group_size;
13484 dd_dev_info(dd,
13485 "RcvArray group count too high, change to %u\n",
13486 dd->rcv_entries.ngroups);
13487 dd->rcv_entries.nctxt_extra = 0;
13488 }
13489 /*
13490 * PIO send contexts
13491 */
13492 ret = init_sc_pools_and_sizes(dd);
13493 if (ret >= 0) { /* success */
13494 dd->num_send_contexts = ret;
13495 dd_dev_info(
13496 dd,
13497 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n",
13498 send_contexts,
13499 dd->num_send_contexts,
13500 dd->sc_sizes[SC_KERNEL].count,
13501 dd->sc_sizes[SC_ACK].count,
13502 dd->sc_sizes[SC_USER].count,
13503 dd->sc_sizes[SC_VL15].count);
13504 ret = 0; /* success */
13505 }
13506
13507 return ret;
13508 }
13509
13510 /*
13511 * Set the device/port partition key table. The MAD code
13512 * will ensure that, at least, the partial management
13513 * partition key is present in the table.
13514 */
set_partition_keys(struct hfi1_pportdata * ppd)13515 static void set_partition_keys(struct hfi1_pportdata *ppd)
13516 {
13517 struct hfi1_devdata *dd = ppd->dd;
13518 u64 reg = 0;
13519 int i;
13520
13521 dd_dev_info(dd, "Setting partition keys\n");
13522 for (i = 0; i < hfi1_get_npkeys(dd); i++) {
13523 reg |= (ppd->pkeys[i] &
13524 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
13525 ((i % 4) *
13526 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
13527 /* Each register holds 4 PKey values. */
13528 if ((i % 4) == 3) {
13529 write_csr(dd, RCV_PARTITION_KEY +
13530 ((i - 3) * 2), reg);
13531 reg = 0;
13532 }
13533 }
13534
13535 /* Always enable HW pkeys check when pkeys table is set */
13536 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
13537 }
13538
13539 /*
13540 * These CSRs and memories are uninitialized on reset and must be
13541 * written before reading to set the ECC/parity bits.
13542 *
13543 * NOTE: All user context CSRs that are not mmaped write-only
13544 * (e.g. the TID flows) must be initialized even if the driver never
13545 * reads them.
13546 */
write_uninitialized_csrs_and_memories(struct hfi1_devdata * dd)13547 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
13548 {
13549 int i, j;
13550
13551 /* CceIntMap */
13552 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13553 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13554
13555 /* SendCtxtCreditReturnAddr */
13556 for (i = 0; i < chip_send_contexts(dd); i++)
13557 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13558
13559 /* PIO Send buffers */
13560 /* SDMA Send buffers */
13561 /*
13562 * These are not normally read, and (presently) have no method
13563 * to be read, so are not pre-initialized
13564 */
13565
13566 /* RcvHdrAddr */
13567 /* RcvHdrTailAddr */
13568 /* RcvTidFlowTable */
13569 for (i = 0; i < chip_rcv_contexts(dd); i++) {
13570 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13571 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13572 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
13573 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
13574 }
13575
13576 /* RcvArray */
13577 for (i = 0; i < chip_rcv_array_count(dd); i++)
13578 hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0);
13579
13580 /* RcvQPMapTable */
13581 for (i = 0; i < 32; i++)
13582 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13583 }
13584
13585 /*
13586 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
13587 */
clear_cce_status(struct hfi1_devdata * dd,u64 status_bits,u64 ctrl_bits)13588 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
13589 u64 ctrl_bits)
13590 {
13591 unsigned long timeout;
13592 u64 reg;
13593
13594 /* is the condition present? */
13595 reg = read_csr(dd, CCE_STATUS);
13596 if ((reg & status_bits) == 0)
13597 return;
13598
13599 /* clear the condition */
13600 write_csr(dd, CCE_CTRL, ctrl_bits);
13601
13602 /* wait for the condition to clear */
13603 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
13604 while (1) {
13605 reg = read_csr(dd, CCE_STATUS);
13606 if ((reg & status_bits) == 0)
13607 return;
13608 if (time_after(jiffies, timeout)) {
13609 dd_dev_err(dd,
13610 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
13611 status_bits, reg & status_bits);
13612 return;
13613 }
13614 udelay(1);
13615 }
13616 }
13617
13618 /* set CCE CSRs to chip reset defaults */
reset_cce_csrs(struct hfi1_devdata * dd)13619 static void reset_cce_csrs(struct hfi1_devdata *dd)
13620 {
13621 int i;
13622
13623 /* CCE_REVISION read-only */
13624 /* CCE_REVISION2 read-only */
13625 /* CCE_CTRL - bits clear automatically */
13626 /* CCE_STATUS read-only, use CceCtrl to clear */
13627 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
13628 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
13629 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
13630 for (i = 0; i < CCE_NUM_SCRATCH; i++)
13631 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
13632 /* CCE_ERR_STATUS read-only */
13633 write_csr(dd, CCE_ERR_MASK, 0);
13634 write_csr(dd, CCE_ERR_CLEAR, ~0ull);
13635 /* CCE_ERR_FORCE leave alone */
13636 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
13637 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
13638 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
13639 /* CCE_PCIE_CTRL leave alone */
13640 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
13641 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
13642 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
13643 CCE_MSIX_TABLE_UPPER_RESETCSR);
13644 }
13645 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
13646 /* CCE_MSIX_PBA read-only */
13647 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
13648 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
13649 }
13650 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13651 write_csr(dd, CCE_INT_MAP, 0);
13652 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
13653 /* CCE_INT_STATUS read-only */
13654 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
13655 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
13656 /* CCE_INT_FORCE leave alone */
13657 /* CCE_INT_BLOCKED read-only */
13658 }
13659 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
13660 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
13661 }
13662
13663 /* set MISC CSRs to chip reset defaults */
reset_misc_csrs(struct hfi1_devdata * dd)13664 static void reset_misc_csrs(struct hfi1_devdata *dd)
13665 {
13666 int i;
13667
13668 for (i = 0; i < 32; i++) {
13669 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13670 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13671 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13672 }
13673 /*
13674 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
13675 * only be written 128-byte chunks
13676 */
13677 /* init RSA engine to clear lingering errors */
13678 write_csr(dd, MISC_CFG_RSA_CMD, 1);
13679 write_csr(dd, MISC_CFG_RSA_MU, 0);
13680 write_csr(dd, MISC_CFG_FW_CTRL, 0);
13681 /* MISC_STS_8051_DIGEST read-only */
13682 /* MISC_STS_SBM_DIGEST read-only */
13683 /* MISC_STS_PCIE_DIGEST read-only */
13684 /* MISC_STS_FAB_DIGEST read-only */
13685 /* MISC_ERR_STATUS read-only */
13686 write_csr(dd, MISC_ERR_MASK, 0);
13687 write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13688 /* MISC_ERR_FORCE leave alone */
13689 }
13690
13691 /* set TXE CSRs to chip reset defaults */
reset_txe_csrs(struct hfi1_devdata * dd)13692 static void reset_txe_csrs(struct hfi1_devdata *dd)
13693 {
13694 int i;
13695
13696 /*
13697 * TXE Kernel CSRs
13698 */
13699 write_csr(dd, SEND_CTRL, 0);
13700 __cm_reset(dd, 0); /* reset CM internal state */
13701 /* SEND_CONTEXTS read-only */
13702 /* SEND_DMA_ENGINES read-only */
13703 /* SEND_PIO_MEM_SIZE read-only */
13704 /* SEND_DMA_MEM_SIZE read-only */
13705 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13706 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */
13707 /* SEND_PIO_ERR_STATUS read-only */
13708 write_csr(dd, SEND_PIO_ERR_MASK, 0);
13709 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13710 /* SEND_PIO_ERR_FORCE leave alone */
13711 /* SEND_DMA_ERR_STATUS read-only */
13712 write_csr(dd, SEND_DMA_ERR_MASK, 0);
13713 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13714 /* SEND_DMA_ERR_FORCE leave alone */
13715 /* SEND_EGRESS_ERR_STATUS read-only */
13716 write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13717 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13718 /* SEND_EGRESS_ERR_FORCE leave alone */
13719 write_csr(dd, SEND_BTH_QP, 0);
13720 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13721 write_csr(dd, SEND_SC2VLT0, 0);
13722 write_csr(dd, SEND_SC2VLT1, 0);
13723 write_csr(dd, SEND_SC2VLT2, 0);
13724 write_csr(dd, SEND_SC2VLT3, 0);
13725 write_csr(dd, SEND_LEN_CHECK0, 0);
13726 write_csr(dd, SEND_LEN_CHECK1, 0);
13727 /* SEND_ERR_STATUS read-only */
13728 write_csr(dd, SEND_ERR_MASK, 0);
13729 write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13730 /* SEND_ERR_FORCE read-only */
13731 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
13732 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
13733 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
13734 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13735 for (i = 0; i < chip_send_contexts(dd) / NUM_CONTEXTS_PER_SET; i++)
13736 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
13737 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
13738 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
13739 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
13740 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
13741 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
13742 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
13743 /* SEND_CM_CREDIT_USED_STATUS read-only */
13744 write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13745 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13746 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13747 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13748 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13749 for (i = 0; i < TXE_NUM_DATA_VL; i++)
13750 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
13751 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13752 /* SEND_CM_CREDIT_USED_VL read-only */
13753 /* SEND_CM_CREDIT_USED_VL15 read-only */
13754 /* SEND_EGRESS_CTXT_STATUS read-only */
13755 /* SEND_EGRESS_SEND_DMA_STATUS read-only */
13756 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13757 /* SEND_EGRESS_ERR_INFO read-only */
13758 /* SEND_EGRESS_ERR_SOURCE read-only */
13759
13760 /*
13761 * TXE Per-Context CSRs
13762 */
13763 for (i = 0; i < chip_send_contexts(dd); i++) {
13764 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13765 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13766 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13767 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13768 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13769 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13770 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13771 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13772 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13773 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13774 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13775 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13776 }
13777
13778 /*
13779 * TXE Per-SDMA CSRs
13780 */
13781 for (i = 0; i < chip_sdma_engines(dd); i++) {
13782 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13783 /* SEND_DMA_STATUS read-only */
13784 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13785 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13786 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13787 /* SEND_DMA_HEAD read-only */
13788 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13789 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13790 /* SEND_DMA_IDLE_CNT read-only */
13791 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13792 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13793 /* SEND_DMA_DESC_FETCHED_CNT read-only */
13794 /* SEND_DMA_ENG_ERR_STATUS read-only */
13795 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13796 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13797 /* SEND_DMA_ENG_ERR_FORCE leave alone */
13798 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13799 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13800 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13801 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13802 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13803 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13804 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13805 }
13806 }
13807
13808 /*
13809 * Expect on entry:
13810 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
13811 */
init_rbufs(struct hfi1_devdata * dd)13812 static void init_rbufs(struct hfi1_devdata *dd)
13813 {
13814 u64 reg;
13815 int count;
13816
13817 /*
13818 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
13819 * clear.
13820 */
13821 count = 0;
13822 while (1) {
13823 reg = read_csr(dd, RCV_STATUS);
13824 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13825 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13826 break;
13827 /*
13828 * Give up after 1ms - maximum wait time.
13829 *
13830 * RBuf size is 136KiB. Slowest possible is PCIe Gen1 x1 at
13831 * 250MB/s bandwidth. Lower rate to 66% for overhead to get:
13832 * 136 KB / (66% * 250MB/s) = 844us
13833 */
13834 if (count++ > 500) {
13835 dd_dev_err(dd,
13836 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13837 __func__, reg);
13838 break;
13839 }
13840 udelay(2); /* do not busy-wait the CSR */
13841 }
13842
13843 /* start the init - expect RcvCtrl to be 0 */
13844 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13845
13846 /*
13847 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief
13848 * period after the write before RcvStatus.RxRbufInitDone is valid.
13849 * The delay in the first run through the loop below is sufficient and
13850 * required before the first read of RcvStatus.RxRbufInintDone.
13851 */
13852 read_csr(dd, RCV_CTRL);
13853
13854 /* wait for the init to finish */
13855 count = 0;
13856 while (1) {
13857 /* delay is required first time through - see above */
13858 udelay(2); /* do not busy-wait the CSR */
13859 reg = read_csr(dd, RCV_STATUS);
13860 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13861 break;
13862
13863 /* give up after 100us - slowest possible at 33MHz is 73us */
13864 if (count++ > 50) {
13865 dd_dev_err(dd,
13866 "%s: RcvStatus.RxRbufInit not set, continuing\n",
13867 __func__);
13868 break;
13869 }
13870 }
13871 }
13872
13873 /* set RXE CSRs to chip reset defaults */
reset_rxe_csrs(struct hfi1_devdata * dd)13874 static void reset_rxe_csrs(struct hfi1_devdata *dd)
13875 {
13876 int i, j;
13877
13878 /*
13879 * RXE Kernel CSRs
13880 */
13881 write_csr(dd, RCV_CTRL, 0);
13882 init_rbufs(dd);
13883 /* RCV_STATUS read-only */
13884 /* RCV_CONTEXTS read-only */
13885 /* RCV_ARRAY_CNT read-only */
13886 /* RCV_BUF_SIZE read-only */
13887 write_csr(dd, RCV_BTH_QP, 0);
13888 write_csr(dd, RCV_MULTICAST, 0);
13889 write_csr(dd, RCV_BYPASS, 0);
13890 write_csr(dd, RCV_VL15, 0);
13891 /* this is a clear-down */
13892 write_csr(dd, RCV_ERR_INFO,
13893 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
13894 /* RCV_ERR_STATUS read-only */
13895 write_csr(dd, RCV_ERR_MASK, 0);
13896 write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13897 /* RCV_ERR_FORCE leave alone */
13898 for (i = 0; i < 32; i++)
13899 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13900 for (i = 0; i < 4; i++)
13901 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13902 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13903 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13904 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13905 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13906 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++)
13907 clear_rsm_rule(dd, i);
13908 for (i = 0; i < 32; i++)
13909 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13910
13911 /*
13912 * RXE Kernel and User Per-Context CSRs
13913 */
13914 for (i = 0; i < chip_rcv_contexts(dd); i++) {
13915 /* kernel */
13916 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13917 /* RCV_CTXT_STATUS read-only */
13918 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13919 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13920 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13921 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13922 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13923 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13924 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13925 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13926 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13927 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13928
13929 /* user */
13930 /* RCV_HDR_TAIL read-only */
13931 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13932 /* RCV_EGR_INDEX_TAIL read-only */
13933 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13934 /* RCV_EGR_OFFSET_TAIL read-only */
13935 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
13936 write_uctxt_csr(dd, i,
13937 RCV_TID_FLOW_TABLE + (8 * j), 0);
13938 }
13939 }
13940 }
13941
13942 /*
13943 * Set sc2vl tables.
13944 *
13945 * They power on to zeros, so to avoid send context errors
13946 * they need to be set:
13947 *
13948 * SC 0-7 -> VL 0-7 (respectively)
13949 * SC 15 -> VL 15
13950 * otherwise
13951 * -> VL 0
13952 */
init_sc2vl_tables(struct hfi1_devdata * dd)13953 static void init_sc2vl_tables(struct hfi1_devdata *dd)
13954 {
13955 int i;
13956 /* init per architecture spec, constrained by hardware capability */
13957
13958 /* HFI maps sent packets */
13959 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13960 0,
13961 0, 0, 1, 1,
13962 2, 2, 3, 3,
13963 4, 4, 5, 5,
13964 6, 6, 7, 7));
13965 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13966 1,
13967 8, 0, 9, 0,
13968 10, 0, 11, 0,
13969 12, 0, 13, 0,
13970 14, 0, 15, 15));
13971 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13972 2,
13973 16, 0, 17, 0,
13974 18, 0, 19, 0,
13975 20, 0, 21, 0,
13976 22, 0, 23, 0));
13977 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13978 3,
13979 24, 0, 25, 0,
13980 26, 0, 27, 0,
13981 28, 0, 29, 0,
13982 30, 0, 31, 0));
13983
13984 /* DC maps received packets */
13985 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13986 15_0,
13987 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
13988 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13989 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13990 31_16,
13991 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
13992 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
13993
13994 /* initialize the cached sc2vl values consistently with h/w */
13995 for (i = 0; i < 32; i++) {
13996 if (i < 8 || i == 15)
13997 *((u8 *)(dd->sc2vl) + i) = (u8)i;
13998 else
13999 *((u8 *)(dd->sc2vl) + i) = 0;
14000 }
14001 }
14002
14003 /*
14004 * Read chip sizes and then reset parts to sane, disabled, values. We cannot
14005 * depend on the chip going through a power-on reset - a driver may be loaded
14006 * and unloaded many times.
14007 *
14008 * Do not write any CSR values to the chip in this routine - there may be
14009 * a reset following the (possible) FLR in this routine.
14010 *
14011 */
init_chip(struct hfi1_devdata * dd)14012 static int init_chip(struct hfi1_devdata *dd)
14013 {
14014 int i;
14015 int ret = 0;
14016
14017 /*
14018 * Put the HFI CSRs in a known state.
14019 * Combine this with a DC reset.
14020 *
14021 * Stop the device from doing anything while we do a
14022 * reset. We know there are no other active users of
14023 * the device since we are now in charge. Turn off
14024 * off all outbound and inbound traffic and make sure
14025 * the device does not generate any interrupts.
14026 */
14027
14028 /* disable send contexts and SDMA engines */
14029 write_csr(dd, SEND_CTRL, 0);
14030 for (i = 0; i < chip_send_contexts(dd); i++)
14031 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
14032 for (i = 0; i < chip_sdma_engines(dd); i++)
14033 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
14034 /* disable port (turn off RXE inbound traffic) and contexts */
14035 write_csr(dd, RCV_CTRL, 0);
14036 for (i = 0; i < chip_rcv_contexts(dd); i++)
14037 write_csr(dd, RCV_CTXT_CTRL, 0);
14038 /* mask all interrupt sources */
14039 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
14040 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
14041
14042 /*
14043 * DC Reset: do a full DC reset before the register clear.
14044 * A recommended length of time to hold is one CSR read,
14045 * so reread the CceDcCtrl. Then, hold the DC in reset
14046 * across the clear.
14047 */
14048 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
14049 (void)read_csr(dd, CCE_DC_CTRL);
14050
14051 if (use_flr) {
14052 /*
14053 * A FLR will reset the SPC core and part of the PCIe.
14054 * The parts that need to be restored have already been
14055 * saved.
14056 */
14057 dd_dev_info(dd, "Resetting CSRs with FLR\n");
14058
14059 /* do the FLR, the DC reset will remain */
14060 pcie_flr(dd->pcidev);
14061
14062 /* restore command and BARs */
14063 ret = restore_pci_variables(dd);
14064 if (ret) {
14065 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
14066 __func__);
14067 return ret;
14068 }
14069
14070 if (is_ax(dd)) {
14071 dd_dev_info(dd, "Resetting CSRs with FLR\n");
14072 pcie_flr(dd->pcidev);
14073 ret = restore_pci_variables(dd);
14074 if (ret) {
14075 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
14076 __func__);
14077 return ret;
14078 }
14079 }
14080 } else {
14081 dd_dev_info(dd, "Resetting CSRs with writes\n");
14082 reset_cce_csrs(dd);
14083 reset_txe_csrs(dd);
14084 reset_rxe_csrs(dd);
14085 reset_misc_csrs(dd);
14086 }
14087 /* clear the DC reset */
14088 write_csr(dd, CCE_DC_CTRL, 0);
14089
14090 /* Set the LED off */
14091 setextled(dd, 0);
14092
14093 /*
14094 * Clear the QSFP reset.
14095 * An FLR enforces a 0 on all out pins. The driver does not touch
14096 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and
14097 * anything plugged constantly in reset, if it pays attention
14098 * to RESET_N.
14099 * Prime examples of this are optical cables. Set all pins high.
14100 * I2CCLK and I2CDAT will change per direction, and INT_N and
14101 * MODPRS_N are input only and their value is ignored.
14102 */
14103 write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
14104 write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
14105 init_chip_resources(dd);
14106 return ret;
14107 }
14108
init_early_variables(struct hfi1_devdata * dd)14109 static void init_early_variables(struct hfi1_devdata *dd)
14110 {
14111 int i;
14112
14113 /* assign link credit variables */
14114 dd->vau = CM_VAU;
14115 dd->link_credits = CM_GLOBAL_CREDITS;
14116 if (is_ax(dd))
14117 dd->link_credits--;
14118 dd->vcu = cu_to_vcu(hfi1_cu);
14119 /* enough room for 8 MAD packets plus header - 17K */
14120 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
14121 if (dd->vl15_init > dd->link_credits)
14122 dd->vl15_init = dd->link_credits;
14123
14124 write_uninitialized_csrs_and_memories(dd);
14125
14126 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
14127 for (i = 0; i < dd->num_pports; i++) {
14128 struct hfi1_pportdata *ppd = &dd->pport[i];
14129
14130 set_partition_keys(ppd);
14131 }
14132 init_sc2vl_tables(dd);
14133 }
14134
init_kdeth_qp(struct hfi1_devdata * dd)14135 static void init_kdeth_qp(struct hfi1_devdata *dd)
14136 {
14137 write_csr(dd, SEND_BTH_QP,
14138 (RVT_KDETH_QP_PREFIX & SEND_BTH_QP_KDETH_QP_MASK) <<
14139 SEND_BTH_QP_KDETH_QP_SHIFT);
14140
14141 write_csr(dd, RCV_BTH_QP,
14142 (RVT_KDETH_QP_PREFIX & RCV_BTH_QP_KDETH_QP_MASK) <<
14143 RCV_BTH_QP_KDETH_QP_SHIFT);
14144 }
14145
14146 /**
14147 * hfi1_get_qp_map - get qp map
14148 * @dd: device data
14149 * @idx: index to read
14150 */
hfi1_get_qp_map(struct hfi1_devdata * dd,u8 idx)14151 u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx)
14152 {
14153 u64 reg = read_csr(dd, RCV_QP_MAP_TABLE + (idx / 8) * 8);
14154
14155 reg >>= (idx % 8) * 8;
14156 return reg;
14157 }
14158
14159 /**
14160 * init_qpmap_table - init qp map
14161 * @dd: device data
14162 * @first_ctxt: first context
14163 * @last_ctxt: first context
14164 *
14165 * This return sets the qpn mapping table that
14166 * is indexed by qpn[8:1].
14167 *
14168 * The routine will round robin the 256 settings
14169 * from first_ctxt to last_ctxt.
14170 *
14171 * The first/last looks ahead to having specialized
14172 * receive contexts for mgmt and bypass. Normal
14173 * verbs traffic will assumed to be on a range
14174 * of receive contexts.
14175 */
init_qpmap_table(struct hfi1_devdata * dd,u32 first_ctxt,u32 last_ctxt)14176 static void init_qpmap_table(struct hfi1_devdata *dd,
14177 u32 first_ctxt,
14178 u32 last_ctxt)
14179 {
14180 u64 reg = 0;
14181 u64 regno = RCV_QP_MAP_TABLE;
14182 int i;
14183 u64 ctxt = first_ctxt;
14184
14185 for (i = 0; i < 256; i++) {
14186 reg |= ctxt << (8 * (i % 8));
14187 ctxt++;
14188 if (ctxt > last_ctxt)
14189 ctxt = first_ctxt;
14190 if (i % 8 == 7) {
14191 write_csr(dd, regno, reg);
14192 reg = 0;
14193 regno += 8;
14194 }
14195 }
14196
14197 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
14198 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
14199 }
14200
14201 struct rsm_map_table {
14202 u64 map[NUM_MAP_REGS];
14203 unsigned int used;
14204 };
14205
14206 struct rsm_rule_data {
14207 u8 offset;
14208 u8 pkt_type;
14209 u32 field1_off;
14210 u32 field2_off;
14211 u32 index1_off;
14212 u32 index1_width;
14213 u32 index2_off;
14214 u32 index2_width;
14215 u32 mask1;
14216 u32 value1;
14217 u32 mask2;
14218 u32 value2;
14219 };
14220
14221 /*
14222 * Return an initialized RMT map table for users to fill in. OK if it
14223 * returns NULL, indicating no table.
14224 */
alloc_rsm_map_table(struct hfi1_devdata * dd)14225 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd)
14226 {
14227 struct rsm_map_table *rmt;
14228 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */
14229
14230 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL);
14231 if (rmt) {
14232 memset(rmt->map, rxcontext, sizeof(rmt->map));
14233 rmt->used = 0;
14234 }
14235
14236 return rmt;
14237 }
14238
14239 /*
14240 * Write the final RMT map table to the chip and free the table. OK if
14241 * table is NULL.
14242 */
complete_rsm_map_table(struct hfi1_devdata * dd,struct rsm_map_table * rmt)14243 static void complete_rsm_map_table(struct hfi1_devdata *dd,
14244 struct rsm_map_table *rmt)
14245 {
14246 int i;
14247
14248 if (rmt) {
14249 /* write table to chip */
14250 for (i = 0; i < NUM_MAP_REGS; i++)
14251 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]);
14252
14253 /* enable RSM */
14254 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14255 }
14256 }
14257
14258 /* Is a receive side mapping rule */
has_rsm_rule(struct hfi1_devdata * dd,u8 rule_index)14259 static bool has_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)
14260 {
14261 return read_csr(dd, RCV_RSM_CFG + (8 * rule_index)) != 0;
14262 }
14263
14264 /*
14265 * Add a receive side mapping rule.
14266 */
add_rsm_rule(struct hfi1_devdata * dd,u8 rule_index,struct rsm_rule_data * rrd)14267 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index,
14268 struct rsm_rule_data *rrd)
14269 {
14270 write_csr(dd, RCV_RSM_CFG + (8 * rule_index),
14271 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT |
14272 1ull << rule_index | /* enable bit */
14273 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
14274 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index),
14275 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
14276 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
14277 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
14278 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
14279 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
14280 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
14281 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index),
14282 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT |
14283 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT |
14284 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT |
14285 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT);
14286 }
14287
14288 /*
14289 * Clear a receive side mapping rule.
14290 */
clear_rsm_rule(struct hfi1_devdata * dd,u8 rule_index)14291 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)
14292 {
14293 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 0);
14294 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 0);
14295 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 0);
14296 }
14297
14298 /* return the number of RSM map table entries that will be used for QOS */
qos_rmt_entries(struct hfi1_devdata * dd,unsigned int * mp,unsigned int * np)14299 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
14300 unsigned int *np)
14301 {
14302 int i;
14303 unsigned int m, n;
14304 u8 max_by_vl = 0;
14305
14306 /* is QOS active at all? */
14307 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
14308 num_vls == 1 ||
14309 krcvqsset <= 1)
14310 goto no_qos;
14311
14312 /* determine bits for qpn */
14313 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++)
14314 if (krcvqs[i] > max_by_vl)
14315 max_by_vl = krcvqs[i];
14316 if (max_by_vl > 32)
14317 goto no_qos;
14318 m = ilog2(__roundup_pow_of_two(max_by_vl));
14319
14320 /* determine bits for vl */
14321 n = ilog2(__roundup_pow_of_two(num_vls));
14322
14323 /* reject if too much is used */
14324 if ((m + n) > 7)
14325 goto no_qos;
14326
14327 if (mp)
14328 *mp = m;
14329 if (np)
14330 *np = n;
14331
14332 return 1 << (m + n);
14333
14334 no_qos:
14335 if (mp)
14336 *mp = 0;
14337 if (np)
14338 *np = 0;
14339 return 0;
14340 }
14341
14342 /**
14343 * init_qos - init RX qos
14344 * @dd: device data
14345 * @rmt: RSM map table
14346 *
14347 * This routine initializes Rule 0 and the RSM map table to implement
14348 * quality of service (qos).
14349 *
14350 * If all of the limit tests succeed, qos is applied based on the array
14351 * interpretation of krcvqs where entry 0 is VL0.
14352 *
14353 * The number of vl bits (n) and the number of qpn bits (m) are computed to
14354 * feed both the RSM map table and the single rule.
14355 */
init_qos(struct hfi1_devdata * dd,struct rsm_map_table * rmt)14356 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)
14357 {
14358 struct rsm_rule_data rrd;
14359 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
14360 unsigned int rmt_entries;
14361 u64 reg;
14362
14363 if (!rmt)
14364 goto bail;
14365 rmt_entries = qos_rmt_entries(dd, &m, &n);
14366 if (rmt_entries == 0)
14367 goto bail;
14368 qpns_per_vl = 1 << m;
14369
14370 /* enough room in the map table? */
14371 rmt_entries = 1 << (m + n);
14372 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES)
14373 goto bail;
14374
14375 /* add qos entries to the RSM map table */
14376 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) {
14377 unsigned tctxt;
14378
14379 for (qpn = 0, tctxt = ctxt;
14380 krcvqs[i] && qpn < qpns_per_vl; qpn++) {
14381 unsigned idx, regoff, regidx;
14382
14383 /* generate the index the hardware will produce */
14384 idx = rmt->used + ((qpn << n) ^ i);
14385 regoff = (idx % 8) * 8;
14386 regidx = idx / 8;
14387 /* replace default with context number */
14388 reg = rmt->map[regidx];
14389 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
14390 << regoff);
14391 reg |= (u64)(tctxt++) << regoff;
14392 rmt->map[regidx] = reg;
14393 if (tctxt == ctxt + krcvqs[i])
14394 tctxt = ctxt;
14395 }
14396 ctxt += krcvqs[i];
14397 }
14398
14399 rrd.offset = rmt->used;
14400 rrd.pkt_type = 2;
14401 rrd.field1_off = LRH_BTH_MATCH_OFFSET;
14402 rrd.field2_off = LRH_SC_MATCH_OFFSET;
14403 rrd.index1_off = LRH_SC_SELECT_OFFSET;
14404 rrd.index1_width = n;
14405 rrd.index2_off = QPN_SELECT_OFFSET;
14406 rrd.index2_width = m + n;
14407 rrd.mask1 = LRH_BTH_MASK;
14408 rrd.value1 = LRH_BTH_VALUE;
14409 rrd.mask2 = LRH_SC_MASK;
14410 rrd.value2 = LRH_SC_VALUE;
14411
14412 /* add rule 0 */
14413 add_rsm_rule(dd, RSM_INS_VERBS, &rrd);
14414
14415 /* mark RSM map entries as used */
14416 rmt->used += rmt_entries;
14417 /* map everything else to the mcast/err/vl15 context */
14418 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT);
14419 dd->qos_shift = n + 1;
14420 return;
14421 bail:
14422 dd->qos_shift = 1;
14423 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
14424 }
14425
init_fecn_handling(struct hfi1_devdata * dd,struct rsm_map_table * rmt)14426 static void init_fecn_handling(struct hfi1_devdata *dd,
14427 struct rsm_map_table *rmt)
14428 {
14429 struct rsm_rule_data rrd;
14430 u64 reg;
14431 int i, idx, regoff, regidx, start;
14432 u8 offset;
14433 u32 total_cnt;
14434
14435 if (HFI1_CAP_IS_KSET(TID_RDMA))
14436 /* Exclude context 0 */
14437 start = 1;
14438 else
14439 start = dd->first_dyn_alloc_ctxt;
14440
14441 total_cnt = dd->num_rcv_contexts - start;
14442
14443 /* there needs to be enough room in the map table */
14444 if (rmt->used + total_cnt >= NUM_MAP_ENTRIES) {
14445 dd_dev_err(dd, "FECN handling disabled - too many contexts allocated\n");
14446 return;
14447 }
14448
14449 /*
14450 * RSM will extract the destination context as an index into the
14451 * map table. The destination contexts are a sequential block
14452 * in the range start...num_rcv_contexts-1 (inclusive).
14453 * Map entries are accessed as offset + extracted value. Adjust
14454 * the added offset so this sequence can be placed anywhere in
14455 * the table - as long as the entries themselves do not wrap.
14456 * There are only enough bits in offset for the table size, so
14457 * start with that to allow for a "negative" offset.
14458 */
14459 offset = (u8)(NUM_MAP_ENTRIES + rmt->used - start);
14460
14461 for (i = start, idx = rmt->used; i < dd->num_rcv_contexts;
14462 i++, idx++) {
14463 /* replace with identity mapping */
14464 regoff = (idx % 8) * 8;
14465 regidx = idx / 8;
14466 reg = rmt->map[regidx];
14467 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff);
14468 reg |= (u64)i << regoff;
14469 rmt->map[regidx] = reg;
14470 }
14471
14472 /*
14473 * For RSM intercept of Expected FECN packets:
14474 * o packet type 0 - expected
14475 * o match on F (bit 95), using select/match 1, and
14476 * o match on SH (bit 133), using select/match 2.
14477 *
14478 * Use index 1 to extract the 8-bit receive context from DestQP
14479 * (start at bit 64). Use that as the RSM map table index.
14480 */
14481 rrd.offset = offset;
14482 rrd.pkt_type = 0;
14483 rrd.field1_off = 95;
14484 rrd.field2_off = 133;
14485 rrd.index1_off = 64;
14486 rrd.index1_width = 8;
14487 rrd.index2_off = 0;
14488 rrd.index2_width = 0;
14489 rrd.mask1 = 1;
14490 rrd.value1 = 1;
14491 rrd.mask2 = 1;
14492 rrd.value2 = 1;
14493
14494 /* add rule 1 */
14495 add_rsm_rule(dd, RSM_INS_FECN, &rrd);
14496
14497 rmt->used += total_cnt;
14498 }
14499
hfi1_is_rmt_full(int start,int spare)14500 static inline bool hfi1_is_rmt_full(int start, int spare)
14501 {
14502 return (start + spare) > NUM_MAP_ENTRIES;
14503 }
14504
hfi1_netdev_update_rmt(struct hfi1_devdata * dd)14505 static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd)
14506 {
14507 u8 i, j;
14508 u8 ctx_id = 0;
14509 u64 reg;
14510 u32 regoff;
14511 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
14512 int ctxt_count = hfi1_netdev_ctxt_count(dd);
14513
14514 /* We already have contexts mapped in RMT */
14515 if (has_rsm_rule(dd, RSM_INS_VNIC) || has_rsm_rule(dd, RSM_INS_AIP)) {
14516 dd_dev_info(dd, "Contexts are already mapped in RMT\n");
14517 return true;
14518 }
14519
14520 if (hfi1_is_rmt_full(rmt_start, NUM_NETDEV_MAP_ENTRIES)) {
14521 dd_dev_err(dd, "Not enough RMT entries used = %d\n",
14522 rmt_start);
14523 return false;
14524 }
14525
14526 dev_dbg(&(dd)->pcidev->dev, "RMT start = %d, end %d\n",
14527 rmt_start,
14528 rmt_start + NUM_NETDEV_MAP_ENTRIES);
14529
14530 /* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */
14531 regoff = RCV_RSM_MAP_TABLE + (rmt_start / 8) * 8;
14532 reg = read_csr(dd, regoff);
14533 for (i = 0; i < NUM_NETDEV_MAP_ENTRIES; i++) {
14534 /* Update map register with netdev context */
14535 j = (rmt_start + i) % 8;
14536 reg &= ~(0xffllu << (j * 8));
14537 reg |= (u64)hfi1_netdev_get_ctxt(dd, ctx_id++)->ctxt << (j * 8);
14538 /* Wrap up netdev ctx index */
14539 ctx_id %= ctxt_count;
14540 /* Write back map register */
14541 if (j == 7 || ((i + 1) == NUM_NETDEV_MAP_ENTRIES)) {
14542 dev_dbg(&(dd)->pcidev->dev,
14543 "RMT[%d] =0x%llx\n",
14544 regoff - RCV_RSM_MAP_TABLE, reg);
14545
14546 write_csr(dd, regoff, reg);
14547 regoff += 8;
14548 if (i < (NUM_NETDEV_MAP_ENTRIES - 1))
14549 reg = read_csr(dd, regoff);
14550 }
14551 }
14552
14553 return true;
14554 }
14555
hfi1_enable_rsm_rule(struct hfi1_devdata * dd,int rule,struct rsm_rule_data * rrd)14556 static void hfi1_enable_rsm_rule(struct hfi1_devdata *dd,
14557 int rule, struct rsm_rule_data *rrd)
14558 {
14559 if (!hfi1_netdev_update_rmt(dd)) {
14560 dd_dev_err(dd, "Failed to update RMT for RSM%d rule\n", rule);
14561 return;
14562 }
14563
14564 add_rsm_rule(dd, rule, rrd);
14565 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14566 }
14567
hfi1_init_aip_rsm(struct hfi1_devdata * dd)14568 void hfi1_init_aip_rsm(struct hfi1_devdata *dd)
14569 {
14570 /*
14571 * go through with the initialisation only if this rule actually doesn't
14572 * exist yet
14573 */
14574 if (atomic_fetch_inc(&dd->ipoib_rsm_usr_num) == 0) {
14575 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
14576 struct rsm_rule_data rrd = {
14577 .offset = rmt_start,
14578 .pkt_type = IB_PACKET_TYPE,
14579 .field1_off = LRH_BTH_MATCH_OFFSET,
14580 .mask1 = LRH_BTH_MASK,
14581 .value1 = LRH_BTH_VALUE,
14582 .field2_off = BTH_DESTQP_MATCH_OFFSET,
14583 .mask2 = BTH_DESTQP_MASK,
14584 .value2 = BTH_DESTQP_VALUE,
14585 .index1_off = DETH_AIP_SQPN_SELECT_OFFSET +
14586 ilog2(NUM_NETDEV_MAP_ENTRIES),
14587 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),
14588 .index2_off = DETH_AIP_SQPN_SELECT_OFFSET,
14589 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)
14590 };
14591
14592 hfi1_enable_rsm_rule(dd, RSM_INS_AIP, &rrd);
14593 }
14594 }
14595
14596 /* Initialize RSM for VNIC */
hfi1_init_vnic_rsm(struct hfi1_devdata * dd)14597 void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)
14598 {
14599 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
14600 struct rsm_rule_data rrd = {
14601 /* Add rule for vnic */
14602 .offset = rmt_start,
14603 .pkt_type = 4,
14604 /* Match 16B packets */
14605 .field1_off = L2_TYPE_MATCH_OFFSET,
14606 .mask1 = L2_TYPE_MASK,
14607 .value1 = L2_16B_VALUE,
14608 /* Match ETH L4 packets */
14609 .field2_off = L4_TYPE_MATCH_OFFSET,
14610 .mask2 = L4_16B_TYPE_MASK,
14611 .value2 = L4_16B_ETH_VALUE,
14612 /* Calc context from veswid and entropy */
14613 .index1_off = L4_16B_HDR_VESWID_OFFSET,
14614 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),
14615 .index2_off = L2_16B_ENTROPY_OFFSET,
14616 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)
14617 };
14618
14619 hfi1_enable_rsm_rule(dd, RSM_INS_VNIC, &rrd);
14620 }
14621
hfi1_deinit_vnic_rsm(struct hfi1_devdata * dd)14622 void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)
14623 {
14624 clear_rsm_rule(dd, RSM_INS_VNIC);
14625 }
14626
hfi1_deinit_aip_rsm(struct hfi1_devdata * dd)14627 void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd)
14628 {
14629 /* only actually clear the rule if it's the last user asking to do so */
14630 if (atomic_fetch_add_unless(&dd->ipoib_rsm_usr_num, -1, 0) == 1)
14631 clear_rsm_rule(dd, RSM_INS_AIP);
14632 }
14633
init_rxe(struct hfi1_devdata * dd)14634 static int init_rxe(struct hfi1_devdata *dd)
14635 {
14636 struct rsm_map_table *rmt;
14637 u64 val;
14638
14639 /* enable all receive errors */
14640 write_csr(dd, RCV_ERR_MASK, ~0ull);
14641
14642 rmt = alloc_rsm_map_table(dd);
14643 if (!rmt)
14644 return -ENOMEM;
14645
14646 /* set up QOS, including the QPN map table */
14647 init_qos(dd, rmt);
14648 init_fecn_handling(dd, rmt);
14649 complete_rsm_map_table(dd, rmt);
14650 /* record number of used rsm map entries for netdev */
14651 hfi1_netdev_set_free_rmt_idx(dd, rmt->used);
14652 kfree(rmt);
14653
14654 /*
14655 * make sure RcvCtrl.RcvWcb <= PCIe Device Control
14656 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
14657 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one
14658 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
14659 * Max_PayLoad_Size set to its minimum of 128.
14660 *
14661 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
14662 * (64 bytes). Max_Payload_Size is possibly modified upward in
14663 * tune_pcie_caps() which is called after this routine.
14664 */
14665
14666 /* Have 16 bytes (4DW) of bypass header available in header queue */
14667 val = read_csr(dd, RCV_BYPASS);
14668 val &= ~RCV_BYPASS_HDR_SIZE_SMASK;
14669 val |= ((4ull & RCV_BYPASS_HDR_SIZE_MASK) <<
14670 RCV_BYPASS_HDR_SIZE_SHIFT);
14671 write_csr(dd, RCV_BYPASS, val);
14672 return 0;
14673 }
14674
init_other(struct hfi1_devdata * dd)14675 static void init_other(struct hfi1_devdata *dd)
14676 {
14677 /* enable all CCE errors */
14678 write_csr(dd, CCE_ERR_MASK, ~0ull);
14679 /* enable *some* Misc errors */
14680 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
14681 /* enable all DC errors, except LCB */
14682 write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
14683 write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
14684 }
14685
14686 /*
14687 * Fill out the given AU table using the given CU. A CU is defined in terms
14688 * AUs. The table is a an encoding: given the index, how many AUs does that
14689 * represent?
14690 *
14691 * NOTE: Assumes that the register layout is the same for the
14692 * local and remote tables.
14693 */
assign_cm_au_table(struct hfi1_devdata * dd,u32 cu,u32 csr0to3,u32 csr4to7)14694 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
14695 u32 csr0to3, u32 csr4to7)
14696 {
14697 write_csr(dd, csr0to3,
14698 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
14699 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
14700 2ull * cu <<
14701 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
14702 4ull * cu <<
14703 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
14704 write_csr(dd, csr4to7,
14705 8ull * cu <<
14706 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
14707 16ull * cu <<
14708 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
14709 32ull * cu <<
14710 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
14711 64ull * cu <<
14712 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
14713 }
14714
assign_local_cm_au_table(struct hfi1_devdata * dd,u8 vcu)14715 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14716 {
14717 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
14718 SEND_CM_LOCAL_AU_TABLE4_TO7);
14719 }
14720
assign_remote_cm_au_table(struct hfi1_devdata * dd,u8 vcu)14721 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14722 {
14723 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
14724 SEND_CM_REMOTE_AU_TABLE4_TO7);
14725 }
14726
init_txe(struct hfi1_devdata * dd)14727 static void init_txe(struct hfi1_devdata *dd)
14728 {
14729 int i;
14730
14731 /* enable all PIO, SDMA, general, and Egress errors */
14732 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
14733 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
14734 write_csr(dd, SEND_ERR_MASK, ~0ull);
14735 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
14736
14737 /* enable all per-context and per-SDMA engine errors */
14738 for (i = 0; i < chip_send_contexts(dd); i++)
14739 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
14740 for (i = 0; i < chip_sdma_engines(dd); i++)
14741 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
14742
14743 /* set the local CU to AU mapping */
14744 assign_local_cm_au_table(dd, dd->vcu);
14745
14746 /*
14747 * Set reasonable default for Credit Return Timer
14748 * Don't set on Simulator - causes it to choke.
14749 */
14750 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
14751 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
14752 }
14753
hfi1_set_ctxt_jkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd,u16 jkey)14754 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14755 u16 jkey)
14756 {
14757 u8 hw_ctxt;
14758 u64 reg;
14759
14760 if (!rcd || !rcd->sc)
14761 return -EINVAL;
14762
14763 hw_ctxt = rcd->sc->hw_context;
14764 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
14765 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
14766 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
14767 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
14768 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
14769 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
14770 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
14771 /*
14772 * Enable send-side J_KEY integrity check, unless this is A0 h/w
14773 */
14774 if (!is_ax(dd)) {
14775 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14776 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14777 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14778 }
14779
14780 /* Enable J_KEY check on receive context. */
14781 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
14782 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
14783 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
14784 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg);
14785
14786 return 0;
14787 }
14788
hfi1_clear_ctxt_jkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd)14789 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
14790 {
14791 u8 hw_ctxt;
14792 u64 reg;
14793
14794 if (!rcd || !rcd->sc)
14795 return -EINVAL;
14796
14797 hw_ctxt = rcd->sc->hw_context;
14798 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
14799 /*
14800 * Disable send-side J_KEY integrity check, unless this is A0 h/w.
14801 * This check would not have been enabled for A0 h/w, see
14802 * set_ctxt_jkey().
14803 */
14804 if (!is_ax(dd)) {
14805 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14806 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14807 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14808 }
14809 /* Turn off the J_KEY on the receive side */
14810 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0);
14811
14812 return 0;
14813 }
14814
hfi1_set_ctxt_pkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd,u16 pkey)14815 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14816 u16 pkey)
14817 {
14818 u8 hw_ctxt;
14819 u64 reg;
14820
14821 if (!rcd || !rcd->sc)
14822 return -EINVAL;
14823
14824 hw_ctxt = rcd->sc->hw_context;
14825 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
14826 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
14827 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
14828 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14829 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14830 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
14831 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14832
14833 return 0;
14834 }
14835
hfi1_clear_ctxt_pkey(struct hfi1_devdata * dd,struct hfi1_ctxtdata * ctxt)14836 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt)
14837 {
14838 u8 hw_ctxt;
14839 u64 reg;
14840
14841 if (!ctxt || !ctxt->sc)
14842 return -EINVAL;
14843
14844 hw_ctxt = ctxt->sc->hw_context;
14845 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14846 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14847 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14848 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
14849
14850 return 0;
14851 }
14852
14853 /*
14854 * Start doing the clean up the chip. Our clean up happens in multiple
14855 * stages and this is just the first.
14856 */
hfi1_start_cleanup(struct hfi1_devdata * dd)14857 void hfi1_start_cleanup(struct hfi1_devdata *dd)
14858 {
14859 aspm_exit(dd);
14860 free_cntrs(dd);
14861 free_rcverr(dd);
14862 finish_chip_resources(dd);
14863 }
14864
14865 #define HFI_BASE_GUID(dev) \
14866 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
14867
14868 /*
14869 * Information can be shared between the two HFIs on the same ASIC
14870 * in the same OS. This function finds the peer device and sets
14871 * up a shared structure.
14872 */
init_asic_data(struct hfi1_devdata * dd)14873 static int init_asic_data(struct hfi1_devdata *dd)
14874 {
14875 unsigned long index;
14876 struct hfi1_devdata *peer;
14877 struct hfi1_asic_data *asic_data;
14878 int ret = 0;
14879
14880 /* pre-allocate the asic structure in case we are the first device */
14881 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL);
14882 if (!asic_data)
14883 return -ENOMEM;
14884
14885 xa_lock_irq(&hfi1_dev_table);
14886 /* Find our peer device */
14887 xa_for_each(&hfi1_dev_table, index, peer) {
14888 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(peer)) &&
14889 dd->unit != peer->unit)
14890 break;
14891 }
14892
14893 if (peer) {
14894 /* use already allocated structure */
14895 dd->asic_data = peer->asic_data;
14896 kfree(asic_data);
14897 } else {
14898 dd->asic_data = asic_data;
14899 mutex_init(&dd->asic_data->asic_resource_mutex);
14900 }
14901 dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
14902 xa_unlock_irq(&hfi1_dev_table);
14903
14904 /* first one through - set up i2c devices */
14905 if (!peer)
14906 ret = set_up_i2c(dd, dd->asic_data);
14907
14908 return ret;
14909 }
14910
14911 /*
14912 * Set dd->boardname. Use a generic name if a name is not returned from
14913 * EFI variable space.
14914 *
14915 * Return 0 on success, -ENOMEM if space could not be allocated.
14916 */
obtain_boardname(struct hfi1_devdata * dd)14917 static int obtain_boardname(struct hfi1_devdata *dd)
14918 {
14919 /* generic board description */
14920 const char generic[] =
14921 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
14922 unsigned long size;
14923 int ret;
14924
14925 ret = read_hfi1_efi_var(dd, "description", &size,
14926 (void **)&dd->boardname);
14927 if (ret) {
14928 dd_dev_info(dd, "Board description not found\n");
14929 /* use generic description */
14930 dd->boardname = kstrdup(generic, GFP_KERNEL);
14931 if (!dd->boardname)
14932 return -ENOMEM;
14933 }
14934 return 0;
14935 }
14936
14937 /*
14938 * Check the interrupt registers to make sure that they are mapped correctly.
14939 * It is intended to help user identify any mismapping by VMM when the driver
14940 * is running in a VM. This function should only be called before interrupt
14941 * is set up properly.
14942 *
14943 * Return 0 on success, -EINVAL on failure.
14944 */
check_int_registers(struct hfi1_devdata * dd)14945 static int check_int_registers(struct hfi1_devdata *dd)
14946 {
14947 u64 reg;
14948 u64 all_bits = ~(u64)0;
14949 u64 mask;
14950
14951 /* Clear CceIntMask[0] to avoid raising any interrupts */
14952 mask = read_csr(dd, CCE_INT_MASK);
14953 write_csr(dd, CCE_INT_MASK, 0ull);
14954 reg = read_csr(dd, CCE_INT_MASK);
14955 if (reg)
14956 goto err_exit;
14957
14958 /* Clear all interrupt status bits */
14959 write_csr(dd, CCE_INT_CLEAR, all_bits);
14960 reg = read_csr(dd, CCE_INT_STATUS);
14961 if (reg)
14962 goto err_exit;
14963
14964 /* Set all interrupt status bits */
14965 write_csr(dd, CCE_INT_FORCE, all_bits);
14966 reg = read_csr(dd, CCE_INT_STATUS);
14967 if (reg != all_bits)
14968 goto err_exit;
14969
14970 /* Restore the interrupt mask */
14971 write_csr(dd, CCE_INT_CLEAR, all_bits);
14972 write_csr(dd, CCE_INT_MASK, mask);
14973
14974 return 0;
14975 err_exit:
14976 write_csr(dd, CCE_INT_MASK, mask);
14977 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n");
14978 return -EINVAL;
14979 }
14980
14981 /**
14982 * hfi1_init_dd() - Initialize most of the dd structure.
14983 * @dd: the dd device
14984 *
14985 * This is global, and is called directly at init to set up the
14986 * chip-specific function pointers for later use.
14987 */
hfi1_init_dd(struct hfi1_devdata * dd)14988 int hfi1_init_dd(struct hfi1_devdata *dd)
14989 {
14990 struct pci_dev *pdev = dd->pcidev;
14991 struct hfi1_pportdata *ppd;
14992 u64 reg;
14993 int i, ret;
14994 static const char * const inames[] = { /* implementation names */
14995 "RTL silicon",
14996 "RTL VCS simulation",
14997 "RTL FPGA emulation",
14998 "Functional simulator"
14999 };
15000 struct pci_dev *parent = pdev->bus->self;
15001 u32 sdma_engines = chip_sdma_engines(dd);
15002
15003 ppd = dd->pport;
15004 for (i = 0; i < dd->num_pports; i++, ppd++) {
15005 int vl;
15006 /* init common fields */
15007 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
15008 /* DC supports 4 link widths */
15009 ppd->link_width_supported =
15010 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
15011 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
15012 ppd->link_width_downgrade_supported =
15013 ppd->link_width_supported;
15014 /* start out enabling only 4X */
15015 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
15016 ppd->link_width_downgrade_enabled =
15017 ppd->link_width_downgrade_supported;
15018 /* link width active is 0 when link is down */
15019 /* link width downgrade active is 0 when link is down */
15020
15021 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
15022 num_vls > HFI1_MAX_VLS_SUPPORTED) {
15023 dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n",
15024 num_vls, HFI1_MAX_VLS_SUPPORTED);
15025 num_vls = HFI1_MAX_VLS_SUPPORTED;
15026 }
15027 ppd->vls_supported = num_vls;
15028 ppd->vls_operational = ppd->vls_supported;
15029 /* Set the default MTU. */
15030 for (vl = 0; vl < num_vls; vl++)
15031 dd->vld[vl].mtu = hfi1_max_mtu;
15032 dd->vld[15].mtu = MAX_MAD_PACKET;
15033 /*
15034 * Set the initial values to reasonable default, will be set
15035 * for real when link is up.
15036 */
15037 ppd->overrun_threshold = 0x4;
15038 ppd->phy_error_threshold = 0xf;
15039 ppd->port_crc_mode_enabled = link_crc_mask;
15040 /* initialize supported LTP CRC mode */
15041 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
15042 /* initialize enabled LTP CRC mode */
15043 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
15044 /* start in offline */
15045 ppd->host_link_state = HLS_DN_OFFLINE;
15046 init_vl_arb_caches(ppd);
15047 }
15048
15049 /*
15050 * Do remaining PCIe setup and save PCIe values in dd.
15051 * Any error printing is already done by the init code.
15052 * On return, we have the chip mapped.
15053 */
15054 ret = hfi1_pcie_ddinit(dd, pdev);
15055 if (ret < 0)
15056 goto bail_free;
15057
15058 /* Save PCI space registers to rewrite after device reset */
15059 ret = save_pci_variables(dd);
15060 if (ret < 0)
15061 goto bail_cleanup;
15062
15063 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
15064 & CCE_REVISION_CHIP_REV_MAJOR_MASK;
15065 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
15066 & CCE_REVISION_CHIP_REV_MINOR_MASK;
15067
15068 /*
15069 * Check interrupt registers mapping if the driver has no access to
15070 * the upstream component. In this case, it is likely that the driver
15071 * is running in a VM.
15072 */
15073 if (!parent) {
15074 ret = check_int_registers(dd);
15075 if (ret)
15076 goto bail_cleanup;
15077 }
15078
15079 /*
15080 * obtain the hardware ID - NOT related to unit, which is a
15081 * software enumeration
15082 */
15083 reg = read_csr(dd, CCE_REVISION2);
15084 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
15085 & CCE_REVISION2_HFI_ID_MASK;
15086 /* the variable size will remove unwanted bits */
15087 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
15088 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
15089 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
15090 dd->icode < ARRAY_SIZE(inames) ?
15091 inames[dd->icode] : "unknown", (int)dd->irev);
15092
15093 /* speeds the hardware can support */
15094 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
15095 /* speeds allowed to run at */
15096 dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
15097 /* give a reasonable active value, will be set on link up */
15098 dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
15099
15100 /* fix up link widths for emulation _p */
15101 ppd = dd->pport;
15102 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
15103 ppd->link_width_supported =
15104 ppd->link_width_enabled =
15105 ppd->link_width_downgrade_supported =
15106 ppd->link_width_downgrade_enabled =
15107 OPA_LINK_WIDTH_1X;
15108 }
15109 /* insure num_vls isn't larger than number of sdma engines */
15110 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > sdma_engines) {
15111 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
15112 num_vls, sdma_engines);
15113 num_vls = sdma_engines;
15114 ppd->vls_supported = sdma_engines;
15115 ppd->vls_operational = ppd->vls_supported;
15116 }
15117
15118 /*
15119 * Convert the ns parameter to the 64 * cclocks used in the CSR.
15120 * Limit the max if larger than the field holds. If timeout is
15121 * non-zero, then the calculated field will be at least 1.
15122 *
15123 * Must be after icode is set up - the cclock rate depends
15124 * on knowing the hardware being used.
15125 */
15126 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
15127 if (dd->rcv_intr_timeout_csr >
15128 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
15129 dd->rcv_intr_timeout_csr =
15130 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
15131 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
15132 dd->rcv_intr_timeout_csr = 1;
15133
15134 /* needs to be done before we look for the peer device */
15135 read_guid(dd);
15136
15137 /* set up shared ASIC data with peer device */
15138 ret = init_asic_data(dd);
15139 if (ret)
15140 goto bail_cleanup;
15141
15142 /* obtain chip sizes, reset chip CSRs */
15143 ret = init_chip(dd);
15144 if (ret)
15145 goto bail_cleanup;
15146
15147 /* read in the PCIe link speed information */
15148 ret = pcie_speeds(dd);
15149 if (ret)
15150 goto bail_cleanup;
15151
15152 /* call before get_platform_config(), after init_chip_resources() */
15153 ret = eprom_init(dd);
15154 if (ret)
15155 goto bail_free_rcverr;
15156
15157 /* Needs to be called before hfi1_firmware_init */
15158 get_platform_config(dd);
15159
15160 /* read in firmware */
15161 ret = hfi1_firmware_init(dd);
15162 if (ret)
15163 goto bail_cleanup;
15164
15165 /*
15166 * In general, the PCIe Gen3 transition must occur after the
15167 * chip has been idled (so it won't initiate any PCIe transactions
15168 * e.g. an interrupt) and before the driver changes any registers
15169 * (the transition will reset the registers).
15170 *
15171 * In particular, place this call after:
15172 * - init_chip() - the chip will not initiate any PCIe transactions
15173 * - pcie_speeds() - reads the current link speed
15174 * - hfi1_firmware_init() - the needed firmware is ready to be
15175 * downloaded
15176 */
15177 ret = do_pcie_gen3_transition(dd);
15178 if (ret)
15179 goto bail_cleanup;
15180
15181 /*
15182 * This should probably occur in hfi1_pcie_init(), but historically
15183 * occurs after the do_pcie_gen3_transition() code.
15184 */
15185 tune_pcie_caps(dd);
15186
15187 /* start setting dd values and adjusting CSRs */
15188 init_early_variables(dd);
15189
15190 parse_platform_config(dd);
15191
15192 ret = obtain_boardname(dd);
15193 if (ret)
15194 goto bail_cleanup;
15195
15196 snprintf(dd->boardversion, BOARD_VERS_MAX,
15197 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
15198 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
15199 (u32)dd->majrev,
15200 (u32)dd->minrev,
15201 (dd->revision >> CCE_REVISION_SW_SHIFT)
15202 & CCE_REVISION_SW_MASK);
15203
15204 /* alloc VNIC/AIP rx data */
15205 ret = hfi1_alloc_rx(dd);
15206 if (ret)
15207 goto bail_cleanup;
15208
15209 ret = set_up_context_variables(dd);
15210 if (ret)
15211 goto bail_cleanup;
15212
15213 /* set initial RXE CSRs */
15214 ret = init_rxe(dd);
15215 if (ret)
15216 goto bail_cleanup;
15217
15218 /* set initial TXE CSRs */
15219 init_txe(dd);
15220 /* set initial non-RXE, non-TXE CSRs */
15221 init_other(dd);
15222 /* set up KDETH QP prefix in both RX and TX CSRs */
15223 init_kdeth_qp(dd);
15224
15225 ret = hfi1_dev_affinity_init(dd);
15226 if (ret)
15227 goto bail_cleanup;
15228
15229 /* send contexts must be set up before receive contexts */
15230 ret = init_send_contexts(dd);
15231 if (ret)
15232 goto bail_cleanup;
15233
15234 ret = hfi1_create_kctxts(dd);
15235 if (ret)
15236 goto bail_cleanup;
15237
15238 /*
15239 * Initialize aspm, to be done after gen3 transition and setting up
15240 * contexts and before enabling interrupts
15241 */
15242 aspm_init(dd);
15243
15244 ret = init_pervl_scs(dd);
15245 if (ret)
15246 goto bail_cleanup;
15247
15248 /* sdma init */
15249 for (i = 0; i < dd->num_pports; ++i) {
15250 ret = sdma_init(dd, i);
15251 if (ret)
15252 goto bail_cleanup;
15253 }
15254
15255 /* use contexts created by hfi1_create_kctxts */
15256 ret = set_up_interrupts(dd);
15257 if (ret)
15258 goto bail_cleanup;
15259
15260 ret = hfi1_comp_vectors_set_up(dd);
15261 if (ret)
15262 goto bail_clear_intr;
15263
15264 /* set up LCB access - must be after set_up_interrupts() */
15265 init_lcb_access(dd);
15266
15267 /*
15268 * Serial number is created from the base guid:
15269 * [27:24] = base guid [38:35]
15270 * [23: 0] = base guid [23: 0]
15271 */
15272 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
15273 (dd->base_guid & 0xFFFFFF) |
15274 ((dd->base_guid >> 11) & 0xF000000));
15275
15276 dd->oui1 = dd->base_guid >> 56 & 0xFF;
15277 dd->oui2 = dd->base_guid >> 48 & 0xFF;
15278 dd->oui3 = dd->base_guid >> 40 & 0xFF;
15279
15280 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
15281 if (ret)
15282 goto bail_clear_intr;
15283
15284 thermal_init(dd);
15285
15286 ret = init_cntrs(dd);
15287 if (ret)
15288 goto bail_clear_intr;
15289
15290 ret = init_rcverr(dd);
15291 if (ret)
15292 goto bail_free_cntrs;
15293
15294 init_completion(&dd->user_comp);
15295
15296 /* The user refcount starts with one to inidicate an active device */
15297 refcount_set(&dd->user_refcount, 1);
15298
15299 goto bail;
15300
15301 bail_free_rcverr:
15302 free_rcverr(dd);
15303 bail_free_cntrs:
15304 free_cntrs(dd);
15305 bail_clear_intr:
15306 hfi1_comp_vectors_clean_up(dd);
15307 msix_clean_up_interrupts(dd);
15308 bail_cleanup:
15309 hfi1_free_rx(dd);
15310 hfi1_pcie_ddcleanup(dd);
15311 bail_free:
15312 hfi1_free_devdata(dd);
15313 bail:
15314 return ret;
15315 }
15316
delay_cycles(struct hfi1_pportdata * ppd,u32 desired_egress_rate,u32 dw_len)15317 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
15318 u32 dw_len)
15319 {
15320 u32 delta_cycles;
15321 u32 current_egress_rate = ppd->current_egress_rate;
15322 /* rates here are in units of 10^6 bits/sec */
15323
15324 if (desired_egress_rate == -1)
15325 return 0; /* shouldn't happen */
15326
15327 if (desired_egress_rate >= current_egress_rate)
15328 return 0; /* we can't help go faster, only slower */
15329
15330 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
15331 egress_cycles(dw_len * 4, current_egress_rate);
15332
15333 return (u16)delta_cycles;
15334 }
15335
15336 /**
15337 * create_pbc - build a pbc for transmission
15338 * @ppd: info of physical Hfi port
15339 * @flags: special case flags or-ed in built pbc
15340 * @srate_mbs: static rate
15341 * @vl: vl
15342 * @dw_len: dword length (header words + data words + pbc words)
15343 *
15344 * Create a PBC with the given flags, rate, VL, and length.
15345 *
15346 * NOTE: The PBC created will not insert any HCRC - all callers but one are
15347 * for verbs, which does not use this PSM feature. The lone other caller
15348 * is for the diagnostic interface which calls this if the user does not
15349 * supply their own PBC.
15350 */
create_pbc(struct hfi1_pportdata * ppd,u64 flags,int srate_mbs,u32 vl,u32 dw_len)15351 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
15352 u32 dw_len)
15353 {
15354 u64 pbc, delay = 0;
15355
15356 if (unlikely(srate_mbs))
15357 delay = delay_cycles(ppd, srate_mbs, dw_len);
15358
15359 pbc = flags
15360 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
15361 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
15362 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
15363 | (dw_len & PBC_LENGTH_DWS_MASK)
15364 << PBC_LENGTH_DWS_SHIFT;
15365
15366 return pbc;
15367 }
15368
15369 #define SBUS_THERMAL 0x4f
15370 #define SBUS_THERM_MONITOR_MODE 0x1
15371
15372 #define THERM_FAILURE(dev, ret, reason) \
15373 dd_dev_err((dd), \
15374 "Thermal sensor initialization failed: %s (%d)\n", \
15375 (reason), (ret))
15376
15377 /*
15378 * Initialize the thermal sensor.
15379 *
15380 * After initialization, enable polling of thermal sensor through
15381 * SBus interface. In order for this to work, the SBus Master
15382 * firmware has to be loaded due to the fact that the HW polling
15383 * logic uses SBus interrupts, which are not supported with
15384 * default firmware. Otherwise, no data will be returned through
15385 * the ASIC_STS_THERM CSR.
15386 */
thermal_init(struct hfi1_devdata * dd)15387 static int thermal_init(struct hfi1_devdata *dd)
15388 {
15389 int ret = 0;
15390
15391 if (dd->icode != ICODE_RTL_SILICON ||
15392 check_chip_resource(dd, CR_THERM_INIT, NULL))
15393 return ret;
15394
15395 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
15396 if (ret) {
15397 THERM_FAILURE(dd, ret, "Acquire SBus");
15398 return ret;
15399 }
15400
15401 dd_dev_info(dd, "Initializing thermal sensor\n");
15402 /* Disable polling of thermal readings */
15403 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
15404 msleep(100);
15405 /* Thermal Sensor Initialization */
15406 /* Step 1: Reset the Thermal SBus Receiver */
15407 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15408 RESET_SBUS_RECEIVER, 0);
15409 if (ret) {
15410 THERM_FAILURE(dd, ret, "Bus Reset");
15411 goto done;
15412 }
15413 /* Step 2: Set Reset bit in Thermal block */
15414 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15415 WRITE_SBUS_RECEIVER, 0x1);
15416 if (ret) {
15417 THERM_FAILURE(dd, ret, "Therm Block Reset");
15418 goto done;
15419 }
15420 /* Step 3: Write clock divider value (100MHz -> 2MHz) */
15421 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
15422 WRITE_SBUS_RECEIVER, 0x32);
15423 if (ret) {
15424 THERM_FAILURE(dd, ret, "Write Clock Div");
15425 goto done;
15426 }
15427 /* Step 4: Select temperature mode */
15428 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
15429 WRITE_SBUS_RECEIVER,
15430 SBUS_THERM_MONITOR_MODE);
15431 if (ret) {
15432 THERM_FAILURE(dd, ret, "Write Mode Sel");
15433 goto done;
15434 }
15435 /* Step 5: De-assert block reset and start conversion */
15436 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15437 WRITE_SBUS_RECEIVER, 0x2);
15438 if (ret) {
15439 THERM_FAILURE(dd, ret, "Write Reset Deassert");
15440 goto done;
15441 }
15442 /* Step 5.1: Wait for first conversion (21.5ms per spec) */
15443 msleep(22);
15444
15445 /* Enable polling of thermal readings */
15446 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
15447
15448 /* Set initialized flag */
15449 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0);
15450 if (ret)
15451 THERM_FAILURE(dd, ret, "Unable to set thermal init flag");
15452
15453 done:
15454 release_chip_resource(dd, CR_SBUS);
15455 return ret;
15456 }
15457
handle_temp_err(struct hfi1_devdata * dd)15458 static void handle_temp_err(struct hfi1_devdata *dd)
15459 {
15460 struct hfi1_pportdata *ppd = &dd->pport[0];
15461 /*
15462 * Thermal Critical Interrupt
15463 * Put the device into forced freeze mode, take link down to
15464 * offline, and put DC into reset.
15465 */
15466 dd_dev_emerg(dd,
15467 "Critical temperature reached! Forcing device into freeze mode!\n");
15468 dd->flags |= HFI1_FORCED_FREEZE;
15469 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
15470 /*
15471 * Shut DC down as much and as quickly as possible.
15472 *
15473 * Step 1: Take the link down to OFFLINE. This will cause the
15474 * 8051 to put the Serdes in reset. However, we don't want to
15475 * go through the entire link state machine since we want to
15476 * shutdown ASAP. Furthermore, this is not a graceful shutdown
15477 * but rather an attempt to save the chip.
15478 * Code below is almost the same as quiet_serdes() but avoids
15479 * all the extra work and the sleeps.
15480 */
15481 ppd->driver_link_ready = 0;
15482 ppd->link_enabled = 0;
15483 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) |
15484 PLS_OFFLINE);
15485 /*
15486 * Step 2: Shutdown LCB and 8051
15487 * After shutdown, do not restore DC_CFG_RESET value.
15488 */
15489 dc_shutdown(dd);
15490 }
15491
