1 /*
2 * Huawei HiNIC PCI Express Linux driver
3 * Copyright(c) 2017 Huawei Technologies Co., Ltd
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * for more details.
13 *
14 */
15
16 #include <linux/kernel.h>
17 #include <linux/types.h>
18 #include <linux/errno.h>
19 #include <linux/pci.h>
20 #include <linux/device.h>
21 #include <linux/workqueue.h>
22 #include <linux/interrupt.h>
23 #include <linux/slab.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/log2.h>
26 #include <asm/byteorder.h>
27 #include <asm/barrier.h>
28
29 #include "hinic_hw_csr.h"
30 #include "hinic_hw_if.h"
31 #include "hinic_hw_eqs.h"
32
33 #define HINIC_EQS_WQ_NAME "hinic_eqs"
34
35 #define GET_EQ_NUM_PAGES(eq, pg_size) \
36 (ALIGN((eq)->q_len * (eq)->elem_size, pg_size) / (pg_size))
37
38 #define GET_EQ_NUM_ELEMS_IN_PG(eq, pg_size) ((pg_size) / (eq)->elem_size)
39
40 #define EQ_CONS_IDX_REG_ADDR(eq) (((eq)->type == HINIC_AEQ) ? \
41 HINIC_CSR_AEQ_CONS_IDX_ADDR((eq)->q_id) : \
42 HINIC_CSR_CEQ_CONS_IDX_ADDR((eq)->q_id))
43
44 #define EQ_PROD_IDX_REG_ADDR(eq) (((eq)->type == HINIC_AEQ) ? \
45 HINIC_CSR_AEQ_PROD_IDX_ADDR((eq)->q_id) : \
46 HINIC_CSR_CEQ_PROD_IDX_ADDR((eq)->q_id))
47
48 #define EQ_HI_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
49 HINIC_CSR_AEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
50 HINIC_CSR_CEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num))
51
52 #define EQ_LO_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
53 HINIC_CSR_AEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
54 HINIC_CSR_CEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num))
55
56 #define GET_EQ_ELEMENT(eq, idx) \
57 ((eq)->virt_addr[(idx) / (eq)->num_elem_in_pg] + \
58 (((idx) & ((eq)->num_elem_in_pg - 1)) * (eq)->elem_size))
59
60 #define GET_AEQ_ELEM(eq, idx) ((struct hinic_aeq_elem *) \
61 GET_EQ_ELEMENT(eq, idx))
62
63 #define GET_CEQ_ELEM(eq, idx) ((u32 *) \
64 GET_EQ_ELEMENT(eq, idx))
65
66 #define GET_CURR_AEQ_ELEM(eq) GET_AEQ_ELEM(eq, (eq)->cons_idx)
67
68 #define GET_CURR_CEQ_ELEM(eq) GET_CEQ_ELEM(eq, (eq)->cons_idx)
69
70 #define PAGE_IN_4K(page_size) ((page_size) >> 12)
71 #define EQ_SET_HW_PAGE_SIZE_VAL(eq) (ilog2(PAGE_IN_4K((eq)->page_size)))
72
73 #define ELEMENT_SIZE_IN_32B(eq) (((eq)->elem_size) >> 5)
74 #define EQ_SET_HW_ELEM_SIZE_VAL(eq) (ilog2(ELEMENT_SIZE_IN_32B(eq)))
75
76 #define EQ_MAX_PAGES 8
77
78 #define CEQE_TYPE_SHIFT 23
79 #define CEQE_TYPE_MASK 0x7
80
81 #define CEQE_TYPE(ceqe) (((ceqe) >> CEQE_TYPE_SHIFT) & \
82 CEQE_TYPE_MASK)
83
84 #define CEQE_DATA_MASK 0x3FFFFFF
85 #define CEQE_DATA(ceqe) ((ceqe) & CEQE_DATA_MASK)
86
87 #define aeq_to_aeqs(eq) \
88 container_of((eq) - (eq)->q_id, struct hinic_aeqs, aeq[0])
89
90 #define ceq_to_ceqs(eq) \
91 container_of((eq) - (eq)->q_id, struct hinic_ceqs, ceq[0])
92
93 #define work_to_aeq_work(work) \
94 container_of(work, struct hinic_eq_work, work)
95
96 #define DMA_ATTR_AEQ_DEFAULT 0
97 #define DMA_ATTR_CEQ_DEFAULT 0
98
99 /* No coalescence */
100 #define THRESH_CEQ_DEFAULT 0
101
102 enum eq_int_mode {
103 EQ_INT_MODE_ARMED,
104 EQ_INT_MODE_ALWAYS
105 };
106
107 enum eq_arm_state {
108 EQ_NOT_ARMED,
109 EQ_ARMED
110 };
111
112 /**
113 * hinic_aeq_register_hw_cb - register AEQ callback for specific event
114 * @aeqs: pointer to Async eqs of the chip
115 * @event: aeq event to register callback for it
116 * @handle: private data will be used by the callback
117 * @hw_handler: callback function
118 **/
hinic_aeq_register_hw_cb(struct hinic_aeqs * aeqs,enum hinic_aeq_type event,void * handle,void (* hwe_handler)(void * handle,void * data,u8 size))119 void hinic_aeq_register_hw_cb(struct hinic_aeqs *aeqs,
120 enum hinic_aeq_type event, void *handle,
121 void (*hwe_handler)(void *handle, void *data,
122 u8 size))
123 {
124 struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];
125
126 hwe_cb->hwe_handler = hwe_handler;
127 hwe_cb->handle = handle;
128 hwe_cb->hwe_state = HINIC_EQE_ENABLED;
129 }
130
131 /**
132 * hinic_aeq_unregister_hw_cb - unregister the AEQ callback for specific event
133 * @aeqs: pointer to Async eqs of the chip
134 * @event: aeq event to unregister callback for it
135 **/
hinic_aeq_unregister_hw_cb(struct hinic_aeqs * aeqs,enum hinic_aeq_type event)136 void hinic_aeq_unregister_hw_cb(struct hinic_aeqs *aeqs,
137 enum hinic_aeq_type event)
138 {
139 struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];
140
141 hwe_cb->hwe_state &= ~HINIC_EQE_ENABLED;
142
143 while (hwe_cb->hwe_state & HINIC_EQE_RUNNING)
144 schedule();
145
146 hwe_cb->hwe_handler = NULL;
147 }
148
149 /**
150 * hinic_ceq_register_cb - register CEQ callback for specific event
151 * @ceqs: pointer to Completion eqs part of the chip
152 * @event: ceq event to register callback for it
153 * @handle: private data will be used by the callback
154 * @handler: callback function
155 **/
hinic_ceq_register_cb(struct hinic_ceqs * ceqs,enum hinic_ceq_type event,void * handle,void (* handler)(void * handle,u32 ceqe_data))156 void hinic_ceq_register_cb(struct hinic_ceqs *ceqs,
157 enum hinic_ceq_type event, void *handle,
158 void (*handler)(void *handle, u32 ceqe_data))
159 {
160 struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];
161
162 ceq_cb->handler = handler;
163 ceq_cb->handle = handle;
164 ceq_cb->ceqe_state = HINIC_EQE_ENABLED;
165 }
166
167 /**
168 * hinic_ceq_unregister_cb - unregister the CEQ callback for specific event
169 * @ceqs: pointer to Completion eqs part of the chip
170 * @event: ceq event to unregister callback for it
171 **/
hinic_ceq_unregister_cb(struct hinic_ceqs * ceqs,enum hinic_ceq_type event)172 void hinic_ceq_unregister_cb(struct hinic_ceqs *ceqs,
173 enum hinic_ceq_type event)
174 {
175 struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];
176
177 ceq_cb->ceqe_state &= ~HINIC_EQE_ENABLED;
178
179 while (ceq_cb->ceqe_state & HINIC_EQE_RUNNING)
180 schedule();
181
182 ceq_cb->handler = NULL;
183 }
184
eq_cons_idx_checksum_set(u32 val)185 static u8 eq_cons_idx_checksum_set(u32 val)
186 {
187 u8 checksum = 0;
188 int idx;
189
190 for (idx = 0; idx < 32; idx += 4)
191 checksum ^= ((val >> idx) & 0xF);
192
193 return (checksum & 0xF);
194 }
195
196 /**
197 * eq_update_ci - update the HW cons idx of event queue
198 * @eq: the event queue to update the cons idx for
199 **/
eq_update_ci(struct hinic_eq * eq)200 static void eq_update_ci(struct hinic_eq *eq)
201 {
202 u32 val, addr = EQ_CONS_IDX_REG_ADDR(eq);
203
204 /* Read Modify Write */
205 val = hinic_hwif_read_reg(eq->hwif, addr);
206
207 val = HINIC_EQ_CI_CLEAR(val, IDX) &
208 HINIC_EQ_CI_CLEAR(val, WRAPPED) &
209 HINIC_EQ_CI_CLEAR(val, INT_ARMED) &
210 HINIC_EQ_CI_CLEAR(val, XOR_CHKSUM);
211
212 val |= HINIC_EQ_CI_SET(eq->cons_idx, IDX) |
213 HINIC_EQ_CI_SET(eq->wrapped, WRAPPED) |
214 HINIC_EQ_CI_SET(EQ_ARMED, INT_ARMED);
215
216 val |= HINIC_EQ_CI_SET(eq_cons_idx_checksum_set(val), XOR_CHKSUM);
217
218 hinic_hwif_write_reg(eq->hwif, addr, val);
219 }
220
221 /**
222 * aeq_irq_handler - handler for the AEQ event
223 * @eq: the Async Event Queue that received the event
224 **/
aeq_irq_handler(struct hinic_eq * eq)225 static void aeq_irq_handler(struct hinic_eq *eq)
226 {
227 struct hinic_aeqs *aeqs = aeq_to_aeqs(eq);
228 struct hinic_hwif *hwif = aeqs->hwif;
229 struct pci_dev *pdev = hwif->pdev;
230 struct hinic_aeq_elem *aeqe_curr;
231 struct hinic_hw_event_cb *hwe_cb;
232 enum hinic_aeq_type event;
233 unsigned long eqe_state;
234 u32 aeqe_desc;
235 int i, size;
236
237 for (i = 0; i < eq->q_len; i++) {
238 aeqe_curr = GET_CURR_AEQ_ELEM(eq);
239
240 /* Data in HW is in Big endian Format */
241 aeqe_desc = be32_to_cpu(aeqe_curr->desc);
242
243 /* HW toggles the wrapped bit, when it adds eq element */
244 if (HINIC_EQ_ELEM_DESC_GET(aeqe_desc, WRAPPED) == eq->wrapped)
245 break;
246
247 event = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, TYPE);
248 if (event >= HINIC_MAX_AEQ_EVENTS) {
249 dev_err(&pdev->dev, "Unknown AEQ Event %d\n", event);
250 return;
251 }
252
253 if (!HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SRC)) {
254 hwe_cb = &aeqs->hwe_cb[event];
255
256 size = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SIZE);
257
258 eqe_state = cmpxchg(&hwe_cb->hwe_state,
259 HINIC_EQE_ENABLED,
260 HINIC_EQE_ENABLED |
261 HINIC_EQE_RUNNING);
262 if ((eqe_state == HINIC_EQE_ENABLED) &&
263 (hwe_cb->hwe_handler))
264 hwe_cb->hwe_handler(hwe_cb->handle,
265 aeqe_curr->data, size);
266 else
267 dev_err(&pdev->dev, "Unhandled AEQ Event %d\n",
268 event);
269
270 hwe_cb->hwe_state &= ~HINIC_EQE_RUNNING;
271 }
272
273 eq->cons_idx++;
274
275 if (eq->cons_idx == eq->q_len) {
276 eq->cons_idx = 0;
277 eq->wrapped = !eq->wrapped;
278 }
279 }
280 }
281
282 /**
283 * ceq_event_handler - handler for the ceq events
284 * @ceqs: ceqs part of the chip
285 * @ceqe: ceq element that describes the event
286 **/
ceq_event_handler(struct hinic_ceqs * ceqs,u32 ceqe)287 static void ceq_event_handler(struct hinic_ceqs *ceqs, u32 ceqe)
288 {
289 struct hinic_hwif *hwif = ceqs->hwif;
290 struct pci_dev *pdev = hwif->pdev;
291 struct hinic_ceq_cb *ceq_cb;
292 enum hinic_ceq_type event;
293 unsigned long eqe_state;
294
295 event = CEQE_TYPE(ceqe);
296 if (event >= HINIC_MAX_CEQ_EVENTS) {
297 dev_err(&pdev->dev, "Unknown CEQ event, event = %d\n", event);
298 return;
299 }
300
301 ceq_cb = &ceqs->ceq_cb[event];
302
303 eqe_state = cmpxchg(&ceq_cb->ceqe_state,
304 HINIC_EQE_ENABLED,
305 HINIC_EQE_ENABLED | HINIC_EQE_RUNNING);
306
307 if ((eqe_state == HINIC_EQE_ENABLED) && (ceq_cb->handler))
308 ceq_cb->handler(ceq_cb->handle, CEQE_DATA(ceqe));
309 else
310 dev_err(&pdev->dev, "Unhandled CEQ Event %d\n", event);
311
312 ceq_cb->ceqe_state &= ~HINIC_EQE_RUNNING;
313 }
314
315 /**
316 * ceq_irq_handler - handler for the CEQ event
317 * @eq: the Completion Event Queue that received the event
318 **/
ceq_irq_handler(struct hinic_eq * eq)319 static void ceq_irq_handler(struct hinic_eq *eq)
320 {
321 struct hinic_ceqs *ceqs = ceq_to_ceqs(eq);
322 u32 ceqe;
323 int i;
324
325 for (i = 0; i < eq->q_len; i++) {
326 ceqe = *(GET_CURR_CEQ_ELEM(eq));
327
328 /* Data in HW is in Big endian Format */
329 ceqe = be32_to_cpu(ceqe);
330
331 /* HW toggles the wrapped bit, when it adds eq element event */
332 if (HINIC_EQ_ELEM_DESC_GET(ceqe, WRAPPED) == eq->wrapped)
333 break;
334
335 ceq_event_handler(ceqs, ceqe);
336
337 eq->cons_idx++;
338
339 if (eq->cons_idx == eq->q_len) {
340 eq->cons_idx = 0;
341 eq->wrapped = !eq->wrapped;
342 }
343 }
344 }
345
346 /**
347 * eq_irq_handler - handler for the EQ event
348 * @data: the Event Queue that received the event
349 **/
eq_irq_handler(void * data)350 static void eq_irq_handler(void *data)
351 {
352 struct hinic_eq *eq = data;
353
354 if (eq->type == HINIC_AEQ)
355 aeq_irq_handler(eq);
356 else if (eq->type == HINIC_CEQ)
357 ceq_irq_handler(eq);
358
359 eq_update_ci(eq);
360 }
361
362 /**
363 * eq_irq_work - the work of the EQ that received the event
364 * @work: the work struct that is associated with the EQ
365 **/
eq_irq_work(struct work_struct * work)366 static void eq_irq_work(struct work_struct *work)
367 {
368 struct hinic_eq_work *aeq_work = work_to_aeq_work(work);
369 struct hinic_eq *aeq;
370
371 aeq = aeq_work->data;
372 eq_irq_handler(aeq);
373 }
374
375 /**
376 * ceq_tasklet - the tasklet of the EQ that received the event
377 * @ceq_data: the eq
378 **/
ceq_tasklet(unsigned long ceq_data)379 static void ceq_tasklet(unsigned long ceq_data)
380 {
381 struct hinic_eq *ceq = (struct hinic_eq *)ceq_data;
382
383 eq_irq_handler(ceq);
384 }
385
386 /**
387 * aeq_interrupt - aeq interrupt handler
388 * @irq: irq number
389 * @data: the Async Event Queue that collected the event
390 **/
aeq_interrupt(int irq,void * data)391 static irqreturn_t aeq_interrupt(int irq, void *data)
392 {
393 struct hinic_eq_work *aeq_work;
394 struct hinic_eq *aeq = data;
395 struct hinic_aeqs *aeqs;
396
397 /* clear resend timer cnt register */
398 hinic_msix_attr_cnt_clear(aeq->hwif, aeq->msix_entry.entry);
399
400 aeq_work = &aeq->aeq_work;
401 aeq_work->data = aeq;
402
403 aeqs = aeq_to_aeqs(aeq);
404 queue_work(aeqs->workq, &aeq_work->work);
405
406 return IRQ_HANDLED;
407 }
408
409 /**
410 * ceq_interrupt - ceq interrupt handler
411 * @irq: irq number
412 * @data: the Completion Event Queue that collected the event
413 **/
ceq_interrupt(int irq,void * data)414 static irqreturn_t ceq_interrupt(int irq, void *data)
415 {
416 struct hinic_eq *ceq = data;
417
418 /* clear resend timer cnt register */
419 hinic_msix_attr_cnt_clear(ceq->hwif, ceq->msix_entry.entry);
420
421 tasklet_schedule(&ceq->ceq_tasklet);
422
423 return IRQ_HANDLED;
424 }
425
set_ctrl0(struct hinic_eq * eq)426 static void set_ctrl0(struct hinic_eq *eq)
427 {
428 struct msix_entry *msix_entry = &eq->msix_entry;
429 enum hinic_eq_type type = eq->type;
430 u32 addr, val, ctrl0;
431
432 if (type == HINIC_AEQ) {
433 /* RMW Ctrl0 */
434 addr = HINIC_CSR_AEQ_CTRL_0_ADDR(eq->q_id);
435
436 val = hinic_hwif_read_reg(eq->hwif, addr);
437
438 val = HINIC_AEQ_CTRL_0_CLEAR(val, INT_IDX) &
439 HINIC_AEQ_CTRL_0_CLEAR(val, DMA_ATTR) &
440 HINIC_AEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
441 HINIC_AEQ_CTRL_0_CLEAR(val, INT_MODE);
442
443 ctrl0 = HINIC_AEQ_CTRL_0_SET(msix_entry->entry, INT_IDX) |
444 HINIC_AEQ_CTRL_0_SET(DMA_ATTR_AEQ_DEFAULT, DMA_ATTR) |
445 HINIC_AEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
446 PCI_INTF_IDX) |
447 HINIC_AEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INT_MODE);
448
449 val |= ctrl0;
450
451 hinic_hwif_write_reg(eq->hwif, addr, val);
452 } else if (type == HINIC_CEQ) {
453 /* RMW Ctrl0 */
454 addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);
455
456 val = hinic_hwif_read_reg(eq->hwif, addr);
457
458 val = HINIC_CEQ_CTRL_0_CLEAR(val, INTR_IDX) &
459 HINIC_CEQ_CTRL_0_CLEAR(val, DMA_ATTR) &
460 HINIC_CEQ_CTRL_0_CLEAR(val, KICK_THRESH) &
461 HINIC_CEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
462 HINIC_CEQ_CTRL_0_CLEAR(val, INTR_MODE);
463
464 ctrl0 = HINIC_CEQ_CTRL_0_SET(msix_entry->entry, INTR_IDX) |
465 HINIC_CEQ_CTRL_0_SET(DMA_ATTR_CEQ_DEFAULT, DMA_ATTR) |
466 HINIC_CEQ_CTRL_0_SET(THRESH_CEQ_DEFAULT, KICK_THRESH) |
467 HINIC_CEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
468 PCI_INTF_IDX) |
469 HINIC_CEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INTR_MODE);
470
471 val |= ctrl0;
472
473 hinic_hwif_write_reg(eq->hwif, addr, val);
474 }
475 }
476
set_ctrl1(struct hinic_eq * eq)477 static void set_ctrl1(struct hinic_eq *eq)
478 {
479 enum hinic_eq_type type = eq->type;
480 u32 page_size_val, elem_size;
481 u32 addr, val, ctrl1;
482
483 if (type == HINIC_AEQ) {
484 /* RMW Ctrl1 */
485 addr = HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id);
486
487 page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);
488 elem_size = EQ_SET_HW_ELEM_SIZE_VAL(eq);
489
490 val = hinic_hwif_read_reg(eq->hwif, addr);
491
492 val = HINIC_AEQ_CTRL_1_CLEAR(val, LEN) &
493 HINIC_AEQ_CTRL_1_CLEAR(val, ELEM_SIZE) &
494 HINIC_AEQ_CTRL_1_CLEAR(val, PAGE_SIZE);
495
496 ctrl1 = HINIC_AEQ_CTRL_1_SET(eq->q_len, LEN) |
497 HINIC_AEQ_CTRL_1_SET(elem_size, ELEM_SIZE) |
498 HINIC_AEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);
499
500 val |= ctrl1;
501
502 hinic_hwif_write_reg(eq->hwif, addr, val);
503 } else if (type == HINIC_CEQ) {
504 /* RMW Ctrl1 */
505 addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);
506
507 page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);
508
509 val = hinic_hwif_read_reg(eq->hwif, addr);
510
511 val = HINIC_CEQ_CTRL_1_CLEAR(val, LEN) &
512 HINIC_CEQ_CTRL_1_CLEAR(val, PAGE_SIZE);
513
514 ctrl1 = HINIC_CEQ_CTRL_1_SET(eq->q_len, LEN) |
515 HINIC_CEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);
516
517 val |= ctrl1;
518
519 hinic_hwif_write_reg(eq->hwif, addr, val);
520 }
521 }
522
523 /**
524 * set_eq_ctrls - setting eq's ctrl registers
525 * @eq: the Event Queue for setting
526 **/
set_eq_ctrls(struct hinic_eq * eq)527 static void set_eq_ctrls(struct hinic_eq *eq)
528 {
529 set_ctrl0(eq);
530 set_ctrl1(eq);
531 }
532
533 /**
534 * aeq_elements_init - initialize all the elements in the aeq
535 * @eq: the Async Event Queue
536 * @init_val: value to initialize the elements with it
537 **/
aeq_elements_init(struct hinic_eq * eq,u32 init_val)538 static void aeq_elements_init(struct hinic_eq *eq, u32 init_val)
539 {
540 struct hinic_aeq_elem *aeqe;
541 int i;
542
543 for (i = 0; i < eq->q_len; i++) {
544 aeqe = GET_AEQ_ELEM(eq, i);
545 aeqe->desc = cpu_to_be32(init_val);
546 }
547
548 wmb(); /* Write the initilzation values */
549 }
550
551 /**
552 * ceq_elements_init - Initialize all the elements in the ceq
553 * @eq: the event queue
554 * @init_val: value to init with it the elements
555 **/
ceq_elements_init(struct hinic_eq * eq,u32 init_val)556 static void ceq_elements_init(struct hinic_eq *eq, u32 init_val)
557 {
558 u32 *ceqe;
559 int i;
560
561 for (i = 0; i < eq->q_len; i++) {
562 ceqe = GET_CEQ_ELEM(eq, i);
563 *(ceqe) = cpu_to_be32(init_val);
564 }
565
566 wmb(); /* Write the initilzation values */
567 }
568
569 /**
570 * alloc_eq_pages - allocate the pages for the queue
571 * @eq: the event queue
572 *
573 * Return 0 - Success, Negative - Failure
574 **/
alloc_eq_pages(struct hinic_eq * eq)575 static int alloc_eq_pages(struct hinic_eq *eq)
576 {
577 struct hinic_hwif *hwif = eq->hwif;
578 struct pci_dev *pdev = hwif->pdev;
579 u32 init_val, addr, val;
580 size_t addr_size;
581 int err, pg;
582
583 addr_size = eq->num_pages * sizeof(*eq->dma_addr);
584 eq->dma_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL);
585 if (!eq->dma_addr)
586 return -ENOMEM;
587
588 addr_size = eq->num_pages * sizeof(*eq->virt_addr);
589 eq->virt_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL);
590 if (!eq->virt_addr) {
591 err = -ENOMEM;
592 goto err_virt_addr_alloc;
593 }
594
595 for (pg = 0; pg < eq->num_pages; pg++) {
596 eq->virt_addr[pg] = dma_zalloc_coherent(&pdev->dev,
597 eq->page_size,
598 &eq->dma_addr[pg],
599 GFP_KERNEL);
600 if (!eq->virt_addr[pg]) {
601 err = -ENOMEM;
602 goto err_dma_alloc;
603 }
604
605 addr = EQ_HI_PHYS_ADDR_REG(eq, pg);
606 val = upper_32_bits(eq->dma_addr[pg]);
607
608 hinic_hwif_write_reg(hwif, addr, val);
609
610 addr = EQ_LO_PHYS_ADDR_REG(eq, pg);
611 val = lower_32_bits(eq->dma_addr[pg]);
612
613 hinic_hwif_write_reg(hwif, addr, val);
614 }
615
616 init_val = HINIC_EQ_ELEM_DESC_SET(eq->wrapped, WRAPPED);
617
618 if (eq->type == HINIC_AEQ)
619 aeq_elements_init(eq, init_val);
620 else if (eq->type == HINIC_CEQ)
621 ceq_elements_init(eq, init_val);
622
623 return 0;
624
625 err_dma_alloc:
626 while (--pg >= 0)
627 dma_free_coherent(&pdev->dev, eq->page_size,
628 eq->virt_addr[pg],
629 eq->dma_addr[pg]);
630
631 devm_kfree(&pdev->dev, eq->virt_addr);
632
633 err_virt_addr_alloc:
634 devm_kfree(&pdev->dev, eq->dma_addr);
635 return err;
636 }
637
638 /**
639 * free_eq_pages - free the pages of the queue
640 * @eq: the Event Queue
641 **/
free_eq_pages(struct hinic_eq * eq)642 static void free_eq_pages(struct hinic_eq *eq)
643 {
644 struct hinic_hwif *hwif = eq->hwif;
645 struct pci_dev *pdev = hwif->pdev;
646 int pg;
647
648 for (pg = 0; pg < eq->num_pages; pg++)
649 dma_free_coherent(&pdev->dev, eq->page_size,
650 eq->virt_addr[pg],
651 eq->dma_addr[pg]);
652
653 devm_kfree(&pdev->dev, eq->virt_addr);
654 devm_kfree(&pdev->dev, eq->dma_addr);
655 }
656
657 /**
658 * init_eq - initialize Event Queue
659 * @eq: the event queue
660 * @hwif: the HW interface of a PCI function device
661 * @type: the type of the event queue, aeq or ceq
662 * @q_id: Queue id number
663 * @q_len: the number of EQ elements
664 * @page_size: the page size of the pages in the event queue
665 * @entry: msix entry associated with the event queue
666 *
667 * Return 0 - Success, Negative - Failure
668 **/
init_eq(struct hinic_eq * eq,struct hinic_hwif * hwif,enum hinic_eq_type type,int q_id,u32 q_len,u32 page_size,struct msix_entry entry)669 static int init_eq(struct hinic_eq *eq, struct hinic_hwif *hwif,
670 enum hinic_eq_type type, int q_id, u32 q_len, u32 page_size,
671 struct msix_entry entry)
672 {
673 struct pci_dev *pdev = hwif->pdev;
674 int err;
675
676 eq->hwif = hwif;
677 eq->type = type;
678 eq->q_id = q_id;
679 eq->q_len = q_len;
680 eq->page_size = page_size;
681
682 /* Clear PI and CI, also clear the ARM bit */
683 hinic_hwif_write_reg(eq->hwif, EQ_CONS_IDX_REG_ADDR(eq), 0);
684 hinic_hwif_write_reg(eq->hwif, EQ_PROD_IDX_REG_ADDR(eq), 0);
685
686 eq->cons_idx = 0;
687 eq->wrapped = 0;
688
689 if (type == HINIC_AEQ) {
690 eq->elem_size = HINIC_AEQE_SIZE;
691 } else if (type == HINIC_CEQ) {
692 eq->elem_size = HINIC_CEQE_SIZE;
693 } else {
694 dev_err(&pdev->dev, "Invalid EQ type\n");
695 return -EINVAL;
696 }
697
698 eq->num_pages = GET_EQ_NUM_PAGES(eq, page_size);
699 eq->num_elem_in_pg = GET_EQ_NUM_ELEMS_IN_PG(eq, page_size);
700
701 eq->msix_entry = entry;
702
703 if (eq->num_elem_in_pg & (eq->num_elem_in_pg - 1)) {
704 dev_err(&pdev->dev, "num elements in eq page != power of 2\n");
705 return -EINVAL;
706 }
707
708 if (eq->num_pages > EQ_MAX_PAGES) {
709 dev_err(&pdev->dev, "too many pages for eq\n");
710 return -EINVAL;
711 }
712
713 set_eq_ctrls(eq);
714 eq_update_ci(eq);
715
716 err = alloc_eq_pages(eq);
717 if (err) {
718 dev_err(&pdev->dev, "Failed to allocate pages for eq\n");
719 return err;
720 }
721
722 if (type == HINIC_AEQ) {
723 struct hinic_eq_work *aeq_work = &eq->aeq_work;
724
725 INIT_WORK(&aeq_work->work, eq_irq_work);
726 } else if (type == HINIC_CEQ) {
727 tasklet_init(&eq->ceq_tasklet, ceq_tasklet,
728 (unsigned long)eq);
729 }
730
731 /* set the attributes of the msix entry */
732 hinic_msix_attr_set(eq->hwif, eq->msix_entry.entry,
733 HINIC_EQ_MSIX_PENDING_LIMIT_DEFAULT,
734 HINIC_EQ_MSIX_COALESC_TIMER_DEFAULT,
735 HINIC_EQ_MSIX_LLI_TIMER_DEFAULT,
736 HINIC_EQ_MSIX_LLI_CREDIT_LIMIT_DEFAULT,
737 HINIC_EQ_MSIX_RESEND_TIMER_DEFAULT);
738
739 if (type == HINIC_AEQ)
740 err = request_irq(entry.vector, aeq_interrupt, 0,
741 "hinic_aeq", eq);
742 else if (type == HINIC_CEQ)
743 err = request_irq(entry.vector, ceq_interrupt, 0,
744 "hinic_ceq", eq);
745
746 if (err) {
747 dev_err(&pdev->dev, "Failed to request irq for the EQ\n");
748 goto err_req_irq;
749 }
750
751 return 0;
752
753 err_req_irq:
754 free_eq_pages(eq);
755 return err;
756 }
757
758 /**
759 * remove_eq - remove Event Queue
760 * @eq: the event queue
761 **/
remove_eq(struct hinic_eq * eq)762 static void remove_eq(struct hinic_eq *eq)
763 {
764 struct msix_entry *entry = &eq->msix_entry;
765
766 free_irq(entry->vector, eq);
767
768 if (eq->type == HINIC_AEQ) {
769 struct hinic_eq_work *aeq_work = &eq->aeq_work;
770
771 cancel_work_sync(&aeq_work->work);
772 } else if (eq->type == HINIC_CEQ) {
773 tasklet_kill(&eq->ceq_tasklet);
774 }
775
776 free_eq_pages(eq);
777 }
778
779 /**
780 * hinic_aeqs_init - initialize all the aeqs
781 * @aeqs: pointer to Async eqs of the chip
782 * @hwif: the HW interface of a PCI function device
783 * @num_aeqs: number of AEQs
784 * @q_len: number of EQ elements
785 * @page_size: the page size of the pages in the event queue
786 * @msix_entries: msix entries associated with the event queues
787 *
788 * Return 0 - Success, negative - Failure
789 **/
hinic_aeqs_init(struct hinic_aeqs * aeqs,struct hinic_hwif * hwif,int num_aeqs,u32 q_len,u32 page_size,struct msix_entry * msix_entries)790 int hinic_aeqs_init(struct hinic_aeqs *aeqs, struct hinic_hwif *hwif,
791 int num_aeqs, u32 q_len, u32 page_size,
792 struct msix_entry *msix_entries)
793 {
794 struct pci_dev *pdev = hwif->pdev;
795 int err, i, q_id;
796
797 aeqs->workq = create_singlethread_workqueue(HINIC_EQS_WQ_NAME);
798 if (!aeqs->workq)
799 return -ENOMEM;
800
801 aeqs->hwif = hwif;
802 aeqs->num_aeqs = num_aeqs;
803
804 for (q_id = 0; q_id < num_aeqs; q_id++) {
805 err = init_eq(&aeqs->aeq[q_id], hwif, HINIC_AEQ, q_id, q_len,
806 page_size, msix_entries[q_id]);
807 if (err) {
808 dev_err(&pdev->dev, "Failed to init aeq %d\n", q_id);
809 goto err_init_aeq;
810 }
811 }
812
813 return 0;
814
815 err_init_aeq:
816 for (i = 0; i < q_id; i++)
817 remove_eq(&aeqs->aeq[i]);
818
819 destroy_workqueue(aeqs->workq);
820 return err;
821 }
822
823 /**
824 * hinic_aeqs_free - free all the aeqs
825 * @aeqs: pointer to Async eqs of the chip
826 **/
hinic_aeqs_free(struct hinic_aeqs * aeqs)827 void hinic_aeqs_free(struct hinic_aeqs *aeqs)
828 {
829 int q_id;
830
831 for (q_id = 0; q_id < aeqs->num_aeqs ; q_id++)
832 remove_eq(&aeqs->aeq[q_id]);
833
834 destroy_workqueue(aeqs->workq);
835 }
836
837 /**
838 * hinic_ceqs_init - init all the ceqs
839 * @ceqs: ceqs part of the chip
840 * @hwif: the hardware interface of a pci function device
841 * @num_ceqs: number of CEQs
842 * @q_len: number of EQ elements
843 * @page_size: the page size of the event queue
844 * @msix_entries: msix entries associated with the event queues
845 *
846 * Return 0 - Success, Negative - Failure
847 **/
hinic_ceqs_init(struct hinic_ceqs * ceqs,struct hinic_hwif * hwif,int num_ceqs,u32 q_len,u32 page_size,struct msix_entry * msix_entries)848 int hinic_ceqs_init(struct hinic_ceqs *ceqs, struct hinic_hwif *hwif,
849 int num_ceqs, u32 q_len, u32 page_size,
850 struct msix_entry *msix_entries)
851 {
852 struct pci_dev *pdev = hwif->pdev;
853 int i, q_id, err;
854
855 ceqs->hwif = hwif;
856 ceqs->num_ceqs = num_ceqs;
857
858 for (q_id = 0; q_id < num_ceqs; q_id++) {
859 err = init_eq(&ceqs->ceq[q_id], hwif, HINIC_CEQ, q_id, q_len,
860 page_size, msix_entries[q_id]);
861 if (err) {
862 dev_err(&pdev->dev, "Failed to init ceq %d\n", q_id);
863 goto err_init_ceq;
864 }
865 }
866
867 return 0;
868
869 err_init_ceq:
870 for (i = 0; i < q_id; i++)
871 remove_eq(&ceqs->ceq[i]);
872
873 return err;
874 }
875
876 /**
877 * hinic_ceqs_free - free all the ceqs
878 * @ceqs: ceqs part of the chip
879 **/
hinic_ceqs_free(struct hinic_ceqs * ceqs)880 void hinic_ceqs_free(struct hinic_ceqs *ceqs)
881 {
882 int q_id;
883
884 for (q_id = 0; q_id < ceqs->num_ceqs; q_id++)
885 remove_eq(&ceqs->ceq[q_id]);
886 }
887
