1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
17 * details.
18 **********************************************************************/
19 #include <linux/pci.h>
20 #include <linux/netdevice.h>
21 #include <linux/vmalloc.h>
22 #include "liquidio_common.h"
23 #include "octeon_droq.h"
24 #include "octeon_iq.h"
25 #include "response_manager.h"
26 #include "octeon_device.h"
27 #include "octeon_main.h"
28 #include "octeon_network.h"
29 #include "cn66xx_device.h"
30 #include "cn23xx_pf_device.h"
31 #include "cn23xx_vf_device.h"
32
33 struct iq_post_status {
34 int status;
35 int index;
36 };
37
38 static void check_db_timeout(struct work_struct *work);
39 static void __check_db_timeout(struct octeon_device *oct, u64 iq_no);
40
41 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *);
42
IQ_INSTR_MODE_64B(struct octeon_device * oct,int iq_no)43 static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)
44 {
45 struct octeon_instr_queue *iq =
46 (struct octeon_instr_queue *)oct->instr_queue[iq_no];
47 return iq->iqcmd_64B;
48 }
49
50 #define IQ_INSTR_MODE_32B(oct, iq_no) (!IQ_INSTR_MODE_64B(oct, iq_no))
51
52 /* Define this to return the request status comaptible to old code */
53 /*#define OCTEON_USE_OLD_REQ_STATUS*/
54
55 /* Return 0 on success, 1 on failure */
octeon_init_instr_queue(struct octeon_device * oct,union oct_txpciq txpciq,u32 num_descs)56 int octeon_init_instr_queue(struct octeon_device *oct,
57 union oct_txpciq txpciq,
58 u32 num_descs)
59 {
60 struct octeon_instr_queue *iq;
61 struct octeon_iq_config *conf = NULL;
62 u32 iq_no = (u32)txpciq.s.q_no;
63 u32 q_size;
64 struct cavium_wq *db_wq;
65 int numa_node = dev_to_node(&oct->pci_dev->dev);
66
67 if (OCTEON_CN6XXX(oct))
68 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn6xxx)));
69 else if (OCTEON_CN23XX_PF(oct))
70 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_pf)));
71 else if (OCTEON_CN23XX_VF(oct))
72 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_vf)));
73
74 if (!conf) {
75 dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n",
76 oct->chip_id);
77 return 1;
78 }
79
80 q_size = (u32)conf->instr_type * num_descs;
81
82 iq = oct->instr_queue[iq_no];
83
84 iq->oct_dev = oct;
85
86 iq->base_addr = lio_dma_alloc(oct, q_size, &iq->base_addr_dma);
87 if (!iq->base_addr) {
88 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n",
89 iq_no);
90 return 1;
91 }
92
93 iq->max_count = num_descs;
94
95 /* Initialize a list to holds requests that have been posted to Octeon
96 * but has yet to be fetched by octeon
97 */
98 iq->request_list = vmalloc_node((sizeof(*iq->request_list) * num_descs),
99 numa_node);
100 if (!iq->request_list)
101 iq->request_list =
102 vmalloc(array_size(num_descs,
103 sizeof(*iq->request_list)));
104 if (!iq->request_list) {
105 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
106 dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n",
107 iq_no);
108 return 1;
109 }
110
111 memset(iq->request_list, 0, sizeof(*iq->request_list) * num_descs);
112
113 dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %pad count: %d\n",
114 iq_no, iq->base_addr, &iq->base_addr_dma, iq->max_count);
115
116 iq->txpciq.u64 = txpciq.u64;
117 iq->fill_threshold = (u32)conf->db_min;
118 iq->fill_cnt = 0;
119 iq->host_write_index = 0;
120 iq->octeon_read_index = 0;
121 iq->flush_index = 0;
122 iq->last_db_time = 0;
123 iq->do_auto_flush = 1;
124 iq->db_timeout = (u32)conf->db_timeout;
125 atomic_set(&iq->instr_pending, 0);
126
127 /* Initialize the spinlock for this instruction queue */
128 spin_lock_init(&iq->lock);
129 if (iq_no == 0) {
130 iq->allow_soft_cmds = true;
131 spin_lock_init(&iq->post_lock);
132 } else {
133 iq->allow_soft_cmds = false;
134 }
135
136 spin_lock_init(&iq->iq_flush_running_lock);
137
138 oct->io_qmask.iq |= BIT_ULL(iq_no);
139
140 /* Set the 32B/64B mode for each input queue */
141 oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);
142 iq->iqcmd_64B = (conf->instr_type == 64);
143
144 oct->fn_list.setup_iq_regs(oct, iq_no);
145
146 oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db",
147 WQ_MEM_RECLAIM,
148 0);
149 if (!oct->check_db_wq[iq_no].wq) {
150 vfree(iq->request_list);
151 iq->request_list = NULL;
152 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
153 dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n",
154 iq_no);
155 return 1;
156 }
157
158 db_wq = &oct->check_db_wq[iq_no];
159
160 INIT_DELAYED_WORK(&db_wq->wk.work, check_db_timeout);
161 db_wq->wk.ctxptr = oct;
162 db_wq->wk.ctxul = iq_no;
163 queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1));
164
165 return 0;
166 }
167
octeon_delete_instr_queue(struct octeon_device * oct,u32 iq_no)168 int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
169 {
170 u64 desc_size = 0, q_size;
171 struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
172
173 cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work);
174 destroy_workqueue(oct->check_db_wq[iq_no].wq);
175
176 if (OCTEON_CN6XXX(oct))
177 desc_size =
178 CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn6xxx));
179 else if (OCTEON_CN23XX_PF(oct))
180 desc_size =
181 CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_pf));
182 else if (OCTEON_CN23XX_VF(oct))
183 desc_size =
184 CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_vf));
185
186 vfree(iq->request_list);
187
188 if (iq->base_addr) {
189 q_size = iq->max_count * desc_size;
190 lio_dma_free(oct, (u32)q_size, iq->base_addr,
191 iq->base_addr_dma);
192 oct->io_qmask.iq &= ~(1ULL << iq_no);
193 vfree(oct->instr_queue[iq_no]);
194 oct->instr_queue[iq_no] = NULL;
195 oct->num_iqs--;
196 return 0;
197 }
198 return 1;
199 }
200
201 /* Return 0 on success, 1 on failure */
octeon_setup_iq(struct octeon_device * oct,int ifidx,int q_index,union oct_txpciq txpciq,u32 num_descs,void * app_ctx)202 int octeon_setup_iq(struct octeon_device *oct,
203 int ifidx,
204 int q_index,
205 union oct_txpciq txpciq,
206 u32 num_descs,
207 void *app_ctx)
208 {
209 u32 iq_no = (u32)txpciq.s.q_no;
210 int numa_node = dev_to_node(&oct->pci_dev->dev);
211
212 if (oct->instr_queue[iq_no]) {
213 dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n",
214 iq_no);
215 oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64;
216 oct->instr_queue[iq_no]->app_ctx = app_ctx;
217 return 0;
218 }
219 oct->instr_queue[iq_no] =
220 vmalloc_node(sizeof(struct octeon_instr_queue), numa_node);
221 if (!oct->instr_queue[iq_no])
222 oct->instr_queue[iq_no] =
223 vmalloc(sizeof(struct octeon_instr_queue));
224 if (!oct->instr_queue[iq_no])
225 return 1;
226
227 memset(oct->instr_queue[iq_no], 0,
228 sizeof(struct octeon_instr_queue));
229
230 oct->instr_queue[iq_no]->q_index = q_index;
231 oct->instr_queue[iq_no]->app_ctx = app_ctx;
232 oct->instr_queue[iq_no]->ifidx = ifidx;
233
234 if (octeon_init_instr_queue(oct, txpciq, num_descs)) {
235 vfree(oct->instr_queue[iq_no]);
236 oct->instr_queue[iq_no] = NULL;
237 return 1;
238 }
239
240 oct->num_iqs++;
241 if (oct->fn_list.enable_io_queues(oct))
242 return 1;
243
244 return 0;
245 }
246
lio_wait_for_instr_fetch(struct octeon_device * oct)247 int lio_wait_for_instr_fetch(struct octeon_device *oct)
248 {
249 int i, retry = 1000, pending, instr_cnt = 0;
250
251 do {
252 instr_cnt = 0;
253
254 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
255 if (!(oct->io_qmask.iq & BIT_ULL(i)))
256 continue;
257 pending =
258 atomic_read(&oct->instr_queue[i]->instr_pending);
259 if (pending)
260 __check_db_timeout(oct, i);
261 instr_cnt += pending;
262 }
263
264 if (instr_cnt == 0)
265 break;
266
267 schedule_timeout_uninterruptible(1);
268
269 } while (retry-- && instr_cnt);
270
271 return instr_cnt;
272 }
273
274 static inline void
ring_doorbell(struct octeon_device * oct,struct octeon_instr_queue * iq)275 ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq)
276 {
277 if (atomic_read(&oct->status) == OCT_DEV_RUNNING) {
278 writel(iq->fill_cnt, iq->doorbell_reg);
279 /* make sure doorbell write goes through */
280 mmiowb();
281 iq->fill_cnt = 0;
282 iq->last_db_time = jiffies;
283 return;
284 }
285 }
286
287 void
octeon_ring_doorbell_locked(struct octeon_device * oct,u32 iq_no)288 octeon_ring_doorbell_locked(struct octeon_device *oct, u32 iq_no)
289 {
290 struct octeon_instr_queue *iq;
291
292 iq = oct->instr_queue[iq_no];
293 spin_lock(&iq->post_lock);
294 if (iq->fill_cnt)
295 ring_doorbell(oct, iq);
296 spin_unlock(&iq->post_lock);
297 }
298
__copy_cmd_into_iq(struct octeon_instr_queue * iq,u8 * cmd)299 static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq,
300 u8 *cmd)
301 {
302 u8 *iqptr, cmdsize;
303
304 cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
305 iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
306
307 memcpy(iqptr, cmd, cmdsize);
308 }
309
310 static inline struct iq_post_status
__post_command2(struct octeon_instr_queue * iq,u8 * cmd)311 __post_command2(struct octeon_instr_queue *iq, u8 *cmd)
312 {
313 struct iq_post_status st;
314
315 st.status = IQ_SEND_OK;
316
317 /* This ensures that the read index does not wrap around to the same
318 * position if queue gets full before Octeon could fetch any instr.
319 */
320 if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) {
321 st.status = IQ_SEND_FAILED;
322 st.index = -1;
323 return st;
324 }
325
326 if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 2))
327 st.status = IQ_SEND_STOP;
328
329 __copy_cmd_into_iq(iq, cmd);
330
331 /* "index" is returned, host_write_index is modified. */
332 st.index = iq->host_write_index;
333 iq->host_write_index = incr_index(iq->host_write_index, 1,
334 iq->max_count);
335 iq->fill_cnt++;
336
337 /* Flush the command into memory. We need to be sure the data is in
338 * memory before indicating that the instruction is pending.
339 */
340 wmb();
341
342 atomic_inc(&iq->instr_pending);
343
344 return st;
345 }
346
347 int
octeon_register_reqtype_free_fn(struct octeon_device * oct,int reqtype,void (* fn)(void *))348 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,
349 void (*fn)(void *))
350 {
351 if (reqtype > REQTYPE_LAST) {
352 dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n",
353 __func__, reqtype);
354 return -EINVAL;
355 }
356
357 reqtype_free_fn[oct->octeon_id][reqtype] = fn;
358
359 return 0;
360 }
361
362 static inline void
__add_to_request_list(struct octeon_instr_queue * iq,int idx,void * buf,int reqtype)363 __add_to_request_list(struct octeon_instr_queue *iq,
364 int idx, void *buf, int reqtype)
365 {
366 iq->request_list[idx].buf = buf;
367 iq->request_list[idx].reqtype = reqtype;
368 }
369
370 /* Can only run in process context */
371 int
lio_process_iq_request_list(struct octeon_device * oct,struct octeon_instr_queue * iq,u32 napi_budget)372 lio_process_iq_request_list(struct octeon_device *oct,
373 struct octeon_instr_queue *iq, u32 napi_budget)
374 {
375 struct cavium_wq *cwq = &oct->dma_comp_wq;
376 int reqtype;
377 void *buf;
378 u32 old = iq->flush_index;
379 u32 inst_count = 0;
380 unsigned int pkts_compl = 0, bytes_compl = 0;
381 struct octeon_soft_command *sc;
382 struct octeon_instr_irh *irh;
383 unsigned long flags;
384
385 while (old != iq->octeon_read_index) {
386 reqtype = iq->request_list[old].reqtype;
387 buf = iq->request_list[old].buf;
388
389 if (reqtype == REQTYPE_NONE)
390 goto skip_this;
391
392 octeon_update_tx_completion_counters(buf, reqtype, &pkts_compl,
393 &bytes_compl);
394
395 switch (reqtype) {
396 case REQTYPE_NORESP_NET:
397 case REQTYPE_NORESP_NET_SG:
398 case REQTYPE_RESP_NET_SG:
399 reqtype_free_fn[oct->octeon_id][reqtype](buf);
400 break;
401 case REQTYPE_RESP_NET:
402 case REQTYPE_SOFT_COMMAND:
403 sc = buf;
404
405 if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct))
406 irh = (struct octeon_instr_irh *)
407 &sc->cmd.cmd3.irh;
408 else
409 irh = (struct octeon_instr_irh *)
410 &sc->cmd.cmd2.irh;
411 if (irh->rflag) {
412 /* We're expecting a response from Octeon.
413 * It's up to lio_process_ordered_list() to
414 * process sc. Add sc to the ordered soft
415 * command response list because we expect
416 * a response from Octeon.
417 */
418 spin_lock_irqsave
419 (&oct->response_list
420 [OCTEON_ORDERED_SC_LIST].lock,
421 flags);
422 atomic_inc(&oct->response_list
423 [OCTEON_ORDERED_SC_LIST].
424 pending_req_count);
425 list_add_tail(&sc->node, &oct->response_list
426 [OCTEON_ORDERED_SC_LIST].head);
427 spin_unlock_irqrestore
428 (&oct->response_list
429 [OCTEON_ORDERED_SC_LIST].lock,
430 flags);
431 } else {
432 if (sc->callback) {
433 /* This callback must not sleep */
434 sc->callback(oct, OCTEON_REQUEST_DONE,
435 sc->callback_arg);
436 }
437 }
438 break;
439 default:
440 dev_err(&oct->pci_dev->dev,
441 "%s Unknown reqtype: %d buf: %p at idx %d\n",
442 __func__, reqtype, buf, old);
443 }
444
445 iq->request_list[old].buf = NULL;
446 iq->request_list[old].reqtype = 0;
447
448 skip_this:
449 inst_count++;
450 old = incr_index(old, 1, iq->max_count);
451
452 if ((napi_budget) && (inst_count >= napi_budget))
453 break;
454 }
455 if (bytes_compl)
456 octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl,
457 bytes_compl);
458 iq->flush_index = old;
459
460 if (atomic_read(&oct->response_list
461 [OCTEON_ORDERED_SC_LIST].pending_req_count))
462 queue_delayed_work(cwq->wq, &cwq->wk.work, msecs_to_jiffies(1));
463
464 return inst_count;
465 }
466
467 /* Can only be called from process context */
468 int
octeon_flush_iq(struct octeon_device * oct,struct octeon_instr_queue * iq,u32 napi_budget)469 octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,
470 u32 napi_budget)
471 {
472 u32 inst_processed = 0;
473 u32 tot_inst_processed = 0;
474 int tx_done = 1;
475
476 if (!spin_trylock(&iq->iq_flush_running_lock))
477 return tx_done;
478
479 spin_lock_bh(&iq->lock);
480
481 iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
482
483 do {
484 /* Process any outstanding IQ packets. */
485 if (iq->flush_index == iq->octeon_read_index)
486 break;
487
488 if (napi_budget)
489 inst_processed =
490 lio_process_iq_request_list(oct, iq,
491 napi_budget -
492 tot_inst_processed);
493 else
494 inst_processed =
495 lio_process_iq_request_list(oct, iq, 0);
496
497 if (inst_processed) {
498 atomic_sub(inst_processed, &iq->instr_pending);
499 iq->stats.instr_processed += inst_processed;
500 }
501
502 tot_inst_processed += inst_processed;
503 } while (tot_inst_processed < napi_budget);
504
505 if (napi_budget && (tot_inst_processed >= napi_budget))
506 tx_done = 0;
507
508 iq->last_db_time = jiffies;
509
510 spin_unlock_bh(&iq->lock);
511
512 spin_unlock(&iq->iq_flush_running_lock);
513
514 return tx_done;
515 }
516
517 /* Process instruction queue after timeout.
518 * This routine gets called from a workqueue or when removing the module.
519 */
__check_db_timeout(struct octeon_device * oct,u64 iq_no)520 static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)
521 {
522 struct octeon_instr_queue *iq;
523 u64 next_time;
524
525 if (!oct)
526 return;
527
528 iq = oct->instr_queue[iq_no];
529 if (!iq)
530 return;
531
532 /* return immediately, if no work pending */
533 if (!atomic_read(&iq->instr_pending))
534 return;
535 /* If jiffies - last_db_time < db_timeout do nothing */
536 next_time = iq->last_db_time + iq->db_timeout;
537 if (!time_after(jiffies, (unsigned long)next_time))
538 return;
539 iq->last_db_time = jiffies;
540
541 /* Flush the instruction queue */
542 octeon_flush_iq(oct, iq, 0);
543
544 lio_enable_irq(NULL, iq);
545 }
546
547 /* Called by the Poll thread at regular intervals to check the instruction
548 * queue for commands to be posted and for commands that were fetched by Octeon.
549 */
check_db_timeout(struct work_struct * work)550 static void check_db_timeout(struct work_struct *work)
551 {
552 struct cavium_wk *wk = (struct cavium_wk *)work;
553 struct octeon_device *oct = (struct octeon_device *)wk->ctxptr;
554 u64 iq_no = wk->ctxul;
555 struct cavium_wq *db_wq = &oct->check_db_wq[iq_no];
556 u32 delay = 10;
557
558 __check_db_timeout(oct, iq_no);
559 queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(delay));
560 }
561
562 int
octeon_send_command(struct octeon_device * oct,u32 iq_no,u32 force_db,void * cmd,void * buf,u32 datasize,u32 reqtype)563 octeon_send_command(struct octeon_device *oct, u32 iq_no,
564 u32 force_db, void *cmd, void *buf,
565 u32 datasize, u32 reqtype)
566 {
567 int xmit_stopped;
568 struct iq_post_status st;
569 struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
570
571 /* Get the lock and prevent other tasks and tx interrupt handler from
572 * running.
573 */
574 if (iq->allow_soft_cmds)
575 spin_lock_bh(&iq->post_lock);
576
577 st = __post_command2(iq, cmd);
578
579 if (st.status != IQ_SEND_FAILED) {
580 xmit_stopped = octeon_report_sent_bytes_to_bql(buf, reqtype);
581 __add_to_request_list(iq, st.index, buf, reqtype);
582 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);
583 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);
584
585 if (iq->fill_cnt >= MAX_OCTEON_FILL_COUNT || force_db ||
586 xmit_stopped || st.status == IQ_SEND_STOP)
587 ring_doorbell(oct, iq);
588 } else {
589 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);
590 }
591
592 if (iq->allow_soft_cmds)
593 spin_unlock_bh(&iq->post_lock);
594
595 /* This is only done here to expedite packets being flushed
596 * for cases where there are no IQ completion interrupts.
597 */
598
599 return st.status;
600 }
601
602 void
octeon_prepare_soft_command(struct octeon_device * oct,struct octeon_soft_command * sc,u8 opcode,u8 subcode,u32 irh_ossp,u64 ossp0,u64 ossp1)603 octeon_prepare_soft_command(struct octeon_device *oct,
604 struct octeon_soft_command *sc,
605 u8 opcode,
606 u8 subcode,
607 u32 irh_ossp,
608 u64 ossp0,
609 u64 ossp1)
610 {
611 struct octeon_config *oct_cfg;
612 struct octeon_instr_ih2 *ih2;
613 struct octeon_instr_ih3 *ih3;
614 struct octeon_instr_pki_ih3 *pki_ih3;
615 struct octeon_instr_irh *irh;
616 struct octeon_instr_rdp *rdp;
617
618 WARN_ON(opcode > 15);
619 WARN_ON(subcode > 127);
620
621 oct_cfg = octeon_get_conf(oct);
622
623 if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {
624 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
625
626 ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;
627
628 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
629
630 pki_ih3->w = 1;
631 pki_ih3->raw = 1;
632 pki_ih3->utag = 1;
633 pki_ih3->uqpg =
634 oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
635 pki_ih3->utt = 1;
636 pki_ih3->tag = LIO_CONTROL;
637 pki_ih3->tagtype = ATOMIC_TAG;
638 pki_ih3->qpg =
639 oct->instr_queue[sc->iq_no]->txpciq.s.ctrl_qpg;
640
641 pki_ih3->pm = 0x7;
642 pki_ih3->sl = 8;
643
644 if (sc->datasize)
645 ih3->dlengsz = sc->datasize;
646
647 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
648 irh->opcode = opcode;
649 irh->subcode = subcode;
650
651 /* opcode/subcode specific parameters (ossp) */
652 irh->ossp = irh_ossp;
653 sc->cmd.cmd3.ossp[0] = ossp0;
654 sc->cmd.cmd3.ossp[1] = ossp1;
655
656 if (sc->rdatasize) {
657 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
658 rdp->pcie_port = oct->pcie_port;
659 rdp->rlen = sc->rdatasize;
660
661 irh->rflag = 1;
662 /*PKI IH3*/
663 /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
664 ih3->fsz = LIO_SOFTCMDRESP_IH3;
665 } else {
666 irh->rflag = 0;
667 /*PKI IH3*/
668 /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
669 ih3->fsz = LIO_PCICMD_O3;
670 }
671
672 } else {
673 ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
674 ih2->tagtype = ATOMIC_TAG;
675 ih2->tag = LIO_CONTROL;
676 ih2->raw = 1;
677 ih2->grp = CFG_GET_CTRL_Q_GRP(oct_cfg);
678
679 if (sc->datasize) {
680 ih2->dlengsz = sc->datasize;
681 ih2->rs = 1;
682 }
683
684 irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
685 irh->opcode = opcode;
686 irh->subcode = subcode;
687
688 /* opcode/subcode specific parameters (ossp) */
689 irh->ossp = irh_ossp;
690 sc->cmd.cmd2.ossp[0] = ossp0;
691 sc->cmd.cmd2.ossp[1] = ossp1;
692
693 if (sc->rdatasize) {
694 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
695 rdp->pcie_port = oct->pcie_port;
696 rdp->rlen = sc->rdatasize;
697
698 irh->rflag = 1;
699 /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
700 ih2->fsz = LIO_SOFTCMDRESP_IH2;
701 } else {
702 irh->rflag = 0;
703 /* irh + ossp[0] + ossp[1] = 24 bytes */
704 ih2->fsz = LIO_PCICMD_O2;
705 }
706 }
707 }
708
octeon_send_soft_command(struct octeon_device * oct,struct octeon_soft_command * sc)709 int octeon_send_soft_command(struct octeon_device *oct,
710 struct octeon_soft_command *sc)
711 {
712 struct octeon_instr_queue *iq;
713 struct octeon_instr_ih2 *ih2;
714 struct octeon_instr_ih3 *ih3;
715 struct octeon_instr_irh *irh;
716 u32 len;
717
718 iq = oct->instr_queue[sc->iq_no];
719 if (!iq->allow_soft_cmds) {
720 dev_err(&oct->pci_dev->dev, "Soft commands are not allowed on Queue %d\n",
721 sc->iq_no);
722 INCR_INSTRQUEUE_PKT_COUNT(oct, sc->iq_no, instr_dropped, 1);
723 return IQ_SEND_FAILED;
724 }
725
726 if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {
727 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
728 if (ih3->dlengsz) {
729 WARN_ON(!sc->dmadptr);
730 sc->cmd.cmd3.dptr = sc->dmadptr;
731 }
732 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
733 if (irh->rflag) {
734 WARN_ON(!sc->dmarptr);
735 WARN_ON(!sc->status_word);
736 *sc->status_word = COMPLETION_WORD_INIT;
737 sc->cmd.cmd3.rptr = sc->dmarptr;
738 }
739 len = (u32)ih3->dlengsz;
740 } else {
741 ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
742 if (ih2->dlengsz) {
743 WARN_ON(!sc->dmadptr);
744 sc->cmd.cmd2.dptr = sc->dmadptr;
745 }
746 irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
747 if (irh->rflag) {
748 WARN_ON(!sc->dmarptr);
749 WARN_ON(!sc->status_word);
750 *sc->status_word = COMPLETION_WORD_INIT;
751 sc->cmd.cmd2.rptr = sc->dmarptr;
752 }
753 len = (u32)ih2->dlengsz;
754 }
755
756 if (sc->wait_time)
757 sc->timeout = jiffies + sc->wait_time;
758
759 return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
760 len, REQTYPE_SOFT_COMMAND));
761 }
762
octeon_setup_sc_buffer_pool(struct octeon_device * oct)763 int octeon_setup_sc_buffer_pool(struct octeon_device *oct)
764 {
765 int i;
766 u64 dma_addr;
767 struct octeon_soft_command *sc;
768
769 INIT_LIST_HEAD(&oct->sc_buf_pool.head);
770 spin_lock_init(&oct->sc_buf_pool.lock);
771 atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0);
772
773 for (i = 0; i < MAX_SOFT_COMMAND_BUFFERS; i++) {
774 sc = (struct octeon_soft_command *)
775 lio_dma_alloc(oct,
776 SOFT_COMMAND_BUFFER_SIZE,
777 (dma_addr_t *)&dma_addr);
778 if (!sc) {
779 octeon_free_sc_buffer_pool(oct);
780 return 1;
781 }
782
783 sc->dma_addr = dma_addr;
784 sc->size = SOFT_COMMAND_BUFFER_SIZE;
785
786 list_add_tail(&sc->node, &oct->sc_buf_pool.head);
787 }
788
789 return 0;
790 }
791
octeon_free_sc_buffer_pool(struct octeon_device * oct)792 int octeon_free_sc_buffer_pool(struct octeon_device *oct)
793 {
794 struct list_head *tmp, *tmp2;
795 struct octeon_soft_command *sc;
796
797 spin_lock_bh(&oct->sc_buf_pool.lock);
798
799 list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) {
800 list_del(tmp);
801
802 sc = (struct octeon_soft_command *)tmp;
803
804 lio_dma_free(oct, sc->size, sc, sc->dma_addr);
805 }
806
807 INIT_LIST_HEAD(&oct->sc_buf_pool.head);
808
809 spin_unlock_bh(&oct->sc_buf_pool.lock);
810
811 return 0;
812 }
813
octeon_alloc_soft_command(struct octeon_device * oct,u32 datasize,u32 rdatasize,u32 ctxsize)814 struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct,
815 u32 datasize,
816 u32 rdatasize,
817 u32 ctxsize)
818 {
819 u64 dma_addr;
820 u32 size;
821 u32 offset = sizeof(struct octeon_soft_command);
822 struct octeon_soft_command *sc = NULL;
823 struct list_head *tmp;
824
825 WARN_ON((offset + datasize + rdatasize + ctxsize) >
826 SOFT_COMMAND_BUFFER_SIZE);
827
828 spin_lock_bh(&oct->sc_buf_pool.lock);
829
830 if (list_empty(&oct->sc_buf_pool.head)) {
831 spin_unlock_bh(&oct->sc_buf_pool.lock);
832 return NULL;
833 }
834
835 list_for_each(tmp, &oct->sc_buf_pool.head)
836 break;
837
838 list_del(tmp);
839
840 atomic_inc(&oct->sc_buf_pool.alloc_buf_count);
841
842 spin_unlock_bh(&oct->sc_buf_pool.lock);
843
844 sc = (struct octeon_soft_command *)tmp;
845
846 dma_addr = sc->dma_addr;
847 size = sc->size;
848
849 memset(sc, 0, sc->size);
850
851 sc->dma_addr = dma_addr;
852 sc->size = size;
853
854 if (ctxsize) {
855 sc->ctxptr = (u8 *)sc + offset;
856 sc->ctxsize = ctxsize;
857 }
858
859 /* Start data at 128 byte boundary */
860 offset = (offset + ctxsize + 127) & 0xffffff80;
861
862 if (datasize) {
863 sc->virtdptr = (u8 *)sc + offset;
864 sc->dmadptr = dma_addr + offset;
865 sc->datasize = datasize;
866 }
867
868 /* Start rdata at 128 byte boundary */
869 offset = (offset + datasize + 127) & 0xffffff80;
870
871 if (rdatasize) {
872 WARN_ON(rdatasize < 16);
873 sc->virtrptr = (u8 *)sc + offset;
874 sc->dmarptr = dma_addr + offset;
875 sc->rdatasize = rdatasize;
876 sc->status_word = (u64 *)((u8 *)(sc->virtrptr) + rdatasize - 8);
877 }
878
879 return sc;
880 }
881
octeon_free_soft_command(struct octeon_device * oct,struct octeon_soft_command * sc)882 void octeon_free_soft_command(struct octeon_device *oct,
883 struct octeon_soft_command *sc)
884 {
885 spin_lock_bh(&oct->sc_buf_pool.lock);
886
887 list_add_tail(&sc->node, &oct->sc_buf_pool.head);
888
889 atomic_dec(&oct->sc_buf_pool.alloc_buf_count);
890
891 spin_unlock_bh(&oct->sc_buf_pool.lock);
892 }
893
