1 /*
2 * cxgb4_uld.c:Chelsio Upper Layer Driver Interface for T4/T5/T6 SGE management
3 *
4 * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written by: Atul Gupta (atul.gupta@chelsio.com)
35 * Written by: Hariprasad Shenai (hariprasad@chelsio.com)
36 */
37
38 #include <linux/kernel.h>
39 #include <linux/module.h>
40 #include <linux/errno.h>
41 #include <linux/types.h>
42 #include <linux/debugfs.h>
43 #include <linux/export.h>
44 #include <linux/list.h>
45 #include <linux/skbuff.h>
46 #include <linux/pci.h>
47
48 #include "cxgb4.h"
49 #include "cxgb4_uld.h"
50 #include "t4_regs.h"
51 #include "t4fw_api.h"
52 #include "t4_msg.h"
53
54 #define for_each_uldrxq(m, i) for (i = 0; i < ((m)->nrxq + (m)->nciq); i++)
55
get_msix_idx_from_bmap(struct adapter * adap)56 static int get_msix_idx_from_bmap(struct adapter *adap)
57 {
58 struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
59 unsigned long flags;
60 unsigned int msix_idx;
61
62 spin_lock_irqsave(&bmap->lock, flags);
63 msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize);
64 if (msix_idx < bmap->mapsize) {
65 __set_bit(msix_idx, bmap->msix_bmap);
66 } else {
67 spin_unlock_irqrestore(&bmap->lock, flags);
68 return -ENOSPC;
69 }
70
71 spin_unlock_irqrestore(&bmap->lock, flags);
72 return msix_idx;
73 }
74
free_msix_idx_in_bmap(struct adapter * adap,unsigned int msix_idx)75 static void free_msix_idx_in_bmap(struct adapter *adap, unsigned int msix_idx)
76 {
77 struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
78 unsigned long flags;
79
80 spin_lock_irqsave(&bmap->lock, flags);
81 __clear_bit(msix_idx, bmap->msix_bmap);
82 spin_unlock_irqrestore(&bmap->lock, flags);
83 }
84
85 /* Flush the aggregated lro sessions */
uldrx_flush_handler(struct sge_rspq * q)86 static void uldrx_flush_handler(struct sge_rspq *q)
87 {
88 struct adapter *adap = q->adap;
89
90 if (adap->uld[q->uld].lro_flush)
91 adap->uld[q->uld].lro_flush(&q->lro_mgr);
92 }
93
94 /**
95 * uldrx_handler - response queue handler for ULD queues
96 * @q: the response queue that received the packet
97 * @rsp: the response queue descriptor holding the offload message
98 * @gl: the gather list of packet fragments
99 *
100 * Deliver an ingress offload packet to a ULD. All processing is done by
101 * the ULD, we just maintain statistics.
102 */
uldrx_handler(struct sge_rspq * q,const __be64 * rsp,const struct pkt_gl * gl)103 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
104 const struct pkt_gl *gl)
105 {
106 struct adapter *adap = q->adap;
107 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
108 int ret;
109
110 /* FW can send CPLs encapsulated in a CPL_FW4_MSG */
111 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
112 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
113 rsp += 2;
114
115 if (q->flush_handler)
116 ret = adap->uld[q->uld].lro_rx_handler(adap->uld[q->uld].handle,
117 rsp, gl, &q->lro_mgr,
118 &q->napi);
119 else
120 ret = adap->uld[q->uld].rx_handler(adap->uld[q->uld].handle,
121 rsp, gl);
122
123 if (ret) {
124 rxq->stats.nomem++;
125 return -1;
126 }
127
128 if (!gl)
129 rxq->stats.imm++;
130 else if (gl == CXGB4_MSG_AN)
131 rxq->stats.an++;
132 else
133 rxq->stats.pkts++;
134 return 0;
135 }
136
alloc_uld_rxqs(struct adapter * adap,struct sge_uld_rxq_info * rxq_info,bool lro)137 static int alloc_uld_rxqs(struct adapter *adap,
138 struct sge_uld_rxq_info *rxq_info, bool lro)
139 {
140 struct sge *s = &adap->sge;
141 unsigned int nq = rxq_info->nrxq + rxq_info->nciq;
142 struct sge_ofld_rxq *q = rxq_info->uldrxq;
143 unsigned short *ids = rxq_info->rspq_id;
144 unsigned int bmap_idx = 0;
145 unsigned int per_chan;
146 int i, err, msi_idx, que_idx = 0;
147
148 per_chan = rxq_info->nrxq / adap->params.nports;
149
150 if (adap->flags & USING_MSIX)
151 msi_idx = 1;
152 else
153 msi_idx = -((int)s->intrq.abs_id + 1);
154
155 for (i = 0; i < nq; i++, q++) {
156 if (i == rxq_info->nrxq) {
157 /* start allocation of concentrator queues */
158 per_chan = rxq_info->nciq / adap->params.nports;
159 que_idx = 0;
160 }
161
162 if (msi_idx >= 0) {
163 bmap_idx = get_msix_idx_from_bmap(adap);
164 msi_idx = adap->msix_info_ulds[bmap_idx].idx;
165 }
166 err = t4_sge_alloc_rxq(adap, &q->rspq, false,
167 adap->port[que_idx++ / per_chan],
168 msi_idx,
169 q->fl.size ? &q->fl : NULL,
170 uldrx_handler,
171 lro ? uldrx_flush_handler : NULL,
172 0);
173 if (err)
174 goto freeout;
175 if (msi_idx >= 0)
176 rxq_info->msix_tbl[i] = bmap_idx;
177 memset(&q->stats, 0, sizeof(q->stats));
178 if (ids)
179 ids[i] = q->rspq.abs_id;
180 }
181 return 0;
182 freeout:
183 q = rxq_info->uldrxq;
184 for ( ; i; i--, q++) {
185 if (q->rspq.desc)
186 free_rspq_fl(adap, &q->rspq,
187 q->fl.size ? &q->fl : NULL);
188 }
189 return err;
190 }
191
192 static int
setup_sge_queues_uld(struct adapter * adap,unsigned int uld_type,bool lro)193 setup_sge_queues_uld(struct adapter *adap, unsigned int uld_type, bool lro)
194 {
195 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
196 int i, ret = 0;
197
198 if (adap->flags & USING_MSIX) {
199 rxq_info->msix_tbl = kcalloc((rxq_info->nrxq + rxq_info->nciq),
200 sizeof(unsigned short),
201 GFP_KERNEL);
202 if (!rxq_info->msix_tbl)
203 return -ENOMEM;
204 }
205
206 ret = !(!alloc_uld_rxqs(adap, rxq_info, lro));
207
208 /* Tell uP to route control queue completions to rdma rspq */
209 if (adap->flags & FULL_INIT_DONE &&
210 !ret && uld_type == CXGB4_ULD_RDMA) {
211 struct sge *s = &adap->sge;
212 unsigned int cmplqid;
213 u32 param, cmdop;
214
215 cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL;
216 for_each_port(adap, i) {
217 cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id;
218 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
219 FW_PARAMS_PARAM_X_V(cmdop) |
220 FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id));
221 ret = t4_set_params(adap, adap->mbox, adap->pf,
222 0, 1, ¶m, &cmplqid);
223 }
224 }
225 return ret;
226 }
227
t4_free_uld_rxqs(struct adapter * adap,int n,struct sge_ofld_rxq * q)228 static void t4_free_uld_rxqs(struct adapter *adap, int n,
229 struct sge_ofld_rxq *q)
230 {
231 for ( ; n; n--, q++) {
232 if (q->rspq.desc)
233 free_rspq_fl(adap, &q->rspq,
234 q->fl.size ? &q->fl : NULL);
235 }
236 }
237
free_sge_queues_uld(struct adapter * adap,unsigned int uld_type)238 static void free_sge_queues_uld(struct adapter *adap, unsigned int uld_type)
239 {
240 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
241
242 if (adap->flags & FULL_INIT_DONE && uld_type == CXGB4_ULD_RDMA) {
243 struct sge *s = &adap->sge;
244 u32 param, cmdop, cmplqid = 0;
245 int i;
246
247 cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL;
248 for_each_port(adap, i) {
249 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
250 FW_PARAMS_PARAM_X_V(cmdop) |
251 FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id));
252 t4_set_params(adap, adap->mbox, adap->pf,
253 0, 1, ¶m, &cmplqid);
254 }
255 }
256
257 if (rxq_info->nciq)
258 t4_free_uld_rxqs(adap, rxq_info->nciq,
259 rxq_info->uldrxq + rxq_info->nrxq);
260 t4_free_uld_rxqs(adap, rxq_info->nrxq, rxq_info->uldrxq);
261 if (adap->flags & USING_MSIX)
262 kfree(rxq_info->msix_tbl);
263 }
264
cfg_queues_uld(struct adapter * adap,unsigned int uld_type,const struct cxgb4_uld_info * uld_info)265 static int cfg_queues_uld(struct adapter *adap, unsigned int uld_type,
266 const struct cxgb4_uld_info *uld_info)
267 {
268 struct sge *s = &adap->sge;
269 struct sge_uld_rxq_info *rxq_info;
270 int i, nrxq, ciq_size;
271
272 rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
273 if (!rxq_info)
274 return -ENOMEM;
275
276 if (adap->flags & USING_MSIX && uld_info->nrxq > s->nqs_per_uld) {
277 i = s->nqs_per_uld;
278 rxq_info->nrxq = roundup(i, adap->params.nports);
279 } else {
280 i = min_t(int, uld_info->nrxq,
281 num_online_cpus());
282 rxq_info->nrxq = roundup(i, adap->params.nports);
283 }
284 if (!uld_info->ciq) {
285 rxq_info->nciq = 0;
286 } else {
287 if (adap->flags & USING_MSIX)
288 rxq_info->nciq = min_t(int, s->nqs_per_uld,
289 num_online_cpus());
290 else
291 rxq_info->nciq = min_t(int, MAX_OFLD_QSETS,
292 num_online_cpus());
293 rxq_info->nciq = ((rxq_info->nciq / adap->params.nports) *
294 adap->params.nports);
295 rxq_info->nciq = max_t(int, rxq_info->nciq,
296 adap->params.nports);
297 }
298
299 nrxq = rxq_info->nrxq + rxq_info->nciq; /* total rxq's */
300 rxq_info->uldrxq = kcalloc(nrxq, sizeof(struct sge_ofld_rxq),
301 GFP_KERNEL);
302 if (!rxq_info->uldrxq) {
303 kfree(rxq_info);
304 return -ENOMEM;
305 }
306
307 rxq_info->rspq_id = kcalloc(nrxq, sizeof(unsigned short), GFP_KERNEL);
308 if (!rxq_info->rspq_id) {
309 kfree(rxq_info->uldrxq);
310 kfree(rxq_info);
311 return -ENOMEM;
312 }
313
314 for (i = 0; i < rxq_info->nrxq; i++) {
315 struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
316
317 init_rspq(adap, &r->rspq, 5, 1, uld_info->rxq_size, 64);
318 r->rspq.uld = uld_type;
319 r->fl.size = 72;
320 }
321
322 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
323 if (ciq_size > SGE_MAX_IQ_SIZE) {
324 dev_warn(adap->pdev_dev, "CIQ size too small for available IQs\n");
325 ciq_size = SGE_MAX_IQ_SIZE;
326 }
327
328 for (i = rxq_info->nrxq; i < nrxq; i++) {
329 struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
330
331 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
332 r->rspq.uld = uld_type;
333 }
334
335 memcpy(rxq_info->name, uld_info->name, IFNAMSIZ);
336 adap->sge.uld_rxq_info[uld_type] = rxq_info;
337
338 return 0;
339 }
340
free_queues_uld(struct adapter * adap,unsigned int uld_type)341 static void free_queues_uld(struct adapter *adap, unsigned int uld_type)
342 {
343 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
344
345 adap->sge.uld_rxq_info[uld_type] = NULL;
346 kfree(rxq_info->rspq_id);
347 kfree(rxq_info->uldrxq);
348 kfree(rxq_info);
349 }
350
351 static int
request_msix_queue_irqs_uld(struct adapter * adap,unsigned int uld_type)352 request_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
353 {
354 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
355 int err = 0;
356 unsigned int idx, bmap_idx;
357
358 for_each_uldrxq(rxq_info, idx) {
359 bmap_idx = rxq_info->msix_tbl[idx];
360 err = request_irq(adap->msix_info_ulds[bmap_idx].vec,
361 t4_sge_intr_msix, 0,
362 adap->msix_info_ulds[bmap_idx].desc,
363 &rxq_info->uldrxq[idx].rspq);
364 if (err)
365 goto unwind;
366 }
367 return 0;
368 unwind:
369 while (idx-- > 0) {
370 bmap_idx = rxq_info->msix_tbl[idx];
371 free_msix_idx_in_bmap(adap, bmap_idx);
372 free_irq(adap->msix_info_ulds[bmap_idx].vec,
373 &rxq_info->uldrxq[idx].rspq);
374 }
375 return err;
376 }
377
378 static void
free_msix_queue_irqs_uld(struct adapter * adap,unsigned int uld_type)379 free_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
380 {
381 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
382 unsigned int idx, bmap_idx;
383
384 for_each_uldrxq(rxq_info, idx) {
385 bmap_idx = rxq_info->msix_tbl[idx];
386
387 free_msix_idx_in_bmap(adap, bmap_idx);
388 free_irq(adap->msix_info_ulds[bmap_idx].vec,
389 &rxq_info->uldrxq[idx].rspq);
390 }
391 }
392
name_msix_vecs_uld(struct adapter * adap,unsigned int uld_type)393 static void name_msix_vecs_uld(struct adapter *adap, unsigned int uld_type)
394 {
395 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
396 int n = sizeof(adap->msix_info_ulds[0].desc);
397 unsigned int idx, bmap_idx;
398
399 for_each_uldrxq(rxq_info, idx) {
400 bmap_idx = rxq_info->msix_tbl[idx];
401
402 snprintf(adap->msix_info_ulds[bmap_idx].desc, n, "%s-%s%d",
403 adap->port[0]->name, rxq_info->name, idx);
404 }
405 }
406
enable_rx(struct adapter * adap,struct sge_rspq * q)407 static void enable_rx(struct adapter *adap, struct sge_rspq *q)
408 {
409 if (!q)
410 return;
411
412 if (q->handler)
413 napi_enable(&q->napi);
414
415 /* 0-increment GTS to start the timer and enable interrupts */
416 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
417 SEINTARM_V(q->intr_params) |
418 INGRESSQID_V(q->cntxt_id));
419 }
420
quiesce_rx(struct adapter * adap,struct sge_rspq * q)421 static void quiesce_rx(struct adapter *adap, struct sge_rspq *q)
422 {
423 if (q && q->handler)
424 napi_disable(&q->napi);
425 }
426
enable_rx_uld(struct adapter * adap,unsigned int uld_type)427 static void enable_rx_uld(struct adapter *adap, unsigned int uld_type)
428 {
429 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
430 int idx;
431
432 for_each_uldrxq(rxq_info, idx)
433 enable_rx(adap, &rxq_info->uldrxq[idx].rspq);
434 }
435
quiesce_rx_uld(struct adapter * adap,unsigned int uld_type)436 static void quiesce_rx_uld(struct adapter *adap, unsigned int uld_type)
437 {
438 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
439 int idx;
440
441 for_each_uldrxq(rxq_info, idx)
442 quiesce_rx(adap, &rxq_info->uldrxq[idx].rspq);
443 }
444
445 static void
free_sge_txq_uld(struct adapter * adap,struct sge_uld_txq_info * txq_info)446 free_sge_txq_uld(struct adapter *adap, struct sge_uld_txq_info *txq_info)
447 {
448 int nq = txq_info->ntxq;
449 int i;
450
451 for (i = 0; i < nq; i++) {
452 struct sge_uld_txq *txq = &txq_info->uldtxq[i];
453
454 if (txq && txq->q.desc) {
455 tasklet_kill(&txq->qresume_tsk);
456 t4_ofld_eq_free(adap, adap->mbox, adap->pf, 0,
457 txq->q.cntxt_id);
458 free_tx_desc(adap, &txq->q, txq->q.in_use, false);
459 kfree(txq->q.sdesc);
460 __skb_queue_purge(&txq->sendq);
461 free_txq(adap, &txq->q);
462 }
463 }
464 }
465
466 static int
alloc_sge_txq_uld(struct adapter * adap,struct sge_uld_txq_info * txq_info,unsigned int uld_type)467 alloc_sge_txq_uld(struct adapter *adap, struct sge_uld_txq_info *txq_info,
468 unsigned int uld_type)
469 {
470 struct sge *s = &adap->sge;
471 int nq = txq_info->ntxq;
472 int i, j, err;
473
474 j = nq / adap->params.nports;
475 for (i = 0; i < nq; i++) {
476 struct sge_uld_txq *txq = &txq_info->uldtxq[i];
477
478 txq->q.size = 1024;
479 err = t4_sge_alloc_uld_txq(adap, txq, adap->port[i / j],
480 s->fw_evtq.cntxt_id, uld_type);
481 if (err)
482 goto freeout;
483 }
484 return 0;
485 freeout:
486 free_sge_txq_uld(adap, txq_info);
487 return err;
488 }
489
490 static void
release_sge_txq_uld(struct adapter * adap,unsigned int uld_type)491 release_sge_txq_uld(struct adapter *adap, unsigned int uld_type)
492 {
493 struct sge_uld_txq_info *txq_info = NULL;
494 int tx_uld_type = TX_ULD(uld_type);
495
496 txq_info = adap->sge.uld_txq_info[tx_uld_type];
497
498 if (txq_info && atomic_dec_and_test(&txq_info->users)) {
499 free_sge_txq_uld(adap, txq_info);
500 kfree(txq_info->uldtxq);
501 kfree(txq_info);
502 adap->sge.uld_txq_info[tx_uld_type] = NULL;
503 }
504 }
505
506 static int
setup_sge_txq_uld(struct adapter * adap,unsigned int uld_type,const struct cxgb4_uld_info * uld_info)507 setup_sge_txq_uld(struct adapter *adap, unsigned int uld_type,
508 const struct cxgb4_uld_info *uld_info)
509 {
510 struct sge_uld_txq_info *txq_info = NULL;
511 int tx_uld_type, i;
512
513 tx_uld_type = TX_ULD(uld_type);
514 txq_info = adap->sge.uld_txq_info[tx_uld_type];
515
516 if ((tx_uld_type == CXGB4_TX_OFLD) && txq_info &&
517 (atomic_inc_return(&txq_info->users) > 1))
518 return 0;
519
520 txq_info = kzalloc(sizeof(*txq_info), GFP_KERNEL);
521 if (!txq_info)
522 return -ENOMEM;
523
524 i = min_t(int, uld_info->ntxq, num_online_cpus());
525 txq_info->ntxq = roundup(i, adap->params.nports);
526
527 txq_info->uldtxq = kcalloc(txq_info->ntxq, sizeof(struct sge_uld_txq),
528 GFP_KERNEL);
529 if (!txq_info->uldtxq) {
530 kfree(txq_info);
531 return -ENOMEM;
532 }
533
534 if (alloc_sge_txq_uld(adap, txq_info, tx_uld_type)) {
535 kfree(txq_info->uldtxq);
536 kfree(txq_info);
537 return -ENOMEM;
538 }
539
540 atomic_inc(&txq_info->users);
541 adap->sge.uld_txq_info[tx_uld_type] = txq_info;
542 return 0;
543 }
544
uld_queue_init(struct adapter * adap,unsigned int uld_type,struct cxgb4_lld_info * lli)545 static void uld_queue_init(struct adapter *adap, unsigned int uld_type,
546 struct cxgb4_lld_info *lli)
547 {
548 struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
549
550 lli->rxq_ids = rxq_info->rspq_id;
551 lli->nrxq = rxq_info->nrxq;
552 lli->ciq_ids = rxq_info->rspq_id + rxq_info->nrxq;
553 lli->nciq = rxq_info->nciq;
554 }
555
t4_uld_mem_alloc(struct adapter * adap)556 int t4_uld_mem_alloc(struct adapter *adap)
557 {
558 struct sge *s = &adap->sge;
559
560 adap->uld = kcalloc(CXGB4_ULD_MAX, sizeof(*adap->uld), GFP_KERNEL);
561 if (!adap->uld)
562 return -ENOMEM;
563
564 s->uld_rxq_info = kcalloc(CXGB4_ULD_MAX,
565 sizeof(struct sge_uld_rxq_info *),
566 GFP_KERNEL);
567 if (!s->uld_rxq_info)
568 goto err_uld;
569
570 s->uld_txq_info = kcalloc(CXGB4_TX_MAX,
571 sizeof(struct sge_uld_txq_info *),
572 GFP_KERNEL);
573 if (!s->uld_txq_info)
574 goto err_uld_rx;
575 return 0;
576
577 err_uld_rx:
578 kfree(s->uld_rxq_info);
579 err_uld:
580 kfree(adap->uld);
581 return -ENOMEM;
582 }
583
t4_uld_mem_free(struct adapter * adap)584 void t4_uld_mem_free(struct adapter *adap)
585 {
586 struct sge *s = &adap->sge;
587
588 kfree(s->uld_txq_info);
589 kfree(s->uld_rxq_info);
590 kfree(adap->uld);
591 }
592
593 /* This function should be called with uld_mutex taken. */
cxgb4_shutdown_uld_adapter(struct adapter * adap,enum cxgb4_uld type)594 static void cxgb4_shutdown_uld_adapter(struct adapter *adap, enum cxgb4_uld type)
595 {
596 if (adap->uld[type].handle) {
597 adap->uld[type].handle = NULL;
598 adap->uld[type].add = NULL;
599 release_sge_txq_uld(adap, type);
600
601 if (adap->flags & FULL_INIT_DONE)
602 quiesce_rx_uld(adap, type);
603
604 if (adap->flags & USING_MSIX)
605 free_msix_queue_irqs_uld(adap, type);
606
607 free_sge_queues_uld(adap, type);
608 free_queues_uld(adap, type);
609 }
610 }
611
t4_uld_clean_up(struct adapter * adap)612 void t4_uld_clean_up(struct adapter *adap)
613 {
614 unsigned int i;
615
616 mutex_lock(&uld_mutex);
617 for (i = 0; i < CXGB4_ULD_MAX; i++) {
618 if (!adap->uld[i].handle)
619 continue;
620
621 cxgb4_shutdown_uld_adapter(adap, i);
622 }
623 mutex_unlock(&uld_mutex);
624 }
625
uld_init(struct adapter * adap,struct cxgb4_lld_info * lld)626 static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
627 {
628 int i;
629
630 lld->pdev = adap->pdev;
631 lld->pf = adap->pf;
632 lld->l2t = adap->l2t;
633 lld->tids = &adap->tids;
634 lld->ports = adap->port;
635 lld->vr = &adap->vres;
636 lld->mtus = adap->params.mtus;
637 lld->ntxq = adap->sge.ofldqsets;
638 lld->nchan = adap->params.nports;
639 lld->nports = adap->params.nports;
640 lld->wr_cred = adap->params.ofldq_wr_cred;
641 lld->crypto = adap->params.crypto;
642 lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
643 lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A);
644 lld->iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A);
645 lld->iscsi_llimit = t4_read_reg(adap, ULP_RX_ISCSI_LLIMIT_A);
646 lld->iscsi_ppm = &adap->iscsi_ppm;
647 lld->adapter_type = adap->params.chip;
648 lld->cclk_ps = 1000000000 / adap->params.vpd.cclk;
649 lld->udb_density = 1 << adap->params.sge.eq_qpp;
650 lld->ucq_density = 1 << adap->params.sge.iq_qpp;
651 lld->filt_mode = adap->params.tp.vlan_pri_map;
652 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
653 for (i = 0; i < NCHAN; i++)
654 lld->tx_modq[i] = i;
655 lld->gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
656 lld->db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
657 lld->fw_vers = adap->params.fw_vers;
658 lld->dbfifo_int_thresh = dbfifo_int_thresh;
659 lld->sge_ingpadboundary = adap->sge.fl_align;
660 lld->sge_egrstatuspagesize = adap->sge.stat_len;
661 lld->sge_pktshift = adap->sge.pktshift;
662 lld->ulp_crypto = adap->params.crypto;
663 lld->enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
664 lld->max_ordird_qp = adap->params.max_ordird_qp;
665 lld->max_ird_adapter = adap->params.max_ird_adapter;
666 lld->ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
667 lld->nodeid = dev_to_node(adap->pdev_dev);
668 lld->fr_nsmr_tpte_wr_support = adap->params.fr_nsmr_tpte_wr_support;
669 lld->write_w_imm_support = adap->params.write_w_imm_support;
670 lld->write_cmpl_support = adap->params.write_cmpl_support;
671 }
672
uld_attach(struct adapter * adap,unsigned int uld)673 static void uld_attach(struct adapter *adap, unsigned int uld)
674 {
675 void *handle;
676 struct cxgb4_lld_info lli;
677
678 uld_init(adap, &lli);
679 uld_queue_init(adap, uld, &lli);
680
681 handle = adap->uld[uld].add(&lli);
682 if (IS_ERR(handle)) {
683 dev_warn(adap->pdev_dev,
684 "could not attach to the %s driver, error %ld\n",
685 adap->uld[uld].name, PTR_ERR(handle));
686 return;
687 }
688
689 adap->uld[uld].handle = handle;
690 t4_register_netevent_notifier();
691
692 if (adap->flags & FULL_INIT_DONE)
693 adap->uld[uld].state_change(handle, CXGB4_STATE_UP);
694 }
695
696 /**
697 * cxgb4_register_uld - register an upper-layer driver
698 * @type: the ULD type
699 * @p: the ULD methods
700 *
701 * Registers an upper-layer driver with this driver and notifies the ULD
702 * about any presently available devices that support its type. Returns
703 * %-EBUSY if a ULD of the same type is already registered.
704 */
cxgb4_register_uld(enum cxgb4_uld type,const struct cxgb4_uld_info * p)705 int cxgb4_register_uld(enum cxgb4_uld type,
706 const struct cxgb4_uld_info *p)
707 {
708 int ret = 0;
709 unsigned int adap_idx = 0;
710 struct adapter *adap;
711
712 if (type >= CXGB4_ULD_MAX)
713 return -EINVAL;
714
715 mutex_lock(&uld_mutex);
716 list_for_each_entry(adap, &adapter_list, list_node) {
717 if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
718 (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
719 continue;
720 if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
721 continue;
722 ret = cfg_queues_uld(adap, type, p);
723 if (ret)
724 goto out;
725 ret = setup_sge_queues_uld(adap, type, p->lro);
726 if (ret)
727 goto free_queues;
728 if (adap->flags & USING_MSIX) {
729 name_msix_vecs_uld(adap, type);
730 ret = request_msix_queue_irqs_uld(adap, type);
731 if (ret)
732 goto free_rxq;
733 }
734 if (adap->flags & FULL_INIT_DONE)
735 enable_rx_uld(adap, type);
736 if (adap->uld[type].add) {
737 ret = -EBUSY;
738 goto free_irq;
739 }
740 ret = setup_sge_txq_uld(adap, type, p);
741 if (ret)
742 goto free_irq;
743 adap->uld[type] = *p;
744 uld_attach(adap, type);
745 adap_idx++;
746 }
747 mutex_unlock(&uld_mutex);
748 return 0;
749
750 free_irq:
751 if (adap->flags & FULL_INIT_DONE)
752 quiesce_rx_uld(adap, type);
753 if (adap->flags & USING_MSIX)
754 free_msix_queue_irqs_uld(adap, type);
755 free_rxq:
756 free_sge_queues_uld(adap, type);
757 free_queues:
758 free_queues_uld(adap, type);
759 out:
760
761 list_for_each_entry(adap, &adapter_list, list_node) {
762 if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
763 (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
764 continue;
765 if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
766 continue;
767 if (!adap_idx)
768 break;
769 adap->uld[type].handle = NULL;
770 adap->uld[type].add = NULL;
771 release_sge_txq_uld(adap, type);
772 if (adap->flags & FULL_INIT_DONE)
773 quiesce_rx_uld(adap, type);
774 if (adap->flags & USING_MSIX)
775 free_msix_queue_irqs_uld(adap, type);
776 free_sge_queues_uld(adap, type);
777 free_queues_uld(adap, type);
778 adap_idx--;
779 }
780 mutex_unlock(&uld_mutex);
781 return ret;
782 }
783 EXPORT_SYMBOL(cxgb4_register_uld);
784
785 /**
786 * cxgb4_unregister_uld - unregister an upper-layer driver
787 * @type: the ULD type
788 *
789 * Unregisters an existing upper-layer driver.
790 */
cxgb4_unregister_uld(enum cxgb4_uld type)791 int cxgb4_unregister_uld(enum cxgb4_uld type)
792 {
793 struct adapter *adap;
794
795 if (type >= CXGB4_ULD_MAX)
796 return -EINVAL;
797
798 mutex_lock(&uld_mutex);
799 list_for_each_entry(adap, &adapter_list, list_node) {
800 if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
801 (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
802 continue;
803 if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
804 continue;
805
806 cxgb4_shutdown_uld_adapter(adap, type);
807 }
808 mutex_unlock(&uld_mutex);
809
810 return 0;
811 }
812 EXPORT_SYMBOL(cxgb4_unregister_uld);
813
