1 /*******************************************************************
2 * This file is part of the Emulex Linux Device Driver for *
3 * Fibre Channel Host Bus Adapters. *
4 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term *
5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *
6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *
7 * EMULEX and SLI are trademarks of Emulex. *
8 * www.broadcom.com *
9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *
10 * *
11 * This program is free software; you can redistribute it and/or *
12 * modify it under the terms of version 2 of the GNU General *
13 * Public License as published by the Free Software Foundation. *
14 * This program is distributed in the hope that it will be useful. *
15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19 * TO BE LEGALLY INVALID. See the GNU General Public License for *
20 * more details, a copy of which can be found in the file COPYING *
21 * included with this package. *
22 ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
30
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
38
39 #include "lpfc_version.h"
40 #include "lpfc_hw4.h"
41 #include "lpfc_hw.h"
42 #include "lpfc_sli.h"
43 #include "lpfc_sli4.h"
44 #include "lpfc_nl.h"
45 #include "lpfc_disc.h"
46 #include "lpfc.h"
47 #include "lpfc_nvme.h"
48 #include "lpfc_scsi.h"
49 #include "lpfc_logmsg.h"
50 #include "lpfc_crtn.h"
51 #include "lpfc_vport.h"
52 #include "lpfc_debugfs.h"
53
54 /* NVME initiator-based functions */
55
56 static struct lpfc_io_buf *
57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58 int idx, int expedite);
59
60 static void
61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62
63 static struct nvme_fc_port_template lpfc_nvme_template;
64
65 static union lpfc_wqe128 lpfc_iread_cmd_template;
66 static union lpfc_wqe128 lpfc_iwrite_cmd_template;
67 static union lpfc_wqe128 lpfc_icmnd_cmd_template;
68
69 /* Setup WQE templates for NVME IOs */
70 void
lpfc_nvme_cmd_template(void)71 lpfc_nvme_cmd_template(void)
72 {
73 union lpfc_wqe128 *wqe;
74
75 /* IREAD template */
76 wqe = &lpfc_iread_cmd_template;
77 memset(wqe, 0, sizeof(union lpfc_wqe128));
78
79 /* Word 0, 1, 2 - BDE is variable */
80
81 /* Word 3 - cmd_buff_len, payload_offset_len is zero */
82
83 /* Word 4 - total_xfer_len is variable */
84
85 /* Word 5 - is zero */
86
87 /* Word 6 - ctxt_tag, xri_tag is variable */
88
89 /* Word 7 */
90 bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
91 bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
92 bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
93 bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
94
95 /* Word 8 - abort_tag is variable */
96
97 /* Word 9 - reqtag is variable */
98
99 /* Word 10 - dbde, wqes is variable */
100 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
101 bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1);
102 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
103 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
104 bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
105 bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
106
107 /* Word 11 - pbde is variable */
108 bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD);
109 bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
110 bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
111
112 /* Word 12 - is zero */
113
114 /* Word 13, 14, 15 - PBDE is variable */
115
116 /* IWRITE template */
117 wqe = &lpfc_iwrite_cmd_template;
118 memset(wqe, 0, sizeof(union lpfc_wqe128));
119
120 /* Word 0, 1, 2 - BDE is variable */
121
122 /* Word 3 - cmd_buff_len, payload_offset_len is zero */
123
124 /* Word 4 - total_xfer_len is variable */
125
126 /* Word 5 - initial_xfer_len is variable */
127
128 /* Word 6 - ctxt_tag, xri_tag is variable */
129
130 /* Word 7 */
131 bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
132 bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
133 bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
134 bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
135
136 /* Word 8 - abort_tag is variable */
137
138 /* Word 9 - reqtag is variable */
139
140 /* Word 10 - dbde, wqes is variable */
141 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
142 bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1);
143 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
144 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
145 bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
146 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
147
148 /* Word 11 - pbde is variable */
149 bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD);
150 bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
151 bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
152
153 /* Word 12 - is zero */
154
155 /* Word 13, 14, 15 - PBDE is variable */
156
157 /* ICMND template */
158 wqe = &lpfc_icmnd_cmd_template;
159 memset(wqe, 0, sizeof(union lpfc_wqe128));
160
161 /* Word 0, 1, 2 - BDE is variable */
162
163 /* Word 3 - payload_offset_len is variable */
164
165 /* Word 4, 5 - is zero */
166
167 /* Word 6 - ctxt_tag, xri_tag is variable */
168
169 /* Word 7 */
170 bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
171 bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
172 bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
173 bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
174
175 /* Word 8 - abort_tag is variable */
176
177 /* Word 9 - reqtag is variable */
178
179 /* Word 10 - dbde, wqes is variable */
180 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
181 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
182 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
183 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
184 bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
185 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
186
187 /* Word 11 */
188 bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND);
189 bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
190 bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
191
192 /* Word 12, 13, 14, 15 - is zero */
193 }
194
195 /**
196 * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry.
197 * @pwqeq: Pointer to command iocb.
198 * @xritag: Tag that uniqely identifies the local exchange resource.
199 * @opt: Option bits -
200 * bit 0 = inhibit sending abts on the link
201 *
202 * This function is called with hbalock held.
203 **/
204 void
lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq * pwqeq,u16 xritag,u8 opt)205 lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt)
206 {
207 union lpfc_wqe128 *wqe = &pwqeq->wqe;
208
209 /* WQEs are reused. Clear stale data and set key fields to
210 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
211 */
212 memset(wqe, 0, sizeof(*wqe));
213
214 if (opt & INHIBIT_ABORT)
215 bf_set(abort_cmd_ia, &wqe->abort_cmd, 1);
216 /* Abort specified xri tag, with the mask deliberately zeroed */
217 bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG);
218
219 bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
220
221 /* Abort the IO associated with this outstanding exchange ID. */
222 wqe->abort_cmd.wqe_com.abort_tag = xritag;
223
224 /* iotag for the wqe completion. */
225 bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag);
226
227 bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1);
228 bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
229
230 bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND);
231 bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1);
232 bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
233 }
234
235 /**
236 * lpfc_nvme_create_queue -
237 * @pnvme_lport: Transport localport that LS is to be issued from
238 * @lpfc_pnvme: Pointer to the driver's nvme instance data
239 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
240 * @qsize: Size of the queue in bytes
241 * @handle: An opaque driver handle used in follow-up calls.
242 *
243 * Driver registers this routine to preallocate and initialize any
244 * internal data structures to bind the @qidx to its internal IO queues.
245 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
246 *
247 * Return value :
248 * 0 - Success
249 * -EINVAL - Unsupported input value.
250 * -ENOMEM - Could not alloc necessary memory
251 **/
252 static int
lpfc_nvme_create_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,u16 qsize,void ** handle)253 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
254 unsigned int qidx, u16 qsize,
255 void **handle)
256 {
257 struct lpfc_nvme_lport *lport;
258 struct lpfc_vport *vport;
259 struct lpfc_nvme_qhandle *qhandle;
260 char *str;
261
262 if (!pnvme_lport->private)
263 return -ENOMEM;
264
265 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
266 vport = lport->vport;
267 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
268 if (qhandle == NULL)
269 return -ENOMEM;
270
271 qhandle->cpu_id = raw_smp_processor_id();
272 qhandle->qidx = qidx;
273 /*
274 * NVME qidx == 0 is the admin queue, so both admin queue
275 * and first IO queue will use MSI-X vector and associated
276 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
277 */
278 if (qidx) {
279 str = "IO "; /* IO queue */
280 qhandle->index = ((qidx - 1) %
281 lpfc_nvme_template.max_hw_queues);
282 } else {
283 str = "ADM"; /* Admin queue */
284 qhandle->index = qidx;
285 }
286
287 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
288 "6073 Binding %s HdwQueue %d (cpu %d) to "
289 "hdw_queue %d qhandle x%px\n", str,
290 qidx, qhandle->cpu_id, qhandle->index, qhandle);
291 *handle = (void *)qhandle;
292 return 0;
293 }
294
295 /**
296 * lpfc_nvme_delete_queue -
297 * @pnvme_lport: Transport localport that LS is to be issued from
298 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
299 * @handle: An opaque driver handle from lpfc_nvme_create_queue
300 *
301 * Driver registers this routine to free
302 * any internal data structures to bind the @qidx to its internal
303 * IO queues.
304 *
305 * Return value :
306 * 0 - Success
307 * TODO: What are the failure codes.
308 **/
309 static void
lpfc_nvme_delete_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,void * handle)310 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
311 unsigned int qidx,
312 void *handle)
313 {
314 struct lpfc_nvme_lport *lport;
315 struct lpfc_vport *vport;
316
317 if (!pnvme_lport->private)
318 return;
319
320 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
321 vport = lport->vport;
322
323 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
324 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
325 lport, qidx, handle);
326 kfree(handle);
327 }
328
329 static void
lpfc_nvme_localport_delete(struct nvme_fc_local_port * localport)330 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
331 {
332 struct lpfc_nvme_lport *lport = localport->private;
333
334 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
335 "6173 localport x%px delete complete\n",
336 lport);
337
338 /* release any threads waiting for the unreg to complete */
339 if (lport->vport->localport)
340 complete(lport->lport_unreg_cmp);
341 }
342
343 /* lpfc_nvme_remoteport_delete
344 *
345 * @remoteport: Pointer to an nvme transport remoteport instance.
346 *
347 * This is a template downcall. NVME transport calls this function
348 * when it has completed the unregistration of a previously
349 * registered remoteport.
350 *
351 * Return value :
352 * None
353 */
354 static void
lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port * remoteport)355 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
356 {
357 struct lpfc_nvme_rport *rport = remoteport->private;
358 struct lpfc_vport *vport;
359 struct lpfc_nodelist *ndlp;
360
361 ndlp = rport->ndlp;
362 if (!ndlp)
363 goto rport_err;
364
365 vport = ndlp->vport;
366 if (!vport)
367 goto rport_err;
368
369 /* Remove this rport from the lport's list - memory is owned by the
370 * transport. Remove the ndlp reference for the NVME transport before
371 * calling state machine to remove the node.
372 */
373 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
374 "6146 remoteport delete of remoteport x%px\n",
375 remoteport);
376 spin_lock_irq(&vport->phba->hbalock);
377
378 /* The register rebind might have occurred before the delete
379 * downcall. Guard against this race.
380 */
381 if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) {
382 ndlp->nrport = NULL;
383 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
384 spin_unlock_irq(&vport->phba->hbalock);
385
386 /* Remove original register reference. The host transport
387 * won't reference this rport/remoteport any further.
388 */
389 lpfc_nlp_put(ndlp);
390 } else {
391 spin_unlock_irq(&vport->phba->hbalock);
392 }
393
394 rport_err:
395 return;
396 }
397
398 /**
399 * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
400 * @phba: pointer to lpfc hba data structure.
401 * @axchg: pointer to exchange context for the NVME LS request
402 *
403 * This routine is used for processing an asychronously received NVME LS
404 * request. Any remaining validation is done and the LS is then forwarded
405 * to the nvme-fc transport via nvme_fc_rcv_ls_req().
406 *
407 * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
408 * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
409 * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
410 *
411 * Returns 0 if LS was handled and delivered to the transport
412 * Returns 1 if LS failed to be handled and should be dropped
413 */
414 int
lpfc_nvme_handle_lsreq(struct lpfc_hba * phba,struct lpfc_async_xchg_ctx * axchg)415 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
416 struct lpfc_async_xchg_ctx *axchg)
417 {
418 #if (IS_ENABLED(CONFIG_NVME_FC))
419 struct lpfc_vport *vport;
420 struct lpfc_nvme_rport *lpfc_rport;
421 struct nvme_fc_remote_port *remoteport;
422 struct lpfc_nvme_lport *lport;
423 uint32_t *payload = axchg->payload;
424 int rc;
425
426 vport = axchg->ndlp->vport;
427 lpfc_rport = axchg->ndlp->nrport;
428 if (!lpfc_rport)
429 return -EINVAL;
430
431 remoteport = lpfc_rport->remoteport;
432 if (!vport->localport)
433 return -EINVAL;
434
435 lport = vport->localport->private;
436 if (!lport)
437 return -EINVAL;
438
439 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
440 axchg->size);
441
442 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
443 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
444 "%08x %08x %08x\n",
445 axchg->size, rc,
446 *payload, *(payload+1), *(payload+2),
447 *(payload+3), *(payload+4), *(payload+5));
448
449 if (!rc)
450 return 0;
451 #endif
452 return 1;
453 }
454
455 /**
456 * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
457 * LS request.
458 * @phba: Pointer to HBA context object
459 * @vport: The local port that issued the LS
460 * @cmdwqe: Pointer to driver command WQE object.
461 * @wcqe: Pointer to driver response CQE object.
462 *
463 * This function is the generic completion handler for NVME LS requests.
464 * The function updates any states and statistics, calls the transport
465 * ls_req done() routine, then tears down the command and buffers used
466 * for the LS request.
467 **/
468 void
__lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_vport * vport,struct lpfc_iocbq * cmdwqe,struct lpfc_wcqe_complete * wcqe)469 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
470 struct lpfc_iocbq *cmdwqe,
471 struct lpfc_wcqe_complete *wcqe)
472 {
473 struct nvmefc_ls_req *pnvme_lsreq;
474 struct lpfc_dmabuf *buf_ptr;
475 struct lpfc_nodelist *ndlp;
476 uint32_t status;
477
478 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
479 ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
480 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
481
482 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
483 "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x "
484 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
485 "ndlp:x%px\n",
486 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
487 cmdwqe->sli4_xritag, status,
488 (wcqe->parameter & 0xffff),
489 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp);
490
491 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
492 cmdwqe->sli4_xritag, status, wcqe->parameter);
493
494 if (cmdwqe->context3) {
495 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3;
496 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
497 kfree(buf_ptr);
498 cmdwqe->context3 = NULL;
499 }
500 if (pnvme_lsreq->done)
501 pnvme_lsreq->done(pnvme_lsreq, status);
502 else
503 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
504 "6046 NVMEx cmpl without done call back? "
505 "Data %px DID %x Xri: %x status %x\n",
506 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
507 cmdwqe->sli4_xritag, status);
508 if (ndlp) {
509 lpfc_nlp_put(ndlp);
510 cmdwqe->context1 = NULL;
511 }
512 lpfc_sli_release_iocbq(phba, cmdwqe);
513 }
514
515 static void
lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_wcqe_complete * wcqe)516 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
517 struct lpfc_wcqe_complete *wcqe)
518 {
519 struct lpfc_vport *vport = cmdwqe->vport;
520 struct lpfc_nvme_lport *lport;
521 uint32_t status;
522
523 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
524
525 if (vport->localport) {
526 lport = (struct lpfc_nvme_lport *)vport->localport->private;
527 if (lport) {
528 atomic_inc(&lport->fc4NvmeLsCmpls);
529 if (status) {
530 if (bf_get(lpfc_wcqe_c_xb, wcqe))
531 atomic_inc(&lport->cmpl_ls_xb);
532 atomic_inc(&lport->cmpl_ls_err);
533 }
534 }
535 }
536
537 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
538 }
539
540 static int
lpfc_nvme_gen_req(struct lpfc_vport * vport,struct lpfc_dmabuf * bmp,struct lpfc_dmabuf * inp,struct nvmefc_ls_req * pnvme_lsreq,void (* cmpl)(struct lpfc_hba *,struct lpfc_iocbq *,struct lpfc_wcqe_complete *),struct lpfc_nodelist * ndlp,uint32_t num_entry,uint32_t tmo,uint8_t retry)541 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
542 struct lpfc_dmabuf *inp,
543 struct nvmefc_ls_req *pnvme_lsreq,
544 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
545 struct lpfc_wcqe_complete *),
546 struct lpfc_nodelist *ndlp, uint32_t num_entry,
547 uint32_t tmo, uint8_t retry)
548 {
549 struct lpfc_hba *phba = vport->phba;
550 union lpfc_wqe128 *wqe;
551 struct lpfc_iocbq *genwqe;
552 struct ulp_bde64 *bpl;
553 struct ulp_bde64 bde;
554 int i, rc, xmit_len, first_len;
555
556 /* Allocate buffer for command WQE */
557 genwqe = lpfc_sli_get_iocbq(phba);
558 if (genwqe == NULL)
559 return 1;
560
561 wqe = &genwqe->wqe;
562 /* Initialize only 64 bytes */
563 memset(wqe, 0, sizeof(union lpfc_wqe));
564
565 genwqe->context3 = (uint8_t *)bmp;
566 genwqe->iocb_flag |= LPFC_IO_NVME_LS;
567
568 /* Save for completion so we can release these resources */
569 genwqe->context1 = lpfc_nlp_get(ndlp);
570 genwqe->context2 = (uint8_t *)pnvme_lsreq;
571 /* Fill in payload, bp points to frame payload */
572
573 if (!tmo)
574 /* FC spec states we need 3 * ratov for CT requests */
575 tmo = (3 * phba->fc_ratov);
576
577 /* For this command calculate the xmit length of the request bde. */
578 xmit_len = 0;
579 first_len = 0;
580 bpl = (struct ulp_bde64 *)bmp->virt;
581 for (i = 0; i < num_entry; i++) {
582 bde.tus.w = bpl[i].tus.w;
583 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
584 break;
585 xmit_len += bde.tus.f.bdeSize;
586 if (i == 0)
587 first_len = xmit_len;
588 }
589
590 genwqe->rsvd2 = num_entry;
591 genwqe->hba_wqidx = 0;
592
593 /* Words 0 - 2 */
594 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
595 wqe->generic.bde.tus.f.bdeSize = first_len;
596 wqe->generic.bde.addrLow = bpl[0].addrLow;
597 wqe->generic.bde.addrHigh = bpl[0].addrHigh;
598
599 /* Word 3 */
600 wqe->gen_req.request_payload_len = first_len;
601
602 /* Word 4 */
603
604 /* Word 5 */
605 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
606 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
607 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
608 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
609 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
610
611 /* Word 6 */
612 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
613 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
614 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
615
616 /* Word 7 */
617 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1));
618 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
619 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
620 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
621
622 /* Word 8 */
623 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
624
625 /* Word 9 */
626 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
627
628 /* Word 10 */
629 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
630 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
631 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
632 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
633 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
634
635 /* Word 11 */
636 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
637 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
638
639
640 /* Issue GEN REQ WQE for NPORT <did> */
641 genwqe->wqe_cmpl = cmpl;
642 genwqe->iocb_cmpl = NULL;
643 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
644 genwqe->vport = vport;
645 genwqe->retry = retry;
646
647 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
648 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
649
650 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
651 if (rc) {
652 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
653 "6045 Issue GEN REQ WQE to NPORT x%x "
654 "Data: x%x x%x rc x%x\n",
655 ndlp->nlp_DID, genwqe->iotag,
656 vport->port_state, rc);
657 lpfc_sli_release_iocbq(phba, genwqe);
658 return 1;
659 }
660
661 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
662 "6050 Issue GEN REQ WQE to NPORT x%x "
663 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
664 "bmp:x%px xmit:%d 1st:%d\n",
665 ndlp->nlp_DID, genwqe->sli4_xritag,
666 vport->port_state,
667 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
668 return 0;
669 }
670
671
672 /**
673 * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
674 * @vport: The local port issuing the LS
675 * @ndlp: The remote port to send the LS to
676 * @pnvme_lsreq: Pointer to LS request structure from the transport
677 * @gen_req_cmp: Completion call-back
678 *
679 * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
680 * WQE to perform the LS operation.
681 *
682 * Return value :
683 * 0 - Success
684 * non-zero: various error codes, in form of -Exxx
685 **/
686 int
__lpfc_nvme_ls_req(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq,void (* gen_req_cmp)(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_wcqe_complete * wcqe))687 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
688 struct nvmefc_ls_req *pnvme_lsreq,
689 void (*gen_req_cmp)(struct lpfc_hba *phba,
690 struct lpfc_iocbq *cmdwqe,
691 struct lpfc_wcqe_complete *wcqe))
692 {
693 struct lpfc_dmabuf *bmp;
694 struct ulp_bde64 *bpl;
695 int ret;
696 uint16_t ntype, nstate;
697
698 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
699 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
700 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
701 "LS Req\n",
702 ndlp);
703 return -ENODEV;
704 }
705
706 ntype = ndlp->nlp_type;
707 nstate = ndlp->nlp_state;
708 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
709 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
710 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
711 "6088 NVMEx LS REQ: Fail DID x%06x not "
712 "ready for IO. Type x%x, State x%x\n",
713 ndlp->nlp_DID, ntype, nstate);
714 return -ENODEV;
715 }
716
717 /*
718 * there are two dma buf in the request, actually there is one and
719 * the second one is just the start address + cmd size.
720 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
721 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
722 * because the nvem layer owns the data bufs.
723 * We do not have to break these packets open, we don't care what is
724 * in them. And we do not have to look at the resonse data, we only
725 * care that we got a response. All of the caring is going to happen
726 * in the nvme-fc layer.
727 */
728
729 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
730 if (!bmp) {
731 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
732 "6044 NVMEx LS REQ: Could not alloc LS buf "
733 "for DID %x\n",
734 ndlp->nlp_DID);
735 return -ENOMEM;
736 }
737
738 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
739 if (!bmp->virt) {
740 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
741 "6042 NVMEx LS REQ: Could not alloc mbuf "
742 "for DID %x\n",
743 ndlp->nlp_DID);
744 kfree(bmp);
745 return -ENOMEM;
746 }
747
748 INIT_LIST_HEAD(&bmp->list);
749
750 bpl = (struct ulp_bde64 *)bmp->virt;
751 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
752 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
753 bpl->tus.f.bdeFlags = 0;
754 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
755 bpl->tus.w = le32_to_cpu(bpl->tus.w);
756 bpl++;
757
758 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
759 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
760 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
761 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
762 bpl->tus.w = le32_to_cpu(bpl->tus.w);
763
764 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
765 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
766 "rqstlen:%d rsplen:%d %pad %pad\n",
767 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
768 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
769 &pnvme_lsreq->rspdma);
770
771 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
772 pnvme_lsreq, gen_req_cmp, ndlp, 2,
773 LPFC_NVME_LS_TIMEOUT, 0);
774 if (ret != WQE_SUCCESS) {
775 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
776 "6052 NVMEx REQ: EXIT. issue ls wqe failed "
777 "lsreq x%px Status %x DID %x\n",
778 pnvme_lsreq, ret, ndlp->nlp_DID);
779 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
780 kfree(bmp);
781 return -EIO;
782 }
783
784 return 0;
785 }
786
787 /**
788 * lpfc_nvme_ls_req - Issue an NVME Link Service request
789 * @pnvme_lport: Transport localport that LS is to be issued from.
790 * @nvme_rport: Transport remoteport that LS is to be sent to.
791 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
792 *
793 * Driver registers this routine to handle any link service request
794 * from the nvme_fc transport to a remote nvme-aware port.
795 *
796 * Return value :
797 * 0 - Success
798 * non-zero: various error codes, in form of -Exxx
799 **/
800 static int
lpfc_nvme_ls_req(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)801 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
802 struct nvme_fc_remote_port *pnvme_rport,
803 struct nvmefc_ls_req *pnvme_lsreq)
804 {
805 struct lpfc_nvme_lport *lport;
806 struct lpfc_nvme_rport *rport;
807 struct lpfc_vport *vport;
808 int ret;
809
810 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
811 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
812 if (unlikely(!lport) || unlikely(!rport))
813 return -EINVAL;
814
815 vport = lport->vport;
816 if (vport->load_flag & FC_UNLOADING)
817 return -ENODEV;
818
819 atomic_inc(&lport->fc4NvmeLsRequests);
820
821 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
822 lpfc_nvme_ls_req_cmp);
823 if (ret)
824 atomic_inc(&lport->xmt_ls_err);
825
826 return ret;
827 }
828
829 /**
830 * __lpfc_nvme_ls_abort - Generic service routine to abort a prior
831 * NVME LS request
832 * @vport: The local port that issued the LS
833 * @ndlp: The remote port the LS was sent to
834 * @pnvme_lsreq: Pointer to LS request structure from the transport
835 *
836 * The driver validates the ndlp, looks for the LS, and aborts the
837 * LS if found.
838 *
839 * Returns:
840 * 0 : if LS found and aborted
841 * non-zero: various error conditions in form -Exxx
842 **/
843 int
__lpfc_nvme_ls_abort(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq)844 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
845 struct nvmefc_ls_req *pnvme_lsreq)
846 {
847 struct lpfc_hba *phba = vport->phba;
848 struct lpfc_sli_ring *pring;
849 struct lpfc_iocbq *wqe, *next_wqe;
850 bool foundit = false;
851
852 if (!ndlp) {
853 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
854 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
855 "x%06x, Failing LS Req\n",
856 ndlp, ndlp ? ndlp->nlp_DID : 0);
857 return -EINVAL;
858 }
859
860 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
861 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
862 "x%p rqstlen:%d rsplen:%d %pad %pad\n",
863 pnvme_lsreq, pnvme_lsreq->rqstlen,
864 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
865 &pnvme_lsreq->rspdma);
866
867 /*
868 * Lock the ELS ring txcmplq and look for the wqe that matches
869 * this ELS. If found, issue an abort on the wqe.
870 */
871 pring = phba->sli4_hba.nvmels_wq->pring;
872 spin_lock_irq(&phba->hbalock);
873 spin_lock(&pring->ring_lock);
874 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
875 if (wqe->context2 == pnvme_lsreq) {
876 wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
877 foundit = true;
878 break;
879 }
880 }
881 spin_unlock(&pring->ring_lock);
882
883 if (foundit)
884 lpfc_sli_issue_abort_iotag(phba, pring, wqe);
885 spin_unlock_irq(&phba->hbalock);
886
887 if (foundit)
888 return 0;
889
890 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
891 "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n",
892 pnvme_lsreq);
893 return -EINVAL;
894 }
895
896 static int
lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port * localport,struct nvme_fc_remote_port * remoteport,struct nvmefc_ls_rsp * ls_rsp)897 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
898 struct nvme_fc_remote_port *remoteport,
899 struct nvmefc_ls_rsp *ls_rsp)
900 {
901 struct lpfc_async_xchg_ctx *axchg =
902 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
903 struct lpfc_nvme_lport *lport;
904 int rc;
905
906 if (axchg->phba->pport->load_flag & FC_UNLOADING)
907 return -ENODEV;
908
909 lport = (struct lpfc_nvme_lport *)localport->private;
910
911 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
912
913 if (rc) {
914 /*
915 * unless the failure is due to having already sent
916 * the response, an abort will be generated for the
917 * exchange if the rsp can't be sent.
918 */
919 if (rc != -EALREADY)
920 atomic_inc(&lport->xmt_ls_abort);
921 return rc;
922 }
923
924 return 0;
925 }
926
927 /**
928 * lpfc_nvme_ls_abort - Abort a prior NVME LS request
929 * @pnvme_lport: Transport localport that LS is to be issued from.
930 * @pnvme_rport: Transport remoteport that LS is to be sent to.
931 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
932 *
933 * Driver registers this routine to abort a NVME LS request that is
934 * in progress (from the transports perspective).
935 **/
936 static void
lpfc_nvme_ls_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)937 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
938 struct nvme_fc_remote_port *pnvme_rport,
939 struct nvmefc_ls_req *pnvme_lsreq)
940 {
941 struct lpfc_nvme_lport *lport;
942 struct lpfc_vport *vport;
943 struct lpfc_hba *phba;
944 struct lpfc_nodelist *ndlp;
945 int ret;
946
947 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
948 if (unlikely(!lport))
949 return;
950 vport = lport->vport;
951 phba = vport->phba;
952
953 if (vport->load_flag & FC_UNLOADING)
954 return;
955
956 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
957
958 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
959 if (!ret)
960 atomic_inc(&lport->xmt_ls_abort);
961 }
962
963 /* Fix up the existing sgls for NVME IO. */
964 static inline void
lpfc_nvme_adj_fcp_sgls(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct nvmefc_fcp_req * nCmd)965 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
966 struct lpfc_io_buf *lpfc_ncmd,
967 struct nvmefc_fcp_req *nCmd)
968 {
969 struct lpfc_hba *phba = vport->phba;
970 struct sli4_sge *sgl;
971 union lpfc_wqe128 *wqe;
972 uint32_t *wptr, *dptr;
973
974 /*
975 * Get a local pointer to the built-in wqe and correct
976 * the cmd size to match NVME's 96 bytes and fix
977 * the dma address.
978 */
979
980 wqe = &lpfc_ncmd->cur_iocbq.wqe;
981
982 /*
983 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
984 * match NVME. NVME sends 96 bytes. Also, use the
985 * nvme commands command and response dma addresses
986 * rather than the virtual memory to ease the restore
987 * operation.
988 */
989 sgl = lpfc_ncmd->dma_sgl;
990 sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
991 if (phba->cfg_nvme_embed_cmd) {
992 sgl->addr_hi = 0;
993 sgl->addr_lo = 0;
994
995 /* Word 0-2 - NVME CMND IU (embedded payload) */
996 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
997 wqe->generic.bde.tus.f.bdeSize = 56;
998 wqe->generic.bde.addrHigh = 0;
999 wqe->generic.bde.addrLow = 64; /* Word 16 */
1000
1001 /* Word 10 - dbde is 0, wqes is 1 in template */
1002
1003 /*
1004 * Embed the payload in the last half of the WQE
1005 * WQE words 16-30 get the NVME CMD IU payload
1006 *
1007 * WQE words 16-19 get payload Words 1-4
1008 * WQE words 20-21 get payload Words 6-7
1009 * WQE words 22-29 get payload Words 16-23
1010 */
1011 wptr = &wqe->words[16]; /* WQE ptr */
1012 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */
1013 dptr++; /* Skip Word 0 in payload */
1014
1015 *wptr++ = *dptr++; /* Word 1 */
1016 *wptr++ = *dptr++; /* Word 2 */
1017 *wptr++ = *dptr++; /* Word 3 */
1018 *wptr++ = *dptr++; /* Word 4 */
1019 dptr++; /* Skip Word 5 in payload */
1020 *wptr++ = *dptr++; /* Word 6 */
1021 *wptr++ = *dptr++; /* Word 7 */
1022 dptr += 8; /* Skip Words 8-15 in payload */
1023 *wptr++ = *dptr++; /* Word 16 */
1024 *wptr++ = *dptr++; /* Word 17 */
1025 *wptr++ = *dptr++; /* Word 18 */
1026 *wptr++ = *dptr++; /* Word 19 */
1027 *wptr++ = *dptr++; /* Word 20 */
1028 *wptr++ = *dptr++; /* Word 21 */
1029 *wptr++ = *dptr++; /* Word 22 */
1030 *wptr = *dptr; /* Word 23 */
1031 } else {
1032 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
1033 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
1034
1035 /* Word 0-2 - NVME CMND IU Inline BDE */
1036 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1037 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
1038 wqe->generic.bde.addrHigh = sgl->addr_hi;
1039 wqe->generic.bde.addrLow = sgl->addr_lo;
1040
1041 /* Word 10 */
1042 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
1043 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
1044 }
1045
1046 sgl++;
1047
1048 /* Setup the physical region for the FCP RSP */
1049 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
1050 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
1051 sgl->word2 = le32_to_cpu(sgl->word2);
1052 if (nCmd->sg_cnt)
1053 bf_set(lpfc_sli4_sge_last, sgl, 0);
1054 else
1055 bf_set(lpfc_sli4_sge_last, sgl, 1);
1056 sgl->word2 = cpu_to_le32(sgl->word2);
1057 sgl->sge_len = cpu_to_le32(nCmd->rsplen);
1058 }
1059
1060
1061 /*
1062 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
1063 *
1064 * Driver registers this routine as it io request handler. This
1065 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1066 * data structure to the rport indicated in @lpfc_nvme_rport.
1067 *
1068 * Return value :
1069 * 0 - Success
1070 * TODO: What are the failure codes.
1071 **/
1072 static void
lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,struct lpfc_wcqe_complete * wcqe)1073 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
1074 struct lpfc_wcqe_complete *wcqe)
1075 {
1076 struct lpfc_io_buf *lpfc_ncmd =
1077 (struct lpfc_io_buf *)pwqeIn->context1;
1078 struct lpfc_vport *vport = pwqeIn->vport;
1079 struct nvmefc_fcp_req *nCmd;
1080 struct nvme_fc_ersp_iu *ep;
1081 struct nvme_fc_cmd_iu *cp;
1082 struct lpfc_nodelist *ndlp;
1083 struct lpfc_nvme_fcpreq_priv *freqpriv;
1084 struct lpfc_nvme_lport *lport;
1085 uint32_t code, status, idx;
1086 uint16_t cid, sqhd, data;
1087 uint32_t *ptr;
1088 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1089 int cpu;
1090 #endif
1091
1092 /* Sanity check on return of outstanding command */
1093 if (!lpfc_ncmd) {
1094 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1095 "6071 Null lpfc_ncmd pointer. No "
1096 "release, skip completion\n");
1097 return;
1098 }
1099
1100 /* Guard against abort handler being called at same time */
1101 spin_lock(&lpfc_ncmd->buf_lock);
1102
1103 if (!lpfc_ncmd->nvmeCmd) {
1104 spin_unlock(&lpfc_ncmd->buf_lock);
1105 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1106 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
1107 "nvmeCmd x%px\n",
1108 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
1109
1110 /* Release the lpfc_ncmd regardless of the missing elements. */
1111 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1112 return;
1113 }
1114 nCmd = lpfc_ncmd->nvmeCmd;
1115 status = bf_get(lpfc_wcqe_c_status, wcqe);
1116
1117 idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
1118 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
1119
1120 if (unlikely(status && vport->localport)) {
1121 lport = (struct lpfc_nvme_lport *)vport->localport->private;
1122 if (lport) {
1123 if (bf_get(lpfc_wcqe_c_xb, wcqe))
1124 atomic_inc(&lport->cmpl_fcp_xb);
1125 atomic_inc(&lport->cmpl_fcp_err);
1126 }
1127 }
1128
1129 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
1130 lpfc_ncmd->cur_iocbq.sli4_xritag,
1131 status, wcqe->parameter);
1132 /*
1133 * Catch race where our node has transitioned, but the
1134 * transport is still transitioning.
1135 */
1136 ndlp = lpfc_ncmd->ndlp;
1137 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1138 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1139 "6062 Ignoring NVME cmpl. No ndlp\n");
1140 goto out_err;
1141 }
1142
1143 code = bf_get(lpfc_wcqe_c_code, wcqe);
1144 if (code == CQE_CODE_NVME_ERSP) {
1145 /* For this type of CQE, we need to rebuild the rsp */
1146 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1147
1148 /*
1149 * Get Command Id from cmd to plug into response. This
1150 * code is not needed in the next NVME Transport drop.
1151 */
1152 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1153 cid = cp->sqe.common.command_id;
1154
1155 /*
1156 * RSN is in CQE word 2
1157 * SQHD is in CQE Word 3 bits 15:0
1158 * Cmd Specific info is in CQE Word 1
1159 * and in CQE Word 0 bits 15:0
1160 */
1161 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1162
1163 /* Now lets build the NVME ERSP IU */
1164 ep->iu_len = cpu_to_be16(8);
1165 ep->rsn = wcqe->parameter;
1166 ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1167 ep->rsvd12 = 0;
1168 ptr = (uint32_t *)&ep->cqe.result.u64;
1169 *ptr++ = wcqe->total_data_placed;
1170 data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1171 *ptr = (uint32_t)data;
1172 ep->cqe.sq_head = sqhd;
1173 ep->cqe.sq_id = nCmd->sqid;
1174 ep->cqe.command_id = cid;
1175 ep->cqe.status = 0;
1176
1177 lpfc_ncmd->status = IOSTAT_SUCCESS;
1178 lpfc_ncmd->result = 0;
1179 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1180 nCmd->transferred_length = nCmd->payload_length;
1181 } else {
1182 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);
1183 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1184
1185 /* For NVME, the only failure path that results in an
1186 * IO error is when the adapter rejects it. All other
1187 * conditions are a success case and resolved by the
1188 * transport.
1189 * IOSTAT_FCP_RSP_ERROR means:
1190 * 1. Length of data received doesn't match total
1191 * transfer length in WQE
1192 * 2. If the RSP payload does NOT match these cases:
1193 * a. RSP length 12/24 bytes and all zeros
1194 * b. NVME ERSP
1195 */
1196 switch (lpfc_ncmd->status) {
1197 case IOSTAT_SUCCESS:
1198 nCmd->transferred_length = wcqe->total_data_placed;
1199 nCmd->rcv_rsplen = 0;
1200 nCmd->status = 0;
1201 break;
1202 case IOSTAT_FCP_RSP_ERROR:
1203 nCmd->transferred_length = wcqe->total_data_placed;
1204 nCmd->rcv_rsplen = wcqe->parameter;
1205 nCmd->status = 0;
1206 /* Sanity check */
1207 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN)
1208 break;
1209 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1210 "6081 NVME Completion Protocol Error: "
1211 "xri %x status x%x result x%x "
1212 "placed x%x\n",
1213 lpfc_ncmd->cur_iocbq.sli4_xritag,
1214 lpfc_ncmd->status, lpfc_ncmd->result,
1215 wcqe->total_data_placed);
1216 break;
1217 case IOSTAT_LOCAL_REJECT:
1218 /* Let fall through to set command final state. */
1219 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1220 lpfc_printf_vlog(vport, KERN_INFO,
1221 LOG_NVME_IOERR,
1222 "6032 Delay Aborted cmd x%px "
1223 "nvme cmd x%px, xri x%x, "
1224 "xb %d\n",
1225 lpfc_ncmd, nCmd,
1226 lpfc_ncmd->cur_iocbq.sli4_xritag,
1227 bf_get(lpfc_wcqe_c_xb, wcqe));
1228 fallthrough;
1229 default:
1230 out_err:
1231 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1232 "6072 NVME Completion Error: xri %x "
1233 "status x%x result x%x [x%x] "
1234 "placed x%x\n",
1235 lpfc_ncmd->cur_iocbq.sli4_xritag,
1236 lpfc_ncmd->status, lpfc_ncmd->result,
1237 wcqe->parameter,
1238 wcqe->total_data_placed);
1239 nCmd->transferred_length = 0;
1240 nCmd->rcv_rsplen = 0;
1241 nCmd->status = NVME_SC_INTERNAL;
1242 }
1243 }
1244
1245 /* pick up SLI4 exhange busy condition */
1246 if (bf_get(lpfc_wcqe_c_xb, wcqe))
1247 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1248 else
1249 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1250
1251 /* Update stats and complete the IO. There is
1252 * no need for dma unprep because the nvme_transport
1253 * owns the dma address.
1254 */
1255 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1256 if (lpfc_ncmd->ts_cmd_start) {
1257 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1258 lpfc_ncmd->ts_data_io = ktime_get_ns();
1259 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1260 lpfc_io_ktime(phba, lpfc_ncmd);
1261 }
1262 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1263 cpu = raw_smp_processor_id();
1264 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1265 if (lpfc_ncmd->cpu != cpu)
1266 lpfc_printf_vlog(vport,
1267 KERN_INFO, LOG_NVME_IOERR,
1268 "6701 CPU Check cmpl: "
1269 "cpu %d expect %d\n",
1270 cpu, lpfc_ncmd->cpu);
1271 }
1272 #endif
1273
1274 /* NVME targets need completion held off until the abort exchange
1275 * completes unless the NVME Rport is getting unregistered.
1276 */
1277
1278 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1279 freqpriv = nCmd->private;
1280 freqpriv->nvme_buf = NULL;
1281 lpfc_ncmd->nvmeCmd = NULL;
1282 spin_unlock(&lpfc_ncmd->buf_lock);
1283 nCmd->done(nCmd);
1284 } else
1285 spin_unlock(&lpfc_ncmd->buf_lock);
1286
1287 /* Call release with XB=1 to queue the IO into the abort list. */
1288 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1289 }
1290
1291
1292 /**
1293 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1294 * @vport: pointer to a host virtual N_Port data structure
1295 * @lpfcn_cmd: Pointer to lpfc scsi command
1296 * @pnode: pointer to a node-list data structure
1297 * @cstat: pointer to the control status structure
1298 *
1299 * Driver registers this routine as it io request handler. This
1300 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1301 * data structure to the rport indicated in @lpfc_nvme_rport.
1302 *
1303 * Return value :
1304 * 0 - Success
1305 * TODO: What are the failure codes.
1306 **/
1307 static int
lpfc_nvme_prep_io_cmd(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct lpfc_nodelist * pnode,struct lpfc_fc4_ctrl_stat * cstat)1308 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1309 struct lpfc_io_buf *lpfc_ncmd,
1310 struct lpfc_nodelist *pnode,
1311 struct lpfc_fc4_ctrl_stat *cstat)
1312 {
1313 struct lpfc_hba *phba = vport->phba;
1314 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1315 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);
1316 union lpfc_wqe128 *wqe = &pwqeq->wqe;
1317 uint32_t req_len;
1318
1319 if (!NLP_CHK_NODE_ACT(pnode))
1320 return -EINVAL;
1321
1322 /*
1323 * There are three possibilities here - use scatter-gather segment, use
1324 * the single mapping, or neither.
1325 */
1326 if (nCmd->sg_cnt) {
1327 if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1328 /* From the iwrite template, initialize words 7 - 11 */
1329 memcpy(&wqe->words[7],
1330 &lpfc_iwrite_cmd_template.words[7],
1331 sizeof(uint32_t) * 5);
1332
1333 /* Word 4 */
1334 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1335
1336 /* Word 5 */
1337 if ((phba->cfg_nvme_enable_fb) &&
1338 (pnode->nlp_flag & NLP_FIRSTBURST)) {
1339 req_len = lpfc_ncmd->nvmeCmd->payload_length;
1340 if (req_len < pnode->nvme_fb_size)
1341 wqe->fcp_iwrite.initial_xfer_len =
1342 req_len;
1343 else
1344 wqe->fcp_iwrite.initial_xfer_len =
1345 pnode->nvme_fb_size;
1346 } else {
1347 wqe->fcp_iwrite.initial_xfer_len = 0;
1348 }
1349 cstat->output_requests++;
1350 } else {
1351 /* From the iread template, initialize words 7 - 11 */
1352 memcpy(&wqe->words[7],
1353 &lpfc_iread_cmd_template.words[7],
1354 sizeof(uint32_t) * 5);
1355
1356 /* Word 4 */
1357 wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1358
1359 /* Word 5 */
1360 wqe->fcp_iread.rsrvd5 = 0;
1361
1362 cstat->input_requests++;
1363 }
1364 } else {
1365 /* From the icmnd template, initialize words 4 - 11 */
1366 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1367 sizeof(uint32_t) * 8);
1368 cstat->control_requests++;
1369 }
1370
1371 if (pnode->nlp_nvme_info & NLP_NVME_NSLER)
1372 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1373 /*
1374 * Finish initializing those WQE fields that are independent
1375 * of the nvme_cmnd request_buffer
1376 */
1377
1378 /* Word 3 */
1379 bf_set(payload_offset_len, &wqe->fcp_icmd,
1380 (nCmd->rsplen + nCmd->cmdlen));
1381
1382 /* Word 6 */
1383 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1384 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1385 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1386
1387 /* Word 8 */
1388 wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1389
1390 /* Word 9 */
1391 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1392
1393 /* Words 13 14 15 are for PBDE support */
1394
1395 pwqeq->vport = vport;
1396 return 0;
1397 }
1398
1399
1400 /**
1401 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1402 * @vport: pointer to a host virtual N_Port data structure
1403 * @lpfcn_cmd: Pointer to lpfc scsi command
1404 *
1405 * Driver registers this routine as it io request handler. This
1406 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1407 * data structure to the rport indicated in @lpfc_nvme_rport.
1408 *
1409 * Return value :
1410 * 0 - Success
1411 * TODO: What are the failure codes.
1412 **/
1413 static int
lpfc_nvme_prep_io_dma(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd)1414 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1415 struct lpfc_io_buf *lpfc_ncmd)
1416 {
1417 struct lpfc_hba *phba = vport->phba;
1418 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1419 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1420 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1421 struct sli4_hybrid_sgl *sgl_xtra = NULL;
1422 struct scatterlist *data_sg;
1423 struct sli4_sge *first_data_sgl;
1424 struct ulp_bde64 *bde;
1425 dma_addr_t physaddr = 0;
1426 uint32_t num_bde = 0;
1427 uint32_t dma_len = 0;
1428 uint32_t dma_offset = 0;
1429 int nseg, i, j;
1430 bool lsp_just_set = false;
1431
1432 /* Fix up the command and response DMA stuff. */
1433 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1434
1435 /*
1436 * There are three possibilities here - use scatter-gather segment, use
1437 * the single mapping, or neither.
1438 */
1439 if (nCmd->sg_cnt) {
1440 /*
1441 * Jump over the cmd and rsp SGEs. The fix routine
1442 * has already adjusted for this.
1443 */
1444 sgl += 2;
1445
1446 first_data_sgl = sgl;
1447 lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1448 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1449 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1450 "6058 Too many sg segments from "
1451 "NVME Transport. Max %d, "
1452 "nvmeIO sg_cnt %d\n",
1453 phba->cfg_nvme_seg_cnt + 1,
1454 lpfc_ncmd->seg_cnt);
1455 lpfc_ncmd->seg_cnt = 0;
1456 return 1;
1457 }
1458
1459 /*
1460 * The driver established a maximum scatter-gather segment count
1461 * during probe that limits the number of sg elements in any
1462 * single nvme command. Just run through the seg_cnt and format
1463 * the sge's.
1464 */
1465 nseg = nCmd->sg_cnt;
1466 data_sg = nCmd->first_sgl;
1467
1468 /* for tracking the segment boundaries */
1469 j = 2;
1470 for (i = 0; i < nseg; i++) {
1471 if (data_sg == NULL) {
1472 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1473 "6059 dptr err %d, nseg %d\n",
1474 i, nseg);
1475 lpfc_ncmd->seg_cnt = 0;
1476 return 1;
1477 }
1478
1479 sgl->word2 = 0;
1480 if ((num_bde + 1) == nseg) {
1481 bf_set(lpfc_sli4_sge_last, sgl, 1);
1482 bf_set(lpfc_sli4_sge_type, sgl,
1483 LPFC_SGE_TYPE_DATA);
1484 } else {
1485 bf_set(lpfc_sli4_sge_last, sgl, 0);
1486
1487 /* expand the segment */
1488 if (!lsp_just_set &&
1489 !((j + 1) % phba->border_sge_num) &&
1490 ((nseg - 1) != i)) {
1491 /* set LSP type */
1492 bf_set(lpfc_sli4_sge_type, sgl,
1493 LPFC_SGE_TYPE_LSP);
1494
1495 sgl_xtra = lpfc_get_sgl_per_hdwq(
1496 phba, lpfc_ncmd);
1497
1498 if (unlikely(!sgl_xtra)) {
1499 lpfc_ncmd->seg_cnt = 0;
1500 return 1;
1501 }
1502 sgl->addr_lo = cpu_to_le32(putPaddrLow(
1503 sgl_xtra->dma_phys_sgl));
1504 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1505 sgl_xtra->dma_phys_sgl));
1506
1507 } else {
1508 bf_set(lpfc_sli4_sge_type, sgl,
1509 LPFC_SGE_TYPE_DATA);
1510 }
1511 }
1512
1513 if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1514 LPFC_SGE_TYPE_LSP)) {
1515 if ((nseg - 1) == i)
1516 bf_set(lpfc_sli4_sge_last, sgl, 1);
1517
1518 physaddr = data_sg->dma_address;
1519 dma_len = data_sg->length;
1520 sgl->addr_lo = cpu_to_le32(
1521 putPaddrLow(physaddr));
1522 sgl->addr_hi = cpu_to_le32(
1523 putPaddrHigh(physaddr));
1524
1525 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1526 sgl->word2 = cpu_to_le32(sgl->word2);
1527 sgl->sge_len = cpu_to_le32(dma_len);
1528
1529 dma_offset += dma_len;
1530 data_sg = sg_next(data_sg);
1531
1532 sgl++;
1533
1534 lsp_just_set = false;
1535 } else {
1536 sgl->word2 = cpu_to_le32(sgl->word2);
1537
1538 sgl->sge_len = cpu_to_le32(
1539 phba->cfg_sg_dma_buf_size);
1540
1541 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1542 i = i - 1;
1543
1544 lsp_just_set = true;
1545 }
1546
1547 j++;
1548 }
1549 if (phba->cfg_enable_pbde) {
1550 /* Use PBDE support for first SGL only, offset == 0 */
1551 /* Words 13-15 */
1552 bde = (struct ulp_bde64 *)
1553 &wqe->words[13];
1554 bde->addrLow = first_data_sgl->addr_lo;
1555 bde->addrHigh = first_data_sgl->addr_hi;
1556 bde->tus.f.bdeSize =
1557 le32_to_cpu(first_data_sgl->sge_len);
1558 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1559 bde->tus.w = cpu_to_le32(bde->tus.w);
1560 /* wqe_pbde is 1 in template */
1561 } else {
1562 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1563 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
1564 }
1565
1566 } else {
1567 lpfc_ncmd->seg_cnt = 0;
1568
1569 /* For this clause to be valid, the payload_length
1570 * and sg_cnt must zero.
1571 */
1572 if (nCmd->payload_length != 0) {
1573 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1574 "6063 NVME DMA Prep Err: sg_cnt %d "
1575 "payload_length x%x\n",
1576 nCmd->sg_cnt, nCmd->payload_length);
1577 return 1;
1578 }
1579 }
1580 return 0;
1581 }
1582
1583 /**
1584 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1585 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1586 * @lpfc_nvme_lport: Pointer to the driver's local port data
1587 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1588 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1589 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1590 *
1591 * Driver registers this routine as it io request handler. This
1592 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1593 * data structure to the rport
1594 indicated in @lpfc_nvme_rport.
1595 *
1596 * Return value :
1597 * 0 - Success
1598 * TODO: What are the failure codes.
1599 **/
1600 static int
lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1601 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1602 struct nvme_fc_remote_port *pnvme_rport,
1603 void *hw_queue_handle,
1604 struct nvmefc_fcp_req *pnvme_fcreq)
1605 {
1606 int ret = 0;
1607 int expedite = 0;
1608 int idx, cpu;
1609 struct lpfc_nvme_lport *lport;
1610 struct lpfc_fc4_ctrl_stat *cstat;
1611 struct lpfc_vport *vport;
1612 struct lpfc_hba *phba;
1613 struct lpfc_nodelist *ndlp;
1614 struct lpfc_io_buf *lpfc_ncmd;
1615 struct lpfc_nvme_rport *rport;
1616 struct lpfc_nvme_qhandle *lpfc_queue_info;
1617 struct lpfc_nvme_fcpreq_priv *freqpriv;
1618 struct nvme_common_command *sqe;
1619 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1620 uint64_t start = 0;
1621 #endif
1622
1623 /* Validate pointers. LLDD fault handling with transport does
1624 * have timing races.
1625 */
1626 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1627 if (unlikely(!lport)) {
1628 ret = -EINVAL;
1629 goto out_fail;
1630 }
1631
1632 vport = lport->vport;
1633
1634 if (unlikely(!hw_queue_handle)) {
1635 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1636 "6117 Fail IO, NULL hw_queue_handle\n");
1637 atomic_inc(&lport->xmt_fcp_err);
1638 ret = -EBUSY;
1639 goto out_fail;
1640 }
1641
1642 phba = vport->phba;
1643
1644 if (unlikely(vport->load_flag & FC_UNLOADING)) {
1645 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1646 "6124 Fail IO, Driver unload\n");
1647 atomic_inc(&lport->xmt_fcp_err);
1648 ret = -ENODEV;
1649 goto out_fail;
1650 }
1651
1652 freqpriv = pnvme_fcreq->private;
1653 if (unlikely(!freqpriv)) {
1654 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1655 "6158 Fail IO, NULL request data\n");
1656 atomic_inc(&lport->xmt_fcp_err);
1657 ret = -EINVAL;
1658 goto out_fail;
1659 }
1660
1661 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1662 if (phba->ktime_on)
1663 start = ktime_get_ns();
1664 #endif
1665 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1666 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1667
1668 /*
1669 * Catch race where our node has transitioned, but the
1670 * transport is still transitioning.
1671 */
1672 ndlp = rport->ndlp;
1673 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1674 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1675 "6053 Busy IO, ndlp not ready: rport x%px "
1676 "ndlp x%px, DID x%06x\n",
1677 rport, ndlp, pnvme_rport->port_id);
1678 atomic_inc(&lport->xmt_fcp_err);
1679 ret = -EBUSY;
1680 goto out_fail;
1681 }
1682
1683 /* The remote node has to be a mapped target or it's an error. */
1684 if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1685 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1686 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1687 "6036 Fail IO, DID x%06x not ready for "
1688 "IO. State x%x, Type x%x Flg x%x\n",
1689 pnvme_rport->port_id,
1690 ndlp->nlp_state, ndlp->nlp_type,
1691 ndlp->upcall_flags);
1692 atomic_inc(&lport->xmt_fcp_bad_ndlp);
1693 ret = -EBUSY;
1694 goto out_fail;
1695
1696 }
1697
1698 /* Currently only NVME Keep alive commands should be expedited
1699 * if the driver runs out of a resource. These should only be
1700 * issued on the admin queue, qidx 0
1701 */
1702 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1703 sqe = &((struct nvme_fc_cmd_iu *)
1704 pnvme_fcreq->cmdaddr)->sqe.common;
1705 if (sqe->opcode == nvme_admin_keep_alive)
1706 expedite = 1;
1707 }
1708
1709 /* The node is shared with FCP IO, make sure the IO pending count does
1710 * not exceed the programmed depth.
1711 */
1712 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1713 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1714 !expedite) {
1715 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1716 "6174 Fail IO, ndlp qdepth exceeded: "
1717 "idx %d DID %x pend %d qdepth %d\n",
1718 lpfc_queue_info->index, ndlp->nlp_DID,
1719 atomic_read(&ndlp->cmd_pending),
1720 ndlp->cmd_qdepth);
1721 atomic_inc(&lport->xmt_fcp_qdepth);
1722 ret = -EBUSY;
1723 goto out_fail;
1724 }
1725 }
1726
1727 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1728 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1729 idx = lpfc_queue_info->index;
1730 } else {
1731 cpu = raw_smp_processor_id();
1732 idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1733 }
1734
1735 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1736 if (lpfc_ncmd == NULL) {
1737 atomic_inc(&lport->xmt_fcp_noxri);
1738 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1739 "6065 Fail IO, driver buffer pool is empty: "
1740 "idx %d DID %x\n",
1741 lpfc_queue_info->index, ndlp->nlp_DID);
1742 ret = -EBUSY;
1743 goto out_fail;
1744 }
1745 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1746 if (start) {
1747 lpfc_ncmd->ts_cmd_start = start;
1748 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1749 } else {
1750 lpfc_ncmd->ts_cmd_start = 0;
1751 }
1752 #endif
1753
1754 /*
1755 * Store the data needed by the driver to issue, abort, and complete
1756 * an IO.
1757 * Do not let the IO hang out forever. There is no midlayer issuing
1758 * an abort so inform the FW of the maximum IO pending time.
1759 */
1760 freqpriv->nvme_buf = lpfc_ncmd;
1761 lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1762 lpfc_ncmd->ndlp = ndlp;
1763 lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1764
1765 /*
1766 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1767 * This identfier was create in our hardware queue create callback
1768 * routine. The driver now is dependent on the IO queue steering from
1769 * the transport. We are trusting the upper NVME layers know which
1770 * index to use and that they have affinitized a CPU to this hardware
1771 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1772 */
1773 lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1774 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1775
1776 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1777 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1778 if (ret) {
1779 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1780 "6175 Fail IO, Prep DMA: "
1781 "idx %d DID %x\n",
1782 lpfc_queue_info->index, ndlp->nlp_DID);
1783 atomic_inc(&lport->xmt_fcp_err);
1784 ret = -ENOMEM;
1785 goto out_free_nvme_buf;
1786 }
1787
1788 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1789 lpfc_ncmd->cur_iocbq.sli4_xritag,
1790 lpfc_queue_info->index, ndlp->nlp_DID);
1791
1792 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1793 if (ret) {
1794 atomic_inc(&lport->xmt_fcp_wqerr);
1795 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1796 "6113 Fail IO, Could not issue WQE err %x "
1797 "sid: x%x did: x%x oxid: x%x\n",
1798 ret, vport->fc_myDID, ndlp->nlp_DID,
1799 lpfc_ncmd->cur_iocbq.sli4_xritag);
1800 goto out_free_nvme_buf;
1801 }
1802
1803 if (phba->cfg_xri_rebalancing)
1804 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1805
1806 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1807 if (lpfc_ncmd->ts_cmd_start)
1808 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1809
1810 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1811 cpu = raw_smp_processor_id();
1812 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1813 lpfc_ncmd->cpu = cpu;
1814 if (idx != cpu)
1815 lpfc_printf_vlog(vport,
1816 KERN_INFO, LOG_NVME_IOERR,
1817 "6702 CPU Check cmd: "
1818 "cpu %d wq %d\n",
1819 lpfc_ncmd->cpu,
1820 lpfc_queue_info->index);
1821 }
1822 #endif
1823 return 0;
1824
1825 out_free_nvme_buf:
1826 if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1827 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1828 cstat->output_requests--;
1829 else
1830 cstat->input_requests--;
1831 } else
1832 cstat->control_requests--;
1833 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1834 out_fail:
1835 return ret;
1836 }
1837
1838 /**
1839 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1840 * @phba: Pointer to HBA context object
1841 * @cmdiocb: Pointer to command iocb object.
1842 * @rspiocb: Pointer to response iocb object.
1843 *
1844 * This is the callback function for any NVME FCP IO that was aborted.
1845 *
1846 * Return value:
1847 * None
1848 **/
1849 void
lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * cmdiocb,struct lpfc_wcqe_complete * abts_cmpl)1850 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1851 struct lpfc_wcqe_complete *abts_cmpl)
1852 {
1853 lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1854 "6145 ABORT_XRI_CN completing on rpi x%x "
1855 "original iotag x%x, abort cmd iotag x%x "
1856 "req_tag x%x, status x%x, hwstatus x%x\n",
1857 cmdiocb->iocb.un.acxri.abortContextTag,
1858 cmdiocb->iocb.un.acxri.abortIoTag,
1859 cmdiocb->iotag,
1860 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1861 bf_get(lpfc_wcqe_c_status, abts_cmpl),
1862 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1863 lpfc_sli_release_iocbq(phba, cmdiocb);
1864 }
1865
1866 /**
1867 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1868 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1869 * @lpfc_nvme_lport: Pointer to the driver's local port data
1870 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1871 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1872 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1873 *
1874 * Driver registers this routine as its nvme request io abort handler. This
1875 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1876 * data structure to the rport indicated in @lpfc_nvme_rport. This routine
1877 * is executed asynchronously - one the target is validated as "MAPPED" and
1878 * ready for IO, the driver issues the abort request and returns.
1879 *
1880 * Return value:
1881 * None
1882 **/
1883 static void
lpfc_nvme_fcp_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1884 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1885 struct nvme_fc_remote_port *pnvme_rport,
1886 void *hw_queue_handle,
1887 struct nvmefc_fcp_req *pnvme_fcreq)
1888 {
1889 struct lpfc_nvme_lport *lport;
1890 struct lpfc_vport *vport;
1891 struct lpfc_hba *phba;
1892 struct lpfc_io_buf *lpfc_nbuf;
1893 struct lpfc_iocbq *abts_buf;
1894 struct lpfc_iocbq *nvmereq_wqe;
1895 struct lpfc_nvme_fcpreq_priv *freqpriv;
1896 unsigned long flags;
1897 int ret_val;
1898
1899 /* Validate pointers. LLDD fault handling with transport does
1900 * have timing races.
1901 */
1902 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1903 if (unlikely(!lport))
1904 return;
1905
1906 vport = lport->vport;
1907
1908 if (unlikely(!hw_queue_handle)) {
1909 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1910 "6129 Fail Abort, HW Queue Handle NULL.\n");
1911 return;
1912 }
1913
1914 phba = vport->phba;
1915 freqpriv = pnvme_fcreq->private;
1916
1917 if (unlikely(!freqpriv))
1918 return;
1919 if (vport->load_flag & FC_UNLOADING)
1920 return;
1921
1922 /* Announce entry to new IO submit field. */
1923 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1924 "6002 Abort Request to rport DID x%06x "
1925 "for nvme_fc_req x%px\n",
1926 pnvme_rport->port_id,
1927 pnvme_fcreq);
1928
1929 /* If the hba is getting reset, this flag is set. It is
1930 * cleared when the reset is complete and rings reestablished.
1931 */
1932 spin_lock_irqsave(&phba->hbalock, flags);
1933 /* driver queued commands are in process of being flushed */
1934 if (phba->hba_flag & HBA_IOQ_FLUSH) {
1935 spin_unlock_irqrestore(&phba->hbalock, flags);
1936 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1937 "6139 Driver in reset cleanup - flushing "
1938 "NVME Req now. hba_flag x%x\n",
1939 phba->hba_flag);
1940 return;
1941 }
1942
1943 lpfc_nbuf = freqpriv->nvme_buf;
1944 if (!lpfc_nbuf) {
1945 spin_unlock_irqrestore(&phba->hbalock, flags);
1946 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1947 "6140 NVME IO req has no matching lpfc nvme "
1948 "io buffer. Skipping abort req.\n");
1949 return;
1950 } else if (!lpfc_nbuf->nvmeCmd) {
1951 spin_unlock_irqrestore(&phba->hbalock, flags);
1952 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1953 "6141 lpfc NVME IO req has no nvme_fcreq "
1954 "io buffer. Skipping abort req.\n");
1955 return;
1956 }
1957 nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1958
1959 /* Guard against IO completion being called at same time */
1960 spin_lock(&lpfc_nbuf->buf_lock);
1961
1962 /*
1963 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1964 * state must match the nvme_fcreq passed by the nvme
1965 * transport. If they don't match, it is likely the driver
1966 * has already completed the NVME IO and the nvme transport
1967 * has not seen it yet.
1968 */
1969 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1970 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1971 "6143 NVME req mismatch: "
1972 "lpfc_nbuf x%px nvmeCmd x%px, "
1973 "pnvme_fcreq x%px. Skipping Abort xri x%x\n",
1974 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1975 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1976 goto out_unlock;
1977 }
1978
1979 /* Don't abort IOs no longer on the pending queue. */
1980 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
1981 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1982 "6142 NVME IO req x%px not queued - skipping "
1983 "abort req xri x%x\n",
1984 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1985 goto out_unlock;
1986 }
1987
1988 atomic_inc(&lport->xmt_fcp_abort);
1989 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1990 nvmereq_wqe->sli4_xritag,
1991 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1992
1993 /* Outstanding abort is in progress */
1994 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
1995 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1996 "6144 Outstanding NVME I/O Abort Request "
1997 "still pending on nvme_fcreq x%px, "
1998 "lpfc_ncmd %px xri x%x\n",
1999 pnvme_fcreq, lpfc_nbuf,
2000 nvmereq_wqe->sli4_xritag);
2001 goto out_unlock;
2002 }
2003
2004 abts_buf = __lpfc_sli_get_iocbq(phba);
2005 if (!abts_buf) {
2006 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2007 "6136 No available abort wqes. Skipping "
2008 "Abts req for nvme_fcreq x%px xri x%x\n",
2009 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
2010 goto out_unlock;
2011 }
2012
2013 /* Ready - mark outstanding as aborted by driver. */
2014 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
2015
2016 lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0);
2017
2018 /* ABTS WQE must go to the same WQ as the WQE to be aborted */
2019 abts_buf->iocb_flag |= LPFC_IO_NVME;
2020 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx;
2021 abts_buf->vport = vport;
2022 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
2023 ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf);
2024 spin_unlock(&lpfc_nbuf->buf_lock);
2025 spin_unlock_irqrestore(&phba->hbalock, flags);
2026 if (ret_val) {
2027 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2028 "6137 Failed abts issue_wqe with status x%x "
2029 "for nvme_fcreq x%px.\n",
2030 ret_val, pnvme_fcreq);
2031 lpfc_sli_release_iocbq(phba, abts_buf);
2032 return;
2033 }
2034
2035 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
2036 "6138 Transport Abort NVME Request Issued for "
2037 "ox_id x%x on reqtag x%x\n",
2038 nvmereq_wqe->sli4_xritag,
2039 abts_buf->iotag);
2040 return;
2041
2042 out_unlock:
2043 spin_unlock(&lpfc_nbuf->buf_lock);
2044 spin_unlock_irqrestore(&phba->hbalock, flags);
2045 return;
2046 }
2047
2048 /* Declare and initialization an instance of the FC NVME template. */
2049 static struct nvme_fc_port_template lpfc_nvme_template = {
2050 /* initiator-based functions */
2051 .localport_delete = lpfc_nvme_localport_delete,
2052 .remoteport_delete = lpfc_nvme_remoteport_delete,
2053 .create_queue = lpfc_nvme_create_queue,
2054 .delete_queue = lpfc_nvme_delete_queue,
2055 .ls_req = lpfc_nvme_ls_req,
2056 .fcp_io = lpfc_nvme_fcp_io_submit,
2057 .ls_abort = lpfc_nvme_ls_abort,
2058 .fcp_abort = lpfc_nvme_fcp_abort,
2059 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp,
2060
2061 .max_hw_queues = 1,
2062 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2063 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2064 .dma_boundary = 0xFFFFFFFF,
2065
2066 /* Sizes of additional private data for data structures.
2067 * No use for the last two sizes at this time.
2068 */
2069 .local_priv_sz = sizeof(struct lpfc_nvme_lport),
2070 .remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2071 .lsrqst_priv_sz = 0,
2072 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2073 };
2074
2075 /**
2076 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2077 * @phba: The HBA for which this call is being executed.
2078 *
2079 * This routine removes a nvme buffer from head of @hdwq io_buf_list
2080 * and returns to caller.
2081 *
2082 * Return codes:
2083 * NULL - Error
2084 * Pointer to lpfc_nvme_buf - Success
2085 **/
2086 static struct lpfc_io_buf *
lpfc_get_nvme_buf(struct lpfc_hba * phba,struct lpfc_nodelist * ndlp,int idx,int expedite)2087 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2088 int idx, int expedite)
2089 {
2090 struct lpfc_io_buf *lpfc_ncmd;
2091 struct lpfc_sli4_hdw_queue *qp;
2092 struct sli4_sge *sgl;
2093 struct lpfc_iocbq *pwqeq;
2094 union lpfc_wqe128 *wqe;
2095
2096 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2097
2098 if (lpfc_ncmd) {
2099 pwqeq = &(lpfc_ncmd->cur_iocbq);
2100 wqe = &pwqeq->wqe;
2101
2102 /* Setup key fields in buffer that may have been changed
2103 * if other protocols used this buffer.
2104 */
2105 pwqeq->iocb_flag = LPFC_IO_NVME;
2106 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl;
2107 lpfc_ncmd->start_time = jiffies;
2108 lpfc_ncmd->flags = 0;
2109
2110 /* Rsp SGE will be filled in when we rcv an IO
2111 * from the NVME Layer to be sent.
2112 * The cmd is going to be embedded so we need a SKIP SGE.
2113 */
2114 sgl = lpfc_ncmd->dma_sgl;
2115 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2116 bf_set(lpfc_sli4_sge_last, sgl, 0);
2117 sgl->word2 = cpu_to_le32(sgl->word2);
2118 /* Fill in word 3 / sgl_len during cmd submission */
2119
2120 /* Initialize 64 bytes only */
2121 memset(wqe, 0, sizeof(union lpfc_wqe));
2122
2123 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2124 atomic_inc(&ndlp->cmd_pending);
2125 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2126 }
2127
2128 } else {
2129 qp = &phba->sli4_hba.hdwq[idx];
2130 qp->empty_io_bufs++;
2131 }
2132
2133 return lpfc_ncmd;
2134 }
2135
2136 /**
2137 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2138 * @phba: The Hba for which this call is being executed.
2139 * @lpfc_ncmd: The nvme buffer which is being released.
2140 *
2141 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2142 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2143 * and cannot be reused for at least RA_TOV amount of time if it was
2144 * aborted.
2145 **/
2146 static void
lpfc_release_nvme_buf(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2147 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2148 {
2149 struct lpfc_sli4_hdw_queue *qp;
2150 unsigned long iflag = 0;
2151
2152 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2153 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2154
2155 lpfc_ncmd->ndlp = NULL;
2156 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2157
2158 qp = lpfc_ncmd->hdwq;
2159 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2160 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2161 "6310 XB release deferred for "
2162 "ox_id x%x on reqtag x%x\n",
2163 lpfc_ncmd->cur_iocbq.sli4_xritag,
2164 lpfc_ncmd->cur_iocbq.iotag);
2165
2166 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2167 list_add_tail(&lpfc_ncmd->list,
2168 &qp->lpfc_abts_io_buf_list);
2169 qp->abts_nvme_io_bufs++;
2170 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2171 } else
2172 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2173 }
2174
2175 /**
2176 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2177 * @pvport - the lpfc_vport instance requesting a localport.
2178 *
2179 * This routine is invoked to create an nvme localport instance to bind
2180 * to the nvme_fc_transport. It is called once during driver load
2181 * like lpfc_create_shost after all other services are initialized.
2182 * It requires a vport, vpi, and wwns at call time. Other localport
2183 * parameters are modified as the driver's FCID and the Fabric WWN
2184 * are established.
2185 *
2186 * Return codes
2187 * 0 - successful
2188 * -ENOMEM - no heap memory available
2189 * other values - from nvme registration upcall
2190 **/
2191 int
lpfc_nvme_create_localport(struct lpfc_vport * vport)2192 lpfc_nvme_create_localport(struct lpfc_vport *vport)
2193 {
2194 int ret = 0;
2195 struct lpfc_hba *phba = vport->phba;
2196 struct nvme_fc_port_info nfcp_info;
2197 struct nvme_fc_local_port *localport;
2198 struct lpfc_nvme_lport *lport;
2199
2200 /* Initialize this localport instance. The vport wwn usage ensures
2201 * that NPIV is accounted for.
2202 */
2203 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2204 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2205 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2206 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2207
2208 /* We need to tell the transport layer + 1 because it takes page
2209 * alignment into account. When space for the SGL is allocated we
2210 * allocate + 3, one for cmd, one for rsp and one for this alignment
2211 */
2212 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2213
2214 /* Advertise how many hw queues we support based on cfg_hdw_queue,
2215 * which will not exceed cpu count.
2216 */
2217 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2218
2219 if (!IS_ENABLED(CONFIG_NVME_FC))
2220 return ret;
2221
2222 /* localport is allocated from the stack, but the registration
2223 * call allocates heap memory as well as the private area.
2224 */
2225
2226 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2227 &vport->phba->pcidev->dev, &localport);
2228 if (!ret) {
2229 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2230 "6005 Successfully registered local "
2231 "NVME port num %d, localP x%px, private "
2232 "x%px, sg_seg %d\n",
2233 localport->port_num, localport,
2234 localport->private,
2235 lpfc_nvme_template.max_sgl_segments);
2236
2237 /* Private is our lport size declared in the template. */
2238 lport = (struct lpfc_nvme_lport *)localport->private;
2239 vport->localport = localport;
2240 lport->vport = vport;
2241 vport->nvmei_support = 1;
2242
2243 atomic_set(&lport->xmt_fcp_noxri, 0);
2244 atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2245 atomic_set(&lport->xmt_fcp_qdepth, 0);
2246 atomic_set(&lport->xmt_fcp_err, 0);
2247 atomic_set(&lport->xmt_fcp_wqerr, 0);
2248 atomic_set(&lport->xmt_fcp_abort, 0);
2249 atomic_set(&lport->xmt_ls_abort, 0);
2250 atomic_set(&lport->xmt_ls_err, 0);
2251 atomic_set(&lport->cmpl_fcp_xb, 0);
2252 atomic_set(&lport->cmpl_fcp_err, 0);
2253 atomic_set(&lport->cmpl_ls_xb, 0);
2254 atomic_set(&lport->cmpl_ls_err, 0);
2255
2256 atomic_set(&lport->fc4NvmeLsRequests, 0);
2257 atomic_set(&lport->fc4NvmeLsCmpls, 0);
2258 }
2259
2260 return ret;
2261 }
2262
2263 #if (IS_ENABLED(CONFIG_NVME_FC))
2264 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2265 *
2266 * The driver has to wait for the host nvme transport to callback
2267 * indicating the localport has successfully unregistered all
2268 * resources. Since this is an uninterruptible wait, loop every ten
2269 * seconds and print a message indicating no progress.
2270 *
2271 * An uninterruptible wait is used because of the risk of transport-to-
2272 * driver state mismatch.
2273 */
2274 static void
lpfc_nvme_lport_unreg_wait(struct lpfc_vport * vport,struct lpfc_nvme_lport * lport,struct completion * lport_unreg_cmp)2275 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2276 struct lpfc_nvme_lport *lport,
2277 struct completion *lport_unreg_cmp)
2278 {
2279 u32 wait_tmo;
2280 int ret, i, pending = 0;
2281 struct lpfc_sli_ring *pring;
2282 struct lpfc_hba *phba = vport->phba;
2283
2284 /* Host transport has to clean up and confirm requiring an indefinite
2285 * wait. Print a message if a 10 second wait expires and renew the
2286 * wait. This is unexpected.
2287 */
2288 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2289 while (true) {
2290 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2291 if (unlikely(!ret)) {
2292 pending = 0;
2293 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2294 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2295 if (!pring)
2296 continue;
2297 if (pring->txcmplq_cnt)
2298 pending += pring->txcmplq_cnt;
2299 }
2300 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2301 "6176 Lport x%px Localport x%px wait "
2302 "timed out. Pending %d. Renewing.\n",
2303 lport, vport->localport, pending);
2304 continue;
2305 }
2306 break;
2307 }
2308 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2309 "6177 Lport x%px Localport x%px Complete Success\n",
2310 lport, vport->localport);
2311 }
2312 #endif
2313
2314 /**
2315 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2316 * @pnvme: pointer to lpfc nvme data structure.
2317 *
2318 * This routine is invoked to destroy all lports bound to the phba.
2319 * The lport memory was allocated by the nvme fc transport and is
2320 * released there. This routine ensures all rports bound to the
2321 * lport have been disconnected.
2322 *
2323 **/
2324 void
lpfc_nvme_destroy_localport(struct lpfc_vport * vport)2325 lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2326 {
2327 #if (IS_ENABLED(CONFIG_NVME_FC))
2328 struct nvme_fc_local_port *localport;
2329 struct lpfc_nvme_lport *lport;
2330 int ret;
2331 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2332
2333 if (vport->nvmei_support == 0)
2334 return;
2335
2336 localport = vport->localport;
2337 lport = (struct lpfc_nvme_lport *)localport->private;
2338
2339 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2340 "6011 Destroying NVME localport x%px\n",
2341 localport);
2342
2343 /* lport's rport list is clear. Unregister
2344 * lport and release resources.
2345 */
2346 lport->lport_unreg_cmp = &lport_unreg_cmp;
2347 ret = nvme_fc_unregister_localport(localport);
2348
2349 /* Wait for completion. This either blocks
2350 * indefinitely or succeeds
2351 */
2352 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2353 vport->localport = NULL;
2354
2355 /* Regardless of the unregister upcall response, clear
2356 * nvmei_support. All rports are unregistered and the
2357 * driver will clean up.
2358 */
2359 vport->nvmei_support = 0;
2360 if (ret == 0) {
2361 lpfc_printf_vlog(vport,
2362 KERN_INFO, LOG_NVME_DISC,
2363 "6009 Unregistered lport Success\n");
2364 } else {
2365 lpfc_printf_vlog(vport,
2366 KERN_INFO, LOG_NVME_DISC,
2367 "6010 Unregistered lport "
2368 "Failed, status x%x\n",
2369 ret);
2370 }
2371 #endif
2372 }
2373
2374 void
lpfc_nvme_update_localport(struct lpfc_vport * vport)2375 lpfc_nvme_update_localport(struct lpfc_vport *vport)
2376 {
2377 #if (IS_ENABLED(CONFIG_NVME_FC))
2378 struct nvme_fc_local_port *localport;
2379 struct lpfc_nvme_lport *lport;
2380
2381 localport = vport->localport;
2382 if (!localport) {
2383 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2384 "6710 Update NVME fail. No localport\n");
2385 return;
2386 }
2387 lport = (struct lpfc_nvme_lport *)localport->private;
2388 if (!lport) {
2389 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2390 "6171 Update NVME fail. localP x%px, No lport\n",
2391 localport);
2392 return;
2393 }
2394 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2395 "6012 Update NVME lport x%px did x%x\n",
2396 localport, vport->fc_myDID);
2397
2398 localport->port_id = vport->fc_myDID;
2399 if (localport->port_id == 0)
2400 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2401 else
2402 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2403
2404 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2405 "6030 bound lport x%px to DID x%06x\n",
2406 lport, localport->port_id);
2407 #endif
2408 }
2409
2410 int
lpfc_nvme_register_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2411 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2412 {
2413 #if (IS_ENABLED(CONFIG_NVME_FC))
2414 int ret = 0;
2415 struct nvme_fc_local_port *localport;
2416 struct lpfc_nvme_lport *lport;
2417 struct lpfc_nvme_rport *rport;
2418 struct lpfc_nvme_rport *oldrport;
2419 struct nvme_fc_remote_port *remote_port;
2420 struct nvme_fc_port_info rpinfo;
2421 struct lpfc_nodelist *prev_ndlp = NULL;
2422 struct fc_rport *srport = ndlp->rport;
2423
2424 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2425 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2426 ndlp->nlp_DID, ndlp->nlp_type);
2427
2428 localport = vport->localport;
2429 if (!localport)
2430 return 0;
2431
2432 lport = (struct lpfc_nvme_lport *)localport->private;
2433
2434 /* NVME rports are not preserved across devloss.
2435 * Just register this instance. Note, rpinfo->dev_loss_tmo
2436 * is left 0 to indicate accept transport defaults. The
2437 * driver communicates port role capabilities consistent
2438 * with the PRLI response data.
2439 */
2440 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2441 rpinfo.port_id = ndlp->nlp_DID;
2442 if (ndlp->nlp_type & NLP_NVME_TARGET)
2443 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2444 if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2445 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2446
2447 if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2448 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2449
2450 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2451 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2452 if (srport)
2453 rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2454 else
2455 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2456
2457 spin_lock_irq(&vport->phba->hbalock);
2458 oldrport = lpfc_ndlp_get_nrport(ndlp);
2459 if (oldrport) {
2460 prev_ndlp = oldrport->ndlp;
2461 spin_unlock_irq(&vport->phba->hbalock);
2462 } else {
2463 spin_unlock_irq(&vport->phba->hbalock);
2464 lpfc_nlp_get(ndlp);
2465 }
2466
2467 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2468 if (!ret) {
2469 /* If the ndlp already has an nrport, this is just
2470 * a resume of the existing rport. Else this is a
2471 * new rport.
2472 */
2473 /* Guard against an unregister/reregister
2474 * race that leaves the WAIT flag set.
2475 */
2476 spin_lock_irq(&vport->phba->hbalock);
2477 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2478 spin_unlock_irq(&vport->phba->hbalock);
2479 rport = remote_port->private;
2480 if (oldrport) {
2481
2482 /* Sever the ndlp<->rport association
2483 * before dropping the ndlp ref from
2484 * register.
2485 */
2486 spin_lock_irq(&vport->phba->hbalock);
2487 ndlp->nrport = NULL;
2488 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2489 spin_unlock_irq(&vport->phba->hbalock);
2490 rport->ndlp = NULL;
2491 rport->remoteport = NULL;
2492
2493 /* Reference only removed if previous NDLP is no longer
2494 * active. It might be just a swap and removing the
2495 * reference would cause a premature cleanup.
2496 */
2497 if (prev_ndlp && prev_ndlp != ndlp) {
2498 if ((!NLP_CHK_NODE_ACT(prev_ndlp)) ||
2499 (!prev_ndlp->nrport))
2500 lpfc_nlp_put(prev_ndlp);
2501 }
2502 }
2503
2504 /* Clean bind the rport to the ndlp. */
2505 rport->remoteport = remote_port;
2506 rport->lport = lport;
2507 rport->ndlp = ndlp;
2508 spin_lock_irq(&vport->phba->hbalock);
2509 ndlp->nrport = rport;
2510 spin_unlock_irq(&vport->phba->hbalock);
2511 lpfc_printf_vlog(vport, KERN_INFO,
2512 LOG_NVME_DISC | LOG_NODE,
2513 "6022 Bind lport x%px to remoteport x%px "
2514 "rport x%px WWNN 0x%llx, "
2515 "Rport WWPN 0x%llx DID "
2516 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2517 lport, remote_port, rport,
2518 rpinfo.node_name, rpinfo.port_name,
2519 rpinfo.port_id, rpinfo.port_role,
2520 ndlp, prev_ndlp);
2521 } else {
2522 lpfc_printf_vlog(vport, KERN_ERR,
2523 LOG_TRACE_EVENT,
2524 "6031 RemotePort Registration failed "
2525 "err: %d, DID x%06x\n",
2526 ret, ndlp->nlp_DID);
2527 }
2528
2529 return ret;
2530 #else
2531 return 0;
2532 #endif
2533 }
2534
2535 /**
2536 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2537 *
2538 * If the ndlp represents an NVME Target, that we are logged into,
2539 * ping the NVME FC Transport layer to initiate a device rescan
2540 * on this remote NPort.
2541 */
2542 void
lpfc_nvme_rescan_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2543 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2544 {
2545 #if (IS_ENABLED(CONFIG_NVME_FC))
2546 struct lpfc_nvme_rport *nrport;
2547 struct nvme_fc_remote_port *remoteport = NULL;
2548
2549 spin_lock_irq(&vport->phba->hbalock);
2550 nrport = lpfc_ndlp_get_nrport(ndlp);
2551 if (nrport)
2552 remoteport = nrport->remoteport;
2553 spin_unlock_irq(&vport->phba->hbalock);
2554
2555 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2556 "6170 Rescan NPort DID x%06x type x%x "
2557 "state x%x nrport x%px remoteport x%px\n",
2558 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2559 nrport, remoteport);
2560
2561 if (!nrport || !remoteport)
2562 goto rescan_exit;
2563
2564 /* Only rescan if we are an NVME target in the MAPPED state */
2565 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2566 ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2567 nvme_fc_rescan_remoteport(remoteport);
2568
2569 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2570 "6172 NVME rescanned DID x%06x "
2571 "port_state x%x\n",
2572 ndlp->nlp_DID, remoteport->port_state);
2573 }
2574 return;
2575 rescan_exit:
2576 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2577 "6169 Skip NVME Rport Rescan, NVME remoteport "
2578 "unregistered\n");
2579 #endif
2580 }
2581
2582 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2583 *
2584 * There is no notion of Devloss or rport recovery from the current
2585 * nvme_transport perspective. Loss of an rport just means IO cannot
2586 * be sent and recovery is completely up to the initator.
2587 * For now, the driver just unbinds the DID and port_role so that
2588 * no further IO can be issued. Changes are planned for later.
2589 *
2590 * Notes - the ndlp reference count is not decremented here since
2591 * since there is no nvme_transport api for devloss. Node ref count
2592 * is only adjusted in driver unload.
2593 */
2594 void
lpfc_nvme_unregister_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2595 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2596 {
2597 #if (IS_ENABLED(CONFIG_NVME_FC))
2598 int ret;
2599 struct nvme_fc_local_port *localport;
2600 struct lpfc_nvme_lport *lport;
2601 struct lpfc_nvme_rport *rport;
2602 struct nvme_fc_remote_port *remoteport = NULL;
2603
2604 localport = vport->localport;
2605
2606 /* This is fundamental error. The localport is always
2607 * available until driver unload. Just exit.
2608 */
2609 if (!localport)
2610 return;
2611
2612 lport = (struct lpfc_nvme_lport *)localport->private;
2613 if (!lport)
2614 goto input_err;
2615
2616 spin_lock_irq(&vport->phba->hbalock);
2617 rport = lpfc_ndlp_get_nrport(ndlp);
2618 if (rport)
2619 remoteport = rport->remoteport;
2620 spin_unlock_irq(&vport->phba->hbalock);
2621 if (!remoteport)
2622 goto input_err;
2623
2624 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2625 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2626 "port_id x%06x, portstate x%x port type x%x\n",
2627 remoteport, remoteport->port_name,
2628 remoteport->port_id, remoteport->port_state,
2629 ndlp->nlp_type);
2630
2631 /* Sanity check ndlp type. Only call for NVME ports. Don't
2632 * clear any rport state until the transport calls back.
2633 */
2634
2635 if (ndlp->nlp_type & NLP_NVME_TARGET) {
2636 /* No concern about the role change on the nvme remoteport.
2637 * The transport will update it.
2638 */
2639 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG;
2640
2641 /* Don't let the host nvme transport keep sending keep-alives
2642 * on this remoteport. Vport is unloading, no recovery. The
2643 * return values is ignored. The upcall is a courtesy to the
2644 * transport.
2645 */
2646 if (vport->load_flag & FC_UNLOADING)
2647 (void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2648
2649 ret = nvme_fc_unregister_remoteport(remoteport);
2650 if (ret != 0) {
2651 lpfc_nlp_put(ndlp);
2652 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2653 "6167 NVME unregister failed %d "
2654 "port_state x%x\n",
2655 ret, remoteport->port_state);
2656 }
2657 }
2658 return;
2659
2660 input_err:
2661 #endif
2662 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2663 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2664 vport->localport, ndlp->rport, ndlp->nlp_DID);
2665 }
2666
2667 /**
2668 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2669 * @phba: pointer to lpfc hba data structure.
2670 * @axri: pointer to the fcp xri abort wcqe structure.
2671 * @lpfc_ncmd: The nvme job structure for the request being aborted.
2672 *
2673 * This routine is invoked by the worker thread to process a SLI4 fast-path
2674 * NVME aborted xri. Aborted NVME IO commands are completed to the transport
2675 * here.
2676 **/
2677 void
lpfc_sli4_nvme_xri_aborted(struct lpfc_hba * phba,struct sli4_wcqe_xri_aborted * axri,struct lpfc_io_buf * lpfc_ncmd)2678 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2679 struct sli4_wcqe_xri_aborted *axri,
2680 struct lpfc_io_buf *lpfc_ncmd)
2681 {
2682 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2683 struct nvmefc_fcp_req *nvme_cmd = NULL;
2684 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2685
2686
2687 if (ndlp)
2688 lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2689
2690 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2691 "6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2692 "xri released\n",
2693 lpfc_ncmd->nvmeCmd, xri,
2694 lpfc_ncmd->cur_iocbq.iotag);
2695
2696 /* Aborted NVME commands are required to not complete
2697 * before the abort exchange command fully completes.
2698 * Once completed, it is available via the put list.
2699 */
2700 if (lpfc_ncmd->nvmeCmd) {
2701 nvme_cmd = lpfc_ncmd->nvmeCmd;
2702 nvme_cmd->done(nvme_cmd);
2703 lpfc_ncmd->nvmeCmd = NULL;
2704 }
2705 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2706 }
2707
2708 /**
2709 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2710 * @phba: Pointer to HBA context object.
2711 *
2712 * This function flushes all wqes in the nvme rings and frees all resources
2713 * in the txcmplq. This function does not issue abort wqes for the IO
2714 * commands in txcmplq, they will just be returned with
2715 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2716 * slot has been permanently disabled.
2717 **/
2718 void
lpfc_nvme_wait_for_io_drain(struct lpfc_hba * phba)2719 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2720 {
2721 struct lpfc_sli_ring *pring;
2722 u32 i, wait_cnt = 0;
2723
2724 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2725 return;
2726
2727 /* Cycle through all IO rings and make sure all outstanding
2728 * WQEs have been removed from the txcmplqs.
2729 */
2730 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2731 if (!phba->sli4_hba.hdwq[i].io_wq)
2732 continue;
2733 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2734
2735 if (!pring)
2736 continue;
2737
2738 /* Retrieve everything on the txcmplq */
2739 while (!list_empty(&pring->txcmplq)) {
2740 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2741 wait_cnt++;
2742
2743 /* The sleep is 10mS. Every ten seconds,
2744 * dump a message. Something is wrong.
2745 */
2746 if ((wait_cnt % 1000) == 0) {
2747 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2748 "6178 NVME IO not empty, "
2749 "cnt %d\n", wait_cnt);
2750 }
2751 }
2752 }
2753 }
2754
2755 void
lpfc_nvme_cancel_iocb(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn)2756 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn)
2757 {
2758 #if (IS_ENABLED(CONFIG_NVME_FC))
2759 struct lpfc_io_buf *lpfc_ncmd;
2760 struct nvmefc_fcp_req *nCmd;
2761 struct lpfc_nvme_fcpreq_priv *freqpriv;
2762
2763 if (!pwqeIn->context1) {
2764 lpfc_sli_release_iocbq(phba, pwqeIn);
2765 return;
2766 }
2767 /* For abort iocb just return, IO iocb will do a done call */
2768 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2769 CMD_ABORT_XRI_CX) {
2770 lpfc_sli_release_iocbq(phba, pwqeIn);
2771 return;
2772 }
2773 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1;
2774
2775 spin_lock(&lpfc_ncmd->buf_lock);
2776 if (!lpfc_ncmd->nvmeCmd) {
2777 spin_unlock(&lpfc_ncmd->buf_lock);
2778 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2779 return;
2780 }
2781
2782 nCmd = lpfc_ncmd->nvmeCmd;
2783 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2784 "6194 NVME Cancel xri %x\n",
2785 lpfc_ncmd->cur_iocbq.sli4_xritag);
2786
2787 nCmd->transferred_length = 0;
2788 nCmd->rcv_rsplen = 0;
2789 nCmd->status = NVME_SC_INTERNAL;
2790 freqpriv = nCmd->private;
2791 freqpriv->nvme_buf = NULL;
2792 lpfc_ncmd->nvmeCmd = NULL;
2793
2794 spin_unlock(&lpfc_ncmd->buf_lock);
2795 nCmd->done(nCmd);
2796
2797 /* Call release with XB=1 to queue the IO into the abort list. */
2798 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2799 #endif
2800 }
2801
