1 /*
2 * SPDX-License-Identifier: Apache-2.0
3 * Copyright (c) 2022 Intel Corp.
4 */
5
6 #ifndef ZEPHYR_DRIVERS_DISK_NVME_NVME_COMMAND_H_
7 #define ZEPHYR_DRIVERS_DISK_NVME_NVME_COMMAND_H_
8
9 #include <zephyr/sys/slist.h>
10 #include <zephyr/sys/byteorder.h>
11
12 struct nvme_command {
13 /* dword 0 */
14 struct _cdw0 {
15 uint8_t opc; /* opcode */
16 uint8_t fuse : 2; /* fused operation */
17 uint8_t rsvd : 4; /* reserved */
18 uint8_t psdt : 2; /* PRP or SGL for Data Transfer */
19 uint16_t cid; /* command identifier */
20 } cdw0;
21
22 /* dword 1 */
23 uint32_t nsid; /* namespace identifier */
24
25 /* dword 2-3 */
26 uint32_t cdw2;
27 uint32_t cdw3;
28
29 /* dword 4-5 */
30 uint64_t mptr; /* metadata pointer */
31
32 /* dword 6-7 and 8-9 */
33 struct _dptr {
34 uint64_t prp1; /* prp entry 1 */
35 uint64_t prp2; /* prp entry 2 */
36 } dptr; /* data pointer */
37
38 /* dword 10 */
39 union {
40 uint32_t cdw10; /* command-specific */
41 uint32_t ndt; /* Number of Dwords in Data transfer */
42 };
43
44 /* dword 11 */
45 union {
46 uint32_t cdw11; /* command-specific */
47 uint32_t ndm; /* Number of Dwords in Metadata transfer */
48 };
49
50 /* dword 12-15 */
51 uint32_t cdw12; /* command-specific */
52 uint32_t cdw13; /* command-specific */
53 uint32_t cdw14; /* command-specific */
54 uint32_t cdw15; /* command-specific */
55 };
56
57 struct nvme_completion {
58 /* dword 0 */
59 uint32_t cdw0; /* command-specific */
60
61 /* dword 1 */
62 uint32_t rsvd;
63
64 /* dword 2 */
65 uint16_t sqhd; /* submission queue head pointer */
66 uint16_t sqid; /* submission queue identifier */
67
68 /* dword 3 */
69 uint16_t cid; /* command identifier */
70 uint16_t status;
71 } __aligned(8);
72
73 struct nvme_completion_poll_status {
74 int status;
75 struct nvme_completion cpl;
76 struct k_sem sem;
77 };
78
79 /* status code types */
80 enum nvme_status_code_type {
81 NVME_SCT_GENERIC = 0x0,
82 NVME_SCT_COMMAND_SPECIFIC = 0x1,
83 NVME_SCT_MEDIA_ERROR = 0x2,
84 NVME_SCT_PATH_RELATED = 0x3,
85 /* 0x3-0x6 - reserved */
86 NVME_SCT_VENDOR_SPECIFIC = 0x7,
87 };
88
89 /* generic command status codes */
90 enum nvme_generic_command_status_code {
91 NVME_SC_SUCCESS = 0x00,
92 NVME_SC_INVALID_OPCODE = 0x01,
93 NVME_SC_INVALID_FIELD = 0x02,
94 NVME_SC_COMMAND_ID_CONFLICT = 0x03,
95 NVME_SC_DATA_TRANSFER_ERROR = 0x04,
96 NVME_SC_ABORTED_POWER_LOSS = 0x05,
97 NVME_SC_INTERNAL_DEVICE_ERROR = 0x06,
98 NVME_SC_ABORTED_BY_REQUEST = 0x07,
99 NVME_SC_ABORTED_SQ_DELETION = 0x08,
100 NVME_SC_ABORTED_FAILED_FUSED = 0x09,
101 NVME_SC_ABORTED_MISSING_FUSED = 0x0a,
102 NVME_SC_INVALID_NAMESPACE_OR_FORMAT = 0x0b,
103 NVME_SC_COMMAND_SEQUENCE_ERROR = 0x0c,
104 NVME_SC_INVALID_SGL_SEGMENT_DESCR = 0x0d,
105 NVME_SC_INVALID_NUMBER_OF_SGL_DESCR = 0x0e,
106 NVME_SC_DATA_SGL_LENGTH_INVALID = 0x0f,
107 NVME_SC_METADATA_SGL_LENGTH_INVALID = 0x10,
108 NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID = 0x11,
109 NVME_SC_INVALID_USE_OF_CMB = 0x12,
110 NVME_SC_PRP_OFFSET_INVALID = 0x13,
111 NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED = 0x14,
112 NVME_SC_OPERATION_DENIED = 0x15,
113 NVME_SC_SGL_OFFSET_INVALID = 0x16,
114 /* 0x17 - reserved */
115 NVME_SC_HOST_ID_INCONSISTENT_FORMAT = 0x18,
116 NVME_SC_KEEP_ALIVE_TIMEOUT_EXPIRED = 0x19,
117 NVME_SC_KEEP_ALIVE_TIMEOUT_INVALID = 0x1a,
118 NVME_SC_ABORTED_DUE_TO_PREEMPT = 0x1b,
119 NVME_SC_SANITIZE_FAILED = 0x1c,
120 NVME_SC_SANITIZE_IN_PROGRESS = 0x1d,
121 NVME_SC_SGL_DATA_BLOCK_GRAN_INVALID = 0x1e,
122 NVME_SC_NOT_SUPPORTED_IN_CMB = 0x1f,
123 NVME_SC_NAMESPACE_IS_WRITE_PROTECTED = 0x20,
124 NVME_SC_COMMAND_INTERRUPTED = 0x21,
125 NVME_SC_TRANSIENT_TRANSPORT_ERROR = 0x22,
126
127 NVME_SC_LBA_OUT_OF_RANGE = 0x80,
128 NVME_SC_CAPACITY_EXCEEDED = 0x81,
129 NVME_SC_NAMESPACE_NOT_READY = 0x82,
130 NVME_SC_RESERVATION_CONFLICT = 0x83,
131 NVME_SC_FORMAT_IN_PROGRESS = 0x84,
132 };
133
134 /* command specific status codes */
135 enum nvme_command_specific_status_code {
136 NVME_SC_COMPLETION_QUEUE_INVALID = 0x00,
137 NVME_SC_INVALID_QUEUE_IDENTIFIER = 0x01,
138 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED = 0x02,
139 NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED = 0x03,
140 /* 0x04 - reserved */
141 NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05,
142 NVME_SC_INVALID_FIRMWARE_SLOT = 0x06,
143 NVME_SC_INVALID_FIRMWARE_IMAGE = 0x07,
144 NVME_SC_INVALID_INTERRUPT_VECTOR = 0x08,
145 NVME_SC_INVALID_LOG_PAGE = 0x09,
146 NVME_SC_INVALID_FORMAT = 0x0a,
147 NVME_SC_FIRMWARE_REQUIRES_RESET = 0x0b,
148 NVME_SC_INVALID_QUEUE_DELETION = 0x0c,
149 NVME_SC_FEATURE_NOT_SAVEABLE = 0x0d,
150 NVME_SC_FEATURE_NOT_CHANGEABLE = 0x0e,
151 NVME_SC_FEATURE_NOT_NS_SPECIFIC = 0x0f,
152 NVME_SC_FW_ACT_REQUIRES_NVMS_RESET = 0x10,
153 NVME_SC_FW_ACT_REQUIRES_RESET = 0x11,
154 NVME_SC_FW_ACT_REQUIRES_TIME = 0x12,
155 NVME_SC_FW_ACT_PROHIBITED = 0x13,
156 NVME_SC_OVERLAPPING_RANGE = 0x14,
157 NVME_SC_NS_INSUFFICIENT_CAPACITY = 0x15,
158 NVME_SC_NS_ID_UNAVAILABLE = 0x16,
159 /* 0x17 - reserved */
160 NVME_SC_NS_ALREADY_ATTACHED = 0x18,
161 NVME_SC_NS_IS_PRIVATE = 0x19,
162 NVME_SC_NS_NOT_ATTACHED = 0x1a,
163 NVME_SC_THIN_PROV_NOT_SUPPORTED = 0x1b,
164 NVME_SC_CTRLR_LIST_INVALID = 0x1c,
165 NVME_SC_SELF_TEST_IN_PROGRESS = 0x1d,
166 NVME_SC_BOOT_PART_WRITE_PROHIB = 0x1e,
167 NVME_SC_INVALID_CTRLR_ID = 0x1f,
168 NVME_SC_INVALID_SEC_CTRLR_STATE = 0x20,
169 NVME_SC_INVALID_NUM_OF_CTRLR_RESRC = 0x21,
170 NVME_SC_INVALID_RESOURCE_ID = 0x22,
171 NVME_SC_SANITIZE_PROHIBITED_WPMRE = 0x23,
172 NVME_SC_ANA_GROUP_ID_INVALID = 0x24,
173 NVME_SC_ANA_ATTACH_FAILED = 0x25,
174
175 NVME_SC_CONFLICTING_ATTRIBUTES = 0x80,
176 NVME_SC_INVALID_PROTECTION_INFO = 0x81,
177 NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE = 0x82,
178 };
179
180 /* media error status codes */
181 enum nvme_media_error_status_code {
182 NVME_SC_WRITE_FAULTS = 0x80,
183 NVME_SC_UNRECOVERED_READ_ERROR = 0x81,
184 NVME_SC_GUARD_CHECK_ERROR = 0x82,
185 NVME_SC_APPLICATION_TAG_CHECK_ERROR = 0x83,
186 NVME_SC_REFERENCE_TAG_CHECK_ERROR = 0x84,
187 NVME_SC_COMPARE_FAILURE = 0x85,
188 NVME_SC_ACCESS_DENIED = 0x86,
189 NVME_SC_DEALLOCATED_OR_UNWRITTEN = 0x87,
190 };
191
192 /* path related status codes */
193 enum nvme_path_related_status_code {
194 NVME_SC_INTERNAL_PATH_ERROR = 0x00,
195 NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS = 0x01,
196 NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE = 0x02,
197 NVME_SC_ASYMMETRIC_ACCESS_TRANSITION = 0x03,
198 NVME_SC_CONTROLLER_PATHING_ERROR = 0x60,
199 NVME_SC_HOST_PATHING_ERROR = 0x70,
200 NVME_SC_COMMAND_ABORTED_BY_HOST = 0x71,
201 };
202
203 /* admin opcodes */
204 enum nvme_admin_opcode {
205 NVME_OPC_DELETE_IO_SQ = 0x00,
206 NVME_OPC_CREATE_IO_SQ = 0x01,
207 NVME_OPC_GET_LOG_PAGE = 0x02,
208 /* 0x03 - reserved */
209 NVME_OPC_DELETE_IO_CQ = 0x04,
210 NVME_OPC_CREATE_IO_CQ = 0x05,
211 NVME_OPC_IDENTIFY = 0x06,
212 /* 0x07 - reserved */
213 NVME_OPC_ABORT = 0x08,
214 NVME_OPC_SET_FEATURES = 0x09,
215 NVME_OPC_GET_FEATURES = 0x0a,
216 /* 0x0b - reserved */
217 NVME_OPC_ASYNC_EVENT_REQUEST = 0x0c,
218 NVME_OPC_NAMESPACE_MANAGEMENT = 0x0d,
219 /* 0x0e-0x0f - reserved */
220 NVME_OPC_FIRMWARE_ACTIVATE = 0x10,
221 NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD = 0x11,
222 /* 0x12-0x13 - reserved */
223 NVME_OPC_DEVICE_SELF_TEST = 0x14,
224 NVME_OPC_NAMESPACE_ATTACHMENT = 0x15,
225 /* 0x16-0x17 - reserved */
226 NVME_OPC_KEEP_ALIVE = 0x18,
227 NVME_OPC_DIRECTIVE_SEND = 0x19,
228 NVME_OPC_DIRECTIVE_RECEIVE = 0x1a,
229 /* 0x1b - reserved */
230 NVME_OPC_VIRTUALIZATION_MANAGEMENT = 0x1c,
231 NVME_OPC_NVME_MI_SEND = 0x1d,
232 NVME_OPC_NVME_MI_RECEIVE = 0x1e,
233 /* 0x1f-0x7b - reserved */
234 NVME_OPC_DOORBELL_BUFFER_CONFIG = 0x7c,
235
236 NVME_OPC_FORMAT_NVM = 0x80,
237 NVME_OPC_SECURITY_SEND = 0x81,
238 NVME_OPC_SECURITY_RECEIVE = 0x82,
239 /* 0x83 - reserved */
240 NVME_OPC_SANITIZE = 0x84,
241 /* 0x85 - reserved */
242 NVME_OPC_GET_LBA_STATUS = 0x86,
243 };
244
245 /* nvme nvm opcodes */
246 enum nvme_nvm_opcode {
247 NVME_OPC_FLUSH = 0x00,
248 NVME_OPC_WRITE = 0x01,
249 NVME_OPC_READ = 0x02,
250 /* 0x03 - reserved */
251 NVME_OPC_WRITE_UNCORRECTABLE = 0x04,
252 NVME_OPC_COMPARE = 0x05,
253 /* 0x06-0x07 - reserved */
254 NVME_OPC_WRITE_ZEROES = 0x08,
255 NVME_OPC_DATASET_MANAGEMENT = 0x09,
256 /* 0x0a-0x0b - reserved */
257 NVME_OPC_VERIFY = 0x0c,
258 NVME_OPC_RESERVATION_REGISTER = 0x0d,
259 NVME_OPC_RESERVATION_REPORT = 0x0e,
260 /* 0x0f-0x10 - reserved */
261 NVME_OPC_RESERVATION_ACQUIRE = 0x11,
262 /* 0x12-0x14 - reserved */
263 NVME_OPC_RESERVATION_RELEASE = 0x15,
264 };
265
266 enum nvme_feature {
267 /* 0x00 - reserved */
268 NVME_FEAT_ARBITRATION = 0x01,
269 NVME_FEAT_POWER_MANAGEMENT = 0x02,
270 NVME_FEAT_LBA_RANGE_TYPE = 0x03,
271 NVME_FEAT_TEMPERATURE_THRESHOLD = 0x04,
272 NVME_FEAT_ERROR_RECOVERY = 0x05,
273 NVME_FEAT_VOLATILE_WRITE_CACHE = 0x06,
274 NVME_FEAT_NUMBER_OF_QUEUES = 0x07,
275 NVME_FEAT_INTERRUPT_COALESCING = 0x08,
276 NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09,
277 NVME_FEAT_WRITE_ATOMICITY = 0x0A,
278 NVME_FEAT_ASYNC_EVENT_CONFIGURATION = 0x0B,
279 NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION = 0x0C,
280 NVME_FEAT_HOST_MEMORY_BUFFER = 0x0D,
281 NVME_FEAT_TIMESTAMP = 0x0E,
282 NVME_FEAT_KEEP_ALIVE_TIMER = 0x0F,
283 NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT = 0x10,
284 NVME_FEAT_NON_OP_POWER_STATE_CONFIG = 0x11,
285 NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG = 0x12,
286 NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG = 0x13,
287 NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW = 0x14,
288 NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES = 0x15,
289 NVME_FEAT_HOST_BEHAVIOR_SUPPORT = 0x16,
290 NVME_FEAT_SANITIZE_CONFIG = 0x17,
291 NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION = 0x18,
292 /* 0x19-0x77 - reserved */
293 /* 0x78-0x7f - NVMe Management Interface */
294 NVME_FEAT_SOFTWARE_PROGRESS_MARKER = 0x80,
295 NVME_FEAT_HOST_IDENTIFIER = 0x81,
296 NVME_FEAT_RESERVATION_NOTIFICATION_MASK = 0x82,
297 NVME_FEAT_RESERVATION_PERSISTENCE = 0x83,
298 NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG = 0x84,
299 /* 0x85-0xBF - command set specific (reserved) */
300 /* 0xC0-0xFF - vendor specific */
301 };
302
303 #if !defined(CONFIG_DCACHE_LINE_SIZE) || (CONFIG_DCACHE_LINE_SIZE == 0)
304 #define CACHE_LINE_SIZE (64)
305 #else
306 #define CACHE_LINE_SIZE CONFIG_DCACHE_LINE_SIZE
307 #endif
308
309 #define NVME_PBAO_MASK (CONFIG_MMU_PAGE_SIZE - 1)
310
311 #define NVME_PRP_NEXT_PAGE(_addr) \
312 ((_addr & ~NVME_PBAO_MASK) + CONFIG_MMU_PAGE_SIZE)
313
314 struct nvme_prp_list {
315 uintptr_t prp[CONFIG_MMU_PAGE_SIZE / sizeof(uintptr_t)]
316 __aligned(CONFIG_MMU_PAGE_SIZE);
317 sys_dnode_t node;
318 };
319
320 struct nvme_cmd_qpair {
321 struct nvme_controller *ctrlr;
322 uint32_t id;
323
324 uint32_t num_entries;
325
326 uint32_t sq_tdbl_off;
327 uint32_t cq_hdbl_off;
328
329 uint32_t phase;
330 uint32_t sq_head;
331 uint32_t sq_tail;
332 uint32_t cq_head;
333
334 int64_t num_cmds;
335 int64_t num_intr_handler_calls;
336 int64_t num_retries;
337 int64_t num_failures;
338 int64_t num_ignored;
339
340 struct nvme_command *cmd;
341 struct nvme_completion *cpl;
342
343 uintptr_t cmd_bus_addr;
344 uintptr_t cpl_bus_addr;
345
346 uint16_t vector;
347 } __aligned(CACHE_LINE_SIZE);
348
349 typedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *);
350
351 enum nvme_request_type {
352 NVME_REQUEST_NULL = 1,
353 NVME_REQUEST_VADDR = 2,
354 };
355
356 struct nvme_request {
357 struct nvme_command cmd;
358 struct nvme_cmd_qpair *qpair;
359
360 uint32_t type;
361 uint32_t req_start;
362 int32_t retries;
363
364 void *payload;
365 uint32_t payload_size;
366 nvme_cb_fn_t cb_fn;
367 void *cb_arg;
368
369 struct nvme_prp_list *prp_list;
370
371 sys_dnode_t node;
372 };
373
374 void nvme_cmd_init(void);
375
376 void nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl);
377
378 #ifdef CONFIG_NVME_LOG_LEVEL_DBG
379 void nvme_completion_print(const struct nvme_completion *cpl);
380 #else
381 #define nvme_completion_print(...)
382 #endif /* CONFIG_NVME_LOG_LEVEL_DBG */
383
384 void nvme_cmd_request_free(struct nvme_request *request);
385
386 struct nvme_request *nvme_cmd_request_alloc(void);
387
388 int nvme_cmd_qpair_setup(struct nvme_cmd_qpair *qpair,
389 struct nvme_controller *ctrlr,
390 uint32_t id);
391
392 void nvme_cmd_qpair_reset(struct nvme_cmd_qpair *qpair);
393
394 int nvme_cmd_qpair_submit_request(struct nvme_cmd_qpair *qpair,
395 struct nvme_request *request);
396
397 int nvme_cmd_identify_controller(struct nvme_controller *ctrlr,
398 void *payload,
399 nvme_cb_fn_t cb_fn,
400 void *cb_arg);
401
402 int nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr,
403 nvme_cb_fn_t cb_fn, void *cb_arg);
404
405 int nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr,
406 uint32_t nsid, void *payload,
407 nvme_cb_fn_t cb_fn, void *cb_arg);
408
409 int nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
410 struct nvme_cmd_qpair *io_queue,
411 nvme_cb_fn_t cb_fn, void *cb_arg);
412
413 int nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
414 struct nvme_cmd_qpair *io_queue,
415 nvme_cb_fn_t cb_fn, void *cb_arg);
416
417 int nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
418 struct nvme_cmd_qpair *io_queue,
419 nvme_cb_fn_t cb_fn, void *cb_arg);
420
421 int nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
422 struct nvme_cmd_qpair *io_queue,
423 nvme_cb_fn_t cb_fn, void *cb_arg);
424
425 int nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
426 uint8_t feature, uint32_t cdw11,
427 uint32_t cdw12, uint32_t cdw13,
428 uint32_t cdw14, uint32_t cdw15,
429 void *payload, uint32_t payload_size,
430 nvme_cb_fn_t cb_fn, void *cb_arg);
431
432 int nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr,
433 uint8_t feature, uint32_t cdw11,
434 void *payload, uint32_t payload_size,
435 nvme_cb_fn_t cb_fn, void *cb_arg);
436
437 int nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
438 uint32_t num_queues,
439 nvme_cb_fn_t cb_fn, void *cb_arg);
440
441 static inline
nvme_allocate_request(nvme_cb_fn_t cb_fn,void * cb_arg)442 struct nvme_request *nvme_allocate_request(nvme_cb_fn_t cb_fn, void *cb_arg)
443 {
444 struct nvme_request *request;
445
446 request = nvme_cmd_request_alloc();
447 if (request != NULL) {
448 request->cb_fn = cb_fn;
449 request->cb_arg = cb_arg;
450 }
451
452 return request;
453 }
454
455 static inline
nvme_allocate_request_vaddr(void * payload,uint32_t payload_size,nvme_cb_fn_t cb_fn,void * cb_arg)456 struct nvme_request *nvme_allocate_request_vaddr(void *payload,
457 uint32_t payload_size,
458 nvme_cb_fn_t cb_fn,
459 void *cb_arg)
460 {
461 struct nvme_request *request;
462
463 request = nvme_allocate_request(cb_fn, cb_arg);
464 if (request != NULL) {
465 request->type = NVME_REQUEST_VADDR;
466 request->payload = payload;
467 request->payload_size = payload_size;
468 }
469
470 return request;
471 }
472
473
474 static inline
nvme_allocate_request_null(nvme_cb_fn_t cb_fn,void * cb_arg)475 struct nvme_request *nvme_allocate_request_null(nvme_cb_fn_t cb_fn,
476 void *cb_arg)
477 {
478 struct nvme_request *request;
479
480 request = nvme_allocate_request(cb_fn, cb_arg);
481 if (request != NULL) {
482 request->type = NVME_REQUEST_NULL;
483 }
484
485 return request;
486 }
487
488 /*
489 * Command building helper functions
490 * These functions assume allocator zeros out cmd structure
491 */
492 static inline
nvme_namespace_flush_cmd(struct nvme_command * cmd,uint32_t nsid)493 void nvme_namespace_flush_cmd(struct nvme_command *cmd, uint32_t nsid)
494 {
495 cmd->cdw0.opc = NVME_OPC_FLUSH;
496 cmd->nsid = sys_cpu_to_le32(nsid);
497 }
498
499 static inline
nvme_namespace_rw_cmd(struct nvme_command * cmd,uint32_t rwcmd,uint32_t nsid,uint64_t lba,uint32_t count)500 void nvme_namespace_rw_cmd(struct nvme_command *cmd, uint32_t rwcmd,
501 uint32_t nsid, uint64_t lba, uint32_t count)
502 {
503 cmd->cdw0.opc = rwcmd;
504 cmd->nsid = sys_cpu_to_le32(nsid);
505 cmd->cdw10 = sys_cpu_to_le32(lba & 0xffffffffu);
506 cmd->cdw11 = sys_cpu_to_le32(lba >> 32);
507 cmd->cdw12 = sys_cpu_to_le32(count-1);
508 }
509
510 static inline
nvme_namespace_write_cmd(struct nvme_command * cmd,uint32_t nsid,uint64_t lba,uint32_t count)511 void nvme_namespace_write_cmd(struct nvme_command *cmd, uint32_t nsid,
512 uint64_t lba, uint32_t count)
513 {
514 nvme_namespace_rw_cmd(cmd, NVME_OPC_WRITE, nsid, lba, count);
515 }
516
517 static inline
nvme_namespace_read_cmd(struct nvme_command * cmd,uint32_t nsid,uint64_t lba,uint32_t count)518 void nvme_namespace_read_cmd(struct nvme_command *cmd, uint32_t nsid,
519 uint64_t lba, uint32_t count)
520 {
521 nvme_namespace_rw_cmd(cmd, NVME_OPC_READ, nsid, lba, count);
522 }
523
nvme_completion_swapbytes(struct nvme_completion * cpl)524 static inline void nvme_completion_swapbytes(struct nvme_completion *cpl)
525 {
526 #if _BYTE_ORDER != _LITTLE_ENDIAN
527 cpl->cdw0 = sys_le32_to_cpu(cpl->cdw0);
528 /* omit rsvd1 */
529 cpl->sqhd = sys_le16_to_cpu(cpl->sqhd);
530 cpl->sqid = sys_le16_to_cpu(cpl->sqid);
531 /* omit cid */
532 cpl->status = sys_le16_to_cpu(s->status);
533 #else
534 ARG_UNUSED(cpl);
535 #endif
536 }
537
538 static inline
nvme_completion_poll(struct nvme_completion_poll_status * status)539 void nvme_completion_poll(struct nvme_completion_poll_status *status)
540 {
541 k_sem_take(&status->sem, K_FOREVER);
542 }
543
544 #define NVME_CPL_STATUS_POLL_INIT(cpl_status) \
545 { \
546 .status = 0, \
547 .sem = Z_SEM_INITIALIZER(cpl_status.sem, 0, 1), \
548 }
549
550 static inline
nvme_cpl_status_poll_init(struct nvme_completion_poll_status * status)551 void nvme_cpl_status_poll_init(struct nvme_completion_poll_status *status)
552 {
553 status->status = 0;
554 k_sem_init(&status->sem, 0, 1);
555 }
556
557 #define nvme_completion_is_error(cpl) \
558 ((NVME_STATUS_GET_SC((cpl)->status) != 0) | \
559 (NVME_STATUS_GET_SCT((cpl)->status) != 0))
560
561 static inline
nvme_cpl_status_is_error(struct nvme_completion_poll_status * status)562 bool nvme_cpl_status_is_error(struct nvme_completion_poll_status *status)
563 {
564 return ((status->status != 0) ||
565 nvme_completion_is_error(&status->cpl));
566 }
567
568 #endif /* ZEPHYR_DRIVERS_DISK_NVME_NVME_COMMAND_H_ */
569
