1 /*
2 * SPDX-License-Identifier: Apache-2.0
3 * Copyright (c) 2022 Intel Corp.
4 */
5
6 #ifndef ZEPHYR_DRIVERS_DISK_NVME_NVME_COMMAND_H_
7 #define ZEPHYR_DRIVERS_DISK_NVME_NVME_COMMAND_H_
8
9 #include <zephyr/sys/slist.h>
10 #include <zephyr/sys/byteorder.h>
11
12 struct nvme_command {
13 /* dword 0 */
14 struct _cdw0 {
15 uint8_t opc; /* opcode */
16 uint8_t fuse : 2; /* fused operation */
17 uint8_t rsvd : 4; /* reserved */
18 uint8_t psdt : 2; /* PRP or SGL for Data Transfer */
19 uint16_t cid; /* command identifier */
20 } cdw0;
21
22 /* dword 1 */
23 uint32_t nsid; /* namespace identifier */
24
25 /* dword 2-3 */
26 uint32_t cdw2;
27 uint32_t cdw3;
28
29 /* dword 4-5 */
30 uint64_t mptr; /* metadata pointer */
31
32 /* dword 6-7 and 8-9 */
33 struct _dptr {
34 uint64_t prp1; /* prp entry 1 */
35 uint64_t prp2; /* prp entry 2 */
36 } dptr; /* data pointer */
37
38 /* dword 10 */
39 union {
40 uint32_t cdw10; /* command-specific */
41 uint32_t ndt; /* Number of Dwords in Data transfer */
42 };
43
44 /* dword 11 */
45 union {
46 uint32_t cdw11; /* command-specific */
47 uint32_t ndm; /* Number of Dwords in Metadata transfer */
48 };
49
50 /* dword 12-15 */
51 uint32_t cdw12; /* command-specific */
52 uint32_t cdw13; /* command-specific */
53 uint32_t cdw14; /* command-specific */
54 uint32_t cdw15; /* command-specific */
55 };
56
57 struct nvme_completion {
58 /* dword 0 */
59 uint32_t cdw0; /* command-specific */
60
61 /* dword 1 */
62 uint32_t rsvd;
63
64 /* dword 2 */
65 uint16_t sqhd; /* submission queue head pointer */
66 uint16_t sqid; /* submission queue identifier */
67
68 /* dword 3 */
69 uint16_t cid; /* command identifier */
70 uint16_t status;
71 } __aligned(8);
72
73 struct nvme_completion_poll_status {
74 int status;
75 struct nvme_completion cpl;
76 struct k_sem sem;
77 };
78
79 /* status code types */
80 enum nvme_status_code_type {
81 NVME_SCT_GENERIC = 0x0,
82 NVME_SCT_COMMAND_SPECIFIC = 0x1,
83 NVME_SCT_MEDIA_ERROR = 0x2,
84 NVME_SCT_PATH_RELATED = 0x3,
85 /* 0x3-0x6 - reserved */
86 NVME_SCT_VENDOR_SPECIFIC = 0x7,
87 };
88
89 /* generic command status codes */
90 enum nvme_generic_command_status_code {
91 NVME_SC_SUCCESS = 0x00,
92 NVME_SC_INVALID_OPCODE = 0x01,
93 NVME_SC_INVALID_FIELD = 0x02,
94 NVME_SC_COMMAND_ID_CONFLICT = 0x03,
95 NVME_SC_DATA_TRANSFER_ERROR = 0x04,
96 NVME_SC_ABORTED_POWER_LOSS = 0x05,
97 NVME_SC_INTERNAL_DEVICE_ERROR = 0x06,
98 NVME_SC_ABORTED_BY_REQUEST = 0x07,
99 NVME_SC_ABORTED_SQ_DELETION = 0x08,
100 NVME_SC_ABORTED_FAILED_FUSED = 0x09,
101 NVME_SC_ABORTED_MISSING_FUSED = 0x0a,
102 NVME_SC_INVALID_NAMESPACE_OR_FORMAT = 0x0b,
103 NVME_SC_COMMAND_SEQUENCE_ERROR = 0x0c,
104 NVME_SC_INVALID_SGL_SEGMENT_DESCR = 0x0d,
105 NVME_SC_INVALID_NUMBER_OF_SGL_DESCR = 0x0e,
106 NVME_SC_DATA_SGL_LENGTH_INVALID = 0x0f,
107 NVME_SC_METADATA_SGL_LENGTH_INVALID = 0x10,
108 NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID = 0x11,
109 NVME_SC_INVALID_USE_OF_CMB = 0x12,
110 NVME_SC_PRP_OFFSET_INVALID = 0x13,
111 NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED = 0x14,
112 NVME_SC_OPERATION_DENIED = 0x15,
113 NVME_SC_SGL_OFFSET_INVALID = 0x16,
114 /* 0x17 - reserved */
115 NVME_SC_HOST_ID_INCONSISTENT_FORMAT = 0x18,
116 NVME_SC_KEEP_ALIVE_TIMEOUT_EXPIRED = 0x19,
117 NVME_SC_KEEP_ALIVE_TIMEOUT_INVALID = 0x1a,
118 NVME_SC_ABORTED_DUE_TO_PREEMPT = 0x1b,
119 NVME_SC_SANITIZE_FAILED = 0x1c,
120 NVME_SC_SANITIZE_IN_PROGRESS = 0x1d,
121 NVME_SC_SGL_DATA_BLOCK_GRAN_INVALID = 0x1e,
122 NVME_SC_NOT_SUPPORTED_IN_CMB = 0x1f,
123 NVME_SC_NAMESPACE_IS_WRITE_PROTECTED = 0x20,
124 NVME_SC_COMMAND_INTERRUPTED = 0x21,
125 NVME_SC_TRANSIENT_TRANSPORT_ERROR = 0x22,
126
127 NVME_SC_LBA_OUT_OF_RANGE = 0x80,
128 NVME_SC_CAPACITY_EXCEEDED = 0x81,
129 NVME_SC_NAMESPACE_NOT_READY = 0x82,
130 NVME_SC_RESERVATION_CONFLICT = 0x83,
131 NVME_SC_FORMAT_IN_PROGRESS = 0x84,
132 };
133
134 /* command specific status codes */
135 enum nvme_command_specific_status_code {
136 NVME_SC_COMPLETION_QUEUE_INVALID = 0x00,
137 NVME_SC_INVALID_QUEUE_IDENTIFIER = 0x01,
138 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED = 0x02,
139 NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED = 0x03,
140 /* 0x04 - reserved */
141 NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05,
142 NVME_SC_INVALID_FIRMWARE_SLOT = 0x06,
143 NVME_SC_INVALID_FIRMWARE_IMAGE = 0x07,
144 NVME_SC_INVALID_INTERRUPT_VECTOR = 0x08,
145 NVME_SC_INVALID_LOG_PAGE = 0x09,
146 NVME_SC_INVALID_FORMAT = 0x0a,
147 NVME_SC_FIRMWARE_REQUIRES_RESET = 0x0b,
148 NVME_SC_INVALID_QUEUE_DELETION = 0x0c,
149 NVME_SC_FEATURE_NOT_SAVEABLE = 0x0d,
150 NVME_SC_FEATURE_NOT_CHANGEABLE = 0x0e,
151 NVME_SC_FEATURE_NOT_NS_SPECIFIC = 0x0f,
152 NVME_SC_FW_ACT_REQUIRES_NVMS_RESET = 0x10,
153 NVME_SC_FW_ACT_REQUIRES_RESET = 0x11,
154 NVME_SC_FW_ACT_REQUIRES_TIME = 0x12,
155 NVME_SC_FW_ACT_PROHIBITED = 0x13,
156 NVME_SC_OVERLAPPING_RANGE = 0x14,
157 NVME_SC_NS_INSUFFICIENT_CAPACITY = 0x15,
158 NVME_SC_NS_ID_UNAVAILABLE = 0x16,
159 /* 0x17 - reserved */
160 NVME_SC_NS_ALREADY_ATTACHED = 0x18,
161 NVME_SC_NS_IS_PRIVATE = 0x19,
162 NVME_SC_NS_NOT_ATTACHED = 0x1a,
163 NVME_SC_THIN_PROV_NOT_SUPPORTED = 0x1b,
164 NVME_SC_CTRLR_LIST_INVALID = 0x1c,
165 NVME_SC_SELF_TEST_IN_PROGRESS = 0x1d,
166 NVME_SC_BOOT_PART_WRITE_PROHIB = 0x1e,
167 NVME_SC_INVALID_CTRLR_ID = 0x1f,
168 NVME_SC_INVALID_SEC_CTRLR_STATE = 0x20,
169 NVME_SC_INVALID_NUM_OF_CTRLR_RESRC = 0x21,
170 NVME_SC_INVALID_RESOURCE_ID = 0x22,
171 NVME_SC_SANITIZE_PROHIBITED_WPMRE = 0x23,
172 NVME_SC_ANA_GROUP_ID_INVALID = 0x24,
173 NVME_SC_ANA_ATTACH_FAILED = 0x25,
174
175 NVME_SC_CONFLICTING_ATTRIBUTES = 0x80,
176 NVME_SC_INVALID_PROTECTION_INFO = 0x81,
177 NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE = 0x82,
178 };
179
180 /* media error status codes */
181 enum nvme_media_error_status_code {
182 NVME_SC_WRITE_FAULTS = 0x80,
183 NVME_SC_UNRECOVERED_READ_ERROR = 0x81,
184 NVME_SC_GUARD_CHECK_ERROR = 0x82,
185 NVME_SC_APPLICATION_TAG_CHECK_ERROR = 0x83,
186 NVME_SC_REFERENCE_TAG_CHECK_ERROR = 0x84,
187 NVME_SC_COMPARE_FAILURE = 0x85,
188 NVME_SC_ACCESS_DENIED = 0x86,
189 NVME_SC_DEALLOCATED_OR_UNWRITTEN = 0x87,
190 };
191
192 /* path related status codes */
193 enum nvme_path_related_status_code {
194 NVME_SC_INTERNAL_PATH_ERROR = 0x00,
195 NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS = 0x01,
196 NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE = 0x02,
197 NVME_SC_ASYMMETRIC_ACCESS_TRANSITION = 0x03,
198 NVME_SC_CONTROLLER_PATHING_ERROR = 0x60,
199 NVME_SC_HOST_PATHING_ERROR = 0x70,
200 NVME_SC_COMMAND_ABOTHED_BY_HOST = 0x71,
201 };
202
203 /* admin opcodes */
204 enum nvme_admin_opcode {
205 NVME_OPC_DELETE_IO_SQ = 0x00,
206 NVME_OPC_CREATE_IO_SQ = 0x01,
207 NVME_OPC_GET_LOG_PAGE = 0x02,
208 /* 0x03 - reserved */
209 NVME_OPC_DELETE_IO_CQ = 0x04,
210 NVME_OPC_CREATE_IO_CQ = 0x05,
211 NVME_OPC_IDENTIFY = 0x06,
212 /* 0x07 - reserved */
213 NVME_OPC_ABORT = 0x08,
214 NVME_OPC_SET_FEATURES = 0x09,
215 NVME_OPC_GET_FEATURES = 0x0a,
216 /* 0x0b - reserved */
217 NVME_OPC_ASYNC_EVENT_REQUEST = 0x0c,
218 NVME_OPC_NAMESPACE_MANAGEMENT = 0x0d,
219 /* 0x0e-0x0f - reserved */
220 NVME_OPC_FIRMWARE_ACTIVATE = 0x10,
221 NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD = 0x11,
222 /* 0x12-0x13 - reserved */
223 NVME_OPC_DEVICE_SELF_TEST = 0x14,
224 NVME_OPC_NAMESPACE_ATTACHMENT = 0x15,
225 /* 0x16-0x17 - reserved */
226 NVME_OPC_KEEP_ALIVE = 0x18,
227 NVME_OPC_DIRECTIVE_SEND = 0x19,
228 NVME_OPC_DIRECTIVE_RECEIVE = 0x1a,
229 /* 0x1b - reserved */
230 NVME_OPC_VIRTUALIZATION_MANAGEMENT = 0x1c,
231 NVME_OPC_NVME_MI_SEND = 0x1d,
232 NVME_OPC_NVME_MI_RECEIVE = 0x1e,
233 /* 0x1f-0x7b - reserved */
234 NVME_OPC_DOORBELL_BUFFER_CONFIG = 0x7c,
235
236 NVME_OPC_FORMAT_NVM = 0x80,
237 NVME_OPC_SECURITY_SEND = 0x81,
238 NVME_OPC_SECURITY_RECEIVE = 0x82,
239 /* 0x83 - reserved */
240 NVME_OPC_SANITIZE = 0x84,
241 /* 0x85 - reserved */
242 NVME_OPC_GET_LBA_STATUS = 0x86,
243 };
244
245 /* nvme nvm opcodes */
246 enum nvme_nvm_opcode {
247 NVME_OPC_FLUSH = 0x00,
248 NVME_OPC_WRITE = 0x01,
249 NVME_OPC_READ = 0x02,
250 /* 0x03 - reserved */
251 NVME_OPC_WRITE_UNCORRECTABLE = 0x04,
252 NVME_OPC_COMPARE = 0x05,
253 /* 0x06-0x07 - reserved */
254 NVME_OPC_WRITE_ZEROES = 0x08,
255 NVME_OPC_DATASET_MANAGEMENT = 0x09,
256 /* 0x0a-0x0b - reserved */
257 NVME_OPC_VERIFY = 0x0c,
258 NVME_OPC_RESERVATION_REGISTER = 0x0d,
259 NVME_OPC_RESERVATION_REPORT = 0x0e,
260 /* 0x0f-0x10 - reserved */
261 NVME_OPC_RESERVATION_ACQUIRE = 0x11,
262 /* 0x12-0x14 - reserved */
263 NVME_OPC_RESERVATION_RELEASE = 0x15,
264 };
265
266 enum nvme_feature {
267 /* 0x00 - reserved */
268 NVME_FEAT_ARBITRATION = 0x01,
269 NVME_FEAT_POWER_MANAGEMENT = 0x02,
270 NVME_FEAT_LBA_RANGE_TYPE = 0x03,
271 NVME_FEAT_TEMPERATURE_THRESHOLD = 0x04,
272 NVME_FEAT_ERROR_RECOVERY = 0x05,
273 NVME_FEAT_VOLATILE_WRITE_CACHE = 0x06,
274 NVME_FEAT_NUMBER_OF_QUEUES = 0x07,
275 NVME_FEAT_INTERRUPT_COALESCING = 0x08,
276 NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09,
277 NVME_FEAT_WRITE_ATOMICITY = 0x0A,
278 NVME_FEAT_ASYNC_EVENT_CONFIGURATION = 0x0B,
279 NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION = 0x0C,
280 NVME_FEAT_HOST_MEMORY_BUFFER = 0x0D,
281 NVME_FEAT_TIMESTAMP = 0x0E,
282 NVME_FEAT_KEEP_ALIVE_TIMER = 0x0F,
283 NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT = 0x10,
284 NVME_FEAT_NON_OP_POWER_STATE_CONFIG = 0x11,
285 NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG = 0x12,
286 NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG = 0x13,
287 NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW = 0x14,
288 NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES = 0x15,
289 NVME_FEAT_HOST_BEHAVIOR_SUPPORT = 0x16,
290 NVME_FEAT_SANITIZE_CONFIG = 0x17,
291 NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION = 0x18,
292 /* 0x19-0x77 - reserved */
293 /* 0x78-0x7f - NVMe Management Interface */
294 NVME_FEAT_SOFTWARE_PROGRESS_MARKER = 0x80,
295 NVME_FEAT_HOST_IDENTIFIER = 0x81,
296 NVME_FEAT_RESERVATION_NOTIFICATION_MASK = 0x82,
297 NVME_FEAT_RESERVATION_PERSISTENCE = 0x83,
298 NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG = 0x84,
299 /* 0x85-0xBF - command set specific (reserved) */
300 /* 0xC0-0xFF - vendor specific */
301 };
302
303 #if !defined(CONFIG_DCACHE_LINE_SIZE) || (CONFIG_DCACHE_LINE_SIZE == 0)
304 #define CACHE_LINE_SIZE (64)
305 #else
306 #define CACHE_LINE_SIZE CONFIG_DCACHE_LINE_SIZE
307 #endif
308
309 /* Assuming page size it always 4Kib
310 * ToDo: define it accorditng to CONFIG_MMU_PAGE_SIZE
311 */
312 #define NVME_PBAO_MASK 0xFFF
313
314 #define NVME_PRP_NEXT_PAGE(_addr) ((_addr & (~NVME_PBAO_MASK)) + 0x1000)
315
316 struct nvme_prp_list {
317 uintptr_t prp[CONFIG_MMU_PAGE_SIZE / sizeof(uintptr_t)]
318 __aligned(0x1000);
319 sys_dnode_t node;
320 };
321
322 struct nvme_cmd_qpair {
323 struct nvme_controller *ctrlr;
324 uint32_t id;
325
326 uint32_t num_entries;
327
328 uint32_t sq_tdbl_off;
329 uint32_t cq_hdbl_off;
330
331 uint32_t phase;
332 uint32_t sq_head;
333 uint32_t sq_tail;
334 uint32_t cq_head;
335
336 int64_t num_cmds;
337 int64_t num_intr_handler_calls;
338 int64_t num_retries;
339 int64_t num_failures;
340 int64_t num_ignored;
341
342 struct nvme_command *cmd;
343 struct nvme_completion *cpl;
344
345 uintptr_t cmd_bus_addr;
346 uintptr_t cpl_bus_addr;
347
348 uint16_t vector;
349 } __aligned(CACHE_LINE_SIZE);
350
351 typedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *);
352
353 enum nvme_request_type {
354 NVME_REQUEST_NULL = 1,
355 NVME_REQUEST_VADDR = 2,
356 };
357
358 struct nvme_request {
359 struct nvme_command cmd;
360 struct nvme_cmd_qpair *qpair;
361
362 uint32_t type;
363 uint32_t req_start;
364 int32_t retries;
365
366 void *payload;
367 uint32_t payload_size;
368 nvme_cb_fn_t cb_fn;
369 void *cb_arg;
370
371 struct nvme_prp_list *prp_list;
372
373 sys_dnode_t node;
374 };
375
376 void nvme_cmd_init(void);
377
378 void nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl);
379
380 void nvme_cmd_request_free(struct nvme_request *request);
381
382 struct nvme_request *nvme_cmd_request_alloc(void);
383
384 int nvme_cmd_qpair_setup(struct nvme_cmd_qpair *qpair,
385 struct nvme_controller *ctrlr,
386 uint32_t id);
387
388 void nvme_cmd_qpair_reset(struct nvme_cmd_qpair *qpair);
389
390 int nvme_cmd_qpair_submit_request(struct nvme_cmd_qpair *qpair,
391 struct nvme_request *request);
392
393 int nvme_cmd_identify_controller(struct nvme_controller *ctrlr,
394 void *payload,
395 nvme_cb_fn_t cb_fn,
396 void *cb_arg);
397
398 int nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr,
399 nvme_cb_fn_t cb_fn, void *cb_arg);
400
401 int nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr,
402 uint32_t nsid, void *payload,
403 nvme_cb_fn_t cb_fn, void *cb_arg);
404
405 int nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
406 struct nvme_cmd_qpair *io_queue,
407 nvme_cb_fn_t cb_fn, void *cb_arg);
408
409 int nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
410 struct nvme_cmd_qpair *io_queue,
411 nvme_cb_fn_t cb_fn, void *cb_arg);
412
413 int nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
414 struct nvme_cmd_qpair *io_queue,
415 nvme_cb_fn_t cb_fn, void *cb_arg);
416
417 int nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
418 struct nvme_cmd_qpair *io_queue,
419 nvme_cb_fn_t cb_fn, void *cb_arg);
420
421 int nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
422 uint8_t feature, uint32_t cdw11,
423 uint32_t cdw12, uint32_t cdw13,
424 uint32_t cdw14, uint32_t cdw15,
425 void *payload, uint32_t payload_size,
426 nvme_cb_fn_t cb_fn, void *cb_arg);
427
428 int nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr,
429 uint8_t feature, uint32_t cdw11,
430 void *payload, uint32_t payload_size,
431 nvme_cb_fn_t cb_fn, void *cb_arg);
432
433 int nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
434 uint32_t num_queues,
435 nvme_cb_fn_t cb_fn, void *cb_arg);
436
437 static inline
nvme_allocate_request(nvme_cb_fn_t cb_fn,void * cb_arg)438 struct nvme_request *nvme_allocate_request(nvme_cb_fn_t cb_fn, void *cb_arg)
439 {
440 struct nvme_request *request;
441
442 request = nvme_cmd_request_alloc();
443 if (request != NULL) {
444 request->cb_fn = cb_fn;
445 request->cb_arg = cb_arg;
446 }
447
448 return request;
449 }
450
451 static inline
nvme_allocate_request_vaddr(void * payload,uint32_t payload_size,nvme_cb_fn_t cb_fn,void * cb_arg)452 struct nvme_request *nvme_allocate_request_vaddr(void *payload,
453 uint32_t payload_size,
454 nvme_cb_fn_t cb_fn,
455 void *cb_arg)
456 {
457 struct nvme_request *request;
458
459 request = nvme_allocate_request(cb_fn, cb_arg);
460 if (request != NULL) {
461 request->type = NVME_REQUEST_VADDR;
462 request->payload = payload;
463 request->payload_size = payload_size;
464 }
465
466 return request;
467 }
468
469
470 static inline
nvme_allocate_request_null(nvme_cb_fn_t cb_fn,void * cb_arg)471 struct nvme_request *nvme_allocate_request_null(nvme_cb_fn_t cb_fn,
472 void *cb_arg)
473 {
474 struct nvme_request *request;
475
476 request = nvme_allocate_request(cb_fn, cb_arg);
477 if (request != NULL) {
478 request->type = NVME_REQUEST_NULL;
479 }
480
481 return request;
482 }
483
484 /*
485 * Command building helper functions
486 * These functions assume allocator zeros out cmd structure
487 */
488 static inline
nvme_namespace_flush_cmd(struct nvme_command * cmd,uint32_t nsid)489 void nvme_namespace_flush_cmd(struct nvme_command *cmd, uint32_t nsid)
490 {
491 cmd->cdw0.opc = NVME_OPC_FLUSH;
492 cmd->nsid = sys_cpu_to_le32(nsid);
493 }
494
495 static inline
nvme_namespace_rw_cmd(struct nvme_command * cmd,uint32_t rwcmd,uint32_t nsid,uint64_t lba,uint32_t count)496 void nvme_namespace_rw_cmd(struct nvme_command *cmd, uint32_t rwcmd,
497 uint32_t nsid, uint64_t lba, uint32_t count)
498 {
499 cmd->cdw0.opc = rwcmd;
500 cmd->nsid = sys_cpu_to_le32(nsid);
501 cmd->cdw10 = sys_cpu_to_le32(lba & 0xffffffffu);
502 cmd->cdw11 = sys_cpu_to_le32(lba >> 32);
503 cmd->cdw12 = sys_cpu_to_le32(count-1);
504 }
505
506 static inline
nvme_namespace_write_cmd(struct nvme_command * cmd,uint32_t nsid,uint64_t lba,uint32_t count)507 void nvme_namespace_write_cmd(struct nvme_command *cmd, uint32_t nsid,
508 uint64_t lba, uint32_t count)
509 {
510 nvme_namespace_rw_cmd(cmd, NVME_OPC_WRITE, nsid, lba, count);
511 }
512
513 static inline
nvme_namespace_read_cmd(struct nvme_command * cmd,uint32_t nsid,uint64_t lba,uint32_t count)514 void nvme_namespace_read_cmd(struct nvme_command *cmd, uint32_t nsid,
515 uint64_t lba, uint32_t count)
516 {
517 nvme_namespace_rw_cmd(cmd, NVME_OPC_READ, nsid, lba, count);
518 }
519
nvme_completion_swapbytes(struct nvme_completion * cpl)520 static inline void nvme_completion_swapbytes(struct nvme_completion *cpl)
521 {
522 #if _BYTE_ORDER != _LITTLE_ENDIAN
523 cpl->cdw0 = sys_le32_to_cpu(cpl->cdw0);
524 /* omit rsvd1 */
525 cpl->sqhd = sys_le16_to_cpu(cpl->sqhd);
526 cpl->sqid = sys_le16_to_cpu(cpl->sqid);
527 /* omit cid */
528 cpl->status = sys_le16_to_cpu(s->status);
529 #else
530 ARG_UNUSED(cpl);
531 #endif
532 }
533
534 static inline
nvme_completion_poll(struct nvme_completion_poll_status * status)535 void nvme_completion_poll(struct nvme_completion_poll_status *status)
536 {
537 k_sem_take(&status->sem, K_FOREVER);
538 }
539
540 #define NVME_CPL_STATUS_POLL_INIT(cpl_status) \
541 { \
542 .status = 0, \
543 .sem = Z_SEM_INITIALIZER(cpl_status.sem, 0, 1), \
544 }
545
546 static inline
nvme_cpl_status_poll_init(struct nvme_completion_poll_status * status)547 void nvme_cpl_status_poll_init(struct nvme_completion_poll_status *status)
548 {
549 status->status = 0;
550 k_sem_init(&status->sem, 0, 1);
551 }
552
553 #define nvme_completion_is_error(cpl) \
554 ((NVME_STATUS_GET_SC((cpl)->status) != 0) | \
555 (NVME_STATUS_GET_SCT((cpl)->status) != 0))
556
557 static inline
nvme_cpl_status_is_error(struct nvme_completion_poll_status * status)558 bool nvme_cpl_status_is_error(struct nvme_completion_poll_status *status)
559 {
560 return ((status->status != 0) ||
561 nvme_completion_is_error(&status->cpl));
562 }
563
564 #endif /* ZEPHYR_DRIVERS_DISK_NVME_NVME_COMMAND_H_ */
565
