3  * SPDX-License-Identifier: Apache-2.0
28 	struct nvme_controller *nvme_ctrlr = dev->data;  in nvme_controller_wait_for_ready()
31 		k_ms_to_ticks_ceil32(nvme_ctrlr->ready_timeout_in_ms);  in nvme_controller_wait_for_ready()
39 			return -EIO;  in nvme_controller_wait_for_ready()
47 		if ((int64_t)timeout - sys_clock_tick_get_32() < 0) {  in nvme_controller_wait_for_ready()
49 			return -EIO;  in nvme_controller_wait_for_ready()
97 	struct nvme_controller *nvme_ctrlr = dev->data;  in nvme_controller_enable()
132 	cc |= nvme_ctrlr->mps << NVME_CC_REG_MPS_SHIFT;  in nvme_controller_enable()
141 	struct nvme_controller *nvme_ctrlr = dev->data;  in nvme_controller_setup_admin_queues()
145 	nvme_cmd_qpair_reset(nvme_ctrlr->adminq);  in nvme_controller_setup_admin_queues()
148 	if (nvme_cmd_qpair_setup(nvme_ctrlr->adminq, nvme_ctrlr, 0) != 0) {  in nvme_controller_setup_admin_queues()
150 		return -EIO;  in nvme_controller_setup_admin_queues()
153 	nvme_mmio_write_8(regs, asq, nvme_ctrlr->adminq->cmd_bus_addr);  in nvme_controller_setup_admin_queues()
154 	nvme_mmio_write_8(regs, acq, nvme_ctrlr->adminq->cpl_bus_addr);  in nvme_controller_setup_admin_queues()
156 	/* acqs and asqs are 0-based. */  in nvme_controller_setup_admin_queues()
157 	qsize = CONFIG_NVME_ADMIN_ENTRIES - 1;  in nvme_controller_setup_admin_queues()
169 	struct nvme_controller *nvme_ctrlr = dev->data;  in nvme_controller_setup_io_queues()
178 					     nvme_ctrlr->num_io_queues,  in nvme_controller_setup_io_queues()
186 		LOG_ERR("Could not set IO num queues to %u",  in nvme_controller_setup_io_queues()
187 			nvme_ctrlr->num_io_queues);  in nvme_controller_setup_io_queues()
189 		return -EIO;  in nvme_controller_setup_io_queues()
193 	 * Data in cdw0 is 0-based.  in nvme_controller_setup_io_queues()
194 	 * Lower 16-bits indicate number of submission queues allocated.  in nvme_controller_setup_io_queues()
195 	 * Upper 16-bits indicate number of completion queues allocated.  in nvme_controller_setup_io_queues()
205 	nvme_ctrlr->num_io_queues = MIN(nvme_ctrlr->num_io_queues,  in nvme_controller_setup_io_queues()
207 	nvme_ctrlr->num_io_queues = MIN(nvme_ctrlr->num_io_queues,  in nvme_controller_setup_io_queues()
210 	for (idx = 0; idx < nvme_ctrlr->num_io_queues; idx++) {  in nvme_controller_setup_io_queues()
211 		io_qpair = &nvme_ctrlr->ioq[idx];  in nvme_controller_setup_io_queues()
214 			return -EIO;  in nvme_controller_setup_io_queues()
232 			return -EIO;  in nvme_controller_setup_io_queues()
248 			return -EIO;  in nvme_controller_setup_io_queues()
257 	struct nvme_controller *nvme_ctrlr = dev->data;  in nvme_controller_gather_info()
262 	nvme_ctrlr->cap_lo = cap_lo = nvme_mmio_read_4(regs, cap_lo);  in nvme_controller_gather_info()
272 	nvme_ctrlr->cap_hi = cap_hi = nvme_mmio_read_4(regs, cap_hi);  in nvme_controller_gather_info()
301 	nvme_ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;  in nvme_controller_gather_info()
303 	nvme_ctrlr->mps = NVME_CAP_HI_MPSMIN(cap_hi);  in nvme_controller_gather_info()
304 	nvme_ctrlr->page_size = 1 << (NVME_MPS_SHIFT + nvme_ctrlr->mps);  in nvme_controller_gather_info()
306 	LOG_DBG("MPS: %u - Page Size: %u bytes",  in nvme_controller_gather_info()
307 		nvme_ctrlr->mps, nvme_ctrlr->page_size);  in nvme_controller_gather_info()
311 	nvme_ctrlr->ready_timeout_in_ms = to * 500;  in nvme_controller_gather_info()
314 	 * page-sized PRP (4KB pages -> 2MB).  in nvme_controller_gather_info()
315 	 * ToDo: it could be less -> take the minimum.  in nvme_controller_gather_info()
317 	nvme_ctrlr->max_xfer_size = nvme_ctrlr->page_size /  in nvme_controller_gather_info()
318 		8 * nvme_ctrlr->page_size;  in nvme_controller_gather_info()
320 	LOG_DBG("Max transfer size: %u bytes", nvme_ctrlr->max_xfer_size);  in nvme_controller_gather_info()
325 	const struct nvme_controller_config *nvme_ctrlr_cfg = dev->config;  in nvme_controller_pcie_configure()
326 	struct nvme_controller *nvme_ctrlr = dev->data;  in nvme_controller_pcie_configure()
330 	if (nvme_ctrlr_cfg->pcie->bdf == PCIE_BDF_NONE) {  in nvme_controller_pcie_configure()
332 		return -ENODEV;  in nvme_controller_pcie_configure()
336 		PCIE_ID_TO_VEND(nvme_ctrlr_cfg->pcie->id),  in nvme_controller_pcie_configure()
337 		PCIE_ID_TO_DEV(nvme_ctrlr_cfg->pcie->id),  in nvme_controller_pcie_configure()
338 		PCIE_BDF_TO_BUS(nvme_ctrlr_cfg->pcie->bdf),  in nvme_controller_pcie_configure()
339 		PCIE_BDF_TO_DEV(nvme_ctrlr_cfg->pcie->bdf),  in nvme_controller_pcie_configure()
340 		PCIE_BDF_TO_FUNC(nvme_ctrlr_cfg->pcie->bdf));  in nvme_controller_pcie_configure()
342 	if (!pcie_get_mbar(nvme_ctrlr_cfg->pcie->bdf,  in nvme_controller_pcie_configure()
345 		return -EIO;  in nvme_controller_pcie_configure()
352 	n_vectors = pcie_msi_vectors_allocate(nvme_ctrlr_cfg->pcie->bdf,  in nvme_controller_pcie_configure()
354 					      nvme_ctrlr->vectors,  in nvme_controller_pcie_configure()
357 		LOG_ERR("Could not allocate %u MSI-X vectors",  in nvme_controller_pcie_configure()
359 		return -EIO;  in nvme_controller_pcie_configure()
362 	/* Enabling MSI-X and the vectors */  in nvme_controller_pcie_configure()
363 	if (!pcie_msi_enable(nvme_ctrlr_cfg->pcie->bdf,  in nvme_controller_pcie_configure()
364 			     nvme_ctrlr->vectors, n_vectors, 0)) {  in nvme_controller_pcie_configure()
365 		LOG_ERR("Could not enable MSI-X");  in nvme_controller_pcie_configure()
366 		return -EIO;  in nvme_controller_pcie_configure()
383 		return -EIO;  in nvme_controller_identify()
386 	nvme_controller_data_swapbytes(&nvme_ctrlr->cdata);  in nvme_controller_identify()
392 	if (nvme_ctrlr->cdata.mdts > 0) {  in nvme_controller_identify()
393 		nvme_ctrlr->max_xfer_size =  in nvme_controller_identify()
394 			MIN(nvme_ctrlr->max_xfer_size,  in nvme_controller_identify()
395 			    1 << (nvme_ctrlr->cdata.mdts + NVME_MPS_SHIFT +  in nvme_controller_identify()
396 				  NVME_CAP_HI_MPSMIN(nvme_ctrlr->cap_hi)));  in nvme_controller_identify()
407 	     i < MIN(nvme_ctrlr->cdata.nn, CONFIG_NVME_MAX_NAMESPACES); i++) {  in nvme_controller_setup_namespaces()
408 		struct nvme_namespace *ns = &nvme_ctrlr->ns[i];  in nvme_controller_setup_namespaces()
420 	struct nvme_controller *nvme_ctrlr = dev->data;  in nvme_controller_init()
423 	k_mutex_init(&nvme_ctrlr->lock);  in nvme_controller_init()
427 	nvme_ctrlr->dev = dev;  in nvme_controller_init()