Lines Matching refs:anv
226 struct apple_nvme *anv = cookie; in apple_nvme_rtkit_crashed() local
228 dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot"); in apple_nvme_rtkit_crashed()
229 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_rtkit_crashed()
235 struct apple_nvme *anv = cookie; in apple_nvme_sart_dma_setup() local
244 dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL); in apple_nvme_sart_dma_setup()
248 ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, bfr->size); in apple_nvme_sart_dma_setup()
250 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); in apple_nvme_sart_dma_setup()
261 struct apple_nvme *anv = cookie; in apple_nvme_sart_dma_destroy() local
263 apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size); in apple_nvme_sart_dma_destroy()
264 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); in apple_nvme_sart_dma_destroy()
275 struct apple_nvme *anv = queue_to_apple_nvme(q); in apple_nvmmu_inval() local
277 writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL); in apple_nvmmu_inval()
278 if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT)) in apple_nvmmu_inval()
279 dev_warn_ratelimited(anv->dev, in apple_nvmmu_inval()
286 struct apple_nvme *anv = queue_to_apple_nvme(q); in apple_nvme_submit_cmd() local
313 spin_lock_irq(&anv->lock); in apple_nvme_submit_cmd()
315 spin_unlock_irq(&anv->lock); in apple_nvme_submit_cmd()
342 static void apple_nvme_free_prps(struct apple_nvme *anv, struct request *req) in apple_nvme_free_prps() argument
353 dma_pool_free(anv->prp_page_pool, prp_list, dma_addr); in apple_nvme_free_prps()
358 static void apple_nvme_unmap_data(struct apple_nvme *anv, struct request *req) in apple_nvme_unmap_data() argument
363 dma_unmap_page(anv->dev, iod->first_dma, iod->dma_len, in apple_nvme_unmap_data()
370 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req)); in apple_nvme_unmap_data()
372 dma_pool_free(anv->prp_small_pool, apple_nvme_iod_list(req)[0], in apple_nvme_unmap_data()
375 apple_nvme_free_prps(anv, req); in apple_nvme_unmap_data()
376 mempool_free(iod->sg, anv->iod_mempool); in apple_nvme_unmap_data()
393 static blk_status_t apple_nvme_setup_prps(struct apple_nvme *anv, in apple_nvme_setup_prps() argument
431 pool = anv->prp_small_pool; in apple_nvme_setup_prps()
434 pool = anv->prp_page_pool; in apple_nvme_setup_prps()
478 apple_nvme_free_prps(anv, req); in apple_nvme_setup_prps()
487 static blk_status_t apple_nvme_setup_prp_simple(struct apple_nvme *anv, in apple_nvme_setup_prp_simple() argument
496 iod->first_dma = dma_map_bvec(anv->dev, bv, rq_dma_dir(req), 0); in apple_nvme_setup_prp_simple()
497 if (dma_mapping_error(anv->dev, iod->first_dma)) in apple_nvme_setup_prp_simple()
507 static blk_status_t apple_nvme_map_data(struct apple_nvme *anv, in apple_nvme_map_data() argument
519 return apple_nvme_setup_prp_simple(anv, req, &cmnd->rw, in apple_nvme_map_data()
524 iod->sg = mempool_alloc(anv->iod_mempool, GFP_ATOMIC); in apple_nvme_map_data()
532 nr_mapped = dma_map_sg_attrs(anv->dev, iod->sg, iod->nents, in apple_nvme_map_data()
537 ret = apple_nvme_setup_prps(anv, req, &cmnd->rw); in apple_nvme_map_data()
543 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req)); in apple_nvme_map_data()
545 mempool_free(iod->sg, anv->iod_mempool); in apple_nvme_map_data()
552 struct apple_nvme *anv = queue_to_apple_nvme(iod->q); in apple_nvme_unmap_rq() local
555 apple_nvme_unmap_data(anv, req); in apple_nvme_unmap_rq()
577 apple_nvme_queue_tagset(struct apple_nvme *anv, struct apple_nvme_queue *q) in apple_nvme_queue_tagset() argument
580 return anv->admin_tagset.tags[0]; in apple_nvme_queue_tagset()
582 return anv->tagset.tags[0]; in apple_nvme_queue_tagset()
588 struct apple_nvme *anv = queue_to_apple_nvme(q); in apple_nvme_handle_cqe() local
595 req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id); in apple_nvme_handle_cqe()
597 dev_warn(anv->dev, "invalid id %d completed", command_id); in apple_nvme_handle_cqe()
660 struct apple_nvme *anv = data; in apple_nvme_irq() local
664 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_irq()
665 if (apple_nvme_handle_cq(&anv->ioq, false)) in apple_nvme_irq()
667 if (apple_nvme_handle_cq(&anv->adminq, false)) in apple_nvme_irq()
669 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_irq()
676 static int apple_nvme_create_cq(struct apple_nvme *anv) in apple_nvme_create_cq() argument
685 c.create_cq.prp1 = cpu_to_le64(anv->ioq.cq_dma_addr); in apple_nvme_create_cq()
691 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_create_cq()
694 static int apple_nvme_remove_cq(struct apple_nvme *anv) in apple_nvme_remove_cq() argument
701 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_remove_cq()
704 static int apple_nvme_create_sq(struct apple_nvme *anv) in apple_nvme_create_sq() argument
713 c.create_sq.prp1 = cpu_to_le64(anv->ioq.sq_dma_addr); in apple_nvme_create_sq()
719 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_create_sq()
722 static int apple_nvme_remove_sq(struct apple_nvme *anv) in apple_nvme_remove_sq() argument
729 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_remove_sq()
737 struct apple_nvme *anv = queue_to_apple_nvme(q); in apple_nvme_queue_rq() local
753 if (!nvme_check_ready(&anv->ctrl, req, true)) in apple_nvme_queue_rq()
754 return nvme_fail_nonready_command(&anv->ctrl, req); in apple_nvme_queue_rq()
761 ret = apple_nvme_map_data(anv, req, cmnd); in apple_nvme_queue_rq()
787 struct apple_nvme *anv = queue_to_apple_nvme(q); in apple_nvme_init_request() local
792 nreq->ctrl = &anv->ctrl; in apple_nvme_init_request()
798 static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) in apple_nvme_disable() argument
800 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); in apple_nvme_disable()
804 if (apple_rtkit_is_crashed(anv->rtk)) in apple_nvme_disable()
811 if (anv->ctrl.state == NVME_CTRL_LIVE || in apple_nvme_disable()
812 anv->ctrl.state == NVME_CTRL_RESETTING) { in apple_nvme_disable()
814 nvme_start_freeze(&anv->ctrl); in apple_nvme_disable()
822 nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT); in apple_nvme_disable()
824 nvme_quiesce_io_queues(&anv->ctrl); in apple_nvme_disable()
827 if (READ_ONCE(anv->ioq.enabled)) { in apple_nvme_disable()
828 apple_nvme_remove_sq(anv); in apple_nvme_disable()
829 apple_nvme_remove_cq(anv); in apple_nvme_disable()
847 nvme_disable_ctrl(&anv->ctrl, shutdown); in apple_nvme_disable()
848 nvme_disable_ctrl(&anv->ctrl, false); in apple_nvme_disable()
851 WRITE_ONCE(anv->ioq.enabled, false); in apple_nvme_disable()
852 WRITE_ONCE(anv->adminq.enabled, false); in apple_nvme_disable()
854 nvme_quiesce_admin_queue(&anv->ctrl); in apple_nvme_disable()
857 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_disable()
858 apple_nvme_handle_cq(&anv->ioq, true); in apple_nvme_disable()
859 apple_nvme_handle_cq(&anv->adminq, true); in apple_nvme_disable()
860 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_disable()
862 nvme_cancel_tagset(&anv->ctrl); in apple_nvme_disable()
863 nvme_cancel_admin_tagset(&anv->ctrl); in apple_nvme_disable()
871 nvme_unquiesce_io_queues(&anv->ctrl); in apple_nvme_disable()
872 nvme_unquiesce_admin_queue(&anv->ctrl); in apple_nvme_disable()
880 struct apple_nvme *anv = queue_to_apple_nvme(q); in apple_nvme_timeout() local
882 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); in apple_nvme_timeout()
884 if (anv->ctrl.state != NVME_CTRL_LIVE) { in apple_nvme_timeout()
899 dev_warn(anv->dev, in apple_nvme_timeout()
912 if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) { in apple_nvme_timeout()
913 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_timeout()
915 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_timeout()
917 dev_warn(anv->dev, in apple_nvme_timeout()
928 dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n", in apple_nvme_timeout()
931 apple_nvme_disable(anv, false); in apple_nvme_timeout()
932 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_timeout()
940 struct apple_nvme *anv = queue_to_apple_nvme(q); in apple_nvme_poll() local
944 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_poll()
946 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_poll()
986 struct apple_nvme *anv = in apple_nvme_reset_work() local
989 if (anv->ctrl.state != NVME_CTRL_RESETTING) { in apple_nvme_reset_work()
990 dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", in apple_nvme_reset_work()
991 anv->ctrl.state); in apple_nvme_reset_work()
997 if (apple_rtkit_is_crashed(anv->rtk)) { in apple_nvme_reset_work()
998 dev_err(anv->dev, in apple_nvme_reset_work()
1005 if (apple_rtkit_is_running(anv->rtk)) { in apple_nvme_reset_work()
1007 if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) in apple_nvme_reset_work()
1008 apple_nvme_disable(anv, false); in apple_nvme_reset_work()
1009 dev_dbg(anv->dev, "Trying to shut down RTKit before reset."); in apple_nvme_reset_work()
1010 ret = apple_rtkit_shutdown(anv->rtk); in apple_nvme_reset_work()
1015 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_reset_work()
1017 ret = reset_control_assert(anv->reset); in apple_nvme_reset_work()
1021 ret = apple_rtkit_reinit(anv->rtk); in apple_nvme_reset_work()
1025 ret = reset_control_deassert(anv->reset); in apple_nvme_reset_work()
1030 anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_reset_work()
1031 ret = apple_rtkit_boot(anv->rtk); in apple_nvme_reset_work()
1033 dev_err(anv->dev, "ANS did not boot"); in apple_nvme_reset_work()
1037 ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS, in apple_nvme_reset_work()
1042 dev_err(anv->dev, "ANS did not initialize"); in apple_nvme_reset_work()
1046 dev_dbg(anv->dev, "ANS booted successfully."); in apple_nvme_reset_work()
1052 anv->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_KB_SZ << 1, in apple_nvme_reset_work()
1053 dma_max_mapping_size(anv->dev) >> 9); in apple_nvme_reset_work()
1054 anv->ctrl.max_segments = NVME_MAX_SEGS; in apple_nvme_reset_work()
1056 dma_set_max_seg_size(anv->dev, 0xffffffff); in apple_nvme_reset_work()
1066 anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); in apple_nvme_reset_work()
1070 anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL); in apple_nvme_reset_work()
1074 anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); in apple_nvme_reset_work()
1082 writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & in apple_nvme_reset_work()
1084 anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); in apple_nvme_reset_work()
1089 writel(aqa, anv->mmio_nvme + NVME_REG_AQA); in apple_nvme_reset_work()
1090 writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); in apple_nvme_reset_work()
1091 writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ); in apple_nvme_reset_work()
1094 writeq(anv->adminq.tcb_dma_addr, in apple_nvme_reset_work()
1095 anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE); in apple_nvme_reset_work()
1096 writeq(anv->ioq.tcb_dma_addr, in apple_nvme_reset_work()
1097 anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE); in apple_nvme_reset_work()
1099 anv->ctrl.sqsize = in apple_nvme_reset_work()
1101 anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); in apple_nvme_reset_work()
1103 dev_dbg(anv->dev, "Enabling controller now"); in apple_nvme_reset_work()
1104 ret = nvme_enable_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1108 dev_dbg(anv->dev, "Starting admin queue"); in apple_nvme_reset_work()
1109 apple_nvme_init_queue(&anv->adminq); in apple_nvme_reset_work()
1110 nvme_unquiesce_admin_queue(&anv->ctrl); in apple_nvme_reset_work()
1112 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_CONNECTING)) { in apple_nvme_reset_work()
1113 dev_warn(anv->ctrl.device, in apple_nvme_reset_work()
1119 ret = nvme_init_ctrl_finish(&anv->ctrl, false); in apple_nvme_reset_work()
1123 dev_dbg(anv->dev, "Creating IOCQ"); in apple_nvme_reset_work()
1124 ret = apple_nvme_create_cq(anv); in apple_nvme_reset_work()
1127 dev_dbg(anv->dev, "Creating IOSQ"); in apple_nvme_reset_work()
1128 ret = apple_nvme_create_sq(anv); in apple_nvme_reset_work()
1132 apple_nvme_init_queue(&anv->ioq); in apple_nvme_reset_work()
1134 ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues); in apple_nvme_reset_work()
1142 anv->ctrl.queue_count = nr_io_queues + 1; in apple_nvme_reset_work()
1144 nvme_unquiesce_io_queues(&anv->ctrl); in apple_nvme_reset_work()
1145 nvme_wait_freeze(&anv->ctrl); in apple_nvme_reset_work()
1146 blk_mq_update_nr_hw_queues(&anv->tagset, 1); in apple_nvme_reset_work()
1147 nvme_unfreeze(&anv->ctrl); in apple_nvme_reset_work()
1149 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_LIVE)) { in apple_nvme_reset_work()
1150 dev_warn(anv->ctrl.device, in apple_nvme_reset_work()
1156 nvme_start_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1158 dev_dbg(anv->dev, "ANS boot and NVMe init completed."); in apple_nvme_reset_work()
1162 apple_nvme_remove_sq(anv); in apple_nvme_reset_work()
1164 apple_nvme_remove_cq(anv); in apple_nvme_reset_work()
1166 dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret); in apple_nvme_reset_work()
1167 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); in apple_nvme_reset_work()
1168 nvme_get_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1169 apple_nvme_disable(anv, false); in apple_nvme_reset_work()
1170 nvme_mark_namespaces_dead(&anv->ctrl); in apple_nvme_reset_work()
1171 if (!queue_work(nvme_wq, &anv->remove_work)) in apple_nvme_reset_work()
1172 nvme_put_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1177 struct apple_nvme *anv = in apple_nvme_remove_dead_ctrl_work() local
1180 nvme_put_ctrl(&anv->ctrl); in apple_nvme_remove_dead_ctrl_work()
1181 device_release_driver(anv->dev); in apple_nvme_remove_dead_ctrl_work()
1211 struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); in apple_nvme_free_ctrl() local
1213 if (anv->ctrl.admin_q) in apple_nvme_free_ctrl()
1214 blk_put_queue(anv->ctrl.admin_q); in apple_nvme_free_ctrl()
1215 put_device(anv->dev); in apple_nvme_free_ctrl()
1231 struct apple_nvme *anv = data; in apple_nvme_async_probe() local
1233 flush_work(&anv->ctrl.reset_work); in apple_nvme_async_probe()
1234 flush_work(&anv->ctrl.scan_work); in apple_nvme_async_probe()
1235 nvme_put_ctrl(&anv->ctrl); in apple_nvme_async_probe()
1243 static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) in apple_nvme_alloc_tagsets() argument
1247 anv->admin_tagset.ops = &apple_nvme_mq_admin_ops; in apple_nvme_alloc_tagsets()
1248 anv->admin_tagset.nr_hw_queues = 1; in apple_nvme_alloc_tagsets()
1249 anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH; in apple_nvme_alloc_tagsets()
1250 anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT; in apple_nvme_alloc_tagsets()
1251 anv->admin_tagset.numa_node = NUMA_NO_NODE; in apple_nvme_alloc_tagsets()
1252 anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod); in apple_nvme_alloc_tagsets()
1253 anv->admin_tagset.flags = BLK_MQ_F_NO_SCHED; in apple_nvme_alloc_tagsets()
1254 anv->admin_tagset.driver_data = &anv->adminq; in apple_nvme_alloc_tagsets()
1256 ret = blk_mq_alloc_tag_set(&anv->admin_tagset); in apple_nvme_alloc_tagsets()
1259 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set, in apple_nvme_alloc_tagsets()
1260 &anv->admin_tagset); in apple_nvme_alloc_tagsets()
1264 anv->tagset.ops = &apple_nvme_mq_ops; in apple_nvme_alloc_tagsets()
1265 anv->tagset.nr_hw_queues = 1; in apple_nvme_alloc_tagsets()
1266 anv->tagset.nr_maps = 1; in apple_nvme_alloc_tagsets()
1272 anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; in apple_nvme_alloc_tagsets()
1273 anv->tagset.queue_depth = APPLE_ANS_MAX_QUEUE_DEPTH - 1; in apple_nvme_alloc_tagsets()
1274 anv->tagset.timeout = NVME_IO_TIMEOUT; in apple_nvme_alloc_tagsets()
1275 anv->tagset.numa_node = NUMA_NO_NODE; in apple_nvme_alloc_tagsets()
1276 anv->tagset.cmd_size = sizeof(struct apple_nvme_iod); in apple_nvme_alloc_tagsets()
1277 anv->tagset.flags = BLK_MQ_F_SHOULD_MERGE; in apple_nvme_alloc_tagsets()
1278 anv->tagset.driver_data = &anv->ioq; in apple_nvme_alloc_tagsets()
1280 ret = blk_mq_alloc_tag_set(&anv->tagset); in apple_nvme_alloc_tagsets()
1283 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set, in apple_nvme_alloc_tagsets()
1284 &anv->tagset); in apple_nvme_alloc_tagsets()
1288 anv->ctrl.admin_tagset = &anv->admin_tagset; in apple_nvme_alloc_tagsets()
1289 anv->ctrl.tagset = &anv->tagset; in apple_nvme_alloc_tagsets()
1294 static int apple_nvme_queue_alloc(struct apple_nvme *anv, in apple_nvme_queue_alloc() argument
1299 q->cqes = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1305 q->sqes = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1315 q->tcbs = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1330 static void apple_nvme_detach_genpd(struct apple_nvme *anv) in apple_nvme_detach_genpd() argument
1334 if (anv->pd_count <= 1) in apple_nvme_detach_genpd()
1337 for (i = anv->pd_count - 1; i >= 0; i--) { in apple_nvme_detach_genpd()
1338 if (anv->pd_link[i]) in apple_nvme_detach_genpd()
1339 device_link_del(anv->pd_link[i]); in apple_nvme_detach_genpd()
1340 if (!IS_ERR_OR_NULL(anv->pd_dev[i])) in apple_nvme_detach_genpd()
1341 dev_pm_domain_detach(anv->pd_dev[i], true); in apple_nvme_detach_genpd()
1345 static int apple_nvme_attach_genpd(struct apple_nvme *anv) in apple_nvme_attach_genpd() argument
1347 struct device *dev = anv->dev; in apple_nvme_attach_genpd()
1350 anv->pd_count = of_count_phandle_with_args( in apple_nvme_attach_genpd()
1352 if (anv->pd_count <= 1) in apple_nvme_attach_genpd()
1355 anv->pd_dev = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_dev), in apple_nvme_attach_genpd()
1357 if (!anv->pd_dev) in apple_nvme_attach_genpd()
1360 anv->pd_link = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_link), in apple_nvme_attach_genpd()
1362 if (!anv->pd_link) in apple_nvme_attach_genpd()
1365 for (i = 0; i < anv->pd_count; i++) { in apple_nvme_attach_genpd()
1366 anv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); in apple_nvme_attach_genpd()
1367 if (IS_ERR(anv->pd_dev[i])) { in apple_nvme_attach_genpd()
1368 apple_nvme_detach_genpd(anv); in apple_nvme_attach_genpd()
1369 return PTR_ERR(anv->pd_dev[i]); in apple_nvme_attach_genpd()
1372 anv->pd_link[i] = device_link_add(dev, anv->pd_dev[i], in apple_nvme_attach_genpd()
1376 if (!anv->pd_link[i]) { in apple_nvme_attach_genpd()
1377 apple_nvme_detach_genpd(anv); in apple_nvme_attach_genpd()
1393 struct apple_nvme *anv; in apple_nvme_probe() local
1396 anv = devm_kzalloc(dev, sizeof(*anv), GFP_KERNEL); in apple_nvme_probe()
1397 if (!anv) in apple_nvme_probe()
1400 anv->dev = get_device(dev); in apple_nvme_probe()
1401 anv->adminq.is_adminq = true; in apple_nvme_probe()
1402 platform_set_drvdata(pdev, anv); in apple_nvme_probe()
1404 ret = apple_nvme_attach_genpd(anv); in apple_nvme_probe()
1414 anv->irq = platform_get_irq(pdev, 0); in apple_nvme_probe()
1415 if (anv->irq < 0) { in apple_nvme_probe()
1416 ret = anv->irq; in apple_nvme_probe()
1419 if (!anv->irq) { in apple_nvme_probe()
1424 anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans"); in apple_nvme_probe()
1425 if (IS_ERR(anv->mmio_coproc)) { in apple_nvme_probe()
1426 ret = PTR_ERR(anv->mmio_coproc); in apple_nvme_probe()
1429 anv->mmio_nvme = devm_platform_ioremap_resource_byname(pdev, "nvme"); in apple_nvme_probe()
1430 if (IS_ERR(anv->mmio_nvme)) { in apple_nvme_probe()
1431 ret = PTR_ERR(anv->mmio_nvme); in apple_nvme_probe()
1435 anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; in apple_nvme_probe()
1436 anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; in apple_nvme_probe()
1437 anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB; in apple_nvme_probe()
1438 anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; in apple_nvme_probe()
1440 anv->sart = devm_apple_sart_get(dev); in apple_nvme_probe()
1441 if (IS_ERR(anv->sart)) { in apple_nvme_probe()
1442 ret = dev_err_probe(dev, PTR_ERR(anv->sart), in apple_nvme_probe()
1447 anv->reset = devm_reset_control_array_get_exclusive(anv->dev); in apple_nvme_probe()
1448 if (IS_ERR(anv->reset)) { in apple_nvme_probe()
1449 ret = dev_err_probe(dev, PTR_ERR(anv->reset), in apple_nvme_probe()
1454 INIT_WORK(&anv->ctrl.reset_work, apple_nvme_reset_work); in apple_nvme_probe()
1455 INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work); in apple_nvme_probe()
1456 spin_lock_init(&anv->lock); in apple_nvme_probe()
1458 ret = apple_nvme_queue_alloc(anv, &anv->adminq); in apple_nvme_probe()
1461 ret = apple_nvme_queue_alloc(anv, &anv->ioq); in apple_nvme_probe()
1465 anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev, in apple_nvme_probe()
1468 if (!anv->prp_page_pool) { in apple_nvme_probe()
1473 anv->prp_small_pool = in apple_nvme_probe()
1474 dmam_pool_create("prp list 256", anv->dev, 256, 256, 0); in apple_nvme_probe()
1475 if (!anv->prp_small_pool) { in apple_nvme_probe()
1481 anv->iod_mempool = in apple_nvme_probe()
1483 if (!anv->iod_mempool) { in apple_nvme_probe()
1487 ret = devm_add_action_or_reset(anv->dev, in apple_nvme_probe()
1488 devm_apple_nvme_mempool_destroy, anv->iod_mempool); in apple_nvme_probe()
1492 ret = apple_nvme_alloc_tagsets(anv); in apple_nvme_probe()
1496 ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0, in apple_nvme_probe()
1497 "nvme-apple", anv); in apple_nvme_probe()
1503 anv->rtk = in apple_nvme_probe()
1504 devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops); in apple_nvme_probe()
1505 if (IS_ERR(anv->rtk)) { in apple_nvme_probe()
1506 ret = dev_err_probe(dev, PTR_ERR(anv->rtk), in apple_nvme_probe()
1511 ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, in apple_nvme_probe()
1518 anv->ctrl.admin_q = blk_mq_init_queue(&anv->admin_tagset); in apple_nvme_probe()
1519 if (IS_ERR(anv->ctrl.admin_q)) { in apple_nvme_probe()
1524 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_probe()
1525 async_schedule(apple_nvme_async_probe, anv); in apple_nvme_probe()
1530 put_device(anv->dev); in apple_nvme_probe()
1536 struct apple_nvme *anv = platform_get_drvdata(pdev); in apple_nvme_remove() local
1538 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); in apple_nvme_remove()
1539 flush_work(&anv->ctrl.reset_work); in apple_nvme_remove()
1540 nvme_stop_ctrl(&anv->ctrl); in apple_nvme_remove()
1541 nvme_remove_namespaces(&anv->ctrl); in apple_nvme_remove()
1542 apple_nvme_disable(anv, true); in apple_nvme_remove()
1543 nvme_uninit_ctrl(&anv->ctrl); in apple_nvme_remove()
1545 if (apple_rtkit_is_running(anv->rtk)) in apple_nvme_remove()
1546 apple_rtkit_shutdown(anv->rtk); in apple_nvme_remove()
1548 apple_nvme_detach_genpd(anv); in apple_nvme_remove()
1555 struct apple_nvme *anv = platform_get_drvdata(pdev); in apple_nvme_shutdown() local
1557 apple_nvme_disable(anv, true); in apple_nvme_shutdown()
1558 if (apple_rtkit_is_running(anv->rtk)) in apple_nvme_shutdown()
1559 apple_rtkit_shutdown(anv->rtk); in apple_nvme_shutdown()
1564 struct apple_nvme *anv = dev_get_drvdata(dev); in apple_nvme_resume() local
1566 return nvme_reset_ctrl(&anv->ctrl); in apple_nvme_resume()
1571 struct apple_nvme *anv = dev_get_drvdata(dev); in apple_nvme_suspend() local
1574 apple_nvme_disable(anv, true); in apple_nvme_suspend()
1576 if (apple_rtkit_is_running(anv->rtk)) in apple_nvme_suspend()
1577 ret = apple_rtkit_shutdown(anv->rtk); in apple_nvme_suspend()
1579 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_suspend()