34 #define cxl_doorbell_busy(cxlds)                                                \  argument
35 	(readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) &                  \
54 static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)  in cxl_pci_mbox_wait_for_doorbell()  argument
59 	while (cxl_doorbell_busy(cxlds)) {  in cxl_pci_mbox_wait_for_doorbell()
64 			if (!cxl_doorbell_busy(cxlds))  in cxl_pci_mbox_wait_for_doorbell()
71 	dev_dbg(cxlds->dev, "Doorbell wait took %dms",  in cxl_pci_mbox_wait_for_doorbell()
109 static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,  in __cxl_pci_mbox_send_cmd()  argument
112 	void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;  in __cxl_pci_mbox_send_cmd()
113 	struct device *dev = cxlds->dev;  in __cxl_pci_mbox_send_cmd()
118 	lockdep_assert_held(&cxlds->mbox_mutex);  in __cxl_pci_mbox_send_cmd()
138 	if (cxl_doorbell_busy(cxlds)) {  in __cxl_pci_mbox_send_cmd()
140 			readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);  in __cxl_pci_mbox_send_cmd()
142 		cxl_cmd_err(cxlds->dev, mbox_cmd, md_status,  in __cxl_pci_mbox_send_cmd()
159 	writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);  in __cxl_pci_mbox_send_cmd()
164 	       cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);  in __cxl_pci_mbox_send_cmd()
167 	rc = cxl_pci_mbox_wait_for_doorbell(cxlds);  in __cxl_pci_mbox_send_cmd()
169 		u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);  in __cxl_pci_mbox_send_cmd()
171 		cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout");  in __cxl_pci_mbox_send_cmd()
176 	status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET);  in __cxl_pci_mbox_send_cmd()
187 	cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);  in __cxl_pci_mbox_send_cmd()
199 		size_t n = min3(mbox_cmd->size_out, cxlds->payload_size, out_len);  in __cxl_pci_mbox_send_cmd()
210 static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)  in cxl_pci_mbox_send()  argument
214 	mutex_lock_io(&cxlds->mbox_mutex);  in cxl_pci_mbox_send()
215 	rc = __cxl_pci_mbox_send_cmd(cxlds, cmd);  in cxl_pci_mbox_send()
216 	mutex_unlock(&cxlds->mbox_mutex);  in cxl_pci_mbox_send()
221 static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)  in cxl_pci_setup_mailbox()  argument
223 	const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);  in cxl_pci_setup_mailbox()
229 		md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);  in cxl_pci_setup_mailbox()
237 		cxl_err(cxlds->dev, md_status,  in cxl_pci_setup_mailbox()
248 	if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) {  in cxl_pci_setup_mailbox()
249 		cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle");  in cxl_pci_setup_mailbox()
253 	cxlds->mbox_send = cxl_pci_mbox_send;  in cxl_pci_setup_mailbox()
254 	cxlds->payload_size =  in cxl_pci_setup_mailbox()
264 	cxlds->payload_size = min_t(size_t, cxlds->payload_size, SZ_1M);  in cxl_pci_setup_mailbox()
265 	if (cxlds->payload_size < 256) {  in cxl_pci_setup_mailbox()
266 		dev_err(cxlds->dev, "Mailbox is too small (%zub)",  in cxl_pci_setup_mailbox()
267 			cxlds->payload_size);  in cxl_pci_setup_mailbox()
271 	dev_dbg(cxlds->dev, "Mailbox payload sized %zu",  in cxl_pci_setup_mailbox()
272 		cxlds->payload_size);  in cxl_pci_setup_mailbox()
350 static int cxl_map_regs(struct cxl_dev_state *cxlds, struct cxl_register_map *map)  in cxl_map_regs()  argument
352 	struct device *dev = cxlds->dev;  in cxl_map_regs()
357 		cxl_map_component_regs(pdev, &cxlds->regs.component, map);  in cxl_map_regs()
361 		cxl_map_device_regs(pdev, &cxlds->regs.device_regs, map);  in cxl_map_regs()
395 static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds)  in devm_cxl_pci_create_doe()  argument
397 	struct device *dev = cxlds->dev;  in devm_cxl_pci_create_doe()
401 	xa_init(&cxlds->doe_mbs);  in devm_cxl_pci_create_doe()
402 	if (devm_add_action(&pdev->dev, cxl_pci_destroy_doe, &cxlds->doe_mbs)) {  in devm_cxl_pci_create_doe()
421 		if (xa_insert(&cxlds->doe_mbs, off, doe_mb, GFP_KERNEL)) {  in devm_cxl_pci_create_doe()
435 	struct cxl_dev_state *cxlds;  in cxl_pci_probe()  local
449 	cxlds = cxl_dev_state_create(&pdev->dev);  in cxl_pci_probe()
450 	if (IS_ERR(cxlds))  in cxl_pci_probe()
451 		return PTR_ERR(cxlds);  in cxl_pci_probe()
453 	cxlds->serial = pci_get_dsn(pdev);  in cxl_pci_probe()
454 	cxlds->cxl_dvsec = pci_find_dvsec_capability(  in cxl_pci_probe()
456 	if (!cxlds->cxl_dvsec)  in cxl_pci_probe()
464 	rc = cxl_map_regs(cxlds, &map);  in cxl_pci_probe()
472 	cxlds->component_reg_phys = CXL_RESOURCE_NONE;  in cxl_pci_probe()
477 	cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);  in cxl_pci_probe()
479 	devm_cxl_pci_create_doe(cxlds);  in cxl_pci_probe()
481 	rc = cxl_pci_setup_mailbox(cxlds);  in cxl_pci_probe()
485 	rc = cxl_enumerate_cmds(cxlds);  in cxl_pci_probe()
489 	rc = cxl_dev_state_identify(cxlds);  in cxl_pci_probe()
493 	rc = cxl_mem_create_range_info(cxlds);  in cxl_pci_probe()
497 	cxlmd = devm_cxl_add_memdev(cxlds);  in cxl_pci_probe()
501 	if (resource_size(&cxlds->pmem_res) && IS_ENABLED(CONFIG_CXL_PMEM))  in cxl_pci_probe()