48 static inline struct pci_dev *ctrl_dev(struct controller *ctrl)  in ctrl_dev()  argument
50 	return ctrl->pcie->port;  in ctrl_dev()
57 static inline int pciehp_request_irq(struct controller *ctrl)  in pciehp_request_irq()  argument
59 	int retval, irq = ctrl->pcie->irq;  in pciehp_request_irq()
62 		ctrl->poll_thread = kthread_run(&pciehp_poll, ctrl,  in pciehp_request_irq()
64 						slot_name(ctrl));  in pciehp_request_irq()
65 		return PTR_ERR_OR_ZERO(ctrl->poll_thread);  in pciehp_request_irq()
70 				      IRQF_SHARED, "pciehp", ctrl);  in pciehp_request_irq()
72 		ctrl_err(ctrl, "Cannot get irq %d for the hotplug controller\n",  in pciehp_request_irq()
77 static inline void pciehp_free_irq(struct controller *ctrl)  in pciehp_free_irq()  argument
80 		kthread_stop(ctrl->poll_thread);  in pciehp_free_irq()
82 		free_irq(ctrl->pcie->irq, ctrl);  in pciehp_free_irq()
85 static int pcie_poll_cmd(struct controller *ctrl, int timeout)  in pcie_poll_cmd()  argument
87 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pcie_poll_cmd()
93 			ctrl_info(ctrl, "%s: no response from device\n",  in pcie_poll_cmd()
101 			ctrl->cmd_busy = 0;  in pcie_poll_cmd()
111 static void pcie_wait_cmd(struct controller *ctrl)  in pcie_wait_cmd()  argument
115 	unsigned long cmd_timeout = ctrl->cmd_started + duration;  in pcie_wait_cmd()
123 	if (NO_CMD_CMPL(ctrl))  in pcie_wait_cmd()
126 	if (!ctrl->cmd_busy)  in pcie_wait_cmd()
139 	if (ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE &&  in pcie_wait_cmd()
140 	    ctrl->slot_ctrl & PCI_EXP_SLTCTL_CCIE)  in pcie_wait_cmd()
141 		rc = wait_event_timeout(ctrl->queue, !ctrl->cmd_busy, timeout);  in pcie_wait_cmd()
143 		rc = pcie_poll_cmd(ctrl, jiffies_to_msecs(timeout));  in pcie_wait_cmd()
146 		ctrl_info(ctrl, "Timeout on hotplug command %#06x (issued %u msec ago)\n",  in pcie_wait_cmd()
147 			  ctrl->slot_ctrl,  in pcie_wait_cmd()
148 			  jiffies_to_msecs(jiffies - ctrl->cmd_started));  in pcie_wait_cmd()
156 static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd,  in pcie_do_write_cmd()  argument
159 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pcie_do_write_cmd()
162 	mutex_lock(&ctrl->ctrl_lock);  in pcie_do_write_cmd()
167 	pcie_wait_cmd(ctrl);  in pcie_do_write_cmd()
171 		ctrl_info(ctrl, "%s: no response from device\n", __func__);  in pcie_do_write_cmd()
178 	ctrl->cmd_busy = 1;  in pcie_do_write_cmd()
180 	ctrl->slot_ctrl = slot_ctrl;  in pcie_do_write_cmd()
182 	ctrl->cmd_started = jiffies;  in pcie_do_write_cmd()
193 		ctrl->cmd_busy = 0;  in pcie_do_write_cmd()
200 		pcie_wait_cmd(ctrl);  in pcie_do_write_cmd()
203 	mutex_unlock(&ctrl->ctrl_lock);  in pcie_do_write_cmd()
208  * @ctrl: controller to which the command is issued
212 static void pcie_write_cmd(struct controller *ctrl, u16 cmd, u16 mask)  in pcie_write_cmd()  argument
214 	pcie_do_write_cmd(ctrl, cmd, mask, true);  in pcie_write_cmd()
218 static void pcie_write_cmd_nowait(struct controller *ctrl, u16 cmd, u16 mask)  in pcie_write_cmd_nowait()  argument
220 	pcie_do_write_cmd(ctrl, cmd, mask, false);  in pcie_write_cmd_nowait()
225  * @ctrl: PCIe hotplug controller
234 int pciehp_check_link_active(struct controller *ctrl)  in pciehp_check_link_active()  argument
236 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_check_link_active()
245 	ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status);  in pciehp_check_link_active()
290 int pciehp_check_link_status(struct controller *ctrl)  in pciehp_check_link_status()  argument
292 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_check_link_status()
297 		ctrl_info(ctrl, "Slot(%s): No link\n", slot_name(ctrl));  in pciehp_check_link_status()
301 	if (ctrl->inband_presence_disabled)  in pciehp_check_link_status()
304 	found = pci_bus_check_dev(ctrl->pcie->port->subordinate,  in pciehp_check_link_status()
310 			   &ctrl->pending_events);  in pciehp_check_link_status()
313 	ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status);  in pciehp_check_link_status()
316 		ctrl_info(ctrl, "Slot(%s): Cannot train link: status %#06x\n",  in pciehp_check_link_status()
317 			  slot_name(ctrl), lnk_status);  in pciehp_check_link_status()
321 	pcie_update_link_speed(ctrl->pcie->port->subordinate, lnk_status);  in pciehp_check_link_status()
324 		ctrl_info(ctrl, "Slot(%s): No device found\n",  in pciehp_check_link_status()
325 			  slot_name(ctrl));  in pciehp_check_link_status()
332 static int __pciehp_link_set(struct controller *ctrl, bool enable)  in __pciehp_link_set()  argument
334 	struct pci_dev *pdev = ctrl_dev(ctrl);  in __pciehp_link_set()
345 	ctrl_dbg(ctrl, "%s: lnk_ctrl = %x\n", __func__, lnk_ctrl);  in __pciehp_link_set()
349 static int pciehp_link_enable(struct controller *ctrl)  in pciehp_link_enable()  argument
351 	return __pciehp_link_set(ctrl, true);  in pciehp_link_enable()
357 	struct controller *ctrl = to_ctrl(hotplug_slot);  in pciehp_get_raw_indicator_status()  local
358 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_get_raw_indicator_status()
370 	struct controller *ctrl = to_ctrl(hotplug_slot);  in pciehp_get_attention_status()  local
371 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_get_attention_status()
377 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x, value read %x\n", __func__,  in pciehp_get_attention_status()
378 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, slot_ctrl);  in pciehp_get_attention_status()
398 void pciehp_get_power_status(struct controller *ctrl, u8 *status)  in pciehp_get_power_status()  argument
400 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_get_power_status()
404 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x value read %x\n", __func__,  in pciehp_get_power_status()
405 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, slot_ctrl);  in pciehp_get_power_status()
420 void pciehp_get_latch_status(struct controller *ctrl, u8 *status)  in pciehp_get_latch_status()  argument
422 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_get_latch_status()
431  * @ctrl: PCIe hotplug controller
441 int pciehp_card_present(struct controller *ctrl)  in pciehp_card_present()  argument
443 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_card_present()
456  * @ctrl: PCIe hotplug controller
466 int pciehp_card_present_or_link_active(struct controller *ctrl)  in pciehp_card_present_or_link_active()  argument
470 	ret = pciehp_card_present(ctrl);  in pciehp_card_present_or_link_active()
474 	return pciehp_check_link_active(ctrl);  in pciehp_card_present_or_link_active()
477 int pciehp_query_power_fault(struct controller *ctrl)  in pciehp_query_power_fault()  argument
479 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_query_power_fault()
489 	struct controller *ctrl = to_ctrl(hotplug_slot);  in pciehp_set_raw_indicator_status()  local
490 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_set_raw_indicator_status()
493 	pcie_write_cmd_nowait(ctrl, status << 6,  in pciehp_set_raw_indicator_status()
501  * @ctrl: PCIe hotplug controller
514 void pciehp_set_indicators(struct controller *ctrl, int pwr, int attn)  in pciehp_set_indicators()  argument
518 	if (PWR_LED(ctrl) && pwr != INDICATOR_NOOP) {  in pciehp_set_indicators()
523 	if (ATTN_LED(ctrl) && attn != INDICATOR_NOOP) {  in pciehp_set_indicators()
529 		pcie_write_cmd_nowait(ctrl, cmd, mask);  in pciehp_set_indicators()
530 		ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,  in pciehp_set_indicators()
531 			 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, cmd);  in pciehp_set_indicators()
535 int pciehp_power_on_slot(struct controller *ctrl)  in pciehp_power_on_slot()  argument
537 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_power_on_slot()
546 	ctrl->power_fault_detected = 0;  in pciehp_power_on_slot()
548 	pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_ON, PCI_EXP_SLTCTL_PCC);  in pciehp_power_on_slot()
549 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,  in pciehp_power_on_slot()
550 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,  in pciehp_power_on_slot()
553 	retval = pciehp_link_enable(ctrl);  in pciehp_power_on_slot()
555 		ctrl_err(ctrl, "%s: Can not enable the link!\n", __func__);  in pciehp_power_on_slot()
560 void pciehp_power_off_slot(struct controller *ctrl)  in pciehp_power_off_slot()  argument
562 	pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_OFF, PCI_EXP_SLTCTL_PCC);  in pciehp_power_off_slot()
563 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,  in pciehp_power_off_slot()
564 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,  in pciehp_power_off_slot()
568 static void pciehp_ignore_dpc_link_change(struct controller *ctrl,  in pciehp_ignore_dpc_link_change()  argument
576 	atomic_and(~PCI_EXP_SLTSTA_DLLSC, &ctrl->pending_events);  in pciehp_ignore_dpc_link_change()
580 	ctrl_info(ctrl, "Slot(%s): Link Down/Up ignored (recovered by DPC)\n",  in pciehp_ignore_dpc_link_change()
581 		  slot_name(ctrl));  in pciehp_ignore_dpc_link_change()
588 	down_read_nested(&ctrl->reset_lock, ctrl->depth);  in pciehp_ignore_dpc_link_change()
589 	if (!pciehp_check_link_active(ctrl))  in pciehp_ignore_dpc_link_change()
590 		pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC);  in pciehp_ignore_dpc_link_change()
591 	up_read(&ctrl->reset_lock);  in pciehp_ignore_dpc_link_change()
596 	struct controller *ctrl = (struct controller *)dev_id;  in pciehp_isr()  local
597 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_isr()
606 	    (!(ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE) && !pciehp_poll_mode))  in pciehp_isr()
619 			atomic_or(RERUN_ISR, &ctrl->pending_events);  in pciehp_isr()
627 		ctrl_info(ctrl, "%s: no response from device\n", __func__);  in pciehp_isr()
645 	if (ctrl->power_fault_detected)  in pciehp_isr()
648 		ctrl->power_fault_detected = true;  in pciehp_isr()
670 	ctrl_dbg(ctrl, "pending interrupts %#06x from Slot Status\n", events);  in pciehp_isr()
679 		ctrl->cmd_busy = 0;  in pciehp_isr()
681 		wake_up(&ctrl->queue);  in pciehp_isr()
690 		ctrl_dbg(ctrl, "ignoring hotplug event %#06x\n", events);  in pciehp_isr()
695 	atomic_or(events, &ctrl->pending_events);  in pciehp_isr()
701 	struct controller *ctrl = (struct controller *)dev_id;  in pciehp_ist()  local
702 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_ist()
706 	ctrl->ist_running = true;  in pciehp_ist()
710 	if (atomic_fetch_and(~RERUN_ISR, &ctrl->pending_events) & RERUN_ISR) {  in pciehp_ist()
718 	events = atomic_xchg(&ctrl->pending_events, 0);  in pciehp_ist()
726 		ctrl_info(ctrl, "Slot(%s): Attention button pressed\n",  in pciehp_ist()
727 			  slot_name(ctrl));  in pciehp_ist()
728 		pciehp_handle_button_press(ctrl);  in pciehp_ist()
733 		ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl));  in pciehp_ist()
734 		pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,  in pciehp_ist()
743 	    ctrl->state == ON_STATE) {  in pciehp_ist()
745 		pciehp_ignore_dpc_link_change(ctrl, pdev, irq);  in pciehp_ist()
752 	down_read_nested(&ctrl->reset_lock, ctrl->depth);  in pciehp_ist()
754 		pciehp_handle_disable_request(ctrl);  in pciehp_ist()
756 		pciehp_handle_presence_or_link_change(ctrl, events);  in pciehp_ist()
757 	up_read(&ctrl->reset_lock);  in pciehp_ist()
762 	ctrl->ist_running = false;  in pciehp_ist()
763 	wake_up(&ctrl->requester);  in pciehp_ist()
769 	struct controller *ctrl = data;  in pciehp_poll()  local
775 		while (pciehp_isr(IRQ_NOTCONNECTED, ctrl) == IRQ_WAKE_THREAD ||  in pciehp_poll()
776 		       atomic_read(&ctrl->pending_events))  in pciehp_poll()
777 			pciehp_ist(IRQ_NOTCONNECTED, ctrl);  in pciehp_poll()
788 static void pcie_enable_notification(struct controller *ctrl)  in pcie_enable_notification()  argument
809 	if (ATTN_BUTTN(ctrl))  in pcie_enable_notification()
821 	pcie_write_cmd_nowait(ctrl, cmd, mask);  in pcie_enable_notification()
822 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,  in pcie_enable_notification()
823 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, cmd);  in pcie_enable_notification()
826 static void pcie_disable_notification(struct controller *ctrl)  in pcie_disable_notification()  argument
834 	pcie_write_cmd(ctrl, 0, mask);  in pcie_disable_notification()
835 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,  in pcie_disable_notification()
836 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, 0);  in pcie_disable_notification()
839 void pcie_clear_hotplug_events(struct controller *ctrl)  in pcie_clear_hotplug_events()  argument
841 	pcie_capability_write_word(ctrl_dev(ctrl), PCI_EXP_SLTSTA,  in pcie_clear_hotplug_events()
845 void pcie_enable_interrupt(struct controller *ctrl)  in pcie_enable_interrupt()  argument
850 	pcie_write_cmd(ctrl, mask, mask);  in pcie_enable_interrupt()
853 void pcie_disable_interrupt(struct controller *ctrl)  in pcie_disable_interrupt()  argument
865 	pcie_write_cmd(ctrl, 0, mask);  in pcie_disable_interrupt()
880 	struct controller *ctrl = get_service_data(dev);  in pciehp_slot_reset()  local
882 	if (ctrl->state != ON_STATE)  in pciehp_slot_reset()
888 	if (!pciehp_check_link_active(ctrl))  in pciehp_slot_reset()
889 		pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC);  in pciehp_slot_reset()
904 	struct controller *ctrl = to_ctrl(hotplug_slot);  in pciehp_reset_slot()  local
905 	struct pci_dev *pdev = ctrl_dev(ctrl);  in pciehp_reset_slot()
912 	down_write_nested(&ctrl->reset_lock, ctrl->depth);  in pciehp_reset_slot()
914 	if (!ATTN_BUTTN(ctrl)) {  in pciehp_reset_slot()
921 	pcie_write_cmd(ctrl, 0, ctrl_mask);  in pciehp_reset_slot()
922 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,  in pciehp_reset_slot()
923 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, 0);  in pciehp_reset_slot()
925 	rc = pci_bridge_secondary_bus_reset(ctrl->pcie->port);  in pciehp_reset_slot()
928 	pcie_write_cmd_nowait(ctrl, ctrl_mask, ctrl_mask);  in pciehp_reset_slot()
929 	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,  in pciehp_reset_slot()
930 		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, ctrl_mask);  in pciehp_reset_slot()
932 	up_write(&ctrl->reset_lock);  in pciehp_reset_slot()
936 int pcie_init_notification(struct controller *ctrl)  in pcie_init_notification()  argument
938 	if (pciehp_request_irq(ctrl))  in pcie_init_notification()
940 	pcie_enable_notification(ctrl);  in pcie_init_notification()
941 	ctrl->notification_enabled = 1;  in pcie_init_notification()
945 void pcie_shutdown_notification(struct controller *ctrl)  in pcie_shutdown_notification()  argument
947 	if (ctrl->notification_enabled) {  in pcie_shutdown_notification()
948 		pcie_disable_notification(ctrl);  in pcie_shutdown_notification()
949 		pciehp_free_irq(ctrl);  in pcie_shutdown_notification()
950 		ctrl->notification_enabled = 0;  in pcie_shutdown_notification()
954 static inline void dbg_ctrl(struct controller *ctrl)  in dbg_ctrl()  argument
956 	struct pci_dev *pdev = ctrl->pcie->port;  in dbg_ctrl()
959 	ctrl_dbg(ctrl, "Slot Capabilities      : 0x%08x\n", ctrl->slot_cap);  in dbg_ctrl()
961 	ctrl_dbg(ctrl, "Slot Status            : 0x%04x\n", reg16);  in dbg_ctrl()
963 	ctrl_dbg(ctrl, "Slot Control           : 0x%04x\n", reg16);  in dbg_ctrl()
984 	struct controller *ctrl;  in pcie_init()  local
990 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);  in pcie_init()
991 	if (!ctrl)  in pcie_init()
994 	ctrl->pcie = dev;  in pcie_init()
995 	ctrl->depth = pcie_hotplug_depth(dev->port);  in pcie_init()
1008 	ctrl->slot_cap = slot_cap;  in pcie_init()
1009 	mutex_init(&ctrl->ctrl_lock);  in pcie_init()
1010 	mutex_init(&ctrl->state_lock);  in pcie_init()
1011 	init_rwsem(&ctrl->reset_lock);  in pcie_init()
1012 	init_waitqueue_head(&ctrl->requester);  in pcie_init()
1013 	init_waitqueue_head(&ctrl->queue);  in pcie_init()
1014 	INIT_DELAYED_WORK(&ctrl->button_work, pciehp_queue_pushbutton_work);  in pcie_init()
1015 	dbg_ctrl(ctrl);  in pcie_init()
1018 	ctrl->state = list_empty(&subordinate->devices) ? OFF_STATE : ON_STATE;  in pcie_init()
1023 		pcie_write_cmd_nowait(ctrl, PCI_EXP_SLTCTL_IBPD_DISABLE,  in pcie_init()
1025 		ctrl->inband_presence_disabled = 1;  in pcie_init()
1029 		ctrl->inband_presence_disabled = 1;  in pcie_init()
1040 …ctrl_info(ctrl, "Slot #%d AttnBtn%c PwrCtrl%c MRL%c AttnInd%c PwrInd%c HotPlug%c Surprise%c Interl…  in pcie_init()
1059 	if (POWER_CTRL(ctrl)) {  in pcie_init()
1060 		pciehp_get_power_status(ctrl, &poweron);  in pcie_init()
1061 		if (!pciehp_card_present_or_link_active(ctrl) && poweron) {  in pcie_init()
1062 			pcie_disable_notification(ctrl);  in pcie_init()
1063 			pciehp_power_off_slot(ctrl);  in pcie_init()
1067 	return ctrl;  in pcie_init()
1070 void pciehp_release_ctrl(struct controller *ctrl)  in pciehp_release_ctrl()  argument
1072 	cancel_delayed_work_sync(&ctrl->button_work);  in pciehp_release_ctrl()
1073 	kfree(ctrl);  in pciehp_release_ctrl()