1 /* SPDX-License-Identifier: GPL-2.0-or-later */
4  * Copyright (C) 2011-2013 Samsung India Software Operations
5  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
16 #include <linux/blk-crypto-profile.h>
17 #include <linux/blk-mq.h>
21 #include <linux/dma-direction.h>
24 #include <ufs/unipro.h>
25 #include <ufs/ufs.h>
26 #include <ufs/ufs_quirks.h>
27 #include <ufs/ufshci.h>
65  * struct uic_command - UIC command structure
89 /* Host <-> Device UniPro Link state */
97 #define ufshcd_is_link_off(hba) ((hba)->uic_link_state == UIC_LINK_OFF_STATE)
98 #define ufshcd_is_link_active(hba) ((hba)->uic_link_state == \
100 #define ufshcd_is_link_hibern8(hba) ((hba)->uic_link_state == \
102 #define ufshcd_is_link_broken(hba) ((hba)->uic_link_state == \
104 #define ufshcd_set_link_off(hba) ((hba)->uic_link_state = UIC_LINK_OFF_STATE)
105 #define ufshcd_set_link_active(hba) ((hba)->uic_link_state = \
107 #define ufshcd_set_link_hibern8(hba) ((hba)->uic_link_state = \
109 #define ufshcd_set_link_broken(hba) ((hba)->uic_link_state = \
113 	((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
115 	((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
117 	((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
119 	((h)->curr_dev_pwr_mode = UFS_DEEPSLEEP_PWR_MODE)
121 	((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
123 	((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
125 	((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
127 	((h)->curr_dev_pwr_mode == UFS_DEEPSLEEP_PWR_MODE)
130  * UFS Power management levels.
152  * struct ufshcd_lrb - local reference block
163  * @command_type: SCSI, UFS, Query.
171  * @crypto_key_slot: the key slot to use for inline crypto (-1 if none)
191 	u8 lun; /* UPIU LUN id field is only 8-bit wide */
206  * struct ufs_query_req - parameters for building a query request
216  * struct ufs_query_resp - UPIU QUERY
225  * struct ufs_query - holds relevant data structures for query request
237  * struct ufs_dev_cmd - all assosiated fields with device management commands
238  * @type: device management command type - Query, NOP OUT
251  * struct ufs_clk_info - UFS clock related info
252  * @list: list headed by hba->clk_list_head
294  * struct ufs_hba_variant_ops - variant specific callbacks
298  * @get_ufs_hci_version: called to get UFS HCI version
302  *                     variant specific Uni-Pro initialization.
304  *                       to allow variant specific Uni-Pro initialization.
319  * @device_reset: called to issue a reset pulse on the UFS device
385  * struct ufs_clk_gating - UFS clock gating related info
417  * struct ufs_clk_scaling - UFS clock scaling related data
419  * devfreq ->target() function is called then schedule "suspend_work" to
425  * @saved_pwr_info: UFS power mode may also be changed during scaling and this
459  * struct ufs_event_hist - keeps history of errors
473  * struct ufs_stats - keeps usage/err statistics
477  *		reset this after link-startup.
492  * enum ufshcd_state - UFS host controller state
523 	 * If UFS host controller is having issue in processing LCC (Line
526 	 * the LCC transmission on UFS device (by clearing TX_LCC_ENABLE
548 	 * is enabled, standard UFS host driver will call the vendor specific
584 	 * auto-hibernate capability but it doesn't work.
618 	 * 64-bit addressing supported capability but it doesn't work.
624 	 * auto-hibernate capability but it's FASTAUTO only.
662 	 * This capability allows host controller driver to use the UFS HCI's
664 	 * CAUTION: Enabling this might reduce overall UFS throughput.
669 	 * This capability allows the device auto-bkops to be always enabled
685 	 * This capability allows the host controller driver to turn-on
706 	 * if it is supported by the UFS device. The host controller driver must
707 	 * support device hardware reset via the hba->device_reset() callback,
714 	 * notification if it is supported by the UFS device.
752  * struct ufshcd_res_info_t - MCQ related resource regions
776  * struct ufshcd_mcq_opr_info_t - Operation and Runtime registers
797  * struct ufs_hba - per adapter private structure
799  * @ucdl_base_addr: UFS Command Descriptor base address
802  * @ucdl_dma_addr: UFS Command Descriptor DMA address
807  * @ufs_device_wlun: WLUN that controls the entire UFS device.
809  * @curr_dev_pwr_mode: active UFS device power mode.
810  * @uic_link_state: active state of the link to the UFS device.
811  * @rpm_lvl: desired UFS power management level during runtime PM.
812  * @spm_lvl: desired UFS power management level during system PM.
814  * @ahit: value of Auto-Hibernate Idle Timer register.
819  * @capabilities: UFS Controller Capabilities
820  * @mcq_capabilities: UFS Multi Circular Queue capabilities
824  * @ufs_version: UFS Version to which controller complies
830  * @is_irq_enabled: whether or not the UFS controller interrupt is enabled.
833  * @dev_quirks: bitmask with information about deviations from the UFS standard.
851  * @eh_work: Worker to handle UFS errors that require s/w attention
854  * @uic_error: UFS interconnect layer error status
861  * @dev_cmd: ufs device management command information
864  * @dev_info: information about the UFS device
866  * @vreg_info: UFS device voltage regulator information
867  * @clk_list_head: UFS host controller clocks list node head
869  * @lanes_per_direction: number of lanes per data direction between the UFS
870  *	controller and the UFS device.
874  * @caps: bitmask with information about UFS controller capabilities
876  * @clk_scaling: frequency scaling information owned by the UFS driver
879  * @is_sys_suspended: UFS device has been suspended because of system suspend
888  * @bsg_queue: BSG queue associated with the UFS controller
890  *	management) after the UFS device has finished a WriteBooster buffer
892  * @monitor: statistics about UFS commands
897  * @debugfs_root: UFS controller debugfs root directory
901  * @luns_avail: number of regular and well known LUNs supported by the UFS
938 	/* Desired UFS power management level during runtime PM */
940 	/* Desired UFS power management level during system PM */
944 	/* Auto-Hibernate Idle Timer register value */
971 	/* Device deviations from standard UFS device spec. */
1013 	/* Keeps information of the UFS device connected to this host */
1076  * struct ufs_hw_queue - per hardware queue structure
1117 	return hba->mcq_enabled;  in is_mcq_enabled()
1123 	return hba->sg_entry_size;  in ufshcd_sg_entry_size()
1129 	hba->sg_entry_size = sg_entry_size;  in ufshcd_set_sg_entry_size()
1149 	return hba->caps & UFSHCD_CAP_CLK_GATING;  in ufshcd_is_clkgating_allowed()
1153 	return hba->caps & UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;  in ufshcd_can_hibern8_during_gating()
1157 	return hba->caps & UFSHCD_CAP_CLK_SCALING;  in ufshcd_is_clkscaling_supported()
1161 	return hba->caps & UFSHCD_CAP_AUTO_BKOPS_SUSPEND;  in ufshcd_can_autobkops_during_suspend()
1165 	return hba->caps & UFSHCD_CAP_RPM_AUTOSUSPEND;  in ufshcd_is_rpm_autosuspend_allowed()
1170 	return (hba->caps & UFSHCD_CAP_INTR_AGGR) &&  in ufshcd_is_intr_aggr_allowed()
1171 		!(hba->quirks & UFSHCD_QUIRK_BROKEN_INTR_AGGR);  in ufshcd_is_intr_aggr_allowed()
1177 		  (hba->caps & UFSHCD_CAP_AGGR_POWER_COLLAPSE));  in ufshcd_can_aggressive_pc()
1182 	return (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) &&  in ufshcd_is_auto_hibern8_supported()
1183 		!(hba->quirks & UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8);  in ufshcd_is_auto_hibern8_supported()
1188 	return FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, hba->ahit);  in ufshcd_is_auto_hibern8_enabled()
1193 	return hba->caps & UFSHCD_CAP_WB_EN;  in ufshcd_is_wb_allowed()
1198 	return hba->caps & UFSHCD_CAP_WB_WITH_CLK_SCALING;  in ufshcd_enable_wb_if_scaling_up()
1202 	writel((val), (hba)->mcq_base + (reg))
1204 	readl((hba)->mcq_base + (reg))
1207 	writel_relaxed((val), (hba)->mcq_base + (reg))
1209 	readl_relaxed((hba)->mcq_base + (reg))
1212 	writel((val), (hba)->mmio_base + (reg))
1214 	readl((hba)->mmio_base + (reg))
1217  * ufshcd_rmwl - perform read/modify/write for a controller register
1257  * ufshcd_set_variant - set variant specific data to the hba
1264 	hba->priv = variant;  in ufshcd_set_variant()
1268  * ufshcd_get_variant - get variant specific data from the hba
1274 	return hba->priv;  in ufshcd_get_variant()
1348 	return (pwr_info->pwr_rx == FAST_MODE ||  in ufshcd_is_hs_mode()
1349 		pwr_info->pwr_rx == FASTAUTO_MODE) &&  in ufshcd_is_hs_mode()
1350 		(pwr_info->pwr_tx == FAST_MODE ||  in ufshcd_is_hs_mode()
1351 		pwr_info->pwr_tx == FASTAUTO_MODE);  in ufshcd_is_hs_mode()
1390 /* Wrapper functions for safely calling variant operations */
1393 	if (hba->vops && hba->vops->init)  in ufshcd_vops_init()
1394 		return hba->vops->init(hba);  in ufshcd_vops_init()
1401 	if (hba->vops && hba->vops->phy_initialization)  in ufshcd_vops_phy_initialization()
1402 		return hba->vops->phy_initialization(hba);  in ufshcd_vops_phy_initialization()