22 #include "ipa.h"
38  * This driver supports the Qualcomm IP Accelerator (IPA), which is a
39  * networking component found in many Qualcomm SoCs.  The IPA is connected
43  * The IPA is the conduit between the AP and the modem that carries network
47  * The IPA provides protocol checksum calculation, offloading this work
48  * from the AP.  The IPA offers additional functionality, including routing,
54  * There are two distinct layers that implement the IPA hardware, and this
56  * interface (GSI) is an integral component of the IPA, providing a
57  * well-defined communication layer between the AP subsystem and the IPA
59  * between the AP and the IPA.
61  * The IPA layer uses GSI channels to implement its "endpoints".  And while
62  * a GSI channel carries data between the AP and the IPA, a pair of IPA
74  * ipa_suspend_handler() - Handle the suspend IPA interrupt
75  * @ipa:	IPA pointer
76  * @irq_id:	IPA interrupt type (unused)
78  * If an RX endpoint is in suspend state, and the IPA has a packet
79  * destined for that endpoint, the IPA generates a SUSPEND interrupt
83 static void ipa_suspend_handler(struct ipa *ipa, enum ipa_irq_id irq_id)  in ipa_suspend_handler()  argument
89 	if (!test_and_set_bit(IPA_FLAG_RESUMED, ipa->flags))  in ipa_suspend_handler()
90 		pm_wakeup_dev_event(&ipa->pdev->dev, 0, true);  in ipa_suspend_handler()
93 	ipa_interrupt_suspend_clear_all(ipa->interrupt);  in ipa_suspend_handler()
97  * ipa_setup() - Set up IPA hardware
98  * @ipa:	IPA pointer
107 int ipa_setup(struct ipa *ipa)  in ipa_setup()  argument
111 	struct device *dev = &ipa->pdev->dev;  in ipa_setup()
114 	/* Setup for IPA v3.5.1 has some slight differences */  in ipa_setup()
115 	ret = gsi_setup(&ipa->gsi, ipa->version == IPA_VERSION_3_5_1);  in ipa_setup()
119 	ipa->interrupt = ipa_interrupt_setup(ipa);  in ipa_setup()
120 	if (IS_ERR(ipa->interrupt)) {  in ipa_setup()
121 		ret = PTR_ERR(ipa->interrupt);  in ipa_setup()
124 	ipa_interrupt_add(ipa->interrupt, IPA_IRQ_TX_SUSPEND,  in ipa_setup()
127 	ipa_uc_setup(ipa);  in ipa_setup()
133 	ipa_endpoint_setup(ipa);  in ipa_setup()
138 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];  in ipa_setup()
143 	ret = ipa_mem_setup(ipa);  in ipa_setup()
147 	ret = ipa_table_setup(ipa);  in ipa_setup()
154 	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];  in ipa_setup()
159 	ipa_endpoint_default_route_set(ipa, exception_endpoint->endpoint_id);  in ipa_setup()
162 	ret = ipa_modem_setup(ipa);  in ipa_setup()
166 	ipa->setup_complete = true;  in ipa_setup()
168 	dev_info(dev, "IPA driver setup completed successfully\n");  in ipa_setup()
173 	ipa_endpoint_default_route_clear(ipa);  in ipa_setup()
176 	ipa_table_teardown(ipa);  in ipa_setup()
178 	ipa_mem_teardown(ipa);  in ipa_setup()
182 	ipa_endpoint_teardown(ipa);  in ipa_setup()
185 	ipa_uc_teardown(ipa);  in ipa_setup()
186 	ipa_interrupt_remove(ipa->interrupt, IPA_IRQ_TX_SUSPEND);  in ipa_setup()
187 	ipa_interrupt_teardown(ipa->interrupt);  in ipa_setup()
189 	gsi_teardown(&ipa->gsi);  in ipa_setup()
196  * @ipa:	IPA pointer
198 static void ipa_teardown(struct ipa *ipa)  in ipa_teardown()  argument
203 	ipa_modem_teardown(ipa);  in ipa_teardown()
204 	ipa_endpoint_default_route_clear(ipa);  in ipa_teardown()
205 	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];  in ipa_teardown()
207 	ipa_table_teardown(ipa);  in ipa_teardown()
208 	ipa_mem_teardown(ipa);  in ipa_teardown()
209 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];  in ipa_teardown()
211 	ipa_endpoint_teardown(ipa);  in ipa_teardown()
212 	(void)device_init_wakeup(&ipa->pdev->dev, false);  in ipa_teardown()
213 	ipa_uc_teardown(ipa);  in ipa_teardown()
214 	ipa_interrupt_remove(ipa->interrupt, IPA_IRQ_TX_SUSPEND);  in ipa_teardown()
215 	ipa_interrupt_teardown(ipa->interrupt);  in ipa_teardown()
216 	gsi_teardown(&ipa->gsi);  in ipa_teardown()
220 static void ipa_hardware_config_comp(struct ipa *ipa)  in ipa_hardware_config_comp()  argument
224 	/* Nothing to configure for IPA v3.5.1 */  in ipa_hardware_config_comp()
225 	if (ipa->version == IPA_VERSION_3_5_1)  in ipa_hardware_config_comp()
228 	val = ioread32(ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET);  in ipa_hardware_config_comp()
230 	if (ipa->version == IPA_VERSION_4_0) {  in ipa_hardware_config_comp()
241 	iowrite32(val, ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET);  in ipa_hardware_config_comp()
245 static void ipa_hardware_config_qsb(struct ipa *ipa)  in ipa_hardware_config_qsb()  argument
251 	if (ipa->version == IPA_VERSION_4_2)  in ipa_hardware_config_qsb()
255 	iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_WRITES_OFFSET);  in ipa_hardware_config_qsb()
257 	if (ipa->version == IPA_VERSION_3_5_1) {  in ipa_hardware_config_qsb()
262 		if (ipa->version == IPA_VERSION_4_2)  in ipa_hardware_config_qsb()
269 	iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_READS_OFFSET);  in ipa_hardware_config_qsb()
272 static void ipa_idle_indication_cfg(struct ipa *ipa,  in ipa_idle_indication_cfg()  argument
284 	offset = ipa_reg_idle_indication_cfg_offset(ipa->version);  in ipa_idle_indication_cfg()
285 	iowrite32(val, ipa->reg_virt + offset);  in ipa_idle_indication_cfg()
289  * ipa_hardware_dcd_config() - Enable dynamic clock division on IPA
290  * @ipa:	IPA pointer
292  * Configures when the IPA signals it is idle to the global clock
296 static void ipa_hardware_dcd_config(struct ipa *ipa)  in ipa_hardware_dcd_config()  argument
298 	/* Recommended values for IPA 3.5 according to IPA HPG */  in ipa_hardware_dcd_config()
299 	ipa_idle_indication_cfg(ipa, 256, false);  in ipa_hardware_dcd_config()
302 static void ipa_hardware_dcd_deconfig(struct ipa *ipa)  in ipa_hardware_dcd_deconfig()  argument
305 	ipa_idle_indication_cfg(ipa, 0, true);  in ipa_hardware_dcd_deconfig()
310  * @ipa:	IPA pointer
312 static void ipa_hardware_config(struct ipa *ipa)  in ipa_hardware_config()  argument
318 	val = ipa_reg_bcr_val(ipa->version);  in ipa_hardware_config()
319 	iowrite32(val, ipa->reg_virt + IPA_REG_BCR_OFFSET);  in ipa_hardware_config()
321 	if (ipa->version != IPA_VERSION_3_5_1) {  in ipa_hardware_config()
325 		iowrite32(val, ipa->reg_virt + IPA_REG_CLKON_CFG_OFFSET);  in ipa_hardware_config()
328 		val = ioread32(ipa->reg_virt + IPA_REG_TX_CFG_OFFSET);  in ipa_hardware_config()
330 		iowrite32(val, ipa->reg_virt + IPA_REG_TX_CFG_OFFSET);  in ipa_hardware_config()
333 	ipa_hardware_config_comp(ipa);  in ipa_hardware_config()
336 	ipa_hardware_config_qsb(ipa);  in ipa_hardware_config()
339 	val = ioread32(ipa->reg_virt + IPA_REG_COUNTER_CFG_OFFSET);  in ipa_hardware_config()
342 	iowrite32(val, ipa->reg_virt + IPA_REG_COUNTER_CFG_OFFSET);  in ipa_hardware_config()
344 	/* Disable hashed IPv4 and IPv6 routing and filtering for IPA v4.2 */  in ipa_hardware_config()
345 	if (ipa->version == IPA_VERSION_4_2)  in ipa_hardware_config()
346 		iowrite32(0, ipa->reg_virt + IPA_REG_FILT_ROUT_HASH_EN_OFFSET);  in ipa_hardware_config()
349 	ipa_hardware_dcd_config(ipa);  in ipa_hardware_config()
354  * @ipa:	IPA pointer
358 static void ipa_hardware_deconfig(struct ipa *ipa)  in ipa_hardware_deconfig()  argument
361 	ipa_hardware_dcd_deconfig(ipa);  in ipa_hardware_deconfig()
366 /* # IPA resources used based on version (see IPA_RESOURCE_GROUP_COUNT) */
367 static int ipa_resource_group_count(struct ipa *ipa)  in ipa_resource_group_count()  argument
369 	switch (ipa->version) {  in ipa_resource_group_count()
385 static bool ipa_resource_limits_valid(struct ipa *ipa,  in ipa_resource_limits_valid()  argument
388 	u32 group_count = ipa_resource_group_count(ipa);  in ipa_resource_limits_valid()
421 static bool ipa_resource_limits_valid(struct ipa *ipa,  in ipa_resource_limits_valid()  argument
430 ipa_resource_config_common(struct ipa *ipa, u32 offset,  in ipa_resource_config_common()  argument
441 	iowrite32(val, ipa->reg_virt + offset);  in ipa_resource_config_common()
444 static void ipa_resource_config_src_01(struct ipa *ipa,  in ipa_resource_config_src_01()  argument
449 	ipa_resource_config_common(ipa, offset,  in ipa_resource_config_src_01()
453 static void ipa_resource_config_src_23(struct ipa *ipa,  in ipa_resource_config_src_23()  argument
458 	ipa_resource_config_common(ipa, offset,  in ipa_resource_config_src_23()
462 static void ipa_resource_config_dst_01(struct ipa *ipa,  in ipa_resource_config_dst_01()  argument
467 	ipa_resource_config_common(ipa, offset,  in ipa_resource_config_dst_01()
471 static void ipa_resource_config_dst_23(struct ipa *ipa,  in ipa_resource_config_dst_23()  argument
476 	ipa_resource_config_common(ipa, offset,  in ipa_resource_config_dst_23()
481 ipa_resource_config(struct ipa *ipa, const struct ipa_resource_data *data)  in ipa_resource_config()  argument
485 	if (!ipa_resource_limits_valid(ipa, data))  in ipa_resource_config()
489 		ipa_resource_config_src_01(ipa, &data->resource_src[i]);  in ipa_resource_config()
490 		ipa_resource_config_src_23(ipa, &data->resource_src[i]);  in ipa_resource_config()
494 		ipa_resource_config_dst_01(ipa, &data->resource_dst[i]);  in ipa_resource_config()
495 		ipa_resource_config_dst_23(ipa, &data->resource_dst[i]);  in ipa_resource_config()
501 static void ipa_resource_deconfig(struct ipa *ipa)  in ipa_resource_deconfig()  argument
507  * ipa_config() - Configure IPA hardware
508  * @ipa:	IPA pointer
509  * @data:	IPA configuration data
511  * Perform initialization requiring IPA clock to be enabled.
513 static int ipa_config(struct ipa *ipa, const struct ipa_data *data)  in ipa_config()  argument
521 	ipa_clock_get(ipa);  in ipa_config()
523 	ipa_hardware_config(ipa);  in ipa_config()
525 	ret = ipa_endpoint_config(ipa);  in ipa_config()
529 	ret = ipa_mem_config(ipa);  in ipa_config()
533 	ipa_table_config(ipa);  in ipa_config()
536 	ret = ipa_resource_config(ipa, data->resource_data);  in ipa_config()
540 	ret = ipa_modem_config(ipa);  in ipa_config()
547 	ipa_resource_deconfig(ipa);  in ipa_config()
549 	ipa_table_deconfig(ipa);  in ipa_config()
550 	ipa_mem_deconfig(ipa);  in ipa_config()
552 	ipa_endpoint_deconfig(ipa);  in ipa_config()
554 	ipa_hardware_deconfig(ipa);  in ipa_config()
555 	ipa_clock_put(ipa);  in ipa_config()
562  * @ipa:	IPA pointer
564 static void ipa_deconfig(struct ipa *ipa)  in ipa_deconfig()  argument
566 	ipa_modem_deconfig(ipa);  in ipa_deconfig()
567 	ipa_resource_deconfig(ipa);  in ipa_deconfig()
568 	ipa_table_deconfig(ipa);  in ipa_deconfig()
569 	ipa_mem_deconfig(ipa);  in ipa_deconfig()
570 	ipa_endpoint_deconfig(ipa);  in ipa_deconfig()
571 	ipa_hardware_deconfig(ipa);  in ipa_deconfig()
572 	ipa_clock_put(ipa);  in ipa_deconfig()
630 		.compatible	= "qcom,sdm845-ipa",
634 		.compatible	= "qcom,sc7180-ipa",
695  * ipa_probe() - IPA platform driver probe function
701  * This is the main entry point for the IPA driver.  Initialization proceeds
704  *     access to the IPA hardware.
705  *   - The "config" stage requires the IPA clock to be active so IPA registers
706  *     can be accessed, but does not require the use of IPA immediate commands.
707  *   - The "setup" stage uses IPA immediate commands, and so requires the GSI
715  * to the AP; this triggers the start if IPA setup.
725 	struct ipa *ipa;  in ipa_probe()  local
767 	/* Allocate and initialize the IPA structure */  in ipa_probe()
768 	ipa = kzalloc(sizeof(*ipa), GFP_KERNEL);  in ipa_probe()
769 	if (!ipa) {  in ipa_probe()
774 	ipa->pdev = pdev;  in ipa_probe()
775 	dev_set_drvdata(dev, ipa);  in ipa_probe()
776 	ipa->modem_rproc = rproc;  in ipa_probe()
777 	ipa->clock = clock;  in ipa_probe()
778 	ipa->version = data->version;  in ipa_probe()
780 	ret = ipa_reg_init(ipa);  in ipa_probe()
784 	ret = ipa_mem_init(ipa, data->mem_data);  in ipa_probe()
788 	/* GSI v2.0+ (IPA v4.0+) uses prefetch for the command channel */  in ipa_probe()
789 	prefetch = ipa->version != IPA_VERSION_3_5_1;  in ipa_probe()
790 	/* IPA v4.2 requires the AP to allocate channels for the modem */  in ipa_probe()
791 	modem_alloc = ipa->version == IPA_VERSION_4_2;  in ipa_probe()
793 	ret = gsi_init(&ipa->gsi, pdev, prefetch, data->endpoint_count,  in ipa_probe()
799 	ipa->filter_map = ipa_endpoint_init(ipa, data->endpoint_count,  in ipa_probe()
801 	if (!ipa->filter_map) {  in ipa_probe()
806 	ret = ipa_table_init(ipa);  in ipa_probe()
810 	ret = ipa_modem_init(ipa, modem_init);  in ipa_probe()
814 	ret = ipa_config(ipa, data);  in ipa_probe()
818 	dev_info(dev, "IPA driver initialized");  in ipa_probe()
834 	ret = ipa_setup(ipa);  in ipa_probe()
841 	ipa_deconfig(ipa);  in ipa_probe()
843 	ipa_modem_exit(ipa);  in ipa_probe()
845 	ipa_table_exit(ipa);  in ipa_probe()
847 	ipa_endpoint_exit(ipa);  in ipa_probe()
849 	gsi_exit(&ipa->gsi);  in ipa_probe()
851 	ipa_mem_exit(ipa);  in ipa_probe()
853 	ipa_reg_exit(ipa);  in ipa_probe()
855 	kfree(ipa);  in ipa_probe()
866 	struct ipa *ipa = dev_get_drvdata(&pdev->dev);  in ipa_remove()  local
867 	struct rproc *rproc = ipa->modem_rproc;  in ipa_remove()
868 	struct ipa_clock *clock = ipa->clock;  in ipa_remove()
871 	if (ipa->setup_complete) {  in ipa_remove()
872 		ret = ipa_modem_stop(ipa);  in ipa_remove()
876 		ipa_teardown(ipa);  in ipa_remove()
879 	ipa_deconfig(ipa);  in ipa_remove()
880 	ipa_modem_exit(ipa);  in ipa_remove()
881 	ipa_table_exit(ipa);  in ipa_remove()
882 	ipa_endpoint_exit(ipa);  in ipa_remove()
883 	gsi_exit(&ipa->gsi);  in ipa_remove()
884 	ipa_mem_exit(ipa);  in ipa_remove()
885 	ipa_reg_exit(ipa);  in ipa_remove()
886 	kfree(ipa);  in ipa_remove()
895  * @dev:	IPA device structure
901  * the IPA clock running until this point.
905 	struct ipa *ipa = dev_get_drvdata(dev);  in ipa_suspend()  local
908 	 * get an IPA SUSPEND interrupt.  We trigger a system resume in  in ipa_suspend()
911 	__clear_bit(IPA_FLAG_RESUMED, ipa->flags);  in ipa_suspend()
913 	ipa_endpoint_suspend(ipa);  in ipa_suspend()
915 	ipa_clock_put(ipa);  in ipa_suspend()
922  * @dev:	IPA device structure
927  * Takes an IPA clock reference to keep the clock running until suspend,
932 	struct ipa *ipa = dev_get_drvdata(dev);  in ipa_resume()  local
934 	/* This clock reference will keep the IPA out of suspend  in ipa_resume()
937 	ipa_clock_get(ipa);  in ipa_resume()
939 	ipa_endpoint_resume(ipa);  in ipa_resume()
953 		.name		= "ipa",