1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2009-2010 Intel Corporation
10  * Some Intel Ibex Peak based platforms support so-called "intelligent
34  *   - dual MCP configs
37  *   - handle CPU hotplug
38  *   - provide turbo enable/disable api
41  *   - CDI 403777, 403778 - Auburndale EDS vol 1 & 2
42  *   - CDI 401376 - Ibex Peak EDS
43  *   - ref 26037, 26641 - IPS BIOS spec
44  *   - ref 26489 - Nehalem BIOS writer's guide
45  *   - ref 26921 - Ibex Peak BIOS Specification
67 #include <linux/io-64-nonatomic-lo-hi.h>
228 #define thm_readb(off) readb(ips->regmap + (off))
229 #define thm_readw(off) readw(ips->regmap + (off))
230 #define thm_readl(off) readl(ips->regmap + (off))
231 #define thm_readq(off) readq(ips->regmap + (off))
233 #define thm_writeb(off, val) writeb((val), ips->regmap + (off))
234 #define thm_writew(off, val) writew((val), ips->regmap + (off))
235 #define thm_writel(off, val) writel((val), ips->regmap + (off))
242 static const int IPS_SAMPLE_WINDOW = 5000; /* 5s moving window of samples */
245 /* Per-SKU limits */
254 /* Max temps are -10 degrees C to avoid PROCHOT# */
338  * ips_cpu_busy - is CPU busy?
355  * ips_cpu_raise - raise CPU power clamp
369 	if (!ips->cpu_turbo_enabled)  in ips_cpu_raise()
378 	if (((new_tdp_limit * 10) / 8) > ips->core_power_limit)  in ips_cpu_raise()
393  * ips_cpu_lower - lower CPU power clamp
410 	new_limit = cur_limit - 8; /* 1W decrease */  in ips_cpu_lower()
413 	if (new_limit  < (ips->orig_turbo_limit & TURBO_TDP_MASK))  in ips_cpu_lower()
414 		new_limit = ips->orig_turbo_limit & TURBO_TDP_MASK;  in ips_cpu_lower()
428  * do_enable_cpu_turbo - internal turbo enable function
447  * ips_enable_cpu_turbo - enable turbo mode on all CPUs
456 	if (ips->__cpu_turbo_on)  in ips_enable_cpu_turbo()
459 	if (ips->turbo_toggle_allowed)  in ips_enable_cpu_turbo()
462 	ips->__cpu_turbo_on = true;  in ips_enable_cpu_turbo()
466  * do_disable_cpu_turbo - internal turbo disable function
485  * ips_disable_cpu_turbo - disable turbo mode on all CPUs
494 	if (!ips->__cpu_turbo_on)  in ips_disable_cpu_turbo()
497 	if (ips->turbo_toggle_allowed)  in ips_disable_cpu_turbo()
500 	ips->__cpu_turbo_on = false;  in ips_disable_cpu_turbo()
504  * ips_gpu_busy - is GPU busy?
518 	return ips->gpu_busy();  in ips_gpu_busy()
522  * ips_gpu_raise - raise GPU power clamp
533 	if (!ips->gpu_raise())  in ips_gpu_raise()
534 		ips->gpu_turbo_enabled = false;  in ips_gpu_raise()
540  * ips_gpu_lower - lower GPU power clamp
550 	if (!ips->gpu_lower())  in ips_gpu_lower()
551 		ips->gpu_turbo_enabled = false;  in ips_gpu_lower()
557  * ips_enable_gpu_turbo - notify the gfx driver turbo is available
565 	if (ips->__gpu_turbo_on)  in ips_enable_gpu_turbo()
567 	ips->__gpu_turbo_on = true;  in ips_enable_gpu_turbo()
571  * ips_disable_gpu_turbo - notify the gfx driver to disable turbo mode
579 	if (!ips->__gpu_turbo_on)  in ips_disable_gpu_turbo()
582 	if (!ips->gpu_turbo_disable())  in ips_disable_gpu_turbo()
583 		dev_err(ips->dev, "failed to disable graphics turbo\n");  in ips_disable_gpu_turbo()
585 		ips->__gpu_turbo_on = false;  in ips_disable_gpu_turbo()
589  * mcp_exceeded - check whether we're outside our thermal & power limits
601 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in mcp_exceeded()
603 	temp_limit = ips->mcp_temp_limit * 100;  in mcp_exceeded()
604 	if (ips->mcp_avg_temp > temp_limit)  in mcp_exceeded()
607 	avg_power = ips->cpu_avg_power + ips->mch_avg_power;  in mcp_exceeded()
608 	if (avg_power > ips->mcp_power_limit)  in mcp_exceeded()
611 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in mcp_exceeded()
617  * cpu_exceeded - check whether a CPU core is outside its limits
626 	int avg;  in cpu_exceeded()  local
629 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in cpu_exceeded()
630 	avg = cpu ? ips->ctv2_avg_temp : ips->ctv1_avg_temp;  in cpu_exceeded()
631 	if (avg > (ips->limits->core_temp_limit * 100))  in cpu_exceeded()
633 	if (ips->cpu_avg_power > ips->core_power_limit * 100)  in cpu_exceeded()
635 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in cpu_exceeded()
638 		dev_info(ips->dev, "CPU power or thermal limit exceeded\n");  in cpu_exceeded()
644  * mch_exceeded - check whether the GPU is over budget
654 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in mch_exceeded()
655 	if (ips->mch_avg_temp > (ips->limits->mch_temp_limit * 100))  in mch_exceeded()
657 	if (ips->mch_avg_power > ips->mch_power_limit)  in mch_exceeded()
659 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in mch_exceeded()
665  * verify_limits - verify BIOS provided limits
668  * BIOS can optionally provide non-default limits for power and temp.  Check
674 	if (ips->mcp_power_limit < ips->limits->mcp_power_limit ||  in verify_limits()
675 	    ips->mcp_power_limit > 35000)  in verify_limits()
676 		ips->mcp_power_limit = ips->limits->mcp_power_limit;  in verify_limits()
678 	if (ips->mcp_temp_limit < ips->limits->core_temp_limit ||  in verify_limits()
679 	    ips->mcp_temp_limit < ips->limits->mch_temp_limit ||  in verify_limits()
680 	    ips->mcp_temp_limit > 150)  in verify_limits()
681 		ips->mcp_temp_limit = min(ips->limits->core_temp_limit,  in verify_limits()
682 					  ips->limits->mch_temp_limit);  in verify_limits()
686  * update_turbo_limits - get various limits & settings from regs
693  * the regs for updates (as a result of AC->DC transition for example).
702 	ips->cpu_turbo_enabled = !(hts & HTS_PCTD_DIS);  in update_turbo_limits()
707 	ips->cpu_turbo_enabled = false;  in update_turbo_limits()
709 	if (ips->gpu_busy)  in update_turbo_limits()
710 		ips->gpu_turbo_enabled = !(hts & HTS_GTD_DIS);  in update_turbo_limits()
712 	ips->core_power_limit = thm_readw(THM_MPCPC);  in update_turbo_limits()
713 	ips->mch_power_limit = thm_readw(THM_MMGPC);  in update_turbo_limits()
714 	ips->mcp_temp_limit = thm_readw(THM_PTL);  in update_turbo_limits()
715 	ips->mcp_power_limit = thm_readw(THM_MPPC);  in update_turbo_limits()
722  * ips_adjust - adjust power clamp based on thermal state
728  *   - do we need to adjust up or down?
729  *   - is CPU busy?
730  *   - is GPU busy?
731  *   - is CPU in turbo?
732  *   - is GPU in turbo?
733  *   - is CPU or GPU preferred? (CPU is default)
736  *   - up (TDP available)
737  *     - CPU not busy, GPU not busy - nothing
738  *     - CPU busy, GPU not busy - adjust CPU up
739  *     - CPU not busy, GPU busy - adjust GPU up
740  *     - CPU busy, GPU busy - adjust preferred unit up, taking headroom from
741  *       non-preferred unit if necessary
742  *   - down (at TDP limit)
743  *     - adjust both CPU and GPU down if possible
745 		cpu+ gpu+	cpu+gpu-	cpu-gpu+	cpu-gpu-
747 cpu < gpu >=	cpu+gpu-(mcp<)	cpu+gpu-(mcp<)	gpu-		gpu-
748 cpu >= gpu <	cpu-gpu+(mcp<)	cpu-		cpu-gpu+(mcp<)	cpu-
749 cpu >= gpu >=	cpu-gpu-	cpu-gpu-	cpu-gpu-	cpu-gpu-
757 	dev_dbg(ips->dev, "starting ips-adjust thread\n");  in ips_adjust()
767 		spin_lock_irqsave(&ips->turbo_status_lock, flags);  in ips_adjust()
768 		if (ips->poll_turbo_status)  in ips_adjust()
770 		spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in ips_adjust()
773 		if (ips->cpu_turbo_enabled)  in ips_adjust()
778 		if (ips->gpu_turbo_enabled)  in ips_adjust()
804 	dev_dbg(ips->dev, "ips-adjust thread stopped\n");  in ips_adjust()
818 	u16 avg;  in calc_avg_temp()  local
825 	avg = (u16)total;  in calc_avg_temp()
827 	return avg;  in calc_avg_temp()
883 	ret = (((val - *last) * 1000) / period);  in get_cpu_power()
891 static u16 update_average_temp(u16 avg, u16 val)  in update_average_temp()  argument
897 		(((temp_decay_factor - 1) * avg) / temp_decay_factor);  in update_average_temp()
902 static u16 update_average_power(u32 avg, u32 val)  in update_average_power()  argument
907 		(((power_decay_factor - 1) * avg) / power_decay_factor);  in update_average_power()
915 	u32 avg;  in calc_avg_power()  local
922 	avg = (u32)total;  in calc_avg_power()
924 	return avg;  in calc_avg_power()
930 	wake_up_process(ips->monitor);  in monitor_timeout()
934  * ips_monitor - temp/power monitoring thread
961 		dev_err(ips->dev,  in ips_monitor()
969 		return -ENOMEM;  in ips_monitor()
999 		if (ips->read_mch_val) {  in ips_monitor()
1000 			mchp = ips->read_mch_val();  in ips_monitor()
1009 	ips->mcp_avg_temp = calc_avg_temp(ips, mcp_samples);  in ips_monitor()
1010 	ips->ctv1_avg_temp = calc_avg_temp(ips, ctv1_samples);  in ips_monitor()
1011 	ips->ctv2_avg_temp = calc_avg_temp(ips, ctv2_samples);  in ips_monitor()
1012 	ips->mch_avg_temp = calc_avg_temp(ips, mch_samples);  in ips_monitor()
1013 	ips->cpu_avg_power = calc_avg_power(ips, cpu_samples);  in ips_monitor()
1014 	ips->mch_avg_power = calc_avg_power(ips, mchp_samples);  in ips_monitor()
1023 	wake_up_process(ips->adjust);  in ips_monitor()
1026 	 * Ok, now we have an initial avg.  From here on out, we track the  in ips_monitor()
1027 	 * running avg using a decaying average calculation.  This allows  in ips_monitor()
1034 	timer_setup(&ips->timer, monitor_timeout, TIMER_DEFERRABLE);  in ips_monitor()
1041 		ips->mcp_avg_temp = update_average_temp(ips->mcp_avg_temp, val);  in ips_monitor()
1045 		ips->ctv1_avg_temp =  in ips_monitor()
1046 			update_average_temp(ips->ctv1_avg_temp, val);  in ips_monitor()
1050 		ips->cpu_avg_power =  in ips_monitor()
1051 			update_average_power(ips->cpu_avg_power, cpu_val);  in ips_monitor()
1053 		if (ips->second_cpu) {  in ips_monitor()
1056 			ips->ctv2_avg_temp =  in ips_monitor()
1057 				update_average_temp(ips->ctv2_avg_temp, val);  in ips_monitor()
1062 		ips->mch_avg_temp = update_average_temp(ips->mch_avg_temp, val);  in ips_monitor()
1064 		if (ips->read_mch_val) {  in ips_monitor()
1065 			mch_val = ips->read_mch_val();  in ips_monitor()
1066 			ips->mch_avg_power =  in ips_monitor()
1067 				update_average_power(ips->mch_avg_power,  in ips_monitor()
1081 			dev_warn(ips->dev,  in ips_monitor()
1092 		mod_timer(&ips->timer, expire);  in ips_monitor()
1096 		last_sample_period = jiffies_to_msecs(jiffies) - last_msecs;  in ips_monitor()
1101 	del_timer_sync(&ips->timer);  in ips_monitor()
1103 	dev_dbg(ips->dev, "ips-monitor thread stopped\n");  in ips_monitor()
1112 	dev_dbg(ips->dev, #reg ": 0x%04x\n", val); \
1117 	dev_dbg(ips->dev, #reg ": 0x%08x\n", val); \
1122 	dev_dbg(ips->dev, #reg ": 0x%016x\n", val); \
1130 	dev_dbg(ips->dev, "Processor temp limit: %d\n", ptl);
1142  * ips_irq_handler - handle temperature triggers and other IPS events
1159 	dev_info(ips->dev, "TSES: 0x%02x\n", tses);  in ips_irq_handler()
1160 	dev_info(ips->dev, "TES: 0x%02x\n", tes);  in ips_irq_handler()
1170 			spin_lock(&ips->turbo_status_lock);  in ips_irq_handler()
1171 			ips->core_power_limit = (sts & STS_PCPL_MASK) >>  in ips_irq_handler()
1173 			ips->mch_power_limit = (sts & STS_GPL_MASK) >>  in ips_irq_handler()
1176 			ips->cpu_turbo_enabled = !(sts & STS_PCTD_DIS);  in ips_irq_handler()
1181 			ips->cpu_turbo_enabled = false;  in ips_irq_handler()
1182 			if (ips->gpu_busy)  in ips_irq_handler()
1183 				ips->gpu_turbo_enabled = !(sts & STS_GTD_DIS);  in ips_irq_handler()
1184 			ips->mcp_temp_limit = (sts & STS_PTL_MASK) >>  in ips_irq_handler()
1186 			ips->mcp_power_limit = (tc1 & STS_PPL_MASK) >>  in ips_irq_handler()
1189 			spin_unlock(&ips->turbo_status_lock);  in ips_irq_handler()
1198 		dev_warn(ips->dev, "thermal trip occurred, tses: 0x%04x\n",  in ips_irq_handler()
1215 	struct ips_driver *ips = m->private;  in cpu_temp_show()
1217 	seq_printf(m, "%d.%02d\n", ips->ctv1_avg_temp / 100,  in cpu_temp_show()
1218 		   ips->ctv1_avg_temp % 100);  in cpu_temp_show()
1226 	struct ips_driver *ips = m->private;  in cpu_power_show()
1228 	seq_printf(m, "%dmW\n", ips->cpu_avg_power);  in cpu_power_show()
1258 	struct ips_driver *ips = m->private;  in mch_temp_show()
1260 	seq_printf(m, "%d.%02d\n", ips->mch_avg_temp / 100,  in mch_temp_show()
1261 		   ips->mch_avg_temp % 100);  in mch_temp_show()
1269 	struct ips_driver *ips = m->private;  in mch_power_show()
1271 	seq_printf(m, "%dmW\n", ips->mch_avg_power);  in mch_power_show()
1279 	debugfs_remove_recursive(ips->debug_root);  in ips_debugfs_cleanup()
1284 	ips->debug_root = debugfs_create_dir("ips", NULL);  in ips_debugfs_init()
1286 	debugfs_create_file("cpu_temp", 0444, ips->debug_root, ips, &cpu_temp_fops);  in ips_debugfs_init()
1287 	debugfs_create_file("cpu_power", 0444, ips->debug_root, ips, &cpu_power_fops);  in ips_debugfs_init()
1288 	debugfs_create_file("cpu_clamp", 0444, ips->debug_root, ips, &cpu_clamp_fops);  in ips_debugfs_init()
1289 	debugfs_create_file("mch_temp", 0444, ips->debug_root, ips, &mch_temp_fops);  in ips_debugfs_init()
1290 	debugfs_create_file("mch_power", 0444, ips->debug_root, ips, &mch_power_fops);  in ips_debugfs_init()
1295  * ips_detect_cpu - detect whether CPU supports IPS
1307 		dev_info(ips->dev, "Non-IPS CPU detected.\n");  in ips_detect_cpu()
1318 		ips->turbo_toggle_allowed = true;  in ips_detect_cpu()
1320 		ips->turbo_toggle_allowed = false;  in ips_detect_cpu()
1329 		dev_info(ips->dev, "No CPUID match found.\n");  in ips_detect_cpu()
1337 	if (limits->core_power_limit != (tdp / 8) * 1000) {  in ips_detect_cpu()
1338 		dev_info(ips->dev,  in ips_detect_cpu()
1340 			 tdp / 8, limits->core_power_limit / 1000);  in ips_detect_cpu()
1341 		limits->core_power_limit = (tdp / 8) * 1000;  in ips_detect_cpu()
1348  * ips_get_i915_syms - try to get GPU control methods from i915 driver
1358 	ips->read_mch_val = symbol_get(i915_read_mch_val);  in ips_get_i915_syms()
1359 	if (!ips->read_mch_val)  in ips_get_i915_syms()
1361 	ips->gpu_raise = symbol_get(i915_gpu_raise);  in ips_get_i915_syms()
1362 	if (!ips->gpu_raise)  in ips_get_i915_syms()
1364 	ips->gpu_lower = symbol_get(i915_gpu_lower);  in ips_get_i915_syms()
1365 	if (!ips->gpu_lower)  in ips_get_i915_syms()
1367 	ips->gpu_busy = symbol_get(i915_gpu_busy);  in ips_get_i915_syms()
1368 	if (!ips->gpu_busy)  in ips_get_i915_syms()
1370 	ips->gpu_turbo_disable = symbol_get(i915_gpu_turbo_disable);  in ips_get_i915_syms()
1371 	if (!ips->gpu_turbo_disable)  in ips_get_i915_syms()
1391 	if (!ips->gpu_busy && late_i915_load) {  in ips_gpu_turbo_enabled()
1393 			dev_info(ips->dev,  in ips_gpu_turbo_enabled()
1395 			ips->gpu_turbo_enabled = !(thm_readl(THM_HTS) & HTS_GTD_DIS);  in ips_gpu_turbo_enabled()
1399 	return ips->gpu_turbo_enabled;  in ips_gpu_turbo_enabled()
1422 	pr_info("Blacklisted intel_ips for %s\n", id->ident);  in ips_blacklist_callback()
1431 			DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1448 		return -ENODEV;  in ips_probe()
1450 	ips = devm_kzalloc(&dev->dev, sizeof(*ips), GFP_KERNEL);  in ips_probe()
1452 		return -ENOMEM;  in ips_probe()
1454 	spin_lock_init(&ips->turbo_status_lock);  in ips_probe()
1455 	ips->dev = &dev->dev;  in ips_probe()
1457 	ips->limits = ips_detect_cpu(ips);  in ips_probe()
1458 	if (!ips->limits) {  in ips_probe()
1459 		dev_info(&dev->dev, "IPS not supported on this CPU\n");  in ips_probe()
1460 		return -ENXIO;  in ips_probe()
1465 		dev_err(&dev->dev, "can't enable PCI device, aborting\n");  in ips_probe()
1471 		dev_err(&dev->dev, "failed to map thermal regs, aborting\n");  in ips_probe()
1474 	ips->regmap = pcim_iomap_table(dev)[0];  in ips_probe()
1480 		dev_err(&dev->dev, "thermal device not enabled (0x%02x), aborting\n", tse);  in ips_probe()
1481 		return -ENXIO;  in ips_probe()
1487 		dev_err(&dev->dev, "thermal reporting for required devices not enabled, aborting\n");  in ips_probe()
1488 		return -ENXIO;  in ips_probe()
1492 		ips->second_cpu = true;  in ips_probe()
1495 	dev_dbg(&dev->dev, "max cpu power clamp: %dW\n",  in ips_probe()
1496 		ips->mcp_power_limit / 10);  in ips_probe()
1497 	dev_dbg(&dev->dev, "max core power clamp: %dW\n",  in ips_probe()
1498 		ips->core_power_limit / 10);  in ips_probe()
1501 		ips->poll_turbo_status = true;  in ips_probe()
1504 		dev_info(&dev->dev, "failed to get i915 symbols, graphics turbo disabled until i915 loads\n");  in ips_probe()
1505 		ips->gpu_turbo_enabled = false;  in ips_probe()
1507 		dev_dbg(&dev->dev, "graphics turbo enabled\n");  in ips_probe()
1508 		ips->gpu_turbo_enabled = true;  in ips_probe()
1517 		dev_err(&dev->dev, "platform indicates TDP override unavailable, aborting\n");  in ips_probe()
1518 		return -ENODEV;  in ips_probe()
1529 	ips->irq = pci_irq_vector(dev, 0);  in ips_probe()
1531 	ret = request_irq(ips->irq, ips_irq_handler, IRQF_SHARED, "ips", ips);  in ips_probe()
1533 		dev_err(&dev->dev, "request irq failed, aborting\n");  in ips_probe()
1543 	ips->cta_val = thm_readw(THM_CTA);  in ips_probe()
1544 	ips->pta_val = thm_readw(THM_PTA);  in ips_probe()
1545 	ips->mgta_val = thm_readw(THM_MGTA);  in ips_probe()
1548 	rdmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);  in ips_probe()
1551 	ips->cpu_turbo_enabled = false;  in ips_probe()
1554 	ips->adjust = kthread_create(ips_adjust, ips, "ips-adjust");  in ips_probe()
1555 	if (IS_ERR(ips->adjust)) {  in ips_probe()
1556 		dev_err(&dev->dev,  in ips_probe()
1558 		ret = -ENOMEM;  in ips_probe()
1567 	ips->monitor = kthread_run(ips_monitor, ips, "ips-monitor");  in ips_probe()
1568 	if (IS_ERR(ips->monitor)) {  in ips_probe()
1569 		dev_err(&dev->dev,  in ips_probe()
1571 		ret = -ENOMEM;  in ips_probe()
1575 	hts = (ips->core_power_limit << HTS_PCPL_SHIFT) |  in ips_probe()
1576 		(ips->mcp_temp_limit << HTS_PTL_SHIFT) | HTS_NVV;  in ips_probe()
1584 	dev_info(&dev->dev, "IPS driver initialized, MCP temp limit %d\n",  in ips_probe()
1585 		 ips->mcp_temp_limit);  in ips_probe()
1589 	kthread_stop(ips->adjust);  in ips_probe()
1591 	free_irq(ips->irq, ips);  in ips_probe()
1604 	if (ips->read_mch_val)  in ips_remove()
1606 	if (ips->gpu_raise)  in ips_remove()
1608 	if (ips->gpu_lower)  in ips_remove()
1610 	if (ips->gpu_busy)  in ips_remove()
1612 	if (ips->gpu_turbo_disable)  in ips_remove()
1618 	wrmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);  in ips_remove()
1620 	free_irq(ips->irq, ips);  in ips_remove()
1622 	if (ips->adjust)  in ips_remove()
1623 		kthread_stop(ips->adjust);  in ips_remove()
1624 	if (ips->monitor)  in ips_remove()
1625 		kthread_stop(ips->monitor);  in ips_remove()
1626 	dev_dbg(&dev->dev, "IPS driver removed\n");  in ips_remove()