1 // SPDX-License-Identifier: GPL-2.0-only
3  * intel_powerclamp.c - package c-state idle injection
16  *              cpu in non-irq context does not reduce irq. for majority of the
105 		pr_err("Out of recommended range %lu, between 6-25ms\n",  in duration_set()
107 		ret = -EINVAL;  in duration_set()
153 		pr_err("Out of recommended window size %lu, between 2-10\n",  in window_size_set()
155 		ret = -EINVAL;  in window_size_set()
201 		(highest_subcstate - 1);  in find_target_mwait()
233 	while (info->msr_index) {  in has_pkg_state_counter()
234 		if (!rdmsrl_safe(info->msr_index, &val))  in has_pkg_state_counter()
248 	while (info->msr_index) {  in pkg_state_counter()
249 		if (!info->skip) {  in pkg_state_counter()
250 			if (!rdmsrl_safe(info->msr_index, &val))  in pkg_state_counter()
253 				info->skip = true;  in pkg_state_counter()
273 	} else if (ratio == MAX_TARGET_RATIO - 1 &&  in get_compensation()
275 		cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&  in get_compensation()
276 		cal_data[ratio - 2].confidence >= CONFIDENCE_OK) {  in get_compensation()
278 			cal_data[ratio - 1].steady_comp +  in get_compensation()
279 			cal_data[ratio - 2].steady_comp) / 3;  in get_compensation()
281 		cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&  in get_compensation()
284 			cal_data[ratio - 1].steady_comp +  in get_compensation()
293 		comp = MAX_TARGET_RATIO - ratio - 1;  in get_compensation()
308 	if (d->confidence >= CONFIDENCE_OK ||  in adjust_compensation()
313 	delta = set_target_ratio - current_ratio;  in adjust_compensation()
316 		if (d->steady_comp)  in adjust_compensation()
317 			d->steady_comp =  in adjust_compensation()
318 				roundup(delta+d->steady_comp, 2)/2;  in adjust_compensation()
320 			d->steady_comp = delta;  in adjust_compensation()
321 		d->confidence++;  in adjust_compensation()
339 	else if (tsc_now-tsc_last) {  in powerclamp_adjust_controls()
340 		val64 = 100*(msr_now-msr_last);  in powerclamp_adjust_controls()
341 		do_div(val64, (tsc_now-tsc_last));  in powerclamp_adjust_controls()
368 	int interval; /* jiffies to sleep for each attempt */  in clamp_balancing_func()  local
378 	w_data->target_ratio = READ_ONCE(set_target_ratio);  in clamp_balancing_func()
379 	w_data->guard = 1 + w_data->target_ratio / 20;  in clamp_balancing_func()
380 	w_data->window_size_now = window_size;  in clamp_balancing_func()
381 	w_data->duration_jiffies = msecs_to_jiffies(duration);  in clamp_balancing_func()
382 	w_data->count++;  in clamp_balancing_func()
386 	 * c-states, thus we need to compensate the injected idle ratio  in clamp_balancing_func()
389 	compensated_ratio = w_data->target_ratio +  in clamp_balancing_func()
390 		get_compensation(w_data->target_ratio);  in clamp_balancing_func()
393 	interval = w_data->duration_jiffies * 100 / compensated_ratio;  in clamp_balancing_func()
396 	target_jiffies = roundup(jiffies, interval);  in clamp_balancing_func()
397 	sleeptime = target_jiffies - jiffies;  in clamp_balancing_func()
401 	if (clamping && w_data->clamping && cpu_online(w_data->cpu))  in clamp_balancing_func()
402 		kthread_queue_delayed_work(w_data->worker,  in clamp_balancing_func()
403 					   &w_data->idle_injection_work,  in clamp_balancing_func()
418 	if (w_data->cpu == control_cpu &&  in clamp_idle_injection_func()
419 	    !(w_data->count % w_data->window_size_now)) {  in clamp_idle_injection_func()
421 			powerclamp_adjust_controls(w_data->target_ratio,  in clamp_idle_injection_func()
422 						   w_data->guard,  in clamp_idle_injection_func()
423 						   w_data->window_size_now);  in clamp_idle_injection_func()
430 	play_idle(jiffies_to_usecs(w_data->duration_jiffies));  in clamp_idle_injection_func()
433 	if (clamping && w_data->clamping && cpu_online(w_data->cpu))  in clamp_idle_injection_func()
434 		kthread_queue_work(w_data->worker, &w_data->balancing_work);  in clamp_idle_injection_func()
439  * to monitor actual idle ratio.
459 		if (tsc_now - tsc_last) {  in poll_pkg_cstate()
460 			val64 = 100 * (msr_now - msr_last);  in poll_pkg_cstate()
461 			do_div(val64, (tsc_now - tsc_last));  in poll_pkg_cstate()
483 	w_data->worker = worker;  in start_power_clamp_worker()
484 	w_data->count = 0;  in start_power_clamp_worker()
485 	w_data->cpu = cpu;  in start_power_clamp_worker()
486 	w_data->clamping = true;  in start_power_clamp_worker()
488 	sched_set_fifo(worker->task);  in start_power_clamp_worker()
489 	kthread_init_work(&w_data->balancing_work, clamp_balancing_func);  in start_power_clamp_worker()
490 	kthread_init_delayed_work(&w_data->idle_injection_work,  in start_power_clamp_worker()
492 	kthread_queue_work(w_data->worker, &w_data->balancing_work);  in start_power_clamp_worker()
499 	if (!w_data->worker)  in stop_power_clamp_worker()
502 	w_data->clamping = false;  in stop_power_clamp_worker()
510 	kthread_cancel_work_sync(&w_data->balancing_work);  in stop_power_clamp_worker()
511 	kthread_cancel_delayed_work_sync(&w_data->idle_injection_work);  in stop_power_clamp_worker()
519 	clear_bit(w_data->cpu, cpu_clamping_mask);  in stop_power_clamp_worker()
520 	kthread_destroy_worker(w_data->worker);  in stop_power_clamp_worker()
522 	w_data->worker = NULL;  in stop_power_clamp_worker()
529 	set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);  in start_power_clamp()
612 		*state = -1; /* indicates invalid state */  in powerclamp_get_cur_state()
623 				(unsigned long) (MAX_TARGET_RATIO-1));  in powerclamp_set_cur_state()
661 		return -ENODEV;  in powerclamp_probe()
666 		pr_info("No package C-state available\n");  in powerclamp_probe()
667 		return -ENODEV;  in powerclamp_probe()
713 		return -ENOMEM;  in powerclamp_init()
733 		retval = -ENOMEM;  in powerclamp_init()
740 		retval = -ENODEV;  in powerclamp_init()
777 MODULE_DESCRIPTION("Package Level C-state Idle Injection for Intel CPUs");