6245 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in intel_rps_limits()  local
6255 		limits = (rps->max_freq_softlimit) << 23;  in intel_rps_limits()
6256 		if (val <= rps->min_freq_softlimit)  in intel_rps_limits()
6257 			limits |= (rps->min_freq_softlimit) << 14;  in intel_rps_limits()
6259 		limits = rps->max_freq_softlimit << 24;  in intel_rps_limits()
6260 		if (val <= rps->min_freq_softlimit)  in intel_rps_limits()
6261 			limits |= rps->min_freq_softlimit << 16;  in intel_rps_limits()
6269 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in rps_set_power()  local
6273 	lockdep_assert_held(&rps->power.mutex);  in rps_set_power()
6275 	if (new_power == rps->power.mode)  in rps_set_power()
6338 	rps->power.mode = new_power;  in rps_set_power()
6339 	rps->power.up_threshold = threshold_up;  in rps_set_power()
6340 	rps->power.down_threshold = threshold_down;  in rps_set_power()
6345 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen6_set_rps_thresholds()  local
6348 	new_power = rps->power.mode;  in gen6_set_rps_thresholds()
6349 	switch (rps->power.mode) {  in gen6_set_rps_thresholds()
6351 		if (val > rps->efficient_freq + 1 &&  in gen6_set_rps_thresholds()
6352 		    val > rps->cur_freq)  in gen6_set_rps_thresholds()
6357 		if (val <= rps->efficient_freq &&  in gen6_set_rps_thresholds()
6358 		    val < rps->cur_freq)  in gen6_set_rps_thresholds()
6360 		else if (val >= rps->rp0_freq &&  in gen6_set_rps_thresholds()
6361 			 val > rps->cur_freq)  in gen6_set_rps_thresholds()
6366 		if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&  in gen6_set_rps_thresholds()
6367 		    val < rps->cur_freq)  in gen6_set_rps_thresholds()
6372 	if (val <= rps->min_freq_softlimit)  in gen6_set_rps_thresholds()
6374 	if (val >= rps->max_freq_softlimit)  in gen6_set_rps_thresholds()
6377 	mutex_lock(&rps->power.mutex);  in gen6_set_rps_thresholds()
6378 	if (rps->power.interactive)  in gen6_set_rps_thresholds()
6381 	mutex_unlock(&rps->power.mutex);  in gen6_set_rps_thresholds()
6382 	rps->last_adj = 0;  in gen6_set_rps_thresholds()
6387 	struct intel_rps *rps = &i915->gt_pm.rps;  in intel_rps_mark_interactive()  local
6392 	mutex_lock(&rps->power.mutex);  in intel_rps_mark_interactive()
6394 		if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))  in intel_rps_mark_interactive()
6397 		GEM_BUG_ON(!rps->power.interactive);  in intel_rps_mark_interactive()
6398 		rps->power.interactive--;  in intel_rps_mark_interactive()
6400 	mutex_unlock(&rps->power.mutex);  in intel_rps_mark_interactive()
6405 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen6_rps_pm_mask()  local
6409 	if (val > rps->min_freq_softlimit)  in gen6_rps_pm_mask()
6411 	if (val < rps->max_freq_softlimit)  in gen6_rps_pm_mask()
6424 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen6_set_rps()  local
6429 	if (val != rps->cur_freq) {  in gen6_set_rps()
6451 	rps->cur_freq = val;  in gen6_set_rps()
6467 	if (val != dev_priv->gt_pm.rps.cur_freq) {  in valleyview_set_rps()
6475 	dev_priv->gt_pm.rps.cur_freq = val;  in valleyview_set_rps()
6490 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in vlv_set_rps_idle()  local
6491 	u32 val = rps->idle_freq;  in vlv_set_rps_idle()
6494 	if (rps->cur_freq <= val)  in vlv_set_rps_idle()
6519 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen6_rps_busy()  local
6522 	if (rps->enabled) {  in gen6_rps_busy()
6528 			   gen6_rps_pm_mask(dev_priv, rps->cur_freq));  in gen6_rps_busy()
6535 		freq = max(rps->cur_freq,  in gen6_rps_busy()
6536 			   rps->efficient_freq);  in gen6_rps_busy()
6540 					rps->min_freq_softlimit,  in gen6_rps_busy()
6541 					rps->max_freq_softlimit)))  in gen6_rps_busy()
6549 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen6_rps_idle()  local
6559 	if (rps->enabled) {  in gen6_rps_idle()
6563 			gen6_set_rps(dev_priv, rps->idle_freq);  in gen6_rps_idle()
6564 		rps->last_adj = 0;  in gen6_rps_idle()
6574 	struct intel_rps *rps = &rq->i915->gt_pm.rps;  in gen6_rps_boost()  local
6581 	if (!rps->enabled)  in gen6_rps_boost()
6591 		boost = !atomic_fetch_inc(&rps->num_waiters);  in gen6_rps_boost()
6598 	if (READ_ONCE(rps->cur_freq) < rps->boost_freq)  in gen6_rps_boost()
6599 		schedule_work(&rps->work);  in gen6_rps_boost()
6601 	atomic_inc(rps_client ? &rps_client->boosts : &rps->boosts);  in gen6_rps_boost()
6606 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in intel_set_rps()  local
6610 	GEM_BUG_ON(val > rps->max_freq);  in intel_set_rps()
6611 	GEM_BUG_ON(val < rps->min_freq);  in intel_set_rps()
6613 	if (!rps->enabled) {  in intel_set_rps()
6614 		rps->cur_freq = val;  in intel_set_rps()
6762 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen6_init_rps_frequencies()  local
6769 		rps->rp0_freq = (rp_state_cap >> 16) & 0xff;  in gen6_init_rps_frequencies()
6770 		rps->rp1_freq = (rp_state_cap >>  8) & 0xff;  in gen6_init_rps_frequencies()
6771 		rps->min_freq = (rp_state_cap >>  0) & 0xff;  in gen6_init_rps_frequencies()
6774 		rps->rp0_freq = (rp_state_cap >>  0) & 0xff;  in gen6_init_rps_frequencies()
6775 		rps->rp1_freq = (rp_state_cap >>  8) & 0xff;  in gen6_init_rps_frequencies()
6776 		rps->min_freq = (rp_state_cap >> 16) & 0xff;  in gen6_init_rps_frequencies()
6779 	rps->max_freq = rps->rp0_freq;  in gen6_init_rps_frequencies()
6781 	rps->efficient_freq = rps->rp1_freq;  in gen6_init_rps_frequencies()
6789 			rps->efficient_freq =  in gen6_init_rps_frequencies()
6792 					rps->min_freq,  in gen6_init_rps_frequencies()
6793 					rps->max_freq);  in gen6_init_rps_frequencies()
6800 		rps->rp0_freq *= GEN9_FREQ_SCALER;  in gen6_init_rps_frequencies()
6801 		rps->rp1_freq *= GEN9_FREQ_SCALER;  in gen6_init_rps_frequencies()
6802 		rps->min_freq *= GEN9_FREQ_SCALER;  in gen6_init_rps_frequencies()
6803 		rps->max_freq *= GEN9_FREQ_SCALER;  in gen6_init_rps_frequencies()
6804 		rps->efficient_freq *= GEN9_FREQ_SCALER;  in gen6_init_rps_frequencies()
6811 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in reset_rps()  local
6812 	u8 freq = rps->cur_freq;  in reset_rps()
6815 	rps->power.mode = -1;  in reset_rps()
6816 	rps->cur_freq = -1;  in reset_rps()
6830 			GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));  in gen9_enable_rps()
6973 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen8_enable_rps()  local
6979 		   HSW_FREQUENCY(rps->rp1_freq));  in gen8_enable_rps()
6981 		   HSW_FREQUENCY(rps->rp1_freq));  in gen8_enable_rps()
6987 		   rps->max_freq_softlimit << 24 |  in gen8_enable_rps()
6988 		   rps->min_freq_softlimit << 16);  in gen8_enable_rps()
7100 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in gen6_update_ring_freq()  local
7110 	if (rps->max_freq <= rps->min_freq)  in gen6_update_ring_freq()
7132 	min_gpu_freq = rps->min_freq;  in gen6_update_ring_freq()
7133 	max_gpu_freq = rps->max_freq;  in gen6_update_ring_freq()
7391 	dev_priv->gt_pm.rps.gpll_ref_freq =  in vlv_init_gpll_ref_freq()
7397 			 dev_priv->gt_pm.rps.gpll_ref_freq);  in vlv_init_gpll_ref_freq()
7402 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in valleyview_init_gt_powersave()  local
7424 	rps->max_freq = valleyview_rps_max_freq(dev_priv);  in valleyview_init_gt_powersave()
7425 	rps->rp0_freq = rps->max_freq;  in valleyview_init_gt_powersave()
7427 			 intel_gpu_freq(dev_priv, rps->max_freq),  in valleyview_init_gt_powersave()
7428 			 rps->max_freq);  in valleyview_init_gt_powersave()
7430 	rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);  in valleyview_init_gt_powersave()
7432 			 intel_gpu_freq(dev_priv, rps->efficient_freq),  in valleyview_init_gt_powersave()
7433 			 rps->efficient_freq);  in valleyview_init_gt_powersave()
7435 	rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);  in valleyview_init_gt_powersave()
7437 			 intel_gpu_freq(dev_priv, rps->rp1_freq),  in valleyview_init_gt_powersave()
7438 			 rps->rp1_freq);  in valleyview_init_gt_powersave()
7440 	rps->min_freq = valleyview_rps_min_freq(dev_priv);  in valleyview_init_gt_powersave()
7442 			 intel_gpu_freq(dev_priv, rps->min_freq),  in valleyview_init_gt_powersave()
7443 			 rps->min_freq);  in valleyview_init_gt_powersave()
7448 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in cherryview_init_gt_powersave()  local
7469 	rps->max_freq = cherryview_rps_max_freq(dev_priv);  in cherryview_init_gt_powersave()
7470 	rps->rp0_freq = rps->max_freq;  in cherryview_init_gt_powersave()
7472 			 intel_gpu_freq(dev_priv, rps->max_freq),  in cherryview_init_gt_powersave()
7473 			 rps->max_freq);  in cherryview_init_gt_powersave()
7475 	rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);  in cherryview_init_gt_powersave()
7477 			 intel_gpu_freq(dev_priv, rps->efficient_freq),  in cherryview_init_gt_powersave()
7478 			 rps->efficient_freq);  in cherryview_init_gt_powersave()
7480 	rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);  in cherryview_init_gt_powersave()
7482 			 intel_gpu_freq(dev_priv, rps->rp1_freq),  in cherryview_init_gt_powersave()
7483 			 rps->rp1_freq);  in cherryview_init_gt_powersave()
7485 	rps->min_freq = cherryview_rps_min_freq(dev_priv);  in cherryview_init_gt_powersave()
7487 			 intel_gpu_freq(dev_priv, rps->min_freq),  in cherryview_init_gt_powersave()
7488 			 rps->min_freq);  in cherryview_init_gt_powersave()
7490 	WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |  in cherryview_init_gt_powersave()
7491 		   rps->min_freq) & 1,  in cherryview_init_gt_powersave()
7867 	pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));  in __i915_gfx_val()
8154 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in intel_init_gt_powersave()  local
8176 	rps->idle_freq = rps->min_freq;  in intel_init_gt_powersave()
8177 	rps->cur_freq = rps->idle_freq;  in intel_init_gt_powersave()
8179 	rps->max_freq_softlimit = rps->max_freq;  in intel_init_gt_powersave()
8180 	rps->min_freq_softlimit = rps->min_freq;  in intel_init_gt_powersave()
8183 		rps->min_freq_softlimit =  in intel_init_gt_powersave()
8185 			      rps->efficient_freq,  in intel_init_gt_powersave()
8196 					 (rps->max_freq & 0xff) * 50,  in intel_init_gt_powersave()
8198 			rps->max_freq = params & 0xff;  in intel_init_gt_powersave()
8203 	rps->boost_freq = rps->max_freq;  in intel_init_gt_powersave()
8235 	dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */  in intel_sanitize_gt_powersave()
8280 	if (!dev_priv->gt_pm.rps.enabled)  in intel_disable_rps()
8294 	dev_priv->gt_pm.rps.enabled = false;  in intel_disable_rps()
8344 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in intel_enable_rps()  local
8348 	if (rps->enabled)  in intel_enable_rps()
8366 	WARN_ON(rps->max_freq < rps->min_freq);  in intel_enable_rps()
8367 	WARN_ON(rps->idle_freq > rps->max_freq);  in intel_enable_rps()
8369 	WARN_ON(rps->efficient_freq < rps->min_freq);  in intel_enable_rps()
8370 	WARN_ON(rps->efficient_freq > rps->max_freq);  in intel_enable_rps()
8372 	rps->enabled = true;  in intel_enable_rps()
9575 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in byt_gpu_freq()  local
9581 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);  in byt_gpu_freq()
9586 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in byt_freq_opcode()  local
9588 	return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;  in byt_freq_opcode()
9593 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in chv_gpu_freq()  local
9599 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);  in chv_gpu_freq()
9604 	struct intel_rps *rps = &dev_priv->gt_pm.rps;  in chv_freq_opcode()  local
9607 	return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;  in chv_freq_opcode()
9639 	mutex_init(&dev_priv->gt_pm.rps.power.mutex);  in intel_pm_setup()
9641 	atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);  in intel_pm_setup()