324 				struct child_latency_info *lat_info,  in scale_cookie_change()  argument
329 	unsigned long old = atomic_read(&lat_info->scale_cookie);  in scale_cookie_change()
338 			atomic_set(&lat_info->scale_cookie,  in scale_cookie_change()
341 			atomic_inc(&lat_info->scale_cookie);  in scale_cookie_change()
343 			atomic_add(scale, &lat_info->scale_cookie);  in scale_cookie_change()
353 				atomic_dec(&lat_info->scale_cookie);  in scale_cookie_change()
355 			atomic_sub(scale, &lat_info->scale_cookie);  in scale_cookie_change()
394 	struct child_latency_info *lat_info;  in check_scale_change()  local
408 	lat_info = &parent->child_lat;  in check_scale_change()
409 	cur_cookie = atomic_read(&lat_info->scale_cookie);  in check_scale_change()
410 	scale_lat = READ_ONCE(lat_info->scale_lat);  in check_scale_change()
437 		samples_thresh = lat_info->nr_samples * 5;  in check_scale_change()
524 	struct child_latency_info *lat_info;  in iolatency_check_latencies()  local
543 	lat_info = &parent->child_lat;  in iolatency_check_latencies()
549 	    atomic_read(&lat_info->scale_cookie) == DEFAULT_SCALE_COOKIE)  in iolatency_check_latencies()
553 	spin_lock_irqsave(&lat_info->lock, flags);  in iolatency_check_latencies()
556 	lat_info->nr_samples -= iolat->nr_samples;  in iolatency_check_latencies()
557 	lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
560 	if ((lat_info->last_scale_event >= now ||  in iolatency_check_latencies()
561 	    now - lat_info->last_scale_event < BLKIOLATENCY_MIN_ADJUST_TIME))  in iolatency_check_latencies()
569 		if (lat_info->scale_grp == iolat) {  in iolatency_check_latencies()
570 			lat_info->last_scale_event = now;  in iolatency_check_latencies()
571 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in iolatency_check_latencies()
573 	} else if (lat_info->scale_lat == 0 ||  in iolatency_check_latencies()
574 		   lat_info->scale_lat >= iolat->min_lat_nsec) {  in iolatency_check_latencies()
575 		lat_info->last_scale_event = now;  in iolatency_check_latencies()
576 		if (!lat_info->scale_grp ||  in iolatency_check_latencies()
577 		    lat_info->scale_lat > iolat->min_lat_nsec) {  in iolatency_check_latencies()
578 			WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);  in iolatency_check_latencies()
579 			lat_info->scale_grp = iolat;  in iolatency_check_latencies()
581 		scale_cookie_change(iolat->blkiolat, lat_info, false);  in iolatency_check_latencies()
585 	spin_unlock_irqrestore(&lat_info->lock, flags);  in iolatency_check_latencies()
668 		struct child_latency_info *lat_info;  in blkiolatency_timer_fn()  local
683 		lat_info = &iolat->child_lat;  in blkiolatency_timer_fn()
684 		cookie = atomic_read(&lat_info->scale_cookie);  in blkiolatency_timer_fn()
689 		spin_lock_irqsave(&lat_info->lock, flags);  in blkiolatency_timer_fn()
690 		if (lat_info->last_scale_event >= now)  in blkiolatency_timer_fn()
697 		if (lat_info->scale_grp == NULL) {  in blkiolatency_timer_fn()
698 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in blkiolatency_timer_fn()
707 		if (now - lat_info->last_scale_event >=  in blkiolatency_timer_fn()
709 			lat_info->scale_grp = NULL;  in blkiolatency_timer_fn()
711 		spin_unlock_irqrestore(&lat_info->lock, flags);  in blkiolatency_timer_fn()
774 		struct child_latency_info *lat_info;  in iolatency_clear_scaling()  local
778 		lat_info = &iolat->child_lat;  in iolatency_clear_scaling()
779 		spin_lock(&lat_info->lock);  in iolatency_clear_scaling()
780 		atomic_set(&lat_info->scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_clear_scaling()
781 		lat_info->last_scale_event = 0;  in iolatency_clear_scaling()
782 		lat_info->scale_grp = NULL;  in iolatency_clear_scaling()
783 		lat_info->scale_lat = 0;  in iolatency_clear_scaling()
784 		spin_unlock(&lat_info->lock);  in iolatency_clear_scaling()