212 static void flush_latency_buckets(struct kyber_queue_data *kqd,  in flush_latency_buckets()  argument
216 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in flush_latency_buckets()
228 static int calculate_percentile(struct kyber_queue_data *kqd,  in calculate_percentile()  argument
232 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in calculate_percentile()
245 	if (!kqd->latency_timeout[sched_domain])  in calculate_percentile()
246 		kqd->latency_timeout[sched_domain] = max(jiffies + HZ, 1UL);  in calculate_percentile()
248 	    time_is_after_jiffies(kqd->latency_timeout[sched_domain])) {  in calculate_percentile()
251 	kqd->latency_timeout[sched_domain] = 0;  in calculate_percentile()
259 	memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type]));  in calculate_percentile()
261 	trace_kyber_latency(kqd->dev, kyber_domain_names[sched_domain],  in calculate_percentile()
268 static void kyber_resize_domain(struct kyber_queue_data *kqd,  in kyber_resize_domain()  argument
272 	if (depth != kqd->domain_tokens[sched_domain].sb.depth) {  in kyber_resize_domain()
273 		sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth);  in kyber_resize_domain()
274 		trace_kyber_adjust(kqd->dev, kyber_domain_names[sched_domain],  in kyber_resize_domain()
281 	struct kyber_queue_data *kqd = from_timer(kqd, t, timer);  in kyber_timer_fn()  local
290 		cpu_latency = per_cpu_ptr(kqd->cpu_latency, cpu);  in kyber_timer_fn()
292 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
294 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
307 		p90 = calculate_percentile(kqd, sched_domain, KYBER_IO_LATENCY,  in kyber_timer_fn()
322 		p99 = calculate_percentile(kqd, sched_domain,  in kyber_timer_fn()
334 				p99 = kqd->domain_p99[sched_domain];  in kyber_timer_fn()
335 			kqd->domain_p99[sched_domain] = -1;  in kyber_timer_fn()
337 			kqd->domain_p99[sched_domain] = p99;  in kyber_timer_fn()
352 			orig_depth = kqd->domain_tokens[sched_domain].sb.depth;  in kyber_timer_fn()
354 			kyber_resize_domain(kqd, sched_domain, depth);  in kyber_timer_fn()
361 	struct kyber_queue_data *kqd;  in kyber_queue_data_alloc()  local
365 	kqd = kzalloc_node(sizeof(*kqd), GFP_KERNEL, q->node);  in kyber_queue_data_alloc()
366 	if (!kqd)  in kyber_queue_data_alloc()
369 	kqd->q = q;  in kyber_queue_data_alloc()
370 	kqd->dev = disk_devt(q->disk);  in kyber_queue_data_alloc()
372 	kqd->cpu_latency = alloc_percpu_gfp(struct kyber_cpu_latency,  in kyber_queue_data_alloc()
374 	if (!kqd->cpu_latency)  in kyber_queue_data_alloc()
377 	timer_setup(&kqd->timer, kyber_timer_fn, 0);  in kyber_queue_data_alloc()
382 		ret = sbitmap_queue_init_node(&kqd->domain_tokens[i],  in kyber_queue_data_alloc()
387 				sbitmap_queue_free(&kqd->domain_tokens[i]);  in kyber_queue_data_alloc()
393 		kqd->domain_p99[i] = -1;  in kyber_queue_data_alloc()
394 		kqd->latency_targets[i] = kyber_latency_targets[i];  in kyber_queue_data_alloc()
397 	return kqd;  in kyber_queue_data_alloc()
400 	free_percpu(kqd->cpu_latency);  in kyber_queue_data_alloc()
402 	kfree(kqd);  in kyber_queue_data_alloc()
409 	struct kyber_queue_data *kqd;  in kyber_init_sched()  local
416 	kqd = kyber_queue_data_alloc(q);  in kyber_init_sched()
417 	if (IS_ERR(kqd)) {  in kyber_init_sched()
419 		return PTR_ERR(kqd);  in kyber_init_sched()
426 	eq->elevator_data = kqd;  in kyber_init_sched()
434 	struct kyber_queue_data *kqd = e->elevator_data;  in kyber_exit_sched()  local
437 	del_timer_sync(&kqd->timer);  in kyber_exit_sched()
438 	blk_stat_disable_accounting(kqd->q);  in kyber_exit_sched()
441 		sbitmap_queue_free(&kqd->domain_tokens[i]);  in kyber_exit_sched()
442 	free_percpu(kqd->cpu_latency);  in kyber_exit_sched()
443 	kfree(kqd);  in kyber_exit_sched()
457 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_depth_updated()  local
461 	kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U;  in kyber_depth_updated()
463 	sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, kqd->async_depth);  in kyber_depth_updated()
542 static void rq_clear_domain_token(struct kyber_queue_data *kqd,  in rq_clear_domain_token()  argument
551 		sbitmap_queue_clear(&kqd->domain_tokens[sched_domain], nr,  in rq_clear_domain_token()
563 		struct kyber_queue_data *kqd = data->q->elevator->elevator_data;  in kyber_limit_depth()  local
565 		data->shallow_depth = kqd->async_depth;  in kyber_limit_depth()
617 	struct kyber_queue_data *kqd = rq->q->elevator->elevator_data;  in kyber_finish_request()  local
619 	rq_clear_domain_token(kqd, rq);  in kyber_finish_request()
642 	struct kyber_queue_data *kqd = rq->q->elevator->elevator_data;  in kyber_completed_request()  local
651 	cpu_latency = get_cpu_ptr(kqd->cpu_latency);  in kyber_completed_request()
652 	target = kqd->latency_targets[sched_domain];  in kyber_completed_request()
657 	put_cpu_ptr(kqd->cpu_latency);  in kyber_completed_request()
659 	timer_reduce(&kqd->timer, jiffies + HZ / 10);  in kyber_completed_request()
707 static int kyber_get_domain_token(struct kyber_queue_data *kqd,  in kyber_get_domain_token()  argument
712 	struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];  in kyber_get_domain_token()
755 kyber_dispatch_cur_domain(struct kyber_queue_data *kqd,  in kyber_dispatch_cur_domain()  argument
775 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
782 			trace_kyber_throttled(kqd->dev,  in kyber_dispatch_cur_domain()
786 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
795 			trace_kyber_throttled(kqd->dev,  in kyber_dispatch_cur_domain()
806 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_dispatch_request()  local
818 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
839 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
868 	struct kyber_queue_data *kqd = e->elevator_data;		\
870 	return sprintf(page, "%llu\n", kqd->latency_targets[domain]);	\
876 	struct kyber_queue_data *kqd = e->elevator_data;		\
884 	kqd->latency_targets[domain] = nsec;				\
905 	struct kyber_queue_data *kqd = q->elevator->elevator_data;	\
907 	sbitmap_queue_show(&kqd->domain_tokens[domain], m);		\
964 	struct kyber_queue_data *kqd = q->elevator->elevator_data;  in KYBER_DEBUGFS_DOMAIN_ATTRS()  local
966 	seq_printf(m, "%u\n", kqd->async_depth);  in KYBER_DEBUGFS_DOMAIN_ATTRS()