68 deadline_rb_root(struct deadline_data *dd, struct request *rq)  in deadline_rb_root()  argument
70 	return &dd->sort_list[rq_data_dir(rq)];  in deadline_rb_root()
88 deadline_add_rq_rb(struct deadline_data *dd, struct request *rq)  in deadline_add_rq_rb()  argument
90 	struct rb_root *root = deadline_rb_root(dd, rq);  in deadline_add_rq_rb()
96 deadline_del_rq_rb(struct deadline_data *dd, struct request *rq)  in deadline_del_rq_rb()  argument
100 	if (dd->next_rq[data_dir] == rq)  in deadline_del_rq_rb()
101 		dd->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_del_rq_rb()
103 	elv_rb_del(deadline_rb_root(dd, rq), rq);  in deadline_del_rq_rb()
111 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_remove_request()  local
119 		deadline_del_rq_rb(dd, rq);  in deadline_remove_request()
129 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_request_merged()  local
135 		elv_rb_del(deadline_rb_root(dd, req), req);  in dd_request_merged()
136 		deadline_add_rq_rb(dd, req);  in dd_request_merged()
165 deadline_move_request(struct deadline_data *dd, struct request *rq)  in deadline_move_request()  argument
169 	dd->next_rq[READ] = NULL;  in deadline_move_request()
170 	dd->next_rq[WRITE] = NULL;  in deadline_move_request()
171 	dd->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_move_request()
183 static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)  in deadline_check_fifo()  argument
185 	struct request *rq = rq_entry_fifo(dd->fifo_list[ddir].next);  in deadline_check_fifo()
201 deadline_fifo_request(struct deadline_data *dd, int data_dir)  in deadline_fifo_request()  argument
209 	if (list_empty(&dd->fifo_list[data_dir]))  in deadline_fifo_request()
212 	rq = rq_entry_fifo(dd->fifo_list[data_dir].next);  in deadline_fifo_request()
220 	spin_lock_irqsave(&dd->zone_lock, flags);  in deadline_fifo_request()
221 	list_for_each_entry(rq, &dd->fifo_list[WRITE], queuelist) {  in deadline_fifo_request()
227 	spin_unlock_irqrestore(&dd->zone_lock, flags);  in deadline_fifo_request()
237 deadline_next_request(struct deadline_data *dd, int data_dir)  in deadline_next_request()  argument
245 	rq = dd->next_rq[data_dir];  in deadline_next_request()
256 	spin_lock_irqsave(&dd->zone_lock, flags);  in deadline_next_request()
262 	spin_unlock_irqrestore(&dd->zone_lock, flags);  in deadline_next_request()
271 static struct request *__dd_dispatch_request(struct deadline_data *dd)  in __dd_dispatch_request()  argument
277 	if (!list_empty(&dd->dispatch)) {  in __dd_dispatch_request()
278 		rq = list_first_entry(&dd->dispatch, struct request, queuelist);  in __dd_dispatch_request()
283 	reads = !list_empty(&dd->fifo_list[READ]);  in __dd_dispatch_request()
284 	writes = !list_empty(&dd->fifo_list[WRITE]);  in __dd_dispatch_request()
289 	rq = deadline_next_request(dd, WRITE);  in __dd_dispatch_request()
291 		rq = deadline_next_request(dd, READ);  in __dd_dispatch_request()
293 	if (rq && dd->batching < dd->fifo_batch)  in __dd_dispatch_request()
303 		BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[READ]));  in __dd_dispatch_request()
305 		if (deadline_fifo_request(dd, WRITE) &&  in __dd_dispatch_request()
306 		    (dd->starved++ >= dd->writes_starved))  in __dd_dispatch_request()
320 		BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[WRITE]));  in __dd_dispatch_request()
322 		dd->starved = 0;  in __dd_dispatch_request()
335 	next_rq = deadline_next_request(dd, data_dir);  in __dd_dispatch_request()
336 	if (deadline_check_fifo(dd, data_dir) || !next_rq) {  in __dd_dispatch_request()
342 		rq = deadline_fifo_request(dd, data_dir);  in __dd_dispatch_request()
358 	dd->batching = 0;  in __dd_dispatch_request()
364 	dd->batching++;  in __dd_dispatch_request()
365 	deadline_move_request(dd, rq);  in __dd_dispatch_request()
383 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;  in dd_dispatch_request()  local
386 	spin_lock(&dd->lock);  in dd_dispatch_request()
387 	rq = __dd_dispatch_request(dd);  in dd_dispatch_request()
388 	spin_unlock(&dd->lock);  in dd_dispatch_request()
395 	struct deadline_data *dd = e->elevator_data;  in dd_exit_queue()  local
397 	BUG_ON(!list_empty(&dd->fifo_list[READ]));  in dd_exit_queue()
398 	BUG_ON(!list_empty(&dd->fifo_list[WRITE]));  in dd_exit_queue()
400 	kfree(dd);  in dd_exit_queue()
408 	struct deadline_data *dd;  in dd_init_queue()  local
415 	dd = kzalloc_node(sizeof(*dd), GFP_KERNEL, q->node);  in dd_init_queue()
416 	if (!dd) {  in dd_init_queue()
420 	eq->elevator_data = dd;  in dd_init_queue()
422 	INIT_LIST_HEAD(&dd->fifo_list[READ]);  in dd_init_queue()
423 	INIT_LIST_HEAD(&dd->fifo_list[WRITE]);  in dd_init_queue()
424 	dd->sort_list[READ] = RB_ROOT;  in dd_init_queue()
425 	dd->sort_list[WRITE] = RB_ROOT;  in dd_init_queue()
426 	dd->fifo_expire[READ] = read_expire;  in dd_init_queue()
427 	dd->fifo_expire[WRITE] = write_expire;  in dd_init_queue()
428 	dd->writes_starved = writes_starved;  in dd_init_queue()
429 	dd->front_merges = 1;  in dd_init_queue()
430 	dd->fifo_batch = fifo_batch;  in dd_init_queue()
431 	spin_lock_init(&dd->lock);  in dd_init_queue()
432 	spin_lock_init(&dd->zone_lock);  in dd_init_queue()
433 	INIT_LIST_HEAD(&dd->dispatch);  in dd_init_queue()
442 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_request_merge()  local
446 	if (!dd->front_merges)  in dd_request_merge()
449 	__rq = elv_rb_find(&dd->sort_list[bio_data_dir(bio)], sector);  in dd_request_merge()
466 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_bio_merge()  local
470 	spin_lock(&dd->lock);  in dd_bio_merge()
472 	spin_unlock(&dd->lock);  in dd_bio_merge()
487 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_insert_request()  local
503 			list_add(&rq->queuelist, &dd->dispatch);  in dd_insert_request()
505 			list_add_tail(&rq->queuelist, &dd->dispatch);  in dd_insert_request()
507 		deadline_add_rq_rb(dd, rq);  in dd_insert_request()
518 		rq->fifo_time = jiffies + dd->fifo_expire[data_dir];  in dd_insert_request()
519 		list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]);  in dd_insert_request()
527 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_insert_requests()  local
529 	spin_lock(&dd->lock);  in dd_insert_requests()
537 	spin_unlock(&dd->lock);  in dd_insert_requests()
567 		struct deadline_data *dd = q->elevator->elevator_data;  in dd_finish_request()  local
570 		spin_lock_irqsave(&dd->zone_lock, flags);  in dd_finish_request()
572 		if (!list_empty(&dd->fifo_list[WRITE]))  in dd_finish_request()
574 		spin_unlock_irqrestore(&dd->zone_lock, flags);  in dd_finish_request()
580 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;  in dd_has_work()  local
582 	return !list_empty_careful(&dd->dispatch) ||  in dd_has_work()
583 		!list_empty_careful(&dd->fifo_list[0]) ||  in dd_has_work()
584 		!list_empty_careful(&dd->fifo_list[1]);  in dd_has_work()
607 	struct deadline_data *dd = e->elevator_data;			\
613 SHOW_FUNCTION(deadline_read_expire_show, dd->fifo_expire[READ], 1);
614 SHOW_FUNCTION(deadline_write_expire_show, dd->fifo_expire[WRITE], 1);
615 SHOW_FUNCTION(deadline_writes_starved_show, dd->writes_starved, 0);
616 SHOW_FUNCTION(deadline_front_merges_show, dd->front_merges, 0);
617 SHOW_FUNCTION(deadline_fifo_batch_show, dd->fifo_batch, 0);
623 	struct deadline_data *dd = e->elevator_data;			\
636 STORE_FUNCTION(deadline_read_expire_store, &dd->fifo_expire[READ], 0, INT_MAX, 1);
637 STORE_FUNCTION(deadline_write_expire_store, &dd->fifo_expire[WRITE], 0, INT_MAX, 1);
638 STORE_FUNCTION(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX, 0);
639 STORE_FUNCTION(deadline_front_merges_store, &dd->front_merges, 0, 1, 0);
640 STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0);
659 	__acquires(&dd->lock)						\
662 	struct deadline_data *dd = q->elevator->elevator_data;		\
664 	spin_lock(&dd->lock);						\
665 	return seq_list_start(&dd->fifo_list[ddir], *pos);		\
672 	struct deadline_data *dd = q->elevator->elevator_data;		\
674 	return seq_list_next(v, &dd->fifo_list[ddir], pos);		\
678 	__releases(&dd->lock)						\
681 	struct deadline_data *dd = q->elevator->elevator_data;		\
683 	spin_unlock(&dd->lock);						\
697 	struct deadline_data *dd = q->elevator->elevator_data;		\
698 	struct request *rq = dd->next_rq[ddir];				\
711 	struct deadline_data *dd = q->elevator->elevator_data;  in DEADLINE_DEBUGFS_DDIR_ATTRS()  local
713 	seq_printf(m, "%u\n", dd->batching);  in DEADLINE_DEBUGFS_DDIR_ATTRS()
720 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_starved_show()  local
722 	seq_printf(m, "%u\n", dd->starved);  in deadline_starved_show()
727 	__acquires(&dd->lock)  in deadline_dispatch_start()
730 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_dispatch_start()  local
732 	spin_lock(&dd->lock);  in deadline_dispatch_start()
733 	return seq_list_start(&dd->dispatch, *pos);  in deadline_dispatch_start()
739 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_dispatch_next()  local
741 	return seq_list_next(v, &dd->dispatch, pos);  in deadline_dispatch_next()
745 	__releases(&dd->lock)  in deadline_dispatch_stop()
748 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_dispatch_stop()  local
750 	spin_unlock(&dd->lock);  in deadline_dispatch_stop()