67 deadline_rb_root(struct deadline_data *dd, struct request *rq)  in deadline_rb_root()  argument
69 	return &dd->sort_list[rq_data_dir(rq)];  in deadline_rb_root()
87 deadline_add_rq_rb(struct deadline_data *dd, struct request *rq)  in deadline_add_rq_rb()  argument
89 	struct rb_root *root = deadline_rb_root(dd, rq);  in deadline_add_rq_rb()
95 deadline_del_rq_rb(struct deadline_data *dd, struct request *rq)  in deadline_del_rq_rb()  argument
99 	if (dd->next_rq[data_dir] == rq)  in deadline_del_rq_rb()
100 		dd->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_del_rq_rb()
102 	elv_rb_del(deadline_rb_root(dd, rq), rq);  in deadline_del_rq_rb()
110 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_remove_request()  local
118 		deadline_del_rq_rb(dd, rq);  in deadline_remove_request()
128 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_request_merged()  local
134 		elv_rb_del(deadline_rb_root(dd, req), req);  in dd_request_merged()
135 		deadline_add_rq_rb(dd, req);  in dd_request_merged()
164 deadline_move_request(struct deadline_data *dd, struct request *rq)  in deadline_move_request()  argument
168 	dd->next_rq[READ] = NULL;  in deadline_move_request()
169 	dd->next_rq[WRITE] = NULL;  in deadline_move_request()
170 	dd->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_move_request()
182 static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)  in deadline_check_fifo()  argument
184 	struct request *rq = rq_entry_fifo(dd->fifo_list[ddir].next);  in deadline_check_fifo()
200 deadline_fifo_request(struct deadline_data *dd, int data_dir)  in deadline_fifo_request()  argument
208 	if (list_empty(&dd->fifo_list[data_dir]))  in deadline_fifo_request()
211 	rq = rq_entry_fifo(dd->fifo_list[data_dir].next);  in deadline_fifo_request()
219 	spin_lock_irqsave(&dd->zone_lock, flags);  in deadline_fifo_request()
220 	list_for_each_entry(rq, &dd->fifo_list[WRITE], queuelist) {  in deadline_fifo_request()
226 	spin_unlock_irqrestore(&dd->zone_lock, flags);  in deadline_fifo_request()
236 deadline_next_request(struct deadline_data *dd, int data_dir)  in deadline_next_request()  argument
244 	rq = dd->next_rq[data_dir];  in deadline_next_request()
255 	spin_lock_irqsave(&dd->zone_lock, flags);  in deadline_next_request()
261 	spin_unlock_irqrestore(&dd->zone_lock, flags);  in deadline_next_request()
270 static struct request *__dd_dispatch_request(struct deadline_data *dd)  in __dd_dispatch_request()  argument
276 	if (!list_empty(&dd->dispatch)) {  in __dd_dispatch_request()
277 		rq = list_first_entry(&dd->dispatch, struct request, queuelist);  in __dd_dispatch_request()
282 	reads = !list_empty(&dd->fifo_list[READ]);  in __dd_dispatch_request()
283 	writes = !list_empty(&dd->fifo_list[WRITE]);  in __dd_dispatch_request()
288 	rq = deadline_next_request(dd, WRITE);  in __dd_dispatch_request()
290 		rq = deadline_next_request(dd, READ);  in __dd_dispatch_request()
292 	if (rq && dd->batching < dd->fifo_batch)  in __dd_dispatch_request()
302 		BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[READ]));  in __dd_dispatch_request()
304 		if (deadline_fifo_request(dd, WRITE) &&  in __dd_dispatch_request()
305 		    (dd->starved++ >= dd->writes_starved))  in __dd_dispatch_request()
319 		BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[WRITE]));  in __dd_dispatch_request()
321 		dd->starved = 0;  in __dd_dispatch_request()
334 	next_rq = deadline_next_request(dd, data_dir);  in __dd_dispatch_request()
335 	if (deadline_check_fifo(dd, data_dir) || !next_rq) {  in __dd_dispatch_request()
341 		rq = deadline_fifo_request(dd, data_dir);  in __dd_dispatch_request()
357 	dd->batching = 0;  in __dd_dispatch_request()
363 	dd->batching++;  in __dd_dispatch_request()
364 	deadline_move_request(dd, rq);  in __dd_dispatch_request()
382 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;  in dd_dispatch_request()  local
385 	spin_lock(&dd->lock);  in dd_dispatch_request()
386 	rq = __dd_dispatch_request(dd);  in dd_dispatch_request()
387 	spin_unlock(&dd->lock);  in dd_dispatch_request()
394 	struct deadline_data *dd = e->elevator_data;  in dd_exit_queue()  local
396 	BUG_ON(!list_empty(&dd->fifo_list[READ]));  in dd_exit_queue()
397 	BUG_ON(!list_empty(&dd->fifo_list[WRITE]));  in dd_exit_queue()
399 	kfree(dd);  in dd_exit_queue()
407 	struct deadline_data *dd;  in dd_init_queue()  local
414 	dd = kzalloc_node(sizeof(*dd), GFP_KERNEL, q->node);  in dd_init_queue()
415 	if (!dd) {  in dd_init_queue()
419 	eq->elevator_data = dd;  in dd_init_queue()
421 	INIT_LIST_HEAD(&dd->fifo_list[READ]);  in dd_init_queue()
422 	INIT_LIST_HEAD(&dd->fifo_list[WRITE]);  in dd_init_queue()
423 	dd->sort_list[READ] = RB_ROOT;  in dd_init_queue()
424 	dd->sort_list[WRITE] = RB_ROOT;  in dd_init_queue()
425 	dd->fifo_expire[READ] = read_expire;  in dd_init_queue()
426 	dd->fifo_expire[WRITE] = write_expire;  in dd_init_queue()
427 	dd->writes_starved = writes_starved;  in dd_init_queue()
428 	dd->front_merges = 1;  in dd_init_queue()
429 	dd->fifo_batch = fifo_batch;  in dd_init_queue()
430 	spin_lock_init(&dd->lock);  in dd_init_queue()
431 	spin_lock_init(&dd->zone_lock);  in dd_init_queue()
432 	INIT_LIST_HEAD(&dd->dispatch);  in dd_init_queue()
441 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_request_merge()  local
445 	if (!dd->front_merges)  in dd_request_merge()
448 	__rq = elv_rb_find(&dd->sort_list[bio_data_dir(bio)], sector);  in dd_request_merge()
464 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_bio_merge()  local
468 	spin_lock(&dd->lock);  in dd_bio_merge()
470 	spin_unlock(&dd->lock);  in dd_bio_merge()
485 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_insert_request()  local
501 			list_add(&rq->queuelist, &dd->dispatch);  in dd_insert_request()
503 			list_add_tail(&rq->queuelist, &dd->dispatch);  in dd_insert_request()
505 		deadline_add_rq_rb(dd, rq);  in dd_insert_request()
516 		rq->fifo_time = jiffies + dd->fifo_expire[data_dir];  in dd_insert_request()
517 		list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]);  in dd_insert_request()
525 	struct deadline_data *dd = q->elevator->elevator_data;  in dd_insert_requests()  local
527 	spin_lock(&dd->lock);  in dd_insert_requests()
535 	spin_unlock(&dd->lock);  in dd_insert_requests()
558 		struct deadline_data *dd = q->elevator->elevator_data;  in dd_finish_request()  local
561 		spin_lock_irqsave(&dd->zone_lock, flags);  in dd_finish_request()
563 		spin_unlock_irqrestore(&dd->zone_lock, flags);  in dd_finish_request()
569 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;  in dd_has_work()  local
571 	return !list_empty_careful(&dd->dispatch) ||  in dd_has_work()
572 		!list_empty_careful(&dd->fifo_list[0]) ||  in dd_has_work()
573 		!list_empty_careful(&dd->fifo_list[1]);  in dd_has_work()
596 	struct deadline_data *dd = e->elevator_data;			\
602 SHOW_FUNCTION(deadline_read_expire_show, dd->fifo_expire[READ], 1);
603 SHOW_FUNCTION(deadline_write_expire_show, dd->fifo_expire[WRITE], 1);
604 SHOW_FUNCTION(deadline_writes_starved_show, dd->writes_starved, 0);
605 SHOW_FUNCTION(deadline_front_merges_show, dd->front_merges, 0);
606 SHOW_FUNCTION(deadline_fifo_batch_show, dd->fifo_batch, 0);
612 	struct deadline_data *dd = e->elevator_data;			\
625 STORE_FUNCTION(deadline_read_expire_store, &dd->fifo_expire[READ], 0, INT_MAX, 1);
626 STORE_FUNCTION(deadline_write_expire_store, &dd->fifo_expire[WRITE], 0, INT_MAX, 1);
627 STORE_FUNCTION(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX, 0);
628 STORE_FUNCTION(deadline_front_merges_store, &dd->front_merges, 0, 1, 0);
629 STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0);
648 	__acquires(&dd->lock)						\
651 	struct deadline_data *dd = q->elevator->elevator_data;		\
653 	spin_lock(&dd->lock);						\
654 	return seq_list_start(&dd->fifo_list[ddir], *pos);		\
661 	struct deadline_data *dd = q->elevator->elevator_data;		\
663 	return seq_list_next(v, &dd->fifo_list[ddir], pos);		\
667 	__releases(&dd->lock)						\
670 	struct deadline_data *dd = q->elevator->elevator_data;		\
672 	spin_unlock(&dd->lock);						\
686 	struct deadline_data *dd = q->elevator->elevator_data;		\
687 	struct request *rq = dd->next_rq[ddir];				\
700 	struct deadline_data *dd = q->elevator->elevator_data;  in DEADLINE_DEBUGFS_DDIR_ATTRS()  local
702 	seq_printf(m, "%u\n", dd->batching);  in DEADLINE_DEBUGFS_DDIR_ATTRS()
709 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_starved_show()  local
711 	seq_printf(m, "%u\n", dd->starved);  in deadline_starved_show()
716 	__acquires(&dd->lock)  in deadline_dispatch_start()
719 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_dispatch_start()  local
721 	spin_lock(&dd->lock);  in deadline_dispatch_start()
722 	return seq_list_start(&dd->dispatch, *pos);  in deadline_dispatch_start()
728 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_dispatch_next()  local
730 	return seq_list_next(v, &dd->dispatch, pos);  in deadline_dispatch_next()
734 	__releases(&dd->lock)  in deadline_dispatch_stop()
737 	struct deadline_data *dd = q->elevator->elevator_data;  in deadline_dispatch_stop()  local
739 	spin_unlock(&dd->lock);  in deadline_dispatch_stop()