Lines Matching +full:mixed +full:- +full:burst
1 // SPDX-License-Identifier: GPL-2.0
7 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
8 * - July2000
9 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
17 #include <linux/backing-dev.h>
20 #include <linux/blk-mq.h>
32 #include <linux/fault-inject.h>
37 #include <linux/blk-cgroup.h>
38 #include <linux/t10-pi.h>
43 #include <linux/blk-crypto.h>
49 #include "blk-mq.h"
50 #include "blk-mq-sched.h"
51 #include "blk-pm.h"
52 #include "blk-rq-qos.h"
75 * blk_queue_flag_set - atomically set a queue flag
81 set_bit(flag, &q->queue_flags); in blk_queue_flag_set()
86 * blk_queue_flag_clear - atomically clear a queue flag
92 clear_bit(flag, &q->queue_flags); in blk_queue_flag_clear()
97 * blk_queue_flag_test_and_set - atomically test and set a queue flag
101 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
106 return test_and_set_bit(flag, &q->queue_flags); in blk_queue_flag_test_and_set()
114 INIT_LIST_HEAD(&rq->queuelist); in blk_rq_init()
115 rq->q = q; in blk_rq_init()
116 rq->__sector = (sector_t) -1; in blk_rq_init()
117 INIT_HLIST_NODE(&rq->hash); in blk_rq_init()
118 RB_CLEAR_NODE(&rq->rb_node); in blk_rq_init()
119 rq->tag = BLK_MQ_NO_TAG; in blk_rq_init()
120 rq->internal_tag = BLK_MQ_NO_TAG; in blk_rq_init()
121 rq->start_time_ns = ktime_get_ns(); in blk_rq_init()
122 rq->part = NULL; in blk_rq_init()
123 refcount_set(&rq->ref, 1); in blk_rq_init()
151 * blk_op_str - Return string XXX in the REQ_OP_XXX.
174 [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" },
175 [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" },
176 [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" },
177 [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" },
178 [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" },
179 [BLK_STS_NEXUS] = { -EBADE, "critical nexus" },
180 [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" },
181 [BLK_STS_PROTECTION] = { -EILSEQ, "protection" },
182 [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" },
183 [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" },
184 [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" },
187 [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" },
190 [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" },
191 [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" },
194 [BLK_STS_IOERR] = { -EIO, "I/O" },
215 return -EIO; in blk_status_to_errno()
232 req->rq_disk ? req->rq_disk->disk_name : "?", in print_req_error()
234 req->cmd_flags & ~REQ_OP_MASK, in print_req_error()
235 req->nr_phys_segments, in print_req_error()
236 IOPRIO_PRIO_CLASS(req->ioprio)); in print_req_error()
243 bio->bi_status = error; in req_bio_endio()
245 if (unlikely(rq->rq_flags & RQF_QUIET)) in req_bio_endio()
255 if (bio->bi_iter.bi_size) in req_bio_endio()
256 bio->bi_status = BLK_STS_IOERR; in req_bio_endio()
258 bio->bi_iter.bi_sector = rq->__sector; in req_bio_endio()
262 if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ)) in req_bio_endio()
269 rq->rq_disk ? rq->rq_disk->disk_name : "?", in blk_dump_rq_flags()
270 (unsigned long long) rq->cmd_flags); in blk_dump_rq_flags()
276 rq->bio, rq->biotail, blk_rq_bytes(rq)); in blk_dump_rq_flags()
281 * blk_sync_queue - cancel any pending callbacks on a queue
290 * that its ->submit_bio will not re-add plugging prior to calling
300 del_timer_sync(&q->timeout); in blk_sync_queue()
301 cancel_work_sync(&q->timeout_work); in blk_sync_queue()
306 * blk_set_pm_only - increment pm_only counter
311 atomic_inc(&q->pm_only); in blk_set_pm_only()
319 pm_only = atomic_dec_return(&q->pm_only); in blk_clear_pm_only()
322 wake_up_all(&q->mq_freeze_wq); in blk_clear_pm_only()
327 * blk_put_queue - decrement the request_queue refcount
338 kobject_put(&q->kobj); in blk_put_queue()
357 wake_up_all(&q->mq_freeze_wq); in blk_set_queue_dying()
362 * blk_cleanup_queue - shutdown a request queue
366 * put it. All future requests will be failed immediately with -ENODEV.
394 /* for synchronous bio-based driver finish in-flight integrity i/o */ in blk_cleanup_queue()
398 del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer); in blk_cleanup_queue()
409 * Queue has become frozen, there can't be any in-queue requests, so in blk_cleanup_queue()
412 mutex_lock(&q->sysfs_lock); in blk_cleanup_queue()
413 if (q->elevator) in blk_cleanup_queue()
415 mutex_unlock(&q->sysfs_lock); in blk_cleanup_queue()
417 percpu_ref_exit(&q->q_usage_counter); in blk_cleanup_queue()
425 * blk_queue_enter() - try to increase q->q_usage_counter
437 if (percpu_ref_tryget_live(&q->q_usage_counter)) { in blk_queue_enter()
446 percpu_ref_put(&q->q_usage_counter); in blk_queue_enter()
455 return -EBUSY; in blk_queue_enter()
466 wait_event(q->mq_freeze_wq, in blk_queue_enter()
467 (!q->mq_freeze_depth && in blk_queue_enter()
472 return -ENODEV; in blk_queue_enter()
478 struct request_queue *q = bio->bi_disk->queue; in bio_queue_enter()
479 bool nowait = bio->bi_opf & REQ_NOWAIT; in bio_queue_enter()
495 percpu_ref_put(&q->q_usage_counter); in blk_queue_exit()
503 wake_up_all(&q->mq_freeze_wq); in blk_queue_usage_counter_release()
510 kblockd_schedule_work(&q->timeout_work); in blk_rq_timed_out_timer()
527 q->last_merge = NULL; in blk_alloc_queue()
529 q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL); in blk_alloc_queue()
530 if (q->id < 0) in blk_alloc_queue()
533 ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); in blk_alloc_queue()
537 q->backing_dev_info = bdi_alloc(node_id); in blk_alloc_queue()
538 if (!q->backing_dev_info) in blk_alloc_queue()
541 q->stats = blk_alloc_queue_stats(); in blk_alloc_queue()
542 if (!q->stats) in blk_alloc_queue()
545 q->node = node_id; in blk_alloc_queue()
547 atomic_set(&q->nr_active_requests_shared_sbitmap, 0); in blk_alloc_queue()
549 timer_setup(&q->backing_dev_info->laptop_mode_wb_timer, in blk_alloc_queue()
551 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0); in blk_alloc_queue()
552 INIT_WORK(&q->timeout_work, blk_timeout_work); in blk_alloc_queue()
553 INIT_LIST_HEAD(&q->icq_list); in blk_alloc_queue()
555 INIT_LIST_HEAD(&q->blkg_list); in blk_alloc_queue()
558 kobject_init(&q->kobj, &blk_queue_ktype); in blk_alloc_queue()
560 mutex_init(&q->debugfs_mutex); in blk_alloc_queue()
561 mutex_init(&q->sysfs_lock); in blk_alloc_queue()
562 mutex_init(&q->sysfs_dir_lock); in blk_alloc_queue()
563 spin_lock_init(&q->queue_lock); in blk_alloc_queue()
565 init_waitqueue_head(&q->mq_freeze_wq); in blk_alloc_queue()
566 mutex_init(&q->mq_freeze_lock); in blk_alloc_queue()
572 if (percpu_ref_init(&q->q_usage_counter, in blk_alloc_queue()
581 blk_set_default_limits(&q->limits); in blk_alloc_queue()
582 q->nr_requests = BLKDEV_MAX_RQ; in blk_alloc_queue()
587 percpu_ref_exit(&q->q_usage_counter); in blk_alloc_queue()
589 blk_free_queue_stats(q->stats); in blk_alloc_queue()
591 bdi_put(q->backing_dev_info); in blk_alloc_queue()
593 bioset_exit(&q->bio_split); in blk_alloc_queue()
595 ida_simple_remove(&blk_queue_ida, q->id); in blk_alloc_queue()
603 * blk_get_queue - increment the request_queue refcount
622 * blk_get_request - allocate a request
636 if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn) in blk_get_request()
637 q->mq_ops->initialize_rq_fn(req); in blk_get_request()
655 bio_devname(bio, b), bio->bi_opf, in handle_bad_sector()
671 return part->make_it_fail && should_fail(&fail_make_request, bytes); in should_fail_request()
698 if (part->policy && op_is_write(op)) { in bio_check_ro()
701 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) in bio_check_ro()
705 "Trying to write to read-only block-device %s (partno %d)\n", in bio_check_ro()
706 bio_devname(bio, b), part->partno); in bio_check_ro()
707 /* Older lvm-tools actually trigger this */ in bio_check_ro()
716 if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size)) in should_fail_bio()
717 return -EIO; in should_fail_bio()
724 * This may well happen - the kernel calls bread() without checking the size of
733 bio->bi_iter.bi_sector > maxsector - nr_sectors)) { in bio_check_eod()
735 return -EIO; in bio_check_eod()
746 int ret = -EIO; in blk_partition_remap()
749 p = __disk_get_part(bio->bi_disk, bio->bi_partno); in blk_partition_remap()
752 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) in blk_partition_remap()
760 bio->bi_iter.bi_sector += p->start_sect; in blk_partition_remap()
761 trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p), in blk_partition_remap()
762 bio->bi_iter.bi_sector - p->start_sect); in blk_partition_remap()
764 bio->bi_partno = 0; in blk_partition_remap()
777 sector_t pos = bio->bi_iter.bi_sector; in blk_check_zone_append()
785 if (pos & (blk_queue_zone_sectors(q) - 1) || in blk_check_zone_append()
791 * split and could result in non-contiguous sectors being written in in blk_check_zone_append()
794 if (nr_sectors > q->limits.chunk_sectors) in blk_check_zone_append()
798 if (nr_sectors > q->limits.max_zone_append_sectors) in blk_check_zone_append()
801 bio->bi_opf |= REQ_NOMERGE; in blk_check_zone_append()
808 struct request_queue *q = bio->bi_disk->queue; in submit_bio_checks()
815 if (plug && plug->nowait) in submit_bio_checks()
816 bio->bi_opf |= REQ_NOWAIT; in submit_bio_checks()
819 * For a REQ_NOWAIT based request, return -EOPNOTSUPP in submit_bio_checks()
822 if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q)) in submit_bio_checks()
828 if (bio->bi_partno) { in submit_bio_checks()
832 if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0))) in submit_bio_checks()
834 if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk)))) in submit_bio_checks()
842 if (op_is_flush(bio->bi_opf) && in submit_bio_checks()
843 !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) { in submit_bio_checks()
844 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA); in submit_bio_checks()
851 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) in submit_bio_checks()
852 bio->bi_opf &= ~REQ_HIPRI; in submit_bio_checks()
864 if (!q->limits.max_write_same_sectors) in submit_bio_checks()
884 if (!q->limits.max_write_zeroes_sectors) in submit_bio_checks()
892 * Various block parts want %current->io_context, so allocate it up in submit_bio_checks()
897 if (unlikely(!current->io_context)) in submit_bio_checks()
898 create_task_io_context(current, GFP_ATOMIC, q->node); in submit_bio_checks()
920 bio->bi_status = status; in submit_bio_checks()
927 struct gendisk *disk = bio->bi_disk; in __submit_bio()
931 if (!disk->fops->submit_bio) in __submit_bio()
933 ret = disk->fops->submit_bio(bio); in __submit_bio()
935 blk_queue_exit(disk->queue); in __submit_bio()
940 * The loop in this function may be a bit non-obvious, and so deserves some
943 * - Before entering the loop, bio->bi_next is NULL (as all callers ensure
945 * - We pretend that we have just taken it off a longer list, so we assign
947 * bio_list of new bios to be added. ->submit_bio() may indeed add some more
949 * non-NULL value in bio_list and re-enter the loop from the top.
950 * - In this case we really did just take the bio of the top of the list (no
951 * pretending) and so remove it from bio_list, and call into ->submit_bio()
954 * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio.
956 * ->submit_bio_bio, but that haven't been processed yet.
963 BUG_ON(bio->bi_next); in __submit_bio_noacct()
966 current->bio_list = bio_list_on_stack; in __submit_bio_noacct()
969 struct request_queue *q = bio->bi_disk->queue; in __submit_bio_noacct()
990 if (q == bio->bi_disk->queue) in __submit_bio_noacct()
1003 current->bio_list = NULL; in __submit_bio_noacct()
1012 current->bio_list = bio_list; in __submit_bio_noacct_mq()
1015 struct gendisk *disk = bio->bi_disk; in __submit_bio_noacct_mq()
1021 blk_queue_exit(disk->queue); in __submit_bio_noacct_mq()
1029 current->bio_list = NULL; in __submit_bio_noacct_mq()
1034 * submit_bio_noacct - re-submit a bio to the block device layer for I/O
1048 * We only want one ->submit_bio to be active at a time, else stack in submit_bio_noacct()
1049 * usage with stacked devices could be a problem. Use current->bio_list in submit_bio_noacct()
1050 * to collect a list of requests submited by a ->submit_bio method while in submit_bio_noacct()
1053 if (current->bio_list) { in submit_bio_noacct()
1054 bio_list_add(¤t->bio_list[0], bio); in submit_bio_noacct()
1058 if (!bio->bi_disk->fops->submit_bio) in submit_bio_noacct()
1065 * submit_bio - submit a bio to the block device layer for I/O
1073 * completion, is delivered asynchronously through the ->bi_end_io() callback
1074 * in @bio. The bio must NOT be touched by thecaller until ->bi_end_io() has
1090 count = queue_logical_block_size(bio->bi_disk->queue) >> 9; in submit_bio()
1097 task_io_account_read(bio->bi_iter.bi_size); in submit_bio()
1104 current->comm, task_pid_nr(current), in submit_bio()
1106 (unsigned long long)bio->bi_iter.bi_sector, in submit_bio()
1114 * the submitting cgroup IO-throttled, submission can be a significant in submit_bio()
1134 * blk_cloned_rq_check_limits - Helper function to check a cloned request
1140 * @rq may have been made based on weaker limitations of upper-level queues
1146 * Request stacking drivers like request-based dm may change the queue
1159 * a non-read/write command (discard, write same,etc.) the in blk_cloned_rq_check_limits()
1160 * low-level device driver will set the relevant queue limit to in blk_cloned_rq_check_limits()
1161 * 0 to prevent blk-lib from issuing more of the offending in blk_cloned_rq_check_limits()
1175 * queue's settings related to segment counting like q->bounce_pfn in blk_cloned_rq_check_limits()
1180 rq->nr_phys_segments = blk_recalc_rq_segments(rq); in blk_cloned_rq_check_limits()
1181 if (rq->nr_phys_segments > queue_max_segments(q)) { in blk_cloned_rq_check_limits()
1183 __func__, rq->nr_phys_segments, queue_max_segments(q)); in blk_cloned_rq_check_limits()
1191 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1203 if (rq->rq_disk && in blk_insert_cloned_request()
1204 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq))) in blk_insert_cloned_request()
1223 * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1237 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; in blk_rq_err_bytes()
1241 if (!(rq->rq_flags & RQF_MIXED_MERGE)) in blk_rq_err_bytes()
1247 * which have all the failfast bits that the first one has - in blk_rq_err_bytes()
1251 for (bio = rq->bio; bio; bio = bio->bi_next) { in blk_rq_err_bytes()
1252 if ((bio->bi_opf & ff) != ff) in blk_rq_err_bytes()
1254 bytes += bio->bi_iter.bi_size; in blk_rq_err_bytes()
1267 stamp = READ_ONCE(part->stamp); in update_io_ticks()
1269 if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) in update_io_ticks()
1270 __part_stat_add(part, io_ticks, end ? now - stamp : 1); in update_io_ticks()
1272 if (part->partno) { in update_io_ticks()
1273 part = &part_to_disk(part)->part0; in update_io_ticks()
1280 if (req->part && blk_do_io_stat(req)) { in blk_account_io_completion()
1285 part = req->part; in blk_account_io_completion()
1298 if (req->part && blk_do_io_stat(req) && in blk_account_io_done()
1299 !(req->rq_flags & RQF_FLUSH_SEQ)) { in blk_account_io_done()
1304 part = req->part; in blk_account_io_done()
1308 part_stat_add(part, nsecs[sgrp], now - req->start_time_ns); in blk_account_io_done()
1320 rq->part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); in blk_account_io_start()
1323 update_io_ticks(rq->part, jiffies, false); in blk_account_io_start()
1346 *part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector); in part_start_io_acct()
1355 return __part_start_io_acct(&disk->part0, sectors, op); in disk_start_io_acct()
1364 unsigned long duration = now - start_time; in __part_end_io_acct()
1384 __part_end_io_acct(&disk->part0, op, start_time); in disk_end_io_acct()
1394 if (rq->bio) { in blk_steal_bios()
1395 if (list->tail) in blk_steal_bios()
1396 list->tail->bi_next = rq->bio; in blk_steal_bios()
1398 list->head = rq->bio; in blk_steal_bios()
1399 list->tail = rq->biotail; in blk_steal_bios()
1401 rq->bio = NULL; in blk_steal_bios()
1402 rq->biotail = NULL; in blk_steal_bios()
1405 rq->__data_len = 0; in blk_steal_bios()
1410 * blk_update_request - Special helper function for request stacking drivers
1421 * (e.g. request-based dm) so that they can handle partial completion.
1432 * %false - this request doesn't have any more data
1433 * %true - this request has more data
1442 if (!req->bio) in blk_update_request()
1448 req->q->integrity.profile->complete_fn(req, nr_bytes); in blk_update_request()
1452 !(req->rq_flags & RQF_QUIET))) in blk_update_request()
1458 while (req->bio) { in blk_update_request()
1459 struct bio *bio = req->bio; in blk_update_request()
1460 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); in blk_update_request()
1462 if (bio_bytes == bio->bi_iter.bi_size) in blk_update_request()
1463 req->bio = bio->bi_next; in blk_update_request()
1470 nr_bytes -= bio_bytes; in blk_update_request()
1479 if (!req->bio) { in blk_update_request()
1485 req->__data_len = 0; in blk_update_request()
1489 req->__data_len -= total_bytes; in blk_update_request()
1493 req->__sector += total_bytes >> 9; in blk_update_request()
1495 /* mixed attributes always follow the first bio */ in blk_update_request()
1496 if (req->rq_flags & RQF_MIXED_MERGE) { in blk_update_request()
1497 req->cmd_flags &= ~REQ_FAILFAST_MASK; in blk_update_request()
1498 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK; in blk_update_request()
1501 if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) { in blk_update_request()
1508 req->__data_len = blk_rq_cur_bytes(req); in blk_update_request()
1512 req->nr_phys_segments = blk_recalc_rq_segments(req); in blk_update_request()
1521 * rq_flush_dcache_pages - Helper function to flush all pages in a request
1539 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
1543 * Check if underlying low-level drivers of a device are busy.
1551 * on burst I/O load.
1554 * 0 - Not busy (The request stacking driver should dispatch request)
1555 * 1 - Busy (The request stacking driver should stop dispatching request)
1559 if (queue_is_mq(q) && q->mq_ops->busy) in blk_lld_busy()
1560 return q->mq_ops->busy(q); in blk_lld_busy()
1567 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
1577 while ((bio = rq->bio) != NULL) { in blk_rq_unprep_clone()
1578 rq->bio = bio->bi_next; in blk_rq_unprep_clone()
1586 * blk_rq_prep_clone - Helper function to setup clone request
1620 if (rq->bio) { in blk_rq_prep_clone()
1621 rq->biotail->bi_next = bio; in blk_rq_prep_clone()
1622 rq->biotail = bio; in blk_rq_prep_clone()
1624 rq->bio = rq->biotail = bio; in blk_rq_prep_clone()
1630 rq->__sector = blk_rq_pos(rq_src); in blk_rq_prep_clone()
1631 rq->__data_len = blk_rq_bytes(rq_src); in blk_rq_prep_clone()
1632 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) { in blk_rq_prep_clone()
1633 rq->rq_flags |= RQF_SPECIAL_PAYLOAD; in blk_rq_prep_clone()
1634 rq->special_vec = rq_src->special_vec; in blk_rq_prep_clone()
1636 rq->nr_phys_segments = rq_src->nr_phys_segments; in blk_rq_prep_clone()
1637 rq->ioprio = rq_src->ioprio; in blk_rq_prep_clone()
1639 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) in blk_rq_prep_clone()
1649 return -ENOMEM; in blk_rq_prep_clone()
1667 * blk_start_plug - initialize blk_plug and track it inside the task_struct
1680 * Tracking blk_plug inside the task_struct will help with auto-flushing the
1696 if (tsk->plug) in blk_start_plug()
1699 INIT_LIST_HEAD(&plug->mq_list); in blk_start_plug()
1700 INIT_LIST_HEAD(&plug->cb_list); in blk_start_plug()
1701 plug->rq_count = 0; in blk_start_plug()
1702 plug->multiple_queues = false; in blk_start_plug()
1703 plug->nowait = false; in blk_start_plug()
1709 tsk->plug = plug; in blk_start_plug()
1717 while (!list_empty(&plug->cb_list)) { in flush_plug_callbacks()
1718 list_splice_init(&plug->cb_list, &callbacks); in flush_plug_callbacks()
1724 list_del(&cb->list); in flush_plug_callbacks()
1725 cb->callback(cb, from_schedule); in flush_plug_callbacks()
1733 struct blk_plug *plug = current->plug; in blk_check_plugged()
1739 list_for_each_entry(cb, &plug->cb_list, list) in blk_check_plugged()
1740 if (cb->callback == unplug && cb->data == data) in blk_check_plugged()
1747 cb->data = data; in blk_check_plugged()
1748 cb->callback = unplug; in blk_check_plugged()
1749 list_add(&cb->list, &plug->cb_list); in blk_check_plugged()
1759 if (!list_empty(&plug->mq_list)) in blk_flush_plug_list()
1764 * blk_finish_plug - mark the end of a batch of submitted I/O
1775 if (plug != current->plug) in blk_finish_plug()
1779 current->plug = NULL; in blk_finish_plug()
1803 /* used for unplugging and affects IO latency/throughput - HIGHPRI */ in blk_dev_init()