1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Functions related to sysfs handling
4 */
5 #include <linux/kernel.h>
6 #include <linux/slab.h>
7 #include <linux/module.h>
8 #include <linux/bio.h>
9 #include <linux/blkdev.h>
10 #include <linux/backing-dev.h>
11 #include <linux/blktrace_api.h>
12 #include <linux/blk-mq.h>
13 #include <linux/blk-cgroup.h>
14 #include <linux/debugfs.h>
15
16 #include "blk.h"
17 #include "blk-mq.h"
18 #include "blk-mq-debugfs.h"
19 #include "blk-wbt.h"
20
21 struct queue_sysfs_entry {
22 struct attribute attr;
23 ssize_t (*show)(struct request_queue *, char *);
24 ssize_t (*store)(struct request_queue *, const char *, size_t);
25 };
26
27 static ssize_t
queue_var_show(unsigned long var,char * page)28 queue_var_show(unsigned long var, char *page)
29 {
30 return sprintf(page, "%lu\n", var);
31 }
32
33 static ssize_t
queue_var_store(unsigned long * var,const char * page,size_t count)34 queue_var_store(unsigned long *var, const char *page, size_t count)
35 {
36 int err;
37 unsigned long v;
38
39 err = kstrtoul(page, 10, &v);
40 if (err || v > UINT_MAX)
41 return -EINVAL;
42
43 *var = v;
44
45 return count;
46 }
47
queue_var_store64(s64 * var,const char * page)48 static ssize_t queue_var_store64(s64 *var, const char *page)
49 {
50 int err;
51 s64 v;
52
53 err = kstrtos64(page, 10, &v);
54 if (err < 0)
55 return err;
56
57 *var = v;
58 return 0;
59 }
60
queue_requests_show(struct request_queue * q,char * page)61 static ssize_t queue_requests_show(struct request_queue *q, char *page)
62 {
63 return queue_var_show(q->nr_requests, page);
64 }
65
66 static ssize_t
queue_requests_store(struct request_queue * q,const char * page,size_t count)67 queue_requests_store(struct request_queue *q, const char *page, size_t count)
68 {
69 unsigned long nr;
70 int ret, err;
71
72 if (!queue_is_mq(q))
73 return -EINVAL;
74
75 ret = queue_var_store(&nr, page, count);
76 if (ret < 0)
77 return ret;
78
79 if (nr < BLKDEV_MIN_RQ)
80 nr = BLKDEV_MIN_RQ;
81
82 err = blk_mq_update_nr_requests(q, nr);
83 if (err)
84 return err;
85
86 return ret;
87 }
88
queue_ra_show(struct request_queue * q,char * page)89 static ssize_t queue_ra_show(struct request_queue *q, char *page)
90 {
91 unsigned long ra_kb;
92
93 if (!q->disk)
94 return -EINVAL;
95 ra_kb = q->disk->bdi->ra_pages << (PAGE_SHIFT - 10);
96 return queue_var_show(ra_kb, page);
97 }
98
99 static ssize_t
queue_ra_store(struct request_queue * q,const char * page,size_t count)100 queue_ra_store(struct request_queue *q, const char *page, size_t count)
101 {
102 unsigned long ra_kb;
103 ssize_t ret;
104
105 if (!q->disk)
106 return -EINVAL;
107 ret = queue_var_store(&ra_kb, page, count);
108 if (ret < 0)
109 return ret;
110 q->disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
111 return ret;
112 }
113
queue_max_sectors_show(struct request_queue * q,char * page)114 static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
115 {
116 int max_sectors_kb = queue_max_sectors(q) >> 1;
117
118 return queue_var_show(max_sectors_kb, page);
119 }
120
queue_max_segments_show(struct request_queue * q,char * page)121 static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
122 {
123 return queue_var_show(queue_max_segments(q), page);
124 }
125
queue_max_discard_segments_show(struct request_queue * q,char * page)126 static ssize_t queue_max_discard_segments_show(struct request_queue *q,
127 char *page)
128 {
129 return queue_var_show(queue_max_discard_segments(q), page);
130 }
131
queue_max_integrity_segments_show(struct request_queue * q,char * page)132 static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
133 {
134 return queue_var_show(q->limits.max_integrity_segments, page);
135 }
136
queue_max_segment_size_show(struct request_queue * q,char * page)137 static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
138 {
139 return queue_var_show(queue_max_segment_size(q), page);
140 }
141
queue_logical_block_size_show(struct request_queue * q,char * page)142 static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
143 {
144 return queue_var_show(queue_logical_block_size(q), page);
145 }
146
queue_physical_block_size_show(struct request_queue * q,char * page)147 static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
148 {
149 return queue_var_show(queue_physical_block_size(q), page);
150 }
151
queue_chunk_sectors_show(struct request_queue * q,char * page)152 static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
153 {
154 return queue_var_show(q->limits.chunk_sectors, page);
155 }
156
queue_io_min_show(struct request_queue * q,char * page)157 static ssize_t queue_io_min_show(struct request_queue *q, char *page)
158 {
159 return queue_var_show(queue_io_min(q), page);
160 }
161
queue_io_opt_show(struct request_queue * q,char * page)162 static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
163 {
164 return queue_var_show(queue_io_opt(q), page);
165 }
166
queue_discard_granularity_show(struct request_queue * q,char * page)167 static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
168 {
169 return queue_var_show(q->limits.discard_granularity, page);
170 }
171
queue_discard_max_hw_show(struct request_queue * q,char * page)172 static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
173 {
174
175 return sprintf(page, "%llu\n",
176 (unsigned long long)q->limits.max_hw_discard_sectors << 9);
177 }
178
queue_discard_max_show(struct request_queue * q,char * page)179 static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
180 {
181 return sprintf(page, "%llu\n",
182 (unsigned long long)q->limits.max_discard_sectors << 9);
183 }
184
queue_discard_max_store(struct request_queue * q,const char * page,size_t count)185 static ssize_t queue_discard_max_store(struct request_queue *q,
186 const char *page, size_t count)
187 {
188 unsigned long max_discard;
189 ssize_t ret = queue_var_store(&max_discard, page, count);
190
191 if (ret < 0)
192 return ret;
193
194 if (max_discard & (q->limits.discard_granularity - 1))
195 return -EINVAL;
196
197 max_discard >>= 9;
198 if (max_discard > UINT_MAX)
199 return -EINVAL;
200
201 if (max_discard > q->limits.max_hw_discard_sectors)
202 max_discard = q->limits.max_hw_discard_sectors;
203
204 q->limits.max_discard_sectors = max_discard;
205 return ret;
206 }
207
queue_discard_zeroes_data_show(struct request_queue * q,char * page)208 static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
209 {
210 return queue_var_show(0, page);
211 }
212
queue_write_same_max_show(struct request_queue * q,char * page)213 static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
214 {
215 return sprintf(page, "%llu\n",
216 (unsigned long long)q->limits.max_write_same_sectors << 9);
217 }
218
queue_write_zeroes_max_show(struct request_queue * q,char * page)219 static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
220 {
221 return sprintf(page, "%llu\n",
222 (unsigned long long)q->limits.max_write_zeroes_sectors << 9);
223 }
224
queue_zone_write_granularity_show(struct request_queue * q,char * page)225 static ssize_t queue_zone_write_granularity_show(struct request_queue *q,
226 char *page)
227 {
228 return queue_var_show(queue_zone_write_granularity(q), page);
229 }
230
queue_zone_append_max_show(struct request_queue * q,char * page)231 static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page)
232 {
233 unsigned long long max_sectors = q->limits.max_zone_append_sectors;
234
235 return sprintf(page, "%llu\n", max_sectors << SECTOR_SHIFT);
236 }
237
238 static ssize_t
queue_max_sectors_store(struct request_queue * q,const char * page,size_t count)239 queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
240 {
241 unsigned long max_sectors_kb,
242 max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
243 page_kb = 1 << (PAGE_SHIFT - 10);
244 ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
245
246 if (ret < 0)
247 return ret;
248
249 max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
250 q->limits.max_dev_sectors >> 1);
251
252 if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
253 return -EINVAL;
254
255 spin_lock_irq(&q->queue_lock);
256 q->limits.max_sectors = max_sectors_kb << 1;
257 if (q->disk)
258 q->disk->bdi->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);
259 spin_unlock_irq(&q->queue_lock);
260
261 return ret;
262 }
263
queue_max_hw_sectors_show(struct request_queue * q,char * page)264 static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
265 {
266 int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
267
268 return queue_var_show(max_hw_sectors_kb, page);
269 }
270
queue_virt_boundary_mask_show(struct request_queue * q,char * page)271 static ssize_t queue_virt_boundary_mask_show(struct request_queue *q, char *page)
272 {
273 return queue_var_show(q->limits.virt_boundary_mask, page);
274 }
275
276 #define QUEUE_SYSFS_BIT_FNS(name, flag, neg) \
277 static ssize_t \
278 queue_##name##_show(struct request_queue *q, char *page) \
279 { \
280 int bit; \
281 bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags); \
282 return queue_var_show(neg ? !bit : bit, page); \
283 } \
284 static ssize_t \
285 queue_##name##_store(struct request_queue *q, const char *page, size_t count) \
286 { \
287 unsigned long val; \
288 ssize_t ret; \
289 ret = queue_var_store(&val, page, count); \
290 if (ret < 0) \
291 return ret; \
292 if (neg) \
293 val = !val; \
294 \
295 if (val) \
296 blk_queue_flag_set(QUEUE_FLAG_##flag, q); \
297 else \
298 blk_queue_flag_clear(QUEUE_FLAG_##flag, q); \
299 return ret; \
300 }
301
302 QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
303 QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
304 QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
305 QUEUE_SYSFS_BIT_FNS(stable_writes, STABLE_WRITES, 0);
306 #undef QUEUE_SYSFS_BIT_FNS
307
queue_zoned_show(struct request_queue * q,char * page)308 static ssize_t queue_zoned_show(struct request_queue *q, char *page)
309 {
310 switch (blk_queue_zoned_model(q)) {
311 case BLK_ZONED_HA:
312 return sprintf(page, "host-aware\n");
313 case BLK_ZONED_HM:
314 return sprintf(page, "host-managed\n");
315 default:
316 return sprintf(page, "none\n");
317 }
318 }
319
queue_nr_zones_show(struct request_queue * q,char * page)320 static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)
321 {
322 return queue_var_show(blk_queue_nr_zones(q), page);
323 }
324
queue_max_open_zones_show(struct request_queue * q,char * page)325 static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page)
326 {
327 return queue_var_show(queue_max_open_zones(q), page);
328 }
329
queue_max_active_zones_show(struct request_queue * q,char * page)330 static ssize_t queue_max_active_zones_show(struct request_queue *q, char *page)
331 {
332 return queue_var_show(queue_max_active_zones(q), page);
333 }
334
queue_nomerges_show(struct request_queue * q,char * page)335 static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
336 {
337 return queue_var_show((blk_queue_nomerges(q) << 1) |
338 blk_queue_noxmerges(q), page);
339 }
340
queue_nomerges_store(struct request_queue * q,const char * page,size_t count)341 static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
342 size_t count)
343 {
344 unsigned long nm;
345 ssize_t ret = queue_var_store(&nm, page, count);
346
347 if (ret < 0)
348 return ret;
349
350 blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
351 blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
352 if (nm == 2)
353 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
354 else if (nm)
355 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
356
357 return ret;
358 }
359
queue_rq_affinity_show(struct request_queue * q,char * page)360 static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
361 {
362 bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
363 bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
364
365 return queue_var_show(set << force, page);
366 }
367
368 static ssize_t
queue_rq_affinity_store(struct request_queue * q,const char * page,size_t count)369 queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
370 {
371 ssize_t ret = -EINVAL;
372 #ifdef CONFIG_SMP
373 unsigned long val;
374
375 ret = queue_var_store(&val, page, count);
376 if (ret < 0)
377 return ret;
378
379 if (val == 2) {
380 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
381 blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
382 } else if (val == 1) {
383 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
384 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
385 } else if (val == 0) {
386 blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
387 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
388 }
389 #endif
390 return ret;
391 }
392
queue_poll_delay_show(struct request_queue * q,char * page)393 static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
394 {
395 int val;
396
397 if (q->poll_nsec == BLK_MQ_POLL_CLASSIC)
398 val = BLK_MQ_POLL_CLASSIC;
399 else
400 val = q->poll_nsec / 1000;
401
402 return sprintf(page, "%d\n", val);
403 }
404
queue_poll_delay_store(struct request_queue * q,const char * page,size_t count)405 static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
406 size_t count)
407 {
408 int err, val;
409
410 if (!q->mq_ops || !q->mq_ops->poll)
411 return -EINVAL;
412
413 err = kstrtoint(page, 10, &val);
414 if (err < 0)
415 return err;
416
417 if (val == BLK_MQ_POLL_CLASSIC)
418 q->poll_nsec = BLK_MQ_POLL_CLASSIC;
419 else if (val >= 0)
420 q->poll_nsec = val * 1000;
421 else
422 return -EINVAL;
423
424 return count;
425 }
426
queue_poll_show(struct request_queue * q,char * page)427 static ssize_t queue_poll_show(struct request_queue *q, char *page)
428 {
429 return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
430 }
431
queue_poll_store(struct request_queue * q,const char * page,size_t count)432 static ssize_t queue_poll_store(struct request_queue *q, const char *page,
433 size_t count)
434 {
435 unsigned long poll_on;
436 ssize_t ret;
437
438 if (!q->tag_set || q->tag_set->nr_maps <= HCTX_TYPE_POLL ||
439 !q->tag_set->map[HCTX_TYPE_POLL].nr_queues)
440 return -EINVAL;
441
442 ret = queue_var_store(&poll_on, page, count);
443 if (ret < 0)
444 return ret;
445
446 if (poll_on) {
447 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
448 } else {
449 blk_mq_freeze_queue(q);
450 blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
451 blk_mq_unfreeze_queue(q);
452 }
453
454 return ret;
455 }
456
queue_io_timeout_show(struct request_queue * q,char * page)457 static ssize_t queue_io_timeout_show(struct request_queue *q, char *page)
458 {
459 return sprintf(page, "%u\n", jiffies_to_msecs(q->rq_timeout));
460 }
461
queue_io_timeout_store(struct request_queue * q,const char * page,size_t count)462 static ssize_t queue_io_timeout_store(struct request_queue *q, const char *page,
463 size_t count)
464 {
465 unsigned int val;
466 int err;
467
468 err = kstrtou32(page, 10, &val);
469 if (err || val == 0)
470 return -EINVAL;
471
472 blk_queue_rq_timeout(q, msecs_to_jiffies(val));
473
474 return count;
475 }
476
queue_wb_lat_show(struct request_queue * q,char * page)477 static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
478 {
479 if (!wbt_rq_qos(q))
480 return -EINVAL;
481
482 return sprintf(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
483 }
484
queue_wb_lat_store(struct request_queue * q,const char * page,size_t count)485 static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
486 size_t count)
487 {
488 struct rq_qos *rqos;
489 ssize_t ret;
490 s64 val;
491
492 ret = queue_var_store64(&val, page);
493 if (ret < 0)
494 return ret;
495 if (val < -1)
496 return -EINVAL;
497
498 rqos = wbt_rq_qos(q);
499 if (!rqos) {
500 ret = wbt_init(q);
501 if (ret)
502 return ret;
503 }
504
505 if (val == -1)
506 val = wbt_default_latency_nsec(q);
507 else if (val >= 0)
508 val *= 1000ULL;
509
510 if (wbt_get_min_lat(q) == val)
511 return count;
512
513 /*
514 * Ensure that the queue is idled, in case the latency update
515 * ends up either enabling or disabling wbt completely. We can't
516 * have IO inflight if that happens.
517 */
518 blk_mq_freeze_queue(q);
519 blk_mq_quiesce_queue(q);
520
521 wbt_set_min_lat(q, val);
522
523 blk_mq_unquiesce_queue(q);
524 blk_mq_unfreeze_queue(q);
525
526 return count;
527 }
528
queue_wc_show(struct request_queue * q,char * page)529 static ssize_t queue_wc_show(struct request_queue *q, char *page)
530 {
531 if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
532 return sprintf(page, "write back\n");
533
534 return sprintf(page, "write through\n");
535 }
536
queue_wc_store(struct request_queue * q,const char * page,size_t count)537 static ssize_t queue_wc_store(struct request_queue *q, const char *page,
538 size_t count)
539 {
540 int set = -1;
541
542 if (!strncmp(page, "write back", 10))
543 set = 1;
544 else if (!strncmp(page, "write through", 13) ||
545 !strncmp(page, "none", 4))
546 set = 0;
547
548 if (set == -1)
549 return -EINVAL;
550
551 if (set)
552 blk_queue_flag_set(QUEUE_FLAG_WC, q);
553 else
554 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
555
556 return count;
557 }
558
queue_fua_show(struct request_queue * q,char * page)559 static ssize_t queue_fua_show(struct request_queue *q, char *page)
560 {
561 return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));
562 }
563
queue_dax_show(struct request_queue * q,char * page)564 static ssize_t queue_dax_show(struct request_queue *q, char *page)
565 {
566 return queue_var_show(blk_queue_dax(q), page);
567 }
568
569 #define QUEUE_RO_ENTRY(_prefix, _name) \
570 static struct queue_sysfs_entry _prefix##_entry = { \
571 .attr = { .name = _name, .mode = 0444 }, \
572 .show = _prefix##_show, \
573 };
574
575 #define QUEUE_RW_ENTRY(_prefix, _name) \
576 static struct queue_sysfs_entry _prefix##_entry = { \
577 .attr = { .name = _name, .mode = 0644 }, \
578 .show = _prefix##_show, \
579 .store = _prefix##_store, \
580 };
581
582 QUEUE_RW_ENTRY(queue_requests, "nr_requests");
583 QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
584 QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
585 QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
586 QUEUE_RO_ENTRY(queue_max_segments, "max_segments");
587 QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
588 QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");
589 QUEUE_RW_ENTRY(elv_iosched, "scheduler");
590
591 QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");
592 QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");
593 QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
594 QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");
595 QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");
596
597 QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
598 QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");
599 QUEUE_RO_ENTRY(queue_discard_max_hw, "discard_max_hw_bytes");
600 QUEUE_RW_ENTRY(queue_discard_max, "discard_max_bytes");
601 QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");
602
603 QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
604 QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes");
605 QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
606 QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
607
608 QUEUE_RO_ENTRY(queue_zoned, "zoned");
609 QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
610 QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");
611 QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");
612
613 QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
614 QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
615 QUEUE_RW_ENTRY(queue_poll, "io_poll");
616 QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
617 QUEUE_RW_ENTRY(queue_wc, "write_cache");
618 QUEUE_RO_ENTRY(queue_fua, "fua");
619 QUEUE_RO_ENTRY(queue_dax, "dax");
620 QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
621 QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
622 QUEUE_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
623
624 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
625 QUEUE_RW_ENTRY(blk_throtl_sample_time, "throttle_sample_time");
626 #endif
627
628 /* legacy alias for logical_block_size: */
629 static struct queue_sysfs_entry queue_hw_sector_size_entry = {
630 .attr = {.name = "hw_sector_size", .mode = 0444 },
631 .show = queue_logical_block_size_show,
632 };
633
634 QUEUE_RW_ENTRY(queue_nonrot, "rotational");
635 QUEUE_RW_ENTRY(queue_iostats, "iostats");
636 QUEUE_RW_ENTRY(queue_random, "add_random");
637 QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");
638
639 static struct attribute *queue_attrs[] = {
640 &queue_requests_entry.attr,
641 &queue_ra_entry.attr,
642 &queue_max_hw_sectors_entry.attr,
643 &queue_max_sectors_entry.attr,
644 &queue_max_segments_entry.attr,
645 &queue_max_discard_segments_entry.attr,
646 &queue_max_integrity_segments_entry.attr,
647 &queue_max_segment_size_entry.attr,
648 &elv_iosched_entry.attr,
649 &queue_hw_sector_size_entry.attr,
650 &queue_logical_block_size_entry.attr,
651 &queue_physical_block_size_entry.attr,
652 &queue_chunk_sectors_entry.attr,
653 &queue_io_min_entry.attr,
654 &queue_io_opt_entry.attr,
655 &queue_discard_granularity_entry.attr,
656 &queue_discard_max_entry.attr,
657 &queue_discard_max_hw_entry.attr,
658 &queue_discard_zeroes_data_entry.attr,
659 &queue_write_same_max_entry.attr,
660 &queue_write_zeroes_max_entry.attr,
661 &queue_zone_append_max_entry.attr,
662 &queue_zone_write_granularity_entry.attr,
663 &queue_nonrot_entry.attr,
664 &queue_zoned_entry.attr,
665 &queue_nr_zones_entry.attr,
666 &queue_max_open_zones_entry.attr,
667 &queue_max_active_zones_entry.attr,
668 &queue_nomerges_entry.attr,
669 &queue_rq_affinity_entry.attr,
670 &queue_iostats_entry.attr,
671 &queue_stable_writes_entry.attr,
672 &queue_random_entry.attr,
673 &queue_poll_entry.attr,
674 &queue_wc_entry.attr,
675 &queue_fua_entry.attr,
676 &queue_dax_entry.attr,
677 &queue_wb_lat_entry.attr,
678 &queue_poll_delay_entry.attr,
679 &queue_io_timeout_entry.attr,
680 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
681 &blk_throtl_sample_time_entry.attr,
682 #endif
683 &queue_virt_boundary_mask_entry.attr,
684 NULL,
685 };
686
queue_attr_visible(struct kobject * kobj,struct attribute * attr,int n)687 static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
688 int n)
689 {
690 struct request_queue *q =
691 container_of(kobj, struct request_queue, kobj);
692
693 if (attr == &queue_io_timeout_entry.attr &&
694 (!q->mq_ops || !q->mq_ops->timeout))
695 return 0;
696
697 if ((attr == &queue_max_open_zones_entry.attr ||
698 attr == &queue_max_active_zones_entry.attr) &&
699 !blk_queue_is_zoned(q))
700 return 0;
701
702 return attr->mode;
703 }
704
705 static struct attribute_group queue_attr_group = {
706 .attrs = queue_attrs,
707 .is_visible = queue_attr_visible,
708 };
709
710
711 #define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
712
713 static ssize_t
queue_attr_show(struct kobject * kobj,struct attribute * attr,char * page)714 queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
715 {
716 struct queue_sysfs_entry *entry = to_queue(attr);
717 struct request_queue *q =
718 container_of(kobj, struct request_queue, kobj);
719 ssize_t res;
720
721 if (!entry->show)
722 return -EIO;
723 mutex_lock(&q->sysfs_lock);
724 res = entry->show(q, page);
725 mutex_unlock(&q->sysfs_lock);
726 return res;
727 }
728
729 static ssize_t
queue_attr_store(struct kobject * kobj,struct attribute * attr,const char * page,size_t length)730 queue_attr_store(struct kobject *kobj, struct attribute *attr,
731 const char *page, size_t length)
732 {
733 struct queue_sysfs_entry *entry = to_queue(attr);
734 struct request_queue *q;
735 ssize_t res;
736
737 if (!entry->store)
738 return -EIO;
739
740 q = container_of(kobj, struct request_queue, kobj);
741 mutex_lock(&q->sysfs_lock);
742 res = entry->store(q, page, length);
743 mutex_unlock(&q->sysfs_lock);
744 return res;
745 }
746
blk_free_queue_rcu(struct rcu_head * rcu_head)747 static void blk_free_queue_rcu(struct rcu_head *rcu_head)
748 {
749 struct request_queue *q = container_of(rcu_head, struct request_queue,
750 rcu_head);
751 kmem_cache_free(blk_requestq_cachep, q);
752 }
753
754 /* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */
blk_exit_queue(struct request_queue * q)755 static void blk_exit_queue(struct request_queue *q)
756 {
757 /*
758 * Since the I/O scheduler exit code may access cgroup information,
759 * perform I/O scheduler exit before disassociating from the block
760 * cgroup controller.
761 */
762 if (q->elevator) {
763 ioc_clear_queue(q);
764 __elevator_exit(q, q->elevator);
765 }
766
767 /*
768 * Remove all references to @q from the block cgroup controller before
769 * restoring @q->queue_lock to avoid that restoring this pointer causes
770 * e.g. blkcg_print_blkgs() to crash.
771 */
772 blkcg_exit_queue(q);
773 }
774
775 /**
776 * blk_release_queue - releases all allocated resources of the request_queue
777 * @kobj: pointer to a kobject, whose container is a request_queue
778 *
779 * This function releases all allocated resources of the request queue.
780 *
781 * The struct request_queue refcount is incremented with blk_get_queue() and
782 * decremented with blk_put_queue(). Once the refcount reaches 0 this function
783 * is called.
784 *
785 * For drivers that have a request_queue on a gendisk and added with
786 * __device_add_disk() the refcount to request_queue will reach 0 with
787 * the last put_disk() called by the driver. For drivers which don't use
788 * __device_add_disk() this happens with blk_cleanup_queue().
789 *
790 * Drivers exist which depend on the release of the request_queue to be
791 * synchronous, it should not be deferred.
792 *
793 * Context: can sleep
794 */
blk_release_queue(struct kobject * kobj)795 static void blk_release_queue(struct kobject *kobj)
796 {
797 struct request_queue *q =
798 container_of(kobj, struct request_queue, kobj);
799
800 might_sleep();
801
802 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
803 blk_stat_remove_callback(q, q->poll_cb);
804 blk_stat_free_callback(q->poll_cb);
805
806 blk_free_queue_stats(q->stats);
807
808 if (queue_is_mq(q)) {
809 struct blk_mq_hw_ctx *hctx;
810 int i;
811
812 cancel_delayed_work_sync(&q->requeue_work);
813
814 queue_for_each_hw_ctx(q, hctx, i)
815 cancel_delayed_work_sync(&hctx->run_work);
816 }
817
818 blk_exit_queue(q);
819
820 blk_queue_free_zone_bitmaps(q);
821
822 if (queue_is_mq(q))
823 blk_mq_release(q);
824
825 blk_trace_shutdown(q);
826 mutex_lock(&q->debugfs_mutex);
827 debugfs_remove_recursive(q->debugfs_dir);
828 mutex_unlock(&q->debugfs_mutex);
829
830 if (queue_is_mq(q))
831 blk_mq_debugfs_unregister(q);
832
833 bioset_exit(&q->bio_split);
834
835 ida_simple_remove(&blk_queue_ida, q->id);
836 call_rcu(&q->rcu_head, blk_free_queue_rcu);
837 }
838
839 static const struct sysfs_ops queue_sysfs_ops = {
840 .show = queue_attr_show,
841 .store = queue_attr_store,
842 };
843
844 struct kobj_type blk_queue_ktype = {
845 .sysfs_ops = &queue_sysfs_ops,
846 .release = blk_release_queue,
847 };
848
849 /**
850 * blk_register_queue - register a block layer queue with sysfs
851 * @disk: Disk of which the request queue should be registered with sysfs.
852 */
blk_register_queue(struct gendisk * disk)853 int blk_register_queue(struct gendisk *disk)
854 {
855 int ret;
856 struct device *dev = disk_to_dev(disk);
857 struct request_queue *q = disk->queue;
858
859 ret = blk_trace_init_sysfs(dev);
860 if (ret)
861 return ret;
862
863 mutex_lock(&q->sysfs_dir_lock);
864
865 ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
866 if (ret < 0) {
867 blk_trace_remove_sysfs(dev);
868 goto unlock;
869 }
870
871 ret = sysfs_create_group(&q->kobj, &queue_attr_group);
872 if (ret) {
873 blk_trace_remove_sysfs(dev);
874 kobject_del(&q->kobj);
875 kobject_put(&dev->kobj);
876 goto unlock;
877 }
878
879 mutex_lock(&q->debugfs_mutex);
880 q->debugfs_dir = debugfs_create_dir(kobject_name(q->kobj.parent),
881 blk_debugfs_root);
882 mutex_unlock(&q->debugfs_mutex);
883
884 if (queue_is_mq(q)) {
885 __blk_mq_register_dev(dev, q);
886 blk_mq_debugfs_register(q);
887 }
888
889 mutex_lock(&q->sysfs_lock);
890 if (q->elevator) {
891 ret = elv_register_queue(q, false);
892 if (ret) {
893 mutex_unlock(&q->sysfs_lock);
894 mutex_unlock(&q->sysfs_dir_lock);
895 kobject_del(&q->kobj);
896 blk_trace_remove_sysfs(dev);
897 kobject_put(&dev->kobj);
898 return ret;
899 }
900 }
901
902 blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
903 wbt_enable_default(q);
904 blk_throtl_register_queue(q);
905
906 /* Now everything is ready and send out KOBJ_ADD uevent */
907 kobject_uevent(&q->kobj, KOBJ_ADD);
908 if (q->elevator)
909 kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
910 mutex_unlock(&q->sysfs_lock);
911
912 ret = 0;
913 unlock:
914 mutex_unlock(&q->sysfs_dir_lock);
915
916 /*
917 * SCSI probing may synchronously create and destroy a lot of
918 * request_queues for non-existent devices. Shutting down a fully
919 * functional queue takes measureable wallclock time as RCU grace
920 * periods are involved. To avoid excessive latency in these
921 * cases, a request_queue starts out in a degraded mode which is
922 * faster to shut down and is made fully functional here as
923 * request_queues for non-existent devices never get registered.
924 */
925 if (!blk_queue_init_done(q)) {
926 blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
927 percpu_ref_switch_to_percpu(&q->q_usage_counter);
928 }
929
930 return ret;
931 }
932
933 /**
934 * blk_unregister_queue - counterpart of blk_register_queue()
935 * @disk: Disk of which the request queue should be unregistered from sysfs.
936 *
937 * Note: the caller is responsible for guaranteeing that this function is called
938 * after blk_register_queue() has finished.
939 */
blk_unregister_queue(struct gendisk * disk)940 void blk_unregister_queue(struct gendisk *disk)
941 {
942 struct request_queue *q = disk->queue;
943
944 if (WARN_ON(!q))
945 return;
946
947 /* Return early if disk->queue was never registered. */
948 if (!blk_queue_registered(q))
949 return;
950
951 /*
952 * Since sysfs_remove_dir() prevents adding new directory entries
953 * before removal of existing entries starts, protect against
954 * concurrent elv_iosched_store() calls.
955 */
956 mutex_lock(&q->sysfs_lock);
957 blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
958 mutex_unlock(&q->sysfs_lock);
959
960 mutex_lock(&q->sysfs_dir_lock);
961 /*
962 * Remove the sysfs attributes before unregistering the queue data
963 * structures that can be modified through sysfs.
964 */
965 if (queue_is_mq(q))
966 blk_mq_unregister_dev(disk_to_dev(disk), q);
967
968 kobject_uevent(&q->kobj, KOBJ_REMOVE);
969 kobject_del(&q->kobj);
970 blk_trace_remove_sysfs(disk_to_dev(disk));
971
972 mutex_lock(&q->sysfs_lock);
973 if (q->elevator)
974 elv_unregister_queue(q);
975 mutex_unlock(&q->sysfs_lock);
976 mutex_unlock(&q->sysfs_dir_lock);
977
978 kobject_put(&disk_to_dev(disk)->kobj);
979 }
980
