1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2009-2011 Red Hat, Inc.
4  *
5  * Author: Mikulas Patocka <mpatocka@redhat.com>
6  *
7  * This file is released under the GPL.
8  */
9 
10 #include <linux/dm-bufio.h>
11 
12 #include <linux/device-mapper.h>
13 #include <linux/dm-io.h>
14 #include <linux/slab.h>
15 #include <linux/sched/mm.h>
16 #include <linux/jiffies.h>
17 #include <linux/vmalloc.h>
18 #include <linux/shrinker.h>
19 #include <linux/module.h>
20 #include <linux/rbtree.h>
21 #include <linux/stacktrace.h>
22 #include <linux/jump_label.h>
23 
24 #include "dm.h"
25 
26 #define DM_MSG_PREFIX "bufio"
27 
28 /*
29  * Memory management policy:
30  *	Limit the number of buffers to DM_BUFIO_MEMORY_PERCENT of main memory
31  *	or DM_BUFIO_VMALLOC_PERCENT of vmalloc memory (whichever is lower).
32  *	Always allocate at least DM_BUFIO_MIN_BUFFERS buffers.
33  *	Start background writeback when there are DM_BUFIO_WRITEBACK_PERCENT
34  *	dirty buffers.
35  */
36 #define DM_BUFIO_MIN_BUFFERS		8
37 
38 #define DM_BUFIO_MEMORY_PERCENT		2
39 #define DM_BUFIO_VMALLOC_PERCENT	25
40 #define DM_BUFIO_WRITEBACK_RATIO	3
41 #define DM_BUFIO_LOW_WATERMARK_RATIO	16
42 
43 /*
44  * Check buffer ages in this interval (seconds)
45  */
46 #define DM_BUFIO_WORK_TIMER_SECS	30
47 
48 /*
49  * Free buffers when they are older than this (seconds)
50  */
51 #define DM_BUFIO_DEFAULT_AGE_SECS	300
52 
53 /*
54  * The nr of bytes of cached data to keep around.
55  */
56 #define DM_BUFIO_DEFAULT_RETAIN_BYTES   (256 * 1024)
57 
58 /*
59  * Align buffer writes to this boundary.
60  * Tests show that SSDs have the highest IOPS when using 4k writes.
61  */
62 #define DM_BUFIO_WRITE_ALIGN		4096
63 
64 /*
65  * dm_buffer->list_mode
66  */
67 #define LIST_CLEAN	0
68 #define LIST_DIRTY	1
69 #define LIST_SIZE	2
70 
71 /*--------------------------------------------------------------*/
72 
73 /*
74  * Rather than use an LRU list, we use a clock algorithm where entries
75  * are held in a circular list.  When an entry is 'hit' a reference bit
76  * is set.  The least recently used entry is approximated by running a
77  * cursor around the list selecting unreferenced entries. Referenced
78  * entries have their reference bit cleared as the cursor passes them.
79  */
80 struct lru_entry {
81 	struct list_head list;
82 	atomic_t referenced;
83 };
84 
85 struct lru_iter {
86 	struct lru *lru;
87 	struct list_head list;
88 	struct lru_entry *stop;
89 	struct lru_entry *e;
90 };
91 
92 struct lru {
93 	struct list_head *cursor;
94 	unsigned long count;
95 
96 	struct list_head iterators;
97 };
98 
99 /*--------------*/
100 
lru_init(struct lru * lru)101 static void lru_init(struct lru *lru)
102 {
103 	lru->cursor = NULL;
104 	lru->count = 0;
105 	INIT_LIST_HEAD(&lru->iterators);
106 }
107 
lru_destroy(struct lru * lru)108 static void lru_destroy(struct lru *lru)
109 {
110 	WARN_ON_ONCE(lru->cursor);
111 	WARN_ON_ONCE(!list_empty(&lru->iterators));
112 }
113 
114 /*
115  * Insert a new entry into the lru.
116  */
lru_insert(struct lru * lru,struct lru_entry * le)117 static void lru_insert(struct lru *lru, struct lru_entry *le)
118 {
119 	/*
120 	 * Don't be tempted to set to 1, makes the lru aspect
121 	 * perform poorly.
122 	 */
123 	atomic_set(&le->referenced, 0);
124 
125 	if (lru->cursor) {
126 		list_add_tail(&le->list, lru->cursor);
127 	} else {
128 		INIT_LIST_HEAD(&le->list);
129 		lru->cursor = &le->list;
130 	}
131 	lru->count++;
132 }
133 
134 /*--------------*/
135 
136 /*
137  * Convert a list_head pointer to an lru_entry pointer.
138  */
to_le(struct list_head * l)139 static inline struct lru_entry *to_le(struct list_head *l)
140 {
141 	return container_of(l, struct lru_entry, list);
142 }
143 
144 /*
145  * Initialize an lru_iter and add it to the list of cursors in the lru.
146  */
lru_iter_begin(struct lru * lru,struct lru_iter * it)147 static void lru_iter_begin(struct lru *lru, struct lru_iter *it)
148 {
149 	it->lru = lru;
150 	it->stop = lru->cursor ? to_le(lru->cursor->prev) : NULL;
151 	it->e = lru->cursor ? to_le(lru->cursor) : NULL;
152 	list_add(&it->list, &lru->iterators);
153 }
154 
155 /*
156  * Remove an lru_iter from the list of cursors in the lru.
157  */
lru_iter_end(struct lru_iter * it)158 static inline void lru_iter_end(struct lru_iter *it)
159 {
160 	list_del(&it->list);
161 }
162 
163 /* Predicate function type to be used with lru_iter_next */
164 typedef bool (*iter_predicate)(struct lru_entry *le, void *context);
165 
166 /*
167  * Advance the cursor to the next entry that passes the
168  * predicate, and return that entry.  Returns NULL if the
169  * iteration is complete.
170  */
lru_iter_next(struct lru_iter * it,iter_predicate pred,void * context)171 static struct lru_entry *lru_iter_next(struct lru_iter *it,
172 				       iter_predicate pred, void *context)
173 {
174 	struct lru_entry *e;
175 
176 	while (it->e) {
177 		e = it->e;
178 
179 		/* advance the cursor */
180 		if (it->e == it->stop)
181 			it->e = NULL;
182 		else
183 			it->e = to_le(it->e->list.next);
184 
185 		if (pred(e, context))
186 			return e;
187 	}
188 
189 	return NULL;
190 }
191 
192 /*
193  * Invalidate a specific lru_entry and update all cursors in
194  * the lru accordingly.
195  */
lru_iter_invalidate(struct lru * lru,struct lru_entry * e)196 static void lru_iter_invalidate(struct lru *lru, struct lru_entry *e)
197 {
198 	struct lru_iter *it;
199 
200 	list_for_each_entry(it, &lru->iterators, list) {
201 		/* Move c->e forwards if necc. */
202 		if (it->e == e) {
203 			it->e = to_le(it->e->list.next);
204 			if (it->e == e)
205 				it->e = NULL;
206 		}
207 
208 		/* Move it->stop backwards if necc. */
209 		if (it->stop == e) {
210 			it->stop = to_le(it->stop->list.prev);
211 			if (it->stop == e)
212 				it->stop = NULL;
213 		}
214 	}
215 }
216 
217 /*--------------*/
218 
219 /*
220  * Remove a specific entry from the lru.
221  */
lru_remove(struct lru * lru,struct lru_entry * le)222 static void lru_remove(struct lru *lru, struct lru_entry *le)
223 {
224 	lru_iter_invalidate(lru, le);
225 	if (lru->count == 1) {
226 		lru->cursor = NULL;
227 	} else {
228 		if (lru->cursor == &le->list)
229 			lru->cursor = lru->cursor->next;
230 		list_del(&le->list);
231 	}
232 	lru->count--;
233 }
234 
235 /*
236  * Mark as referenced.
237  */
lru_reference(struct lru_entry * le)238 static inline void lru_reference(struct lru_entry *le)
239 {
240 	atomic_set(&le->referenced, 1);
241 }
242 
243 /*--------------*/
244 
245 /*
246  * Remove the least recently used entry (approx), that passes the predicate.
247  * Returns NULL on failure.
248  */
249 enum evict_result {
250 	ER_EVICT,
251 	ER_DONT_EVICT,
252 	ER_STOP, /* stop looking for something to evict */
253 };
254 
255 typedef enum evict_result (*le_predicate)(struct lru_entry *le, void *context);
256 
257 static struct lru_entry *lru_evict(struct lru *lru, le_predicate pred, void *context)
258 {
259 	unsigned long tested = 0;
260 	struct list_head *h = lru->cursor;
261 	struct lru_entry *le;
262 
263 	if (!h)
264 		return NULL;
265 	/*
266 	 * In the worst case we have to loop around twice. Once to clear
267 	 * the reference flags, and then again to discover the predicate
268 	 * fails for all entries.
269 	 */
270 	while (tested < lru->count) {
271 		le = container_of(h, struct lru_entry, list);
272 
273 		if (atomic_read(&le->referenced)) {
274 			atomic_set(&le->referenced, 0);
275 		} else {
276 			tested++;
277 			switch (pred(le, context)) {
278 			case ER_EVICT:
279 				/*
280 				 * Adjust the cursor, so we start the next
281 				 * search from here.
282 				 */
283 				lru->cursor = le->list.next;
284 				lru_remove(lru, le);
285 				return le;
286 
287 			case ER_DONT_EVICT:
288 				break;
289 
290 			case ER_STOP:
291 				lru->cursor = le->list.next;
292 				return NULL;
293 			}
294 		}
295 
296 		h = h->next;
297 
298 		cond_resched();
299 	}
300 
301 	return NULL;
302 }
303 
304 /*--------------------------------------------------------------*/
305 
306 /*
307  * Buffer state bits.
308  */
309 #define B_READING	0
310 #define B_WRITING	1
311 #define B_DIRTY		2
312 
313 /*
314  * Describes how the block was allocated:
315  * kmem_cache_alloc(), __get_free_pages() or vmalloc().
316  * See the comment at alloc_buffer_data.
317  */
318 enum data_mode {
319 	DATA_MODE_SLAB = 0,
320 	DATA_MODE_GET_FREE_PAGES = 1,
321 	DATA_MODE_VMALLOC = 2,
322 	DATA_MODE_LIMIT = 3
323 };
324 
325 struct dm_buffer {
326 	/* protected by the locks in dm_buffer_cache */
327 	struct rb_node node;
328 
329 	/* immutable, so don't need protecting */
330 	sector_t block;
331 	void *data;
332 	unsigned char data_mode;		/* DATA_MODE_* */
333 
334 	/*
335 	 * These two fields are used in isolation, so do not need
336 	 * a surrounding lock.
337 	 */
338 	atomic_t hold_count;
339 	unsigned long last_accessed;
340 
341 	/*
342 	 * Everything else is protected by the mutex in
343 	 * dm_bufio_client
344 	 */
345 	unsigned long state;
346 	struct lru_entry lru;
347 	unsigned char list_mode;		/* LIST_* */
348 	blk_status_t read_error;
349 	blk_status_t write_error;
350 	unsigned int dirty_start;
351 	unsigned int dirty_end;
352 	unsigned int write_start;
353 	unsigned int write_end;
354 	struct list_head write_list;
355 	struct dm_bufio_client *c;
356 	void (*end_io)(struct dm_buffer *b, blk_status_t bs);
357 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
358 #define MAX_STACK 10
359 	unsigned int stack_len;
360 	unsigned long stack_entries[MAX_STACK];
361 #endif
362 };
363 
364 /*--------------------------------------------------------------*/
365 
366 /*
367  * The buffer cache manages buffers, particularly:
368  *  - inc/dec of holder count
369  *  - setting the last_accessed field
370  *  - maintains clean/dirty state along with lru
371  *  - selecting buffers that match predicates
372  *
373  * It does *not* handle:
374  *  - allocation/freeing of buffers.
375  *  - IO
376  *  - Eviction or cache sizing.
377  *
378  * cache_get() and cache_put() are threadsafe, you do not need to
379  * protect these calls with a surrounding mutex.  All the other
380  * methods are not threadsafe; they do use locking primitives, but
381  * only enough to ensure get/put are threadsafe.
382  */
383 
384 struct buffer_tree {
385 	struct rw_semaphore lock;
386 	struct rb_root root;
387 } ____cacheline_aligned_in_smp;
388 
389 struct dm_buffer_cache {
390 	struct lru lru[LIST_SIZE];
391 	/*
392 	 * We spread entries across multiple trees to reduce contention
393 	 * on the locks.
394 	 */
395 	unsigned int num_locks;
396 	struct buffer_tree trees[];
397 };
398 
cache_index(sector_t block,unsigned int num_locks)399 static inline unsigned int cache_index(sector_t block, unsigned int num_locks)
400 {
401 	return dm_hash_locks_index(block, num_locks);
402 }
403 
cache_read_lock(struct dm_buffer_cache * bc,sector_t block)404 static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block)
405 {
406 	down_read(&bc->trees[cache_index(block, bc->num_locks)].lock);
407 }
408 
cache_read_unlock(struct dm_buffer_cache * bc,sector_t block)409 static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block)
410 {
411 	up_read(&bc->trees[cache_index(block, bc->num_locks)].lock);
412 }
413 
cache_write_lock(struct dm_buffer_cache * bc,sector_t block)414 static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block)
415 {
416 	down_write(&bc->trees[cache_index(block, bc->num_locks)].lock);
417 }
418 
cache_write_unlock(struct dm_buffer_cache * bc,sector_t block)419 static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block)
420 {
421 	up_write(&bc->trees[cache_index(block, bc->num_locks)].lock);
422 }
423 
424 /*
425  * Sometimes we want to repeatedly get and drop locks as part of an iteration.
426  * This struct helps avoid redundant drop and gets of the same lock.
427  */
428 struct lock_history {
429 	struct dm_buffer_cache *cache;
430 	bool write;
431 	unsigned int previous;
432 	unsigned int no_previous;
433 };
434 
lh_init(struct lock_history * lh,struct dm_buffer_cache * cache,bool write)435 static void lh_init(struct lock_history *lh, struct dm_buffer_cache *cache, bool write)
436 {
437 	lh->cache = cache;
438 	lh->write = write;
439 	lh->no_previous = cache->num_locks;
440 	lh->previous = lh->no_previous;
441 }
442 
__lh_lock(struct lock_history * lh,unsigned int index)443 static void __lh_lock(struct lock_history *lh, unsigned int index)
444 {
445 	if (lh->write)
446 		down_write(&lh->cache->trees[index].lock);
447 	else
448 		down_read(&lh->cache->trees[index].lock);
449 }
450 
__lh_unlock(struct lock_history * lh,unsigned int index)451 static void __lh_unlock(struct lock_history *lh, unsigned int index)
452 {
453 	if (lh->write)
454 		up_write(&lh->cache->trees[index].lock);
455 	else
456 		up_read(&lh->cache->trees[index].lock);
457 }
458 
459 /*
460  * Make sure you call this since it will unlock the final lock.
461  */
lh_exit(struct lock_history * lh)462 static void lh_exit(struct lock_history *lh)
463 {
464 	if (lh->previous != lh->no_previous) {
465 		__lh_unlock(lh, lh->previous);
466 		lh->previous = lh->no_previous;
467 	}
468 }
469 
470 /*
471  * Named 'next' because there is no corresponding
472  * 'up/unlock' call since it's done automatically.
473  */
lh_next(struct lock_history * lh,sector_t b)474 static void lh_next(struct lock_history *lh, sector_t b)
475 {
476 	unsigned int index = cache_index(b, lh->no_previous); /* no_previous is num_locks */
477 
478 	if (lh->previous != lh->no_previous) {
479 		if (lh->previous != index) {
480 			__lh_unlock(lh, lh->previous);
481 			__lh_lock(lh, index);
482 			lh->previous = index;
483 		}
484 	} else {
485 		__lh_lock(lh, index);
486 		lh->previous = index;
487 	}
488 }
489 
le_to_buffer(struct lru_entry * le)490 static inline struct dm_buffer *le_to_buffer(struct lru_entry *le)
491 {
492 	return container_of(le, struct dm_buffer, lru);
493 }
494 
list_to_buffer(struct list_head * l)495 static struct dm_buffer *list_to_buffer(struct list_head *l)
496 {
497 	struct lru_entry *le = list_entry(l, struct lru_entry, list);
498 
499 	if (!le)
500 		return NULL;
501 
502 	return le_to_buffer(le);
503 }
504 
cache_init(struct dm_buffer_cache * bc,unsigned int num_locks)505 static void cache_init(struct dm_buffer_cache *bc, unsigned int num_locks)
506 {
507 	unsigned int i;
508 
509 	bc->num_locks = num_locks;
510 
511 	for (i = 0; i < bc->num_locks; i++) {
512 		init_rwsem(&bc->trees[i].lock);
513 		bc->trees[i].root = RB_ROOT;
514 	}
515 
516 	lru_init(&bc->lru[LIST_CLEAN]);
517 	lru_init(&bc->lru[LIST_DIRTY]);
518 }
519 
cache_destroy(struct dm_buffer_cache * bc)520 static void cache_destroy(struct dm_buffer_cache *bc)
521 {
522 	unsigned int i;
523 
524 	for (i = 0; i < bc->num_locks; i++)
525 		WARN_ON_ONCE(!RB_EMPTY_ROOT(&bc->trees[i].root));
526 
527 	lru_destroy(&bc->lru[LIST_CLEAN]);
528 	lru_destroy(&bc->lru[LIST_DIRTY]);
529 }
530 
531 /*--------------*/
532 
533 /*
534  * not threadsafe, or racey depending how you look at it
535  */
cache_count(struct dm_buffer_cache * bc,int list_mode)536 static inline unsigned long cache_count(struct dm_buffer_cache *bc, int list_mode)
537 {
538 	return bc->lru[list_mode].count;
539 }
540 
cache_total(struct dm_buffer_cache * bc)541 static inline unsigned long cache_total(struct dm_buffer_cache *bc)
542 {
543 	return cache_count(bc, LIST_CLEAN) + cache_count(bc, LIST_DIRTY);
544 }
545 
546 /*--------------*/
547 
548 /*
549  * Gets a specific buffer, indexed by block.
550  * If the buffer is found then its holder count will be incremented and
551  * lru_reference will be called.
552  *
553  * threadsafe
554  */
__cache_get(const struct rb_root * root,sector_t block)555 static struct dm_buffer *__cache_get(const struct rb_root *root, sector_t block)
556 {
557 	struct rb_node *n = root->rb_node;
558 	struct dm_buffer *b;
559 
560 	while (n) {
561 		b = container_of(n, struct dm_buffer, node);
562 
563 		if (b->block == block)
564 			return b;
565 
566 		n = block < b->block ? n->rb_left : n->rb_right;
567 	}
568 
569 	return NULL;
570 }
571 
__cache_inc_buffer(struct dm_buffer * b)572 static void __cache_inc_buffer(struct dm_buffer *b)
573 {
574 	atomic_inc(&b->hold_count);
575 	WRITE_ONCE(b->last_accessed, jiffies);
576 }
577 
cache_get(struct dm_buffer_cache * bc,sector_t block)578 static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block)
579 {
580 	struct dm_buffer *b;
581 
582 	cache_read_lock(bc, block);
583 	b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block);
584 	if (b) {
585 		lru_reference(&b->lru);
586 		__cache_inc_buffer(b);
587 	}
588 	cache_read_unlock(bc, block);
589 
590 	return b;
591 }
592 
593 /*--------------*/
594 
595 /*
596  * Returns true if the hold count hits zero.
597  * threadsafe
598  */
cache_put(struct dm_buffer_cache * bc,struct dm_buffer * b)599 static bool cache_put(struct dm_buffer_cache *bc, struct dm_buffer *b)
600 {
601 	bool r;
602 
603 	cache_read_lock(bc, b->block);
604 	BUG_ON(!atomic_read(&b->hold_count));
605 	r = atomic_dec_and_test(&b->hold_count);
606 	cache_read_unlock(bc, b->block);
607 
608 	return r;
609 }
610 
611 /*--------------*/
612 
613 typedef enum evict_result (*b_predicate)(struct dm_buffer *, void *);
614 
615 /*
616  * Evicts a buffer based on a predicate.  The oldest buffer that
617  * matches the predicate will be selected.  In addition to the
618  * predicate the hold_count of the selected buffer will be zero.
619  */
620 struct evict_wrapper {
621 	struct lock_history *lh;
622 	b_predicate pred;
623 	void *context;
624 };
625 
626 /*
627  * Wraps the buffer predicate turning it into an lru predicate.  Adds
628  * extra test for hold_count.
629  */
__evict_pred(struct lru_entry * le,void * context)630 static enum evict_result __evict_pred(struct lru_entry *le, void *context)
631 {
632 	struct evict_wrapper *w = context;
633 	struct dm_buffer *b = le_to_buffer(le);
634 
635 	lh_next(w->lh, b->block);
636 
637 	if (atomic_read(&b->hold_count))
638 		return ER_DONT_EVICT;
639 
640 	return w->pred(b, w->context);
641 }
642 
__cache_evict(struct dm_buffer_cache * bc,int list_mode,b_predicate pred,void * context,struct lock_history * lh)643 static struct dm_buffer *__cache_evict(struct dm_buffer_cache *bc, int list_mode,
644 				       b_predicate pred, void *context,
645 				       struct lock_history *lh)
646 {
647 	struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context};
648 	struct lru_entry *le;
649 	struct dm_buffer *b;
650 
651 	le = lru_evict(&bc->lru[list_mode], __evict_pred, &w);
652 	if (!le)
653 		return NULL;
654 
655 	b = le_to_buffer(le);
656 	/* __evict_pred will have locked the appropriate tree. */
657 	rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root);
658 
659 	return b;
660 }
661 
cache_evict(struct dm_buffer_cache * bc,int list_mode,b_predicate pred,void * context)662 static struct dm_buffer *cache_evict(struct dm_buffer_cache *bc, int list_mode,
663 				     b_predicate pred, void *context)
664 {
665 	struct dm_buffer *b;
666 	struct lock_history lh;
667 
668 	lh_init(&lh, bc, true);
669 	b = __cache_evict(bc, list_mode, pred, context, &lh);
670 	lh_exit(&lh);
671 
672 	return b;
673 }
674 
675 /*--------------*/
676 
677 /*
678  * Mark a buffer as clean or dirty. Not threadsafe.
679  */
cache_mark(struct dm_buffer_cache * bc,struct dm_buffer * b,int list_mode)680 static void cache_mark(struct dm_buffer_cache *bc, struct dm_buffer *b, int list_mode)
681 {
682 	cache_write_lock(bc, b->block);
683 	if (list_mode != b->list_mode) {
684 		lru_remove(&bc->lru[b->list_mode], &b->lru);
685 		b->list_mode = list_mode;
686 		lru_insert(&bc->lru[b->list_mode], &b->lru);
687 	}
688 	cache_write_unlock(bc, b->block);
689 }
690 
691 /*--------------*/
692 
693 /*
694  * Runs through the lru associated with 'old_mode', if the predicate matches then
695  * it moves them to 'new_mode'.  Not threadsafe.
696  */
__cache_mark_many(struct dm_buffer_cache * bc,int old_mode,int new_mode,b_predicate pred,void * context,struct lock_history * lh)697 static void __cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode,
698 			      b_predicate pred, void *context, struct lock_history *lh)
699 {
700 	struct lru_entry *le;
701 	struct dm_buffer *b;
702 	struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context};
703 
704 	while (true) {
705 		le = lru_evict(&bc->lru[old_mode], __evict_pred, &w);
706 		if (!le)
707 			break;
708 
709 		b = le_to_buffer(le);
710 		b->list_mode = new_mode;
711 		lru_insert(&bc->lru[b->list_mode], &b->lru);
712 	}
713 }
714 
cache_mark_many(struct dm_buffer_cache * bc,int old_mode,int new_mode,b_predicate pred,void * context)715 static void cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode,
716 			    b_predicate pred, void *context)
717 {
718 	struct lock_history lh;
719 
720 	lh_init(&lh, bc, true);
721 	__cache_mark_many(bc, old_mode, new_mode, pred, context, &lh);
722 	lh_exit(&lh);
723 }
724 
725 /*--------------*/
726 
727 /*
728  * Iterates through all clean or dirty entries calling a function for each
729  * entry.  The callback may terminate the iteration early.  Not threadsafe.
730  */
731 
732 /*
733  * Iterator functions should return one of these actions to indicate
734  * how the iteration should proceed.
735  */
736 enum it_action {
737 	IT_NEXT,
738 	IT_COMPLETE,
739 };
740 
741 typedef enum it_action (*iter_fn)(struct dm_buffer *b, void *context);
742 
743 static void __cache_iterate(struct dm_buffer_cache *bc, int list_mode,
744 			    iter_fn fn, void *context, struct lock_history *lh)
745 {
746 	struct lru *lru = &bc->lru[list_mode];
747 	struct lru_entry *le, *first;
748 
749 	if (!lru->cursor)
750 		return;
751 
752 	first = le = to_le(lru->cursor);
753 	do {
754 		struct dm_buffer *b = le_to_buffer(le);
755 
756 		lh_next(lh, b->block);
757 
758 		switch (fn(b, context)) {
759 		case IT_NEXT:
760 			break;
761 
762 		case IT_COMPLETE:
763 			return;
764 		}
765 		cond_resched();
766 
767 		le = to_le(le->list.next);
768 	} while (le != first);
769 }
770 
cache_iterate(struct dm_buffer_cache * bc,int list_mode,iter_fn fn,void * context)771 static void cache_iterate(struct dm_buffer_cache *bc, int list_mode,
772 			  iter_fn fn, void *context)
773 {
774 	struct lock_history lh;
775 
776 	lh_init(&lh, bc, false);
777 	__cache_iterate(bc, list_mode, fn, context, &lh);
778 	lh_exit(&lh);
779 }
780 
781 /*--------------*/
782 
783 /*
784  * Passes ownership of the buffer to the cache. Returns false if the
785  * buffer was already present (in which case ownership does not pass).
786  * eg, a race with another thread.
787  *
788  * Holder count should be 1 on insertion.
789  *
790  * Not threadsafe.
791  */
__cache_insert(struct rb_root * root,struct dm_buffer * b)792 static bool __cache_insert(struct rb_root *root, struct dm_buffer *b)
793 {
794 	struct rb_node **new = &root->rb_node, *parent = NULL;
795 	struct dm_buffer *found;
796 
797 	while (*new) {
798 		found = container_of(*new, struct dm_buffer, node);
799 
800 		if (found->block == b->block)
801 			return false;
802 
803 		parent = *new;
804 		new = b->block < found->block ?
805 			&found->node.rb_left : &found->node.rb_right;
806 	}
807 
808 	rb_link_node(&b->node, parent, new);
809 	rb_insert_color(&b->node, root);
810 
811 	return true;
812 }
813 
cache_insert(struct dm_buffer_cache * bc,struct dm_buffer * b)814 static bool cache_insert(struct dm_buffer_cache *bc, struct dm_buffer *b)
815 {
816 	bool r;
817 
818 	if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE))
819 		return false;
820 
821 	cache_write_lock(bc, b->block);
822 	BUG_ON(atomic_read(&b->hold_count) != 1);
823 	r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b);
824 	if (r)
825 		lru_insert(&bc->lru[b->list_mode], &b->lru);
826 	cache_write_unlock(bc, b->block);
827 
828 	return r;
829 }
830 
831 /*--------------*/
832 
833 /*
834  * Removes buffer from cache, ownership of the buffer passes back to the caller.
835  * Fails if the hold_count is not one (ie. the caller holds the only reference).
836  *
837  * Not threadsafe.
838  */
cache_remove(struct dm_buffer_cache * bc,struct dm_buffer * b)839 static bool cache_remove(struct dm_buffer_cache *bc, struct dm_buffer *b)
840 {
841 	bool r;
842 
843 	cache_write_lock(bc, b->block);
844 
845 	if (atomic_read(&b->hold_count) != 1) {
846 		r = false;
847 	} else {
848 		r = true;
849 		rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root);
850 		lru_remove(&bc->lru[b->list_mode], &b->lru);
851 	}
852 
853 	cache_write_unlock(bc, b->block);
854 
855 	return r;
856 }
857 
858 /*--------------*/
859 
860 typedef void (*b_release)(struct dm_buffer *);
861 
__find_next(struct rb_root * root,sector_t block)862 static struct dm_buffer *__find_next(struct rb_root *root, sector_t block)
863 {
864 	struct rb_node *n = root->rb_node;
865 	struct dm_buffer *b;
866 	struct dm_buffer *best = NULL;
867 
868 	while (n) {
869 		b = container_of(n, struct dm_buffer, node);
870 
871 		if (b->block == block)
872 			return b;
873 
874 		if (block <= b->block) {
875 			n = n->rb_left;
876 			best = b;
877 		} else {
878 			n = n->rb_right;
879 		}
880 	}
881 
882 	return best;
883 }
884 
__remove_range(struct dm_buffer_cache * bc,struct rb_root * root,sector_t begin,sector_t end,b_predicate pred,b_release release)885 static void __remove_range(struct dm_buffer_cache *bc,
886 			   struct rb_root *root,
887 			   sector_t begin, sector_t end,
888 			   b_predicate pred, b_release release)
889 {
890 	struct dm_buffer *b;
891 
892 	while (true) {
893 		cond_resched();
894 
895 		b = __find_next(root, begin);
896 		if (!b || (b->block >= end))
897 			break;
898 
899 		begin = b->block + 1;
900 
901 		if (atomic_read(&b->hold_count))
902 			continue;
903 
904 		if (pred(b, NULL) == ER_EVICT) {
905 			rb_erase(&b->node, root);
906 			lru_remove(&bc->lru[b->list_mode], &b->lru);
907 			release(b);
908 		}
909 	}
910 }
911 
cache_remove_range(struct dm_buffer_cache * bc,sector_t begin,sector_t end,b_predicate pred,b_release release)912 static void cache_remove_range(struct dm_buffer_cache *bc,
913 			       sector_t begin, sector_t end,
914 			       b_predicate pred, b_release release)
915 {
916 	unsigned int i;
917 
918 	for (i = 0; i < bc->num_locks; i++) {
919 		down_write(&bc->trees[i].lock);
920 		__remove_range(bc, &bc->trees[i].root, begin, end, pred, release);
921 		up_write(&bc->trees[i].lock);
922 	}
923 }
924 
925 /*----------------------------------------------------------------*/
926 
927 /*
928  * Linking of buffers:
929  *	All buffers are linked to buffer_cache with their node field.
930  *
931  *	Clean buffers that are not being written (B_WRITING not set)
932  *	are linked to lru[LIST_CLEAN] with their lru_list field.
933  *
934  *	Dirty and clean buffers that are being written are linked to
935  *	lru[LIST_DIRTY] with their lru_list field. When the write
936  *	finishes, the buffer cannot be relinked immediately (because we
937  *	are in an interrupt context and relinking requires process
938  *	context), so some clean-not-writing buffers can be held on
939  *	dirty_lru too.  They are later added to lru in the process
940  *	context.
941  */
942 struct dm_bufio_client {
943 	struct block_device *bdev;
944 	unsigned int block_size;
945 	s8 sectors_per_block_bits;
946 
947 	bool no_sleep;
948 	struct mutex lock;
949 	spinlock_t spinlock;
950 
951 	int async_write_error;
952 
953 	void (*alloc_callback)(struct dm_buffer *buf);
954 	void (*write_callback)(struct dm_buffer *buf);
955 	struct kmem_cache *slab_buffer;
956 	struct kmem_cache *slab_cache;
957 	struct dm_io_client *dm_io;
958 
959 	struct list_head reserved_buffers;
960 	unsigned int need_reserved_buffers;
961 
962 	unsigned int minimum_buffers;
963 
964 	sector_t start;
965 
966 	struct shrinker shrinker;
967 	struct work_struct shrink_work;
968 	atomic_long_t need_shrink;
969 
970 	wait_queue_head_t free_buffer_wait;
971 
972 	struct list_head client_list;
973 
974 	/*
975 	 * Used by global_cleanup to sort the clients list.
976 	 */
977 	unsigned long oldest_buffer;
978 
979 	struct dm_buffer_cache cache; /* must be last member */
980 };
981 
982 static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled);
983 
984 /*----------------------------------------------------------------*/
985 
986 #define dm_bufio_in_request()	(!!current->bio_list)
987 
dm_bufio_lock(struct dm_bufio_client * c)988 static void dm_bufio_lock(struct dm_bufio_client *c)
989 {
990 	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
991 		spin_lock_bh(&c->spinlock);
992 	else
993 		mutex_lock_nested(&c->lock, dm_bufio_in_request());
994 }
995 
dm_bufio_unlock(struct dm_bufio_client * c)996 static void dm_bufio_unlock(struct dm_bufio_client *c)
997 {
998 	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
999 		spin_unlock_bh(&c->spinlock);
1000 	else
1001 		mutex_unlock(&c->lock);
1002 }
1003 
1004 /*----------------------------------------------------------------*/
1005 
1006 /*
1007  * Default cache size: available memory divided by the ratio.
1008  */
1009 static unsigned long dm_bufio_default_cache_size;
1010 
1011 /*
1012  * Total cache size set by the user.
1013  */
1014 static unsigned long dm_bufio_cache_size;
1015 
1016 /*
1017  * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change
1018  * at any time.  If it disagrees, the user has changed cache size.
1019  */
1020 static unsigned long dm_bufio_cache_size_latch;
1021 
1022 static DEFINE_SPINLOCK(global_spinlock);
1023 
1024 /*
1025  * Buffers are freed after this timeout
1026  */
1027 static unsigned int dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS;
1028 static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES;
1029 
1030 static unsigned long dm_bufio_peak_allocated;
1031 static unsigned long dm_bufio_allocated_kmem_cache;
1032 static unsigned long dm_bufio_allocated_get_free_pages;
1033 static unsigned long dm_bufio_allocated_vmalloc;
1034 static unsigned long dm_bufio_current_allocated;
1035 
1036 /*----------------------------------------------------------------*/
1037 
1038 /*
1039  * The current number of clients.
1040  */
1041 static int dm_bufio_client_count;
1042 
1043 /*
1044  * The list of all clients.
1045  */
1046 static LIST_HEAD(dm_bufio_all_clients);
1047 
1048 /*
1049  * This mutex protects dm_bufio_cache_size_latch and dm_bufio_client_count
1050  */
1051 static DEFINE_MUTEX(dm_bufio_clients_lock);
1052 
1053 static struct workqueue_struct *dm_bufio_wq;
1054 static struct delayed_work dm_bufio_cleanup_old_work;
1055 static struct work_struct dm_bufio_replacement_work;
1056 
1057 
1058 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
buffer_record_stack(struct dm_buffer * b)1059 static void buffer_record_stack(struct dm_buffer *b)
1060 {
1061 	b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2);
1062 }
1063 #endif
1064 
1065 /*----------------------------------------------------------------*/
1066 
adjust_total_allocated(struct dm_buffer * b,bool unlink)1067 static void adjust_total_allocated(struct dm_buffer *b, bool unlink)
1068 {
1069 	unsigned char data_mode;
1070 	long diff;
1071 
1072 	static unsigned long * const class_ptr[DATA_MODE_LIMIT] = {
1073 		&dm_bufio_allocated_kmem_cache,
1074 		&dm_bufio_allocated_get_free_pages,
1075 		&dm_bufio_allocated_vmalloc,
1076 	};
1077 
1078 	data_mode = b->data_mode;
1079 	diff = (long)b->c->block_size;
1080 	if (unlink)
1081 		diff = -diff;
1082 
1083 	spin_lock(&global_spinlock);
1084 
1085 	*class_ptr[data_mode] += diff;
1086 
1087 	dm_bufio_current_allocated += diff;
1088 
1089 	if (dm_bufio_current_allocated > dm_bufio_peak_allocated)
1090 		dm_bufio_peak_allocated = dm_bufio_current_allocated;
1091 
1092 	if (!unlink) {
1093 		if (dm_bufio_current_allocated > dm_bufio_cache_size)
1094 			queue_work(dm_bufio_wq, &dm_bufio_replacement_work);
1095 	}
1096 
1097 	spin_unlock(&global_spinlock);
1098 }
1099 
1100 /*
1101  * Change the number of clients and recalculate per-client limit.
1102  */
__cache_size_refresh(void)1103 static void __cache_size_refresh(void)
1104 {
1105 	if (WARN_ON(!mutex_is_locked(&dm_bufio_clients_lock)))
1106 		return;
1107 	if (WARN_ON(dm_bufio_client_count < 0))
1108 		return;
1109 
1110 	dm_bufio_cache_size_latch = READ_ONCE(dm_bufio_cache_size);
1111 
1112 	/*
1113 	 * Use default if set to 0 and report the actual cache size used.
1114 	 */
1115 	if (!dm_bufio_cache_size_latch) {
1116 		(void)cmpxchg(&dm_bufio_cache_size, 0,
1117 			      dm_bufio_default_cache_size);
1118 		dm_bufio_cache_size_latch = dm_bufio_default_cache_size;
1119 	}
1120 }
1121 
1122 /*
1123  * Allocating buffer data.
1124  *
1125  * Small buffers are allocated with kmem_cache, to use space optimally.
1126  *
1127  * For large buffers, we choose between get_free_pages and vmalloc.
1128  * Each has advantages and disadvantages.
1129  *
1130  * __get_free_pages can randomly fail if the memory is fragmented.
1131  * __vmalloc won't randomly fail, but vmalloc space is limited (it may be
1132  * as low as 128M) so using it for caching is not appropriate.
1133  *
1134  * If the allocation may fail we use __get_free_pages. Memory fragmentation
1135  * won't have a fatal effect here, but it just causes flushes of some other
1136  * buffers and more I/O will be performed. Don't use __get_free_pages if it
1137  * always fails (i.e. order > MAX_ORDER).
1138  *
1139  * If the allocation shouldn't fail we use __vmalloc. This is only for the
1140  * initial reserve allocation, so there's no risk of wasting all vmalloc
1141  * space.
1142  */
alloc_buffer_data(struct dm_bufio_client * c,gfp_t gfp_mask,unsigned char * data_mode)1143 static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask,
1144 			       unsigned char *data_mode)
1145 {
1146 	if (unlikely(c->slab_cache != NULL)) {
1147 		*data_mode = DATA_MODE_SLAB;
1148 		return kmem_cache_alloc(c->slab_cache, gfp_mask);
1149 	}
1150 
1151 	if (c->block_size <= KMALLOC_MAX_SIZE &&
1152 	    gfp_mask & __GFP_NORETRY) {
1153 		*data_mode = DATA_MODE_GET_FREE_PAGES;
1154 		return (void *)__get_free_pages(gfp_mask,
1155 						c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
1156 	}
1157 
1158 	*data_mode = DATA_MODE_VMALLOC;
1159 
1160 	return __vmalloc(c->block_size, gfp_mask);
1161 }
1162 
1163 /*
1164  * Free buffer's data.
1165  */
free_buffer_data(struct dm_bufio_client * c,void * data,unsigned char data_mode)1166 static void free_buffer_data(struct dm_bufio_client *c,
1167 			     void *data, unsigned char data_mode)
1168 {
1169 	switch (data_mode) {
1170 	case DATA_MODE_SLAB:
1171 		kmem_cache_free(c->slab_cache, data);
1172 		break;
1173 
1174 	case DATA_MODE_GET_FREE_PAGES:
1175 		free_pages((unsigned long)data,
1176 			   c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
1177 		break;
1178 
1179 	case DATA_MODE_VMALLOC:
1180 		vfree(data);
1181 		break;
1182 
1183 	default:
1184 		DMCRIT("dm_bufio_free_buffer_data: bad data mode: %d",
1185 		       data_mode);
1186 		BUG();
1187 	}
1188 }
1189 
1190 /*
1191  * Allocate buffer and its data.
1192  */
alloc_buffer(struct dm_bufio_client * c,gfp_t gfp_mask)1193 static struct dm_buffer *alloc_buffer(struct dm_bufio_client *c, gfp_t gfp_mask)
1194 {
1195 	struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask);
1196 
1197 	if (!b)
1198 		return NULL;
1199 
1200 	b->c = c;
1201 
1202 	b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode);
1203 	if (!b->data) {
1204 		kmem_cache_free(c->slab_buffer, b);
1205 		return NULL;
1206 	}
1207 	adjust_total_allocated(b, false);
1208 
1209 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1210 	b->stack_len = 0;
1211 #endif
1212 	return b;
1213 }
1214 
1215 /*
1216  * Free buffer and its data.
1217  */
free_buffer(struct dm_buffer * b)1218 static void free_buffer(struct dm_buffer *b)
1219 {
1220 	struct dm_bufio_client *c = b->c;
1221 
1222 	adjust_total_allocated(b, true);
1223 	free_buffer_data(c, b->data, b->data_mode);
1224 	kmem_cache_free(c->slab_buffer, b);
1225 }
1226 
1227 /*
1228  *--------------------------------------------------------------------------
1229  * Submit I/O on the buffer.
1230  *
1231  * Bio interface is faster but it has some problems:
1232  *	the vector list is limited (increasing this limit increases
1233  *	memory-consumption per buffer, so it is not viable);
1234  *
1235  *	the memory must be direct-mapped, not vmalloced;
1236  *
1237  * If the buffer is small enough (up to DM_BUFIO_INLINE_VECS pages) and
1238  * it is not vmalloced, try using the bio interface.
1239  *
1240  * If the buffer is big, if it is vmalloced or if the underlying device
1241  * rejects the bio because it is too large, use dm-io layer to do the I/O.
1242  * The dm-io layer splits the I/O into multiple requests, avoiding the above
1243  * shortcomings.
1244  *--------------------------------------------------------------------------
1245  */
1246 
1247 /*
1248  * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
1249  * that the request was handled directly with bio interface.
1250  */
dmio_complete(unsigned long error,void * context)1251 static void dmio_complete(unsigned long error, void *context)
1252 {
1253 	struct dm_buffer *b = context;
1254 
1255 	b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0);
1256 }
1257 
use_dmio(struct dm_buffer * b,enum req_op op,sector_t sector,unsigned int n_sectors,unsigned int offset)1258 static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
1259 		     unsigned int n_sectors, unsigned int offset)
1260 {
1261 	int r;
1262 	struct dm_io_request io_req = {
1263 		.bi_opf = op,
1264 		.notify.fn = dmio_complete,
1265 		.notify.context = b,
1266 		.client = b->c->dm_io,
1267 	};
1268 	struct dm_io_region region = {
1269 		.bdev = b->c->bdev,
1270 		.sector = sector,
1271 		.count = n_sectors,
1272 	};
1273 
1274 	if (b->data_mode != DATA_MODE_VMALLOC) {
1275 		io_req.mem.type = DM_IO_KMEM;
1276 		io_req.mem.ptr.addr = (char *)b->data + offset;
1277 	} else {
1278 		io_req.mem.type = DM_IO_VMA;
1279 		io_req.mem.ptr.vma = (char *)b->data + offset;
1280 	}
1281 
1282 	r = dm_io(&io_req, 1, &region, NULL);
1283 	if (unlikely(r))
1284 		b->end_io(b, errno_to_blk_status(r));
1285 }
1286 
bio_complete(struct bio * bio)1287 static void bio_complete(struct bio *bio)
1288 {
1289 	struct dm_buffer *b = bio->bi_private;
1290 	blk_status_t status = bio->bi_status;
1291 
1292 	bio_uninit(bio);
1293 	kfree(bio);
1294 	b->end_io(b, status);
1295 }
1296 
use_bio(struct dm_buffer * b,enum req_op op,sector_t sector,unsigned int n_sectors,unsigned int offset)1297 static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
1298 		    unsigned int n_sectors, unsigned int offset)
1299 {
1300 	struct bio *bio;
1301 	char *ptr;
1302 	unsigned int len;
1303 
1304 	bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN);
1305 	if (!bio) {
1306 		use_dmio(b, op, sector, n_sectors, offset);
1307 		return;
1308 	}
1309 	bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op);
1310 	bio->bi_iter.bi_sector = sector;
1311 	bio->bi_end_io = bio_complete;
1312 	bio->bi_private = b;
1313 
1314 	ptr = (char *)b->data + offset;
1315 	len = n_sectors << SECTOR_SHIFT;
1316 
1317 	__bio_add_page(bio, virt_to_page(ptr), len, offset_in_page(ptr));
1318 
1319 	submit_bio(bio);
1320 }
1321 
block_to_sector(struct dm_bufio_client * c,sector_t block)1322 static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block)
1323 {
1324 	sector_t sector;
1325 
1326 	if (likely(c->sectors_per_block_bits >= 0))
1327 		sector = block << c->sectors_per_block_bits;
1328 	else
1329 		sector = block * (c->block_size >> SECTOR_SHIFT);
1330 	sector += c->start;
1331 
1332 	return sector;
1333 }
1334 
submit_io(struct dm_buffer * b,enum req_op op,void (* end_io)(struct dm_buffer *,blk_status_t))1335 static void submit_io(struct dm_buffer *b, enum req_op op,
1336 		      void (*end_io)(struct dm_buffer *, blk_status_t))
1337 {
1338 	unsigned int n_sectors;
1339 	sector_t sector;
1340 	unsigned int offset, end;
1341 
1342 	b->end_io = end_io;
1343 
1344 	sector = block_to_sector(b->c, b->block);
1345 
1346 	if (op != REQ_OP_WRITE) {
1347 		n_sectors = b->c->block_size >> SECTOR_SHIFT;
1348 		offset = 0;
1349 	} else {
1350 		if (b->c->write_callback)
1351 			b->c->write_callback(b);
1352 		offset = b->write_start;
1353 		end = b->write_end;
1354 		offset &= -DM_BUFIO_WRITE_ALIGN;
1355 		end += DM_BUFIO_WRITE_ALIGN - 1;
1356 		end &= -DM_BUFIO_WRITE_ALIGN;
1357 		if (unlikely(end > b->c->block_size))
1358 			end = b->c->block_size;
1359 
1360 		sector += offset >> SECTOR_SHIFT;
1361 		n_sectors = (end - offset) >> SECTOR_SHIFT;
1362 	}
1363 
1364 	if (b->data_mode != DATA_MODE_VMALLOC)
1365 		use_bio(b, op, sector, n_sectors, offset);
1366 	else
1367 		use_dmio(b, op, sector, n_sectors, offset);
1368 }
1369 
1370 /*
1371  *--------------------------------------------------------------
1372  * Writing dirty buffers
1373  *--------------------------------------------------------------
1374  */
1375 
1376 /*
1377  * The endio routine for write.
1378  *
1379  * Set the error, clear B_WRITING bit and wake anyone who was waiting on
1380  * it.
1381  */
write_endio(struct dm_buffer * b,blk_status_t status)1382 static void write_endio(struct dm_buffer *b, blk_status_t status)
1383 {
1384 	b->write_error = status;
1385 	if (unlikely(status)) {
1386 		struct dm_bufio_client *c = b->c;
1387 
1388 		(void)cmpxchg(&c->async_write_error, 0,
1389 				blk_status_to_errno(status));
1390 	}
1391 
1392 	BUG_ON(!test_bit(B_WRITING, &b->state));
1393 
1394 	smp_mb__before_atomic();
1395 	clear_bit(B_WRITING, &b->state);
1396 	smp_mb__after_atomic();
1397 
1398 	wake_up_bit(&b->state, B_WRITING);
1399 }
1400 
1401 /*
1402  * Initiate a write on a dirty buffer, but don't wait for it.
1403  *
1404  * - If the buffer is not dirty, exit.
1405  * - If there some previous write going on, wait for it to finish (we can't
1406  *   have two writes on the same buffer simultaneously).
1407  * - Submit our write and don't wait on it. We set B_WRITING indicating
1408  *   that there is a write in progress.
1409  */
__write_dirty_buffer(struct dm_buffer * b,struct list_head * write_list)1410 static void __write_dirty_buffer(struct dm_buffer *b,
1411 				 struct list_head *write_list)
1412 {
1413 	if (!test_bit(B_DIRTY, &b->state))
1414 		return;
1415 
1416 	clear_bit(B_DIRTY, &b->state);
1417 	wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
1418 
1419 	b->write_start = b->dirty_start;
1420 	b->write_end = b->dirty_end;
1421 
1422 	if (!write_list)
1423 		submit_io(b, REQ_OP_WRITE, write_endio);
1424 	else
1425 		list_add_tail(&b->write_list, write_list);
1426 }
1427 
__flush_write_list(struct list_head * write_list)1428 static void __flush_write_list(struct list_head *write_list)
1429 {
1430 	struct blk_plug plug;
1431 
1432 	blk_start_plug(&plug);
1433 	while (!list_empty(write_list)) {
1434 		struct dm_buffer *b =
1435 			list_entry(write_list->next, struct dm_buffer, write_list);
1436 		list_del(&b->write_list);
1437 		submit_io(b, REQ_OP_WRITE, write_endio);
1438 		cond_resched();
1439 	}
1440 	blk_finish_plug(&plug);
1441 }
1442 
1443 /*
1444  * Wait until any activity on the buffer finishes.  Possibly write the
1445  * buffer if it is dirty.  When this function finishes, there is no I/O
1446  * running on the buffer and the buffer is not dirty.
1447  */
__make_buffer_clean(struct dm_buffer * b)1448 static void __make_buffer_clean(struct dm_buffer *b)
1449 {
1450 	BUG_ON(atomic_read(&b->hold_count));
1451 
1452 	/* smp_load_acquire() pairs with read_endio()'s smp_mb__before_atomic() */
1453 	if (!smp_load_acquire(&b->state))	/* fast case */
1454 		return;
1455 
1456 	wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
1457 	__write_dirty_buffer(b, NULL);
1458 	wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
1459 }
1460 
is_clean(struct dm_buffer * b,void * context)1461 static enum evict_result is_clean(struct dm_buffer *b, void *context)
1462 {
1463 	struct dm_bufio_client *c = context;
1464 
1465 	/* These should never happen */
1466 	if (WARN_ON_ONCE(test_bit(B_WRITING, &b->state)))
1467 		return ER_DONT_EVICT;
1468 	if (WARN_ON_ONCE(test_bit(B_DIRTY, &b->state)))
1469 		return ER_DONT_EVICT;
1470 	if (WARN_ON_ONCE(b->list_mode != LIST_CLEAN))
1471 		return ER_DONT_EVICT;
1472 
1473 	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep &&
1474 	    unlikely(test_bit(B_READING, &b->state)))
1475 		return ER_DONT_EVICT;
1476 
1477 	return ER_EVICT;
1478 }
1479 
is_dirty(struct dm_buffer * b,void * context)1480 static enum evict_result is_dirty(struct dm_buffer *b, void *context)
1481 {
1482 	/* These should never happen */
1483 	if (WARN_ON_ONCE(test_bit(B_READING, &b->state)))
1484 		return ER_DONT_EVICT;
1485 	if (WARN_ON_ONCE(b->list_mode != LIST_DIRTY))
1486 		return ER_DONT_EVICT;
1487 
1488 	return ER_EVICT;
1489 }
1490 
1491 /*
1492  * Find some buffer that is not held by anybody, clean it, unlink it and
1493  * return it.
1494  */
__get_unclaimed_buffer(struct dm_bufio_client * c)1495 static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c)
1496 {
1497 	struct dm_buffer *b;
1498 
1499 	b = cache_evict(&c->cache, LIST_CLEAN, is_clean, c);
1500 	if (b) {
1501 		/* this also waits for pending reads */
1502 		__make_buffer_clean(b);
1503 		return b;
1504 	}
1505 
1506 	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
1507 		return NULL;
1508 
1509 	b = cache_evict(&c->cache, LIST_DIRTY, is_dirty, NULL);
1510 	if (b) {
1511 		__make_buffer_clean(b);
1512 		return b;
1513 	}
1514 
1515 	return NULL;
1516 }
1517 
1518 /*
1519  * Wait until some other threads free some buffer or release hold count on
1520  * some buffer.
1521  *
1522  * This function is entered with c->lock held, drops it and regains it
1523  * before exiting.
1524  */
__wait_for_free_buffer(struct dm_bufio_client * c)1525 static void __wait_for_free_buffer(struct dm_bufio_client *c)
1526 {
1527 	DECLARE_WAITQUEUE(wait, current);
1528 
1529 	add_wait_queue(&c->free_buffer_wait, &wait);
1530 	set_current_state(TASK_UNINTERRUPTIBLE);
1531 	dm_bufio_unlock(c);
1532 
1533 	/*
1534 	 * It's possible to miss a wake up event since we don't always
1535 	 * hold c->lock when wake_up is called.  So we have a timeout here,
1536 	 * just in case.
1537 	 */
1538 	io_schedule_timeout(5 * HZ);
1539 
1540 	remove_wait_queue(&c->free_buffer_wait, &wait);
1541 
1542 	dm_bufio_lock(c);
1543 }
1544 
1545 enum new_flag {
1546 	NF_FRESH = 0,
1547 	NF_READ = 1,
1548 	NF_GET = 2,
1549 	NF_PREFETCH = 3
1550 };
1551 
1552 /*
1553  * Allocate a new buffer. If the allocation is not possible, wait until
1554  * some other thread frees a buffer.
1555  *
1556  * May drop the lock and regain it.
1557  */
__alloc_buffer_wait_no_callback(struct dm_bufio_client * c,enum new_flag nf)1558 static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
1559 {
1560 	struct dm_buffer *b;
1561 	bool tried_noio_alloc = false;
1562 
1563 	/*
1564 	 * dm-bufio is resistant to allocation failures (it just keeps
1565 	 * one buffer reserved in cases all the allocations fail).
1566 	 * So set flags to not try too hard:
1567 	 *	GFP_NOWAIT: don't wait; if we need to sleep we'll release our
1568 	 *		    mutex and wait ourselves.
1569 	 *	__GFP_NORETRY: don't retry and rather return failure
1570 	 *	__GFP_NOMEMALLOC: don't use emergency reserves
1571 	 *	__GFP_NOWARN: don't print a warning in case of failure
1572 	 *
1573 	 * For debugging, if we set the cache size to 1, no new buffers will
1574 	 * be allocated.
1575 	 */
1576 	while (1) {
1577 		if (dm_bufio_cache_size_latch != 1) {
1578 			b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1579 			if (b)
1580 				return b;
1581 		}
1582 
1583 		if (nf == NF_PREFETCH)
1584 			return NULL;
1585 
1586 		if (dm_bufio_cache_size_latch != 1 && !tried_noio_alloc) {
1587 			dm_bufio_unlock(c);
1588 			b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1589 			dm_bufio_lock(c);
1590 			if (b)
1591 				return b;
1592 			tried_noio_alloc = true;
1593 		}
1594 
1595 		if (!list_empty(&c->reserved_buffers)) {
1596 			b = list_to_buffer(c->reserved_buffers.next);
1597 			list_del(&b->lru.list);
1598 			c->need_reserved_buffers++;
1599 
1600 			return b;
1601 		}
1602 
1603 		b = __get_unclaimed_buffer(c);
1604 		if (b)
1605 			return b;
1606 
1607 		__wait_for_free_buffer(c);
1608 	}
1609 }
1610 
__alloc_buffer_wait(struct dm_bufio_client * c,enum new_flag nf)1611 static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
1612 {
1613 	struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
1614 
1615 	if (!b)
1616 		return NULL;
1617 
1618 	if (c->alloc_callback)
1619 		c->alloc_callback(b);
1620 
1621 	return b;
1622 }
1623 
1624 /*
1625  * Free a buffer and wake other threads waiting for free buffers.
1626  */
__free_buffer_wake(struct dm_buffer * b)1627 static void __free_buffer_wake(struct dm_buffer *b)
1628 {
1629 	struct dm_bufio_client *c = b->c;
1630 
1631 	b->block = -1;
1632 	if (!c->need_reserved_buffers)
1633 		free_buffer(b);
1634 	else {
1635 		list_add(&b->lru.list, &c->reserved_buffers);
1636 		c->need_reserved_buffers--;
1637 	}
1638 
1639 	/*
1640 	 * We hold the bufio lock here, so no one can add entries to the
1641 	 * wait queue anyway.
1642 	 */
1643 	if (unlikely(waitqueue_active(&c->free_buffer_wait)))
1644 		wake_up(&c->free_buffer_wait);
1645 }
1646 
cleaned(struct dm_buffer * b,void * context)1647 static enum evict_result cleaned(struct dm_buffer *b, void *context)
1648 {
1649 	if (WARN_ON_ONCE(test_bit(B_READING, &b->state)))
1650 		return ER_DONT_EVICT; /* should never happen */
1651 
1652 	if (test_bit(B_DIRTY, &b->state) || test_bit(B_WRITING, &b->state))
1653 		return ER_DONT_EVICT;
1654 	else
1655 		return ER_EVICT;
1656 }
1657 
__move_clean_buffers(struct dm_bufio_client * c)1658 static void __move_clean_buffers(struct dm_bufio_client *c)
1659 {
1660 	cache_mark_many(&c->cache, LIST_DIRTY, LIST_CLEAN, cleaned, NULL);
1661 }
1662 
1663 struct write_context {
1664 	int no_wait;
1665 	struct list_head *write_list;
1666 };
1667 
write_one(struct dm_buffer * b,void * context)1668 static enum it_action write_one(struct dm_buffer *b, void *context)
1669 {
1670 	struct write_context *wc = context;
1671 
1672 	if (wc->no_wait && test_bit(B_WRITING, &b->state))
1673 		return IT_COMPLETE;
1674 
1675 	__write_dirty_buffer(b, wc->write_list);
1676 	return IT_NEXT;
1677 }
1678 
__write_dirty_buffers_async(struct dm_bufio_client * c,int no_wait,struct list_head * write_list)1679 static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
1680 					struct list_head *write_list)
1681 {
1682 	struct write_context wc = {.no_wait = no_wait, .write_list = write_list};
1683 
1684 	__move_clean_buffers(c);
1685 	cache_iterate(&c->cache, LIST_DIRTY, write_one, &wc);
1686 }
1687 
1688 /*
1689  * Check if we're over watermark.
1690  * If we are over threshold_buffers, start freeing buffers.
1691  * If we're over "limit_buffers", block until we get under the limit.
1692  */
__check_watermark(struct dm_bufio_client * c,struct list_head * write_list)1693 static void __check_watermark(struct dm_bufio_client *c,
1694 			      struct list_head *write_list)
1695 {
1696 	if (cache_count(&c->cache, LIST_DIRTY) >
1697 	    cache_count(&c->cache, LIST_CLEAN) * DM_BUFIO_WRITEBACK_RATIO)
1698 		__write_dirty_buffers_async(c, 1, write_list);
1699 }
1700 
1701 /*
1702  *--------------------------------------------------------------
1703  * Getting a buffer
1704  *--------------------------------------------------------------
1705  */
1706 
cache_put_and_wake(struct dm_bufio_client * c,struct dm_buffer * b)1707 static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer *b)
1708 {
1709 	/*
1710 	 * Relying on waitqueue_active() is racey, but we sleep
1711 	 * with schedule_timeout anyway.
1712 	 */
1713 	if (cache_put(&c->cache, b) &&
1714 	    unlikely(waitqueue_active(&c->free_buffer_wait)))
1715 		wake_up(&c->free_buffer_wait);
1716 }
1717 
1718 /*
1719  * This assumes you have already checked the cache to see if the buffer
1720  * is already present (it will recheck after dropping the lock for allocation).
1721  */
__bufio_new(struct dm_bufio_client * c,sector_t block,enum new_flag nf,int * need_submit,struct list_head * write_list)1722 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
1723 				     enum new_flag nf, int *need_submit,
1724 				     struct list_head *write_list)
1725 {
1726 	struct dm_buffer *b, *new_b = NULL;
1727 
1728 	*need_submit = 0;
1729 
1730 	/* This can't be called with NF_GET */
1731 	if (WARN_ON_ONCE(nf == NF_GET))
1732 		return NULL;
1733 
1734 	new_b = __alloc_buffer_wait(c, nf);
1735 	if (!new_b)
1736 		return NULL;
1737 
1738 	/*
1739 	 * We've had a period where the mutex was unlocked, so need to
1740 	 * recheck the buffer tree.
1741 	 */
1742 	b = cache_get(&c->cache, block);
1743 	if (b) {
1744 		__free_buffer_wake(new_b);
1745 		goto found_buffer;
1746 	}
1747 
1748 	__check_watermark(c, write_list);
1749 
1750 	b = new_b;
1751 	atomic_set(&b->hold_count, 1);
1752 	WRITE_ONCE(b->last_accessed, jiffies);
1753 	b->block = block;
1754 	b->read_error = 0;
1755 	b->write_error = 0;
1756 	b->list_mode = LIST_CLEAN;
1757 
1758 	if (nf == NF_FRESH)
1759 		b->state = 0;
1760 	else {
1761 		b->state = 1 << B_READING;
1762 		*need_submit = 1;
1763 	}
1764 
1765 	/*
1766 	 * We mustn't insert into the cache until the B_READING state
1767 	 * is set.  Otherwise another thread could get it and use
1768 	 * it before it had been read.
1769 	 */
1770 	cache_insert(&c->cache, b);
1771 
1772 	return b;
1773 
1774 found_buffer:
1775 	if (nf == NF_PREFETCH) {
1776 		cache_put_and_wake(c, b);
1777 		return NULL;
1778 	}
1779 
1780 	/*
1781 	 * Note: it is essential that we don't wait for the buffer to be
1782 	 * read if dm_bufio_get function is used. Both dm_bufio_get and
1783 	 * dm_bufio_prefetch can be used in the driver request routine.
1784 	 * If the user called both dm_bufio_prefetch and dm_bufio_get on
1785 	 * the same buffer, it would deadlock if we waited.
1786 	 */
1787 	if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) {
1788 		cache_put_and_wake(c, b);
1789 		return NULL;
1790 	}
1791 
1792 	return b;
1793 }
1794 
1795 /*
1796  * The endio routine for reading: set the error, clear the bit and wake up
1797  * anyone waiting on the buffer.
1798  */
read_endio(struct dm_buffer * b,blk_status_t status)1799 static void read_endio(struct dm_buffer *b, blk_status_t status)
1800 {
1801 	b->read_error = status;
1802 
1803 	BUG_ON(!test_bit(B_READING, &b->state));
1804 
1805 	smp_mb__before_atomic();
1806 	clear_bit(B_READING, &b->state);
1807 	smp_mb__after_atomic();
1808 
1809 	wake_up_bit(&b->state, B_READING);
1810 }
1811 
1812 /*
1813  * A common routine for dm_bufio_new and dm_bufio_read.  Operation of these
1814  * functions is similar except that dm_bufio_new doesn't read the
1815  * buffer from the disk (assuming that the caller overwrites all the data
1816  * and uses dm_bufio_mark_buffer_dirty to write new data back).
1817  */
new_read(struct dm_bufio_client * c,sector_t block,enum new_flag nf,struct dm_buffer ** bp)1818 static void *new_read(struct dm_bufio_client *c, sector_t block,
1819 		      enum new_flag nf, struct dm_buffer **bp)
1820 {
1821 	int need_submit = 0;
1822 	struct dm_buffer *b;
1823 
1824 	LIST_HEAD(write_list);
1825 
1826 	*bp = NULL;
1827 
1828 	/*
1829 	 * Fast path, hopefully the block is already in the cache.  No need
1830 	 * to get the client lock for this.
1831 	 */
1832 	b = cache_get(&c->cache, block);
1833 	if (b) {
1834 		if (nf == NF_PREFETCH) {
1835 			cache_put_and_wake(c, b);
1836 			return NULL;
1837 		}
1838 
1839 		/*
1840 		 * Note: it is essential that we don't wait for the buffer to be
1841 		 * read if dm_bufio_get function is used. Both dm_bufio_get and
1842 		 * dm_bufio_prefetch can be used in the driver request routine.
1843 		 * If the user called both dm_bufio_prefetch and dm_bufio_get on
1844 		 * the same buffer, it would deadlock if we waited.
1845 		 */
1846 		if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) {
1847 			cache_put_and_wake(c, b);
1848 			return NULL;
1849 		}
1850 	}
1851 
1852 	if (!b) {
1853 		if (nf == NF_GET)
1854 			return NULL;
1855 
1856 		dm_bufio_lock(c);
1857 		b = __bufio_new(c, block, nf, &need_submit, &write_list);
1858 		dm_bufio_unlock(c);
1859 	}
1860 
1861 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1862 	if (b && (atomic_read(&b->hold_count) == 1))
1863 		buffer_record_stack(b);
1864 #endif
1865 
1866 	__flush_write_list(&write_list);
1867 
1868 	if (!b)
1869 		return NULL;
1870 
1871 	if (need_submit)
1872 		submit_io(b, REQ_OP_READ, read_endio);
1873 
1874 	wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
1875 
1876 	if (b->read_error) {
1877 		int error = blk_status_to_errno(b->read_error);
1878 
1879 		dm_bufio_release(b);
1880 
1881 		return ERR_PTR(error);
1882 	}
1883 
1884 	*bp = b;
1885 
1886 	return b->data;
1887 }
1888 
dm_bufio_get(struct dm_bufio_client * c,sector_t block,struct dm_buffer ** bp)1889 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block,
1890 		   struct dm_buffer **bp)
1891 {
1892 	return new_read(c, block, NF_GET, bp);
1893 }
1894 EXPORT_SYMBOL_GPL(dm_bufio_get);
1895 
dm_bufio_read(struct dm_bufio_client * c,sector_t block,struct dm_buffer ** bp)1896 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
1897 		    struct dm_buffer **bp)
1898 {
1899 	if (WARN_ON_ONCE(dm_bufio_in_request()))
1900 		return ERR_PTR(-EINVAL);
1901 
1902 	return new_read(c, block, NF_READ, bp);
1903 }
1904 EXPORT_SYMBOL_GPL(dm_bufio_read);
1905 
dm_bufio_new(struct dm_bufio_client * c,sector_t block,struct dm_buffer ** bp)1906 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
1907 		   struct dm_buffer **bp)
1908 {
1909 	if (WARN_ON_ONCE(dm_bufio_in_request()))
1910 		return ERR_PTR(-EINVAL);
1911 
1912 	return new_read(c, block, NF_FRESH, bp);
1913 }
1914 EXPORT_SYMBOL_GPL(dm_bufio_new);
1915 
dm_bufio_prefetch(struct dm_bufio_client * c,sector_t block,unsigned int n_blocks)1916 void dm_bufio_prefetch(struct dm_bufio_client *c,
1917 		       sector_t block, unsigned int n_blocks)
1918 {
1919 	struct blk_plug plug;
1920 
1921 	LIST_HEAD(write_list);
1922 
1923 	if (WARN_ON_ONCE(dm_bufio_in_request()))
1924 		return; /* should never happen */
1925 
1926 	blk_start_plug(&plug);
1927 
1928 	for (; n_blocks--; block++) {
1929 		int need_submit;
1930 		struct dm_buffer *b;
1931 
1932 		b = cache_get(&c->cache, block);
1933 		if (b) {
1934 			/* already in cache */
1935 			cache_put_and_wake(c, b);
1936 			continue;
1937 		}
1938 
1939 		dm_bufio_lock(c);
1940 		b = __bufio_new(c, block, NF_PREFETCH, &need_submit,
1941 				&write_list);
1942 		if (unlikely(!list_empty(&write_list))) {
1943 			dm_bufio_unlock(c);
1944 			blk_finish_plug(&plug);
1945 			__flush_write_list(&write_list);
1946 			blk_start_plug(&plug);
1947 			dm_bufio_lock(c);
1948 		}
1949 		if (unlikely(b != NULL)) {
1950 			dm_bufio_unlock(c);
1951 
1952 			if (need_submit)
1953 				submit_io(b, REQ_OP_READ, read_endio);
1954 			dm_bufio_release(b);
1955 
1956 			cond_resched();
1957 
1958 			if (!n_blocks)
1959 				goto flush_plug;
1960 			dm_bufio_lock(c);
1961 		}
1962 		dm_bufio_unlock(c);
1963 	}
1964 
1965 flush_plug:
1966 	blk_finish_plug(&plug);
1967 }
1968 EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
1969 
dm_bufio_release(struct dm_buffer * b)1970 void dm_bufio_release(struct dm_buffer *b)
1971 {
1972 	struct dm_bufio_client *c = b->c;
1973 
1974 	/*
1975 	 * If there were errors on the buffer, and the buffer is not
1976 	 * to be written, free the buffer. There is no point in caching
1977 	 * invalid buffer.
1978 	 */
1979 	if ((b->read_error || b->write_error) &&
1980 	    !test_bit_acquire(B_READING, &b->state) &&
1981 	    !test_bit(B_WRITING, &b->state) &&
1982 	    !test_bit(B_DIRTY, &b->state)) {
1983 		dm_bufio_lock(c);
1984 
1985 		/* cache remove can fail if there are other holders */
1986 		if (cache_remove(&c->cache, b)) {
1987 			__free_buffer_wake(b);
1988 			dm_bufio_unlock(c);
1989 			return;
1990 		}
1991 
1992 		dm_bufio_unlock(c);
1993 	}
1994 
1995 	cache_put_and_wake(c, b);
1996 }
1997 EXPORT_SYMBOL_GPL(dm_bufio_release);
1998 
dm_bufio_mark_partial_buffer_dirty(struct dm_buffer * b,unsigned int start,unsigned int end)1999 void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
2000 					unsigned int start, unsigned int end)
2001 {
2002 	struct dm_bufio_client *c = b->c;
2003 
2004 	BUG_ON(start >= end);
2005 	BUG_ON(end > b->c->block_size);
2006 
2007 	dm_bufio_lock(c);
2008 
2009 	BUG_ON(test_bit(B_READING, &b->state));
2010 
2011 	if (!test_and_set_bit(B_DIRTY, &b->state)) {
2012 		b->dirty_start = start;
2013 		b->dirty_end = end;
2014 		cache_mark(&c->cache, b, LIST_DIRTY);
2015 	} else {
2016 		if (start < b->dirty_start)
2017 			b->dirty_start = start;
2018 		if (end > b->dirty_end)
2019 			b->dirty_end = end;
2020 	}
2021 
2022 	dm_bufio_unlock(c);
2023 }
2024 EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty);
2025 
dm_bufio_mark_buffer_dirty(struct dm_buffer * b)2026 void dm_bufio_mark_buffer_dirty(struct dm_buffer *b)
2027 {
2028 	dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size);
2029 }
2030 EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty);
2031 
dm_bufio_write_dirty_buffers_async(struct dm_bufio_client * c)2032 void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c)
2033 {
2034 	LIST_HEAD(write_list);
2035 
2036 	if (WARN_ON_ONCE(dm_bufio_in_request()))
2037 		return; /* should never happen */
2038 
2039 	dm_bufio_lock(c);
2040 	__write_dirty_buffers_async(c, 0, &write_list);
2041 	dm_bufio_unlock(c);
2042 	__flush_write_list(&write_list);
2043 }
2044 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async);
2045 
2046 /*
2047  * For performance, it is essential that the buffers are written asynchronously
2048  * and simultaneously (so that the block layer can merge the writes) and then
2049  * waited upon.
2050  *
2051  * Finally, we flush hardware disk cache.
2052  */
is_writing(struct lru_entry * e,void * context)2053 static bool is_writing(struct lru_entry *e, void *context)
2054 {
2055 	struct dm_buffer *b = le_to_buffer(e);
2056 
2057 	return test_bit(B_WRITING, &b->state);
2058 }
2059 
dm_bufio_write_dirty_buffers(struct dm_bufio_client * c)2060 int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c)
2061 {
2062 	int a, f;
2063 	unsigned long nr_buffers;
2064 	struct lru_entry *e;
2065 	struct lru_iter it;
2066 
2067 	LIST_HEAD(write_list);
2068 
2069 	dm_bufio_lock(c);
2070 	__write_dirty_buffers_async(c, 0, &write_list);
2071 	dm_bufio_unlock(c);
2072 	__flush_write_list(&write_list);
2073 	dm_bufio_lock(c);
2074 
2075 	nr_buffers = cache_count(&c->cache, LIST_DIRTY);
2076 	lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it);
2077 	while ((e = lru_iter_next(&it, is_writing, c))) {
2078 		struct dm_buffer *b = le_to_buffer(e);
2079 		__cache_inc_buffer(b);
2080 
2081 		BUG_ON(test_bit(B_READING, &b->state));
2082 
2083 		if (nr_buffers) {
2084 			nr_buffers--;
2085 			dm_bufio_unlock(c);
2086 			wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
2087 			dm_bufio_lock(c);
2088 		} else {
2089 			wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
2090 		}
2091 
2092 		if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state))
2093 			cache_mark(&c->cache, b, LIST_CLEAN);
2094 
2095 		cache_put_and_wake(c, b);
2096 
2097 		cond_resched();
2098 	}
2099 	lru_iter_end(&it);
2100 
2101 	wake_up(&c->free_buffer_wait);
2102 	dm_bufio_unlock(c);
2103 
2104 	a = xchg(&c->async_write_error, 0);
2105 	f = dm_bufio_issue_flush(c);
2106 	if (a)
2107 		return a;
2108 
2109 	return f;
2110 }
2111 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers);
2112 
2113 /*
2114  * Use dm-io to send an empty barrier to flush the device.
2115  */
dm_bufio_issue_flush(struct dm_bufio_client * c)2116 int dm_bufio_issue_flush(struct dm_bufio_client *c)
2117 {
2118 	struct dm_io_request io_req = {
2119 		.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC,
2120 		.mem.type = DM_IO_KMEM,
2121 		.mem.ptr.addr = NULL,
2122 		.client = c->dm_io,
2123 	};
2124 	struct dm_io_region io_reg = {
2125 		.bdev = c->bdev,
2126 		.sector = 0,
2127 		.count = 0,
2128 	};
2129 
2130 	if (WARN_ON_ONCE(dm_bufio_in_request()))
2131 		return -EINVAL;
2132 
2133 	return dm_io(&io_req, 1, &io_reg, NULL);
2134 }
2135 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
2136 
2137 /*
2138  * Use dm-io to send a discard request to flush the device.
2139  */
dm_bufio_issue_discard(struct dm_bufio_client * c,sector_t block,sector_t count)2140 int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count)
2141 {
2142 	struct dm_io_request io_req = {
2143 		.bi_opf = REQ_OP_DISCARD | REQ_SYNC,
2144 		.mem.type = DM_IO_KMEM,
2145 		.mem.ptr.addr = NULL,
2146 		.client = c->dm_io,
2147 	};
2148 	struct dm_io_region io_reg = {
2149 		.bdev = c->bdev,
2150 		.sector = block_to_sector(c, block),
2151 		.count = block_to_sector(c, count),
2152 	};
2153 
2154 	if (WARN_ON_ONCE(dm_bufio_in_request()))
2155 		return -EINVAL; /* discards are optional */
2156 
2157 	return dm_io(&io_req, 1, &io_reg, NULL);
2158 }
2159 EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
2160 
forget_buffer(struct dm_bufio_client * c,sector_t block)2161 static bool forget_buffer(struct dm_bufio_client *c, sector_t block)
2162 {
2163 	struct dm_buffer *b;
2164 
2165 	b = cache_get(&c->cache, block);
2166 	if (b) {
2167 		if (likely(!smp_load_acquire(&b->state))) {
2168 			if (cache_remove(&c->cache, b))
2169 				__free_buffer_wake(b);
2170 			else
2171 				cache_put_and_wake(c, b);
2172 		} else {
2173 			cache_put_and_wake(c, b);
2174 		}
2175 	}
2176 
2177 	return b ? true : false;
2178 }
2179 
2180 /*
2181  * Free the given buffer.
2182  *
2183  * This is just a hint, if the buffer is in use or dirty, this function
2184  * does nothing.
2185  */
dm_bufio_forget(struct dm_bufio_client * c,sector_t block)2186 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block)
2187 {
2188 	dm_bufio_lock(c);
2189 	forget_buffer(c, block);
2190 	dm_bufio_unlock(c);
2191 }
2192 EXPORT_SYMBOL_GPL(dm_bufio_forget);
2193 
idle(struct dm_buffer * b,void * context)2194 static enum evict_result idle(struct dm_buffer *b, void *context)
2195 {
2196 	return b->state ? ER_DONT_EVICT : ER_EVICT;
2197 }
2198 
dm_bufio_forget_buffers(struct dm_bufio_client * c,sector_t block,sector_t n_blocks)2199 void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks)
2200 {
2201 	dm_bufio_lock(c);
2202 	cache_remove_range(&c->cache, block, block + n_blocks, idle, __free_buffer_wake);
2203 	dm_bufio_unlock(c);
2204 }
2205 EXPORT_SYMBOL_GPL(dm_bufio_forget_buffers);
2206 
dm_bufio_set_minimum_buffers(struct dm_bufio_client * c,unsigned int n)2207 void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned int n)
2208 {
2209 	c->minimum_buffers = n;
2210 }
2211 EXPORT_SYMBOL_GPL(dm_bufio_set_minimum_buffers);
2212 
dm_bufio_get_block_size(struct dm_bufio_client * c)2213 unsigned int dm_bufio_get_block_size(struct dm_bufio_client *c)
2214 {
2215 	return c->block_size;
2216 }
2217 EXPORT_SYMBOL_GPL(dm_bufio_get_block_size);
2218 
dm_bufio_get_device_size(struct dm_bufio_client * c)2219 sector_t dm_bufio_get_device_size(struct dm_bufio_client *c)
2220 {
2221 	sector_t s = bdev_nr_sectors(c->bdev);
2222 
2223 	if (s >= c->start)
2224 		s -= c->start;
2225 	else
2226 		s = 0;
2227 	if (likely(c->sectors_per_block_bits >= 0))
2228 		s >>= c->sectors_per_block_bits;
2229 	else
2230 		sector_div(s, c->block_size >> SECTOR_SHIFT);
2231 	return s;
2232 }
2233 EXPORT_SYMBOL_GPL(dm_bufio_get_device_size);
2234 
dm_bufio_get_dm_io_client(struct dm_bufio_client * c)2235 struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c)
2236 {
2237 	return c->dm_io;
2238 }
2239 EXPORT_SYMBOL_GPL(dm_bufio_get_dm_io_client);
2240 
dm_bufio_get_block_number(struct dm_buffer * b)2241 sector_t dm_bufio_get_block_number(struct dm_buffer *b)
2242 {
2243 	return b->block;
2244 }
2245 EXPORT_SYMBOL_GPL(dm_bufio_get_block_number);
2246 
dm_bufio_get_block_data(struct dm_buffer * b)2247 void *dm_bufio_get_block_data(struct dm_buffer *b)
2248 {
2249 	return b->data;
2250 }
2251 EXPORT_SYMBOL_GPL(dm_bufio_get_block_data);
2252 
dm_bufio_get_aux_data(struct dm_buffer * b)2253 void *dm_bufio_get_aux_data(struct dm_buffer *b)
2254 {
2255 	return b + 1;
2256 }
2257 EXPORT_SYMBOL_GPL(dm_bufio_get_aux_data);
2258 
dm_bufio_get_client(struct dm_buffer * b)2259 struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b)
2260 {
2261 	return b->c;
2262 }
2263 EXPORT_SYMBOL_GPL(dm_bufio_get_client);
2264 
warn_leak(struct dm_buffer * b,void * context)2265 static enum it_action warn_leak(struct dm_buffer *b, void *context)
2266 {
2267 	bool *warned = context;
2268 
2269 	WARN_ON(!(*warned));
2270 	*warned = true;
2271 	DMERR("leaked buffer %llx, hold count %u, list %d",
2272 	      (unsigned long long)b->block, atomic_read(&b->hold_count), b->list_mode);
2273 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
2274 	stack_trace_print(b->stack_entries, b->stack_len, 1);
2275 	/* mark unclaimed to avoid WARN_ON at end of drop_buffers() */
2276 	atomic_set(&b->hold_count, 0);
2277 #endif
2278 	return IT_NEXT;
2279 }
2280 
drop_buffers(struct dm_bufio_client * c)2281 static void drop_buffers(struct dm_bufio_client *c)
2282 {
2283 	int i;
2284 	struct dm_buffer *b;
2285 
2286 	if (WARN_ON(dm_bufio_in_request()))
2287 		return; /* should never happen */
2288 
2289 	/*
2290 	 * An optimization so that the buffers are not written one-by-one.
2291 	 */
2292 	dm_bufio_write_dirty_buffers_async(c);
2293 
2294 	dm_bufio_lock(c);
2295 
2296 	while ((b = __get_unclaimed_buffer(c)))
2297 		__free_buffer_wake(b);
2298 
2299 	for (i = 0; i < LIST_SIZE; i++) {
2300 		bool warned = false;
2301 
2302 		cache_iterate(&c->cache, i, warn_leak, &warned);
2303 	}
2304 
2305 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
2306 	while ((b = __get_unclaimed_buffer(c)))
2307 		__free_buffer_wake(b);
2308 #endif
2309 
2310 	for (i = 0; i < LIST_SIZE; i++)
2311 		WARN_ON(cache_count(&c->cache, i));
2312 
2313 	dm_bufio_unlock(c);
2314 }
2315 
get_retain_buffers(struct dm_bufio_client * c)2316 static unsigned long get_retain_buffers(struct dm_bufio_client *c)
2317 {
2318 	unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes);
2319 
2320 	if (likely(c->sectors_per_block_bits >= 0))
2321 		retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT;
2322 	else
2323 		retain_bytes /= c->block_size;
2324 
2325 	return retain_bytes;
2326 }
2327 
__scan(struct dm_bufio_client * c)2328 static void __scan(struct dm_bufio_client *c)
2329 {
2330 	int l;
2331 	struct dm_buffer *b;
2332 	unsigned long freed = 0;
2333 	unsigned long retain_target = get_retain_buffers(c);
2334 	unsigned long count = cache_total(&c->cache);
2335 
2336 	for (l = 0; l < LIST_SIZE; l++) {
2337 		while (true) {
2338 			if (count - freed <= retain_target)
2339 				atomic_long_set(&c->need_shrink, 0);
2340 			if (!atomic_long_read(&c->need_shrink))
2341 				break;
2342 
2343 			b = cache_evict(&c->cache, l,
2344 					l == LIST_CLEAN ? is_clean : is_dirty, c);
2345 			if (!b)
2346 				break;
2347 
2348 			__make_buffer_clean(b);
2349 			__free_buffer_wake(b);
2350 
2351 			atomic_long_dec(&c->need_shrink);
2352 			freed++;
2353 			cond_resched();
2354 		}
2355 	}
2356 }
2357 
shrink_work(struct work_struct * w)2358 static void shrink_work(struct work_struct *w)
2359 {
2360 	struct dm_bufio_client *c = container_of(w, struct dm_bufio_client, shrink_work);
2361 
2362 	dm_bufio_lock(c);
2363 	__scan(c);
2364 	dm_bufio_unlock(c);
2365 }
2366 
dm_bufio_shrink_scan(struct shrinker * shrink,struct shrink_control * sc)2367 static unsigned long dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
2368 {
2369 	struct dm_bufio_client *c;
2370 
2371 	c = container_of(shrink, struct dm_bufio_client, shrinker);
2372 	atomic_long_add(sc->nr_to_scan, &c->need_shrink);
2373 	queue_work(dm_bufio_wq, &c->shrink_work);
2374 
2375 	return sc->nr_to_scan;
2376 }
2377 
dm_bufio_shrink_count(struct shrinker * shrink,struct shrink_control * sc)2378 static unsigned long dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
2379 {
2380 	struct dm_bufio_client *c = container_of(shrink, struct dm_bufio_client, shrinker);
2381 	unsigned long count = cache_total(&c->cache);
2382 	unsigned long retain_target = get_retain_buffers(c);
2383 	unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink);
2384 
2385 	if (unlikely(count < retain_target))
2386 		count = 0;
2387 	else
2388 		count -= retain_target;
2389 
2390 	if (unlikely(count < queued_for_cleanup))
2391 		count = 0;
2392 	else
2393 		count -= queued_for_cleanup;
2394 
2395 	return count;
2396 }
2397 
2398 /*
2399  * Create the buffering interface
2400  */
dm_bufio_client_create(struct block_device * bdev,unsigned int block_size,unsigned int reserved_buffers,unsigned int aux_size,void (* alloc_callback)(struct dm_buffer *),void (* write_callback)(struct dm_buffer *),unsigned int flags)2401 struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsigned int block_size,
2402 					       unsigned int reserved_buffers, unsigned int aux_size,
2403 					       void (*alloc_callback)(struct dm_buffer *),
2404 					       void (*write_callback)(struct dm_buffer *),
2405 					       unsigned int flags)
2406 {
2407 	int r;
2408 	unsigned int num_locks;
2409 	struct dm_bufio_client *c;
2410 	char slab_name[27];
2411 
2412 	if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) {
2413 		DMERR("%s: block size not specified or is not multiple of 512b", __func__);
2414 		r = -EINVAL;
2415 		goto bad_client;
2416 	}
2417 
2418 	num_locks = dm_num_hash_locks();
2419 	c = kzalloc(sizeof(*c) + (num_locks * sizeof(struct buffer_tree)), GFP_KERNEL);
2420 	if (!c) {
2421 		r = -ENOMEM;
2422 		goto bad_client;
2423 	}
2424 	cache_init(&c->cache, num_locks);
2425 
2426 	c->bdev = bdev;
2427 	c->block_size = block_size;
2428 	if (is_power_of_2(block_size))
2429 		c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT;
2430 	else
2431 		c->sectors_per_block_bits = -1;
2432 
2433 	c->alloc_callback = alloc_callback;
2434 	c->write_callback = write_callback;
2435 
2436 	if (flags & DM_BUFIO_CLIENT_NO_SLEEP) {
2437 		c->no_sleep = true;
2438 		static_branch_inc(&no_sleep_enabled);
2439 	}
2440 
2441 	mutex_init(&c->lock);
2442 	spin_lock_init(&c->spinlock);
2443 	INIT_LIST_HEAD(&c->reserved_buffers);
2444 	c->need_reserved_buffers = reserved_buffers;
2445 
2446 	dm_bufio_set_minimum_buffers(c, DM_BUFIO_MIN_BUFFERS);
2447 
2448 	init_waitqueue_head(&c->free_buffer_wait);
2449 	c->async_write_error = 0;
2450 
2451 	c->dm_io = dm_io_client_create();
2452 	if (IS_ERR(c->dm_io)) {
2453 		r = PTR_ERR(c->dm_io);
2454 		goto bad_dm_io;
2455 	}
2456 
2457 	if (block_size <= KMALLOC_MAX_SIZE &&
2458 	    (block_size < PAGE_SIZE || !is_power_of_2(block_size))) {
2459 		unsigned int align = min(1U << __ffs(block_size), (unsigned int)PAGE_SIZE);
2460 
2461 		snprintf(slab_name, sizeof(slab_name), "dm_bufio_cache-%u", block_size);
2462 		c->slab_cache = kmem_cache_create(slab_name, block_size, align,
2463 						  SLAB_RECLAIM_ACCOUNT, NULL);
2464 		if (!c->slab_cache) {
2465 			r = -ENOMEM;
2466 			goto bad;
2467 		}
2468 	}
2469 	if (aux_size)
2470 		snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u", aux_size);
2471 	else
2472 		snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer");
2473 	c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size,
2474 					   0, SLAB_RECLAIM_ACCOUNT, NULL);
2475 	if (!c->slab_buffer) {
2476 		r = -ENOMEM;
2477 		goto bad;
2478 	}
2479 
2480 	while (c->need_reserved_buffers) {
2481 		struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL);
2482 
2483 		if (!b) {
2484 			r = -ENOMEM;
2485 			goto bad;
2486 		}
2487 		__free_buffer_wake(b);
2488 	}
2489 
2490 	INIT_WORK(&c->shrink_work, shrink_work);
2491 	atomic_long_set(&c->need_shrink, 0);
2492 
2493 	c->shrinker.count_objects = dm_bufio_shrink_count;
2494 	c->shrinker.scan_objects = dm_bufio_shrink_scan;
2495 	c->shrinker.seeks = 1;
2496 	c->shrinker.batch = 0;
2497 	r = register_shrinker(&c->shrinker, "dm-bufio:(%u:%u)",
2498 			      MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev));
2499 	if (r)
2500 		goto bad;
2501 
2502 	mutex_lock(&dm_bufio_clients_lock);
2503 	dm_bufio_client_count++;
2504 	list_add(&c->client_list, &dm_bufio_all_clients);
2505 	__cache_size_refresh();
2506 	mutex_unlock(&dm_bufio_clients_lock);
2507 
2508 	return c;
2509 
2510 bad:
2511 	while (!list_empty(&c->reserved_buffers)) {
2512 		struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next);
2513 
2514 		list_del(&b->lru.list);
2515 		free_buffer(b);
2516 	}
2517 	kmem_cache_destroy(c->slab_cache);
2518 	kmem_cache_destroy(c->slab_buffer);
2519 	dm_io_client_destroy(c->dm_io);
2520 bad_dm_io:
2521 	mutex_destroy(&c->lock);
2522 	if (c->no_sleep)
2523 		static_branch_dec(&no_sleep_enabled);
2524 	kfree(c);
2525 bad_client:
2526 	return ERR_PTR(r);
2527 }
2528 EXPORT_SYMBOL_GPL(dm_bufio_client_create);
2529 
2530 /*
2531  * Free the buffering interface.
2532  * It is required that there are no references on any buffers.
2533  */
dm_bufio_client_destroy(struct dm_bufio_client * c)2534 void dm_bufio_client_destroy(struct dm_bufio_client *c)
2535 {
2536 	unsigned int i;
2537 
2538 	drop_buffers(c);
2539 
2540 	unregister_shrinker(&c->shrinker);
2541 	flush_work(&c->shrink_work);
2542 
2543 	mutex_lock(&dm_bufio_clients_lock);
2544 
2545 	list_del(&c->client_list);
2546 	dm_bufio_client_count--;
2547 	__cache_size_refresh();
2548 
2549 	mutex_unlock(&dm_bufio_clients_lock);
2550 
2551 	WARN_ON(c->need_reserved_buffers);
2552 
2553 	while (!list_empty(&c->reserved_buffers)) {
2554 		struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next);
2555 
2556 		list_del(&b->lru.list);
2557 		free_buffer(b);
2558 	}
2559 
2560 	for (i = 0; i < LIST_SIZE; i++)
2561 		if (cache_count(&c->cache, i))
2562 			DMERR("leaked buffer count %d: %lu", i, cache_count(&c->cache, i));
2563 
2564 	for (i = 0; i < LIST_SIZE; i++)
2565 		WARN_ON(cache_count(&c->cache, i));
2566 
2567 	cache_destroy(&c->cache);
2568 	kmem_cache_destroy(c->slab_cache);
2569 	kmem_cache_destroy(c->slab_buffer);
2570 	dm_io_client_destroy(c->dm_io);
2571 	mutex_destroy(&c->lock);
2572 	if (c->no_sleep)
2573 		static_branch_dec(&no_sleep_enabled);
2574 	kfree(c);
2575 }
2576 EXPORT_SYMBOL_GPL(dm_bufio_client_destroy);
2577 
dm_bufio_client_reset(struct dm_bufio_client * c)2578 void dm_bufio_client_reset(struct dm_bufio_client *c)
2579 {
2580 	drop_buffers(c);
2581 	flush_work(&c->shrink_work);
2582 }
2583 EXPORT_SYMBOL_GPL(dm_bufio_client_reset);
2584 
dm_bufio_set_sector_offset(struct dm_bufio_client * c,sector_t start)2585 void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start)
2586 {
2587 	c->start = start;
2588 }
2589 EXPORT_SYMBOL_GPL(dm_bufio_set_sector_offset);
2590 
2591 /*--------------------------------------------------------------*/
2592 
get_max_age_hz(void)2593 static unsigned int get_max_age_hz(void)
2594 {
2595 	unsigned int max_age = READ_ONCE(dm_bufio_max_age);
2596 
2597 	if (max_age > UINT_MAX / HZ)
2598 		max_age = UINT_MAX / HZ;
2599 
2600 	return max_age * HZ;
2601 }
2602 
older_than(struct dm_buffer * b,unsigned long age_hz)2603 static bool older_than(struct dm_buffer *b, unsigned long age_hz)
2604 {
2605 	return time_after_eq(jiffies, READ_ONCE(b->last_accessed) + age_hz);
2606 }
2607 
2608 struct evict_params {
2609 	gfp_t gfp;
2610 	unsigned long age_hz;
2611 
2612 	/*
2613 	 * This gets updated with the largest last_accessed (ie. most
2614 	 * recently used) of the evicted buffers.  It will not be reinitialised
2615 	 * by __evict_many(), so you can use it across multiple invocations.
2616 	 */
2617 	unsigned long last_accessed;
2618 };
2619 
2620 /*
2621  * We may not be able to evict this buffer if IO pending or the client
2622  * is still using it.
2623  *
2624  * And if GFP_NOFS is used, we must not do any I/O because we hold
2625  * dm_bufio_clients_lock and we would risk deadlock if the I/O gets
2626  * rerouted to different bufio client.
2627  */
select_for_evict(struct dm_buffer * b,void * context)2628 static enum evict_result select_for_evict(struct dm_buffer *b, void *context)
2629 {
2630 	struct evict_params *params = context;
2631 
2632 	if (!(params->gfp & __GFP_FS) ||
2633 	    (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) {
2634 		if (test_bit_acquire(B_READING, &b->state) ||
2635 		    test_bit(B_WRITING, &b->state) ||
2636 		    test_bit(B_DIRTY, &b->state))
2637 			return ER_DONT_EVICT;
2638 	}
2639 
2640 	return older_than(b, params->age_hz) ? ER_EVICT : ER_STOP;
2641 }
2642 
__evict_many(struct dm_bufio_client * c,struct evict_params * params,int list_mode,unsigned long max_count)2643 static unsigned long __evict_many(struct dm_bufio_client *c,
2644 				  struct evict_params *params,
2645 				  int list_mode, unsigned long max_count)
2646 {
2647 	unsigned long count;
2648 	unsigned long last_accessed;
2649 	struct dm_buffer *b;
2650 
2651 	for (count = 0; count < max_count; count++) {
2652 		b = cache_evict(&c->cache, list_mode, select_for_evict, params);
2653 		if (!b)
2654 			break;
2655 
2656 		last_accessed = READ_ONCE(b->last_accessed);
2657 		if (time_after_eq(params->last_accessed, last_accessed))
2658 			params->last_accessed = last_accessed;
2659 
2660 		__make_buffer_clean(b);
2661 		__free_buffer_wake(b);
2662 
2663 		cond_resched();
2664 	}
2665 
2666 	return count;
2667 }
2668 
evict_old_buffers(struct dm_bufio_client * c,unsigned long age_hz)2669 static void evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
2670 {
2671 	struct evict_params params = {.gfp = 0, .age_hz = age_hz, .last_accessed = 0};
2672 	unsigned long retain = get_retain_buffers(c);
2673 	unsigned long count;
2674 	LIST_HEAD(write_list);
2675 
2676 	dm_bufio_lock(c);
2677 
2678 	__check_watermark(c, &write_list);
2679 	if (unlikely(!list_empty(&write_list))) {
2680 		dm_bufio_unlock(c);
2681 		__flush_write_list(&write_list);
2682 		dm_bufio_lock(c);
2683 	}
2684 
2685 	count = cache_total(&c->cache);
2686 	if (count > retain)
2687 		__evict_many(c, &params, LIST_CLEAN, count - retain);
2688 
2689 	dm_bufio_unlock(c);
2690 }
2691 
cleanup_old_buffers(void)2692 static void cleanup_old_buffers(void)
2693 {
2694 	unsigned long max_age_hz = get_max_age_hz();
2695 	struct dm_bufio_client *c;
2696 
2697 	mutex_lock(&dm_bufio_clients_lock);
2698 
2699 	__cache_size_refresh();
2700 
2701 	list_for_each_entry(c, &dm_bufio_all_clients, client_list)
2702 		evict_old_buffers(c, max_age_hz);
2703 
2704 	mutex_unlock(&dm_bufio_clients_lock);
2705 }
2706 
work_fn(struct work_struct * w)2707 static void work_fn(struct work_struct *w)
2708 {
2709 	cleanup_old_buffers();
2710 
2711 	queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
2712 			   DM_BUFIO_WORK_TIMER_SECS * HZ);
2713 }
2714 
2715 /*--------------------------------------------------------------*/
2716 
2717 /*
2718  * Global cleanup tries to evict the oldest buffers from across _all_
2719  * the clients.  It does this by repeatedly evicting a few buffers from
2720  * the client that holds the oldest buffer.  It's approximate, but hopefully
2721  * good enough.
2722  */
__pop_client(void)2723 static struct dm_bufio_client *__pop_client(void)
2724 {
2725 	struct list_head *h;
2726 
2727 	if (list_empty(&dm_bufio_all_clients))
2728 		return NULL;
2729 
2730 	h = dm_bufio_all_clients.next;
2731 	list_del(h);
2732 	return container_of(h, struct dm_bufio_client, client_list);
2733 }
2734 
2735 /*
2736  * Inserts the client in the global client list based on its
2737  * 'oldest_buffer' field.
2738  */
__insert_client(struct dm_bufio_client * new_client)2739 static void __insert_client(struct dm_bufio_client *new_client)
2740 {
2741 	struct dm_bufio_client *c;
2742 	struct list_head *h = dm_bufio_all_clients.next;
2743 
2744 	while (h != &dm_bufio_all_clients) {
2745 		c = container_of(h, struct dm_bufio_client, client_list);
2746 		if (time_after_eq(c->oldest_buffer, new_client->oldest_buffer))
2747 			break;
2748 		h = h->next;
2749 	}
2750 
2751 	list_add_tail(&new_client->client_list, h);
2752 }
2753 
__evict_a_few(unsigned long nr_buffers)2754 static unsigned long __evict_a_few(unsigned long nr_buffers)
2755 {
2756 	unsigned long count;
2757 	struct dm_bufio_client *c;
2758 	struct evict_params params = {
2759 		.gfp = GFP_KERNEL,
2760 		.age_hz = 0,
2761 		/* set to jiffies in case there are no buffers in this client */
2762 		.last_accessed = jiffies
2763 	};
2764 
2765 	c = __pop_client();
2766 	if (!c)
2767 		return 0;
2768 
2769 	dm_bufio_lock(c);
2770 	count = __evict_many(c, &params, LIST_CLEAN, nr_buffers);
2771 	dm_bufio_unlock(c);
2772 
2773 	if (count)
2774 		c->oldest_buffer = params.last_accessed;
2775 	__insert_client(c);
2776 
2777 	return count;
2778 }
2779 
check_watermarks(void)2780 static void check_watermarks(void)
2781 {
2782 	LIST_HEAD(write_list);
2783 	struct dm_bufio_client *c;
2784 
2785 	mutex_lock(&dm_bufio_clients_lock);
2786 	list_for_each_entry(c, &dm_bufio_all_clients, client_list) {
2787 		dm_bufio_lock(c);
2788 		__check_watermark(c, &write_list);
2789 		dm_bufio_unlock(c);
2790 	}
2791 	mutex_unlock(&dm_bufio_clients_lock);
2792 
2793 	__flush_write_list(&write_list);
2794 }
2795 
evict_old(void)2796 static void evict_old(void)
2797 {
2798 	unsigned long threshold = dm_bufio_cache_size -
2799 		dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO;
2800 
2801 	mutex_lock(&dm_bufio_clients_lock);
2802 	while (dm_bufio_current_allocated > threshold) {
2803 		if (!__evict_a_few(64))
2804 			break;
2805 		cond_resched();
2806 	}
2807 	mutex_unlock(&dm_bufio_clients_lock);
2808 }
2809 
do_global_cleanup(struct work_struct * w)2810 static void do_global_cleanup(struct work_struct *w)
2811 {
2812 	check_watermarks();
2813 	evict_old();
2814 }
2815 
2816 /*
2817  *--------------------------------------------------------------
2818  * Module setup
2819  *--------------------------------------------------------------
2820  */
2821 
2822 /*
2823  * This is called only once for the whole dm_bufio module.
2824  * It initializes memory limit.
2825  */
dm_bufio_init(void)2826 static int __init dm_bufio_init(void)
2827 {
2828 	__u64 mem;
2829 
2830 	dm_bufio_allocated_kmem_cache = 0;
2831 	dm_bufio_allocated_get_free_pages = 0;
2832 	dm_bufio_allocated_vmalloc = 0;
2833 	dm_bufio_current_allocated = 0;
2834 
2835 	mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(),
2836 			       DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT;
2837 
2838 	if (mem > ULONG_MAX)
2839 		mem = ULONG_MAX;
2840 
2841 #ifdef CONFIG_MMU
2842 	if (mem > mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100))
2843 		mem = mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100);
2844 #endif
2845 
2846 	dm_bufio_default_cache_size = mem;
2847 
2848 	mutex_lock(&dm_bufio_clients_lock);
2849 	__cache_size_refresh();
2850 	mutex_unlock(&dm_bufio_clients_lock);
2851 
2852 	dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0);
2853 	if (!dm_bufio_wq)
2854 		return -ENOMEM;
2855 
2856 	INIT_DELAYED_WORK(&dm_bufio_cleanup_old_work, work_fn);
2857 	INIT_WORK(&dm_bufio_replacement_work, do_global_cleanup);
2858 	queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
2859 			   DM_BUFIO_WORK_TIMER_SECS * HZ);
2860 
2861 	return 0;
2862 }
2863 
2864 /*
2865  * This is called once when unloading the dm_bufio module.
2866  */
dm_bufio_exit(void)2867 static void __exit dm_bufio_exit(void)
2868 {
2869 	int bug = 0;
2870 
2871 	cancel_delayed_work_sync(&dm_bufio_cleanup_old_work);
2872 	destroy_workqueue(dm_bufio_wq);
2873 
2874 	if (dm_bufio_client_count) {
2875 		DMCRIT("%s: dm_bufio_client_count leaked: %d",
2876 			__func__, dm_bufio_client_count);
2877 		bug = 1;
2878 	}
2879 
2880 	if (dm_bufio_current_allocated) {
2881 		DMCRIT("%s: dm_bufio_current_allocated leaked: %lu",
2882 			__func__, dm_bufio_current_allocated);
2883 		bug = 1;
2884 	}
2885 
2886 	if (dm_bufio_allocated_get_free_pages) {
2887 		DMCRIT("%s: dm_bufio_allocated_get_free_pages leaked: %lu",
2888 		       __func__, dm_bufio_allocated_get_free_pages);
2889 		bug = 1;
2890 	}
2891 
2892 	if (dm_bufio_allocated_vmalloc) {
2893 		DMCRIT("%s: dm_bufio_vmalloc leaked: %lu",
2894 		       __func__, dm_bufio_allocated_vmalloc);
2895 		bug = 1;
2896 	}
2897 
2898 	WARN_ON(bug); /* leaks are not worth crashing the system */
2899 }
2900 
2901 module_init(dm_bufio_init)
2902 module_exit(dm_bufio_exit)
2903 
2904 module_param_named(max_cache_size_bytes, dm_bufio_cache_size, ulong, 0644);
2905 MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache");
2906 
2907 module_param_named(max_age_seconds, dm_bufio_max_age, uint, 0644);
2908 MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds");
2909 
2910 module_param_named(retain_bytes, dm_bufio_retain_bytes, ulong, 0644);
2911 MODULE_PARM_DESC(retain_bytes, "Try to keep at least this many bytes cached in memory");
2912 
2913 module_param_named(peak_allocated_bytes, dm_bufio_peak_allocated, ulong, 0644);
2914 MODULE_PARM_DESC(peak_allocated_bytes, "Tracks the maximum allocated memory");
2915 
2916 module_param_named(allocated_kmem_cache_bytes, dm_bufio_allocated_kmem_cache, ulong, 0444);
2917 MODULE_PARM_DESC(allocated_kmem_cache_bytes, "Memory allocated with kmem_cache_alloc");
2918 
2919 module_param_named(allocated_get_free_pages_bytes, dm_bufio_allocated_get_free_pages, ulong, 0444);
2920 MODULE_PARM_DESC(allocated_get_free_pages_bytes, "Memory allocated with get_free_pages");
2921 
2922 module_param_named(allocated_vmalloc_bytes, dm_bufio_allocated_vmalloc, ulong, 0444);
2923 MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc");
2924 
2925 module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, 0444);
2926 MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache");
2927 
2928 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
2929 MODULE_DESCRIPTION(DM_NAME " buffered I/O library");
2930 MODULE_LICENSE("GPL");
2931