Lines Matching refs:b
239 static void buffer_record_stack(struct dm_buffer *b) in buffer_record_stack() argument
241 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2); in buffer_record_stack()
251 struct dm_buffer *b; in __find() local
254 b = container_of(n, struct dm_buffer, node); in __find()
256 if (b->block == block) in __find()
257 return b; in __find()
259 n = (b->block < block) ? n->rb_left : n->rb_right; in __find()
265 static void __insert(struct dm_bufio_client *c, struct dm_buffer *b) in __insert() argument
273 if (found->block == b->block) { in __insert()
274 BUG_ON(found != b); in __insert()
279 new = (found->block < b->block) ? in __insert()
283 rb_link_node(&b->node, parent, new); in __insert()
284 rb_insert_color(&b->node, &c->buffer_tree); in __insert()
287 static void __remove(struct dm_bufio_client *c, struct dm_buffer *b) in __remove() argument
289 rb_erase(&b->node, &c->buffer_tree); in __remove()
294 static void adjust_total_allocated(struct dm_buffer *b, bool unlink) in adjust_total_allocated() argument
305 data_mode = b->data_mode; in adjust_total_allocated()
306 diff = (long)b->c->block_size; in adjust_total_allocated()
319 b->accessed = 1; in adjust_total_allocated()
322 list_add(&b->global_list, &global_queue); in adjust_total_allocated()
327 list_del(&b->global_list); in adjust_total_allocated()
444 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask); in alloc_buffer() local
446 if (!b) in alloc_buffer()
449 b->c = c; in alloc_buffer()
451 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode); in alloc_buffer()
452 if (!b->data) { in alloc_buffer()
453 kmem_cache_free(c->slab_buffer, b); in alloc_buffer()
458 b->stack_len = 0; in alloc_buffer()
460 return b; in alloc_buffer()
466 static void free_buffer(struct dm_buffer *b) in free_buffer() argument
468 struct dm_bufio_client *c = b->c; in free_buffer()
470 free_buffer_data(c, b->data, b->data_mode); in free_buffer()
471 kmem_cache_free(c->slab_buffer, b); in free_buffer()
477 static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty) in __link_buffer() argument
479 struct dm_bufio_client *c = b->c; in __link_buffer()
482 b->block = block; in __link_buffer()
483 b->list_mode = dirty; in __link_buffer()
484 list_add(&b->lru_list, &c->lru[dirty]); in __link_buffer()
485 __insert(b->c, b); in __link_buffer()
486 b->last_accessed = jiffies; in __link_buffer()
488 adjust_total_allocated(b, false); in __link_buffer()
494 static void __unlink_buffer(struct dm_buffer *b) in __unlink_buffer() argument
496 struct dm_bufio_client *c = b->c; in __unlink_buffer()
498 BUG_ON(!c->n_buffers[b->list_mode]); in __unlink_buffer()
500 c->n_buffers[b->list_mode]--; in __unlink_buffer()
501 __remove(b->c, b); in __unlink_buffer()
502 list_del(&b->lru_list); in __unlink_buffer()
504 adjust_total_allocated(b, true); in __unlink_buffer()
510 static void __relink_lru(struct dm_buffer *b, int dirty) in __relink_lru() argument
512 struct dm_bufio_client *c = b->c; in __relink_lru()
514 b->accessed = 1; in __relink_lru()
516 BUG_ON(!c->n_buffers[b->list_mode]); in __relink_lru()
518 c->n_buffers[b->list_mode]--; in __relink_lru()
520 b->list_mode = dirty; in __relink_lru()
521 list_move(&b->lru_list, &c->lru[dirty]); in __relink_lru()
522 b->last_accessed = jiffies; in __relink_lru()
549 struct dm_buffer *b = context; in dmio_complete() local
551 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); in dmio_complete()
554 static void use_dmio(struct dm_buffer *b, int rw, sector_t sector, in use_dmio() argument
562 .notify.context = b, in use_dmio()
563 .client = b->c->dm_io, in use_dmio()
566 .bdev = b->c->bdev, in use_dmio()
571 if (b->data_mode != DATA_MODE_VMALLOC) { in use_dmio()
573 io_req.mem.ptr.addr = (char *)b->data + offset; in use_dmio()
576 io_req.mem.ptr.vma = (char *)b->data + offset; in use_dmio()
581 b->end_io(b, errno_to_blk_status(r)); in use_dmio()
586 struct dm_buffer *b = bio->bi_private; in bio_complete() local
589 b->end_io(b, status); in bio_complete()
592 static void use_bio(struct dm_buffer *b, int rw, sector_t sector, in use_bio() argument
599 vec_size = b->c->block_size >> PAGE_SHIFT; in use_bio()
600 if (unlikely(b->c->sectors_per_block_bits < PAGE_SHIFT - SECTOR_SHIFT)) in use_bio()
606 use_dmio(b, rw, sector, n_sectors, offset); in use_bio()
611 bio_set_dev(bio, b->c->bdev); in use_bio()
614 bio->bi_private = b; in use_bio()
616 ptr = (char *)b->data + offset; in use_bio()
634 static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buffer *, blk_status_t)) in submit_io() argument
640 b->end_io = end_io; in submit_io()
642 if (likely(b->c->sectors_per_block_bits >= 0)) in submit_io()
643 sector = b->block << b->c->sectors_per_block_bits; in submit_io()
645 sector = b->block * (b->c->block_size >> SECTOR_SHIFT); in submit_io()
646 sector += b->c->start; in submit_io()
649 n_sectors = b->c->block_size >> SECTOR_SHIFT; in submit_io()
652 if (b->c->write_callback) in submit_io()
653 b->c->write_callback(b); in submit_io()
654 offset = b->write_start; in submit_io()
655 end = b->write_end; in submit_io()
659 if (unlikely(end > b->c->block_size)) in submit_io()
660 end = b->c->block_size; in submit_io()
666 if (b->data_mode != DATA_MODE_VMALLOC) in submit_io()
667 use_bio(b, rw, sector, n_sectors, offset); in submit_io()
669 use_dmio(b, rw, sector, n_sectors, offset); in submit_io()
682 static void write_endio(struct dm_buffer *b, blk_status_t status) in write_endio() argument
684 b->write_error = status; in write_endio()
686 struct dm_bufio_client *c = b->c; in write_endio()
692 BUG_ON(!test_bit(B_WRITING, &b->state)); in write_endio()
695 clear_bit(B_WRITING, &b->state); in write_endio()
698 wake_up_bit(&b->state, B_WRITING); in write_endio()
710 static void __write_dirty_buffer(struct dm_buffer *b, in __write_dirty_buffer() argument
713 if (!test_bit(B_DIRTY, &b->state)) in __write_dirty_buffer()
716 clear_bit(B_DIRTY, &b->state); in __write_dirty_buffer()
717 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __write_dirty_buffer()
719 b->write_start = b->dirty_start; in __write_dirty_buffer()
720 b->write_end = b->dirty_end; in __write_dirty_buffer()
723 submit_io(b, REQ_OP_WRITE, write_endio); in __write_dirty_buffer()
725 list_add_tail(&b->write_list, write_list); in __write_dirty_buffer()
733 struct dm_buffer *b = in __flush_write_list() local
735 list_del(&b->write_list); in __flush_write_list()
736 submit_io(b, REQ_OP_WRITE, write_endio); in __flush_write_list()
747 static void __make_buffer_clean(struct dm_buffer *b) in __make_buffer_clean() argument
749 BUG_ON(b->hold_count); in __make_buffer_clean()
751 if (!b->state) /* fast case */ in __make_buffer_clean()
754 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
755 __write_dirty_buffer(b, NULL); in __make_buffer_clean()
756 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
765 struct dm_buffer *b; in __get_unclaimed_buffer() local
767 list_for_each_entry_reverse(b, &c->lru[LIST_CLEAN], lru_list) { in __get_unclaimed_buffer()
768 BUG_ON(test_bit(B_WRITING, &b->state)); in __get_unclaimed_buffer()
769 BUG_ON(test_bit(B_DIRTY, &b->state)); in __get_unclaimed_buffer()
771 if (!b->hold_count) { in __get_unclaimed_buffer()
772 __make_buffer_clean(b); in __get_unclaimed_buffer()
773 __unlink_buffer(b); in __get_unclaimed_buffer()
774 return b; in __get_unclaimed_buffer()
779 list_for_each_entry_reverse(b, &c->lru[LIST_DIRTY], lru_list) { in __get_unclaimed_buffer()
780 BUG_ON(test_bit(B_READING, &b->state)); in __get_unclaimed_buffer()
782 if (!b->hold_count) { in __get_unclaimed_buffer()
783 __make_buffer_clean(b); in __get_unclaimed_buffer()
784 __unlink_buffer(b); in __get_unclaimed_buffer()
785 return b; in __get_unclaimed_buffer()
830 struct dm_buffer *b; in __alloc_buffer_wait_no_callback() local
848 b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); in __alloc_buffer_wait_no_callback()
849 if (b) in __alloc_buffer_wait_no_callback()
850 return b; in __alloc_buffer_wait_no_callback()
858 b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); in __alloc_buffer_wait_no_callback()
860 if (b) in __alloc_buffer_wait_no_callback()
861 return b; in __alloc_buffer_wait_no_callback()
866 b = list_entry(c->reserved_buffers.next, in __alloc_buffer_wait_no_callback()
868 list_del(&b->lru_list); in __alloc_buffer_wait_no_callback()
871 return b; in __alloc_buffer_wait_no_callback()
874 b = __get_unclaimed_buffer(c); in __alloc_buffer_wait_no_callback()
875 if (b) in __alloc_buffer_wait_no_callback()
876 return b; in __alloc_buffer_wait_no_callback()
884 struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf); in __alloc_buffer_wait() local
886 if (!b) in __alloc_buffer_wait()
890 c->alloc_callback(b); in __alloc_buffer_wait()
892 return b; in __alloc_buffer_wait()
898 static void __free_buffer_wake(struct dm_buffer *b) in __free_buffer_wake() argument
900 struct dm_bufio_client *c = b->c; in __free_buffer_wake()
903 free_buffer(b); in __free_buffer_wake()
905 list_add(&b->lru_list, &c->reserved_buffers); in __free_buffer_wake()
915 struct dm_buffer *b, *tmp; in __write_dirty_buffers_async() local
917 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) { in __write_dirty_buffers_async()
918 BUG_ON(test_bit(B_READING, &b->state)); in __write_dirty_buffers_async()
920 if (!test_bit(B_DIRTY, &b->state) && in __write_dirty_buffers_async()
921 !test_bit(B_WRITING, &b->state)) { in __write_dirty_buffers_async()
922 __relink_lru(b, LIST_CLEAN); in __write_dirty_buffers_async()
926 if (no_wait && test_bit(B_WRITING, &b->state)) in __write_dirty_buffers_async()
929 __write_dirty_buffer(b, write_list); in __write_dirty_buffers_async()
954 struct dm_buffer *b, *new_b = NULL; in __bufio_new() local
958 b = __find(c, block); in __bufio_new()
959 if (b) in __bufio_new()
973 b = __find(c, block); in __bufio_new()
974 if (b) { in __bufio_new()
981 b = new_b; in __bufio_new()
982 b->hold_count = 1; in __bufio_new()
983 b->read_error = 0; in __bufio_new()
984 b->write_error = 0; in __bufio_new()
985 __link_buffer(b, block, LIST_CLEAN); in __bufio_new()
988 b->state = 0; in __bufio_new()
989 return b; in __bufio_new()
992 b->state = 1 << B_READING; in __bufio_new()
995 return b; in __bufio_new()
1007 if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state))) in __bufio_new()
1010 b->hold_count++; in __bufio_new()
1011 __relink_lru(b, test_bit(B_DIRTY, &b->state) || in __bufio_new()
1012 test_bit(B_WRITING, &b->state)); in __bufio_new()
1013 return b; in __bufio_new()
1020 static void read_endio(struct dm_buffer *b, blk_status_t status) in read_endio() argument
1022 b->read_error = status; in read_endio()
1024 BUG_ON(!test_bit(B_READING, &b->state)); in read_endio()
1027 clear_bit(B_READING, &b->state); in read_endio()
1030 wake_up_bit(&b->state, B_READING); in read_endio()
1043 struct dm_buffer *b; in new_read() local
1048 b = __bufio_new(c, block, nf, &need_submit, &write_list); in new_read()
1050 if (b && b->hold_count == 1) in new_read()
1051 buffer_record_stack(b); in new_read()
1057 if (!b) in new_read()
1061 submit_io(b, REQ_OP_READ, read_endio); in new_read()
1063 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in new_read()
1065 if (b->read_error) { in new_read()
1066 int error = blk_status_to_errno(b->read_error); in new_read()
1068 dm_bufio_release(b); in new_read()
1073 *bp = b; in new_read()
1075 return b->data; in new_read()
1117 struct dm_buffer *b; in dm_bufio_prefetch() local
1118 b = __bufio_new(c, block, NF_PREFETCH, &need_submit, in dm_bufio_prefetch()
1127 if (unlikely(b != NULL)) { in dm_bufio_prefetch()
1131 submit_io(b, REQ_OP_READ, read_endio); in dm_bufio_prefetch()
1132 dm_bufio_release(b); in dm_bufio_prefetch()
1149 void dm_bufio_release(struct dm_buffer *b) in dm_bufio_release() argument
1151 struct dm_bufio_client *c = b->c; in dm_bufio_release()
1155 BUG_ON(!b->hold_count); in dm_bufio_release()
1157 b->hold_count--; in dm_bufio_release()
1158 if (!b->hold_count) { in dm_bufio_release()
1166 if ((b->read_error || b->write_error) && in dm_bufio_release()
1167 !test_bit(B_READING, &b->state) && in dm_bufio_release()
1168 !test_bit(B_WRITING, &b->state) && in dm_bufio_release()
1169 !test_bit(B_DIRTY, &b->state)) { in dm_bufio_release()
1170 __unlink_buffer(b); in dm_bufio_release()
1171 __free_buffer_wake(b); in dm_bufio_release()
1179 void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b, in dm_bufio_mark_partial_buffer_dirty() argument
1182 struct dm_bufio_client *c = b->c; in dm_bufio_mark_partial_buffer_dirty()
1185 BUG_ON(end > b->c->block_size); in dm_bufio_mark_partial_buffer_dirty()
1189 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_mark_partial_buffer_dirty()
1191 if (!test_and_set_bit(B_DIRTY, &b->state)) { in dm_bufio_mark_partial_buffer_dirty()
1192 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
1193 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
1194 __relink_lru(b, LIST_DIRTY); in dm_bufio_mark_partial_buffer_dirty()
1196 if (start < b->dirty_start) in dm_bufio_mark_partial_buffer_dirty()
1197 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
1198 if (end > b->dirty_end) in dm_bufio_mark_partial_buffer_dirty()
1199 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
1206 void dm_bufio_mark_buffer_dirty(struct dm_buffer *b) in dm_bufio_mark_buffer_dirty() argument
1208 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size); in dm_bufio_mark_buffer_dirty()
1236 struct dm_buffer *b, *tmp; in dm_bufio_write_dirty_buffers() local
1247 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) { in dm_bufio_write_dirty_buffers()
1253 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_write_dirty_buffers()
1255 if (test_bit(B_WRITING, &b->state)) { in dm_bufio_write_dirty_buffers()
1258 b->hold_count++; in dm_bufio_write_dirty_buffers()
1260 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_write_dirty_buffers()
1263 b->hold_count--; in dm_bufio_write_dirty_buffers()
1265 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_write_dirty_buffers()
1269 if (!test_bit(B_DIRTY, &b->state) && in dm_bufio_write_dirty_buffers()
1270 !test_bit(B_WRITING, &b->state)) in dm_bufio_write_dirty_buffers()
1271 __relink_lru(b, LIST_CLEAN); in dm_bufio_write_dirty_buffers()
1340 void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block) in dm_bufio_release_move() argument
1342 struct dm_bufio_client *c = b->c; in dm_bufio_release_move()
1366 BUG_ON(!b->hold_count); in dm_bufio_release_move()
1367 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_release_move()
1369 __write_dirty_buffer(b, NULL); in dm_bufio_release_move()
1370 if (b->hold_count == 1) { in dm_bufio_release_move()
1371 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_release_move()
1373 set_bit(B_DIRTY, &b->state); in dm_bufio_release_move()
1374 b->dirty_start = 0; in dm_bufio_release_move()
1375 b->dirty_end = c->block_size; in dm_bufio_release_move()
1376 __unlink_buffer(b); in dm_bufio_release_move()
1377 __link_buffer(b, new_block, LIST_DIRTY); in dm_bufio_release_move()
1380 wait_on_bit_lock_io(&b->state, B_WRITING, in dm_bufio_release_move()
1389 old_block = b->block; in dm_bufio_release_move()
1390 __unlink_buffer(b); in dm_bufio_release_move()
1391 __link_buffer(b, new_block, b->list_mode); in dm_bufio_release_move()
1392 submit_io(b, REQ_OP_WRITE, write_endio); in dm_bufio_release_move()
1393 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_release_move()
1395 __unlink_buffer(b); in dm_bufio_release_move()
1396 __link_buffer(b, old_block, b->list_mode); in dm_bufio_release_move()
1400 dm_bufio_release(b); in dm_bufio_release_move()
1412 struct dm_buffer *b; in dm_bufio_forget() local
1416 b = __find(c, block); in dm_bufio_forget()
1417 if (b && likely(!b->hold_count) && likely(!b->state)) { in dm_bufio_forget()
1418 __unlink_buffer(b); in dm_bufio_forget()
1419 __free_buffer_wake(b); in dm_bufio_forget()
1449 sector_t dm_bufio_get_block_number(struct dm_buffer *b) in dm_bufio_get_block_number() argument
1451 return b->block; in dm_bufio_get_block_number()
1455 void *dm_bufio_get_block_data(struct dm_buffer *b) in dm_bufio_get_block_data() argument
1457 return b->data; in dm_bufio_get_block_data()
1461 void *dm_bufio_get_aux_data(struct dm_buffer *b) in dm_bufio_get_aux_data() argument
1463 return b + 1; in dm_bufio_get_aux_data()
1467 struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b) in dm_bufio_get_client() argument
1469 return b->c; in dm_bufio_get_client()
1475 struct dm_buffer *b; in drop_buffers() local
1488 while ((b = __get_unclaimed_buffer(c))) in drop_buffers()
1489 __free_buffer_wake(b); in drop_buffers()
1492 list_for_each_entry(b, &c->lru[i], lru_list) { in drop_buffers()
1496 (unsigned long long)b->block, b->hold_count, i); in drop_buffers()
1498 stack_trace_print(b->stack_entries, b->stack_len, 1); in drop_buffers()
1500 b->hold_count = 0; in drop_buffers()
1505 while ((b = __get_unclaimed_buffer(c))) in drop_buffers()
1506 __free_buffer_wake(b); in drop_buffers()
1523 static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp) in __try_evict_buffer() argument
1526 if (test_bit(B_READING, &b->state) || in __try_evict_buffer()
1527 test_bit(B_WRITING, &b->state) || in __try_evict_buffer()
1528 test_bit(B_DIRTY, &b->state)) in __try_evict_buffer()
1532 if (b->hold_count) in __try_evict_buffer()
1535 __make_buffer_clean(b); in __try_evict_buffer()
1536 __unlink_buffer(b); in __try_evict_buffer()
1537 __free_buffer_wake(b); in __try_evict_buffer()
1556 struct dm_buffer *b, *tmp; in __scan() local
1563 list_for_each_entry_safe_reverse(b, tmp, &c->lru[l], lru_list) { in __scan()
1564 if (__try_evict_buffer(b, gfp_mask)) in __scan()
1681 struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL); in dm_bufio_client_create() local
1683 if (!b) { in dm_bufio_client_create()
1687 __free_buffer_wake(b); in dm_bufio_client_create()
1708 struct dm_buffer *b = list_entry(c->reserved_buffers.next, in dm_bufio_client_create() local
1710 list_del(&b->lru_list); in dm_bufio_client_create()
1711 free_buffer(b); in dm_bufio_client_create()
1748 struct dm_buffer *b = list_entry(c->reserved_buffers.next, in dm_bufio_client_destroy() local
1750 list_del(&b->lru_list); in dm_bufio_client_destroy()
1751 free_buffer(b); in dm_bufio_client_destroy()
1785 static bool older_than(struct dm_buffer *b, unsigned long age_hz) in older_than() argument
1787 return time_after_eq(jiffies, b->last_accessed + age_hz); in older_than()
1792 struct dm_buffer *b, *tmp; in __evict_old_buffers() local
1807 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_CLEAN], lru_list) { in __evict_old_buffers()
1811 if (!older_than(b, age_hz)) in __evict_old_buffers()
1814 if (__try_evict_buffer(b, 0)) in __evict_old_buffers()
1827 struct dm_buffer *b; in do_global_cleanup() local
1848 b = list_entry(global_queue.prev, struct dm_buffer, global_list); in do_global_cleanup()
1850 if (b->accessed) { in do_global_cleanup()
1851 b->accessed = 0; in do_global_cleanup()
1852 list_move(&b->global_list, &global_queue); in do_global_cleanup()
1859 current_client = b->c; in do_global_cleanup()
1876 if (unlikely(!__try_evict_buffer(b, GFP_KERNEL))) { in do_global_cleanup()
1878 list_move(&b->global_list, &global_queue); in do_global_cleanup()