Lines Matching refs:bp
43 struct xfs_buf *bp) in xfs_buf_is_vmapped() argument
52 return bp->b_addr && bp->b_page_count > 1; in xfs_buf_is_vmapped()
57 struct xfs_buf *bp) in xfs_buf_vmap_len() argument
59 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset; in xfs_buf_vmap_len()
77 struct xfs_buf *bp) in xfs_buf_ioacct_inc() argument
79 if (bp->b_flags & XBF_NO_IOACCT) in xfs_buf_ioacct_inc()
82 ASSERT(bp->b_flags & XBF_ASYNC); in xfs_buf_ioacct_inc()
83 spin_lock(&bp->b_lock); in xfs_buf_ioacct_inc()
84 if (!(bp->b_state & XFS_BSTATE_IN_FLIGHT)) { in xfs_buf_ioacct_inc()
85 bp->b_state |= XFS_BSTATE_IN_FLIGHT; in xfs_buf_ioacct_inc()
86 percpu_counter_inc(&bp->b_target->bt_io_count); in xfs_buf_ioacct_inc()
88 spin_unlock(&bp->b_lock); in xfs_buf_ioacct_inc()
97 struct xfs_buf *bp) in __xfs_buf_ioacct_dec() argument
99 lockdep_assert_held(&bp->b_lock); in __xfs_buf_ioacct_dec()
101 if (bp->b_state & XFS_BSTATE_IN_FLIGHT) { in __xfs_buf_ioacct_dec()
102 bp->b_state &= ~XFS_BSTATE_IN_FLIGHT; in __xfs_buf_ioacct_dec()
103 percpu_counter_dec(&bp->b_target->bt_io_count); in __xfs_buf_ioacct_dec()
109 struct xfs_buf *bp) in xfs_buf_ioacct_dec() argument
111 spin_lock(&bp->b_lock); in xfs_buf_ioacct_dec()
112 __xfs_buf_ioacct_dec(bp); in xfs_buf_ioacct_dec()
113 spin_unlock(&bp->b_lock); in xfs_buf_ioacct_dec()
126 struct xfs_buf *bp) in xfs_buf_stale() argument
128 ASSERT(xfs_buf_islocked(bp)); in xfs_buf_stale()
130 bp->b_flags |= XBF_STALE; in xfs_buf_stale()
137 bp->b_flags &= ~_XBF_DELWRI_Q; in xfs_buf_stale()
145 spin_lock(&bp->b_lock); in xfs_buf_stale()
146 __xfs_buf_ioacct_dec(bp); in xfs_buf_stale()
148 atomic_set(&bp->b_lru_ref, 0); in xfs_buf_stale()
149 if (!(bp->b_state & XFS_BSTATE_DISPOSE) && in xfs_buf_stale()
150 (list_lru_del(&bp->b_target->bt_lru, &bp->b_lru))) in xfs_buf_stale()
151 atomic_dec(&bp->b_hold); in xfs_buf_stale()
153 ASSERT(atomic_read(&bp->b_hold) >= 1); in xfs_buf_stale()
154 spin_unlock(&bp->b_lock); in xfs_buf_stale()
159 struct xfs_buf *bp, in xfs_buf_get_maps() argument
162 ASSERT(bp->b_maps == NULL); in xfs_buf_get_maps()
163 bp->b_map_count = map_count; in xfs_buf_get_maps()
166 bp->b_maps = &bp->__b_map; in xfs_buf_get_maps()
170 bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map), in xfs_buf_get_maps()
172 if (!bp->b_maps) in xfs_buf_get_maps()
182 struct xfs_buf *bp) in xfs_buf_free_maps() argument
184 if (bp->b_maps != &bp->__b_map) { in xfs_buf_free_maps()
185 kmem_free(bp->b_maps); in xfs_buf_free_maps()
186 bp->b_maps = NULL; in xfs_buf_free_maps()
197 struct xfs_buf *bp; in _xfs_buf_alloc() local
201 bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS); in _xfs_buf_alloc()
202 if (unlikely(!bp)) in _xfs_buf_alloc()
211 atomic_set(&bp->b_hold, 1); in _xfs_buf_alloc()
212 atomic_set(&bp->b_lru_ref, 1); in _xfs_buf_alloc()
213 init_completion(&bp->b_iowait); in _xfs_buf_alloc()
214 INIT_LIST_HEAD(&bp->b_lru); in _xfs_buf_alloc()
215 INIT_LIST_HEAD(&bp->b_list); in _xfs_buf_alloc()
216 INIT_LIST_HEAD(&bp->b_li_list); in _xfs_buf_alloc()
217 sema_init(&bp->b_sema, 0); /* held, no waiters */ in _xfs_buf_alloc()
218 spin_lock_init(&bp->b_lock); in _xfs_buf_alloc()
219 bp->b_target = target; in _xfs_buf_alloc()
220 bp->b_flags = flags; in _xfs_buf_alloc()
227 error = xfs_buf_get_maps(bp, nmaps); in _xfs_buf_alloc()
229 kmem_zone_free(xfs_buf_zone, bp); in _xfs_buf_alloc()
233 bp->b_bn = map[0].bm_bn; in _xfs_buf_alloc()
234 bp->b_length = 0; in _xfs_buf_alloc()
236 bp->b_maps[i].bm_bn = map[i].bm_bn; in _xfs_buf_alloc()
237 bp->b_maps[i].bm_len = map[i].bm_len; in _xfs_buf_alloc()
238 bp->b_length += map[i].bm_len; in _xfs_buf_alloc()
240 bp->b_io_length = bp->b_length; in _xfs_buf_alloc()
242 atomic_set(&bp->b_pin_count, 0); in _xfs_buf_alloc()
243 init_waitqueue_head(&bp->b_waiters); in _xfs_buf_alloc()
246 trace_xfs_buf_init(bp, _RET_IP_); in _xfs_buf_alloc()
248 return bp; in _xfs_buf_alloc()
257 xfs_buf_t *bp, in _xfs_buf_get_pages() argument
261 if (bp->b_pages == NULL) { in _xfs_buf_get_pages()
262 bp->b_page_count = page_count; in _xfs_buf_get_pages()
264 bp->b_pages = bp->b_page_array; in _xfs_buf_get_pages()
266 bp->b_pages = kmem_alloc(sizeof(struct page *) * in _xfs_buf_get_pages()
268 if (bp->b_pages == NULL) in _xfs_buf_get_pages()
271 memset(bp->b_pages, 0, sizeof(struct page *) * page_count); in _xfs_buf_get_pages()
281 xfs_buf_t *bp) in _xfs_buf_free_pages() argument
283 if (bp->b_pages != bp->b_page_array) { in _xfs_buf_free_pages()
284 kmem_free(bp->b_pages); in _xfs_buf_free_pages()
285 bp->b_pages = NULL; in _xfs_buf_free_pages()
298 xfs_buf_t *bp) in xfs_buf_free() argument
300 trace_xfs_buf_free(bp, _RET_IP_); in xfs_buf_free()
302 ASSERT(list_empty(&bp->b_lru)); in xfs_buf_free()
304 if (bp->b_flags & _XBF_PAGES) { in xfs_buf_free()
307 if (xfs_buf_is_vmapped(bp)) in xfs_buf_free()
308 vm_unmap_ram(bp->b_addr - bp->b_offset, in xfs_buf_free()
309 bp->b_page_count); in xfs_buf_free()
311 for (i = 0; i < bp->b_page_count; i++) { in xfs_buf_free()
312 struct page *page = bp->b_pages[i]; in xfs_buf_free()
316 } else if (bp->b_flags & _XBF_KMEM) in xfs_buf_free()
317 kmem_free(bp->b_addr); in xfs_buf_free()
318 _xfs_buf_free_pages(bp); in xfs_buf_free()
319 xfs_buf_free_maps(bp); in xfs_buf_free()
320 kmem_zone_free(xfs_buf_zone, bp); in xfs_buf_free()
328 xfs_buf_t *bp, in xfs_buf_allocate_memory() argument
343 size = BBTOB(bp->b_length); in xfs_buf_allocate_memory()
345 bp->b_addr = kmem_alloc(size, KM_NOFS); in xfs_buf_allocate_memory()
346 if (!bp->b_addr) { in xfs_buf_allocate_memory()
351 if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) != in xfs_buf_allocate_memory()
352 ((unsigned long)bp->b_addr & PAGE_MASK)) { in xfs_buf_allocate_memory()
354 kmem_free(bp->b_addr); in xfs_buf_allocate_memory()
355 bp->b_addr = NULL; in xfs_buf_allocate_memory()
358 bp->b_offset = offset_in_page(bp->b_addr); in xfs_buf_allocate_memory()
359 bp->b_pages = bp->b_page_array; in xfs_buf_allocate_memory()
360 bp->b_pages[0] = virt_to_page(bp->b_addr); in xfs_buf_allocate_memory()
361 bp->b_page_count = 1; in xfs_buf_allocate_memory()
362 bp->b_flags |= _XBF_KMEM; in xfs_buf_allocate_memory()
367 start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT; in xfs_buf_allocate_memory()
368 end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1) in xfs_buf_allocate_memory()
371 error = _xfs_buf_get_pages(bp, page_count); in xfs_buf_allocate_memory()
375 offset = bp->b_offset; in xfs_buf_allocate_memory()
376 bp->b_flags |= _XBF_PAGES; in xfs_buf_allocate_memory()
378 for (i = 0; i < bp->b_page_count; i++) { in xfs_buf_allocate_memory()
385 bp->b_page_count = i; in xfs_buf_allocate_memory()
402 XFS_STATS_INC(bp->b_target->bt_mount, xb_page_retries); in xfs_buf_allocate_memory()
407 XFS_STATS_INC(bp->b_target->bt_mount, xb_page_found); in xfs_buf_allocate_memory()
411 bp->b_pages[i] = page; in xfs_buf_allocate_memory()
417 for (i = 0; i < bp->b_page_count; i++) in xfs_buf_allocate_memory()
418 __free_page(bp->b_pages[i]); in xfs_buf_allocate_memory()
419 bp->b_flags &= ~_XBF_PAGES; in xfs_buf_allocate_memory()
428 xfs_buf_t *bp, in _xfs_buf_map_pages() argument
431 ASSERT(bp->b_flags & _XBF_PAGES); in _xfs_buf_map_pages()
432 if (bp->b_page_count == 1) { in _xfs_buf_map_pages()
434 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset; in _xfs_buf_map_pages()
436 bp->b_addr = NULL; in _xfs_buf_map_pages()
451 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count, in _xfs_buf_map_pages()
453 if (bp->b_addr) in _xfs_buf_map_pages()
459 if (!bp->b_addr) in _xfs_buf_map_pages()
461 bp->b_addr += bp->b_offset; in _xfs_buf_map_pages()
476 const struct xfs_buf *bp = obj; in _xfs_buf_obj_cmp() local
484 if (bp->b_bn != map->bm_bn) in _xfs_buf_obj_cmp()
487 if (unlikely(bp->b_length != map->bm_len)) { in _xfs_buf_obj_cmp()
496 ASSERT(bp->b_flags & XBF_STALE); in _xfs_buf_obj_cmp()
555 xfs_buf_t *bp; in xfs_buf_find() local
586 bp = rhashtable_lookup_fast(&pag->pag_buf_hash, &cmap, in xfs_buf_find()
588 if (bp) { in xfs_buf_find()
589 atomic_inc(&bp->b_hold); in xfs_buf_find()
613 if (!xfs_buf_trylock(bp)) { in xfs_buf_find()
615 xfs_buf_rele(bp); in xfs_buf_find()
619 xfs_buf_lock(bp); in xfs_buf_find()
628 if (bp->b_flags & XBF_STALE) { in xfs_buf_find()
629 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0); in xfs_buf_find()
630 ASSERT(bp->b_iodone == NULL); in xfs_buf_find()
631 bp->b_flags &= _XBF_KMEM | _XBF_PAGES; in xfs_buf_find()
632 bp->b_ops = NULL; in xfs_buf_find()
635 trace_xfs_buf_find(bp, flags, _RET_IP_); in xfs_buf_find()
637 *found_bp = bp; in xfs_buf_find()
648 struct xfs_buf *bp; in xfs_buf_incore() local
652 error = xfs_buf_find(target, &map, 1, flags, NULL, &bp); in xfs_buf_incore()
655 return bp; in xfs_buf_incore()
670 struct xfs_buf *bp; in xfs_buf_get_map() local
674 error = xfs_buf_find(target, map, nmaps, flags, NULL, &bp); in xfs_buf_get_map()
706 error = xfs_buf_find(target, map, nmaps, flags, new_bp, &bp); in xfs_buf_get_map()
712 if (bp != new_bp) in xfs_buf_get_map()
716 if (!bp->b_addr) { in xfs_buf_get_map()
717 error = _xfs_buf_map_pages(bp, flags); in xfs_buf_get_map()
721 xfs_buf_relse(bp); in xfs_buf_get_map()
731 xfs_buf_ioerror(bp, 0); in xfs_buf_get_map()
734 trace_xfs_buf_get(bp, flags, _RET_IP_); in xfs_buf_get_map()
735 return bp; in xfs_buf_get_map()
740 xfs_buf_t *bp, in _xfs_buf_read() argument
744 ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL); in _xfs_buf_read()
746 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD); in _xfs_buf_read()
747 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD); in _xfs_buf_read()
749 return xfs_buf_submit(bp); in _xfs_buf_read()
760 struct xfs_buf *bp; in xfs_buf_read_map() local
764 bp = xfs_buf_get_map(target, map, nmaps, flags); in xfs_buf_read_map()
765 if (bp) { in xfs_buf_read_map()
766 trace_xfs_buf_read(bp, flags, _RET_IP_); in xfs_buf_read_map()
768 if (!(bp->b_flags & XBF_DONE)) { in xfs_buf_read_map()
770 bp->b_ops = ops; in xfs_buf_read_map()
771 _xfs_buf_read(bp, flags); in xfs_buf_read_map()
777 xfs_buf_relse(bp); in xfs_buf_read_map()
781 bp->b_flags &= ~XBF_READ; in xfs_buf_read_map()
785 return bp; in xfs_buf_read_map()
819 struct xfs_buf *bp; in xfs_buf_read_uncached() local
823 bp = xfs_buf_get_uncached(target, numblks, flags); in xfs_buf_read_uncached()
824 if (!bp) in xfs_buf_read_uncached()
828 ASSERT(bp->b_map_count == 1); in xfs_buf_read_uncached()
829 bp->b_bn = XFS_BUF_DADDR_NULL; /* always null for uncached buffers */ in xfs_buf_read_uncached()
830 bp->b_maps[0].bm_bn = daddr; in xfs_buf_read_uncached()
831 bp->b_flags |= XBF_READ; in xfs_buf_read_uncached()
832 bp->b_ops = ops; in xfs_buf_read_uncached()
834 xfs_buf_submit(bp); in xfs_buf_read_uncached()
835 if (bp->b_error) { in xfs_buf_read_uncached()
836 int error = bp->b_error; in xfs_buf_read_uncached()
837 xfs_buf_relse(bp); in xfs_buf_read_uncached()
841 *bpp = bp; in xfs_buf_read_uncached()
851 struct xfs_buf *bp, in xfs_buf_set_empty() argument
854 if (bp->b_pages) in xfs_buf_set_empty()
855 _xfs_buf_free_pages(bp); in xfs_buf_set_empty()
857 bp->b_pages = NULL; in xfs_buf_set_empty()
858 bp->b_page_count = 0; in xfs_buf_set_empty()
859 bp->b_addr = NULL; in xfs_buf_set_empty()
860 bp->b_length = numblks; in xfs_buf_set_empty()
861 bp->b_io_length = numblks; in xfs_buf_set_empty()
863 ASSERT(bp->b_map_count == 1); in xfs_buf_set_empty()
864 bp->b_bn = XFS_BUF_DADDR_NULL; in xfs_buf_set_empty()
865 bp->b_maps[0].bm_bn = XFS_BUF_DADDR_NULL; in xfs_buf_set_empty()
866 bp->b_maps[0].bm_len = bp->b_length; in xfs_buf_set_empty()
882 xfs_buf_t *bp, in xfs_buf_associate_memory() argument
899 if (bp->b_pages) in xfs_buf_associate_memory()
900 _xfs_buf_free_pages(bp); in xfs_buf_associate_memory()
902 bp->b_pages = NULL; in xfs_buf_associate_memory()
903 bp->b_addr = mem; in xfs_buf_associate_memory()
905 rval = _xfs_buf_get_pages(bp, page_count); in xfs_buf_associate_memory()
909 bp->b_offset = offset; in xfs_buf_associate_memory()
911 for (i = 0; i < bp->b_page_count; i++) { in xfs_buf_associate_memory()
912 bp->b_pages[i] = mem_to_page((void *)pageaddr); in xfs_buf_associate_memory()
916 bp->b_io_length = BTOBB(len); in xfs_buf_associate_memory()
917 bp->b_length = BTOBB(buflen); in xfs_buf_associate_memory()
930 struct xfs_buf *bp; in xfs_buf_get_uncached() local
934 bp = _xfs_buf_alloc(target, &map, 1, flags & XBF_NO_IOACCT); in xfs_buf_get_uncached()
935 if (unlikely(bp == NULL)) in xfs_buf_get_uncached()
939 error = _xfs_buf_get_pages(bp, page_count); in xfs_buf_get_uncached()
944 bp->b_pages[i] = alloc_page(xb_to_gfp(flags)); in xfs_buf_get_uncached()
945 if (!bp->b_pages[i]) in xfs_buf_get_uncached()
948 bp->b_flags |= _XBF_PAGES; in xfs_buf_get_uncached()
950 error = _xfs_buf_map_pages(bp, 0); in xfs_buf_get_uncached()
957 trace_xfs_buf_get_uncached(bp, _RET_IP_); in xfs_buf_get_uncached()
958 return bp; in xfs_buf_get_uncached()
962 __free_page(bp->b_pages[i]); in xfs_buf_get_uncached()
963 _xfs_buf_free_pages(bp); in xfs_buf_get_uncached()
965 xfs_buf_free_maps(bp); in xfs_buf_get_uncached()
966 kmem_zone_free(xfs_buf_zone, bp); in xfs_buf_get_uncached()
978 xfs_buf_t *bp) in xfs_buf_hold() argument
980 trace_xfs_buf_hold(bp, _RET_IP_); in xfs_buf_hold()
981 atomic_inc(&bp->b_hold); in xfs_buf_hold()
990 xfs_buf_t *bp) in xfs_buf_rele() argument
992 struct xfs_perag *pag = bp->b_pag; in xfs_buf_rele()
996 trace_xfs_buf_rele(bp, _RET_IP_); in xfs_buf_rele()
999 ASSERT(list_empty(&bp->b_lru)); in xfs_buf_rele()
1000 if (atomic_dec_and_test(&bp->b_hold)) { in xfs_buf_rele()
1001 xfs_buf_ioacct_dec(bp); in xfs_buf_rele()
1002 xfs_buf_free(bp); in xfs_buf_rele()
1007 ASSERT(atomic_read(&bp->b_hold) > 0); in xfs_buf_rele()
1009 release = atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock); in xfs_buf_rele()
1010 spin_lock(&bp->b_lock); in xfs_buf_rele()
1018 if ((atomic_read(&bp->b_hold) == 1) && !list_empty(&bp->b_lru)) in xfs_buf_rele()
1019 __xfs_buf_ioacct_dec(bp); in xfs_buf_rele()
1024 __xfs_buf_ioacct_dec(bp); in xfs_buf_rele()
1025 if (!(bp->b_flags & XBF_STALE) && atomic_read(&bp->b_lru_ref)) { in xfs_buf_rele()
1031 if (list_lru_add(&bp->b_target->bt_lru, &bp->b_lru)) { in xfs_buf_rele()
1032 bp->b_state &= ~XFS_BSTATE_DISPOSE; in xfs_buf_rele()
1033 atomic_inc(&bp->b_hold); in xfs_buf_rele()
1043 if (!(bp->b_state & XFS_BSTATE_DISPOSE)) { in xfs_buf_rele()
1044 list_lru_del(&bp->b_target->bt_lru, &bp->b_lru); in xfs_buf_rele()
1046 ASSERT(list_empty(&bp->b_lru)); in xfs_buf_rele()
1049 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); in xfs_buf_rele()
1050 rhashtable_remove_fast(&pag->pag_buf_hash, &bp->b_rhash_head, in xfs_buf_rele()
1058 spin_unlock(&bp->b_lock); in xfs_buf_rele()
1061 xfs_buf_free(bp); in xfs_buf_rele()
1078 struct xfs_buf *bp) in xfs_buf_trylock() argument
1082 locked = down_trylock(&bp->b_sema) == 0; in xfs_buf_trylock()
1084 trace_xfs_buf_trylock(bp, _RET_IP_); in xfs_buf_trylock()
1086 trace_xfs_buf_trylock_fail(bp, _RET_IP_); in xfs_buf_trylock()
1101 struct xfs_buf *bp) in xfs_buf_lock() argument
1103 trace_xfs_buf_lock(bp, _RET_IP_); in xfs_buf_lock()
1105 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE)) in xfs_buf_lock()
1106 xfs_log_force(bp->b_target->bt_mount, 0); in xfs_buf_lock()
1107 down(&bp->b_sema); in xfs_buf_lock()
1109 trace_xfs_buf_lock_done(bp, _RET_IP_); in xfs_buf_lock()
1114 struct xfs_buf *bp) in xfs_buf_unlock() argument
1116 ASSERT(xfs_buf_islocked(bp)); in xfs_buf_unlock()
1118 up(&bp->b_sema); in xfs_buf_unlock()
1119 trace_xfs_buf_unlock(bp, _RET_IP_); in xfs_buf_unlock()
1124 xfs_buf_t *bp) in xfs_buf_wait_unpin() argument
1128 if (atomic_read(&bp->b_pin_count) == 0) in xfs_buf_wait_unpin()
1131 add_wait_queue(&bp->b_waiters, &wait); in xfs_buf_wait_unpin()
1134 if (atomic_read(&bp->b_pin_count) == 0) in xfs_buf_wait_unpin()
1138 remove_wait_queue(&bp->b_waiters, &wait); in xfs_buf_wait_unpin()
1148 struct xfs_buf *bp) in xfs_buf_ioend() argument
1150 bool read = bp->b_flags & XBF_READ; in xfs_buf_ioend()
1152 trace_xfs_buf_iodone(bp, _RET_IP_); in xfs_buf_ioend()
1154 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD); in xfs_buf_ioend()
1160 if (!bp->b_error && bp->b_io_error) in xfs_buf_ioend()
1161 xfs_buf_ioerror(bp, bp->b_io_error); in xfs_buf_ioend()
1164 if (read && !bp->b_error && bp->b_ops) { in xfs_buf_ioend()
1165 ASSERT(!bp->b_iodone); in xfs_buf_ioend()
1166 bp->b_ops->verify_read(bp); in xfs_buf_ioend()
1169 if (!bp->b_error) in xfs_buf_ioend()
1170 bp->b_flags |= XBF_DONE; in xfs_buf_ioend()
1172 if (bp->b_iodone) in xfs_buf_ioend()
1173 (*(bp->b_iodone))(bp); in xfs_buf_ioend()
1174 else if (bp->b_flags & XBF_ASYNC) in xfs_buf_ioend()
1175 xfs_buf_relse(bp); in xfs_buf_ioend()
1177 complete(&bp->b_iowait); in xfs_buf_ioend()
1184 struct xfs_buf *bp = in xfs_buf_ioend_work() local
1187 xfs_buf_ioend(bp); in xfs_buf_ioend_work()
1192 struct xfs_buf *bp) in xfs_buf_ioend_async() argument
1194 INIT_WORK(&bp->b_ioend_work, xfs_buf_ioend_work); in xfs_buf_ioend_async()
1195 queue_work(bp->b_ioend_wq, &bp->b_ioend_work); in xfs_buf_ioend_async()
1200 xfs_buf_t *bp, in __xfs_buf_ioerror() argument
1205 bp->b_error = error; in __xfs_buf_ioerror()
1206 trace_xfs_buf_ioerror(bp, error, failaddr); in __xfs_buf_ioerror()
1211 struct xfs_buf *bp, in xfs_buf_ioerror_alert() argument
1214 xfs_alert(bp->b_target->bt_mount, in xfs_buf_ioerror_alert()
1216 func, (uint64_t)XFS_BUF_ADDR(bp), bp->b_length, in xfs_buf_ioerror_alert()
1217 -bp->b_error); in xfs_buf_ioerror_alert()
1222 struct xfs_buf *bp) in xfs_bwrite() argument
1226 ASSERT(xfs_buf_islocked(bp)); in xfs_bwrite()
1228 bp->b_flags |= XBF_WRITE; in xfs_bwrite()
1229 bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | in xfs_bwrite()
1232 error = xfs_buf_submit(bp); in xfs_bwrite()
1234 xfs_force_shutdown(bp->b_target->bt_mount, in xfs_bwrite()
1244 struct xfs_buf *bp = (struct xfs_buf *)bio->bi_private; in xfs_buf_bio_end_io() local
1253 cmpxchg(&bp->b_io_error, 0, error); in xfs_buf_bio_end_io()
1256 if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ)) in xfs_buf_bio_end_io()
1257 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp)); in xfs_buf_bio_end_io()
1259 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) in xfs_buf_bio_end_io()
1260 xfs_buf_ioend_async(bp); in xfs_buf_bio_end_io()
1266 struct xfs_buf *bp, in xfs_buf_ioapply_map() argument
1274 int total_nr_pages = bp->b_page_count; in xfs_buf_ioapply_map()
1277 sector_t sector = bp->b_maps[map].bm_bn; in xfs_buf_ioapply_map()
1293 size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count); in xfs_buf_ioapply_map()
1298 atomic_inc(&bp->b_io_remaining); in xfs_buf_ioapply_map()
1302 bio_set_dev(bio, bp->b_target->bt_bdev); in xfs_buf_ioapply_map()
1305 bio->bi_private = bp; in xfs_buf_ioapply_map()
1314 rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes, in xfs_buf_ioapply_map()
1326 if (xfs_buf_is_vmapped(bp)) { in xfs_buf_ioapply_map()
1327 flush_kernel_vmap_range(bp->b_addr, in xfs_buf_ioapply_map()
1328 xfs_buf_vmap_len(bp)); in xfs_buf_ioapply_map()
1338 atomic_dec(&bp->b_io_remaining); in xfs_buf_ioapply_map()
1339 xfs_buf_ioerror(bp, -EIO); in xfs_buf_ioapply_map()
1347 struct xfs_buf *bp) in _xfs_buf_ioapply() argument
1360 bp->b_error = 0; in _xfs_buf_ioapply()
1366 if (!bp->b_ioend_wq) in _xfs_buf_ioapply()
1367 bp->b_ioend_wq = bp->b_target->bt_mount->m_buf_workqueue; in _xfs_buf_ioapply()
1369 if (bp->b_flags & XBF_WRITE) { in _xfs_buf_ioapply()
1371 if (bp->b_flags & XBF_SYNCIO) in _xfs_buf_ioapply()
1373 if (bp->b_flags & XBF_FUA) in _xfs_buf_ioapply()
1375 if (bp->b_flags & XBF_FLUSH) in _xfs_buf_ioapply()
1383 if (bp->b_ops) { in _xfs_buf_ioapply()
1384 bp->b_ops->verify_write(bp); in _xfs_buf_ioapply()
1385 if (bp->b_error) { in _xfs_buf_ioapply()
1386 xfs_force_shutdown(bp->b_target->bt_mount, in _xfs_buf_ioapply()
1390 } else if (bp->b_bn != XFS_BUF_DADDR_NULL) { in _xfs_buf_ioapply()
1391 struct xfs_mount *mp = bp->b_target->bt_mount; in _xfs_buf_ioapply()
1400 __func__, bp->b_bn, bp->b_length); in _xfs_buf_ioapply()
1401 xfs_hex_dump(bp->b_addr, in _xfs_buf_ioapply()
1406 } else if (bp->b_flags & XBF_READ_AHEAD) { in _xfs_buf_ioapply()
1422 offset = bp->b_offset; in _xfs_buf_ioapply()
1423 size = BBTOB(bp->b_io_length); in _xfs_buf_ioapply()
1425 for (i = 0; i < bp->b_map_count; i++) { in _xfs_buf_ioapply()
1426 xfs_buf_ioapply_map(bp, i, &offset, &size, op, op_flags); in _xfs_buf_ioapply()
1427 if (bp->b_error) in _xfs_buf_ioapply()
1440 struct xfs_buf *bp) in xfs_buf_iowait() argument
1442 ASSERT(!(bp->b_flags & XBF_ASYNC)); in xfs_buf_iowait()
1444 trace_xfs_buf_iowait(bp, _RET_IP_); in xfs_buf_iowait()
1445 wait_for_completion(&bp->b_iowait); in xfs_buf_iowait()
1446 trace_xfs_buf_iowait_done(bp, _RET_IP_); in xfs_buf_iowait()
1448 return bp->b_error; in xfs_buf_iowait()
1459 struct xfs_buf *bp, in __xfs_buf_submit() argument
1464 trace_xfs_buf_submit(bp, _RET_IP_); in __xfs_buf_submit()
1466 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); in __xfs_buf_submit()
1469 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) { in __xfs_buf_submit()
1470 xfs_buf_ioerror(bp, -EIO); in __xfs_buf_submit()
1471 bp->b_flags &= ~XBF_DONE; in __xfs_buf_submit()
1472 xfs_buf_stale(bp); in __xfs_buf_submit()
1473 if (bp->b_flags & XBF_ASYNC) in __xfs_buf_submit()
1474 xfs_buf_ioend(bp); in __xfs_buf_submit()
1483 xfs_buf_hold(bp); in __xfs_buf_submit()
1485 if (bp->b_flags & XBF_WRITE) in __xfs_buf_submit()
1486 xfs_buf_wait_unpin(bp); in __xfs_buf_submit()
1489 bp->b_io_error = 0; in __xfs_buf_submit()
1496 atomic_set(&bp->b_io_remaining, 1); in __xfs_buf_submit()
1497 if (bp->b_flags & XBF_ASYNC) in __xfs_buf_submit()
1498 xfs_buf_ioacct_inc(bp); in __xfs_buf_submit()
1499 _xfs_buf_ioapply(bp); in __xfs_buf_submit()
1506 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) { in __xfs_buf_submit()
1507 if (bp->b_error || !(bp->b_flags & XBF_ASYNC)) in __xfs_buf_submit()
1508 xfs_buf_ioend(bp); in __xfs_buf_submit()
1510 xfs_buf_ioend_async(bp); in __xfs_buf_submit()
1514 error = xfs_buf_iowait(bp); in __xfs_buf_submit()
1521 xfs_buf_rele(bp); in __xfs_buf_submit()
1527 struct xfs_buf *bp, in xfs_buf_offset() argument
1532 if (bp->b_addr) in xfs_buf_offset()
1533 return bp->b_addr + offset; in xfs_buf_offset()
1535 offset += bp->b_offset; in xfs_buf_offset()
1536 page = bp->b_pages[offset >> PAGE_SHIFT]; in xfs_buf_offset()
1545 xfs_buf_t *bp, /* buffer to process */ in xfs_buf_iomove() argument
1558 page_index = (boff + bp->b_offset) >> PAGE_SHIFT; in xfs_buf_iomove()
1559 page_offset = (boff + bp->b_offset) & ~PAGE_MASK; in xfs_buf_iomove()
1560 page = bp->b_pages[page_index]; in xfs_buf_iomove()
1562 BBTOB(bp->b_io_length) - boff); in xfs_buf_iomove()
1599 struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru); in xfs_buftarg_wait_rele() local
1602 if (atomic_read(&bp->b_hold) > 1) { in xfs_buftarg_wait_rele()
1604 trace_xfs_buf_wait_buftarg(bp, _RET_IP_); in xfs_buftarg_wait_rele()
1607 if (!spin_trylock(&bp->b_lock)) in xfs_buftarg_wait_rele()
1614 atomic_set(&bp->b_lru_ref, 0); in xfs_buftarg_wait_rele()
1615 bp->b_state |= XFS_BSTATE_DISPOSE; in xfs_buftarg_wait_rele()
1617 spin_unlock(&bp->b_lock); in xfs_buftarg_wait_rele()
1650 struct xfs_buf *bp; in xfs_wait_buftarg() local
1651 bp = list_first_entry(&dispose, struct xfs_buf, b_lru); in xfs_wait_buftarg()
1652 list_del_init(&bp->b_lru); in xfs_wait_buftarg()
1653 if (bp->b_flags & XBF_WRITE_FAIL) { in xfs_wait_buftarg()
1656 (long long)bp->b_bn); in xfs_wait_buftarg()
1660 xfs_buf_rele(bp); in xfs_wait_buftarg()
1674 struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru); in xfs_buftarg_isolate() local
1681 if (!spin_trylock(&bp->b_lock)) in xfs_buftarg_isolate()
1688 if (atomic_add_unless(&bp->b_lru_ref, -1, 0)) { in xfs_buftarg_isolate()
1689 spin_unlock(&bp->b_lock); in xfs_buftarg_isolate()
1693 bp->b_state |= XFS_BSTATE_DISPOSE; in xfs_buftarg_isolate()
1695 spin_unlock(&bp->b_lock); in xfs_buftarg_isolate()
1713 struct xfs_buf *bp; in xfs_buftarg_shrink_scan() local
1714 bp = list_first_entry(&dispose, struct xfs_buf, b_lru); in xfs_buftarg_shrink_scan()
1715 list_del_init(&bp->b_lru); in xfs_buftarg_shrink_scan()
1716 xfs_buf_rele(bp); in xfs_buftarg_shrink_scan()
1833 struct xfs_buf *bp; in xfs_buf_delwri_cancel() local
1836 bp = list_first_entry(list, struct xfs_buf, b_list); in xfs_buf_delwri_cancel()
1838 xfs_buf_lock(bp); in xfs_buf_delwri_cancel()
1839 bp->b_flags &= ~_XBF_DELWRI_Q; in xfs_buf_delwri_cancel()
1840 list_del_init(&bp->b_list); in xfs_buf_delwri_cancel()
1841 xfs_buf_relse(bp); in xfs_buf_delwri_cancel()
1858 struct xfs_buf *bp, in xfs_buf_delwri_queue() argument
1861 ASSERT(xfs_buf_islocked(bp)); in xfs_buf_delwri_queue()
1862 ASSERT(!(bp->b_flags & XBF_READ)); in xfs_buf_delwri_queue()
1869 if (bp->b_flags & _XBF_DELWRI_Q) { in xfs_buf_delwri_queue()
1870 trace_xfs_buf_delwri_queued(bp, _RET_IP_); in xfs_buf_delwri_queue()
1874 trace_xfs_buf_delwri_queue(bp, _RET_IP_); in xfs_buf_delwri_queue()
1884 bp->b_flags |= _XBF_DELWRI_Q; in xfs_buf_delwri_queue()
1885 if (list_empty(&bp->b_list)) { in xfs_buf_delwri_queue()
1886 atomic_inc(&bp->b_hold); in xfs_buf_delwri_queue()
1887 list_add_tail(&bp->b_list, list); in xfs_buf_delwri_queue()
1905 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list); in xfs_buf_cmp() local
1908 diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn; in xfs_buf_cmp()
1928 struct xfs_buf *bp, *n; in xfs_buf_delwri_submit_buffers() local
1936 list_for_each_entry_safe(bp, n, buffer_list, b_list) { in xfs_buf_delwri_submit_buffers()
1938 if (xfs_buf_ispinned(bp)) { in xfs_buf_delwri_submit_buffers()
1942 if (!xfs_buf_trylock(bp)) in xfs_buf_delwri_submit_buffers()
1945 xfs_buf_lock(bp); in xfs_buf_delwri_submit_buffers()
1954 if (!(bp->b_flags & _XBF_DELWRI_Q)) { in xfs_buf_delwri_submit_buffers()
1955 list_del_init(&bp->b_list); in xfs_buf_delwri_submit_buffers()
1956 xfs_buf_relse(bp); in xfs_buf_delwri_submit_buffers()
1960 trace_xfs_buf_delwri_split(bp, _RET_IP_); in xfs_buf_delwri_submit_buffers()
1968 bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_WRITE_FAIL); in xfs_buf_delwri_submit_buffers()
1969 bp->b_flags |= XBF_WRITE; in xfs_buf_delwri_submit_buffers()
1971 bp->b_flags &= ~XBF_ASYNC; in xfs_buf_delwri_submit_buffers()
1972 list_move_tail(&bp->b_list, wait_list); in xfs_buf_delwri_submit_buffers()
1974 bp->b_flags |= XBF_ASYNC; in xfs_buf_delwri_submit_buffers()
1975 list_del_init(&bp->b_list); in xfs_buf_delwri_submit_buffers()
1977 __xfs_buf_submit(bp, false); in xfs_buf_delwri_submit_buffers()
2014 struct xfs_buf *bp; in xfs_buf_delwri_submit() local
2020 bp = list_first_entry(&wait_list, struct xfs_buf, b_list); in xfs_buf_delwri_submit()
2022 list_del_init(&bp->b_list); in xfs_buf_delwri_submit()
2028 error2 = xfs_buf_iowait(bp); in xfs_buf_delwri_submit()
2029 xfs_buf_relse(bp); in xfs_buf_delwri_submit()
2054 struct xfs_buf *bp, in xfs_buf_delwri_pushbuf() argument
2060 ASSERT(bp->b_flags & _XBF_DELWRI_Q); in xfs_buf_delwri_pushbuf()
2062 trace_xfs_buf_delwri_pushbuf(bp, _RET_IP_); in xfs_buf_delwri_pushbuf()
2068 xfs_buf_lock(bp); in xfs_buf_delwri_pushbuf()
2069 list_move(&bp->b_list, &submit_list); in xfs_buf_delwri_pushbuf()
2070 xfs_buf_unlock(bp); in xfs_buf_delwri_pushbuf()
2085 error = xfs_buf_iowait(bp); in xfs_buf_delwri_pushbuf()
2086 bp->b_flags |= _XBF_DELWRI_Q; in xfs_buf_delwri_pushbuf()
2087 xfs_buf_unlock(bp); in xfs_buf_delwri_pushbuf()
2112 void xfs_buf_set_ref(struct xfs_buf *bp, int lru_ref) in xfs_buf_set_ref() argument
2119 if (XFS_TEST_ERROR(false, bp->b_target->bt_mount, in xfs_buf_set_ref()
2123 atomic_set(&bp->b_lru_ref, lru_ref); in xfs_buf_set_ref()