Lines Matching refs:bitmap
35 static inline char *bmname(struct bitmap *bitmap) in bmname() argument
37 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX"; in bmname()
50 static int md_bitmap_checkpage(struct bitmap_counts *bitmap, in md_bitmap_checkpage() argument
52 __releases(bitmap->lock) in md_bitmap_checkpage()
53 __acquires(bitmap->lock) in md_bitmap_checkpage()
57 if (page >= bitmap->pages) { in md_bitmap_checkpage()
65 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */ in md_bitmap_checkpage()
68 if (bitmap->bp[page].map) /* page is already allocated, just return */ in md_bitmap_checkpage()
76 spin_unlock_irq(&bitmap->lock); in md_bitmap_checkpage()
91 spin_lock_irq(&bitmap->lock); in md_bitmap_checkpage()
100 if (!bitmap->bp[page].map) in md_bitmap_checkpage()
101 bitmap->bp[page].hijacked = 1; in md_bitmap_checkpage()
102 } else if (bitmap->bp[page].map || in md_bitmap_checkpage()
103 bitmap->bp[page].hijacked) { in md_bitmap_checkpage()
110 bitmap->bp[page].map = mappage; in md_bitmap_checkpage()
111 bitmap->missing_pages--; in md_bitmap_checkpage()
119 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page) in md_bitmap_checkfree() argument
123 if (bitmap->bp[page].count) /* page is still busy */ in md_bitmap_checkfree()
128 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */ in md_bitmap_checkfree()
129 bitmap->bp[page].hijacked = 0; in md_bitmap_checkfree()
130 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
133 ptr = bitmap->bp[page].map; in md_bitmap_checkfree()
134 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
135 bitmap->missing_pages++; in md_bitmap_checkfree()
212 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait) in write_sb_page() argument
216 struct mddev *mddev = bitmap->mddev; in write_sb_page()
217 struct bitmap_storage *store = &bitmap->storage; in write_sb_page()
284 static void md_bitmap_file_kick(struct bitmap *bitmap);
288 static void write_page(struct bitmap *bitmap, struct page *page, int wait) in write_page() argument
292 if (bitmap->storage.file == NULL) { in write_page()
293 switch (write_sb_page(bitmap, page, wait)) { in write_page()
295 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in write_page()
302 atomic_inc(&bitmap->pending_writes); in write_page()
310 wait_event(bitmap->write_wait, in write_page()
311 atomic_read(&bitmap->pending_writes)==0); in write_page()
313 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in write_page()
314 md_bitmap_file_kick(bitmap); in write_page()
319 struct bitmap *bitmap = bh->b_private; in end_bitmap_write() local
322 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in end_bitmap_write()
323 if (atomic_dec_and_test(&bitmap->pending_writes)) in end_bitmap_write()
324 wake_up(&bitmap->write_wait); in end_bitmap_write()
352 struct bitmap *bitmap, in read_page() argument
393 bh->b_private = bitmap; in read_page()
394 atomic_inc(&bitmap->pending_writes); in read_page()
404 wait_event(bitmap->write_wait, in read_page()
405 atomic_read(&bitmap->pending_writes)==0); in read_page()
406 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in read_page()
426 static void md_bitmap_wait_writes(struct bitmap *bitmap) in md_bitmap_wait_writes() argument
428 if (bitmap->storage.file) in md_bitmap_wait_writes()
429 wait_event(bitmap->write_wait, in md_bitmap_wait_writes()
430 atomic_read(&bitmap->pending_writes)==0); in md_bitmap_wait_writes()
439 md_super_wait(bitmap->mddev); in md_bitmap_wait_writes()
444 void md_bitmap_update_sb(struct bitmap *bitmap) in md_bitmap_update_sb() argument
448 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */ in md_bitmap_update_sb()
450 if (bitmap->mddev->bitmap_info.external) in md_bitmap_update_sb()
452 if (!bitmap->storage.sb_page) /* no superblock */ in md_bitmap_update_sb()
454 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_update_sb()
455 sb->events = cpu_to_le64(bitmap->mddev->events); in md_bitmap_update_sb()
456 if (bitmap->mddev->events < bitmap->events_cleared) in md_bitmap_update_sb()
458 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_update_sb()
459 sb->events_cleared = cpu_to_le64(bitmap->events_cleared); in md_bitmap_update_sb()
464 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR)); in md_bitmap_update_sb()
466 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ); in md_bitmap_update_sb()
467 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind); in md_bitmap_update_sb()
469 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_update_sb()
470 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize); in md_bitmap_update_sb()
471 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes); in md_bitmap_update_sb()
472 sb->sectors_reserved = cpu_to_le32(bitmap->mddev-> in md_bitmap_update_sb()
475 write_page(bitmap, bitmap->storage.sb_page, 1); in md_bitmap_update_sb()
480 void md_bitmap_print_sb(struct bitmap *bitmap) in md_bitmap_print_sb() argument
484 if (!bitmap || !bitmap->storage.sb_page) in md_bitmap_print_sb()
486 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_print_sb()
487 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap)); in md_bitmap_print_sb()
519 static int md_bitmap_new_disk_sb(struct bitmap *bitmap) in md_bitmap_new_disk_sb() argument
524 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO); in md_bitmap_new_disk_sb()
525 if (bitmap->storage.sb_page == NULL) in md_bitmap_new_disk_sb()
527 bitmap->storage.sb_page->index = 0; in md_bitmap_new_disk_sb()
529 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_new_disk_sb()
534 chunksize = bitmap->mddev->bitmap_info.chunksize; in md_bitmap_new_disk_sb()
543 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep; in md_bitmap_new_disk_sb()
549 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_new_disk_sb()
555 write_behind = bitmap->mddev->bitmap_info.max_write_behind; in md_bitmap_new_disk_sb()
559 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_new_disk_sb()
562 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_new_disk_sb()
564 memcpy(sb->uuid, bitmap->mddev->uuid, 16); in md_bitmap_new_disk_sb()
566 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_new_disk_sb()
567 sb->state = cpu_to_le32(bitmap->flags); in md_bitmap_new_disk_sb()
568 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_new_disk_sb()
569 sb->events_cleared = cpu_to_le64(bitmap->mddev->events); in md_bitmap_new_disk_sb()
570 bitmap->mddev->bitmap_info.nodes = 0; in md_bitmap_new_disk_sb()
578 static int md_bitmap_read_sb(struct bitmap *bitmap) in md_bitmap_read_sb() argument
588 loff_t offset = bitmap->mddev->bitmap_info.offset; in md_bitmap_read_sb()
590 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) { in md_bitmap_read_sb()
594 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
602 bitmap->storage.sb_page = sb_page; in md_bitmap_read_sb()
606 if (bitmap->cluster_slot >= 0) { in md_bitmap_read_sb()
607 sector_t bm_blocks = bitmap->mddev->resync_max_sectors; in md_bitmap_read_sb()
610 (bitmap->mddev->bitmap_info.chunksize >> 9)); in md_bitmap_read_sb()
615 offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3)); in md_bitmap_read_sb()
617 bitmap->cluster_slot, offset); in md_bitmap_read_sb()
620 if (bitmap->storage.file) { in md_bitmap_read_sb()
621 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host); in md_bitmap_read_sb()
624 err = read_page(bitmap->storage.file, 0, in md_bitmap_read_sb()
625 bitmap, bytes, sb_page); in md_bitmap_read_sb()
627 err = read_sb_page(bitmap->mddev, in md_bitmap_read_sb()
647 strlcpy(bitmap->mddev->bitmap_info.cluster_name, in md_bitmap_read_sb()
667 bmname(bitmap), reason); in md_bitmap_read_sb()
672 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_read_sb()
674 if (bitmap->mddev->persistent) { in md_bitmap_read_sb()
679 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) { in md_bitmap_read_sb()
681 bmname(bitmap)); in md_bitmap_read_sb()
685 if (!nodes && (events < bitmap->mddev->events)) { in md_bitmap_read_sb()
687 bmname(bitmap), events, in md_bitmap_read_sb()
688 (unsigned long long) bitmap->mddev->events); in md_bitmap_read_sb()
689 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
694 bitmap->flags |= le32_to_cpu(sb->state); in md_bitmap_read_sb()
696 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags); in md_bitmap_read_sb()
697 bitmap->events_cleared = le64_to_cpu(sb->events_cleared); in md_bitmap_read_sb()
698 strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64); in md_bitmap_read_sb()
704 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
705 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) { in md_bitmap_read_sb()
706 err = md_setup_cluster(bitmap->mddev, nodes); in md_bitmap_read_sb()
709 bmname(bitmap), err); in md_bitmap_read_sb()
712 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev); in md_bitmap_read_sb()
718 if (test_bit(BITMAP_STALE, &bitmap->flags)) in md_bitmap_read_sb()
719 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_read_sb()
720 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
721 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_read_sb()
722 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_read_sb()
723 bitmap->mddev->bitmap_info.nodes = nodes; in md_bitmap_read_sb()
724 if (bitmap->mddev->bitmap_info.space == 0 || in md_bitmap_read_sb()
725 bitmap->mddev->bitmap_info.space > sectors_reserved) in md_bitmap_read_sb()
726 bitmap->mddev->bitmap_info.space = sectors_reserved; in md_bitmap_read_sb()
728 md_bitmap_print_sb(bitmap); in md_bitmap_read_sb()
729 if (bitmap->cluster_slot < 0) in md_bitmap_read_sb()
730 md_cluster_stop(bitmap->mddev); in md_bitmap_read_sb()
863 static void md_bitmap_file_kick(struct bitmap *bitmap) in md_bitmap_file_kick() argument
867 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) { in md_bitmap_file_kick()
868 md_bitmap_update_sb(bitmap); in md_bitmap_file_kick()
870 if (bitmap->storage.file) { in md_bitmap_file_kick()
873 ptr = file_path(bitmap->storage.file, in md_bitmap_file_kick()
877 bmname(bitmap), IS_ERR(ptr) ? "" : ptr); in md_bitmap_file_kick()
882 bmname(bitmap)); in md_bitmap_file_kick()
893 static inline void set_page_attr(struct bitmap *bitmap, int pnum, in set_page_attr() argument
896 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in set_page_attr()
899 static inline void clear_page_attr(struct bitmap *bitmap, int pnum, in clear_page_attr() argument
902 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in clear_page_attr()
905 static inline int test_page_attr(struct bitmap *bitmap, int pnum, in test_page_attr() argument
908 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in test_page_attr()
911 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum, in test_and_clear_page_attr() argument
915 bitmap->storage.filemap_attr); in test_and_clear_page_attr()
924 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_set_bit() argument
929 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_set_bit()
930 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_set_bit()
933 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_set_bit()
934 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_set_bit()
936 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
939 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
943 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_set_bit()
950 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_DIRTY); in md_bitmap_file_set_bit()
953 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_clear_bit() argument
958 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_clear_bit()
959 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_clear_bit()
962 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_clear_bit()
963 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_clear_bit()
965 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
968 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
970 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_clear_bit()
975 if (!test_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_NEEDWRITE)) { in md_bitmap_file_clear_bit()
976 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_PENDING); in md_bitmap_file_clear_bit()
977 bitmap->allclean = 0; in md_bitmap_file_clear_bit()
981 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_test_bit() argument
986 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_test_bit()
989 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
992 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
994 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_test_bit()
1006 void md_bitmap_unplug(struct bitmap *bitmap) in md_bitmap_unplug() argument
1012 if (!bitmap || !bitmap->storage.filemap || in md_bitmap_unplug()
1013 test_bit(BITMAP_STALE, &bitmap->flags)) in md_bitmap_unplug()
1018 for (i = 0; i < bitmap->storage.file_pages; i++) { in md_bitmap_unplug()
1019 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in md_bitmap_unplug()
1020 need_write = test_and_clear_page_attr(bitmap, i, in md_bitmap_unplug()
1024 md_bitmap_wait_writes(bitmap); in md_bitmap_unplug()
1025 if (bitmap->mddev->queue) in md_bitmap_unplug()
1026 blk_add_trace_msg(bitmap->mddev->queue, in md_bitmap_unplug()
1029 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING); in md_bitmap_unplug()
1030 write_page(bitmap, bitmap->storage.filemap[i], 0); in md_bitmap_unplug()
1035 md_bitmap_wait_writes(bitmap); in md_bitmap_unplug()
1037 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in md_bitmap_unplug()
1038 md_bitmap_file_kick(bitmap); in md_bitmap_unplug()
1042 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1054 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) in md_bitmap_init_from_disk() argument
1064 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_init_from_disk()
1066 chunks = bitmap->counts.chunks; in md_bitmap_init_from_disk()
1069 if (!file && !bitmap->mddev->bitmap_info.offset) { in md_bitmap_init_from_disk()
1075 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift) in md_bitmap_init_from_disk()
1077 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1078 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1084 outofdate = test_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_init_from_disk()
1086 pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap)); in md_bitmap_init_from_disk()
1090 bmname(bitmap), in md_bitmap_init_from_disk()
1098 if (!bitmap->mddev->bitmap_info.external) in md_bitmap_init_from_disk()
1101 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_init_from_disk()
1102 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE)); in md_bitmap_init_from_disk()
1106 index = file_page_index(&bitmap->storage, i); in md_bitmap_init_from_disk()
1107 bit = file_page_offset(&bitmap->storage, i); in md_bitmap_init_from_disk()
1117 ret = read_page(file, index, bitmap, in md_bitmap_init_from_disk()
1121 bitmap->mddev, in md_bitmap_init_from_disk()
1122 bitmap->mddev->bitmap_info.offset, in md_bitmap_init_from_disk()
1140 write_page(bitmap, page, 1); in md_bitmap_init_from_disk()
1144 &bitmap->flags)) in md_bitmap_init_from_disk()
1149 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_init_from_disk()
1156 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift in md_bitmap_init_from_disk()
1158 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1159 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1167 bmname(bitmap), store->file_pages, in md_bitmap_init_from_disk()
1174 bmname(bitmap), ret); in md_bitmap_init_from_disk()
1178 void md_bitmap_write_all(struct bitmap *bitmap) in md_bitmap_write_all() argument
1185 if (!bitmap || !bitmap->storage.filemap) in md_bitmap_write_all()
1187 if (bitmap->storage.file) in md_bitmap_write_all()
1191 for (i = 0; i < bitmap->storage.file_pages; i++) in md_bitmap_write_all()
1192 set_page_attr(bitmap, i, in md_bitmap_write_all()
1194 bitmap->allclean = 0; in md_bitmap_write_all()
1197 static void md_bitmap_count_page(struct bitmap_counts *bitmap, in md_bitmap_count_page() argument
1200 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_count_page()
1202 bitmap->bp[page].count += inc; in md_bitmap_count_page()
1203 md_bitmap_checkfree(bitmap, page); in md_bitmap_count_page()
1206 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset) in md_bitmap_set_pending() argument
1208 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_set_pending()
1210 struct bitmap_page *bp = &bitmap->bp[page]; in md_bitmap_set_pending()
1216 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1227 struct bitmap *bitmap; in md_bitmap_daemon_work() local
1237 bitmap = mddev->bitmap; in md_bitmap_daemon_work()
1238 if (bitmap == NULL) { in md_bitmap_daemon_work()
1242 if (time_before(jiffies, bitmap->daemon_lastrun in md_bitmap_daemon_work()
1246 bitmap->daemon_lastrun = jiffies; in md_bitmap_daemon_work()
1247 if (bitmap->allclean) { in md_bitmap_daemon_work()
1251 bitmap->allclean = 1; in md_bitmap_daemon_work()
1253 if (bitmap->mddev->queue) in md_bitmap_daemon_work()
1254 blk_add_trace_msg(bitmap->mddev->queue, in md_bitmap_daemon_work()
1261 for (j = 0; j < bitmap->storage.file_pages; j++) in md_bitmap_daemon_work()
1262 if (test_and_clear_page_attr(bitmap, j, in md_bitmap_daemon_work()
1264 set_page_attr(bitmap, j, in md_bitmap_daemon_work()
1267 if (bitmap->need_sync && in md_bitmap_daemon_work()
1272 bitmap->need_sync = 0; in md_bitmap_daemon_work()
1273 if (bitmap->storage.filemap) { in md_bitmap_daemon_work()
1274 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_daemon_work()
1276 cpu_to_le64(bitmap->events_cleared); in md_bitmap_daemon_work()
1278 set_page_attr(bitmap, 0, in md_bitmap_daemon_work()
1285 counts = &bitmap->counts; in md_bitmap_daemon_work()
1306 if (*bmc == 1 && !bitmap->need_sync) { in md_bitmap_daemon_work()
1310 md_bitmap_file_clear_bit(bitmap, block); in md_bitmap_daemon_work()
1314 bitmap->allclean = 0; in md_bitmap_daemon_work()
1319 md_bitmap_wait_writes(bitmap); in md_bitmap_daemon_work()
1329 j < bitmap->storage.file_pages in md_bitmap_daemon_work()
1330 && !test_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_daemon_work()
1332 if (test_page_attr(bitmap, j, in md_bitmap_daemon_work()
1336 if (bitmap->storage.filemap && in md_bitmap_daemon_work()
1337 test_and_clear_page_attr(bitmap, j, in md_bitmap_daemon_work()
1339 write_page(bitmap, bitmap->storage.filemap[j], 0); in md_bitmap_daemon_work()
1344 if (bitmap->allclean == 0) in md_bitmap_daemon_work()
1350 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap, in md_bitmap_get_counter() argument
1353 __releases(bitmap->lock) in md_bitmap_get_counter()
1354 __acquires(bitmap->lock) in md_bitmap_get_counter()
1360 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_get_counter()
1366 err = md_bitmap_checkpage(bitmap, page, create, 0); in md_bitmap_get_counter()
1368 if (bitmap->bp[page].hijacked || in md_bitmap_get_counter()
1369 bitmap->bp[page].map == NULL) in md_bitmap_get_counter()
1370 csize = ((sector_t)1) << (bitmap->chunkshift + in md_bitmap_get_counter()
1373 csize = ((sector_t)1) << bitmap->chunkshift; in md_bitmap_get_counter()
1381 if (bitmap->bp[page].hijacked) { /* hijacked pointer */ in md_bitmap_get_counter()
1386 &bitmap->bp[page].map)[hi]; in md_bitmap_get_counter()
1389 &(bitmap->bp[page].map[pageoff]); in md_bitmap_get_counter()
1392 int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind) in md_bitmap_startwrite() argument
1394 if (!bitmap) in md_bitmap_startwrite()
1399 atomic_inc(&bitmap->behind_writes); in md_bitmap_startwrite()
1400 bw = atomic_read(&bitmap->behind_writes); in md_bitmap_startwrite()
1401 if (bw > bitmap->behind_writes_used) in md_bitmap_startwrite()
1402 bitmap->behind_writes_used = bw; in md_bitmap_startwrite()
1405 bw, bitmap->mddev->bitmap_info.max_write_behind); in md_bitmap_startwrite()
1412 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1413 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1); in md_bitmap_startwrite()
1415 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1425 prepare_to_wait(&bitmap->overflow_wait, &__wait, in md_bitmap_startwrite()
1427 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1429 finish_wait(&bitmap->overflow_wait, &__wait); in md_bitmap_startwrite()
1435 md_bitmap_file_set_bit(bitmap, offset); in md_bitmap_startwrite()
1436 md_bitmap_count_page(&bitmap->counts, offset, 1); in md_bitmap_startwrite()
1444 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1456 void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset, in md_bitmap_endwrite() argument
1459 if (!bitmap) in md_bitmap_endwrite()
1462 if (atomic_dec_and_test(&bitmap->behind_writes)) in md_bitmap_endwrite()
1463 wake_up(&bitmap->behind_wait); in md_bitmap_endwrite()
1465 atomic_read(&bitmap->behind_writes), in md_bitmap_endwrite()
1466 bitmap->mddev->bitmap_info.max_write_behind); in md_bitmap_endwrite()
1474 spin_lock_irqsave(&bitmap->counts.lock, flags); in md_bitmap_endwrite()
1475 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0); in md_bitmap_endwrite()
1477 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in md_bitmap_endwrite()
1481 if (success && !bitmap->mddev->degraded && in md_bitmap_endwrite()
1482 bitmap->events_cleared < bitmap->mddev->events) { in md_bitmap_endwrite()
1483 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_endwrite()
1484 bitmap->need_sync = 1; in md_bitmap_endwrite()
1485 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear); in md_bitmap_endwrite()
1492 wake_up(&bitmap->overflow_wait); in md_bitmap_endwrite()
1496 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_endwrite()
1497 bitmap->allclean = 0; in md_bitmap_endwrite()
1499 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in md_bitmap_endwrite()
1509 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, in __bitmap_start_sync() argument
1514 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */ in __bitmap_start_sync()
1518 spin_lock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1519 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in __bitmap_start_sync()
1533 spin_unlock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1537 int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, in md_bitmap_start_sync() argument
1552 rv |= __bitmap_start_sync(bitmap, offset, in md_bitmap_start_sync()
1561 void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted) in md_bitmap_end_sync() argument
1566 if (bitmap == NULL) { in md_bitmap_end_sync()
1570 spin_lock_irqsave(&bitmap->counts.lock, flags); in md_bitmap_end_sync()
1571 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in md_bitmap_end_sync()
1582 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_end_sync()
1583 bitmap->allclean = 0; in md_bitmap_end_sync()
1588 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in md_bitmap_end_sync()
1592 void md_bitmap_close_sync(struct bitmap *bitmap) in md_bitmap_close_sync() argument
1600 if (!bitmap) in md_bitmap_close_sync()
1602 while (sector < bitmap->mddev->resync_max_sectors) { in md_bitmap_close_sync()
1603 md_bitmap_end_sync(bitmap, sector, &blocks, 0); in md_bitmap_close_sync()
1609 void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force) in md_bitmap_cond_end_sync() argument
1614 if (!bitmap) in md_bitmap_cond_end_sync()
1617 bitmap->last_end_sync = jiffies; in md_bitmap_cond_end_sync()
1620 if (!force && time_before(jiffies, (bitmap->last_end_sync in md_bitmap_cond_end_sync()
1621 + bitmap->mddev->bitmap_info.daemon_sleep))) in md_bitmap_cond_end_sync()
1623 wait_event(bitmap->mddev->recovery_wait, in md_bitmap_cond_end_sync()
1624 atomic_read(&bitmap->mddev->recovery_active) == 0); in md_bitmap_cond_end_sync()
1626 bitmap->mddev->curr_resync_completed = sector; in md_bitmap_cond_end_sync()
1627 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags); in md_bitmap_cond_end_sync()
1628 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1); in md_bitmap_cond_end_sync()
1630 while (s < sector && s < bitmap->mddev->resync_max_sectors) { in md_bitmap_cond_end_sync()
1631 md_bitmap_end_sync(bitmap, s, &blocks, 0); in md_bitmap_cond_end_sync()
1634 bitmap->last_end_sync = jiffies; in md_bitmap_cond_end_sync()
1635 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed); in md_bitmap_cond_end_sync()
1643 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_sync_with_cluster() local
1647 md_bitmap_end_sync(bitmap, sector, &blocks, 0); in md_bitmap_sync_with_cluster()
1653 md_bitmap_start_sync(bitmap, sector, &blocks, 0); in md_bitmap_sync_with_cluster()
1660 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed) in md_bitmap_set_memory_bits() argument
1669 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1670 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1); in md_bitmap_set_memory_bits()
1672 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1677 md_bitmap_count_page(&bitmap->counts, offset, 1); in md_bitmap_set_memory_bits()
1678 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_set_memory_bits()
1679 bitmap->allclean = 0; in md_bitmap_set_memory_bits()
1683 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1687 void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e) in md_bitmap_dirty_bits() argument
1692 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift; in md_bitmap_dirty_bits()
1693 md_bitmap_set_memory_bits(bitmap, sec, 1); in md_bitmap_dirty_bits()
1694 md_bitmap_file_set_bit(bitmap, sec); in md_bitmap_dirty_bits()
1695 if (sec < bitmap->mddev->recovery_cp) in md_bitmap_dirty_bits()
1700 bitmap->mddev->recovery_cp = sec; in md_bitmap_dirty_bits()
1709 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_flush() local
1712 if (!bitmap) /* there was no bitmap */ in md_bitmap_flush()
1719 bitmap->daemon_lastrun -= sleep; in md_bitmap_flush()
1721 bitmap->daemon_lastrun -= sleep; in md_bitmap_flush()
1723 bitmap->daemon_lastrun -= sleep; in md_bitmap_flush()
1725 md_bitmap_update_sb(bitmap); in md_bitmap_flush()
1731 void md_bitmap_free(struct bitmap *bitmap) in md_bitmap_free() argument
1736 if (!bitmap) /* there was no bitmap */ in md_bitmap_free()
1739 if (bitmap->sysfs_can_clear) in md_bitmap_free()
1740 sysfs_put(bitmap->sysfs_can_clear); in md_bitmap_free()
1742 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info && in md_bitmap_free()
1743 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev)) in md_bitmap_free()
1744 md_cluster_stop(bitmap->mddev); in md_bitmap_free()
1747 wait_event(bitmap->write_wait, in md_bitmap_free()
1748 atomic_read(&bitmap->pending_writes) == 0); in md_bitmap_free()
1751 md_bitmap_file_unmap(&bitmap->storage); in md_bitmap_free()
1753 bp = bitmap->counts.bp; in md_bitmap_free()
1754 pages = bitmap->counts.pages; in md_bitmap_free()
1763 kfree(bitmap); in md_bitmap_free()
1769 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_wait_behind_writes() local
1772 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { in md_bitmap_wait_behind_writes()
1776 wait_event(bitmap->behind_wait, in md_bitmap_wait_behind_writes()
1777 atomic_read(&bitmap->behind_writes) == 0); in md_bitmap_wait_behind_writes()
1783 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_destroy() local
1785 if (!bitmap) /* there was no bitmap */ in md_bitmap_destroy()
1794 mddev->bitmap = NULL; /* disconnect from the md device */ in md_bitmap_destroy()
1800 md_bitmap_free(bitmap); in md_bitmap_destroy()
1808 struct bitmap *md_bitmap_create(struct mddev *mddev, int slot) in md_bitmap_create()
1810 struct bitmap *bitmap; in md_bitmap_create() local
1826 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); in md_bitmap_create()
1827 if (!bitmap) in md_bitmap_create()
1830 spin_lock_init(&bitmap->counts.lock); in md_bitmap_create()
1831 atomic_set(&bitmap->pending_writes, 0); in md_bitmap_create()
1832 init_waitqueue_head(&bitmap->write_wait); in md_bitmap_create()
1833 init_waitqueue_head(&bitmap->overflow_wait); in md_bitmap_create()
1834 init_waitqueue_head(&bitmap->behind_wait); in md_bitmap_create()
1836 bitmap->mddev = mddev; in md_bitmap_create()
1837 bitmap->cluster_slot = slot; in md_bitmap_create()
1842 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear"); in md_bitmap_create()
1845 bitmap->sysfs_can_clear = NULL; in md_bitmap_create()
1847 bitmap->storage.file = file; in md_bitmap_create()
1863 err = md_bitmap_new_disk_sb(bitmap); in md_bitmap_create()
1865 err = md_bitmap_read_sb(bitmap); in md_bitmap_create()
1877 bitmap->daemon_lastrun = jiffies; in md_bitmap_create()
1878 err = md_bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1); in md_bitmap_create()
1883 bitmap->counts.pages, bmname(bitmap)); in md_bitmap_create()
1885 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0; in md_bitmap_create()
1889 return bitmap; in md_bitmap_create()
1891 md_bitmap_free(bitmap); in md_bitmap_create()
1900 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_load() local
1903 if (!bitmap) in md_bitmap_load()
1919 md_bitmap_start_sync(bitmap, sector, &blocks, 0); in md_bitmap_load()
1922 md_bitmap_close_sync(bitmap); in md_bitmap_load()
1925 || bitmap->events_cleared == mddev->events) in md_bitmap_load()
1931 err = md_bitmap_init_from_disk(bitmap, start); in md_bitmap_load()
1936 clear_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_load()
1939 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery); in md_bitmap_load()
1944 md_bitmap_update_sb(bitmap); in md_bitmap_load()
1946 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in md_bitmap_load()
1954 struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot) in get_bitmap_from_slot()
1957 struct bitmap *bitmap; in get_bitmap_from_slot() local
1959 bitmap = md_bitmap_create(mddev, slot); in get_bitmap_from_slot()
1960 if (IS_ERR(bitmap)) { in get_bitmap_from_slot()
1961 rv = PTR_ERR(bitmap); in get_bitmap_from_slot()
1965 rv = md_bitmap_init_from_disk(bitmap, 0); in get_bitmap_from_slot()
1967 md_bitmap_free(bitmap); in get_bitmap_from_slot()
1971 return bitmap; in get_bitmap_from_slot()
1984 struct bitmap *bitmap; in md_bitmap_copy_from_slot() local
1986 bitmap = get_bitmap_from_slot(mddev, slot); in md_bitmap_copy_from_slot()
1987 if (IS_ERR(bitmap)) { in md_bitmap_copy_from_slot()
1992 counts = &bitmap->counts; in md_bitmap_copy_from_slot()
1995 if (md_bitmap_file_test_bit(bitmap, block)) { in md_bitmap_copy_from_slot()
1999 md_bitmap_file_clear_bit(bitmap, block); in md_bitmap_copy_from_slot()
2000 md_bitmap_set_memory_bits(mddev->bitmap, block, 1); in md_bitmap_copy_from_slot()
2001 md_bitmap_file_set_bit(mddev->bitmap, block); in md_bitmap_copy_from_slot()
2006 md_bitmap_update_sb(bitmap); in md_bitmap_copy_from_slot()
2009 for (i = 0; i < bitmap->storage.file_pages; i++) in md_bitmap_copy_from_slot()
2010 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING)) in md_bitmap_copy_from_slot()
2011 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE); in md_bitmap_copy_from_slot()
2012 md_bitmap_unplug(bitmap); in md_bitmap_copy_from_slot()
2014 md_bitmap_unplug(mddev->bitmap); in md_bitmap_copy_from_slot()
2017 md_bitmap_free(bitmap); in md_bitmap_copy_from_slot()
2024 void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap) in md_bitmap_status() argument
2029 if (!bitmap) in md_bitmap_status()
2032 counts = &bitmap->counts; in md_bitmap_status()
2034 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10; in md_bitmap_status()
2041 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize, in md_bitmap_status()
2043 if (bitmap->storage.file) { in md_bitmap_status()
2045 seq_file_path(seq, bitmap->storage.file, " \t\n"); in md_bitmap_status()
2051 int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks, in md_bitmap_resize() argument
2074 if (bitmap->storage.file && !init) { in md_bitmap_resize()
2084 long space = bitmap->mddev->bitmap_info.space; in md_bitmap_resize()
2090 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8); in md_bitmap_resize()
2091 if (!bitmap->mddev->bitmap_info.external) in md_bitmap_resize()
2094 bitmap->mddev->bitmap_info.space = space; in md_bitmap_resize()
2096 chunkshift = bitmap->counts.chunkshift; in md_bitmap_resize()
2103 if (!bitmap->mddev->bitmap_info.external) in md_bitmap_resize()
2111 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file) in md_bitmap_resize()
2113 !bitmap->mddev->bitmap_info.external, in md_bitmap_resize()
2114 mddev_is_clustered(bitmap->mddev) in md_bitmap_resize()
2115 ? bitmap->cluster_slot : 0); in md_bitmap_resize()
2131 bitmap->mddev->pers->quiesce(bitmap->mddev, 1); in md_bitmap_resize()
2133 store.file = bitmap->storage.file; in md_bitmap_resize()
2134 bitmap->storage.file = NULL; in md_bitmap_resize()
2136 if (store.sb_page && bitmap->storage.sb_page) in md_bitmap_resize()
2138 page_address(bitmap->storage.sb_page), in md_bitmap_resize()
2140 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_resize()
2141 md_bitmap_file_unmap(&bitmap->storage); in md_bitmap_resize()
2142 bitmap->storage = store; in md_bitmap_resize()
2144 old_counts = bitmap->counts; in md_bitmap_resize()
2145 bitmap->counts.bp = new_bp; in md_bitmap_resize()
2146 bitmap->counts.pages = pages; in md_bitmap_resize()
2147 bitmap->counts.missing_pages = pages; in md_bitmap_resize()
2148 bitmap->counts.chunkshift = chunkshift; in md_bitmap_resize()
2149 bitmap->counts.chunks = chunks; in md_bitmap_resize()
2150 bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift + in md_bitmap_resize()
2157 if (mddev_is_clustered(bitmap->mddev)) { in md_bitmap_resize()
2160 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1); in md_bitmap_resize()
2171 bitmap->counts.bp = old_counts.bp; in md_bitmap_resize()
2172 bitmap->counts.pages = old_counts.pages; in md_bitmap_resize()
2173 bitmap->counts.missing_pages = old_counts.pages; in md_bitmap_resize()
2174 bitmap->counts.chunkshift = old_counts.chunkshift; in md_bitmap_resize()
2175 bitmap->counts.chunks = old_counts.chunks; in md_bitmap_resize()
2176 bitmap->mddev->bitmap_info.chunksize = 1 << (old_counts.chunkshift + in md_bitmap_resize()
2182 bitmap->counts.bp[page].count += 1; in md_bitmap_resize()
2194 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in md_bitmap_resize()
2201 md_bitmap_file_set_bit(bitmap, block); in md_bitmap_resize()
2205 md_bitmap_count_page(&bitmap->counts, block, 1); in md_bitmap_resize()
2206 md_bitmap_set_pending(&bitmap->counts, block); in md_bitmap_resize()
2215 if (bitmap->counts.bp != old_counts.bp) { in md_bitmap_resize()
2227 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in md_bitmap_resize()
2234 md_bitmap_count_page(&bitmap->counts, block, 1); in md_bitmap_resize()
2235 md_bitmap_set_pending(&bitmap->counts, block); in md_bitmap_resize()
2240 for (i = 0; i < bitmap->storage.file_pages; i++) in md_bitmap_resize()
2241 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in md_bitmap_resize()
2243 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_resize()
2246 md_bitmap_unplug(bitmap); in md_bitmap_resize()
2247 bitmap->mddev->pers->quiesce(bitmap->mddev, 0); in md_bitmap_resize()
2288 if (mddev->bitmap || mddev->bitmap_info.file || in location_store()
2334 struct bitmap *bitmap; in location_store() local
2335 bitmap = md_bitmap_create(mddev, -1); in location_store()
2337 if (IS_ERR(bitmap)) in location_store()
2338 rv = PTR_ERR(bitmap); in location_store()
2340 mddev->bitmap = bitmap; in location_store()
2395 if (mddev->bitmap && in space_store()
2396 sectors < (mddev->bitmap->storage.bytes + 511) >> 9) in space_store()
2488 md_bitmap_update_sb(mddev->bitmap); in backlog_store()
2507 if (mddev->bitmap) in chunksize_store()
2532 if (mddev->bitmap || in metadata_store()
2553 if (mddev->bitmap) in can_clear_show()
2554 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ? in can_clear_show()
2564 if (mddev->bitmap == NULL) in can_clear_store()
2567 mddev->bitmap->need_sync = 1; in can_clear_store()
2571 mddev->bitmap->need_sync = 0; in can_clear_store()
2585 if (mddev->bitmap == NULL) in behind_writes_used_show()
2589 mddev->bitmap->behind_writes_used); in behind_writes_used_show()
2597 if (mddev->bitmap) in behind_writes_used_reset()
2598 mddev->bitmap->behind_writes_used = 0; in behind_writes_used_reset()