15 #include <linux/bitmap.h>
41  * 1 << (38 - 3)	bitmap bytes needed
43  *			(that's 32 GiB of bitmap for 1 PiB storage)
52  *	22 --> we need that much 4KiB pages of bitmap.
61  * bitmap storage and IO:
62  *	Bitmap is stored little endian on disk, and is kept little endian in
63  *	core memory. Currently we still hold the full bitmap in core as long
67  *	We plan to reduce the amount of in-core bitmap pages by paging them in
115 	struct drbd_bitmap *b = device->bitmap;  in __bm_print_lock_info()
118 	drbd_err(device, "FIXME %s[%d] in %s, bitmap locked for '%s' by %s[%d]\n",  in __bm_print_lock_info()
126 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_lock()
130 		drbd_err(device, "FIXME no bitmap in drbd_bm_lock!?\n");  in drbd_bm_lock()
137 		drbd_warn(device, "%s[%d] going to '%s' but bitmap already locked for '%s' by %s[%d]\n",  in drbd_bm_lock()
144 		drbd_err(device, "FIXME bitmap already locked in bm_lock\n");  in drbd_bm_lock()
153 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_unlock()
155 		drbd_err(device, "FIXME no bitmap in drbd_bm_unlock!?\n");  in drbd_bm_unlock()
159 	if (!(BM_LOCKED_MASK & device->bitmap->bm_flags))  in drbd_bm_unlock()
160 		drbd_err(device, "FIXME bitmap not locked in bm_unlock\n");  in drbd_bm_unlock()
169 /* at a granularity of 4k storage per bitmap bit:
172  *  1<<23 4k bitmap pages.
194  * use atomic bit manipulation, as set_out_of_sync (and therefore bitmap
209  * context, we can get away with a bit per page and one wait queue per bitmap.
213 	struct drbd_bitmap *b = device->bitmap;  in bm_page_lock_io()
220 	struct drbd_bitmap *b = device->bitmap;  in bm_page_unlock_io()
223 	wake_up(&device->bitmap->bm_io_wait);  in bm_page_unlock_io()
242 	device->bitmap->n_bitmap_hints = 0;  in drbd_bm_reset_al_hints()
248  * @page_nr:	the bitmap page to mark with the "hint" flag
256 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_mark_for_writeout()
258 	if (page_nr >= device->bitmap->bm_number_of_pages) {  in drbd_bm_mark_for_writeout()
260 			 page_nr, (int)device->bitmap->bm_number_of_pages);  in drbd_bm_mark_for_writeout()
263 	page = device->bitmap->bm_pages[page_nr];  in drbd_bm_mark_for_writeout()
335 /* word offset from start of bitmap to word number _in_page_
431  * allocates the drbd_bitmap and stores it in device->bitmap.
435 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_init()
444 	device->bitmap = b;  in drbd_bm_init()
451 	if (!expect(device->bitmap))  in drbd_bm_capacity()
453 	return device->bitmap->bm_dev_capacity;  in drbd_bm_capacity()
460 	if (!expect(device->bitmap))  in drbd_bm_cleanup()
462 	bm_free_pages(device->bitmap->bm_pages, device->bitmap->bm_number_of_pages);  in drbd_bm_cleanup()
463 	bm_vk_free(device->bitmap->bm_pages);  in drbd_bm_cleanup()
464 	kfree(device->bitmap);  in drbd_bm_cleanup()
465 	device->bitmap = NULL;  in drbd_bm_cleanup()
493 	/* bitmap is always stored little endian,  in bm_clear_surplus()
529 	/* bitmap is always stored little endian,  in bm_set_surplus()
552 /* you better not modify the bitmap while this is running,
623  * make sure the bitmap has enough room for the attached storage,
627  * In case this is actually a resize, we copy the old bitmap into the new one.
628  * Otherwise, the bitmap is initialized to all bits set.
632 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_resize()
739 	drbd_info(device, "resync bitmap: bits=%lu words=%lu pages=%lu\n", bits, words, want);  in drbd_bm_resize()
756 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_total_weight()
785 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_words()
796 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_bits()
803 /* merge number words from buffer into the bitmap starting at offset.
805  * bitmap must be locked by drbd_bm_lock.
811 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_merge_lel()
854 /* copy number words from the bitmap starting at offset into the buffer.
860 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_get_lel()
893 /* set all bits in the bitmap */
896 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_set_all()
909 /* clear all bits in the bitmap */
912 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_clear_all()
941 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_endio()
946 		drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx);  in drbd_bm_endio()
956 			drbd_err(device, "IO ERROR %d on bitmap page idx %u\n",  in drbd_bm_endio()
960 		dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx);  in drbd_bm_endio()
994 	struct drbd_bitmap *b = device->bitmap;  in bm_page_io_async()
1017 			drbd_err(device, "Invalid offset during on-disk bitmap access: "  in bm_page_io_async()
1063  * bm_rw: read/write the whole bitmap from/to its on disk location.
1068 	struct drbd_bitmap *b = device->bitmap;  in bm_rw()
1075 	 * We are protected against bitmap disappearing/resizing by holding an  in bm_rw()
1077 	 * For read/write, we are protected against changes to the bitmap by  in bm_rw()
1078 	 * the bitmap lock (see drbd_bitmap_io).  in bm_rw()
1079 	 * For lazy writeout, we don't care for ongoing changes to the bitmap,  in bm_rw()
1180 	/* summary for global bitmap IO */  in bm_rw()
1184 			drbd_info(device, "bitmap %s of %u pages took %u ms\n",  in bm_rw()
1191 		drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n");  in bm_rw()
1216  * drbd_bm_read() - Read the whole bitmap from its on disk location.
1225  * drbd_bm_write() - Write the whole bitmap to its on disk location.
1236  * drbd_bm_write_all() - Write the whole bitmap to its on disk location.
1247  * drbd_bm_write_lazy() - Write bitmap pages 0 to @upper_idx-1, if they have changed.
1257  * drbd_bm_write_copy_pages() - Write the whole bitmap to its on disk location.
1261  * In contrast to drbd_bm_write(), this will copy the bitmap pages
1263  * while still allowing the bitmap to change, for example if a resync or online
1273  * drbd_bm_write_hinted() - Write bitmap pages with "hint" marks, if they have changed.
1292 	struct drbd_bitmap *b = device->bitmap;  in __bm_find_next()
1332 	struct drbd_bitmap *b = device->bitmap;  in bm_find_next()
1382  * Must hold bitmap lock already. */
1386 	struct drbd_bitmap *b = device->bitmap;  in __bm_change_bits_to()
1436 	struct drbd_bitmap *b = device->bitmap;  in bm_change_bits_to()
1486 		 * remote bitmap as well, and is reconstructed during the next  in bm_set_full_words_within_one_page()
1487 		 * bitmap exchange, if lost locally due to a crash. */  in bm_set_full_words_within_one_page()
1496  * Can be called to set the whole bitmap in one go.
1508 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_set_bits()
1543 		bm_set_full_words_within_one_page(device->bitmap, page_nr, first_word, last_word);  in _drbd_bm_set_bits()
1552 	/* consider bitmap->bm_bits = 32768, bitmap->bm_number_of_pages = 1. (or multiples).  in _drbd_bm_set_bits()
1556 	 * as we did not allocate it, it is not present in bitmap->bm_pages.  in _drbd_bm_set_bits()
1559 		bm_set_full_words_within_one_page(device->bitmap, last_page, first_word, last_word);  in _drbd_bm_set_bits()
1581 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_test_bit()
1612 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_count_bits()
1618 	/* If this is called without a bitmap, that is a bug.  But just to be  in drbd_bm_count_bits()
1621 	 * local read there (no bitmap probably implies no disk) */  in drbd_bm_count_bits()
1654  * value is zero, or we already "locked" this "bitmap extent" by other means.
1657  * worth of the bitmap a "bitmap extent".
1661  * reference count of some bitmap extent element from some lru instead...
1666 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_e_weight()