115 	struct drbd_bitmap *b = device->bitmap;  in __bm_print_lock_info()  local
120 		 func, b->bm_why ?: "?",  in __bm_print_lock_info()
121 		 b->bm_task->comm, task_pid_nr(b->bm_task));  in __bm_print_lock_info()
126 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_lock()  local
129 	if (!b) {  in drbd_bm_lock()
134 	trylock_failed = !mutex_trylock(&b->bm_change);  in drbd_bm_lock()
139 			  why, b->bm_why ?: "?",  in drbd_bm_lock()
140 			  b->bm_task->comm, task_pid_nr(b->bm_task));  in drbd_bm_lock()
141 		mutex_lock(&b->bm_change);  in drbd_bm_lock()
143 	if (BM_LOCKED_MASK & b->bm_flags)  in drbd_bm_lock()
145 	b->bm_flags |= flags & BM_LOCKED_MASK;  in drbd_bm_lock()
147 	b->bm_why  = why;  in drbd_bm_lock()
148 	b->bm_task = current;  in drbd_bm_lock()
153 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_unlock()  local
154 	if (!b) {  in drbd_bm_unlock()
162 	b->bm_flags &= ~BM_LOCKED_MASK;  in drbd_bm_unlock()
163 	b->bm_why  = NULL;  in drbd_bm_unlock()
164 	b->bm_task = NULL;  in drbd_bm_unlock()
165 	mutex_unlock(&b->bm_change);  in drbd_bm_unlock()
213 	struct drbd_bitmap *b = device->bitmap;  in bm_page_lock_io()  local
214 	void *addr = &page_private(b->bm_pages[page_nr]);  in bm_page_lock_io()
215 	wait_event(b->bm_io_wait, !test_and_set_bit(BM_PAGE_IO_LOCK, addr));  in bm_page_lock_io()
220 	struct drbd_bitmap *b = device->bitmap;  in bm_page_unlock_io()  local
221 	void *addr = &page_private(b->bm_pages[page_nr]);  in bm_page_unlock_io()
256 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_mark_for_writeout()  local
264 	BUG_ON(b->n_bitmap_hints >= ARRAY_SIZE(b->al_bitmap_hints));  in drbd_bm_mark_for_writeout()
266 		b->al_bitmap_hints[b->n_bitmap_hints++] = page_nr;  in drbd_bm_mark_for_writeout()
296 static unsigned int bm_word_to_page_idx(struct drbd_bitmap *b, unsigned long long_nr)  in bm_word_to_page_idx()  argument
300 	BUG_ON(page_nr >= b->bm_number_of_pages);  in bm_word_to_page_idx()
304 static unsigned int bm_bit_to_page_idx(struct drbd_bitmap *b, u64 bitnr)  in bm_bit_to_page_idx()  argument
308 	BUG_ON(page_nr >= b->bm_number_of_pages);  in bm_bit_to_page_idx()
312 static unsigned long *__bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)  in __bm_map_pidx()  argument
314 	struct page *page = b->bm_pages[idx];  in __bm_map_pidx()
318 static unsigned long *bm_map_pidx(struct drbd_bitmap *b, unsigned int idx)  in bm_map_pidx()  argument
320 	return __bm_map_pidx(b, idx);  in bm_map_pidx()
378 static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)  in bm_realloc_pages()  argument
380 	struct page **old_pages = b->bm_pages;  in bm_realloc_pages()
383 	unsigned long have = b->bm_number_of_pages;  in bm_realloc_pages()
437 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_init()  local
438 	WARN_ON(b != NULL);  in drbd_bm_init()
439 	b = kzalloc(sizeof(struct drbd_bitmap), GFP_KERNEL);  in drbd_bm_init()
440 	if (!b)  in drbd_bm_init()
442 	spin_lock_init(&b->bm_lock);  in drbd_bm_init()
443 	mutex_init(&b->bm_change);  in drbd_bm_init()
444 	init_waitqueue_head(&b->bm_io_wait);  in drbd_bm_init()
446 	device->bitmap = b;  in drbd_bm_init()
484 static int bm_clear_surplus(struct drbd_bitmap *b)  in bm_clear_surplus()  argument
492 	tmp = (b->bm_bits & BITS_PER_PAGE_MASK);  in bm_clear_surplus()
499 	p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);  in bm_clear_surplus()
521 static void bm_set_surplus(struct drbd_bitmap *b)  in bm_set_surplus()  argument
528 	tmp = (b->bm_bits & BITS_PER_PAGE_MASK);  in bm_set_surplus()
535 	p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);  in bm_set_surplus()
556 static unsigned long bm_count_bits(struct drbd_bitmap *b)  in bm_count_bits()  argument
560 	unsigned long mask = (1UL << (b->bm_bits & BITS_PER_LONG_MASK)) -1;  in bm_count_bits()
564 	for (idx = 0; idx < b->bm_number_of_pages - 1; idx++) {  in bm_count_bits()
565 		p_addr = __bm_map_pidx(b, idx);  in bm_count_bits()
571 	last_word = ((b->bm_bits - 1) & BITS_PER_PAGE_MASK) >> LN2_BPL;  in bm_count_bits()
572 	p_addr = __bm_map_pidx(b, idx);  in bm_count_bits()
584 static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len)  in bm_memset()  argument
592 	if (end > b->bm_words) {  in bm_memset()
599 		idx = bm_word_to_page_idx(b, offset);  in bm_memset()
600 		p_addr = bm_map_pidx(b, idx);  in bm_memset()
608 		bm_set_page_need_writeout(b->bm_pages[idx]);  in bm_memset()
634 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_resize()  local
641 	if (!expect(b))  in drbd_bm_resize()
649 	if (capacity == b->bm_dev_capacity)  in drbd_bm_resize()
653 		spin_lock_irq(&b->bm_lock);  in drbd_bm_resize()
654 		opages = b->bm_pages;  in drbd_bm_resize()
655 		onpages = b->bm_number_of_pages;  in drbd_bm_resize()
656 		owords = b->bm_words;  in drbd_bm_resize()
657 		b->bm_pages = NULL;  in drbd_bm_resize()
658 		b->bm_number_of_pages =  in drbd_bm_resize()
659 		b->bm_set   =  in drbd_bm_resize()
660 		b->bm_bits  =  in drbd_bm_resize()
661 		b->bm_words =  in drbd_bm_resize()
662 		b->bm_dev_capacity = 0;  in drbd_bm_resize()
663 		spin_unlock_irq(&b->bm_lock);  in drbd_bm_resize()
689 	have = b->bm_number_of_pages;  in drbd_bm_resize()
691 		D_ASSERT(device, b->bm_pages != NULL);  in drbd_bm_resize()
692 		npages = b->bm_pages;  in drbd_bm_resize()
697 			npages = bm_realloc_pages(b, want);  in drbd_bm_resize()
705 	spin_lock_irq(&b->bm_lock);  in drbd_bm_resize()
706 	opages = b->bm_pages;  in drbd_bm_resize()
707 	owords = b->bm_words;  in drbd_bm_resize()
708 	obits  = b->bm_bits;  in drbd_bm_resize()
712 		bm_set_surplus(b);  in drbd_bm_resize()
714 	b->bm_pages = npages;  in drbd_bm_resize()
715 	b->bm_number_of_pages = want;  in drbd_bm_resize()
716 	b->bm_bits  = bits;  in drbd_bm_resize()
717 	b->bm_words = words;  in drbd_bm_resize()
718 	b->bm_dev_capacity = capacity;  in drbd_bm_resize()
722 			bm_memset(b, owords, 0xff, words-owords);  in drbd_bm_resize()
723 			b->bm_set += bits - obits;  in drbd_bm_resize()
725 			bm_memset(b, owords, 0x00, words-owords);  in drbd_bm_resize()
734 	(void)bm_clear_surplus(b);  in drbd_bm_resize()
736 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_resize()
740 		b->bm_set = bm_count_bits(b);  in drbd_bm_resize()
758 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_total_weight()  local
762 	if (!expect(b))  in _drbd_bm_total_weight()
764 	if (!expect(b->bm_pages))  in _drbd_bm_total_weight()
767 	spin_lock_irqsave(&b->bm_lock, flags);  in _drbd_bm_total_weight()
768 	s = b->bm_set;  in _drbd_bm_total_weight()
769 	spin_unlock_irqrestore(&b->bm_lock, flags);  in _drbd_bm_total_weight()
787 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_words()  local
788 	if (!expect(b))  in drbd_bm_words()
790 	if (!expect(b->bm_pages))  in drbd_bm_words()
793 	return b->bm_words;  in drbd_bm_words()
798 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_bits()  local
799 	if (!expect(b))  in drbd_bm_bits()
802 	return b->bm_bits;  in drbd_bm_bits()
813 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_merge_lel()  local
821 	if (!expect(b))  in drbd_bm_merge_lel()
823 	if (!expect(b->bm_pages))  in drbd_bm_merge_lel()
827 	WARN_ON(offset >= b->bm_words);  in drbd_bm_merge_lel()
828 	WARN_ON(end    >  b->bm_words);  in drbd_bm_merge_lel()
830 	spin_lock_irq(&b->bm_lock);  in drbd_bm_merge_lel()
833 		idx = bm_word_to_page_idx(b, offset);  in drbd_bm_merge_lel()
834 		p_addr = bm_map_pidx(b, idx);  in drbd_bm_merge_lel()
841 			b->bm_set += hweight_long(word) - bits;  in drbd_bm_merge_lel()
844 		bm_set_page_need_writeout(b->bm_pages[idx]);  in drbd_bm_merge_lel()
851 	if (end == b->bm_words)  in drbd_bm_merge_lel()
852 		b->bm_set -= bm_clear_surplus(b);  in drbd_bm_merge_lel()
853 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_merge_lel()
862 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_get_lel()  local
868 	if (!expect(b))  in drbd_bm_get_lel()
870 	if (!expect(b->bm_pages))  in drbd_bm_get_lel()
873 	spin_lock_irq(&b->bm_lock);  in drbd_bm_get_lel()
874 	if ((offset >= b->bm_words) ||  in drbd_bm_get_lel()
875 	    (end    >  b->bm_words) ||  in drbd_bm_get_lel()
880 			(unsigned long) b->bm_words);  in drbd_bm_get_lel()
884 			p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, offset));  in drbd_bm_get_lel()
892 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_get_lel()
898 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_set_all()  local
899 	if (!expect(b))  in drbd_bm_set_all()
901 	if (!expect(b->bm_pages))  in drbd_bm_set_all()
904 	spin_lock_irq(&b->bm_lock);  in drbd_bm_set_all()
905 	bm_memset(b, 0, 0xff, b->bm_words);  in drbd_bm_set_all()
906 	(void)bm_clear_surplus(b);  in drbd_bm_set_all()
907 	b->bm_set = b->bm_bits;  in drbd_bm_set_all()
908 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_set_all()
914 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_clear_all()  local
915 	if (!expect(b))  in drbd_bm_clear_all()
917 	if (!expect(b->bm_pages))  in drbd_bm_clear_all()
920 	spin_lock_irq(&b->bm_lock);  in drbd_bm_clear_all()
921 	bm_memset(b, 0, 0, b->bm_words);  in drbd_bm_clear_all()
922 	b->bm_set = 0;  in drbd_bm_clear_all()
923 	spin_unlock_irq(&b->bm_lock);  in drbd_bm_clear_all()
943 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_endio()  local
947 	    !bm_test_page_unchanged(b->bm_pages[idx]))  in drbd_bm_endio()
954 		bm_set_page_io_err(b->bm_pages[idx]);  in drbd_bm_endio()
961 		bm_clear_page_io_err(b->bm_pages[idx]);  in drbd_bm_endio()
983 	struct drbd_bitmap *b = device->bitmap;  in bm_page_io_async()  local
1002 	bm_set_page_unchanged(b->bm_pages[page_nr]);  in bm_page_io_async()
1007 		copy_highpage(page, b->bm_pages[page_nr]);  in bm_page_io_async()
1010 		page = b->bm_pages[page_nr];  in bm_page_io_async()
1036 	struct drbd_bitmap *b = device->bitmap;  in bm_rw()  local
1074 		WARN_ON(!(BM_LOCKED_MASK & b->bm_flags));  in bm_rw()
1080 	num_pages = b->bm_number_of_pages;  in bm_rw()
1096 		for (hint = 0; hint < b->n_bitmap_hints; hint++) {  in bm_rw()
1097 			i = b->al_bitmap_hints[hint];  in bm_rw()
1102 			    &page_private(b->bm_pages[i])))  in bm_rw()
1105 			if (bm_test_page_unchanged(b->bm_pages[i]))  in bm_rw()
1117 			    bm_test_page_unchanged(b->bm_pages[i])) {  in bm_rw()
1124 			    !bm_test_page_lazy_writeout(b->bm_pages[i])) {  in bm_rw()
1169 		b->bm_set = bm_count_bits(b);  in bm_rw()
1173 	now = b->bm_set;  in bm_rw()
1260 	struct drbd_bitmap *b = device->bitmap;  in __bm_find_next()  local
1266 	if (bm_fo > b->bm_bits) {  in __bm_find_next()
1267 		drbd_err(device, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits);  in __bm_find_next()
1270 		while (bm_fo < b->bm_bits) {  in __bm_find_next()
1273 			p_addr = __bm_map_pidx(b, bm_bit_to_page_idx(b, bm_fo));  in __bm_find_next()
1285 				if (bm_fo >= b->bm_bits)  in __bm_find_next()
1300 	struct drbd_bitmap *b = device->bitmap;  in bm_find_next()  local
1303 	if (!expect(b))  in bm_find_next()
1305 	if (!expect(b->bm_pages))  in bm_find_next()
1308 	spin_lock_irq(&b->bm_lock);  in bm_find_next()
1309 	if (BM_DONT_TEST & b->bm_flags)  in bm_find_next()
1314 	spin_unlock_irq(&b->bm_lock);  in bm_find_next()
1354 	struct drbd_bitmap *b = device->bitmap;  in __bm_change_bits_to()  local
1361 	if (e >= b->bm_bits) {  in __bm_change_bits_to()
1363 				s, e, b->bm_bits);  in __bm_change_bits_to()
1364 		e = b->bm_bits ? b->bm_bits -1 : 0;  in __bm_change_bits_to()
1367 		unsigned int page_nr = bm_bit_to_page_idx(b, bitnr);  in __bm_change_bits_to()
1372 				bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1374 				bm_set_page_need_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1377 			p_addr = __bm_map_pidx(b, page_nr);  in __bm_change_bits_to()
1388 		bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1390 		bm_set_page_need_writeout(b->bm_pages[last_page_nr]);  in __bm_change_bits_to()
1392 	b->bm_set += changed_total;  in __bm_change_bits_to()
1404 	struct drbd_bitmap *b = device->bitmap;  in bm_change_bits_to()  local
1407 	if (!expect(b))  in bm_change_bits_to()
1409 	if (!expect(b->bm_pages))  in bm_change_bits_to()
1412 	spin_lock_irqsave(&b->bm_lock, flags);  in bm_change_bits_to()
1413 	if ((val ? BM_DONT_SET : BM_DONT_CLEAR) & b->bm_flags)  in bm_change_bits_to()
1418 	spin_unlock_irqrestore(&b->bm_lock, flags);  in bm_change_bits_to()
1436 static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b,  in bm_set_full_words_within_one_page()  argument
1442 	unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr]);  in bm_set_full_words_within_one_page()
1456 		bm_set_page_lazy_writeout(b->bm_pages[page_nr]);  in bm_set_full_words_within_one_page()
1457 		b->bm_set += changed;  in bm_set_full_words_within_one_page()
1476 	struct drbd_bitmap *b = device->bitmap;  in _drbd_bm_set_bits()  local
1487 		spin_lock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1489 		spin_unlock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1495 	spin_lock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1512 		spin_unlock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1515 		spin_lock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1536 	spin_unlock_irq(&b->bm_lock);  in _drbd_bm_set_bits()
1549 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_test_bit()  local
1553 	if (!expect(b))  in drbd_bm_test_bit()
1555 	if (!expect(b->bm_pages))  in drbd_bm_test_bit()
1558 	spin_lock_irqsave(&b->bm_lock, flags);  in drbd_bm_test_bit()
1559 	if (BM_DONT_TEST & b->bm_flags)  in drbd_bm_test_bit()
1561 	if (bitnr < b->bm_bits) {  in drbd_bm_test_bit()
1562 		p_addr = bm_map_pidx(b, bm_bit_to_page_idx(b, bitnr));  in drbd_bm_test_bit()
1565 	} else if (bitnr == b->bm_bits) {  in drbd_bm_test_bit()
1568 		drbd_err(device, "bitnr=%lu > bm_bits=%lu\n", bitnr, b->bm_bits);  in drbd_bm_test_bit()
1572 	spin_unlock_irqrestore(&b->bm_lock, flags);  in drbd_bm_test_bit()
1580 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_count_bits()  local
1590 	if (!expect(b))  in drbd_bm_count_bits()
1592 	if (!expect(b->bm_pages))  in drbd_bm_count_bits()
1595 	spin_lock_irqsave(&b->bm_lock, flags);  in drbd_bm_count_bits()
1596 	if (BM_DONT_TEST & b->bm_flags)  in drbd_bm_count_bits()
1599 		unsigned int idx = bm_bit_to_page_idx(b, bitnr);  in drbd_bm_count_bits()
1604 			p_addr = bm_map_pidx(b, idx);  in drbd_bm_count_bits()
1606 		if (expect(bitnr < b->bm_bits))  in drbd_bm_count_bits()
1609 			drbd_err(device, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits);  in drbd_bm_count_bits()
1613 	spin_unlock_irqrestore(&b->bm_lock, flags);  in drbd_bm_count_bits()
1634 	struct drbd_bitmap *b = device->bitmap;  in drbd_bm_e_weight()  local
1639 	if (!expect(b))  in drbd_bm_e_weight()
1641 	if (!expect(b->bm_pages))  in drbd_bm_e_weight()
1644 	spin_lock_irqsave(&b->bm_lock, flags);  in drbd_bm_e_weight()
1645 	if (BM_DONT_TEST & b->bm_flags)  in drbd_bm_e_weight()
1649 	e = min((size_t)S2W(enr+1), b->bm_words);  in drbd_bm_e_weight()
1651 	if (s < b->bm_words) {  in drbd_bm_e_weight()
1653 		p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, s));  in drbd_bm_e_weight()
1660 	spin_unlock_irqrestore(&b->bm_lock, flags);  in drbd_bm_e_weight()