119 static inline int raid6_d0(struct stripe_head *sh)  in raid6_d0()  argument
121 	if (sh->ddf_layout)  in raid6_d0()
125 	if (sh->qd_idx == sh->disks - 1)  in raid6_d0()
128 		return sh->qd_idx + 1;  in raid6_d0()
141 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,  in raid6_idx_to_slot()  argument
146 	if (sh->ddf_layout)  in raid6_idx_to_slot()
148 	if (idx == sh->pd_idx)  in raid6_idx_to_slot()
150 	if (idx == sh->qd_idx)  in raid6_idx_to_slot()
152 	if (!sh->ddf_layout)  in raid6_idx_to_slot()
159 static int stripe_operations_active(struct stripe_head *sh)  in stripe_operations_active()  argument
161 	return sh->check_state || sh->reconstruct_state ||  in stripe_operations_active()
162 	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||  in stripe_operations_active()
163 	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);  in stripe_operations_active()
166 static bool stripe_is_lowprio(struct stripe_head *sh)  in stripe_is_lowprio()  argument
168 	return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||  in stripe_is_lowprio()
169 		test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&  in stripe_is_lowprio()
170 	       !test_bit(STRIPE_R5C_CACHING, &sh->state);  in stripe_is_lowprio()
173 static void raid5_wakeup_stripe_thread(struct stripe_head *sh)  in raid5_wakeup_stripe_thread()  argument
174 	__must_hold(&sh->raid_conf->device_lock)  in raid5_wakeup_stripe_thread()
176 	struct r5conf *conf = sh->raid_conf;  in raid5_wakeup_stripe_thread()
179 	int i, cpu = sh->cpu;  in raid5_wakeup_stripe_thread()
183 		sh->cpu = cpu;  in raid5_wakeup_stripe_thread()
186 	if (list_empty(&sh->lru)) {  in raid5_wakeup_stripe_thread()
189 		if (stripe_is_lowprio(sh))  in raid5_wakeup_stripe_thread()
190 			list_add_tail(&sh->lru, &group->loprio_list);  in raid5_wakeup_stripe_thread()
192 			list_add_tail(&sh->lru, &group->handle_list);  in raid5_wakeup_stripe_thread()
194 		sh->group = group;  in raid5_wakeup_stripe_thread()
202 	group = conf->worker_groups + cpu_to_group(sh->cpu);  in raid5_wakeup_stripe_thread()
206 	queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);  in raid5_wakeup_stripe_thread()
213 			queue_work_on(sh->cpu, raid5_wq,  in raid5_wakeup_stripe_thread()
220 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,  in do_release_stripe()  argument
227 	BUG_ON(!list_empty(&sh->lru));  in do_release_stripe()
231 		for (i = sh->disks; i--; )  in do_release_stripe()
232 			if (test_bit(R5_InJournal, &sh->dev[i].flags))  in do_release_stripe()
241 	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||  in do_release_stripe()
243 	     !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {  in do_release_stripe()
244 		if (test_bit(STRIPE_R5C_CACHING, &sh->state))  in do_release_stripe()
245 			r5c_make_stripe_write_out(sh);  in do_release_stripe()
246 		set_bit(STRIPE_HANDLE, &sh->state);  in do_release_stripe()
249 	if (test_bit(STRIPE_HANDLE, &sh->state)) {  in do_release_stripe()
250 		if (test_bit(STRIPE_DELAYED, &sh->state) &&  in do_release_stripe()
251 		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in do_release_stripe()
252 			list_add_tail(&sh->lru, &conf->delayed_list);  in do_release_stripe()
253 		else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&  in do_release_stripe()
254 			   sh->bm_seq - conf->seq_write > 0)  in do_release_stripe()
255 			list_add_tail(&sh->lru, &conf->bitmap_list);  in do_release_stripe()
257 			clear_bit(STRIPE_DELAYED, &sh->state);  in do_release_stripe()
258 			clear_bit(STRIPE_BIT_DELAY, &sh->state);  in do_release_stripe()
260 				if (stripe_is_lowprio(sh))  in do_release_stripe()
261 					list_add_tail(&sh->lru,  in do_release_stripe()
264 					list_add_tail(&sh->lru,  in do_release_stripe()
267 				raid5_wakeup_stripe_thread(sh);  in do_release_stripe()
273 		BUG_ON(stripe_operations_active(sh));  in do_release_stripe()
274 		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in do_release_stripe()
279 		if (!test_bit(STRIPE_EXPANDING, &sh->state)) {  in do_release_stripe()
281 				list_add_tail(&sh->lru, temp_inactive_list);  in do_release_stripe()
283 				WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));  in do_release_stripe()
285 					list_add_tail(&sh->lru, temp_inactive_list);  in do_release_stripe()
288 					if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))  in do_release_stripe()
290 					if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))  in do_release_stripe()
292 					list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);  in do_release_stripe()
300 					list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);  in do_release_stripe()
306 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,  in __release_stripe()  argument
310 	if (atomic_dec_and_test(&sh->count))  in __release_stripe()
311 		do_release_stripe(conf, sh, temp_inactive_list);  in __release_stripe()
367 	struct stripe_head *sh, *t;  in release_stripe_list()  local
373 	llist_for_each_entry_safe(sh, t, head, release_list) {  in release_stripe_list()
378 		clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);  in release_stripe_list()
384 		hash = sh->hash_lock_index;  in release_stripe_list()
385 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in release_stripe_list()
392 void raid5_release_stripe(struct stripe_head *sh)  in raid5_release_stripe()  argument
394 	struct r5conf *conf = sh->raid_conf;  in raid5_release_stripe()
402 	if (atomic_add_unless(&sh->count, -1, 1))  in raid5_release_stripe()
406 		test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))  in raid5_release_stripe()
408 	wakeup = llist_add(&sh->release_list, &conf->released_stripes);  in raid5_release_stripe()
414 	if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {  in raid5_release_stripe()
416 		hash = sh->hash_lock_index;  in raid5_release_stripe()
417 		do_release_stripe(conf, sh, &list);  in raid5_release_stripe()
423 static inline void remove_hash(struct stripe_head *sh)  in remove_hash()  argument
426 		(unsigned long long)sh->sector);  in remove_hash()
428 	hlist_del_init(&sh->hash);  in remove_hash()
431 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)  in insert_hash()  argument
433 	struct hlist_head *hp = stripe_hash(conf, sh->sector);  in insert_hash()
436 		(unsigned long long)sh->sector);  in insert_hash()
438 	hlist_add_head(&sh->hash, hp);  in insert_hash()
444 	struct stripe_head *sh = NULL;  in get_free_stripe()  local
450 	sh = list_entry(first, struct stripe_head, lru);  in get_free_stripe()
452 	remove_hash(sh);  in get_free_stripe()
454 	BUG_ON(hash != sh->hash_lock_index);  in get_free_stripe()
458 	return sh;  in get_free_stripe()
462 static void free_stripe_pages(struct stripe_head *sh)  in free_stripe_pages()  argument
468 	if (!sh->pages)  in free_stripe_pages()
471 	for (i = 0; i < sh->nr_pages; i++) {  in free_stripe_pages()
472 		p = sh->pages[i];  in free_stripe_pages()
475 		sh->pages[i] = NULL;  in free_stripe_pages()
479 static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)  in alloc_stripe_pages()  argument
484 	for (i = 0; i < sh->nr_pages; i++) {  in alloc_stripe_pages()
486 		if (sh->pages[i])  in alloc_stripe_pages()
491 			free_stripe_pages(sh);  in alloc_stripe_pages()
494 		sh->pages[i] = p;  in alloc_stripe_pages()
500 init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)  in init_stripe_shared_pages()  argument
504 	if (sh->pages)  in init_stripe_shared_pages()
511 	sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);  in init_stripe_shared_pages()
512 	if (!sh->pages)  in init_stripe_shared_pages()
514 	sh->nr_pages = nr_pages;  in init_stripe_shared_pages()
515 	sh->stripes_per_page = cnt;  in init_stripe_shared_pages()
520 static void shrink_buffers(struct stripe_head *sh)  in shrink_buffers()  argument
523 	int num = sh->raid_conf->pool_size;  in shrink_buffers()
529 		WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);  in shrink_buffers()
530 		p = sh->dev[i].page;  in shrink_buffers()
533 		sh->dev[i].page = NULL;  in shrink_buffers()
538 		sh->dev[i].page = NULL;  in shrink_buffers()
539 	free_stripe_pages(sh); /* Free pages */  in shrink_buffers()
543 static int grow_buffers(struct stripe_head *sh, gfp_t gfp)  in grow_buffers()  argument
546 	int num = sh->raid_conf->pool_size;  in grow_buffers()
555 		sh->dev[i].page = page;  in grow_buffers()
556 		sh->dev[i].orig_page = page;  in grow_buffers()
557 		sh->dev[i].offset = 0;  in grow_buffers()
560 	if (alloc_stripe_pages(sh, gfp))  in grow_buffers()
564 		sh->dev[i].page = raid5_get_dev_page(sh, i);  in grow_buffers()
565 		sh->dev[i].orig_page = sh->dev[i].page;  in grow_buffers()
566 		sh->dev[i].offset = raid5_get_page_offset(sh, i);  in grow_buffers()
573 			    struct stripe_head *sh);
575 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)  in init_stripe()  argument
577 	struct r5conf *conf = sh->raid_conf;  in init_stripe()
580 	BUG_ON(atomic_read(&sh->count) != 0);  in init_stripe()
581 	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));  in init_stripe()
582 	BUG_ON(stripe_operations_active(sh));  in init_stripe()
583 	BUG_ON(sh->batch_head);  in init_stripe()
589 	sh->generation = conf->generation - previous;  in init_stripe()
590 	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;  in init_stripe()
591 	sh->sector = sector;  in init_stripe()
592 	stripe_set_idx(sector, conf, previous, sh);  in init_stripe()
593 	sh->state = 0;  in init_stripe()
595 	for (i = sh->disks; i--; ) {  in init_stripe()
596 		struct r5dev *dev = &sh->dev[i];  in init_stripe()
601 			       (unsigned long long)sh->sector, i, dev->toread,  in init_stripe()
607 		dev->sector = raid5_compute_blocknr(sh, i, previous);  in init_stripe()
611 	sh->overwrite_disks = 0;  in init_stripe()
612 	insert_hash(conf, sh);  in init_stripe()
613 	sh->cpu = smp_processor_id();  in init_stripe()
614 	set_bit(STRIPE_BATCH_READY, &sh->state);  in init_stripe()
620 	struct stripe_head *sh;  in __find_stripe()  local
623 	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)  in __find_stripe()
624 		if (sh->sector == sector && sh->generation == generation)  in __find_stripe()
625 			return sh;  in __find_stripe()
634 	struct stripe_head *sh;  in find_get_stripe()  local
636 	sh = __find_stripe(conf, sector, generation);  in find_get_stripe()
637 	if (!sh)  in find_get_stripe()
640 	if (atomic_inc_not_zero(&sh->count))  in find_get_stripe()
641 		return sh;  in find_get_stripe()
650 	if (!atomic_read(&sh->count)) {  in find_get_stripe()
651 		if (!test_bit(STRIPE_HANDLE, &sh->state))  in find_get_stripe()
653 		BUG_ON(list_empty(&sh->lru) &&  in find_get_stripe()
654 		       !test_bit(STRIPE_EXPANDING, &sh->state));  in find_get_stripe()
658 		list_del_init(&sh->lru);  in find_get_stripe()
662 		if (sh->group) {  in find_get_stripe()
663 			sh->group->stripes_cnt--;  in find_get_stripe()
664 			sh->group = NULL;  in find_get_stripe()
667 	atomic_inc(&sh->count);  in find_get_stripe()
670 	return sh;  in find_get_stripe()
807 	struct stripe_head *sh;  in raid5_get_active_stripe()  local
834 		sh = find_get_stripe(conf, sector, conf->generation - previous,  in raid5_get_active_stripe()
836 		if (sh)  in raid5_get_active_stripe()
840 			sh = get_free_stripe(conf, hash);  in raid5_get_active_stripe()
841 			if (sh) {  in raid5_get_active_stripe()
843 				init_stripe(sh, sector, previous);  in raid5_get_active_stripe()
844 				atomic_inc(&sh->count);  in raid5_get_active_stripe()
864 	return sh;  in raid5_get_active_stripe()
867 static bool is_full_stripe_write(struct stripe_head *sh)  in is_full_stripe_write()  argument
869 	BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));  in is_full_stripe_write()
870 	return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);  in is_full_stripe_write()
895 static bool stripe_can_batch(struct stripe_head *sh)  in stripe_can_batch()  argument
897 	struct r5conf *conf = sh->raid_conf;  in stripe_can_batch()
901 	return test_bit(STRIPE_BATCH_READY, &sh->state) &&  in stripe_can_batch()
902 		!test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&  in stripe_can_batch()
903 		is_full_stripe_write(sh);  in stripe_can_batch()
908 		struct stripe_head *sh, struct stripe_head *last_sh)  in stripe_add_to_batch_list()  argument
916 	tmp_sec = sh->sector;  in stripe_add_to_batch_list()
919 	head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);  in stripe_add_to_batch_list()
936 	lock_two_stripes(head, sh);  in stripe_add_to_batch_list()
938 	if (!stripe_can_batch(head) || !stripe_can_batch(sh))  in stripe_add_to_batch_list()
941 	if (sh->batch_head)  in stripe_add_to_batch_list()
945 	while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in stripe_add_to_batch_list()
947 	if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||  in stripe_add_to_batch_list()
948 	    bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))  in stripe_add_to_batch_list()
965 		sh->batch_head = head->batch_head;  in stripe_add_to_batch_list()
971 		list_add(&sh->batch_list, &head->batch_list);  in stripe_add_to_batch_list()
975 		sh->batch_head = head->batch_head;  in stripe_add_to_batch_list()
977 		list_add_tail(&sh->batch_list, &head->batch_list);  in stripe_add_to_batch_list()
981 	if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in stripe_add_to_batch_list()
986 	if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {  in stripe_add_to_batch_list()
987 		int seq = sh->bm_seq;  in stripe_add_to_batch_list()
988 		if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&  in stripe_add_to_batch_list()
989 		    sh->batch_head->bm_seq > seq)  in stripe_add_to_batch_list()
990 			seq = sh->batch_head->bm_seq;  in stripe_add_to_batch_list()
991 		set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);  in stripe_add_to_batch_list()
992 		sh->batch_head->bm_seq = seq;  in stripe_add_to_batch_list()
995 	atomic_inc(&sh->count);  in stripe_add_to_batch_list()
997 	unlock_two_stripes(head, sh);  in stripe_add_to_batch_list()
1005 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)  in use_new_offset()  argument
1015 	if (sh->generation == conf->generation - 1)  in use_new_offset()
1132 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)  in ops_run_io()  argument
1134 	struct r5conf *conf = sh->raid_conf;  in ops_run_io()
1135 	int i, disks = sh->disks;  in ops_run_io()
1136 	struct stripe_head *head_sh = sh;  in ops_run_io()
1143 	if (log_stripe(sh, s) == 0)  in ops_run_io()
1155 		sh = head_sh;  in ops_run_io()
1156 		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {  in ops_run_io()
1158 			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))  in ops_run_io()
1160 			if (test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_io()
1162 		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))  in ops_run_io()
1165 					    &sh->dev[i].flags)) {  in ops_run_io()
1170 		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))  in ops_run_io()
1174 		dev = &sh->dev[i];  in ops_run_io()
1216 			int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in ops_run_io()
1250 			set_bit(STRIPE_IO_STARTED, &sh->state);  in ops_run_io()
1256 			bi->bi_private = sh;  in ops_run_io()
1259 				__func__, (unsigned long long)sh->sector,  in ops_run_io()
1261 			atomic_inc(&sh->count);  in ops_run_io()
1262 			if (sh != head_sh)  in ops_run_io()
1264 			if (use_new_offset(conf, sh))  in ops_run_io()
1265 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1268 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1273 			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))  in ops_run_io()
1274 				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in ops_run_io()
1277 			    test_bit(R5_InJournal, &sh->dev[i].flags))  in ops_run_io()
1283 				sh->dev[i].vec.bv_page = sh->dev[i].orig_page;  in ops_run_io()
1285 				sh->dev[i].vec.bv_page = sh->dev[i].page;  in ops_run_io()
1288 			bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;  in ops_run_io()
1297 				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);  in ops_run_io()
1302 						sh->dev[i].sector);  in ops_run_io()
1313 			set_bit(STRIPE_IO_STARTED, &sh->state);  in ops_run_io()
1318 			rbi->bi_private = sh;  in ops_run_io()
1322 				__func__, (unsigned long long)sh->sector,  in ops_run_io()
1324 			atomic_inc(&sh->count);  in ops_run_io()
1325 			if (sh != head_sh)  in ops_run_io()
1327 			if (use_new_offset(conf, sh))  in ops_run_io()
1328 				rbi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1331 				rbi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1333 			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))  in ops_run_io()
1334 				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in ops_run_io()
1335 			sh->dev[i].rvec.bv_page = sh->dev[i].page;  in ops_run_io()
1338 			rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;  in ops_run_io()
1349 						sh->dev[i].sector);  in ops_run_io()
1357 				set_bit(STRIPE_DEGRADED, &sh->state);  in ops_run_io()
1359 				bi->bi_opf, i, (unsigned long long)sh->sector);  in ops_run_io()
1360 			clear_bit(R5_LOCKED, &sh->dev[i].flags);  in ops_run_io()
1361 			set_bit(STRIPE_HANDLE, &sh->state);  in ops_run_io()
1366 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_io()
1368 		if (sh != head_sh)  in ops_run_io()
1379 	struct stripe_head *sh, int no_skipcopy)  in async_copy_data()  argument
1387 	struct r5conf *conf = sh->raid_conf;  in async_copy_data()
1443 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_biofill()  local
1445 	struct r5conf *conf = sh->raid_conf;  in ops_complete_biofill()
1448 		(unsigned long long)sh->sector);  in ops_complete_biofill()
1451 	for (i = sh->disks; i--; ) {  in ops_complete_biofill()
1452 		struct r5dev *dev = &sh->dev[i];  in ops_complete_biofill()
1473 	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);  in ops_complete_biofill()
1475 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_biofill()
1476 	raid5_release_stripe(sh);  in ops_complete_biofill()
1479 static void ops_run_biofill(struct stripe_head *sh)  in ops_run_biofill()  argument
1484 	struct r5conf *conf = sh->raid_conf;  in ops_run_biofill()
1486 	BUG_ON(sh->batch_head);  in ops_run_biofill()
1488 		(unsigned long long)sh->sector);  in ops_run_biofill()
1490 	for (i = sh->disks; i--; ) {  in ops_run_biofill()
1491 		struct r5dev *dev = &sh->dev[i];  in ops_run_biofill()
1494 			spin_lock_irq(&sh->stripe_lock);  in ops_run_biofill()
1497 			spin_unlock_irq(&sh->stripe_lock);  in ops_run_biofill()
1502 						     dev->sector, tx, sh, 0);  in ops_run_biofill()
1508 	atomic_inc(&sh->count);  in ops_run_biofill()
1509 	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);  in ops_run_biofill()
1513 static void mark_target_uptodate(struct stripe_head *sh, int target)  in mark_target_uptodate()  argument
1520 	tgt = &sh->dev[target];  in mark_target_uptodate()
1528 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_compute()  local
1531 		(unsigned long long)sh->sector);  in ops_complete_compute()
1534 	mark_target_uptodate(sh, sh->ops.target);  in ops_complete_compute()
1535 	mark_target_uptodate(sh, sh->ops.target2);  in ops_complete_compute()
1537 	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);  in ops_complete_compute()
1538 	if (sh->check_state == check_state_compute_run)  in ops_complete_compute()
1539 		sh->check_state = check_state_compute_result;  in ops_complete_compute()
1540 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_compute()
1541 	raid5_release_stripe(sh);  in ops_complete_compute()
1551 static addr_conv_t *to_addr_conv(struct stripe_head *sh,  in to_addr_conv()  argument
1554 	return (void *) (to_addr_page(percpu, i) + sh->disks + 2);  in to_addr_conv()
1561 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)  in to_addr_offs()  argument
1563 	return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);  in to_addr_offs()
1567 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute5()  argument
1569 	int disks = sh->disks;  in ops_run_compute5()
1571 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_compute5()
1572 	int target = sh->ops.target;  in ops_run_compute5()
1573 	struct r5dev *tgt = &sh->dev[target];  in ops_run_compute5()
1581 	BUG_ON(sh->batch_head);  in ops_run_compute5()
1584 		__func__, (unsigned long long)sh->sector, target);  in ops_run_compute5()
1589 			off_srcs[count] = sh->dev[i].offset;  in ops_run_compute5()
1590 			xor_srcs[count++] = sh->dev[i].page;  in ops_run_compute5()
1594 	atomic_inc(&sh->count);  in ops_run_compute5()
1597 			  ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));  in ops_run_compute5()
1600 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute5()
1603 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute5()
1620 				struct stripe_head *sh,  in set_syndrome_sources()  argument
1623 	int disks = sh->disks;  in set_syndrome_sources()
1624 	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);  in set_syndrome_sources()
1625 	int d0_idx = raid6_d0(sh);  in set_syndrome_sources()
1635 		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);  in set_syndrome_sources()
1636 		struct r5dev *dev = &sh->dev[i];  in set_syndrome_sources()
1638 		if (i == sh->qd_idx || i == sh->pd_idx ||  in set_syndrome_sources()
1647 				srcs[slot] = sh->dev[i].orig_page;  in set_syndrome_sources()
1649 				srcs[slot] = sh->dev[i].page;  in set_syndrome_sources()
1655 			offs[slot] = sh->dev[i].offset;  in set_syndrome_sources()
1664 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_1()  argument
1666 	int disks = sh->disks;  in ops_run_compute6_1()
1668 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_1()
1670 	int qd_idx = sh->qd_idx;  in ops_run_compute6_1()
1679 	BUG_ON(sh->batch_head);  in ops_run_compute6_1()
1680 	if (sh->ops.target < 0)  in ops_run_compute6_1()
1681 		target = sh->ops.target2;  in ops_run_compute6_1()
1682 	else if (sh->ops.target2 < 0)  in ops_run_compute6_1()
1683 		target = sh->ops.target;  in ops_run_compute6_1()
1689 		__func__, (unsigned long long)sh->sector, target);  in ops_run_compute6_1()
1691 	tgt = &sh->dev[target];  in ops_run_compute6_1()
1696 	atomic_inc(&sh->count);  in ops_run_compute6_1()
1699 		count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);  in ops_run_compute6_1()
1703 				  ops_complete_compute, sh,  in ops_run_compute6_1()
1704 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1706 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute6_1()
1713 			offs[count] = sh->dev[i].offset;  in ops_run_compute6_1()
1714 			blocks[count++] = sh->dev[i].page;  in ops_run_compute6_1()
1718 				  NULL, ops_complete_compute, sh,  in ops_run_compute6_1()
1719 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1721 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute6_1()
1728 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_2()  argument
1730 	int i, count, disks = sh->disks;  in ops_run_compute6_2()
1731 	int syndrome_disks = sh->ddf_layout ? disks : disks-2;  in ops_run_compute6_2()
1732 	int d0_idx = raid6_d0(sh);  in ops_run_compute6_2()
1734 	int target = sh->ops.target;  in ops_run_compute6_2()
1735 	int target2 = sh->ops.target2;  in ops_run_compute6_2()
1736 	struct r5dev *tgt = &sh->dev[target];  in ops_run_compute6_2()
1737 	struct r5dev *tgt2 = &sh->dev[target2];  in ops_run_compute6_2()
1740 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_2()
1743 	BUG_ON(sh->batch_head);  in ops_run_compute6_2()
1745 		 __func__, (unsigned long long)sh->sector, target, target2);  in ops_run_compute6_2()
1760 		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);  in ops_run_compute6_2()
1762 		offs[slot] = sh->dev[i].offset;  in ops_run_compute6_2()
1763 		blocks[slot] = sh->dev[i].page;  in ops_run_compute6_2()
1776 		 __func__, (unsigned long long)sh->sector, faila, failb);  in ops_run_compute6_2()
1778 	atomic_inc(&sh->count);  in ops_run_compute6_2()
1785 					  ops_complete_compute, sh,  in ops_run_compute6_2()
1786 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1788 						  RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1794 			int qd_idx = sh->qd_idx;  in ops_run_compute6_2()
1806 				offs[count] = sh->dev[i].offset;  in ops_run_compute6_2()
1807 				blocks[count++] = sh->dev[i].page;  in ops_run_compute6_2()
1809 			dest = sh->dev[data_target].page;  in ops_run_compute6_2()
1810 			dest_off = sh->dev[data_target].offset;  in ops_run_compute6_2()
1814 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1816 				       RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1819 			count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);  in ops_run_compute6_2()
1821 					  ops_complete_compute, sh,  in ops_run_compute6_2()
1822 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1824 						  RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1829 				  ops_complete_compute, sh,  in ops_run_compute6_2()
1830 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1834 						RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1840 						RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1849 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_prexor()  local
1852 		(unsigned long long)sh->sector);  in ops_complete_prexor()
1854 	if (r5c_is_writeback(sh->raid_conf->log))  in ops_complete_prexor()
1859 		r5c_release_extra_page(sh);  in ops_complete_prexor()
1863 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor5()  argument
1866 	int disks = sh->disks;  in ops_run_prexor5()
1868 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_prexor5()
1869 	int count = 0, pd_idx = sh->pd_idx, i;  in ops_run_prexor5()
1873 	unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;  in ops_run_prexor5()
1874 	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;  in ops_run_prexor5()
1876 	BUG_ON(sh->batch_head);  in ops_run_prexor5()
1878 		(unsigned long long)sh->sector);  in ops_run_prexor5()
1881 		struct r5dev *dev = &sh->dev[i];  in ops_run_prexor5()
1897 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor5()
1899 			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_prexor5()
1905 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor6()  argument
1909 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_prexor6()
1914 		(unsigned long long)sh->sector);  in ops_run_prexor6()
1916 	count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);  in ops_run_prexor6()
1919 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor6()
1921 			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_prexor6()
1927 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)  in ops_run_biodrain()  argument
1929 	struct r5conf *conf = sh->raid_conf;  in ops_run_biodrain()
1930 	int disks = sh->disks;  in ops_run_biodrain()
1932 	struct stripe_head *head_sh = sh;  in ops_run_biodrain()
1935 		(unsigned long long)sh->sector);  in ops_run_biodrain()
1941 		sh = head_sh;  in ops_run_biodrain()
1946 			dev = &sh->dev[i];  in ops_run_biodrain()
1952 			spin_lock_irq(&sh->stripe_lock);  in ops_run_biodrain()
1955 			sh->overwrite_disks = 0;  in ops_run_biodrain()
1958 			spin_unlock_irq(&sh->stripe_lock);  in ops_run_biodrain()
1972 							     dev->sector, tx, sh,  in ops_run_biodrain()
1985 				sh = list_first_entry(&sh->batch_list,  in ops_run_biodrain()
1988 				if (sh == head_sh)  in ops_run_biodrain()
2000 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_reconstruct()  local
2001 	int disks = sh->disks;  in ops_complete_reconstruct()
2002 	int pd_idx = sh->pd_idx;  in ops_complete_reconstruct()
2003 	int qd_idx = sh->qd_idx;  in ops_complete_reconstruct()
2008 		(unsigned long long)sh->sector);  in ops_complete_reconstruct()
2011 		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);  in ops_complete_reconstruct()
2012 		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);  in ops_complete_reconstruct()
2013 		discard |= test_bit(R5_Discard, &sh->dev[i].flags);  in ops_complete_reconstruct()
2017 		struct r5dev *dev = &sh->dev[i];  in ops_complete_reconstruct()
2022 				if (test_bit(STRIPE_EXPAND_READY, &sh->state))  in ops_complete_reconstruct()
2032 	if (sh->reconstruct_state == reconstruct_state_drain_run)  in ops_complete_reconstruct()
2033 		sh->reconstruct_state = reconstruct_state_drain_result;  in ops_complete_reconstruct()
2034 	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)  in ops_complete_reconstruct()
2035 		sh->reconstruct_state = reconstruct_state_prexor_drain_result;  in ops_complete_reconstruct()
2037 		BUG_ON(sh->reconstruct_state != reconstruct_state_run);  in ops_complete_reconstruct()
2038 		sh->reconstruct_state = reconstruct_state_result;  in ops_complete_reconstruct()
2041 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_reconstruct()
2042 	raid5_release_stripe(sh);  in ops_complete_reconstruct()
2046 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct5()  argument
2049 	int disks = sh->disks;  in ops_run_reconstruct5()
2053 	int count, pd_idx = sh->pd_idx, i;  in ops_run_reconstruct5()
2059 	struct stripe_head *head_sh = sh;  in ops_run_reconstruct5()
2063 		(unsigned long long)sh->sector);  in ops_run_reconstruct5()
2065 	for (i = 0; i < sh->disks; i++) {  in ops_run_reconstruct5()
2068 		if (!test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_reconstruct5()
2071 	if (i >= sh->disks) {  in ops_run_reconstruct5()
2072 		atomic_inc(&sh->count);  in ops_run_reconstruct5()
2073 		set_bit(R5_Discard, &sh->dev[pd_idx].flags);  in ops_run_reconstruct5()
2074 		ops_complete_reconstruct(sh);  in ops_run_reconstruct5()
2080 	off_srcs = to_addr_offs(sh, percpu);  in ops_run_reconstruct5()
2086 		off_dest = off_srcs[count] = sh->dev[pd_idx].offset;  in ops_run_reconstruct5()
2087 		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;  in ops_run_reconstruct5()
2089 			struct r5dev *dev = &sh->dev[i];  in ops_run_reconstruct5()
2097 		xor_dest = sh->dev[pd_idx].page;  in ops_run_reconstruct5()
2098 		off_dest = sh->dev[pd_idx].offset;  in ops_run_reconstruct5()
2100 			struct r5dev *dev = &sh->dev[i];  in ops_run_reconstruct5()
2114 		list_first_entry(&sh->batch_list,  in ops_run_reconstruct5()
2122 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2126 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2131 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_reconstruct5()
2134 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_reconstruct5()
2137 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_reconstruct5()
2144 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct6()  argument
2151 	struct stripe_head *head_sh = sh;  in ops_run_reconstruct6()
2156 	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);  in ops_run_reconstruct6()
2158 	for (i = 0; i < sh->disks; i++) {  in ops_run_reconstruct6()
2159 		if (sh->pd_idx == i || sh->qd_idx == i)  in ops_run_reconstruct6()
2161 		if (!test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_reconstruct6()
2164 	if (i >= sh->disks) {  in ops_run_reconstruct6()
2165 		atomic_inc(&sh->count);  in ops_run_reconstruct6()
2166 		set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);  in ops_run_reconstruct6()
2167 		set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);  in ops_run_reconstruct6()
2168 		ops_complete_reconstruct(sh);  in ops_run_reconstruct6()
2174 	offs = to_addr_offs(sh, percpu);  in ops_run_reconstruct6()
2176 	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {  in ops_run_reconstruct6()
2184 	count = set_syndrome_sources(blocks, offs, sh, synflags);  in ops_run_reconstruct6()
2186 		list_first_entry(&sh->batch_list,  in ops_run_reconstruct6()
2192 				  head_sh, to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2195 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2197 			RAID5_STRIPE_SIZE(sh->raid_conf),  &submit);  in ops_run_reconstruct6()
2200 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_reconstruct6()
2208 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_check()  local
2211 		(unsigned long long)sh->sector);  in ops_complete_check()
2213 	sh->check_state = check_state_check_result;  in ops_complete_check()
2214 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_check()
2215 	raid5_release_stripe(sh);  in ops_complete_check()
2218 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_check_p()  argument
2220 	int disks = sh->disks;  in ops_run_check_p()
2221 	int pd_idx = sh->pd_idx;  in ops_run_check_p()
2222 	int qd_idx = sh->qd_idx;  in ops_run_check_p()
2226 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_check_p()
2233 		(unsigned long long)sh->sector);  in ops_run_check_p()
2235 	BUG_ON(sh->batch_head);  in ops_run_check_p()
2237 	xor_dest = sh->dev[pd_idx].page;  in ops_run_check_p()
2238 	off_dest = sh->dev[pd_idx].offset;  in ops_run_check_p()
2244 		off_srcs[count] = sh->dev[i].offset;  in ops_run_check_p()
2245 		xor_srcs[count++] = sh->dev[i].page;  in ops_run_check_p()
2249 			  to_addr_conv(sh, percpu, 0));  in ops_run_check_p()
2251 			   RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_check_p()
2252 			   &sh->ops.zero_sum_result, &submit);  in ops_run_check_p()
2254 	atomic_inc(&sh->count);  in ops_run_check_p()
2255 	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);  in ops_run_check_p()
2259 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)  in ops_run_check_pq()  argument
2262 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_check_pq()
2267 		(unsigned long long)sh->sector, checkp);  in ops_run_check_pq()
2269 	BUG_ON(sh->batch_head);  in ops_run_check_pq()
2270 	count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);  in ops_run_check_pq()
2274 	atomic_inc(&sh->count);  in ops_run_check_pq()
2276 			  sh, to_addr_conv(sh, percpu, 0));  in ops_run_check_pq()
2278 			   RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_check_pq()
2279 			   &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);  in ops_run_check_pq()
2282 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)  in raid_run_ops()  argument
2284 	int overlap_clear = 0, i, disks = sh->disks;  in raid_run_ops()
2286 	struct r5conf *conf = sh->raid_conf;  in raid_run_ops()
2293 		ops_run_biofill(sh);  in raid_run_ops()
2299 			tx = ops_run_compute5(sh, percpu);  in raid_run_ops()
2301 			if (sh->ops.target2 < 0 || sh->ops.target < 0)  in raid_run_ops()
2302 				tx = ops_run_compute6_1(sh, percpu);  in raid_run_ops()
2304 				tx = ops_run_compute6_2(sh, percpu);  in raid_run_ops()
2313 			tx = ops_run_prexor5(sh, percpu, tx);  in raid_run_ops()
2315 			tx = ops_run_prexor6(sh, percpu, tx);  in raid_run_ops()
2319 		tx = ops_run_partial_parity(sh, percpu, tx);  in raid_run_ops()
2322 		tx = ops_run_biodrain(sh, tx);  in raid_run_ops()
2328 			ops_run_reconstruct5(sh, percpu, tx);  in raid_run_ops()
2330 			ops_run_reconstruct6(sh, percpu, tx);  in raid_run_ops()
2334 		if (sh->check_state == check_state_run)  in raid_run_ops()
2335 			ops_run_check_p(sh, percpu);  in raid_run_ops()
2336 		else if (sh->check_state == check_state_run_q)  in raid_run_ops()
2337 			ops_run_check_pq(sh, percpu, 0);  in raid_run_ops()
2338 		else if (sh->check_state == check_state_run_pq)  in raid_run_ops()
2339 			ops_run_check_pq(sh, percpu, 1);  in raid_run_ops()
2344 	if (overlap_clear && !sh->batch_head) {  in raid_run_ops()
2346 			struct r5dev *dev = &sh->dev[i];  in raid_run_ops()
2348 				wake_up(&sh->raid_conf->wait_for_overlap);  in raid_run_ops()
2354 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)  in free_stripe()  argument
2357 	kfree(sh->pages);  in free_stripe()
2359 	if (sh->ppl_page)  in free_stripe()
2360 		__free_page(sh->ppl_page);  in free_stripe()
2361 	kmem_cache_free(sc, sh);  in free_stripe()
2367 	struct stripe_head *sh;  in alloc_stripe()  local
2369 	sh = kmem_cache_zalloc(sc, gfp);  in alloc_stripe()
2370 	if (sh) {  in alloc_stripe()
2371 		spin_lock_init(&sh->stripe_lock);  in alloc_stripe()
2372 		spin_lock_init(&sh->batch_lock);  in alloc_stripe()
2373 		INIT_LIST_HEAD(&sh->batch_list);  in alloc_stripe()
2374 		INIT_LIST_HEAD(&sh->lru);  in alloc_stripe()
2375 		INIT_LIST_HEAD(&sh->r5c);  in alloc_stripe()
2376 		INIT_LIST_HEAD(&sh->log_list);  in alloc_stripe()
2377 		atomic_set(&sh->count, 1);  in alloc_stripe()
2378 		sh->raid_conf = conf;  in alloc_stripe()
2379 		sh->log_start = MaxSector;  in alloc_stripe()
2382 			sh->ppl_page = alloc_page(gfp);  in alloc_stripe()
2383 			if (!sh->ppl_page) {  in alloc_stripe()
2384 				free_stripe(sc, sh);  in alloc_stripe()
2389 		if (init_stripe_shared_pages(sh, conf, disks)) {  in alloc_stripe()
2390 			free_stripe(sc, sh);  in alloc_stripe()
2395 	return sh;  in alloc_stripe()
2399 	struct stripe_head *sh;  in grow_one_stripe()  local
2401 	sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);  in grow_one_stripe()
2402 	if (!sh)  in grow_one_stripe()
2405 	if (grow_buffers(sh, gfp)) {  in grow_one_stripe()
2406 		shrink_buffers(sh);  in grow_one_stripe()
2407 		free_stripe(conf->slab_cache, sh);  in grow_one_stripe()
2410 	sh->hash_lock_index =  in grow_one_stripe()
2415 	raid5_release_stripe(sh);  in grow_one_stripe()
2701 	struct stripe_head *sh;  in drop_one_stripe()  local
2705 	sh = get_free_stripe(conf, hash);  in drop_one_stripe()
2707 	if (!sh)  in drop_one_stripe()
2709 	BUG_ON(atomic_read(&sh->count));  in drop_one_stripe()
2710 	shrink_buffers(sh);  in drop_one_stripe()
2711 	free_stripe(conf->slab_cache, sh);  in drop_one_stripe()
2751 	struct stripe_head *sh = bi->bi_private;  in raid5_end_read_request()  local
2752 	struct r5conf *conf = sh->raid_conf;  in raid5_end_read_request()
2753 	int disks = sh->disks, i;  in raid5_end_read_request()
2758 		if (bi == &sh->dev[i].req)  in raid5_end_read_request()
2762 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),  in raid5_end_read_request()
2768 	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))  in raid5_end_read_request()
2778 	if (use_new_offset(conf, sh))  in raid5_end_read_request()
2779 		s = sh->sector + rdev->new_data_offset;  in raid5_end_read_request()
2781 		s = sh->sector + rdev->data_offset;  in raid5_end_read_request()
2783 		set_bit(R5_UPTODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2784 		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {  in raid5_end_read_request()
2795 			clear_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2796 			clear_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_read_request()
2797 		} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))  in raid5_end_read_request()
2798 			clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2800 		if (test_bit(R5_InJournal, &sh->dev[i].flags))  in raid5_end_read_request()
2805 			set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2813 		clear_bit(R5_UPTODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2816 		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))  in raid5_end_read_request()
2829 		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {  in raid5_end_read_request()
2850 		    && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))  in raid5_end_read_request()
2853 			if (sh->qd_idx >= 0 && sh->pd_idx == i)  in raid5_end_read_request()
2854 				set_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2855 			else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {  in raid5_end_read_request()
2856 				set_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2857 				clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2859 				set_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2861 			clear_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2862 			clear_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_read_request()
2866 				      rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))  in raid5_end_read_request()
2872 	clear_bit(R5_LOCKED, &sh->dev[i].flags);  in raid5_end_read_request()
2873 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_end_read_request()
2874 	raid5_release_stripe(sh);  in raid5_end_read_request()
2879 	struct stripe_head *sh = bi->bi_private;  in raid5_end_write_request()  local
2880 	struct r5conf *conf = sh->raid_conf;  in raid5_end_write_request()
2881 	int disks = sh->disks, i;  in raid5_end_write_request()
2888 		if (bi == &sh->dev[i].req) {  in raid5_end_write_request()
2892 		if (bi == &sh->dev[i].rreq) {  in raid5_end_write_request()
2906 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),  in raid5_end_write_request()
2916 		else if (is_badblock(rdev, sh->sector,  in raid5_end_write_request()
2919 			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);  in raid5_end_write_request()
2922 			set_bit(STRIPE_DEGRADED, &sh->state);  in raid5_end_write_request()
2924 			set_bit(R5_WriteError, &sh->dev[i].flags);  in raid5_end_write_request()
2928 		} else if (is_badblock(rdev, sh->sector,  in raid5_end_write_request()
2931 			set_bit(R5_MadeGood, &sh->dev[i].flags);  in raid5_end_write_request()
2932 			if (test_bit(R5_ReadError, &sh->dev[i].flags))  in raid5_end_write_request()
2937 				set_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_write_request()
2942 	if (sh->batch_head && bi->bi_status && !replacement)  in raid5_end_write_request()
2943 		set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);  in raid5_end_write_request()
2946 	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))  in raid5_end_write_request()
2947 		clear_bit(R5_LOCKED, &sh->dev[i].flags);  in raid5_end_write_request()
2948 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_end_write_request()
2950 	if (sh->batch_head && sh != sh->batch_head)  in raid5_end_write_request()
2951 		raid5_release_stripe(sh->batch_head);  in raid5_end_write_request()
2952 	raid5_release_stripe(sh);  in raid5_end_write_request()
2995 			      struct stripe_head *sh)  in raid5_compute_sector()  argument
3183 	if (sh) {  in raid5_compute_sector()
3184 		sh->pd_idx = pd_idx;  in raid5_compute_sector()
3185 		sh->qd_idx = qd_idx;  in raid5_compute_sector()
3186 		sh->ddf_layout = ddf_layout;  in raid5_compute_sector()
3195 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)  in raid5_compute_blocknr()  argument
3197 	struct r5conf *conf = sh->raid_conf;  in raid5_compute_blocknr()
3198 	int raid_disks = sh->disks;  in raid5_compute_blocknr()
3200 	sector_t new_sector = sh->sector, check;  in raid5_compute_blocknr()
3215 	if (i == sh->pd_idx)  in raid5_compute_blocknr()
3223 			if (i > sh->pd_idx)  in raid5_compute_blocknr()
3228 			if (i < sh->pd_idx)  in raid5_compute_blocknr()
3230 			i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3242 		if (i == sh->qd_idx)  in raid5_compute_blocknr()
3249 			if (sh->pd_idx == raid_disks-1)  in raid5_compute_blocknr()
3251 			else if (i > sh->pd_idx)  in raid5_compute_blocknr()
3256 			if (sh->pd_idx == raid_disks-1)  in raid5_compute_blocknr()
3260 				if (i < sh->pd_idx)  in raid5_compute_blocknr()
3262 				i -= (sh->pd_idx + 2);  in raid5_compute_blocknr()
3272 			if (sh->pd_idx == 0)  in raid5_compute_blocknr()
3276 				if (i < sh->pd_idx)  in raid5_compute_blocknr()
3278 				i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3283 			if (i > sh->pd_idx)  in raid5_compute_blocknr()
3288 			if (i < sh->pd_idx)  in raid5_compute_blocknr()
3290 			i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3306 	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx  in raid5_compute_blocknr()
3307 		|| sh2.qd_idx != sh->qd_idx) {  in raid5_compute_blocknr()
3372 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,  in schedule_reconstruction()  argument
3375 	int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;  in schedule_reconstruction()
3376 	struct r5conf *conf = sh->raid_conf;  in schedule_reconstruction()
3386 		r5c_release_extra_page(sh);  in schedule_reconstruction()
3389 			struct r5dev *dev = &sh->dev[i];  in schedule_reconstruction()
3410 			sh->reconstruct_state = reconstruct_state_drain_run;  in schedule_reconstruction()
3413 			sh->reconstruct_state = reconstruct_state_run;  in schedule_reconstruction()
3418 			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))  in schedule_reconstruction()
3421 		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||  in schedule_reconstruction()
3422 			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));  in schedule_reconstruction()
3424 			(!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||  in schedule_reconstruction()
3425 			   test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));  in schedule_reconstruction()
3428 			struct r5dev *dev = &sh->dev[i];  in schedule_reconstruction()
3447 		sh->reconstruct_state = reconstruct_state_prexor_drain_run;  in schedule_reconstruction()
3456 	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);  in schedule_reconstruction()
3457 	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);  in schedule_reconstruction()
3461 		int qd_idx = sh->qd_idx;  in schedule_reconstruction()
3462 		struct r5dev *dev = &sh->dev[qd_idx];  in schedule_reconstruction()
3469 	if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&  in schedule_reconstruction()
3471 	    !test_bit(STRIPE_FULL_WRITE, &sh->state) &&  in schedule_reconstruction()
3472 	    test_bit(R5_Insync, &sh->dev[pd_idx].flags))  in schedule_reconstruction()
3476 		__func__, (unsigned long long)sh->sector,  in schedule_reconstruction()
3480 static bool stripe_bio_overlaps(struct stripe_head *sh, struct bio *bi,  in stripe_bio_overlaps()  argument
3483 	struct r5conf *conf = sh->raid_conf;  in stripe_bio_overlaps()
3487 		 bi->bi_iter.bi_sector, sh->sector);  in stripe_bio_overlaps()
3490 	if (sh->batch_head)  in stripe_bio_overlaps()
3494 		bip = &sh->dev[dd_idx].towrite;  in stripe_bio_overlaps()
3496 		bip = &sh->dev[dd_idx].toread;  in stripe_bio_overlaps()
3521 		for (i = 0; i < sh->disks; i++) {  in stripe_bio_overlaps()
3522 			if (i != sh->pd_idx &&  in stripe_bio_overlaps()
3523 			    (i == dd_idx || sh->dev[i].towrite)) {  in stripe_bio_overlaps()
3524 				sector = sh->dev[i].sector;  in stripe_bio_overlaps()
3540 static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,  in __add_stripe_bio()  argument
3543 	struct r5conf *conf = sh->raid_conf;  in __add_stripe_bio()
3548 		bip = &sh->dev[dd_idx].towrite;  in __add_stripe_bio()
3552 		bip = &sh->dev[dd_idx].toread;  in __add_stripe_bio()
3559 		clear_bit(STRIPE_BATCH_READY, &sh->state);  in __add_stripe_bio()
3570 		sector_t sector = sh->dev[dd_idx].sector;  in __add_stripe_bio()
3571 		for (bi=sh->dev[dd_idx].towrite;  in __add_stripe_bio()
3572 		     sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&  in __add_stripe_bio()
3574 		     bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {  in __add_stripe_bio()
3578 		if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))  in __add_stripe_bio()
3579 			if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))  in __add_stripe_bio()
3580 				sh->overwrite_disks++;  in __add_stripe_bio()
3584 		 (*bip)->bi_iter.bi_sector, sh->sector, dd_idx,  in __add_stripe_bio()
3585 		 sh->dev[dd_idx].sector);  in __add_stripe_bio()
3600 		set_bit(STRIPE_BITMAP_PENDING, &sh->state);  in __add_stripe_bio()
3601 		spin_unlock_irq(&sh->stripe_lock);  in __add_stripe_bio()
3602 		md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,  in __add_stripe_bio()
3604 		spin_lock_irq(&sh->stripe_lock);  in __add_stripe_bio()
3605 		clear_bit(STRIPE_BITMAP_PENDING, &sh->state);  in __add_stripe_bio()
3606 		if (!sh->batch_head) {  in __add_stripe_bio()
3607 			sh->bm_seq = conf->seq_flush+1;  in __add_stripe_bio()
3608 			set_bit(STRIPE_BIT_DELAY, &sh->state);  in __add_stripe_bio()
3618 static bool add_stripe_bio(struct stripe_head *sh, struct bio *bi,  in add_stripe_bio()  argument
3621 	spin_lock_irq(&sh->stripe_lock);  in add_stripe_bio()
3623 	if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {  in add_stripe_bio()
3624 		set_bit(R5_Overlap, &sh->dev[dd_idx].flags);  in add_stripe_bio()
3625 		spin_unlock_irq(&sh->stripe_lock);  in add_stripe_bio()
3629 	__add_stripe_bio(sh, bi, dd_idx, forwrite, previous);  in add_stripe_bio()
3630 	spin_unlock_irq(&sh->stripe_lock);  in add_stripe_bio()
3637 			    struct stripe_head *sh)  in stripe_set_idx()  argument
3649 			     &dd_idx, sh);  in stripe_set_idx()
3653 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_stripe()  argument
3657 	BUG_ON(sh->batch_head);  in handle_failed_stripe()
3662 		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {  in handle_failed_stripe()
3675 					    sh->sector,  in handle_failed_stripe()
3681 		spin_lock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3683 		bi = sh->dev[i].towrite;  in handle_failed_stripe()
3684 		sh->dev[i].towrite = NULL;  in handle_failed_stripe()
3685 		sh->overwrite_disks = 0;  in handle_failed_stripe()
3686 		spin_unlock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3690 		log_stripe_write_finished(sh);  in handle_failed_stripe()
3692 		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in handle_failed_stripe()
3696 			sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3697 			struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3704 			md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_failed_stripe()
3708 		bi = sh->dev[i].written;  in handle_failed_stripe()
3709 		sh->dev[i].written = NULL;  in handle_failed_stripe()
3710 		if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {  in handle_failed_stripe()
3711 			WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in handle_failed_stripe()
3712 			sh->dev[i].page = sh->dev[i].orig_page;  in handle_failed_stripe()
3717 		       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3718 			struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3728 		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&  in handle_failed_stripe()
3730 		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||  in handle_failed_stripe()
3731 		      test_bit(R5_ReadError, &sh->dev[i].flags))) {  in handle_failed_stripe()
3732 			spin_lock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3733 			bi = sh->dev[i].toread;  in handle_failed_stripe()
3734 			sh->dev[i].toread = NULL;  in handle_failed_stripe()
3735 			spin_unlock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3736 			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in handle_failed_stripe()
3741 			       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3743 					r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3750 			md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_failed_stripe()
3755 		clear_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_failed_stripe()
3760 	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))  in handle_failed_stripe()
3766 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_sync()  argument
3772 	BUG_ON(sh->batch_head);  in handle_failed_sync()
3773 	clear_bit(STRIPE_SYNCING, &sh->state);  in handle_failed_sync()
3774 	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))  in handle_failed_sync()
3795 			    && !rdev_set_badblocks(rdev, sh->sector,  in handle_failed_sync()
3802 			    && !rdev_set_badblocks(rdev, sh->sector,  in handle_failed_sync()
3814 static int want_replace(struct stripe_head *sh, int disk_idx)  in want_replace()  argument
3820 	rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement);  in want_replace()
3824 	    && (rdev->recovery_offset <= sh->sector  in want_replace()
3825 		|| rdev->mddev->recovery_cp <= sh->sector))  in want_replace()
3831 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,  in need_this_block()  argument
3834 	struct r5dev *dev = &sh->dev[disk_idx];  in need_this_block()
3835 	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],  in need_this_block()
3836 				  &sh->dev[s->failed_num[1]] };  in need_this_block()
3838 	bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);  in need_this_block()
3854 	    (s->replacing && want_replace(sh, disk_idx)))  in need_this_block()
3879 	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in need_this_block()
3900 		     s->failed_num[i] == sh->pd_idx ||  in need_this_block()
3901 		     s->failed_num[i] == sh->qd_idx) &&  in need_this_block()
3918 	    sh->sector < sh->raid_conf->mddev->recovery_cp)  in need_this_block()
3922 		if (s->failed_num[i] != sh->pd_idx &&  in need_this_block()
3923 		    s->failed_num[i] != sh->qd_idx &&  in need_this_block()
3938 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,  in fetch_block()  argument
3941 	struct r5dev *dev = &sh->dev[disk_idx];  in fetch_block()
3944 	if (need_this_block(sh, s, disk_idx, disks)) {  in fetch_block()
3950 		BUG_ON(sh->batch_head);  in fetch_block()
3962 		    ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||  in fetch_block()
3969 			       (unsigned long long)sh->sector, disk_idx);  in fetch_block()
3970 			set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in fetch_block()
3973 			sh->ops.target = disk_idx;  in fetch_block()
3974 			sh->ops.target2 = -1; /* no 2nd target */  in fetch_block()
3993 				      &sh->dev[other].flags))  in fetch_block()
3998 			       (unsigned long long)sh->sector,  in fetch_block()
4000 			set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in fetch_block()
4002 			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);  in fetch_block()
4003 			set_bit(R5_Wantcompute, &sh->dev[other].flags);  in fetch_block()
4004 			sh->ops.target = disk_idx;  in fetch_block()
4005 			sh->ops.target2 = other;  in fetch_block()
4024 static void handle_stripe_fill(struct stripe_head *sh,  in handle_stripe_fill()  argument
4034 	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&  in handle_stripe_fill()
4035 	    !sh->reconstruct_state) {  in handle_stripe_fill()
4045 			if (test_bit(STRIPE_R5C_CACHING, &sh->state))  in handle_stripe_fill()
4046 				r5c_make_stripe_write_out(sh);  in handle_stripe_fill()
4051 			if (fetch_block(sh, s, i, disks))  in handle_stripe_fill()
4055 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_fill()
4066 	struct stripe_head *sh, int disks)  in handle_stripe_clean_event()  argument
4071 	struct stripe_head *head_sh = sh;  in handle_stripe_clean_event()
4075 		if (sh->dev[i].written) {  in handle_stripe_clean_event()
4076 			dev = &sh->dev[i];  in handle_stripe_clean_event()
4102 				md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_stripe_clean_event()
4104 						   !test_bit(STRIPE_DEGRADED, &sh->state),  in handle_stripe_clean_event()
4107 					sh = list_first_entry(&sh->batch_list,  in handle_stripe_clean_event()
4110 					if (sh != head_sh) {  in handle_stripe_clean_event()
4111 						dev = &sh->dev[i];  in handle_stripe_clean_event()
4115 				sh = head_sh;  in handle_stripe_clean_event()
4116 				dev = &sh->dev[i];  in handle_stripe_clean_event()
4121 	log_stripe_write_finished(sh);  in handle_stripe_clean_event()
4124 	    test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {  in handle_stripe_clean_event()
4126 		clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);  in handle_stripe_clean_event()
4127 		clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);  in handle_stripe_clean_event()
4128 		if (sh->qd_idx >= 0) {  in handle_stripe_clean_event()
4129 			clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);  in handle_stripe_clean_event()
4130 			clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);  in handle_stripe_clean_event()
4133 		clear_bit(STRIPE_DISCARD, &sh->state);  in handle_stripe_clean_event()
4140 		hash = sh->hash_lock_index;  in handle_stripe_clean_event()
4142 		remove_hash(sh);  in handle_stripe_clean_event()
4145 			sh = list_first_entry(&sh->batch_list,  in handle_stripe_clean_event()
4147 			if (sh != head_sh)  in handle_stripe_clean_event()
4150 		sh = head_sh;  in handle_stripe_clean_event()
4152 		if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))  in handle_stripe_clean_event()
4153 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_clean_event()
4157 	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))  in handle_stripe_clean_event()
4181 				  struct stripe_head *sh,  in handle_stripe_dirtying()  argument
4196 	    (recovery_cp < MaxSector && sh->sector >= recovery_cp &&  in handle_stripe_dirtying()
4204 			 (unsigned long long)sh->sector);  in handle_stripe_dirtying()
4207 		struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4209 		     i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe_dirtying()
4221 		    i != sh->pd_idx && i != sh->qd_idx &&  in handle_stripe_dirtying()
4233 		 (unsigned long long)sh->sector, sh->state, rmw, rcw);  in handle_stripe_dirtying()
4234 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_dirtying()
4240 					  (unsigned long long)sh->sector, rmw);  in handle_stripe_dirtying()
4242 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4245 			    !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {  in handle_stripe_dirtying()
4260 					r5c_use_extra_page(sh);  in handle_stripe_dirtying()
4265 				set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4272 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4274 			     i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe_dirtying()
4281 					     &sh->state)) {  in handle_stripe_dirtying()
4288 					set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4297 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4299 			    i != sh->pd_idx && i != sh->qd_idx &&  in handle_stripe_dirtying()
4306 					     &sh->state)) {  in handle_stripe_dirtying()
4314 					set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4319 					  (unsigned long long)sh->sector,  in handle_stripe_dirtying()
4320 					  rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));  in handle_stripe_dirtying()
4324 	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in handle_stripe_dirtying()
4325 		set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4337 	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&  in handle_stripe_dirtying()
4339 	     !test_bit(STRIPE_BIT_DELAY, &sh->state)))  in handle_stripe_dirtying()
4340 		schedule_reconstruction(sh, s, rcw == 0, 0);  in handle_stripe_dirtying()
4344 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks5()  argument
4349 	BUG_ON(sh->batch_head);  in handle_parity_checks5()
4350 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_parity_checks5()
4352 	switch (sh->check_state) {  in handle_parity_checks5()
4357 			sh->check_state = check_state_run;  in handle_parity_checks5()
4359 			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);  in handle_parity_checks5()
4363 		dev = &sh->dev[s->failed_num[0]];  in handle_parity_checks5()
4366 		sh->check_state = check_state_idle;  in handle_parity_checks5()
4368 			dev = &sh->dev[sh->pd_idx];  in handle_parity_checks5()
4371 		if (test_bit(STRIPE_INSYNC, &sh->state))  in handle_parity_checks5()
4382 		clear_bit(STRIPE_DEGRADED, &sh->state);  in handle_parity_checks5()
4383 		set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4388 		sh->check_state = check_state_idle;  in handle_parity_checks5()
4400 		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)  in handle_parity_checks5()
4404 			set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4409 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4412 						    (unsigned long long) sh->sector,  in handle_parity_checks5()
4413 						    (unsigned long long) sh->sector +  in handle_parity_checks5()
4416 				sh->check_state = check_state_compute_run;  in handle_parity_checks5()
4417 				set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in handle_parity_checks5()
4420 					&sh->dev[sh->pd_idx].flags);  in handle_parity_checks5()
4421 				sh->ops.target = sh->pd_idx;  in handle_parity_checks5()
4422 				sh->ops.target2 = -1;  in handle_parity_checks5()
4431 		       __func__, sh->check_state,  in handle_parity_checks5()
4432 		       (unsigned long long) sh->sector);  in handle_parity_checks5()
4437 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks6()  argument
4441 	int pd_idx = sh->pd_idx;  in handle_parity_checks6()
4442 	int qd_idx = sh->qd_idx;  in handle_parity_checks6()
4445 	BUG_ON(sh->batch_head);  in handle_parity_checks6()
4446 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_parity_checks6()
4456 	switch (sh->check_state) {  in handle_parity_checks6()
4464 			sh->check_state = check_state_run;  in handle_parity_checks6()
4470 			if (sh->check_state == check_state_run)  in handle_parity_checks6()
4471 				sh->check_state = check_state_run_pq;  in handle_parity_checks6()
4473 				sh->check_state = check_state_run_q;  in handle_parity_checks6()
4477 		sh->ops.zero_sum_result = 0;  in handle_parity_checks6()
4479 		if (sh->check_state == check_state_run) {  in handle_parity_checks6()
4481 			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);  in handle_parity_checks6()
4484 		if (sh->check_state >= check_state_run &&  in handle_parity_checks6()
4485 		    sh->check_state <= check_state_run_pq) {  in handle_parity_checks6()
4497 		sh->check_state = check_state_idle;  in handle_parity_checks6()
4500 		if (test_bit(STRIPE_INSYNC, &sh->state))  in handle_parity_checks6()
4508 			dev = &sh->dev[s->failed_num[1]];  in handle_parity_checks6()
4514 			dev = &sh->dev[s->failed_num[0]];  in handle_parity_checks6()
4519 		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {  in handle_parity_checks6()
4520 			dev = &sh->dev[pd_idx];  in handle_parity_checks6()
4525 		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {  in handle_parity_checks6()
4526 			dev = &sh->dev[qd_idx];  in handle_parity_checks6()
4534 			      dev - (struct r5dev *) &sh->dev)) {  in handle_parity_checks6()
4539 		clear_bit(STRIPE_DEGRADED, &sh->state);  in handle_parity_checks6()
4541 		set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4548 		sh->check_state = check_state_idle;  in handle_parity_checks6()
4554 		if (sh->ops.zero_sum_result == 0) {  in handle_parity_checks6()
4557 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4563 				sh->check_state = check_state_compute_result;  in handle_parity_checks6()
4574 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4577 						    (unsigned long long) sh->sector,  in handle_parity_checks6()
4578 						    (unsigned long long) sh->sector +  in handle_parity_checks6()
4581 				int *target = &sh->ops.target;  in handle_parity_checks6()
4583 				sh->ops.target = -1;  in handle_parity_checks6()
4584 				sh->ops.target2 = -1;  in handle_parity_checks6()
4585 				sh->check_state = check_state_compute_run;  in handle_parity_checks6()
4586 				set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in handle_parity_checks6()
4588 				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {  in handle_parity_checks6()
4590 						&sh->dev[pd_idx].flags);  in handle_parity_checks6()
4592 					target = &sh->ops.target2;  in handle_parity_checks6()
4595 				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {  in handle_parity_checks6()
4597 						&sh->dev[qd_idx].flags);  in handle_parity_checks6()
4608 			__func__, sh->check_state,  in handle_parity_checks6()
4609 			(unsigned long long) sh->sector);  in handle_parity_checks6()
4614 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)  in handle_stripe_expansion()  argument
4622 	BUG_ON(sh->batch_head);  in handle_stripe_expansion()
4623 	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);  in handle_stripe_expansion()
4624 	for (i = 0; i < sh->disks; i++)  in handle_stripe_expansion()
4625 		if (i != sh->pd_idx && i != sh->qd_idx) {  in handle_stripe_expansion()
4630 			sector_t bn = raid5_compute_blocknr(sh, i, 1);  in handle_stripe_expansion()
4651 					  sh->dev[i].page, sh2->dev[dd_idx].offset,  in handle_stripe_expansion()
4652 					  sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),  in handle_stripe_expansion()
4687 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)  in analyse_stripe()  argument
4689 	struct r5conf *conf = sh->raid_conf;  in analyse_stripe()
4690 	int disks = sh->disks;  in analyse_stripe()
4697 	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;  in analyse_stripe()
4698 	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;  in analyse_stripe()
4711 		dev = &sh->dev[i];  in analyse_stripe()
4722 		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))  in analyse_stripe()
4751 		    rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&  in analyse_stripe()
4752 		    !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in analyse_stripe()
4766 			is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in analyse_stripe()
4793 		else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)  in analyse_stripe()
4863 	if (test_bit(STRIPE_SYNCING, &sh->state)) {  in analyse_stripe()
4873 		    sh->sector >= conf->mddev->recovery_cp ||  in analyse_stripe()
4886 static int clear_batch_ready(struct stripe_head *sh)  in clear_batch_ready()  argument
4889 	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))  in clear_batch_ready()
4890 		return (sh->batch_head && sh->batch_head != sh);  in clear_batch_ready()
4891 	spin_lock(&sh->stripe_lock);  in clear_batch_ready()
4892 	if (!sh->batch_head) {  in clear_batch_ready()
4893 		spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4901 	if (sh->batch_head != sh) {  in clear_batch_ready()
4902 		spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4905 	spin_lock(&sh->batch_lock);  in clear_batch_ready()
4906 	list_for_each_entry(tmp, &sh->batch_list, batch_list)  in clear_batch_ready()
4908 	spin_unlock(&sh->batch_lock);  in clear_batch_ready()
4909 	spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4921 	struct stripe_head *sh, *next;  in break_stripe_batch_list()  local
4925 	list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {  in break_stripe_batch_list()
4927 		list_del_init(&sh->batch_list);  in break_stripe_batch_list()
4929 		WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |  in break_stripe_batch_list()
4941 			"stripe state: %lx\n", sh->state);  in break_stripe_batch_list()
4946 		set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |  in break_stripe_batch_list()
4952 		sh->check_state = head_sh->check_state;  in break_stripe_batch_list()
4953 		sh->reconstruct_state = head_sh->reconstruct_state;  in break_stripe_batch_list()
4954 		spin_lock_irq(&sh->stripe_lock);  in break_stripe_batch_list()
4955 		sh->batch_head = NULL;  in break_stripe_batch_list()
4956 		spin_unlock_irq(&sh->stripe_lock);  in break_stripe_batch_list()
4957 		for (i = 0; i < sh->disks; i++) {  in break_stripe_batch_list()
4958 			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in break_stripe_batch_list()
4960 			sh->dev[i].flags = head_sh->dev[i].flags &  in break_stripe_batch_list()
4964 		    sh->state & handle_flags)  in break_stripe_batch_list()
4965 			set_bit(STRIPE_HANDLE, &sh->state);  in break_stripe_batch_list()
4966 		raid5_release_stripe(sh);  in break_stripe_batch_list()
4981 static void handle_stripe(struct stripe_head *sh)  in handle_stripe()  argument
4984 	struct r5conf *conf = sh->raid_conf;  in handle_stripe()
4987 	int disks = sh->disks;  in handle_stripe()
4990 	clear_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
4998 	if (clear_batch_ready(sh))  in handle_stripe()
5001 	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {  in handle_stripe()
5004 		set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5008 	if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))  in handle_stripe()
5009 		break_stripe_batch_list(sh, 0);  in handle_stripe()
5011 	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {  in handle_stripe()
5012 		spin_lock(&sh->stripe_lock);  in handle_stripe()
5017 		if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&  in handle_stripe()
5018 		    !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&  in handle_stripe()
5019 		    !test_bit(STRIPE_DISCARD, &sh->state) &&  in handle_stripe()
5020 		    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {  in handle_stripe()
5021 			set_bit(STRIPE_SYNCING, &sh->state);  in handle_stripe()
5022 			clear_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5023 			clear_bit(STRIPE_REPLACED, &sh->state);  in handle_stripe()
5025 		spin_unlock(&sh->stripe_lock);  in handle_stripe()
5027 	clear_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe()
5031 	       (unsigned long long)sh->sector, sh->state,  in handle_stripe()
5032 	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,  in handle_stripe()
5033 	       sh->check_state, sh->reconstruct_state);  in handle_stripe()
5035 	analyse_stripe(sh, &s);  in handle_stripe()
5037 	if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))  in handle_stripe()
5042 		set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5049 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5057 	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {  in handle_stripe()
5059 		set_bit(STRIPE_BIOFILL_RUN, &sh->state);  in handle_stripe()
5075 		sh->check_state = 0;  in handle_stripe()
5076 		sh->reconstruct_state = 0;  in handle_stripe()
5077 		break_stripe_batch_list(sh, 0);  in handle_stripe()
5079 			handle_failed_stripe(conf, sh, &s, disks);  in handle_stripe()
5081 			handle_failed_sync(conf, sh, &s);  in handle_stripe()
5088 	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)  in handle_stripe()
5090 	if (sh->reconstruct_state == reconstruct_state_drain_result ||  in handle_stripe()
5091 	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {  in handle_stripe()
5092 		sh->reconstruct_state = reconstruct_state_idle;  in handle_stripe()
5097 		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&  in handle_stripe()
5098 		       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));  in handle_stripe()
5099 		BUG_ON(sh->qd_idx >= 0 &&  in handle_stripe()
5100 		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&  in handle_stripe()
5101 		       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));  in handle_stripe()
5103 			struct r5dev *dev = &sh->dev[i];  in handle_stripe()
5105 				(i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe()
5115 				    ((i == sh->pd_idx || i == sh->qd_idx)  &&  in handle_stripe()
5117 					set_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5120 		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in handle_stripe()
5128 	pdev = &sh->dev[sh->pd_idx];  in handle_stripe()
5129 	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)  in handle_stripe()
5130 		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);  in handle_stripe()
5131 	qdev = &sh->dev[sh->qd_idx];  in handle_stripe()
5132 	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)  in handle_stripe()
5133 		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)  in handle_stripe()
5145 		handle_stripe_clean_event(conf, sh, disks);  in handle_stripe()
5148 		r5c_handle_cached_data_endio(conf, sh, disks);  in handle_stripe()
5149 	log_stripe_write_finished(sh);  in handle_stripe()
5160 		handle_stripe_fill(sh, &s, disks);  in handle_stripe()
5167 	r5c_finish_stripe_write_out(conf, sh, &s);  in handle_stripe()
5178 	if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {  in handle_stripe()
5181 				handle_stripe_dirtying(conf, sh, &s, disks);  in handle_stripe()
5187 				ret = r5c_try_caching_write(conf, sh, &s,  in handle_stripe()
5198 			    (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&  in handle_stripe()
5200 				ret = handle_stripe_dirtying(conf, sh, &s,  in handle_stripe()
5213 	if (sh->check_state ||  in handle_stripe()
5215 	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&  in handle_stripe()
5216 	     !test_bit(STRIPE_INSYNC, &sh->state))) {  in handle_stripe()
5218 			handle_parity_checks6(conf, sh, &s, disks);  in handle_stripe()
5220 			handle_parity_checks5(conf, sh, &s, disks);  in handle_stripe()
5224 	    && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)  in handle_stripe()
5225 	    && !test_bit(STRIPE_REPLACED, &sh->state)) {  in handle_stripe()
5228 			if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {  in handle_stripe()
5229 				WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));  in handle_stripe()
5230 				set_bit(R5_WantReplace, &sh->dev[i].flags);  in handle_stripe()
5231 				set_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_stripe()
5235 			set_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5236 		set_bit(STRIPE_REPLACED, &sh->state);  in handle_stripe()
5239 	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&  in handle_stripe()
5240 	    test_bit(STRIPE_INSYNC, &sh->state)) {  in handle_stripe()
5242 		clear_bit(STRIPE_SYNCING, &sh->state);  in handle_stripe()
5243 		if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))  in handle_stripe()
5252 			struct r5dev *dev = &sh->dev[s.failed_num[i]];  in handle_stripe()
5269 	if (sh->reconstruct_state == reconstruct_state_result) {  in handle_stripe()
5271 			= raid5_get_active_stripe(conf, NULL, sh->sector,  in handle_stripe()
5278 			set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe()
5279 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5289 		sh->reconstruct_state = reconstruct_state_idle;  in handle_stripe()
5290 		clear_bit(STRIPE_EXPANDING, &sh->state);  in handle_stripe()
5292 			set_bit(R5_Wantwrite, &sh->dev[i].flags);  in handle_stripe()
5293 			set_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_stripe()
5298 	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&  in handle_stripe()
5299 	    !sh->reconstruct_state) {  in handle_stripe()
5301 		sh->disks = conf->raid_disks;  in handle_stripe()
5302 		stripe_set_idx(sh->sector, conf, 0, sh);  in handle_stripe()
5303 		schedule_reconstruction(sh, &s, 1, 1);  in handle_stripe()
5304 	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {  in handle_stripe()
5305 		clear_bit(STRIPE_EXPAND_READY, &sh->state);  in handle_stripe()
5312 	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))  in handle_stripe()
5313 		handle_stripe_expansion(conf, sh);  in handle_stripe()
5333 			struct r5dev *dev = &sh->dev[i];  in handle_stripe()
5337 				if (!rdev_set_badblocks(rdev, sh->sector,  in handle_stripe()
5344 				rdev_clear_badblocks(rdev, sh->sector,  in handle_stripe()
5353 				rdev_clear_badblocks(rdev, sh->sector,  in handle_stripe()
5360 		raid_run_ops(sh, s.ops_request);  in handle_stripe()
5362 	ops_run_io(sh, &s);  in handle_stripe()
5375 	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);  in handle_stripe()
5384 			struct stripe_head *sh;  in raid5_activate_delayed()  local
5385 			sh = list_entry(l, struct stripe_head, lru);  in raid5_activate_delayed()
5387 			clear_bit(STRIPE_DELAYED, &sh->state);  in raid5_activate_delayed()
5388 			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in raid5_activate_delayed()
5390 			list_add_tail(&sh->lru, &conf->hold_list);  in raid5_activate_delayed()
5391 			raid5_wakeup_stripe_thread(sh);  in raid5_activate_delayed()
5404 		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);  in activate_bit_delay()  local
5406 		list_del_init(&sh->lru);  in activate_bit_delay()
5407 		atomic_inc(&sh->count);  in activate_bit_delay()
5408 		hash = sh->hash_lock_index;  in activate_bit_delay()
5409 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in activate_bit_delay()
5625 	struct stripe_head *sh, *tmp;  in __get_priority_stripe()  local
5635 	sh = NULL;  in __get_priority_stripe()
5661 		sh = list_entry(handle_list->next, typeof(*sh), lru);  in __get_priority_stripe()
5665 		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {  in __get_priority_stripe()
5685 				sh = tmp;  in __get_priority_stripe()
5690 		if (sh) {  in __get_priority_stripe()
5698 	if (!sh) {  in __get_priority_stripe()
5708 		sh->group = NULL;  in __get_priority_stripe()
5710 	list_del_init(&sh->lru);  in __get_priority_stripe()
5711 	BUG_ON(atomic_inc_return(&sh->count) != 1);  in __get_priority_stripe()
5712 	return sh;  in __get_priority_stripe()
5725 	struct stripe_head *sh;  in raid5_unplug()  local
5734 			sh = list_first_entry(&cb->list, struct stripe_head, lru);  in raid5_unplug()
5735 			list_del_init(&sh->lru);  in raid5_unplug()
5742 			clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);  in raid5_unplug()
5747 			hash = sh->hash_lock_index;  in raid5_unplug()
5748 			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);  in raid5_unplug()
5761 				struct stripe_head *sh)  in release_stripe_plug()  argument
5769 		raid5_release_stripe(sh);  in release_stripe_plug()
5782 	if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))  in release_stripe_plug()
5783 		list_add_tail(&sh->lru, &cb->list);  in release_stripe_plug()
5785 		raid5_release_stripe(sh);  in release_stripe_plug()
5792 	struct stripe_head *sh;  in make_discard_request()  local
5822 		sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0);  in make_discard_request()
5825 		set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);  in make_discard_request()
5826 		if (test_bit(STRIPE_SYNCING, &sh->state)) {  in make_discard_request()
5827 			raid5_release_stripe(sh);  in make_discard_request()
5831 		clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);  in make_discard_request()
5832 		spin_lock_irq(&sh->stripe_lock);  in make_discard_request()
5834 			if (d == sh->pd_idx || d == sh->qd_idx)  in make_discard_request()
5836 			if (sh->dev[d].towrite || sh->dev[d].toread) {  in make_discard_request()
5837 				set_bit(R5_Overlap, &sh->dev[d].flags);  in make_discard_request()
5838 				spin_unlock_irq(&sh->stripe_lock);  in make_discard_request()
5839 				raid5_release_stripe(sh);  in make_discard_request()
5844 		set_bit(STRIPE_DISCARD, &sh->state);  in make_discard_request()
5846 		sh->overwrite_disks = 0;  in make_discard_request()
5848 			if (d == sh->pd_idx || d == sh->qd_idx)  in make_discard_request()
5850 			sh->dev[d].towrite = bi;  in make_discard_request()
5851 			set_bit(R5_OVERWRITE, &sh->dev[d].flags);  in make_discard_request()
5854 			sh->overwrite_disks++;  in make_discard_request()
5856 		spin_unlock_irq(&sh->stripe_lock);  in make_discard_request()
5862 						     sh->sector,  in make_discard_request()
5865 			sh->bm_seq = conf->seq_flush + 1;  in make_discard_request()
5866 			set_bit(STRIPE_BIT_DELAY, &sh->state);  in make_discard_request()
5869 		set_bit(STRIPE_HANDLE, &sh->state);  in make_discard_request()
5870 		clear_bit(STRIPE_DELAYED, &sh->state);  in make_discard_request()
5871 		if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in make_discard_request()
5873 		release_stripe_plug(mddev, sh);  in make_discard_request()
5894 				    struct stripe_head *sh)  in stripe_ahead_of_reshape()  argument
5900 	for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {  in stripe_ahead_of_reshape()
5901 		if (dd_idx == sh->pd_idx)  in stripe_ahead_of_reshape()
5904 		min_sector = min(min_sector, sh->dev[dd_idx].sector);  in stripe_ahead_of_reshape()
5905 		max_sector = min(max_sector, sh->dev[dd_idx].sector);  in stripe_ahead_of_reshape()
5921 		struct stripe_request_ctx *ctx, struct stripe_head *sh,  in add_all_stripe_bios()  argument
5927 	spin_lock_irq(&sh->stripe_lock);  in add_all_stripe_bios()
5929 	for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {  in add_all_stripe_bios()
5930 		struct r5dev *dev = &sh->dev[dd_idx];  in add_all_stripe_bios()
5932 		if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in add_all_stripe_bios()
5939 		if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {  in add_all_stripe_bios()
5949 	for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {  in add_all_stripe_bios()
5950 		struct r5dev *dev = &sh->dev[dd_idx];  in add_all_stripe_bios()
5952 		if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in add_all_stripe_bios()
5959 		__add_stripe_bio(sh, bi, dd_idx, forwrite, previous);  in add_all_stripe_bios()
5965 	spin_unlock_irq(&sh->stripe_lock);  in add_all_stripe_bios()
5975 	struct stripe_head *sh;  in make_stripe_request()  local
6015 	sh = raid5_get_active_stripe(conf, ctx, new_sector, flags);  in make_stripe_request()
6016 	if (unlikely(!sh)) {  in make_stripe_request()
6023 	    stripe_ahead_of_reshape(mddev, conf, sh)) {  in make_stripe_request()
6042 	if (test_bit(STRIPE_EXPANDING, &sh->state) ||  in make_stripe_request()
6043 	    !add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) {  in make_stripe_request()
6053 	if (stripe_can_batch(sh)) {  in make_stripe_request()
6054 		stripe_add_to_batch_list(conf, sh, ctx->batch_last);  in make_stripe_request()
6057 		atomic_inc(&sh->count);  in make_stripe_request()
6058 		ctx->batch_last = sh;  in make_stripe_request()
6062 		set_bit(STRIPE_R5C_PREFLUSH, &sh->state);  in make_stripe_request()
6067 	set_bit(STRIPE_HANDLE, &sh->state);  in make_stripe_request()
6068 	clear_bit(STRIPE_DELAYED, &sh->state);  in make_stripe_request()
6069 	if ((!sh->batch_head || sh == sh->batch_head) &&  in make_stripe_request()
6071 	    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in make_stripe_request()
6074 	release_stripe_plug(mddev, sh);  in make_stripe_request()
6078 	raid5_release_stripe(sh);  in make_stripe_request()
6213 	struct stripe_head *sh;  in reshape_request()  local
6364 		sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i,  in reshape_request()
6366 		set_bit(STRIPE_EXPANDING, &sh->state);  in reshape_request()
6371 		for (j=sh->disks; j--;) {  in reshape_request()
6373 			if (j == sh->pd_idx)  in reshape_request()
6376 			    j == sh->qd_idx)  in reshape_request()
6378 			s = raid5_compute_blocknr(sh, j, 0);  in reshape_request()
6383 			memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));  in reshape_request()
6384 			set_bit(R5_Expanded, &sh->dev[j].flags);  in reshape_request()
6385 			set_bit(R5_UPTODATE, &sh->dev[j].flags);  in reshape_request()
6388 			set_bit(STRIPE_EXPAND_READY, &sh->state);  in reshape_request()
6389 			set_bit(STRIPE_HANDLE, &sh->state);  in reshape_request()
6391 		list_add(&sh->lru, &stripes);  in reshape_request()
6414 		sh = raid5_get_active_stripe(conf, NULL, first_sector,  in reshape_request()
6416 		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);  in reshape_request()
6417 		set_bit(STRIPE_HANDLE, &sh->state);  in reshape_request()
6418 		raid5_release_stripe(sh);  in reshape_request()
6425 		sh = list_entry(stripes.next, struct stripe_head, lru);  in reshape_request()
6426 		list_del_init(&sh->lru);  in reshape_request()
6427 		raid5_release_stripe(sh);  in reshape_request()
6476 	struct stripe_head *sh;  in raid5_sync_request()  local
6535 	sh = raid5_get_active_stripe(conf, NULL, sector_nr,  in raid5_sync_request()
6537 	if (sh == NULL) {  in raid5_sync_request()
6538 		sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0);  in raid5_sync_request()
6559 	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);  in raid5_sync_request()
6560 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_sync_request()
6562 	raid5_release_stripe(sh);  in raid5_sync_request()
6580 	struct stripe_head *sh;  in retry_aligned_read()  local
6601 		sh = raid5_get_active_stripe(conf, NULL, sector,  in retry_aligned_read()
6603 		if (!sh) {  in retry_aligned_read()
6610 		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {  in retry_aligned_read()
6611 			raid5_release_stripe(sh);  in retry_aligned_read()
6617 		set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);  in retry_aligned_read()
6618 		handle_stripe(sh);  in retry_aligned_read()
6619 		raid5_release_stripe(sh);  in retry_aligned_read()
6635 	struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;  in handle_active_stripes()  local
6640 			(sh = __get_priority_stripe(conf, group)) != NULL)  in handle_active_stripes()
6641 		batch[batch_size++] = sh;  in handle_active_stripes()