112 static inline int raid6_d0(struct stripe_head *sh)  in raid6_d0()  argument
114 	if (sh->ddf_layout)  in raid6_d0()
118 	if (sh->qd_idx == sh->disks - 1)  in raid6_d0()
121 		return sh->qd_idx + 1;  in raid6_d0()
134 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,  in raid6_idx_to_slot()  argument
139 	if (sh->ddf_layout)  in raid6_idx_to_slot()
141 	if (idx == sh->pd_idx)  in raid6_idx_to_slot()
143 	if (idx == sh->qd_idx)  in raid6_idx_to_slot()
145 	if (!sh->ddf_layout)  in raid6_idx_to_slot()
152 static int stripe_operations_active(struct stripe_head *sh)  in stripe_operations_active()  argument
154 	return sh->check_state || sh->reconstruct_state ||  in stripe_operations_active()
155 	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||  in stripe_operations_active()
156 	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);  in stripe_operations_active()
159 static bool stripe_is_lowprio(struct stripe_head *sh)  in stripe_is_lowprio()  argument
161 	return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||  in stripe_is_lowprio()
162 		test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&  in stripe_is_lowprio()
163 	       !test_bit(STRIPE_R5C_CACHING, &sh->state);  in stripe_is_lowprio()
166 static void raid5_wakeup_stripe_thread(struct stripe_head *sh)  in raid5_wakeup_stripe_thread()  argument
168 	struct r5conf *conf = sh->raid_conf;  in raid5_wakeup_stripe_thread()
171 	int i, cpu = sh->cpu;  in raid5_wakeup_stripe_thread()
175 		sh->cpu = cpu;  in raid5_wakeup_stripe_thread()
178 	if (list_empty(&sh->lru)) {  in raid5_wakeup_stripe_thread()
181 		if (stripe_is_lowprio(sh))  in raid5_wakeup_stripe_thread()
182 			list_add_tail(&sh->lru, &group->loprio_list);  in raid5_wakeup_stripe_thread()
184 			list_add_tail(&sh->lru, &group->handle_list);  in raid5_wakeup_stripe_thread()
186 		sh->group = group;  in raid5_wakeup_stripe_thread()
194 	group = conf->worker_groups + cpu_to_group(sh->cpu);  in raid5_wakeup_stripe_thread()
198 	queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);  in raid5_wakeup_stripe_thread()
205 			queue_work_on(sh->cpu, raid5_wq,  in raid5_wakeup_stripe_thread()
212 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,  in do_release_stripe()  argument
218 	BUG_ON(!list_empty(&sh->lru));  in do_release_stripe()
222 		for (i = sh->disks; i--; )  in do_release_stripe()
223 			if (test_bit(R5_InJournal, &sh->dev[i].flags))  in do_release_stripe()
232 	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||  in do_release_stripe()
234 	     !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {  in do_release_stripe()
235 		if (test_bit(STRIPE_R5C_CACHING, &sh->state))  in do_release_stripe()
236 			r5c_make_stripe_write_out(sh);  in do_release_stripe()
237 		set_bit(STRIPE_HANDLE, &sh->state);  in do_release_stripe()
240 	if (test_bit(STRIPE_HANDLE, &sh->state)) {  in do_release_stripe()
241 		if (test_bit(STRIPE_DELAYED, &sh->state) &&  in do_release_stripe()
242 		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in do_release_stripe()
243 			list_add_tail(&sh->lru, &conf->delayed_list);  in do_release_stripe()
244 		else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&  in do_release_stripe()
245 			   sh->bm_seq - conf->seq_write > 0)  in do_release_stripe()
246 			list_add_tail(&sh->lru, &conf->bitmap_list);  in do_release_stripe()
248 			clear_bit(STRIPE_DELAYED, &sh->state);  in do_release_stripe()
249 			clear_bit(STRIPE_BIT_DELAY, &sh->state);  in do_release_stripe()
251 				if (stripe_is_lowprio(sh))  in do_release_stripe()
252 					list_add_tail(&sh->lru,  in do_release_stripe()
255 					list_add_tail(&sh->lru,  in do_release_stripe()
258 				raid5_wakeup_stripe_thread(sh);  in do_release_stripe()
264 		BUG_ON(stripe_operations_active(sh));  in do_release_stripe()
265 		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in do_release_stripe()
270 		if (!test_bit(STRIPE_EXPANDING, &sh->state)) {  in do_release_stripe()
272 				list_add_tail(&sh->lru, temp_inactive_list);  in do_release_stripe()
274 				WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));  in do_release_stripe()
276 					list_add_tail(&sh->lru, temp_inactive_list);  in do_release_stripe()
279 					if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))  in do_release_stripe()
281 					if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))  in do_release_stripe()
283 					list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);  in do_release_stripe()
291 					list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);  in do_release_stripe()
297 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,  in __release_stripe()  argument
300 	if (atomic_dec_and_test(&sh->count))  in __release_stripe()
301 		do_release_stripe(conf, sh, temp_inactive_list);  in __release_stripe()
357 	struct stripe_head *sh, *t;  in release_stripe_list()  local
363 	llist_for_each_entry_safe(sh, t, head, release_list) {  in release_stripe_list()
368 		clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);  in release_stripe_list()
374 		hash = sh->hash_lock_index;  in release_stripe_list()
375 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in release_stripe_list()
382 void raid5_release_stripe(struct stripe_head *sh)  in raid5_release_stripe()  argument
384 	struct r5conf *conf = sh->raid_conf;  in raid5_release_stripe()
392 	if (atomic_add_unless(&sh->count, -1, 1))  in raid5_release_stripe()
396 		test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))  in raid5_release_stripe()
398 	wakeup = llist_add(&sh->release_list, &conf->released_stripes);  in raid5_release_stripe()
404 	if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {  in raid5_release_stripe()
406 		hash = sh->hash_lock_index;  in raid5_release_stripe()
407 		do_release_stripe(conf, sh, &list);  in raid5_release_stripe()
413 static inline void remove_hash(struct stripe_head *sh)  in remove_hash()  argument
416 		(unsigned long long)sh->sector);  in remove_hash()
418 	hlist_del_init(&sh->hash);  in remove_hash()
421 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)  in insert_hash()  argument
423 	struct hlist_head *hp = stripe_hash(conf, sh->sector);  in insert_hash()
426 		(unsigned long long)sh->sector);  in insert_hash()
428 	hlist_add_head(&sh->hash, hp);  in insert_hash()
434 	struct stripe_head *sh = NULL;  in get_free_stripe()  local
440 	sh = list_entry(first, struct stripe_head, lru);  in get_free_stripe()
442 	remove_hash(sh);  in get_free_stripe()
444 	BUG_ON(hash != sh->hash_lock_index);  in get_free_stripe()
448 	return sh;  in get_free_stripe()
452 static void free_stripe_pages(struct stripe_head *sh)  in free_stripe_pages()  argument
458 	if (!sh->pages)  in free_stripe_pages()
461 	for (i = 0; i < sh->nr_pages; i++) {  in free_stripe_pages()
462 		p = sh->pages[i];  in free_stripe_pages()
465 		sh->pages[i] = NULL;  in free_stripe_pages()
469 static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)  in alloc_stripe_pages()  argument
474 	for (i = 0; i < sh->nr_pages; i++) {  in alloc_stripe_pages()
476 		if (sh->pages[i])  in alloc_stripe_pages()
481 			free_stripe_pages(sh);  in alloc_stripe_pages()
484 		sh->pages[i] = p;  in alloc_stripe_pages()
490 init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)  in init_stripe_shared_pages()  argument
494 	if (sh->pages)  in init_stripe_shared_pages()
501 	sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);  in init_stripe_shared_pages()
502 	if (!sh->pages)  in init_stripe_shared_pages()
504 	sh->nr_pages = nr_pages;  in init_stripe_shared_pages()
505 	sh->stripes_per_page = cnt;  in init_stripe_shared_pages()
510 static void shrink_buffers(struct stripe_head *sh)  in shrink_buffers()  argument
513 	int num = sh->raid_conf->pool_size;  in shrink_buffers()
519 		WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);  in shrink_buffers()
520 		p = sh->dev[i].page;  in shrink_buffers()
523 		sh->dev[i].page = NULL;  in shrink_buffers()
528 		sh->dev[i].page = NULL;  in shrink_buffers()
529 	free_stripe_pages(sh); /* Free pages */  in shrink_buffers()
533 static int grow_buffers(struct stripe_head *sh, gfp_t gfp)  in grow_buffers()  argument
536 	int num = sh->raid_conf->pool_size;  in grow_buffers()
545 		sh->dev[i].page = page;  in grow_buffers()
546 		sh->dev[i].orig_page = page;  in grow_buffers()
547 		sh->dev[i].offset = 0;  in grow_buffers()
550 	if (alloc_stripe_pages(sh, gfp))  in grow_buffers()
554 		sh->dev[i].page = raid5_get_dev_page(sh, i);  in grow_buffers()
555 		sh->dev[i].orig_page = sh->dev[i].page;  in grow_buffers()
556 		sh->dev[i].offset = raid5_get_page_offset(sh, i);  in grow_buffers()
563 			    struct stripe_head *sh);
565 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)  in init_stripe()  argument
567 	struct r5conf *conf = sh->raid_conf;  in init_stripe()
570 	BUG_ON(atomic_read(&sh->count) != 0);  in init_stripe()
571 	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));  in init_stripe()
572 	BUG_ON(stripe_operations_active(sh));  in init_stripe()
573 	BUG_ON(sh->batch_head);  in init_stripe()
579 	sh->generation = conf->generation - previous;  in init_stripe()
580 	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;  in init_stripe()
581 	sh->sector = sector;  in init_stripe()
582 	stripe_set_idx(sector, conf, previous, sh);  in init_stripe()
583 	sh->state = 0;  in init_stripe()
585 	for (i = sh->disks; i--; ) {  in init_stripe()
586 		struct r5dev *dev = &sh->dev[i];  in init_stripe()
591 			       (unsigned long long)sh->sector, i, dev->toread,  in init_stripe()
597 		dev->sector = raid5_compute_blocknr(sh, i, previous);  in init_stripe()
601 	sh->overwrite_disks = 0;  in init_stripe()
602 	insert_hash(conf, sh);  in init_stripe()
603 	sh->cpu = smp_processor_id();  in init_stripe()
604 	set_bit(STRIPE_BATCH_READY, &sh->state);  in init_stripe()
610 	struct stripe_head *sh;  in __find_stripe()  local
613 	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)  in __find_stripe()
614 		if (sh->sector == sector && sh->generation == generation)  in __find_stripe()
615 			return sh;  in __find_stripe()
706 	struct stripe_head *sh;  in raid5_get_active_stripe()  local
718 		sh = __find_stripe(conf, sector, conf->generation - previous);  in raid5_get_active_stripe()
719 		if (!sh) {  in raid5_get_active_stripe()
721 				sh = get_free_stripe(conf, hash);  in raid5_get_active_stripe()
722 				if (!sh && !test_bit(R5_DID_ALLOC,  in raid5_get_active_stripe()
727 			if (noblock && sh == NULL)  in raid5_get_active_stripe()
731 			if (!sh) {  in raid5_get_active_stripe()
746 				init_stripe(sh, sector, previous);  in raid5_get_active_stripe()
747 				atomic_inc(&sh->count);  in raid5_get_active_stripe()
749 		} else if (!atomic_inc_not_zero(&sh->count)) {  in raid5_get_active_stripe()
751 			if (!atomic_read(&sh->count)) {  in raid5_get_active_stripe()
752 				if (!test_bit(STRIPE_HANDLE, &sh->state))  in raid5_get_active_stripe()
754 				BUG_ON(list_empty(&sh->lru) &&  in raid5_get_active_stripe()
755 				       !test_bit(STRIPE_EXPANDING, &sh->state));  in raid5_get_active_stripe()
759 				list_del_init(&sh->lru);  in raid5_get_active_stripe()
762 				if (sh->group) {  in raid5_get_active_stripe()
763 					sh->group->stripes_cnt--;  in raid5_get_active_stripe()
764 					sh->group = NULL;  in raid5_get_active_stripe()
767 			atomic_inc(&sh->count);  in raid5_get_active_stripe()
770 	} while (sh == NULL);  in raid5_get_active_stripe()
773 	return sh;  in raid5_get_active_stripe()
776 static bool is_full_stripe_write(struct stripe_head *sh)  in is_full_stripe_write()  argument
778 	BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));  in is_full_stripe_write()
779 	return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);  in is_full_stripe_write()
804 static bool stripe_can_batch(struct stripe_head *sh)  in stripe_can_batch()  argument
806 	struct r5conf *conf = sh->raid_conf;  in stripe_can_batch()
810 	return test_bit(STRIPE_BATCH_READY, &sh->state) &&  in stripe_can_batch()
811 		!test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&  in stripe_can_batch()
812 		is_full_stripe_write(sh);  in stripe_can_batch()
816 static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)  in stripe_add_to_batch_list()  argument
825 	tmp_sec = sh->sector;  in stripe_add_to_batch_list()
828 	head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);  in stripe_add_to_batch_list()
861 	lock_two_stripes(head, sh);  in stripe_add_to_batch_list()
863 	if (!stripe_can_batch(head) || !stripe_can_batch(sh))  in stripe_add_to_batch_list()
866 	if (sh->batch_head)  in stripe_add_to_batch_list()
870 	while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)  in stripe_add_to_batch_list()
872 	if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||  in stripe_add_to_batch_list()
873 	    bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))  in stripe_add_to_batch_list()
890 		sh->batch_head = head->batch_head;  in stripe_add_to_batch_list()
896 		list_add(&sh->batch_list, &head->batch_list);  in stripe_add_to_batch_list()
900 		sh->batch_head = head->batch_head;  in stripe_add_to_batch_list()
902 		list_add_tail(&sh->batch_list, &head->batch_list);  in stripe_add_to_batch_list()
906 	if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in stripe_add_to_batch_list()
911 	if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {  in stripe_add_to_batch_list()
912 		int seq = sh->bm_seq;  in stripe_add_to_batch_list()
913 		if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&  in stripe_add_to_batch_list()
914 		    sh->batch_head->bm_seq > seq)  in stripe_add_to_batch_list()
915 			seq = sh->batch_head->bm_seq;  in stripe_add_to_batch_list()
916 		set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);  in stripe_add_to_batch_list()
917 		sh->batch_head->bm_seq = seq;  in stripe_add_to_batch_list()
920 	atomic_inc(&sh->count);  in stripe_add_to_batch_list()
922 	unlock_two_stripes(head, sh);  in stripe_add_to_batch_list()
930 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)  in use_new_offset()  argument
940 	if (sh->generation == conf->generation - 1)  in use_new_offset()
1056 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)  in ops_run_io()  argument
1058 	struct r5conf *conf = sh->raid_conf;  in ops_run_io()
1059 	int i, disks = sh->disks;  in ops_run_io()
1060 	struct stripe_head *head_sh = sh;  in ops_run_io()
1066 	if (log_stripe(sh, s) == 0)  in ops_run_io()
1077 		sh = head_sh;  in ops_run_io()
1078 		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {  in ops_run_io()
1080 			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))  in ops_run_io()
1082 			if (test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_io()
1084 		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))  in ops_run_io()
1087 					    &sh->dev[i].flags)) {  in ops_run_io()
1092 		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))  in ops_run_io()
1096 		bi = &sh->dev[i].req;  in ops_run_io()
1097 		rbi = &sh->dev[i].rreq; /* For writing to replacement */  in ops_run_io()
1137 			int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in ops_run_io()
1171 			set_bit(STRIPE_IO_STARTED, &sh->state);  in ops_run_io()
1178 			bi->bi_private = sh;  in ops_run_io()
1181 				__func__, (unsigned long long)sh->sector,  in ops_run_io()
1183 			atomic_inc(&sh->count);  in ops_run_io()
1184 			if (sh != head_sh)  in ops_run_io()
1186 			if (use_new_offset(conf, sh))  in ops_run_io()
1187 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1190 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1195 			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))  in ops_run_io()
1196 				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in ops_run_io()
1199 			    test_bit(R5_InJournal, &sh->dev[i].flags))  in ops_run_io()
1205 				sh->dev[i].vec.bv_page = sh->dev[i].orig_page;  in ops_run_io()
1207 				sh->dev[i].vec.bv_page = sh->dev[i].page;  in ops_run_io()
1210 			bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;  in ops_run_io()
1212 			bi->bi_write_hint = sh->dev[i].write_hint;  in ops_run_io()
1214 				sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;  in ops_run_io()
1222 				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);  in ops_run_io()
1227 						      sh->dev[i].sector);  in ops_run_io()
1238 			set_bit(STRIPE_IO_STARTED, &sh->state);  in ops_run_io()
1244 			rbi->bi_private = sh;  in ops_run_io()
1248 				__func__, (unsigned long long)sh->sector,  in ops_run_io()
1250 			atomic_inc(&sh->count);  in ops_run_io()
1251 			if (sh != head_sh)  in ops_run_io()
1253 			if (use_new_offset(conf, sh))  in ops_run_io()
1254 				rbi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1257 				rbi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1259 			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))  in ops_run_io()
1260 				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in ops_run_io()
1261 			sh->dev[i].rvec.bv_page = sh->dev[i].page;  in ops_run_io()
1264 			rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;  in ops_run_io()
1266 			rbi->bi_write_hint = sh->dev[i].write_hint;  in ops_run_io()
1267 			sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;  in ops_run_io()
1277 						      sh->dev[i].sector);  in ops_run_io()
1285 				set_bit(STRIPE_DEGRADED, &sh->state);  in ops_run_io()
1287 				bi->bi_opf, i, (unsigned long long)sh->sector);  in ops_run_io()
1288 			clear_bit(R5_LOCKED, &sh->dev[i].flags);  in ops_run_io()
1289 			set_bit(STRIPE_HANDLE, &sh->state);  in ops_run_io()
1294 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_io()
1296 		if (sh != head_sh)  in ops_run_io()
1307 	struct stripe_head *sh, int no_skipcopy)  in async_copy_data()  argument
1315 	struct r5conf *conf = sh->raid_conf;  in async_copy_data()
1371 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_biofill()  local
1373 	struct r5conf *conf = sh->raid_conf;  in ops_complete_biofill()
1376 		(unsigned long long)sh->sector);  in ops_complete_biofill()
1379 	for (i = sh->disks; i--; ) {  in ops_complete_biofill()
1380 		struct r5dev *dev = &sh->dev[i];  in ops_complete_biofill()
1401 	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);  in ops_complete_biofill()
1403 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_biofill()
1404 	raid5_release_stripe(sh);  in ops_complete_biofill()
1407 static void ops_run_biofill(struct stripe_head *sh)  in ops_run_biofill()  argument
1412 	struct r5conf *conf = sh->raid_conf;  in ops_run_biofill()
1414 	BUG_ON(sh->batch_head);  in ops_run_biofill()
1416 		(unsigned long long)sh->sector);  in ops_run_biofill()
1418 	for (i = sh->disks; i--; ) {  in ops_run_biofill()
1419 		struct r5dev *dev = &sh->dev[i];  in ops_run_biofill()
1422 			spin_lock_irq(&sh->stripe_lock);  in ops_run_biofill()
1425 			spin_unlock_irq(&sh->stripe_lock);  in ops_run_biofill()
1430 						     dev->sector, tx, sh, 0);  in ops_run_biofill()
1436 	atomic_inc(&sh->count);  in ops_run_biofill()
1437 	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);  in ops_run_biofill()
1441 static void mark_target_uptodate(struct stripe_head *sh, int target)  in mark_target_uptodate()  argument
1448 	tgt = &sh->dev[target];  in mark_target_uptodate()
1456 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_compute()  local
1459 		(unsigned long long)sh->sector);  in ops_complete_compute()
1462 	mark_target_uptodate(sh, sh->ops.target);  in ops_complete_compute()
1463 	mark_target_uptodate(sh, sh->ops.target2);  in ops_complete_compute()
1465 	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);  in ops_complete_compute()
1466 	if (sh->check_state == check_state_compute_run)  in ops_complete_compute()
1467 		sh->check_state = check_state_compute_result;  in ops_complete_compute()
1468 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_compute()
1469 	raid5_release_stripe(sh);  in ops_complete_compute()
1479 static addr_conv_t *to_addr_conv(struct stripe_head *sh,  in to_addr_conv()  argument
1482 	return (void *) (to_addr_page(percpu, i) + sh->disks + 2);  in to_addr_conv()
1489 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)  in to_addr_offs()  argument
1491 	return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);  in to_addr_offs()
1495 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute5()  argument
1497 	int disks = sh->disks;  in ops_run_compute5()
1499 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_compute5()
1500 	int target = sh->ops.target;  in ops_run_compute5()
1501 	struct r5dev *tgt = &sh->dev[target];  in ops_run_compute5()
1509 	BUG_ON(sh->batch_head);  in ops_run_compute5()
1512 		__func__, (unsigned long long)sh->sector, target);  in ops_run_compute5()
1517 			off_srcs[count] = sh->dev[i].offset;  in ops_run_compute5()
1518 			xor_srcs[count++] = sh->dev[i].page;  in ops_run_compute5()
1522 	atomic_inc(&sh->count);  in ops_run_compute5()
1525 			  ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));  in ops_run_compute5()
1528 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute5()
1531 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute5()
1548 				struct stripe_head *sh,  in set_syndrome_sources()  argument
1551 	int disks = sh->disks;  in set_syndrome_sources()
1552 	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);  in set_syndrome_sources()
1553 	int d0_idx = raid6_d0(sh);  in set_syndrome_sources()
1563 		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);  in set_syndrome_sources()
1564 		struct r5dev *dev = &sh->dev[i];  in set_syndrome_sources()
1566 		if (i == sh->qd_idx || i == sh->pd_idx ||  in set_syndrome_sources()
1575 				srcs[slot] = sh->dev[i].orig_page;  in set_syndrome_sources()
1577 				srcs[slot] = sh->dev[i].page;  in set_syndrome_sources()
1583 			offs[slot] = sh->dev[i].offset;  in set_syndrome_sources()
1592 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_1()  argument
1594 	int disks = sh->disks;  in ops_run_compute6_1()
1596 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_1()
1598 	int qd_idx = sh->qd_idx;  in ops_run_compute6_1()
1607 	BUG_ON(sh->batch_head);  in ops_run_compute6_1()
1608 	if (sh->ops.target < 0)  in ops_run_compute6_1()
1609 		target = sh->ops.target2;  in ops_run_compute6_1()
1610 	else if (sh->ops.target2 < 0)  in ops_run_compute6_1()
1611 		target = sh->ops.target;  in ops_run_compute6_1()
1617 		__func__, (unsigned long long)sh->sector, target);  in ops_run_compute6_1()
1619 	tgt = &sh->dev[target];  in ops_run_compute6_1()
1624 	atomic_inc(&sh->count);  in ops_run_compute6_1()
1627 		count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);  in ops_run_compute6_1()
1631 				  ops_complete_compute, sh,  in ops_run_compute6_1()
1632 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1634 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute6_1()
1641 			offs[count] = sh->dev[i].offset;  in ops_run_compute6_1()
1642 			blocks[count++] = sh->dev[i].page;  in ops_run_compute6_1()
1646 				  NULL, ops_complete_compute, sh,  in ops_run_compute6_1()
1647 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_1()
1649 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_compute6_1()
1656 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_compute6_2()  argument
1658 	int i, count, disks = sh->disks;  in ops_run_compute6_2()
1659 	int syndrome_disks = sh->ddf_layout ? disks : disks-2;  in ops_run_compute6_2()
1660 	int d0_idx = raid6_d0(sh);  in ops_run_compute6_2()
1662 	int target = sh->ops.target;  in ops_run_compute6_2()
1663 	int target2 = sh->ops.target2;  in ops_run_compute6_2()
1664 	struct r5dev *tgt = &sh->dev[target];  in ops_run_compute6_2()
1665 	struct r5dev *tgt2 = &sh->dev[target2];  in ops_run_compute6_2()
1668 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_compute6_2()
1671 	BUG_ON(sh->batch_head);  in ops_run_compute6_2()
1673 		 __func__, (unsigned long long)sh->sector, target, target2);  in ops_run_compute6_2()
1688 		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);  in ops_run_compute6_2()
1690 		offs[slot] = sh->dev[i].offset;  in ops_run_compute6_2()
1691 		blocks[slot] = sh->dev[i].page;  in ops_run_compute6_2()
1704 		 __func__, (unsigned long long)sh->sector, faila, failb);  in ops_run_compute6_2()
1706 	atomic_inc(&sh->count);  in ops_run_compute6_2()
1713 					  ops_complete_compute, sh,  in ops_run_compute6_2()
1714 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1716 						  RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1722 			int qd_idx = sh->qd_idx;  in ops_run_compute6_2()
1734 				offs[count] = sh->dev[i].offset;  in ops_run_compute6_2()
1735 				blocks[count++] = sh->dev[i].page;  in ops_run_compute6_2()
1737 			dest = sh->dev[data_target].page;  in ops_run_compute6_2()
1738 			dest_off = sh->dev[data_target].offset;  in ops_run_compute6_2()
1742 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1744 				       RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1747 			count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);  in ops_run_compute6_2()
1749 					  ops_complete_compute, sh,  in ops_run_compute6_2()
1750 					  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1752 						  RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1757 				  ops_complete_compute, sh,  in ops_run_compute6_2()
1758 				  to_addr_conv(sh, percpu, 0));  in ops_run_compute6_2()
1762 						RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1768 						RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_compute6_2()
1777 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_prexor()  local
1780 		(unsigned long long)sh->sector);  in ops_complete_prexor()
1782 	if (r5c_is_writeback(sh->raid_conf->log))  in ops_complete_prexor()
1787 		r5c_release_extra_page(sh);  in ops_complete_prexor()
1791 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor5()  argument
1794 	int disks = sh->disks;  in ops_run_prexor5()
1796 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_prexor5()
1797 	int count = 0, pd_idx = sh->pd_idx, i;  in ops_run_prexor5()
1801 	unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;  in ops_run_prexor5()
1802 	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;  in ops_run_prexor5()
1804 	BUG_ON(sh->batch_head);  in ops_run_prexor5()
1806 		(unsigned long long)sh->sector);  in ops_run_prexor5()
1809 		struct r5dev *dev = &sh->dev[i];  in ops_run_prexor5()
1825 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor5()
1827 			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_prexor5()
1833 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_prexor6()  argument
1837 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_prexor6()
1842 		(unsigned long long)sh->sector);  in ops_run_prexor6()
1844 	count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);  in ops_run_prexor6()
1847 			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));  in ops_run_prexor6()
1849 			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_prexor6()
1855 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)  in ops_run_biodrain()  argument
1857 	struct r5conf *conf = sh->raid_conf;  in ops_run_biodrain()
1858 	int disks = sh->disks;  in ops_run_biodrain()
1860 	struct stripe_head *head_sh = sh;  in ops_run_biodrain()
1863 		(unsigned long long)sh->sector);  in ops_run_biodrain()
1869 		sh = head_sh;  in ops_run_biodrain()
1874 			dev = &sh->dev[i];  in ops_run_biodrain()
1880 			spin_lock_irq(&sh->stripe_lock);  in ops_run_biodrain()
1883 			sh->overwrite_disks = 0;  in ops_run_biodrain()
1886 			spin_unlock_irq(&sh->stripe_lock);  in ops_run_biodrain()
1900 							     dev->sector, tx, sh,  in ops_run_biodrain()
1913 				sh = list_first_entry(&sh->batch_list,  in ops_run_biodrain()
1916 				if (sh == head_sh)  in ops_run_biodrain()
1928 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_reconstruct()  local
1929 	int disks = sh->disks;  in ops_complete_reconstruct()
1930 	int pd_idx = sh->pd_idx;  in ops_complete_reconstruct()
1931 	int qd_idx = sh->qd_idx;  in ops_complete_reconstruct()
1936 		(unsigned long long)sh->sector);  in ops_complete_reconstruct()
1939 		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);  in ops_complete_reconstruct()
1940 		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);  in ops_complete_reconstruct()
1941 		discard |= test_bit(R5_Discard, &sh->dev[i].flags);  in ops_complete_reconstruct()
1945 		struct r5dev *dev = &sh->dev[i];  in ops_complete_reconstruct()
1950 				if (test_bit(STRIPE_EXPAND_READY, &sh->state))  in ops_complete_reconstruct()
1960 	if (sh->reconstruct_state == reconstruct_state_drain_run)  in ops_complete_reconstruct()
1961 		sh->reconstruct_state = reconstruct_state_drain_result;  in ops_complete_reconstruct()
1962 	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)  in ops_complete_reconstruct()
1963 		sh->reconstruct_state = reconstruct_state_prexor_drain_result;  in ops_complete_reconstruct()
1965 		BUG_ON(sh->reconstruct_state != reconstruct_state_run);  in ops_complete_reconstruct()
1966 		sh->reconstruct_state = reconstruct_state_result;  in ops_complete_reconstruct()
1969 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_reconstruct()
1970 	raid5_release_stripe(sh);  in ops_complete_reconstruct()
1974 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct5()  argument
1977 	int disks = sh->disks;  in ops_run_reconstruct5()
1981 	int count, pd_idx = sh->pd_idx, i;  in ops_run_reconstruct5()
1987 	struct stripe_head *head_sh = sh;  in ops_run_reconstruct5()
1991 		(unsigned long long)sh->sector);  in ops_run_reconstruct5()
1993 	for (i = 0; i < sh->disks; i++) {  in ops_run_reconstruct5()
1996 		if (!test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_reconstruct5()
1999 	if (i >= sh->disks) {  in ops_run_reconstruct5()
2000 		atomic_inc(&sh->count);  in ops_run_reconstruct5()
2001 		set_bit(R5_Discard, &sh->dev[pd_idx].flags);  in ops_run_reconstruct5()
2002 		ops_complete_reconstruct(sh);  in ops_run_reconstruct5()
2008 	off_srcs = to_addr_offs(sh, percpu);  in ops_run_reconstruct5()
2014 		off_dest = off_srcs[count] = sh->dev[pd_idx].offset;  in ops_run_reconstruct5()
2015 		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;  in ops_run_reconstruct5()
2017 			struct r5dev *dev = &sh->dev[i];  in ops_run_reconstruct5()
2025 		xor_dest = sh->dev[pd_idx].page;  in ops_run_reconstruct5()
2026 		off_dest = sh->dev[pd_idx].offset;  in ops_run_reconstruct5()
2028 			struct r5dev *dev = &sh->dev[i];  in ops_run_reconstruct5()
2042 		list_first_entry(&sh->batch_list,  in ops_run_reconstruct5()
2050 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2054 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct5()
2059 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_reconstruct5()
2062 				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);  in ops_run_reconstruct5()
2065 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_reconstruct5()
2072 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,  in ops_run_reconstruct6()  argument
2079 	struct stripe_head *head_sh = sh;  in ops_run_reconstruct6()
2084 	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);  in ops_run_reconstruct6()
2086 	for (i = 0; i < sh->disks; i++) {  in ops_run_reconstruct6()
2087 		if (sh->pd_idx == i || sh->qd_idx == i)  in ops_run_reconstruct6()
2089 		if (!test_bit(R5_Discard, &sh->dev[i].flags))  in ops_run_reconstruct6()
2092 	if (i >= sh->disks) {  in ops_run_reconstruct6()
2093 		atomic_inc(&sh->count);  in ops_run_reconstruct6()
2094 		set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);  in ops_run_reconstruct6()
2095 		set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);  in ops_run_reconstruct6()
2096 		ops_complete_reconstruct(sh);  in ops_run_reconstruct6()
2102 	offs = to_addr_offs(sh, percpu);  in ops_run_reconstruct6()
2104 	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {  in ops_run_reconstruct6()
2112 	count = set_syndrome_sources(blocks, offs, sh, synflags);  in ops_run_reconstruct6()
2114 		list_first_entry(&sh->batch_list,  in ops_run_reconstruct6()
2120 				  head_sh, to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2123 				  to_addr_conv(sh, percpu, j));  in ops_run_reconstruct6()
2125 			RAID5_STRIPE_SIZE(sh->raid_conf),  &submit);  in ops_run_reconstruct6()
2128 		sh = list_first_entry(&sh->batch_list, struct stripe_head,  in ops_run_reconstruct6()
2136 	struct stripe_head *sh = stripe_head_ref;  in ops_complete_check()  local
2139 		(unsigned long long)sh->sector);  in ops_complete_check()
2141 	sh->check_state = check_state_check_result;  in ops_complete_check()
2142 	set_bit(STRIPE_HANDLE, &sh->state);  in ops_complete_check()
2143 	raid5_release_stripe(sh);  in ops_complete_check()
2146 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)  in ops_run_check_p()  argument
2148 	int disks = sh->disks;  in ops_run_check_p()
2149 	int pd_idx = sh->pd_idx;  in ops_run_check_p()
2150 	int qd_idx = sh->qd_idx;  in ops_run_check_p()
2154 	unsigned int *off_srcs = to_addr_offs(sh, percpu);  in ops_run_check_p()
2161 		(unsigned long long)sh->sector);  in ops_run_check_p()
2163 	BUG_ON(sh->batch_head);  in ops_run_check_p()
2165 	xor_dest = sh->dev[pd_idx].page;  in ops_run_check_p()
2166 	off_dest = sh->dev[pd_idx].offset;  in ops_run_check_p()
2172 		off_srcs[count] = sh->dev[i].offset;  in ops_run_check_p()
2173 		xor_srcs[count++] = sh->dev[i].page;  in ops_run_check_p()
2177 			  to_addr_conv(sh, percpu, 0));  in ops_run_check_p()
2179 			   RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_check_p()
2180 			   &sh->ops.zero_sum_result, &submit);  in ops_run_check_p()
2182 	atomic_inc(&sh->count);  in ops_run_check_p()
2183 	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);  in ops_run_check_p()
2187 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)  in ops_run_check_pq()  argument
2190 	unsigned int *offs = to_addr_offs(sh, percpu);  in ops_run_check_pq()
2195 		(unsigned long long)sh->sector, checkp);  in ops_run_check_pq()
2197 	BUG_ON(sh->batch_head);  in ops_run_check_pq()
2198 	count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);  in ops_run_check_pq()
2202 	atomic_inc(&sh->count);  in ops_run_check_pq()
2204 			  sh, to_addr_conv(sh, percpu, 0));  in ops_run_check_pq()
2206 			   RAID5_STRIPE_SIZE(sh->raid_conf),  in ops_run_check_pq()
2207 			   &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);  in ops_run_check_pq()
2210 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)  in raid_run_ops()  argument
2212 	int overlap_clear = 0, i, disks = sh->disks;  in raid_run_ops()
2214 	struct r5conf *conf = sh->raid_conf;  in raid_run_ops()
2222 		ops_run_biofill(sh);  in raid_run_ops()
2228 			tx = ops_run_compute5(sh, percpu);  in raid_run_ops()
2230 			if (sh->ops.target2 < 0 || sh->ops.target < 0)  in raid_run_ops()
2231 				tx = ops_run_compute6_1(sh, percpu);  in raid_run_ops()
2233 				tx = ops_run_compute6_2(sh, percpu);  in raid_run_ops()
2242 			tx = ops_run_prexor5(sh, percpu, tx);  in raid_run_ops()
2244 			tx = ops_run_prexor6(sh, percpu, tx);  in raid_run_ops()
2248 		tx = ops_run_partial_parity(sh, percpu, tx);  in raid_run_ops()
2251 		tx = ops_run_biodrain(sh, tx);  in raid_run_ops()
2257 			ops_run_reconstruct5(sh, percpu, tx);  in raid_run_ops()
2259 			ops_run_reconstruct6(sh, percpu, tx);  in raid_run_ops()
2263 		if (sh->check_state == check_state_run)  in raid_run_ops()
2264 			ops_run_check_p(sh, percpu);  in raid_run_ops()
2265 		else if (sh->check_state == check_state_run_q)  in raid_run_ops()
2266 			ops_run_check_pq(sh, percpu, 0);  in raid_run_ops()
2267 		else if (sh->check_state == check_state_run_pq)  in raid_run_ops()
2268 			ops_run_check_pq(sh, percpu, 1);  in raid_run_ops()
2273 	if (overlap_clear && !sh->batch_head)  in raid_run_ops()
2275 			struct r5dev *dev = &sh->dev[i];  in raid_run_ops()
2277 				wake_up(&sh->raid_conf->wait_for_overlap);  in raid_run_ops()
2282 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)  in free_stripe()  argument
2285 	kfree(sh->pages);  in free_stripe()
2287 	if (sh->ppl_page)  in free_stripe()
2288 		__free_page(sh->ppl_page);  in free_stripe()
2289 	kmem_cache_free(sc, sh);  in free_stripe()
2295 	struct stripe_head *sh;  in alloc_stripe()  local
2298 	sh = kmem_cache_zalloc(sc, gfp);  in alloc_stripe()
2299 	if (sh) {  in alloc_stripe()
2300 		spin_lock_init(&sh->stripe_lock);  in alloc_stripe()
2301 		spin_lock_init(&sh->batch_lock);  in alloc_stripe()
2302 		INIT_LIST_HEAD(&sh->batch_list);  in alloc_stripe()
2303 		INIT_LIST_HEAD(&sh->lru);  in alloc_stripe()
2304 		INIT_LIST_HEAD(&sh->r5c);  in alloc_stripe()
2305 		INIT_LIST_HEAD(&sh->log_list);  in alloc_stripe()
2306 		atomic_set(&sh->count, 1);  in alloc_stripe()
2307 		sh->raid_conf = conf;  in alloc_stripe()
2308 		sh->log_start = MaxSector;  in alloc_stripe()
2310 			struct r5dev *dev = &sh->dev[i];  in alloc_stripe()
2317 			sh->ppl_page = alloc_page(gfp);  in alloc_stripe()
2318 			if (!sh->ppl_page) {  in alloc_stripe()
2319 				free_stripe(sc, sh);  in alloc_stripe()
2324 		if (init_stripe_shared_pages(sh, conf, disks)) {  in alloc_stripe()
2325 			free_stripe(sc, sh);  in alloc_stripe()
2330 	return sh;  in alloc_stripe()
2334 	struct stripe_head *sh;  in grow_one_stripe()  local
2336 	sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);  in grow_one_stripe()
2337 	if (!sh)  in grow_one_stripe()
2340 	if (grow_buffers(sh, gfp)) {  in grow_one_stripe()
2341 		shrink_buffers(sh);  in grow_one_stripe()
2342 		free_stripe(conf->slab_cache, sh);  in grow_one_stripe()
2345 	sh->hash_lock_index =  in grow_one_stripe()
2350 	raid5_release_stripe(sh);  in grow_one_stripe()
2636 	struct stripe_head *sh;  in drop_one_stripe()  local
2640 	sh = get_free_stripe(conf, hash);  in drop_one_stripe()
2642 	if (!sh)  in drop_one_stripe()
2644 	BUG_ON(atomic_read(&sh->count));  in drop_one_stripe()
2645 	shrink_buffers(sh);  in drop_one_stripe()
2646 	free_stripe(conf->slab_cache, sh);  in drop_one_stripe()
2664 	struct stripe_head *sh = bi->bi_private;  in raid5_end_read_request()  local
2665 	struct r5conf *conf = sh->raid_conf;  in raid5_end_read_request()
2666 	int disks = sh->disks, i;  in raid5_end_read_request()
2672 		if (bi == &sh->dev[i].req)  in raid5_end_read_request()
2676 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),  in raid5_end_read_request()
2683 	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))  in raid5_end_read_request()
2693 	if (use_new_offset(conf, sh))  in raid5_end_read_request()
2694 		s = sh->sector + rdev->new_data_offset;  in raid5_end_read_request()
2696 		s = sh->sector + rdev->data_offset;  in raid5_end_read_request()
2698 		set_bit(R5_UPTODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2699 		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {  in raid5_end_read_request()
2710 			clear_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2711 			clear_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_read_request()
2712 		} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))  in raid5_end_read_request()
2713 			clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2715 		if (test_bit(R5_InJournal, &sh->dev[i].flags))  in raid5_end_read_request()
2720 			set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2729 		clear_bit(R5_UPTODATE, &sh->dev[i].flags);  in raid5_end_read_request()
2732 		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))  in raid5_end_read_request()
2745 		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {  in raid5_end_read_request()
2766 		    && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))  in raid5_end_read_request()
2769 			if (sh->qd_idx >= 0 && sh->pd_idx == i)  in raid5_end_read_request()
2770 				set_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2771 			else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {  in raid5_end_read_request()
2772 				set_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2773 				clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2775 				set_bit(R5_ReadNoMerge, &sh->dev[i].flags);  in raid5_end_read_request()
2777 			clear_bit(R5_ReadError, &sh->dev[i].flags);  in raid5_end_read_request()
2778 			clear_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_read_request()
2782 				      rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))  in raid5_end_read_request()
2788 	clear_bit(R5_LOCKED, &sh->dev[i].flags);  in raid5_end_read_request()
2789 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_end_read_request()
2790 	raid5_release_stripe(sh);  in raid5_end_read_request()
2795 	struct stripe_head *sh = bi->bi_private;  in raid5_end_write_request()  local
2796 	struct r5conf *conf = sh->raid_conf;  in raid5_end_write_request()
2797 	int disks = sh->disks, i;  in raid5_end_write_request()
2804 		if (bi == &sh->dev[i].req) {  in raid5_end_write_request()
2808 		if (bi == &sh->dev[i].rreq) {  in raid5_end_write_request()
2822 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),  in raid5_end_write_request()
2833 		else if (is_badblock(rdev, sh->sector,  in raid5_end_write_request()
2836 			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);  in raid5_end_write_request()
2839 			set_bit(STRIPE_DEGRADED, &sh->state);  in raid5_end_write_request()
2841 			set_bit(R5_WriteError, &sh->dev[i].flags);  in raid5_end_write_request()
2845 		} else if (is_badblock(rdev, sh->sector,  in raid5_end_write_request()
2848 			set_bit(R5_MadeGood, &sh->dev[i].flags);  in raid5_end_write_request()
2849 			if (test_bit(R5_ReadError, &sh->dev[i].flags))  in raid5_end_write_request()
2854 				set_bit(R5_ReWrite, &sh->dev[i].flags);  in raid5_end_write_request()
2859 	if (sh->batch_head && bi->bi_status && !replacement)  in raid5_end_write_request()
2860 		set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);  in raid5_end_write_request()
2863 	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))  in raid5_end_write_request()
2864 		clear_bit(R5_LOCKED, &sh->dev[i].flags);  in raid5_end_write_request()
2865 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_end_write_request()
2866 	raid5_release_stripe(sh);  in raid5_end_write_request()
2868 	if (sh->batch_head && sh != sh->batch_head)  in raid5_end_write_request()
2869 		raid5_release_stripe(sh->batch_head);  in raid5_end_write_request()
2916 			      struct stripe_head *sh)  in raid5_compute_sector()  argument
3104 	if (sh) {  in raid5_compute_sector()
3105 		sh->pd_idx = pd_idx;  in raid5_compute_sector()
3106 		sh->qd_idx = qd_idx;  in raid5_compute_sector()
3107 		sh->ddf_layout = ddf_layout;  in raid5_compute_sector()
3116 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)  in raid5_compute_blocknr()  argument
3118 	struct r5conf *conf = sh->raid_conf;  in raid5_compute_blocknr()
3119 	int raid_disks = sh->disks;  in raid5_compute_blocknr()
3121 	sector_t new_sector = sh->sector, check;  in raid5_compute_blocknr()
3136 	if (i == sh->pd_idx)  in raid5_compute_blocknr()
3144 			if (i > sh->pd_idx)  in raid5_compute_blocknr()
3149 			if (i < sh->pd_idx)  in raid5_compute_blocknr()
3151 			i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3163 		if (i == sh->qd_idx)  in raid5_compute_blocknr()
3170 			if (sh->pd_idx == raid_disks-1)  in raid5_compute_blocknr()
3172 			else if (i > sh->pd_idx)  in raid5_compute_blocknr()
3177 			if (sh->pd_idx == raid_disks-1)  in raid5_compute_blocknr()
3181 				if (i < sh->pd_idx)  in raid5_compute_blocknr()
3183 				i -= (sh->pd_idx + 2);  in raid5_compute_blocknr()
3193 			if (sh->pd_idx == 0)  in raid5_compute_blocknr()
3197 				if (i < sh->pd_idx)  in raid5_compute_blocknr()
3199 				i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3204 			if (i > sh->pd_idx)  in raid5_compute_blocknr()
3209 			if (i < sh->pd_idx)  in raid5_compute_blocknr()
3211 			i -= (sh->pd_idx + 1);  in raid5_compute_blocknr()
3227 	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx  in raid5_compute_blocknr()
3228 		|| sh2.qd_idx != sh->qd_idx) {  in raid5_compute_blocknr()
3293 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,  in schedule_reconstruction()  argument
3296 	int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;  in schedule_reconstruction()
3297 	struct r5conf *conf = sh->raid_conf;  in schedule_reconstruction()
3307 		r5c_release_extra_page(sh);  in schedule_reconstruction()
3310 			struct r5dev *dev = &sh->dev[i];  in schedule_reconstruction()
3331 			sh->reconstruct_state = reconstruct_state_drain_run;  in schedule_reconstruction()
3334 			sh->reconstruct_state = reconstruct_state_run;  in schedule_reconstruction()
3339 			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))  in schedule_reconstruction()
3342 		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||  in schedule_reconstruction()
3343 			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));  in schedule_reconstruction()
3345 			(!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||  in schedule_reconstruction()
3346 			   test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));  in schedule_reconstruction()
3349 			struct r5dev *dev = &sh->dev[i];  in schedule_reconstruction()
3368 		sh->reconstruct_state = reconstruct_state_prexor_drain_run;  in schedule_reconstruction()
3377 	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);  in schedule_reconstruction()
3378 	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);  in schedule_reconstruction()
3382 		int qd_idx = sh->qd_idx;  in schedule_reconstruction()
3383 		struct r5dev *dev = &sh->dev[qd_idx];  in schedule_reconstruction()
3390 	if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&  in schedule_reconstruction()
3392 	    !test_bit(STRIPE_FULL_WRITE, &sh->state) &&  in schedule_reconstruction()
3393 	    test_bit(R5_Insync, &sh->dev[pd_idx].flags))  in schedule_reconstruction()
3397 		__func__, (unsigned long long)sh->sector,  in schedule_reconstruction()
3406 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,  in add_stripe_bio()  argument
3410 	struct r5conf *conf = sh->raid_conf;  in add_stripe_bio()
3415 		(unsigned long long)sh->sector);  in add_stripe_bio()
3417 	spin_lock_irq(&sh->stripe_lock);  in add_stripe_bio()
3418 	sh->dev[dd_idx].write_hint = bi->bi_write_hint;  in add_stripe_bio()
3420 	if (sh->batch_head)  in add_stripe_bio()
3423 		bip = &sh->dev[dd_idx].towrite;  in add_stripe_bio()
3427 		bip = &sh->dev[dd_idx].toread;  in add_stripe_bio()
3450 		for (i = 0; i < sh->disks; i++) {  in add_stripe_bio()
3451 			if (i != sh->pd_idx &&  in add_stripe_bio()
3452 			    (i == dd_idx || sh->dev[i].towrite)) {  in add_stripe_bio()
3453 				sector = sh->dev[i].sector;  in add_stripe_bio()
3467 		clear_bit(STRIPE_BATCH_READY, &sh->state);  in add_stripe_bio()
3478 		sector_t sector = sh->dev[dd_idx].sector;  in add_stripe_bio()
3479 		for (bi=sh->dev[dd_idx].towrite;  in add_stripe_bio()
3480 		     sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&  in add_stripe_bio()
3482 		     bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {  in add_stripe_bio()
3486 		if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))  in add_stripe_bio()
3487 			if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))  in add_stripe_bio()
3488 				sh->overwrite_disks++;  in add_stripe_bio()
3493 		(unsigned long long)sh->sector, dd_idx);  in add_stripe_bio()
3508 		set_bit(STRIPE_BITMAP_PENDING, &sh->state);  in add_stripe_bio()
3509 		spin_unlock_irq(&sh->stripe_lock);  in add_stripe_bio()
3510 		md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,  in add_stripe_bio()
3512 		spin_lock_irq(&sh->stripe_lock);  in add_stripe_bio()
3513 		clear_bit(STRIPE_BITMAP_PENDING, &sh->state);  in add_stripe_bio()
3514 		if (!sh->batch_head) {  in add_stripe_bio()
3515 			sh->bm_seq = conf->seq_flush+1;  in add_stripe_bio()
3516 			set_bit(STRIPE_BIT_DELAY, &sh->state);  in add_stripe_bio()
3519 	spin_unlock_irq(&sh->stripe_lock);  in add_stripe_bio()
3521 	if (stripe_can_batch(sh))  in add_stripe_bio()
3522 		stripe_add_to_batch_list(conf, sh);  in add_stripe_bio()
3526 	set_bit(R5_Overlap, &sh->dev[dd_idx].flags);  in add_stripe_bio()
3527 	spin_unlock_irq(&sh->stripe_lock);  in add_stripe_bio()
3534 			    struct stripe_head *sh)  in stripe_set_idx()  argument
3546 			     &dd_idx, sh);  in stripe_set_idx()
3550 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_stripe()  argument
3554 	BUG_ON(sh->batch_head);  in handle_failed_stripe()
3559 		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {  in handle_failed_stripe()
3572 					    sh->sector,  in handle_failed_stripe()
3578 		spin_lock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3580 		bi = sh->dev[i].towrite;  in handle_failed_stripe()
3581 		sh->dev[i].towrite = NULL;  in handle_failed_stripe()
3582 		sh->overwrite_disks = 0;  in handle_failed_stripe()
3583 		spin_unlock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3587 		log_stripe_write_finished(sh);  in handle_failed_stripe()
3589 		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in handle_failed_stripe()
3593 			sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3594 			struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3601 			md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_failed_stripe()
3605 		bi = sh->dev[i].written;  in handle_failed_stripe()
3606 		sh->dev[i].written = NULL;  in handle_failed_stripe()
3607 		if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {  in handle_failed_stripe()
3608 			WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));  in handle_failed_stripe()
3609 			sh->dev[i].page = sh->dev[i].orig_page;  in handle_failed_stripe()
3614 		       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3615 			struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3625 		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&  in handle_failed_stripe()
3627 		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||  in handle_failed_stripe()
3628 		      test_bit(R5_ReadError, &sh->dev[i].flags))) {  in handle_failed_stripe()
3629 			spin_lock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3630 			bi = sh->dev[i].toread;  in handle_failed_stripe()
3631 			sh->dev[i].toread = NULL;  in handle_failed_stripe()
3632 			spin_unlock_irq(&sh->stripe_lock);  in handle_failed_stripe()
3633 			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in handle_failed_stripe()
3638 			       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3640 					r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3647 			md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_failed_stripe()
3652 		clear_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_failed_stripe()
3657 	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))  in handle_failed_stripe()
3663 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_sync()  argument
3669 	BUG_ON(sh->batch_head);  in handle_failed_sync()
3670 	clear_bit(STRIPE_SYNCING, &sh->state);  in handle_failed_sync()
3671 	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))  in handle_failed_sync()
3692 			    && !rdev_set_badblocks(rdev, sh->sector,  in handle_failed_sync()
3699 			    && !rdev_set_badblocks(rdev, sh->sector,  in handle_failed_sync()
3711 static int want_replace(struct stripe_head *sh, int disk_idx)  in want_replace()  argument
3717 	rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement);  in want_replace()
3721 	    && (rdev->recovery_offset <= sh->sector  in want_replace()
3722 		|| rdev->mddev->recovery_cp <= sh->sector))  in want_replace()
3728 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,  in need_this_block()  argument
3731 	struct r5dev *dev = &sh->dev[disk_idx];  in need_this_block()
3732 	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],  in need_this_block()
3733 				  &sh->dev[s->failed_num[1]] };  in need_this_block()
3735 	bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);  in need_this_block()
3751 	    (s->replacing && want_replace(sh, disk_idx)))  in need_this_block()
3776 	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in need_this_block()
3797 		     s->failed_num[i] == sh->pd_idx ||  in need_this_block()
3798 		     s->failed_num[i] == sh->qd_idx) &&  in need_this_block()
3815 	    sh->sector < sh->raid_conf->mddev->recovery_cp)  in need_this_block()
3819 		if (s->failed_num[i] != sh->pd_idx &&  in need_this_block()
3820 		    s->failed_num[i] != sh->qd_idx &&  in need_this_block()
3835 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,  in fetch_block()  argument
3838 	struct r5dev *dev = &sh->dev[disk_idx];  in fetch_block()
3841 	if (need_this_block(sh, s, disk_idx, disks)) {  in fetch_block()
3847 		BUG_ON(sh->batch_head);  in fetch_block()
3859 		    ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||  in fetch_block()
3866 			       (unsigned long long)sh->sector, disk_idx);  in fetch_block()
3867 			set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in fetch_block()
3870 			sh->ops.target = disk_idx;  in fetch_block()
3871 			sh->ops.target2 = -1; /* no 2nd target */  in fetch_block()
3890 				      &sh->dev[other].flags))  in fetch_block()
3895 			       (unsigned long long)sh->sector,  in fetch_block()
3897 			set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in fetch_block()
3899 			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);  in fetch_block()
3900 			set_bit(R5_Wantcompute, &sh->dev[other].flags);  in fetch_block()
3901 			sh->ops.target = disk_idx;  in fetch_block()
3902 			sh->ops.target2 = other;  in fetch_block()
3921 static void handle_stripe_fill(struct stripe_head *sh,  in handle_stripe_fill()  argument
3931 	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&  in handle_stripe_fill()
3932 	    !sh->reconstruct_state) {  in handle_stripe_fill()
3942 			if (test_bit(STRIPE_R5C_CACHING, &sh->state))  in handle_stripe_fill()
3943 				r5c_make_stripe_write_out(sh);  in handle_stripe_fill()
3948 			if (fetch_block(sh, s, i, disks))  in handle_stripe_fill()
3952 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_fill()
3963 	struct stripe_head *sh, int disks)  in handle_stripe_clean_event()  argument
3968 	struct stripe_head *head_sh = sh;  in handle_stripe_clean_event()
3972 		if (sh->dev[i].written) {  in handle_stripe_clean_event()
3973 			dev = &sh->dev[i];  in handle_stripe_clean_event()
3999 				md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,  in handle_stripe_clean_event()
4001 						   !test_bit(STRIPE_DEGRADED, &sh->state),  in handle_stripe_clean_event()
4004 					sh = list_first_entry(&sh->batch_list,  in handle_stripe_clean_event()
4007 					if (sh != head_sh) {  in handle_stripe_clean_event()
4008 						dev = &sh->dev[i];  in handle_stripe_clean_event()
4012 				sh = head_sh;  in handle_stripe_clean_event()
4013 				dev = &sh->dev[i];  in handle_stripe_clean_event()
4018 	log_stripe_write_finished(sh);  in handle_stripe_clean_event()
4021 	    test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {  in handle_stripe_clean_event()
4023 		clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);  in handle_stripe_clean_event()
4024 		clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);  in handle_stripe_clean_event()
4025 		if (sh->qd_idx >= 0) {  in handle_stripe_clean_event()
4026 			clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);  in handle_stripe_clean_event()
4027 			clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);  in handle_stripe_clean_event()
4030 		clear_bit(STRIPE_DISCARD, &sh->state);  in handle_stripe_clean_event()
4037 		hash = sh->hash_lock_index;  in handle_stripe_clean_event()
4039 		remove_hash(sh);  in handle_stripe_clean_event()
4042 			sh = list_first_entry(&sh->batch_list,  in handle_stripe_clean_event()
4044 			if (sh != head_sh)  in handle_stripe_clean_event()
4047 		sh = head_sh;  in handle_stripe_clean_event()
4049 		if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))  in handle_stripe_clean_event()
4050 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_clean_event()
4054 	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))  in handle_stripe_clean_event()
4078 				  struct stripe_head *sh,  in handle_stripe_dirtying()  argument
4093 	    (recovery_cp < MaxSector && sh->sector >= recovery_cp &&  in handle_stripe_dirtying()
4101 			 (unsigned long long)sh->sector);  in handle_stripe_dirtying()
4104 		struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4106 		     i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe_dirtying()
4118 		    i != sh->pd_idx && i != sh->qd_idx &&  in handle_stripe_dirtying()
4130 		 (unsigned long long)sh->sector, sh->state, rmw, rcw);  in handle_stripe_dirtying()
4131 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe_dirtying()
4137 					  (unsigned long long)sh->sector, rmw);  in handle_stripe_dirtying()
4139 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4142 			    !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {  in handle_stripe_dirtying()
4157 					r5c_use_extra_page(sh);  in handle_stripe_dirtying()
4162 				set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4169 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4171 			     i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe_dirtying()
4178 					     &sh->state)) {  in handle_stripe_dirtying()
4185 					set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4194 			struct r5dev *dev = &sh->dev[i];  in handle_stripe_dirtying()
4196 			    i != sh->pd_idx && i != sh->qd_idx &&  in handle_stripe_dirtying()
4203 					     &sh->state)) {  in handle_stripe_dirtying()
4211 					set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4216 					  (unsigned long long)sh->sector,  in handle_stripe_dirtying()
4217 					  rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));  in handle_stripe_dirtying()
4221 	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in handle_stripe_dirtying()
4222 		set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe_dirtying()
4234 	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&  in handle_stripe_dirtying()
4236 	     !test_bit(STRIPE_BIT_DELAY, &sh->state)))  in handle_stripe_dirtying()
4237 		schedule_reconstruction(sh, s, rcw == 0, 0);  in handle_stripe_dirtying()
4241 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks5()  argument
4246 	BUG_ON(sh->batch_head);  in handle_parity_checks5()
4247 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_parity_checks5()
4249 	switch (sh->check_state) {  in handle_parity_checks5()
4254 			sh->check_state = check_state_run;  in handle_parity_checks5()
4256 			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);  in handle_parity_checks5()
4260 		dev = &sh->dev[s->failed_num[0]];  in handle_parity_checks5()
4263 		sh->check_state = check_state_idle;  in handle_parity_checks5()
4265 			dev = &sh->dev[sh->pd_idx];  in handle_parity_checks5()
4268 		if (test_bit(STRIPE_INSYNC, &sh->state))  in handle_parity_checks5()
4279 		clear_bit(STRIPE_DEGRADED, &sh->state);  in handle_parity_checks5()
4280 		set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4285 		sh->check_state = check_state_idle;  in handle_parity_checks5()
4297 		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)  in handle_parity_checks5()
4301 			set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4306 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks5()
4309 						    (unsigned long long) sh->sector,  in handle_parity_checks5()
4310 						    (unsigned long long) sh->sector +  in handle_parity_checks5()
4313 				sh->check_state = check_state_compute_run;  in handle_parity_checks5()
4314 				set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in handle_parity_checks5()
4317 					&sh->dev[sh->pd_idx].flags);  in handle_parity_checks5()
4318 				sh->ops.target = sh->pd_idx;  in handle_parity_checks5()
4319 				sh->ops.target2 = -1;  in handle_parity_checks5()
4328 		       __func__, sh->check_state,  in handle_parity_checks5()
4329 		       (unsigned long long) sh->sector);  in handle_parity_checks5()
4334 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks6()  argument
4338 	int pd_idx = sh->pd_idx;  in handle_parity_checks6()
4339 	int qd_idx = sh->qd_idx;  in handle_parity_checks6()
4342 	BUG_ON(sh->batch_head);  in handle_parity_checks6()
4343 	set_bit(STRIPE_HANDLE, &sh->state);  in handle_parity_checks6()
4353 	switch (sh->check_state) {  in handle_parity_checks6()
4361 			sh->check_state = check_state_run;  in handle_parity_checks6()
4367 			if (sh->check_state == check_state_run)  in handle_parity_checks6()
4368 				sh->check_state = check_state_run_pq;  in handle_parity_checks6()
4370 				sh->check_state = check_state_run_q;  in handle_parity_checks6()
4374 		sh->ops.zero_sum_result = 0;  in handle_parity_checks6()
4376 		if (sh->check_state == check_state_run) {  in handle_parity_checks6()
4378 			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);  in handle_parity_checks6()
4381 		if (sh->check_state >= check_state_run &&  in handle_parity_checks6()
4382 		    sh->check_state <= check_state_run_pq) {  in handle_parity_checks6()
4394 		sh->check_state = check_state_idle;  in handle_parity_checks6()
4397 		if (test_bit(STRIPE_INSYNC, &sh->state))  in handle_parity_checks6()
4405 			dev = &sh->dev[s->failed_num[1]];  in handle_parity_checks6()
4411 			dev = &sh->dev[s->failed_num[0]];  in handle_parity_checks6()
4416 		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {  in handle_parity_checks6()
4417 			dev = &sh->dev[pd_idx];  in handle_parity_checks6()
4422 		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {  in handle_parity_checks6()
4423 			dev = &sh->dev[qd_idx];  in handle_parity_checks6()
4431 			      dev - (struct r5dev *) &sh->dev)) {  in handle_parity_checks6()
4436 		clear_bit(STRIPE_DEGRADED, &sh->state);  in handle_parity_checks6()
4438 		set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4445 		sh->check_state = check_state_idle;  in handle_parity_checks6()
4451 		if (sh->ops.zero_sum_result == 0) {  in handle_parity_checks6()
4454 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4460 				sh->check_state = check_state_compute_result;  in handle_parity_checks6()
4471 				set_bit(STRIPE_INSYNC, &sh->state);  in handle_parity_checks6()
4474 						    (unsigned long long) sh->sector,  in handle_parity_checks6()
4475 						    (unsigned long long) sh->sector +  in handle_parity_checks6()
4478 				int *target = &sh->ops.target;  in handle_parity_checks6()
4480 				sh->ops.target = -1;  in handle_parity_checks6()
4481 				sh->ops.target2 = -1;  in handle_parity_checks6()
4482 				sh->check_state = check_state_compute_run;  in handle_parity_checks6()
4483 				set_bit(STRIPE_COMPUTE_RUN, &sh->state);  in handle_parity_checks6()
4485 				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {  in handle_parity_checks6()
4487 						&sh->dev[pd_idx].flags);  in handle_parity_checks6()
4489 					target = &sh->ops.target2;  in handle_parity_checks6()
4492 				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {  in handle_parity_checks6()
4494 						&sh->dev[qd_idx].flags);  in handle_parity_checks6()
4505 			__func__, sh->check_state,  in handle_parity_checks6()
4506 			(unsigned long long) sh->sector);  in handle_parity_checks6()
4511 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)  in handle_stripe_expansion()  argument
4519 	BUG_ON(sh->batch_head);  in handle_stripe_expansion()
4520 	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);  in handle_stripe_expansion()
4521 	for (i = 0; i < sh->disks; i++)  in handle_stripe_expansion()
4522 		if (i != sh->pd_idx && i != sh->qd_idx) {  in handle_stripe_expansion()
4527 			sector_t bn = raid5_compute_blocknr(sh, i, 1);  in handle_stripe_expansion()
4547 					  sh->dev[i].page, sh2->dev[dd_idx].offset,  in handle_stripe_expansion()
4548 					  sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),  in handle_stripe_expansion()
4583 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)  in analyse_stripe()  argument
4585 	struct r5conf *conf = sh->raid_conf;  in analyse_stripe()
4586 	int disks = sh->disks;  in analyse_stripe()
4593 	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;  in analyse_stripe()
4594 	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;  in analyse_stripe()
4607 		dev = &sh->dev[i];  in analyse_stripe()
4618 		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))  in analyse_stripe()
4647 		    rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&  in analyse_stripe()
4648 		    !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in analyse_stripe()
4662 			is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),  in analyse_stripe()
4689 		else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)  in analyse_stripe()
4759 	if (test_bit(STRIPE_SYNCING, &sh->state)) {  in analyse_stripe()
4769 		    sh->sector >= conf->mddev->recovery_cp ||  in analyse_stripe()
4782 static int clear_batch_ready(struct stripe_head *sh)  in clear_batch_ready()  argument
4785 	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))  in clear_batch_ready()
4786 		return (sh->batch_head && sh->batch_head != sh);  in clear_batch_ready()
4787 	spin_lock(&sh->stripe_lock);  in clear_batch_ready()
4788 	if (!sh->batch_head) {  in clear_batch_ready()
4789 		spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4797 	if (sh->batch_head != sh) {  in clear_batch_ready()
4798 		spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4801 	spin_lock(&sh->batch_lock);  in clear_batch_ready()
4802 	list_for_each_entry(tmp, &sh->batch_list, batch_list)  in clear_batch_ready()
4804 	spin_unlock(&sh->batch_lock);  in clear_batch_ready()
4805 	spin_unlock(&sh->stripe_lock);  in clear_batch_ready()
4817 	struct stripe_head *sh, *next;  in break_stripe_batch_list()  local
4821 	list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {  in break_stripe_batch_list()
4823 		list_del_init(&sh->batch_list);  in break_stripe_batch_list()
4825 		WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |  in break_stripe_batch_list()
4837 			"stripe state: %lx\n", sh->state);  in break_stripe_batch_list()
4842 		set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |  in break_stripe_batch_list()
4848 		sh->check_state = head_sh->check_state;  in break_stripe_batch_list()
4849 		sh->reconstruct_state = head_sh->reconstruct_state;  in break_stripe_batch_list()
4850 		spin_lock_irq(&sh->stripe_lock);  in break_stripe_batch_list()
4851 		sh->batch_head = NULL;  in break_stripe_batch_list()
4852 		spin_unlock_irq(&sh->stripe_lock);  in break_stripe_batch_list()
4853 		for (i = 0; i < sh->disks; i++) {  in break_stripe_batch_list()
4854 			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))  in break_stripe_batch_list()
4856 			sh->dev[i].flags = head_sh->dev[i].flags &  in break_stripe_batch_list()
4860 		    sh->state & handle_flags)  in break_stripe_batch_list()
4861 			set_bit(STRIPE_HANDLE, &sh->state);  in break_stripe_batch_list()
4862 		raid5_release_stripe(sh);  in break_stripe_batch_list()
4877 static void handle_stripe(struct stripe_head *sh)  in handle_stripe()  argument
4880 	struct r5conf *conf = sh->raid_conf;  in handle_stripe()
4883 	int disks = sh->disks;  in handle_stripe()
4886 	clear_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
4894 	if (clear_batch_ready(sh))  in handle_stripe()
4897 	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {  in handle_stripe()
4900 		set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
4904 	if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))  in handle_stripe()
4905 		break_stripe_batch_list(sh, 0);  in handle_stripe()
4907 	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {  in handle_stripe()
4908 		spin_lock(&sh->stripe_lock);  in handle_stripe()
4913 		if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&  in handle_stripe()
4914 		    !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&  in handle_stripe()
4915 		    !test_bit(STRIPE_DISCARD, &sh->state) &&  in handle_stripe()
4916 		    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {  in handle_stripe()
4917 			set_bit(STRIPE_SYNCING, &sh->state);  in handle_stripe()
4918 			clear_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
4919 			clear_bit(STRIPE_REPLACED, &sh->state);  in handle_stripe()
4921 		spin_unlock(&sh->stripe_lock);  in handle_stripe()
4923 	clear_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe()
4927 	       (unsigned long long)sh->sector, sh->state,  in handle_stripe()
4928 	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,  in handle_stripe()
4929 	       sh->check_state, sh->reconstruct_state);  in handle_stripe()
4931 	analyse_stripe(sh, &s);  in handle_stripe()
4933 	if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))  in handle_stripe()
4938 		set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
4945 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
4953 	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {  in handle_stripe()
4955 		set_bit(STRIPE_BIOFILL_RUN, &sh->state);  in handle_stripe()
4971 		sh->check_state = 0;  in handle_stripe()
4972 		sh->reconstruct_state = 0;  in handle_stripe()
4973 		break_stripe_batch_list(sh, 0);  in handle_stripe()
4975 			handle_failed_stripe(conf, sh, &s, disks);  in handle_stripe()
4977 			handle_failed_sync(conf, sh, &s);  in handle_stripe()
4984 	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)  in handle_stripe()
4986 	if (sh->reconstruct_state == reconstruct_state_drain_result ||  in handle_stripe()
4987 	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {  in handle_stripe()
4988 		sh->reconstruct_state = reconstruct_state_idle;  in handle_stripe()
4993 		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&  in handle_stripe()
4994 		       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));  in handle_stripe()
4995 		BUG_ON(sh->qd_idx >= 0 &&  in handle_stripe()
4996 		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&  in handle_stripe()
4997 		       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));  in handle_stripe()
4999 			struct r5dev *dev = &sh->dev[i];  in handle_stripe()
5001 				(i == sh->pd_idx || i == sh->qd_idx ||  in handle_stripe()
5011 				    ((i == sh->pd_idx || i == sh->qd_idx)  &&  in handle_stripe()
5013 					set_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5016 		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in handle_stripe()
5024 	pdev = &sh->dev[sh->pd_idx];  in handle_stripe()
5025 	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)  in handle_stripe()
5026 		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);  in handle_stripe()
5027 	qdev = &sh->dev[sh->qd_idx];  in handle_stripe()
5028 	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)  in handle_stripe()
5029 		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)  in handle_stripe()
5041 		handle_stripe_clean_event(conf, sh, disks);  in handle_stripe()
5044 		r5c_handle_cached_data_endio(conf, sh, disks);  in handle_stripe()
5045 	log_stripe_write_finished(sh);  in handle_stripe()
5056 		handle_stripe_fill(sh, &s, disks);  in handle_stripe()
5063 	r5c_finish_stripe_write_out(conf, sh, &s);  in handle_stripe()
5074 	if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {  in handle_stripe()
5077 				handle_stripe_dirtying(conf, sh, &s, disks);  in handle_stripe()
5083 				ret = r5c_try_caching_write(conf, sh, &s,  in handle_stripe()
5094 			    (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&  in handle_stripe()
5096 				ret = handle_stripe_dirtying(conf, sh, &s,  in handle_stripe()
5109 	if (sh->check_state ||  in handle_stripe()
5111 	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&  in handle_stripe()
5112 	     !test_bit(STRIPE_INSYNC, &sh->state))) {  in handle_stripe()
5114 			handle_parity_checks6(conf, sh, &s, disks);  in handle_stripe()
5116 			handle_parity_checks5(conf, sh, &s, disks);  in handle_stripe()
5120 	    && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)  in handle_stripe()
5121 	    && !test_bit(STRIPE_REPLACED, &sh->state)) {  in handle_stripe()
5124 			if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {  in handle_stripe()
5125 				WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));  in handle_stripe()
5126 				set_bit(R5_WantReplace, &sh->dev[i].flags);  in handle_stripe()
5127 				set_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_stripe()
5131 			set_bit(STRIPE_INSYNC, &sh->state);  in handle_stripe()
5132 		set_bit(STRIPE_REPLACED, &sh->state);  in handle_stripe()
5135 	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&  in handle_stripe()
5136 	    test_bit(STRIPE_INSYNC, &sh->state)) {  in handle_stripe()
5138 		clear_bit(STRIPE_SYNCING, &sh->state);  in handle_stripe()
5139 		if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))  in handle_stripe()
5148 			struct r5dev *dev = &sh->dev[s.failed_num[i]];  in handle_stripe()
5165 	if (sh->reconstruct_state == reconstruct_state_result) {  in handle_stripe()
5167 			= raid5_get_active_stripe(conf, sh->sector, 1, 1, 1);  in handle_stripe()
5172 			set_bit(STRIPE_DELAYED, &sh->state);  in handle_stripe()
5173 			set_bit(STRIPE_HANDLE, &sh->state);  in handle_stripe()
5183 		sh->reconstruct_state = reconstruct_state_idle;  in handle_stripe()
5184 		clear_bit(STRIPE_EXPANDING, &sh->state);  in handle_stripe()
5186 			set_bit(R5_Wantwrite, &sh->dev[i].flags);  in handle_stripe()
5187 			set_bit(R5_LOCKED, &sh->dev[i].flags);  in handle_stripe()
5192 	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&  in handle_stripe()
5193 	    !sh->reconstruct_state) {  in handle_stripe()
5195 		sh->disks = conf->raid_disks;  in handle_stripe()
5196 		stripe_set_idx(sh->sector, conf, 0, sh);  in handle_stripe()
5197 		schedule_reconstruction(sh, &s, 1, 1);  in handle_stripe()
5198 	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {  in handle_stripe()
5199 		clear_bit(STRIPE_EXPAND_READY, &sh->state);  in handle_stripe()
5206 	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))  in handle_stripe()
5207 		handle_stripe_expansion(conf, sh);  in handle_stripe()
5227 			struct r5dev *dev = &sh->dev[i];  in handle_stripe()
5231 				if (!rdev_set_badblocks(rdev, sh->sector,  in handle_stripe()
5238 				rdev_clear_badblocks(rdev, sh->sector,  in handle_stripe()
5247 				rdev_clear_badblocks(rdev, sh->sector,  in handle_stripe()
5254 		raid_run_ops(sh, s.ops_request);  in handle_stripe()
5256 	ops_run_io(sh, &s);  in handle_stripe()
5269 	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);  in handle_stripe()
5277 			struct stripe_head *sh;  in raid5_activate_delayed()  local
5278 			sh = list_entry(l, struct stripe_head, lru);  in raid5_activate_delayed()
5280 			clear_bit(STRIPE_DELAYED, &sh->state);  in raid5_activate_delayed()
5281 			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in raid5_activate_delayed()
5283 			list_add_tail(&sh->lru, &conf->hold_list);  in raid5_activate_delayed()
5284 			raid5_wakeup_stripe_thread(sh);  in raid5_activate_delayed()
5297 		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);  in activate_bit_delay()  local
5299 		list_del_init(&sh->lru);  in activate_bit_delay()
5300 		atomic_inc(&sh->count);  in activate_bit_delay()
5301 		hash = sh->hash_lock_index;  in activate_bit_delay()
5302 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in activate_bit_delay()
5516 	struct stripe_head *sh, *tmp;  in __get_priority_stripe()  local
5526 	sh = NULL;  in __get_priority_stripe()
5552 		sh = list_entry(handle_list->next, typeof(*sh), lru);  in __get_priority_stripe()
5556 		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {  in __get_priority_stripe()
5576 				sh = tmp;  in __get_priority_stripe()
5581 		if (sh) {  in __get_priority_stripe()
5589 	if (!sh) {  in __get_priority_stripe()
5599 		sh->group = NULL;  in __get_priority_stripe()
5601 	list_del_init(&sh->lru);  in __get_priority_stripe()
5602 	BUG_ON(atomic_inc_return(&sh->count) != 1);  in __get_priority_stripe()
5603 	return sh;  in __get_priority_stripe()
5616 	struct stripe_head *sh;  in raid5_unplug()  local
5625 			sh = list_first_entry(&cb->list, struct stripe_head, lru);  in raid5_unplug()
5626 			list_del_init(&sh->lru);  in raid5_unplug()
5633 			clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);  in raid5_unplug()
5638 			hash = sh->hash_lock_index;  in raid5_unplug()
5639 			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);  in raid5_unplug()
5652 				struct stripe_head *sh)  in release_stripe_plug()  argument
5660 		raid5_release_stripe(sh);  in release_stripe_plug()
5673 	if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))  in release_stripe_plug()
5674 		list_add_tail(&sh->lru, &cb->list);  in release_stripe_plug()
5676 		raid5_release_stripe(sh);  in release_stripe_plug()
5683 	struct stripe_head *sh;  in make_discard_request()  local
5709 		sh = raid5_get_active_stripe(conf, logical_sector, 0, 0, 0);  in make_discard_request()
5712 		set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);  in make_discard_request()
5713 		if (test_bit(STRIPE_SYNCING, &sh->state)) {  in make_discard_request()
5714 			raid5_release_stripe(sh);  in make_discard_request()
5718 		clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);  in make_discard_request()
5719 		spin_lock_irq(&sh->stripe_lock);  in make_discard_request()
5721 			if (d == sh->pd_idx || d == sh->qd_idx)  in make_discard_request()
5723 			if (sh->dev[d].towrite || sh->dev[d].toread) {  in make_discard_request()
5724 				set_bit(R5_Overlap, &sh->dev[d].flags);  in make_discard_request()
5725 				spin_unlock_irq(&sh->stripe_lock);  in make_discard_request()
5726 				raid5_release_stripe(sh);  in make_discard_request()
5731 		set_bit(STRIPE_DISCARD, &sh->state);  in make_discard_request()
5733 		sh->overwrite_disks = 0;  in make_discard_request()
5735 			if (d == sh->pd_idx || d == sh->qd_idx)  in make_discard_request()
5737 			sh->dev[d].towrite = bi;  in make_discard_request()
5738 			set_bit(R5_OVERWRITE, &sh->dev[d].flags);  in make_discard_request()
5741 			sh->overwrite_disks++;  in make_discard_request()
5743 		spin_unlock_irq(&sh->stripe_lock);  in make_discard_request()
5749 						     sh->sector,  in make_discard_request()
5752 			sh->bm_seq = conf->seq_flush + 1;  in make_discard_request()
5753 			set_bit(STRIPE_BIT_DELAY, &sh->state);  in make_discard_request()
5756 		set_bit(STRIPE_HANDLE, &sh->state);  in make_discard_request()
5757 		clear_bit(STRIPE_DELAYED, &sh->state);  in make_discard_request()
5758 		if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in make_discard_request()
5760 		release_stripe_plug(mddev, sh);  in make_discard_request()
5772 	struct stripe_head *sh;  in raid5_make_request()  local
5865 		sh = raid5_get_active_stripe(conf, new_sector, previous,  in raid5_make_request()
5867 		if (sh) {  in raid5_make_request()
5886 					raid5_release_stripe(sh);  in raid5_make_request()
5896 				raid5_release_stripe(sh);  in raid5_make_request()
5900 			if (test_bit(STRIPE_EXPANDING, &sh->state) ||  in raid5_make_request()
5901 			    !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {  in raid5_make_request()
5907 				raid5_release_stripe(sh);  in raid5_make_request()
5913 				set_bit(STRIPE_R5C_PREFLUSH, &sh->state);  in raid5_make_request()
5918 			set_bit(STRIPE_HANDLE, &sh->state);  in raid5_make_request()
5919 			clear_bit(STRIPE_DELAYED, &sh->state);  in raid5_make_request()
5920 			if ((!sh->batch_head || sh == sh->batch_head) &&  in raid5_make_request()
5922 			    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))  in raid5_make_request()
5924 			release_stripe_plug(mddev, sh);  in raid5_make_request()
5953 	struct stripe_head *sh;  in reshape_request()  local
6104 		sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);  in reshape_request()
6105 		set_bit(STRIPE_EXPANDING, &sh->state);  in reshape_request()
6110 		for (j=sh->disks; j--;) {  in reshape_request()
6112 			if (j == sh->pd_idx)  in reshape_request()
6115 			    j == sh->qd_idx)  in reshape_request()
6117 			s = raid5_compute_blocknr(sh, j, 0);  in reshape_request()
6122 			memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));  in reshape_request()
6123 			set_bit(R5_Expanded, &sh->dev[j].flags);  in reshape_request()
6124 			set_bit(R5_UPTODATE, &sh->dev[j].flags);  in reshape_request()
6127 			set_bit(STRIPE_EXPAND_READY, &sh->state);  in reshape_request()
6128 			set_bit(STRIPE_HANDLE, &sh->state);  in reshape_request()
6130 		list_add(&sh->lru, &stripes);  in reshape_request()
6153 		sh = raid5_get_active_stripe(conf, first_sector, 1, 0, 1);  in reshape_request()
6154 		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);  in reshape_request()
6155 		set_bit(STRIPE_HANDLE, &sh->state);  in reshape_request()
6156 		raid5_release_stripe(sh);  in reshape_request()
6163 		sh = list_entry(stripes.next, struct stripe_head, lru);  in reshape_request()
6164 		list_del_init(&sh->lru);  in reshape_request()
6165 		raid5_release_stripe(sh);  in reshape_request()
6214 	struct stripe_head *sh;  in raid5_sync_request()  local
6273 	sh = raid5_get_active_stripe(conf, sector_nr, 0, 1, 0);  in raid5_sync_request()
6274 	if (sh == NULL) {  in raid5_sync_request()
6275 		sh = raid5_get_active_stripe(conf, sector_nr, 0, 0, 0);  in raid5_sync_request()
6296 	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);  in raid5_sync_request()
6297 	set_bit(STRIPE_HANDLE, &sh->state);  in raid5_sync_request()
6299 	raid5_release_stripe(sh);  in raid5_sync_request()
6317 	struct stripe_head *sh;  in retry_aligned_read()  local
6338 		sh = raid5_get_active_stripe(conf, sector, 0, 1, 1);  in retry_aligned_read()
6340 		if (!sh) {  in retry_aligned_read()
6347 		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {  in retry_aligned_read()
6348 			raid5_release_stripe(sh);  in retry_aligned_read()
6354 		set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);  in retry_aligned_read()
6355 		handle_stripe(sh);  in retry_aligned_read()
6356 		raid5_release_stripe(sh);  in retry_aligned_read()
6373 	struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;  in handle_active_stripes()  local
6378 			(sh = __get_priority_stripe(conf, group)) != NULL)  in handle_active_stripes()
6379 		batch[batch_size++] = sh;  in handle_active_stripes()