62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
329 	struct pnfs_layout_segment *lseg;  in pnfs_clear_layoutreturn_info()  local
333 	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {  in pnfs_clear_layoutreturn_info()
334 		if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))  in pnfs_clear_layoutreturn_info()
336 		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);  in pnfs_clear_layoutreturn_info()
350 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg,  in pnfs_clear_lseg_state()  argument
353 	clear_bit(NFS_LSEG_ROC, &lseg->pls_flags);  in pnfs_clear_lseg_state()
354 	clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);  in pnfs_clear_lseg_state()
355 	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags))  in pnfs_clear_lseg_state()
356 		pnfs_lseg_dec_and_remove_zero(lseg, free_me);  in pnfs_clear_lseg_state()
357 	if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))  in pnfs_clear_lseg_state()
358 		pnfs_lseg_dec_and_remove_zero(lseg, free_me);  in pnfs_clear_lseg_state()
419 	struct pnfs_layout_segment *lseg, *next;  in pnfs_mark_layout_stateid_invalid()  local
422 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)  in pnfs_mark_layout_stateid_invalid()
423 		pnfs_clear_lseg_state(lseg, lseg_list);  in pnfs_mark_layout_stateid_invalid()
493 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,  in pnfs_init_lseg()  argument
497 	INIT_LIST_HEAD(&lseg->pls_list);  in pnfs_init_lseg()
498 	INIT_LIST_HEAD(&lseg->pls_lc_list);  in pnfs_init_lseg()
499 	refcount_set(&lseg->pls_refcount, 1);  in pnfs_init_lseg()
500 	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);  in pnfs_init_lseg()
501 	lseg->pls_layout = lo;  in pnfs_init_lseg()
502 	lseg->pls_range = *range;  in pnfs_init_lseg()
503 	lseg->pls_seq = be32_to_cpu(stateid->seqid);  in pnfs_init_lseg()
506 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)  in pnfs_free_lseg()  argument
508 	if (lseg != NULL) {  in pnfs_free_lseg()
509 		struct inode *inode = lseg->pls_layout->plh_inode;  in pnfs_free_lseg()
510 		NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg);  in pnfs_free_lseg()
516 		struct pnfs_layout_segment *lseg)  in pnfs_layout_remove_lseg()  argument
518 	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));  in pnfs_layout_remove_lseg()
519 	list_del_init(&lseg->pls_list);  in pnfs_layout_remove_lseg()
522 	if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))  in pnfs_layout_remove_lseg()
535 		struct pnfs_layout_segment *lseg)  in pnfs_cache_lseg_for_layoutreturn()  argument
537 	if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&  in pnfs_cache_lseg_for_layoutreturn()
539 		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);  in pnfs_cache_lseg_for_layoutreturn()
540 		list_move_tail(&lseg->pls_list, &lo->plh_return_segs);  in pnfs_cache_lseg_for_layoutreturn()
547 pnfs_put_lseg(struct pnfs_layout_segment *lseg)  in pnfs_put_lseg()  argument
552 	if (!lseg)  in pnfs_put_lseg()
555 	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,  in pnfs_put_lseg()
556 		refcount_read(&lseg->pls_refcount),  in pnfs_put_lseg()
557 		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));  in pnfs_put_lseg()
559 	lo = lseg->pls_layout;  in pnfs_put_lseg()
562 	if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {  in pnfs_put_lseg()
563 		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {  in pnfs_put_lseg()
568 		pnfs_layout_remove_lseg(lo, lseg);  in pnfs_put_lseg()
569 		if (pnfs_cache_lseg_for_layoutreturn(lo, lseg))  in pnfs_put_lseg()
570 			lseg = NULL;  in pnfs_put_lseg()
572 		pnfs_free_lseg(lseg);  in pnfs_put_lseg()
597 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,  in pnfs_lseg_dec_and_remove_zero()  argument
600 	if (!refcount_dec_and_test(&lseg->pls_refcount))  in pnfs_lseg_dec_and_remove_zero()
602 	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);  in pnfs_lseg_dec_and_remove_zero()
603 	list_add(&lseg->pls_list, tmp_list);  in pnfs_lseg_dec_and_remove_zero()
608 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,  in mark_lseg_invalid()  argument
613 	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {  in mark_lseg_invalid()
618 		dprintk("%s: lseg %p ref %d\n", __func__, lseg,  in mark_lseg_invalid()
619 			refcount_read(&lseg->pls_refcount));  in mark_lseg_invalid()
620 		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))  in mark_lseg_invalid()
645 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg,  in pnfs_match_lseg_recall()  argument
649 	if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq))  in pnfs_match_lseg_recall()
653 	return pnfs_should_free_range(&lseg->pls_range, recall_range);  in pnfs_match_lseg_recall()
677 	struct pnfs_layout_segment *lseg, *next;  in pnfs_mark_matching_lsegs_invalid()  local
684 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)  in pnfs_mark_matching_lsegs_invalid()
685 		if (pnfs_match_lseg_recall(lseg, recall_range, seq)) {  in pnfs_mark_matching_lsegs_invalid()
688 				lseg, lseg->pls_range.iomode, lseg->pls_seq,  in pnfs_mark_matching_lsegs_invalid()
689 				lseg->pls_range.offset, lseg->pls_range.length);  in pnfs_mark_matching_lsegs_invalid()
690 			if (!mark_lseg_invalid(lseg, tmp_list))  in pnfs_mark_matching_lsegs_invalid()
703 	struct pnfs_layout_segment *lseg, *next;  in pnfs_free_returned_lsegs()  local
705 	list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) {  in pnfs_free_returned_lsegs()
706 		if (pnfs_match_lseg_recall(lseg, range, seq))  in pnfs_free_returned_lsegs()
707 			list_move_tail(&lseg->pls_list, free_me);  in pnfs_free_returned_lsegs()
715 	struct pnfs_layout_segment *lseg, *tmp;  in pnfs_free_lseg_list()  local
720 	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {  in pnfs_free_lseg_list()
721 		list_del(&lseg->pls_list);  in pnfs_free_lseg_list()
722 		pnfs_free_lseg(lseg);  in pnfs_free_lseg_list()
1084 	struct pnfs_layout_segment *lseg, *tmp;  in pnfs_clear_layoutcommit()  local
1088 	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {  in pnfs_clear_layoutcommit()
1089 		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))  in pnfs_clear_layoutcommit()
1091 		pnfs_lseg_dec_and_remove_zero(lseg, head);  in pnfs_clear_layoutcommit()
1356 	struct pnfs_layout_segment *lseg, *next;  in pnfs_roc()  local
1402 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) {  in pnfs_roc()
1403 		if (skip_read && lseg->pls_range.iomode == IOMODE_READ)  in pnfs_roc()
1406 		if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags))  in pnfs_roc()
1412 		set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);  in pnfs_roc()
1413 		if (!mark_lseg_invalid(lseg, &lo->plh_return_segs))  in pnfs_roc()
1415 		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);  in pnfs_roc()
1577 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,  in pnfs_lseg_no_merge()  argument
1585 		   struct pnfs_layout_segment *lseg,  in pnfs_generic_layout_insert_lseg()  argument
1599 		if (do_merge(lseg, lp)) {  in pnfs_generic_layout_insert_lseg()
1603 		if (is_after(&lseg->pls_range, &lp->pls_range))  in pnfs_generic_layout_insert_lseg()
1605 		list_add_tail(&lseg->pls_list, &lp->pls_list);  in pnfs_generic_layout_insert_lseg()
1609 			__func__, lseg, lseg->pls_range.iomode,  in pnfs_generic_layout_insert_lseg()
1610 			lseg->pls_range.offset, lseg->pls_range.length,  in pnfs_generic_layout_insert_lseg()
1615 	list_add_tail(&lseg->pls_list, &lo->plh_segs);  in pnfs_generic_layout_insert_lseg()
1618 		__func__, lseg, lseg->pls_range.iomode,  in pnfs_generic_layout_insert_lseg()
1619 		lseg->pls_range.offset, lseg->pls_range.length);  in pnfs_generic_layout_insert_lseg()
1629 		   struct pnfs_layout_segment *lseg,  in pnfs_layout_insert_lseg()  argument
1636 		ld->add_lseg(lo, lseg, free_me);  in pnfs_layout_insert_lseg()
1638 		pnfs_generic_layout_insert_lseg(lo, lseg,  in pnfs_layout_insert_lseg()
1734 	struct pnfs_layout_segment *lseg, *ret = NULL;  in pnfs_find_lseg()  local
1738 	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {  in pnfs_find_lseg()
1739 		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&  in pnfs_find_lseg()
1740 		    pnfs_lseg_range_match(&lseg->pls_range, range,  in pnfs_find_lseg()
1742 			ret = pnfs_get_lseg(lseg);  in pnfs_find_lseg()
1897 	struct pnfs_layout_segment *lseg = NULL;  in pnfs_update_layout()  local
1905 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
1911 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
1917 	lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp));  in pnfs_update_layout()
1918 	if (IS_ERR(lseg))  in pnfs_update_layout()
1925 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
1932 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
1940 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
1952 		lseg = ERR_PTR(wait_var_event_killable(&lo->plh_outstanding,  in pnfs_update_layout()
1954 		if (IS_ERR(lseg))  in pnfs_update_layout()
1960 	lseg = pnfs_find_lseg(lo, &arg, strict_iomode);  in pnfs_update_layout()
1961 	if (lseg) {  in pnfs_update_layout()
1962 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
1982 			lseg = ERR_PTR(wait_on_bit(&lo->plh_flags,  in pnfs_update_layout()
1985 			if (IS_ERR(lseg))  in pnfs_update_layout()
1997 			lseg = ERR_PTR(status);  in pnfs_update_layout()
1999 					iomode, lo, lseg,  in pnfs_update_layout()
2020 		lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo));  in pnfs_update_layout()
2021 		if (!IS_ERR(lseg)) {  in pnfs_update_layout()
2027 					lseg, PNFS_UPDATE_LAYOUT_RETRY);  in pnfs_update_layout()
2030 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
2036 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
2061 	lseg = nfs4_proc_layoutget(lgp, &timeout);  in pnfs_update_layout()
2062 	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
2065 	if (IS_ERR(lseg)) {  in pnfs_update_layout()
2066 		switch(PTR_ERR(lseg)) {  in pnfs_update_layout()
2069 				lseg = NULL;  in pnfs_update_layout()
2075 			if (!nfs_error_is_fatal(PTR_ERR(lseg))) {  in pnfs_update_layout()
2077 				lseg = NULL;  in pnfs_update_layout()
2081 		if (lseg) {  in pnfs_update_layout()
2085 				iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY);  in pnfs_update_layout()
2096 	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,  in pnfs_update_layout()
2104 			IS_ERR_OR_NULL(lseg) ? "not found" : "found",  in pnfs_update_layout()
2108 	return lseg;  in pnfs_update_layout()
2239 	struct pnfs_layout_segment *lseg;  in pnfs_parse_lgopen()  local
2276 	lseg = pnfs_layout_process(lgp);  in pnfs_parse_lgopen()
2277 	if (!IS_ERR(lseg)) {  in pnfs_parse_lgopen()
2280 		pnfs_put_lseg(lseg);  in pnfs_parse_lgopen()
2302 	struct pnfs_layout_segment *lseg;  in pnfs_layout_process()  local
2310 	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);  in pnfs_layout_process()
2311 	if (IS_ERR_OR_NULL(lseg)) {  in pnfs_layout_process()
2312 		if (!lseg)  in pnfs_layout_process()
2313 			lseg = ERR_PTR(-ENOMEM);  in pnfs_layout_process()
2316 		       __func__, PTR_ERR(lseg));  in pnfs_layout_process()
2317 		return lseg;  in pnfs_layout_process()
2320 	pnfs_init_lseg(lo, lseg, &res->range, &res->stateid);  in pnfs_layout_process()
2347 	pnfs_get_lseg(lseg);  in pnfs_layout_process()
2348 	pnfs_layout_insert_lseg(lo, lseg, &free_me);  in pnfs_layout_process()
2352 		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);  in pnfs_layout_process()
2356 	return lseg;  in pnfs_layout_process()
2360 	lseg->pls_layout = lo;  in pnfs_layout_process()
2361 	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);  in pnfs_layout_process()
2366 mark_lseg_invalid_or_return(struct pnfs_layout_segment *lseg,  in mark_lseg_invalid_or_return()  argument
2369 	if (!mark_lseg_invalid(lseg, tmp_list))  in mark_lseg_invalid_or_return()
2371 	pnfs_cache_lseg_for_layoutreturn(lseg->pls_layout, lseg);  in mark_lseg_invalid_or_return()
2397 	struct pnfs_layout_segment *lseg, *next;  in pnfs_mark_matching_lsegs_return()  local
2404 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)  in pnfs_mark_matching_lsegs_return()
2405 		if (pnfs_match_lseg_recall(lseg, return_range, seq)) {  in pnfs_mark_matching_lsegs_return()
2408 				lseg, lseg->pls_range.iomode,  in pnfs_mark_matching_lsegs_return()
2409 				lseg->pls_range.offset,  in pnfs_mark_matching_lsegs_return()
2410 				lseg->pls_range.length);  in pnfs_mark_matching_lsegs_return()
2411 			if (mark_lseg_invalid_or_return(lseg, tmp_list))  in pnfs_mark_matching_lsegs_return()
2414 			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);  in pnfs_mark_matching_lsegs_return()
2431 				       struct pnfs_layout_segment *lseg)  in pnfs_error_mark_layout_for_return()  argument
2435 		.iomode = lseg->pls_range.iomode,  in pnfs_error_mark_layout_for_return()
2636 		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,  in pnfs_ld_write_done()
2664 			struct pnfs_layout_segment *lseg,  in pnfs_try_to_write_data()  argument
2687 	struct pnfs_layout_segment *lseg = desc->pg_lseg;  in pnfs_do_write()  local
2690 	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);  in pnfs_do_write()
2709 	pnfs_put_lseg(hdr->lseg);  in pnfs_writehdr_free()
2726 	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);  in pnfs_generic_pg_writepages()
2790 		       struct pnfs_layout_segment *lseg)  in pnfs_try_to_read_data()  argument
2815 		pnfs_put_lseg(hdr->lseg);  in pnfs_read_resend_pnfs()
2816 		hdr->lseg = NULL;  in pnfs_read_resend_pnfs()
2829 	struct pnfs_layout_segment *lseg = desc->pg_lseg;  in pnfs_do_read()  local
2832 	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);  in pnfs_do_read()
2851 	pnfs_put_lseg(hdr->lseg);  in pnfs_readhdr_free()
2867 	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);  in pnfs_generic_pg_readpages()
2889 	struct pnfs_layout_segment *lseg;  in pnfs_list_write_lseg()  local
2891 	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {  in pnfs_list_write_lseg()
2892 		if (lseg->pls_range.iomode == IOMODE_RW &&  in pnfs_list_write_lseg()
2893 		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))  in pnfs_list_write_lseg()
2894 			list_add(&lseg->pls_lc_list, listp);  in pnfs_list_write_lseg()
2900 	struct pnfs_layout_segment *lseg, *tmp;  in pnfs_list_write_lseg_done()  local
2903 	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {  in pnfs_list_write_lseg_done()
2904 		list_del_init(&lseg->pls_lc_list);  in pnfs_list_write_lseg_done()
2905 		pnfs_put_lseg(lseg);  in pnfs_list_write_lseg_done()
2911 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)  in pnfs_set_lo_fail()  argument
2913 	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);  in pnfs_set_lo_fail()
2918 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,  in pnfs_set_layoutcommit()  argument
2932 	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {  in pnfs_set_layoutcommit()
2934 		pnfs_get_lseg(lseg);  in pnfs_set_layoutcommit()
2938 		__func__, lseg, nfsi->layout->plh_lwb);  in pnfs_set_layoutcommit()