1281 static void nfs4_put_copy(struct nfsd4_copy *copy)  in nfs4_put_copy()  argument
1283 	if (!refcount_dec_and_test(&copy->refcount))  in nfs4_put_copy()
1285 	kfree(copy->cp_src);  in nfs4_put_copy()
1286 	kfree(copy);  in nfs4_put_copy()
1289 static void nfsd4_stop_copy(struct nfsd4_copy *copy)  in nfsd4_stop_copy()  argument
1291 	if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, &copy->cp_flags))  in nfsd4_stop_copy()
1292 		kthread_stop(copy->copy_task);  in nfsd4_stop_copy()
1293 	nfs4_put_copy(copy);  in nfsd4_stop_copy()
1298 	struct nfsd4_copy *copy = NULL;  in nfsd4_get_copy()  local
1302 		copy = list_first_entry(&clp->async_copies, struct nfsd4_copy,  in nfsd4_get_copy()
1304 		refcount_inc(&copy->refcount);  in nfsd4_get_copy()
1307 	return copy;  in nfsd4_get_copy()
1312 	struct nfsd4_copy *copy;  in nfsd4_shutdown_copy()  local
1314 	while ((copy = nfsd4_get_copy(clp)) != NULL)  in nfsd4_shutdown_copy()
1315 		nfsd4_stop_copy(copy);  in nfsd4_shutdown_copy()
1510 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1513 	stateid_t *s_stid = &copy->cp_src_stateid;  in nfsd4_setup_inter_ssc()
1518 					    &copy->cp_dst_stateid,  in nfsd4_setup_inter_ssc()
1519 					    WR_STATE, &copy->nf_dst, NULL);  in nfsd4_setup_inter_ssc()
1523 	status = nfsd4_interssc_connect(copy->cp_src, rqstp, &copy->ss_nsui);  in nfsd4_setup_inter_ssc()
1529 	copy->c_fh.size = s_fh->fh_handle.fh_size;  in nfsd4_setup_inter_ssc()
1530 	memcpy(copy->c_fh.data, &s_fh->fh_handle.fh_raw, copy->c_fh.size);  in nfsd4_setup_inter_ssc()
1531 	copy->stateid.seqid = cpu_to_be32(s_stid->si_generation);  in nfsd4_setup_inter_ssc()
1532 	memcpy(copy->stateid.other, (void *)&s_stid->si_opaque,  in nfsd4_setup_inter_ssc()
1568 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1590 		      struct nfsd4_copy *copy)  in nfsd4_setup_intra_ssc()  argument
1592 	return nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,  in nfsd4_setup_intra_ssc()
1593 				 &copy->nf_src, &copy->cp_dst_stateid,  in nfsd4_setup_intra_ssc()
1594 				 &copy->nf_dst);  in nfsd4_setup_intra_ssc()
1620 static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)  in nfsd4_init_copy_res()  argument
1622 	copy->cp_res.wr_stable_how =  in nfsd4_init_copy_res()
1623 		test_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags) ?  in nfsd4_init_copy_res()
1625 	nfsd4_copy_set_sync(copy, sync);  in nfsd4_init_copy_res()
1626 	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->cp_clp->net);  in nfsd4_init_copy_res()
1629 static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy,  in _nfsd_copy_file_range()  argument
1635 	u64 bytes_total = copy->cp_count;  in _nfsd_copy_file_range()
1636 	u64 src_pos = copy->cp_src_pos;  in _nfsd_copy_file_range()
1637 	u64 dst_pos = copy->cp_dst_pos;  in _nfsd_copy_file_range()
1652 		copy->cp_res.wr_bytes_written += bytes_copied;  in _nfsd_copy_file_range()
1655 	} while (bytes_total > 0 && nfsd4_copy_is_async(copy));  in _nfsd_copy_file_range()
1657 	if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) {  in _nfsd_copy_file_range()
1659 		end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1;  in _nfsd_copy_file_range()
1660 		status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0);  in _nfsd_copy_file_range()
1664 			set_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags);  in _nfsd_copy_file_range()
1669 static __be32 nfsd4_do_copy(struct nfsd4_copy *copy,  in nfsd4_do_copy()  argument
1676 	bytes = _nfsd_copy_file_range(copy, dst, src);  in nfsd4_do_copy()
1681 	if (bytes < 0 && !copy->cp_res.wr_bytes_written)  in nfsd4_do_copy()
1684 		nfsd4_init_copy_res(copy, sync);  in nfsd4_do_copy()
1711 static void release_copy_files(struct nfsd4_copy *copy)  in release_copy_files()  argument
1713 	if (copy->nf_src)  in release_copy_files()
1714 		nfsd_file_put(copy->nf_src);  in release_copy_files()
1715 	if (copy->nf_dst)  in release_copy_files()
1716 		nfsd_file_put(copy->nf_dst);  in release_copy_files()
1719 static void cleanup_async_copy(struct nfsd4_copy *copy)  in cleanup_async_copy()  argument
1721 	nfs4_free_copy_state(copy);  in cleanup_async_copy()
1722 	release_copy_files(copy);  in cleanup_async_copy()
1723 	if (copy->cp_clp) {  in cleanup_async_copy()
1724 		spin_lock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1725 		if (!list_empty(&copy->copies))  in cleanup_async_copy()
1726 			list_del_init(&copy->copies);  in cleanup_async_copy()
1727 		spin_unlock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1729 	nfs4_put_copy(copy);  in cleanup_async_copy()
1732 static void nfsd4_send_cb_offload(struct nfsd4_copy *copy, __be32 nfserr)  in nfsd4_send_cb_offload()  argument
1740 	memcpy(&cbo->co_res, &copy->cp_res, sizeof(copy->cp_res));  in nfsd4_send_cb_offload()
1741 	memcpy(&cbo->co_fh, &copy->fh, sizeof(copy->fh));  in nfsd4_send_cb_offload()
1744 	nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops,  in nfsd4_send_cb_offload()
1746 	trace_nfsd_cb_offload(copy->cp_clp, &cbo->co_res.cb_stateid,  in nfsd4_send_cb_offload()
1747 			      &cbo->co_fh, copy->cp_count, nfserr);  in nfsd4_send_cb_offload()
1760 	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;  in nfsd4_do_async_copy()  local
1763 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_do_async_copy()
1766 		filp = nfs42_ssc_open(copy->ss_nsui->nsui_vfsmount,  in nfsd4_do_async_copy()
1767 				      &copy->c_fh, &copy->stateid);  in nfsd4_do_async_copy()
1779 		nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file,  in nfsd4_do_async_copy()
1781 		nfsd4_cleanup_inter_ssc(copy->ss_nsui, filp, copy->nf_dst);  in nfsd4_do_async_copy()
1783 		nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_do_async_copy()
1784 				       copy->nf_dst->nf_file, false);  in nfsd4_do_async_copy()
1788 	nfsd4_send_cb_offload(copy, nfserr);  in nfsd4_do_async_copy()
1789 	cleanup_async_copy(copy);  in nfsd4_do_async_copy()
1797 	struct nfsd4_copy *copy = &u->copy;  in nfsd4_copy()  local
1801 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1802 		if (!inter_copy_offload_enable || nfsd4_copy_is_sync(copy)) {  in nfsd4_copy()
1806 		status = nfsd4_setup_inter_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1810 		status = nfsd4_setup_intra_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1815 	copy->cp_clp = cstate->clp;  in nfsd4_copy()
1816 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,  in nfsd4_copy()
1818 	if (nfsd4_copy_is_async(copy)) {  in nfsd4_copy()
1830 		if (!nfs4_init_copy_state(nn, copy))  in nfsd4_copy()
1832 		memcpy(&copy->cp_res.cb_stateid, &copy->cp_stateid.cs_stid,  in nfsd4_copy()
1833 			sizeof(copy->cp_res.cb_stateid));  in nfsd4_copy()
1834 		dup_copy_fields(copy, async_copy);  in nfsd4_copy()
1846 		status = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_copy()
1847 				       copy->nf_dst->nf_file, true);  in nfsd4_copy()
1850 	release_copy_files(copy);  in nfsd4_copy()
1853 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1859 		refcount_dec(&copy->ss_nsui->nsui_refcnt);  in nfsd4_copy()
1870 	struct nfsd4_copy *copy;  in find_async_copy_locked()  local
1874 	list_for_each_entry(copy, &clp->async_copies, copies) {  in find_async_copy_locked()
1875 		if (memcmp(&copy->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE))  in find_async_copy_locked()
1877 		return copy;  in find_async_copy_locked()
1885 	struct nfsd4_copy *copy;  in find_async_copy()  local
1888 	copy = find_async_copy_locked(clp, stateid);  in find_async_copy()
1889 	if (copy)  in find_async_copy()
1890 		refcount_inc(&copy->refcount);  in find_async_copy()
1892 	return copy;  in find_async_copy()
1901 	struct nfsd4_copy *copy;  in nfsd4_offload_cancel()  local
1904 	copy = find_async_copy(clp, &os->stateid);  in nfsd4_offload_cancel()
1905 	if (!copy) {  in nfsd4_offload_cancel()
1910 		nfsd4_stop_copy(copy);  in nfsd4_offload_cancel()
1987 	struct nfsd4_copy *copy;  in nfsd4_offload_status()  local
1991 	copy = find_async_copy_locked(clp, &os->stateid);  in nfsd4_offload_status()
1992 	if (copy)  in nfsd4_offload_status()
1993 		os->count = copy->cp_res.wr_bytes_written;  in nfsd4_offload_status()
2578 	struct nfsd4_copy *copy;  in check_if_stalefh_allowed()  local
2594 			copy = (struct nfsd4_copy *)&op->u;  in check_if_stalefh_allowed()
2600 			if (nfsd4_ssc_is_inter(copy))  in check_if_stalefh_allowed()