58 		 "Enable inter server to server copy offload. Default: false");
923 	 * If we do a zero copy read, then a client will see read data  in nfsd4_read()
927 	 * To ensure proper ordering, we therefore turn off zero copy if  in nfsd4_read()
1247 static void nfs4_put_copy(struct nfsd4_copy *copy)  in nfs4_put_copy()  argument
1249 	if (!refcount_dec_and_test(&copy->refcount))  in nfs4_put_copy()
1251 	kfree(copy->cp_src);  in nfs4_put_copy()
1252 	kfree(copy);  in nfs4_put_copy()
1255 static void nfsd4_stop_copy(struct nfsd4_copy *copy)  in nfsd4_stop_copy()  argument
1257 	if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, &copy->cp_flags))  in nfsd4_stop_copy()
1258 		kthread_stop(copy->copy_task);  in nfsd4_stop_copy()
1259 	nfs4_put_copy(copy);  in nfsd4_stop_copy()
1264 	struct nfsd4_copy *copy = NULL;  in nfsd4_get_copy()  local
1268 		copy = list_first_entry(&clp->async_copies, struct nfsd4_copy,  in nfsd4_get_copy()
1270 		refcount_inc(&copy->refcount);  in nfsd4_get_copy()
1273 	return copy;  in nfsd4_get_copy()
1278 	struct nfsd4_copy *copy;  in nfsd4_shutdown_copy()  local
1280 	while ((copy = nfsd4_get_copy(clp)) != NULL)  in nfsd4_shutdown_copy()
1281 		nfsd4_stop_copy(copy);  in nfsd4_shutdown_copy()
1370  * Support one copy source server for now.
1474  * Verify COPY destination stateid.
1478  * Called with COPY cstate:
1485 		      struct nfsd4_copy *copy, struct vfsmount **mount)  in nfsd4_setup_inter_ssc()  argument
1488 	stateid_t *s_stid = &copy->cp_src_stateid;  in nfsd4_setup_inter_ssc()
1493 					    &copy->cp_dst_stateid,  in nfsd4_setup_inter_ssc()
1494 					    WR_STATE, &copy->nf_dst, NULL);  in nfsd4_setup_inter_ssc()
1498 	status = nfsd4_interssc_connect(copy->cp_src, rqstp, mount);  in nfsd4_setup_inter_ssc()
1504 	copy->c_fh.size = s_fh->fh_handle.fh_size;  in nfsd4_setup_inter_ssc()
1505 	memcpy(copy->c_fh.data, &s_fh->fh_handle.fh_raw, copy->c_fh.size);  in nfsd4_setup_inter_ssc()
1506 	copy->stateid.seqid = cpu_to_be32(s_stid->si_generation);  in nfsd4_setup_inter_ssc()
1507 	memcpy(copy->stateid.other, (void *)&s_stid->si_opaque,  in nfsd4_setup_inter_ssc()
1562 		      struct nfsd4_copy *copy,  in nfsd4_setup_inter_ssc()  argument
1591 		      struct nfsd4_copy *copy)  in nfsd4_setup_intra_ssc()  argument
1593 	return nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,  in nfsd4_setup_intra_ssc()
1594 				 &copy->nf_src, &copy->cp_dst_stateid,  in nfsd4_setup_intra_ssc()
1595 				 &copy->nf_dst);  in nfsd4_setup_intra_ssc()
1628 static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)  in nfsd4_init_copy_res()  argument
1630 	copy->cp_res.wr_stable_how =  in nfsd4_init_copy_res()
1631 		test_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags) ?  in nfsd4_init_copy_res()
1633 	nfsd4_copy_set_sync(copy, sync);  in nfsd4_init_copy_res()
1634 	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->cp_clp->net);  in nfsd4_init_copy_res()
1637 static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy,  in _nfsd_copy_file_range()  argument
1643 	u64 bytes_total = copy->cp_count;  in _nfsd_copy_file_range()
1644 	u64 src_pos = copy->cp_src_pos;  in _nfsd_copy_file_range()
1645 	u64 dst_pos = copy->cp_dst_pos;  in _nfsd_copy_file_range()
1659 		copy->cp_res.wr_bytes_written += bytes_copied;  in _nfsd_copy_file_range()
1662 	} while (bytes_total > 0 && nfsd4_copy_is_async(copy));  in _nfsd_copy_file_range()
1663 	/* for a non-zero asynchronous copy do a commit of data */  in _nfsd_copy_file_range()
1664 	if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) {  in _nfsd_copy_file_range()
1666 		status = vfs_fsync_range(dst, copy->cp_dst_pos,  in _nfsd_copy_file_range()
1667 					 copy->cp_res.wr_bytes_written, 0);  in _nfsd_copy_file_range()
1671 			set_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags);  in _nfsd_copy_file_range()
1676 static __be32 nfsd4_do_copy(struct nfsd4_copy *copy,  in nfsd4_do_copy()  argument
1683 	bytes = _nfsd_copy_file_range(copy, dst, src);  in nfsd4_do_copy()
1685 	/* for async copy, we ignore the error, client can always retry  in nfsd4_do_copy()
1688 	if (bytes < 0 && !copy->cp_res.wr_bytes_written)  in nfsd4_do_copy()
1691 		nfsd4_init_copy_res(copy, sync);  in nfsd4_do_copy()
1718 static void cleanup_async_copy(struct nfsd4_copy *copy)  in cleanup_async_copy()  argument
1720 	nfs4_free_copy_state(copy);  in cleanup_async_copy()
1721 	nfsd_file_put(copy->nf_dst);  in cleanup_async_copy()
1722 	if (!nfsd4_ssc_is_inter(copy))  in cleanup_async_copy()
1723 		nfsd_file_put(copy->nf_src);  in cleanup_async_copy()
1724 	spin_lock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1725 	list_del(&copy->copies);  in cleanup_async_copy()
1726 	spin_unlock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1727 	nfs4_put_copy(copy);  in cleanup_async_copy()
1730 static void nfsd4_send_cb_offload(struct nfsd4_copy *copy, __be32 nfserr)  in nfsd4_send_cb_offload()  argument
1738 	memcpy(&cbo->co_res, &copy->cp_res, sizeof(copy->cp_res));  in nfsd4_send_cb_offload()
1739 	memcpy(&cbo->co_fh, &copy->fh, sizeof(copy->fh));  in nfsd4_send_cb_offload()
1742 	nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops,  in nfsd4_send_cb_offload()
1744 	trace_nfsd_cb_offload(copy->cp_clp, &cbo->co_res.cb_stateid,  in nfsd4_send_cb_offload()
1745 			      &cbo->co_fh, copy->cp_count, nfserr);  in nfsd4_send_cb_offload()
1750  * nfsd4_do_async_copy - kthread function for background server-side COPY
1751  * @data: arguments for COPY operation
1754  *   %0: Copy operation is done.
1758 	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;  in nfsd4_do_async_copy()  local
1761 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_do_async_copy()
1764 		filp = nfs42_ssc_open(copy->ss_mnt, &copy->c_fh,  in nfsd4_do_async_copy()
1765 				      &copy->stateid);  in nfsd4_do_async_copy()
1774 			nfsd4_interssc_disconnect(copy->ss_mnt);  in nfsd4_do_async_copy()
1777 		nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file,  in nfsd4_do_async_copy()
1779 		nfsd4_cleanup_inter_ssc(copy->ss_mnt, filp, copy->nf_dst);  in nfsd4_do_async_copy()
1781 		nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_do_async_copy()
1782 				       copy->nf_dst->nf_file, false);  in nfsd4_do_async_copy()
1783 		nfsd4_cleanup_intra_ssc(copy->nf_src, copy->nf_dst);  in nfsd4_do_async_copy()
1787 	nfsd4_send_cb_offload(copy, nfserr);  in nfsd4_do_async_copy()
1788 	cleanup_async_copy(copy);  in nfsd4_do_async_copy()
1796 	struct nfsd4_copy *copy = &u->copy;  in nfsd4_copy()  local
1800 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1801 		if (!inter_copy_offload_enable || nfsd4_copy_is_sync(copy)) {  in nfsd4_copy()
1805 		status = nfsd4_setup_inter_ssc(rqstp, cstate, copy,  in nfsd4_copy()
1806 				&copy->ss_mnt);  in nfsd4_copy()
1810 		status = nfsd4_setup_intra_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1815 	copy->cp_clp = cstate->clp;  in nfsd4_copy()
1816 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,  in nfsd4_copy()
1818 	if (nfsd4_copy_is_async(copy)) {  in nfsd4_copy()
1828 		if (!nfs4_init_copy_state(nn, copy))  in nfsd4_copy()
1831 		memcpy(&copy->cp_res.cb_stateid, &copy->cp_stateid.cs_stid,  in nfsd4_copy()
1832 			sizeof(copy->cp_res.cb_stateid));  in nfsd4_copy()
1833 		dup_copy_fields(copy, async_copy);  in nfsd4_copy()
1835 				async_copy, "%s", "copy thread");  in nfsd4_copy()
1845 		status = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_copy()
1846 				       copy->nf_dst->nf_file, true);  in nfsd4_copy()
1847 		nfsd4_cleanup_intra_ssc(copy->nf_src, copy->nf_dst);  in nfsd4_copy()
1855 	if (nfsd4_ssc_is_inter(copy))  in nfsd4_copy()
1856 		nfsd4_interssc_disconnect(copy->ss_mnt);  in nfsd4_copy()
1863 	struct nfsd4_copy *copy;  in find_async_copy()  local
1866 	list_for_each_entry(copy, &clp->async_copies, copies) {  in find_async_copy()
1867 		if (memcmp(&copy->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE))  in find_async_copy()
1869 		refcount_inc(&copy->refcount);  in find_async_copy()
1871 		return copy;  in find_async_copy()
1883 	struct nfsd4_copy *copy;  in nfsd4_offload_cancel()  local
1886 	copy = find_async_copy(clp, &os->stateid);  in nfsd4_offload_cancel()
1887 	if (!copy) {  in nfsd4_offload_cancel()
1892 		nfsd4_stop_copy(copy);  in nfsd4_offload_cancel()
1970 	struct nfsd4_copy *copy;  in nfsd4_offload_status()  local
1973 	copy = find_async_copy(clp, &os->stateid);  in nfsd4_offload_status()
1974 	if (copy) {  in nfsd4_offload_status()
1975 		os->count = copy->cp_res.wr_bytes_written;  in nfsd4_offload_status()
1976 		nfs4_put_copy(copy);  in nfsd4_offload_status()
2429 	 * Get the entire list, then copy out only the user attributes  in nfsd4_listxattrs()
2562 	struct nfsd4_copy *copy;  in check_if_stalefh_allowed()  local
2566 	/* traverse all operation and if it's a COPY compound, mark the  in check_if_stalefh_allowed()
2578 			copy = (struct nfsd4_copy *)&op->u;  in check_if_stalefh_allowed()
2584 			if (nfsd4_ssc_is_inter(copy))  in check_if_stalefh_allowed()