560 int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_args()  argument
564 	struct rm_rdma_op *op = &rm->rdma;  in rds_cmsg_rdma_args()
574 	    || rm->rdma.op_active)  in rds_cmsg_rdma_args()
623 	op->op_sg = rds_message_alloc_sgs(rm, nr_pages);  in rds_cmsg_rdma_args()
648 		if (rm->data.op_sg) {  in rds_cmsg_rdma_args()
649 			rm->rdma.op_notify = 0;  in rds_cmsg_rdma_args()
650 			rm->data.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);  in rds_cmsg_rdma_args()
738 int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_dest()  argument
747 	    rm->m_rdma_cookie != 0)  in rds_cmsg_rdma_dest()
750 	memcpy(&rm->m_rdma_cookie, CMSG_DATA(cmsg), sizeof(rm->m_rdma_cookie));  in rds_cmsg_rdma_dest()
757 	r_key = rds_rdma_cookie_key(rm->m_rdma_cookie);  in rds_cmsg_rdma_dest()
769 		rm->rdma.op_rdma_mr = mr;  in rds_cmsg_rdma_dest()
780 int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_rdma_map()  argument
784 	    rm->m_rdma_cookie != 0)  in rds_cmsg_rdma_map()
787 	return __rds_rdma_map(rs, CMSG_DATA(cmsg), &rm->m_rdma_cookie,  in rds_cmsg_rdma_map()
788 			      &rm->rdma.op_rdma_mr, rm->m_conn_path);  in rds_cmsg_rdma_map()
794 int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,  in rds_cmsg_atomic()  argument
802 	 || rm->atomic.op_active)  in rds_cmsg_atomic()
810 		rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;  in rds_cmsg_atomic()
811 		rm->atomic.op_m_fadd.add = args->fadd.add;  in rds_cmsg_atomic()
812 		rm->atomic.op_m_fadd.nocarry_mask = 0;  in rds_cmsg_atomic()
815 		rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;  in rds_cmsg_atomic()
816 		rm->atomic.op_m_fadd.add = args->m_fadd.add;  in rds_cmsg_atomic()
817 		rm->atomic.op_m_fadd.nocarry_mask = args->m_fadd.nocarry_mask;  in rds_cmsg_atomic()
820 		rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;  in rds_cmsg_atomic()
821 		rm->atomic.op_m_cswp.compare = args->cswp.compare;  in rds_cmsg_atomic()
822 		rm->atomic.op_m_cswp.swap = args->cswp.swap;  in rds_cmsg_atomic()
823 		rm->atomic.op_m_cswp.compare_mask = ~0;  in rds_cmsg_atomic()
824 		rm->atomic.op_m_cswp.swap_mask = ~0;  in rds_cmsg_atomic()
827 		rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;  in rds_cmsg_atomic()
828 		rm->atomic.op_m_cswp.compare = args->m_cswp.compare;  in rds_cmsg_atomic()
829 		rm->atomic.op_m_cswp.swap = args->m_cswp.swap;  in rds_cmsg_atomic()
830 		rm->atomic.op_m_cswp.compare_mask = args->m_cswp.compare_mask;  in rds_cmsg_atomic()
831 		rm->atomic.op_m_cswp.swap_mask = args->m_cswp.swap_mask;  in rds_cmsg_atomic()
837 	rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);  in rds_cmsg_atomic()
838 	rm->atomic.op_silent = !!(args->flags & RDS_RDMA_SILENT);  in rds_cmsg_atomic()
839 	rm->atomic.op_active = 1;  in rds_cmsg_atomic()
840 	rm->atomic.op_recverr = rs->rs_recverr;  in rds_cmsg_atomic()
841 	rm->atomic.op_sg = rds_message_alloc_sgs(rm, 1);  in rds_cmsg_atomic()
842 	if (!rm->atomic.op_sg) {  in rds_cmsg_atomic()
858 	sg_set_page(rm->atomic.op_sg, page, 8, offset_in_page(args->local_addr));  in rds_cmsg_atomic()
860 	if (rm->atomic.op_notify || rm->atomic.op_recverr) {  in rds_cmsg_atomic()
866 		rm->atomic.op_notifier = kmalloc(sizeof(*rm->atomic.op_notifier), GFP_KERNEL);  in rds_cmsg_atomic()
867 		if (!rm->atomic.op_notifier) {  in rds_cmsg_atomic()
872 		rm->atomic.op_notifier->n_user_token = args->user_token;  in rds_cmsg_atomic()
873 		rm->atomic.op_notifier->n_status = RDS_RDMA_SUCCESS;  in rds_cmsg_atomic()
876 	rm->atomic.op_rkey = rds_rdma_cookie_key(args->cookie);  in rds_cmsg_atomic()
877 	rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie);  in rds_cmsg_atomic()
883 	rm->atomic.op_active = 0;  in rds_cmsg_atomic()
884 	kfree(rm->atomic.op_notifier);  in rds_cmsg_atomic()