Lines Matching refs:rm
67 struct rds_message *rm, *tmp; in rds_send_path_reset() local
71 rm = cp->cp_xmit_rm; in rds_send_path_reset()
77 rds_message_unmapped(rm); in rds_send_path_reset()
78 rds_message_put(rm); in rds_send_path_reset()
95 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_reset()
96 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_path_reset()
97 set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags); in rds_send_path_reset()
140 struct rds_message *rm; in rds_send_xmit() local
201 rm = cp->cp_xmit_rm; in rds_send_xmit()
207 if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) { in rds_send_xmit()
208 rm = rds_cong_update_alloc(conn); in rds_send_xmit()
209 if (IS_ERR(rm)) { in rds_send_xmit()
210 ret = PTR_ERR(rm); in rds_send_xmit()
213 rm->data.op_active = 1; in rds_send_xmit()
214 rm->m_inc.i_conn_path = cp; in rds_send_xmit()
215 rm->m_inc.i_conn = cp->cp_conn; in rds_send_xmit()
217 cp->cp_xmit_rm = rm; in rds_send_xmit()
227 if (!rm) { in rds_send_xmit()
243 rm = list_entry(cp->cp_send_queue.next, in rds_send_xmit()
246 rds_message_addref(rm); in rds_send_xmit()
252 list_move_tail(&rm->m_conn_item, in rds_send_xmit()
258 if (!rm) in rds_send_xmit()
268 if (test_bit(RDS_MSG_FLUSH, &rm->m_flags) || in rds_send_xmit()
269 (rm->rdma.op_active && in rds_send_xmit()
270 test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags))) { in rds_send_xmit()
272 if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) in rds_send_xmit()
273 list_move(&rm->m_conn_item, &to_be_dropped); in rds_send_xmit()
279 len = ntohl(rm->m_inc.i_hdr.h_len); in rds_send_xmit()
282 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_xmit()
294 cp->cp_xmit_rm = rm; in rds_send_xmit()
298 if (rm->rdma.op_active && !cp->cp_xmit_rdma_sent) { in rds_send_xmit()
299 rm->m_final_op = &rm->rdma; in rds_send_xmit()
303 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
304 ret = conn->c_trans->xmit_rdma(conn, &rm->rdma); in rds_send_xmit()
306 clear_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
307 wake_up_interruptible(&rm->m_flush_wait); in rds_send_xmit()
314 if (rm->atomic.op_active && !cp->cp_xmit_atomic_sent) { in rds_send_xmit()
315 rm->m_final_op = &rm->atomic; in rds_send_xmit()
319 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
320 ret = conn->c_trans->xmit_atomic(conn, &rm->atomic); in rds_send_xmit()
322 clear_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
323 wake_up_interruptible(&rm->m_flush_wait); in rds_send_xmit()
337 if (rm->data.op_nents == 0) { in rds_send_xmit()
341 ops_present = (rm->atomic.op_active || rm->rdma.op_active); in rds_send_xmit()
342 if (rm->atomic.op_active && !rm->atomic.op_silent) in rds_send_xmit()
344 if (rm->rdma.op_active && !rm->rdma.op_silent) in rds_send_xmit()
348 && !rm->m_rdma_cookie) in rds_send_xmit()
349 rm->data.op_active = 0; in rds_send_xmit()
352 if (rm->data.op_active && !cp->cp_xmit_data_sent) { in rds_send_xmit()
353 rm->m_final_op = &rm->data; in rds_send_xmit()
355 ret = conn->c_trans->xmit(conn, rm, in rds_send_xmit()
370 sg = &rm->data.op_sg[cp->cp_xmit_sg]; in rds_send_xmit()
381 rm->data.op_nents); in rds_send_xmit()
386 (cp->cp_xmit_sg == rm->data.op_nents)) in rds_send_xmit()
395 if (!rm->data.op_active || cp->cp_xmit_data_sent) { in rds_send_xmit()
404 rds_message_put(rm); in rds_send_xmit()
416 list_for_each_entry(rm, &to_be_dropped, m_conn_item) in rds_send_xmit()
417 rds_message_put(rm); in rds_send_xmit()
461 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm) in rds_send_sndbuf_remove() argument
463 u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_sndbuf_remove()
474 static inline int rds_send_is_acked(struct rds_message *rm, u64 ack, in rds_send_is_acked() argument
478 return is_acked(rm, ack); in rds_send_is_acked()
479 return be64_to_cpu(rm->m_inc.i_hdr.h_sequence) <= ack; in rds_send_is_acked()
488 void rds_rdma_send_complete(struct rds_message *rm, int status) in rds_rdma_send_complete() argument
496 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
498 notify = rm->rdma.op_notify | rm->data.op_notify; in rds_rdma_send_complete()
499 ro = &rm->rdma; in rds_rdma_send_complete()
500 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) && in rds_rdma_send_complete()
503 rs = rm->m_rs; in rds_rdma_send_complete()
514 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
526 void rds_atomic_send_complete(struct rds_message *rm, int status) in rds_atomic_send_complete() argument
533 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
535 ao = &rm->atomic; in rds_atomic_send_complete()
536 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) in rds_atomic_send_complete()
539 rs = rm->m_rs; in rds_atomic_send_complete()
550 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
565 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status) in __rds_send_complete() argument
570 ro = &rm->rdma; in __rds_send_complete()
577 ao = &rm->atomic; in __rds_send_complete()
599 struct rds_message *rm; in rds_send_remove_from_sock() local
604 rm = list_entry(messages->next, struct rds_message, in rds_send_remove_from_sock()
606 list_del_init(&rm->m_conn_item); in rds_send_remove_from_sock()
618 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
619 if (!test_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) in rds_send_remove_from_sock()
622 if (rs != rm->m_rs) { in rds_send_remove_from_sock()
627 rs = rm->m_rs; in rds_send_remove_from_sock()
635 if (test_and_clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) { in rds_send_remove_from_sock()
636 struct rm_rdma_op *ro = &rm->rdma; in rds_send_remove_from_sock()
639 list_del_init(&rm->m_sock_item); in rds_send_remove_from_sock()
640 rds_send_sndbuf_remove(rs, rm); in rds_send_remove_from_sock()
649 rm->rdma.op_notifier = NULL; in rds_send_remove_from_sock()
656 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
657 rds_message_put(rm); in rds_send_remove_from_sock()
659 rds_message_put(rm); in rds_send_remove_from_sock()
679 struct rds_message *rm, *tmp; in rds_send_path_drop_acked() local
685 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_drop_acked()
686 if (!rds_send_is_acked(rm, ack, is_acked)) in rds_send_path_drop_acked()
689 list_move(&rm->m_conn_item, &list); in rds_send_path_drop_acked()
690 clear_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_path_drop_acked()
714 struct rds_message *rm, *tmp; in rds_send_drop_to() local
723 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) { in rds_send_drop_to()
725 (!ipv6_addr_equal(&dest->sin6_addr, &rm->m_daddr) || in rds_send_drop_to()
726 dest->sin6_port != rm->m_inc.i_hdr.h_dport)) in rds_send_drop_to()
729 list_move(&rm->m_sock_item, &list); in rds_send_drop_to()
730 rds_send_sndbuf_remove(rs, rm); in rds_send_drop_to()
731 clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_drop_to()
743 list_for_each_entry(rm, &list, m_sock_item) { in rds_send_drop_to()
745 conn = rm->m_inc.i_conn; in rds_send_drop_to()
747 cp = rm->m_inc.i_conn_path; in rds_send_drop_to()
757 if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) { in rds_send_drop_to()
761 list_del_init(&rm->m_conn_item); in rds_send_drop_to()
768 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
771 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
774 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
776 rds_message_put(rm); in rds_send_drop_to()
782 rm = list_entry(list.next, struct rds_message, m_sock_item); in rds_send_drop_to()
783 list_del_init(&rm->m_sock_item); in rds_send_drop_to()
784 rds_message_wait(rm); in rds_send_drop_to()
791 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
794 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
797 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
799 rds_message_put(rm); in rds_send_drop_to()
810 struct rds_message *rm, __be16 sport, in rds_send_queue_rm() argument
819 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_queue_rm()
842 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_queue_rm()
844 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue); in rds_send_queue_rm()
845 set_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_queue_rm()
846 rds_message_addref(rm); in rds_send_queue_rm()
848 rm->m_rs = rs; in rds_send_queue_rm()
852 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0); in rds_send_queue_rm()
853 rm->m_inc.i_conn = conn; in rds_send_queue_rm()
854 rm->m_inc.i_conn_path = cp; in rds_send_queue_rm()
855 rds_message_addref(rm); in rds_send_queue_rm()
858 rm->m_inc.i_hdr.h_sequence = cpu_to_be64(cp->cp_next_tx_seq++); in rds_send_queue_rm()
859 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_queue_rm()
860 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_queue_rm()
864 rm, len, rs, rs->rs_snd_bytes, in rds_send_queue_rm()
865 (unsigned long long)be64_to_cpu(rm->m_inc.i_hdr.h_sequence)); in rds_send_queue_rm()
940 static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_zcopy() argument
946 !rm->data.op_mmp_znotifier) in rds_cmsg_zcopy()
949 rm->data.op_mmp_znotifier->z_cookie = *cookie; in rds_cmsg_zcopy()
953 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_send() argument
971 ret = rds_cmsg_rdma_args(rs, rm, cmsg); in rds_cmsg_send()
975 ret = rds_cmsg_rdma_dest(rs, rm, cmsg); in rds_cmsg_send()
979 ret = rds_cmsg_rdma_map(rs, rm, cmsg); in rds_cmsg_send()
992 ret = rds_cmsg_atomic(rs, rm, cmsg); in rds_cmsg_send()
996 ret = rds_cmsg_zcopy(rs, rm, cmsg); in rds_cmsg_send()
1073 struct rds_message *rm = NULL; in rds_sendmsg() local
1227 rm = rds_message_alloc(ret, GFP_KERNEL); in rds_sendmsg()
1228 if (!rm) { in rds_sendmsg()
1235 rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs); in rds_sendmsg()
1236 if (!rm->data.op_sg) { in rds_sendmsg()
1240 ret = rds_message_copy_from_user(rm, &msg->msg_iter, zcopy); in rds_sendmsg()
1244 rm->data.op_active = 1; in rds_sendmsg()
1246 rm->m_daddr = daddr; in rds_sendmsg()
1270 rm->m_conn_path = cpath; in rds_sendmsg()
1273 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr); in rds_sendmsg()
1281 if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) { in rds_sendmsg()
1283 &rm->rdma, conn->c_trans->xmit_rdma); in rds_sendmsg()
1288 if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) { in rds_sendmsg()
1290 &rm->atomic, conn->c_trans->xmit_atomic); in rds_sendmsg()
1307 while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port, in rds_sendmsg()
1317 rds_send_queue_rm(rs, conn, cpath, rm, in rds_sendmsg()
1350 rds_message_put(rm); in rds_sendmsg()
1358 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1); in rds_sendmsg()
1360 if (rm) in rds_sendmsg()
1361 rds_message_put(rm); in rds_sendmsg()
1377 struct rds_message *rm; in rds_send_probe() local
1381 rm = rds_message_alloc(0, GFP_ATOMIC); in rds_send_probe()
1382 if (!rm) { in rds_send_probe()
1387 rm->m_daddr = cp->cp_conn->c_faddr; in rds_send_probe()
1388 rm->data.op_active = 1; in rds_send_probe()
1397 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_probe()
1398 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_probe()
1399 rds_message_addref(rm); in rds_send_probe()
1400 rm->m_inc.i_conn = cp->cp_conn; in rds_send_probe()
1401 rm->m_inc.i_conn_path = cp; in rds_send_probe()
1403 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, in rds_send_probe()
1405 rm->m_inc.i_hdr.h_flags |= h_flags; in rds_send_probe()
1413 rds_message_add_extension(&rm->m_inc.i_hdr, in rds_send_probe()
1416 rds_message_add_extension(&rm->m_inc.i_hdr, in rds_send_probe()
1432 rds_message_put(rm); in rds_send_probe()
1436 if (rm) in rds_send_probe()
1437 rds_message_put(rm); in rds_send_probe()