Lines Matching full:rs
473 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm) in rds_send_sndbuf_remove() argument
477 assert_spin_locked(&rs->rs_lock); in rds_send_sndbuf_remove()
479 BUG_ON(rs->rs_snd_bytes < len); in rds_send_sndbuf_remove()
480 rs->rs_snd_bytes -= len; in rds_send_sndbuf_remove()
482 if (rs->rs_snd_bytes == 0) in rds_send_sndbuf_remove()
502 struct rds_sock *rs = NULL; in rds_rdma_send_complete() local
513 rs = rm->m_rs; in rds_rdma_send_complete()
514 sock_hold(rds_rs_to_sk(rs)); in rds_rdma_send_complete()
517 spin_lock(&rs->rs_lock); in rds_rdma_send_complete()
518 list_add_tail(¬ifier->n_list, &rs->rs_notify_queue); in rds_rdma_send_complete()
519 spin_unlock(&rs->rs_lock); in rds_rdma_send_complete()
526 if (rs) { in rds_rdma_send_complete()
527 rds_wake_sk_sleep(rs); in rds_rdma_send_complete()
528 sock_put(rds_rs_to_sk(rs)); in rds_rdma_send_complete()
538 struct rds_sock *rs = NULL; in rds_atomic_send_complete() local
549 rs = rm->m_rs; in rds_atomic_send_complete()
550 sock_hold(rds_rs_to_sk(rs)); in rds_atomic_send_complete()
553 spin_lock(&rs->rs_lock); in rds_atomic_send_complete()
554 list_add_tail(¬ifier->n_list, &rs->rs_notify_queue); in rds_atomic_send_complete()
555 spin_unlock(&rs->rs_lock); in rds_atomic_send_complete()
562 if (rs) { in rds_atomic_send_complete()
563 rds_wake_sk_sleep(rs); in rds_atomic_send_complete()
564 sock_put(rds_rs_to_sk(rs)); in rds_atomic_send_complete()
575 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status) in __rds_send_complete() argument
583 list_add_tail(&ro->op_notifier->n_list, &rs->rs_notify_queue); in __rds_send_complete()
590 list_add_tail(&ao->op_notifier->n_list, &rs->rs_notify_queue); in __rds_send_complete()
608 struct rds_sock *rs = NULL; in rds_send_remove_from_sock() local
632 if (rs != rm->m_rs) { in rds_send_remove_from_sock()
633 if (rs) { in rds_send_remove_from_sock()
634 rds_wake_sk_sleep(rs); in rds_send_remove_from_sock()
635 sock_put(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
637 rs = rm->m_rs; in rds_send_remove_from_sock()
638 if (rs) in rds_send_remove_from_sock()
639 sock_hold(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
641 if (!rs) in rds_send_remove_from_sock()
643 spin_lock(&rs->rs_lock); in rds_send_remove_from_sock()
650 rds_send_sndbuf_remove(rs, rm); in rds_send_remove_from_sock()
656 &rs->rs_notify_queue); in rds_send_remove_from_sock()
663 spin_unlock(&rs->rs_lock); in rds_send_remove_from_sock()
672 if (rs) { in rds_send_remove_from_sock()
673 rds_wake_sk_sleep(rs); in rds_send_remove_from_sock()
674 sock_put(rds_rs_to_sk(rs)); in rds_send_remove_from_sock()
722 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest) in rds_send_drop_to() argument
730 /* get all the messages we're dropping under the rs lock */ in rds_send_drop_to()
731 spin_lock_irqsave(&rs->rs_lock, flags); in rds_send_drop_to()
733 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) { in rds_send_drop_to()
740 rds_send_sndbuf_remove(rs, rm); in rds_send_drop_to()
744 /* order flag updates with the rs lock */ in rds_send_drop_to()
747 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_send_drop_to()
780 spin_lock(&rs->rs_lock); in rds_send_drop_to()
781 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
782 spin_unlock(&rs->rs_lock); in rds_send_drop_to()
789 rds_wake_sk_sleep(rs); in rds_send_drop_to()
803 spin_lock(&rs->rs_lock); in rds_send_drop_to()
804 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
805 spin_unlock(&rs->rs_lock); in rds_send_drop_to()
818 static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn, in rds_send_queue_rm() argument
833 spin_lock_irqsave(&rs->rs_lock, flags); in rds_send_queue_rm()
843 if (rs->rs_snd_bytes < rds_sk_sndbuf(rs)) { in rds_send_queue_rm()
844 rs->rs_snd_bytes += len; in rds_send_queue_rm()
851 if (rs->rs_snd_bytes >= rds_sk_sndbuf(rs) / 2) in rds_send_queue_rm()
854 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue); in rds_send_queue_rm()
857 sock_hold(rds_rs_to_sk(rs)); in rds_send_queue_rm()
858 rm->m_rs = rs; in rds_send_queue_rm()
873 rdsdebug("queued msg %p len %d, rs %p bytes %d seq %llu\n", in rds_send_queue_rm()
874 rm, len, rs, rs->rs_snd_bytes, in rds_send_queue_rm()
880 spin_unlock_irqrestore(&rs->rs_lock, flags); in rds_send_queue_rm()
971 static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_zcopy() argument
984 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_send() argument
1005 ret = rds_cmsg_rdma_args(rs, rm, cmsg, &vct->vec[ind]); in rds_cmsg_send()
1010 ret = rds_cmsg_rdma_dest(rs, rm, cmsg); in rds_cmsg_send()
1014 ret = rds_cmsg_rdma_map(rs, rm, cmsg); in rds_cmsg_send()
1027 ret = rds_cmsg_atomic(rs, rm, cmsg); in rds_cmsg_send()
1031 ret = rds_cmsg_zcopy(rs, rm, cmsg); in rds_cmsg_send()
1045 static int rds_send_mprds_hash(struct rds_sock *rs, in rds_send_mprds_hash() argument
1051 hash = RDS_MPATH_HASH(rs, RDS_MPATH_WORKERS); in rds_send_mprds_hash()
1053 hash = RDS_MPATH_HASH(rs, conn->c_npaths); in rds_send_mprds_hash()
1104 struct rds_sock *rs = rds_sk_to_rs(sk); in rds_sendmsg() local
1119 sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY)); in rds_sendmsg()
1204 daddr = rs->rs_conn_addr; in rds_sendmsg()
1205 dport = rs->rs_conn_port; in rds_sendmsg()
1206 scope_id = rs->rs_bound_scope_id; in rds_sendmsg()
1211 if (ipv6_addr_any(&rs->rs_bound_addr) || ipv6_addr_any(&daddr)) { in rds_sendmsg()
1221 ipv6_addr_v4mapped(&rs->rs_bound_addr)) { in rds_sendmsg()
1230 if (scope_id != rs->rs_bound_scope_id) { in rds_sendmsg()
1232 scope_id = rs->rs_bound_scope_id; in rds_sendmsg()
1233 } else if (rs->rs_bound_scope_id) { in rds_sendmsg()
1252 if (payload_len > rds_sk_sndbuf(rs)) { in rds_sendmsg()
1258 if (rs->rs_transport->t_type != RDS_TRANS_TCP) { in rds_sendmsg()
1292 if (rs->rs_conn && ipv6_addr_equal(&rs->rs_conn->c_faddr, &daddr) && in rds_sendmsg()
1293 rs->rs_tos == rs->rs_conn->c_tos) { in rds_sendmsg()
1294 conn = rs->rs_conn; in rds_sendmsg()
1297 &rs->rs_bound_addr, &daddr, in rds_sendmsg()
1298 rs->rs_transport, rs->rs_tos, in rds_sendmsg()
1305 rs->rs_conn = conn; in rds_sendmsg()
1309 cpath = &conn->c_path[rds_send_mprds_hash(rs, conn, nonblock)]; in rds_sendmsg()
1316 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr, &vct); in rds_sendmsg()
1346 ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs); in rds_sendmsg()
1348 rs->rs_seen_congestion = 1; in rds_sendmsg()
1351 while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port, in rds_sendmsg()
1361 rds_send_queue_rm(rs, conn, cpath, rm, in rds_sendmsg()
1362 rs->rs_bound_port, in rds_sendmsg()
1411 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1); in rds_sendmsg()