2415 static void qp_lock(const struct vmci_qp *qpair)  in qp_lock()  argument
2417 	qp_acquire_queue_mutex(qpair->produce_q);  in qp_lock()
2424 static void qp_unlock(const struct vmci_qp *qpair)  in qp_unlock()  argument
2426 	qp_release_queue_mutex(qpair->produce_q);  in qp_unlock()
2456 static int qp_get_queue_headers(const struct vmci_qp *qpair,  in qp_get_queue_headers()  argument
2462 	result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);  in qp_get_queue_headers()
2464 		*produce_q_header = qpair->produce_q->q_header;  in qp_get_queue_headers()
2465 		*consume_q_header = qpair->consume_q->q_header;  in qp_get_queue_headers()
2466 	} else if (qpair->produce_q->saved_header &&  in qp_get_queue_headers()
2467 		   qpair->consume_q->saved_header) {  in qp_get_queue_headers()
2468 		*produce_q_header = qpair->produce_q->saved_header;  in qp_get_queue_headers()
2469 		*consume_q_header = qpair->consume_q->saved_header;  in qp_get_queue_headers()
2483 	struct vmci_qp *qpair = (struct vmci_qp *)client_data;  in qp_wakeup_cb()  local
2485 	qp_lock(qpair);  in qp_wakeup_cb()
2486 	while (qpair->blocked > 0) {  in qp_wakeup_cb()
2487 		qpair->blocked--;  in qp_wakeup_cb()
2488 		qpair->generation++;  in qp_wakeup_cb()
2489 		wake_up(&qpair->event);  in qp_wakeup_cb()
2491 	qp_unlock(qpair);  in qp_wakeup_cb()
2501 static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)  in qp_wait_for_ready_queue()  argument
2505 	qpair->blocked++;  in qp_wait_for_ready_queue()
2506 	generation = qpair->generation;  in qp_wait_for_ready_queue()
2507 	qp_unlock(qpair);  in qp_wait_for_ready_queue()
2508 	wait_event(qpair->event, generation != qpair->generation);  in qp_wait_for_ready_queue()
2509 	qp_lock(qpair);  in qp_wait_for_ready_queue()
2653 int vmci_qpair_alloc(struct vmci_qp **qpair,  in vmci_qpair_alloc()  argument
2742 	*qpair = my_qpair;  in vmci_qpair_alloc()
2757 int vmci_qpair_detach(struct vmci_qp **qpair)  in vmci_qpair_detach()  argument
2762 	if (!qpair || !(*qpair))  in vmci_qpair_detach()
2765 	old_qpair = *qpair;  in vmci_qpair_detach()
2782 	*qpair = NULL;  in vmci_qpair_detach()
2797 int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,  in vmci_qpair_get_produce_indexes()  argument
2805 	if (!qpair)  in vmci_qpair_get_produce_indexes()
2808 	qp_lock(qpair);  in vmci_qpair_get_produce_indexes()
2810 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);  in vmci_qpair_get_produce_indexes()
2814 	qp_unlock(qpair);  in vmci_qpair_get_produce_indexes()
2817 	    ((producer_tail && *producer_tail >= qpair->produce_q_size) ||  in vmci_qpair_get_produce_indexes()
2818 	     (consumer_head && *consumer_head >= qpair->produce_q_size)))  in vmci_qpair_get_produce_indexes()
2834 int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,  in vmci_qpair_get_consume_indexes()  argument
2842 	if (!qpair)  in vmci_qpair_get_consume_indexes()
2845 	qp_lock(qpair);  in vmci_qpair_get_consume_indexes()
2847 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);  in vmci_qpair_get_consume_indexes()
2851 	qp_unlock(qpair);  in vmci_qpair_get_consume_indexes()
2854 	    ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||  in vmci_qpair_get_consume_indexes()
2855 	     (producer_head && *producer_head >= qpair->consume_q_size)))  in vmci_qpair_get_consume_indexes()
2871 s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)  in vmci_qpair_produce_free_space()  argument
2877 	if (!qpair)  in vmci_qpair_produce_free_space()
2880 	qp_lock(qpair);  in vmci_qpair_produce_free_space()
2882 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);  in vmci_qpair_produce_free_space()
2886 						  qpair->produce_q_size);  in vmci_qpair_produce_free_space()
2890 	qp_unlock(qpair);  in vmci_qpair_produce_free_space()
2905 s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)  in vmci_qpair_consume_free_space()  argument
2911 	if (!qpair)  in vmci_qpair_consume_free_space()
2914 	qp_lock(qpair);  in vmci_qpair_consume_free_space()
2916 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);  in vmci_qpair_consume_free_space()
2920 						  qpair->consume_q_size);  in vmci_qpair_consume_free_space()
2924 	qp_unlock(qpair);  in vmci_qpair_consume_free_space()
2940 s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)  in vmci_qpair_produce_buf_ready()  argument
2946 	if (!qpair)  in vmci_qpair_produce_buf_ready()
2949 	qp_lock(qpair);  in vmci_qpair_produce_buf_ready()
2951 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);  in vmci_qpair_produce_buf_ready()
2955 						 qpair->produce_q_size);  in vmci_qpair_produce_buf_ready()
2959 	qp_unlock(qpair);  in vmci_qpair_produce_buf_ready()
2975 s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)  in vmci_qpair_consume_buf_ready()  argument
2981 	if (!qpair)  in vmci_qpair_consume_buf_ready()
2984 	qp_lock(qpair);  in vmci_qpair_consume_buf_ready()
2986 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);  in vmci_qpair_consume_buf_ready()
2990 						 qpair->consume_q_size);  in vmci_qpair_consume_buf_ready()
2994 	qp_unlock(qpair);  in vmci_qpair_consume_buf_ready()
3010 ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,  in vmci_qpair_enqueue()  argument
3019 	if (!qpair || !buf)  in vmci_qpair_enqueue()
3024 	qp_lock(qpair);  in vmci_qpair_enqueue()
3027 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enqueue()
3028 					   qpair->consume_q,  in vmci_qpair_enqueue()
3029 					   qpair->produce_q_size,  in vmci_qpair_enqueue()
3033 		    !qp_wait_for_ready_queue(qpair))  in vmci_qpair_enqueue()
3038 	qp_unlock(qpair);  in vmci_qpair_enqueue()
3054 ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,  in vmci_qpair_dequeue()  argument
3063 	if (!qpair || !buf)  in vmci_qpair_dequeue()
3068 	qp_lock(qpair);  in vmci_qpair_dequeue()
3071 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequeue()
3072 					   qpair->consume_q,  in vmci_qpair_dequeue()
3073 					   qpair->consume_q_size,  in vmci_qpair_dequeue()
3077 		    !qp_wait_for_ready_queue(qpair))  in vmci_qpair_dequeue()
3082 	qp_unlock(qpair);  in vmci_qpair_dequeue()
3099 ssize_t vmci_qpair_peek(struct vmci_qp *qpair,  in vmci_qpair_peek()  argument
3108 	if (!qpair || !buf)  in vmci_qpair_peek()
3113 	qp_lock(qpair);  in vmci_qpair_peek()
3116 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peek()
3117 					   qpair->consume_q,  in vmci_qpair_peek()
3118 					   qpair->consume_q_size,  in vmci_qpair_peek()
3122 		    !qp_wait_for_ready_queue(qpair))  in vmci_qpair_peek()
3127 	qp_unlock(qpair);  in vmci_qpair_peek()
3144 ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,  in vmci_qpair_enquev()  argument
3151 	if (!qpair)  in vmci_qpair_enquev()
3154 	qp_lock(qpair);  in vmci_qpair_enquev()
3157 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enquev()
3158 					   qpair->consume_q,  in vmci_qpair_enquev()
3159 					   qpair->produce_q_size,  in vmci_qpair_enquev()
3163 		    !qp_wait_for_ready_queue(qpair))  in vmci_qpair_enquev()
3168 	qp_unlock(qpair);  in vmci_qpair_enquev()
3185 ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,  in vmci_qpair_dequev()  argument
3192 	if (!qpair)  in vmci_qpair_dequev()
3195 	qp_lock(qpair);  in vmci_qpair_dequev()
3198 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequev()
3199 					   qpair->consume_q,  in vmci_qpair_dequev()
3200 					   qpair->consume_q_size,  in vmci_qpair_dequev()
3204 		    !qp_wait_for_ready_queue(qpair))  in vmci_qpair_dequev()
3209 	qp_unlock(qpair);  in vmci_qpair_dequev()
3227 ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,  in vmci_qpair_peekv()  argument
3234 	if (!qpair)  in vmci_qpair_peekv()
3237 	qp_lock(qpair);  in vmci_qpair_peekv()
3240 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peekv()
3241 					   qpair->consume_q,  in vmci_qpair_peekv()
3242 					   qpair->consume_q_size,  in vmci_qpair_peekv()
3246 		    !qp_wait_for_ready_queue(qpair))  in vmci_qpair_peekv()
3251 	qp_unlock(qpair);  in vmci_qpair_peekv()