147 	struct vmci_queue *produce_q;  member
203 	struct vmci_queue *produce_q;  member
216 	void *produce_q;  member
434 	struct vmci_queue *produce_q = prod_q;  in qp_alloc_ppn_set()  local
438 	if (!produce_q || !num_produce_pages || !consume_q ||  in qp_alloc_ppn_set()
461 			produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;  in qp_alloc_ppn_set()
578 static void qp_init_queue_mutex(struct vmci_queue *produce_q,  in qp_init_queue_mutex()  argument
586 	if (produce_q->kernel_if->host) {  in qp_init_queue_mutex()
587 		produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
588 		consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
589 		mutex_init(produce_q->kernel_if->mutex);  in qp_init_queue_mutex()
596 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,  in qp_cleanup_queue_mutex()  argument
599 	if (produce_q->kernel_if->host) {  in qp_cleanup_queue_mutex()
600 		produce_q->kernel_if->mutex = NULL;  in qp_cleanup_queue_mutex()
652 				   struct vmci_queue *produce_q,  in qp_host_get_user_memory()  argument
659 				     produce_q->kernel_if->num_pages,  in qp_host_get_user_memory()
661 				     produce_q->kernel_if->u.h.header_page);  in qp_host_get_user_memory()
662 	if (retval < (int)produce_q->kernel_if->num_pages) {  in qp_host_get_user_memory()
666 			qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
682 		qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
683 				 produce_q->kernel_if->num_pages, false);  in qp_host_get_user_memory()
697 					struct vmci_queue *produce_q,  in qp_host_register_user_memory()  argument
710 	    produce_q->kernel_if->num_pages * PAGE_SIZE;  in qp_host_register_user_memory()
711 	return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,  in qp_host_register_user_memory()
720 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,  in qp_host_unregister_user_memory()  argument
723 	qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_unregister_user_memory()
724 			 produce_q->kernel_if->num_pages, true);  in qp_host_unregister_user_memory()
725 	memset(produce_q->kernel_if->u.h.header_page, 0,  in qp_host_unregister_user_memory()
726 	       sizeof(*produce_q->kernel_if->u.h.header_page) *  in qp_host_unregister_user_memory()
727 	       produce_q->kernel_if->num_pages);  in qp_host_unregister_user_memory()
743 static int qp_host_map_queues(struct vmci_queue *produce_q,  in qp_host_map_queues()  argument
748 	if (!produce_q->q_header || !consume_q->q_header) {  in qp_host_map_queues()
751 		if (produce_q->q_header != consume_q->q_header)  in qp_host_map_queues()
754 		if (produce_q->kernel_if->u.h.header_page == NULL ||  in qp_host_map_queues()
755 		    *produce_q->kernel_if->u.h.header_page == NULL)  in qp_host_map_queues()
758 		headers[0] = *produce_q->kernel_if->u.h.header_page;  in qp_host_map_queues()
761 		produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);  in qp_host_map_queues()
762 		if (produce_q->q_header != NULL) {  in qp_host_map_queues()
765 							 produce_q->q_header +  in qp_host_map_queues()
784 				struct vmci_queue *produce_q,  in qp_host_unmap_queues()  argument
787 	if (produce_q->q_header) {  in qp_host_unmap_queues()
788 		if (produce_q->q_header < consume_q->q_header)  in qp_host_unmap_queues()
789 			vunmap(produce_q->q_header);  in qp_host_unmap_queues()
793 		produce_q->q_header = NULL;  in qp_host_unmap_queues()
883 			 void *produce_q,  in qp_guest_endpoint_create()  argument
906 		entry->produce_q = produce_q;  in qp_guest_endpoint_create()
932 	qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in qp_guest_endpoint_destroy()
933 	qp_free_queue(entry->produce_q, entry->qp.produce_size);  in qp_guest_endpoint_destroy()
1094 			       struct vmci_queue **produce_q,  in qp_alloc_guest_work()  argument
1146 			my_consume_q = queue_pair_entry->produce_q;  in qp_alloc_guest_work()
1229 	*produce_q = (struct vmci_queue *)my_produce_q;  in qp_alloc_guest_work()
1239 		vmci_q_header_init((*produce_q)->q_header, *handle);  in qp_alloc_guest_work()
1357 	entry->produce_q = qp_host_alloc_queue(guest_produce_size);  in qp_broker_create()
1358 	if (entry->produce_q == NULL) {  in qp_broker_create()
1368 	qp_init_queue_mutex(entry->produce_q, entry->consume_q);  in qp_broker_create()
1382 		entry->produce_q->q_header = entry->local_mem;  in qp_broker_create()
1392 						      entry->produce_q,  in qp_broker_create()
1425 		vmci_q_header_init(entry->produce_q->q_header,  in qp_broker_create()
1437 		qp_host_free_queue(entry->produce_q, guest_produce_size);  in qp_broker_create()
1639 							      entry->produce_q,  in qp_broker_attach()
1771 			      struct vmci_queue **produce_q,  in qp_alloc_host_work()  argument
1807 			*produce_q = entry->consume_q;  in qp_alloc_host_work()
1808 			*consume_q = entry->produce_q;  in qp_alloc_host_work()
1810 			*produce_q = entry->produce_q;  in qp_alloc_host_work()
1830 		  struct vmci_queue **produce_q,  in vmci_qp_alloc()  argument
1841 	if (!handle || !produce_q || !consume_q ||  in vmci_qp_alloc()
1846 		return qp_alloc_guest_work(handle, produce_q,  in vmci_qp_alloc()
1851 		return qp_alloc_host_work(handle, produce_q,  in vmci_qp_alloc()
2019 					 entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2023 	result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2025 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_set_page_store()
2060 	entry->produce_q->saved_header = NULL;  in qp_reset_saved_headers()
2139 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2140 		headers_mapped = entry->produce_q->q_header ||  in vmci_qp_broker_detach()
2145 						 entry->produce_q,  in vmci_qp_broker_detach()
2152 			qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_detach()
2160 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2178 		qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in vmci_qp_broker_detach()
2179 		qp_host_free_queue(entry->produce_q, entry->qp.produce_size);  in vmci_qp_broker_detach()
2255 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2259 						 entry->produce_q,  in vmci_qp_broker_map()
2261 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2288 	if (entry->produce_q->saved_header != NULL &&  in qp_save_headers()
2299 	if (NULL == entry->produce_q->q_header ||  in qp_save_headers()
2301 		result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in qp_save_headers()
2306 	memcpy(&entry->saved_produce_q, entry->produce_q->q_header,  in qp_save_headers()
2308 	entry->produce_q->saved_header = &entry->saved_produce_q;  in qp_save_headers()
2358 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2364 		qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);  in vmci_qp_broker_unmap()
2373 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_unmap()
2381 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2431 	qp_acquire_queue_mutex(qpair->produce_q);  in qp_lock()
2440 	qp_release_queue_mutex(qpair->produce_q);  in qp_unlock()
2447 static int qp_map_queue_headers(struct vmci_queue *produce_q,  in qp_map_queue_headers()  argument
2452 	if (NULL == produce_q->q_header || NULL == consume_q->q_header) {  in qp_map_queue_headers()
2453 		result = qp_host_map_queues(produce_q, consume_q);  in qp_map_queue_headers()
2455 			return (produce_q->saved_header &&  in qp_map_queue_headers()
2476 	result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);  in qp_get_queue_headers()
2478 		*produce_q_header = qpair->produce_q->q_header;  in qp_get_queue_headers()
2480 	} else if (qpair->produce_q->saved_header &&  in qp_get_queue_headers()
2482 		*produce_q_header = qpair->produce_q->saved_header;  in qp_get_queue_headers()
2540 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,  in qp_enqueue_locked()  argument
2551 	result = qp_map_queue_headers(produce_q, consume_q);  in qp_enqueue_locked()
2555 	free_space = vmci_q_header_free_space(produce_q->q_header,  in qp_enqueue_locked()
2565 	tail = vmci_q_header_producer_tail(produce_q->q_header);  in qp_enqueue_locked()
2567 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);  in qp_enqueue_locked()
2573 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);  in qp_enqueue_locked()
2575 			result = qp_memcpy_to_queue_iter(produce_q, 0, from,  in qp_enqueue_locked()
2588 	vmci_q_header_add_producer_tail(produce_q->q_header, written,  in qp_enqueue_locked()
2606 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,  in qp_dequeue_locked()  argument
2618 	result = qp_map_queue_headers(produce_q, consume_q);  in qp_dequeue_locked()
2623 					    produce_q->q_header,  in qp_dequeue_locked()
2638 	head = vmci_q_header_consumer_head(produce_q->q_header);  in qp_dequeue_locked()
2657 		vmci_q_header_add_consumer_head(produce_q->q_header,  in qp_dequeue_locked()
2752 			       &my_qpair->produce_q,  in vmci_qpair_alloc()
3052 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enqueue()
3096 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequeue()
3141 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peek()
3182 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enquev()
3223 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequev()
3265 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peekv()