221 	void (*add_page)(struct vmballoon *b, int idx, struct page *p);
222 	int (*lock)(struct vmballoon *b, unsigned int num_pages,
224 	int (*unlock)(struct vmballoon *b, unsigned int num_pages,
279 static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)  in vmballoon_send_start()  argument
284 	STATS_INC(b->stats.start);  in vmballoon_send_start()
290 		b->capabilities = capabilities;  in vmballoon_send_start()
294 		b->capabilities = VMW_BALLOON_BASIC_CMDS;  in vmballoon_send_start()
306 	if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&  in vmballoon_send_start()
307 	    (b->capabilities & VMW_BALLOON_BATCHED_CMDS))  in vmballoon_send_start()
308 		b->supported_page_sizes = 2;  in vmballoon_send_start()
310 		b->supported_page_sizes = 1;  in vmballoon_send_start()
314 		STATS_INC(b->stats.start_fail);  in vmballoon_send_start()
319 static bool vmballoon_check_status(struct vmballoon *b, unsigned long status)  in vmballoon_check_status()  argument
326 		b->reset_required = true;  in vmballoon_check_status()
340 static bool vmballoon_send_guest_id(struct vmballoon *b)  in vmballoon_send_guest_id()  argument
347 	STATS_INC(b->stats.guest_type);  in vmballoon_send_guest_id()
349 	if (vmballoon_check_status(b, status))  in vmballoon_send_guest_id()
353 	STATS_INC(b->stats.guest_type_fail);  in vmballoon_send_guest_id()
368 static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target)  in vmballoon_send_get_target()  argument
381 	si_meminfo(&b->sysinfo);  in vmballoon_send_get_target()
382 	limit = b->sysinfo.totalram;  in vmballoon_send_get_target()
390 	STATS_INC(b->stats.target);  in vmballoon_send_get_target()
393 	if (vmballoon_check_status(b, status)) {  in vmballoon_send_get_target()
399 	STATS_INC(b->stats.target_fail);  in vmballoon_send_get_target()
408 static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,  in vmballoon_send_lock_page()  argument
418 	STATS_INC(b->stats.lock[false]);  in vmballoon_send_lock_page()
421 	if (vmballoon_check_status(b, status))  in vmballoon_send_lock_page()
425 	STATS_INC(b->stats.lock_fail[false]);  in vmballoon_send_lock_page()
429 static int vmballoon_send_batched_lock(struct vmballoon *b,  in vmballoon_send_batched_lock()  argument
433 	unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));  in vmballoon_send_batched_lock()
435 	STATS_INC(b->stats.lock[is_2m_pages]);  in vmballoon_send_batched_lock()
444 	if (vmballoon_check_status(b, status))  in vmballoon_send_batched_lock()
448 	STATS_INC(b->stats.lock_fail[is_2m_pages]);  in vmballoon_send_batched_lock()
456 static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn,  in vmballoon_send_unlock_page()  argument
466 	STATS_INC(b->stats.unlock[false]);  in vmballoon_send_unlock_page()
469 	if (vmballoon_check_status(b, status))  in vmballoon_send_unlock_page()
473 	STATS_INC(b->stats.unlock_fail[false]);  in vmballoon_send_unlock_page()
477 static bool vmballoon_send_batched_unlock(struct vmballoon *b,  in vmballoon_send_batched_unlock()  argument
481 	unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page));  in vmballoon_send_batched_unlock()
483 	STATS_INC(b->stats.unlock[is_2m_pages]);  in vmballoon_send_batched_unlock()
492 	if (vmballoon_check_status(b, status))  in vmballoon_send_batched_unlock()
496 	STATS_INC(b->stats.unlock_fail[is_2m_pages]);  in vmballoon_send_batched_unlock()
522 static void vmballoon_pop(struct vmballoon *b)  in vmballoon_pop()  argument
530 				&b->page_sizes[is_2m_pages];  in vmballoon_pop()
536 			STATS_INC(b->stats.free[is_2m_pages]);  in vmballoon_pop()
537 			b->size -= size_per_page;  in vmballoon_pop()
543 	free_page((unsigned long)b->batch_page);  in vmballoon_pop()
544 	b->batch_page = NULL;  in vmballoon_pop()
552 static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages,  in vmballoon_lock_page()  argument
556 	struct page *page = b->page;  in vmballoon_lock_page()
557 	struct vmballoon_page_size *page_size = &b->page_sizes[false];  in vmballoon_lock_page()
561 	locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status,  in vmballoon_lock_page()
564 		STATS_INC(b->stats.refused_alloc[false]);  in vmballoon_lock_page()
591 	b->size++;  in vmballoon_lock_page()
596 static int vmballoon_lock_batched_page(struct vmballoon *b,  in vmballoon_lock_batched_page()  argument
602 	locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages,  in vmballoon_lock_batched_page()
606 			u64 pa = vmballoon_batch_get_pa(b->batch_page, i);  in vmballoon_lock_batched_page()
616 		u64 pa = vmballoon_batch_get_pa(b->batch_page, i);  in vmballoon_lock_batched_page()
619 				&b->page_sizes[is_2m_pages];  in vmballoon_lock_batched_page()
621 		locked = vmballoon_batch_get_status(b->batch_page, i);  in vmballoon_lock_batched_page()
626 			b->size += size_per_page;  in vmballoon_lock_batched_page()
655 static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages,  in vmballoon_unlock_page()  argument
658 	struct page *page = b->page;  in vmballoon_unlock_page()
659 	struct vmballoon_page_size *page_size = &b->page_sizes[false];  in vmballoon_unlock_page()
663 	if (!vmballoon_send_unlock_page(b, page_to_pfn(page), target)) {  in vmballoon_unlock_page()
670 	STATS_INC(b->stats.free[false]);  in vmballoon_unlock_page()
673 	b->size--;  in vmballoon_unlock_page()
678 static int vmballoon_unlock_batched_page(struct vmballoon *b,  in vmballoon_unlock_batched_page()  argument
686 	hv_success = vmballoon_send_batched_unlock(b, num_pages, is_2m_pages,  in vmballoon_unlock_batched_page()
692 		u64 pa = vmballoon_batch_get_pa(b->batch_page, i);  in vmballoon_unlock_batched_page()
695 				&b->page_sizes[is_2m_pages];  in vmballoon_unlock_batched_page()
697 		locked = vmballoon_batch_get_status(b->batch_page, i);  in vmballoon_unlock_batched_page()
708 			STATS_INC(b->stats.free[is_2m_pages]);  in vmballoon_unlock_batched_page()
711 			b->size -= size_per_page;  in vmballoon_unlock_batched_page()
722 static void vmballoon_release_refused_pages(struct vmballoon *b,  in vmballoon_release_refused_pages()  argument
727 			&b->page_sizes[is_2m_pages];  in vmballoon_release_refused_pages()
732 		STATS_INC(b->stats.refused_free[is_2m_pages]);  in vmballoon_release_refused_pages()
738 static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p)  in vmballoon_add_page()  argument
740 	b->page = p;  in vmballoon_add_page()
743 static void vmballoon_add_batched_page(struct vmballoon *b, int idx,  in vmballoon_add_batched_page()  argument
746 	vmballoon_batch_set_pa(b->batch_page, idx,  in vmballoon_add_batched_page()
755 static void vmballoon_inflate(struct vmballoon *b)  in vmballoon_inflate()  argument
762 	pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);  in vmballoon_inflate()
783 	is_2m_pages = b->supported_page_sizes == VMW_BALLOON_NUM_PAGE_SIZES;  in vmballoon_inflate()
785 	pr_debug("%s - goal: %d",  __func__, b->target - b->size);  in vmballoon_inflate()
787 	while (!b->reset_required &&  in vmballoon_inflate()
788 		b->size + num_pages * vmballoon_page_size(is_2m_pages)  in vmballoon_inflate()
789 		< b->target) {  in vmballoon_inflate()
793 			STATS_INC(b->stats.alloc[is_2m_pages]);  in vmballoon_inflate()
795 			STATS_INC(b->stats.sleep_alloc);  in vmballoon_inflate()
799 			STATS_INC(b->stats.alloc_fail[is_2m_pages]);  in vmballoon_inflate()
802 				b->ops->lock(b, num_pages, true, &b->target);  in vmballoon_inflate()
822 				STATS_INC(b->stats.sleep_alloc_fail);  in vmballoon_inflate()
839 		b->ops->add_page(b, num_pages++, page);  in vmballoon_inflate()
840 		if (num_pages == b->batch_max_pages) {  in vmballoon_inflate()
841 			error = b->ops->lock(b, num_pages, is_2m_pages,  in vmballoon_inflate()
842 					&b->target);  in vmballoon_inflate()
852 		b->ops->lock(b, num_pages, is_2m_pages, &b->target);  in vmballoon_inflate()
854 	vmballoon_release_refused_pages(b, true);  in vmballoon_inflate()
855 	vmballoon_release_refused_pages(b, false);  in vmballoon_inflate()
861 static void vmballoon_deflate(struct vmballoon *b)  in vmballoon_deflate()  argument
865 	pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);  in vmballoon_deflate()
868 	for (is_2m_pages = 0; is_2m_pages < b->supported_page_sizes;  in vmballoon_deflate()
873 				&b->page_sizes[is_2m_pages];  in vmballoon_deflate()
876 			if (b->reset_required ||  in vmballoon_deflate()
877 				(b->target > 0 &&  in vmballoon_deflate()
878 					b->size - num_pages  in vmballoon_deflate()
880 				< b->target + vmballoon_page_size(true)))  in vmballoon_deflate()
884 			b->ops->add_page(b, num_pages++, page);  in vmballoon_deflate()
886 			if (num_pages == b->batch_max_pages) {  in vmballoon_deflate()
889 				error = b->ops->unlock(b, num_pages,  in vmballoon_deflate()
890 						is_2m_pages, &b->target);  in vmballoon_deflate()
900 			b->ops->unlock(b, num_pages, is_2m_pages, &b->target);  in vmballoon_deflate()
916 static bool vmballoon_init_batching(struct vmballoon *b)  in vmballoon_init_batching()  argument
924 	b->batch_page = page_address(page);  in vmballoon_init_batching()
933 	struct vmballoon *b = client_data;  in vmballoon_doorbell()  local
935 	STATS_INC(b->stats.doorbell);  in vmballoon_doorbell()
937 	mod_delayed_work(system_freezable_wq, &b->dwork, 0);  in vmballoon_doorbell()
943 static void vmballoon_vmci_cleanup(struct vmballoon *b)  in vmballoon_vmci_cleanup()  argument
949 	STATS_INC(b->stats.doorbell_unset);  in vmballoon_vmci_cleanup()
951 	if (!vmci_handle_is_invalid(b->vmci_doorbell)) {  in vmballoon_vmci_cleanup()
952 		vmci_doorbell_destroy(b->vmci_doorbell);  in vmballoon_vmci_cleanup()
953 		b->vmci_doorbell = VMCI_INVALID_HANDLE;  in vmballoon_vmci_cleanup()
960 static int vmballoon_vmci_init(struct vmballoon *b)  in vmballoon_vmci_init()  argument
964 	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)  in vmballoon_vmci_init()
967 	error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,  in vmballoon_vmci_init()
969 				     vmballoon_doorbell, b);  in vmballoon_vmci_init()
974 	error = VMWARE_BALLOON_CMD(VMCI_DOORBELL_SET, b->vmci_doorbell.context,  in vmballoon_vmci_init()
975 				   b->vmci_doorbell.resource, dummy);  in vmballoon_vmci_init()
977 	STATS_INC(b->stats.doorbell_set);  in vmballoon_vmci_init()
984 	vmballoon_vmci_cleanup(b);  in vmballoon_vmci_init()
993 static void vmballoon_reset(struct vmballoon *b)  in vmballoon_reset()  argument
997 	vmballoon_vmci_cleanup(b);  in vmballoon_reset()
1000 	vmballoon_pop(b);  in vmballoon_reset()
1002 	if (!vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))  in vmballoon_reset()
1005 	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {  in vmballoon_reset()
1006 		b->ops = &vmballoon_batched_ops;  in vmballoon_reset()
1007 		b->batch_max_pages = VMW_BALLOON_BATCH_MAX_PAGES;  in vmballoon_reset()
1008 		if (!vmballoon_init_batching(b)) {  in vmballoon_reset()
1015 			vmballoon_send_start(b, 0);  in vmballoon_reset()
1018 	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {  in vmballoon_reset()
1019 		b->ops = &vmballoon_basic_ops;  in vmballoon_reset()
1020 		b->batch_max_pages = 1;  in vmballoon_reset()
1023 	b->reset_required = false;  in vmballoon_reset()
1025 	error = vmballoon_vmci_init(b);  in vmballoon_reset()
1029 	if (!vmballoon_send_guest_id(b))  in vmballoon_reset()
1040 	struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);  in vmballoon_work()  local
1043 	STATS_INC(b->stats.timer);  in vmballoon_work()
1045 	if (b->reset_required)  in vmballoon_work()
1046 		vmballoon_reset(b);  in vmballoon_work()
1048 	if (!b->reset_required && vmballoon_send_get_target(b, &target)) {  in vmballoon_work()
1050 		b->target = target;  in vmballoon_work()
1052 		if (b->size < target)  in vmballoon_work()
1053 			vmballoon_inflate(b);  in vmballoon_work()
1055 				b->size > target + vmballoon_page_size(true))  in vmballoon_work()
1056 			vmballoon_deflate(b);  in vmballoon_work()
1074 	struct vmballoon *b = f->private;  in vmballoon_debug_show()  local
1075 	struct vmballoon_stats *stats = &b->stats;  in vmballoon_debug_show()
1082 		   VMW_BALLOON_CAPABILITIES, b->capabilities,  in vmballoon_debug_show()
1083 		   b->reset_required ? 'y' : 'n');  in vmballoon_debug_show()
1089 		   b->target, b->size);  in vmballoon_debug_show()
1147 static int __init vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1151 	b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,  in vmballoon_debugfs_init()
1153 	if (IS_ERR(b->dbg_entry)) {  in vmballoon_debugfs_init()
1154 		error = PTR_ERR(b->dbg_entry);  in vmballoon_debugfs_init()
1162 static void __exit vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1164 	debugfs_remove(b->dbg_entry);  in vmballoon_debugfs_exit()
1169 static inline int vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1174 static inline void vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument