1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved.
46 …"Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performanc…
54 /* Magic number for the balloon mount-point */
80 #define VMW_BALLOON_2M_ORDER		(PMD_SHIFT - PAGE_SHIFT)
83  * 64-bit targets are only supported in 64-bit
118  * enum vmballoon_cmd_type - backdoor commands.
140  * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size.
141  * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page.
142  * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about
158  *				       size changes.
209 	[VMW_BALLOON_CMD_BATCHED_2M_LOCK]	= "2m-lock",
210 	[VMW_BALLOON_CMD_BATCHED_2M_UNLOCK]	= "2m-unlock",
250  * struct vmballoon_batch_entry - a batch entry for lock or unlock.
254  * @pfn: the physical frame number of the page to be locked or unlocked.
258 	u64 reserved : PAGE_SHIFT - 5;
264 	 * @max_page_size: maximum supported page size for ballooning.
271 	 * @size: balloon actual size in basic page size (frames).
273 	 * While we currently do not support size which is bigger than 32-bit,
274 	 * in preparation for future support, use 64-bits.
276 	atomic64_t size;  member
279 	 * @target: balloon target size in basic page size (frames).
307 	 * @batch_page: pointer to communication batch page.
309 	 * When batching is used, batch_page points to a page, which holds up to
319 	 * disabled, only a single page can be locked/unlock on each operation.
326 	 * @page: page to be locked/unlocked by the hypervisor
328 	 * @page is only used when batching is disabled and a single page is
333 	struct page *page;  member
355 	 * @huge_pages - list of the inflated 2MB pages.
376 	 * Lock ordering: @conf_sem -> @comm_lock .
381 	 * @shrinker: shrinker interface that is used to avoid over-inflation.
419 		atomic64_inc(&b->stats->ops[op][type]);  in vmballoon_stats_op_inc()
426 		atomic64_inc(&b->stats->general_stat[stat]);  in vmballoon_stats_gen_inc()
434 		atomic64_add(val, &b->stats->general_stat[stat]);  in vmballoon_stats_gen_add()
439 					    enum vmballoon_page_size_type size)  in vmballoon_stats_page_inc()  argument
442 		atomic64_inc(&b->stats->page_stat[stat][size]);  in vmballoon_stats_page_inc()
447 					    enum vmballoon_page_size_type size,  in vmballoon_stats_page_add()  argument
451 		atomic64_add(val, &b->stats->page_stat[stat][size]);  in vmballoon_stats_page_add()
483 		WRITE_ONCE(b->target, local_result);  in __vmballoon_cmd()
495 		b->reset_required = true;  in __vmballoon_cmd()
522 		b->capabilities = capabilities;  in vmballoon_send_start()
525 		b->capabilities = VMW_BALLOON_BASIC_CMDS;  in vmballoon_send_start()
528 		return -EIO;  in vmballoon_send_start()
536 	b->max_page_size = VMW_BALLOON_4K_PAGE;  in vmballoon_send_start()
537 	if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&  in vmballoon_send_start()
538 	    (b->capabilities & VMW_BALLOON_BATCHED_CMDS))  in vmballoon_send_start()
539 		b->max_page_size = VMW_BALLOON_2M_PAGE;  in vmballoon_send_start()
546  * vmballoon_send_guest_id - communicate guest type to the host.
564 	return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;  in vmballoon_send_guest_id()
568  * vmballoon_page_order() - return the order of the page
569  * @page_size: the size of the page.
580  * vmballoon_page_in_frames() - returns the number of frames in a page.
581  * @page_size: the size of the page.
592  * vmballoon_mark_page_offline() - mark a page as offline
593  * @page: pointer for the page.
594  * @page_size: the size of the page.
597 vmballoon_mark_page_offline(struct page *page,  in vmballoon_mark_page_offline()  argument
603 		__SetPageOffline(page + i);  in vmballoon_mark_page_offline()
607  * vmballoon_mark_page_online() - mark a page as online
608  * @page: pointer for the page.
609  * @page_size: the size of the page.
612 vmballoon_mark_page_online(struct page *page,  in vmballoon_mark_page_online()  argument
618 		__ClearPageOffline(page + i);  in vmballoon_mark_page_online()
622  * vmballoon_send_get_target() - Retrieve desired balloon size from the host.
626  * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required
627  * by the host-guest protocol and EIO if an error occurred in communicating with
637 	/* Ensure limit fits in 32-bits if 64-bit targets are not supported */  in vmballoon_send_get_target()
638 	if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&  in vmballoon_send_get_target()
640 		return -EINVAL;  in vmballoon_send_get_target()
644 	return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;  in vmballoon_send_get_target()
648  * vmballoon_alloc_page_list - allocates a list of pages.
663 	struct page *page;  in vmballoon_alloc_page_list()  local
669 		 * before. This happens when 2MB page rejected during inflation  in vmballoon_alloc_page_list()
672 		if (!list_empty(&ctl->prealloc_pages)) {  in vmballoon_alloc_page_list()
673 			page = list_first_entry(&ctl->prealloc_pages,  in vmballoon_alloc_page_list()
674 						struct page, lru);  in vmballoon_alloc_page_list()
675 			list_del(&page->lru);  in vmballoon_alloc_page_list()
677 			if (ctl->page_size == VMW_BALLOON_2M_PAGE)  in vmballoon_alloc_page_list()
678 				page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|  in vmballoon_alloc_page_list()
681 				page = balloon_page_alloc();  in vmballoon_alloc_page_list()
684 						 ctl->page_size);  in vmballoon_alloc_page_list()
687 		if (page) {  in vmballoon_alloc_page_list()
688 			/* Success. Add the page to the list and continue. */  in vmballoon_alloc_page_list()
689 			list_add(&page->lru, &ctl->pages);  in vmballoon_alloc_page_list()
695 					 ctl->page_size);  in vmballoon_alloc_page_list()
699 	ctl->n_pages = i;  in vmballoon_alloc_page_list()
701 	return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;  in vmballoon_alloc_page_list()
705  * vmballoon_handle_one_result - Handle lock/unlock result for a single page.
708  * @page: pointer for the page whose result should be handled.
709  * @page_size: size of the page.
712 static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,  in vmballoon_handle_one_result()  argument
716 	/* On success do nothing. The page is already on the balloon list. */  in vmballoon_handle_one_result()
721 		 page_to_pfn(page), status,  in vmballoon_handle_one_result()
728 	return -EIO;  in vmballoon_handle_one_result()
732  * vmballoon_status_page - returns the status of (un)lock operation
735  * @idx: index for the page for which the operation is performed.
736  * @p: pointer to where the page struct is returned.
739  * an individual page. Provides the page that the operation was performed on on
740  * the @page argument.
742  * Returns: The status of a lock or unlock operation for an individual page.
745 					   struct page **p)  in vmballoon_status_page()
749 		*p = pfn_to_page(b->batch_page[idx].pfn);  in vmballoon_status_page()
750 		return b->batch_page[idx].status;  in vmballoon_status_page()
753 	/* non-batching mode */  in vmballoon_status_page()
754 	*p = b->page;  in vmballoon_status_page()
759 	 * page status. So for non-batching mode, the indication is always of  in vmballoon_status_page()
766  * vmballoon_lock_op - notifies the host about inflated/deflated pages.
769  * @page_size: size of the page.
772  * Notify the host about page(s) that were ballooned (or removed from the
775  * check the return value and maybe submit a different page. The pages that are
776  * inflated/deflated are pointed by @b->page.
787 	lockdep_assert_held(&b->comm_lock);  in vmballoon_lock_op()
799 		pfn = PHYS_PFN(virt_to_phys(b->batch_page));  in vmballoon_lock_op()
803 		pfn = page_to_pfn(b->page);  in vmballoon_lock_op()
805 		/* In non-batching mode, PFNs must fit in 32-bit */  in vmballoon_lock_op()
814  * vmballoon_add_page - adds a page towards lock/unlock operation.
817  * @idx: index of the page to be ballooned in this batch.
818  * @p: pointer to the page that is about to be ballooned.
820  * Adds the page to be ballooned. Must be called while holding @comm_lock.
823 			       struct page *p)  in vmballoon_add_page()
825 	lockdep_assert_held(&b->comm_lock);  in vmballoon_add_page()
828 		b->batch_page[idx] = (struct vmballoon_batch_entry)  in vmballoon_add_page()
831 		b->page = p;  in vmballoon_add_page()
835  * vmballoon_lock - lock or unlock a batch of pages.
841  * according to @ctl). If the host rejects the page put it on the
842  * @ctl refuse list. These refused page are then released when moving to the
843  * next size of pages.
845  * Note that we neither free any @page here nor put them back on the ballooned
847  * reasons. First, we do not want to free the page under the lock. Second, it
856 	struct page *page;  in vmballoon_lock()  local
859 	num_pages = ctl->n_pages;  in vmballoon_lock()
864 	spin_lock(&b->comm_lock);  in vmballoon_lock()
867 	list_for_each_entry(page, &ctl->pages, lru)  in vmballoon_lock()
868 		vmballoon_add_page(b, i++, page);  in vmballoon_lock()
870 	batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,  in vmballoon_lock()
871 					 ctl->op);  in vmballoon_lock()
875 	 * @ctl->n_pages we are saving the original value in @num_pages and  in vmballoon_lock()
881 		status = vmballoon_status_page(b, i, &page);  in vmballoon_lock()
891 		if (!vmballoon_handle_one_result(b, page, ctl->page_size,  in vmballoon_lock()
899 		list_move(&page->lru, &ctl->refused_pages);  in vmballoon_lock()
900 		ctl->n_pages--;  in vmballoon_lock()
901 		ctl->n_refused_pages++;  in vmballoon_lock()
904 	spin_unlock(&b->comm_lock);  in vmballoon_lock()
906 	return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;  in vmballoon_lock()
910  * vmballoon_release_page_list() - Releases a page list
922 	struct page *page, *tmp;  in vmballoon_release_page_list()  local
924 	list_for_each_entry_safe(page, tmp, page_list, lru) {  in vmballoon_release_page_list()
925 		list_del(&page->lru);  in vmballoon_release_page_list()
926 		__free_pages(page, vmballoon_page_order(page_size));  in vmballoon_release_page_list()
942 				 ctl->page_size);  in vmballoon_release_refused_pages()
944 	vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages,  in vmballoon_release_refused_pages()
945 				    ctl->page_size);  in vmballoon_release_refused_pages()
949  * vmballoon_change - retrieve the required balloon change
953  * Return: the required change for the balloon size. A positive number
958 	int64_t size, target;  in vmballoon_change()  local
960 	size = atomic64_read(&b->size);  in vmballoon_change()
961 	target = READ_ONCE(b->target);  in vmballoon_change()
968 	if (b->reset_required)  in vmballoon_change()
972 	if (target < size && target != 0 &&  in vmballoon_change()
973 	    size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))  in vmballoon_change()
976 	/* If an out-of-memory recently occurred, inflation is disallowed. */  in vmballoon_change()
977 	if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))  in vmballoon_change()
980 	return target - size;  in vmballoon_change()
984  * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.
991  * Enqueues the provides list of pages in the ballooned page list, clears the
1000 	struct page *page;  in vmballoon_enqueue_page_list()  local
1003 		balloon_page_list_enqueue(&b->b_dev_info, pages);  in vmballoon_enqueue_page_list()
1009 		spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1011 		list_for_each_entry(page, pages, lru) {  in vmballoon_enqueue_page_list()
1012 			vmballoon_mark_page_offline(page, VMW_BALLOON_2M_PAGE);  in vmballoon_enqueue_page_list()
1015 		list_splice_init(pages, &b->huge_pages);  in vmballoon_enqueue_page_list()
1018 		spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1025  * vmballoon_dequeue_page_list() - Dequeues page lists for deflation.
1035  * size are available.
1043 	struct page *page, *tmp;  in vmballoon_dequeue_page_list()  local
1049 		*n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,  in vmballoon_dequeue_page_list()
1055 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1056 	list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {  in vmballoon_dequeue_page_list()
1057 		vmballoon_mark_page_online(page, VMW_BALLOON_2M_PAGE);  in vmballoon_dequeue_page_list()
1059 		list_move(&page->lru, pages);  in vmballoon_dequeue_page_list()
1066 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1071  * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k.
1082 	struct page *page, *tmp;  in vmballoon_split_refused_pages()  local
1085 	order = vmballoon_page_order(ctl->page_size);  in vmballoon_split_refused_pages()
1087 	list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) {  in vmballoon_split_refused_pages()
1088 		list_del(&page->lru);  in vmballoon_split_refused_pages()
1089 		split_page(page, order);  in vmballoon_split_refused_pages()
1091 			list_add(&page[i].lru, &ctl->prealloc_pages);  in vmballoon_split_refused_pages()
1093 	ctl->n_refused_pages = 0;  in vmballoon_split_refused_pages()
1097  * vmballoon_inflate() - Inflate the balloon towards its target size.
1108 		.page_size = b->max_page_size,  in vmballoon_inflate()
1121 		to_inflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_inflate()
1139 		/* Update the balloon size */  in vmballoon_inflate()
1140 		atomic64_add(ctl.n_pages * page_in_frames, &b->size);  in vmballoon_inflate()
1147 		 * the maximum allowed, move to the next page size.  in vmballoon_inflate()
1159 			ctl.page_size--;  in vmballoon_inflate()
1177  * vmballoon_deflate() - Decrease the size of the balloon.
1181  * calculated according to the target size.
1184  * Decrease the size of the balloon allowing guest to use more memory.
1186  * Return: The number of deflated frames (i.e., basic page size units)
1217 		 * performed according to the target and balloon size.  in vmballoon_deflate()
1219 		to_deflate_frames = n_frames ? n_frames - tried_frames :  in vmballoon_deflate()
1220 					       -vmballoon_change(b);  in vmballoon_deflate()
1227 		 * Calculate the number of frames based on current page size,  in vmballoon_deflate()
1230 		to_deflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_deflate()
1249 		 * page size, and would eventually try again later.  in vmballoon_deflate()
1255 		 * Check if we deflated enough. We will move to the next page  in vmballoon_deflate()
1256 		 * size if we did not manage to do so. This calculation takes  in vmballoon_deflate()
1264 		atomic64_sub(n_unlocked_frames, &b->size);  in vmballoon_deflate()
1279 		/* If we failed to unlock all the pages, move to next size. */  in vmballoon_deflate()
1281 			if (ctl.page_size == b->max_page_size)  in vmballoon_deflate()
1293  * vmballoon_deinit_batching - disables batching mode.
1297  * Disables batching, by deallocating the page for communication with the
1302 	free_page((unsigned long)b->batch_page);  in vmballoon_deinit_batching()
1303 	b->batch_page = NULL;  in vmballoon_deinit_batching()
1305 	b->batch_max_pages = 1;  in vmballoon_deinit_batching()
1309  * vmballoon_init_batching - enable batching mode.
1313  * Enables batching, by allocating a page for communication with the hypervisor
1316  * Return: zero on success or an appropriate error-code.
1320 	struct page *page;  in vmballoon_init_batching()  local
1322 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);  in vmballoon_init_batching()
1323 	if (!page)  in vmballoon_init_batching()
1324 		return -ENOMEM;  in vmballoon_init_batching()
1326 	b->batch_page = page_address(page);  in vmballoon_init_batching()
1327 	b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);  in vmballoon_init_batching()
1343 	mod_delayed_work(system_freezable_wq, &b->dwork, 0);  in vmballoon_doorbell()
1354 	if (!vmci_handle_is_invalid(b->vmci_doorbell)) {  in vmballoon_vmci_cleanup()
1355 		vmci_doorbell_destroy(b->vmci_doorbell);  in vmballoon_vmci_cleanup()
1356 		b->vmci_doorbell = VMCI_INVALID_HANDLE;  in vmballoon_vmci_cleanup()
1361  * vmballoon_vmci_init - Initialize vmci doorbell.
1365  * Return: zero on success or when wakeup command not supported. Error-code
1374 	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)  in vmballoon_vmci_init()
1377 	error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,  in vmballoon_vmci_init()
1385 				b->vmci_doorbell.context,  in vmballoon_vmci_init()
1386 				b->vmci_doorbell.resource, NULL);  in vmballoon_vmci_init()
1394 	return -EIO;  in vmballoon_vmci_init()
1398  * vmballoon_pop - Quickly release all pages allocate for the balloon.
1408 	unsigned long size;  in vmballoon_pop()  local
1410 	while ((size = atomic64_read(&b->size)))  in vmballoon_pop()
1411 		vmballoon_deflate(b, size, false);  in vmballoon_pop()
1423 	down_write(&b->conf_sem);  in vmballoon_reset()
1433 	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {  in vmballoon_reset()
1444 	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {  in vmballoon_reset()
1449 	b->reset_required = false;  in vmballoon_reset()
1459 	up_write(&b->conf_sem);  in vmballoon_reset()
1463  * vmballoon_work - periodic balloon worker for reset, inflation and deflation.
1467  * Resets the protocol if needed, gets the new size and adjusts balloon as
1476 	if (b->reset_required)  in vmballoon_work()
1479 	down_read(&b->conf_sem);  in vmballoon_work()
1492 		pr_debug("%s - size: %llu, target %lu\n", __func__,  in vmballoon_work()
1493 			 atomic64_read(&b->size), READ_ONCE(b->target));  in vmballoon_work()
1501 	up_read(&b->conf_sem);  in vmballoon_work()
1513  * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure.
1515  * @sc: page reclaim information.
1525 	pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));  in vmballoon_shrinker_scan()
1533 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_shrinker_scan()
1536 	deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);  in vmballoon_shrinker_scan()
1546 	WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);  in vmballoon_shrinker_scan()
1548 	up_read(&b->conf_sem);  in vmballoon_shrinker_scan()
1554  * vmballoon_shrinker_count() - return the number of ballooned pages.
1556  * @sc: page reclaim information.
1566 	return atomic64_read(&b->size);  in vmballoon_shrinker_count()
1571 	if (b->shrinker_registered)  in vmballoon_unregister_shrinker()
1572 		unregister_shrinker(&b->shrinker);  in vmballoon_unregister_shrinker()
1573 	b->shrinker_registered = false;  in vmballoon_unregister_shrinker()
1584 	b->shrinker.scan_objects = vmballoon_shrinker_scan;  in vmballoon_register_shrinker()
1585 	b->shrinker.count_objects = vmballoon_shrinker_count;  in vmballoon_register_shrinker()
1586 	b->shrinker.seeks = DEFAULT_SEEKS;  in vmballoon_register_shrinker()
1588 	r = register_shrinker(&b->shrinker, "vmw-balloon");  in vmballoon_register_shrinker()
1591 		b->shrinker_registered = true;  in vmballoon_register_shrinker()
1621 	down_write(&b->conf_sem);  in vmballoon_enable_stats()
1624 	if (b->stats)  in vmballoon_enable_stats()
1627 	b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);  in vmballoon_enable_stats()
1629 	if (!b->stats) {  in vmballoon_enable_stats()
1631 		r = -ENOMEM;  in vmballoon_enable_stats()
1636 	up_write(&b->conf_sem);  in vmballoon_enable_stats()
1641  * vmballoon_debug_show - shows statistics of balloon operations.
1646  * To avoid the overhead - mainly that of memory - of collecting the statistics,
1653 	struct vmballoon *b = f->private;  in vmballoon_debug_show()
1657 	if (!b->stats) {  in vmballoon_debug_show()
1665 	seq_printf(f, "%-22s: %#16x\n", "balloon capabilities",  in vmballoon_debug_show()
1667 	seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);  in vmballoon_debug_show()
1668 	seq_printf(f, "%-22s: %16s\n", "is resetting",  in vmballoon_debug_show()
1669 		   b->reset_required ? "y" : "n");  in vmballoon_debug_show()
1671 	/* format size info */  in vmballoon_debug_show()
1672 	seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));  in vmballoon_debug_show()
1673 	seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));  in vmballoon_debug_show()
1679 		seq_printf(f, "%-22s: %16llu (%llu failed)\n",  in vmballoon_debug_show()
1681 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),  in vmballoon_debug_show()
1682 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));  in vmballoon_debug_show()
1686 		seq_printf(f, "%-22s: %16llu\n",  in vmballoon_debug_show()
1688 			   atomic64_read(&b->stats->general_stat[i]));  in vmballoon_debug_show()
1692 			seq_printf(f, "%-18s(%s): %16llu\n",  in vmballoon_debug_show()
1695 				   atomic64_read(&b->stats->page_stat[i][j]));  in vmballoon_debug_show()
1713 	kfree(b->stats);  in vmballoon_debugfs_exit()
1714 	b->stats = NULL;  in vmballoon_debugfs_exit()
1732  * vmballoon_migratepage() - migrates a balloon page.
1734  * @newpage: the page to which @page should be migrated.
1735  * @page: a ballooned page that should be migrated.
1738  * This function is really open-coded, but that is according to the interface
1741  * Return: zero on success, -EAGAIN when migration cannot be performed
1742  *	   momentarily, and -EBUSY if migration failed and should be retried
1743  *	   with that specific page.
1746 				 struct page *newpage, struct page *page,  in vmballoon_migratepage()  argument
1759 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_migratepage()
1760 		return -EAGAIN;  in vmballoon_migratepage()
1762 	spin_lock(&b->comm_lock);  in vmballoon_migratepage()
1765 	 * hypervisor may tell us that it has enough memory and the new page is  in vmballoon_migratepage()
1766 	 * not needed. Since the old page is isolated, we cannot use the list  in vmballoon_migratepage()
1770 	vmballoon_add_page(b, 0, page);  in vmballoon_migratepage()
1775 		status = vmballoon_status_page(b, 0, &page);  in vmballoon_migratepage()
1782 		spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1783 		ret = -EBUSY;  in vmballoon_migratepage()
1788 	 * The page is isolated, so it is safe to delete it without holding  in vmballoon_migratepage()
1792 	balloon_page_delete(page);  in vmballoon_migratepage()
1794 	put_page(page);  in vmballoon_migratepage()
1804 	spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1808 		 * A failure happened. While we can deflate the page we just  in vmballoon_migratepage()
1810 		 * we will decrease the size of the balloon to reflect the  in vmballoon_migratepage()
1813 		atomic64_dec(&b->size);  in vmballoon_migratepage()
1814 		ret = -EBUSY;  in vmballoon_migratepage()
1817 		 * Success. Take a reference for the page, and we will add it to  in vmballoon_migratepage()
1825 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1833 		balloon_page_insert(&b->b_dev_info, newpage);  in vmballoon_migratepage()
1841 	b->b_dev_info.isolated_pages--;  in vmballoon_migratepage()
1842 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1845 	up_read(&b->conf_sem);  in vmballoon_migratepage()
1850  * vmballoon_compaction_init() - initialized compaction for the balloon.
1862 	b->b_dev_info.migratepage = vmballoon_migratepage;  in vmballoon_compaction_init()
1880 		return -ENODEV;  in vmballoon_init()
1900 	balloon.page = NULL;  in vmballoon_init()