422 static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,  in vmballoon_stats_op_inc()  argument
426 		atomic64_inc(&b->stats->ops[op][type]);  in vmballoon_stats_op_inc()
429 static inline void vmballoon_stats_gen_inc(struct vmballoon *b,  in vmballoon_stats_gen_inc()  argument
433 		atomic64_inc(&b->stats->general_stat[stat]);  in vmballoon_stats_gen_inc()
436 static inline void vmballoon_stats_gen_add(struct vmballoon *b,  in vmballoon_stats_gen_add()  argument
441 		atomic64_add(val, &b->stats->general_stat[stat]);  in vmballoon_stats_gen_add()
444 static inline void vmballoon_stats_page_inc(struct vmballoon *b,  in vmballoon_stats_page_inc()  argument
449 		atomic64_inc(&b->stats->page_stat[stat][size]);  in vmballoon_stats_page_inc()
452 static inline void vmballoon_stats_page_add(struct vmballoon *b,  in vmballoon_stats_page_add()  argument
458 		atomic64_add(val, &b->stats->page_stat[stat][size]);  in vmballoon_stats_page_add()
462 __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,  in __vmballoon_cmd()  argument
467 	vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);  in __vmballoon_cmd()
490 		WRITE_ONCE(b->target, local_result);  in __vmballoon_cmd()
494 		vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);  in __vmballoon_cmd()
502 		b->reset_required = true;  in __vmballoon_cmd()
508 vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,  in vmballoon_cmd()  argument
513 	return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);  in vmballoon_cmd()
520 static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)  in vmballoon_send_start()  argument
524 	status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,  in vmballoon_send_start()
529 		b->capabilities = capabilities;  in vmballoon_send_start()
532 		b->capabilities = VMW_BALLOON_BASIC_CMDS;  in vmballoon_send_start()
543 	b->max_page_size = VMW_BALLOON_4K_PAGE;  in vmballoon_send_start()
544 	if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&  in vmballoon_send_start()
545 	    (b->capabilities & VMW_BALLOON_BATCHED_CMDS))  in vmballoon_send_start()
546 		b->max_page_size = VMW_BALLOON_2M_PAGE;  in vmballoon_send_start()
564 static int vmballoon_send_guest_id(struct vmballoon *b)  in vmballoon_send_guest_id()  argument
568 	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,  in vmballoon_send_guest_id()
637 static int vmballoon_send_get_target(struct vmballoon *b)  in vmballoon_send_get_target()  argument
645 	if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&  in vmballoon_send_get_target()
649 	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);  in vmballoon_send_get_target()
666 static int vmballoon_alloc_page_list(struct vmballoon *b,  in vmballoon_alloc_page_list()  argument
690 			vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,  in vmballoon_alloc_page_list()
701 		vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,  in vmballoon_alloc_page_list()
719 static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,  in vmballoon_handle_one_result()  argument
732 	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,  in vmballoon_handle_one_result()
751 static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,  in vmballoon_status_page()  argument
756 		*p = pfn_to_page(b->batch_page[idx].pfn);  in vmballoon_status_page()
757 		return b->batch_page[idx].status;  in vmballoon_status_page()
761 	*p = b->page;  in vmballoon_status_page()
787 static unsigned long vmballoon_lock_op(struct vmballoon *b,  in vmballoon_lock_op()  argument
794 	lockdep_assert_held(&b->comm_lock);  in vmballoon_lock_op()
806 		pfn = PHYS_PFN(virt_to_phys(b->batch_page));  in vmballoon_lock_op()
810 		pfn = page_to_pfn(b->page);  in vmballoon_lock_op()
817 	return vmballoon_cmd(b, cmd, pfn, num_pages);  in vmballoon_lock_op()
829 static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,  in vmballoon_add_page()  argument
832 	lockdep_assert_held(&b->comm_lock);  in vmballoon_add_page()
835 		b->batch_page[idx] = (struct vmballoon_batch_entry)  in vmballoon_add_page()
838 		b->page = p;  in vmballoon_add_page()
860 static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)  in vmballoon_lock()  argument
871 	spin_lock(&b->comm_lock);  in vmballoon_lock()
875 		vmballoon_add_page(b, i++, page);  in vmballoon_lock()
877 	batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,  in vmballoon_lock()
888 		status = vmballoon_status_page(b, i, &page);  in vmballoon_lock()
898 		if (!vmballoon_handle_one_result(b, page, ctl->page_size,  in vmballoon_lock()
911 	spin_unlock(&b->comm_lock);  in vmballoon_lock()
945 static void vmballoon_release_refused_pages(struct vmballoon *b,  in vmballoon_release_refused_pages()  argument
948 	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,  in vmballoon_release_refused_pages()
963 static int64_t vmballoon_change(struct vmballoon *b)  in vmballoon_change()  argument
967 	size = atomic64_read(&b->size);  in vmballoon_change()
968 	target = READ_ONCE(b->target);  in vmballoon_change()
975 	if (b->reset_required)  in vmballoon_change()
984 	if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))  in vmballoon_change()
1001 static void vmballoon_enqueue_page_list(struct vmballoon *b,  in vmballoon_enqueue_page_list()  argument
1010 		balloon_page_list_enqueue(&b->b_dev_info, pages);  in vmballoon_enqueue_page_list()
1016 		spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1022 		list_splice_init(pages, &b->huge_pages);  in vmballoon_enqueue_page_list()
1025 		spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1044 static void vmballoon_dequeue_page_list(struct vmballoon *b,  in vmballoon_dequeue_page_list()  argument
1056 		*n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,  in vmballoon_dequeue_page_list()
1062 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1063 	list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {  in vmballoon_dequeue_page_list()
1073 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1108 static void vmballoon_inflate(struct vmballoon *b)  in vmballoon_inflate()  argument
1115 		.page_size = b->max_page_size,  in vmballoon_inflate()
1119 	while ((to_inflate_frames = vmballoon_change(b)) > 0) {  in vmballoon_inflate()
1128 		to_inflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_inflate()
1133 		alloc_error = vmballoon_alloc_page_list(b, &ctl,  in vmballoon_inflate()
1137 		lock_error = vmballoon_lock(b, &ctl);  in vmballoon_inflate()
1147 		atomic64_add(ctl.n_pages * page_in_frames, &b->size);  in vmballoon_inflate()
1149 		vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,  in vmballoon_inflate()
1178 		vmballoon_release_refused_pages(b, &ctl);  in vmballoon_inflate()
1195 static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,  in vmballoon_deflate()  argument
1227 					       -vmballoon_change(b);  in vmballoon_deflate()
1237 		to_deflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_deflate()
1242 		vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,  in vmballoon_deflate()
1259 			vmballoon_lock(b, &ctl);  in vmballoon_deflate()
1271 		atomic64_sub(n_unlocked_frames, &b->size);  in vmballoon_deflate()
1274 		vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,  in vmballoon_deflate()
1282 		vmballoon_enqueue_page_list(b, &ctl.refused_pages,  in vmballoon_deflate()
1288 			if (ctl.page_size == b->max_page_size)  in vmballoon_deflate()
1307 static void vmballoon_deinit_batching(struct vmballoon *b)  in vmballoon_deinit_batching()  argument
1309 	free_page((unsigned long)b->batch_page);  in vmballoon_deinit_batching()
1310 	b->batch_page = NULL;  in vmballoon_deinit_batching()
1312 	b->batch_max_pages = 1;  in vmballoon_deinit_batching()
1325 static int vmballoon_init_batching(struct vmballoon *b)  in vmballoon_init_batching()  argument
1333 	b->batch_page = page_address(page);  in vmballoon_init_batching()
1334 	b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);  in vmballoon_init_batching()
1346 	struct vmballoon *b = client_data;  in vmballoon_doorbell()  local
1348 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL);  in vmballoon_doorbell()
1350 	mod_delayed_work(system_freezable_wq, &b->dwork, 0);  in vmballoon_doorbell()
1356 static void vmballoon_vmci_cleanup(struct vmballoon *b)  in vmballoon_vmci_cleanup()  argument
1358 	vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,  in vmballoon_vmci_cleanup()
1361 	if (!vmci_handle_is_invalid(b->vmci_doorbell)) {  in vmballoon_vmci_cleanup()
1362 		vmci_doorbell_destroy(b->vmci_doorbell);  in vmballoon_vmci_cleanup()
1363 		b->vmci_doorbell = VMCI_INVALID_HANDLE;  in vmballoon_vmci_cleanup()
1377 static int vmballoon_vmci_init(struct vmballoon *b)  in vmballoon_vmci_init()  argument
1381 	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)  in vmballoon_vmci_init()
1384 	error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,  in vmballoon_vmci_init()
1386 				     vmballoon_doorbell, b);  in vmballoon_vmci_init()
1391 	error =	__vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,  in vmballoon_vmci_init()
1392 				b->vmci_doorbell.context,  in vmballoon_vmci_init()
1393 				b->vmci_doorbell.resource, NULL);  in vmballoon_vmci_init()
1400 	vmballoon_vmci_cleanup(b);  in vmballoon_vmci_init()
1413 static void vmballoon_pop(struct vmballoon *b)  in vmballoon_pop()  argument
1417 	while ((size = atomic64_read(&b->size)))  in vmballoon_pop()
1418 		vmballoon_deflate(b, size, false);  in vmballoon_pop()
1426 static void vmballoon_reset(struct vmballoon *b)  in vmballoon_reset()  argument
1430 	down_write(&b->conf_sem);  in vmballoon_reset()
1432 	vmballoon_vmci_cleanup(b);  in vmballoon_reset()
1435 	vmballoon_pop(b);  in vmballoon_reset()
1437 	if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))  in vmballoon_reset()
1440 	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {  in vmballoon_reset()
1441 		if (vmballoon_init_batching(b)) {  in vmballoon_reset()
1448 			vmballoon_send_start(b, 0);  in vmballoon_reset()
1451 	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {  in vmballoon_reset()
1452 		vmballoon_deinit_batching(b);  in vmballoon_reset()
1455 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET);  in vmballoon_reset()
1456 	b->reset_required = false;  in vmballoon_reset()
1458 	error = vmballoon_vmci_init(b);  in vmballoon_reset()
1462 	if (vmballoon_send_guest_id(b))  in vmballoon_reset()
1466 	up_write(&b->conf_sem);  in vmballoon_reset()
1480 	struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);  in vmballoon_work()  local
1483 	if (b->reset_required)  in vmballoon_work()
1484 		vmballoon_reset(b);  in vmballoon_work()
1486 	down_read(&b->conf_sem);  in vmballoon_work()
1493 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER);  in vmballoon_work()
1495 	if (!vmballoon_send_get_target(b))  in vmballoon_work()
1496 		change = vmballoon_change(b);  in vmballoon_work()
1500 			 atomic64_read(&b->size), READ_ONCE(b->target));  in vmballoon_work()
1503 			vmballoon_inflate(b);  in vmballoon_work()
1505 			vmballoon_deflate(b, 0, true);  in vmballoon_work()
1508 	up_read(&b->conf_sem);  in vmballoon_work()
1529 	struct vmballoon *b = &balloon;  in vmballoon_shrinker_scan()  local
1532 	pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));  in vmballoon_shrinker_scan()
1534 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK);  in vmballoon_shrinker_scan()
1540 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_shrinker_scan()
1543 	deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);  in vmballoon_shrinker_scan()
1545 	vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE,  in vmballoon_shrinker_scan()
1553 	WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);  in vmballoon_shrinker_scan()
1555 	up_read(&b->conf_sem);  in vmballoon_shrinker_scan()
1571 	struct vmballoon *b = &balloon;  in vmballoon_shrinker_count()  local
1573 	return atomic64_read(&b->size);  in vmballoon_shrinker_count()
1576 static void vmballoon_unregister_shrinker(struct vmballoon *b)  in vmballoon_unregister_shrinker()  argument
1578 	if (b->shrinker_registered)  in vmballoon_unregister_shrinker()
1579 		unregister_shrinker(&b->shrinker);  in vmballoon_unregister_shrinker()
1580 	b->shrinker_registered = false;  in vmballoon_unregister_shrinker()
1583 static int vmballoon_register_shrinker(struct vmballoon *b)  in vmballoon_register_shrinker()  argument
1591 	b->shrinker.scan_objects = vmballoon_shrinker_scan;  in vmballoon_register_shrinker()
1592 	b->shrinker.count_objects = vmballoon_shrinker_count;  in vmballoon_register_shrinker()
1593 	b->shrinker.seeks = DEFAULT_SEEKS;  in vmballoon_register_shrinker()
1595 	r = register_shrinker(&b->shrinker);  in vmballoon_register_shrinker()
1598 		b->shrinker_registered = true;  in vmballoon_register_shrinker()
1624 static int vmballoon_enable_stats(struct vmballoon *b)  in vmballoon_enable_stats()  argument
1628 	down_write(&b->conf_sem);  in vmballoon_enable_stats()
1631 	if (b->stats)  in vmballoon_enable_stats()
1634 	b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);  in vmballoon_enable_stats()
1636 	if (!b->stats) {  in vmballoon_enable_stats()
1643 	up_write(&b->conf_sem);  in vmballoon_enable_stats()
1660 	struct vmballoon *b = f->private;  in vmballoon_debug_show()  local
1664 	if (!b->stats) {  in vmballoon_debug_show()
1665 		int r = vmballoon_enable_stats(b);  in vmballoon_debug_show()
1674 	seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);  in vmballoon_debug_show()
1676 		   b->reset_required ? "y" : "n");  in vmballoon_debug_show()
1679 	seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));  in vmballoon_debug_show()
1680 	seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));  in vmballoon_debug_show()
1688 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),  in vmballoon_debug_show()
1689 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));  in vmballoon_debug_show()
1695 			   atomic64_read(&b->stats->general_stat[i]));  in vmballoon_debug_show()
1702 				   atomic64_read(&b->stats->page_stat[i][j]));  in vmballoon_debug_show()
1710 static void __init vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1712 	b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,  in vmballoon_debugfs_init()
1716 static void __exit vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1719 	debugfs_remove(b->dbg_entry);  in vmballoon_debugfs_exit()
1720 	kfree(b->stats);  in vmballoon_debugfs_exit()
1721 	b->stats = NULL;  in vmballoon_debugfs_exit()
1726 static inline void vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1730 static inline void vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1771 	struct vmballoon *b;  in vmballoon_migratepage()  local
1774 	b = container_of(b_dev_info, struct vmballoon, b_dev_info);  in vmballoon_migratepage()
1780 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_migratepage()
1783 	spin_lock(&b->comm_lock);  in vmballoon_migratepage()
1791 	vmballoon_add_page(b, 0, page);  in vmballoon_migratepage()
1792 	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,  in vmballoon_migratepage()
1796 		status = vmballoon_status_page(b, 0, &page);  in vmballoon_migratepage()
1803 		spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1818 	vmballoon_add_page(b, 0, newpage);  in vmballoon_migratepage()
1819 	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,  in vmballoon_migratepage()
1823 		status = vmballoon_status_page(b, 0, &newpage);  in vmballoon_migratepage()
1825 	spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1834 		atomic64_dec(&b->size);  in vmballoon_migratepage()
1846 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1854 		balloon_page_insert(&b->b_dev_info, newpage);  in vmballoon_migratepage()
1862 	b->b_dev_info.isolated_pages--;  in vmballoon_migratepage()
1863 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1866 	up_read(&b->conf_sem);  in vmballoon_migratepage()
1875 static void vmballoon_compaction_deinit(struct vmballoon *b)  in vmballoon_compaction_deinit()  argument
1877 	if (!IS_ERR(b->b_dev_info.inode))  in vmballoon_compaction_deinit()
1878 		iput(b->b_dev_info.inode);  in vmballoon_compaction_deinit()
1880 	b->b_dev_info.inode = NULL;  in vmballoon_compaction_deinit()
1896 static __init int vmballoon_compaction_init(struct vmballoon *b)  in vmballoon_compaction_init()  argument
1902 	b->b_dev_info.migratepage = vmballoon_migratepage;  in vmballoon_compaction_init()
1903 	b->b_dev_info.inode = alloc_anon_inode(vmballoon_mnt->mnt_sb);  in vmballoon_compaction_init()
1905 	if (IS_ERR(b->b_dev_info.inode))  in vmballoon_compaction_init()
1906 		return PTR_ERR(b->b_dev_info.inode);  in vmballoon_compaction_init()
1908 	b->b_dev_info.inode->i_mapping->a_ops = &balloon_aops;  in vmballoon_compaction_init()
1914 static void vmballoon_compaction_deinit(struct vmballoon *b)  in vmballoon_compaction_deinit()  argument
1918 static int vmballoon_compaction_init(struct vmballoon *b)  in vmballoon_compaction_init()  argument