421 static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,  in vmballoon_stats_op_inc()  argument
425 		atomic64_inc(&b->stats->ops[op][type]);  in vmballoon_stats_op_inc()
428 static inline void vmballoon_stats_gen_inc(struct vmballoon *b,  in vmballoon_stats_gen_inc()  argument
432 		atomic64_inc(&b->stats->general_stat[stat]);  in vmballoon_stats_gen_inc()
435 static inline void vmballoon_stats_gen_add(struct vmballoon *b,  in vmballoon_stats_gen_add()  argument
440 		atomic64_add(val, &b->stats->general_stat[stat]);  in vmballoon_stats_gen_add()
443 static inline void vmballoon_stats_page_inc(struct vmballoon *b,  in vmballoon_stats_page_inc()  argument
448 		atomic64_inc(&b->stats->page_stat[stat][size]);  in vmballoon_stats_page_inc()
451 static inline void vmballoon_stats_page_add(struct vmballoon *b,  in vmballoon_stats_page_add()  argument
457 		atomic64_add(val, &b->stats->page_stat[stat][size]);  in vmballoon_stats_page_add()
461 __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,  in __vmballoon_cmd()  argument
466 	vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);  in __vmballoon_cmd()
489 		WRITE_ONCE(b->target, local_result);  in __vmballoon_cmd()
493 		vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);  in __vmballoon_cmd()
501 		b->reset_required = true;  in __vmballoon_cmd()
507 vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,  in vmballoon_cmd()  argument
512 	return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);  in vmballoon_cmd()
519 static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)  in vmballoon_send_start()  argument
523 	status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,  in vmballoon_send_start()
528 		b->capabilities = capabilities;  in vmballoon_send_start()
531 		b->capabilities = VMW_BALLOON_BASIC_CMDS;  in vmballoon_send_start()
542 	b->max_page_size = VMW_BALLOON_4K_PAGE;  in vmballoon_send_start()
543 	if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&  in vmballoon_send_start()
544 	    (b->capabilities & VMW_BALLOON_BATCHED_CMDS))  in vmballoon_send_start()
545 		b->max_page_size = VMW_BALLOON_2M_PAGE;  in vmballoon_send_start()
563 static int vmballoon_send_guest_id(struct vmballoon *b)  in vmballoon_send_guest_id()  argument
567 	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,  in vmballoon_send_guest_id()
636 static int vmballoon_send_get_target(struct vmballoon *b)  in vmballoon_send_get_target()  argument
644 	if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&  in vmballoon_send_get_target()
648 	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);  in vmballoon_send_get_target()
665 static int vmballoon_alloc_page_list(struct vmballoon *b,  in vmballoon_alloc_page_list()  argument
689 			vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,  in vmballoon_alloc_page_list()
700 		vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,  in vmballoon_alloc_page_list()
718 static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,  in vmballoon_handle_one_result()  argument
731 	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,  in vmballoon_handle_one_result()
750 static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,  in vmballoon_status_page()  argument
755 		*p = pfn_to_page(b->batch_page[idx].pfn);  in vmballoon_status_page()
756 		return b->batch_page[idx].status;  in vmballoon_status_page()
760 	*p = b->page;  in vmballoon_status_page()
786 static unsigned long vmballoon_lock_op(struct vmballoon *b,  in vmballoon_lock_op()  argument
793 	lockdep_assert_held(&b->comm_lock);  in vmballoon_lock_op()
805 		pfn = PHYS_PFN(virt_to_phys(b->batch_page));  in vmballoon_lock_op()
809 		pfn = page_to_pfn(b->page);  in vmballoon_lock_op()
816 	return vmballoon_cmd(b, cmd, pfn, num_pages);  in vmballoon_lock_op()
828 static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,  in vmballoon_add_page()  argument
831 	lockdep_assert_held(&b->comm_lock);  in vmballoon_add_page()
834 		b->batch_page[idx] = (struct vmballoon_batch_entry)  in vmballoon_add_page()
837 		b->page = p;  in vmballoon_add_page()
859 static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)  in vmballoon_lock()  argument
870 	spin_lock(&b->comm_lock);  in vmballoon_lock()
874 		vmballoon_add_page(b, i++, page);  in vmballoon_lock()
876 	batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,  in vmballoon_lock()
887 		status = vmballoon_status_page(b, i, &page);  in vmballoon_lock()
897 		if (!vmballoon_handle_one_result(b, page, ctl->page_size,  in vmballoon_lock()
910 	spin_unlock(&b->comm_lock);  in vmballoon_lock()
944 static void vmballoon_release_refused_pages(struct vmballoon *b,  in vmballoon_release_refused_pages()  argument
947 	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,  in vmballoon_release_refused_pages()
962 static int64_t vmballoon_change(struct vmballoon *b)  in vmballoon_change()  argument
966 	size = atomic64_read(&b->size);  in vmballoon_change()
967 	target = READ_ONCE(b->target);  in vmballoon_change()
974 	if (b->reset_required)  in vmballoon_change()
983 	if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))  in vmballoon_change()
1000 static void vmballoon_enqueue_page_list(struct vmballoon *b,  in vmballoon_enqueue_page_list()  argument
1009 		balloon_page_list_enqueue(&b->b_dev_info, pages);  in vmballoon_enqueue_page_list()
1015 		spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1021 		list_splice_init(pages, &b->huge_pages);  in vmballoon_enqueue_page_list()
1024 		spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1043 static void vmballoon_dequeue_page_list(struct vmballoon *b,  in vmballoon_dequeue_page_list()  argument
1055 		*n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,  in vmballoon_dequeue_page_list()
1061 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1062 	list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {  in vmballoon_dequeue_page_list()
1072 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1107 static void vmballoon_inflate(struct vmballoon *b)  in vmballoon_inflate()  argument
1114 		.page_size = b->max_page_size,  in vmballoon_inflate()
1118 	while ((to_inflate_frames = vmballoon_change(b)) > 0) {  in vmballoon_inflate()
1127 		to_inflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_inflate()
1132 		alloc_error = vmballoon_alloc_page_list(b, &ctl,  in vmballoon_inflate()
1136 		lock_error = vmballoon_lock(b, &ctl);  in vmballoon_inflate()
1146 		atomic64_add(ctl.n_pages * page_in_frames, &b->size);  in vmballoon_inflate()
1148 		vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,  in vmballoon_inflate()
1177 		vmballoon_release_refused_pages(b, &ctl);  in vmballoon_inflate()
1194 static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,  in vmballoon_deflate()  argument
1226 					       -vmballoon_change(b);  in vmballoon_deflate()
1236 		to_deflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_deflate()
1241 		vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,  in vmballoon_deflate()
1258 			vmballoon_lock(b, &ctl);  in vmballoon_deflate()
1270 		atomic64_sub(n_unlocked_frames, &b->size);  in vmballoon_deflate()
1273 		vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,  in vmballoon_deflate()
1281 		vmballoon_enqueue_page_list(b, &ctl.refused_pages,  in vmballoon_deflate()
1287 			if (ctl.page_size == b->max_page_size)  in vmballoon_deflate()
1306 static void vmballoon_deinit_batching(struct vmballoon *b)  in vmballoon_deinit_batching()  argument
1308 	free_page((unsigned long)b->batch_page);  in vmballoon_deinit_batching()
1309 	b->batch_page = NULL;  in vmballoon_deinit_batching()
1311 	b->batch_max_pages = 1;  in vmballoon_deinit_batching()
1324 static int vmballoon_init_batching(struct vmballoon *b)  in vmballoon_init_batching()  argument
1332 	b->batch_page = page_address(page);  in vmballoon_init_batching()
1333 	b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);  in vmballoon_init_batching()
1345 	struct vmballoon *b = client_data;  in vmballoon_doorbell()  local
1347 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL);  in vmballoon_doorbell()
1349 	mod_delayed_work(system_freezable_wq, &b->dwork, 0);  in vmballoon_doorbell()
1355 static void vmballoon_vmci_cleanup(struct vmballoon *b)  in vmballoon_vmci_cleanup()  argument
1357 	vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,  in vmballoon_vmci_cleanup()
1360 	if (!vmci_handle_is_invalid(b->vmci_doorbell)) {  in vmballoon_vmci_cleanup()
1361 		vmci_doorbell_destroy(b->vmci_doorbell);  in vmballoon_vmci_cleanup()
1362 		b->vmci_doorbell = VMCI_INVALID_HANDLE;  in vmballoon_vmci_cleanup()
1376 static int vmballoon_vmci_init(struct vmballoon *b)  in vmballoon_vmci_init()  argument
1380 	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)  in vmballoon_vmci_init()
1383 	error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,  in vmballoon_vmci_init()
1385 				     vmballoon_doorbell, b);  in vmballoon_vmci_init()
1390 	error =	__vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,  in vmballoon_vmci_init()
1391 				b->vmci_doorbell.context,  in vmballoon_vmci_init()
1392 				b->vmci_doorbell.resource, NULL);  in vmballoon_vmci_init()
1399 	vmballoon_vmci_cleanup(b);  in vmballoon_vmci_init()
1412 static void vmballoon_pop(struct vmballoon *b)  in vmballoon_pop()  argument
1416 	while ((size = atomic64_read(&b->size)))  in vmballoon_pop()
1417 		vmballoon_deflate(b, size, false);  in vmballoon_pop()
1425 static void vmballoon_reset(struct vmballoon *b)  in vmballoon_reset()  argument
1429 	down_write(&b->conf_sem);  in vmballoon_reset()
1431 	vmballoon_vmci_cleanup(b);  in vmballoon_reset()
1434 	vmballoon_pop(b);  in vmballoon_reset()
1436 	if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))  in vmballoon_reset()
1439 	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {  in vmballoon_reset()
1440 		if (vmballoon_init_batching(b)) {  in vmballoon_reset()
1447 			vmballoon_send_start(b, 0);  in vmballoon_reset()
1450 	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {  in vmballoon_reset()
1451 		vmballoon_deinit_batching(b);  in vmballoon_reset()
1454 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET);  in vmballoon_reset()
1455 	b->reset_required = false;  in vmballoon_reset()
1457 	error = vmballoon_vmci_init(b);  in vmballoon_reset()
1461 	if (vmballoon_send_guest_id(b))  in vmballoon_reset()
1465 	up_write(&b->conf_sem);  in vmballoon_reset()
1479 	struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);  in vmballoon_work()  local
1482 	if (b->reset_required)  in vmballoon_work()
1483 		vmballoon_reset(b);  in vmballoon_work()
1485 	down_read(&b->conf_sem);  in vmballoon_work()
1492 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER);  in vmballoon_work()
1494 	if (!vmballoon_send_get_target(b))  in vmballoon_work()
1495 		change = vmballoon_change(b);  in vmballoon_work()
1499 			 atomic64_read(&b->size), READ_ONCE(b->target));  in vmballoon_work()
1502 			vmballoon_inflate(b);  in vmballoon_work()
1504 			vmballoon_deflate(b, 0, true);  in vmballoon_work()
1507 	up_read(&b->conf_sem);  in vmballoon_work()
1528 	struct vmballoon *b = &balloon;  in vmballoon_shrinker_scan()  local
1531 	pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));  in vmballoon_shrinker_scan()
1533 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK);  in vmballoon_shrinker_scan()
1539 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_shrinker_scan()
1542 	deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);  in vmballoon_shrinker_scan()
1544 	vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE,  in vmballoon_shrinker_scan()
1552 	WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);  in vmballoon_shrinker_scan()
1554 	up_read(&b->conf_sem);  in vmballoon_shrinker_scan()
1570 	struct vmballoon *b = &balloon;  in vmballoon_shrinker_count()  local
1572 	return atomic64_read(&b->size);  in vmballoon_shrinker_count()
1575 static void vmballoon_unregister_shrinker(struct vmballoon *b)  in vmballoon_unregister_shrinker()  argument
1577 	if (b->shrinker_registered)  in vmballoon_unregister_shrinker()
1578 		unregister_shrinker(&b->shrinker);  in vmballoon_unregister_shrinker()
1579 	b->shrinker_registered = false;  in vmballoon_unregister_shrinker()
1582 static int vmballoon_register_shrinker(struct vmballoon *b)  in vmballoon_register_shrinker()  argument
1590 	b->shrinker.scan_objects = vmballoon_shrinker_scan;  in vmballoon_register_shrinker()
1591 	b->shrinker.count_objects = vmballoon_shrinker_count;  in vmballoon_register_shrinker()
1592 	b->shrinker.seeks = DEFAULT_SEEKS;  in vmballoon_register_shrinker()
1594 	r = register_shrinker(&b->shrinker);  in vmballoon_register_shrinker()
1597 		b->shrinker_registered = true;  in vmballoon_register_shrinker()
1623 static int vmballoon_enable_stats(struct vmballoon *b)  in vmballoon_enable_stats()  argument
1627 	down_write(&b->conf_sem);  in vmballoon_enable_stats()
1630 	if (b->stats)  in vmballoon_enable_stats()
1633 	b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);  in vmballoon_enable_stats()
1635 	if (!b->stats) {  in vmballoon_enable_stats()
1642 	up_write(&b->conf_sem);  in vmballoon_enable_stats()
1659 	struct vmballoon *b = f->private;  in vmballoon_debug_show()  local
1663 	if (!b->stats) {  in vmballoon_debug_show()
1664 		int r = vmballoon_enable_stats(b);  in vmballoon_debug_show()
1673 	seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);  in vmballoon_debug_show()
1675 		   b->reset_required ? "y" : "n");  in vmballoon_debug_show()
1678 	seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));  in vmballoon_debug_show()
1679 	seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));  in vmballoon_debug_show()
1687 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),  in vmballoon_debug_show()
1688 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));  in vmballoon_debug_show()
1694 			   atomic64_read(&b->stats->general_stat[i]));  in vmballoon_debug_show()
1701 				   atomic64_read(&b->stats->page_stat[i][j]));  in vmballoon_debug_show()
1709 static void __init vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1711 	b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,  in vmballoon_debugfs_init()
1715 static void __exit vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1718 	debugfs_remove(b->dbg_entry);  in vmballoon_debugfs_exit()
1719 	kfree(b->stats);  in vmballoon_debugfs_exit()
1720 	b->stats = NULL;  in vmballoon_debugfs_exit()
1725 static inline void vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1729 static inline void vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1770 	struct vmballoon *b;  in vmballoon_migratepage()  local
1773 	b = container_of(b_dev_info, struct vmballoon, b_dev_info);  in vmballoon_migratepage()
1779 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_migratepage()
1782 	spin_lock(&b->comm_lock);  in vmballoon_migratepage()
1790 	vmballoon_add_page(b, 0, page);  in vmballoon_migratepage()
1791 	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,  in vmballoon_migratepage()
1795 		status = vmballoon_status_page(b, 0, &page);  in vmballoon_migratepage()
1802 		spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1817 	vmballoon_add_page(b, 0, newpage);  in vmballoon_migratepage()
1818 	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,  in vmballoon_migratepage()
1822 		status = vmballoon_status_page(b, 0, &newpage);  in vmballoon_migratepage()
1824 	spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1833 		atomic64_dec(&b->size);  in vmballoon_migratepage()
1845 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1853 		balloon_page_insert(&b->b_dev_info, newpage);  in vmballoon_migratepage()
1861 	b->b_dev_info.isolated_pages--;  in vmballoon_migratepage()
1862 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1865 	up_read(&b->conf_sem);  in vmballoon_migratepage()
1874 static void vmballoon_compaction_deinit(struct vmballoon *b)  in vmballoon_compaction_deinit()  argument
1876 	if (!IS_ERR(b->b_dev_info.inode))  in vmballoon_compaction_deinit()
1877 		iput(b->b_dev_info.inode);  in vmballoon_compaction_deinit()
1879 	b->b_dev_info.inode = NULL;  in vmballoon_compaction_deinit()
1895 static __init int vmballoon_compaction_init(struct vmballoon *b)  in vmballoon_compaction_init()  argument
1901 	b->b_dev_info.migratepage = vmballoon_migratepage;  in vmballoon_compaction_init()
1902 	b->b_dev_info.inode = alloc_anon_inode(vmballoon_mnt->mnt_sb);  in vmballoon_compaction_init()
1904 	if (IS_ERR(b->b_dev_info.inode))  in vmballoon_compaction_init()
1905 		return PTR_ERR(b->b_dev_info.inode);  in vmballoon_compaction_init()
1907 	b->b_dev_info.inode->i_mapping->a_ops = &balloon_aops;  in vmballoon_compaction_init()
1913 static void vmballoon_compaction_deinit(struct vmballoon *b)  in vmballoon_compaction_deinit()  argument
1917 static int vmballoon_compaction_init(struct vmballoon *b)  in vmballoon_compaction_init()  argument