147 	struct virtio_balloon *vb = vq->vdev->priv;  in balloon_ack()  local
149 	wake_up(&vb->acked);  in balloon_ack()
152 static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)  in tell_host()  argument
157 	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);  in tell_host()
160 	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);  in tell_host()
164 	wait_event(vb->acked, virtqueue_get_buf(vq, &len));  in tell_host()
171 	struct virtio_balloon *vb =  in virtballoon_free_page_report()  local
173 	struct virtqueue *vq = vb->reporting_vq;  in virtballoon_free_page_report()
177 	err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT | __GFP_NOWARN);  in virtballoon_free_page_report()
190 	wait_event(vb->acked, virtqueue_get_buf(vq, &unused));  in virtballoon_free_page_report()
195 static void set_page_pfns(struct virtio_balloon *vb,  in set_page_pfns()  argument
207 		pfns[i] = cpu_to_virtio32(vb->vdev,  in set_page_pfns()
211 static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)  in fill_balloon()  argument
219 	num = min(num, ARRAY_SIZE(vb->pfns));  in fill_balloon()
226 			dev_info_ratelimited(&vb->vdev->dev,  in fill_balloon()
237 	mutex_lock(&vb->balloon_lock);  in fill_balloon()
239 	vb->num_pfns = 0;  in fill_balloon()
242 		balloon_page_enqueue(&vb->vb_dev_info, page);  in fill_balloon()
244 		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);  in fill_balloon()
245 		vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;  in fill_balloon()
246 		if (!virtio_has_feature(vb->vdev,  in fill_balloon()
249 		vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE;  in fill_balloon()
252 	num_allocated_pages = vb->num_pfns;  in fill_balloon()
254 	if (vb->num_pfns != 0)  in fill_balloon()
255 		tell_host(vb, vb->inflate_vq);  in fill_balloon()
256 	mutex_unlock(&vb->balloon_lock);  in fill_balloon()
261 static void release_pages_balloon(struct virtio_balloon *vb,  in release_pages_balloon()  argument
267 		if (!virtio_has_feature(vb->vdev,  in release_pages_balloon()
275 static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)  in leak_balloon()  argument
279 	struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;  in leak_balloon()
283 	num = min(num, ARRAY_SIZE(vb->pfns));  in leak_balloon()
285 	mutex_lock(&vb->balloon_lock);  in leak_balloon()
287 	num = min(num, (size_t)vb->num_pages);  in leak_balloon()
288 	for (vb->num_pfns = 0; vb->num_pfns < num;  in leak_balloon()
289 	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {  in leak_balloon()
293 		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);  in leak_balloon()
295 		vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;  in leak_balloon()
298 	num_freed_pages = vb->num_pfns;  in leak_balloon()
304 	if (vb->num_pfns != 0)  in leak_balloon()
305 		tell_host(vb, vb->deflate_vq);  in leak_balloon()
306 	release_pages_balloon(vb, &pages);  in leak_balloon()
307 	mutex_unlock(&vb->balloon_lock);  in leak_balloon()
311 static inline void update_stat(struct virtio_balloon *vb, int idx,  in update_stat()  argument
315 	vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);  in update_stat()
316 	vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);  in update_stat()
321 static unsigned int update_balloon_stats(struct virtio_balloon *vb)  in update_balloon_stats()  argument
336 	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,  in update_balloon_stats()
338 	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,  in update_balloon_stats()
340 	update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);  in update_balloon_stats()
341 	update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);  in update_balloon_stats()
343 	update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC,  in update_balloon_stats()
345 	update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL,  in update_balloon_stats()
349 	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,  in update_balloon_stats()
351 	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,  in update_balloon_stats()
353 	update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,  in update_balloon_stats()
355 	update_stat(vb, idx++, VIRTIO_BALLOON_S_CACHES,  in update_balloon_stats()
372 	struct virtio_balloon *vb = vq->vdev->priv;  in stats_request()  local
374 	spin_lock(&vb->stop_update_lock);  in stats_request()
375 	if (!vb->stop_update)  in stats_request()
376 		queue_work(system_freezable_wq, &vb->update_balloon_stats_work);  in stats_request()
377 	spin_unlock(&vb->stop_update_lock);  in stats_request()
380 static void stats_handle_request(struct virtio_balloon *vb)  in stats_handle_request()  argument
386 	num_stats = update_balloon_stats(vb);  in stats_handle_request()
388 	vq = vb->stats_vq;  in stats_handle_request()
391 	sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);  in stats_handle_request()
392 	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);  in stats_handle_request()
396 static inline s64 towards_target(struct virtio_balloon *vb)  in towards_target()  argument
402 	virtio_cread_le(vb->vdev, struct virtio_balloon_config, num_pages,  in towards_target()
406 	return target - vb->num_pages;  in towards_target()
410 static unsigned long return_free_pages_to_mm(struct virtio_balloon *vb,  in return_free_pages_to_mm()  argument
416 	spin_lock_irq(&vb->free_page_list_lock);  in return_free_pages_to_mm()
418 		page = balloon_page_pop(&vb->free_page_list);  in return_free_pages_to_mm()
424 	vb->num_free_page_blocks -= num_returned;  in return_free_pages_to_mm()
425 	spin_unlock_irq(&vb->free_page_list_lock);  in return_free_pages_to_mm()
430 static void virtio_balloon_queue_free_page_work(struct virtio_balloon *vb)  in virtio_balloon_queue_free_page_work()  argument
432 	if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))  in virtio_balloon_queue_free_page_work()
437 			     &vb->config_read_bitmap))  in virtio_balloon_queue_free_page_work()
440 	queue_work(vb->balloon_wq, &vb->report_free_page_work);  in virtio_balloon_queue_free_page_work()
445 	struct virtio_balloon *vb = vdev->priv;  in virtballoon_changed()  local
448 	spin_lock_irqsave(&vb->stop_update_lock, flags);  in virtballoon_changed()
449 	if (!vb->stop_update) {  in virtballoon_changed()
451 			   &vb->update_balloon_size_work);  in virtballoon_changed()
452 		virtio_balloon_queue_free_page_work(vb);  in virtballoon_changed()
454 	spin_unlock_irqrestore(&vb->stop_update_lock, flags);  in virtballoon_changed()
457 static void update_balloon_size(struct virtio_balloon *vb)  in update_balloon_size()  argument
459 	u32 actual = vb->num_pages;  in update_balloon_size()
462 	virtio_cwrite_le(vb->vdev, struct virtio_balloon_config, actual,  in update_balloon_size()
468 	struct virtio_balloon *vb;  in update_balloon_stats_func()  local
470 	vb = container_of(work, struct virtio_balloon,  in update_balloon_stats_func()
472 	stats_handle_request(vb);  in update_balloon_stats_func()
477 	struct virtio_balloon *vb;  in update_balloon_size_func()  local
480 	vb = container_of(work, struct virtio_balloon,  in update_balloon_size_func()
482 	diff = towards_target(vb);  in update_balloon_size_func()
488 		diff -= fill_balloon(vb, diff);  in update_balloon_size_func()
490 		diff += leak_balloon(vb, -diff);  in update_balloon_size_func()
491 	update_balloon_size(vb);  in update_balloon_size_func()
497 static int init_vqs(struct virtio_balloon *vb)  in init_vqs()  argument
519 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {  in init_vqs()
524 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {  in init_vqs()
529 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) {  in init_vqs()
534 	err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX,  in init_vqs()
539 	vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE];  in init_vqs()
540 	vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE];  in init_vqs()
541 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {  in init_vqs()
544 		vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS];  in init_vqs()
550 		num_stats = update_balloon_stats(vb);  in init_vqs()
552 		sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);  in init_vqs()
553 		err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb,  in init_vqs()
556 			dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n",  in init_vqs()
560 		virtqueue_kick(vb->stats_vq);  in init_vqs()
563 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))  in init_vqs()
564 		vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE];  in init_vqs()
566 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING))  in init_vqs()
567 		vb->reporting_vq = vqs[VIRTIO_BALLOON_VQ_REPORTING];  in init_vqs()
572 static u32 virtio_balloon_cmd_id_received(struct virtio_balloon *vb)  in virtio_balloon_cmd_id_received()  argument
575 			       &vb->config_read_bitmap)) {  in virtio_balloon_cmd_id_received()
577 		virtio_cread_le(vb->vdev, struct virtio_balloon_config,  in virtio_balloon_cmd_id_received()
579 				&vb->cmd_id_received_cache);  in virtio_balloon_cmd_id_received()
582 	return vb->cmd_id_received_cache;  in virtio_balloon_cmd_id_received()
585 static int send_cmd_id_start(struct virtio_balloon *vb)  in send_cmd_id_start()  argument
588 	struct virtqueue *vq = vb->free_page_vq;  in send_cmd_id_start()
595 	vb->cmd_id_active = cpu_to_virtio32(vb->vdev,  in send_cmd_id_start()
596 					virtio_balloon_cmd_id_received(vb));  in send_cmd_id_start()
597 	sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active));  in send_cmd_id_start()
598 	err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL);  in send_cmd_id_start()
604 static int send_cmd_id_stop(struct virtio_balloon *vb)  in send_cmd_id_stop()  argument
607 	struct virtqueue *vq = vb->free_page_vq;  in send_cmd_id_stop()
614 	sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop));  in send_cmd_id_stop()
615 	err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL);  in send_cmd_id_stop()
621 static int get_free_page_and_send(struct virtio_balloon *vb)  in get_free_page_and_send()  argument
623 	struct virtqueue *vq = vb->free_page_vq;  in get_free_page_and_send()
653 		spin_lock_irq(&vb->free_page_list_lock);  in get_free_page_and_send()
654 		balloon_page_push(&vb->free_page_list, page);  in get_free_page_and_send()
655 		vb->num_free_page_blocks++;  in get_free_page_and_send()
656 		spin_unlock_irq(&vb->free_page_list_lock);  in get_free_page_and_send()
668 static int send_free_pages(struct virtio_balloon *vb)  in send_free_pages()  argument
678 		cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active);  in send_free_pages()
680 			     virtio_balloon_cmd_id_received(vb)))  in send_free_pages()
687 		err = get_free_page_and_send(vb);  in send_free_pages()
697 static void virtio_balloon_report_free_page(struct virtio_balloon *vb)  in virtio_balloon_report_free_page()  argument
700 	struct device *dev = &vb->vdev->dev;  in virtio_balloon_report_free_page()
703 	err = send_cmd_id_start(vb);  in virtio_balloon_report_free_page()
707 	err = send_free_pages(vb);  in virtio_balloon_report_free_page()
712 	err = send_cmd_id_stop(vb);  in virtio_balloon_report_free_page()
719 	struct virtio_balloon *vb = container_of(work, struct virtio_balloon,  in report_free_page_func()  local
723 	cmd_id_received = virtio_balloon_cmd_id_received(vb);  in report_free_page_func()
726 		return_free_pages_to_mm(vb, ULONG_MAX);  in report_free_page_func()
729 		   virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) {  in report_free_page_func()
730 		virtio_balloon_report_free_page(vb);  in report_free_page_func()
747  *  1) insert newpage into vb->pages list and update the host about it;
748  *  2) update the host about the old page removed from vb->pages list;
756 	struct virtio_balloon *vb = container_of(vb_dev_info,  in virtballoon_migratepage()  local
768 	if (!mutex_trylock(&vb->balloon_lock))  in virtballoon_migratepage()
778 	if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM) &&  in virtballoon_migratepage()
790 	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;  in virtballoon_migratepage()
791 	set_page_pfns(vb, vb->pfns, newpage);  in virtballoon_migratepage()
792 	tell_host(vb, vb->inflate_vq);  in virtballoon_migratepage()
798 	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;  in virtballoon_migratepage()
799 	set_page_pfns(vb, vb->pfns, page);  in virtballoon_migratepage()
800 	tell_host(vb, vb->deflate_vq);  in virtballoon_migratepage()
802 	mutex_unlock(&vb->balloon_lock);  in virtballoon_migratepage()
822 static unsigned long shrink_free_pages(struct virtio_balloon *vb,  in shrink_free_pages()  argument
830 	blocks_freed = return_free_pages_to_mm(vb, blocks_to_free);  in shrink_free_pages()
838 	struct virtio_balloon *vb = container_of(shrinker,  in virtio_balloon_shrinker_scan()  local
841 	return shrink_free_pages(vb, sc->nr_to_scan);  in virtio_balloon_shrinker_scan()
847 	struct virtio_balloon *vb = container_of(shrinker,  in virtio_balloon_shrinker_count()  local
850 	return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES;  in virtio_balloon_shrinker_count()
856 	struct virtio_balloon *vb = container_of(nb,  in virtio_balloon_oom_notify()  local
860 	*freed += leak_balloon(vb, VIRTIO_BALLOON_OOM_NR_PAGES) /  in virtio_balloon_oom_notify()
862 	update_balloon_size(vb);  in virtio_balloon_oom_notify()
867 static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb)  in virtio_balloon_unregister_shrinker()  argument
869 	unregister_shrinker(&vb->shrinker);  in virtio_balloon_unregister_shrinker()
872 static int virtio_balloon_register_shrinker(struct virtio_balloon *vb)  in virtio_balloon_register_shrinker()  argument
874 	vb->shrinker.scan_objects = virtio_balloon_shrinker_scan;  in virtio_balloon_register_shrinker()
875 	vb->shrinker.count_objects = virtio_balloon_shrinker_count;  in virtio_balloon_register_shrinker()
876 	vb->shrinker.seeks = DEFAULT_SEEKS;  in virtio_balloon_register_shrinker()
878 	return register_shrinker(&vb->shrinker);  in virtio_balloon_register_shrinker()
883 	struct virtio_balloon *vb;  in virtballoon_probe()  local
892 	vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL);  in virtballoon_probe()
893 	if (!vb) {  in virtballoon_probe()
898 	INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);  in virtballoon_probe()
899 	INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);  in virtballoon_probe()
900 	spin_lock_init(&vb->stop_update_lock);  in virtballoon_probe()
901 	mutex_init(&vb->balloon_lock);  in virtballoon_probe()
902 	init_waitqueue_head(&vb->acked);  in virtballoon_probe()
903 	vb->vdev = vdev;  in virtballoon_probe()
905 	balloon_devinfo_init(&vb->vb_dev_info);  in virtballoon_probe()
907 	err = init_vqs(vb);  in virtballoon_probe()
918 	vb->vb_dev_info.migratepage = virtballoon_migratepage;  in virtballoon_probe()
919 	vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);  in virtballoon_probe()
920 	if (IS_ERR(vb->vb_dev_info.inode)) {  in virtballoon_probe()
921 		err = PTR_ERR(vb->vb_dev_info.inode);  in virtballoon_probe()
924 	vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;  in virtballoon_probe()
931 		if (virtqueue_get_vring_size(vb->free_page_vq) < 2) {  in virtballoon_probe()
935 		vb->balloon_wq = alloc_workqueue("balloon-wq",  in virtballoon_probe()
937 		if (!vb->balloon_wq) {  in virtballoon_probe()
941 		INIT_WORK(&vb->report_free_page_work, report_free_page_func);  in virtballoon_probe()
942 		vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP;  in virtballoon_probe()
943 		vb->cmd_id_active = cpu_to_virtio32(vb->vdev,  in virtballoon_probe()
945 		vb->cmd_id_stop = cpu_to_virtio32(vb->vdev,  in virtballoon_probe()
947 		spin_lock_init(&vb->free_page_list_lock);  in virtballoon_probe()
948 		INIT_LIST_HEAD(&vb->free_page_list);  in virtballoon_probe()
953 		err = virtio_balloon_register_shrinker(vb);  in virtballoon_probe()
958 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) {  in virtballoon_probe()
959 		vb->oom_nb.notifier_call = virtio_balloon_oom_notify;  in virtballoon_probe()
960 		vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY;  in virtballoon_probe()
961 		err = register_oom_notifier(&vb->oom_nb);  in virtballoon_probe()
982 		virtio_cwrite_le(vb->vdev, struct virtio_balloon_config,  in virtballoon_probe()
986 	vb->pr_dev_info.report = virtballoon_free_page_report;  in virtballoon_probe()
987 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) {  in virtballoon_probe()
990 		capacity = virtqueue_get_vring_size(vb->reporting_vq);  in virtballoon_probe()
996 		err = page_reporting_register(&vb->pr_dev_info);  in virtballoon_probe()
1003 	if (towards_target(vb))  in virtballoon_probe()
1008 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))  in virtballoon_probe()
1009 		unregister_oom_notifier(&vb->oom_nb);  in virtballoon_probe()
1011 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))  in virtballoon_probe()
1012 		virtio_balloon_unregister_shrinker(vb);  in virtballoon_probe()
1015 		destroy_workqueue(vb->balloon_wq);  in virtballoon_probe()
1018 	iput(vb->vb_dev_info.inode);  in virtballoon_probe()
1025 	kfree(vb);  in virtballoon_probe()
1030 static void remove_common(struct virtio_balloon *vb)  in remove_common()  argument
1033 	while (vb->num_pages)  in remove_common()
1034 		leak_balloon(vb, vb->num_pages);  in remove_common()
1035 	update_balloon_size(vb);  in remove_common()
1038 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))  in remove_common()
1039 		return_free_pages_to_mm(vb, ULONG_MAX);  in remove_common()
1042 	vb->vdev->config->reset(vb->vdev);  in remove_common()
1044 	vb->vdev->config->del_vqs(vb->vdev);  in remove_common()
1049 	struct virtio_balloon *vb = vdev->priv;  in virtballoon_remove()  local
1051 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING))  in virtballoon_remove()
1052 		page_reporting_unregister(&vb->pr_dev_info);  in virtballoon_remove()
1053 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))  in virtballoon_remove()
1054 		unregister_oom_notifier(&vb->oom_nb);  in virtballoon_remove()
1055 	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))  in virtballoon_remove()
1056 		virtio_balloon_unregister_shrinker(vb);  in virtballoon_remove()
1057 	spin_lock_irq(&vb->stop_update_lock);  in virtballoon_remove()
1058 	vb->stop_update = true;  in virtballoon_remove()
1059 	spin_unlock_irq(&vb->stop_update_lock);  in virtballoon_remove()
1060 	cancel_work_sync(&vb->update_balloon_size_work);  in virtballoon_remove()
1061 	cancel_work_sync(&vb->update_balloon_stats_work);  in virtballoon_remove()
1064 		cancel_work_sync(&vb->report_free_page_work);  in virtballoon_remove()
1065 		destroy_workqueue(vb->balloon_wq);  in virtballoon_remove()
1068 	remove_common(vb);  in virtballoon_remove()
1070 	if (vb->vb_dev_info.inode)  in virtballoon_remove()
1071 		iput(vb->vb_dev_info.inode);  in virtballoon_remove()
1075 	kfree(vb);  in virtballoon_remove()
1081 	struct virtio_balloon *vb = vdev->priv;  in virtballoon_freeze()  local
1087 	remove_common(vb);  in virtballoon_freeze()
1093 	struct virtio_balloon *vb = vdev->priv;  in virtballoon_restore()  local
1102 	if (towards_target(vb))  in virtballoon_restore()
1104 	update_balloon_size(vb);  in virtballoon_restore()