Lines Matching refs:vb
134 struct virtio_balloon *vb = vq->vdev->priv; in balloon_ack() local
136 wake_up(&vb->acked); in balloon_ack()
139 static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq) in tell_host() argument
144 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns); in tell_host()
147 virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL); in tell_host()
151 wait_event(vb->acked, virtqueue_get_buf(vq, &len)); in tell_host()
155 static void set_page_pfns(struct virtio_balloon *vb, in set_page_pfns() argument
165 pfns[i] = cpu_to_virtio32(vb->vdev, in set_page_pfns()
169 static unsigned fill_balloon(struct virtio_balloon *vb, size_t num) in fill_balloon() argument
177 num = min(num, ARRAY_SIZE(vb->pfns)); in fill_balloon()
184 dev_info_ratelimited(&vb->vdev->dev, in fill_balloon()
195 mutex_lock(&vb->balloon_lock); in fill_balloon()
197 vb->num_pfns = 0; in fill_balloon()
200 balloon_page_enqueue(&vb->vb_dev_info, page); in fill_balloon()
202 set_page_pfns(vb, vb->pfns + vb->num_pfns, page); in fill_balloon()
203 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE; in fill_balloon()
204 if (!virtio_has_feature(vb->vdev, in fill_balloon()
207 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE; in fill_balloon()
210 num_allocated_pages = vb->num_pfns; in fill_balloon()
212 if (vb->num_pfns != 0) in fill_balloon()
213 tell_host(vb, vb->inflate_vq); in fill_balloon()
214 mutex_unlock(&vb->balloon_lock); in fill_balloon()
219 static void release_pages_balloon(struct virtio_balloon *vb, in release_pages_balloon() argument
225 if (!virtio_has_feature(vb->vdev, in release_pages_balloon()
233 static unsigned leak_balloon(struct virtio_balloon *vb, size_t num) in leak_balloon() argument
237 struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info; in leak_balloon()
241 num = min(num, ARRAY_SIZE(vb->pfns)); in leak_balloon()
243 mutex_lock(&vb->balloon_lock); in leak_balloon()
245 num = min(num, (size_t)vb->num_pages); in leak_balloon()
246 for (vb->num_pfns = 0; vb->num_pfns < num; in leak_balloon()
247 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { in leak_balloon()
251 set_page_pfns(vb, vb->pfns + vb->num_pfns, page); in leak_balloon()
253 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE; in leak_balloon()
256 num_freed_pages = vb->num_pfns; in leak_balloon()
262 if (vb->num_pfns != 0) in leak_balloon()
263 tell_host(vb, vb->deflate_vq); in leak_balloon()
264 release_pages_balloon(vb, &pages); in leak_balloon()
265 mutex_unlock(&vb->balloon_lock); in leak_balloon()
269 static inline void update_stat(struct virtio_balloon *vb, int idx, in update_stat() argument
273 vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag); in update_stat()
274 vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val); in update_stat()
279 static unsigned int update_balloon_stats(struct virtio_balloon *vb) in update_balloon_stats() argument
294 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN, in update_balloon_stats()
296 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT, in update_balloon_stats()
298 update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]); in update_balloon_stats()
299 update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]); in update_balloon_stats()
301 update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC, in update_balloon_stats()
303 update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL, in update_balloon_stats()
307 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE, in update_balloon_stats()
309 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT, in update_balloon_stats()
311 update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL, in update_balloon_stats()
313 update_stat(vb, idx++, VIRTIO_BALLOON_S_CACHES, in update_balloon_stats()
330 struct virtio_balloon *vb = vq->vdev->priv; in stats_request() local
332 spin_lock(&vb->stop_update_lock); in stats_request()
333 if (!vb->stop_update) in stats_request()
334 queue_work(system_freezable_wq, &vb->update_balloon_stats_work); in stats_request()
335 spin_unlock(&vb->stop_update_lock); in stats_request()
338 static void stats_handle_request(struct virtio_balloon *vb) in stats_handle_request() argument
344 num_stats = update_balloon_stats(vb); in stats_handle_request()
346 vq = vb->stats_vq; in stats_handle_request()
349 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); in stats_handle_request()
350 virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL); in stats_handle_request()
354 static inline s64 towards_target(struct virtio_balloon *vb) in towards_target() argument
359 virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages, in towards_target()
363 if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1)) in towards_target()
367 return target - vb->num_pages; in towards_target()
371 static unsigned long return_free_pages_to_mm(struct virtio_balloon *vb, in return_free_pages_to_mm() argument
377 spin_lock_irq(&vb->free_page_list_lock); in return_free_pages_to_mm()
379 page = balloon_page_pop(&vb->free_page_list); in return_free_pages_to_mm()
385 vb->num_free_page_blocks -= num_returned; in return_free_pages_to_mm()
386 spin_unlock_irq(&vb->free_page_list_lock); in return_free_pages_to_mm()
391 static void virtio_balloon_queue_free_page_work(struct virtio_balloon *vb) in virtio_balloon_queue_free_page_work() argument
393 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtio_balloon_queue_free_page_work()
398 &vb->config_read_bitmap)) in virtio_balloon_queue_free_page_work()
401 queue_work(vb->balloon_wq, &vb->report_free_page_work); in virtio_balloon_queue_free_page_work()
406 struct virtio_balloon *vb = vdev->priv; in virtballoon_changed() local
409 spin_lock_irqsave(&vb->stop_update_lock, flags); in virtballoon_changed()
410 if (!vb->stop_update) { in virtballoon_changed()
412 &vb->update_balloon_size_work); in virtballoon_changed()
413 virtio_balloon_queue_free_page_work(vb); in virtballoon_changed()
415 spin_unlock_irqrestore(&vb->stop_update_lock, flags); in virtballoon_changed()
418 static void update_balloon_size(struct virtio_balloon *vb) in update_balloon_size() argument
420 u32 actual = vb->num_pages; in update_balloon_size()
423 if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1)) in update_balloon_size()
426 virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual, in update_balloon_size()
432 struct virtio_balloon *vb; in update_balloon_stats_func() local
434 vb = container_of(work, struct virtio_balloon, in update_balloon_stats_func()
436 stats_handle_request(vb); in update_balloon_stats_func()
441 struct virtio_balloon *vb; in update_balloon_size_func() local
444 vb = container_of(work, struct virtio_balloon, in update_balloon_size_func()
446 diff = towards_target(vb); in update_balloon_size_func()
452 diff -= fill_balloon(vb, diff); in update_balloon_size_func()
454 diff += leak_balloon(vb, -diff); in update_balloon_size_func()
455 update_balloon_size(vb); in update_balloon_size_func()
461 static int init_vqs(struct virtio_balloon *vb) in init_vqs() argument
480 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { in init_vqs()
485 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { in init_vqs()
490 err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, in init_vqs()
495 vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE]; in init_vqs()
496 vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE]; in init_vqs()
497 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { in init_vqs()
500 vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS]; in init_vqs()
506 num_stats = update_balloon_stats(vb); in init_vqs()
508 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); in init_vqs()
509 err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, in init_vqs()
512 dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n", in init_vqs()
516 virtqueue_kick(vb->stats_vq); in init_vqs()
519 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in init_vqs()
520 vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE]; in init_vqs()
525 static u32 virtio_balloon_cmd_id_received(struct virtio_balloon *vb) in virtio_balloon_cmd_id_received() argument
528 &vb->config_read_bitmap)) in virtio_balloon_cmd_id_received()
529 virtio_cread(vb->vdev, struct virtio_balloon_config, in virtio_balloon_cmd_id_received()
531 &vb->cmd_id_received_cache); in virtio_balloon_cmd_id_received()
533 return vb->cmd_id_received_cache; in virtio_balloon_cmd_id_received()
536 static int send_cmd_id_start(struct virtio_balloon *vb) in send_cmd_id_start() argument
539 struct virtqueue *vq = vb->free_page_vq; in send_cmd_id_start()
546 vb->cmd_id_active = virtio32_to_cpu(vb->vdev, in send_cmd_id_start()
547 virtio_balloon_cmd_id_received(vb)); in send_cmd_id_start()
548 sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active)); in send_cmd_id_start()
549 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL); in send_cmd_id_start()
555 static int send_cmd_id_stop(struct virtio_balloon *vb) in send_cmd_id_stop() argument
558 struct virtqueue *vq = vb->free_page_vq; in send_cmd_id_stop()
565 sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop)); in send_cmd_id_stop()
566 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL); in send_cmd_id_stop()
572 static int get_free_page_and_send(struct virtio_balloon *vb) in get_free_page_and_send() argument
574 struct virtqueue *vq = vb->free_page_vq; in get_free_page_and_send()
604 spin_lock_irq(&vb->free_page_list_lock); in get_free_page_and_send()
605 balloon_page_push(&vb->free_page_list, page); in get_free_page_and_send()
606 vb->num_free_page_blocks++; in get_free_page_and_send()
607 spin_unlock_irq(&vb->free_page_list_lock); in get_free_page_and_send()
619 static int send_free_pages(struct virtio_balloon *vb) in send_free_pages() argument
629 cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active); in send_free_pages()
631 virtio_balloon_cmd_id_received(vb))) in send_free_pages()
638 err = get_free_page_and_send(vb); in send_free_pages()
648 static void virtio_balloon_report_free_page(struct virtio_balloon *vb) in virtio_balloon_report_free_page() argument
651 struct device *dev = &vb->vdev->dev; in virtio_balloon_report_free_page()
654 err = send_cmd_id_start(vb); in virtio_balloon_report_free_page()
658 err = send_free_pages(vb); in virtio_balloon_report_free_page()
663 err = send_cmd_id_stop(vb); in virtio_balloon_report_free_page()
670 struct virtio_balloon *vb = container_of(work, struct virtio_balloon, in report_free_page_func() local
674 cmd_id_received = virtio_balloon_cmd_id_received(vb); in report_free_page_func()
677 return_free_pages_to_mm(vb, ULONG_MAX); in report_free_page_func()
680 virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) { in report_free_page_func()
681 virtio_balloon_report_free_page(vb); in report_free_page_func()
707 struct virtio_balloon *vb = container_of(vb_dev_info, in virtballoon_migratepage() local
719 if (!mutex_trylock(&vb->balloon_lock)) in virtballoon_migratepage()
730 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; in virtballoon_migratepage()
731 set_page_pfns(vb, vb->pfns, newpage); in virtballoon_migratepage()
732 tell_host(vb, vb->inflate_vq); in virtballoon_migratepage()
738 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; in virtballoon_migratepage()
739 set_page_pfns(vb, vb->pfns, page); in virtballoon_migratepage()
740 tell_host(vb, vb->deflate_vq); in virtballoon_migratepage()
742 mutex_unlock(&vb->balloon_lock); in virtballoon_migratepage()
762 static unsigned long shrink_free_pages(struct virtio_balloon *vb, in shrink_free_pages() argument
770 blocks_freed = return_free_pages_to_mm(vb, blocks_to_free); in shrink_free_pages()
775 static unsigned long leak_balloon_pages(struct virtio_balloon *vb, in leak_balloon_pages() argument
778 return leak_balloon(vb, pages_to_free * VIRTIO_BALLOON_PAGES_PER_PAGE) / in leak_balloon_pages()
782 static unsigned long shrink_balloon_pages(struct virtio_balloon *vb, in shrink_balloon_pages() argument
792 while (vb->num_pages && pages_freed < pages_to_free) in shrink_balloon_pages()
793 pages_freed += leak_balloon_pages(vb, in shrink_balloon_pages()
796 update_balloon_size(vb); in shrink_balloon_pages()
805 struct virtio_balloon *vb = container_of(shrinker, in virtio_balloon_shrinker_scan() local
810 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtio_balloon_shrinker_scan()
811 pages_freed = shrink_free_pages(vb, pages_to_free); in virtio_balloon_shrinker_scan()
816 pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed); in virtio_balloon_shrinker_scan()
824 struct virtio_balloon *vb = container_of(shrinker, in virtio_balloon_shrinker_count() local
828 count = vb->num_pages / VIRTIO_BALLOON_PAGES_PER_PAGE; in virtio_balloon_shrinker_count()
829 count += vb->num_free_page_blocks << VIRTIO_BALLOON_FREE_PAGE_ORDER; in virtio_balloon_shrinker_count()
834 static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb) in virtio_balloon_unregister_shrinker() argument
836 unregister_shrinker(&vb->shrinker); in virtio_balloon_unregister_shrinker()
839 static int virtio_balloon_register_shrinker(struct virtio_balloon *vb) in virtio_balloon_register_shrinker() argument
841 vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; in virtio_balloon_register_shrinker()
842 vb->shrinker.count_objects = virtio_balloon_shrinker_count; in virtio_balloon_register_shrinker()
843 vb->shrinker.seeks = DEFAULT_SEEKS; in virtio_balloon_register_shrinker()
845 return register_shrinker(&vb->shrinker); in virtio_balloon_register_shrinker()
850 struct virtio_balloon *vb; in virtballoon_probe() local
860 vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL); in virtballoon_probe()
861 if (!vb) { in virtballoon_probe()
866 INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func); in virtballoon_probe()
867 INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func); in virtballoon_probe()
868 spin_lock_init(&vb->stop_update_lock); in virtballoon_probe()
869 mutex_init(&vb->balloon_lock); in virtballoon_probe()
870 init_waitqueue_head(&vb->acked); in virtballoon_probe()
871 vb->vdev = vdev; in virtballoon_probe()
873 balloon_devinfo_init(&vb->vb_dev_info); in virtballoon_probe()
875 err = init_vqs(vb); in virtballoon_probe()
886 vb->vb_dev_info.migratepage = virtballoon_migratepage; in virtballoon_probe()
887 vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb); in virtballoon_probe()
888 if (IS_ERR(vb->vb_dev_info.inode)) { in virtballoon_probe()
889 err = PTR_ERR(vb->vb_dev_info.inode); in virtballoon_probe()
893 vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops; in virtballoon_probe()
900 if (virtqueue_get_vring_size(vb->free_page_vq) < 2) { in virtballoon_probe()
904 vb->balloon_wq = alloc_workqueue("balloon-wq", in virtballoon_probe()
906 if (!vb->balloon_wq) { in virtballoon_probe()
910 INIT_WORK(&vb->report_free_page_work, report_free_page_func); in virtballoon_probe()
911 vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP; in virtballoon_probe()
912 vb->cmd_id_active = cpu_to_virtio32(vb->vdev, in virtballoon_probe()
914 vb->cmd_id_stop = cpu_to_virtio32(vb->vdev, in virtballoon_probe()
916 spin_lock_init(&vb->free_page_list_lock); in virtballoon_probe()
917 INIT_LIST_HEAD(&vb->free_page_list); in virtballoon_probe()
920 virtio_cwrite(vb->vdev, struct virtio_balloon_config, in virtballoon_probe()
928 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { in virtballoon_probe()
929 err = virtio_balloon_register_shrinker(vb); in virtballoon_probe()
935 if (towards_target(vb)) in virtballoon_probe()
941 destroy_workqueue(vb->balloon_wq); in virtballoon_probe()
945 kfree(vb); in virtballoon_probe()
950 static void remove_common(struct virtio_balloon *vb) in remove_common() argument
953 while (vb->num_pages) in remove_common()
954 leak_balloon(vb, vb->num_pages); in remove_common()
955 update_balloon_size(vb); in remove_common()
958 vb->vdev->config->reset(vb->vdev); in remove_common()
960 vb->vdev->config->del_vqs(vb->vdev); in remove_common()
965 struct virtio_balloon *vb = vdev->priv; in virtballoon_remove() local
967 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) in virtballoon_remove()
968 virtio_balloon_unregister_shrinker(vb); in virtballoon_remove()
969 spin_lock_irq(&vb->stop_update_lock); in virtballoon_remove()
970 vb->stop_update = true; in virtballoon_remove()
971 spin_unlock_irq(&vb->stop_update_lock); in virtballoon_remove()
972 cancel_work_sync(&vb->update_balloon_size_work); in virtballoon_remove()
973 cancel_work_sync(&vb->update_balloon_stats_work); in virtballoon_remove()
976 cancel_work_sync(&vb->report_free_page_work); in virtballoon_remove()
977 destroy_workqueue(vb->balloon_wq); in virtballoon_remove()
980 remove_common(vb); in virtballoon_remove()
982 if (vb->vb_dev_info.inode) in virtballoon_remove()
983 iput(vb->vb_dev_info.inode); in virtballoon_remove()
987 kfree(vb); in virtballoon_remove()
993 struct virtio_balloon *vb = vdev->priv; in virtballoon_freeze() local
999 remove_common(vb); in virtballoon_freeze()
1005 struct virtio_balloon *vb = vdev->priv; in virtballoon_restore() local
1014 if (towards_target(vb)) in virtballoon_restore()
1016 update_balloon_size(vb); in virtballoon_restore()