Lines Matching refs:si
108 __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset) in __try_to_reclaim_swap() argument
110 swp_entry_t entry = swp_entry(si->type, offset); in __try_to_reclaim_swap()
136 static int discard_swap(struct swap_info_struct *si) in discard_swap() argument
144 se = &si->first_swap_extent; in discard_swap()
148 err = blkdev_issue_discard(si->bdev, start_block, in discard_swap()
155 list_for_each_entry(se, &si->first_swap_extent.list, list) { in discard_swap()
159 err = blkdev_issue_discard(si->bdev, start_block, in discard_swap()
173 static void discard_swap_cluster(struct swap_info_struct *si, in discard_swap_cluster() argument
176 struct swap_extent *se = si->curr_swap_extent; in discard_swap_cluster()
192 si->curr_swap_extent = se; in discard_swap_cluster()
196 if (blkdev_issue_discard(si->bdev, start_block, in discard_swap_cluster()
290 static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, in lock_cluster() argument
295 ci = si->cluster_info; in lock_cluster()
314 struct swap_info_struct *si, unsigned long offset) in lock_cluster_or_swap_info() argument
319 ci = lock_cluster(si, offset); in lock_cluster_or_swap_info()
322 spin_lock(&si->lock); in lock_cluster_or_swap_info()
327 static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si, in unlock_cluster_or_swap_info() argument
333 spin_unlock(&si->lock); in unlock_cluster_or_swap_info()
392 static void swap_cluster_schedule_discard(struct swap_info_struct *si, in swap_cluster_schedule_discard() argument
401 memset(si->swap_map + idx * SWAPFILE_CLUSTER, in swap_cluster_schedule_discard()
404 cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx); in swap_cluster_schedule_discard()
406 schedule_work(&si->discard_work); in swap_cluster_schedule_discard()
409 static void __free_cluster(struct swap_info_struct *si, unsigned long idx) in __free_cluster() argument
411 struct swap_cluster_info *ci = si->cluster_info; in __free_cluster()
414 cluster_list_add_tail(&si->free_clusters, ci, idx); in __free_cluster()
421 static void swap_do_scheduled_discard(struct swap_info_struct *si) in swap_do_scheduled_discard() argument
426 info = si->cluster_info; in swap_do_scheduled_discard()
428 while (!cluster_list_empty(&si->discard_clusters)) { in swap_do_scheduled_discard()
429 idx = cluster_list_del_first(&si->discard_clusters, info); in swap_do_scheduled_discard()
430 spin_unlock(&si->lock); in swap_do_scheduled_discard()
432 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER, in swap_do_scheduled_discard()
435 spin_lock(&si->lock); in swap_do_scheduled_discard()
436 ci = lock_cluster(si, idx * SWAPFILE_CLUSTER); in swap_do_scheduled_discard()
437 __free_cluster(si, idx); in swap_do_scheduled_discard()
438 memset(si->swap_map + idx * SWAPFILE_CLUSTER, in swap_do_scheduled_discard()
446 struct swap_info_struct *si; in swap_discard_work() local
448 si = container_of(work, struct swap_info_struct, discard_work); in swap_discard_work()
450 spin_lock(&si->lock); in swap_discard_work()
451 swap_do_scheduled_discard(si); in swap_discard_work()
452 spin_unlock(&si->lock); in swap_discard_work()
455 static void alloc_cluster(struct swap_info_struct *si, unsigned long idx) in alloc_cluster() argument
457 struct swap_cluster_info *ci = si->cluster_info; in alloc_cluster()
459 VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx); in alloc_cluster()
460 cluster_list_del_first(&si->free_clusters, ci); in alloc_cluster()
464 static void free_cluster(struct swap_info_struct *si, unsigned long idx) in free_cluster() argument
466 struct swap_cluster_info *ci = si->cluster_info + idx; in free_cluster()
474 if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) == in free_cluster()
476 swap_cluster_schedule_discard(si, idx); in free_cluster()
480 __free_cluster(si, idx); in free_cluster()
528 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, in scan_swap_map_ssd_cluster_conflict() argument
535 conflict = !cluster_list_empty(&si->free_clusters) && in scan_swap_map_ssd_cluster_conflict()
536 offset != cluster_list_first(&si->free_clusters) && in scan_swap_map_ssd_cluster_conflict()
537 cluster_is_free(&si->cluster_info[offset]); in scan_swap_map_ssd_cluster_conflict()
542 percpu_cluster = this_cpu_ptr(si->percpu_cluster); in scan_swap_map_ssd_cluster_conflict()
551 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, in scan_swap_map_try_ssd_cluster() argument
560 cluster = this_cpu_ptr(si->percpu_cluster); in scan_swap_map_try_ssd_cluster()
562 if (!cluster_list_empty(&si->free_clusters)) { in scan_swap_map_try_ssd_cluster()
563 cluster->index = si->free_clusters.head; in scan_swap_map_try_ssd_cluster()
566 } else if (!cluster_list_empty(&si->discard_clusters)) { in scan_swap_map_try_ssd_cluster()
571 swap_do_scheduled_discard(si); in scan_swap_map_try_ssd_cluster()
572 *scan_base = *offset = si->cluster_next; in scan_swap_map_try_ssd_cluster()
585 max = min_t(unsigned long, si->max, in scan_swap_map_try_ssd_cluster()
591 ci = lock_cluster(si, tmp); in scan_swap_map_try_ssd_cluster()
593 if (!si->swap_map[tmp]) { in scan_swap_map_try_ssd_cluster()
625 static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset, in swap_range_alloc() argument
630 if (offset == si->lowest_bit) in swap_range_alloc()
631 si->lowest_bit += nr_entries; in swap_range_alloc()
632 if (end == si->highest_bit) in swap_range_alloc()
633 si->highest_bit -= nr_entries; in swap_range_alloc()
634 si->inuse_pages += nr_entries; in swap_range_alloc()
635 if (si->inuse_pages == si->pages) { in swap_range_alloc()
636 si->lowest_bit = si->max; in swap_range_alloc()
637 si->highest_bit = 0; in swap_range_alloc()
638 del_from_avail_list(si); in swap_range_alloc()
654 static void swap_range_free(struct swap_info_struct *si, unsigned long offset, in swap_range_free() argument
660 if (offset < si->lowest_bit) in swap_range_free()
661 si->lowest_bit = offset; in swap_range_free()
662 if (end > si->highest_bit) { in swap_range_free()
663 bool was_full = !si->highest_bit; in swap_range_free()
665 si->highest_bit = end; in swap_range_free()
666 if (was_full && (si->flags & SWP_WRITEOK)) in swap_range_free()
667 add_to_avail_list(si); in swap_range_free()
670 si->inuse_pages -= nr_entries; in swap_range_free()
671 if (si->flags & SWP_BLKDEV) in swap_range_free()
673 si->bdev->bd_disk->fops->swap_slot_free_notify; in swap_range_free()
677 frontswap_invalidate_page(si->type, offset); in swap_range_free()
679 swap_slot_free_notify(si->bdev, offset); in swap_range_free()
684 static int scan_swap_map_slots(struct swap_info_struct *si, in scan_swap_map_slots() argument
709 si->flags += SWP_SCANNING; in scan_swap_map_slots()
710 scan_base = offset = si->cluster_next; in scan_swap_map_slots()
713 if (si->cluster_info) { in scan_swap_map_slots()
714 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) in scan_swap_map_slots()
720 if (unlikely(!si->cluster_nr--)) { in scan_swap_map_slots()
721 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { in scan_swap_map_slots()
722 si->cluster_nr = SWAPFILE_CLUSTER - 1; in scan_swap_map_slots()
726 spin_unlock(&si->lock); in scan_swap_map_slots()
734 scan_base = offset = si->lowest_bit; in scan_swap_map_slots()
738 for (; last_in_cluster <= si->highest_bit; offset++) { in scan_swap_map_slots()
739 if (si->swap_map[offset]) in scan_swap_map_slots()
742 spin_lock(&si->lock); in scan_swap_map_slots()
744 si->cluster_next = offset; in scan_swap_map_slots()
745 si->cluster_nr = SWAPFILE_CLUSTER - 1; in scan_swap_map_slots()
755 spin_lock(&si->lock); in scan_swap_map_slots()
756 si->cluster_nr = SWAPFILE_CLUSTER - 1; in scan_swap_map_slots()
760 if (si->cluster_info) { in scan_swap_map_slots()
761 while (scan_swap_map_ssd_cluster_conflict(si, offset)) { in scan_swap_map_slots()
765 if (!scan_swap_map_try_ssd_cluster(si, &offset, in scan_swap_map_slots()
770 if (!(si->flags & SWP_WRITEOK)) in scan_swap_map_slots()
772 if (!si->highest_bit) in scan_swap_map_slots()
774 if (offset > si->highest_bit) in scan_swap_map_slots()
775 scan_base = offset = si->lowest_bit; in scan_swap_map_slots()
777 ci = lock_cluster(si, offset); in scan_swap_map_slots()
779 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { in scan_swap_map_slots()
782 spin_unlock(&si->lock); in scan_swap_map_slots()
783 swap_was_freed = __try_to_reclaim_swap(si, offset); in scan_swap_map_slots()
784 spin_lock(&si->lock); in scan_swap_map_slots()
791 if (si->swap_map[offset]) { in scan_swap_map_slots()
798 si->swap_map[offset] = usage; in scan_swap_map_slots()
799 inc_cluster_info_page(si, si->cluster_info, offset); in scan_swap_map_slots()
802 swap_range_alloc(si, offset, 1); in scan_swap_map_slots()
803 si->cluster_next = offset + 1; in scan_swap_map_slots()
804 slots[n_ret++] = swp_entry(si->type, offset); in scan_swap_map_slots()
807 if ((n_ret == nr) || (offset >= si->highest_bit)) in scan_swap_map_slots()
816 spin_unlock(&si->lock); in scan_swap_map_slots()
818 spin_lock(&si->lock); in scan_swap_map_slots()
823 if (si->cluster_info) { in scan_swap_map_slots()
824 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) in scan_swap_map_slots()
833 if (si->cluster_nr && !si->swap_map[offset]) { in scan_swap_map_slots()
834 --si->cluster_nr; in scan_swap_map_slots()
839 si->flags -= SWP_SCANNING; in scan_swap_map_slots()
843 spin_unlock(&si->lock); in scan_swap_map_slots()
844 while (++offset <= si->highest_bit) { in scan_swap_map_slots()
845 if (!si->swap_map[offset]) { in scan_swap_map_slots()
846 spin_lock(&si->lock); in scan_swap_map_slots()
849 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { in scan_swap_map_slots()
850 spin_lock(&si->lock); in scan_swap_map_slots()
858 offset = si->lowest_bit; in scan_swap_map_slots()
860 if (!si->swap_map[offset]) { in scan_swap_map_slots()
861 spin_lock(&si->lock); in scan_swap_map_slots()
864 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { in scan_swap_map_slots()
865 spin_lock(&si->lock); in scan_swap_map_slots()
874 spin_lock(&si->lock); in scan_swap_map_slots()
877 si->flags -= SWP_SCANNING; in scan_swap_map_slots()
881 static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) in swap_alloc_cluster() argument
897 if (cluster_list_empty(&si->free_clusters)) in swap_alloc_cluster()
900 idx = cluster_list_first(&si->free_clusters); in swap_alloc_cluster()
902 ci = lock_cluster(si, offset); in swap_alloc_cluster()
903 alloc_cluster(si, idx); in swap_alloc_cluster()
906 map = si->swap_map + offset; in swap_alloc_cluster()
910 swap_range_alloc(si, offset, SWAPFILE_CLUSTER); in swap_alloc_cluster()
911 *slot = swp_entry(si->type, offset); in swap_alloc_cluster()
916 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) in swap_free_cluster() argument
921 ci = lock_cluster(si, offset); in swap_free_cluster()
923 free_cluster(si, idx); in swap_free_cluster()
925 swap_range_free(si, offset, SWAPFILE_CLUSTER); in swap_free_cluster()
928 static unsigned long scan_swap_map(struct swap_info_struct *si, in scan_swap_map() argument
934 n_ret = scan_swap_map_slots(si, usage, 1, &entry); in scan_swap_map()
946 struct swap_info_struct *si, *next; in get_swap_pages() local
970 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) { in get_swap_pages()
972 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]); in get_swap_pages()
974 spin_lock(&si->lock); in get_swap_pages()
975 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) { in get_swap_pages()
977 if (plist_node_empty(&si->avail_lists[node])) { in get_swap_pages()
978 spin_unlock(&si->lock); in get_swap_pages()
981 WARN(!si->highest_bit, in get_swap_pages()
983 si->type); in get_swap_pages()
984 WARN(!(si->flags & SWP_WRITEOK), in get_swap_pages()
986 si->type); in get_swap_pages()
987 __del_from_avail_list(si); in get_swap_pages()
988 spin_unlock(&si->lock); in get_swap_pages()
992 if (!(si->flags & SWP_FILE)) in get_swap_pages()
993 n_ret = swap_alloc_cluster(si, swp_entries); in get_swap_pages()
995 n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, in get_swap_pages()
997 spin_unlock(&si->lock); in get_swap_pages()
1001 si->type); in get_swap_pages()
1033 struct swap_info_struct *si; in get_swap_page_of_type() local
1036 si = swap_info[type]; in get_swap_page_of_type()
1037 spin_lock(&si->lock); in get_swap_page_of_type()
1038 if (si && (si->flags & SWP_WRITEOK)) { in get_swap_page_of_type()
1041 offset = scan_swap_map(si, 1); in get_swap_page_of_type()
1043 spin_unlock(&si->lock); in get_swap_page_of_type()
1048 spin_unlock(&si->lock); in get_swap_page_of_type()
1216 struct swap_info_struct *si; in put_swap_page() local
1222 si = _swap_info_get(entry); in put_swap_page()
1223 if (!si) in put_swap_page()
1226 ci = lock_cluster_or_swap_info(si, offset); in put_swap_page()
1229 map = si->swap_map + offset; in put_swap_page()
1238 unlock_cluster_or_swap_info(si, ci); in put_swap_page()
1239 spin_lock(&si->lock); in put_swap_page()
1240 ci = lock_cluster(si, offset); in put_swap_page()
1244 swap_free_cluster(si, idx); in put_swap_page()
1245 spin_unlock(&si->lock); in put_swap_page()
1250 if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) { in put_swap_page()
1251 unlock_cluster_or_swap_info(si, ci); in put_swap_page()
1255 lock_cluster_or_swap_info(si, offset); in put_swap_page()
1258 unlock_cluster_or_swap_info(si, ci); in put_swap_page()
1264 struct swap_info_struct *si; in split_swap_cluster() local
1268 si = _swap_info_get(entry); in split_swap_cluster()
1269 if (!si) in split_swap_cluster()
1271 ci = lock_cluster(si, offset); in split_swap_cluster()
1337 int __swap_count(struct swap_info_struct *si, swp_entry_t entry) in __swap_count() argument
1341 return swap_count(si->swap_map[offset]); in __swap_count()
1344 static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) in swap_swapcount() argument
1350 ci = lock_cluster_or_swap_info(si, offset); in swap_swapcount()
1351 count = swap_count(si->swap_map[offset]); in swap_swapcount()
1352 unlock_cluster_or_swap_info(si, ci); in swap_swapcount()
1364 struct swap_info_struct *si; in __swp_swapcount() local
1366 si = __swap_info_get(entry); in __swp_swapcount()
1367 if (si) in __swp_swapcount()
1368 count = swap_swapcount(si, entry); in __swp_swapcount()
1418 static bool swap_page_trans_huge_swapped(struct swap_info_struct *si, in swap_page_trans_huge_swapped() argument
1422 unsigned char *map = si->swap_map; in swap_page_trans_huge_swapped()
1428 ci = lock_cluster_or_swap_info(si, offset); in swap_page_trans_huge_swapped()
1441 unlock_cluster_or_swap_info(si, ci); in swap_page_trans_huge_swapped()
1448 struct swap_info_struct *si; in page_swapped() local
1455 si = _swap_info_get(entry); in page_swapped()
1456 if (si) in page_swapped()
1457 return swap_page_trans_huge_swapped(si, entry); in page_swapped()
1466 struct swap_info_struct *si; in page_trans_huge_map_swapcount() local
1490 si = _swap_info_get(entry); in page_trans_huge_map_swapcount()
1491 if (si) { in page_trans_huge_map_swapcount()
1492 map = si->swap_map; in page_trans_huge_map_swapcount()
1497 ci = lock_cluster(si, offset); in page_trans_huge_map_swapcount()
1969 static unsigned int find_next_to_unuse(struct swap_info_struct *si, in find_next_to_unuse() argument
1972 unsigned int max = si->max; in find_next_to_unuse()
1996 count = READ_ONCE(si->swap_map[i]); in find_next_to_unuse()
1998 if (!frontswap || frontswap_test(si, i)) in find_next_to_unuse()
2017 struct swap_info_struct *si = swap_info[type]; in try_to_unuse() local
2052 while ((i = find_next_to_unuse(si, i, frontswap)) != 0) { in try_to_unuse()
2063 swap_map = &si->swap_map[i]; in try_to_unuse()
2568 struct swap_info_struct *si = p; in SYSCALL_DEFINE1() local
2571 plist_for_each_entry_continue(si, &swap_active_head, list) { in SYSCALL_DEFINE1()
2572 si->prio++; in SYSCALL_DEFINE1()
2573 si->list.prio--; in SYSCALL_DEFINE1()
2575 if (si->avail_lists[nid].prio != 1) in SYSCALL_DEFINE1()
2576 si->avail_lists[nid].prio--; in SYSCALL_DEFINE1()
2700 struct swap_info_struct *si; in swap_start() local
2711 si = swap_info[type]; in swap_start()
2712 if (!(si->flags & SWP_USED) || !si->swap_map) in swap_start()
2715 return si; in swap_start()
2723 struct swap_info_struct *si = v; in swap_next() local
2729 type = si->type + 1; in swap_next()
2733 si = swap_info[type]; in swap_next()
2734 if (!(si->flags & SWP_USED) || !si->swap_map) in swap_next()
2737 return si; in swap_next()
2750 struct swap_info_struct *si = v; in swap_show() local
2754 if (si == SEQ_START_TOKEN) { in swap_show()
2759 file = si->swap_file; in swap_show()
2765 si->pages << (PAGE_SHIFT - 10), in swap_show()
2766 si->inuse_pages << (PAGE_SHIFT - 10), in swap_show()
2767 si->prio); in swap_show()
3085 static bool swap_discardable(struct swap_info_struct *si) in swap_discardable() argument
3087 struct request_queue *q = bdev_get_queue(si->bdev); in swap_discardable()
3341 struct swap_info_struct *si = swap_info[type]; in si_swapinfo() local
3343 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) in si_swapinfo()
3344 nr_to_be_unused += si->inuse_pages; in si_swapinfo()
3516 struct swap_info_struct *si; in add_swap_count_continuation() local
3530 si = swap_info_get(entry); in add_swap_count_continuation()
3531 if (!si) { in add_swap_count_continuation()
3542 ci = lock_cluster(si, offset); in add_swap_count_continuation()
3544 count = si->swap_map[offset] & ~SWAP_HAS_CACHE; in add_swap_count_continuation()
3557 spin_unlock(&si->lock); in add_swap_count_continuation()
3566 head = vmalloc_to_page(si->swap_map + offset); in add_swap_count_continuation()
3569 spin_lock(&si->cont_lock); in add_swap_count_continuation()
3578 si->flags |= SWP_CONTINUED; in add_swap_count_continuation()
3606 spin_unlock(&si->cont_lock); in add_swap_count_continuation()
3609 spin_unlock(&si->lock); in add_swap_count_continuation()
3625 static bool swap_count_continued(struct swap_info_struct *si, in swap_count_continued() argument
3633 head = vmalloc_to_page(si->swap_map + offset); in swap_count_continued()
3639 spin_lock(&si->cont_lock); in swap_count_continued()
3705 spin_unlock(&si->cont_lock); in swap_count_continued()
3713 static void free_swap_count_continuations(struct swap_info_struct *si) in free_swap_count_continuations() argument
3717 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { in free_swap_count_continuations()
3719 head = vmalloc_to_page(si->swap_map + offset); in free_swap_count_continuations()
3735 struct swap_info_struct *si, *next; in mem_cgroup_throttle_swaprate() local
3750 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], in mem_cgroup_throttle_swaprate()
3752 if (si->bdev) { in mem_cgroup_throttle_swaprate()
3753 blkcg_schedule_throttle(bdev_get_queue(si->bdev), in mem_cgroup_throttle_swaprate()