Lines Matching refs:si
133 static int __try_to_reclaim_swap(struct swap_info_struct *si, in __try_to_reclaim_swap() argument
136 swp_entry_t entry = swp_entry(si->type, offset); in __try_to_reclaim_swap()
177 static int discard_swap(struct swap_info_struct *si) in discard_swap() argument
185 se = first_se(si); in discard_swap()
189 err = blkdev_issue_discard(si->bdev, start_block, in discard_swap()
200 err = blkdev_issue_discard(si->bdev, start_block, in discard_swap()
247 static void discard_swap_cluster(struct swap_info_struct *si, in discard_swap_cluster() argument
250 struct swap_extent *se = offset_to_swap_extent(si, start_page); in discard_swap_cluster()
264 if (blkdev_issue_discard(si->bdev, start_block, in discard_swap_cluster()
357 static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, in lock_cluster() argument
362 ci = si->cluster_info; in lock_cluster()
381 struct swap_info_struct *si, unsigned long offset) in lock_cluster_or_swap_info() argument
386 ci = lock_cluster(si, offset); in lock_cluster_or_swap_info()
389 spin_lock(&si->lock); in lock_cluster_or_swap_info()
394 static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si, in unlock_cluster_or_swap_info() argument
400 spin_unlock(&si->lock); in unlock_cluster_or_swap_info()
459 static void swap_cluster_schedule_discard(struct swap_info_struct *si, in swap_cluster_schedule_discard() argument
468 memset(si->swap_map + idx * SWAPFILE_CLUSTER, in swap_cluster_schedule_discard()
471 cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx); in swap_cluster_schedule_discard()
473 schedule_work(&si->discard_work); in swap_cluster_schedule_discard()
476 static void __free_cluster(struct swap_info_struct *si, unsigned long idx) in __free_cluster() argument
478 struct swap_cluster_info *ci = si->cluster_info; in __free_cluster()
481 cluster_list_add_tail(&si->free_clusters, ci, idx); in __free_cluster()
488 static void swap_do_scheduled_discard(struct swap_info_struct *si) in swap_do_scheduled_discard() argument
493 info = si->cluster_info; in swap_do_scheduled_discard()
495 while (!cluster_list_empty(&si->discard_clusters)) { in swap_do_scheduled_discard()
496 idx = cluster_list_del_first(&si->discard_clusters, info); in swap_do_scheduled_discard()
497 spin_unlock(&si->lock); in swap_do_scheduled_discard()
499 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER, in swap_do_scheduled_discard()
502 spin_lock(&si->lock); in swap_do_scheduled_discard()
503 ci = lock_cluster(si, idx * SWAPFILE_CLUSTER); in swap_do_scheduled_discard()
504 __free_cluster(si, idx); in swap_do_scheduled_discard()
505 memset(si->swap_map + idx * SWAPFILE_CLUSTER, in swap_do_scheduled_discard()
513 struct swap_info_struct *si; in swap_discard_work() local
515 si = container_of(work, struct swap_info_struct, discard_work); in swap_discard_work()
517 spin_lock(&si->lock); in swap_discard_work()
518 swap_do_scheduled_discard(si); in swap_discard_work()
519 spin_unlock(&si->lock); in swap_discard_work()
524 struct swap_info_struct *si; in swap_users_ref_free() local
526 si = container_of(ref, struct swap_info_struct, users); in swap_users_ref_free()
527 complete(&si->comp); in swap_users_ref_free()
530 static void alloc_cluster(struct swap_info_struct *si, unsigned long idx) in alloc_cluster() argument
532 struct swap_cluster_info *ci = si->cluster_info; in alloc_cluster()
534 VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx); in alloc_cluster()
535 cluster_list_del_first(&si->free_clusters, ci); in alloc_cluster()
539 static void free_cluster(struct swap_info_struct *si, unsigned long idx) in free_cluster() argument
541 struct swap_cluster_info *ci = si->cluster_info + idx; in free_cluster()
549 if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) == in free_cluster()
551 swap_cluster_schedule_discard(si, idx); in free_cluster()
555 __free_cluster(si, idx); in free_cluster()
603 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, in scan_swap_map_ssd_cluster_conflict() argument
610 conflict = !cluster_list_empty(&si->free_clusters) && in scan_swap_map_ssd_cluster_conflict()
611 offset != cluster_list_first(&si->free_clusters) && in scan_swap_map_ssd_cluster_conflict()
612 cluster_is_free(&si->cluster_info[offset]); in scan_swap_map_ssd_cluster_conflict()
617 percpu_cluster = this_cpu_ptr(si->percpu_cluster); in scan_swap_map_ssd_cluster_conflict()
626 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, in scan_swap_map_try_ssd_cluster() argument
634 cluster = this_cpu_ptr(si->percpu_cluster); in scan_swap_map_try_ssd_cluster()
636 if (!cluster_list_empty(&si->free_clusters)) { in scan_swap_map_try_ssd_cluster()
637 cluster->index = si->free_clusters.head; in scan_swap_map_try_ssd_cluster()
640 } else if (!cluster_list_empty(&si->discard_clusters)) { in scan_swap_map_try_ssd_cluster()
646 swap_do_scheduled_discard(si); in scan_swap_map_try_ssd_cluster()
647 *scan_base = this_cpu_read(*si->cluster_next_cpu); in scan_swap_map_try_ssd_cluster()
659 max = min_t(unsigned long, si->max, in scan_swap_map_try_ssd_cluster()
662 ci = lock_cluster(si, tmp); in scan_swap_map_try_ssd_cluster()
664 if (!si->swap_map[tmp]) in scan_swap_map_try_ssd_cluster()
696 static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset, in swap_range_alloc() argument
701 if (offset == si->lowest_bit) in swap_range_alloc()
702 si->lowest_bit += nr_entries; in swap_range_alloc()
703 if (end == si->highest_bit) in swap_range_alloc()
704 WRITE_ONCE(si->highest_bit, si->highest_bit - nr_entries); in swap_range_alloc()
705 WRITE_ONCE(si->inuse_pages, si->inuse_pages + nr_entries); in swap_range_alloc()
706 if (si->inuse_pages == si->pages) { in swap_range_alloc()
707 si->lowest_bit = si->max; in swap_range_alloc()
708 si->highest_bit = 0; in swap_range_alloc()
709 del_from_avail_list(si); in swap_range_alloc()
723 static void swap_range_free(struct swap_info_struct *si, unsigned long offset, in swap_range_free() argument
730 if (offset < si->lowest_bit) in swap_range_free()
731 si->lowest_bit = offset; in swap_range_free()
732 if (end > si->highest_bit) { in swap_range_free()
733 bool was_full = !si->highest_bit; in swap_range_free()
735 WRITE_ONCE(si->highest_bit, end); in swap_range_free()
736 if (was_full && (si->flags & SWP_WRITEOK)) in swap_range_free()
737 add_to_avail_list(si); in swap_range_free()
740 WRITE_ONCE(si->inuse_pages, si->inuse_pages - nr_entries); in swap_range_free()
741 if (si->flags & SWP_BLKDEV) in swap_range_free()
743 si->bdev->bd_disk->fops->swap_slot_free_notify; in swap_range_free()
747 arch_swap_invalidate_page(si->type, offset); in swap_range_free()
748 zswap_invalidate(si->type, offset); in swap_range_free()
750 swap_slot_free_notify(si->bdev, offset); in swap_range_free()
753 clear_shadow_from_swap_cache(si->type, begin, end); in swap_range_free()
756 static void set_cluster_next(struct swap_info_struct *si, unsigned long next) in set_cluster_next() argument
760 if (!(si->flags & SWP_SOLIDSTATE)) { in set_cluster_next()
761 si->cluster_next = next; in set_cluster_next()
765 prev = this_cpu_read(*si->cluster_next_cpu); in set_cluster_next()
774 if (si->highest_bit <= si->lowest_bit) in set_cluster_next()
776 next = get_random_u32_inclusive(si->lowest_bit, si->highest_bit); in set_cluster_next()
778 next = max_t(unsigned int, next, si->lowest_bit); in set_cluster_next()
780 this_cpu_write(*si->cluster_next_cpu, next); in set_cluster_next()
783 static bool swap_offset_available_and_locked(struct swap_info_struct *si, in swap_offset_available_and_locked() argument
786 if (data_race(!si->swap_map[offset])) { in swap_offset_available_and_locked()
787 spin_lock(&si->lock); in swap_offset_available_and_locked()
791 if (vm_swap_full() && READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { in swap_offset_available_and_locked()
792 spin_lock(&si->lock); in swap_offset_available_and_locked()
799 static int scan_swap_map_slots(struct swap_info_struct *si, in scan_swap_map_slots() argument
822 si->flags += SWP_SCANNING; in scan_swap_map_slots()
828 if (si->flags & SWP_SOLIDSTATE) in scan_swap_map_slots()
829 scan_base = this_cpu_read(*si->cluster_next_cpu); in scan_swap_map_slots()
831 scan_base = si->cluster_next; in scan_swap_map_slots()
835 if (si->cluster_info) { in scan_swap_map_slots()
836 if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) in scan_swap_map_slots()
838 } else if (unlikely(!si->cluster_nr--)) { in scan_swap_map_slots()
839 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { in scan_swap_map_slots()
840 si->cluster_nr = SWAPFILE_CLUSTER - 1; in scan_swap_map_slots()
844 spin_unlock(&si->lock); in scan_swap_map_slots()
852 scan_base = offset = si->lowest_bit; in scan_swap_map_slots()
856 for (; last_in_cluster <= si->highest_bit; offset++) { in scan_swap_map_slots()
857 if (si->swap_map[offset]) in scan_swap_map_slots()
860 spin_lock(&si->lock); in scan_swap_map_slots()
862 si->cluster_next = offset; in scan_swap_map_slots()
863 si->cluster_nr = SWAPFILE_CLUSTER - 1; in scan_swap_map_slots()
873 spin_lock(&si->lock); in scan_swap_map_slots()
874 si->cluster_nr = SWAPFILE_CLUSTER - 1; in scan_swap_map_slots()
878 if (si->cluster_info) { in scan_swap_map_slots()
879 while (scan_swap_map_ssd_cluster_conflict(si, offset)) { in scan_swap_map_slots()
883 if (!scan_swap_map_try_ssd_cluster(si, &offset, in scan_swap_map_slots()
888 if (!(si->flags & SWP_WRITEOK)) in scan_swap_map_slots()
890 if (!si->highest_bit) in scan_swap_map_slots()
892 if (offset > si->highest_bit) in scan_swap_map_slots()
893 scan_base = offset = si->lowest_bit; in scan_swap_map_slots()
895 ci = lock_cluster(si, offset); in scan_swap_map_slots()
897 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { in scan_swap_map_slots()
900 spin_unlock(&si->lock); in scan_swap_map_slots()
901 swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY); in scan_swap_map_slots()
902 spin_lock(&si->lock); in scan_swap_map_slots()
909 if (si->swap_map[offset]) { in scan_swap_map_slots()
916 WRITE_ONCE(si->swap_map[offset], usage); in scan_swap_map_slots()
917 inc_cluster_info_page(si, si->cluster_info, offset); in scan_swap_map_slots()
920 swap_range_alloc(si, offset, 1); in scan_swap_map_slots()
921 slots[n_ret++] = swp_entry(si->type, offset); in scan_swap_map_slots()
924 if ((n_ret == nr) || (offset >= si->highest_bit)) in scan_swap_map_slots()
933 spin_unlock(&si->lock); in scan_swap_map_slots()
935 spin_lock(&si->lock); in scan_swap_map_slots()
940 if (si->cluster_info) { in scan_swap_map_slots()
941 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) in scan_swap_map_slots()
943 } else if (si->cluster_nr && !si->swap_map[++offset]) { in scan_swap_map_slots()
945 --si->cluster_nr; in scan_swap_map_slots()
960 scan_limit = si->highest_bit; in scan_swap_map_slots()
963 if (!si->swap_map[offset]) in scan_swap_map_slots()
969 set_cluster_next(si, offset + 1); in scan_swap_map_slots()
970 si->flags -= SWP_SCANNING; in scan_swap_map_slots()
974 spin_unlock(&si->lock); in scan_swap_map_slots()
975 while (++offset <= READ_ONCE(si->highest_bit)) { in scan_swap_map_slots()
981 if (swap_offset_available_and_locked(si, offset)) in scan_swap_map_slots()
984 offset = si->lowest_bit; in scan_swap_map_slots()
991 if (swap_offset_available_and_locked(si, offset)) in scan_swap_map_slots()
995 spin_lock(&si->lock); in scan_swap_map_slots()
998 si->flags -= SWP_SCANNING; in scan_swap_map_slots()
1002 static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) in swap_alloc_cluster() argument
1017 if (cluster_list_empty(&si->free_clusters)) in swap_alloc_cluster()
1020 idx = cluster_list_first(&si->free_clusters); in swap_alloc_cluster()
1022 ci = lock_cluster(si, offset); in swap_alloc_cluster()
1023 alloc_cluster(si, idx); in swap_alloc_cluster()
1026 memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER); in swap_alloc_cluster()
1028 swap_range_alloc(si, offset, SWAPFILE_CLUSTER); in swap_alloc_cluster()
1029 *slot = swp_entry(si->type, offset); in swap_alloc_cluster()
1034 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) in swap_free_cluster() argument
1039 ci = lock_cluster(si, offset); in swap_free_cluster()
1040 memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER); in swap_free_cluster()
1042 free_cluster(si, idx); in swap_free_cluster()
1044 swap_range_free(si, offset, SWAPFILE_CLUSTER); in swap_free_cluster()
1050 struct swap_info_struct *si, *next; in get_swap_pages() local
1072 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) { in get_swap_pages()
1074 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]); in get_swap_pages()
1076 spin_lock(&si->lock); in get_swap_pages()
1077 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) { in get_swap_pages()
1079 if (plist_node_empty(&si->avail_lists[node])) { in get_swap_pages()
1080 spin_unlock(&si->lock); in get_swap_pages()
1083 WARN(!si->highest_bit, in get_swap_pages()
1085 si->type); in get_swap_pages()
1086 WARN(!(si->flags & SWP_WRITEOK), in get_swap_pages()
1088 si->type); in get_swap_pages()
1089 __del_from_avail_list(si); in get_swap_pages()
1090 spin_unlock(&si->lock); in get_swap_pages()
1094 if (si->flags & SWP_BLKDEV) in get_swap_pages()
1095 n_ret = swap_alloc_cluster(si, swp_entries); in get_swap_pages()
1097 n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, in get_swap_pages()
1099 spin_unlock(&si->lock); in get_swap_pages()
1258 struct swap_info_struct *si; in get_swap_device() local
1263 si = swp_swap_info(entry); in get_swap_device()
1264 if (!si) in get_swap_device()
1266 if (!percpu_ref_tryget_live(&si->users)) in get_swap_device()
1277 if (offset >= si->max) in get_swap_device()
1280 return si; in get_swap_device()
1287 percpu_ref_put(&si->users); in get_swap_device()
1345 struct swap_info_struct *si; in put_swap_folio() local
1351 si = _swap_info_get(entry); in put_swap_folio()
1352 if (!si) in put_swap_folio()
1355 ci = lock_cluster_or_swap_info(si, offset); in put_swap_folio()
1358 map = si->swap_map + offset; in put_swap_folio()
1367 unlock_cluster_or_swap_info(si, ci); in put_swap_folio()
1368 spin_lock(&si->lock); in put_swap_folio()
1370 swap_free_cluster(si, idx); in put_swap_folio()
1371 spin_unlock(&si->lock); in put_swap_folio()
1376 if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) { in put_swap_folio()
1377 unlock_cluster_or_swap_info(si, ci); in put_swap_folio()
1381 lock_cluster_or_swap_info(si, offset); in put_swap_folio()
1384 unlock_cluster_or_swap_info(si, ci); in put_swap_folio()
1390 struct swap_info_struct *si; in split_swap_cluster() local
1394 si = _swap_info_get(entry); in split_swap_cluster()
1395 if (!si) in split_swap_cluster()
1397 ci = lock_cluster(si, offset); in split_swap_cluster()
1441 struct swap_info_struct *si = swp_swap_info(entry); in __swap_count() local
1444 return swap_count(si->swap_map[offset]); in __swap_count()
1452 int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) in swap_swapcount() argument
1458 ci = lock_cluster_or_swap_info(si, offset); in swap_swapcount()
1459 count = swap_count(si->swap_map[offset]); in swap_swapcount()
1460 unlock_cluster_or_swap_info(si, ci); in swap_swapcount()
1510 static bool swap_page_trans_huge_swapped(struct swap_info_struct *si, in swap_page_trans_huge_swapped() argument
1514 unsigned char *map = si->swap_map; in swap_page_trans_huge_swapped()
1520 ci = lock_cluster_or_swap_info(si, offset); in swap_page_trans_huge_swapped()
1533 unlock_cluster_or_swap_info(si, ci); in swap_page_trans_huge_swapped()
1540 struct swap_info_struct *si = _swap_info_get(entry); in folio_swapped() local
1542 if (!si) in folio_swapped()
1546 return swap_swapcount(si, entry) != 0; in folio_swapped()
1548 return swap_page_trans_huge_swapped(si, entry); in folio_swapped()
1621 struct swap_info_struct *si = swap_type_to_swap_info(type); in get_swap_page_of_type() local
1624 if (!si) in get_swap_page_of_type()
1628 spin_lock(&si->lock); in get_swap_page_of_type()
1629 if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry)) in get_swap_page_of_type()
1631 spin_unlock(&si->lock); in get_swap_page_of_type()
1695 struct swap_info_struct *si = swap_type_to_swap_info(type); in swapdev_block() local
1698 if (!si || !(si->flags & SWP_WRITEOK)) in swapdev_block()
1700 se = offset_to_swap_extent(si, offset); in swapdev_block()
1836 struct swap_info_struct *si; in unuse_pte_range() local
1838 si = swap_info[type]; in unuse_pte_range()
1882 swp_count = READ_ONCE(si->swap_map[offset]); in unuse_pte_range()
2012 static unsigned int find_next_to_unuse(struct swap_info_struct *si, in find_next_to_unuse() argument
2024 for (i = prev + 1; i < si->max; i++) { in find_next_to_unuse()
2025 count = READ_ONCE(si->swap_map[i]); in find_next_to_unuse()
2032 if (i == si->max) in find_next_to_unuse()
2044 struct swap_info_struct *si = swap_info[type]; in try_to_unuse() local
2049 if (!READ_ONCE(si->inuse_pages)) in try_to_unuse()
2062 while (READ_ONCE(si->inuse_pages) && in try_to_unuse()
2090 while (READ_ONCE(si->inuse_pages) && in try_to_unuse()
2092 (i = find_next_to_unuse(si, i)) != 0) { in try_to_unuse()
2125 if (READ_ONCE(si->inuse_pages)) { in try_to_unuse()
2439 struct swap_info_struct *si = p; in SYSCALL_DEFINE1() local
2442 plist_for_each_entry_continue(si, &swap_active_head, list) { in SYSCALL_DEFINE1()
2443 si->prio++; in SYSCALL_DEFINE1()
2444 si->list.prio--; in SYSCALL_DEFINE1()
2446 if (si->avail_lists[nid].prio != 1) in SYSCALL_DEFINE1()
2447 si->avail_lists[nid].prio--; in SYSCALL_DEFINE1()
2583 struct swap_info_struct *si; in swap_start() local
2592 for (type = 0; (si = swap_type_to_swap_info(type)); type++) { in swap_start()
2593 if (!(si->flags & SWP_USED) || !si->swap_map) in swap_start()
2596 return si; in swap_start()
2604 struct swap_info_struct *si = v; in swap_next() local
2610 type = si->type + 1; in swap_next()
2613 for (; (si = swap_type_to_swap_info(type)); type++) { in swap_next()
2614 if (!(si->flags & SWP_USED) || !si->swap_map) in swap_next()
2616 return si; in swap_next()
2629 struct swap_info_struct *si = v; in swap_show() local
2634 if (si == SEQ_START_TOKEN) { in swap_show()
2639 bytes = K(si->pages); in swap_show()
2640 inuse = K(READ_ONCE(si->inuse_pages)); in swap_show()
2642 file = si->swap_file; in swap_show()
2650 si->prio); in swap_show()
3247 struct swap_info_struct *si = swap_info[type]; in si_swapinfo() local
3249 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) in si_swapinfo()
3250 nr_to_be_unused += READ_ONCE(si->inuse_pages); in si_swapinfo()
3365 void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry) in swapcache_clear() argument
3371 ci = lock_cluster_or_swap_info(si, offset); in swapcache_clear()
3372 usage = __swap_entry_free_locked(si, offset, SWAP_HAS_CACHE); in swapcache_clear()
3373 unlock_cluster_or_swap_info(si, ci); in swapcache_clear()
3422 struct swap_info_struct *si; in add_swap_count_continuation() local
3437 si = get_swap_device(entry); in add_swap_count_continuation()
3438 if (!si) { in add_swap_count_continuation()
3445 spin_lock(&si->lock); in add_swap_count_continuation()
3449 ci = lock_cluster(si, offset); in add_swap_count_continuation()
3451 count = swap_count(si->swap_map[offset]); in add_swap_count_continuation()
3467 head = vmalloc_to_page(si->swap_map + offset); in add_swap_count_continuation()
3470 spin_lock(&si->cont_lock); in add_swap_count_continuation()
3479 si->flags |= SWP_CONTINUED; in add_swap_count_continuation()
3507 spin_unlock(&si->cont_lock); in add_swap_count_continuation()
3510 spin_unlock(&si->lock); in add_swap_count_continuation()
3511 put_swap_device(si); in add_swap_count_continuation()
3527 static bool swap_count_continued(struct swap_info_struct *si, in swap_count_continued() argument
3535 head = vmalloc_to_page(si->swap_map + offset); in swap_count_continued()
3541 spin_lock(&si->cont_lock); in swap_count_continued()
3603 spin_unlock(&si->cont_lock); in swap_count_continued()
3611 static void free_swap_count_continuations(struct swap_info_struct *si) in free_swap_count_continuations() argument
3615 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { in free_swap_count_continuations()
3617 head = vmalloc_to_page(si->swap_map + offset); in free_swap_count_continuations()
3632 struct swap_info_struct *si, *next; in __folio_throttle_swaprate() local
3649 plist_for_each_entry_safe(si, next, &swap_avail_heads[nid], in __folio_throttle_swaprate()
3651 if (si->bdev) { in __folio_throttle_swaprate()
3652 blkcg_schedule_throttle(si->bdev->bd_disk, true); in __folio_throttle_swaprate()