1 /*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
9 * Copyright (C) 2002-2011 Hugh Dickins.
10 * Copyright (C) 2011 Google Inc.
11 * Copyright (C) 2002-2005 VERITAS Software Corporation.
12 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 *
14 * Extended attribute support for tmpfs:
15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 *
18 * tiny-shmem:
19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20 *
21 * This file is released under the GPL.
22 */
23
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/fileattr.h>
32 #include <linux/mm.h>
33 #include <linux/random.h>
34 #include <linux/sched/signal.h>
35 #include <linux/export.h>
36 #include <linux/swap.h>
37 #include <linux/uio.h>
38 #include <linux/hugetlb.h>
39 #include <linux/fs_parser.h>
40 #include <linux/swapfile.h>
41 #include <linux/iversion.h>
42 #include "swap.h"
43
44 static struct vfsmount *shm_mnt;
45
46 #ifdef CONFIG_SHMEM
47 /*
48 * This virtual memory filesystem is heavily based on the ramfs. It
49 * extends ramfs by the ability to use swap and honor resource limits
50 * which makes it a completely usable filesystem.
51 */
52
53 #include <linux/xattr.h>
54 #include <linux/exportfs.h>
55 #include <linux/posix_acl.h>
56 #include <linux/posix_acl_xattr.h>
57 #include <linux/mman.h>
58 #include <linux/string.h>
59 #include <linux/slab.h>
60 #include <linux/backing-dev.h>
61 #include <linux/shmem_fs.h>
62 #include <linux/writeback.h>
63 #include <linux/pagevec.h>
64 #include <linux/percpu_counter.h>
65 #include <linux/falloc.h>
66 #include <linux/splice.h>
67 #include <linux/security.h>
68 #include <linux/swapops.h>
69 #include <linux/mempolicy.h>
70 #include <linux/namei.h>
71 #include <linux/ctype.h>
72 #include <linux/migrate.h>
73 #include <linux/highmem.h>
74 #include <linux/seq_file.h>
75 #include <linux/magic.h>
76 #include <linux/syscalls.h>
77 #include <linux/fcntl.h>
78 #include <uapi/linux/memfd.h>
79 #include <linux/userfaultfd_k.h>
80 #include <linux/rmap.h>
81 #include <linux/uuid.h>
82
83 #include <linux/uaccess.h>
84
85 #include "internal.h"
86
87 #define BLOCKS_PER_PAGE (PAGE_SIZE/512)
88 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT)
89
90 /* Pretend that each entry is of this size in directory's i_size */
91 #define BOGO_DIRENT_SIZE 20
92
93 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
94 #define SHORT_SYMLINK_LEN 128
95
96 /*
97 * shmem_fallocate communicates with shmem_fault or shmem_writepage via
98 * inode->i_private (with i_rwsem making sure that it has only one user at
99 * a time): we would prefer not to enlarge the shmem inode just for that.
100 */
101 struct shmem_falloc {
102 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
103 pgoff_t start; /* start of range currently being fallocated */
104 pgoff_t next; /* the next page offset to be fallocated */
105 pgoff_t nr_falloced; /* how many new pages have been fallocated */
106 pgoff_t nr_unswapped; /* how often writepage refused to swap out */
107 };
108
109 struct shmem_options {
110 unsigned long long blocks;
111 unsigned long long inodes;
112 struct mempolicy *mpol;
113 kuid_t uid;
114 kgid_t gid;
115 umode_t mode;
116 bool full_inums;
117 int huge;
118 int seen;
119 #define SHMEM_SEEN_BLOCKS 1
120 #define SHMEM_SEEN_INODES 2
121 #define SHMEM_SEEN_HUGE 4
122 #define SHMEM_SEEN_INUMS 8
123 };
124
125 #ifdef CONFIG_TMPFS
shmem_default_max_blocks(void)126 static unsigned long shmem_default_max_blocks(void)
127 {
128 return totalram_pages() / 2;
129 }
130
shmem_default_max_inodes(void)131 static unsigned long shmem_default_max_inodes(void)
132 {
133 unsigned long nr_pages = totalram_pages();
134
135 return min(nr_pages - totalhigh_pages(), nr_pages / 2);
136 }
137 #endif
138
139 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
140 struct folio **foliop, enum sgp_type sgp,
141 gfp_t gfp, struct vm_area_struct *vma,
142 vm_fault_t *fault_type);
143
SHMEM_SB(struct super_block * sb)144 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
145 {
146 return sb->s_fs_info;
147 }
148
149 /*
150 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
151 * for shared memory and for shared anonymous (/dev/zero) mappings
152 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
153 * consistent with the pre-accounting of private mappings ...
154 */
shmem_acct_size(unsigned long flags,loff_t size)155 static inline int shmem_acct_size(unsigned long flags, loff_t size)
156 {
157 return (flags & VM_NORESERVE) ?
158 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
159 }
160
shmem_unacct_size(unsigned long flags,loff_t size)161 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
162 {
163 if (!(flags & VM_NORESERVE))
164 vm_unacct_memory(VM_ACCT(size));
165 }
166
shmem_reacct_size(unsigned long flags,loff_t oldsize,loff_t newsize)167 static inline int shmem_reacct_size(unsigned long flags,
168 loff_t oldsize, loff_t newsize)
169 {
170 if (!(flags & VM_NORESERVE)) {
171 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
172 return security_vm_enough_memory_mm(current->mm,
173 VM_ACCT(newsize) - VM_ACCT(oldsize));
174 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
175 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
176 }
177 return 0;
178 }
179
180 /*
181 * ... whereas tmpfs objects are accounted incrementally as
182 * pages are allocated, in order to allow large sparse files.
183 * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
184 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
185 */
shmem_acct_block(unsigned long flags,long pages)186 static inline int shmem_acct_block(unsigned long flags, long pages)
187 {
188 if (!(flags & VM_NORESERVE))
189 return 0;
190
191 return security_vm_enough_memory_mm(current->mm,
192 pages * VM_ACCT(PAGE_SIZE));
193 }
194
shmem_unacct_blocks(unsigned long flags,long pages)195 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
196 {
197 if (flags & VM_NORESERVE)
198 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
199 }
200
shmem_inode_acct_block(struct inode * inode,long pages)201 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
202 {
203 struct shmem_inode_info *info = SHMEM_I(inode);
204 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
205
206 if (shmem_acct_block(info->flags, pages))
207 return false;
208
209 if (sbinfo->max_blocks) {
210 if (percpu_counter_compare(&sbinfo->used_blocks,
211 sbinfo->max_blocks - pages) > 0)
212 goto unacct;
213 percpu_counter_add(&sbinfo->used_blocks, pages);
214 }
215
216 return true;
217
218 unacct:
219 shmem_unacct_blocks(info->flags, pages);
220 return false;
221 }
222
shmem_inode_unacct_blocks(struct inode * inode,long pages)223 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
224 {
225 struct shmem_inode_info *info = SHMEM_I(inode);
226 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
227
228 if (sbinfo->max_blocks)
229 percpu_counter_sub(&sbinfo->used_blocks, pages);
230 shmem_unacct_blocks(info->flags, pages);
231 }
232
233 static const struct super_operations shmem_ops;
234 const struct address_space_operations shmem_aops;
235 static const struct file_operations shmem_file_operations;
236 static const struct inode_operations shmem_inode_operations;
237 static const struct inode_operations shmem_dir_inode_operations;
238 static const struct inode_operations shmem_special_inode_operations;
239 static const struct vm_operations_struct shmem_vm_ops;
240 static struct file_system_type shmem_fs_type;
241
vma_is_shmem(struct vm_area_struct * vma)242 bool vma_is_shmem(struct vm_area_struct *vma)
243 {
244 return vma->vm_ops == &shmem_vm_ops;
245 }
246
247 static LIST_HEAD(shmem_swaplist);
248 static DEFINE_MUTEX(shmem_swaplist_mutex);
249
250 /*
251 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
252 * produces a novel ino for the newly allocated inode.
253 *
254 * It may also be called when making a hard link to permit the space needed by
255 * each dentry. However, in that case, no new inode number is needed since that
256 * internally draws from another pool of inode numbers (currently global
257 * get_next_ino()). This case is indicated by passing NULL as inop.
258 */
259 #define SHMEM_INO_BATCH 1024
shmem_reserve_inode(struct super_block * sb,ino_t * inop)260 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
261 {
262 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
263 ino_t ino;
264
265 if (!(sb->s_flags & SB_KERNMOUNT)) {
266 raw_spin_lock(&sbinfo->stat_lock);
267 if (sbinfo->max_inodes) {
268 if (!sbinfo->free_inodes) {
269 raw_spin_unlock(&sbinfo->stat_lock);
270 return -ENOSPC;
271 }
272 sbinfo->free_inodes--;
273 }
274 if (inop) {
275 ino = sbinfo->next_ino++;
276 if (unlikely(is_zero_ino(ino)))
277 ino = sbinfo->next_ino++;
278 if (unlikely(!sbinfo->full_inums &&
279 ino > UINT_MAX)) {
280 /*
281 * Emulate get_next_ino uint wraparound for
282 * compatibility
283 */
284 if (IS_ENABLED(CONFIG_64BIT))
285 pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
286 __func__, MINOR(sb->s_dev));
287 sbinfo->next_ino = 1;
288 ino = sbinfo->next_ino++;
289 }
290 *inop = ino;
291 }
292 raw_spin_unlock(&sbinfo->stat_lock);
293 } else if (inop) {
294 /*
295 * __shmem_file_setup, one of our callers, is lock-free: it
296 * doesn't hold stat_lock in shmem_reserve_inode since
297 * max_inodes is always 0, and is called from potentially
298 * unknown contexts. As such, use a per-cpu batched allocator
299 * which doesn't require the per-sb stat_lock unless we are at
300 * the batch boundary.
301 *
302 * We don't need to worry about inode{32,64} since SB_KERNMOUNT
303 * shmem mounts are not exposed to userspace, so we don't need
304 * to worry about things like glibc compatibility.
305 */
306 ino_t *next_ino;
307
308 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
309 ino = *next_ino;
310 if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
311 raw_spin_lock(&sbinfo->stat_lock);
312 ino = sbinfo->next_ino;
313 sbinfo->next_ino += SHMEM_INO_BATCH;
314 raw_spin_unlock(&sbinfo->stat_lock);
315 if (unlikely(is_zero_ino(ino)))
316 ino++;
317 }
318 *inop = ino;
319 *next_ino = ++ino;
320 put_cpu();
321 }
322
323 return 0;
324 }
325
shmem_free_inode(struct super_block * sb)326 static void shmem_free_inode(struct super_block *sb)
327 {
328 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
329 if (sbinfo->max_inodes) {
330 raw_spin_lock(&sbinfo->stat_lock);
331 sbinfo->free_inodes++;
332 raw_spin_unlock(&sbinfo->stat_lock);
333 }
334 }
335
336 /**
337 * shmem_recalc_inode - recalculate the block usage of an inode
338 * @inode: inode to recalc
339 *
340 * We have to calculate the free blocks since the mm can drop
341 * undirtied hole pages behind our back.
342 *
343 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
344 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
345 *
346 * It has to be called with the spinlock held.
347 */
shmem_recalc_inode(struct inode * inode)348 static void shmem_recalc_inode(struct inode *inode)
349 {
350 struct shmem_inode_info *info = SHMEM_I(inode);
351 long freed;
352
353 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
354 if (freed > 0) {
355 info->alloced -= freed;
356 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
357 shmem_inode_unacct_blocks(inode, freed);
358 }
359 }
360
shmem_charge(struct inode * inode,long pages)361 bool shmem_charge(struct inode *inode, long pages)
362 {
363 struct shmem_inode_info *info = SHMEM_I(inode);
364 unsigned long flags;
365
366 if (!shmem_inode_acct_block(inode, pages))
367 return false;
368
369 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
370 inode->i_mapping->nrpages += pages;
371
372 spin_lock_irqsave(&info->lock, flags);
373 info->alloced += pages;
374 inode->i_blocks += pages * BLOCKS_PER_PAGE;
375 shmem_recalc_inode(inode);
376 spin_unlock_irqrestore(&info->lock, flags);
377
378 return true;
379 }
380
shmem_uncharge(struct inode * inode,long pages)381 void shmem_uncharge(struct inode *inode, long pages)
382 {
383 struct shmem_inode_info *info = SHMEM_I(inode);
384 unsigned long flags;
385
386 /* nrpages adjustment done by __filemap_remove_folio() or caller */
387
388 spin_lock_irqsave(&info->lock, flags);
389 info->alloced -= pages;
390 inode->i_blocks -= pages * BLOCKS_PER_PAGE;
391 shmem_recalc_inode(inode);
392 spin_unlock_irqrestore(&info->lock, flags);
393
394 shmem_inode_unacct_blocks(inode, pages);
395 }
396
397 /*
398 * Replace item expected in xarray by a new item, while holding xa_lock.
399 */
shmem_replace_entry(struct address_space * mapping,pgoff_t index,void * expected,void * replacement)400 static int shmem_replace_entry(struct address_space *mapping,
401 pgoff_t index, void *expected, void *replacement)
402 {
403 XA_STATE(xas, &mapping->i_pages, index);
404 void *item;
405
406 VM_BUG_ON(!expected);
407 VM_BUG_ON(!replacement);
408 item = xas_load(&xas);
409 if (item != expected)
410 return -ENOENT;
411 xas_store(&xas, replacement);
412 return 0;
413 }
414
415 /*
416 * Sometimes, before we decide whether to proceed or to fail, we must check
417 * that an entry was not already brought back from swap by a racing thread.
418 *
419 * Checking page is not enough: by the time a SwapCache page is locked, it
420 * might be reused, and again be SwapCache, using the same swap as before.
421 */
shmem_confirm_swap(struct address_space * mapping,pgoff_t index,swp_entry_t swap)422 static bool shmem_confirm_swap(struct address_space *mapping,
423 pgoff_t index, swp_entry_t swap)
424 {
425 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
426 }
427
428 /*
429 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
430 *
431 * SHMEM_HUGE_NEVER:
432 * disables huge pages for the mount;
433 * SHMEM_HUGE_ALWAYS:
434 * enables huge pages for the mount;
435 * SHMEM_HUGE_WITHIN_SIZE:
436 * only allocate huge pages if the page will be fully within i_size,
437 * also respect fadvise()/madvise() hints;
438 * SHMEM_HUGE_ADVISE:
439 * only allocate huge pages if requested with fadvise()/madvise();
440 */
441
442 #define SHMEM_HUGE_NEVER 0
443 #define SHMEM_HUGE_ALWAYS 1
444 #define SHMEM_HUGE_WITHIN_SIZE 2
445 #define SHMEM_HUGE_ADVISE 3
446
447 /*
448 * Special values.
449 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
450 *
451 * SHMEM_HUGE_DENY:
452 * disables huge on shm_mnt and all mounts, for emergency use;
453 * SHMEM_HUGE_FORCE:
454 * enables huge on shm_mnt and all mounts, w/o needing option, for testing;
455 *
456 */
457 #define SHMEM_HUGE_DENY (-1)
458 #define SHMEM_HUGE_FORCE (-2)
459
460 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
461 /* ifdef here to avoid bloating shmem.o when not necessary */
462
463 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
464
shmem_is_huge(struct vm_area_struct * vma,struct inode * inode,pgoff_t index,bool shmem_huge_force)465 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
466 pgoff_t index, bool shmem_huge_force)
467 {
468 loff_t i_size;
469
470 if (!S_ISREG(inode->i_mode))
471 return false;
472 if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) ||
473 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)))
474 return false;
475 if (shmem_huge_force)
476 return true;
477 if (shmem_huge == SHMEM_HUGE_FORCE)
478 return true;
479 if (shmem_huge == SHMEM_HUGE_DENY)
480 return false;
481
482 switch (SHMEM_SB(inode->i_sb)->huge) {
483 case SHMEM_HUGE_ALWAYS:
484 return true;
485 case SHMEM_HUGE_WITHIN_SIZE:
486 index = round_up(index + 1, HPAGE_PMD_NR);
487 i_size = round_up(i_size_read(inode), PAGE_SIZE);
488 if (i_size >> PAGE_SHIFT >= index)
489 return true;
490 fallthrough;
491 case SHMEM_HUGE_ADVISE:
492 if (vma && (vma->vm_flags & VM_HUGEPAGE))
493 return true;
494 fallthrough;
495 default:
496 return false;
497 }
498 }
499
500 #if defined(CONFIG_SYSFS)
shmem_parse_huge(const char * str)501 static int shmem_parse_huge(const char *str)
502 {
503 if (!strcmp(str, "never"))
504 return SHMEM_HUGE_NEVER;
505 if (!strcmp(str, "always"))
506 return SHMEM_HUGE_ALWAYS;
507 if (!strcmp(str, "within_size"))
508 return SHMEM_HUGE_WITHIN_SIZE;
509 if (!strcmp(str, "advise"))
510 return SHMEM_HUGE_ADVISE;
511 if (!strcmp(str, "deny"))
512 return SHMEM_HUGE_DENY;
513 if (!strcmp(str, "force"))
514 return SHMEM_HUGE_FORCE;
515 return -EINVAL;
516 }
517 #endif
518
519 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
shmem_format_huge(int huge)520 static const char *shmem_format_huge(int huge)
521 {
522 switch (huge) {
523 case SHMEM_HUGE_NEVER:
524 return "never";
525 case SHMEM_HUGE_ALWAYS:
526 return "always";
527 case SHMEM_HUGE_WITHIN_SIZE:
528 return "within_size";
529 case SHMEM_HUGE_ADVISE:
530 return "advise";
531 case SHMEM_HUGE_DENY:
532 return "deny";
533 case SHMEM_HUGE_FORCE:
534 return "force";
535 default:
536 VM_BUG_ON(1);
537 return "bad_val";
538 }
539 }
540 #endif
541
shmem_unused_huge_shrink(struct shmem_sb_info * sbinfo,struct shrink_control * sc,unsigned long nr_to_split)542 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
543 struct shrink_control *sc, unsigned long nr_to_split)
544 {
545 LIST_HEAD(list), *pos, *next;
546 LIST_HEAD(to_remove);
547 struct inode *inode;
548 struct shmem_inode_info *info;
549 struct folio *folio;
550 unsigned long batch = sc ? sc->nr_to_scan : 128;
551 int split = 0;
552
553 if (list_empty(&sbinfo->shrinklist))
554 return SHRINK_STOP;
555
556 spin_lock(&sbinfo->shrinklist_lock);
557 list_for_each_safe(pos, next, &sbinfo->shrinklist) {
558 info = list_entry(pos, struct shmem_inode_info, shrinklist);
559
560 /* pin the inode */
561 inode = igrab(&info->vfs_inode);
562
563 /* inode is about to be evicted */
564 if (!inode) {
565 list_del_init(&info->shrinklist);
566 goto next;
567 }
568
569 /* Check if there's anything to gain */
570 if (round_up(inode->i_size, PAGE_SIZE) ==
571 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
572 list_move(&info->shrinklist, &to_remove);
573 goto next;
574 }
575
576 list_move(&info->shrinklist, &list);
577 next:
578 sbinfo->shrinklist_len--;
579 if (!--batch)
580 break;
581 }
582 spin_unlock(&sbinfo->shrinklist_lock);
583
584 list_for_each_safe(pos, next, &to_remove) {
585 info = list_entry(pos, struct shmem_inode_info, shrinklist);
586 inode = &info->vfs_inode;
587 list_del_init(&info->shrinklist);
588 iput(inode);
589 }
590
591 list_for_each_safe(pos, next, &list) {
592 int ret;
593 pgoff_t index;
594
595 info = list_entry(pos, struct shmem_inode_info, shrinklist);
596 inode = &info->vfs_inode;
597
598 if (nr_to_split && split >= nr_to_split)
599 goto move_back;
600
601 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
602 folio = filemap_get_folio(inode->i_mapping, index);
603 if (!folio)
604 goto drop;
605
606 /* No huge page at the end of the file: nothing to split */
607 if (!folio_test_large(folio)) {
608 folio_put(folio);
609 goto drop;
610 }
611
612 /*
613 * Move the inode on the list back to shrinklist if we failed
614 * to lock the page at this time.
615 *
616 * Waiting for the lock may lead to deadlock in the
617 * reclaim path.
618 */
619 if (!folio_trylock(folio)) {
620 folio_put(folio);
621 goto move_back;
622 }
623
624 ret = split_folio(folio);
625 folio_unlock(folio);
626 folio_put(folio);
627
628 /* If split failed move the inode on the list back to shrinklist */
629 if (ret)
630 goto move_back;
631
632 split++;
633 drop:
634 list_del_init(&info->shrinklist);
635 goto put;
636 move_back:
637 /*
638 * Make sure the inode is either on the global list or deleted
639 * from any local list before iput() since it could be deleted
640 * in another thread once we put the inode (then the local list
641 * is corrupted).
642 */
643 spin_lock(&sbinfo->shrinklist_lock);
644 list_move(&info->shrinklist, &sbinfo->shrinklist);
645 sbinfo->shrinklist_len++;
646 spin_unlock(&sbinfo->shrinklist_lock);
647 put:
648 iput(inode);
649 }
650
651 return split;
652 }
653
shmem_unused_huge_scan(struct super_block * sb,struct shrink_control * sc)654 static long shmem_unused_huge_scan(struct super_block *sb,
655 struct shrink_control *sc)
656 {
657 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
658
659 if (!READ_ONCE(sbinfo->shrinklist_len))
660 return SHRINK_STOP;
661
662 return shmem_unused_huge_shrink(sbinfo, sc, 0);
663 }
664
shmem_unused_huge_count(struct super_block * sb,struct shrink_control * sc)665 static long shmem_unused_huge_count(struct super_block *sb,
666 struct shrink_control *sc)
667 {
668 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
669 return READ_ONCE(sbinfo->shrinklist_len);
670 }
671 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
672
673 #define shmem_huge SHMEM_HUGE_DENY
674
shmem_is_huge(struct vm_area_struct * vma,struct inode * inode,pgoff_t index,bool shmem_huge_force)675 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
676 pgoff_t index, bool shmem_huge_force)
677 {
678 return false;
679 }
680
shmem_unused_huge_shrink(struct shmem_sb_info * sbinfo,struct shrink_control * sc,unsigned long nr_to_split)681 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
682 struct shrink_control *sc, unsigned long nr_to_split)
683 {
684 return 0;
685 }
686 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
687
688 /*
689 * Like filemap_add_folio, but error if expected item has gone.
690 */
shmem_add_to_page_cache(struct folio * folio,struct address_space * mapping,pgoff_t index,void * expected,gfp_t gfp,struct mm_struct * charge_mm)691 static int shmem_add_to_page_cache(struct folio *folio,
692 struct address_space *mapping,
693 pgoff_t index, void *expected, gfp_t gfp,
694 struct mm_struct *charge_mm)
695 {
696 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
697 long nr = folio_nr_pages(folio);
698 int error;
699
700 VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
701 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
702 VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
703 VM_BUG_ON(expected && folio_test_large(folio));
704
705 folio_ref_add(folio, nr);
706 folio->mapping = mapping;
707 folio->index = index;
708
709 if (!folio_test_swapcache(folio)) {
710 error = mem_cgroup_charge(folio, charge_mm, gfp);
711 if (error) {
712 if (folio_test_pmd_mappable(folio)) {
713 count_vm_event(THP_FILE_FALLBACK);
714 count_vm_event(THP_FILE_FALLBACK_CHARGE);
715 }
716 goto error;
717 }
718 }
719 folio_throttle_swaprate(folio, gfp);
720
721 do {
722 xas_lock_irq(&xas);
723 if (expected != xas_find_conflict(&xas)) {
724 xas_set_err(&xas, -EEXIST);
725 goto unlock;
726 }
727 if (expected && xas_find_conflict(&xas)) {
728 xas_set_err(&xas, -EEXIST);
729 goto unlock;
730 }
731 xas_store(&xas, folio);
732 if (xas_error(&xas))
733 goto unlock;
734 if (folio_test_pmd_mappable(folio)) {
735 count_vm_event(THP_FILE_ALLOC);
736 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
737 }
738 mapping->nrpages += nr;
739 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
740 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
741 unlock:
742 xas_unlock_irq(&xas);
743 } while (xas_nomem(&xas, gfp));
744
745 if (xas_error(&xas)) {
746 error = xas_error(&xas);
747 goto error;
748 }
749
750 return 0;
751 error:
752 folio->mapping = NULL;
753 folio_ref_sub(folio, nr);
754 return error;
755 }
756
757 /*
758 * Like delete_from_page_cache, but substitutes swap for @folio.
759 */
shmem_delete_from_page_cache(struct folio * folio,void * radswap)760 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
761 {
762 struct address_space *mapping = folio->mapping;
763 long nr = folio_nr_pages(folio);
764 int error;
765
766 xa_lock_irq(&mapping->i_pages);
767 error = shmem_replace_entry(mapping, folio->index, folio, radswap);
768 folio->mapping = NULL;
769 mapping->nrpages -= nr;
770 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
771 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
772 xa_unlock_irq(&mapping->i_pages);
773 folio_put(folio);
774 BUG_ON(error);
775 }
776
777 /*
778 * Remove swap entry from page cache, free the swap and its page cache.
779 */
shmem_free_swap(struct address_space * mapping,pgoff_t index,void * radswap)780 static int shmem_free_swap(struct address_space *mapping,
781 pgoff_t index, void *radswap)
782 {
783 void *old;
784
785 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
786 if (old != radswap)
787 return -ENOENT;
788 free_swap_and_cache(radix_to_swp_entry(radswap));
789 return 0;
790 }
791
792 /*
793 * Determine (in bytes) how many of the shmem object's pages mapped by the
794 * given offsets are swapped out.
795 *
796 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
797 * as long as the inode doesn't go away and racy results are not a problem.
798 */
shmem_partial_swap_usage(struct address_space * mapping,pgoff_t start,pgoff_t end)799 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
800 pgoff_t start, pgoff_t end)
801 {
802 XA_STATE(xas, &mapping->i_pages, start);
803 struct page *page;
804 unsigned long swapped = 0;
805
806 rcu_read_lock();
807 xas_for_each(&xas, page, end - 1) {
808 if (xas_retry(&xas, page))
809 continue;
810 if (xa_is_value(page))
811 swapped++;
812
813 if (need_resched()) {
814 xas_pause(&xas);
815 cond_resched_rcu();
816 }
817 }
818
819 rcu_read_unlock();
820
821 return swapped << PAGE_SHIFT;
822 }
823
824 /*
825 * Determine (in bytes) how many of the shmem object's pages mapped by the
826 * given vma is swapped out.
827 *
828 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
829 * as long as the inode doesn't go away and racy results are not a problem.
830 */
shmem_swap_usage(struct vm_area_struct * vma)831 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
832 {
833 struct inode *inode = file_inode(vma->vm_file);
834 struct shmem_inode_info *info = SHMEM_I(inode);
835 struct address_space *mapping = inode->i_mapping;
836 unsigned long swapped;
837
838 /* Be careful as we don't hold info->lock */
839 swapped = READ_ONCE(info->swapped);
840
841 /*
842 * The easier cases are when the shmem object has nothing in swap, or
843 * the vma maps it whole. Then we can simply use the stats that we
844 * already track.
845 */
846 if (!swapped)
847 return 0;
848
849 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
850 return swapped << PAGE_SHIFT;
851
852 /* Here comes the more involved part */
853 return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
854 vma->vm_pgoff + vma_pages(vma));
855 }
856
857 /*
858 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
859 */
shmem_unlock_mapping(struct address_space * mapping)860 void shmem_unlock_mapping(struct address_space *mapping)
861 {
862 struct folio_batch fbatch;
863 pgoff_t index = 0;
864
865 folio_batch_init(&fbatch);
866 /*
867 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
868 */
869 while (!mapping_unevictable(mapping) &&
870 filemap_get_folios(mapping, &index, ~0UL, &fbatch)) {
871 check_move_unevictable_folios(&fbatch);
872 folio_batch_release(&fbatch);
873 cond_resched();
874 }
875 }
876
shmem_get_partial_folio(struct inode * inode,pgoff_t index)877 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
878 {
879 struct folio *folio;
880
881 /*
882 * At first avoid shmem_get_folio(,,,SGP_READ): that fails
883 * beyond i_size, and reports fallocated pages as holes.
884 */
885 folio = __filemap_get_folio(inode->i_mapping, index,
886 FGP_ENTRY | FGP_LOCK, 0);
887 if (!xa_is_value(folio))
888 return folio;
889 /*
890 * But read a page back from swap if any of it is within i_size
891 * (although in some cases this is just a waste of time).
892 */
893 folio = NULL;
894 shmem_get_folio(inode, index, &folio, SGP_READ);
895 return folio;
896 }
897
898 /*
899 * Remove range of pages and swap entries from page cache, and free them.
900 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
901 */
shmem_undo_range(struct inode * inode,loff_t lstart,loff_t lend,bool unfalloc)902 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
903 bool unfalloc)
904 {
905 struct address_space *mapping = inode->i_mapping;
906 struct shmem_inode_info *info = SHMEM_I(inode);
907 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
908 pgoff_t end = (lend + 1) >> PAGE_SHIFT;
909 struct folio_batch fbatch;
910 pgoff_t indices[PAGEVEC_SIZE];
911 struct folio *folio;
912 bool same_folio;
913 long nr_swaps_freed = 0;
914 pgoff_t index;
915 int i;
916
917 if (lend == -1)
918 end = -1; /* unsigned, so actually very big */
919
920 if (info->fallocend > start && info->fallocend <= end && !unfalloc)
921 info->fallocend = start;
922
923 folio_batch_init(&fbatch);
924 index = start;
925 while (index < end && find_lock_entries(mapping, index, end - 1,
926 &fbatch, indices)) {
927 for (i = 0; i < folio_batch_count(&fbatch); i++) {
928 folio = fbatch.folios[i];
929
930 index = indices[i];
931
932 if (xa_is_value(folio)) {
933 if (unfalloc)
934 continue;
935 nr_swaps_freed += !shmem_free_swap(mapping,
936 index, folio);
937 continue;
938 }
939 index += folio_nr_pages(folio) - 1;
940
941 if (!unfalloc || !folio_test_uptodate(folio))
942 truncate_inode_folio(mapping, folio);
943 folio_unlock(folio);
944 }
945 folio_batch_remove_exceptionals(&fbatch);
946 folio_batch_release(&fbatch);
947 cond_resched();
948 index++;
949 }
950
951 /*
952 * When undoing a failed fallocate, we want none of the partial folio
953 * zeroing and splitting below, but shall want to truncate the whole
954 * folio when !uptodate indicates that it was added by this fallocate,
955 * even when [lstart, lend] covers only a part of the folio.
956 */
957 if (unfalloc)
958 goto whole_folios;
959
960 same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
961 folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
962 if (folio) {
963 same_folio = lend < folio_pos(folio) + folio_size(folio);
964 folio_mark_dirty(folio);
965 if (!truncate_inode_partial_folio(folio, lstart, lend)) {
966 start = folio->index + folio_nr_pages(folio);
967 if (same_folio)
968 end = folio->index;
969 }
970 folio_unlock(folio);
971 folio_put(folio);
972 folio = NULL;
973 }
974
975 if (!same_folio)
976 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
977 if (folio) {
978 folio_mark_dirty(folio);
979 if (!truncate_inode_partial_folio(folio, lstart, lend))
980 end = folio->index;
981 folio_unlock(folio);
982 folio_put(folio);
983 }
984
985 whole_folios:
986
987 index = start;
988 while (index < end) {
989 cond_resched();
990
991 if (!find_get_entries(mapping, index, end - 1, &fbatch,
992 indices)) {
993 /* If all gone or hole-punch or unfalloc, we're done */
994 if (index == start || end != -1)
995 break;
996 /* But if truncating, restart to make sure all gone */
997 index = start;
998 continue;
999 }
1000 for (i = 0; i < folio_batch_count(&fbatch); i++) {
1001 folio = fbatch.folios[i];
1002
1003 index = indices[i];
1004 if (xa_is_value(folio)) {
1005 if (unfalloc)
1006 continue;
1007 if (shmem_free_swap(mapping, index, folio)) {
1008 /* Swap was replaced by page: retry */
1009 index--;
1010 break;
1011 }
1012 nr_swaps_freed++;
1013 continue;
1014 }
1015
1016 folio_lock(folio);
1017
1018 if (!unfalloc || !folio_test_uptodate(folio)) {
1019 if (folio_mapping(folio) != mapping) {
1020 /* Page was replaced by swap: retry */
1021 folio_unlock(folio);
1022 index--;
1023 break;
1024 }
1025 VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1026 folio);
1027 truncate_inode_folio(mapping, folio);
1028 }
1029 index = folio->index + folio_nr_pages(folio) - 1;
1030 folio_unlock(folio);
1031 }
1032 folio_batch_remove_exceptionals(&fbatch);
1033 folio_batch_release(&fbatch);
1034 index++;
1035 }
1036
1037 spin_lock_irq(&info->lock);
1038 info->swapped -= nr_swaps_freed;
1039 shmem_recalc_inode(inode);
1040 spin_unlock_irq(&info->lock);
1041 }
1042
shmem_truncate_range(struct inode * inode,loff_t lstart,loff_t lend)1043 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1044 {
1045 shmem_undo_range(inode, lstart, lend, false);
1046 inode->i_ctime = inode->i_mtime = current_time(inode);
1047 inode_inc_iversion(inode);
1048 }
1049 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1050
shmem_getattr(struct user_namespace * mnt_userns,const struct path * path,struct kstat * stat,u32 request_mask,unsigned int query_flags)1051 static int shmem_getattr(struct user_namespace *mnt_userns,
1052 const struct path *path, struct kstat *stat,
1053 u32 request_mask, unsigned int query_flags)
1054 {
1055 struct inode *inode = path->dentry->d_inode;
1056 struct shmem_inode_info *info = SHMEM_I(inode);
1057
1058 if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1059 spin_lock_irq(&info->lock);
1060 shmem_recalc_inode(inode);
1061 spin_unlock_irq(&info->lock);
1062 }
1063 if (info->fsflags & FS_APPEND_FL)
1064 stat->attributes |= STATX_ATTR_APPEND;
1065 if (info->fsflags & FS_IMMUTABLE_FL)
1066 stat->attributes |= STATX_ATTR_IMMUTABLE;
1067 if (info->fsflags & FS_NODUMP_FL)
1068 stat->attributes |= STATX_ATTR_NODUMP;
1069 stat->attributes_mask |= (STATX_ATTR_APPEND |
1070 STATX_ATTR_IMMUTABLE |
1071 STATX_ATTR_NODUMP);
1072 generic_fillattr(&init_user_ns, inode, stat);
1073
1074 if (shmem_is_huge(NULL, inode, 0, false))
1075 stat->blksize = HPAGE_PMD_SIZE;
1076
1077 if (request_mask & STATX_BTIME) {
1078 stat->result_mask |= STATX_BTIME;
1079 stat->btime.tv_sec = info->i_crtime.tv_sec;
1080 stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1081 }
1082
1083 return 0;
1084 }
1085
shmem_setattr(struct user_namespace * mnt_userns,struct dentry * dentry,struct iattr * attr)1086 static int shmem_setattr(struct user_namespace *mnt_userns,
1087 struct dentry *dentry, struct iattr *attr)
1088 {
1089 struct inode *inode = d_inode(dentry);
1090 struct shmem_inode_info *info = SHMEM_I(inode);
1091 int error;
1092 bool update_mtime = false;
1093 bool update_ctime = true;
1094
1095 error = setattr_prepare(&init_user_ns, dentry, attr);
1096 if (error)
1097 return error;
1098
1099 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1100 loff_t oldsize = inode->i_size;
1101 loff_t newsize = attr->ia_size;
1102
1103 /* protected by i_rwsem */
1104 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1105 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1106 return -EPERM;
1107
1108 if (newsize != oldsize) {
1109 error = shmem_reacct_size(SHMEM_I(inode)->flags,
1110 oldsize, newsize);
1111 if (error)
1112 return error;
1113 i_size_write(inode, newsize);
1114 update_mtime = true;
1115 } else {
1116 update_ctime = false;
1117 }
1118 if (newsize <= oldsize) {
1119 loff_t holebegin = round_up(newsize, PAGE_SIZE);
1120 if (oldsize > holebegin)
1121 unmap_mapping_range(inode->i_mapping,
1122 holebegin, 0, 1);
1123 if (info->alloced)
1124 shmem_truncate_range(inode,
1125 newsize, (loff_t)-1);
1126 /* unmap again to remove racily COWed private pages */
1127 if (oldsize > holebegin)
1128 unmap_mapping_range(inode->i_mapping,
1129 holebegin, 0, 1);
1130 }
1131 }
1132
1133 setattr_copy(&init_user_ns, inode, attr);
1134 if (attr->ia_valid & ATTR_MODE)
1135 error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode);
1136 if (!error && update_ctime) {
1137 inode->i_ctime = current_time(inode);
1138 if (update_mtime)
1139 inode->i_mtime = inode->i_ctime;
1140 inode_inc_iversion(inode);
1141 }
1142 return error;
1143 }
1144
shmem_evict_inode(struct inode * inode)1145 static void shmem_evict_inode(struct inode *inode)
1146 {
1147 struct shmem_inode_info *info = SHMEM_I(inode);
1148 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1149
1150 if (shmem_mapping(inode->i_mapping)) {
1151 shmem_unacct_size(info->flags, inode->i_size);
1152 inode->i_size = 0;
1153 mapping_set_exiting(inode->i_mapping);
1154 shmem_truncate_range(inode, 0, (loff_t)-1);
1155 if (!list_empty(&info->shrinklist)) {
1156 spin_lock(&sbinfo->shrinklist_lock);
1157 if (!list_empty(&info->shrinklist)) {
1158 list_del_init(&info->shrinklist);
1159 sbinfo->shrinklist_len--;
1160 }
1161 spin_unlock(&sbinfo->shrinklist_lock);
1162 }
1163 while (!list_empty(&info->swaplist)) {
1164 /* Wait while shmem_unuse() is scanning this inode... */
1165 wait_var_event(&info->stop_eviction,
1166 !atomic_read(&info->stop_eviction));
1167 mutex_lock(&shmem_swaplist_mutex);
1168 /* ...but beware of the race if we peeked too early */
1169 if (!atomic_read(&info->stop_eviction))
1170 list_del_init(&info->swaplist);
1171 mutex_unlock(&shmem_swaplist_mutex);
1172 }
1173 }
1174
1175 simple_xattrs_free(&info->xattrs);
1176 WARN_ON(inode->i_blocks);
1177 shmem_free_inode(inode->i_sb);
1178 clear_inode(inode);
1179 }
1180
shmem_find_swap_entries(struct address_space * mapping,pgoff_t start,struct folio_batch * fbatch,pgoff_t * indices,unsigned int type)1181 static int shmem_find_swap_entries(struct address_space *mapping,
1182 pgoff_t start, struct folio_batch *fbatch,
1183 pgoff_t *indices, unsigned int type)
1184 {
1185 XA_STATE(xas, &mapping->i_pages, start);
1186 struct folio *folio;
1187 swp_entry_t entry;
1188
1189 rcu_read_lock();
1190 xas_for_each(&xas, folio, ULONG_MAX) {
1191 if (xas_retry(&xas, folio))
1192 continue;
1193
1194 if (!xa_is_value(folio))
1195 continue;
1196
1197 entry = radix_to_swp_entry(folio);
1198 /*
1199 * swapin error entries can be found in the mapping. But they're
1200 * deliberately ignored here as we've done everything we can do.
1201 */
1202 if (swp_type(entry) != type)
1203 continue;
1204
1205 indices[folio_batch_count(fbatch)] = xas.xa_index;
1206 if (!folio_batch_add(fbatch, folio))
1207 break;
1208
1209 if (need_resched()) {
1210 xas_pause(&xas);
1211 cond_resched_rcu();
1212 }
1213 }
1214 rcu_read_unlock();
1215
1216 return xas.xa_index;
1217 }
1218
1219 /*
1220 * Move the swapped pages for an inode to page cache. Returns the count
1221 * of pages swapped in, or the error in case of failure.
1222 */
shmem_unuse_swap_entries(struct inode * inode,struct folio_batch * fbatch,pgoff_t * indices)1223 static int shmem_unuse_swap_entries(struct inode *inode,
1224 struct folio_batch *fbatch, pgoff_t *indices)
1225 {
1226 int i = 0;
1227 int ret = 0;
1228 int error = 0;
1229 struct address_space *mapping = inode->i_mapping;
1230
1231 for (i = 0; i < folio_batch_count(fbatch); i++) {
1232 struct folio *folio = fbatch->folios[i];
1233
1234 if (!xa_is_value(folio))
1235 continue;
1236 error = shmem_swapin_folio(inode, indices[i],
1237 &folio, SGP_CACHE,
1238 mapping_gfp_mask(mapping),
1239 NULL, NULL);
1240 if (error == 0) {
1241 folio_unlock(folio);
1242 folio_put(folio);
1243 ret++;
1244 }
1245 if (error == -ENOMEM)
1246 break;
1247 error = 0;
1248 }
1249 return error ? error : ret;
1250 }
1251
1252 /*
1253 * If swap found in inode, free it and move page from swapcache to filecache.
1254 */
shmem_unuse_inode(struct inode * inode,unsigned int type)1255 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1256 {
1257 struct address_space *mapping = inode->i_mapping;
1258 pgoff_t start = 0;
1259 struct folio_batch fbatch;
1260 pgoff_t indices[PAGEVEC_SIZE];
1261 int ret = 0;
1262
1263 do {
1264 folio_batch_init(&fbatch);
1265 shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1266 if (folio_batch_count(&fbatch) == 0) {
1267 ret = 0;
1268 break;
1269 }
1270
1271 ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1272 if (ret < 0)
1273 break;
1274
1275 start = indices[folio_batch_count(&fbatch) - 1];
1276 } while (true);
1277
1278 return ret;
1279 }
1280
1281 /*
1282 * Read all the shared memory data that resides in the swap
1283 * device 'type' back into memory, so the swap device can be
1284 * unused.
1285 */
shmem_unuse(unsigned int type)1286 int shmem_unuse(unsigned int type)
1287 {
1288 struct shmem_inode_info *info, *next;
1289 int error = 0;
1290
1291 if (list_empty(&shmem_swaplist))
1292 return 0;
1293
1294 mutex_lock(&shmem_swaplist_mutex);
1295 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1296 if (!info->swapped) {
1297 list_del_init(&info->swaplist);
1298 continue;
1299 }
1300 /*
1301 * Drop the swaplist mutex while searching the inode for swap;
1302 * but before doing so, make sure shmem_evict_inode() will not
1303 * remove placeholder inode from swaplist, nor let it be freed
1304 * (igrab() would protect from unlink, but not from unmount).
1305 */
1306 atomic_inc(&info->stop_eviction);
1307 mutex_unlock(&shmem_swaplist_mutex);
1308
1309 error = shmem_unuse_inode(&info->vfs_inode, type);
1310 cond_resched();
1311
1312 mutex_lock(&shmem_swaplist_mutex);
1313 next = list_next_entry(info, swaplist);
1314 if (!info->swapped)
1315 list_del_init(&info->swaplist);
1316 if (atomic_dec_and_test(&info->stop_eviction))
1317 wake_up_var(&info->stop_eviction);
1318 if (error)
1319 break;
1320 }
1321 mutex_unlock(&shmem_swaplist_mutex);
1322
1323 return error;
1324 }
1325
1326 /*
1327 * Move the page from the page cache to the swap cache.
1328 */
shmem_writepage(struct page * page,struct writeback_control * wbc)1329 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1330 {
1331 struct folio *folio = page_folio(page);
1332 struct shmem_inode_info *info;
1333 struct address_space *mapping;
1334 struct inode *inode;
1335 swp_entry_t swap;
1336 pgoff_t index;
1337
1338 /*
1339 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1340 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1341 * and its shmem_writeback() needs them to be split when swapping.
1342 */
1343 if (folio_test_large(folio)) {
1344 /* Ensure the subpages are still dirty */
1345 folio_test_set_dirty(folio);
1346 if (split_huge_page(page) < 0)
1347 goto redirty;
1348 folio = page_folio(page);
1349 folio_clear_dirty(folio);
1350 }
1351
1352 BUG_ON(!folio_test_locked(folio));
1353 mapping = folio->mapping;
1354 index = folio->index;
1355 inode = mapping->host;
1356 info = SHMEM_I(inode);
1357 if (info->flags & VM_LOCKED)
1358 goto redirty;
1359 if (!total_swap_pages)
1360 goto redirty;
1361
1362 /*
1363 * Our capabilities prevent regular writeback or sync from ever calling
1364 * shmem_writepage; but a stacking filesystem might use ->writepage of
1365 * its underlying filesystem, in which case tmpfs should write out to
1366 * swap only in response to memory pressure, and not for the writeback
1367 * threads or sync.
1368 */
1369 if (!wbc->for_reclaim) {
1370 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */
1371 goto redirty;
1372 }
1373
1374 /*
1375 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1376 * value into swapfile.c, the only way we can correctly account for a
1377 * fallocated folio arriving here is now to initialize it and write it.
1378 *
1379 * That's okay for a folio already fallocated earlier, but if we have
1380 * not yet completed the fallocation, then (a) we want to keep track
1381 * of this folio in case we have to undo it, and (b) it may not be a
1382 * good idea to continue anyway, once we're pushing into swap. So
1383 * reactivate the folio, and let shmem_fallocate() quit when too many.
1384 */
1385 if (!folio_test_uptodate(folio)) {
1386 if (inode->i_private) {
1387 struct shmem_falloc *shmem_falloc;
1388 spin_lock(&inode->i_lock);
1389 shmem_falloc = inode->i_private;
1390 if (shmem_falloc &&
1391 !shmem_falloc->waitq &&
1392 index >= shmem_falloc->start &&
1393 index < shmem_falloc->next)
1394 shmem_falloc->nr_unswapped++;
1395 else
1396 shmem_falloc = NULL;
1397 spin_unlock(&inode->i_lock);
1398 if (shmem_falloc)
1399 goto redirty;
1400 }
1401 folio_zero_range(folio, 0, folio_size(folio));
1402 flush_dcache_folio(folio);
1403 folio_mark_uptodate(folio);
1404 }
1405
1406 swap = folio_alloc_swap(folio);
1407 if (!swap.val)
1408 goto redirty;
1409
1410 /*
1411 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1412 * if it's not already there. Do it now before the folio is
1413 * moved to swap cache, when its pagelock no longer protects
1414 * the inode from eviction. But don't unlock the mutex until
1415 * we've incremented swapped, because shmem_unuse_inode() will
1416 * prune a !swapped inode from the swaplist under this mutex.
1417 */
1418 mutex_lock(&shmem_swaplist_mutex);
1419 if (list_empty(&info->swaplist))
1420 list_add(&info->swaplist, &shmem_swaplist);
1421
1422 if (add_to_swap_cache(folio, swap,
1423 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1424 NULL) == 0) {
1425 spin_lock_irq(&info->lock);
1426 shmem_recalc_inode(inode);
1427 info->swapped++;
1428 spin_unlock_irq(&info->lock);
1429
1430 swap_shmem_alloc(swap);
1431 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
1432
1433 mutex_unlock(&shmem_swaplist_mutex);
1434 BUG_ON(folio_mapped(folio));
1435 swap_writepage(&folio->page, wbc);
1436 return 0;
1437 }
1438
1439 mutex_unlock(&shmem_swaplist_mutex);
1440 put_swap_folio(folio, swap);
1441 redirty:
1442 folio_mark_dirty(folio);
1443 if (wbc->for_reclaim)
1444 return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */
1445 folio_unlock(folio);
1446 return 0;
1447 }
1448
1449 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
shmem_show_mpol(struct seq_file * seq,struct mempolicy * mpol)1450 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1451 {
1452 char buffer[64];
1453
1454 if (!mpol || mpol->mode == MPOL_DEFAULT)
1455 return; /* show nothing */
1456
1457 mpol_to_str(buffer, sizeof(buffer), mpol);
1458
1459 seq_printf(seq, ",mpol=%s", buffer);
1460 }
1461
shmem_get_sbmpol(struct shmem_sb_info * sbinfo)1462 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1463 {
1464 struct mempolicy *mpol = NULL;
1465 if (sbinfo->mpol) {
1466 raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
1467 mpol = sbinfo->mpol;
1468 mpol_get(mpol);
1469 raw_spin_unlock(&sbinfo->stat_lock);
1470 }
1471 return mpol;
1472 }
1473 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
shmem_show_mpol(struct seq_file * seq,struct mempolicy * mpol)1474 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1475 {
1476 }
shmem_get_sbmpol(struct shmem_sb_info * sbinfo)1477 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1478 {
1479 return NULL;
1480 }
1481 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1482 #ifndef CONFIG_NUMA
1483 #define vm_policy vm_private_data
1484 #endif
1485
shmem_pseudo_vma_init(struct vm_area_struct * vma,struct shmem_inode_info * info,pgoff_t index)1486 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1487 struct shmem_inode_info *info, pgoff_t index)
1488 {
1489 /* Create a pseudo vma that just contains the policy */
1490 vma_init(vma, NULL);
1491 /* Bias interleave by inode number to distribute better across nodes */
1492 vma->vm_pgoff = index + info->vfs_inode.i_ino;
1493 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1494 }
1495
shmem_pseudo_vma_destroy(struct vm_area_struct * vma)1496 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1497 {
1498 /* Drop reference taken by mpol_shared_policy_lookup() */
1499 mpol_cond_put(vma->vm_policy);
1500 }
1501
shmem_swapin(swp_entry_t swap,gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1502 static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1503 struct shmem_inode_info *info, pgoff_t index)
1504 {
1505 struct vm_area_struct pvma;
1506 struct page *page;
1507 struct vm_fault vmf = {
1508 .vma = &pvma,
1509 };
1510
1511 shmem_pseudo_vma_init(&pvma, info, index);
1512 page = swap_cluster_readahead(swap, gfp, &vmf);
1513 shmem_pseudo_vma_destroy(&pvma);
1514
1515 if (!page)
1516 return NULL;
1517 return page_folio(page);
1518 }
1519
1520 /*
1521 * Make sure huge_gfp is always more limited than limit_gfp.
1522 * Some of the flags set permissions, while others set limitations.
1523 */
limit_gfp_mask(gfp_t huge_gfp,gfp_t limit_gfp)1524 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1525 {
1526 gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1527 gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1528 gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1529 gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1530
1531 /* Allow allocations only from the originally specified zones. */
1532 result |= zoneflags;
1533
1534 /*
1535 * Minimize the result gfp by taking the union with the deny flags,
1536 * and the intersection of the allow flags.
1537 */
1538 result |= (limit_gfp & denyflags);
1539 result |= (huge_gfp & limit_gfp) & allowflags;
1540
1541 return result;
1542 }
1543
shmem_alloc_hugefolio(gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1544 static struct folio *shmem_alloc_hugefolio(gfp_t gfp,
1545 struct shmem_inode_info *info, pgoff_t index)
1546 {
1547 struct vm_area_struct pvma;
1548 struct address_space *mapping = info->vfs_inode.i_mapping;
1549 pgoff_t hindex;
1550 struct folio *folio;
1551
1552 hindex = round_down(index, HPAGE_PMD_NR);
1553 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1554 XA_PRESENT))
1555 return NULL;
1556
1557 shmem_pseudo_vma_init(&pvma, info, hindex);
1558 folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1559 shmem_pseudo_vma_destroy(&pvma);
1560 if (!folio)
1561 count_vm_event(THP_FILE_FALLBACK);
1562 return folio;
1563 }
1564
shmem_alloc_folio(gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1565 static struct folio *shmem_alloc_folio(gfp_t gfp,
1566 struct shmem_inode_info *info, pgoff_t index)
1567 {
1568 struct vm_area_struct pvma;
1569 struct folio *folio;
1570
1571 shmem_pseudo_vma_init(&pvma, info, index);
1572 folio = vma_alloc_folio(gfp, 0, &pvma, 0, false);
1573 shmem_pseudo_vma_destroy(&pvma);
1574
1575 return folio;
1576 }
1577
shmem_alloc_and_acct_folio(gfp_t gfp,struct inode * inode,pgoff_t index,bool huge)1578 static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode,
1579 pgoff_t index, bool huge)
1580 {
1581 struct shmem_inode_info *info = SHMEM_I(inode);
1582 struct folio *folio;
1583 int nr;
1584 int err = -ENOSPC;
1585
1586 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1587 huge = false;
1588 nr = huge ? HPAGE_PMD_NR : 1;
1589
1590 if (!shmem_inode_acct_block(inode, nr))
1591 goto failed;
1592
1593 if (huge)
1594 folio = shmem_alloc_hugefolio(gfp, info, index);
1595 else
1596 folio = shmem_alloc_folio(gfp, info, index);
1597 if (folio) {
1598 __folio_set_locked(folio);
1599 __folio_set_swapbacked(folio);
1600 return folio;
1601 }
1602
1603 err = -ENOMEM;
1604 shmem_inode_unacct_blocks(inode, nr);
1605 failed:
1606 return ERR_PTR(err);
1607 }
1608
1609 /*
1610 * When a page is moved from swapcache to shmem filecache (either by the
1611 * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
1612 * shmem_unuse_inode()), it may have been read in earlier from swap, in
1613 * ignorance of the mapping it belongs to. If that mapping has special
1614 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1615 * we may need to copy to a suitable page before moving to filecache.
1616 *
1617 * In a future release, this may well be extended to respect cpuset and
1618 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1619 * but for now it is a simple matter of zone.
1620 */
shmem_should_replace_folio(struct folio * folio,gfp_t gfp)1621 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1622 {
1623 return folio_zonenum(folio) > gfp_zone(gfp);
1624 }
1625
shmem_replace_folio(struct folio ** foliop,gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1626 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
1627 struct shmem_inode_info *info, pgoff_t index)
1628 {
1629 struct folio *old, *new;
1630 struct address_space *swap_mapping;
1631 swp_entry_t entry;
1632 pgoff_t swap_index;
1633 int error;
1634
1635 old = *foliop;
1636 entry = folio_swap_entry(old);
1637 swap_index = swp_offset(entry);
1638 swap_mapping = swap_address_space(entry);
1639
1640 /*
1641 * We have arrived here because our zones are constrained, so don't
1642 * limit chance of success by further cpuset and node constraints.
1643 */
1644 gfp &= ~GFP_CONSTRAINT_MASK;
1645 VM_BUG_ON_FOLIO(folio_test_large(old), old);
1646 new = shmem_alloc_folio(gfp, info, index);
1647 if (!new)
1648 return -ENOMEM;
1649
1650 folio_get(new);
1651 folio_copy(new, old);
1652 flush_dcache_folio(new);
1653
1654 __folio_set_locked(new);
1655 __folio_set_swapbacked(new);
1656 folio_mark_uptodate(new);
1657 folio_set_swap_entry(new, entry);
1658 folio_set_swapcache(new);
1659
1660 /*
1661 * Our caller will very soon move newpage out of swapcache, but it's
1662 * a nice clean interface for us to replace oldpage by newpage there.
1663 */
1664 xa_lock_irq(&swap_mapping->i_pages);
1665 error = shmem_replace_entry(swap_mapping, swap_index, old, new);
1666 if (!error) {
1667 mem_cgroup_migrate(old, new);
1668 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
1669 __lruvec_stat_mod_folio(new, NR_SHMEM, 1);
1670 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
1671 __lruvec_stat_mod_folio(old, NR_SHMEM, -1);
1672 }
1673 xa_unlock_irq(&swap_mapping->i_pages);
1674
1675 if (unlikely(error)) {
1676 /*
1677 * Is this possible? I think not, now that our callers check
1678 * both PageSwapCache and page_private after getting page lock;
1679 * but be defensive. Reverse old to newpage for clear and free.
1680 */
1681 old = new;
1682 } else {
1683 folio_add_lru(new);
1684 *foliop = new;
1685 }
1686
1687 folio_clear_swapcache(old);
1688 old->private = NULL;
1689
1690 folio_unlock(old);
1691 folio_put_refs(old, 2);
1692 return error;
1693 }
1694
shmem_set_folio_swapin_error(struct inode * inode,pgoff_t index,struct folio * folio,swp_entry_t swap)1695 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
1696 struct folio *folio, swp_entry_t swap)
1697 {
1698 struct address_space *mapping = inode->i_mapping;
1699 struct shmem_inode_info *info = SHMEM_I(inode);
1700 swp_entry_t swapin_error;
1701 void *old;
1702
1703 swapin_error = make_swapin_error_entry(&folio->page);
1704 old = xa_cmpxchg_irq(&mapping->i_pages, index,
1705 swp_to_radix_entry(swap),
1706 swp_to_radix_entry(swapin_error), 0);
1707 if (old != swp_to_radix_entry(swap))
1708 return;
1709
1710 folio_wait_writeback(folio);
1711 delete_from_swap_cache(folio);
1712 spin_lock_irq(&info->lock);
1713 /*
1714 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks won't
1715 * be 0 when inode is released and thus trigger WARN_ON(inode->i_blocks) in
1716 * shmem_evict_inode.
1717 */
1718 info->alloced--;
1719 info->swapped--;
1720 shmem_recalc_inode(inode);
1721 spin_unlock_irq(&info->lock);
1722 swap_free(swap);
1723 }
1724
1725 /*
1726 * Swap in the folio pointed to by *foliop.
1727 * Caller has to make sure that *foliop contains a valid swapped folio.
1728 * Returns 0 and the folio in foliop if success. On failure, returns the
1729 * error code and NULL in *foliop.
1730 */
shmem_swapin_folio(struct inode * inode,pgoff_t index,struct folio ** foliop,enum sgp_type sgp,gfp_t gfp,struct vm_area_struct * vma,vm_fault_t * fault_type)1731 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1732 struct folio **foliop, enum sgp_type sgp,
1733 gfp_t gfp, struct vm_area_struct *vma,
1734 vm_fault_t *fault_type)
1735 {
1736 struct address_space *mapping = inode->i_mapping;
1737 struct shmem_inode_info *info = SHMEM_I(inode);
1738 struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1739 struct folio *folio = NULL;
1740 swp_entry_t swap;
1741 int error;
1742
1743 VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1744 swap = radix_to_swp_entry(*foliop);
1745 *foliop = NULL;
1746
1747 if (is_swapin_error_entry(swap))
1748 return -EIO;
1749
1750 /* Look it up and read it in.. */
1751 folio = swap_cache_get_folio(swap, NULL, 0);
1752 if (!folio) {
1753 /* Or update major stats only when swapin succeeds?? */
1754 if (fault_type) {
1755 *fault_type |= VM_FAULT_MAJOR;
1756 count_vm_event(PGMAJFAULT);
1757 count_memcg_event_mm(charge_mm, PGMAJFAULT);
1758 }
1759 /* Here we actually start the io */
1760 folio = shmem_swapin(swap, gfp, info, index);
1761 if (!folio) {
1762 error = -ENOMEM;
1763 goto failed;
1764 }
1765 }
1766
1767 /* We have to do this with folio locked to prevent races */
1768 folio_lock(folio);
1769 if (!folio_test_swapcache(folio) ||
1770 folio_swap_entry(folio).val != swap.val ||
1771 !shmem_confirm_swap(mapping, index, swap)) {
1772 error = -EEXIST;
1773 goto unlock;
1774 }
1775 if (!folio_test_uptodate(folio)) {
1776 error = -EIO;
1777 goto failed;
1778 }
1779 folio_wait_writeback(folio);
1780
1781 /*
1782 * Some architectures may have to restore extra metadata to the
1783 * folio after reading from swap.
1784 */
1785 arch_swap_restore(swap, folio);
1786
1787 if (shmem_should_replace_folio(folio, gfp)) {
1788 error = shmem_replace_folio(&folio, gfp, info, index);
1789 if (error)
1790 goto failed;
1791 }
1792
1793 error = shmem_add_to_page_cache(folio, mapping, index,
1794 swp_to_radix_entry(swap), gfp,
1795 charge_mm);
1796 if (error)
1797 goto failed;
1798
1799 spin_lock_irq(&info->lock);
1800 info->swapped--;
1801 shmem_recalc_inode(inode);
1802 spin_unlock_irq(&info->lock);
1803
1804 if (sgp == SGP_WRITE)
1805 folio_mark_accessed(folio);
1806
1807 delete_from_swap_cache(folio);
1808 folio_mark_dirty(folio);
1809 swap_free(swap);
1810
1811 *foliop = folio;
1812 return 0;
1813 failed:
1814 if (!shmem_confirm_swap(mapping, index, swap))
1815 error = -EEXIST;
1816 if (error == -EIO)
1817 shmem_set_folio_swapin_error(inode, index, folio, swap);
1818 unlock:
1819 if (folio) {
1820 folio_unlock(folio);
1821 folio_put(folio);
1822 }
1823
1824 return error;
1825 }
1826
1827 /*
1828 * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
1829 *
1830 * If we allocate a new one we do not mark it dirty. That's up to the
1831 * vm. If we swap it in we mark it dirty since we also free the swap
1832 * entry since a page cannot live in both the swap and page cache.
1833 *
1834 * vma, vmf, and fault_type are only supplied by shmem_fault:
1835 * otherwise they are NULL.
1836 */
shmem_get_folio_gfp(struct inode * inode,pgoff_t index,struct folio ** foliop,enum sgp_type sgp,gfp_t gfp,struct vm_area_struct * vma,struct vm_fault * vmf,vm_fault_t * fault_type)1837 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
1838 struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
1839 struct vm_area_struct *vma, struct vm_fault *vmf,
1840 vm_fault_t *fault_type)
1841 {
1842 struct address_space *mapping = inode->i_mapping;
1843 struct shmem_inode_info *info = SHMEM_I(inode);
1844 struct shmem_sb_info *sbinfo;
1845 struct mm_struct *charge_mm;
1846 struct folio *folio;
1847 pgoff_t hindex = index;
1848 gfp_t huge_gfp;
1849 int error;
1850 int once = 0;
1851 int alloced = 0;
1852
1853 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1854 return -EFBIG;
1855 repeat:
1856 if (sgp <= SGP_CACHE &&
1857 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1858 return -EINVAL;
1859 }
1860
1861 sbinfo = SHMEM_SB(inode->i_sb);
1862 charge_mm = vma ? vma->vm_mm : NULL;
1863
1864 folio = __filemap_get_folio(mapping, index, FGP_ENTRY | FGP_LOCK, 0);
1865 if (folio && vma && userfaultfd_minor(vma)) {
1866 if (!xa_is_value(folio)) {
1867 folio_unlock(folio);
1868 folio_put(folio);
1869 }
1870 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1871 return 0;
1872 }
1873
1874 if (xa_is_value(folio)) {
1875 error = shmem_swapin_folio(inode, index, &folio,
1876 sgp, gfp, vma, fault_type);
1877 if (error == -EEXIST)
1878 goto repeat;
1879
1880 *foliop = folio;
1881 return error;
1882 }
1883
1884 if (folio) {
1885 hindex = folio->index;
1886 if (sgp == SGP_WRITE)
1887 folio_mark_accessed(folio);
1888 if (folio_test_uptodate(folio))
1889 goto out;
1890 /* fallocated folio */
1891 if (sgp != SGP_READ)
1892 goto clear;
1893 folio_unlock(folio);
1894 folio_put(folio);
1895 }
1896
1897 /*
1898 * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
1899 * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
1900 */
1901 *foliop = NULL;
1902 if (sgp == SGP_READ)
1903 return 0;
1904 if (sgp == SGP_NOALLOC)
1905 return -ENOENT;
1906
1907 /*
1908 * Fast cache lookup and swap lookup did not find it: allocate.
1909 */
1910
1911 if (vma && userfaultfd_missing(vma)) {
1912 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1913 return 0;
1914 }
1915
1916 if (!shmem_is_huge(vma, inode, index, false))
1917 goto alloc_nohuge;
1918
1919 huge_gfp = vma_thp_gfp_mask(vma);
1920 huge_gfp = limit_gfp_mask(huge_gfp, gfp);
1921 folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true);
1922 if (IS_ERR(folio)) {
1923 alloc_nohuge:
1924 folio = shmem_alloc_and_acct_folio(gfp, inode, index, false);
1925 }
1926 if (IS_ERR(folio)) {
1927 int retry = 5;
1928
1929 error = PTR_ERR(folio);
1930 folio = NULL;
1931 if (error != -ENOSPC)
1932 goto unlock;
1933 /*
1934 * Try to reclaim some space by splitting a large folio
1935 * beyond i_size on the filesystem.
1936 */
1937 while (retry--) {
1938 int ret;
1939
1940 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1941 if (ret == SHRINK_STOP)
1942 break;
1943 if (ret)
1944 goto alloc_nohuge;
1945 }
1946 goto unlock;
1947 }
1948
1949 hindex = round_down(index, folio_nr_pages(folio));
1950
1951 if (sgp == SGP_WRITE)
1952 __folio_set_referenced(folio);
1953
1954 error = shmem_add_to_page_cache(folio, mapping, hindex,
1955 NULL, gfp & GFP_RECLAIM_MASK,
1956 charge_mm);
1957 if (error)
1958 goto unacct;
1959 folio_add_lru(folio);
1960
1961 spin_lock_irq(&info->lock);
1962 info->alloced += folio_nr_pages(folio);
1963 inode->i_blocks += (blkcnt_t)BLOCKS_PER_PAGE << folio_order(folio);
1964 shmem_recalc_inode(inode);
1965 spin_unlock_irq(&info->lock);
1966 alloced = true;
1967
1968 if (folio_test_pmd_mappable(folio) &&
1969 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1970 folio_next_index(folio) - 1) {
1971 /*
1972 * Part of the large folio is beyond i_size: subject
1973 * to shrink under memory pressure.
1974 */
1975 spin_lock(&sbinfo->shrinklist_lock);
1976 /*
1977 * _careful to defend against unlocked access to
1978 * ->shrink_list in shmem_unused_huge_shrink()
1979 */
1980 if (list_empty_careful(&info->shrinklist)) {
1981 list_add_tail(&info->shrinklist,
1982 &sbinfo->shrinklist);
1983 sbinfo->shrinklist_len++;
1984 }
1985 spin_unlock(&sbinfo->shrinklist_lock);
1986 }
1987
1988 /*
1989 * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
1990 */
1991 if (sgp == SGP_FALLOC)
1992 sgp = SGP_WRITE;
1993 clear:
1994 /*
1995 * Let SGP_WRITE caller clear ends if write does not fill folio;
1996 * but SGP_FALLOC on a folio fallocated earlier must initialize
1997 * it now, lest undo on failure cancel our earlier guarantee.
1998 */
1999 if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
2000 long i, n = folio_nr_pages(folio);
2001
2002 for (i = 0; i < n; i++)
2003 clear_highpage(folio_page(folio, i));
2004 flush_dcache_folio(folio);
2005 folio_mark_uptodate(folio);
2006 }
2007
2008 /* Perhaps the file has been truncated since we checked */
2009 if (sgp <= SGP_CACHE &&
2010 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
2011 if (alloced) {
2012 folio_clear_dirty(folio);
2013 filemap_remove_folio(folio);
2014 spin_lock_irq(&info->lock);
2015 shmem_recalc_inode(inode);
2016 spin_unlock_irq(&info->lock);
2017 }
2018 error = -EINVAL;
2019 goto unlock;
2020 }
2021 out:
2022 *foliop = folio;
2023 return 0;
2024
2025 /*
2026 * Error recovery.
2027 */
2028 unacct:
2029 shmem_inode_unacct_blocks(inode, folio_nr_pages(folio));
2030
2031 if (folio_test_large(folio)) {
2032 folio_unlock(folio);
2033 folio_put(folio);
2034 goto alloc_nohuge;
2035 }
2036 unlock:
2037 if (folio) {
2038 folio_unlock(folio);
2039 folio_put(folio);
2040 }
2041 if (error == -ENOSPC && !once++) {
2042 spin_lock_irq(&info->lock);
2043 shmem_recalc_inode(inode);
2044 spin_unlock_irq(&info->lock);
2045 goto repeat;
2046 }
2047 if (error == -EEXIST)
2048 goto repeat;
2049 return error;
2050 }
2051
shmem_get_folio(struct inode * inode,pgoff_t index,struct folio ** foliop,enum sgp_type sgp)2052 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
2053 enum sgp_type sgp)
2054 {
2055 return shmem_get_folio_gfp(inode, index, foliop, sgp,
2056 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
2057 }
2058
2059 /*
2060 * This is like autoremove_wake_function, but it removes the wait queue
2061 * entry unconditionally - even if something else had already woken the
2062 * target.
2063 */
synchronous_wake_function(wait_queue_entry_t * wait,unsigned mode,int sync,void * key)2064 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2065 {
2066 int ret = default_wake_function(wait, mode, sync, key);
2067 list_del_init(&wait->entry);
2068 return ret;
2069 }
2070
shmem_fault(struct vm_fault * vmf)2071 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2072 {
2073 struct vm_area_struct *vma = vmf->vma;
2074 struct inode *inode = file_inode(vma->vm_file);
2075 gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2076 struct folio *folio = NULL;
2077 int err;
2078 vm_fault_t ret = VM_FAULT_LOCKED;
2079
2080 /*
2081 * Trinity finds that probing a hole which tmpfs is punching can
2082 * prevent the hole-punch from ever completing: which in turn
2083 * locks writers out with its hold on i_rwsem. So refrain from
2084 * faulting pages into the hole while it's being punched. Although
2085 * shmem_undo_range() does remove the additions, it may be unable to
2086 * keep up, as each new page needs its own unmap_mapping_range() call,
2087 * and the i_mmap tree grows ever slower to scan if new vmas are added.
2088 *
2089 * It does not matter if we sometimes reach this check just before the
2090 * hole-punch begins, so that one fault then races with the punch:
2091 * we just need to make racing faults a rare case.
2092 *
2093 * The implementation below would be much simpler if we just used a
2094 * standard mutex or completion: but we cannot take i_rwsem in fault,
2095 * and bloating every shmem inode for this unlikely case would be sad.
2096 */
2097 if (unlikely(inode->i_private)) {
2098 struct shmem_falloc *shmem_falloc;
2099
2100 spin_lock(&inode->i_lock);
2101 shmem_falloc = inode->i_private;
2102 if (shmem_falloc &&
2103 shmem_falloc->waitq &&
2104 vmf->pgoff >= shmem_falloc->start &&
2105 vmf->pgoff < shmem_falloc->next) {
2106 struct file *fpin;
2107 wait_queue_head_t *shmem_falloc_waitq;
2108 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2109
2110 ret = VM_FAULT_NOPAGE;
2111 fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2112 if (fpin)
2113 ret = VM_FAULT_RETRY;
2114
2115 shmem_falloc_waitq = shmem_falloc->waitq;
2116 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2117 TASK_UNINTERRUPTIBLE);
2118 spin_unlock(&inode->i_lock);
2119 schedule();
2120
2121 /*
2122 * shmem_falloc_waitq points into the shmem_fallocate()
2123 * stack of the hole-punching task: shmem_falloc_waitq
2124 * is usually invalid by the time we reach here, but
2125 * finish_wait() does not dereference it in that case;
2126 * though i_lock needed lest racing with wake_up_all().
2127 */
2128 spin_lock(&inode->i_lock);
2129 finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2130 spin_unlock(&inode->i_lock);
2131
2132 if (fpin)
2133 fput(fpin);
2134 return ret;
2135 }
2136 spin_unlock(&inode->i_lock);
2137 }
2138
2139 err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
2140 gfp, vma, vmf, &ret);
2141 if (err)
2142 return vmf_error(err);
2143 if (folio)
2144 vmf->page = folio_file_page(folio, vmf->pgoff);
2145 return ret;
2146 }
2147
shmem_get_unmapped_area(struct file * file,unsigned long uaddr,unsigned long len,unsigned long pgoff,unsigned long flags)2148 unsigned long shmem_get_unmapped_area(struct file *file,
2149 unsigned long uaddr, unsigned long len,
2150 unsigned long pgoff, unsigned long flags)
2151 {
2152 unsigned long (*get_area)(struct file *,
2153 unsigned long, unsigned long, unsigned long, unsigned long);
2154 unsigned long addr;
2155 unsigned long offset;
2156 unsigned long inflated_len;
2157 unsigned long inflated_addr;
2158 unsigned long inflated_offset;
2159
2160 if (len > TASK_SIZE)
2161 return -ENOMEM;
2162
2163 get_area = current->mm->get_unmapped_area;
2164 addr = get_area(file, uaddr, len, pgoff, flags);
2165
2166 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2167 return addr;
2168 if (IS_ERR_VALUE(addr))
2169 return addr;
2170 if (addr & ~PAGE_MASK)
2171 return addr;
2172 if (addr > TASK_SIZE - len)
2173 return addr;
2174
2175 if (shmem_huge == SHMEM_HUGE_DENY)
2176 return addr;
2177 if (len < HPAGE_PMD_SIZE)
2178 return addr;
2179 if (flags & MAP_FIXED)
2180 return addr;
2181 /*
2182 * Our priority is to support MAP_SHARED mapped hugely;
2183 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2184 * But if caller specified an address hint and we allocated area there
2185 * successfully, respect that as before.
2186 */
2187 if (uaddr == addr)
2188 return addr;
2189
2190 if (shmem_huge != SHMEM_HUGE_FORCE) {
2191 struct super_block *sb;
2192
2193 if (file) {
2194 VM_BUG_ON(file->f_op != &shmem_file_operations);
2195 sb = file_inode(file)->i_sb;
2196 } else {
2197 /*
2198 * Called directly from mm/mmap.c, or drivers/char/mem.c
2199 * for "/dev/zero", to create a shared anonymous object.
2200 */
2201 if (IS_ERR(shm_mnt))
2202 return addr;
2203 sb = shm_mnt->mnt_sb;
2204 }
2205 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2206 return addr;
2207 }
2208
2209 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2210 if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2211 return addr;
2212 if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2213 return addr;
2214
2215 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2216 if (inflated_len > TASK_SIZE)
2217 return addr;
2218 if (inflated_len < len)
2219 return addr;
2220
2221 inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2222 if (IS_ERR_VALUE(inflated_addr))
2223 return addr;
2224 if (inflated_addr & ~PAGE_MASK)
2225 return addr;
2226
2227 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2228 inflated_addr += offset - inflated_offset;
2229 if (inflated_offset > offset)
2230 inflated_addr += HPAGE_PMD_SIZE;
2231
2232 if (inflated_addr > TASK_SIZE - len)
2233 return addr;
2234 return inflated_addr;
2235 }
2236
2237 #ifdef CONFIG_NUMA
shmem_set_policy(struct vm_area_struct * vma,struct mempolicy * mpol)2238 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2239 {
2240 struct inode *inode = file_inode(vma->vm_file);
2241 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2242 }
2243
shmem_get_policy(struct vm_area_struct * vma,unsigned long addr)2244 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2245 unsigned long addr)
2246 {
2247 struct inode *inode = file_inode(vma->vm_file);
2248 pgoff_t index;
2249
2250 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2251 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2252 }
2253 #endif
2254
shmem_lock(struct file * file,int lock,struct ucounts * ucounts)2255 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2256 {
2257 struct inode *inode = file_inode(file);
2258 struct shmem_inode_info *info = SHMEM_I(inode);
2259 int retval = -ENOMEM;
2260
2261 /*
2262 * What serializes the accesses to info->flags?
2263 * ipc_lock_object() when called from shmctl_do_lock(),
2264 * no serialization needed when called from shm_destroy().
2265 */
2266 if (lock && !(info->flags & VM_LOCKED)) {
2267 if (!user_shm_lock(inode->i_size, ucounts))
2268 goto out_nomem;
2269 info->flags |= VM_LOCKED;
2270 mapping_set_unevictable(file->f_mapping);
2271 }
2272 if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2273 user_shm_unlock(inode->i_size, ucounts);
2274 info->flags &= ~VM_LOCKED;
2275 mapping_clear_unevictable(file->f_mapping);
2276 }
2277 retval = 0;
2278
2279 out_nomem:
2280 return retval;
2281 }
2282
shmem_mmap(struct file * file,struct vm_area_struct * vma)2283 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2284 {
2285 struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2286 int ret;
2287
2288 ret = seal_check_future_write(info->seals, vma);
2289 if (ret)
2290 return ret;
2291
2292 /* arm64 - allow memory tagging on RAM-based files */
2293 vma->vm_flags |= VM_MTE_ALLOWED;
2294
2295 file_accessed(file);
2296 vma->vm_ops = &shmem_vm_ops;
2297 return 0;
2298 }
2299
2300 #ifdef CONFIG_TMPFS_XATTR
2301 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2302
2303 /*
2304 * chattr's fsflags are unrelated to extended attributes,
2305 * but tmpfs has chosen to enable them under the same config option.
2306 */
shmem_set_inode_flags(struct inode * inode,unsigned int fsflags)2307 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2308 {
2309 unsigned int i_flags = 0;
2310
2311 if (fsflags & FS_NOATIME_FL)
2312 i_flags |= S_NOATIME;
2313 if (fsflags & FS_APPEND_FL)
2314 i_flags |= S_APPEND;
2315 if (fsflags & FS_IMMUTABLE_FL)
2316 i_flags |= S_IMMUTABLE;
2317 /*
2318 * But FS_NODUMP_FL does not require any action in i_flags.
2319 */
2320 inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE);
2321 }
2322 #else
shmem_set_inode_flags(struct inode * inode,unsigned int fsflags)2323 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2324 {
2325 }
2326 #define shmem_initxattrs NULL
2327 #endif
2328
shmem_get_inode(struct super_block * sb,struct inode * dir,umode_t mode,dev_t dev,unsigned long flags)2329 static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir,
2330 umode_t mode, dev_t dev, unsigned long flags)
2331 {
2332 struct inode *inode;
2333 struct shmem_inode_info *info;
2334 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2335 ino_t ino;
2336
2337 if (shmem_reserve_inode(sb, &ino))
2338 return NULL;
2339
2340 inode = new_inode(sb);
2341 if (inode) {
2342 inode->i_ino = ino;
2343 inode_init_owner(&init_user_ns, inode, dir, mode);
2344 inode->i_blocks = 0;
2345 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2346 inode->i_generation = get_random_u32();
2347 info = SHMEM_I(inode);
2348 memset(info, 0, (char *)inode - (char *)info);
2349 spin_lock_init(&info->lock);
2350 atomic_set(&info->stop_eviction, 0);
2351 info->seals = F_SEAL_SEAL;
2352 info->flags = flags & VM_NORESERVE;
2353 info->i_crtime = inode->i_mtime;
2354 info->fsflags = (dir == NULL) ? 0 :
2355 SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
2356 if (info->fsflags)
2357 shmem_set_inode_flags(inode, info->fsflags);
2358 INIT_LIST_HEAD(&info->shrinklist);
2359 INIT_LIST_HEAD(&info->swaplist);
2360 simple_xattrs_init(&info->xattrs);
2361 cache_no_acl(inode);
2362 mapping_set_large_folios(inode->i_mapping);
2363
2364 switch (mode & S_IFMT) {
2365 default:
2366 inode->i_op = &shmem_special_inode_operations;
2367 init_special_inode(inode, mode, dev);
2368 break;
2369 case S_IFREG:
2370 inode->i_mapping->a_ops = &shmem_aops;
2371 inode->i_op = &shmem_inode_operations;
2372 inode->i_fop = &shmem_file_operations;
2373 mpol_shared_policy_init(&info->policy,
2374 shmem_get_sbmpol(sbinfo));
2375 break;
2376 case S_IFDIR:
2377 inc_nlink(inode);
2378 /* Some things misbehave if size == 0 on a directory */
2379 inode->i_size = 2 * BOGO_DIRENT_SIZE;
2380 inode->i_op = &shmem_dir_inode_operations;
2381 inode->i_fop = &simple_dir_operations;
2382 break;
2383 case S_IFLNK:
2384 /*
2385 * Must not load anything in the rbtree,
2386 * mpol_free_shared_policy will not be called.
2387 */
2388 mpol_shared_policy_init(&info->policy, NULL);
2389 break;
2390 }
2391
2392 lockdep_annotate_inode_mutex_key(inode);
2393 } else
2394 shmem_free_inode(sb);
2395 return inode;
2396 }
2397
2398 #ifdef CONFIG_USERFAULTFD
shmem_mfill_atomic_pte(struct mm_struct * dst_mm,pmd_t * dst_pmd,struct vm_area_struct * dst_vma,unsigned long dst_addr,unsigned long src_addr,bool zeropage,bool wp_copy,struct page ** pagep)2399 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2400 pmd_t *dst_pmd,
2401 struct vm_area_struct *dst_vma,
2402 unsigned long dst_addr,
2403 unsigned long src_addr,
2404 bool zeropage, bool wp_copy,
2405 struct page **pagep)
2406 {
2407 struct inode *inode = file_inode(dst_vma->vm_file);
2408 struct shmem_inode_info *info = SHMEM_I(inode);
2409 struct address_space *mapping = inode->i_mapping;
2410 gfp_t gfp = mapping_gfp_mask(mapping);
2411 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2412 void *page_kaddr;
2413 struct folio *folio;
2414 int ret;
2415 pgoff_t max_off;
2416
2417 if (!shmem_inode_acct_block(inode, 1)) {
2418 /*
2419 * We may have got a page, returned -ENOENT triggering a retry,
2420 * and now we find ourselves with -ENOMEM. Release the page, to
2421 * avoid a BUG_ON in our caller.
2422 */
2423 if (unlikely(*pagep)) {
2424 put_page(*pagep);
2425 *pagep = NULL;
2426 }
2427 return -ENOMEM;
2428 }
2429
2430 if (!*pagep) {
2431 ret = -ENOMEM;
2432 folio = shmem_alloc_folio(gfp, info, pgoff);
2433 if (!folio)
2434 goto out_unacct_blocks;
2435
2436 if (!zeropage) { /* COPY */
2437 page_kaddr = kmap_local_folio(folio, 0);
2438 /*
2439 * The read mmap_lock is held here. Despite the
2440 * mmap_lock being read recursive a deadlock is still
2441 * possible if a writer has taken a lock. For example:
2442 *
2443 * process A thread 1 takes read lock on own mmap_lock
2444 * process A thread 2 calls mmap, blocks taking write lock
2445 * process B thread 1 takes page fault, read lock on own mmap lock
2446 * process B thread 2 calls mmap, blocks taking write lock
2447 * process A thread 1 blocks taking read lock on process B
2448 * process B thread 1 blocks taking read lock on process A
2449 *
2450 * Disable page faults to prevent potential deadlock
2451 * and retry the copy outside the mmap_lock.
2452 */
2453 pagefault_disable();
2454 ret = copy_from_user(page_kaddr,
2455 (const void __user *)src_addr,
2456 PAGE_SIZE);
2457 pagefault_enable();
2458 kunmap_local(page_kaddr);
2459
2460 /* fallback to copy_from_user outside mmap_lock */
2461 if (unlikely(ret)) {
2462 *pagep = &folio->page;
2463 ret = -ENOENT;
2464 /* don't free the page */
2465 goto out_unacct_blocks;
2466 }
2467
2468 flush_dcache_folio(folio);
2469 } else { /* ZEROPAGE */
2470 clear_user_highpage(&folio->page, dst_addr);
2471 }
2472 } else {
2473 folio = page_folio(*pagep);
2474 VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2475 *pagep = NULL;
2476 }
2477
2478 VM_BUG_ON(folio_test_locked(folio));
2479 VM_BUG_ON(folio_test_swapbacked(folio));
2480 __folio_set_locked(folio);
2481 __folio_set_swapbacked(folio);
2482 __folio_mark_uptodate(folio);
2483
2484 ret = -EFAULT;
2485 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2486 if (unlikely(pgoff >= max_off))
2487 goto out_release;
2488
2489 ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL,
2490 gfp & GFP_RECLAIM_MASK, dst_mm);
2491 if (ret)
2492 goto out_release;
2493
2494 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
2495 &folio->page, true, wp_copy);
2496 if (ret)
2497 goto out_delete_from_cache;
2498
2499 spin_lock_irq(&info->lock);
2500 info->alloced++;
2501 inode->i_blocks += BLOCKS_PER_PAGE;
2502 shmem_recalc_inode(inode);
2503 spin_unlock_irq(&info->lock);
2504
2505 folio_unlock(folio);
2506 return 0;
2507 out_delete_from_cache:
2508 filemap_remove_folio(folio);
2509 out_release:
2510 folio_unlock(folio);
2511 folio_put(folio);
2512 out_unacct_blocks:
2513 shmem_inode_unacct_blocks(inode, 1);
2514 return ret;
2515 }
2516 #endif /* CONFIG_USERFAULTFD */
2517
2518 #ifdef CONFIG_TMPFS
2519 static const struct inode_operations shmem_symlink_inode_operations;
2520 static const struct inode_operations shmem_short_symlink_operations;
2521
2522 static int
shmem_write_begin(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,struct page ** pagep,void ** fsdata)2523 shmem_write_begin(struct file *file, struct address_space *mapping,
2524 loff_t pos, unsigned len,
2525 struct page **pagep, void **fsdata)
2526 {
2527 struct inode *inode = mapping->host;
2528 struct shmem_inode_info *info = SHMEM_I(inode);
2529 pgoff_t index = pos >> PAGE_SHIFT;
2530 struct folio *folio;
2531 int ret = 0;
2532
2533 /* i_rwsem is held by caller */
2534 if (unlikely(info->seals & (F_SEAL_GROW |
2535 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2536 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2537 return -EPERM;
2538 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2539 return -EPERM;
2540 }
2541
2542 ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
2543
2544 if (ret)
2545 return ret;
2546
2547 *pagep = folio_file_page(folio, index);
2548 if (PageHWPoison(*pagep)) {
2549 folio_unlock(folio);
2550 folio_put(folio);
2551 *pagep = NULL;
2552 return -EIO;
2553 }
2554
2555 return 0;
2556 }
2557
2558 static int
shmem_write_end(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct page * page,void * fsdata)2559 shmem_write_end(struct file *file, struct address_space *mapping,
2560 loff_t pos, unsigned len, unsigned copied,
2561 struct page *page, void *fsdata)
2562 {
2563 struct inode *inode = mapping->host;
2564
2565 if (pos + copied > inode->i_size)
2566 i_size_write(inode, pos + copied);
2567
2568 if (!PageUptodate(page)) {
2569 struct page *head = compound_head(page);
2570 if (PageTransCompound(page)) {
2571 int i;
2572
2573 for (i = 0; i < HPAGE_PMD_NR; i++) {
2574 if (head + i == page)
2575 continue;
2576 clear_highpage(head + i);
2577 flush_dcache_page(head + i);
2578 }
2579 }
2580 if (copied < PAGE_SIZE) {
2581 unsigned from = pos & (PAGE_SIZE - 1);
2582 zero_user_segments(page, 0, from,
2583 from + copied, PAGE_SIZE);
2584 }
2585 SetPageUptodate(head);
2586 }
2587 set_page_dirty(page);
2588 unlock_page(page);
2589 put_page(page);
2590
2591 return copied;
2592 }
2593
shmem_file_read_iter(struct kiocb * iocb,struct iov_iter * to)2594 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2595 {
2596 struct file *file = iocb->ki_filp;
2597 struct inode *inode = file_inode(file);
2598 struct address_space *mapping = inode->i_mapping;
2599 pgoff_t index;
2600 unsigned long offset;
2601 int error = 0;
2602 ssize_t retval = 0;
2603 loff_t *ppos = &iocb->ki_pos;
2604
2605 index = *ppos >> PAGE_SHIFT;
2606 offset = *ppos & ~PAGE_MASK;
2607
2608 for (;;) {
2609 struct folio *folio = NULL;
2610 struct page *page = NULL;
2611 pgoff_t end_index;
2612 unsigned long nr, ret;
2613 loff_t i_size = i_size_read(inode);
2614
2615 end_index = i_size >> PAGE_SHIFT;
2616 if (index > end_index)
2617 break;
2618 if (index == end_index) {
2619 nr = i_size & ~PAGE_MASK;
2620 if (nr <= offset)
2621 break;
2622 }
2623
2624 error = shmem_get_folio(inode, index, &folio, SGP_READ);
2625 if (error) {
2626 if (error == -EINVAL)
2627 error = 0;
2628 break;
2629 }
2630 if (folio) {
2631 folio_unlock(folio);
2632
2633 page = folio_file_page(folio, index);
2634 if (PageHWPoison(page)) {
2635 folio_put(folio);
2636 error = -EIO;
2637 break;
2638 }
2639 }
2640
2641 /*
2642 * We must evaluate after, since reads (unlike writes)
2643 * are called without i_rwsem protection against truncate
2644 */
2645 nr = PAGE_SIZE;
2646 i_size = i_size_read(inode);
2647 end_index = i_size >> PAGE_SHIFT;
2648 if (index == end_index) {
2649 nr = i_size & ~PAGE_MASK;
2650 if (nr <= offset) {
2651 if (folio)
2652 folio_put(folio);
2653 break;
2654 }
2655 }
2656 nr -= offset;
2657
2658 if (folio) {
2659 /*
2660 * If users can be writing to this page using arbitrary
2661 * virtual addresses, take care about potential aliasing
2662 * before reading the page on the kernel side.
2663 */
2664 if (mapping_writably_mapped(mapping))
2665 flush_dcache_page(page);
2666 /*
2667 * Mark the page accessed if we read the beginning.
2668 */
2669 if (!offset)
2670 folio_mark_accessed(folio);
2671 /*
2672 * Ok, we have the page, and it's up-to-date, so
2673 * now we can copy it to user space...
2674 */
2675 ret = copy_page_to_iter(page, offset, nr, to);
2676 folio_put(folio);
2677
2678 } else if (user_backed_iter(to)) {
2679 /*
2680 * Copy to user tends to be so well optimized, but
2681 * clear_user() not so much, that it is noticeably
2682 * faster to copy the zero page instead of clearing.
2683 */
2684 ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
2685 } else {
2686 /*
2687 * But submitting the same page twice in a row to
2688 * splice() - or others? - can result in confusion:
2689 * so don't attempt that optimization on pipes etc.
2690 */
2691 ret = iov_iter_zero(nr, to);
2692 }
2693
2694 retval += ret;
2695 offset += ret;
2696 index += offset >> PAGE_SHIFT;
2697 offset &= ~PAGE_MASK;
2698
2699 if (!iov_iter_count(to))
2700 break;
2701 if (ret < nr) {
2702 error = -EFAULT;
2703 break;
2704 }
2705 cond_resched();
2706 }
2707
2708 *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2709 file_accessed(file);
2710 return retval ? retval : error;
2711 }
2712
shmem_file_llseek(struct file * file,loff_t offset,int whence)2713 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2714 {
2715 struct address_space *mapping = file->f_mapping;
2716 struct inode *inode = mapping->host;
2717
2718 if (whence != SEEK_DATA && whence != SEEK_HOLE)
2719 return generic_file_llseek_size(file, offset, whence,
2720 MAX_LFS_FILESIZE, i_size_read(inode));
2721 if (offset < 0)
2722 return -ENXIO;
2723
2724 inode_lock(inode);
2725 /* We're holding i_rwsem so we can access i_size directly */
2726 offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
2727 if (offset >= 0)
2728 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2729 inode_unlock(inode);
2730 return offset;
2731 }
2732
shmem_fallocate(struct file * file,int mode,loff_t offset,loff_t len)2733 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2734 loff_t len)
2735 {
2736 struct inode *inode = file_inode(file);
2737 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2738 struct shmem_inode_info *info = SHMEM_I(inode);
2739 struct shmem_falloc shmem_falloc;
2740 pgoff_t start, index, end, undo_fallocend;
2741 int error;
2742
2743 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2744 return -EOPNOTSUPP;
2745
2746 inode_lock(inode);
2747
2748 if (mode & FALLOC_FL_PUNCH_HOLE) {
2749 struct address_space *mapping = file->f_mapping;
2750 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2751 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2752 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2753
2754 /* protected by i_rwsem */
2755 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2756 error = -EPERM;
2757 goto out;
2758 }
2759
2760 shmem_falloc.waitq = &shmem_falloc_waitq;
2761 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2762 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2763 spin_lock(&inode->i_lock);
2764 inode->i_private = &shmem_falloc;
2765 spin_unlock(&inode->i_lock);
2766
2767 if ((u64)unmap_end > (u64)unmap_start)
2768 unmap_mapping_range(mapping, unmap_start,
2769 1 + unmap_end - unmap_start, 0);
2770 shmem_truncate_range(inode, offset, offset + len - 1);
2771 /* No need to unmap again: hole-punching leaves COWed pages */
2772
2773 spin_lock(&inode->i_lock);
2774 inode->i_private = NULL;
2775 wake_up_all(&shmem_falloc_waitq);
2776 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2777 spin_unlock(&inode->i_lock);
2778 error = 0;
2779 goto out;
2780 }
2781
2782 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2783 error = inode_newsize_ok(inode, offset + len);
2784 if (error)
2785 goto out;
2786
2787 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2788 error = -EPERM;
2789 goto out;
2790 }
2791
2792 start = offset >> PAGE_SHIFT;
2793 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2794 /* Try to avoid a swapstorm if len is impossible to satisfy */
2795 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2796 error = -ENOSPC;
2797 goto out;
2798 }
2799
2800 shmem_falloc.waitq = NULL;
2801 shmem_falloc.start = start;
2802 shmem_falloc.next = start;
2803 shmem_falloc.nr_falloced = 0;
2804 shmem_falloc.nr_unswapped = 0;
2805 spin_lock(&inode->i_lock);
2806 inode->i_private = &shmem_falloc;
2807 spin_unlock(&inode->i_lock);
2808
2809 /*
2810 * info->fallocend is only relevant when huge pages might be
2811 * involved: to prevent split_huge_page() freeing fallocated
2812 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
2813 */
2814 undo_fallocend = info->fallocend;
2815 if (info->fallocend < end)
2816 info->fallocend = end;
2817
2818 for (index = start; index < end; ) {
2819 struct folio *folio;
2820
2821 /*
2822 * Good, the fallocate(2) manpage permits EINTR: we may have
2823 * been interrupted because we are using up too much memory.
2824 */
2825 if (signal_pending(current))
2826 error = -EINTR;
2827 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2828 error = -ENOMEM;
2829 else
2830 error = shmem_get_folio(inode, index, &folio,
2831 SGP_FALLOC);
2832 if (error) {
2833 info->fallocend = undo_fallocend;
2834 /* Remove the !uptodate folios we added */
2835 if (index > start) {
2836 shmem_undo_range(inode,
2837 (loff_t)start << PAGE_SHIFT,
2838 ((loff_t)index << PAGE_SHIFT) - 1, true);
2839 }
2840 goto undone;
2841 }
2842
2843 /*
2844 * Here is a more important optimization than it appears:
2845 * a second SGP_FALLOC on the same large folio will clear it,
2846 * making it uptodate and un-undoable if we fail later.
2847 */
2848 index = folio_next_index(folio);
2849 /* Beware 32-bit wraparound */
2850 if (!index)
2851 index--;
2852
2853 /*
2854 * Inform shmem_writepage() how far we have reached.
2855 * No need for lock or barrier: we have the page lock.
2856 */
2857 if (!folio_test_uptodate(folio))
2858 shmem_falloc.nr_falloced += index - shmem_falloc.next;
2859 shmem_falloc.next = index;
2860
2861 /*
2862 * If !uptodate, leave it that way so that freeable folios
2863 * can be recognized if we need to rollback on error later.
2864 * But mark it dirty so that memory pressure will swap rather
2865 * than free the folios we are allocating (and SGP_CACHE folios
2866 * might still be clean: we now need to mark those dirty too).
2867 */
2868 folio_mark_dirty(folio);
2869 folio_unlock(folio);
2870 folio_put(folio);
2871 cond_resched();
2872 }
2873
2874 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2875 i_size_write(inode, offset + len);
2876 undone:
2877 spin_lock(&inode->i_lock);
2878 inode->i_private = NULL;
2879 spin_unlock(&inode->i_lock);
2880 out:
2881 if (!error)
2882 file_modified(file);
2883 inode_unlock(inode);
2884 return error;
2885 }
2886
shmem_statfs(struct dentry * dentry,struct kstatfs * buf)2887 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2888 {
2889 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2890
2891 buf->f_type = TMPFS_MAGIC;
2892 buf->f_bsize = PAGE_SIZE;
2893 buf->f_namelen = NAME_MAX;
2894 if (sbinfo->max_blocks) {
2895 buf->f_blocks = sbinfo->max_blocks;
2896 buf->f_bavail =
2897 buf->f_bfree = sbinfo->max_blocks -
2898 percpu_counter_sum(&sbinfo->used_blocks);
2899 }
2900 if (sbinfo->max_inodes) {
2901 buf->f_files = sbinfo->max_inodes;
2902 buf->f_ffree = sbinfo->free_inodes;
2903 }
2904 /* else leave those fields 0 like simple_statfs */
2905
2906 buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
2907
2908 return 0;
2909 }
2910
2911 /*
2912 * File creation. Allocate an inode, and we're done..
2913 */
2914 static int
shmem_mknod(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,umode_t mode,dev_t dev)2915 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir,
2916 struct dentry *dentry, umode_t mode, dev_t dev)
2917 {
2918 struct inode *inode;
2919 int error = -ENOSPC;
2920
2921 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2922 if (inode) {
2923 error = simple_acl_create(dir, inode);
2924 if (error)
2925 goto out_iput;
2926 error = security_inode_init_security(inode, dir,
2927 &dentry->d_name,
2928 shmem_initxattrs, NULL);
2929 if (error && error != -EOPNOTSUPP)
2930 goto out_iput;
2931
2932 error = 0;
2933 dir->i_size += BOGO_DIRENT_SIZE;
2934 dir->i_ctime = dir->i_mtime = current_time(dir);
2935 inode_inc_iversion(dir);
2936 d_instantiate(dentry, inode);
2937 dget(dentry); /* Extra count - pin the dentry in core */
2938 }
2939 return error;
2940 out_iput:
2941 iput(inode);
2942 return error;
2943 }
2944
2945 static int
shmem_tmpfile(struct user_namespace * mnt_userns,struct inode * dir,struct file * file,umode_t mode)2946 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
2947 struct file *file, umode_t mode)
2948 {
2949 struct inode *inode;
2950 int error = -ENOSPC;
2951
2952 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2953 if (inode) {
2954 error = security_inode_init_security(inode, dir,
2955 NULL,
2956 shmem_initxattrs, NULL);
2957 if (error && error != -EOPNOTSUPP)
2958 goto out_iput;
2959 error = simple_acl_create(dir, inode);
2960 if (error)
2961 goto out_iput;
2962 d_tmpfile(file, inode);
2963 }
2964 return finish_open_simple(file, error);
2965 out_iput:
2966 iput(inode);
2967 return error;
2968 }
2969
shmem_mkdir(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,umode_t mode)2970 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
2971 struct dentry *dentry, umode_t mode)
2972 {
2973 int error;
2974
2975 if ((error = shmem_mknod(&init_user_ns, dir, dentry,
2976 mode | S_IFDIR, 0)))
2977 return error;
2978 inc_nlink(dir);
2979 return 0;
2980 }
2981
shmem_create(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,umode_t mode,bool excl)2982 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir,
2983 struct dentry *dentry, umode_t mode, bool excl)
2984 {
2985 return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0);
2986 }
2987
2988 /*
2989 * Link a file..
2990 */
shmem_link(struct dentry * old_dentry,struct inode * dir,struct dentry * dentry)2991 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2992 {
2993 struct inode *inode = d_inode(old_dentry);
2994 int ret = 0;
2995
2996 /*
2997 * No ordinary (disk based) filesystem counts links as inodes;
2998 * but each new link needs a new dentry, pinning lowmem, and
2999 * tmpfs dentries cannot be pruned until they are unlinked.
3000 * But if an O_TMPFILE file is linked into the tmpfs, the
3001 * first link must skip that, to get the accounting right.
3002 */
3003 if (inode->i_nlink) {
3004 ret = shmem_reserve_inode(inode->i_sb, NULL);
3005 if (ret)
3006 goto out;
3007 }
3008
3009 dir->i_size += BOGO_DIRENT_SIZE;
3010 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3011 inode_inc_iversion(dir);
3012 inc_nlink(inode);
3013 ihold(inode); /* New dentry reference */
3014 dget(dentry); /* Extra pinning count for the created dentry */
3015 d_instantiate(dentry, inode);
3016 out:
3017 return ret;
3018 }
3019
shmem_unlink(struct inode * dir,struct dentry * dentry)3020 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3021 {
3022 struct inode *inode = d_inode(dentry);
3023
3024 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3025 shmem_free_inode(inode->i_sb);
3026
3027 dir->i_size -= BOGO_DIRENT_SIZE;
3028 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3029 inode_inc_iversion(dir);
3030 drop_nlink(inode);
3031 dput(dentry); /* Undo the count from "create" - this does all the work */
3032 return 0;
3033 }
3034
shmem_rmdir(struct inode * dir,struct dentry * dentry)3035 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3036 {
3037 if (!simple_empty(dentry))
3038 return -ENOTEMPTY;
3039
3040 drop_nlink(d_inode(dentry));
3041 drop_nlink(dir);
3042 return shmem_unlink(dir, dentry);
3043 }
3044
shmem_whiteout(struct user_namespace * mnt_userns,struct inode * old_dir,struct dentry * old_dentry)3045 static int shmem_whiteout(struct user_namespace *mnt_userns,
3046 struct inode *old_dir, struct dentry *old_dentry)
3047 {
3048 struct dentry *whiteout;
3049 int error;
3050
3051 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3052 if (!whiteout)
3053 return -ENOMEM;
3054
3055 error = shmem_mknod(&init_user_ns, old_dir, whiteout,
3056 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3057 dput(whiteout);
3058 if (error)
3059 return error;
3060
3061 /*
3062 * Cheat and hash the whiteout while the old dentry is still in
3063 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3064 *
3065 * d_lookup() will consistently find one of them at this point,
3066 * not sure which one, but that isn't even important.
3067 */
3068 d_rehash(whiteout);
3069 return 0;
3070 }
3071
3072 /*
3073 * The VFS layer already does all the dentry stuff for rename,
3074 * we just have to decrement the usage count for the target if
3075 * it exists so that the VFS layer correctly free's it when it
3076 * gets overwritten.
3077 */
shmem_rename2(struct user_namespace * mnt_userns,struct inode * old_dir,struct dentry * old_dentry,struct inode * new_dir,struct dentry * new_dentry,unsigned int flags)3078 static int shmem_rename2(struct user_namespace *mnt_userns,
3079 struct inode *old_dir, struct dentry *old_dentry,
3080 struct inode *new_dir, struct dentry *new_dentry,
3081 unsigned int flags)
3082 {
3083 struct inode *inode = d_inode(old_dentry);
3084 int they_are_dirs = S_ISDIR(inode->i_mode);
3085
3086 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3087 return -EINVAL;
3088
3089 if (flags & RENAME_EXCHANGE)
3090 return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
3091
3092 if (!simple_empty(new_dentry))
3093 return -ENOTEMPTY;
3094
3095 if (flags & RENAME_WHITEOUT) {
3096 int error;
3097
3098 error = shmem_whiteout(&init_user_ns, old_dir, old_dentry);
3099 if (error)
3100 return error;
3101 }
3102
3103 if (d_really_is_positive(new_dentry)) {
3104 (void) shmem_unlink(new_dir, new_dentry);
3105 if (they_are_dirs) {
3106 drop_nlink(d_inode(new_dentry));
3107 drop_nlink(old_dir);
3108 }
3109 } else if (they_are_dirs) {
3110 drop_nlink(old_dir);
3111 inc_nlink(new_dir);
3112 }
3113
3114 old_dir->i_size -= BOGO_DIRENT_SIZE;
3115 new_dir->i_size += BOGO_DIRENT_SIZE;
3116 old_dir->i_ctime = old_dir->i_mtime =
3117 new_dir->i_ctime = new_dir->i_mtime =
3118 inode->i_ctime = current_time(old_dir);
3119 inode_inc_iversion(old_dir);
3120 inode_inc_iversion(new_dir);
3121 return 0;
3122 }
3123
shmem_symlink(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,const char * symname)3124 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir,
3125 struct dentry *dentry, const char *symname)
3126 {
3127 int error;
3128 int len;
3129 struct inode *inode;
3130 struct folio *folio;
3131
3132 len = strlen(symname) + 1;
3133 if (len > PAGE_SIZE)
3134 return -ENAMETOOLONG;
3135
3136 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3137 VM_NORESERVE);
3138 if (!inode)
3139 return -ENOSPC;
3140
3141 error = security_inode_init_security(inode, dir, &dentry->d_name,
3142 shmem_initxattrs, NULL);
3143 if (error && error != -EOPNOTSUPP) {
3144 iput(inode);
3145 return error;
3146 }
3147
3148 inode->i_size = len-1;
3149 if (len <= SHORT_SYMLINK_LEN) {
3150 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3151 if (!inode->i_link) {
3152 iput(inode);
3153 return -ENOMEM;
3154 }
3155 inode->i_op = &shmem_short_symlink_operations;
3156 } else {
3157 inode_nohighmem(inode);
3158 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
3159 if (error) {
3160 iput(inode);
3161 return error;
3162 }
3163 inode->i_mapping->a_ops = &shmem_aops;
3164 inode->i_op = &shmem_symlink_inode_operations;
3165 memcpy(folio_address(folio), symname, len);
3166 folio_mark_uptodate(folio);
3167 folio_mark_dirty(folio);
3168 folio_unlock(folio);
3169 folio_put(folio);
3170 }
3171 dir->i_size += BOGO_DIRENT_SIZE;
3172 dir->i_ctime = dir->i_mtime = current_time(dir);
3173 inode_inc_iversion(dir);
3174 d_instantiate(dentry, inode);
3175 dget(dentry);
3176 return 0;
3177 }
3178
shmem_put_link(void * arg)3179 static void shmem_put_link(void *arg)
3180 {
3181 folio_mark_accessed(arg);
3182 folio_put(arg);
3183 }
3184
shmem_get_link(struct dentry * dentry,struct inode * inode,struct delayed_call * done)3185 static const char *shmem_get_link(struct dentry *dentry,
3186 struct inode *inode,
3187 struct delayed_call *done)
3188 {
3189 struct folio *folio = NULL;
3190 int error;
3191
3192 if (!dentry) {
3193 folio = filemap_get_folio(inode->i_mapping, 0);
3194 if (!folio)
3195 return ERR_PTR(-ECHILD);
3196 if (PageHWPoison(folio_page(folio, 0)) ||
3197 !folio_test_uptodate(folio)) {
3198 folio_put(folio);
3199 return ERR_PTR(-ECHILD);
3200 }
3201 } else {
3202 error = shmem_get_folio(inode, 0, &folio, SGP_READ);
3203 if (error)
3204 return ERR_PTR(error);
3205 if (!folio)
3206 return ERR_PTR(-ECHILD);
3207 if (PageHWPoison(folio_page(folio, 0))) {
3208 folio_unlock(folio);
3209 folio_put(folio);
3210 return ERR_PTR(-ECHILD);
3211 }
3212 folio_unlock(folio);
3213 }
3214 set_delayed_call(done, shmem_put_link, folio);
3215 return folio_address(folio);
3216 }
3217
3218 #ifdef CONFIG_TMPFS_XATTR
3219
shmem_fileattr_get(struct dentry * dentry,struct fileattr * fa)3220 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
3221 {
3222 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3223
3224 fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE);
3225
3226 return 0;
3227 }
3228
shmem_fileattr_set(struct user_namespace * mnt_userns,struct dentry * dentry,struct fileattr * fa)3229 static int shmem_fileattr_set(struct user_namespace *mnt_userns,
3230 struct dentry *dentry, struct fileattr *fa)
3231 {
3232 struct inode *inode = d_inode(dentry);
3233 struct shmem_inode_info *info = SHMEM_I(inode);
3234
3235 if (fileattr_has_fsx(fa))
3236 return -EOPNOTSUPP;
3237 if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
3238 return -EOPNOTSUPP;
3239
3240 info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
3241 (fa->flags & SHMEM_FL_USER_MODIFIABLE);
3242
3243 shmem_set_inode_flags(inode, info->fsflags);
3244 inode->i_ctime = current_time(inode);
3245 inode_inc_iversion(inode);
3246 return 0;
3247 }
3248
3249 /*
3250 * Superblocks without xattr inode operations may get some security.* xattr
3251 * support from the LSM "for free". As soon as we have any other xattrs
3252 * like ACLs, we also need to implement the security.* handlers at
3253 * filesystem level, though.
3254 */
3255
3256 /*
3257 * Callback for security_inode_init_security() for acquiring xattrs.
3258 */
shmem_initxattrs(struct inode * inode,const struct xattr * xattr_array,void * fs_info)3259 static int shmem_initxattrs(struct inode *inode,
3260 const struct xattr *xattr_array,
3261 void *fs_info)
3262 {
3263 struct shmem_inode_info *info = SHMEM_I(inode);
3264 const struct xattr *xattr;
3265 struct simple_xattr *new_xattr;
3266 size_t len;
3267
3268 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3269 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3270 if (!new_xattr)
3271 return -ENOMEM;
3272
3273 len = strlen(xattr->name) + 1;
3274 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3275 GFP_KERNEL);
3276 if (!new_xattr->name) {
3277 kvfree(new_xattr);
3278 return -ENOMEM;
3279 }
3280
3281 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3282 XATTR_SECURITY_PREFIX_LEN);
3283 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3284 xattr->name, len);
3285
3286 simple_xattr_list_add(&info->xattrs, new_xattr);
3287 }
3288
3289 return 0;
3290 }
3291
shmem_xattr_handler_get(const struct xattr_handler * handler,struct dentry * unused,struct inode * inode,const char * name,void * buffer,size_t size)3292 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3293 struct dentry *unused, struct inode *inode,
3294 const char *name, void *buffer, size_t size)
3295 {
3296 struct shmem_inode_info *info = SHMEM_I(inode);
3297
3298 name = xattr_full_name(handler, name);
3299 return simple_xattr_get(&info->xattrs, name, buffer, size);
3300 }
3301
shmem_xattr_handler_set(const struct xattr_handler * handler,struct user_namespace * mnt_userns,struct dentry * unused,struct inode * inode,const char * name,const void * value,size_t size,int flags)3302 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3303 struct user_namespace *mnt_userns,
3304 struct dentry *unused, struct inode *inode,
3305 const char *name, const void *value,
3306 size_t size, int flags)
3307 {
3308 struct shmem_inode_info *info = SHMEM_I(inode);
3309 int err;
3310
3311 name = xattr_full_name(handler, name);
3312 err = simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3313 if (!err) {
3314 inode->i_ctime = current_time(inode);
3315 inode_inc_iversion(inode);
3316 }
3317 return err;
3318 }
3319
3320 static const struct xattr_handler shmem_security_xattr_handler = {
3321 .prefix = XATTR_SECURITY_PREFIX,
3322 .get = shmem_xattr_handler_get,
3323 .set = shmem_xattr_handler_set,
3324 };
3325
3326 static const struct xattr_handler shmem_trusted_xattr_handler = {
3327 .prefix = XATTR_TRUSTED_PREFIX,
3328 .get = shmem_xattr_handler_get,
3329 .set = shmem_xattr_handler_set,
3330 };
3331
3332 static const struct xattr_handler *shmem_xattr_handlers[] = {
3333 #ifdef CONFIG_TMPFS_POSIX_ACL
3334 &posix_acl_access_xattr_handler,
3335 &posix_acl_default_xattr_handler,
3336 #endif
3337 &shmem_security_xattr_handler,
3338 &shmem_trusted_xattr_handler,
3339 NULL
3340 };
3341
shmem_listxattr(struct dentry * dentry,char * buffer,size_t size)3342 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3343 {
3344 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3345 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3346 }
3347 #endif /* CONFIG_TMPFS_XATTR */
3348
3349 static const struct inode_operations shmem_short_symlink_operations = {
3350 .getattr = shmem_getattr,
3351 .get_link = simple_get_link,
3352 #ifdef CONFIG_TMPFS_XATTR
3353 .listxattr = shmem_listxattr,
3354 #endif
3355 };
3356
3357 static const struct inode_operations shmem_symlink_inode_operations = {
3358 .getattr = shmem_getattr,
3359 .get_link = shmem_get_link,
3360 #ifdef CONFIG_TMPFS_XATTR
3361 .listxattr = shmem_listxattr,
3362 #endif
3363 };
3364
shmem_get_parent(struct dentry * child)3365 static struct dentry *shmem_get_parent(struct dentry *child)
3366 {
3367 return ERR_PTR(-ESTALE);
3368 }
3369
shmem_match(struct inode * ino,void * vfh)3370 static int shmem_match(struct inode *ino, void *vfh)
3371 {
3372 __u32 *fh = vfh;
3373 __u64 inum = fh[2];
3374 inum = (inum << 32) | fh[1];
3375 return ino->i_ino == inum && fh[0] == ino->i_generation;
3376 }
3377
3378 /* Find any alias of inode, but prefer a hashed alias */
shmem_find_alias(struct inode * inode)3379 static struct dentry *shmem_find_alias(struct inode *inode)
3380 {
3381 struct dentry *alias = d_find_alias(inode);
3382
3383 return alias ?: d_find_any_alias(inode);
3384 }
3385
3386
shmem_fh_to_dentry(struct super_block * sb,struct fid * fid,int fh_len,int fh_type)3387 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3388 struct fid *fid, int fh_len, int fh_type)
3389 {
3390 struct inode *inode;
3391 struct dentry *dentry = NULL;
3392 u64 inum;
3393
3394 if (fh_len < 3)
3395 return NULL;
3396
3397 inum = fid->raw[2];
3398 inum = (inum << 32) | fid->raw[1];
3399
3400 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3401 shmem_match, fid->raw);
3402 if (inode) {
3403 dentry = shmem_find_alias(inode);
3404 iput(inode);
3405 }
3406
3407 return dentry;
3408 }
3409
shmem_encode_fh(struct inode * inode,__u32 * fh,int * len,struct inode * parent)3410 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3411 struct inode *parent)
3412 {
3413 if (*len < 3) {
3414 *len = 3;
3415 return FILEID_INVALID;
3416 }
3417
3418 if (inode_unhashed(inode)) {
3419 /* Unfortunately insert_inode_hash is not idempotent,
3420 * so as we hash inodes here rather than at creation
3421 * time, we need a lock to ensure we only try
3422 * to do it once
3423 */
3424 static DEFINE_SPINLOCK(lock);
3425 spin_lock(&lock);
3426 if (inode_unhashed(inode))
3427 __insert_inode_hash(inode,
3428 inode->i_ino + inode->i_generation);
3429 spin_unlock(&lock);
3430 }
3431
3432 fh[0] = inode->i_generation;
3433 fh[1] = inode->i_ino;
3434 fh[2] = ((__u64)inode->i_ino) >> 32;
3435
3436 *len = 3;
3437 return 1;
3438 }
3439
3440 static const struct export_operations shmem_export_ops = {
3441 .get_parent = shmem_get_parent,
3442 .encode_fh = shmem_encode_fh,
3443 .fh_to_dentry = shmem_fh_to_dentry,
3444 };
3445
3446 enum shmem_param {
3447 Opt_gid,
3448 Opt_huge,
3449 Opt_mode,
3450 Opt_mpol,
3451 Opt_nr_blocks,
3452 Opt_nr_inodes,
3453 Opt_size,
3454 Opt_uid,
3455 Opt_inode32,
3456 Opt_inode64,
3457 };
3458
3459 static const struct constant_table shmem_param_enums_huge[] = {
3460 {"never", SHMEM_HUGE_NEVER },
3461 {"always", SHMEM_HUGE_ALWAYS },
3462 {"within_size", SHMEM_HUGE_WITHIN_SIZE },
3463 {"advise", SHMEM_HUGE_ADVISE },
3464 {}
3465 };
3466
3467 const struct fs_parameter_spec shmem_fs_parameters[] = {
3468 fsparam_u32 ("gid", Opt_gid),
3469 fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge),
3470 fsparam_u32oct("mode", Opt_mode),
3471 fsparam_string("mpol", Opt_mpol),
3472 fsparam_string("nr_blocks", Opt_nr_blocks),
3473 fsparam_string("nr_inodes", Opt_nr_inodes),
3474 fsparam_string("size", Opt_size),
3475 fsparam_u32 ("uid", Opt_uid),
3476 fsparam_flag ("inode32", Opt_inode32),
3477 fsparam_flag ("inode64", Opt_inode64),
3478 {}
3479 };
3480
shmem_parse_one(struct fs_context * fc,struct fs_parameter * param)3481 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3482 {
3483 struct shmem_options *ctx = fc->fs_private;
3484 struct fs_parse_result result;
3485 unsigned long long size;
3486 char *rest;
3487 int opt;
3488
3489 opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3490 if (opt < 0)
3491 return opt;
3492
3493 switch (opt) {
3494 case Opt_size:
3495 size = memparse(param->string, &rest);
3496 if (*rest == '%') {
3497 size <<= PAGE_SHIFT;
3498 size *= totalram_pages();
3499 do_div(size, 100);
3500 rest++;
3501 }
3502 if (*rest)
3503 goto bad_value;
3504 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3505 ctx->seen |= SHMEM_SEEN_BLOCKS;
3506 break;
3507 case Opt_nr_blocks:
3508 ctx->blocks = memparse(param->string, &rest);
3509 if (*rest || ctx->blocks > S64_MAX)
3510 goto bad_value;
3511 ctx->seen |= SHMEM_SEEN_BLOCKS;
3512 break;
3513 case Opt_nr_inodes:
3514 ctx->inodes = memparse(param->string, &rest);
3515 if (*rest)
3516 goto bad_value;
3517 ctx->seen |= SHMEM_SEEN_INODES;
3518 break;
3519 case Opt_mode:
3520 ctx->mode = result.uint_32 & 07777;
3521 break;
3522 case Opt_uid:
3523 ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3524 if (!uid_valid(ctx->uid))
3525 goto bad_value;
3526 break;
3527 case Opt_gid:
3528 ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3529 if (!gid_valid(ctx->gid))
3530 goto bad_value;
3531 break;
3532 case Opt_huge:
3533 ctx->huge = result.uint_32;
3534 if (ctx->huge != SHMEM_HUGE_NEVER &&
3535 !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3536 has_transparent_hugepage()))
3537 goto unsupported_parameter;
3538 ctx->seen |= SHMEM_SEEN_HUGE;
3539 break;
3540 case Opt_mpol:
3541 if (IS_ENABLED(CONFIG_NUMA)) {
3542 mpol_put(ctx->mpol);
3543 ctx->mpol = NULL;
3544 if (mpol_parse_str(param->string, &ctx->mpol))
3545 goto bad_value;
3546 break;
3547 }
3548 goto unsupported_parameter;
3549 case Opt_inode32:
3550 ctx->full_inums = false;
3551 ctx->seen |= SHMEM_SEEN_INUMS;
3552 break;
3553 case Opt_inode64:
3554 if (sizeof(ino_t) < 8) {
3555 return invalfc(fc,
3556 "Cannot use inode64 with <64bit inums in kernel\n");
3557 }
3558 ctx->full_inums = true;
3559 ctx->seen |= SHMEM_SEEN_INUMS;
3560 break;
3561 }
3562 return 0;
3563
3564 unsupported_parameter:
3565 return invalfc(fc, "Unsupported parameter '%s'", param->key);
3566 bad_value:
3567 return invalfc(fc, "Bad value for '%s'", param->key);
3568 }
3569
shmem_parse_options(struct fs_context * fc,void * data)3570 static int shmem_parse_options(struct fs_context *fc, void *data)
3571 {
3572 char *options = data;
3573
3574 if (options) {
3575 int err = security_sb_eat_lsm_opts(options, &fc->security);
3576 if (err)
3577 return err;
3578 }
3579
3580 while (options != NULL) {
3581 char *this_char = options;
3582 for (;;) {
3583 /*
3584 * NUL-terminate this option: unfortunately,
3585 * mount options form a comma-separated list,
3586 * but mpol's nodelist may also contain commas.
3587 */
3588 options = strchr(options, ',');
3589 if (options == NULL)
3590 break;
3591 options++;
3592 if (!isdigit(*options)) {
3593 options[-1] = '\0';
3594 break;
3595 }
3596 }
3597 if (*this_char) {
3598 char *value = strchr(this_char, '=');
3599 size_t len = 0;
3600 int err;
3601
3602 if (value) {
3603 *value++ = '\0';
3604 len = strlen(value);
3605 }
3606 err = vfs_parse_fs_string(fc, this_char, value, len);
3607 if (err < 0)
3608 return err;
3609 }
3610 }
3611 return 0;
3612 }
3613
3614 /*
3615 * Reconfigure a shmem filesystem.
3616 *
3617 * Note that we disallow change from limited->unlimited blocks/inodes while any
3618 * are in use; but we must separately disallow unlimited->limited, because in
3619 * that case we have no record of how much is already in use.
3620 */
shmem_reconfigure(struct fs_context * fc)3621 static int shmem_reconfigure(struct fs_context *fc)
3622 {
3623 struct shmem_options *ctx = fc->fs_private;
3624 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3625 unsigned long inodes;
3626 struct mempolicy *mpol = NULL;
3627 const char *err;
3628
3629 raw_spin_lock(&sbinfo->stat_lock);
3630 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3631
3632 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3633 if (!sbinfo->max_blocks) {
3634 err = "Cannot retroactively limit size";
3635 goto out;
3636 }
3637 if (percpu_counter_compare(&sbinfo->used_blocks,
3638 ctx->blocks) > 0) {
3639 err = "Too small a size for current use";
3640 goto out;
3641 }
3642 }
3643 if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
3644 if (!sbinfo->max_inodes) {
3645 err = "Cannot retroactively limit inodes";
3646 goto out;
3647 }
3648 if (ctx->inodes < inodes) {
3649 err = "Too few inodes for current use";
3650 goto out;
3651 }
3652 }
3653
3654 if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
3655 sbinfo->next_ino > UINT_MAX) {
3656 err = "Current inum too high to switch to 32-bit inums";
3657 goto out;
3658 }
3659
3660 if (ctx->seen & SHMEM_SEEN_HUGE)
3661 sbinfo->huge = ctx->huge;
3662 if (ctx->seen & SHMEM_SEEN_INUMS)
3663 sbinfo->full_inums = ctx->full_inums;
3664 if (ctx->seen & SHMEM_SEEN_BLOCKS)
3665 sbinfo->max_blocks = ctx->blocks;
3666 if (ctx->seen & SHMEM_SEEN_INODES) {
3667 sbinfo->max_inodes = ctx->inodes;
3668 sbinfo->free_inodes = ctx->inodes - inodes;
3669 }
3670
3671 /*
3672 * Preserve previous mempolicy unless mpol remount option was specified.
3673 */
3674 if (ctx->mpol) {
3675 mpol = sbinfo->mpol;
3676 sbinfo->mpol = ctx->mpol; /* transfers initial ref */
3677 ctx->mpol = NULL;
3678 }
3679 raw_spin_unlock(&sbinfo->stat_lock);
3680 mpol_put(mpol);
3681 return 0;
3682 out:
3683 raw_spin_unlock(&sbinfo->stat_lock);
3684 return invalfc(fc, "%s", err);
3685 }
3686
shmem_show_options(struct seq_file * seq,struct dentry * root)3687 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3688 {
3689 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3690
3691 if (sbinfo->max_blocks != shmem_default_max_blocks())
3692 seq_printf(seq, ",size=%luk",
3693 sbinfo->max_blocks << (PAGE_SHIFT - 10));
3694 if (sbinfo->max_inodes != shmem_default_max_inodes())
3695 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3696 if (sbinfo->mode != (0777 | S_ISVTX))
3697 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3698 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3699 seq_printf(seq, ",uid=%u",
3700 from_kuid_munged(&init_user_ns, sbinfo->uid));
3701 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3702 seq_printf(seq, ",gid=%u",
3703 from_kgid_munged(&init_user_ns, sbinfo->gid));
3704
3705 /*
3706 * Showing inode{64,32} might be useful even if it's the system default,
3707 * since then people don't have to resort to checking both here and
3708 * /proc/config.gz to confirm 64-bit inums were successfully applied
3709 * (which may not even exist if IKCONFIG_PROC isn't enabled).
3710 *
3711 * We hide it when inode64 isn't the default and we are using 32-bit
3712 * inodes, since that probably just means the feature isn't even under
3713 * consideration.
3714 *
3715 * As such:
3716 *
3717 * +-----------------+-----------------+
3718 * | TMPFS_INODE64=y | TMPFS_INODE64=n |
3719 * +------------------+-----------------+-----------------+
3720 * | full_inums=true | show | show |
3721 * | full_inums=false | show | hide |
3722 * +------------------+-----------------+-----------------+
3723 *
3724 */
3725 if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
3726 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
3727 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3728 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3729 if (sbinfo->huge)
3730 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3731 #endif
3732 shmem_show_mpol(seq, sbinfo->mpol);
3733 return 0;
3734 }
3735
3736 #endif /* CONFIG_TMPFS */
3737
shmem_put_super(struct super_block * sb)3738 static void shmem_put_super(struct super_block *sb)
3739 {
3740 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3741
3742 free_percpu(sbinfo->ino_batch);
3743 percpu_counter_destroy(&sbinfo->used_blocks);
3744 mpol_put(sbinfo->mpol);
3745 kfree(sbinfo);
3746 sb->s_fs_info = NULL;
3747 }
3748
shmem_fill_super(struct super_block * sb,struct fs_context * fc)3749 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
3750 {
3751 struct shmem_options *ctx = fc->fs_private;
3752 struct inode *inode;
3753 struct shmem_sb_info *sbinfo;
3754
3755 /* Round up to L1_CACHE_BYTES to resist false sharing */
3756 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3757 L1_CACHE_BYTES), GFP_KERNEL);
3758 if (!sbinfo)
3759 return -ENOMEM;
3760
3761 sb->s_fs_info = sbinfo;
3762
3763 #ifdef CONFIG_TMPFS
3764 /*
3765 * Per default we only allow half of the physical ram per
3766 * tmpfs instance, limiting inodes to one per page of lowmem;
3767 * but the internal instance is left unlimited.
3768 */
3769 if (!(sb->s_flags & SB_KERNMOUNT)) {
3770 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
3771 ctx->blocks = shmem_default_max_blocks();
3772 if (!(ctx->seen & SHMEM_SEEN_INODES))
3773 ctx->inodes = shmem_default_max_inodes();
3774 if (!(ctx->seen & SHMEM_SEEN_INUMS))
3775 ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
3776 } else {
3777 sb->s_flags |= SB_NOUSER;
3778 }
3779 sb->s_export_op = &shmem_export_ops;
3780 sb->s_flags |= SB_NOSEC | SB_I_VERSION;
3781 #else
3782 sb->s_flags |= SB_NOUSER;
3783 #endif
3784 sbinfo->max_blocks = ctx->blocks;
3785 sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
3786 if (sb->s_flags & SB_KERNMOUNT) {
3787 sbinfo->ino_batch = alloc_percpu(ino_t);
3788 if (!sbinfo->ino_batch)
3789 goto failed;
3790 }
3791 sbinfo->uid = ctx->uid;
3792 sbinfo->gid = ctx->gid;
3793 sbinfo->full_inums = ctx->full_inums;
3794 sbinfo->mode = ctx->mode;
3795 sbinfo->huge = ctx->huge;
3796 sbinfo->mpol = ctx->mpol;
3797 ctx->mpol = NULL;
3798
3799 raw_spin_lock_init(&sbinfo->stat_lock);
3800 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3801 goto failed;
3802 spin_lock_init(&sbinfo->shrinklist_lock);
3803 INIT_LIST_HEAD(&sbinfo->shrinklist);
3804
3805 sb->s_maxbytes = MAX_LFS_FILESIZE;
3806 sb->s_blocksize = PAGE_SIZE;
3807 sb->s_blocksize_bits = PAGE_SHIFT;
3808 sb->s_magic = TMPFS_MAGIC;
3809 sb->s_op = &shmem_ops;
3810 sb->s_time_gran = 1;
3811 #ifdef CONFIG_TMPFS_XATTR
3812 sb->s_xattr = shmem_xattr_handlers;
3813 #endif
3814 #ifdef CONFIG_TMPFS_POSIX_ACL
3815 sb->s_flags |= SB_POSIXACL;
3816 #endif
3817 uuid_gen(&sb->s_uuid);
3818
3819 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3820 if (!inode)
3821 goto failed;
3822 inode->i_uid = sbinfo->uid;
3823 inode->i_gid = sbinfo->gid;
3824 sb->s_root = d_make_root(inode);
3825 if (!sb->s_root)
3826 goto failed;
3827 return 0;
3828
3829 failed:
3830 shmem_put_super(sb);
3831 return -ENOMEM;
3832 }
3833
shmem_get_tree(struct fs_context * fc)3834 static int shmem_get_tree(struct fs_context *fc)
3835 {
3836 return get_tree_nodev(fc, shmem_fill_super);
3837 }
3838
shmem_free_fc(struct fs_context * fc)3839 static void shmem_free_fc(struct fs_context *fc)
3840 {
3841 struct shmem_options *ctx = fc->fs_private;
3842
3843 if (ctx) {
3844 mpol_put(ctx->mpol);
3845 kfree(ctx);
3846 }
3847 }
3848
3849 static const struct fs_context_operations shmem_fs_context_ops = {
3850 .free = shmem_free_fc,
3851 .get_tree = shmem_get_tree,
3852 #ifdef CONFIG_TMPFS
3853 .parse_monolithic = shmem_parse_options,
3854 .parse_param = shmem_parse_one,
3855 .reconfigure = shmem_reconfigure,
3856 #endif
3857 };
3858
3859 static struct kmem_cache *shmem_inode_cachep;
3860
shmem_alloc_inode(struct super_block * sb)3861 static struct inode *shmem_alloc_inode(struct super_block *sb)
3862 {
3863 struct shmem_inode_info *info;
3864 info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
3865 if (!info)
3866 return NULL;
3867 return &info->vfs_inode;
3868 }
3869
shmem_free_in_core_inode(struct inode * inode)3870 static void shmem_free_in_core_inode(struct inode *inode)
3871 {
3872 if (S_ISLNK(inode->i_mode))
3873 kfree(inode->i_link);
3874 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3875 }
3876
shmem_destroy_inode(struct inode * inode)3877 static void shmem_destroy_inode(struct inode *inode)
3878 {
3879 if (S_ISREG(inode->i_mode))
3880 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3881 }
3882
shmem_init_inode(void * foo)3883 static void shmem_init_inode(void *foo)
3884 {
3885 struct shmem_inode_info *info = foo;
3886 inode_init_once(&info->vfs_inode);
3887 }
3888
shmem_init_inodecache(void)3889 static void shmem_init_inodecache(void)
3890 {
3891 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3892 sizeof(struct shmem_inode_info),
3893 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3894 }
3895
shmem_destroy_inodecache(void)3896 static void shmem_destroy_inodecache(void)
3897 {
3898 kmem_cache_destroy(shmem_inode_cachep);
3899 }
3900
3901 /* Keep the page in page cache instead of truncating it */
shmem_error_remove_page(struct address_space * mapping,struct page * page)3902 static int shmem_error_remove_page(struct address_space *mapping,
3903 struct page *page)
3904 {
3905 return 0;
3906 }
3907
3908 const struct address_space_operations shmem_aops = {
3909 .writepage = shmem_writepage,
3910 .dirty_folio = noop_dirty_folio,
3911 #ifdef CONFIG_TMPFS
3912 .write_begin = shmem_write_begin,
3913 .write_end = shmem_write_end,
3914 #endif
3915 #ifdef CONFIG_MIGRATION
3916 .migrate_folio = migrate_folio,
3917 #endif
3918 .error_remove_page = shmem_error_remove_page,
3919 };
3920 EXPORT_SYMBOL(shmem_aops);
3921
3922 static const struct file_operations shmem_file_operations = {
3923 .mmap = shmem_mmap,
3924 .get_unmapped_area = shmem_get_unmapped_area,
3925 #ifdef CONFIG_TMPFS
3926 .llseek = shmem_file_llseek,
3927 .read_iter = shmem_file_read_iter,
3928 .write_iter = generic_file_write_iter,
3929 .fsync = noop_fsync,
3930 .splice_read = generic_file_splice_read,
3931 .splice_write = iter_file_splice_write,
3932 .fallocate = shmem_fallocate,
3933 #endif
3934 };
3935
3936 static const struct inode_operations shmem_inode_operations = {
3937 .getattr = shmem_getattr,
3938 .setattr = shmem_setattr,
3939 #ifdef CONFIG_TMPFS_XATTR
3940 .listxattr = shmem_listxattr,
3941 .set_acl = simple_set_acl,
3942 .fileattr_get = shmem_fileattr_get,
3943 .fileattr_set = shmem_fileattr_set,
3944 #endif
3945 };
3946
3947 static const struct inode_operations shmem_dir_inode_operations = {
3948 #ifdef CONFIG_TMPFS
3949 .getattr = shmem_getattr,
3950 .create = shmem_create,
3951 .lookup = simple_lookup,
3952 .link = shmem_link,
3953 .unlink = shmem_unlink,
3954 .symlink = shmem_symlink,
3955 .mkdir = shmem_mkdir,
3956 .rmdir = shmem_rmdir,
3957 .mknod = shmem_mknod,
3958 .rename = shmem_rename2,
3959 .tmpfile = shmem_tmpfile,
3960 #endif
3961 #ifdef CONFIG_TMPFS_XATTR
3962 .listxattr = shmem_listxattr,
3963 .fileattr_get = shmem_fileattr_get,
3964 .fileattr_set = shmem_fileattr_set,
3965 #endif
3966 #ifdef CONFIG_TMPFS_POSIX_ACL
3967 .setattr = shmem_setattr,
3968 .set_acl = simple_set_acl,
3969 #endif
3970 };
3971
3972 static const struct inode_operations shmem_special_inode_operations = {
3973 .getattr = shmem_getattr,
3974 #ifdef CONFIG_TMPFS_XATTR
3975 .listxattr = shmem_listxattr,
3976 #endif
3977 #ifdef CONFIG_TMPFS_POSIX_ACL
3978 .setattr = shmem_setattr,
3979 .set_acl = simple_set_acl,
3980 #endif
3981 };
3982
3983 static const struct super_operations shmem_ops = {
3984 .alloc_inode = shmem_alloc_inode,
3985 .free_inode = shmem_free_in_core_inode,
3986 .destroy_inode = shmem_destroy_inode,
3987 #ifdef CONFIG_TMPFS
3988 .statfs = shmem_statfs,
3989 .show_options = shmem_show_options,
3990 #endif
3991 .evict_inode = shmem_evict_inode,
3992 .drop_inode = generic_delete_inode,
3993 .put_super = shmem_put_super,
3994 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3995 .nr_cached_objects = shmem_unused_huge_count,
3996 .free_cached_objects = shmem_unused_huge_scan,
3997 #endif
3998 };
3999
4000 static const struct vm_operations_struct shmem_vm_ops = {
4001 .fault = shmem_fault,
4002 .map_pages = filemap_map_pages,
4003 #ifdef CONFIG_NUMA
4004 .set_policy = shmem_set_policy,
4005 .get_policy = shmem_get_policy,
4006 #endif
4007 };
4008
shmem_init_fs_context(struct fs_context * fc)4009 int shmem_init_fs_context(struct fs_context *fc)
4010 {
4011 struct shmem_options *ctx;
4012
4013 ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
4014 if (!ctx)
4015 return -ENOMEM;
4016
4017 ctx->mode = 0777 | S_ISVTX;
4018 ctx->uid = current_fsuid();
4019 ctx->gid = current_fsgid();
4020
4021 fc->fs_private = ctx;
4022 fc->ops = &shmem_fs_context_ops;
4023 return 0;
4024 }
4025
4026 static struct file_system_type shmem_fs_type = {
4027 .owner = THIS_MODULE,
4028 .name = "tmpfs",
4029 .init_fs_context = shmem_init_fs_context,
4030 #ifdef CONFIG_TMPFS
4031 .parameters = shmem_fs_parameters,
4032 #endif
4033 .kill_sb = kill_litter_super,
4034 .fs_flags = FS_USERNS_MOUNT,
4035 };
4036
shmem_init(void)4037 void __init shmem_init(void)
4038 {
4039 int error;
4040
4041 shmem_init_inodecache();
4042
4043 error = register_filesystem(&shmem_fs_type);
4044 if (error) {
4045 pr_err("Could not register tmpfs\n");
4046 goto out2;
4047 }
4048
4049 shm_mnt = kern_mount(&shmem_fs_type);
4050 if (IS_ERR(shm_mnt)) {
4051 error = PTR_ERR(shm_mnt);
4052 pr_err("Could not kern_mount tmpfs\n");
4053 goto out1;
4054 }
4055
4056 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4057 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4058 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4059 else
4060 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
4061 #endif
4062 return;
4063
4064 out1:
4065 unregister_filesystem(&shmem_fs_type);
4066 out2:
4067 shmem_destroy_inodecache();
4068 shm_mnt = ERR_PTR(error);
4069 }
4070
4071 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
shmem_enabled_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)4072 static ssize_t shmem_enabled_show(struct kobject *kobj,
4073 struct kobj_attribute *attr, char *buf)
4074 {
4075 static const int values[] = {
4076 SHMEM_HUGE_ALWAYS,
4077 SHMEM_HUGE_WITHIN_SIZE,
4078 SHMEM_HUGE_ADVISE,
4079 SHMEM_HUGE_NEVER,
4080 SHMEM_HUGE_DENY,
4081 SHMEM_HUGE_FORCE,
4082 };
4083 int len = 0;
4084 int i;
4085
4086 for (i = 0; i < ARRAY_SIZE(values); i++) {
4087 len += sysfs_emit_at(buf, len,
4088 shmem_huge == values[i] ? "%s[%s]" : "%s%s",
4089 i ? " " : "",
4090 shmem_format_huge(values[i]));
4091 }
4092
4093 len += sysfs_emit_at(buf, len, "\n");
4094
4095 return len;
4096 }
4097
shmem_enabled_store(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)4098 static ssize_t shmem_enabled_store(struct kobject *kobj,
4099 struct kobj_attribute *attr, const char *buf, size_t count)
4100 {
4101 char tmp[16];
4102 int huge;
4103
4104 if (count + 1 > sizeof(tmp))
4105 return -EINVAL;
4106 memcpy(tmp, buf, count);
4107 tmp[count] = '\0';
4108 if (count && tmp[count - 1] == '\n')
4109 tmp[count - 1] = '\0';
4110
4111 huge = shmem_parse_huge(tmp);
4112 if (huge == -EINVAL)
4113 return -EINVAL;
4114 if (!has_transparent_hugepage() &&
4115 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4116 return -EINVAL;
4117
4118 shmem_huge = huge;
4119 if (shmem_huge > SHMEM_HUGE_DENY)
4120 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4121 return count;
4122 }
4123
4124 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
4125 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4126
4127 #else /* !CONFIG_SHMEM */
4128
4129 /*
4130 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
4131 *
4132 * This is intended for small system where the benefits of the full
4133 * shmem code (swap-backed and resource-limited) are outweighed by
4134 * their complexity. On systems without swap this code should be
4135 * effectively equivalent, but much lighter weight.
4136 */
4137
4138 static struct file_system_type shmem_fs_type = {
4139 .name = "tmpfs",
4140 .init_fs_context = ramfs_init_fs_context,
4141 .parameters = ramfs_fs_parameters,
4142 .kill_sb = kill_litter_super,
4143 .fs_flags = FS_USERNS_MOUNT,
4144 };
4145
shmem_init(void)4146 void __init shmem_init(void)
4147 {
4148 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4149
4150 shm_mnt = kern_mount(&shmem_fs_type);
4151 BUG_ON(IS_ERR(shm_mnt));
4152 }
4153
shmem_unuse(unsigned int type)4154 int shmem_unuse(unsigned int type)
4155 {
4156 return 0;
4157 }
4158
shmem_lock(struct file * file,int lock,struct ucounts * ucounts)4159 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
4160 {
4161 return 0;
4162 }
4163
shmem_unlock_mapping(struct address_space * mapping)4164 void shmem_unlock_mapping(struct address_space *mapping)
4165 {
4166 }
4167
4168 #ifdef CONFIG_MMU
shmem_get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)4169 unsigned long shmem_get_unmapped_area(struct file *file,
4170 unsigned long addr, unsigned long len,
4171 unsigned long pgoff, unsigned long flags)
4172 {
4173 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4174 }
4175 #endif
4176
shmem_truncate_range(struct inode * inode,loff_t lstart,loff_t lend)4177 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4178 {
4179 truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4180 }
4181 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4182
4183 #define shmem_vm_ops generic_file_vm_ops
4184 #define shmem_file_operations ramfs_file_operations
4185 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
4186 #define shmem_acct_size(flags, size) 0
4187 #define shmem_unacct_size(flags, size) do {} while (0)
4188
4189 #endif /* CONFIG_SHMEM */
4190
4191 /* common code */
4192
__shmem_file_setup(struct vfsmount * mnt,const char * name,loff_t size,unsigned long flags,unsigned int i_flags)4193 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4194 unsigned long flags, unsigned int i_flags)
4195 {
4196 struct inode *inode;
4197 struct file *res;
4198
4199 if (IS_ERR(mnt))
4200 return ERR_CAST(mnt);
4201
4202 if (size < 0 || size > MAX_LFS_FILESIZE)
4203 return ERR_PTR(-EINVAL);
4204
4205 if (shmem_acct_size(flags, size))
4206 return ERR_PTR(-ENOMEM);
4207
4208 inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
4209 flags);
4210 if (unlikely(!inode)) {
4211 shmem_unacct_size(flags, size);
4212 return ERR_PTR(-ENOSPC);
4213 }
4214 inode->i_flags |= i_flags;
4215 inode->i_size = size;
4216 clear_nlink(inode); /* It is unlinked */
4217 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4218 if (!IS_ERR(res))
4219 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4220 &shmem_file_operations);
4221 if (IS_ERR(res))
4222 iput(inode);
4223 return res;
4224 }
4225
4226 /**
4227 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4228 * kernel internal. There will be NO LSM permission checks against the
4229 * underlying inode. So users of this interface must do LSM checks at a
4230 * higher layer. The users are the big_key and shm implementations. LSM
4231 * checks are provided at the key or shm level rather than the inode.
4232 * @name: name for dentry (to be seen in /proc/<pid>/maps
4233 * @size: size to be set for the file
4234 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4235 */
shmem_kernel_file_setup(const char * name,loff_t size,unsigned long flags)4236 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4237 {
4238 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4239 }
4240
4241 /**
4242 * shmem_file_setup - get an unlinked file living in tmpfs
4243 * @name: name for dentry (to be seen in /proc/<pid>/maps
4244 * @size: size to be set for the file
4245 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4246 */
shmem_file_setup(const char * name,loff_t size,unsigned long flags)4247 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4248 {
4249 return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4250 }
4251 EXPORT_SYMBOL_GPL(shmem_file_setup);
4252
4253 /**
4254 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4255 * @mnt: the tmpfs mount where the file will be created
4256 * @name: name for dentry (to be seen in /proc/<pid>/maps
4257 * @size: size to be set for the file
4258 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4259 */
shmem_file_setup_with_mnt(struct vfsmount * mnt,const char * name,loff_t size,unsigned long flags)4260 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4261 loff_t size, unsigned long flags)
4262 {
4263 return __shmem_file_setup(mnt, name, size, flags, 0);
4264 }
4265 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4266
4267 /**
4268 * shmem_zero_setup - setup a shared anonymous mapping
4269 * @vma: the vma to be mmapped is prepared by do_mmap
4270 */
shmem_zero_setup(struct vm_area_struct * vma)4271 int shmem_zero_setup(struct vm_area_struct *vma)
4272 {
4273 struct file *file;
4274 loff_t size = vma->vm_end - vma->vm_start;
4275
4276 /*
4277 * Cloning a new file under mmap_lock leads to a lock ordering conflict
4278 * between XFS directory reading and selinux: since this file is only
4279 * accessible to the user through its mapping, use S_PRIVATE flag to
4280 * bypass file security, in the same way as shmem_kernel_file_setup().
4281 */
4282 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4283 if (IS_ERR(file))
4284 return PTR_ERR(file);
4285
4286 if (vma->vm_file)
4287 fput(vma->vm_file);
4288 vma->vm_file = file;
4289 vma->vm_ops = &shmem_vm_ops;
4290
4291 return 0;
4292 }
4293
4294 /**
4295 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4296 * @mapping: the page's address_space
4297 * @index: the page index
4298 * @gfp: the page allocator flags to use if allocating
4299 *
4300 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4301 * with any new page allocations done using the specified allocation flags.
4302 * But read_cache_page_gfp() uses the ->read_folio() method: which does not
4303 * suit tmpfs, since it may have pages in swapcache, and needs to find those
4304 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4305 *
4306 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4307 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4308 */
shmem_read_mapping_page_gfp(struct address_space * mapping,pgoff_t index,gfp_t gfp)4309 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4310 pgoff_t index, gfp_t gfp)
4311 {
4312 #ifdef CONFIG_SHMEM
4313 struct inode *inode = mapping->host;
4314 struct folio *folio;
4315 struct page *page;
4316 int error;
4317
4318 BUG_ON(!shmem_mapping(mapping));
4319 error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
4320 gfp, NULL, NULL, NULL);
4321 if (error)
4322 return ERR_PTR(error);
4323
4324 folio_unlock(folio);
4325 page = folio_file_page(folio, index);
4326 if (PageHWPoison(page)) {
4327 folio_put(folio);
4328 return ERR_PTR(-EIO);
4329 }
4330
4331 return page;
4332 #else
4333 /*
4334 * The tiny !SHMEM case uses ramfs without swap
4335 */
4336 return read_cache_page_gfp(mapping, index, gfp);
4337 #endif
4338 }
4339 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
4340