1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *		 2000 Transmeta Corp.
6  *		 2000-2001 Christoph Rohland
7  *		 2000-2001 SAP AG
8  *		 2002 Red Hat Inc.
9  * Copyright (C) 2002-2011 Hugh Dickins.
10  * Copyright (C) 2011 Google Inc.
11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
13  *
14  * Extended attribute support for tmpfs:
15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17  *
18  * tiny-shmem:
19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20  *
21  * This file is released under the GPL.
22  */
23 
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/fileattr.h>
32 #include <linux/mm.h>
33 #include <linux/random.h>
34 #include <linux/sched/signal.h>
35 #include <linux/export.h>
36 #include <linux/swap.h>
37 #include <linux/uio.h>
38 #include <linux/hugetlb.h>
39 #include <linux/fs_parser.h>
40 #include <linux/swapfile.h>
41 #include <linux/iversion.h>
42 #include "swap.h"
43 
44 static struct vfsmount *shm_mnt;
45 
46 #ifdef CONFIG_SHMEM
47 /*
48  * This virtual memory filesystem is heavily based on the ramfs. It
49  * extends ramfs by the ability to use swap and honor resource limits
50  * which makes it a completely usable filesystem.
51  */
52 
53 #include <linux/xattr.h>
54 #include <linux/exportfs.h>
55 #include <linux/posix_acl.h>
56 #include <linux/posix_acl_xattr.h>
57 #include <linux/mman.h>
58 #include <linux/string.h>
59 #include <linux/slab.h>
60 #include <linux/backing-dev.h>
61 #include <linux/shmem_fs.h>
62 #include <linux/writeback.h>
63 #include <linux/pagevec.h>
64 #include <linux/percpu_counter.h>
65 #include <linux/falloc.h>
66 #include <linux/splice.h>
67 #include <linux/security.h>
68 #include <linux/swapops.h>
69 #include <linux/mempolicy.h>
70 #include <linux/namei.h>
71 #include <linux/ctype.h>
72 #include <linux/migrate.h>
73 #include <linux/highmem.h>
74 #include <linux/seq_file.h>
75 #include <linux/magic.h>
76 #include <linux/syscalls.h>
77 #include <linux/fcntl.h>
78 #include <uapi/linux/memfd.h>
79 #include <linux/userfaultfd_k.h>
80 #include <linux/rmap.h>
81 #include <linux/uuid.h>
82 
83 #include <linux/uaccess.h>
84 
85 #include "internal.h"
86 
87 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
88 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
89 
90 /* Pretend that each entry is of this size in directory's i_size */
91 #define BOGO_DIRENT_SIZE 20
92 
93 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
94 #define SHORT_SYMLINK_LEN 128
95 
96 /*
97  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
98  * inode->i_private (with i_rwsem making sure that it has only one user at
99  * a time): we would prefer not to enlarge the shmem inode just for that.
100  */
101 struct shmem_falloc {
102 	wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
103 	pgoff_t start;		/* start of range currently being fallocated */
104 	pgoff_t next;		/* the next page offset to be fallocated */
105 	pgoff_t nr_falloced;	/* how many new pages have been fallocated */
106 	pgoff_t nr_unswapped;	/* how often writepage refused to swap out */
107 };
108 
109 struct shmem_options {
110 	unsigned long long blocks;
111 	unsigned long long inodes;
112 	struct mempolicy *mpol;
113 	kuid_t uid;
114 	kgid_t gid;
115 	umode_t mode;
116 	bool full_inums;
117 	int huge;
118 	int seen;
119 #define SHMEM_SEEN_BLOCKS 1
120 #define SHMEM_SEEN_INODES 2
121 #define SHMEM_SEEN_HUGE 4
122 #define SHMEM_SEEN_INUMS 8
123 };
124 
125 #ifdef CONFIG_TMPFS
shmem_default_max_blocks(void)126 static unsigned long shmem_default_max_blocks(void)
127 {
128 	return totalram_pages() / 2;
129 }
130 
shmem_default_max_inodes(void)131 static unsigned long shmem_default_max_inodes(void)
132 {
133 	unsigned long nr_pages = totalram_pages();
134 
135 	return min(nr_pages - totalhigh_pages(), nr_pages / 2);
136 }
137 #endif
138 
139 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
140 			     struct folio **foliop, enum sgp_type sgp,
141 			     gfp_t gfp, struct vm_area_struct *vma,
142 			     vm_fault_t *fault_type);
143 
SHMEM_SB(struct super_block * sb)144 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
145 {
146 	return sb->s_fs_info;
147 }
148 
149 /*
150  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
151  * for shared memory and for shared anonymous (/dev/zero) mappings
152  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
153  * consistent with the pre-accounting of private mappings ...
154  */
shmem_acct_size(unsigned long flags,loff_t size)155 static inline int shmem_acct_size(unsigned long flags, loff_t size)
156 {
157 	return (flags & VM_NORESERVE) ?
158 		0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
159 }
160 
shmem_unacct_size(unsigned long flags,loff_t size)161 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
162 {
163 	if (!(flags & VM_NORESERVE))
164 		vm_unacct_memory(VM_ACCT(size));
165 }
166 
shmem_reacct_size(unsigned long flags,loff_t oldsize,loff_t newsize)167 static inline int shmem_reacct_size(unsigned long flags,
168 		loff_t oldsize, loff_t newsize)
169 {
170 	if (!(flags & VM_NORESERVE)) {
171 		if (VM_ACCT(newsize) > VM_ACCT(oldsize))
172 			return security_vm_enough_memory_mm(current->mm,
173 					VM_ACCT(newsize) - VM_ACCT(oldsize));
174 		else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
175 			vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
176 	}
177 	return 0;
178 }
179 
180 /*
181  * ... whereas tmpfs objects are accounted incrementally as
182  * pages are allocated, in order to allow large sparse files.
183  * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
184  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
185  */
shmem_acct_block(unsigned long flags,long pages)186 static inline int shmem_acct_block(unsigned long flags, long pages)
187 {
188 	if (!(flags & VM_NORESERVE))
189 		return 0;
190 
191 	return security_vm_enough_memory_mm(current->mm,
192 			pages * VM_ACCT(PAGE_SIZE));
193 }
194 
shmem_unacct_blocks(unsigned long flags,long pages)195 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
196 {
197 	if (flags & VM_NORESERVE)
198 		vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
199 }
200 
shmem_inode_acct_block(struct inode * inode,long pages)201 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
202 {
203 	struct shmem_inode_info *info = SHMEM_I(inode);
204 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
205 
206 	if (shmem_acct_block(info->flags, pages))
207 		return false;
208 
209 	if (sbinfo->max_blocks) {
210 		if (percpu_counter_compare(&sbinfo->used_blocks,
211 					   sbinfo->max_blocks - pages) > 0)
212 			goto unacct;
213 		percpu_counter_add(&sbinfo->used_blocks, pages);
214 	}
215 
216 	return true;
217 
218 unacct:
219 	shmem_unacct_blocks(info->flags, pages);
220 	return false;
221 }
222 
shmem_inode_unacct_blocks(struct inode * inode,long pages)223 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
224 {
225 	struct shmem_inode_info *info = SHMEM_I(inode);
226 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
227 
228 	if (sbinfo->max_blocks)
229 		percpu_counter_sub(&sbinfo->used_blocks, pages);
230 	shmem_unacct_blocks(info->flags, pages);
231 }
232 
233 static const struct super_operations shmem_ops;
234 const struct address_space_operations shmem_aops;
235 static const struct file_operations shmem_file_operations;
236 static const struct inode_operations shmem_inode_operations;
237 static const struct inode_operations shmem_dir_inode_operations;
238 static const struct inode_operations shmem_special_inode_operations;
239 static const struct vm_operations_struct shmem_vm_ops;
240 static struct file_system_type shmem_fs_type;
241 
vma_is_shmem(struct vm_area_struct * vma)242 bool vma_is_shmem(struct vm_area_struct *vma)
243 {
244 	return vma->vm_ops == &shmem_vm_ops;
245 }
246 
247 static LIST_HEAD(shmem_swaplist);
248 static DEFINE_MUTEX(shmem_swaplist_mutex);
249 
250 /*
251  * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
252  * produces a novel ino for the newly allocated inode.
253  *
254  * It may also be called when making a hard link to permit the space needed by
255  * each dentry. However, in that case, no new inode number is needed since that
256  * internally draws from another pool of inode numbers (currently global
257  * get_next_ino()). This case is indicated by passing NULL as inop.
258  */
259 #define SHMEM_INO_BATCH 1024
shmem_reserve_inode(struct super_block * sb,ino_t * inop)260 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
261 {
262 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
263 	ino_t ino;
264 
265 	if (!(sb->s_flags & SB_KERNMOUNT)) {
266 		raw_spin_lock(&sbinfo->stat_lock);
267 		if (sbinfo->max_inodes) {
268 			if (!sbinfo->free_inodes) {
269 				raw_spin_unlock(&sbinfo->stat_lock);
270 				return -ENOSPC;
271 			}
272 			sbinfo->free_inodes--;
273 		}
274 		if (inop) {
275 			ino = sbinfo->next_ino++;
276 			if (unlikely(is_zero_ino(ino)))
277 				ino = sbinfo->next_ino++;
278 			if (unlikely(!sbinfo->full_inums &&
279 				     ino > UINT_MAX)) {
280 				/*
281 				 * Emulate get_next_ino uint wraparound for
282 				 * compatibility
283 				 */
284 				if (IS_ENABLED(CONFIG_64BIT))
285 					pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
286 						__func__, MINOR(sb->s_dev));
287 				sbinfo->next_ino = 1;
288 				ino = sbinfo->next_ino++;
289 			}
290 			*inop = ino;
291 		}
292 		raw_spin_unlock(&sbinfo->stat_lock);
293 	} else if (inop) {
294 		/*
295 		 * __shmem_file_setup, one of our callers, is lock-free: it
296 		 * doesn't hold stat_lock in shmem_reserve_inode since
297 		 * max_inodes is always 0, and is called from potentially
298 		 * unknown contexts. As such, use a per-cpu batched allocator
299 		 * which doesn't require the per-sb stat_lock unless we are at
300 		 * the batch boundary.
301 		 *
302 		 * We don't need to worry about inode{32,64} since SB_KERNMOUNT
303 		 * shmem mounts are not exposed to userspace, so we don't need
304 		 * to worry about things like glibc compatibility.
305 		 */
306 		ino_t *next_ino;
307 
308 		next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
309 		ino = *next_ino;
310 		if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
311 			raw_spin_lock(&sbinfo->stat_lock);
312 			ino = sbinfo->next_ino;
313 			sbinfo->next_ino += SHMEM_INO_BATCH;
314 			raw_spin_unlock(&sbinfo->stat_lock);
315 			if (unlikely(is_zero_ino(ino)))
316 				ino++;
317 		}
318 		*inop = ino;
319 		*next_ino = ++ino;
320 		put_cpu();
321 	}
322 
323 	return 0;
324 }
325 
shmem_free_inode(struct super_block * sb)326 static void shmem_free_inode(struct super_block *sb)
327 {
328 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
329 	if (sbinfo->max_inodes) {
330 		raw_spin_lock(&sbinfo->stat_lock);
331 		sbinfo->free_inodes++;
332 		raw_spin_unlock(&sbinfo->stat_lock);
333 	}
334 }
335 
336 /**
337  * shmem_recalc_inode - recalculate the block usage of an inode
338  * @inode: inode to recalc
339  *
340  * We have to calculate the free blocks since the mm can drop
341  * undirtied hole pages behind our back.
342  *
343  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
344  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
345  *
346  * It has to be called with the spinlock held.
347  */
shmem_recalc_inode(struct inode * inode)348 static void shmem_recalc_inode(struct inode *inode)
349 {
350 	struct shmem_inode_info *info = SHMEM_I(inode);
351 	long freed;
352 
353 	freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
354 	if (freed > 0) {
355 		info->alloced -= freed;
356 		inode->i_blocks -= freed * BLOCKS_PER_PAGE;
357 		shmem_inode_unacct_blocks(inode, freed);
358 	}
359 }
360 
shmem_charge(struct inode * inode,long pages)361 bool shmem_charge(struct inode *inode, long pages)
362 {
363 	struct shmem_inode_info *info = SHMEM_I(inode);
364 	unsigned long flags;
365 
366 	if (!shmem_inode_acct_block(inode, pages))
367 		return false;
368 
369 	/* nrpages adjustment first, then shmem_recalc_inode() when balanced */
370 	inode->i_mapping->nrpages += pages;
371 
372 	spin_lock_irqsave(&info->lock, flags);
373 	info->alloced += pages;
374 	inode->i_blocks += pages * BLOCKS_PER_PAGE;
375 	shmem_recalc_inode(inode);
376 	spin_unlock_irqrestore(&info->lock, flags);
377 
378 	return true;
379 }
380 
shmem_uncharge(struct inode * inode,long pages)381 void shmem_uncharge(struct inode *inode, long pages)
382 {
383 	struct shmem_inode_info *info = SHMEM_I(inode);
384 	unsigned long flags;
385 
386 	/* nrpages adjustment done by __filemap_remove_folio() or caller */
387 
388 	spin_lock_irqsave(&info->lock, flags);
389 	info->alloced -= pages;
390 	inode->i_blocks -= pages * BLOCKS_PER_PAGE;
391 	shmem_recalc_inode(inode);
392 	spin_unlock_irqrestore(&info->lock, flags);
393 
394 	shmem_inode_unacct_blocks(inode, pages);
395 }
396 
397 /*
398  * Replace item expected in xarray by a new item, while holding xa_lock.
399  */
shmem_replace_entry(struct address_space * mapping,pgoff_t index,void * expected,void * replacement)400 static int shmem_replace_entry(struct address_space *mapping,
401 			pgoff_t index, void *expected, void *replacement)
402 {
403 	XA_STATE(xas, &mapping->i_pages, index);
404 	void *item;
405 
406 	VM_BUG_ON(!expected);
407 	VM_BUG_ON(!replacement);
408 	item = xas_load(&xas);
409 	if (item != expected)
410 		return -ENOENT;
411 	xas_store(&xas, replacement);
412 	return 0;
413 }
414 
415 /*
416  * Sometimes, before we decide whether to proceed or to fail, we must check
417  * that an entry was not already brought back from swap by a racing thread.
418  *
419  * Checking page is not enough: by the time a SwapCache page is locked, it
420  * might be reused, and again be SwapCache, using the same swap as before.
421  */
shmem_confirm_swap(struct address_space * mapping,pgoff_t index,swp_entry_t swap)422 static bool shmem_confirm_swap(struct address_space *mapping,
423 			       pgoff_t index, swp_entry_t swap)
424 {
425 	return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
426 }
427 
428 /*
429  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
430  *
431  * SHMEM_HUGE_NEVER:
432  *	disables huge pages for the mount;
433  * SHMEM_HUGE_ALWAYS:
434  *	enables huge pages for the mount;
435  * SHMEM_HUGE_WITHIN_SIZE:
436  *	only allocate huge pages if the page will be fully within i_size,
437  *	also respect fadvise()/madvise() hints;
438  * SHMEM_HUGE_ADVISE:
439  *	only allocate huge pages if requested with fadvise()/madvise();
440  */
441 
442 #define SHMEM_HUGE_NEVER	0
443 #define SHMEM_HUGE_ALWAYS	1
444 #define SHMEM_HUGE_WITHIN_SIZE	2
445 #define SHMEM_HUGE_ADVISE	3
446 
447 /*
448  * Special values.
449  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
450  *
451  * SHMEM_HUGE_DENY:
452  *	disables huge on shm_mnt and all mounts, for emergency use;
453  * SHMEM_HUGE_FORCE:
454  *	enables huge on shm_mnt and all mounts, w/o needing option, for testing;
455  *
456  */
457 #define SHMEM_HUGE_DENY		(-1)
458 #define SHMEM_HUGE_FORCE	(-2)
459 
460 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
461 /* ifdef here to avoid bloating shmem.o when not necessary */
462 
463 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
464 
shmem_is_huge(struct vm_area_struct * vma,struct inode * inode,pgoff_t index,bool shmem_huge_force)465 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
466 		   pgoff_t index, bool shmem_huge_force)
467 {
468 	loff_t i_size;
469 
470 	if (!S_ISREG(inode->i_mode))
471 		return false;
472 	if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) ||
473 	    test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)))
474 		return false;
475 	if (shmem_huge_force)
476 		return true;
477 	if (shmem_huge == SHMEM_HUGE_FORCE)
478 		return true;
479 	if (shmem_huge == SHMEM_HUGE_DENY)
480 		return false;
481 
482 	switch (SHMEM_SB(inode->i_sb)->huge) {
483 	case SHMEM_HUGE_ALWAYS:
484 		return true;
485 	case SHMEM_HUGE_WITHIN_SIZE:
486 		index = round_up(index + 1, HPAGE_PMD_NR);
487 		i_size = round_up(i_size_read(inode), PAGE_SIZE);
488 		if (i_size >> PAGE_SHIFT >= index)
489 			return true;
490 		fallthrough;
491 	case SHMEM_HUGE_ADVISE:
492 		if (vma && (vma->vm_flags & VM_HUGEPAGE))
493 			return true;
494 		fallthrough;
495 	default:
496 		return false;
497 	}
498 }
499 
500 #if defined(CONFIG_SYSFS)
shmem_parse_huge(const char * str)501 static int shmem_parse_huge(const char *str)
502 {
503 	if (!strcmp(str, "never"))
504 		return SHMEM_HUGE_NEVER;
505 	if (!strcmp(str, "always"))
506 		return SHMEM_HUGE_ALWAYS;
507 	if (!strcmp(str, "within_size"))
508 		return SHMEM_HUGE_WITHIN_SIZE;
509 	if (!strcmp(str, "advise"))
510 		return SHMEM_HUGE_ADVISE;
511 	if (!strcmp(str, "deny"))
512 		return SHMEM_HUGE_DENY;
513 	if (!strcmp(str, "force"))
514 		return SHMEM_HUGE_FORCE;
515 	return -EINVAL;
516 }
517 #endif
518 
519 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
shmem_format_huge(int huge)520 static const char *shmem_format_huge(int huge)
521 {
522 	switch (huge) {
523 	case SHMEM_HUGE_NEVER:
524 		return "never";
525 	case SHMEM_HUGE_ALWAYS:
526 		return "always";
527 	case SHMEM_HUGE_WITHIN_SIZE:
528 		return "within_size";
529 	case SHMEM_HUGE_ADVISE:
530 		return "advise";
531 	case SHMEM_HUGE_DENY:
532 		return "deny";
533 	case SHMEM_HUGE_FORCE:
534 		return "force";
535 	default:
536 		VM_BUG_ON(1);
537 		return "bad_val";
538 	}
539 }
540 #endif
541 
shmem_unused_huge_shrink(struct shmem_sb_info * sbinfo,struct shrink_control * sc,unsigned long nr_to_split)542 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
543 		struct shrink_control *sc, unsigned long nr_to_split)
544 {
545 	LIST_HEAD(list), *pos, *next;
546 	LIST_HEAD(to_remove);
547 	struct inode *inode;
548 	struct shmem_inode_info *info;
549 	struct folio *folio;
550 	unsigned long batch = sc ? sc->nr_to_scan : 128;
551 	int split = 0;
552 
553 	if (list_empty(&sbinfo->shrinklist))
554 		return SHRINK_STOP;
555 
556 	spin_lock(&sbinfo->shrinklist_lock);
557 	list_for_each_safe(pos, next, &sbinfo->shrinklist) {
558 		info = list_entry(pos, struct shmem_inode_info, shrinklist);
559 
560 		/* pin the inode */
561 		inode = igrab(&info->vfs_inode);
562 
563 		/* inode is about to be evicted */
564 		if (!inode) {
565 			list_del_init(&info->shrinklist);
566 			goto next;
567 		}
568 
569 		/* Check if there's anything to gain */
570 		if (round_up(inode->i_size, PAGE_SIZE) ==
571 				round_up(inode->i_size, HPAGE_PMD_SIZE)) {
572 			list_move(&info->shrinklist, &to_remove);
573 			goto next;
574 		}
575 
576 		list_move(&info->shrinklist, &list);
577 next:
578 		sbinfo->shrinklist_len--;
579 		if (!--batch)
580 			break;
581 	}
582 	spin_unlock(&sbinfo->shrinklist_lock);
583 
584 	list_for_each_safe(pos, next, &to_remove) {
585 		info = list_entry(pos, struct shmem_inode_info, shrinklist);
586 		inode = &info->vfs_inode;
587 		list_del_init(&info->shrinklist);
588 		iput(inode);
589 	}
590 
591 	list_for_each_safe(pos, next, &list) {
592 		int ret;
593 		pgoff_t index;
594 
595 		info = list_entry(pos, struct shmem_inode_info, shrinklist);
596 		inode = &info->vfs_inode;
597 
598 		if (nr_to_split && split >= nr_to_split)
599 			goto move_back;
600 
601 		index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
602 		folio = filemap_get_folio(inode->i_mapping, index);
603 		if (!folio)
604 			goto drop;
605 
606 		/* No huge page at the end of the file: nothing to split */
607 		if (!folio_test_large(folio)) {
608 			folio_put(folio);
609 			goto drop;
610 		}
611 
612 		/*
613 		 * Move the inode on the list back to shrinklist if we failed
614 		 * to lock the page at this time.
615 		 *
616 		 * Waiting for the lock may lead to deadlock in the
617 		 * reclaim path.
618 		 */
619 		if (!folio_trylock(folio)) {
620 			folio_put(folio);
621 			goto move_back;
622 		}
623 
624 		ret = split_folio(folio);
625 		folio_unlock(folio);
626 		folio_put(folio);
627 
628 		/* If split failed move the inode on the list back to shrinklist */
629 		if (ret)
630 			goto move_back;
631 
632 		split++;
633 drop:
634 		list_del_init(&info->shrinklist);
635 		goto put;
636 move_back:
637 		/*
638 		 * Make sure the inode is either on the global list or deleted
639 		 * from any local list before iput() since it could be deleted
640 		 * in another thread once we put the inode (then the local list
641 		 * is corrupted).
642 		 */
643 		spin_lock(&sbinfo->shrinklist_lock);
644 		list_move(&info->shrinklist, &sbinfo->shrinklist);
645 		sbinfo->shrinklist_len++;
646 		spin_unlock(&sbinfo->shrinklist_lock);
647 put:
648 		iput(inode);
649 	}
650 
651 	return split;
652 }
653 
shmem_unused_huge_scan(struct super_block * sb,struct shrink_control * sc)654 static long shmem_unused_huge_scan(struct super_block *sb,
655 		struct shrink_control *sc)
656 {
657 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
658 
659 	if (!READ_ONCE(sbinfo->shrinklist_len))
660 		return SHRINK_STOP;
661 
662 	return shmem_unused_huge_shrink(sbinfo, sc, 0);
663 }
664 
shmem_unused_huge_count(struct super_block * sb,struct shrink_control * sc)665 static long shmem_unused_huge_count(struct super_block *sb,
666 		struct shrink_control *sc)
667 {
668 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
669 	return READ_ONCE(sbinfo->shrinklist_len);
670 }
671 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
672 
673 #define shmem_huge SHMEM_HUGE_DENY
674 
shmem_is_huge(struct vm_area_struct * vma,struct inode * inode,pgoff_t index,bool shmem_huge_force)675 bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
676 		   pgoff_t index, bool shmem_huge_force)
677 {
678 	return false;
679 }
680 
shmem_unused_huge_shrink(struct shmem_sb_info * sbinfo,struct shrink_control * sc,unsigned long nr_to_split)681 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
682 		struct shrink_control *sc, unsigned long nr_to_split)
683 {
684 	return 0;
685 }
686 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
687 
688 /*
689  * Like filemap_add_folio, but error if expected item has gone.
690  */
shmem_add_to_page_cache(struct folio * folio,struct address_space * mapping,pgoff_t index,void * expected,gfp_t gfp,struct mm_struct * charge_mm)691 static int shmem_add_to_page_cache(struct folio *folio,
692 				   struct address_space *mapping,
693 				   pgoff_t index, void *expected, gfp_t gfp,
694 				   struct mm_struct *charge_mm)
695 {
696 	XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
697 	long nr = folio_nr_pages(folio);
698 	int error;
699 
700 	VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
701 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
702 	VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
703 	VM_BUG_ON(expected && folio_test_large(folio));
704 
705 	folio_ref_add(folio, nr);
706 	folio->mapping = mapping;
707 	folio->index = index;
708 
709 	if (!folio_test_swapcache(folio)) {
710 		error = mem_cgroup_charge(folio, charge_mm, gfp);
711 		if (error) {
712 			if (folio_test_pmd_mappable(folio)) {
713 				count_vm_event(THP_FILE_FALLBACK);
714 				count_vm_event(THP_FILE_FALLBACK_CHARGE);
715 			}
716 			goto error;
717 		}
718 	}
719 	folio_throttle_swaprate(folio, gfp);
720 
721 	do {
722 		xas_lock_irq(&xas);
723 		if (expected != xas_find_conflict(&xas)) {
724 			xas_set_err(&xas, -EEXIST);
725 			goto unlock;
726 		}
727 		if (expected && xas_find_conflict(&xas)) {
728 			xas_set_err(&xas, -EEXIST);
729 			goto unlock;
730 		}
731 		xas_store(&xas, folio);
732 		if (xas_error(&xas))
733 			goto unlock;
734 		if (folio_test_pmd_mappable(folio)) {
735 			count_vm_event(THP_FILE_ALLOC);
736 			__lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
737 		}
738 		mapping->nrpages += nr;
739 		__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
740 		__lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
741 unlock:
742 		xas_unlock_irq(&xas);
743 	} while (xas_nomem(&xas, gfp));
744 
745 	if (xas_error(&xas)) {
746 		error = xas_error(&xas);
747 		goto error;
748 	}
749 
750 	return 0;
751 error:
752 	folio->mapping = NULL;
753 	folio_ref_sub(folio, nr);
754 	return error;
755 }
756 
757 /*
758  * Like delete_from_page_cache, but substitutes swap for @folio.
759  */
shmem_delete_from_page_cache(struct folio * folio,void * radswap)760 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
761 {
762 	struct address_space *mapping = folio->mapping;
763 	long nr = folio_nr_pages(folio);
764 	int error;
765 
766 	xa_lock_irq(&mapping->i_pages);
767 	error = shmem_replace_entry(mapping, folio->index, folio, radswap);
768 	folio->mapping = NULL;
769 	mapping->nrpages -= nr;
770 	__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
771 	__lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
772 	xa_unlock_irq(&mapping->i_pages);
773 	folio_put(folio);
774 	BUG_ON(error);
775 }
776 
777 /*
778  * Remove swap entry from page cache, free the swap and its page cache.
779  */
shmem_free_swap(struct address_space * mapping,pgoff_t index,void * radswap)780 static int shmem_free_swap(struct address_space *mapping,
781 			   pgoff_t index, void *radswap)
782 {
783 	void *old;
784 
785 	old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
786 	if (old != radswap)
787 		return -ENOENT;
788 	free_swap_and_cache(radix_to_swp_entry(radswap));
789 	return 0;
790 }
791 
792 /*
793  * Determine (in bytes) how many of the shmem object's pages mapped by the
794  * given offsets are swapped out.
795  *
796  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
797  * as long as the inode doesn't go away and racy results are not a problem.
798  */
shmem_partial_swap_usage(struct address_space * mapping,pgoff_t start,pgoff_t end)799 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
800 						pgoff_t start, pgoff_t end)
801 {
802 	XA_STATE(xas, &mapping->i_pages, start);
803 	struct page *page;
804 	unsigned long swapped = 0;
805 
806 	rcu_read_lock();
807 	xas_for_each(&xas, page, end - 1) {
808 		if (xas_retry(&xas, page))
809 			continue;
810 		if (xa_is_value(page))
811 			swapped++;
812 
813 		if (need_resched()) {
814 			xas_pause(&xas);
815 			cond_resched_rcu();
816 		}
817 	}
818 
819 	rcu_read_unlock();
820 
821 	return swapped << PAGE_SHIFT;
822 }
823 
824 /*
825  * Determine (in bytes) how many of the shmem object's pages mapped by the
826  * given vma is swapped out.
827  *
828  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
829  * as long as the inode doesn't go away and racy results are not a problem.
830  */
shmem_swap_usage(struct vm_area_struct * vma)831 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
832 {
833 	struct inode *inode = file_inode(vma->vm_file);
834 	struct shmem_inode_info *info = SHMEM_I(inode);
835 	struct address_space *mapping = inode->i_mapping;
836 	unsigned long swapped;
837 
838 	/* Be careful as we don't hold info->lock */
839 	swapped = READ_ONCE(info->swapped);
840 
841 	/*
842 	 * The easier cases are when the shmem object has nothing in swap, or
843 	 * the vma maps it whole. Then we can simply use the stats that we
844 	 * already track.
845 	 */
846 	if (!swapped)
847 		return 0;
848 
849 	if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
850 		return swapped << PAGE_SHIFT;
851 
852 	/* Here comes the more involved part */
853 	return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
854 					vma->vm_pgoff + vma_pages(vma));
855 }
856 
857 /*
858  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
859  */
shmem_unlock_mapping(struct address_space * mapping)860 void shmem_unlock_mapping(struct address_space *mapping)
861 {
862 	struct folio_batch fbatch;
863 	pgoff_t index = 0;
864 
865 	folio_batch_init(&fbatch);
866 	/*
867 	 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
868 	 */
869 	while (!mapping_unevictable(mapping) &&
870 	       filemap_get_folios(mapping, &index, ~0UL, &fbatch)) {
871 		check_move_unevictable_folios(&fbatch);
872 		folio_batch_release(&fbatch);
873 		cond_resched();
874 	}
875 }
876 
shmem_get_partial_folio(struct inode * inode,pgoff_t index)877 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
878 {
879 	struct folio *folio;
880 
881 	/*
882 	 * At first avoid shmem_get_folio(,,,SGP_READ): that fails
883 	 * beyond i_size, and reports fallocated pages as holes.
884 	 */
885 	folio = __filemap_get_folio(inode->i_mapping, index,
886 					FGP_ENTRY | FGP_LOCK, 0);
887 	if (!xa_is_value(folio))
888 		return folio;
889 	/*
890 	 * But read a page back from swap if any of it is within i_size
891 	 * (although in some cases this is just a waste of time).
892 	 */
893 	folio = NULL;
894 	shmem_get_folio(inode, index, &folio, SGP_READ);
895 	return folio;
896 }
897 
898 /*
899  * Remove range of pages and swap entries from page cache, and free them.
900  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
901  */
shmem_undo_range(struct inode * inode,loff_t lstart,loff_t lend,bool unfalloc)902 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
903 								 bool unfalloc)
904 {
905 	struct address_space *mapping = inode->i_mapping;
906 	struct shmem_inode_info *info = SHMEM_I(inode);
907 	pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
908 	pgoff_t end = (lend + 1) >> PAGE_SHIFT;
909 	struct folio_batch fbatch;
910 	pgoff_t indices[PAGEVEC_SIZE];
911 	struct folio *folio;
912 	bool same_folio;
913 	long nr_swaps_freed = 0;
914 	pgoff_t index;
915 	int i;
916 
917 	if (lend == -1)
918 		end = -1;	/* unsigned, so actually very big */
919 
920 	if (info->fallocend > start && info->fallocend <= end && !unfalloc)
921 		info->fallocend = start;
922 
923 	folio_batch_init(&fbatch);
924 	index = start;
925 	while (index < end && find_lock_entries(mapping, index, end - 1,
926 			&fbatch, indices)) {
927 		for (i = 0; i < folio_batch_count(&fbatch); i++) {
928 			folio = fbatch.folios[i];
929 
930 			index = indices[i];
931 
932 			if (xa_is_value(folio)) {
933 				if (unfalloc)
934 					continue;
935 				nr_swaps_freed += !shmem_free_swap(mapping,
936 								index, folio);
937 				continue;
938 			}
939 			index += folio_nr_pages(folio) - 1;
940 
941 			if (!unfalloc || !folio_test_uptodate(folio))
942 				truncate_inode_folio(mapping, folio);
943 			folio_unlock(folio);
944 		}
945 		folio_batch_remove_exceptionals(&fbatch);
946 		folio_batch_release(&fbatch);
947 		cond_resched();
948 		index++;
949 	}
950 
951 	/*
952 	 * When undoing a failed fallocate, we want none of the partial folio
953 	 * zeroing and splitting below, but shall want to truncate the whole
954 	 * folio when !uptodate indicates that it was added by this fallocate,
955 	 * even when [lstart, lend] covers only a part of the folio.
956 	 */
957 	if (unfalloc)
958 		goto whole_folios;
959 
960 	same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
961 	folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
962 	if (folio) {
963 		same_folio = lend < folio_pos(folio) + folio_size(folio);
964 		folio_mark_dirty(folio);
965 		if (!truncate_inode_partial_folio(folio, lstart, lend)) {
966 			start = folio->index + folio_nr_pages(folio);
967 			if (same_folio)
968 				end = folio->index;
969 		}
970 		folio_unlock(folio);
971 		folio_put(folio);
972 		folio = NULL;
973 	}
974 
975 	if (!same_folio)
976 		folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
977 	if (folio) {
978 		folio_mark_dirty(folio);
979 		if (!truncate_inode_partial_folio(folio, lstart, lend))
980 			end = folio->index;
981 		folio_unlock(folio);
982 		folio_put(folio);
983 	}
984 
985 whole_folios:
986 
987 	index = start;
988 	while (index < end) {
989 		cond_resched();
990 
991 		if (!find_get_entries(mapping, index, end - 1, &fbatch,
992 				indices)) {
993 			/* If all gone or hole-punch or unfalloc, we're done */
994 			if (index == start || end != -1)
995 				break;
996 			/* But if truncating, restart to make sure all gone */
997 			index = start;
998 			continue;
999 		}
1000 		for (i = 0; i < folio_batch_count(&fbatch); i++) {
1001 			folio = fbatch.folios[i];
1002 
1003 			index = indices[i];
1004 			if (xa_is_value(folio)) {
1005 				if (unfalloc)
1006 					continue;
1007 				if (shmem_free_swap(mapping, index, folio)) {
1008 					/* Swap was replaced by page: retry */
1009 					index--;
1010 					break;
1011 				}
1012 				nr_swaps_freed++;
1013 				continue;
1014 			}
1015 
1016 			folio_lock(folio);
1017 
1018 			if (!unfalloc || !folio_test_uptodate(folio)) {
1019 				if (folio_mapping(folio) != mapping) {
1020 					/* Page was replaced by swap: retry */
1021 					folio_unlock(folio);
1022 					index--;
1023 					break;
1024 				}
1025 				VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1026 						folio);
1027 				truncate_inode_folio(mapping, folio);
1028 			}
1029 			index = folio->index + folio_nr_pages(folio) - 1;
1030 			folio_unlock(folio);
1031 		}
1032 		folio_batch_remove_exceptionals(&fbatch);
1033 		folio_batch_release(&fbatch);
1034 		index++;
1035 	}
1036 
1037 	spin_lock_irq(&info->lock);
1038 	info->swapped -= nr_swaps_freed;
1039 	shmem_recalc_inode(inode);
1040 	spin_unlock_irq(&info->lock);
1041 }
1042 
shmem_truncate_range(struct inode * inode,loff_t lstart,loff_t lend)1043 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1044 {
1045 	shmem_undo_range(inode, lstart, lend, false);
1046 	inode->i_ctime = inode->i_mtime = current_time(inode);
1047 	inode_inc_iversion(inode);
1048 }
1049 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1050 
shmem_getattr(struct user_namespace * mnt_userns,const struct path * path,struct kstat * stat,u32 request_mask,unsigned int query_flags)1051 static int shmem_getattr(struct user_namespace *mnt_userns,
1052 			 const struct path *path, struct kstat *stat,
1053 			 u32 request_mask, unsigned int query_flags)
1054 {
1055 	struct inode *inode = path->dentry->d_inode;
1056 	struct shmem_inode_info *info = SHMEM_I(inode);
1057 
1058 	if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1059 		spin_lock_irq(&info->lock);
1060 		shmem_recalc_inode(inode);
1061 		spin_unlock_irq(&info->lock);
1062 	}
1063 	if (info->fsflags & FS_APPEND_FL)
1064 		stat->attributes |= STATX_ATTR_APPEND;
1065 	if (info->fsflags & FS_IMMUTABLE_FL)
1066 		stat->attributes |= STATX_ATTR_IMMUTABLE;
1067 	if (info->fsflags & FS_NODUMP_FL)
1068 		stat->attributes |= STATX_ATTR_NODUMP;
1069 	stat->attributes_mask |= (STATX_ATTR_APPEND |
1070 			STATX_ATTR_IMMUTABLE |
1071 			STATX_ATTR_NODUMP);
1072 	generic_fillattr(&init_user_ns, inode, stat);
1073 
1074 	if (shmem_is_huge(NULL, inode, 0, false))
1075 		stat->blksize = HPAGE_PMD_SIZE;
1076 
1077 	if (request_mask & STATX_BTIME) {
1078 		stat->result_mask |= STATX_BTIME;
1079 		stat->btime.tv_sec = info->i_crtime.tv_sec;
1080 		stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1081 	}
1082 
1083 	return 0;
1084 }
1085 
shmem_setattr(struct user_namespace * mnt_userns,struct dentry * dentry,struct iattr * attr)1086 static int shmem_setattr(struct user_namespace *mnt_userns,
1087 			 struct dentry *dentry, struct iattr *attr)
1088 {
1089 	struct inode *inode = d_inode(dentry);
1090 	struct shmem_inode_info *info = SHMEM_I(inode);
1091 	int error;
1092 	bool update_mtime = false;
1093 	bool update_ctime = true;
1094 
1095 	error = setattr_prepare(&init_user_ns, dentry, attr);
1096 	if (error)
1097 		return error;
1098 
1099 	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1100 		loff_t oldsize = inode->i_size;
1101 		loff_t newsize = attr->ia_size;
1102 
1103 		/* protected by i_rwsem */
1104 		if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1105 		    (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1106 			return -EPERM;
1107 
1108 		if (newsize != oldsize) {
1109 			error = shmem_reacct_size(SHMEM_I(inode)->flags,
1110 					oldsize, newsize);
1111 			if (error)
1112 				return error;
1113 			i_size_write(inode, newsize);
1114 			update_mtime = true;
1115 		} else {
1116 			update_ctime = false;
1117 		}
1118 		if (newsize <= oldsize) {
1119 			loff_t holebegin = round_up(newsize, PAGE_SIZE);
1120 			if (oldsize > holebegin)
1121 				unmap_mapping_range(inode->i_mapping,
1122 							holebegin, 0, 1);
1123 			if (info->alloced)
1124 				shmem_truncate_range(inode,
1125 							newsize, (loff_t)-1);
1126 			/* unmap again to remove racily COWed private pages */
1127 			if (oldsize > holebegin)
1128 				unmap_mapping_range(inode->i_mapping,
1129 							holebegin, 0, 1);
1130 		}
1131 	}
1132 
1133 	setattr_copy(&init_user_ns, inode, attr);
1134 	if (attr->ia_valid & ATTR_MODE)
1135 		error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode);
1136 	if (!error && update_ctime) {
1137 		inode->i_ctime = current_time(inode);
1138 		if (update_mtime)
1139 			inode->i_mtime = inode->i_ctime;
1140 		inode_inc_iversion(inode);
1141 	}
1142 	return error;
1143 }
1144 
shmem_evict_inode(struct inode * inode)1145 static void shmem_evict_inode(struct inode *inode)
1146 {
1147 	struct shmem_inode_info *info = SHMEM_I(inode);
1148 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1149 
1150 	if (shmem_mapping(inode->i_mapping)) {
1151 		shmem_unacct_size(info->flags, inode->i_size);
1152 		inode->i_size = 0;
1153 		mapping_set_exiting(inode->i_mapping);
1154 		shmem_truncate_range(inode, 0, (loff_t)-1);
1155 		if (!list_empty(&info->shrinklist)) {
1156 			spin_lock(&sbinfo->shrinklist_lock);
1157 			if (!list_empty(&info->shrinklist)) {
1158 				list_del_init(&info->shrinklist);
1159 				sbinfo->shrinklist_len--;
1160 			}
1161 			spin_unlock(&sbinfo->shrinklist_lock);
1162 		}
1163 		while (!list_empty(&info->swaplist)) {
1164 			/* Wait while shmem_unuse() is scanning this inode... */
1165 			wait_var_event(&info->stop_eviction,
1166 				       !atomic_read(&info->stop_eviction));
1167 			mutex_lock(&shmem_swaplist_mutex);
1168 			/* ...but beware of the race if we peeked too early */
1169 			if (!atomic_read(&info->stop_eviction))
1170 				list_del_init(&info->swaplist);
1171 			mutex_unlock(&shmem_swaplist_mutex);
1172 		}
1173 	}
1174 
1175 	simple_xattrs_free(&info->xattrs);
1176 	WARN_ON(inode->i_blocks);
1177 	shmem_free_inode(inode->i_sb);
1178 	clear_inode(inode);
1179 }
1180 
shmem_find_swap_entries(struct address_space * mapping,pgoff_t start,struct folio_batch * fbatch,pgoff_t * indices,unsigned int type)1181 static int shmem_find_swap_entries(struct address_space *mapping,
1182 				   pgoff_t start, struct folio_batch *fbatch,
1183 				   pgoff_t *indices, unsigned int type)
1184 {
1185 	XA_STATE(xas, &mapping->i_pages, start);
1186 	struct folio *folio;
1187 	swp_entry_t entry;
1188 
1189 	rcu_read_lock();
1190 	xas_for_each(&xas, folio, ULONG_MAX) {
1191 		if (xas_retry(&xas, folio))
1192 			continue;
1193 
1194 		if (!xa_is_value(folio))
1195 			continue;
1196 
1197 		entry = radix_to_swp_entry(folio);
1198 		/*
1199 		 * swapin error entries can be found in the mapping. But they're
1200 		 * deliberately ignored here as we've done everything we can do.
1201 		 */
1202 		if (swp_type(entry) != type)
1203 			continue;
1204 
1205 		indices[folio_batch_count(fbatch)] = xas.xa_index;
1206 		if (!folio_batch_add(fbatch, folio))
1207 			break;
1208 
1209 		if (need_resched()) {
1210 			xas_pause(&xas);
1211 			cond_resched_rcu();
1212 		}
1213 	}
1214 	rcu_read_unlock();
1215 
1216 	return xas.xa_index;
1217 }
1218 
1219 /*
1220  * Move the swapped pages for an inode to page cache. Returns the count
1221  * of pages swapped in, or the error in case of failure.
1222  */
shmem_unuse_swap_entries(struct inode * inode,struct folio_batch * fbatch,pgoff_t * indices)1223 static int shmem_unuse_swap_entries(struct inode *inode,
1224 		struct folio_batch *fbatch, pgoff_t *indices)
1225 {
1226 	int i = 0;
1227 	int ret = 0;
1228 	int error = 0;
1229 	struct address_space *mapping = inode->i_mapping;
1230 
1231 	for (i = 0; i < folio_batch_count(fbatch); i++) {
1232 		struct folio *folio = fbatch->folios[i];
1233 
1234 		if (!xa_is_value(folio))
1235 			continue;
1236 		error = shmem_swapin_folio(inode, indices[i],
1237 					  &folio, SGP_CACHE,
1238 					  mapping_gfp_mask(mapping),
1239 					  NULL, NULL);
1240 		if (error == 0) {
1241 			folio_unlock(folio);
1242 			folio_put(folio);
1243 			ret++;
1244 		}
1245 		if (error == -ENOMEM)
1246 			break;
1247 		error = 0;
1248 	}
1249 	return error ? error : ret;
1250 }
1251 
1252 /*
1253  * If swap found in inode, free it and move page from swapcache to filecache.
1254  */
shmem_unuse_inode(struct inode * inode,unsigned int type)1255 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1256 {
1257 	struct address_space *mapping = inode->i_mapping;
1258 	pgoff_t start = 0;
1259 	struct folio_batch fbatch;
1260 	pgoff_t indices[PAGEVEC_SIZE];
1261 	int ret = 0;
1262 
1263 	do {
1264 		folio_batch_init(&fbatch);
1265 		shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1266 		if (folio_batch_count(&fbatch) == 0) {
1267 			ret = 0;
1268 			break;
1269 		}
1270 
1271 		ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1272 		if (ret < 0)
1273 			break;
1274 
1275 		start = indices[folio_batch_count(&fbatch) - 1];
1276 	} while (true);
1277 
1278 	return ret;
1279 }
1280 
1281 /*
1282  * Read all the shared memory data that resides in the swap
1283  * device 'type' back into memory, so the swap device can be
1284  * unused.
1285  */
shmem_unuse(unsigned int type)1286 int shmem_unuse(unsigned int type)
1287 {
1288 	struct shmem_inode_info *info, *next;
1289 	int error = 0;
1290 
1291 	if (list_empty(&shmem_swaplist))
1292 		return 0;
1293 
1294 	mutex_lock(&shmem_swaplist_mutex);
1295 	list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1296 		if (!info->swapped) {
1297 			list_del_init(&info->swaplist);
1298 			continue;
1299 		}
1300 		/*
1301 		 * Drop the swaplist mutex while searching the inode for swap;
1302 		 * but before doing so, make sure shmem_evict_inode() will not
1303 		 * remove placeholder inode from swaplist, nor let it be freed
1304 		 * (igrab() would protect from unlink, but not from unmount).
1305 		 */
1306 		atomic_inc(&info->stop_eviction);
1307 		mutex_unlock(&shmem_swaplist_mutex);
1308 
1309 		error = shmem_unuse_inode(&info->vfs_inode, type);
1310 		cond_resched();
1311 
1312 		mutex_lock(&shmem_swaplist_mutex);
1313 		next = list_next_entry(info, swaplist);
1314 		if (!info->swapped)
1315 			list_del_init(&info->swaplist);
1316 		if (atomic_dec_and_test(&info->stop_eviction))
1317 			wake_up_var(&info->stop_eviction);
1318 		if (error)
1319 			break;
1320 	}
1321 	mutex_unlock(&shmem_swaplist_mutex);
1322 
1323 	return error;
1324 }
1325 
1326 /*
1327  * Move the page from the page cache to the swap cache.
1328  */
shmem_writepage(struct page * page,struct writeback_control * wbc)1329 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1330 {
1331 	struct folio *folio = page_folio(page);
1332 	struct shmem_inode_info *info;
1333 	struct address_space *mapping;
1334 	struct inode *inode;
1335 	swp_entry_t swap;
1336 	pgoff_t index;
1337 
1338 	/*
1339 	 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1340 	 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1341 	 * and its shmem_writeback() needs them to be split when swapping.
1342 	 */
1343 	if (folio_test_large(folio)) {
1344 		/* Ensure the subpages are still dirty */
1345 		folio_test_set_dirty(folio);
1346 		if (split_huge_page(page) < 0)
1347 			goto redirty;
1348 		folio = page_folio(page);
1349 		folio_clear_dirty(folio);
1350 	}
1351 
1352 	BUG_ON(!folio_test_locked(folio));
1353 	mapping = folio->mapping;
1354 	index = folio->index;
1355 	inode = mapping->host;
1356 	info = SHMEM_I(inode);
1357 	if (info->flags & VM_LOCKED)
1358 		goto redirty;
1359 	if (!total_swap_pages)
1360 		goto redirty;
1361 
1362 	/*
1363 	 * Our capabilities prevent regular writeback or sync from ever calling
1364 	 * shmem_writepage; but a stacking filesystem might use ->writepage of
1365 	 * its underlying filesystem, in which case tmpfs should write out to
1366 	 * swap only in response to memory pressure, and not for the writeback
1367 	 * threads or sync.
1368 	 */
1369 	if (!wbc->for_reclaim) {
1370 		WARN_ON_ONCE(1);	/* Still happens? Tell us about it! */
1371 		goto redirty;
1372 	}
1373 
1374 	/*
1375 	 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1376 	 * value into swapfile.c, the only way we can correctly account for a
1377 	 * fallocated folio arriving here is now to initialize it and write it.
1378 	 *
1379 	 * That's okay for a folio already fallocated earlier, but if we have
1380 	 * not yet completed the fallocation, then (a) we want to keep track
1381 	 * of this folio in case we have to undo it, and (b) it may not be a
1382 	 * good idea to continue anyway, once we're pushing into swap.  So
1383 	 * reactivate the folio, and let shmem_fallocate() quit when too many.
1384 	 */
1385 	if (!folio_test_uptodate(folio)) {
1386 		if (inode->i_private) {
1387 			struct shmem_falloc *shmem_falloc;
1388 			spin_lock(&inode->i_lock);
1389 			shmem_falloc = inode->i_private;
1390 			if (shmem_falloc &&
1391 			    !shmem_falloc->waitq &&
1392 			    index >= shmem_falloc->start &&
1393 			    index < shmem_falloc->next)
1394 				shmem_falloc->nr_unswapped++;
1395 			else
1396 				shmem_falloc = NULL;
1397 			spin_unlock(&inode->i_lock);
1398 			if (shmem_falloc)
1399 				goto redirty;
1400 		}
1401 		folio_zero_range(folio, 0, folio_size(folio));
1402 		flush_dcache_folio(folio);
1403 		folio_mark_uptodate(folio);
1404 	}
1405 
1406 	swap = folio_alloc_swap(folio);
1407 	if (!swap.val)
1408 		goto redirty;
1409 
1410 	/*
1411 	 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1412 	 * if it's not already there.  Do it now before the folio is
1413 	 * moved to swap cache, when its pagelock no longer protects
1414 	 * the inode from eviction.  But don't unlock the mutex until
1415 	 * we've incremented swapped, because shmem_unuse_inode() will
1416 	 * prune a !swapped inode from the swaplist under this mutex.
1417 	 */
1418 	mutex_lock(&shmem_swaplist_mutex);
1419 	if (list_empty(&info->swaplist))
1420 		list_add(&info->swaplist, &shmem_swaplist);
1421 
1422 	if (add_to_swap_cache(folio, swap,
1423 			__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1424 			NULL) == 0) {
1425 		spin_lock_irq(&info->lock);
1426 		shmem_recalc_inode(inode);
1427 		info->swapped++;
1428 		spin_unlock_irq(&info->lock);
1429 
1430 		swap_shmem_alloc(swap);
1431 		shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
1432 
1433 		mutex_unlock(&shmem_swaplist_mutex);
1434 		BUG_ON(folio_mapped(folio));
1435 		swap_writepage(&folio->page, wbc);
1436 		return 0;
1437 	}
1438 
1439 	mutex_unlock(&shmem_swaplist_mutex);
1440 	put_swap_folio(folio, swap);
1441 redirty:
1442 	folio_mark_dirty(folio);
1443 	if (wbc->for_reclaim)
1444 		return AOP_WRITEPAGE_ACTIVATE;	/* Return with folio locked */
1445 	folio_unlock(folio);
1446 	return 0;
1447 }
1448 
1449 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
shmem_show_mpol(struct seq_file * seq,struct mempolicy * mpol)1450 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1451 {
1452 	char buffer[64];
1453 
1454 	if (!mpol || mpol->mode == MPOL_DEFAULT)
1455 		return;		/* show nothing */
1456 
1457 	mpol_to_str(buffer, sizeof(buffer), mpol);
1458 
1459 	seq_printf(seq, ",mpol=%s", buffer);
1460 }
1461 
shmem_get_sbmpol(struct shmem_sb_info * sbinfo)1462 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1463 {
1464 	struct mempolicy *mpol = NULL;
1465 	if (sbinfo->mpol) {
1466 		raw_spin_lock(&sbinfo->stat_lock);	/* prevent replace/use races */
1467 		mpol = sbinfo->mpol;
1468 		mpol_get(mpol);
1469 		raw_spin_unlock(&sbinfo->stat_lock);
1470 	}
1471 	return mpol;
1472 }
1473 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
shmem_show_mpol(struct seq_file * seq,struct mempolicy * mpol)1474 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1475 {
1476 }
shmem_get_sbmpol(struct shmem_sb_info * sbinfo)1477 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1478 {
1479 	return NULL;
1480 }
1481 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1482 #ifndef CONFIG_NUMA
1483 #define vm_policy vm_private_data
1484 #endif
1485 
shmem_pseudo_vma_init(struct vm_area_struct * vma,struct shmem_inode_info * info,pgoff_t index)1486 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1487 		struct shmem_inode_info *info, pgoff_t index)
1488 {
1489 	/* Create a pseudo vma that just contains the policy */
1490 	vma_init(vma, NULL);
1491 	/* Bias interleave by inode number to distribute better across nodes */
1492 	vma->vm_pgoff = index + info->vfs_inode.i_ino;
1493 	vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1494 }
1495 
shmem_pseudo_vma_destroy(struct vm_area_struct * vma)1496 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1497 {
1498 	/* Drop reference taken by mpol_shared_policy_lookup() */
1499 	mpol_cond_put(vma->vm_policy);
1500 }
1501 
shmem_swapin(swp_entry_t swap,gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1502 static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1503 			struct shmem_inode_info *info, pgoff_t index)
1504 {
1505 	struct vm_area_struct pvma;
1506 	struct page *page;
1507 	struct vm_fault vmf = {
1508 		.vma = &pvma,
1509 	};
1510 
1511 	shmem_pseudo_vma_init(&pvma, info, index);
1512 	page = swap_cluster_readahead(swap, gfp, &vmf);
1513 	shmem_pseudo_vma_destroy(&pvma);
1514 
1515 	if (!page)
1516 		return NULL;
1517 	return page_folio(page);
1518 }
1519 
1520 /*
1521  * Make sure huge_gfp is always more limited than limit_gfp.
1522  * Some of the flags set permissions, while others set limitations.
1523  */
limit_gfp_mask(gfp_t huge_gfp,gfp_t limit_gfp)1524 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1525 {
1526 	gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1527 	gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1528 	gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1529 	gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1530 
1531 	/* Allow allocations only from the originally specified zones. */
1532 	result |= zoneflags;
1533 
1534 	/*
1535 	 * Minimize the result gfp by taking the union with the deny flags,
1536 	 * and the intersection of the allow flags.
1537 	 */
1538 	result |= (limit_gfp & denyflags);
1539 	result |= (huge_gfp & limit_gfp) & allowflags;
1540 
1541 	return result;
1542 }
1543 
shmem_alloc_hugefolio(gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1544 static struct folio *shmem_alloc_hugefolio(gfp_t gfp,
1545 		struct shmem_inode_info *info, pgoff_t index)
1546 {
1547 	struct vm_area_struct pvma;
1548 	struct address_space *mapping = info->vfs_inode.i_mapping;
1549 	pgoff_t hindex;
1550 	struct folio *folio;
1551 
1552 	hindex = round_down(index, HPAGE_PMD_NR);
1553 	if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1554 								XA_PRESENT))
1555 		return NULL;
1556 
1557 	shmem_pseudo_vma_init(&pvma, info, hindex);
1558 	folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1559 	shmem_pseudo_vma_destroy(&pvma);
1560 	if (!folio)
1561 		count_vm_event(THP_FILE_FALLBACK);
1562 	return folio;
1563 }
1564 
shmem_alloc_folio(gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1565 static struct folio *shmem_alloc_folio(gfp_t gfp,
1566 			struct shmem_inode_info *info, pgoff_t index)
1567 {
1568 	struct vm_area_struct pvma;
1569 	struct folio *folio;
1570 
1571 	shmem_pseudo_vma_init(&pvma, info, index);
1572 	folio = vma_alloc_folio(gfp, 0, &pvma, 0, false);
1573 	shmem_pseudo_vma_destroy(&pvma);
1574 
1575 	return folio;
1576 }
1577 
shmem_alloc_and_acct_folio(gfp_t gfp,struct inode * inode,pgoff_t index,bool huge)1578 static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode,
1579 		pgoff_t index, bool huge)
1580 {
1581 	struct shmem_inode_info *info = SHMEM_I(inode);
1582 	struct folio *folio;
1583 	int nr;
1584 	int err = -ENOSPC;
1585 
1586 	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1587 		huge = false;
1588 	nr = huge ? HPAGE_PMD_NR : 1;
1589 
1590 	if (!shmem_inode_acct_block(inode, nr))
1591 		goto failed;
1592 
1593 	if (huge)
1594 		folio = shmem_alloc_hugefolio(gfp, info, index);
1595 	else
1596 		folio = shmem_alloc_folio(gfp, info, index);
1597 	if (folio) {
1598 		__folio_set_locked(folio);
1599 		__folio_set_swapbacked(folio);
1600 		return folio;
1601 	}
1602 
1603 	err = -ENOMEM;
1604 	shmem_inode_unacct_blocks(inode, nr);
1605 failed:
1606 	return ERR_PTR(err);
1607 }
1608 
1609 /*
1610  * When a page is moved from swapcache to shmem filecache (either by the
1611  * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
1612  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1613  * ignorance of the mapping it belongs to.  If that mapping has special
1614  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1615  * we may need to copy to a suitable page before moving to filecache.
1616  *
1617  * In a future release, this may well be extended to respect cpuset and
1618  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1619  * but for now it is a simple matter of zone.
1620  */
shmem_should_replace_folio(struct folio * folio,gfp_t gfp)1621 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1622 {
1623 	return folio_zonenum(folio) > gfp_zone(gfp);
1624 }
1625 
shmem_replace_folio(struct folio ** foliop,gfp_t gfp,struct shmem_inode_info * info,pgoff_t index)1626 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
1627 				struct shmem_inode_info *info, pgoff_t index)
1628 {
1629 	struct folio *old, *new;
1630 	struct address_space *swap_mapping;
1631 	swp_entry_t entry;
1632 	pgoff_t swap_index;
1633 	int error;
1634 
1635 	old = *foliop;
1636 	entry = folio_swap_entry(old);
1637 	swap_index = swp_offset(entry);
1638 	swap_mapping = swap_address_space(entry);
1639 
1640 	/*
1641 	 * We have arrived here because our zones are constrained, so don't
1642 	 * limit chance of success by further cpuset and node constraints.
1643 	 */
1644 	gfp &= ~GFP_CONSTRAINT_MASK;
1645 	VM_BUG_ON_FOLIO(folio_test_large(old), old);
1646 	new = shmem_alloc_folio(gfp, info, index);
1647 	if (!new)
1648 		return -ENOMEM;
1649 
1650 	folio_get(new);
1651 	folio_copy(new, old);
1652 	flush_dcache_folio(new);
1653 
1654 	__folio_set_locked(new);
1655 	__folio_set_swapbacked(new);
1656 	folio_mark_uptodate(new);
1657 	folio_set_swap_entry(new, entry);
1658 	folio_set_swapcache(new);
1659 
1660 	/*
1661 	 * Our caller will very soon move newpage out of swapcache, but it's
1662 	 * a nice clean interface for us to replace oldpage by newpage there.
1663 	 */
1664 	xa_lock_irq(&swap_mapping->i_pages);
1665 	error = shmem_replace_entry(swap_mapping, swap_index, old, new);
1666 	if (!error) {
1667 		mem_cgroup_migrate(old, new);
1668 		__lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
1669 		__lruvec_stat_mod_folio(new, NR_SHMEM, 1);
1670 		__lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
1671 		__lruvec_stat_mod_folio(old, NR_SHMEM, -1);
1672 	}
1673 	xa_unlock_irq(&swap_mapping->i_pages);
1674 
1675 	if (unlikely(error)) {
1676 		/*
1677 		 * Is this possible?  I think not, now that our callers check
1678 		 * both PageSwapCache and page_private after getting page lock;
1679 		 * but be defensive.  Reverse old to newpage for clear and free.
1680 		 */
1681 		old = new;
1682 	} else {
1683 		folio_add_lru(new);
1684 		*foliop = new;
1685 	}
1686 
1687 	folio_clear_swapcache(old);
1688 	old->private = NULL;
1689 
1690 	folio_unlock(old);
1691 	folio_put_refs(old, 2);
1692 	return error;
1693 }
1694 
shmem_set_folio_swapin_error(struct inode * inode,pgoff_t index,struct folio * folio,swp_entry_t swap)1695 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
1696 					 struct folio *folio, swp_entry_t swap)
1697 {
1698 	struct address_space *mapping = inode->i_mapping;
1699 	struct shmem_inode_info *info = SHMEM_I(inode);
1700 	swp_entry_t swapin_error;
1701 	void *old;
1702 
1703 	swapin_error = make_swapin_error_entry(&folio->page);
1704 	old = xa_cmpxchg_irq(&mapping->i_pages, index,
1705 			     swp_to_radix_entry(swap),
1706 			     swp_to_radix_entry(swapin_error), 0);
1707 	if (old != swp_to_radix_entry(swap))
1708 		return;
1709 
1710 	folio_wait_writeback(folio);
1711 	delete_from_swap_cache(folio);
1712 	spin_lock_irq(&info->lock);
1713 	/*
1714 	 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks won't
1715 	 * be 0 when inode is released and thus trigger WARN_ON(inode->i_blocks) in
1716 	 * shmem_evict_inode.
1717 	 */
1718 	info->alloced--;
1719 	info->swapped--;
1720 	shmem_recalc_inode(inode);
1721 	spin_unlock_irq(&info->lock);
1722 	swap_free(swap);
1723 }
1724 
1725 /*
1726  * Swap in the folio pointed to by *foliop.
1727  * Caller has to make sure that *foliop contains a valid swapped folio.
1728  * Returns 0 and the folio in foliop if success. On failure, returns the
1729  * error code and NULL in *foliop.
1730  */
shmem_swapin_folio(struct inode * inode,pgoff_t index,struct folio ** foliop,enum sgp_type sgp,gfp_t gfp,struct vm_area_struct * vma,vm_fault_t * fault_type)1731 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1732 			     struct folio **foliop, enum sgp_type sgp,
1733 			     gfp_t gfp, struct vm_area_struct *vma,
1734 			     vm_fault_t *fault_type)
1735 {
1736 	struct address_space *mapping = inode->i_mapping;
1737 	struct shmem_inode_info *info = SHMEM_I(inode);
1738 	struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1739 	struct folio *folio = NULL;
1740 	swp_entry_t swap;
1741 	int error;
1742 
1743 	VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1744 	swap = radix_to_swp_entry(*foliop);
1745 	*foliop = NULL;
1746 
1747 	if (is_swapin_error_entry(swap))
1748 		return -EIO;
1749 
1750 	/* Look it up and read it in.. */
1751 	folio = swap_cache_get_folio(swap, NULL, 0);
1752 	if (!folio) {
1753 		/* Or update major stats only when swapin succeeds?? */
1754 		if (fault_type) {
1755 			*fault_type |= VM_FAULT_MAJOR;
1756 			count_vm_event(PGMAJFAULT);
1757 			count_memcg_event_mm(charge_mm, PGMAJFAULT);
1758 		}
1759 		/* Here we actually start the io */
1760 		folio = shmem_swapin(swap, gfp, info, index);
1761 		if (!folio) {
1762 			error = -ENOMEM;
1763 			goto failed;
1764 		}
1765 	}
1766 
1767 	/* We have to do this with folio locked to prevent races */
1768 	folio_lock(folio);
1769 	if (!folio_test_swapcache(folio) ||
1770 	    folio_swap_entry(folio).val != swap.val ||
1771 	    !shmem_confirm_swap(mapping, index, swap)) {
1772 		error = -EEXIST;
1773 		goto unlock;
1774 	}
1775 	if (!folio_test_uptodate(folio)) {
1776 		error = -EIO;
1777 		goto failed;
1778 	}
1779 	folio_wait_writeback(folio);
1780 
1781 	/*
1782 	 * Some architectures may have to restore extra metadata to the
1783 	 * folio after reading from swap.
1784 	 */
1785 	arch_swap_restore(swap, folio);
1786 
1787 	if (shmem_should_replace_folio(folio, gfp)) {
1788 		error = shmem_replace_folio(&folio, gfp, info, index);
1789 		if (error)
1790 			goto failed;
1791 	}
1792 
1793 	error = shmem_add_to_page_cache(folio, mapping, index,
1794 					swp_to_radix_entry(swap), gfp,
1795 					charge_mm);
1796 	if (error)
1797 		goto failed;
1798 
1799 	spin_lock_irq(&info->lock);
1800 	info->swapped--;
1801 	shmem_recalc_inode(inode);
1802 	spin_unlock_irq(&info->lock);
1803 
1804 	if (sgp == SGP_WRITE)
1805 		folio_mark_accessed(folio);
1806 
1807 	delete_from_swap_cache(folio);
1808 	folio_mark_dirty(folio);
1809 	swap_free(swap);
1810 
1811 	*foliop = folio;
1812 	return 0;
1813 failed:
1814 	if (!shmem_confirm_swap(mapping, index, swap))
1815 		error = -EEXIST;
1816 	if (error == -EIO)
1817 		shmem_set_folio_swapin_error(inode, index, folio, swap);
1818 unlock:
1819 	if (folio) {
1820 		folio_unlock(folio);
1821 		folio_put(folio);
1822 	}
1823 
1824 	return error;
1825 }
1826 
1827 /*
1828  * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
1829  *
1830  * If we allocate a new one we do not mark it dirty. That's up to the
1831  * vm. If we swap it in we mark it dirty since we also free the swap
1832  * entry since a page cannot live in both the swap and page cache.
1833  *
1834  * vma, vmf, and fault_type are only supplied by shmem_fault:
1835  * otherwise they are NULL.
1836  */
shmem_get_folio_gfp(struct inode * inode,pgoff_t index,struct folio ** foliop,enum sgp_type sgp,gfp_t gfp,struct vm_area_struct * vma,struct vm_fault * vmf,vm_fault_t * fault_type)1837 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
1838 		struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
1839 		struct vm_area_struct *vma, struct vm_fault *vmf,
1840 		vm_fault_t *fault_type)
1841 {
1842 	struct address_space *mapping = inode->i_mapping;
1843 	struct shmem_inode_info *info = SHMEM_I(inode);
1844 	struct shmem_sb_info *sbinfo;
1845 	struct mm_struct *charge_mm;
1846 	struct folio *folio;
1847 	pgoff_t hindex = index;
1848 	gfp_t huge_gfp;
1849 	int error;
1850 	int once = 0;
1851 	int alloced = 0;
1852 
1853 	if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1854 		return -EFBIG;
1855 repeat:
1856 	if (sgp <= SGP_CACHE &&
1857 	    ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1858 		return -EINVAL;
1859 	}
1860 
1861 	sbinfo = SHMEM_SB(inode->i_sb);
1862 	charge_mm = vma ? vma->vm_mm : NULL;
1863 
1864 	folio = __filemap_get_folio(mapping, index, FGP_ENTRY | FGP_LOCK, 0);
1865 	if (folio && vma && userfaultfd_minor(vma)) {
1866 		if (!xa_is_value(folio)) {
1867 			folio_unlock(folio);
1868 			folio_put(folio);
1869 		}
1870 		*fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1871 		return 0;
1872 	}
1873 
1874 	if (xa_is_value(folio)) {
1875 		error = shmem_swapin_folio(inode, index, &folio,
1876 					  sgp, gfp, vma, fault_type);
1877 		if (error == -EEXIST)
1878 			goto repeat;
1879 
1880 		*foliop = folio;
1881 		return error;
1882 	}
1883 
1884 	if (folio) {
1885 		hindex = folio->index;
1886 		if (sgp == SGP_WRITE)
1887 			folio_mark_accessed(folio);
1888 		if (folio_test_uptodate(folio))
1889 			goto out;
1890 		/* fallocated folio */
1891 		if (sgp != SGP_READ)
1892 			goto clear;
1893 		folio_unlock(folio);
1894 		folio_put(folio);
1895 	}
1896 
1897 	/*
1898 	 * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
1899 	 * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
1900 	 */
1901 	*foliop = NULL;
1902 	if (sgp == SGP_READ)
1903 		return 0;
1904 	if (sgp == SGP_NOALLOC)
1905 		return -ENOENT;
1906 
1907 	/*
1908 	 * Fast cache lookup and swap lookup did not find it: allocate.
1909 	 */
1910 
1911 	if (vma && userfaultfd_missing(vma)) {
1912 		*fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1913 		return 0;
1914 	}
1915 
1916 	if (!shmem_is_huge(vma, inode, index, false))
1917 		goto alloc_nohuge;
1918 
1919 	huge_gfp = vma_thp_gfp_mask(vma);
1920 	huge_gfp = limit_gfp_mask(huge_gfp, gfp);
1921 	folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true);
1922 	if (IS_ERR(folio)) {
1923 alloc_nohuge:
1924 		folio = shmem_alloc_and_acct_folio(gfp, inode, index, false);
1925 	}
1926 	if (IS_ERR(folio)) {
1927 		int retry = 5;
1928 
1929 		error = PTR_ERR(folio);
1930 		folio = NULL;
1931 		if (error != -ENOSPC)
1932 			goto unlock;
1933 		/*
1934 		 * Try to reclaim some space by splitting a large folio
1935 		 * beyond i_size on the filesystem.
1936 		 */
1937 		while (retry--) {
1938 			int ret;
1939 
1940 			ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1941 			if (ret == SHRINK_STOP)
1942 				break;
1943 			if (ret)
1944 				goto alloc_nohuge;
1945 		}
1946 		goto unlock;
1947 	}
1948 
1949 	hindex = round_down(index, folio_nr_pages(folio));
1950 
1951 	if (sgp == SGP_WRITE)
1952 		__folio_set_referenced(folio);
1953 
1954 	error = shmem_add_to_page_cache(folio, mapping, hindex,
1955 					NULL, gfp & GFP_RECLAIM_MASK,
1956 					charge_mm);
1957 	if (error)
1958 		goto unacct;
1959 	folio_add_lru(folio);
1960 
1961 	spin_lock_irq(&info->lock);
1962 	info->alloced += folio_nr_pages(folio);
1963 	inode->i_blocks += (blkcnt_t)BLOCKS_PER_PAGE << folio_order(folio);
1964 	shmem_recalc_inode(inode);
1965 	spin_unlock_irq(&info->lock);
1966 	alloced = true;
1967 
1968 	if (folio_test_pmd_mappable(folio) &&
1969 	    DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1970 					folio_next_index(folio) - 1) {
1971 		/*
1972 		 * Part of the large folio is beyond i_size: subject
1973 		 * to shrink under memory pressure.
1974 		 */
1975 		spin_lock(&sbinfo->shrinklist_lock);
1976 		/*
1977 		 * _careful to defend against unlocked access to
1978 		 * ->shrink_list in shmem_unused_huge_shrink()
1979 		 */
1980 		if (list_empty_careful(&info->shrinklist)) {
1981 			list_add_tail(&info->shrinklist,
1982 				      &sbinfo->shrinklist);
1983 			sbinfo->shrinklist_len++;
1984 		}
1985 		spin_unlock(&sbinfo->shrinklist_lock);
1986 	}
1987 
1988 	/*
1989 	 * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
1990 	 */
1991 	if (sgp == SGP_FALLOC)
1992 		sgp = SGP_WRITE;
1993 clear:
1994 	/*
1995 	 * Let SGP_WRITE caller clear ends if write does not fill folio;
1996 	 * but SGP_FALLOC on a folio fallocated earlier must initialize
1997 	 * it now, lest undo on failure cancel our earlier guarantee.
1998 	 */
1999 	if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
2000 		long i, n = folio_nr_pages(folio);
2001 
2002 		for (i = 0; i < n; i++)
2003 			clear_highpage(folio_page(folio, i));
2004 		flush_dcache_folio(folio);
2005 		folio_mark_uptodate(folio);
2006 	}
2007 
2008 	/* Perhaps the file has been truncated since we checked */
2009 	if (sgp <= SGP_CACHE &&
2010 	    ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
2011 		if (alloced) {
2012 			folio_clear_dirty(folio);
2013 			filemap_remove_folio(folio);
2014 			spin_lock_irq(&info->lock);
2015 			shmem_recalc_inode(inode);
2016 			spin_unlock_irq(&info->lock);
2017 		}
2018 		error = -EINVAL;
2019 		goto unlock;
2020 	}
2021 out:
2022 	*foliop = folio;
2023 	return 0;
2024 
2025 	/*
2026 	 * Error recovery.
2027 	 */
2028 unacct:
2029 	shmem_inode_unacct_blocks(inode, folio_nr_pages(folio));
2030 
2031 	if (folio_test_large(folio)) {
2032 		folio_unlock(folio);
2033 		folio_put(folio);
2034 		goto alloc_nohuge;
2035 	}
2036 unlock:
2037 	if (folio) {
2038 		folio_unlock(folio);
2039 		folio_put(folio);
2040 	}
2041 	if (error == -ENOSPC && !once++) {
2042 		spin_lock_irq(&info->lock);
2043 		shmem_recalc_inode(inode);
2044 		spin_unlock_irq(&info->lock);
2045 		goto repeat;
2046 	}
2047 	if (error == -EEXIST)
2048 		goto repeat;
2049 	return error;
2050 }
2051 
shmem_get_folio(struct inode * inode,pgoff_t index,struct folio ** foliop,enum sgp_type sgp)2052 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
2053 		enum sgp_type sgp)
2054 {
2055 	return shmem_get_folio_gfp(inode, index, foliop, sgp,
2056 			mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
2057 }
2058 
2059 /*
2060  * This is like autoremove_wake_function, but it removes the wait queue
2061  * entry unconditionally - even if something else had already woken the
2062  * target.
2063  */
synchronous_wake_function(wait_queue_entry_t * wait,unsigned mode,int sync,void * key)2064 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2065 {
2066 	int ret = default_wake_function(wait, mode, sync, key);
2067 	list_del_init(&wait->entry);
2068 	return ret;
2069 }
2070 
shmem_fault(struct vm_fault * vmf)2071 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2072 {
2073 	struct vm_area_struct *vma = vmf->vma;
2074 	struct inode *inode = file_inode(vma->vm_file);
2075 	gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2076 	struct folio *folio = NULL;
2077 	int err;
2078 	vm_fault_t ret = VM_FAULT_LOCKED;
2079 
2080 	/*
2081 	 * Trinity finds that probing a hole which tmpfs is punching can
2082 	 * prevent the hole-punch from ever completing: which in turn
2083 	 * locks writers out with its hold on i_rwsem.  So refrain from
2084 	 * faulting pages into the hole while it's being punched.  Although
2085 	 * shmem_undo_range() does remove the additions, it may be unable to
2086 	 * keep up, as each new page needs its own unmap_mapping_range() call,
2087 	 * and the i_mmap tree grows ever slower to scan if new vmas are added.
2088 	 *
2089 	 * It does not matter if we sometimes reach this check just before the
2090 	 * hole-punch begins, so that one fault then races with the punch:
2091 	 * we just need to make racing faults a rare case.
2092 	 *
2093 	 * The implementation below would be much simpler if we just used a
2094 	 * standard mutex or completion: but we cannot take i_rwsem in fault,
2095 	 * and bloating every shmem inode for this unlikely case would be sad.
2096 	 */
2097 	if (unlikely(inode->i_private)) {
2098 		struct shmem_falloc *shmem_falloc;
2099 
2100 		spin_lock(&inode->i_lock);
2101 		shmem_falloc = inode->i_private;
2102 		if (shmem_falloc &&
2103 		    shmem_falloc->waitq &&
2104 		    vmf->pgoff >= shmem_falloc->start &&
2105 		    vmf->pgoff < shmem_falloc->next) {
2106 			struct file *fpin;
2107 			wait_queue_head_t *shmem_falloc_waitq;
2108 			DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2109 
2110 			ret = VM_FAULT_NOPAGE;
2111 			fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2112 			if (fpin)
2113 				ret = VM_FAULT_RETRY;
2114 
2115 			shmem_falloc_waitq = shmem_falloc->waitq;
2116 			prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2117 					TASK_UNINTERRUPTIBLE);
2118 			spin_unlock(&inode->i_lock);
2119 			schedule();
2120 
2121 			/*
2122 			 * shmem_falloc_waitq points into the shmem_fallocate()
2123 			 * stack of the hole-punching task: shmem_falloc_waitq
2124 			 * is usually invalid by the time we reach here, but
2125 			 * finish_wait() does not dereference it in that case;
2126 			 * though i_lock needed lest racing with wake_up_all().
2127 			 */
2128 			spin_lock(&inode->i_lock);
2129 			finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2130 			spin_unlock(&inode->i_lock);
2131 
2132 			if (fpin)
2133 				fput(fpin);
2134 			return ret;
2135 		}
2136 		spin_unlock(&inode->i_lock);
2137 	}
2138 
2139 	err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
2140 				  gfp, vma, vmf, &ret);
2141 	if (err)
2142 		return vmf_error(err);
2143 	if (folio)
2144 		vmf->page = folio_file_page(folio, vmf->pgoff);
2145 	return ret;
2146 }
2147 
shmem_get_unmapped_area(struct file * file,unsigned long uaddr,unsigned long len,unsigned long pgoff,unsigned long flags)2148 unsigned long shmem_get_unmapped_area(struct file *file,
2149 				      unsigned long uaddr, unsigned long len,
2150 				      unsigned long pgoff, unsigned long flags)
2151 {
2152 	unsigned long (*get_area)(struct file *,
2153 		unsigned long, unsigned long, unsigned long, unsigned long);
2154 	unsigned long addr;
2155 	unsigned long offset;
2156 	unsigned long inflated_len;
2157 	unsigned long inflated_addr;
2158 	unsigned long inflated_offset;
2159 
2160 	if (len > TASK_SIZE)
2161 		return -ENOMEM;
2162 
2163 	get_area = current->mm->get_unmapped_area;
2164 	addr = get_area(file, uaddr, len, pgoff, flags);
2165 
2166 	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2167 		return addr;
2168 	if (IS_ERR_VALUE(addr))
2169 		return addr;
2170 	if (addr & ~PAGE_MASK)
2171 		return addr;
2172 	if (addr > TASK_SIZE - len)
2173 		return addr;
2174 
2175 	if (shmem_huge == SHMEM_HUGE_DENY)
2176 		return addr;
2177 	if (len < HPAGE_PMD_SIZE)
2178 		return addr;
2179 	if (flags & MAP_FIXED)
2180 		return addr;
2181 	/*
2182 	 * Our priority is to support MAP_SHARED mapped hugely;
2183 	 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2184 	 * But if caller specified an address hint and we allocated area there
2185 	 * successfully, respect that as before.
2186 	 */
2187 	if (uaddr == addr)
2188 		return addr;
2189 
2190 	if (shmem_huge != SHMEM_HUGE_FORCE) {
2191 		struct super_block *sb;
2192 
2193 		if (file) {
2194 			VM_BUG_ON(file->f_op != &shmem_file_operations);
2195 			sb = file_inode(file)->i_sb;
2196 		} else {
2197 			/*
2198 			 * Called directly from mm/mmap.c, or drivers/char/mem.c
2199 			 * for "/dev/zero", to create a shared anonymous object.
2200 			 */
2201 			if (IS_ERR(shm_mnt))
2202 				return addr;
2203 			sb = shm_mnt->mnt_sb;
2204 		}
2205 		if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2206 			return addr;
2207 	}
2208 
2209 	offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2210 	if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2211 		return addr;
2212 	if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2213 		return addr;
2214 
2215 	inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2216 	if (inflated_len > TASK_SIZE)
2217 		return addr;
2218 	if (inflated_len < len)
2219 		return addr;
2220 
2221 	inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2222 	if (IS_ERR_VALUE(inflated_addr))
2223 		return addr;
2224 	if (inflated_addr & ~PAGE_MASK)
2225 		return addr;
2226 
2227 	inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2228 	inflated_addr += offset - inflated_offset;
2229 	if (inflated_offset > offset)
2230 		inflated_addr += HPAGE_PMD_SIZE;
2231 
2232 	if (inflated_addr > TASK_SIZE - len)
2233 		return addr;
2234 	return inflated_addr;
2235 }
2236 
2237 #ifdef CONFIG_NUMA
shmem_set_policy(struct vm_area_struct * vma,struct mempolicy * mpol)2238 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2239 {
2240 	struct inode *inode = file_inode(vma->vm_file);
2241 	return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2242 }
2243 
shmem_get_policy(struct vm_area_struct * vma,unsigned long addr)2244 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2245 					  unsigned long addr)
2246 {
2247 	struct inode *inode = file_inode(vma->vm_file);
2248 	pgoff_t index;
2249 
2250 	index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2251 	return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2252 }
2253 #endif
2254 
shmem_lock(struct file * file,int lock,struct ucounts * ucounts)2255 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2256 {
2257 	struct inode *inode = file_inode(file);
2258 	struct shmem_inode_info *info = SHMEM_I(inode);
2259 	int retval = -ENOMEM;
2260 
2261 	/*
2262 	 * What serializes the accesses to info->flags?
2263 	 * ipc_lock_object() when called from shmctl_do_lock(),
2264 	 * no serialization needed when called from shm_destroy().
2265 	 */
2266 	if (lock && !(info->flags & VM_LOCKED)) {
2267 		if (!user_shm_lock(inode->i_size, ucounts))
2268 			goto out_nomem;
2269 		info->flags |= VM_LOCKED;
2270 		mapping_set_unevictable(file->f_mapping);
2271 	}
2272 	if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2273 		user_shm_unlock(inode->i_size, ucounts);
2274 		info->flags &= ~VM_LOCKED;
2275 		mapping_clear_unevictable(file->f_mapping);
2276 	}
2277 	retval = 0;
2278 
2279 out_nomem:
2280 	return retval;
2281 }
2282 
shmem_mmap(struct file * file,struct vm_area_struct * vma)2283 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2284 {
2285 	struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2286 	int ret;
2287 
2288 	ret = seal_check_future_write(info->seals, vma);
2289 	if (ret)
2290 		return ret;
2291 
2292 	/* arm64 - allow memory tagging on RAM-based files */
2293 	vma->vm_flags |= VM_MTE_ALLOWED;
2294 
2295 	file_accessed(file);
2296 	vma->vm_ops = &shmem_vm_ops;
2297 	return 0;
2298 }
2299 
2300 #ifdef CONFIG_TMPFS_XATTR
2301 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2302 
2303 /*
2304  * chattr's fsflags are unrelated to extended attributes,
2305  * but tmpfs has chosen to enable them under the same config option.
2306  */
shmem_set_inode_flags(struct inode * inode,unsigned int fsflags)2307 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2308 {
2309 	unsigned int i_flags = 0;
2310 
2311 	if (fsflags & FS_NOATIME_FL)
2312 		i_flags |= S_NOATIME;
2313 	if (fsflags & FS_APPEND_FL)
2314 		i_flags |= S_APPEND;
2315 	if (fsflags & FS_IMMUTABLE_FL)
2316 		i_flags |= S_IMMUTABLE;
2317 	/*
2318 	 * But FS_NODUMP_FL does not require any action in i_flags.
2319 	 */
2320 	inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE);
2321 }
2322 #else
shmem_set_inode_flags(struct inode * inode,unsigned int fsflags)2323 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2324 {
2325 }
2326 #define shmem_initxattrs NULL
2327 #endif
2328 
shmem_get_inode(struct super_block * sb,struct inode * dir,umode_t mode,dev_t dev,unsigned long flags)2329 static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir,
2330 				     umode_t mode, dev_t dev, unsigned long flags)
2331 {
2332 	struct inode *inode;
2333 	struct shmem_inode_info *info;
2334 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2335 	ino_t ino;
2336 
2337 	if (shmem_reserve_inode(sb, &ino))
2338 		return NULL;
2339 
2340 	inode = new_inode(sb);
2341 	if (inode) {
2342 		inode->i_ino = ino;
2343 		inode_init_owner(&init_user_ns, inode, dir, mode);
2344 		inode->i_blocks = 0;
2345 		inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2346 		inode->i_generation = get_random_u32();
2347 		info = SHMEM_I(inode);
2348 		memset(info, 0, (char *)inode - (char *)info);
2349 		spin_lock_init(&info->lock);
2350 		atomic_set(&info->stop_eviction, 0);
2351 		info->seals = F_SEAL_SEAL;
2352 		info->flags = flags & VM_NORESERVE;
2353 		info->i_crtime = inode->i_mtime;
2354 		info->fsflags = (dir == NULL) ? 0 :
2355 			SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
2356 		if (info->fsflags)
2357 			shmem_set_inode_flags(inode, info->fsflags);
2358 		INIT_LIST_HEAD(&info->shrinklist);
2359 		INIT_LIST_HEAD(&info->swaplist);
2360 		simple_xattrs_init(&info->xattrs);
2361 		cache_no_acl(inode);
2362 		mapping_set_large_folios(inode->i_mapping);
2363 
2364 		switch (mode & S_IFMT) {
2365 		default:
2366 			inode->i_op = &shmem_special_inode_operations;
2367 			init_special_inode(inode, mode, dev);
2368 			break;
2369 		case S_IFREG:
2370 			inode->i_mapping->a_ops = &shmem_aops;
2371 			inode->i_op = &shmem_inode_operations;
2372 			inode->i_fop = &shmem_file_operations;
2373 			mpol_shared_policy_init(&info->policy,
2374 						 shmem_get_sbmpol(sbinfo));
2375 			break;
2376 		case S_IFDIR:
2377 			inc_nlink(inode);
2378 			/* Some things misbehave if size == 0 on a directory */
2379 			inode->i_size = 2 * BOGO_DIRENT_SIZE;
2380 			inode->i_op = &shmem_dir_inode_operations;
2381 			inode->i_fop = &simple_dir_operations;
2382 			break;
2383 		case S_IFLNK:
2384 			/*
2385 			 * Must not load anything in the rbtree,
2386 			 * mpol_free_shared_policy will not be called.
2387 			 */
2388 			mpol_shared_policy_init(&info->policy, NULL);
2389 			break;
2390 		}
2391 
2392 		lockdep_annotate_inode_mutex_key(inode);
2393 	} else
2394 		shmem_free_inode(sb);
2395 	return inode;
2396 }
2397 
2398 #ifdef CONFIG_USERFAULTFD
shmem_mfill_atomic_pte(struct mm_struct * dst_mm,pmd_t * dst_pmd,struct vm_area_struct * dst_vma,unsigned long dst_addr,unsigned long src_addr,bool zeropage,bool wp_copy,struct page ** pagep)2399 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2400 			   pmd_t *dst_pmd,
2401 			   struct vm_area_struct *dst_vma,
2402 			   unsigned long dst_addr,
2403 			   unsigned long src_addr,
2404 			   bool zeropage, bool wp_copy,
2405 			   struct page **pagep)
2406 {
2407 	struct inode *inode = file_inode(dst_vma->vm_file);
2408 	struct shmem_inode_info *info = SHMEM_I(inode);
2409 	struct address_space *mapping = inode->i_mapping;
2410 	gfp_t gfp = mapping_gfp_mask(mapping);
2411 	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2412 	void *page_kaddr;
2413 	struct folio *folio;
2414 	int ret;
2415 	pgoff_t max_off;
2416 
2417 	if (!shmem_inode_acct_block(inode, 1)) {
2418 		/*
2419 		 * We may have got a page, returned -ENOENT triggering a retry,
2420 		 * and now we find ourselves with -ENOMEM. Release the page, to
2421 		 * avoid a BUG_ON in our caller.
2422 		 */
2423 		if (unlikely(*pagep)) {
2424 			put_page(*pagep);
2425 			*pagep = NULL;
2426 		}
2427 		return -ENOMEM;
2428 	}
2429 
2430 	if (!*pagep) {
2431 		ret = -ENOMEM;
2432 		folio = shmem_alloc_folio(gfp, info, pgoff);
2433 		if (!folio)
2434 			goto out_unacct_blocks;
2435 
2436 		if (!zeropage) {	/* COPY */
2437 			page_kaddr = kmap_local_folio(folio, 0);
2438 			/*
2439 			 * The read mmap_lock is held here.  Despite the
2440 			 * mmap_lock being read recursive a deadlock is still
2441 			 * possible if a writer has taken a lock.  For example:
2442 			 *
2443 			 * process A thread 1 takes read lock on own mmap_lock
2444 			 * process A thread 2 calls mmap, blocks taking write lock
2445 			 * process B thread 1 takes page fault, read lock on own mmap lock
2446 			 * process B thread 2 calls mmap, blocks taking write lock
2447 			 * process A thread 1 blocks taking read lock on process B
2448 			 * process B thread 1 blocks taking read lock on process A
2449 			 *
2450 			 * Disable page faults to prevent potential deadlock
2451 			 * and retry the copy outside the mmap_lock.
2452 			 */
2453 			pagefault_disable();
2454 			ret = copy_from_user(page_kaddr,
2455 					     (const void __user *)src_addr,
2456 					     PAGE_SIZE);
2457 			pagefault_enable();
2458 			kunmap_local(page_kaddr);
2459 
2460 			/* fallback to copy_from_user outside mmap_lock */
2461 			if (unlikely(ret)) {
2462 				*pagep = &folio->page;
2463 				ret = -ENOENT;
2464 				/* don't free the page */
2465 				goto out_unacct_blocks;
2466 			}
2467 
2468 			flush_dcache_folio(folio);
2469 		} else {		/* ZEROPAGE */
2470 			clear_user_highpage(&folio->page, dst_addr);
2471 		}
2472 	} else {
2473 		folio = page_folio(*pagep);
2474 		VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2475 		*pagep = NULL;
2476 	}
2477 
2478 	VM_BUG_ON(folio_test_locked(folio));
2479 	VM_BUG_ON(folio_test_swapbacked(folio));
2480 	__folio_set_locked(folio);
2481 	__folio_set_swapbacked(folio);
2482 	__folio_mark_uptodate(folio);
2483 
2484 	ret = -EFAULT;
2485 	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2486 	if (unlikely(pgoff >= max_off))
2487 		goto out_release;
2488 
2489 	ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL,
2490 				      gfp & GFP_RECLAIM_MASK, dst_mm);
2491 	if (ret)
2492 		goto out_release;
2493 
2494 	ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
2495 				       &folio->page, true, wp_copy);
2496 	if (ret)
2497 		goto out_delete_from_cache;
2498 
2499 	spin_lock_irq(&info->lock);
2500 	info->alloced++;
2501 	inode->i_blocks += BLOCKS_PER_PAGE;
2502 	shmem_recalc_inode(inode);
2503 	spin_unlock_irq(&info->lock);
2504 
2505 	folio_unlock(folio);
2506 	return 0;
2507 out_delete_from_cache:
2508 	filemap_remove_folio(folio);
2509 out_release:
2510 	folio_unlock(folio);
2511 	folio_put(folio);
2512 out_unacct_blocks:
2513 	shmem_inode_unacct_blocks(inode, 1);
2514 	return ret;
2515 }
2516 #endif /* CONFIG_USERFAULTFD */
2517 
2518 #ifdef CONFIG_TMPFS
2519 static const struct inode_operations shmem_symlink_inode_operations;
2520 static const struct inode_operations shmem_short_symlink_operations;
2521 
2522 static int
shmem_write_begin(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,struct page ** pagep,void ** fsdata)2523 shmem_write_begin(struct file *file, struct address_space *mapping,
2524 			loff_t pos, unsigned len,
2525 			struct page **pagep, void **fsdata)
2526 {
2527 	struct inode *inode = mapping->host;
2528 	struct shmem_inode_info *info = SHMEM_I(inode);
2529 	pgoff_t index = pos >> PAGE_SHIFT;
2530 	struct folio *folio;
2531 	int ret = 0;
2532 
2533 	/* i_rwsem is held by caller */
2534 	if (unlikely(info->seals & (F_SEAL_GROW |
2535 				   F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2536 		if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2537 			return -EPERM;
2538 		if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2539 			return -EPERM;
2540 	}
2541 
2542 	ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
2543 
2544 	if (ret)
2545 		return ret;
2546 
2547 	*pagep = folio_file_page(folio, index);
2548 	if (PageHWPoison(*pagep)) {
2549 		folio_unlock(folio);
2550 		folio_put(folio);
2551 		*pagep = NULL;
2552 		return -EIO;
2553 	}
2554 
2555 	return 0;
2556 }
2557 
2558 static int
shmem_write_end(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct page * page,void * fsdata)2559 shmem_write_end(struct file *file, struct address_space *mapping,
2560 			loff_t pos, unsigned len, unsigned copied,
2561 			struct page *page, void *fsdata)
2562 {
2563 	struct inode *inode = mapping->host;
2564 
2565 	if (pos + copied > inode->i_size)
2566 		i_size_write(inode, pos + copied);
2567 
2568 	if (!PageUptodate(page)) {
2569 		struct page *head = compound_head(page);
2570 		if (PageTransCompound(page)) {
2571 			int i;
2572 
2573 			for (i = 0; i < HPAGE_PMD_NR; i++) {
2574 				if (head + i == page)
2575 					continue;
2576 				clear_highpage(head + i);
2577 				flush_dcache_page(head + i);
2578 			}
2579 		}
2580 		if (copied < PAGE_SIZE) {
2581 			unsigned from = pos & (PAGE_SIZE - 1);
2582 			zero_user_segments(page, 0, from,
2583 					from + copied, PAGE_SIZE);
2584 		}
2585 		SetPageUptodate(head);
2586 	}
2587 	set_page_dirty(page);
2588 	unlock_page(page);
2589 	put_page(page);
2590 
2591 	return copied;
2592 }
2593 
shmem_file_read_iter(struct kiocb * iocb,struct iov_iter * to)2594 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2595 {
2596 	struct file *file = iocb->ki_filp;
2597 	struct inode *inode = file_inode(file);
2598 	struct address_space *mapping = inode->i_mapping;
2599 	pgoff_t index;
2600 	unsigned long offset;
2601 	int error = 0;
2602 	ssize_t retval = 0;
2603 	loff_t *ppos = &iocb->ki_pos;
2604 
2605 	index = *ppos >> PAGE_SHIFT;
2606 	offset = *ppos & ~PAGE_MASK;
2607 
2608 	for (;;) {
2609 		struct folio *folio = NULL;
2610 		struct page *page = NULL;
2611 		pgoff_t end_index;
2612 		unsigned long nr, ret;
2613 		loff_t i_size = i_size_read(inode);
2614 
2615 		end_index = i_size >> PAGE_SHIFT;
2616 		if (index > end_index)
2617 			break;
2618 		if (index == end_index) {
2619 			nr = i_size & ~PAGE_MASK;
2620 			if (nr <= offset)
2621 				break;
2622 		}
2623 
2624 		error = shmem_get_folio(inode, index, &folio, SGP_READ);
2625 		if (error) {
2626 			if (error == -EINVAL)
2627 				error = 0;
2628 			break;
2629 		}
2630 		if (folio) {
2631 			folio_unlock(folio);
2632 
2633 			page = folio_file_page(folio, index);
2634 			if (PageHWPoison(page)) {
2635 				folio_put(folio);
2636 				error = -EIO;
2637 				break;
2638 			}
2639 		}
2640 
2641 		/*
2642 		 * We must evaluate after, since reads (unlike writes)
2643 		 * are called without i_rwsem protection against truncate
2644 		 */
2645 		nr = PAGE_SIZE;
2646 		i_size = i_size_read(inode);
2647 		end_index = i_size >> PAGE_SHIFT;
2648 		if (index == end_index) {
2649 			nr = i_size & ~PAGE_MASK;
2650 			if (nr <= offset) {
2651 				if (folio)
2652 					folio_put(folio);
2653 				break;
2654 			}
2655 		}
2656 		nr -= offset;
2657 
2658 		if (folio) {
2659 			/*
2660 			 * If users can be writing to this page using arbitrary
2661 			 * virtual addresses, take care about potential aliasing
2662 			 * before reading the page on the kernel side.
2663 			 */
2664 			if (mapping_writably_mapped(mapping))
2665 				flush_dcache_page(page);
2666 			/*
2667 			 * Mark the page accessed if we read the beginning.
2668 			 */
2669 			if (!offset)
2670 				folio_mark_accessed(folio);
2671 			/*
2672 			 * Ok, we have the page, and it's up-to-date, so
2673 			 * now we can copy it to user space...
2674 			 */
2675 			ret = copy_page_to_iter(page, offset, nr, to);
2676 			folio_put(folio);
2677 
2678 		} else if (user_backed_iter(to)) {
2679 			/*
2680 			 * Copy to user tends to be so well optimized, but
2681 			 * clear_user() not so much, that it is noticeably
2682 			 * faster to copy the zero page instead of clearing.
2683 			 */
2684 			ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
2685 		} else {
2686 			/*
2687 			 * But submitting the same page twice in a row to
2688 			 * splice() - or others? - can result in confusion:
2689 			 * so don't attempt that optimization on pipes etc.
2690 			 */
2691 			ret = iov_iter_zero(nr, to);
2692 		}
2693 
2694 		retval += ret;
2695 		offset += ret;
2696 		index += offset >> PAGE_SHIFT;
2697 		offset &= ~PAGE_MASK;
2698 
2699 		if (!iov_iter_count(to))
2700 			break;
2701 		if (ret < nr) {
2702 			error = -EFAULT;
2703 			break;
2704 		}
2705 		cond_resched();
2706 	}
2707 
2708 	*ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2709 	file_accessed(file);
2710 	return retval ? retval : error;
2711 }
2712 
shmem_file_llseek(struct file * file,loff_t offset,int whence)2713 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2714 {
2715 	struct address_space *mapping = file->f_mapping;
2716 	struct inode *inode = mapping->host;
2717 
2718 	if (whence != SEEK_DATA && whence != SEEK_HOLE)
2719 		return generic_file_llseek_size(file, offset, whence,
2720 					MAX_LFS_FILESIZE, i_size_read(inode));
2721 	if (offset < 0)
2722 		return -ENXIO;
2723 
2724 	inode_lock(inode);
2725 	/* We're holding i_rwsem so we can access i_size directly */
2726 	offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
2727 	if (offset >= 0)
2728 		offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2729 	inode_unlock(inode);
2730 	return offset;
2731 }
2732 
shmem_fallocate(struct file * file,int mode,loff_t offset,loff_t len)2733 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2734 							 loff_t len)
2735 {
2736 	struct inode *inode = file_inode(file);
2737 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2738 	struct shmem_inode_info *info = SHMEM_I(inode);
2739 	struct shmem_falloc shmem_falloc;
2740 	pgoff_t start, index, end, undo_fallocend;
2741 	int error;
2742 
2743 	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2744 		return -EOPNOTSUPP;
2745 
2746 	inode_lock(inode);
2747 
2748 	if (mode & FALLOC_FL_PUNCH_HOLE) {
2749 		struct address_space *mapping = file->f_mapping;
2750 		loff_t unmap_start = round_up(offset, PAGE_SIZE);
2751 		loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2752 		DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2753 
2754 		/* protected by i_rwsem */
2755 		if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2756 			error = -EPERM;
2757 			goto out;
2758 		}
2759 
2760 		shmem_falloc.waitq = &shmem_falloc_waitq;
2761 		shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2762 		shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2763 		spin_lock(&inode->i_lock);
2764 		inode->i_private = &shmem_falloc;
2765 		spin_unlock(&inode->i_lock);
2766 
2767 		if ((u64)unmap_end > (u64)unmap_start)
2768 			unmap_mapping_range(mapping, unmap_start,
2769 					    1 + unmap_end - unmap_start, 0);
2770 		shmem_truncate_range(inode, offset, offset + len - 1);
2771 		/* No need to unmap again: hole-punching leaves COWed pages */
2772 
2773 		spin_lock(&inode->i_lock);
2774 		inode->i_private = NULL;
2775 		wake_up_all(&shmem_falloc_waitq);
2776 		WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2777 		spin_unlock(&inode->i_lock);
2778 		error = 0;
2779 		goto out;
2780 	}
2781 
2782 	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2783 	error = inode_newsize_ok(inode, offset + len);
2784 	if (error)
2785 		goto out;
2786 
2787 	if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2788 		error = -EPERM;
2789 		goto out;
2790 	}
2791 
2792 	start = offset >> PAGE_SHIFT;
2793 	end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2794 	/* Try to avoid a swapstorm if len is impossible to satisfy */
2795 	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2796 		error = -ENOSPC;
2797 		goto out;
2798 	}
2799 
2800 	shmem_falloc.waitq = NULL;
2801 	shmem_falloc.start = start;
2802 	shmem_falloc.next  = start;
2803 	shmem_falloc.nr_falloced = 0;
2804 	shmem_falloc.nr_unswapped = 0;
2805 	spin_lock(&inode->i_lock);
2806 	inode->i_private = &shmem_falloc;
2807 	spin_unlock(&inode->i_lock);
2808 
2809 	/*
2810 	 * info->fallocend is only relevant when huge pages might be
2811 	 * involved: to prevent split_huge_page() freeing fallocated
2812 	 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
2813 	 */
2814 	undo_fallocend = info->fallocend;
2815 	if (info->fallocend < end)
2816 		info->fallocend = end;
2817 
2818 	for (index = start; index < end; ) {
2819 		struct folio *folio;
2820 
2821 		/*
2822 		 * Good, the fallocate(2) manpage permits EINTR: we may have
2823 		 * been interrupted because we are using up too much memory.
2824 		 */
2825 		if (signal_pending(current))
2826 			error = -EINTR;
2827 		else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2828 			error = -ENOMEM;
2829 		else
2830 			error = shmem_get_folio(inode, index, &folio,
2831 						SGP_FALLOC);
2832 		if (error) {
2833 			info->fallocend = undo_fallocend;
2834 			/* Remove the !uptodate folios we added */
2835 			if (index > start) {
2836 				shmem_undo_range(inode,
2837 				    (loff_t)start << PAGE_SHIFT,
2838 				    ((loff_t)index << PAGE_SHIFT) - 1, true);
2839 			}
2840 			goto undone;
2841 		}
2842 
2843 		/*
2844 		 * Here is a more important optimization than it appears:
2845 		 * a second SGP_FALLOC on the same large folio will clear it,
2846 		 * making it uptodate and un-undoable if we fail later.
2847 		 */
2848 		index = folio_next_index(folio);
2849 		/* Beware 32-bit wraparound */
2850 		if (!index)
2851 			index--;
2852 
2853 		/*
2854 		 * Inform shmem_writepage() how far we have reached.
2855 		 * No need for lock or barrier: we have the page lock.
2856 		 */
2857 		if (!folio_test_uptodate(folio))
2858 			shmem_falloc.nr_falloced += index - shmem_falloc.next;
2859 		shmem_falloc.next = index;
2860 
2861 		/*
2862 		 * If !uptodate, leave it that way so that freeable folios
2863 		 * can be recognized if we need to rollback on error later.
2864 		 * But mark it dirty so that memory pressure will swap rather
2865 		 * than free the folios we are allocating (and SGP_CACHE folios
2866 		 * might still be clean: we now need to mark those dirty too).
2867 		 */
2868 		folio_mark_dirty(folio);
2869 		folio_unlock(folio);
2870 		folio_put(folio);
2871 		cond_resched();
2872 	}
2873 
2874 	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2875 		i_size_write(inode, offset + len);
2876 undone:
2877 	spin_lock(&inode->i_lock);
2878 	inode->i_private = NULL;
2879 	spin_unlock(&inode->i_lock);
2880 out:
2881 	if (!error)
2882 		file_modified(file);
2883 	inode_unlock(inode);
2884 	return error;
2885 }
2886 
shmem_statfs(struct dentry * dentry,struct kstatfs * buf)2887 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2888 {
2889 	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2890 
2891 	buf->f_type = TMPFS_MAGIC;
2892 	buf->f_bsize = PAGE_SIZE;
2893 	buf->f_namelen = NAME_MAX;
2894 	if (sbinfo->max_blocks) {
2895 		buf->f_blocks = sbinfo->max_blocks;
2896 		buf->f_bavail =
2897 		buf->f_bfree  = sbinfo->max_blocks -
2898 				percpu_counter_sum(&sbinfo->used_blocks);
2899 	}
2900 	if (sbinfo->max_inodes) {
2901 		buf->f_files = sbinfo->max_inodes;
2902 		buf->f_ffree = sbinfo->free_inodes;
2903 	}
2904 	/* else leave those fields 0 like simple_statfs */
2905 
2906 	buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
2907 
2908 	return 0;
2909 }
2910 
2911 /*
2912  * File creation. Allocate an inode, and we're done..
2913  */
2914 static int
shmem_mknod(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,umode_t mode,dev_t dev)2915 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir,
2916 	    struct dentry *dentry, umode_t mode, dev_t dev)
2917 {
2918 	struct inode *inode;
2919 	int error = -ENOSPC;
2920 
2921 	inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2922 	if (inode) {
2923 		error = simple_acl_create(dir, inode);
2924 		if (error)
2925 			goto out_iput;
2926 		error = security_inode_init_security(inode, dir,
2927 						     &dentry->d_name,
2928 						     shmem_initxattrs, NULL);
2929 		if (error && error != -EOPNOTSUPP)
2930 			goto out_iput;
2931 
2932 		error = 0;
2933 		dir->i_size += BOGO_DIRENT_SIZE;
2934 		dir->i_ctime = dir->i_mtime = current_time(dir);
2935 		inode_inc_iversion(dir);
2936 		d_instantiate(dentry, inode);
2937 		dget(dentry); /* Extra count - pin the dentry in core */
2938 	}
2939 	return error;
2940 out_iput:
2941 	iput(inode);
2942 	return error;
2943 }
2944 
2945 static int
shmem_tmpfile(struct user_namespace * mnt_userns,struct inode * dir,struct file * file,umode_t mode)2946 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
2947 	      struct file *file, umode_t mode)
2948 {
2949 	struct inode *inode;
2950 	int error = -ENOSPC;
2951 
2952 	inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2953 	if (inode) {
2954 		error = security_inode_init_security(inode, dir,
2955 						     NULL,
2956 						     shmem_initxattrs, NULL);
2957 		if (error && error != -EOPNOTSUPP)
2958 			goto out_iput;
2959 		error = simple_acl_create(dir, inode);
2960 		if (error)
2961 			goto out_iput;
2962 		d_tmpfile(file, inode);
2963 	}
2964 	return finish_open_simple(file, error);
2965 out_iput:
2966 	iput(inode);
2967 	return error;
2968 }
2969 
shmem_mkdir(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,umode_t mode)2970 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
2971 		       struct dentry *dentry, umode_t mode)
2972 {
2973 	int error;
2974 
2975 	if ((error = shmem_mknod(&init_user_ns, dir, dentry,
2976 				 mode | S_IFDIR, 0)))
2977 		return error;
2978 	inc_nlink(dir);
2979 	return 0;
2980 }
2981 
shmem_create(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,umode_t mode,bool excl)2982 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir,
2983 			struct dentry *dentry, umode_t mode, bool excl)
2984 {
2985 	return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0);
2986 }
2987 
2988 /*
2989  * Link a file..
2990  */
shmem_link(struct dentry * old_dentry,struct inode * dir,struct dentry * dentry)2991 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2992 {
2993 	struct inode *inode = d_inode(old_dentry);
2994 	int ret = 0;
2995 
2996 	/*
2997 	 * No ordinary (disk based) filesystem counts links as inodes;
2998 	 * but each new link needs a new dentry, pinning lowmem, and
2999 	 * tmpfs dentries cannot be pruned until they are unlinked.
3000 	 * But if an O_TMPFILE file is linked into the tmpfs, the
3001 	 * first link must skip that, to get the accounting right.
3002 	 */
3003 	if (inode->i_nlink) {
3004 		ret = shmem_reserve_inode(inode->i_sb, NULL);
3005 		if (ret)
3006 			goto out;
3007 	}
3008 
3009 	dir->i_size += BOGO_DIRENT_SIZE;
3010 	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3011 	inode_inc_iversion(dir);
3012 	inc_nlink(inode);
3013 	ihold(inode);	/* New dentry reference */
3014 	dget(dentry);		/* Extra pinning count for the created dentry */
3015 	d_instantiate(dentry, inode);
3016 out:
3017 	return ret;
3018 }
3019 
shmem_unlink(struct inode * dir,struct dentry * dentry)3020 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3021 {
3022 	struct inode *inode = d_inode(dentry);
3023 
3024 	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3025 		shmem_free_inode(inode->i_sb);
3026 
3027 	dir->i_size -= BOGO_DIRENT_SIZE;
3028 	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3029 	inode_inc_iversion(dir);
3030 	drop_nlink(inode);
3031 	dput(dentry);	/* Undo the count from "create" - this does all the work */
3032 	return 0;
3033 }
3034 
shmem_rmdir(struct inode * dir,struct dentry * dentry)3035 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3036 {
3037 	if (!simple_empty(dentry))
3038 		return -ENOTEMPTY;
3039 
3040 	drop_nlink(d_inode(dentry));
3041 	drop_nlink(dir);
3042 	return shmem_unlink(dir, dentry);
3043 }
3044 
shmem_whiteout(struct user_namespace * mnt_userns,struct inode * old_dir,struct dentry * old_dentry)3045 static int shmem_whiteout(struct user_namespace *mnt_userns,
3046 			  struct inode *old_dir, struct dentry *old_dentry)
3047 {
3048 	struct dentry *whiteout;
3049 	int error;
3050 
3051 	whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3052 	if (!whiteout)
3053 		return -ENOMEM;
3054 
3055 	error = shmem_mknod(&init_user_ns, old_dir, whiteout,
3056 			    S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3057 	dput(whiteout);
3058 	if (error)
3059 		return error;
3060 
3061 	/*
3062 	 * Cheat and hash the whiteout while the old dentry is still in
3063 	 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3064 	 *
3065 	 * d_lookup() will consistently find one of them at this point,
3066 	 * not sure which one, but that isn't even important.
3067 	 */
3068 	d_rehash(whiteout);
3069 	return 0;
3070 }
3071 
3072 /*
3073  * The VFS layer already does all the dentry stuff for rename,
3074  * we just have to decrement the usage count for the target if
3075  * it exists so that the VFS layer correctly free's it when it
3076  * gets overwritten.
3077  */
shmem_rename2(struct user_namespace * mnt_userns,struct inode * old_dir,struct dentry * old_dentry,struct inode * new_dir,struct dentry * new_dentry,unsigned int flags)3078 static int shmem_rename2(struct user_namespace *mnt_userns,
3079 			 struct inode *old_dir, struct dentry *old_dentry,
3080 			 struct inode *new_dir, struct dentry *new_dentry,
3081 			 unsigned int flags)
3082 {
3083 	struct inode *inode = d_inode(old_dentry);
3084 	int they_are_dirs = S_ISDIR(inode->i_mode);
3085 
3086 	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3087 		return -EINVAL;
3088 
3089 	if (flags & RENAME_EXCHANGE)
3090 		return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
3091 
3092 	if (!simple_empty(new_dentry))
3093 		return -ENOTEMPTY;
3094 
3095 	if (flags & RENAME_WHITEOUT) {
3096 		int error;
3097 
3098 		error = shmem_whiteout(&init_user_ns, old_dir, old_dentry);
3099 		if (error)
3100 			return error;
3101 	}
3102 
3103 	if (d_really_is_positive(new_dentry)) {
3104 		(void) shmem_unlink(new_dir, new_dentry);
3105 		if (they_are_dirs) {
3106 			drop_nlink(d_inode(new_dentry));
3107 			drop_nlink(old_dir);
3108 		}
3109 	} else if (they_are_dirs) {
3110 		drop_nlink(old_dir);
3111 		inc_nlink(new_dir);
3112 	}
3113 
3114 	old_dir->i_size -= BOGO_DIRENT_SIZE;
3115 	new_dir->i_size += BOGO_DIRENT_SIZE;
3116 	old_dir->i_ctime = old_dir->i_mtime =
3117 	new_dir->i_ctime = new_dir->i_mtime =
3118 	inode->i_ctime = current_time(old_dir);
3119 	inode_inc_iversion(old_dir);
3120 	inode_inc_iversion(new_dir);
3121 	return 0;
3122 }
3123 
shmem_symlink(struct user_namespace * mnt_userns,struct inode * dir,struct dentry * dentry,const char * symname)3124 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir,
3125 			 struct dentry *dentry, const char *symname)
3126 {
3127 	int error;
3128 	int len;
3129 	struct inode *inode;
3130 	struct folio *folio;
3131 
3132 	len = strlen(symname) + 1;
3133 	if (len > PAGE_SIZE)
3134 		return -ENAMETOOLONG;
3135 
3136 	inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3137 				VM_NORESERVE);
3138 	if (!inode)
3139 		return -ENOSPC;
3140 
3141 	error = security_inode_init_security(inode, dir, &dentry->d_name,
3142 					     shmem_initxattrs, NULL);
3143 	if (error && error != -EOPNOTSUPP) {
3144 		iput(inode);
3145 		return error;
3146 	}
3147 
3148 	inode->i_size = len-1;
3149 	if (len <= SHORT_SYMLINK_LEN) {
3150 		inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3151 		if (!inode->i_link) {
3152 			iput(inode);
3153 			return -ENOMEM;
3154 		}
3155 		inode->i_op = &shmem_short_symlink_operations;
3156 	} else {
3157 		inode_nohighmem(inode);
3158 		error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
3159 		if (error) {
3160 			iput(inode);
3161 			return error;
3162 		}
3163 		inode->i_mapping->a_ops = &shmem_aops;
3164 		inode->i_op = &shmem_symlink_inode_operations;
3165 		memcpy(folio_address(folio), symname, len);
3166 		folio_mark_uptodate(folio);
3167 		folio_mark_dirty(folio);
3168 		folio_unlock(folio);
3169 		folio_put(folio);
3170 	}
3171 	dir->i_size += BOGO_DIRENT_SIZE;
3172 	dir->i_ctime = dir->i_mtime = current_time(dir);
3173 	inode_inc_iversion(dir);
3174 	d_instantiate(dentry, inode);
3175 	dget(dentry);
3176 	return 0;
3177 }
3178 
shmem_put_link(void * arg)3179 static void shmem_put_link(void *arg)
3180 {
3181 	folio_mark_accessed(arg);
3182 	folio_put(arg);
3183 }
3184 
shmem_get_link(struct dentry * dentry,struct inode * inode,struct delayed_call * done)3185 static const char *shmem_get_link(struct dentry *dentry,
3186 				  struct inode *inode,
3187 				  struct delayed_call *done)
3188 {
3189 	struct folio *folio = NULL;
3190 	int error;
3191 
3192 	if (!dentry) {
3193 		folio = filemap_get_folio(inode->i_mapping, 0);
3194 		if (!folio)
3195 			return ERR_PTR(-ECHILD);
3196 		if (PageHWPoison(folio_page(folio, 0)) ||
3197 		    !folio_test_uptodate(folio)) {
3198 			folio_put(folio);
3199 			return ERR_PTR(-ECHILD);
3200 		}
3201 	} else {
3202 		error = shmem_get_folio(inode, 0, &folio, SGP_READ);
3203 		if (error)
3204 			return ERR_PTR(error);
3205 		if (!folio)
3206 			return ERR_PTR(-ECHILD);
3207 		if (PageHWPoison(folio_page(folio, 0))) {
3208 			folio_unlock(folio);
3209 			folio_put(folio);
3210 			return ERR_PTR(-ECHILD);
3211 		}
3212 		folio_unlock(folio);
3213 	}
3214 	set_delayed_call(done, shmem_put_link, folio);
3215 	return folio_address(folio);
3216 }
3217 
3218 #ifdef CONFIG_TMPFS_XATTR
3219 
shmem_fileattr_get(struct dentry * dentry,struct fileattr * fa)3220 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
3221 {
3222 	struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3223 
3224 	fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE);
3225 
3226 	return 0;
3227 }
3228 
shmem_fileattr_set(struct user_namespace * mnt_userns,struct dentry * dentry,struct fileattr * fa)3229 static int shmem_fileattr_set(struct user_namespace *mnt_userns,
3230 			      struct dentry *dentry, struct fileattr *fa)
3231 {
3232 	struct inode *inode = d_inode(dentry);
3233 	struct shmem_inode_info *info = SHMEM_I(inode);
3234 
3235 	if (fileattr_has_fsx(fa))
3236 		return -EOPNOTSUPP;
3237 	if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
3238 		return -EOPNOTSUPP;
3239 
3240 	info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
3241 		(fa->flags & SHMEM_FL_USER_MODIFIABLE);
3242 
3243 	shmem_set_inode_flags(inode, info->fsflags);
3244 	inode->i_ctime = current_time(inode);
3245 	inode_inc_iversion(inode);
3246 	return 0;
3247 }
3248 
3249 /*
3250  * Superblocks without xattr inode operations may get some security.* xattr
3251  * support from the LSM "for free". As soon as we have any other xattrs
3252  * like ACLs, we also need to implement the security.* handlers at
3253  * filesystem level, though.
3254  */
3255 
3256 /*
3257  * Callback for security_inode_init_security() for acquiring xattrs.
3258  */
shmem_initxattrs(struct inode * inode,const struct xattr * xattr_array,void * fs_info)3259 static int shmem_initxattrs(struct inode *inode,
3260 			    const struct xattr *xattr_array,
3261 			    void *fs_info)
3262 {
3263 	struct shmem_inode_info *info = SHMEM_I(inode);
3264 	const struct xattr *xattr;
3265 	struct simple_xattr *new_xattr;
3266 	size_t len;
3267 
3268 	for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3269 		new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3270 		if (!new_xattr)
3271 			return -ENOMEM;
3272 
3273 		len = strlen(xattr->name) + 1;
3274 		new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3275 					  GFP_KERNEL);
3276 		if (!new_xattr->name) {
3277 			kvfree(new_xattr);
3278 			return -ENOMEM;
3279 		}
3280 
3281 		memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3282 		       XATTR_SECURITY_PREFIX_LEN);
3283 		memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3284 		       xattr->name, len);
3285 
3286 		simple_xattr_list_add(&info->xattrs, new_xattr);
3287 	}
3288 
3289 	return 0;
3290 }
3291 
shmem_xattr_handler_get(const struct xattr_handler * handler,struct dentry * unused,struct inode * inode,const char * name,void * buffer,size_t size)3292 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3293 				   struct dentry *unused, struct inode *inode,
3294 				   const char *name, void *buffer, size_t size)
3295 {
3296 	struct shmem_inode_info *info = SHMEM_I(inode);
3297 
3298 	name = xattr_full_name(handler, name);
3299 	return simple_xattr_get(&info->xattrs, name, buffer, size);
3300 }
3301 
shmem_xattr_handler_set(const struct xattr_handler * handler,struct user_namespace * mnt_userns,struct dentry * unused,struct inode * inode,const char * name,const void * value,size_t size,int flags)3302 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3303 				   struct user_namespace *mnt_userns,
3304 				   struct dentry *unused, struct inode *inode,
3305 				   const char *name, const void *value,
3306 				   size_t size, int flags)
3307 {
3308 	struct shmem_inode_info *info = SHMEM_I(inode);
3309 	int err;
3310 
3311 	name = xattr_full_name(handler, name);
3312 	err = simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3313 	if (!err) {
3314 		inode->i_ctime = current_time(inode);
3315 		inode_inc_iversion(inode);
3316 	}
3317 	return err;
3318 }
3319 
3320 static const struct xattr_handler shmem_security_xattr_handler = {
3321 	.prefix = XATTR_SECURITY_PREFIX,
3322 	.get = shmem_xattr_handler_get,
3323 	.set = shmem_xattr_handler_set,
3324 };
3325 
3326 static const struct xattr_handler shmem_trusted_xattr_handler = {
3327 	.prefix = XATTR_TRUSTED_PREFIX,
3328 	.get = shmem_xattr_handler_get,
3329 	.set = shmem_xattr_handler_set,
3330 };
3331 
3332 static const struct xattr_handler *shmem_xattr_handlers[] = {
3333 #ifdef CONFIG_TMPFS_POSIX_ACL
3334 	&posix_acl_access_xattr_handler,
3335 	&posix_acl_default_xattr_handler,
3336 #endif
3337 	&shmem_security_xattr_handler,
3338 	&shmem_trusted_xattr_handler,
3339 	NULL
3340 };
3341 
shmem_listxattr(struct dentry * dentry,char * buffer,size_t size)3342 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3343 {
3344 	struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3345 	return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3346 }
3347 #endif /* CONFIG_TMPFS_XATTR */
3348 
3349 static const struct inode_operations shmem_short_symlink_operations = {
3350 	.getattr	= shmem_getattr,
3351 	.get_link	= simple_get_link,
3352 #ifdef CONFIG_TMPFS_XATTR
3353 	.listxattr	= shmem_listxattr,
3354 #endif
3355 };
3356 
3357 static const struct inode_operations shmem_symlink_inode_operations = {
3358 	.getattr	= shmem_getattr,
3359 	.get_link	= shmem_get_link,
3360 #ifdef CONFIG_TMPFS_XATTR
3361 	.listxattr	= shmem_listxattr,
3362 #endif
3363 };
3364 
shmem_get_parent(struct dentry * child)3365 static struct dentry *shmem_get_parent(struct dentry *child)
3366 {
3367 	return ERR_PTR(-ESTALE);
3368 }
3369 
shmem_match(struct inode * ino,void * vfh)3370 static int shmem_match(struct inode *ino, void *vfh)
3371 {
3372 	__u32 *fh = vfh;
3373 	__u64 inum = fh[2];
3374 	inum = (inum << 32) | fh[1];
3375 	return ino->i_ino == inum && fh[0] == ino->i_generation;
3376 }
3377 
3378 /* Find any alias of inode, but prefer a hashed alias */
shmem_find_alias(struct inode * inode)3379 static struct dentry *shmem_find_alias(struct inode *inode)
3380 {
3381 	struct dentry *alias = d_find_alias(inode);
3382 
3383 	return alias ?: d_find_any_alias(inode);
3384 }
3385 
3386 
shmem_fh_to_dentry(struct super_block * sb,struct fid * fid,int fh_len,int fh_type)3387 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3388 		struct fid *fid, int fh_len, int fh_type)
3389 {
3390 	struct inode *inode;
3391 	struct dentry *dentry = NULL;
3392 	u64 inum;
3393 
3394 	if (fh_len < 3)
3395 		return NULL;
3396 
3397 	inum = fid->raw[2];
3398 	inum = (inum << 32) | fid->raw[1];
3399 
3400 	inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3401 			shmem_match, fid->raw);
3402 	if (inode) {
3403 		dentry = shmem_find_alias(inode);
3404 		iput(inode);
3405 	}
3406 
3407 	return dentry;
3408 }
3409 
shmem_encode_fh(struct inode * inode,__u32 * fh,int * len,struct inode * parent)3410 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3411 				struct inode *parent)
3412 {
3413 	if (*len < 3) {
3414 		*len = 3;
3415 		return FILEID_INVALID;
3416 	}
3417 
3418 	if (inode_unhashed(inode)) {
3419 		/* Unfortunately insert_inode_hash is not idempotent,
3420 		 * so as we hash inodes here rather than at creation
3421 		 * time, we need a lock to ensure we only try
3422 		 * to do it once
3423 		 */
3424 		static DEFINE_SPINLOCK(lock);
3425 		spin_lock(&lock);
3426 		if (inode_unhashed(inode))
3427 			__insert_inode_hash(inode,
3428 					    inode->i_ino + inode->i_generation);
3429 		spin_unlock(&lock);
3430 	}
3431 
3432 	fh[0] = inode->i_generation;
3433 	fh[1] = inode->i_ino;
3434 	fh[2] = ((__u64)inode->i_ino) >> 32;
3435 
3436 	*len = 3;
3437 	return 1;
3438 }
3439 
3440 static const struct export_operations shmem_export_ops = {
3441 	.get_parent     = shmem_get_parent,
3442 	.encode_fh      = shmem_encode_fh,
3443 	.fh_to_dentry	= shmem_fh_to_dentry,
3444 };
3445 
3446 enum shmem_param {
3447 	Opt_gid,
3448 	Opt_huge,
3449 	Opt_mode,
3450 	Opt_mpol,
3451 	Opt_nr_blocks,
3452 	Opt_nr_inodes,
3453 	Opt_size,
3454 	Opt_uid,
3455 	Opt_inode32,
3456 	Opt_inode64,
3457 };
3458 
3459 static const struct constant_table shmem_param_enums_huge[] = {
3460 	{"never",	SHMEM_HUGE_NEVER },
3461 	{"always",	SHMEM_HUGE_ALWAYS },
3462 	{"within_size",	SHMEM_HUGE_WITHIN_SIZE },
3463 	{"advise",	SHMEM_HUGE_ADVISE },
3464 	{}
3465 };
3466 
3467 const struct fs_parameter_spec shmem_fs_parameters[] = {
3468 	fsparam_u32   ("gid",		Opt_gid),
3469 	fsparam_enum  ("huge",		Opt_huge,  shmem_param_enums_huge),
3470 	fsparam_u32oct("mode",		Opt_mode),
3471 	fsparam_string("mpol",		Opt_mpol),
3472 	fsparam_string("nr_blocks",	Opt_nr_blocks),
3473 	fsparam_string("nr_inodes",	Opt_nr_inodes),
3474 	fsparam_string("size",		Opt_size),
3475 	fsparam_u32   ("uid",		Opt_uid),
3476 	fsparam_flag  ("inode32",	Opt_inode32),
3477 	fsparam_flag  ("inode64",	Opt_inode64),
3478 	{}
3479 };
3480 
shmem_parse_one(struct fs_context * fc,struct fs_parameter * param)3481 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3482 {
3483 	struct shmem_options *ctx = fc->fs_private;
3484 	struct fs_parse_result result;
3485 	unsigned long long size;
3486 	char *rest;
3487 	int opt;
3488 
3489 	opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3490 	if (opt < 0)
3491 		return opt;
3492 
3493 	switch (opt) {
3494 	case Opt_size:
3495 		size = memparse(param->string, &rest);
3496 		if (*rest == '%') {
3497 			size <<= PAGE_SHIFT;
3498 			size *= totalram_pages();
3499 			do_div(size, 100);
3500 			rest++;
3501 		}
3502 		if (*rest)
3503 			goto bad_value;
3504 		ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3505 		ctx->seen |= SHMEM_SEEN_BLOCKS;
3506 		break;
3507 	case Opt_nr_blocks:
3508 		ctx->blocks = memparse(param->string, &rest);
3509 		if (*rest || ctx->blocks > S64_MAX)
3510 			goto bad_value;
3511 		ctx->seen |= SHMEM_SEEN_BLOCKS;
3512 		break;
3513 	case Opt_nr_inodes:
3514 		ctx->inodes = memparse(param->string, &rest);
3515 		if (*rest)
3516 			goto bad_value;
3517 		ctx->seen |= SHMEM_SEEN_INODES;
3518 		break;
3519 	case Opt_mode:
3520 		ctx->mode = result.uint_32 & 07777;
3521 		break;
3522 	case Opt_uid:
3523 		ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3524 		if (!uid_valid(ctx->uid))
3525 			goto bad_value;
3526 		break;
3527 	case Opt_gid:
3528 		ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3529 		if (!gid_valid(ctx->gid))
3530 			goto bad_value;
3531 		break;
3532 	case Opt_huge:
3533 		ctx->huge = result.uint_32;
3534 		if (ctx->huge != SHMEM_HUGE_NEVER &&
3535 		    !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3536 		      has_transparent_hugepage()))
3537 			goto unsupported_parameter;
3538 		ctx->seen |= SHMEM_SEEN_HUGE;
3539 		break;
3540 	case Opt_mpol:
3541 		if (IS_ENABLED(CONFIG_NUMA)) {
3542 			mpol_put(ctx->mpol);
3543 			ctx->mpol = NULL;
3544 			if (mpol_parse_str(param->string, &ctx->mpol))
3545 				goto bad_value;
3546 			break;
3547 		}
3548 		goto unsupported_parameter;
3549 	case Opt_inode32:
3550 		ctx->full_inums = false;
3551 		ctx->seen |= SHMEM_SEEN_INUMS;
3552 		break;
3553 	case Opt_inode64:
3554 		if (sizeof(ino_t) < 8) {
3555 			return invalfc(fc,
3556 				       "Cannot use inode64 with <64bit inums in kernel\n");
3557 		}
3558 		ctx->full_inums = true;
3559 		ctx->seen |= SHMEM_SEEN_INUMS;
3560 		break;
3561 	}
3562 	return 0;
3563 
3564 unsupported_parameter:
3565 	return invalfc(fc, "Unsupported parameter '%s'", param->key);
3566 bad_value:
3567 	return invalfc(fc, "Bad value for '%s'", param->key);
3568 }
3569 
shmem_parse_options(struct fs_context * fc,void * data)3570 static int shmem_parse_options(struct fs_context *fc, void *data)
3571 {
3572 	char *options = data;
3573 
3574 	if (options) {
3575 		int err = security_sb_eat_lsm_opts(options, &fc->security);
3576 		if (err)
3577 			return err;
3578 	}
3579 
3580 	while (options != NULL) {
3581 		char *this_char = options;
3582 		for (;;) {
3583 			/*
3584 			 * NUL-terminate this option: unfortunately,
3585 			 * mount options form a comma-separated list,
3586 			 * but mpol's nodelist may also contain commas.
3587 			 */
3588 			options = strchr(options, ',');
3589 			if (options == NULL)
3590 				break;
3591 			options++;
3592 			if (!isdigit(*options)) {
3593 				options[-1] = '\0';
3594 				break;
3595 			}
3596 		}
3597 		if (*this_char) {
3598 			char *value = strchr(this_char, '=');
3599 			size_t len = 0;
3600 			int err;
3601 
3602 			if (value) {
3603 				*value++ = '\0';
3604 				len = strlen(value);
3605 			}
3606 			err = vfs_parse_fs_string(fc, this_char, value, len);
3607 			if (err < 0)
3608 				return err;
3609 		}
3610 	}
3611 	return 0;
3612 }
3613 
3614 /*
3615  * Reconfigure a shmem filesystem.
3616  *
3617  * Note that we disallow change from limited->unlimited blocks/inodes while any
3618  * are in use; but we must separately disallow unlimited->limited, because in
3619  * that case we have no record of how much is already in use.
3620  */
shmem_reconfigure(struct fs_context * fc)3621 static int shmem_reconfigure(struct fs_context *fc)
3622 {
3623 	struct shmem_options *ctx = fc->fs_private;
3624 	struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3625 	unsigned long inodes;
3626 	struct mempolicy *mpol = NULL;
3627 	const char *err;
3628 
3629 	raw_spin_lock(&sbinfo->stat_lock);
3630 	inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3631 
3632 	if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3633 		if (!sbinfo->max_blocks) {
3634 			err = "Cannot retroactively limit size";
3635 			goto out;
3636 		}
3637 		if (percpu_counter_compare(&sbinfo->used_blocks,
3638 					   ctx->blocks) > 0) {
3639 			err = "Too small a size for current use";
3640 			goto out;
3641 		}
3642 	}
3643 	if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
3644 		if (!sbinfo->max_inodes) {
3645 			err = "Cannot retroactively limit inodes";
3646 			goto out;
3647 		}
3648 		if (ctx->inodes < inodes) {
3649 			err = "Too few inodes for current use";
3650 			goto out;
3651 		}
3652 	}
3653 
3654 	if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
3655 	    sbinfo->next_ino > UINT_MAX) {
3656 		err = "Current inum too high to switch to 32-bit inums";
3657 		goto out;
3658 	}
3659 
3660 	if (ctx->seen & SHMEM_SEEN_HUGE)
3661 		sbinfo->huge = ctx->huge;
3662 	if (ctx->seen & SHMEM_SEEN_INUMS)
3663 		sbinfo->full_inums = ctx->full_inums;
3664 	if (ctx->seen & SHMEM_SEEN_BLOCKS)
3665 		sbinfo->max_blocks  = ctx->blocks;
3666 	if (ctx->seen & SHMEM_SEEN_INODES) {
3667 		sbinfo->max_inodes  = ctx->inodes;
3668 		sbinfo->free_inodes = ctx->inodes - inodes;
3669 	}
3670 
3671 	/*
3672 	 * Preserve previous mempolicy unless mpol remount option was specified.
3673 	 */
3674 	if (ctx->mpol) {
3675 		mpol = sbinfo->mpol;
3676 		sbinfo->mpol = ctx->mpol;	/* transfers initial ref */
3677 		ctx->mpol = NULL;
3678 	}
3679 	raw_spin_unlock(&sbinfo->stat_lock);
3680 	mpol_put(mpol);
3681 	return 0;
3682 out:
3683 	raw_spin_unlock(&sbinfo->stat_lock);
3684 	return invalfc(fc, "%s", err);
3685 }
3686 
shmem_show_options(struct seq_file * seq,struct dentry * root)3687 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3688 {
3689 	struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3690 
3691 	if (sbinfo->max_blocks != shmem_default_max_blocks())
3692 		seq_printf(seq, ",size=%luk",
3693 			sbinfo->max_blocks << (PAGE_SHIFT - 10));
3694 	if (sbinfo->max_inodes != shmem_default_max_inodes())
3695 		seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3696 	if (sbinfo->mode != (0777 | S_ISVTX))
3697 		seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3698 	if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3699 		seq_printf(seq, ",uid=%u",
3700 				from_kuid_munged(&init_user_ns, sbinfo->uid));
3701 	if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3702 		seq_printf(seq, ",gid=%u",
3703 				from_kgid_munged(&init_user_ns, sbinfo->gid));
3704 
3705 	/*
3706 	 * Showing inode{64,32} might be useful even if it's the system default,
3707 	 * since then people don't have to resort to checking both here and
3708 	 * /proc/config.gz to confirm 64-bit inums were successfully applied
3709 	 * (which may not even exist if IKCONFIG_PROC isn't enabled).
3710 	 *
3711 	 * We hide it when inode64 isn't the default and we are using 32-bit
3712 	 * inodes, since that probably just means the feature isn't even under
3713 	 * consideration.
3714 	 *
3715 	 * As such:
3716 	 *
3717 	 *                     +-----------------+-----------------+
3718 	 *                     | TMPFS_INODE64=y | TMPFS_INODE64=n |
3719 	 *  +------------------+-----------------+-----------------+
3720 	 *  | full_inums=true  | show            | show            |
3721 	 *  | full_inums=false | show            | hide            |
3722 	 *  +------------------+-----------------+-----------------+
3723 	 *
3724 	 */
3725 	if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
3726 		seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
3727 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3728 	/* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3729 	if (sbinfo->huge)
3730 		seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3731 #endif
3732 	shmem_show_mpol(seq, sbinfo->mpol);
3733 	return 0;
3734 }
3735 
3736 #endif /* CONFIG_TMPFS */
3737 
shmem_put_super(struct super_block * sb)3738 static void shmem_put_super(struct super_block *sb)
3739 {
3740 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3741 
3742 	free_percpu(sbinfo->ino_batch);
3743 	percpu_counter_destroy(&sbinfo->used_blocks);
3744 	mpol_put(sbinfo->mpol);
3745 	kfree(sbinfo);
3746 	sb->s_fs_info = NULL;
3747 }
3748 
shmem_fill_super(struct super_block * sb,struct fs_context * fc)3749 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
3750 {
3751 	struct shmem_options *ctx = fc->fs_private;
3752 	struct inode *inode;
3753 	struct shmem_sb_info *sbinfo;
3754 
3755 	/* Round up to L1_CACHE_BYTES to resist false sharing */
3756 	sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3757 				L1_CACHE_BYTES), GFP_KERNEL);
3758 	if (!sbinfo)
3759 		return -ENOMEM;
3760 
3761 	sb->s_fs_info = sbinfo;
3762 
3763 #ifdef CONFIG_TMPFS
3764 	/*
3765 	 * Per default we only allow half of the physical ram per
3766 	 * tmpfs instance, limiting inodes to one per page of lowmem;
3767 	 * but the internal instance is left unlimited.
3768 	 */
3769 	if (!(sb->s_flags & SB_KERNMOUNT)) {
3770 		if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
3771 			ctx->blocks = shmem_default_max_blocks();
3772 		if (!(ctx->seen & SHMEM_SEEN_INODES))
3773 			ctx->inodes = shmem_default_max_inodes();
3774 		if (!(ctx->seen & SHMEM_SEEN_INUMS))
3775 			ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
3776 	} else {
3777 		sb->s_flags |= SB_NOUSER;
3778 	}
3779 	sb->s_export_op = &shmem_export_ops;
3780 	sb->s_flags |= SB_NOSEC | SB_I_VERSION;
3781 #else
3782 	sb->s_flags |= SB_NOUSER;
3783 #endif
3784 	sbinfo->max_blocks = ctx->blocks;
3785 	sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
3786 	if (sb->s_flags & SB_KERNMOUNT) {
3787 		sbinfo->ino_batch = alloc_percpu(ino_t);
3788 		if (!sbinfo->ino_batch)
3789 			goto failed;
3790 	}
3791 	sbinfo->uid = ctx->uid;
3792 	sbinfo->gid = ctx->gid;
3793 	sbinfo->full_inums = ctx->full_inums;
3794 	sbinfo->mode = ctx->mode;
3795 	sbinfo->huge = ctx->huge;
3796 	sbinfo->mpol = ctx->mpol;
3797 	ctx->mpol = NULL;
3798 
3799 	raw_spin_lock_init(&sbinfo->stat_lock);
3800 	if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3801 		goto failed;
3802 	spin_lock_init(&sbinfo->shrinklist_lock);
3803 	INIT_LIST_HEAD(&sbinfo->shrinklist);
3804 
3805 	sb->s_maxbytes = MAX_LFS_FILESIZE;
3806 	sb->s_blocksize = PAGE_SIZE;
3807 	sb->s_blocksize_bits = PAGE_SHIFT;
3808 	sb->s_magic = TMPFS_MAGIC;
3809 	sb->s_op = &shmem_ops;
3810 	sb->s_time_gran = 1;
3811 #ifdef CONFIG_TMPFS_XATTR
3812 	sb->s_xattr = shmem_xattr_handlers;
3813 #endif
3814 #ifdef CONFIG_TMPFS_POSIX_ACL
3815 	sb->s_flags |= SB_POSIXACL;
3816 #endif
3817 	uuid_gen(&sb->s_uuid);
3818 
3819 	inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3820 	if (!inode)
3821 		goto failed;
3822 	inode->i_uid = sbinfo->uid;
3823 	inode->i_gid = sbinfo->gid;
3824 	sb->s_root = d_make_root(inode);
3825 	if (!sb->s_root)
3826 		goto failed;
3827 	return 0;
3828 
3829 failed:
3830 	shmem_put_super(sb);
3831 	return -ENOMEM;
3832 }
3833 
shmem_get_tree(struct fs_context * fc)3834 static int shmem_get_tree(struct fs_context *fc)
3835 {
3836 	return get_tree_nodev(fc, shmem_fill_super);
3837 }
3838 
shmem_free_fc(struct fs_context * fc)3839 static void shmem_free_fc(struct fs_context *fc)
3840 {
3841 	struct shmem_options *ctx = fc->fs_private;
3842 
3843 	if (ctx) {
3844 		mpol_put(ctx->mpol);
3845 		kfree(ctx);
3846 	}
3847 }
3848 
3849 static const struct fs_context_operations shmem_fs_context_ops = {
3850 	.free			= shmem_free_fc,
3851 	.get_tree		= shmem_get_tree,
3852 #ifdef CONFIG_TMPFS
3853 	.parse_monolithic	= shmem_parse_options,
3854 	.parse_param		= shmem_parse_one,
3855 	.reconfigure		= shmem_reconfigure,
3856 #endif
3857 };
3858 
3859 static struct kmem_cache *shmem_inode_cachep;
3860 
shmem_alloc_inode(struct super_block * sb)3861 static struct inode *shmem_alloc_inode(struct super_block *sb)
3862 {
3863 	struct shmem_inode_info *info;
3864 	info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
3865 	if (!info)
3866 		return NULL;
3867 	return &info->vfs_inode;
3868 }
3869 
shmem_free_in_core_inode(struct inode * inode)3870 static void shmem_free_in_core_inode(struct inode *inode)
3871 {
3872 	if (S_ISLNK(inode->i_mode))
3873 		kfree(inode->i_link);
3874 	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3875 }
3876 
shmem_destroy_inode(struct inode * inode)3877 static void shmem_destroy_inode(struct inode *inode)
3878 {
3879 	if (S_ISREG(inode->i_mode))
3880 		mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3881 }
3882 
shmem_init_inode(void * foo)3883 static void shmem_init_inode(void *foo)
3884 {
3885 	struct shmem_inode_info *info = foo;
3886 	inode_init_once(&info->vfs_inode);
3887 }
3888 
shmem_init_inodecache(void)3889 static void shmem_init_inodecache(void)
3890 {
3891 	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3892 				sizeof(struct shmem_inode_info),
3893 				0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3894 }
3895 
shmem_destroy_inodecache(void)3896 static void shmem_destroy_inodecache(void)
3897 {
3898 	kmem_cache_destroy(shmem_inode_cachep);
3899 }
3900 
3901 /* Keep the page in page cache instead of truncating it */
shmem_error_remove_page(struct address_space * mapping,struct page * page)3902 static int shmem_error_remove_page(struct address_space *mapping,
3903 				   struct page *page)
3904 {
3905 	return 0;
3906 }
3907 
3908 const struct address_space_operations shmem_aops = {
3909 	.writepage	= shmem_writepage,
3910 	.dirty_folio	= noop_dirty_folio,
3911 #ifdef CONFIG_TMPFS
3912 	.write_begin	= shmem_write_begin,
3913 	.write_end	= shmem_write_end,
3914 #endif
3915 #ifdef CONFIG_MIGRATION
3916 	.migrate_folio	= migrate_folio,
3917 #endif
3918 	.error_remove_page = shmem_error_remove_page,
3919 };
3920 EXPORT_SYMBOL(shmem_aops);
3921 
3922 static const struct file_operations shmem_file_operations = {
3923 	.mmap		= shmem_mmap,
3924 	.get_unmapped_area = shmem_get_unmapped_area,
3925 #ifdef CONFIG_TMPFS
3926 	.llseek		= shmem_file_llseek,
3927 	.read_iter	= shmem_file_read_iter,
3928 	.write_iter	= generic_file_write_iter,
3929 	.fsync		= noop_fsync,
3930 	.splice_read	= generic_file_splice_read,
3931 	.splice_write	= iter_file_splice_write,
3932 	.fallocate	= shmem_fallocate,
3933 #endif
3934 };
3935 
3936 static const struct inode_operations shmem_inode_operations = {
3937 	.getattr	= shmem_getattr,
3938 	.setattr	= shmem_setattr,
3939 #ifdef CONFIG_TMPFS_XATTR
3940 	.listxattr	= shmem_listxattr,
3941 	.set_acl	= simple_set_acl,
3942 	.fileattr_get	= shmem_fileattr_get,
3943 	.fileattr_set	= shmem_fileattr_set,
3944 #endif
3945 };
3946 
3947 static const struct inode_operations shmem_dir_inode_operations = {
3948 #ifdef CONFIG_TMPFS
3949 	.getattr	= shmem_getattr,
3950 	.create		= shmem_create,
3951 	.lookup		= simple_lookup,
3952 	.link		= shmem_link,
3953 	.unlink		= shmem_unlink,
3954 	.symlink	= shmem_symlink,
3955 	.mkdir		= shmem_mkdir,
3956 	.rmdir		= shmem_rmdir,
3957 	.mknod		= shmem_mknod,
3958 	.rename		= shmem_rename2,
3959 	.tmpfile	= shmem_tmpfile,
3960 #endif
3961 #ifdef CONFIG_TMPFS_XATTR
3962 	.listxattr	= shmem_listxattr,
3963 	.fileattr_get	= shmem_fileattr_get,
3964 	.fileattr_set	= shmem_fileattr_set,
3965 #endif
3966 #ifdef CONFIG_TMPFS_POSIX_ACL
3967 	.setattr	= shmem_setattr,
3968 	.set_acl	= simple_set_acl,
3969 #endif
3970 };
3971 
3972 static const struct inode_operations shmem_special_inode_operations = {
3973 	.getattr	= shmem_getattr,
3974 #ifdef CONFIG_TMPFS_XATTR
3975 	.listxattr	= shmem_listxattr,
3976 #endif
3977 #ifdef CONFIG_TMPFS_POSIX_ACL
3978 	.setattr	= shmem_setattr,
3979 	.set_acl	= simple_set_acl,
3980 #endif
3981 };
3982 
3983 static const struct super_operations shmem_ops = {
3984 	.alloc_inode	= shmem_alloc_inode,
3985 	.free_inode	= shmem_free_in_core_inode,
3986 	.destroy_inode	= shmem_destroy_inode,
3987 #ifdef CONFIG_TMPFS
3988 	.statfs		= shmem_statfs,
3989 	.show_options	= shmem_show_options,
3990 #endif
3991 	.evict_inode	= shmem_evict_inode,
3992 	.drop_inode	= generic_delete_inode,
3993 	.put_super	= shmem_put_super,
3994 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3995 	.nr_cached_objects	= shmem_unused_huge_count,
3996 	.free_cached_objects	= shmem_unused_huge_scan,
3997 #endif
3998 };
3999 
4000 static const struct vm_operations_struct shmem_vm_ops = {
4001 	.fault		= shmem_fault,
4002 	.map_pages	= filemap_map_pages,
4003 #ifdef CONFIG_NUMA
4004 	.set_policy     = shmem_set_policy,
4005 	.get_policy     = shmem_get_policy,
4006 #endif
4007 };
4008 
shmem_init_fs_context(struct fs_context * fc)4009 int shmem_init_fs_context(struct fs_context *fc)
4010 {
4011 	struct shmem_options *ctx;
4012 
4013 	ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
4014 	if (!ctx)
4015 		return -ENOMEM;
4016 
4017 	ctx->mode = 0777 | S_ISVTX;
4018 	ctx->uid = current_fsuid();
4019 	ctx->gid = current_fsgid();
4020 
4021 	fc->fs_private = ctx;
4022 	fc->ops = &shmem_fs_context_ops;
4023 	return 0;
4024 }
4025 
4026 static struct file_system_type shmem_fs_type = {
4027 	.owner		= THIS_MODULE,
4028 	.name		= "tmpfs",
4029 	.init_fs_context = shmem_init_fs_context,
4030 #ifdef CONFIG_TMPFS
4031 	.parameters	= shmem_fs_parameters,
4032 #endif
4033 	.kill_sb	= kill_litter_super,
4034 	.fs_flags	= FS_USERNS_MOUNT,
4035 };
4036 
shmem_init(void)4037 void __init shmem_init(void)
4038 {
4039 	int error;
4040 
4041 	shmem_init_inodecache();
4042 
4043 	error = register_filesystem(&shmem_fs_type);
4044 	if (error) {
4045 		pr_err("Could not register tmpfs\n");
4046 		goto out2;
4047 	}
4048 
4049 	shm_mnt = kern_mount(&shmem_fs_type);
4050 	if (IS_ERR(shm_mnt)) {
4051 		error = PTR_ERR(shm_mnt);
4052 		pr_err("Could not kern_mount tmpfs\n");
4053 		goto out1;
4054 	}
4055 
4056 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4057 	if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4058 		SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4059 	else
4060 		shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
4061 #endif
4062 	return;
4063 
4064 out1:
4065 	unregister_filesystem(&shmem_fs_type);
4066 out2:
4067 	shmem_destroy_inodecache();
4068 	shm_mnt = ERR_PTR(error);
4069 }
4070 
4071 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
shmem_enabled_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)4072 static ssize_t shmem_enabled_show(struct kobject *kobj,
4073 				  struct kobj_attribute *attr, char *buf)
4074 {
4075 	static const int values[] = {
4076 		SHMEM_HUGE_ALWAYS,
4077 		SHMEM_HUGE_WITHIN_SIZE,
4078 		SHMEM_HUGE_ADVISE,
4079 		SHMEM_HUGE_NEVER,
4080 		SHMEM_HUGE_DENY,
4081 		SHMEM_HUGE_FORCE,
4082 	};
4083 	int len = 0;
4084 	int i;
4085 
4086 	for (i = 0; i < ARRAY_SIZE(values); i++) {
4087 		len += sysfs_emit_at(buf, len,
4088 				     shmem_huge == values[i] ? "%s[%s]" : "%s%s",
4089 				     i ? " " : "",
4090 				     shmem_format_huge(values[i]));
4091 	}
4092 
4093 	len += sysfs_emit_at(buf, len, "\n");
4094 
4095 	return len;
4096 }
4097 
shmem_enabled_store(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)4098 static ssize_t shmem_enabled_store(struct kobject *kobj,
4099 		struct kobj_attribute *attr, const char *buf, size_t count)
4100 {
4101 	char tmp[16];
4102 	int huge;
4103 
4104 	if (count + 1 > sizeof(tmp))
4105 		return -EINVAL;
4106 	memcpy(tmp, buf, count);
4107 	tmp[count] = '\0';
4108 	if (count && tmp[count - 1] == '\n')
4109 		tmp[count - 1] = '\0';
4110 
4111 	huge = shmem_parse_huge(tmp);
4112 	if (huge == -EINVAL)
4113 		return -EINVAL;
4114 	if (!has_transparent_hugepage() &&
4115 			huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4116 		return -EINVAL;
4117 
4118 	shmem_huge = huge;
4119 	if (shmem_huge > SHMEM_HUGE_DENY)
4120 		SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4121 	return count;
4122 }
4123 
4124 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
4125 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4126 
4127 #else /* !CONFIG_SHMEM */
4128 
4129 /*
4130  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
4131  *
4132  * This is intended for small system where the benefits of the full
4133  * shmem code (swap-backed and resource-limited) are outweighed by
4134  * their complexity. On systems without swap this code should be
4135  * effectively equivalent, but much lighter weight.
4136  */
4137 
4138 static struct file_system_type shmem_fs_type = {
4139 	.name		= "tmpfs",
4140 	.init_fs_context = ramfs_init_fs_context,
4141 	.parameters	= ramfs_fs_parameters,
4142 	.kill_sb	= kill_litter_super,
4143 	.fs_flags	= FS_USERNS_MOUNT,
4144 };
4145 
shmem_init(void)4146 void __init shmem_init(void)
4147 {
4148 	BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4149 
4150 	shm_mnt = kern_mount(&shmem_fs_type);
4151 	BUG_ON(IS_ERR(shm_mnt));
4152 }
4153 
shmem_unuse(unsigned int type)4154 int shmem_unuse(unsigned int type)
4155 {
4156 	return 0;
4157 }
4158 
shmem_lock(struct file * file,int lock,struct ucounts * ucounts)4159 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
4160 {
4161 	return 0;
4162 }
4163 
shmem_unlock_mapping(struct address_space * mapping)4164 void shmem_unlock_mapping(struct address_space *mapping)
4165 {
4166 }
4167 
4168 #ifdef CONFIG_MMU
shmem_get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)4169 unsigned long shmem_get_unmapped_area(struct file *file,
4170 				      unsigned long addr, unsigned long len,
4171 				      unsigned long pgoff, unsigned long flags)
4172 {
4173 	return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4174 }
4175 #endif
4176 
shmem_truncate_range(struct inode * inode,loff_t lstart,loff_t lend)4177 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4178 {
4179 	truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4180 }
4181 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4182 
4183 #define shmem_vm_ops				generic_file_vm_ops
4184 #define shmem_file_operations			ramfs_file_operations
4185 #define shmem_get_inode(sb, dir, mode, dev, flags)	ramfs_get_inode(sb, dir, mode, dev)
4186 #define shmem_acct_size(flags, size)		0
4187 #define shmem_unacct_size(flags, size)		do {} while (0)
4188 
4189 #endif /* CONFIG_SHMEM */
4190 
4191 /* common code */
4192 
__shmem_file_setup(struct vfsmount * mnt,const char * name,loff_t size,unsigned long flags,unsigned int i_flags)4193 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4194 				       unsigned long flags, unsigned int i_flags)
4195 {
4196 	struct inode *inode;
4197 	struct file *res;
4198 
4199 	if (IS_ERR(mnt))
4200 		return ERR_CAST(mnt);
4201 
4202 	if (size < 0 || size > MAX_LFS_FILESIZE)
4203 		return ERR_PTR(-EINVAL);
4204 
4205 	if (shmem_acct_size(flags, size))
4206 		return ERR_PTR(-ENOMEM);
4207 
4208 	inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
4209 				flags);
4210 	if (unlikely(!inode)) {
4211 		shmem_unacct_size(flags, size);
4212 		return ERR_PTR(-ENOSPC);
4213 	}
4214 	inode->i_flags |= i_flags;
4215 	inode->i_size = size;
4216 	clear_nlink(inode);	/* It is unlinked */
4217 	res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4218 	if (!IS_ERR(res))
4219 		res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4220 				&shmem_file_operations);
4221 	if (IS_ERR(res))
4222 		iput(inode);
4223 	return res;
4224 }
4225 
4226 /**
4227  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4228  * 	kernel internal.  There will be NO LSM permission checks against the
4229  * 	underlying inode.  So users of this interface must do LSM checks at a
4230  *	higher layer.  The users are the big_key and shm implementations.  LSM
4231  *	checks are provided at the key or shm level rather than the inode.
4232  * @name: name for dentry (to be seen in /proc/<pid>/maps
4233  * @size: size to be set for the file
4234  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4235  */
shmem_kernel_file_setup(const char * name,loff_t size,unsigned long flags)4236 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4237 {
4238 	return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4239 }
4240 
4241 /**
4242  * shmem_file_setup - get an unlinked file living in tmpfs
4243  * @name: name for dentry (to be seen in /proc/<pid>/maps
4244  * @size: size to be set for the file
4245  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4246  */
shmem_file_setup(const char * name,loff_t size,unsigned long flags)4247 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4248 {
4249 	return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4250 }
4251 EXPORT_SYMBOL_GPL(shmem_file_setup);
4252 
4253 /**
4254  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4255  * @mnt: the tmpfs mount where the file will be created
4256  * @name: name for dentry (to be seen in /proc/<pid>/maps
4257  * @size: size to be set for the file
4258  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4259  */
shmem_file_setup_with_mnt(struct vfsmount * mnt,const char * name,loff_t size,unsigned long flags)4260 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4261 				       loff_t size, unsigned long flags)
4262 {
4263 	return __shmem_file_setup(mnt, name, size, flags, 0);
4264 }
4265 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4266 
4267 /**
4268  * shmem_zero_setup - setup a shared anonymous mapping
4269  * @vma: the vma to be mmapped is prepared by do_mmap
4270  */
shmem_zero_setup(struct vm_area_struct * vma)4271 int shmem_zero_setup(struct vm_area_struct *vma)
4272 {
4273 	struct file *file;
4274 	loff_t size = vma->vm_end - vma->vm_start;
4275 
4276 	/*
4277 	 * Cloning a new file under mmap_lock leads to a lock ordering conflict
4278 	 * between XFS directory reading and selinux: since this file is only
4279 	 * accessible to the user through its mapping, use S_PRIVATE flag to
4280 	 * bypass file security, in the same way as shmem_kernel_file_setup().
4281 	 */
4282 	file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4283 	if (IS_ERR(file))
4284 		return PTR_ERR(file);
4285 
4286 	if (vma->vm_file)
4287 		fput(vma->vm_file);
4288 	vma->vm_file = file;
4289 	vma->vm_ops = &shmem_vm_ops;
4290 
4291 	return 0;
4292 }
4293 
4294 /**
4295  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4296  * @mapping:	the page's address_space
4297  * @index:	the page index
4298  * @gfp:	the page allocator flags to use if allocating
4299  *
4300  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4301  * with any new page allocations done using the specified allocation flags.
4302  * But read_cache_page_gfp() uses the ->read_folio() method: which does not
4303  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4304  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4305  *
4306  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4307  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4308  */
shmem_read_mapping_page_gfp(struct address_space * mapping,pgoff_t index,gfp_t gfp)4309 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4310 					 pgoff_t index, gfp_t gfp)
4311 {
4312 #ifdef CONFIG_SHMEM
4313 	struct inode *inode = mapping->host;
4314 	struct folio *folio;
4315 	struct page *page;
4316 	int error;
4317 
4318 	BUG_ON(!shmem_mapping(mapping));
4319 	error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
4320 				  gfp, NULL, NULL, NULL);
4321 	if (error)
4322 		return ERR_PTR(error);
4323 
4324 	folio_unlock(folio);
4325 	page = folio_file_page(folio, index);
4326 	if (PageHWPoison(page)) {
4327 		folio_put(folio);
4328 		return ERR_PTR(-EIO);
4329 	}
4330 
4331 	return page;
4332 #else
4333 	/*
4334 	 * The tiny !SHMEM case uses ramfs without swap
4335 	 */
4336 	return read_cache_page_gfp(mapping, index, gfp);
4337 #endif
4338 }
4339 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
4340