Version:  2.0.40 2.2.26 2.4.37 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16

Linux/mm/shmem.c

  1 /*
  2  * Resizable virtual memory filesystem for Linux.
  3  *
  4  * Copyright (C) 2000 Linus Torvalds.
  5  *               2000 Transmeta Corp.
  6  *               2000-2001 Christoph Rohland
  7  *               2000-2001 SAP AG
  8  *               2002 Red Hat Inc.
  9  * Copyright (C) 2002-2011 Hugh Dickins.
 10  * Copyright (C) 2011 Google Inc.
 11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
 12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
 13  *
 14  * Extended attribute support for tmpfs:
 15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
 16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
 17  *
 18  * tiny-shmem:
 19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
 20  *
 21  * This file is released under the GPL.
 22  */
 23 
 24 #include <linux/fs.h>
 25 #include <linux/init.h>
 26 #include <linux/vfs.h>
 27 #include <linux/mount.h>
 28 #include <linux/ramfs.h>
 29 #include <linux/pagemap.h>
 30 #include <linux/file.h>
 31 #include <linux/mm.h>
 32 #include <linux/export.h>
 33 #include <linux/swap.h>
 34 #include <linux/aio.h>
 35 
 36 static struct vfsmount *shm_mnt;
 37 
 38 #ifdef CONFIG_SHMEM
 39 /*
 40  * This virtual memory filesystem is heavily based on the ramfs. It
 41  * extends ramfs by the ability to use swap and honor resource limits
 42  * which makes it a completely usable filesystem.
 43  */
 44 
 45 #include <linux/xattr.h>
 46 #include <linux/exportfs.h>
 47 #include <linux/posix_acl.h>
 48 #include <linux/posix_acl_xattr.h>
 49 #include <linux/mman.h>
 50 #include <linux/string.h>
 51 #include <linux/slab.h>
 52 #include <linux/backing-dev.h>
 53 #include <linux/shmem_fs.h>
 54 #include <linux/writeback.h>
 55 #include <linux/blkdev.h>
 56 #include <linux/pagevec.h>
 57 #include <linux/percpu_counter.h>
 58 #include <linux/falloc.h>
 59 #include <linux/splice.h>
 60 #include <linux/security.h>
 61 #include <linux/swapops.h>
 62 #include <linux/mempolicy.h>
 63 #include <linux/namei.h>
 64 #include <linux/ctype.h>
 65 #include <linux/migrate.h>
 66 #include <linux/highmem.h>
 67 #include <linux/seq_file.h>
 68 #include <linux/magic.h>
 69 
 70 #include <asm/uaccess.h>
 71 #include <asm/pgtable.h>
 72 
 73 #define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
 74 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
 75 
 76 /* Pretend that each entry is of this size in directory's i_size */
 77 #define BOGO_DIRENT_SIZE 20
 78 
 79 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
 80 #define SHORT_SYMLINK_LEN 128
 81 
 82 /*
 83  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
 84  * inode->i_private (with i_mutex making sure that it has only one user at
 85  * a time): we would prefer not to enlarge the shmem inode just for that.
 86  */
 87 struct shmem_falloc {
 88         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
 89         pgoff_t start;          /* start of range currently being fallocated */
 90         pgoff_t next;           /* the next page offset to be fallocated */
 91         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
 92         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
 93 };
 94 
 95 /* Flag allocation requirements to shmem_getpage */
 96 enum sgp_type {
 97         SGP_READ,       /* don't exceed i_size, don't allocate page */
 98         SGP_CACHE,      /* don't exceed i_size, may allocate page */
 99         SGP_DIRTY,      /* like SGP_CACHE, but set new page dirty */
100         SGP_WRITE,      /* may exceed i_size, may allocate !Uptodate page */
101         SGP_FALLOC,     /* like SGP_WRITE, but make existing page Uptodate */
102 };
103 
104 #ifdef CONFIG_TMPFS
105 static unsigned long shmem_default_max_blocks(void)
106 {
107         return totalram_pages / 2;
108 }
109 
110 static unsigned long shmem_default_max_inodes(void)
111 {
112         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
113 }
114 #endif
115 
116 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
117 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
118                                 struct shmem_inode_info *info, pgoff_t index);
119 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
120         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
121 
122 static inline int shmem_getpage(struct inode *inode, pgoff_t index,
123         struct page **pagep, enum sgp_type sgp, int *fault_type)
124 {
125         return shmem_getpage_gfp(inode, index, pagep, sgp,
126                         mapping_gfp_mask(inode->i_mapping), fault_type);
127 }
128 
129 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
130 {
131         return sb->s_fs_info;
132 }
133 
134 /*
135  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
136  * for shared memory and for shared anonymous (/dev/zero) mappings
137  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
138  * consistent with the pre-accounting of private mappings ...
139  */
140 static inline int shmem_acct_size(unsigned long flags, loff_t size)
141 {
142         return (flags & VM_NORESERVE) ?
143                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
144 }
145 
146 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
147 {
148         if (!(flags & VM_NORESERVE))
149                 vm_unacct_memory(VM_ACCT(size));
150 }
151 
152 /*
153  * ... whereas tmpfs objects are accounted incrementally as
154  * pages are allocated, in order to allow huge sparse files.
155  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
156  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
157  */
158 static inline int shmem_acct_block(unsigned long flags)
159 {
160         return (flags & VM_NORESERVE) ?
161                 security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_CACHE_SIZE)) : 0;
162 }
163 
164 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
165 {
166         if (flags & VM_NORESERVE)
167                 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
168 }
169 
170 static const struct super_operations shmem_ops;
171 static const struct address_space_operations shmem_aops;
172 static const struct file_operations shmem_file_operations;
173 static const struct inode_operations shmem_inode_operations;
174 static const struct inode_operations shmem_dir_inode_operations;
175 static const struct inode_operations shmem_special_inode_operations;
176 static const struct vm_operations_struct shmem_vm_ops;
177 
178 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
179         .ra_pages       = 0,    /* No readahead */
180         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
181 };
182 
183 static LIST_HEAD(shmem_swaplist);
184 static DEFINE_MUTEX(shmem_swaplist_mutex);
185 
186 static int shmem_reserve_inode(struct super_block *sb)
187 {
188         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
189         if (sbinfo->max_inodes) {
190                 spin_lock(&sbinfo->stat_lock);
191                 if (!sbinfo->free_inodes) {
192                         spin_unlock(&sbinfo->stat_lock);
193                         return -ENOSPC;
194                 }
195                 sbinfo->free_inodes--;
196                 spin_unlock(&sbinfo->stat_lock);
197         }
198         return 0;
199 }
200 
201 static void shmem_free_inode(struct super_block *sb)
202 {
203         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
204         if (sbinfo->max_inodes) {
205                 spin_lock(&sbinfo->stat_lock);
206                 sbinfo->free_inodes++;
207                 spin_unlock(&sbinfo->stat_lock);
208         }
209 }
210 
211 /**
212  * shmem_recalc_inode - recalculate the block usage of an inode
213  * @inode: inode to recalc
214  *
215  * We have to calculate the free blocks since the mm can drop
216  * undirtied hole pages behind our back.
217  *
218  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
219  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
220  *
221  * It has to be called with the spinlock held.
222  */
223 static void shmem_recalc_inode(struct inode *inode)
224 {
225         struct shmem_inode_info *info = SHMEM_I(inode);
226         long freed;
227 
228         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
229         if (freed > 0) {
230                 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
231                 if (sbinfo->max_blocks)
232                         percpu_counter_add(&sbinfo->used_blocks, -freed);
233                 info->alloced -= freed;
234                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
235                 shmem_unacct_blocks(info->flags, freed);
236         }
237 }
238 
239 /*
240  * Replace item expected in radix tree by a new item, while holding tree lock.
241  */
242 static int shmem_radix_tree_replace(struct address_space *mapping,
243                         pgoff_t index, void *expected, void *replacement)
244 {
245         void **pslot;
246         void *item;
247 
248         VM_BUG_ON(!expected);
249         VM_BUG_ON(!replacement);
250         pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
251         if (!pslot)
252                 return -ENOENT;
253         item = radix_tree_deref_slot_protected(pslot, &mapping->tree_lock);
254         if (item != expected)
255                 return -ENOENT;
256         radix_tree_replace_slot(pslot, replacement);
257         return 0;
258 }
259 
260 /*
261  * Sometimes, before we decide whether to proceed or to fail, we must check
262  * that an entry was not already brought back from swap by a racing thread.
263  *
264  * Checking page is not enough: by the time a SwapCache page is locked, it
265  * might be reused, and again be SwapCache, using the same swap as before.
266  */
267 static bool shmem_confirm_swap(struct address_space *mapping,
268                                pgoff_t index, swp_entry_t swap)
269 {
270         void *item;
271 
272         rcu_read_lock();
273         item = radix_tree_lookup(&mapping->page_tree, index);
274         rcu_read_unlock();
275         return item == swp_to_radix_entry(swap);
276 }
277 
278 /*
279  * Like add_to_page_cache_locked, but error if expected item has gone.
280  */
281 static int shmem_add_to_page_cache(struct page *page,
282                                    struct address_space *mapping,
283                                    pgoff_t index, gfp_t gfp, void *expected)
284 {
285         int error;
286 
287         VM_BUG_ON_PAGE(!PageLocked(page), page);
288         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
289 
290         page_cache_get(page);
291         page->mapping = mapping;
292         page->index = index;
293 
294         spin_lock_irq(&mapping->tree_lock);
295         if (!expected)
296                 error = radix_tree_insert(&mapping->page_tree, index, page);
297         else
298                 error = shmem_radix_tree_replace(mapping, index, expected,
299                                                                  page);
300         if (!error) {
301                 mapping->nrpages++;
302                 __inc_zone_page_state(page, NR_FILE_PAGES);
303                 __inc_zone_page_state(page, NR_SHMEM);
304                 spin_unlock_irq(&mapping->tree_lock);
305         } else {
306                 page->mapping = NULL;
307                 spin_unlock_irq(&mapping->tree_lock);
308                 page_cache_release(page);
309         }
310         return error;
311 }
312 
313 /*
314  * Like delete_from_page_cache, but substitutes swap for page.
315  */
316 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
317 {
318         struct address_space *mapping = page->mapping;
319         int error;
320 
321         spin_lock_irq(&mapping->tree_lock);
322         error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
323         page->mapping = NULL;
324         mapping->nrpages--;
325         __dec_zone_page_state(page, NR_FILE_PAGES);
326         __dec_zone_page_state(page, NR_SHMEM);
327         spin_unlock_irq(&mapping->tree_lock);
328         page_cache_release(page);
329         BUG_ON(error);
330 }
331 
332 /*
333  * Remove swap entry from radix tree, free the swap and its page cache.
334  */
335 static int shmem_free_swap(struct address_space *mapping,
336                            pgoff_t index, void *radswap)
337 {
338         void *old;
339 
340         spin_lock_irq(&mapping->tree_lock);
341         old = radix_tree_delete_item(&mapping->page_tree, index, radswap);
342         spin_unlock_irq(&mapping->tree_lock);
343         if (old != radswap)
344                 return -ENOENT;
345         free_swap_and_cache(radix_to_swp_entry(radswap));
346         return 0;
347 }
348 
349 /*
350  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
351  */
352 void shmem_unlock_mapping(struct address_space *mapping)
353 {
354         struct pagevec pvec;
355         pgoff_t indices[PAGEVEC_SIZE];
356         pgoff_t index = 0;
357 
358         pagevec_init(&pvec, 0);
359         /*
360          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
361          */
362         while (!mapping_unevictable(mapping)) {
363                 /*
364                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
365                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
366                  */
367                 pvec.nr = find_get_entries(mapping, index,
368                                            PAGEVEC_SIZE, pvec.pages, indices);
369                 if (!pvec.nr)
370                         break;
371                 index = indices[pvec.nr - 1] + 1;
372                 pagevec_remove_exceptionals(&pvec);
373                 check_move_unevictable_pages(pvec.pages, pvec.nr);
374                 pagevec_release(&pvec);
375                 cond_resched();
376         }
377 }
378 
379 /*
380  * Remove range of pages and swap entries from radix tree, and free them.
381  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
382  */
383 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
384                                                                  bool unfalloc)
385 {
386         struct address_space *mapping = inode->i_mapping;
387         struct shmem_inode_info *info = SHMEM_I(inode);
388         pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
389         pgoff_t end = (lend + 1) >> PAGE_CACHE_SHIFT;
390         unsigned int partial_start = lstart & (PAGE_CACHE_SIZE - 1);
391         unsigned int partial_end = (lend + 1) & (PAGE_CACHE_SIZE - 1);
392         struct pagevec pvec;
393         pgoff_t indices[PAGEVEC_SIZE];
394         long nr_swaps_freed = 0;
395         pgoff_t index;
396         int i;
397 
398         if (lend == -1)
399                 end = -1;       /* unsigned, so actually very big */
400 
401         pagevec_init(&pvec, 0);
402         index = start;
403         while (index < end) {
404                 pvec.nr = find_get_entries(mapping, index,
405                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
406                         pvec.pages, indices);
407                 if (!pvec.nr)
408                         break;
409                 mem_cgroup_uncharge_start();
410                 for (i = 0; i < pagevec_count(&pvec); i++) {
411                         struct page *page = pvec.pages[i];
412 
413                         index = indices[i];
414                         if (index >= end)
415                                 break;
416 
417                         if (radix_tree_exceptional_entry(page)) {
418                                 if (unfalloc)
419                                         continue;
420                                 nr_swaps_freed += !shmem_free_swap(mapping,
421                                                                 index, page);
422                                 continue;
423                         }
424 
425                         if (!trylock_page(page))
426                                 continue;
427                         if (!unfalloc || !PageUptodate(page)) {
428                                 if (page->mapping == mapping) {
429                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
430                                         truncate_inode_page(mapping, page);
431                                 }
432                         }
433                         unlock_page(page);
434                 }
435                 pagevec_remove_exceptionals(&pvec);
436                 pagevec_release(&pvec);
437                 mem_cgroup_uncharge_end();
438                 cond_resched();
439                 index++;
440         }
441 
442         if (partial_start) {
443                 struct page *page = NULL;
444                 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
445                 if (page) {
446                         unsigned int top = PAGE_CACHE_SIZE;
447                         if (start > end) {
448                                 top = partial_end;
449                                 partial_end = 0;
450                         }
451                         zero_user_segment(page, partial_start, top);
452                         set_page_dirty(page);
453                         unlock_page(page);
454                         page_cache_release(page);
455                 }
456         }
457         if (partial_end) {
458                 struct page *page = NULL;
459                 shmem_getpage(inode, end, &page, SGP_READ, NULL);
460                 if (page) {
461                         zero_user_segment(page, 0, partial_end);
462                         set_page_dirty(page);
463                         unlock_page(page);
464                         page_cache_release(page);
465                 }
466         }
467         if (start >= end)
468                 return;
469 
470         index = start;
471         while (index < end) {
472                 cond_resched();
473 
474                 pvec.nr = find_get_entries(mapping, index,
475                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
476                                 pvec.pages, indices);
477                 if (!pvec.nr) {
478                         /* If all gone or hole-punch or unfalloc, we're done */
479                         if (index == start || end != -1)
480                                 break;
481                         /* But if truncating, restart to make sure all gone */
482                         index = start;
483                         continue;
484                 }
485                 mem_cgroup_uncharge_start();
486                 for (i = 0; i < pagevec_count(&pvec); i++) {
487                         struct page *page = pvec.pages[i];
488 
489                         index = indices[i];
490                         if (index >= end)
491                                 break;
492 
493                         if (radix_tree_exceptional_entry(page)) {
494                                 if (unfalloc)
495                                         continue;
496                                 if (shmem_free_swap(mapping, index, page)) {
497                                         /* Swap was replaced by page: retry */
498                                         index--;
499                                         break;
500                                 }
501                                 nr_swaps_freed++;
502                                 continue;
503                         }
504 
505                         lock_page(page);
506                         if (!unfalloc || !PageUptodate(page)) {
507                                 if (page->mapping == mapping) {
508                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
509                                         truncate_inode_page(mapping, page);
510                                 } else {
511                                         /* Page was replaced by swap: retry */
512                                         unlock_page(page);
513                                         index--;
514                                         break;
515                                 }
516                         }
517                         unlock_page(page);
518                 }
519                 pagevec_remove_exceptionals(&pvec);
520                 pagevec_release(&pvec);
521                 mem_cgroup_uncharge_end();
522                 index++;
523         }
524 
525         spin_lock(&info->lock);
526         info->swapped -= nr_swaps_freed;
527         shmem_recalc_inode(inode);
528         spin_unlock(&info->lock);
529 }
530 
531 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
532 {
533         shmem_undo_range(inode, lstart, lend, false);
534         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
535 }
536 EXPORT_SYMBOL_GPL(shmem_truncate_range);
537 
538 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
539 {
540         struct inode *inode = dentry->d_inode;
541         int error;
542 
543         error = inode_change_ok(inode, attr);
544         if (error)
545                 return error;
546 
547         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
548                 loff_t oldsize = inode->i_size;
549                 loff_t newsize = attr->ia_size;
550 
551                 if (newsize != oldsize) {
552                         i_size_write(inode, newsize);
553                         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
554                 }
555                 if (newsize < oldsize) {
556                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
557                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
558                         shmem_truncate_range(inode, newsize, (loff_t)-1);
559                         /* unmap again to remove racily COWed private pages */
560                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
561                 }
562         }
563 
564         setattr_copy(inode, attr);
565         if (attr->ia_valid & ATTR_MODE)
566                 error = posix_acl_chmod(inode, inode->i_mode);
567         return error;
568 }
569 
570 static void shmem_evict_inode(struct inode *inode)
571 {
572         struct shmem_inode_info *info = SHMEM_I(inode);
573 
574         if (inode->i_mapping->a_ops == &shmem_aops) {
575                 shmem_unacct_size(info->flags, inode->i_size);
576                 inode->i_size = 0;
577                 shmem_truncate_range(inode, 0, (loff_t)-1);
578                 if (!list_empty(&info->swaplist)) {
579                         mutex_lock(&shmem_swaplist_mutex);
580                         list_del_init(&info->swaplist);
581                         mutex_unlock(&shmem_swaplist_mutex);
582                 }
583         } else
584                 kfree(info->symlink);
585 
586         simple_xattrs_free(&info->xattrs);
587         WARN_ON(inode->i_blocks);
588         shmem_free_inode(inode->i_sb);
589         clear_inode(inode);
590 }
591 
592 /*
593  * If swap found in inode, free it and move page from swapcache to filecache.
594  */
595 static int shmem_unuse_inode(struct shmem_inode_info *info,
596                              swp_entry_t swap, struct page **pagep)
597 {
598         struct address_space *mapping = info->vfs_inode.i_mapping;
599         void *radswap;
600         pgoff_t index;
601         gfp_t gfp;
602         int error = 0;
603 
604         radswap = swp_to_radix_entry(swap);
605         index = radix_tree_locate_item(&mapping->page_tree, radswap);
606         if (index == -1)
607                 return 0;
608 
609         /*
610          * Move _head_ to start search for next from here.
611          * But be careful: shmem_evict_inode checks list_empty without taking
612          * mutex, and there's an instant in list_move_tail when info->swaplist
613          * would appear empty, if it were the only one on shmem_swaplist.
614          */
615         if (shmem_swaplist.next != &info->swaplist)
616                 list_move_tail(&shmem_swaplist, &info->swaplist);
617 
618         gfp = mapping_gfp_mask(mapping);
619         if (shmem_should_replace_page(*pagep, gfp)) {
620                 mutex_unlock(&shmem_swaplist_mutex);
621                 error = shmem_replace_page(pagep, gfp, info, index);
622                 mutex_lock(&shmem_swaplist_mutex);
623                 /*
624                  * We needed to drop mutex to make that restrictive page
625                  * allocation, but the inode might have been freed while we
626                  * dropped it: although a racing shmem_evict_inode() cannot
627                  * complete without emptying the radix_tree, our page lock
628                  * on this swapcache page is not enough to prevent that -
629                  * free_swap_and_cache() of our swap entry will only
630                  * trylock_page(), removing swap from radix_tree whatever.
631                  *
632                  * We must not proceed to shmem_add_to_page_cache() if the
633                  * inode has been freed, but of course we cannot rely on
634                  * inode or mapping or info to check that.  However, we can
635                  * safely check if our swap entry is still in use (and here
636                  * it can't have got reused for another page): if it's still
637                  * in use, then the inode cannot have been freed yet, and we
638                  * can safely proceed (if it's no longer in use, that tells
639                  * nothing about the inode, but we don't need to unuse swap).
640                  */
641                 if (!page_swapcount(*pagep))
642                         error = -ENOENT;
643         }
644 
645         /*
646          * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
647          * but also to hold up shmem_evict_inode(): so inode cannot be freed
648          * beneath us (pagelock doesn't help until the page is in pagecache).
649          */
650         if (!error)
651                 error = shmem_add_to_page_cache(*pagep, mapping, index,
652                                                 GFP_NOWAIT, radswap);
653         if (error != -ENOMEM) {
654                 /*
655                  * Truncation and eviction use free_swap_and_cache(), which
656                  * only does trylock page: if we raced, best clean up here.
657                  */
658                 delete_from_swap_cache(*pagep);
659                 set_page_dirty(*pagep);
660                 if (!error) {
661                         spin_lock(&info->lock);
662                         info->swapped--;
663                         spin_unlock(&info->lock);
664                         swap_free(swap);
665                 }
666                 error = 1;      /* not an error, but entry was found */
667         }
668         return error;
669 }
670 
671 /*
672  * Search through swapped inodes to find and replace swap by page.
673  */
674 int shmem_unuse(swp_entry_t swap, struct page *page)
675 {
676         struct list_head *this, *next;
677         struct shmem_inode_info *info;
678         int found = 0;
679         int error = 0;
680 
681         /*
682          * There's a faint possibility that swap page was replaced before
683          * caller locked it: caller will come back later with the right page.
684          */
685         if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
686                 goto out;
687 
688         /*
689          * Charge page using GFP_KERNEL while we can wait, before taking
690          * the shmem_swaplist_mutex which might hold up shmem_writepage().
691          * Charged back to the user (not to caller) when swap account is used.
692          */
693         error = mem_cgroup_charge_file(page, current->mm, GFP_KERNEL);
694         if (error)
695                 goto out;
696         /* No radix_tree_preload: swap entry keeps a place for page in tree */
697 
698         mutex_lock(&shmem_swaplist_mutex);
699         list_for_each_safe(this, next, &shmem_swaplist) {
700                 info = list_entry(this, struct shmem_inode_info, swaplist);
701                 if (info->swapped)
702                         found = shmem_unuse_inode(info, swap, &page);
703                 else
704                         list_del_init(&info->swaplist);
705                 cond_resched();
706                 if (found)
707                         break;
708         }
709         mutex_unlock(&shmem_swaplist_mutex);
710 
711         if (found < 0)
712                 error = found;
713 out:
714         unlock_page(page);
715         page_cache_release(page);
716         return error;
717 }
718 
719 /*
720  * Move the page from the page cache to the swap cache.
721  */
722 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
723 {
724         struct shmem_inode_info *info;
725         struct address_space *mapping;
726         struct inode *inode;
727         swp_entry_t swap;
728         pgoff_t index;
729 
730         BUG_ON(!PageLocked(page));
731         mapping = page->mapping;
732         index = page->index;
733         inode = mapping->host;
734         info = SHMEM_I(inode);
735         if (info->flags & VM_LOCKED)
736                 goto redirty;
737         if (!total_swap_pages)
738                 goto redirty;
739 
740         /*
741          * shmem_backing_dev_info's capabilities prevent regular writeback or
742          * sync from ever calling shmem_writepage; but a stacking filesystem
743          * might use ->writepage of its underlying filesystem, in which case
744          * tmpfs should write out to swap only in response to memory pressure,
745          * and not for the writeback threads or sync.
746          */
747         if (!wbc->for_reclaim) {
748                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
749                 goto redirty;
750         }
751 
752         /*
753          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
754          * value into swapfile.c, the only way we can correctly account for a
755          * fallocated page arriving here is now to initialize it and write it.
756          *
757          * That's okay for a page already fallocated earlier, but if we have
758          * not yet completed the fallocation, then (a) we want to keep track
759          * of this page in case we have to undo it, and (b) it may not be a
760          * good idea to continue anyway, once we're pushing into swap.  So
761          * reactivate the page, and let shmem_fallocate() quit when too many.
762          */
763         if (!PageUptodate(page)) {
764                 if (inode->i_private) {
765                         struct shmem_falloc *shmem_falloc;
766                         spin_lock(&inode->i_lock);
767                         shmem_falloc = inode->i_private;
768                         if (shmem_falloc &&
769                             !shmem_falloc->waitq &&
770                             index >= shmem_falloc->start &&
771                             index < shmem_falloc->next)
772                                 shmem_falloc->nr_unswapped++;
773                         else
774                                 shmem_falloc = NULL;
775                         spin_unlock(&inode->i_lock);
776                         if (shmem_falloc)
777                                 goto redirty;
778                 }
779                 clear_highpage(page);
780                 flush_dcache_page(page);
781                 SetPageUptodate(page);
782         }
783 
784         swap = get_swap_page();
785         if (!swap.val)
786                 goto redirty;
787 
788         /*
789          * Add inode to shmem_unuse()'s list of swapped-out inodes,
790          * if it's not already there.  Do it now before the page is
791          * moved to swap cache, when its pagelock no longer protects
792          * the inode from eviction.  But don't unlock the mutex until
793          * we've incremented swapped, because shmem_unuse_inode() will
794          * prune a !swapped inode from the swaplist under this mutex.
795          */
796         mutex_lock(&shmem_swaplist_mutex);
797         if (list_empty(&info->swaplist))
798                 list_add_tail(&info->swaplist, &shmem_swaplist);
799 
800         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
801                 swap_shmem_alloc(swap);
802                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
803 
804                 spin_lock(&info->lock);
805                 info->swapped++;
806                 shmem_recalc_inode(inode);
807                 spin_unlock(&info->lock);
808 
809                 mutex_unlock(&shmem_swaplist_mutex);
810                 BUG_ON(page_mapped(page));
811                 swap_writepage(page, wbc);
812                 return 0;
813         }
814 
815         mutex_unlock(&shmem_swaplist_mutex);
816         swapcache_free(swap, NULL);
817 redirty:
818         set_page_dirty(page);
819         if (wbc->for_reclaim)
820                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
821         unlock_page(page);
822         return 0;
823 }
824 
825 #ifdef CONFIG_NUMA
826 #ifdef CONFIG_TMPFS
827 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
828 {
829         char buffer[64];
830 
831         if (!mpol || mpol->mode == MPOL_DEFAULT)
832                 return;         /* show nothing */
833 
834         mpol_to_str(buffer, sizeof(buffer), mpol);
835 
836         seq_printf(seq, ",mpol=%s", buffer);
837 }
838 
839 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
840 {
841         struct mempolicy *mpol = NULL;
842         if (sbinfo->mpol) {
843                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
844                 mpol = sbinfo->mpol;
845                 mpol_get(mpol);
846                 spin_unlock(&sbinfo->stat_lock);
847         }
848         return mpol;
849 }
850 #endif /* CONFIG_TMPFS */
851 
852 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
853                         struct shmem_inode_info *info, pgoff_t index)
854 {
855         struct vm_area_struct pvma;
856         struct page *page;
857 
858         /* Create a pseudo vma that just contains the policy */
859         pvma.vm_start = 0;
860         /* Bias interleave by inode number to distribute better across nodes */
861         pvma.vm_pgoff = index + info->vfs_inode.i_ino;
862         pvma.vm_ops = NULL;
863         pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
864 
865         page = swapin_readahead(swap, gfp, &pvma, 0);
866 
867         /* Drop reference taken by mpol_shared_policy_lookup() */
868         mpol_cond_put(pvma.vm_policy);
869 
870         return page;
871 }
872 
873 static struct page *shmem_alloc_page(gfp_t gfp,
874                         struct shmem_inode_info *info, pgoff_t index)
875 {
876         struct vm_area_struct pvma;
877         struct page *page;
878 
879         /* Create a pseudo vma that just contains the policy */
880         pvma.vm_start = 0;
881         /* Bias interleave by inode number to distribute better across nodes */
882         pvma.vm_pgoff = index + info->vfs_inode.i_ino;
883         pvma.vm_ops = NULL;
884         pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
885 
886         page = alloc_page_vma(gfp, &pvma, 0);
887 
888         /* Drop reference taken by mpol_shared_policy_lookup() */
889         mpol_cond_put(pvma.vm_policy);
890 
891         return page;
892 }
893 #else /* !CONFIG_NUMA */
894 #ifdef CONFIG_TMPFS
895 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
896 {
897 }
898 #endif /* CONFIG_TMPFS */
899 
900 static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
901                         struct shmem_inode_info *info, pgoff_t index)
902 {
903         return swapin_readahead(swap, gfp, NULL, 0);
904 }
905 
906 static inline struct page *shmem_alloc_page(gfp_t gfp,
907                         struct shmem_inode_info *info, pgoff_t index)
908 {
909         return alloc_page(gfp);
910 }
911 #endif /* CONFIG_NUMA */
912 
913 #if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
914 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
915 {
916         return NULL;
917 }
918 #endif
919 
920 /*
921  * When a page is moved from swapcache to shmem filecache (either by the
922  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
923  * shmem_unuse_inode()), it may have been read in earlier from swap, in
924  * ignorance of the mapping it belongs to.  If that mapping has special
925  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
926  * we may need to copy to a suitable page before moving to filecache.
927  *
928  * In a future release, this may well be extended to respect cpuset and
929  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
930  * but for now it is a simple matter of zone.
931  */
932 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
933 {
934         return page_zonenum(page) > gfp_zone(gfp);
935 }
936 
937 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
938                                 struct shmem_inode_info *info, pgoff_t index)
939 {
940         struct page *oldpage, *newpage;
941         struct address_space *swap_mapping;
942         pgoff_t swap_index;
943         int error;
944 
945         oldpage = *pagep;
946         swap_index = page_private(oldpage);
947         swap_mapping = page_mapping(oldpage);
948 
949         /*
950          * We have arrived here because our zones are constrained, so don't
951          * limit chance of success by further cpuset and node constraints.
952          */
953         gfp &= ~GFP_CONSTRAINT_MASK;
954         newpage = shmem_alloc_page(gfp, info, index);
955         if (!newpage)
956                 return -ENOMEM;
957 
958         page_cache_get(newpage);
959         copy_highpage(newpage, oldpage);
960         flush_dcache_page(newpage);
961 
962         __set_page_locked(newpage);
963         SetPageUptodate(newpage);
964         SetPageSwapBacked(newpage);
965         set_page_private(newpage, swap_index);
966         SetPageSwapCache(newpage);
967 
968         /*
969          * Our caller will very soon move newpage out of swapcache, but it's
970          * a nice clean interface for us to replace oldpage by newpage there.
971          */
972         spin_lock_irq(&swap_mapping->tree_lock);
973         error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
974                                                                    newpage);
975         if (!error) {
976                 __inc_zone_page_state(newpage, NR_FILE_PAGES);
977                 __dec_zone_page_state(oldpage, NR_FILE_PAGES);
978         }
979         spin_unlock_irq(&swap_mapping->tree_lock);
980 
981         if (unlikely(error)) {
982                 /*
983                  * Is this possible?  I think not, now that our callers check
984                  * both PageSwapCache and page_private after getting page lock;
985                  * but be defensive.  Reverse old to newpage for clear and free.
986                  */
987                 oldpage = newpage;
988         } else {
989                 mem_cgroup_replace_page_cache(oldpage, newpage);
990                 lru_cache_add_anon(newpage);
991                 *pagep = newpage;
992         }
993 
994         ClearPageSwapCache(oldpage);
995         set_page_private(oldpage, 0);
996 
997         unlock_page(oldpage);
998         page_cache_release(oldpage);
999         page_cache_release(oldpage);
1000         return error;
1001 }
1002 
1003 /*
1004  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1005  *
1006  * If we allocate a new one we do not mark it dirty. That's up to the
1007  * vm. If we swap it in we mark it dirty since we also free the swap
1008  * entry since a page cannot live in both the swap and page cache
1009  */
1010 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1011         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
1012 {
1013         struct address_space *mapping = inode->i_mapping;
1014         struct shmem_inode_info *info;
1015         struct shmem_sb_info *sbinfo;
1016         struct page *page;
1017         swp_entry_t swap;
1018         int error;
1019         int once = 0;
1020         int alloced = 0;
1021 
1022         if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
1023                 return -EFBIG;
1024 repeat:
1025         swap.val = 0;
1026         page = find_lock_entry(mapping, index);
1027         if (radix_tree_exceptional_entry(page)) {
1028                 swap = radix_to_swp_entry(page);
1029                 page = NULL;
1030         }
1031 
1032         if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1033             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1034                 error = -EINVAL;
1035                 goto failed;
1036         }
1037 
1038         if (page && sgp == SGP_WRITE)
1039                 mark_page_accessed(page);
1040 
1041         /* fallocated page? */
1042         if (page && !PageUptodate(page)) {
1043                 if (sgp != SGP_READ)
1044                         goto clear;
1045                 unlock_page(page);
1046                 page_cache_release(page);
1047                 page = NULL;
1048         }
1049         if (page || (sgp == SGP_READ && !swap.val)) {
1050                 *pagep = page;
1051                 return 0;
1052         }
1053 
1054         /*
1055          * Fast cache lookup did not find it:
1056          * bring it back from swap or allocate.
1057          */
1058         info = SHMEM_I(inode);
1059         sbinfo = SHMEM_SB(inode->i_sb);
1060 
1061         if (swap.val) {
1062                 /* Look it up and read it in.. */
1063                 page = lookup_swap_cache(swap);
1064                 if (!page) {
1065                         /* here we actually do the io */
1066                         if (fault_type)
1067                                 *fault_type |= VM_FAULT_MAJOR;
1068                         page = shmem_swapin(swap, gfp, info, index);
1069                         if (!page) {
1070                                 error = -ENOMEM;
1071                                 goto failed;
1072                         }
1073                 }
1074 
1075                 /* We have to do this with page locked to prevent races */
1076                 lock_page(page);
1077                 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1078                     !shmem_confirm_swap(mapping, index, swap)) {
1079                         error = -EEXIST;        /* try again */
1080                         goto unlock;
1081                 }
1082                 if (!PageUptodate(page)) {
1083                         error = -EIO;
1084                         goto failed;
1085                 }
1086                 wait_on_page_writeback(page);
1087 
1088                 if (shmem_should_replace_page(page, gfp)) {
1089                         error = shmem_replace_page(&page, gfp, info, index);
1090                         if (error)
1091                                 goto failed;
1092                 }
1093 
1094                 error = mem_cgroup_charge_file(page, current->mm,
1095                                                 gfp & GFP_RECLAIM_MASK);
1096                 if (!error) {
1097                         error = shmem_add_to_page_cache(page, mapping, index,
1098                                                 gfp, swp_to_radix_entry(swap));
1099                         /*
1100                          * We already confirmed swap under page lock, and make
1101                          * no memory allocation here, so usually no possibility
1102                          * of error; but free_swap_and_cache() only trylocks a
1103                          * page, so it is just possible that the entry has been
1104                          * truncated or holepunched since swap was confirmed.
1105                          * shmem_undo_range() will have done some of the
1106                          * unaccounting, now delete_from_swap_cache() will do
1107                          * the rest (including mem_cgroup_uncharge_swapcache).
1108                          * Reset swap.val? No, leave it so "failed" goes back to
1109                          * "repeat": reading a hole and writing should succeed.
1110                          */
1111                         if (error)
1112                                 delete_from_swap_cache(page);
1113                 }
1114                 if (error)
1115                         goto failed;
1116 
1117                 spin_lock(&info->lock);
1118                 info->swapped--;
1119                 shmem_recalc_inode(inode);
1120                 spin_unlock(&info->lock);
1121 
1122                 if (sgp == SGP_WRITE)
1123                         mark_page_accessed(page);
1124 
1125                 delete_from_swap_cache(page);
1126                 set_page_dirty(page);
1127                 swap_free(swap);
1128 
1129         } else {
1130                 if (shmem_acct_block(info->flags)) {
1131                         error = -ENOSPC;
1132                         goto failed;
1133                 }
1134                 if (sbinfo->max_blocks) {
1135                         if (percpu_counter_compare(&sbinfo->used_blocks,
1136                                                 sbinfo->max_blocks) >= 0) {
1137                                 error = -ENOSPC;
1138                                 goto unacct;
1139                         }
1140                         percpu_counter_inc(&sbinfo->used_blocks);
1141                 }
1142 
1143                 page = shmem_alloc_page(gfp, info, index);
1144                 if (!page) {
1145                         error = -ENOMEM;
1146                         goto decused;
1147                 }
1148 
1149                 __SetPageSwapBacked(page);
1150                 __set_page_locked(page);
1151                 if (sgp == SGP_WRITE)
1152                         init_page_accessed(page);
1153 
1154                 error = mem_cgroup_charge_file(page, current->mm,
1155                                                 gfp & GFP_RECLAIM_MASK);
1156                 if (error)
1157                         goto decused;
1158                 error = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
1159                 if (!error) {
1160                         error = shmem_add_to_page_cache(page, mapping, index,
1161                                                         gfp, NULL);
1162                         radix_tree_preload_end();
1163                 }
1164                 if (error) {
1165                         mem_cgroup_uncharge_cache_page(page);
1166                         goto decused;
1167                 }
1168                 lru_cache_add_anon(page);
1169 
1170                 spin_lock(&info->lock);
1171                 info->alloced++;
1172                 inode->i_blocks += BLOCKS_PER_PAGE;
1173                 shmem_recalc_inode(inode);
1174                 spin_unlock(&info->lock);
1175                 alloced = true;
1176 
1177                 /*
1178                  * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1179                  */
1180                 if (sgp == SGP_FALLOC)
1181                         sgp = SGP_WRITE;
1182 clear:
1183                 /*
1184                  * Let SGP_WRITE caller clear ends if write does not fill page;
1185                  * but SGP_FALLOC on a page fallocated earlier must initialize
1186                  * it now, lest undo on failure cancel our earlier guarantee.
1187                  */
1188                 if (sgp != SGP_WRITE) {
1189                         clear_highpage(page);
1190                         flush_dcache_page(page);
1191                         SetPageUptodate(page);
1192                 }
1193                 if (sgp == SGP_DIRTY)
1194                         set_page_dirty(page);
1195         }
1196 
1197         /* Perhaps the file has been truncated since we checked */
1198         if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1199             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1200                 error = -EINVAL;
1201                 if (alloced)
1202                         goto trunc;
1203                 else
1204                         goto failed;
1205         }
1206         *pagep = page;
1207         return 0;
1208 
1209         /*
1210          * Error recovery.
1211          */
1212 trunc:
1213         info = SHMEM_I(inode);
1214         ClearPageDirty(page);
1215         delete_from_page_cache(page);
1216         spin_lock(&info->lock);
1217         info->alloced--;
1218         inode->i_blocks -= BLOCKS_PER_PAGE;
1219         spin_unlock(&info->lock);
1220 decused:
1221         sbinfo = SHMEM_SB(inode->i_sb);
1222         if (sbinfo->max_blocks)
1223                 percpu_counter_add(&sbinfo->used_blocks, -1);
1224 unacct:
1225         shmem_unacct_blocks(info->flags, 1);
1226 failed:
1227         if (swap.val && error != -EINVAL &&
1228             !shmem_confirm_swap(mapping, index, swap))
1229                 error = -EEXIST;
1230 unlock:
1231         if (page) {
1232                 unlock_page(page);
1233                 page_cache_release(page);
1234         }
1235         if (error == -ENOSPC && !once++) {
1236                 info = SHMEM_I(inode);
1237                 spin_lock(&info->lock);
1238                 shmem_recalc_inode(inode);
1239                 spin_unlock(&info->lock);
1240                 goto repeat;
1241         }
1242         if (error == -EEXIST)   /* from above or from radix_tree_insert */
1243                 goto repeat;
1244         return error;
1245 }
1246 
1247 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1248 {
1249         struct inode *inode = file_inode(vma->vm_file);
1250         int error;
1251         int ret = VM_FAULT_LOCKED;
1252 
1253         /*
1254          * Trinity finds that probing a hole which tmpfs is punching can
1255          * prevent the hole-punch from ever completing: which in turn
1256          * locks writers out with its hold on i_mutex.  So refrain from
1257          * faulting pages into the hole while it's being punched.  Although
1258          * shmem_undo_range() does remove the additions, it may be unable to
1259          * keep up, as each new page needs its own unmap_mapping_range() call,
1260          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1261          *
1262          * It does not matter if we sometimes reach this check just before the
1263          * hole-punch begins, so that one fault then races with the punch:
1264          * we just need to make racing faults a rare case.
1265          *
1266          * The implementation below would be much simpler if we just used a
1267          * standard mutex or completion: but we cannot take i_mutex in fault,
1268          * and bloating every shmem inode for this unlikely case would be sad.
1269          */
1270         if (unlikely(inode->i_private)) {
1271                 struct shmem_falloc *shmem_falloc;
1272 
1273                 spin_lock(&inode->i_lock);
1274                 shmem_falloc = inode->i_private;
1275                 if (shmem_falloc &&
1276                     shmem_falloc->waitq &&
1277                     vmf->pgoff >= shmem_falloc->start &&
1278                     vmf->pgoff < shmem_falloc->next) {
1279                         wait_queue_head_t *shmem_falloc_waitq;
1280                         DEFINE_WAIT(shmem_fault_wait);
1281 
1282                         ret = VM_FAULT_NOPAGE;
1283                         if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
1284                            !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
1285                                 /* It's polite to up mmap_sem if we can */
1286                                 up_read(&vma->vm_mm->mmap_sem);
1287                                 ret = VM_FAULT_RETRY;
1288                         }
1289 
1290                         shmem_falloc_waitq = shmem_falloc->waitq;
1291                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
1292                                         TASK_UNINTERRUPTIBLE);
1293                         spin_unlock(&inode->i_lock);
1294                         schedule();
1295 
1296                         /*
1297                          * shmem_falloc_waitq points into the shmem_fallocate()
1298                          * stack of the hole-punching task: shmem_falloc_waitq
1299                          * is usually invalid by the time we reach here, but
1300                          * finish_wait() does not dereference it in that case;
1301                          * though i_lock needed lest racing with wake_up_all().
1302                          */
1303                         spin_lock(&inode->i_lock);
1304                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
1305                         spin_unlock(&inode->i_lock);
1306                         return ret;
1307                 }
1308                 spin_unlock(&inode->i_lock);
1309         }
1310 
1311         error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1312         if (error)
1313                 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1314 
1315         if (ret & VM_FAULT_MAJOR) {
1316                 count_vm_event(PGMAJFAULT);
1317                 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1318         }
1319         return ret;
1320 }
1321 
1322 #ifdef CONFIG_NUMA
1323 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
1324 {
1325         struct inode *inode = file_inode(vma->vm_file);
1326         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
1327 }
1328 
1329 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1330                                           unsigned long addr)
1331 {
1332         struct inode *inode = file_inode(vma->vm_file);
1333         pgoff_t index;
1334 
1335         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1336         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
1337 }
1338 #endif
1339 
1340 int shmem_lock(struct file *file, int lock, struct user_struct *user)
1341 {
1342         struct inode *inode = file_inode(file);
1343         struct shmem_inode_info *info = SHMEM_I(inode);
1344         int retval = -ENOMEM;
1345 
1346         spin_lock(&info->lock);
1347         if (lock && !(info->flags & VM_LOCKED)) {
1348                 if (!user_shm_lock(inode->i_size, user))
1349                         goto out_nomem;
1350                 info->flags |= VM_LOCKED;
1351                 mapping_set_unevictable(file->f_mapping);
1352         }
1353         if (!lock && (info->flags & VM_LOCKED) && user) {
1354                 user_shm_unlock(inode->i_size, user);
1355                 info->flags &= ~VM_LOCKED;
1356                 mapping_clear_unevictable(file->f_mapping);
1357         }
1358         retval = 0;
1359 
1360 out_nomem:
1361         spin_unlock(&info->lock);
1362         return retval;
1363 }
1364 
1365 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1366 {
1367         file_accessed(file);
1368         vma->vm_ops = &shmem_vm_ops;
1369         return 0;
1370 }
1371 
1372 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1373                                      umode_t mode, dev_t dev, unsigned long flags)
1374 {
1375         struct inode *inode;
1376         struct shmem_inode_info *info;
1377         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1378 
1379         if (shmem_reserve_inode(sb))
1380                 return NULL;
1381 
1382         inode = new_inode(sb);
1383         if (inode) {
1384                 inode->i_ino = get_next_ino();
1385                 inode_init_owner(inode, dir, mode);
1386                 inode->i_blocks = 0;
1387                 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1388                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1389                 inode->i_generation = get_seconds();
1390                 info = SHMEM_I(inode);
1391                 memset(info, 0, (char *)inode - (char *)info);
1392                 spin_lock_init(&info->lock);
1393                 info->flags = flags & VM_NORESERVE;
1394                 INIT_LIST_HEAD(&info->swaplist);
1395                 simple_xattrs_init(&info->xattrs);
1396                 cache_no_acl(inode);
1397 
1398                 switch (mode & S_IFMT) {
1399                 default:
1400                         inode->i_op = &shmem_special_inode_operations;
1401                         init_special_inode(inode, mode, dev);
1402                         break;
1403                 case S_IFREG:
1404                         inode->i_mapping->a_ops = &shmem_aops;
1405                         inode->i_op = &shmem_inode_operations;
1406                         inode->i_fop = &shmem_file_operations;
1407                         mpol_shared_policy_init(&info->policy,
1408                                                  shmem_get_sbmpol(sbinfo));
1409                         break;
1410                 case S_IFDIR:
1411                         inc_nlink(inode);
1412                         /* Some things misbehave if size == 0 on a directory */
1413                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
1414                         inode->i_op = &shmem_dir_inode_operations;
1415                         inode->i_fop = &simple_dir_operations;
1416                         break;
1417                 case S_IFLNK:
1418                         /*
1419                          * Must not load anything in the rbtree,
1420                          * mpol_free_shared_policy will not be called.
1421                          */
1422                         mpol_shared_policy_init(&info->policy, NULL);
1423                         break;
1424                 }
1425         } else
1426                 shmem_free_inode(sb);
1427         return inode;
1428 }
1429 
1430 bool shmem_mapping(struct address_space *mapping)
1431 {
1432         return mapping->backing_dev_info == &shmem_backing_dev_info;
1433 }
1434 
1435 #ifdef CONFIG_TMPFS
1436 static const struct inode_operations shmem_symlink_inode_operations;
1437 static const struct inode_operations shmem_short_symlink_operations;
1438 
1439 #ifdef CONFIG_TMPFS_XATTR
1440 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
1441 #else
1442 #define shmem_initxattrs NULL
1443 #endif
1444 
1445 static int
1446 shmem_write_begin(struct file *file, struct address_space *mapping,
1447                         loff_t pos, unsigned len, unsigned flags,
1448                         struct page **pagep, void **fsdata)
1449 {
1450         struct inode *inode = mapping->host;
1451         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1452         return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1453 }
1454 
1455 static int
1456 shmem_write_end(struct file *file, struct address_space *mapping,
1457                         loff_t pos, unsigned len, unsigned copied,
1458                         struct page *page, void *fsdata)
1459 {
1460         struct inode *inode = mapping->host;
1461 
1462         if (pos + copied > inode->i_size)
1463                 i_size_write(inode, pos + copied);
1464 
1465         if (!PageUptodate(page)) {
1466                 if (copied < PAGE_CACHE_SIZE) {
1467                         unsigned from = pos & (PAGE_CACHE_SIZE - 1);
1468                         zero_user_segments(page, 0, from,
1469                                         from + copied, PAGE_CACHE_SIZE);
1470                 }
1471                 SetPageUptodate(page);
1472         }
1473         set_page_dirty(page);
1474         unlock_page(page);
1475         page_cache_release(page);
1476 
1477         return copied;
1478 }
1479 
1480 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1481 {
1482         struct file *file = iocb->ki_filp;
1483         struct inode *inode = file_inode(file);
1484         struct address_space *mapping = inode->i_mapping;
1485         pgoff_t index;
1486         unsigned long offset;
1487         enum sgp_type sgp = SGP_READ;
1488         int error = 0;
1489         ssize_t retval = 0;
1490         loff_t *ppos = &iocb->ki_pos;
1491 
1492         /*
1493          * Might this read be for a stacking filesystem?  Then when reading
1494          * holes of a sparse file, we actually need to allocate those pages,
1495          * and even mark them dirty, so it cannot exceed the max_blocks limit.
1496          */
1497         if (segment_eq(get_fs(), KERNEL_DS))
1498                 sgp = SGP_DIRTY;
1499 
1500         index = *ppos >> PAGE_CACHE_SHIFT;
1501         offset = *ppos & ~PAGE_CACHE_MASK;
1502 
1503         for (;;) {
1504                 struct page *page = NULL;
1505                 pgoff_t end_index;
1506                 unsigned long nr, ret;
1507                 loff_t i_size = i_size_read(inode);
1508 
1509                 end_index = i_size >> PAGE_CACHE_SHIFT;
1510                 if (index > end_index)
1511                         break;
1512                 if (index == end_index) {
1513                         nr = i_size & ~PAGE_CACHE_MASK;
1514                         if (nr <= offset)
1515                                 break;
1516                 }
1517 
1518                 error = shmem_getpage(inode, index, &page, sgp, NULL);
1519                 if (error) {
1520                         if (error == -EINVAL)
1521                                 error = 0;
1522                         break;
1523                 }
1524                 if (page)
1525                         unlock_page(page);
1526 
1527                 /*
1528                  * We must evaluate after, since reads (unlike writes)
1529                  * are called without i_mutex protection against truncate
1530                  */
1531                 nr = PAGE_CACHE_SIZE;
1532                 i_size = i_size_read(inode);
1533                 end_index = i_size >> PAGE_CACHE_SHIFT;
1534                 if (index == end_index) {
1535                         nr = i_size & ~PAGE_CACHE_MASK;
1536                         if (nr <= offset) {
1537                                 if (page)
1538                                         page_cache_release(page);
1539                                 break;
1540                         }
1541                 }
1542                 nr -= offset;
1543 
1544                 if (page) {
1545                         /*
1546                          * If users can be writing to this page using arbitrary
1547                          * virtual addresses, take care about potential aliasing
1548                          * before reading the page on the kernel side.
1549                          */
1550                         if (mapping_writably_mapped(mapping))
1551                                 flush_dcache_page(page);
1552                         /*
1553                          * Mark the page accessed if we read the beginning.
1554                          */
1555                         if (!offset)
1556                                 mark_page_accessed(page);
1557                 } else {
1558                         page = ZERO_PAGE(0);
1559                         page_cache_get(page);
1560                 }
1561 
1562                 /*
1563                  * Ok, we have the page, and it's up-to-date, so
1564                  * now we can copy it to user space...
1565                  */
1566                 ret = copy_page_to_iter(page, offset, nr, to);
1567                 retval += ret;
1568                 offset += ret;
1569                 index += offset >> PAGE_CACHE_SHIFT;
1570                 offset &= ~PAGE_CACHE_MASK;
1571 
1572                 page_cache_release(page);
1573                 if (!iov_iter_count(to))
1574                         break;
1575                 if (ret < nr) {
1576                         error = -EFAULT;
1577                         break;
1578                 }
1579                 cond_resched();
1580         }
1581 
1582         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1583         file_accessed(file);
1584         return retval ? retval : error;
1585 }
1586 
1587 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
1588                                 struct pipe_inode_info *pipe, size_t len,
1589                                 unsigned int flags)
1590 {
1591         struct address_space *mapping = in->f_mapping;
1592         struct inode *inode = mapping->host;
1593         unsigned int loff, nr_pages, req_pages;
1594         struct page *pages[PIPE_DEF_BUFFERS];
1595         struct partial_page partial[PIPE_DEF_BUFFERS];
1596         struct page *page;
1597         pgoff_t index, end_index;
1598         loff_t isize, left;
1599         int error, page_nr;
1600         struct splice_pipe_desc spd = {
1601                 .pages = pages,
1602                 .partial = partial,
1603                 .nr_pages_max = PIPE_DEF_BUFFERS,
1604                 .flags = flags,
1605                 .ops = &page_cache_pipe_buf_ops,
1606                 .spd_release = spd_release_page,
1607         };
1608 
1609         isize = i_size_read(inode);
1610         if (unlikely(*ppos >= isize))
1611                 return 0;
1612 
1613         left = isize - *ppos;
1614         if (unlikely(left < len))
1615                 len = left;
1616 
1617         if (splice_grow_spd(pipe, &spd))
1618                 return -ENOMEM;
1619 
1620         index = *ppos >> PAGE_CACHE_SHIFT;
1621         loff = *ppos & ~PAGE_CACHE_MASK;
1622         req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1623         nr_pages = min(req_pages, spd.nr_pages_max);
1624 
1625         spd.nr_pages = find_get_pages_contig(mapping, index,
1626                                                 nr_pages, spd.pages);
1627         index += spd.nr_pages;
1628         error = 0;
1629 
1630         while (spd.nr_pages < nr_pages) {
1631                 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1632                 if (error)
1633                         break;
1634                 unlock_page(page);
1635                 spd.pages[spd.nr_pages++] = page;
1636                 index++;
1637         }
1638 
1639         index = *ppos >> PAGE_CACHE_SHIFT;
1640         nr_pages = spd.nr_pages;
1641         spd.nr_pages = 0;
1642 
1643         for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1644                 unsigned int this_len;
1645 
1646                 if (!len)
1647                         break;
1648 
1649                 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
1650                 page = spd.pages[page_nr];
1651 
1652                 if (!PageUptodate(page) || page->mapping != mapping) {
1653                         error = shmem_getpage(inode, index, &page,
1654                                                         SGP_CACHE, NULL);
1655                         if (error)
1656                                 break;
1657                         unlock_page(page);
1658                         page_cache_release(spd.pages[page_nr]);
1659                         spd.pages[page_nr] = page;
1660                 }
1661 
1662                 isize = i_size_read(inode);
1663                 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1664                 if (unlikely(!isize || index > end_index))
1665                         break;
1666 
1667                 if (end_index == index) {
1668                         unsigned int plen;
1669 
1670                         plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1671                         if (plen <= loff)
1672                                 break;
1673 
1674                         this_len = min(this_len, plen - loff);
1675                         len = this_len;
1676                 }
1677 
1678                 spd.partial[page_nr].offset = loff;
1679                 spd.partial[page_nr].len = this_len;
1680                 len -= this_len;
1681                 loff = 0;
1682                 spd.nr_pages++;
1683                 index++;
1684         }
1685 
1686         while (page_nr < nr_pages)
1687                 page_cache_release(spd.pages[page_nr++]);
1688 
1689         if (spd.nr_pages)
1690                 error = splice_to_pipe(pipe, &spd);
1691 
1692         splice_shrink_spd(&spd);
1693 
1694         if (error > 0) {
1695                 *ppos += error;
1696                 file_accessed(in);
1697         }
1698         return error;
1699 }
1700 
1701 /*
1702  * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
1703  */
1704 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
1705                                     pgoff_t index, pgoff_t end, int whence)
1706 {
1707         struct page *page;
1708         struct pagevec pvec;
1709         pgoff_t indices[PAGEVEC_SIZE];
1710         bool done = false;
1711         int i;
1712 
1713         pagevec_init(&pvec, 0);
1714         pvec.nr = 1;            /* start small: we may be there already */
1715         while (!done) {
1716                 pvec.nr = find_get_entries(mapping, index,
1717                                         pvec.nr, pvec.pages, indices);
1718                 if (!pvec.nr) {
1719                         if (whence == SEEK_DATA)
1720                                 index = end;
1721                         break;
1722                 }
1723                 for (i = 0; i < pvec.nr; i++, index++) {
1724                         if (index < indices[i]) {
1725                                 if (whence == SEEK_HOLE) {
1726                                         done = true;
1727                                         break;
1728                                 }
1729                                 index = indices[i];
1730                         }
1731                         page = pvec.pages[i];
1732                         if (page && !radix_tree_exceptional_entry(page)) {
1733                                 if (!PageUptodate(page))
1734                                         page = NULL;
1735                         }
1736                         if (index >= end ||
1737                             (page && whence == SEEK_DATA) ||
1738                             (!page && whence == SEEK_HOLE)) {
1739                                 done = true;
1740                                 break;
1741                         }
1742                 }
1743                 pagevec_remove_exceptionals(&pvec);
1744                 pagevec_release(&pvec);
1745                 pvec.nr = PAGEVEC_SIZE;
1746                 cond_resched();
1747         }
1748         return index;
1749 }
1750 
1751 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
1752 {
1753         struct address_space *mapping = file->f_mapping;
1754         struct inode *inode = mapping->host;
1755         pgoff_t start, end;
1756         loff_t new_offset;
1757 
1758         if (whence != SEEK_DATA && whence != SEEK_HOLE)
1759                 return generic_file_llseek_size(file, offset, whence,
1760                                         MAX_LFS_FILESIZE, i_size_read(inode));
1761         mutex_lock(&inode->i_mutex);
1762         /* We're holding i_mutex so we can access i_size directly */
1763 
1764         if (offset < 0)
1765                 offset = -EINVAL;
1766         else if (offset >= inode->i_size)
1767                 offset = -ENXIO;
1768         else {
1769                 start = offset >> PAGE_CACHE_SHIFT;
1770                 end = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1771                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
1772                 new_offset <<= PAGE_CACHE_SHIFT;
1773                 if (new_offset > offset) {
1774                         if (new_offset < inode->i_size)
1775                                 offset = new_offset;
1776                         else if (whence == SEEK_DATA)
1777                                 offset = -ENXIO;
1778                         else
1779                                 offset = inode->i_size;
1780                 }
1781         }
1782 
1783         if (offset >= 0)
1784                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
1785         mutex_unlock(&inode->i_mutex);
1786         return offset;
1787 }
1788 
1789 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
1790                                                          loff_t len)
1791 {
1792         struct inode *inode = file_inode(file);
1793         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1794         struct shmem_falloc shmem_falloc;
1795         pgoff_t start, index, end;
1796         int error;
1797 
1798         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
1799                 return -EOPNOTSUPP;
1800 
1801         mutex_lock(&inode->i_mutex);
1802 
1803         if (mode & FALLOC_FL_PUNCH_HOLE) {
1804                 struct address_space *mapping = file->f_mapping;
1805                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
1806                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
1807                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
1808 
1809                 shmem_falloc.waitq = &shmem_falloc_waitq;
1810                 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
1811                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
1812                 spin_lock(&inode->i_lock);
1813                 inode->i_private = &shmem_falloc;
1814                 spin_unlock(&inode->i_lock);
1815 
1816                 if ((u64)unmap_end > (u64)unmap_start)
1817                         unmap_mapping_range(mapping, unmap_start,
1818                                             1 + unmap_end - unmap_start, 0);
1819                 shmem_truncate_range(inode, offset, offset + len - 1);
1820                 /* No need to unmap again: hole-punching leaves COWed pages */
1821 
1822                 spin_lock(&inode->i_lock);
1823                 inode->i_private = NULL;
1824                 wake_up_all(&shmem_falloc_waitq);
1825                 spin_unlock(&inode->i_lock);
1826                 error = 0;
1827                 goto out;
1828         }
1829 
1830         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
1831         error = inode_newsize_ok(inode, offset + len);
1832         if (error)
1833                 goto out;
1834 
1835         start = offset >> PAGE_CACHE_SHIFT;
1836         end = (offset + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1837         /* Try to avoid a swapstorm if len is impossible to satisfy */
1838         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
1839                 error = -ENOSPC;
1840                 goto out;
1841         }
1842 
1843         shmem_falloc.waitq = NULL;
1844         shmem_falloc.start = start;
1845         shmem_falloc.next  = start;
1846         shmem_falloc.nr_falloced = 0;
1847         shmem_falloc.nr_unswapped = 0;
1848         spin_lock(&inode->i_lock);
1849         inode->i_private = &shmem_falloc;
1850         spin_unlock(&inode->i_lock);
1851 
1852         for (index = start; index < end; index++) {
1853                 struct page *page;
1854 
1855                 /*
1856                  * Good, the fallocate(2) manpage permits EINTR: we may have
1857                  * been interrupted because we are using up too much memory.
1858                  */
1859                 if (signal_pending(current))
1860                         error = -EINTR;
1861                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
1862                         error = -ENOMEM;
1863                 else
1864                         error = shmem_getpage(inode, index, &page, SGP_FALLOC,
1865                                                                         NULL);
1866                 if (error) {
1867                         /* Remove the !PageUptodate pages we added */
1868                         shmem_undo_range(inode,
1869                                 (loff_t)start << PAGE_CACHE_SHIFT,
1870                                 (loff_t)index << PAGE_CACHE_SHIFT, true);
1871                         goto undone;
1872                 }
1873 
1874                 /*
1875                  * Inform shmem_writepage() how far we have reached.
1876                  * No need for lock or barrier: we have the page lock.
1877                  */
1878                 shmem_falloc.next++;
1879                 if (!PageUptodate(page))
1880                         shmem_falloc.nr_falloced++;
1881 
1882                 /*
1883                  * If !PageUptodate, leave it that way so that freeable pages
1884                  * can be recognized if we need to rollback on error later.
1885                  * But set_page_dirty so that memory pressure will swap rather
1886                  * than free the pages we are allocating (and SGP_CACHE pages
1887                  * might still be clean: we now need to mark those dirty too).
1888                  */
1889                 set_page_dirty(page);
1890                 unlock_page(page);
1891                 page_cache_release(page);
1892                 cond_resched();
1893         }
1894 
1895         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
1896                 i_size_write(inode, offset + len);
1897         inode->i_ctime = CURRENT_TIME;
1898 undone:
1899         spin_lock(&inode->i_lock);
1900         inode->i_private = NULL;
1901         spin_unlock(&inode->i_lock);
1902 out:
1903         mutex_unlock(&inode->i_mutex);
1904         return error;
1905 }
1906 
1907 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1908 {
1909         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1910 
1911         buf->f_type = TMPFS_MAGIC;
1912         buf->f_bsize = PAGE_CACHE_SIZE;
1913         buf->f_namelen = NAME_MAX;
1914         if (sbinfo->max_blocks) {
1915                 buf->f_blocks = sbinfo->max_blocks;
1916                 buf->f_bavail =
1917                 buf->f_bfree  = sbinfo->max_blocks -
1918                                 percpu_counter_sum(&sbinfo->used_blocks);
1919         }
1920         if (sbinfo->max_inodes) {
1921                 buf->f_files = sbinfo->max_inodes;
1922                 buf->f_ffree = sbinfo->free_inodes;
1923         }
1924         /* else leave those fields 0 like simple_statfs */
1925         return 0;
1926 }
1927 
1928 /*
1929  * File creation. Allocate an inode, and we're done..
1930  */
1931 static int
1932 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
1933 {
1934         struct inode *inode;
1935         int error = -ENOSPC;
1936 
1937         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1938         if (inode) {
1939                 error = simple_acl_create(dir, inode);
1940                 if (error)
1941                         goto out_iput;
1942                 error = security_inode_init_security(inode, dir,
1943                                                      &dentry->d_name,
1944                                                      shmem_initxattrs, NULL);
1945                 if (error && error != -EOPNOTSUPP)
1946                         goto out_iput;
1947 
1948                 error = 0;
1949                 dir->i_size += BOGO_DIRENT_SIZE;
1950                 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1951                 d_instantiate(dentry, inode);
1952                 dget(dentry); /* Extra count - pin the dentry in core */
1953         }
1954         return error;
1955 out_iput:
1956         iput(inode);
1957         return error;
1958 }
1959 
1960 static int
1961 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
1962 {
1963         struct inode *inode;
1964         int error = -ENOSPC;
1965 
1966         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
1967         if (inode) {
1968                 error = security_inode_init_security(inode, dir,
1969                                                      NULL,
1970                                                      shmem_initxattrs, NULL);
1971                 if (error && error != -EOPNOTSUPP)
1972                         goto out_iput;
1973                 error = simple_acl_create(dir, inode);
1974                 if (error)
1975                         goto out_iput;
1976                 d_tmpfile(dentry, inode);
1977         }
1978         return error;
1979 out_iput:
1980         iput(inode);
1981         return error;
1982 }
1983 
1984 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
1985 {
1986         int error;
1987 
1988         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1989                 return error;
1990         inc_nlink(dir);
1991         return 0;
1992 }
1993 
1994 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
1995                 bool excl)
1996 {
1997         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1998 }
1999 
2000 /*
2001  * Link a file..
2002  */
2003 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2004 {
2005         struct inode *inode = old_dentry->d_inode;
2006         int ret;
2007 
2008         /*
2009          * No ordinary (disk based) filesystem counts links as inodes;
2010          * but each new link needs a new dentry, pinning lowmem, and
2011          * tmpfs dentries cannot be pruned until they are unlinked.
2012          */
2013         ret = shmem_reserve_inode(inode->i_sb);
2014         if (ret)
2015                 goto out;
2016 
2017         dir->i_size += BOGO_DIRENT_SIZE;
2018         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2019         inc_nlink(inode);
2020         ihold(inode);   /* New dentry reference */
2021         dget(dentry);           /* Extra pinning count for the created dentry */
2022         d_instantiate(dentry, inode);
2023 out:
2024         return ret;
2025 }
2026 
2027 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2028 {
2029         struct inode *inode = dentry->d_inode;
2030 
2031         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2032                 shmem_free_inode(inode->i_sb);
2033 
2034         dir->i_size -= BOGO_DIRENT_SIZE;
2035         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2036         drop_nlink(inode);
2037         dput(dentry);   /* Undo the count from "create" - this does all the work */
2038         return 0;
2039 }
2040 
2041 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2042 {
2043         if (!simple_empty(dentry))
2044                 return -ENOTEMPTY;
2045 
2046         drop_nlink(dentry->d_inode);
2047         drop_nlink(dir);
2048         return shmem_unlink(dir, dentry);
2049 }
2050 
2051 /*
2052  * The VFS layer already does all the dentry stuff for rename,
2053  * we just have to decrement the usage count for the target if
2054  * it exists so that the VFS layer correctly free's it when it
2055  * gets overwritten.
2056  */
2057 static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2058 {
2059         struct inode *inode = old_dentry->d_inode;
2060         int they_are_dirs = S_ISDIR(inode->i_mode);
2061 
2062         if (!simple_empty(new_dentry))
2063                 return -ENOTEMPTY;
2064 
2065         if (new_dentry->d_inode) {
2066                 (void) shmem_unlink(new_dir, new_dentry);
2067                 if (they_are_dirs)
2068                         drop_nlink(old_dir);
2069         } else if (they_are_dirs) {
2070                 drop_nlink(old_dir);
2071                 inc_nlink(new_dir);
2072         }
2073 
2074         old_dir->i_size -= BOGO_DIRENT_SIZE;
2075         new_dir->i_size += BOGO_DIRENT_SIZE;
2076         old_dir->i_ctime = old_dir->i_mtime =
2077         new_dir->i_ctime = new_dir->i_mtime =
2078         inode->i_ctime = CURRENT_TIME;
2079         return 0;
2080 }
2081 
2082 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
2083 {
2084         int error;
2085         int len;
2086         struct inode *inode;
2087         struct page *page;
2088         char *kaddr;
2089         struct shmem_inode_info *info;
2090 
2091         len = strlen(symname) + 1;
2092         if (len > PAGE_CACHE_SIZE)
2093                 return -ENAMETOOLONG;
2094 
2095         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
2096         if (!inode)
2097                 return -ENOSPC;
2098 
2099         error = security_inode_init_security(inode, dir, &dentry->d_name,
2100                                              shmem_initxattrs, NULL);
2101         if (error) {
2102                 if (error != -EOPNOTSUPP) {
2103                         iput(inode);
2104                         return error;
2105                 }
2106                 error = 0;
2107         }
2108 
2109         info = SHMEM_I(inode);
2110         inode->i_size = len-1;
2111         if (len <= SHORT_SYMLINK_LEN) {
2112                 info->symlink = kmemdup(symname, len, GFP_KERNEL);
2113                 if (!info->symlink) {
2114                         iput(inode);
2115                         return -ENOMEM;
2116                 }
2117                 inode->i_op = &shmem_short_symlink_operations;
2118         } else {
2119                 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
2120                 if (error) {
2121                         iput(inode);
2122                         return error;
2123                 }
2124                 inode->i_mapping->a_ops = &shmem_aops;
2125                 inode->i_op = &shmem_symlink_inode_operations;
2126                 kaddr = kmap_atomic(page);
2127                 memcpy(kaddr, symname, len);
2128                 kunmap_atomic(kaddr);
2129                 SetPageUptodate(page);
2130                 set_page_dirty(page);
2131                 unlock_page(page);
2132                 page_cache_release(page);
2133         }
2134         dir->i_size += BOGO_DIRENT_SIZE;
2135         dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2136         d_instantiate(dentry, inode);
2137         dget(dentry);
2138         return 0;
2139 }
2140 
2141 static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd)
2142 {
2143         nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink);
2144         return NULL;
2145 }
2146 
2147 static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
2148 {
2149         struct page *page = NULL;
2150         int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
2151         nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
2152         if (page)
2153                 unlock_page(page);
2154         return page;
2155 }
2156 
2157 static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
2158 {
2159         if (!IS_ERR(nd_get_link(nd))) {
2160                 struct page *page = cookie;
2161                 kunmap(page);
2162                 mark_page_accessed(page);
2163                 page_cache_release(page);
2164         }
2165 }
2166 
2167 #ifdef CONFIG_TMPFS_XATTR
2168 /*
2169  * Superblocks without xattr inode operations may get some security.* xattr
2170  * support from the LSM "for free". As soon as we have any other xattrs
2171  * like ACLs, we also need to implement the security.* handlers at
2172  * filesystem level, though.
2173  */
2174 
2175 /*
2176  * Callback for security_inode_init_security() for acquiring xattrs.
2177  */
2178 static int shmem_initxattrs(struct inode *inode,
2179                             const struct xattr *xattr_array,
2180                             void *fs_info)
2181 {
2182         struct shmem_inode_info *info = SHMEM_I(inode);
2183         const struct xattr *xattr;
2184         struct simple_xattr *new_xattr;
2185         size_t len;
2186 
2187         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
2188                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
2189                 if (!new_xattr)
2190                         return -ENOMEM;
2191 
2192                 len = strlen(xattr->name) + 1;
2193                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
2194                                           GFP_KERNEL);
2195                 if (!new_xattr->name) {
2196                         kfree(new_xattr);
2197                         return -ENOMEM;
2198                 }
2199 
2200                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
2201                        XATTR_SECURITY_PREFIX_LEN);
2202                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
2203                        xattr->name, len);
2204 
2205                 simple_xattr_list_add(&info->xattrs, new_xattr);
2206         }
2207 
2208         return 0;
2209 }
2210 
2211 static const struct xattr_handler *shmem_xattr_handlers[] = {
2212 #ifdef CONFIG_TMPFS_POSIX_ACL
2213         &posix_acl_access_xattr_handler,
2214         &posix_acl_default_xattr_handler,
2215 #endif
2216         NULL
2217 };
2218 
2219 static int shmem_xattr_validate(const char *name)
2220 {
2221         struct { const char *prefix; size_t len; } arr[] = {
2222                 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
2223                 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
2224         };
2225         int i;
2226 
2227         for (i = 0; i < ARRAY_SIZE(arr); i++) {
2228                 size_t preflen = arr[i].len;
2229                 if (strncmp(name, arr[i].prefix, preflen) == 0) {
2230                         if (!name[preflen])
2231                                 return -EINVAL;
2232                         return 0;
2233                 }
2234         }
2235         return -EOPNOTSUPP;
2236 }
2237 
2238 static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
2239                               void *buffer, size_t size)
2240 {
2241         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2242         int err;
2243 
2244         /*
2245          * If this is a request for a synthetic attribute in the system.*
2246          * namespace use the generic infrastructure to resolve a handler
2247          * for it via sb->s_xattr.
2248          */
2249         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2250                 return generic_getxattr(dentry, name, buffer, size);
2251 
2252         err = shmem_xattr_validate(name);
2253         if (err)
2254                 return err;
2255 
2256         return simple_xattr_get(&info->xattrs, name, buffer, size);
2257 }
2258 
2259 static int shmem_setxattr(struct dentry *dentry, const char *name,
2260                           const void *value, size_t size, int flags)
2261 {
2262         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2263         int err;
2264 
2265         /*
2266          * If this is a request for a synthetic attribute in the system.*
2267          * namespace use the generic infrastructure to resolve a handler
2268          * for it via sb->s_xattr.
2269          */
2270         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2271                 return generic_setxattr(dentry, name, value, size, flags);
2272 
2273         err = shmem_xattr_validate(name);
2274         if (err)
2275                 return err;
2276 
2277         return simple_xattr_set(&info->xattrs, name, value, size, flags);
2278 }
2279 
2280 static int shmem_removexattr(struct dentry *dentry, const char *name)
2281 {
2282         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2283         int err;
2284 
2285         /*
2286          * If this is a request for a synthetic attribute in the system.*
2287          * namespace use the generic infrastructure to resolve a handler
2288          * for it via sb->s_xattr.
2289          */
2290         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2291                 return generic_removexattr(dentry, name);
2292 
2293         err = shmem_xattr_validate(name);
2294         if (err)
2295                 return err;
2296 
2297         return simple_xattr_remove(&info->xattrs, name);
2298 }
2299 
2300 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
2301 {
2302         struct shmem_inode_info *info = SHMEM_I(dentry->d_inode);
2303         return simple_xattr_list(&info->xattrs, buffer, size);
2304 }
2305 #endif /* CONFIG_TMPFS_XATTR */
2306 
2307 static const struct inode_operations shmem_short_symlink_operations = {
2308         .readlink       = generic_readlink,
2309         .follow_link    = shmem_follow_short_symlink,
2310 #ifdef CONFIG_TMPFS_XATTR
2311         .setxattr       = shmem_setxattr,
2312         .getxattr       = shmem_getxattr,
2313         .listxattr      = shmem_listxattr,
2314         .removexattr    = shmem_removexattr,
2315 #endif
2316 };
2317 
2318 static const struct inode_operations shmem_symlink_inode_operations = {
2319         .readlink       = generic_readlink,
2320         .follow_link    = shmem_follow_link,
2321         .put_link       = shmem_put_link,
2322 #ifdef CONFIG_TMPFS_XATTR
2323         .setxattr       = shmem_setxattr,
2324         .getxattr       = shmem_getxattr,
2325         .listxattr      = shmem_listxattr,
2326         .removexattr    = shmem_removexattr,
2327 #endif
2328 };
2329 
2330 static struct dentry *shmem_get_parent(struct dentry *child)
2331 {
2332         return ERR_PTR(-ESTALE);
2333 }
2334 
2335 static int shmem_match(struct inode *ino, void *vfh)
2336 {
2337         __u32 *fh = vfh;
2338         __u64 inum = fh[2];
2339         inum = (inum << 32) | fh[1];
2340         return ino->i_ino == inum && fh[0] == ino->i_generation;
2341 }
2342 
2343 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2344                 struct fid *fid, int fh_len, int fh_type)
2345 {
2346         struct inode *inode;
2347         struct dentry *dentry = NULL;
2348         u64 inum;
2349 
2350         if (fh_len < 3)
2351                 return NULL;
2352 
2353         inum = fid->raw[2];
2354         inum = (inum << 32) | fid->raw[1];
2355 
2356         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2357                         shmem_match, fid->raw);
2358         if (inode) {
2359                 dentry = d_find_alias(inode);
2360                 iput(inode);
2361         }
2362 
2363         return dentry;
2364 }
2365 
2366 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
2367                                 struct inode *parent)
2368 {
2369         if (*len < 3) {
2370                 *len = 3;
2371                 return FILEID_INVALID;
2372         }
2373 
2374         if (inode_unhashed(inode)) {
2375                 /* Unfortunately insert_inode_hash is not idempotent,
2376                  * so as we hash inodes here rather than at creation
2377                  * time, we need a lock to ensure we only try
2378                  * to do it once
2379                  */
2380                 static DEFINE_SPINLOCK(lock);
2381                 spin_lock(&lock);
2382                 if (inode_unhashed(inode))
2383                         __insert_inode_hash(inode,
2384                                             inode->i_ino + inode->i_generation);
2385                 spin_unlock(&lock);
2386         }
2387 
2388         fh[0] = inode->i_generation;
2389         fh[1] = inode->i_ino;
2390         fh[2] = ((__u64)inode->i_ino) >> 32;
2391 
2392         *len = 3;
2393         return 1;
2394 }
2395 
2396 static const struct export_operations shmem_export_ops = {
2397         .get_parent     = shmem_get_parent,
2398         .encode_fh      = shmem_encode_fh,
2399         .fh_to_dentry   = shmem_fh_to_dentry,
2400 };
2401 
2402 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2403                                bool remount)
2404 {
2405         char *this_char, *value, *rest;
2406         struct mempolicy *mpol = NULL;
2407         uid_t uid;
2408         gid_t gid;
2409 
2410         while (options != NULL) {
2411                 this_char = options;
2412                 for (;;) {
2413                         /*
2414                          * NUL-terminate this option: unfortunately,
2415                          * mount options form a comma-separated list,
2416                          * but mpol's nodelist may also contain commas.
2417                          */
2418                         options = strchr(options, ',');
2419                         if (options == NULL)
2420                                 break;
2421                         options++;
2422                         if (!isdigit(*options)) {
2423                                 options[-1] = '\0';
2424                                 break;
2425                         }
2426                 }
2427                 if (!*this_char)
2428                         continue;
2429                 if ((value = strchr(this_char,'=')) != NULL) {
2430                         *value++ = 0;
2431                 } else {
2432                         printk(KERN_ERR
2433                             "tmpfs: No value for mount option '%s'\n",
2434                             this_char);
2435                         goto error;
2436                 }
2437 
2438                 if (!strcmp(this_char,"size")) {
2439                         unsigned long long size;
2440                         size = memparse(value,&rest);
2441                         if (*rest == '%') {
2442                                 size <<= PAGE_SHIFT;
2443                                 size *= totalram_pages;
2444                                 do_div(size, 100);
2445                                 rest++;
2446                         }
2447                         if (*rest)
2448                                 goto bad_val;
2449                         sbinfo->max_blocks =
2450                                 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2451                 } else if (!strcmp(this_char,"nr_blocks")) {
2452                         sbinfo->max_blocks = memparse(value, &rest);
2453                         if (*rest)
2454                                 goto bad_val;
2455                 } else if (!strcmp(this_char,"nr_inodes")) {
2456                         sbinfo->max_inodes = memparse(value, &rest);
2457                         if (*rest)
2458                                 goto bad_val;
2459                 } else if (!strcmp(this_char,"mode")) {
2460                         if (remount)
2461                                 continue;
2462                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2463                         if (*rest)
2464                                 goto bad_val;
2465                 } else if (!strcmp(this_char,"uid")) {
2466                         if (remount)
2467                                 continue;
2468                         uid = simple_strtoul(value, &rest, 0);
2469                         if (*rest)
2470                                 goto bad_val;
2471                         sbinfo->uid = make_kuid(current_user_ns(), uid);
2472                         if (!uid_valid(sbinfo->uid))
2473                                 goto bad_val;
2474                 } else if (!strcmp(this_char,"gid")) {
2475                         if (remount)
2476                                 continue;
2477                         gid = simple_strtoul(value, &rest, 0);
2478                         if (*rest)
2479                                 goto bad_val;
2480                         sbinfo->gid = make_kgid(current_user_ns(), gid);
2481                         if (!gid_valid(sbinfo->gid))
2482                                 goto bad_val;
2483                 } else if (!strcmp(this_char,"mpol")) {
2484                         mpol_put(mpol);
2485                         mpol = NULL;
2486                         if (mpol_parse_str(value, &mpol))
2487                                 goto bad_val;
2488                 } else {
2489                         printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2490                                this_char);
2491                         goto error;
2492                 }
2493         }
2494         sbinfo->mpol = mpol;
2495         return 0;
2496 
2497 bad_val:
2498         printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2499                value, this_char);
2500 error:
2501         mpol_put(mpol);
2502         return 1;
2503 
2504 }
2505 
2506 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2507 {
2508         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2509         struct shmem_sb_info config = *sbinfo;
2510         unsigned long inodes;
2511         int error = -EINVAL;
2512 
2513         config.mpol = NULL;
2514         if (shmem_parse_options(data, &config, true))
2515                 return error;
2516 
2517         spin_lock(&sbinfo->stat_lock);
2518         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2519         if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2520                 goto out;
2521         if (config.max_inodes < inodes)
2522                 goto out;
2523         /*
2524          * Those tests disallow limited->unlimited while any are in use;
2525          * but we must separately disallow unlimited->limited, because
2526          * in that case we have no record of how much is already in use.
2527          */
2528         if (config.max_blocks && !sbinfo->max_blocks)
2529                 goto out;
2530         if (config.max_inodes && !sbinfo->max_inodes)
2531                 goto out;
2532 
2533         error = 0;
2534         sbinfo->max_blocks  = config.max_blocks;
2535         sbinfo->max_inodes  = config.max_inodes;
2536         sbinfo->free_inodes = config.max_inodes - inodes;
2537 
2538         /*
2539          * Preserve previous mempolicy unless mpol remount option was specified.
2540          */
2541         if (config.mpol) {
2542                 mpol_put(sbinfo->mpol);
2543                 sbinfo->mpol = config.mpol;     /* transfers initial ref */
2544         }
2545 out:
2546         spin_unlock(&sbinfo->stat_lock);
2547         return error;
2548 }
2549 
2550 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
2551 {
2552         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
2553 
2554         if (sbinfo->max_blocks != shmem_default_max_blocks())
2555                 seq_printf(seq, ",size=%luk",
2556                         sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2557         if (sbinfo->max_inodes != shmem_default_max_inodes())
2558                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2559         if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2560                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
2561         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
2562                 seq_printf(seq, ",uid=%u",
2563                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
2564         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
2565                 seq_printf(seq, ",gid=%u",
2566                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
2567         shmem_show_mpol(seq, sbinfo->mpol);
2568         return 0;
2569 }
2570 #endif /* CONFIG_TMPFS */
2571 
2572 static void shmem_put_super(struct super_block *sb)
2573 {
2574         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2575 
2576         percpu_counter_destroy(&sbinfo->used_blocks);
2577         mpol_put(sbinfo->mpol);
2578         kfree(sbinfo);
2579         sb->s_fs_info = NULL;
2580 }
2581 
2582 int shmem_fill_super(struct super_block *sb, void *data, int silent)
2583 {
2584         struct inode *inode;
2585         struct shmem_sb_info *sbinfo;
2586         int err = -ENOMEM;
2587 
2588         /* Round up to L1_CACHE_BYTES to resist false sharing */
2589         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2590                                 L1_CACHE_BYTES), GFP_KERNEL);
2591         if (!sbinfo)
2592                 return -ENOMEM;
2593 
2594         sbinfo->mode = S_IRWXUGO | S_ISVTX;
2595         sbinfo->uid = current_fsuid();
2596         sbinfo->gid = current_fsgid();
2597         sb->s_fs_info = sbinfo;
2598 
2599 #ifdef CONFIG_TMPFS
2600         /*
2601          * Per default we only allow half of the physical ram per
2602          * tmpfs instance, limiting inodes to one per page of lowmem;
2603          * but the internal instance is left unlimited.
2604          */
2605         if (!(sb->s_flags & MS_KERNMOUNT)) {
2606                 sbinfo->max_blocks = shmem_default_max_blocks();
2607                 sbinfo->max_inodes = shmem_default_max_inodes();
2608                 if (shmem_parse_options(data, sbinfo, false)) {
2609                         err = -EINVAL;
2610                         goto failed;
2611                 }
2612         } else {
2613                 sb->s_flags |= MS_NOUSER;
2614         }
2615         sb->s_export_op = &shmem_export_ops;
2616         sb->s_flags |= MS_NOSEC;
2617 #else
2618         sb->s_flags |= MS_NOUSER;
2619 #endif
2620 
2621         spin_lock_init(&sbinfo->stat_lock);
2622         if (percpu_counter_init(&sbinfo->used_blocks, 0))
2623                 goto failed;
2624         sbinfo->free_inodes = sbinfo->max_inodes;
2625 
2626         sb->s_maxbytes = MAX_LFS_FILESIZE;
2627         sb->s_blocksize = PAGE_CACHE_SIZE;
2628         sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2629         sb->s_magic = TMPFS_MAGIC;
2630         sb->s_op = &shmem_ops;
2631         sb->s_time_gran = 1;
2632 #ifdef CONFIG_TMPFS_XATTR
2633         sb->s_xattr = shmem_xattr_handlers;
2634 #endif
2635 #ifdef CONFIG_TMPFS_POSIX_ACL
2636         sb->s_flags |= MS_POSIXACL;
2637 #endif
2638 
2639         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2640         if (!inode)
2641                 goto failed;
2642         inode->i_uid = sbinfo->uid;
2643         inode->i_gid = sbinfo->gid;
2644         sb->s_root = d_make_root(inode);
2645         if (!sb->s_root)
2646                 goto failed;
2647         return 0;
2648 
2649 failed:
2650         shmem_put_super(sb);
2651         return err;
2652 }
2653 
2654 static struct kmem_cache *shmem_inode_cachep;
2655 
2656 static struct inode *shmem_alloc_inode(struct super_block *sb)
2657 {
2658         struct shmem_inode_info *info;
2659         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2660         if (!info)
2661                 return NULL;
2662         return &info->vfs_inode;
2663 }
2664 
2665 static void shmem_destroy_callback(struct rcu_head *head)
2666 {
2667         struct inode *inode = container_of(head, struct inode, i_rcu);
2668         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2669 }
2670 
2671 static void shmem_destroy_inode(struct inode *inode)
2672 {
2673         if (S_ISREG(inode->i_mode))
2674                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2675         call_rcu(&inode->i_rcu, shmem_destroy_callback);
2676 }
2677 
2678 static void shmem_init_inode(void *foo)
2679 {
2680         struct shmem_inode_info *info = foo;
2681         inode_init_once(&info->vfs_inode);
2682 }
2683 
2684 static int shmem_init_inodecache(void)
2685 {
2686         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2687                                 sizeof(struct shmem_inode_info),
2688                                 0, SLAB_PANIC, shmem_init_inode);
2689         return 0;
2690 }
2691 
2692 static void shmem_destroy_inodecache(void)
2693 {
2694         kmem_cache_destroy(shmem_inode_cachep);
2695 }
2696 
2697 static const struct address_space_operations shmem_aops = {
2698         .writepage      = shmem_writepage,
2699         .set_page_dirty = __set_page_dirty_no_writeback,
2700 #ifdef CONFIG_TMPFS
2701         .write_begin    = shmem_write_begin,
2702         .write_end      = shmem_write_end,
2703 #endif
2704         .migratepage    = migrate_page,
2705         .error_remove_page = generic_error_remove_page,
2706 };
2707 
2708 static const struct file_operations shmem_file_operations = {
2709         .mmap           = shmem_mmap,
2710 #ifdef CONFIG_TMPFS
2711         .llseek         = shmem_file_llseek,
2712         .read           = new_sync_read,
2713         .write          = new_sync_write,
2714         .read_iter      = shmem_file_read_iter,
2715         .write_iter     = generic_file_write_iter,
2716         .fsync          = noop_fsync,
2717         .splice_read    = shmem_file_splice_read,
2718         .splice_write   = iter_file_splice_write,
2719         .fallocate      = shmem_fallocate,
2720 #endif
2721 };
2722 
2723 static const struct inode_operations shmem_inode_operations = {
2724         .setattr        = shmem_setattr,
2725 #ifdef CONFIG_TMPFS_XATTR
2726         .setxattr       = shmem_setxattr,
2727         .getxattr       = shmem_getxattr,
2728         .listxattr      = shmem_listxattr,
2729         .removexattr    = shmem_removexattr,
2730         .set_acl        = simple_set_acl,
2731 #endif
2732 };
2733 
2734 static const struct inode_operations shmem_dir_inode_operations = {
2735 #ifdef CONFIG_TMPFS
2736         .create         = shmem_create,
2737         .lookup         = simple_lookup,
2738         .link           = shmem_link,
2739         .unlink         = shmem_unlink,
2740         .symlink        = shmem_symlink,
2741         .mkdir          = shmem_mkdir,
2742         .rmdir          = shmem_rmdir,
2743         .mknod          = shmem_mknod,
2744         .rename         = shmem_rename,
2745         .tmpfile        = shmem_tmpfile,
2746 #endif
2747 #ifdef CONFIG_TMPFS_XATTR
2748         .setxattr       = shmem_setxattr,
2749         .getxattr       = shmem_getxattr,
2750         .listxattr      = shmem_listxattr,
2751         .removexattr    = shmem_removexattr,
2752 #endif
2753 #ifdef CONFIG_TMPFS_POSIX_ACL
2754         .setattr        = shmem_setattr,
2755         .set_acl        = simple_set_acl,
2756 #endif
2757 };
2758 
2759 static const struct inode_operations shmem_special_inode_operations = {
2760 #ifdef CONFIG_TMPFS_XATTR
2761         .setxattr       = shmem_setxattr,
2762         .getxattr       = shmem_getxattr,
2763         .listxattr      = shmem_listxattr,
2764         .removexattr    = shmem_removexattr,
2765 #endif
2766 #ifdef CONFIG_TMPFS_POSIX_ACL
2767         .setattr        = shmem_setattr,
2768         .set_acl        = simple_set_acl,
2769 #endif
2770 };
2771 
2772 static const struct super_operations shmem_ops = {
2773         .alloc_inode    = shmem_alloc_inode,
2774         .destroy_inode  = shmem_destroy_inode,
2775 #ifdef CONFIG_TMPFS
2776         .statfs         = shmem_statfs,
2777         .remount_fs     = shmem_remount_fs,
2778         .show_options   = shmem_show_options,
2779 #endif
2780         .evict_inode    = shmem_evict_inode,
2781         .drop_inode     = generic_delete_inode,
2782         .put_super      = shmem_put_super,
2783 };
2784 
2785 static const struct vm_operations_struct shmem_vm_ops = {
2786         .fault          = shmem_fault,
2787         .map_pages      = filemap_map_pages,
2788 #ifdef CONFIG_NUMA
2789         .set_policy     = shmem_set_policy,
2790         .get_policy     = shmem_get_policy,
2791 #endif
2792         .remap_pages    = generic_file_remap_pages,
2793 };
2794 
2795 static struct dentry *shmem_mount(struct file_system_type *fs_type,
2796         int flags, const char *dev_name, void *data)
2797 {
2798         return mount_nodev(fs_type, flags, data, shmem_fill_super);
2799 }
2800 
2801 static struct file_system_type shmem_fs_type = {
2802         .owner          = THIS_MODULE,
2803         .name           = "tmpfs",
2804         .mount          = shmem_mount,
2805         .kill_sb        = kill_litter_super,
2806         .fs_flags       = FS_USERNS_MOUNT,
2807 };
2808 
2809 int __init shmem_init(void)
2810 {
2811         int error;
2812 
2813         /* If rootfs called this, don't re-init */
2814         if (shmem_inode_cachep)
2815                 return 0;
2816 
2817         error = bdi_init(&shmem_backing_dev_info);
2818         if (error)
2819                 goto out4;
2820 
2821         error = shmem_init_inodecache();
2822         if (error)
2823                 goto out3;
2824 
2825         error = register_filesystem(&shmem_fs_type);
2826         if (error) {
2827                 printk(KERN_ERR "Could not register tmpfs\n");
2828                 goto out2;
2829         }
2830 
2831         shm_mnt = kern_mount(&shmem_fs_type);
2832         if (IS_ERR(shm_mnt)) {
2833                 error = PTR_ERR(shm_mnt);
2834                 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2835                 goto out1;
2836         }
2837         return 0;
2838 
2839 out1:
2840         unregister_filesystem(&shmem_fs_type);
2841 out2:
2842         shmem_destroy_inodecache();
2843 out3:
2844         bdi_destroy(&shmem_backing_dev_info);
2845 out4:
2846         shm_mnt = ERR_PTR(error);
2847         return error;
2848 }
2849 
2850 #else /* !CONFIG_SHMEM */
2851 
2852 /*
2853  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2854  *
2855  * This is intended for small system where the benefits of the full
2856  * shmem code (swap-backed and resource-limited) are outweighed by
2857  * their complexity. On systems without swap this code should be
2858  * effectively equivalent, but much lighter weight.
2859  */
2860 
2861 static struct file_system_type shmem_fs_type = {
2862         .name           = "tmpfs",
2863         .mount          = ramfs_mount,
2864         .kill_sb        = kill_litter_super,
2865         .fs_flags       = FS_USERNS_MOUNT,
2866 };
2867 
2868 int __init shmem_init(void)
2869 {
2870         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
2871 
2872         shm_mnt = kern_mount(&shmem_fs_type);
2873         BUG_ON(IS_ERR(shm_mnt));
2874 
2875         return 0;
2876 }
2877 
2878 int shmem_unuse(swp_entry_t swap, struct page *page)
2879 {
2880         return 0;
2881 }
2882 
2883 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2884 {
2885         return 0;
2886 }
2887 
2888 void shmem_unlock_mapping(struct address_space *mapping)
2889 {
2890 }
2891 
2892 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
2893 {
2894         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
2895 }
2896 EXPORT_SYMBOL_GPL(shmem_truncate_range);
2897 
2898 #define shmem_vm_ops                            generic_file_vm_ops
2899 #define shmem_file_operations                   ramfs_file_operations
2900 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
2901 #define shmem_acct_size(flags, size)            0
2902 #define shmem_unacct_size(flags, size)          do {} while (0)
2903 
2904 #endif /* CONFIG_SHMEM */
2905 
2906 /* common code */
2907 
2908 static struct dentry_operations anon_ops = {
2909         .d_dname = simple_dname
2910 };
2911 
2912 static struct file *__shmem_file_setup(const char *name, loff_t size,
2913                                        unsigned long flags, unsigned int i_flags)
2914 {
2915         struct file *res;
2916         struct inode *inode;
2917         struct path path;
2918         struct super_block *sb;
2919         struct qstr this;
2920 
2921         if (IS_ERR(shm_mnt))
2922                 return ERR_CAST(shm_mnt);
2923 
2924         if (size < 0 || size > MAX_LFS_FILESIZE)
2925                 return ERR_PTR(-EINVAL);
2926 
2927         if (shmem_acct_size(flags, size))
2928                 return ERR_PTR(-ENOMEM);
2929 
2930         res = ERR_PTR(-ENOMEM);
2931         this.name = name;
2932         this.len = strlen(name);
2933         this.hash = 0; /* will go */
2934         sb = shm_mnt->mnt_sb;
2935         path.dentry = d_alloc_pseudo(sb, &this);
2936         if (!path.dentry)
2937                 goto put_memory;
2938         d_set_d_op(path.dentry, &anon_ops);
2939         path.mnt = mntget(shm_mnt);
2940 
2941         res = ERR_PTR(-ENOSPC);
2942         inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2943         if (!inode)
2944                 goto put_dentry;
2945 
2946         inode->i_flags |= i_flags;
2947         d_instantiate(path.dentry, inode);
2948         inode->i_size = size;
2949         clear_nlink(inode);     /* It is unlinked */
2950         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
2951         if (IS_ERR(res))
2952                 goto put_dentry;
2953 
2954         res = alloc_file(&path, FMODE_WRITE | FMODE_READ,
2955                   &shmem_file_operations);
2956         if (IS_ERR(res))
2957                 goto put_dentry;
2958 
2959         return res;
2960 
2961 put_dentry:
2962         path_put(&path);
2963 put_memory:
2964         shmem_unacct_size(flags, size);
2965         return res;
2966 }
2967 
2968 /**
2969  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
2970  *      kernel internal.  There will be NO LSM permission checks against the
2971  *      underlying inode.  So users of this interface must do LSM checks at a
2972  *      higher layer.  The one user is the big_key implementation.  LSM checks
2973  *      are provided at the key level rather than the inode level.
2974  * @name: name for dentry (to be seen in /proc/<pid>/maps
2975  * @size: size to be set for the file
2976  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2977  */
2978 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
2979 {
2980         return __shmem_file_setup(name, size, flags, S_PRIVATE);
2981 }
2982 
2983 /**
2984  * shmem_file_setup - get an unlinked file living in tmpfs
2985  * @name: name for dentry (to be seen in /proc/<pid>/maps
2986  * @size: size to be set for the file
2987  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2988  */
2989 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2990 {
2991         return __shmem_file_setup(name, size, flags, 0);
2992 }
2993 EXPORT_SYMBOL_GPL(shmem_file_setup);
2994 
2995 /**
2996  * shmem_zero_setup - setup a shared anonymous mapping
2997  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2998  */
2999 int shmem_zero_setup(struct vm_area_struct *vma)
3000 {
3001         struct file *file;
3002         loff_t size = vma->vm_end - vma->vm_start;
3003 
3004         file = shmem_file_setup("dev/zero", size, vma->vm_flags);
3005         if (IS_ERR(file))
3006                 return PTR_ERR(file);
3007 
3008         if (vma->vm_file)
3009                 fput(vma->vm_file);
3010         vma->vm_file = file;
3011         vma->vm_ops = &shmem_vm_ops;
3012         return 0;
3013 }
3014 
3015 /**
3016  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
3017  * @mapping:    the page's address_space
3018  * @index:      the page index
3019  * @gfp:        the page allocator flags to use if allocating
3020  *
3021  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
3022  * with any new page allocations done using the specified allocation flags.
3023  * But read_cache_page_gfp() uses the ->readpage() method: which does not
3024  * suit tmpfs, since it may have pages in swapcache, and needs to find those
3025  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
3026  *
3027  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
3028  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
3029  */
3030 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
3031                                          pgoff_t index, gfp_t gfp)
3032 {
3033 #ifdef CONFIG_SHMEM
3034         struct inode *inode = mapping->host;
3035         struct page *page;
3036         int error;
3037 
3038         BUG_ON(mapping->a_ops != &shmem_aops);
3039         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
3040         if (error)
3041                 page = ERR_PTR(error);
3042         else
3043                 unlock_page(page);
3044         return page;
3045 #else
3046         /*
3047          * The tiny !SHMEM case uses ramfs without swap
3048          */
3049         return read_cache_page_gfp(mapping, index, gfp);
3050 #endif
3051 }
3052 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
3053 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us