Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/mm/mmap.c

  1 /*
  2  * mm/mmap.c
  3  *
  4  * Written by obz.
  5  *
  6  * Address space accounting code        <alan@lxorguk.ukuu.org.uk>
  7  */
  8 
  9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 10 
 11 #include <linux/kernel.h>
 12 #include <linux/slab.h>
 13 #include <linux/backing-dev.h>
 14 #include <linux/mm.h>
 15 #include <linux/vmacache.h>
 16 #include <linux/shm.h>
 17 #include <linux/mman.h>
 18 #include <linux/pagemap.h>
 19 #include <linux/swap.h>
 20 #include <linux/syscalls.h>
 21 #include <linux/capability.h>
 22 #include <linux/init.h>
 23 #include <linux/file.h>
 24 #include <linux/fs.h>
 25 #include <linux/personality.h>
 26 #include <linux/security.h>
 27 #include <linux/hugetlb.h>
 28 #include <linux/shmem_fs.h>
 29 #include <linux/profile.h>
 30 #include <linux/export.h>
 31 #include <linux/mount.h>
 32 #include <linux/mempolicy.h>
 33 #include <linux/rmap.h>
 34 #include <linux/mmu_notifier.h>
 35 #include <linux/mmdebug.h>
 36 #include <linux/perf_event.h>
 37 #include <linux/audit.h>
 38 #include <linux/khugepaged.h>
 39 #include <linux/uprobes.h>
 40 #include <linux/rbtree_augmented.h>
 41 #include <linux/notifier.h>
 42 #include <linux/memory.h>
 43 #include <linux/printk.h>
 44 #include <linux/userfaultfd_k.h>
 45 #include <linux/moduleparam.h>
 46 #include <linux/pkeys.h>
 47 
 48 #include <linux/uaccess.h>
 49 #include <asm/cacheflush.h>
 50 #include <asm/tlb.h>
 51 #include <asm/mmu_context.h>
 52 
 53 #include "internal.h"
 54 
 55 #ifndef arch_mmap_check
 56 #define arch_mmap_check(addr, len, flags)       (0)
 57 #endif
 58 
 59 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
 60 const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
 61 const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX;
 62 int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
 63 #endif
 64 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
 65 const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
 66 const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
 67 int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
 68 #endif
 69 
 70 static bool ignore_rlimit_data;
 71 core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
 72 
 73 static void unmap_region(struct mm_struct *mm,
 74                 struct vm_area_struct *vma, struct vm_area_struct *prev,
 75                 unsigned long start, unsigned long end);
 76 
 77 /* description of effects of mapping type and prot in current implementation.
 78  * this is due to the limited x86 page protection hardware.  The expected
 79  * behavior is in parens:
 80  *
 81  * map_type     prot
 82  *              PROT_NONE       PROT_READ       PROT_WRITE      PROT_EXEC
 83  * MAP_SHARED   r: (no) no      r: (yes) yes    r: (no) yes     r: (no) yes
 84  *              w: (no) no      w: (no) no      w: (yes) yes    w: (no) no
 85  *              x: (no) no      x: (no) yes     x: (no) yes     x: (yes) yes
 86  *
 87  * MAP_PRIVATE  r: (no) no      r: (yes) yes    r: (no) yes     r: (no) yes
 88  *              w: (no) no      w: (no) no      w: (copy) copy  w: (no) no
 89  *              x: (no) no      x: (no) yes     x: (no) yes     x: (yes) yes
 90  *
 91  * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and
 92  * MAP_PRIVATE:
 93  *                                                              r: (no) no
 94  *                                                              w: (no) no
 95  *                                                              x: (yes) yes
 96  */
 97 pgprot_t protection_map[16] = {
 98         __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
 99         __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
100 };
101 
102 pgprot_t vm_get_page_prot(unsigned long vm_flags)
103 {
104         return __pgprot(pgprot_val(protection_map[vm_flags &
105                                 (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
106                         pgprot_val(arch_vm_get_page_prot(vm_flags)));
107 }
108 EXPORT_SYMBOL(vm_get_page_prot);
109 
110 static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
111 {
112         return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
113 }
114 
115 /* Update vma->vm_page_prot to reflect vma->vm_flags. */
116 void vma_set_page_prot(struct vm_area_struct *vma)
117 {
118         unsigned long vm_flags = vma->vm_flags;
119         pgprot_t vm_page_prot;
120 
121         vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
122         if (vma_wants_writenotify(vma, vm_page_prot)) {
123                 vm_flags &= ~VM_SHARED;
124                 vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
125         }
126         /* remove_protection_ptes reads vma->vm_page_prot without mmap_sem */
127         WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
128 }
129 
130 /*
131  * Requires inode->i_mapping->i_mmap_rwsem
132  */
133 static void __remove_shared_vm_struct(struct vm_area_struct *vma,
134                 struct file *file, struct address_space *mapping)
135 {
136         if (vma->vm_flags & VM_DENYWRITE)
137                 atomic_inc(&file_inode(file)->i_writecount);
138         if (vma->vm_flags & VM_SHARED)
139                 mapping_unmap_writable(mapping);
140 
141         flush_dcache_mmap_lock(mapping);
142         vma_interval_tree_remove(vma, &mapping->i_mmap);
143         flush_dcache_mmap_unlock(mapping);
144 }
145 
146 /*
147  * Unlink a file-based vm structure from its interval tree, to hide
148  * vma from rmap and vmtruncate before freeing its page tables.
149  */
150 void unlink_file_vma(struct vm_area_struct *vma)
151 {
152         struct file *file = vma->vm_file;
153 
154         if (file) {
155                 struct address_space *mapping = file->f_mapping;
156                 i_mmap_lock_write(mapping);
157                 __remove_shared_vm_struct(vma, file, mapping);
158                 i_mmap_unlock_write(mapping);
159         }
160 }
161 
162 /*
163  * Close a vm structure and free it, returning the next.
164  */
165 static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
166 {
167         struct vm_area_struct *next = vma->vm_next;
168 
169         might_sleep();
170         if (vma->vm_ops && vma->vm_ops->close)
171                 vma->vm_ops->close(vma);
172         if (vma->vm_file)
173                 fput(vma->vm_file);
174         mpol_put(vma_policy(vma));
175         kmem_cache_free(vm_area_cachep, vma);
176         return next;
177 }
178 
179 static int do_brk(unsigned long addr, unsigned long len);
180 
181 SYSCALL_DEFINE1(brk, unsigned long, brk)
182 {
183         unsigned long retval;
184         unsigned long newbrk, oldbrk;
185         struct mm_struct *mm = current->mm;
186         unsigned long min_brk;
187         bool populate;
188 
189         if (down_write_killable(&mm->mmap_sem))
190                 return -EINTR;
191 
192 #ifdef CONFIG_COMPAT_BRK
193         /*
194          * CONFIG_COMPAT_BRK can still be overridden by setting
195          * randomize_va_space to 2, which will still cause mm->start_brk
196          * to be arbitrarily shifted
197          */
198         if (current->brk_randomized)
199                 min_brk = mm->start_brk;
200         else
201                 min_brk = mm->end_data;
202 #else
203         min_brk = mm->start_brk;
204 #endif
205         if (brk < min_brk)
206                 goto out;
207 
208         /*
209          * Check against rlimit here. If this check is done later after the test
210          * of oldbrk with newbrk then it can escape the test and let the data
211          * segment grow beyond its set limit the in case where the limit is
212          * not page aligned -Ram Gupta
213          */
214         if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
215                               mm->end_data, mm->start_data))
216                 goto out;
217 
218         newbrk = PAGE_ALIGN(brk);
219         oldbrk = PAGE_ALIGN(mm->brk);
220         if (oldbrk == newbrk)
221                 goto set_brk;
222 
223         /* Always allow shrinking brk. */
224         if (brk <= mm->brk) {
225                 if (!do_munmap(mm, newbrk, oldbrk-newbrk))
226                         goto set_brk;
227                 goto out;
228         }
229 
230         /* Check against existing mmap mappings. */
231         if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
232                 goto out;
233 
234         /* Ok, looks good - let it rip. */
235         if (do_brk(oldbrk, newbrk-oldbrk) < 0)
236                 goto out;
237 
238 set_brk:
239         mm->brk = brk;
240         populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
241         up_write(&mm->mmap_sem);
242         if (populate)
243                 mm_populate(oldbrk, newbrk - oldbrk);
244         return brk;
245 
246 out:
247         retval = mm->brk;
248         up_write(&mm->mmap_sem);
249         return retval;
250 }
251 
252 static long vma_compute_subtree_gap(struct vm_area_struct *vma)
253 {
254         unsigned long max, subtree_gap;
255         max = vma->vm_start;
256         if (vma->vm_prev)
257                 max -= vma->vm_prev->vm_end;
258         if (vma->vm_rb.rb_left) {
259                 subtree_gap = rb_entry(vma->vm_rb.rb_left,
260                                 struct vm_area_struct, vm_rb)->rb_subtree_gap;
261                 if (subtree_gap > max)
262                         max = subtree_gap;
263         }
264         if (vma->vm_rb.rb_right) {
265                 subtree_gap = rb_entry(vma->vm_rb.rb_right,
266                                 struct vm_area_struct, vm_rb)->rb_subtree_gap;
267                 if (subtree_gap > max)
268                         max = subtree_gap;
269         }
270         return max;
271 }
272 
273 #ifdef CONFIG_DEBUG_VM_RB
274 static int browse_rb(struct mm_struct *mm)
275 {
276         struct rb_root *root = &mm->mm_rb;
277         int i = 0, j, bug = 0;
278         struct rb_node *nd, *pn = NULL;
279         unsigned long prev = 0, pend = 0;
280 
281         for (nd = rb_first(root); nd; nd = rb_next(nd)) {
282                 struct vm_area_struct *vma;
283                 vma = rb_entry(nd, struct vm_area_struct, vm_rb);
284                 if (vma->vm_start < prev) {
285                         pr_emerg("vm_start %lx < prev %lx\n",
286                                   vma->vm_start, prev);
287                         bug = 1;
288                 }
289                 if (vma->vm_start < pend) {
290                         pr_emerg("vm_start %lx < pend %lx\n",
291                                   vma->vm_start, pend);
292                         bug = 1;
293                 }
294                 if (vma->vm_start > vma->vm_end) {
295                         pr_emerg("vm_start %lx > vm_end %lx\n",
296                                   vma->vm_start, vma->vm_end);
297                         bug = 1;
298                 }
299                 spin_lock(&mm->page_table_lock);
300                 if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
301                         pr_emerg("free gap %lx, correct %lx\n",
302                                vma->rb_subtree_gap,
303                                vma_compute_subtree_gap(vma));
304                         bug = 1;
305                 }
306                 spin_unlock(&mm->page_table_lock);
307                 i++;
308                 pn = nd;
309                 prev = vma->vm_start;
310                 pend = vma->vm_end;
311         }
312         j = 0;
313         for (nd = pn; nd; nd = rb_prev(nd))
314                 j++;
315         if (i != j) {
316                 pr_emerg("backwards %d, forwards %d\n", j, i);
317                 bug = 1;
318         }
319         return bug ? -1 : i;
320 }
321 
322 static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
323 {
324         struct rb_node *nd;
325 
326         for (nd = rb_first(root); nd; nd = rb_next(nd)) {
327                 struct vm_area_struct *vma;
328                 vma = rb_entry(nd, struct vm_area_struct, vm_rb);
329                 VM_BUG_ON_VMA(vma != ignore &&
330                         vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
331                         vma);
332         }
333 }
334 
335 static void validate_mm(struct mm_struct *mm)
336 {
337         int bug = 0;
338         int i = 0;
339         unsigned long highest_address = 0;
340         struct vm_area_struct *vma = mm->mmap;
341 
342         while (vma) {
343                 struct anon_vma *anon_vma = vma->anon_vma;
344                 struct anon_vma_chain *avc;
345 
346                 if (anon_vma) {
347                         anon_vma_lock_read(anon_vma);
348                         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
349                                 anon_vma_interval_tree_verify(avc);
350                         anon_vma_unlock_read(anon_vma);
351                 }
352 
353                 highest_address = vma->vm_end;
354                 vma = vma->vm_next;
355                 i++;
356         }
357         if (i != mm->map_count) {
358                 pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
359                 bug = 1;
360         }
361         if (highest_address != mm->highest_vm_end) {
362                 pr_emerg("mm->highest_vm_end %lx, found %lx\n",
363                           mm->highest_vm_end, highest_address);
364                 bug = 1;
365         }
366         i = browse_rb(mm);
367         if (i != mm->map_count) {
368                 if (i != -1)
369                         pr_emerg("map_count %d rb %d\n", mm->map_count, i);
370                 bug = 1;
371         }
372         VM_BUG_ON_MM(bug, mm);
373 }
374 #else
375 #define validate_mm_rb(root, ignore) do { } while (0)
376 #define validate_mm(mm) do { } while (0)
377 #endif
378 
379 RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
380                      unsigned long, rb_subtree_gap, vma_compute_subtree_gap)
381 
382 /*
383  * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
384  * vma->vm_prev->vm_end values changed, without modifying the vma's position
385  * in the rbtree.
386  */
387 static void vma_gap_update(struct vm_area_struct *vma)
388 {
389         /*
390          * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
391          * function that does exacltly what we want.
392          */
393         vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
394 }
395 
396 static inline void vma_rb_insert(struct vm_area_struct *vma,
397                                  struct rb_root *root)
398 {
399         /* All rb_subtree_gap values must be consistent prior to insertion */
400         validate_mm_rb(root, NULL);
401 
402         rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
403 }
404 
405 static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
406 {
407         /*
408          * Note rb_erase_augmented is a fairly large inline function,
409          * so make sure we instantiate it only once with our desired
410          * augmented rbtree callbacks.
411          */
412         rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
413 }
414 
415 static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma,
416                                                 struct rb_root *root,
417                                                 struct vm_area_struct *ignore)
418 {
419         /*
420          * All rb_subtree_gap values must be consistent prior to erase,
421          * with the possible exception of the "next" vma being erased if
422          * next->vm_start was reduced.
423          */
424         validate_mm_rb(root, ignore);
425 
426         __vma_rb_erase(vma, root);
427 }
428 
429 static __always_inline void vma_rb_erase(struct vm_area_struct *vma,
430                                          struct rb_root *root)
431 {
432         /*
433          * All rb_subtree_gap values must be consistent prior to erase,
434          * with the possible exception of the vma being erased.
435          */
436         validate_mm_rb(root, vma);
437 
438         __vma_rb_erase(vma, root);
439 }
440 
441 /*
442  * vma has some anon_vma assigned, and is already inserted on that
443  * anon_vma's interval trees.
444  *
445  * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
446  * vma must be removed from the anon_vma's interval trees using
447  * anon_vma_interval_tree_pre_update_vma().
448  *
449  * After the update, the vma will be reinserted using
450  * anon_vma_interval_tree_post_update_vma().
451  *
452  * The entire update must be protected by exclusive mmap_sem and by
453  * the root anon_vma's mutex.
454  */
455 static inline void
456 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
457 {
458         struct anon_vma_chain *avc;
459 
460         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
461                 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
462 }
463 
464 static inline void
465 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
466 {
467         struct anon_vma_chain *avc;
468 
469         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
470                 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
471 }
472 
473 static int find_vma_links(struct mm_struct *mm, unsigned long addr,
474                 unsigned long end, struct vm_area_struct **pprev,
475                 struct rb_node ***rb_link, struct rb_node **rb_parent)
476 {
477         struct rb_node **__rb_link, *__rb_parent, *rb_prev;
478 
479         __rb_link = &mm->mm_rb.rb_node;
480         rb_prev = __rb_parent = NULL;
481 
482         while (*__rb_link) {
483                 struct vm_area_struct *vma_tmp;
484 
485                 __rb_parent = *__rb_link;
486                 vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
487 
488                 if (vma_tmp->vm_end > addr) {
489                         /* Fail if an existing vma overlaps the area */
490                         if (vma_tmp->vm_start < end)
491                                 return -ENOMEM;
492                         __rb_link = &__rb_parent->rb_left;
493                 } else {
494                         rb_prev = __rb_parent;
495                         __rb_link = &__rb_parent->rb_right;
496                 }
497         }
498 
499         *pprev = NULL;
500         if (rb_prev)
501                 *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
502         *rb_link = __rb_link;
503         *rb_parent = __rb_parent;
504         return 0;
505 }
506 
507 static unsigned long count_vma_pages_range(struct mm_struct *mm,
508                 unsigned long addr, unsigned long end)
509 {
510         unsigned long nr_pages = 0;
511         struct vm_area_struct *vma;
512 
513         /* Find first overlaping mapping */
514         vma = find_vma_intersection(mm, addr, end);
515         if (!vma)
516                 return 0;
517 
518         nr_pages = (min(end, vma->vm_end) -
519                 max(addr, vma->vm_start)) >> PAGE_SHIFT;
520 
521         /* Iterate over the rest of the overlaps */
522         for (vma = vma->vm_next; vma; vma = vma->vm_next) {
523                 unsigned long overlap_len;
524 
525                 if (vma->vm_start > end)
526                         break;
527 
528                 overlap_len = min(end, vma->vm_end) - vma->vm_start;
529                 nr_pages += overlap_len >> PAGE_SHIFT;
530         }
531 
532         return nr_pages;
533 }
534 
535 void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
536                 struct rb_node **rb_link, struct rb_node *rb_parent)
537 {
538         /* Update tracking information for the gap following the new vma. */
539         if (vma->vm_next)
540                 vma_gap_update(vma->vm_next);
541         else
542                 mm->highest_vm_end = vma->vm_end;
543 
544         /*
545          * vma->vm_prev wasn't known when we followed the rbtree to find the
546          * correct insertion point for that vma. As a result, we could not
547          * update the vma vm_rb parents rb_subtree_gap values on the way down.
548          * So, we first insert the vma with a zero rb_subtree_gap value
549          * (to be consistent with what we did on the way down), and then
550          * immediately update the gap to the correct value. Finally we
551          * rebalance the rbtree after all augmented values have been set.
552          */
553         rb_link_node(&vma->vm_rb, rb_parent, rb_link);
554         vma->rb_subtree_gap = 0;
555         vma_gap_update(vma);
556         vma_rb_insert(vma, &mm->mm_rb);
557 }
558 
559 static void __vma_link_file(struct vm_area_struct *vma)
560 {
561         struct file *file;
562 
563         file = vma->vm_file;
564         if (file) {
565                 struct address_space *mapping = file->f_mapping;
566 
567                 if (vma->vm_flags & VM_DENYWRITE)
568                         atomic_dec(&file_inode(file)->i_writecount);
569                 if (vma->vm_flags & VM_SHARED)
570                         atomic_inc(&mapping->i_mmap_writable);
571 
572                 flush_dcache_mmap_lock(mapping);
573                 vma_interval_tree_insert(vma, &mapping->i_mmap);
574                 flush_dcache_mmap_unlock(mapping);
575         }
576 }
577 
578 static void
579 __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
580         struct vm_area_struct *prev, struct rb_node **rb_link,
581         struct rb_node *rb_parent)
582 {
583         __vma_link_list(mm, vma, prev, rb_parent);
584         __vma_link_rb(mm, vma, rb_link, rb_parent);
585 }
586 
587 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
588                         struct vm_area_struct *prev, struct rb_node **rb_link,
589                         struct rb_node *rb_parent)
590 {
591         struct address_space *mapping = NULL;
592 
593         if (vma->vm_file) {
594                 mapping = vma->vm_file->f_mapping;
595                 i_mmap_lock_write(mapping);
596         }
597 
598         __vma_link(mm, vma, prev, rb_link, rb_parent);
599         __vma_link_file(vma);
600 
601         if (mapping)
602                 i_mmap_unlock_write(mapping);
603 
604         mm->map_count++;
605         validate_mm(mm);
606 }
607 
608 /*
609  * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
610  * mm's list and rbtree.  It has already been inserted into the interval tree.
611  */
612 static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
613 {
614         struct vm_area_struct *prev;
615         struct rb_node **rb_link, *rb_parent;
616 
617         if (find_vma_links(mm, vma->vm_start, vma->vm_end,
618                            &prev, &rb_link, &rb_parent))
619                 BUG();
620         __vma_link(mm, vma, prev, rb_link, rb_parent);
621         mm->map_count++;
622 }
623 
624 static __always_inline void __vma_unlink_common(struct mm_struct *mm,
625                                                 struct vm_area_struct *vma,
626                                                 struct vm_area_struct *prev,
627                                                 bool has_prev,
628                                                 struct vm_area_struct *ignore)
629 {
630         struct vm_area_struct *next;
631 
632         vma_rb_erase_ignore(vma, &mm->mm_rb, ignore);
633         next = vma->vm_next;
634         if (has_prev)
635                 prev->vm_next = next;
636         else {
637                 prev = vma->vm_prev;
638                 if (prev)
639                         prev->vm_next = next;
640                 else
641                         mm->mmap = next;
642         }
643         if (next)
644                 next->vm_prev = prev;
645 
646         /* Kill the cache */
647         vmacache_invalidate(mm);
648 }
649 
650 static inline void __vma_unlink_prev(struct mm_struct *mm,
651                                      struct vm_area_struct *vma,
652                                      struct vm_area_struct *prev)
653 {
654         __vma_unlink_common(mm, vma, prev, true, vma);
655 }
656 
657 /*
658  * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
659  * is already present in an i_mmap tree without adjusting the tree.
660  * The following helper function should be used when such adjustments
661  * are necessary.  The "insert" vma (if any) is to be inserted
662  * before we drop the necessary locks.
663  */
664 int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
665         unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
666         struct vm_area_struct *expand)
667 {
668         struct mm_struct *mm = vma->vm_mm;
669         struct vm_area_struct *next = vma->vm_next, *orig_vma = vma;
670         struct address_space *mapping = NULL;
671         struct rb_root *root = NULL;
672         struct anon_vma *anon_vma = NULL;
673         struct file *file = vma->vm_file;
674         bool start_changed = false, end_changed = false;
675         long adjust_next = 0;
676         int remove_next = 0;
677 
678         if (next && !insert) {
679                 struct vm_area_struct *exporter = NULL, *importer = NULL;
680 
681                 if (end >= next->vm_end) {
682                         /*
683                          * vma expands, overlapping all the next, and
684                          * perhaps the one after too (mprotect case 6).
685                          * The only other cases that gets here are
686                          * case 1, case 7 and case 8.
687                          */
688                         if (next == expand) {
689                                 /*
690                                  * The only case where we don't expand "vma"
691                                  * and we expand "next" instead is case 8.
692                                  */
693                                 VM_WARN_ON(end != next->vm_end);
694                                 /*
695                                  * remove_next == 3 means we're
696                                  * removing "vma" and that to do so we
697                                  * swapped "vma" and "next".
698                                  */
699                                 remove_next = 3;
700                                 VM_WARN_ON(file != next->vm_file);
701                                 swap(vma, next);
702                         } else {
703                                 VM_WARN_ON(expand != vma);
704                                 /*
705                                  * case 1, 6, 7, remove_next == 2 is case 6,
706                                  * remove_next == 1 is case 1 or 7.
707                                  */
708                                 remove_next = 1 + (end > next->vm_end);
709                                 VM_WARN_ON(remove_next == 2 &&
710                                            end != next->vm_next->vm_end);
711                                 VM_WARN_ON(remove_next == 1 &&
712                                            end != next->vm_end);
713                                 /* trim end to next, for case 6 first pass */
714                                 end = next->vm_end;
715                         }
716 
717                         exporter = next;
718                         importer = vma;
719 
720                         /*
721                          * If next doesn't have anon_vma, import from vma after
722                          * next, if the vma overlaps with it.
723                          */
724                         if (remove_next == 2 && !next->anon_vma)
725                                 exporter = next->vm_next;
726 
727                 } else if (end > next->vm_start) {
728                         /*
729                          * vma expands, overlapping part of the next:
730                          * mprotect case 5 shifting the boundary up.
731                          */
732                         adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
733                         exporter = next;
734                         importer = vma;
735                         VM_WARN_ON(expand != importer);
736                 } else if (end < vma->vm_end) {
737                         /*
738                          * vma shrinks, and !insert tells it's not
739                          * split_vma inserting another: so it must be
740                          * mprotect case 4 shifting the boundary down.
741                          */
742                         adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT);
743                         exporter = vma;
744                         importer = next;
745                         VM_WARN_ON(expand != importer);
746                 }
747 
748                 /*
749                  * Easily overlooked: when mprotect shifts the boundary,
750                  * make sure the expanding vma has anon_vma set if the
751                  * shrinking vma had, to cover any anon pages imported.
752                  */
753                 if (exporter && exporter->anon_vma && !importer->anon_vma) {
754                         int error;
755 
756                         importer->anon_vma = exporter->anon_vma;
757                         error = anon_vma_clone(importer, exporter);
758                         if (error)
759                                 return error;
760                 }
761         }
762 again:
763         vma_adjust_trans_huge(orig_vma, start, end, adjust_next);
764 
765         if (file) {
766                 mapping = file->f_mapping;
767                 root = &mapping->i_mmap;
768                 uprobe_munmap(vma, vma->vm_start, vma->vm_end);
769 
770                 if (adjust_next)
771                         uprobe_munmap(next, next->vm_start, next->vm_end);
772 
773                 i_mmap_lock_write(mapping);
774                 if (insert) {
775                         /*
776                          * Put into interval tree now, so instantiated pages
777                          * are visible to arm/parisc __flush_dcache_page
778                          * throughout; but we cannot insert into address
779                          * space until vma start or end is updated.
780                          */
781                         __vma_link_file(insert);
782                 }
783         }
784 
785         anon_vma = vma->anon_vma;
786         if (!anon_vma && adjust_next)
787                 anon_vma = next->anon_vma;
788         if (anon_vma) {
789                 VM_WARN_ON(adjust_next && next->anon_vma &&
790                            anon_vma != next->anon_vma);
791                 anon_vma_lock_write(anon_vma);
792                 anon_vma_interval_tree_pre_update_vma(vma);
793                 if (adjust_next)
794                         anon_vma_interval_tree_pre_update_vma(next);
795         }
796 
797         if (root) {
798                 flush_dcache_mmap_lock(mapping);
799                 vma_interval_tree_remove(vma, root);
800                 if (adjust_next)
801                         vma_interval_tree_remove(next, root);
802         }
803 
804         if (start != vma->vm_start) {
805                 vma->vm_start = start;
806                 start_changed = true;
807         }
808         if (end != vma->vm_end) {
809                 vma->vm_end = end;
810                 end_changed = true;
811         }
812         vma->vm_pgoff = pgoff;
813         if (adjust_next) {
814                 next->vm_start += adjust_next << PAGE_SHIFT;
815                 next->vm_pgoff += adjust_next;
816         }
817 
818         if (root) {
819                 if (adjust_next)
820                         vma_interval_tree_insert(next, root);
821                 vma_interval_tree_insert(vma, root);
822                 flush_dcache_mmap_unlock(mapping);
823         }
824 
825         if (remove_next) {
826                 /*
827                  * vma_merge has merged next into vma, and needs
828                  * us to remove next before dropping the locks.
829                  */
830                 if (remove_next != 3)
831                         __vma_unlink_prev(mm, next, vma);
832                 else
833                         /*
834                          * vma is not before next if they've been
835                          * swapped.
836                          *
837                          * pre-swap() next->vm_start was reduced so
838                          * tell validate_mm_rb to ignore pre-swap()
839                          * "next" (which is stored in post-swap()
840                          * "vma").
841                          */
842                         __vma_unlink_common(mm, next, NULL, false, vma);
843                 if (file)
844                         __remove_shared_vm_struct(next, file, mapping);
845         } else if (insert) {
846                 /*
847                  * split_vma has split insert from vma, and needs
848                  * us to insert it before dropping the locks
849                  * (it may either follow vma or precede it).
850                  */
851                 __insert_vm_struct(mm, insert);
852         } else {
853                 if (start_changed)
854                         vma_gap_update(vma);
855                 if (end_changed) {
856                         if (!next)
857                                 mm->highest_vm_end = end;
858                         else if (!adjust_next)
859                                 vma_gap_update(next);
860                 }
861         }
862 
863         if (anon_vma) {
864                 anon_vma_interval_tree_post_update_vma(vma);
865                 if (adjust_next)
866                         anon_vma_interval_tree_post_update_vma(next);
867                 anon_vma_unlock_write(anon_vma);
868         }
869         if (mapping)
870                 i_mmap_unlock_write(mapping);
871 
872         if (root) {
873                 uprobe_mmap(vma);
874 
875                 if (adjust_next)
876                         uprobe_mmap(next);
877         }
878 
879         if (remove_next) {
880                 if (file) {
881                         uprobe_munmap(next, next->vm_start, next->vm_end);
882                         fput(file);
883                 }
884                 if (next->anon_vma)
885                         anon_vma_merge(vma, next);
886                 mm->map_count--;
887                 mpol_put(vma_policy(next));
888                 kmem_cache_free(vm_area_cachep, next);
889                 /*
890                  * In mprotect's case 6 (see comments on vma_merge),
891                  * we must remove another next too. It would clutter
892                  * up the code too much to do both in one go.
893                  */
894                 if (remove_next != 3) {
895                         /*
896                          * If "next" was removed and vma->vm_end was
897                          * expanded (up) over it, in turn
898                          * "next->vm_prev->vm_end" changed and the
899                          * "vma->vm_next" gap must be updated.
900                          */
901                         next = vma->vm_next;
902                 } else {
903                         /*
904                          * For the scope of the comment "next" and
905                          * "vma" considered pre-swap(): if "vma" was
906                          * removed, next->vm_start was expanded (down)
907                          * over it and the "next" gap must be updated.
908                          * Because of the swap() the post-swap() "vma"
909                          * actually points to pre-swap() "next"
910                          * (post-swap() "next" as opposed is now a
911                          * dangling pointer).
912                          */
913                         next = vma;
914                 }
915                 if (remove_next == 2) {
916                         remove_next = 1;
917                         end = next->vm_end;
918                         goto again;
919                 }
920                 else if (next)
921                         vma_gap_update(next);
922                 else {
923                         /*
924                          * If remove_next == 2 we obviously can't
925                          * reach this path.
926                          *
927                          * If remove_next == 3 we can't reach this
928                          * path because pre-swap() next is always not
929                          * NULL. pre-swap() "next" is not being
930                          * removed and its next->vm_end is not altered
931                          * (and furthermore "end" already matches
932                          * next->vm_end in remove_next == 3).
933                          *
934                          * We reach this only in the remove_next == 1
935                          * case if the "next" vma that was removed was
936                          * the highest vma of the mm. However in such
937                          * case next->vm_end == "end" and the extended
938                          * "vma" has vma->vm_end == next->vm_end so
939                          * mm->highest_vm_end doesn't need any update
940                          * in remove_next == 1 case.
941                          */
942                         VM_WARN_ON(mm->highest_vm_end != end);
943                 }
944         }
945         if (insert && file)
946                 uprobe_mmap(insert);
947 
948         validate_mm(mm);
949 
950         return 0;
951 }
952 
953 /*
954  * If the vma has a ->close operation then the driver probably needs to release
955  * per-vma resources, so we don't attempt to merge those.
956  */
957 static inline int is_mergeable_vma(struct vm_area_struct *vma,
958                                 struct file *file, unsigned long vm_flags,
959                                 struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
960 {
961         /*
962          * VM_SOFTDIRTY should not prevent from VMA merging, if we
963          * match the flags but dirty bit -- the caller should mark
964          * merged VMA as dirty. If dirty bit won't be excluded from
965          * comparison, we increase pressue on the memory system forcing
966          * the kernel to generate new VMAs when old one could be
967          * extended instead.
968          */
969         if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
970                 return 0;
971         if (vma->vm_file != file)
972                 return 0;
973         if (vma->vm_ops && vma->vm_ops->close)
974                 return 0;
975         if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
976                 return 0;
977         return 1;
978 }
979 
980 static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
981                                         struct anon_vma *anon_vma2,
982                                         struct vm_area_struct *vma)
983 {
984         /*
985          * The list_is_singular() test is to avoid merging VMA cloned from
986          * parents. This can improve scalability caused by anon_vma lock.
987          */
988         if ((!anon_vma1 || !anon_vma2) && (!vma ||
989                 list_is_singular(&vma->anon_vma_chain)))
990                 return 1;
991         return anon_vma1 == anon_vma2;
992 }
993 
994 /*
995  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
996  * in front of (at a lower virtual address and file offset than) the vma.
997  *
998  * We cannot merge two vmas if they have differently assigned (non-NULL)
999  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
1000  *
1001  * We don't check here for the merged mmap wrapping around the end of pagecache
1002  * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
1003  * wrap, nor mmaps which cover the final page at index -1UL.
1004  */
1005 static int
1006 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
1007                      struct anon_vma *anon_vma, struct file *file,
1008                      pgoff_t vm_pgoff,
1009                      struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
1010 {
1011         if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
1012             is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
1013                 if (vma->vm_pgoff == vm_pgoff)
1014                         return 1;
1015         }
1016         return 0;
1017 }
1018 
1019 /*
1020  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
1021  * beyond (at a higher virtual address and file offset than) the vma.
1022  *
1023  * We cannot merge two vmas if they have differently assigned (non-NULL)
1024  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
1025  */
1026 static int
1027 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
1028                     struct anon_vma *anon_vma, struct file *file,
1029                     pgoff_t vm_pgoff,
1030                     struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
1031 {
1032         if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
1033             is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
1034                 pgoff_t vm_pglen;
1035                 vm_pglen = vma_pages(vma);
1036                 if (vma->vm_pgoff + vm_pglen == vm_pgoff)
1037                         return 1;
1038         }
1039         return 0;
1040 }
1041 
1042 /*
1043  * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
1044  * whether that can be merged with its predecessor or its successor.
1045  * Or both (it neatly fills a hole).
1046  *
1047  * In most cases - when called for mmap, brk or mremap - [addr,end) is
1048  * certain not to be mapped by the time vma_merge is called; but when
1049  * called for mprotect, it is certain to be already mapped (either at
1050  * an offset within prev, or at the start of next), and the flags of
1051  * this area are about to be changed to vm_flags - and the no-change
1052  * case has already been eliminated.
1053  *
1054  * The following mprotect cases have to be considered, where AAAA is
1055  * the area passed down from mprotect_fixup, never extending beyond one
1056  * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
1057  *
1058  *     AAAA             AAAA                AAAA          AAAA
1059  *    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPNNNNXXXX
1060  *    cannot merge    might become    might become    might become
1061  *                    PPNNNNNNNNNN    PPPPPPPPPPNN    PPPPPPPPPPPP 6 or
1062  *    mmap, brk or    case 4 below    case 5 below    PPPPPPPPXXXX 7 or
1063  *    mremap move:                                    PPPPXXXXXXXX 8
1064  *        AAAA
1065  *    PPPP    NNNN    PPPPPPPPPPPP    PPPPPPPPNNNN    PPPPNNNNNNNN
1066  *    might become    case 1 below    case 2 below    case 3 below
1067  *
1068  * It is important for case 8 that the the vma NNNN overlapping the
1069  * region AAAA is never going to extended over XXXX. Instead XXXX must
1070  * be extended in region AAAA and NNNN must be removed. This way in
1071  * all cases where vma_merge succeeds, the moment vma_adjust drops the
1072  * rmap_locks, the properties of the merged vma will be already
1073  * correct for the whole merged range. Some of those properties like
1074  * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
1075  * be correct for the whole merged range immediately after the
1076  * rmap_locks are released. Otherwise if XXXX would be removed and
1077  * NNNN would be extended over the XXXX range, remove_migration_ptes
1078  * or other rmap walkers (if working on addresses beyond the "end"
1079  * parameter) may establish ptes with the wrong permissions of NNNN
1080  * instead of the right permissions of XXXX.
1081  */
1082 struct vm_area_struct *vma_merge(struct mm_struct *mm,
1083                         struct vm_area_struct *prev, unsigned long addr,
1084                         unsigned long end, unsigned long vm_flags,
1085                         struct anon_vma *anon_vma, struct file *file,
1086                         pgoff_t pgoff, struct mempolicy *policy,
1087                         struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
1088 {
1089         pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
1090         struct vm_area_struct *area, *next;
1091         int err;
1092 
1093         /*
1094          * We later require that vma->vm_flags == vm_flags,
1095          * so this tests vma->vm_flags & VM_SPECIAL, too.
1096          */
1097         if (vm_flags & VM_SPECIAL)
1098                 return NULL;
1099 
1100         if (prev)
1101                 next = prev->vm_next;
1102         else
1103                 next = mm->mmap;
1104         area = next;
1105         if (area && area->vm_end == end)                /* cases 6, 7, 8 */
1106                 next = next->vm_next;
1107 
1108         /* verify some invariant that must be enforced by the caller */
1109         VM_WARN_ON(prev && addr <= prev->vm_start);
1110         VM_WARN_ON(area && end > area->vm_end);
1111         VM_WARN_ON(addr >= end);
1112 
1113         /*
1114          * Can it merge with the predecessor?
1115          */
1116         if (prev && prev->vm_end == addr &&
1117                         mpol_equal(vma_policy(prev), policy) &&
1118                         can_vma_merge_after(prev, vm_flags,
1119                                             anon_vma, file, pgoff,
1120                                             vm_userfaultfd_ctx)) {
1121                 /*
1122                  * OK, it can.  Can we now merge in the successor as well?
1123                  */
1124                 if (next && end == next->vm_start &&
1125                                 mpol_equal(policy, vma_policy(next)) &&
1126                                 can_vma_merge_before(next, vm_flags,
1127                                                      anon_vma, file,
1128                                                      pgoff+pglen,
1129                                                      vm_userfaultfd_ctx) &&
1130                                 is_mergeable_anon_vma(prev->anon_vma,
1131                                                       next->anon_vma, NULL)) {
1132                                                         /* cases 1, 6 */
1133                         err = __vma_adjust(prev, prev->vm_start,
1134                                          next->vm_end, prev->vm_pgoff, NULL,
1135                                          prev);
1136                 } else                                  /* cases 2, 5, 7 */
1137                         err = __vma_adjust(prev, prev->vm_start,
1138                                          end, prev->vm_pgoff, NULL, prev);
1139                 if (err)
1140                         return NULL;
1141                 khugepaged_enter_vma_merge(prev, vm_flags);
1142                 return prev;
1143         }
1144 
1145         /*
1146          * Can this new request be merged in front of next?
1147          */
1148         if (next && end == next->vm_start &&
1149                         mpol_equal(policy, vma_policy(next)) &&
1150                         can_vma_merge_before(next, vm_flags,
1151                                              anon_vma, file, pgoff+pglen,
1152                                              vm_userfaultfd_ctx)) {
1153                 if (prev && addr < prev->vm_end)        /* case 4 */
1154                         err = __vma_adjust(prev, prev->vm_start,
1155                                          addr, prev->vm_pgoff, NULL, next);
1156                 else {                                  /* cases 3, 8 */
1157                         err = __vma_adjust(area, addr, next->vm_end,
1158                                          next->vm_pgoff - pglen, NULL, next);
1159                         /*
1160                          * In case 3 area is already equal to next and
1161                          * this is a noop, but in case 8 "area" has
1162                          * been removed and next was expanded over it.
1163                          */
1164                         area = next;
1165                 }
1166                 if (err)
1167                         return NULL;
1168                 khugepaged_enter_vma_merge(area, vm_flags);
1169                 return area;
1170         }
1171 
1172         return NULL;
1173 }
1174 
1175 /*
1176  * Rough compatbility check to quickly see if it's even worth looking
1177  * at sharing an anon_vma.
1178  *
1179  * They need to have the same vm_file, and the flags can only differ
1180  * in things that mprotect may change.
1181  *
1182  * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1183  * we can merge the two vma's. For example, we refuse to merge a vma if
1184  * there is a vm_ops->close() function, because that indicates that the
1185  * driver is doing some kind of reference counting. But that doesn't
1186  * really matter for the anon_vma sharing case.
1187  */
1188 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
1189 {
1190         return a->vm_end == b->vm_start &&
1191                 mpol_equal(vma_policy(a), vma_policy(b)) &&
1192                 a->vm_file == b->vm_file &&
1193                 !((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) &&
1194                 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
1195 }
1196 
1197 /*
1198  * Do some basic sanity checking to see if we can re-use the anon_vma
1199  * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1200  * the same as 'old', the other will be the new one that is trying
1201  * to share the anon_vma.
1202  *
1203  * NOTE! This runs with mm_sem held for reading, so it is possible that
1204  * the anon_vma of 'old' is concurrently in the process of being set up
1205  * by another page fault trying to merge _that_. But that's ok: if it
1206  * is being set up, that automatically means that it will be a singleton
1207  * acceptable for merging, so we can do all of this optimistically. But
1208  * we do that READ_ONCE() to make sure that we never re-load the pointer.
1209  *
1210  * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1211  * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1212  * is to return an anon_vma that is "complex" due to having gone through
1213  * a fork).
1214  *
1215  * We also make sure that the two vma's are compatible (adjacent,
1216  * and with the same memory policies). That's all stable, even with just
1217  * a read lock on the mm_sem.
1218  */
1219 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
1220 {
1221         if (anon_vma_compatible(a, b)) {
1222                 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
1223 
1224                 if (anon_vma && list_is_singular(&old->anon_vma_chain))
1225                         return anon_vma;
1226         }
1227         return NULL;
1228 }
1229 
1230 /*
1231  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1232  * neighbouring vmas for a suitable anon_vma, before it goes off
1233  * to allocate a new anon_vma.  It checks because a repetitive
1234  * sequence of mprotects and faults may otherwise lead to distinct
1235  * anon_vmas being allocated, preventing vma merge in subsequent
1236  * mprotect.
1237  */
1238 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
1239 {
1240         struct anon_vma *anon_vma;
1241         struct vm_area_struct *near;
1242 
1243         near = vma->vm_next;
1244         if (!near)
1245                 goto try_prev;
1246 
1247         anon_vma = reusable_anon_vma(near, vma, near);
1248         if (anon_vma)
1249                 return anon_vma;
1250 try_prev:
1251         near = vma->vm_prev;
1252         if (!near)
1253                 goto none;
1254 
1255         anon_vma = reusable_anon_vma(near, near, vma);
1256         if (anon_vma)
1257                 return anon_vma;
1258 none:
1259         /*
1260          * There's no absolute need to look only at touching neighbours:
1261          * we could search further afield for "compatible" anon_vmas.
1262          * But it would probably just be a waste of time searching,
1263          * or lead to too many vmas hanging off the same anon_vma.
1264          * We're trying to allow mprotect remerging later on,
1265          * not trying to minimize memory used for anon_vmas.
1266          */
1267         return NULL;
1268 }
1269 
1270 /*
1271  * If a hint addr is less than mmap_min_addr change hint to be as
1272  * low as possible but still greater than mmap_min_addr
1273  */
1274 static inline unsigned long round_hint_to_min(unsigned long hint)
1275 {
1276         hint &= PAGE_MASK;
1277         if (((void *)hint != NULL) &&
1278             (hint < mmap_min_addr))
1279                 return PAGE_ALIGN(mmap_min_addr);
1280         return hint;
1281 }
1282 
1283 static inline int mlock_future_check(struct mm_struct *mm,
1284                                      unsigned long flags,
1285                                      unsigned long len)
1286 {
1287         unsigned long locked, lock_limit;
1288 
1289         /*  mlock MCL_FUTURE? */
1290         if (flags & VM_LOCKED) {
1291                 locked = len >> PAGE_SHIFT;
1292                 locked += mm->locked_vm;
1293                 lock_limit = rlimit(RLIMIT_MEMLOCK);
1294                 lock_limit >>= PAGE_SHIFT;
1295                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
1296                         return -EAGAIN;
1297         }
1298         return 0;
1299 }
1300 
1301 /*
1302  * The caller must hold down_write(&current->mm->mmap_sem).
1303  */
1304 unsigned long do_mmap(struct file *file, unsigned long addr,
1305                         unsigned long len, unsigned long prot,
1306                         unsigned long flags, vm_flags_t vm_flags,
1307                         unsigned long pgoff, unsigned long *populate)
1308 {
1309         struct mm_struct *mm = current->mm;
1310         int pkey = 0;
1311 
1312         *populate = 0;
1313 
1314         if (!len)
1315                 return -EINVAL;
1316 
1317         /*
1318          * Does the application expect PROT_READ to imply PROT_EXEC?
1319          *
1320          * (the exception is when the underlying filesystem is noexec
1321          *  mounted, in which case we dont add PROT_EXEC.)
1322          */
1323         if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
1324                 if (!(file && path_noexec(&file->f_path)))
1325                         prot |= PROT_EXEC;
1326 
1327         if (!(flags & MAP_FIXED))
1328                 addr = round_hint_to_min(addr);
1329 
1330         /* Careful about overflows.. */
1331         len = PAGE_ALIGN(len);
1332         if (!len)
1333                 return -ENOMEM;
1334 
1335         /* offset overflow? */
1336         if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
1337                 return -EOVERFLOW;
1338 
1339         /* Too many mappings? */
1340         if (mm->map_count > sysctl_max_map_count)
1341                 return -ENOMEM;
1342 
1343         /* Obtain the address to map to. we verify (or select) it and ensure
1344          * that it represents a valid section of the address space.
1345          */
1346         addr = get_unmapped_area(file, addr, len, pgoff, flags);
1347         if (offset_in_page(addr))
1348                 return addr;
1349 
1350         if (prot == PROT_EXEC) {
1351                 pkey = execute_only_pkey(mm);
1352                 if (pkey < 0)
1353                         pkey = 0;
1354         }
1355 
1356         /* Do simple checking here so the lower-level routines won't have
1357          * to. we assume access permissions have been handled by the open
1358          * of the memory object, so we don't do any here.
1359          */
1360         vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
1361                         mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1362 
1363         if (flags & MAP_LOCKED)
1364                 if (!can_do_mlock())
1365                         return -EPERM;
1366 
1367         if (mlock_future_check(mm, vm_flags, len))
1368                 return -EAGAIN;
1369 
1370         if (file) {
1371                 struct inode *inode = file_inode(file);
1372 
1373                 switch (flags & MAP_TYPE) {
1374                 case MAP_SHARED:
1375                         if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
1376                                 return -EACCES;
1377 
1378                         /*
1379                          * Make sure we don't allow writing to an append-only
1380                          * file..
1381                          */
1382                         if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1383                                 return -EACCES;
1384 
1385                         /*
1386                          * Make sure there are no mandatory locks on the file.
1387                          */
1388                         if (locks_verify_locked(file))
1389                                 return -EAGAIN;
1390 
1391                         vm_flags |= VM_SHARED | VM_MAYSHARE;
1392                         if (!(file->f_mode & FMODE_WRITE))
1393                                 vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
1394 
1395                         /* fall through */
1396                 case MAP_PRIVATE:
1397                         if (!(file->f_mode & FMODE_READ))
1398                                 return -EACCES;
1399                         if (path_noexec(&file->f_path)) {
1400                                 if (vm_flags & VM_EXEC)
1401                                         return -EPERM;
1402                                 vm_flags &= ~VM_MAYEXEC;
1403                         }
1404 
1405                         if (!file->f_op->mmap)
1406                                 return -ENODEV;
1407                         if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1408                                 return -EINVAL;
1409                         break;
1410 
1411                 default:
1412                         return -EINVAL;
1413                 }
1414         } else {
1415                 switch (flags & MAP_TYPE) {
1416                 case MAP_SHARED:
1417                         if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1418                                 return -EINVAL;
1419                         /*
1420                          * Ignore pgoff.
1421                          */
1422                         pgoff = 0;
1423                         vm_flags |= VM_SHARED | VM_MAYSHARE;
1424                         break;
1425                 case MAP_PRIVATE:
1426                         /*
1427                          * Set pgoff according to addr for anon_vma.
1428                          */
1429                         pgoff = addr >> PAGE_SHIFT;
1430                         break;
1431                 default:
1432                         return -EINVAL;
1433                 }
1434         }
1435 
1436         /*
1437          * Set 'VM_NORESERVE' if we should not account for the
1438          * memory use of this mapping.
1439          */
1440         if (flags & MAP_NORESERVE) {
1441                 /* We honor MAP_NORESERVE if allowed to overcommit */
1442                 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1443                         vm_flags |= VM_NORESERVE;
1444 
1445                 /* hugetlb applies strict overcommit unless MAP_NORESERVE */
1446                 if (file && is_file_hugepages(file))
1447                         vm_flags |= VM_NORESERVE;
1448         }
1449 
1450         addr = mmap_region(file, addr, len, vm_flags, pgoff);
1451         if (!IS_ERR_VALUE(addr) &&
1452             ((vm_flags & VM_LOCKED) ||
1453              (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
1454                 *populate = len;
1455         return addr;
1456 }
1457 
1458 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1459                 unsigned long, prot, unsigned long, flags,
1460                 unsigned long, fd, unsigned long, pgoff)
1461 {
1462         struct file *file = NULL;
1463         unsigned long retval;
1464 
1465         if (!(flags & MAP_ANONYMOUS)) {
1466                 audit_mmap_fd(fd, flags);
1467                 file = fget(fd);
1468                 if (!file)
1469                         return -EBADF;
1470                 if (is_file_hugepages(file))
1471                         len = ALIGN(len, huge_page_size(hstate_file(file)));
1472                 retval = -EINVAL;
1473                 if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
1474                         goto out_fput;
1475         } else if (flags & MAP_HUGETLB) {
1476                 struct user_struct *user = NULL;
1477                 struct hstate *hs;
1478 
1479                 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & SHM_HUGE_MASK);
1480                 if (!hs)
1481                         return -EINVAL;
1482 
1483                 len = ALIGN(len, huge_page_size(hs));
1484                 /*
1485                  * VM_NORESERVE is used because the reservations will be
1486                  * taken when vm_ops->mmap() is called
1487                  * A dummy user value is used because we are not locking
1488                  * memory so no accounting is necessary
1489                  */
1490                 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1491                                 VM_NORESERVE,
1492                                 &user, HUGETLB_ANONHUGE_INODE,
1493                                 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1494                 if (IS_ERR(file))
1495                         return PTR_ERR(file);
1496         }
1497 
1498         flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1499 
1500         retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1501 out_fput:
1502         if (file)
1503                 fput(file);
1504         return retval;
1505 }
1506 
1507 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1508 struct mmap_arg_struct {
1509         unsigned long addr;
1510         unsigned long len;
1511         unsigned long prot;
1512         unsigned long flags;
1513         unsigned long fd;
1514         unsigned long offset;
1515 };
1516 
1517 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1518 {
1519         struct mmap_arg_struct a;
1520 
1521         if (copy_from_user(&a, arg, sizeof(a)))
1522                 return -EFAULT;
1523         if (offset_in_page(a.offset))
1524                 return -EINVAL;
1525 
1526         return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1527                               a.offset >> PAGE_SHIFT);
1528 }
1529 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1530 
1531 /*
1532  * Some shared mappigns will want the pages marked read-only
1533  * to track write events. If so, we'll downgrade vm_page_prot
1534  * to the private version (using protection_map[] without the
1535  * VM_SHARED bit).
1536  */
1537 int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
1538 {
1539         vm_flags_t vm_flags = vma->vm_flags;
1540         const struct vm_operations_struct *vm_ops = vma->vm_ops;
1541 
1542         /* If it was private or non-writable, the write bit is already clear */
1543         if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
1544                 return 0;
1545 
1546         /* The backer wishes to know when pages are first written to? */
1547         if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite))
1548                 return 1;
1549 
1550         /* The open routine did something to the protections that pgprot_modify
1551          * won't preserve? */
1552         if (pgprot_val(vm_page_prot) !=
1553             pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags)))
1554                 return 0;
1555 
1556         /* Do we need to track softdirty? */
1557         if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
1558                 return 1;
1559 
1560         /* Specialty mapping? */
1561         if (vm_flags & VM_PFNMAP)
1562                 return 0;
1563 
1564         /* Can the mapping track the dirty pages? */
1565         return vma->vm_file && vma->vm_file->f_mapping &&
1566                 mapping_cap_account_dirty(vma->vm_file->f_mapping);
1567 }
1568 
1569 /*
1570  * We account for memory if it's a private writeable mapping,
1571  * not hugepages and VM_NORESERVE wasn't set.
1572  */
1573 static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
1574 {
1575         /*
1576          * hugetlb has its own accounting separate from the core VM
1577          * VM_HUGETLB may not be set yet so we cannot check for that flag.
1578          */
1579         if (file && is_file_hugepages(file))
1580                 return 0;
1581 
1582         return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
1583 }
1584 
1585 unsigned long mmap_region(struct file *file, unsigned long addr,
1586                 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
1587 {
1588         struct mm_struct *mm = current->mm;
1589         struct vm_area_struct *vma, *prev;
1590         int error;
1591         struct rb_node **rb_link, *rb_parent;
1592         unsigned long charged = 0;
1593 
1594         /* Check against address space limit. */
1595         if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
1596                 unsigned long nr_pages;
1597 
1598                 /*
1599                  * MAP_FIXED may remove pages of mappings that intersects with
1600                  * requested mapping. Account for the pages it would unmap.
1601                  */
1602                 nr_pages = count_vma_pages_range(mm, addr, addr + len);
1603 
1604                 if (!may_expand_vm(mm, vm_flags,
1605                                         (len >> PAGE_SHIFT) - nr_pages))
1606                         return -ENOMEM;
1607         }
1608 
1609         /* Clear old maps */
1610         while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
1611                               &rb_parent)) {
1612                 if (do_munmap(mm, addr, len))
1613                         return -ENOMEM;
1614         }
1615 
1616         /*
1617          * Private writable mapping: check memory availability
1618          */
1619         if (accountable_mapping(file, vm_flags)) {
1620                 charged = len >> PAGE_SHIFT;
1621                 if (security_vm_enough_memory_mm(mm, charged))
1622                         return -ENOMEM;
1623                 vm_flags |= VM_ACCOUNT;
1624         }
1625 
1626         /*
1627          * Can we just expand an old mapping?
1628          */
1629         vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
1630                         NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
1631         if (vma)
1632                 goto out;
1633 
1634         /*
1635          * Determine the object being mapped and call the appropriate
1636          * specific mapper. the address has already been validated, but
1637          * not unmapped, but the maps are removed from the list.
1638          */
1639         vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1640         if (!vma) {
1641                 error = -ENOMEM;
1642                 goto unacct_error;
1643         }
1644 
1645         vma->vm_mm = mm;
1646         vma->vm_start = addr;
1647         vma->vm_end = addr + len;
1648         vma->vm_flags = vm_flags;
1649         vma->vm_page_prot = vm_get_page_prot(vm_flags);
1650         vma->vm_pgoff = pgoff;
1651         INIT_LIST_HEAD(&vma->anon_vma_chain);
1652 
1653         if (file) {
1654                 if (vm_flags & VM_DENYWRITE) {
1655                         error = deny_write_access(file);
1656                         if (error)
1657                                 goto free_vma;
1658                 }
1659                 if (vm_flags & VM_SHARED) {
1660                         error = mapping_map_writable(file->f_mapping);
1661                         if (error)
1662                                 goto allow_write_and_free_vma;
1663                 }
1664 
1665                 /* ->mmap() can change vma->vm_file, but must guarantee that
1666                  * vma_link() below can deny write-access if VM_DENYWRITE is set
1667                  * and map writably if VM_SHARED is set. This usually means the
1668                  * new file must not have been exposed to user-space, yet.
1669                  */
1670                 vma->vm_file = get_file(file);
1671                 error = file->f_op->mmap(file, vma);
1672                 if (error)
1673                         goto unmap_and_free_vma;
1674 
1675                 /* Can addr have changed??
1676                  *
1677                  * Answer: Yes, several device drivers can do it in their
1678                  *         f_op->mmap method. -DaveM
1679                  * Bug: If addr is changed, prev, rb_link, rb_parent should
1680                  *      be updated for vma_link()
1681                  */
1682                 WARN_ON_ONCE(addr != vma->vm_start);
1683 
1684                 addr = vma->vm_start;
1685                 vm_flags = vma->vm_flags;
1686         } else if (vm_flags & VM_SHARED) {
1687                 error = shmem_zero_setup(vma);
1688                 if (error)
1689                         goto free_vma;
1690         }
1691 
1692         vma_link(mm, vma, prev, rb_link, rb_parent);
1693         /* Once vma denies write, undo our temporary denial count */
1694         if (file) {
1695                 if (vm_flags & VM_SHARED)
1696                         mapping_unmap_writable(file->f_mapping);
1697                 if (vm_flags & VM_DENYWRITE)
1698                         allow_write_access(file);
1699         }
1700         file = vma->vm_file;
1701 out:
1702         perf_event_mmap(vma);
1703 
1704         vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
1705         if (vm_flags & VM_LOCKED) {
1706                 if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
1707                                         vma == get_gate_vma(current->mm)))
1708                         mm->locked_vm += (len >> PAGE_SHIFT);
1709                 else
1710                         vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
1711         }
1712 
1713         if (file)
1714                 uprobe_mmap(vma);
1715 
1716         /*
1717          * New (or expanded) vma always get soft dirty status.
1718          * Otherwise user-space soft-dirty page tracker won't
1719          * be able to distinguish situation when vma area unmapped,
1720          * then new mapped in-place (which must be aimed as
1721          * a completely new data area).
1722          */
1723         vma->vm_flags |= VM_SOFTDIRTY;
1724 
1725         vma_set_page_prot(vma);
1726 
1727         return addr;
1728 
1729 unmap_and_free_vma:
1730         vma->vm_file = NULL;
1731         fput(file);
1732 
1733         /* Undo any partial mapping done by a device driver. */
1734         unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
1735         charged = 0;
1736         if (vm_flags & VM_SHARED)
1737                 mapping_unmap_writable(file->f_mapping);
1738 allow_write_and_free_vma:
1739         if (vm_flags & VM_DENYWRITE)
1740                 allow_write_access(file);
1741 free_vma:
1742         kmem_cache_free(vm_area_cachep, vma);
1743 unacct_error:
1744         if (charged)
1745                 vm_unacct_memory(charged);
1746         return error;
1747 }
1748 
1749 unsigned long unmapped_area(struct vm_unmapped_area_info *info)
1750 {
1751         /*
1752          * We implement the search by looking for an rbtree node that
1753          * immediately follows a suitable gap. That is,
1754          * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
1755          * - gap_end   = vma->vm_start        >= info->low_limit  + length;
1756          * - gap_end - gap_start >= length
1757          */
1758 
1759         struct mm_struct *mm = current->mm;
1760         struct vm_area_struct *vma;
1761         unsigned long length, low_limit, high_limit, gap_start, gap_end;
1762 
1763         /* Adjust search length to account for worst case alignment overhead */
1764         length = info->length + info->align_mask;
1765         if (length < info->length)
1766                 return -ENOMEM;
1767 
1768         /* Adjust search limits by the desired length */
1769         if (info->high_limit < length)
1770                 return -ENOMEM;
1771         high_limit = info->high_limit - length;
1772 
1773         if (info->low_limit > high_limit)
1774                 return -ENOMEM;
1775         low_limit = info->low_limit + length;
1776 
1777         /* Check if rbtree root looks promising */
1778         if (RB_EMPTY_ROOT(&mm->mm_rb))
1779                 goto check_highest;
1780         vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1781         if (vma->rb_subtree_gap < length)
1782                 goto check_highest;
1783 
1784         while (true) {
1785                 /* Visit left subtree if it looks promising */
1786                 gap_end = vma->vm_start;
1787                 if (gap_end >= low_limit && vma->vm_rb.rb_left) {
1788                         struct vm_area_struct *left =
1789                                 rb_entry(vma->vm_rb.rb_left,
1790                                          struct vm_area_struct, vm_rb);
1791                         if (left->rb_subtree_gap >= length) {
1792                                 vma = left;
1793                                 continue;
1794                         }
1795                 }
1796 
1797                 gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1798 check_current:
1799                 /* Check if current node has a suitable gap */
1800                 if (gap_start > high_limit)
1801                         return -ENOMEM;
1802                 if (gap_end >= low_limit && gap_end - gap_start >= length)
1803                         goto found;
1804 
1805                 /* Visit right subtree if it looks promising */
1806                 if (vma->vm_rb.rb_right) {
1807                         struct vm_area_struct *right =
1808                                 rb_entry(vma->vm_rb.rb_right,
1809                                          struct vm_area_struct, vm_rb);
1810                         if (right->rb_subtree_gap >= length) {
1811                                 vma = right;
1812                                 continue;
1813                         }
1814                 }
1815 
1816                 /* Go back up the rbtree to find next candidate node */
1817                 while (true) {
1818                         struct rb_node *prev = &vma->vm_rb;
1819                         if (!rb_parent(prev))
1820                                 goto check_highest;
1821                         vma = rb_entry(rb_parent(prev),
1822                                        struct vm_area_struct, vm_rb);
1823                         if (prev == vma->vm_rb.rb_left) {
1824                                 gap_start = vma->vm_prev->vm_end;
1825                                 gap_end = vma->vm_start;
1826                                 goto check_current;
1827                         }
1828                 }
1829         }
1830 
1831 check_highest:
1832         /* Check highest gap, which does not precede any rbtree node */
1833         gap_start = mm->highest_vm_end;
1834         gap_end = ULONG_MAX;  /* Only for VM_BUG_ON below */
1835         if (gap_start > high_limit)
1836                 return -ENOMEM;
1837 
1838 found:
1839         /* We found a suitable gap. Clip it with the original low_limit. */
1840         if (gap_start < info->low_limit)
1841                 gap_start = info->low_limit;
1842 
1843         /* Adjust gap address to the desired alignment */
1844         gap_start += (info->align_offset - gap_start) & info->align_mask;
1845 
1846         VM_BUG_ON(gap_start + info->length > info->high_limit);
1847         VM_BUG_ON(gap_start + info->length > gap_end);
1848         return gap_start;
1849 }
1850 
1851 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
1852 {
1853         struct mm_struct *mm = current->mm;
1854         struct vm_area_struct *vma;
1855         unsigned long length, low_limit, high_limit, gap_start, gap_end;
1856 
1857         /* Adjust search length to account for worst case alignment overhead */
1858         length = info->length + info->align_mask;
1859         if (length < info->length)
1860                 return -ENOMEM;
1861 
1862         /*
1863          * Adjust search limits by the desired length.
1864          * See implementation comment at top of unmapped_area().
1865          */
1866         gap_end = info->high_limit;
1867         if (gap_end < length)
1868                 return -ENOMEM;
1869         high_limit = gap_end - length;
1870 
1871         if (info->low_limit > high_limit)
1872                 return -ENOMEM;
1873         low_limit = info->low_limit + length;
1874 
1875         /* Check highest gap, which does not precede any rbtree node */
1876         gap_start = mm->highest_vm_end;
1877         if (gap_start <= high_limit)
1878                 goto found_highest;
1879 
1880         /* Check if rbtree root looks promising */
1881         if (RB_EMPTY_ROOT(&mm->mm_rb))
1882                 return -ENOMEM;
1883         vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1884         if (vma->rb_subtree_gap < length)
1885                 return -ENOMEM;
1886 
1887         while (true) {
1888                 /* Visit right subtree if it looks promising */
1889                 gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1890                 if (gap_start <= high_limit && vma->vm_rb.rb_right) {
1891                         struct vm_area_struct *right =
1892                                 rb_entry(vma->vm_rb.rb_right,
1893                                          struct vm_area_struct, vm_rb);
1894                         if (right->rb_subtree_gap >= length) {
1895                                 vma = right;
1896                                 continue;
1897                         }
1898                 }
1899 
1900 check_current:
1901                 /* Check if current node has a suitable gap */
1902                 gap_end = vma->vm_start;
1903                 if (gap_end < low_limit)
1904                         return -ENOMEM;
1905                 if (gap_start <= high_limit && gap_end - gap_start >= length)
1906                         goto found;
1907 
1908                 /* Visit left subtree if it looks promising */
1909                 if (vma->vm_rb.rb_left) {
1910                         struct vm_area_struct *left =
1911                                 rb_entry(vma->vm_rb.rb_left,
1912                                          struct vm_area_struct, vm_rb);
1913                         if (left->rb_subtree_gap >= length) {
1914                                 vma = left;
1915                                 continue;
1916                         }
1917                 }
1918 
1919                 /* Go back up the rbtree to find next candidate node */
1920                 while (true) {
1921                         struct rb_node *prev = &vma->vm_rb;
1922                         if (!rb_parent(prev))
1923                                 return -ENOMEM;
1924                         vma = rb_entry(rb_parent(prev),
1925                                        struct vm_area_struct, vm_rb);
1926                         if (prev == vma->vm_rb.rb_right) {
1927                                 gap_start = vma->vm_prev ?
1928                                         vma->vm_prev->vm_end : 0;
1929                                 goto check_current;
1930                         }
1931                 }
1932         }
1933 
1934 found:
1935         /* We found a suitable gap. Clip it with the original high_limit. */
1936         if (gap_end > info->high_limit)
1937                 gap_end = info->high_limit;
1938 
1939 found_highest:
1940         /* Compute highest gap address at the desired alignment */
1941         gap_end -= info->length;
1942         gap_end -= (gap_end - info->align_offset) & info->align_mask;
1943 
1944         VM_BUG_ON(gap_end < info->low_limit);
1945         VM_BUG_ON(gap_end < gap_start);
1946         return gap_end;
1947 }
1948 
1949 /* Get an address range which is currently unmapped.
1950  * For shmat() with addr=0.
1951  *
1952  * Ugly calling convention alert:
1953  * Return value with the low bits set means error value,
1954  * ie
1955  *      if (ret & ~PAGE_MASK)
1956  *              error = ret;
1957  *
1958  * This function "knows" that -ENOMEM has the bits set.
1959  */
1960 #ifndef HAVE_ARCH_UNMAPPED_AREA
1961 unsigned long
1962 arch_get_unmapped_area(struct file *filp, unsigned long addr,
1963                 unsigned long len, unsigned long pgoff, unsigned long flags)
1964 {
1965         struct mm_struct *mm = current->mm;
1966         struct vm_area_struct *vma;
1967         struct vm_unmapped_area_info info;
1968 
1969         if (len > TASK_SIZE - mmap_min_addr)
1970                 return -ENOMEM;
1971 
1972         if (flags & MAP_FIXED)
1973                 return addr;
1974 
1975         if (addr) {
1976                 addr = PAGE_ALIGN(addr);
1977                 vma = find_vma(mm, addr);
1978                 if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
1979                     (!vma || addr + len <= vma->vm_start))
1980                         return addr;
1981         }
1982 
1983         info.flags = 0;
1984         info.length = len;
1985         info.low_limit = mm->mmap_base;
1986         info.high_limit = TASK_SIZE;
1987         info.align_mask = 0;
1988         return vm_unmapped_area(&info);
1989 }
1990 #endif
1991 
1992 /*
1993  * This mmap-allocator allocates new areas top-down from below the
1994  * stack's low limit (the base):
1995  */
1996 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1997 unsigned long
1998 arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
1999                           const unsigned long len, const unsigned long pgoff,
2000                           const unsigned long flags)
2001 {
2002         struct vm_area_struct *vma;
2003         struct mm_struct *mm = current->mm;
2004         unsigned long addr = addr0;
2005         struct vm_unmapped_area_info info;
2006 
2007         /* requested length too big for entire address space */
2008         if (len > TASK_SIZE - mmap_min_addr)
2009                 return -ENOMEM;
2010 
2011         if (flags & MAP_FIXED)
2012                 return addr;
2013 
2014         /* requesting a specific address */
2015         if (addr) {
2016                 addr = PAGE_ALIGN(addr);
2017                 vma = find_vma(mm, addr);
2018                 if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
2019                                 (!vma || addr + len <= vma->vm_start))
2020                         return addr;
2021         }
2022 
2023         info.flags = VM_UNMAPPED_AREA_TOPDOWN;
2024         info.length = len;
2025         info.low_limit = max(PAGE_SIZE, mmap_min_addr);
2026         info.high_limit = mm->mmap_base;
2027         info.align_mask = 0;
2028         addr = vm_unmapped_area(&info);
2029 
2030         /*
2031          * A failed mmap() very likely causes application failure,
2032          * so fall back to the bottom-up function here. This scenario
2033          * can happen with large stack limits and large mmap()
2034          * allocations.
2035          */
2036         if (offset_in_page(addr)) {
2037                 VM_BUG_ON(addr != -ENOMEM);
2038                 info.flags = 0;
2039                 info.low_limit = TASK_UNMAPPED_BASE;
2040                 info.high_limit = TASK_SIZE;
2041                 addr = vm_unmapped_area(&info);
2042         }
2043 
2044         return addr;
2045 }
2046 #endif
2047 
2048 unsigned long
2049 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
2050                 unsigned long pgoff, unsigned long flags)
2051 {
2052         unsigned long (*get_area)(struct file *, unsigned long,
2053                                   unsigned long, unsigned long, unsigned long);
2054 
2055         unsigned long error = arch_mmap_check(addr, len, flags);
2056         if (error)
2057                 return error;
2058 
2059         /* Careful about overflows.. */
2060         if (len > TASK_SIZE)
2061                 return -ENOMEM;
2062 
2063         get_area = current->mm->get_unmapped_area;
2064         if (file) {
2065                 if (file->f_op->get_unmapped_area)
2066                         get_area = file->f_op->get_unmapped_area;
2067         } else if (flags & MAP_SHARED) {
2068                 /*
2069                  * mmap_region() will call shmem_zero_setup() to create a file,
2070                  * so use shmem's get_unmapped_area in case it can be huge.
2071                  * do_mmap_pgoff() will clear pgoff, so match alignment.
2072                  */
2073                 pgoff = 0;
2074                 get_area = shmem_get_unmapped_area;
2075         }
2076 
2077         addr = get_area(file, addr, len, pgoff, flags);
2078         if (IS_ERR_VALUE(addr))
2079                 return addr;
2080 
2081         if (addr > TASK_SIZE - len)
2082                 return -ENOMEM;
2083         if (offset_in_page(addr))
2084                 return -EINVAL;
2085 
2086         error = security_mmap_addr(addr);
2087         return error ? error : addr;
2088 }
2089 
2090 EXPORT_SYMBOL(get_unmapped_area);
2091 
2092 /* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
2093 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
2094 {
2095         struct rb_node *rb_node;
2096         struct vm_area_struct *vma;
2097 
2098         /* Check the cache first. */
2099         vma = vmacache_find(mm, addr);
2100         if (likely(vma))
2101                 return vma;
2102 
2103         rb_node = mm->mm_rb.rb_node;
2104 
2105         while (rb_node) {
2106                 struct vm_area_struct *tmp;
2107 
2108                 tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2109 
2110                 if (tmp->vm_end > addr) {
2111                         vma = tmp;
2112                         if (tmp->vm_start <= addr)
2113                                 break;
2114                         rb_node = rb_node->rb_left;
2115                 } else
2116                         rb_node = rb_node->rb_right;
2117         }
2118 
2119         if (vma)
2120                 vmacache_update(addr, vma);
2121         return vma;
2122 }
2123 
2124 EXPORT_SYMBOL(find_vma);
2125 
2126 /*
2127  * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
2128  */
2129 struct vm_area_struct *
2130 find_vma_prev(struct mm_struct *mm, unsigned long addr,
2131                         struct vm_area_struct **pprev)
2132 {
2133         struct vm_area_struct *vma;
2134 
2135         vma = find_vma(mm, addr);
2136         if (vma) {
2137                 *pprev = vma->vm_prev;
2138         } else {
2139                 struct rb_node *rb_node = mm->mm_rb.rb_node;
2140                 *pprev = NULL;
2141                 while (rb_node) {
2142                         *pprev = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2143                         rb_node = rb_node->rb_right;
2144                 }
2145         }
2146         return vma;
2147 }
2148 
2149 /*
2150  * Verify that the stack growth is acceptable and
2151  * update accounting. This is shared with both the
2152  * grow-up and grow-down cases.
2153  */
2154 static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
2155 {
2156         struct mm_struct *mm = vma->vm_mm;
2157         struct rlimit *rlim = current->signal->rlim;
2158         unsigned long new_start, actual_size;
2159 
2160         /* address space limit tests */
2161         if (!may_expand_vm(mm, vma->vm_flags, grow))
2162                 return -ENOMEM;
2163 
2164         /* Stack limit test */
2165         actual_size = size;
2166         if (size && (vma->vm_flags & (VM_GROWSUP | VM_GROWSDOWN)))
2167                 actual_size -= PAGE_SIZE;
2168         if (actual_size > READ_ONCE(rlim[RLIMIT_STACK].rlim_cur))
2169                 return -ENOMEM;
2170 
2171         /* mlock limit tests */
2172         if (vma->vm_flags & VM_LOCKED) {
2173                 unsigned long locked;
2174                 unsigned long limit;
2175                 locked = mm->locked_vm + grow;
2176                 limit = READ_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
2177                 limit >>= PAGE_SHIFT;
2178                 if (locked > limit && !capable(CAP_IPC_LOCK))
2179                         return -ENOMEM;
2180         }
2181 
2182         /* Check to ensure the stack will not grow into a hugetlb-only region */
2183         new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
2184                         vma->vm_end - size;
2185         if (is_hugepage_only_range(vma->vm_mm, new_start, size))
2186                 return -EFAULT;
2187 
2188         /*
2189          * Overcommit..  This must be the final test, as it will
2190          * update security statistics.
2191          */
2192         if (security_vm_enough_memory_mm(mm, grow))
2193                 return -ENOMEM;
2194 
2195         return 0;
2196 }
2197 
2198 #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
2199 /*
2200  * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
2201  * vma is the last one with address > vma->vm_end.  Have to extend vma.
2202  */
2203 int expand_upwards(struct vm_area_struct *vma, unsigned long address)
2204 {
2205         struct mm_struct *mm = vma->vm_mm;
2206         int error = 0;
2207 
2208         if (!(vma->vm_flags & VM_GROWSUP))
2209                 return -EFAULT;
2210 
2211         /* Guard against wrapping around to address 0. */
2212         if (address < PAGE_ALIGN(address+4))
2213                 address = PAGE_ALIGN(address+4);
2214         else
2215                 return -ENOMEM;
2216 
2217         /* We must make sure the anon_vma is allocated. */
2218         if (unlikely(anon_vma_prepare(vma)))
2219                 return -ENOMEM;
2220 
2221         /*
2222          * vma->vm_start/vm_end cannot change under us because the caller
2223          * is required to hold the mmap_sem in read mode.  We need the
2224          * anon_vma lock to serialize against concurrent expand_stacks.
2225          */
2226         anon_vma_lock_write(vma->anon_vma);
2227 
2228         /* Somebody else might have raced and expanded it already */
2229         if (address > vma->vm_end) {
2230                 unsigned long size, grow;
2231 
2232                 size = address - vma->vm_start;
2233                 grow = (address - vma->vm_end) >> PAGE_SHIFT;
2234 
2235                 error = -ENOMEM;
2236                 if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
2237                         error = acct_stack_growth(vma, size, grow);
2238                         if (!error) {
2239                                 /*
2240                                  * vma_gap_update() doesn't support concurrent
2241                                  * updates, but we only hold a shared mmap_sem
2242                                  * lock here, so we need to protect against
2243                                  * concurrent vma expansions.
2244                                  * anon_vma_lock_write() doesn't help here, as
2245                                  * we don't guarantee that all growable vmas
2246                                  * in a mm share the same root anon vma.
2247                                  * So, we reuse mm->page_table_lock to guard
2248                                  * against concurrent vma expansions.
2249                                  */
2250                                 spin_lock(&mm->page_table_lock);
2251                                 if (vma->vm_flags & VM_LOCKED)
2252                                         mm->locked_vm += grow;
2253                                 vm_stat_account(mm, vma->vm_flags, grow);
2254                                 anon_vma_interval_tree_pre_update_vma(vma);
2255                                 vma->vm_end = address;
2256                                 anon_vma_interval_tree_post_update_vma(vma);
2257                                 if (vma->vm_next)
2258                                         vma_gap_update(vma->vm_next);
2259                                 else
2260                                         mm->highest_vm_end = address;
2261                                 spin_unlock(&mm->page_table_lock);
2262 
2263                                 perf_event_mmap(vma);
2264                         }
2265                 }
2266         }
2267         anon_vma_unlock_write(vma->anon_vma);
2268         khugepaged_enter_vma_merge(vma, vma->vm_flags);
2269         validate_mm(mm);
2270         return error;
2271 }
2272 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
2273 
2274 /*
2275  * vma is the first one with address < vma->vm_start.  Have to extend vma.
2276  */
2277 int expand_downwards(struct vm_area_struct *vma,
2278                                    unsigned long address)
2279 {
2280         struct mm_struct *mm = vma->vm_mm;
2281         int error;
2282 
2283         address &= PAGE_MASK;
2284         error = security_mmap_addr(address);
2285         if (error)
2286                 return error;
2287 
2288         /* We must make sure the anon_vma is allocated. */
2289         if (unlikely(anon_vma_prepare(vma)))
2290                 return -ENOMEM;
2291 
2292         /*
2293          * vma->vm_start/vm_end cannot change under us because the caller
2294          * is required to hold the mmap_sem in read mode.  We need the
2295          * anon_vma lock to serialize against concurrent expand_stacks.
2296          */
2297         anon_vma_lock_write(vma->anon_vma);
2298 
2299         /* Somebody else might have raced and expanded it already */
2300         if (address < vma->vm_start) {
2301                 unsigned long size, grow;
2302 
2303                 size = vma->vm_end - address;
2304                 grow = (vma->vm_start - address) >> PAGE_SHIFT;
2305 
2306                 error = -ENOMEM;
2307                 if (grow <= vma->vm_pgoff) {
2308                         error = acct_stack_growth(vma, size, grow);
2309                         if (!error) {
2310                                 /*
2311                                  * vma_gap_update() doesn't support concurrent
2312                                  * updates, but we only hold a shared mmap_sem
2313                                  * lock here, so we need to protect against
2314                                  * concurrent vma expansions.
2315                                  * anon_vma_lock_write() doesn't help here, as
2316                                  * we don't guarantee that all growable vmas
2317                                  * in a mm share the same root anon vma.
2318                                  * So, we reuse mm->page_table_lock to guard
2319                                  * against concurrent vma expansions.
2320                                  */
2321                                 spin_lock(&mm->page_table_lock);
2322                                 if (vma->vm_flags & VM_LOCKED)
2323                                         mm->locked_vm += grow;
2324                                 vm_stat_account(mm, vma->vm_flags, grow);
2325                                 anon_vma_interval_tree_pre_update_vma(vma);
2326                                 vma->vm_start = address;
2327                                 vma->vm_pgoff -= grow;
2328                                 anon_vma_interval_tree_post_update_vma(vma);
2329                                 vma_gap_update(vma);
2330                                 spin_unlock(&mm->page_table_lock);
2331 
2332                                 perf_event_mmap(vma);
2333                         }
2334                 }
2335         }
2336         anon_vma_unlock_write(vma->anon_vma);
2337         khugepaged_enter_vma_merge(vma, vma->vm_flags);
2338         validate_mm(mm);
2339         return error;
2340 }
2341 
2342 /*
2343  * Note how expand_stack() refuses to expand the stack all the way to
2344  * abut the next virtual mapping, *unless* that mapping itself is also
2345  * a stack mapping. We want to leave room for a guard page, after all
2346  * (the guard page itself is not added here, that is done by the
2347  * actual page faulting logic)
2348  *
2349  * This matches the behavior of the guard page logic (see mm/memory.c:
2350  * check_stack_guard_page()), which only allows the guard page to be
2351  * removed under these circumstances.
2352  */
2353 #ifdef CONFIG_STACK_GROWSUP
2354 int expand_stack(struct vm_area_struct *vma, unsigned long address)
2355 {
2356         struct vm_area_struct *next;
2357 
2358         address &= PAGE_MASK;
2359         next = vma->vm_next;
2360         if (next && next->vm_start == address + PAGE_SIZE) {
2361                 if (!(next->vm_flags & VM_GROWSUP))
2362                         return -ENOMEM;
2363         }
2364         return expand_upwards(vma, address);
2365 }
2366 
2367 struct vm_area_struct *
2368 find_extend_vma(struct mm_struct *mm, unsigned long addr)
2369 {
2370         struct vm_area_struct *vma, *prev;
2371 
2372         addr &= PAGE_MASK;
2373         vma = find_vma_prev(mm, addr, &prev);
2374         if (vma && (vma->vm_start <= addr))
2375                 return vma;
2376         if (!prev || expand_stack(prev, addr))
2377                 return NULL;
2378         if (prev->vm_flags & VM_LOCKED)
2379                 populate_vma_page_range(prev, addr, prev->vm_end, NULL);
2380         return prev;
2381 }
2382 #else
2383 int expand_stack(struct vm_area_struct *vma, unsigned long address)
2384 {
2385         struct vm_area_struct *prev;
2386 
2387         address &= PAGE_MASK;
2388         prev = vma->vm_prev;
2389         if (prev && prev->vm_end == address) {
2390                 if (!(prev->vm_flags & VM_GROWSDOWN))
2391                         return -ENOMEM;
2392         }
2393         return expand_downwards(vma, address);
2394 }
2395 
2396 struct vm_area_struct *
2397 find_extend_vma(struct mm_struct *mm, unsigned long addr)
2398 {
2399         struct vm_area_struct *vma;
2400         unsigned long start;
2401 
2402         addr &= PAGE_MASK;
2403         vma = find_vma(mm, addr);
2404         if (!vma)
2405                 return NULL;
2406         if (vma->vm_start <= addr)
2407                 return vma;
2408         if (!(vma->vm_flags & VM_GROWSDOWN))
2409                 return NULL;
2410         start = vma->vm_start;
2411         if (expand_stack(vma, addr))
2412                 return NULL;
2413         if (vma->vm_flags & VM_LOCKED)
2414                 populate_vma_page_range(vma, addr, start, NULL);
2415         return vma;
2416 }
2417 #endif
2418 
2419 EXPORT_SYMBOL_GPL(find_extend_vma);
2420 
2421 /*
2422  * Ok - we have the memory areas we should free on the vma list,
2423  * so release them, and do the vma updates.
2424  *
2425  * Called with the mm semaphore held.
2426  */
2427 static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
2428 {
2429         unsigned long nr_accounted = 0;
2430 
2431         /* Update high watermark before we lower total_vm */
2432         update_hiwater_vm(mm);
2433         do {
2434                 long nrpages = vma_pages(vma);
2435 
2436                 if (vma->vm_flags & VM_ACCOUNT)
2437                         nr_accounted += nrpages;
2438                 vm_stat_account(mm, vma->vm_flags, -nrpages);
2439                 vma = remove_vma(vma);
2440         } while (vma);
2441         vm_unacct_memory(nr_accounted);
2442         validate_mm(mm);
2443 }
2444 
2445 /*
2446  * Get rid of page table information in the indicated region.
2447  *
2448  * Called with the mm semaphore held.
2449  */
2450 static void unmap_region(struct mm_struct *mm,
2451                 struct vm_area_struct *vma, struct vm_area_struct *prev,
2452                 unsigned long start, unsigned long end)
2453 {
2454         struct vm_area_struct *next = prev ? prev->vm_next : mm->mmap;
2455         struct mmu_gather tlb;
2456 
2457         lru_add_drain();
2458         tlb_gather_mmu(&tlb, mm, start, end);
2459         update_hiwater_rss(mm);
2460         unmap_vmas(&tlb, vma, start, end);
2461         free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
2462                                  next ? next->vm_start : USER_PGTABLES_CEILING);
2463         tlb_finish_mmu(&tlb, start, end);
2464 }
2465 
2466 /*
2467  * Create a list of vma's touched by the unmap, removing them from the mm's
2468  * vma list as we go..
2469  */
2470 static void
2471 detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
2472         struct vm_area_struct *prev, unsigned long end)
2473 {
2474         struct vm_area_struct **insertion_point;
2475         struct vm_area_struct *tail_vma = NULL;
2476 
2477         insertion_point = (prev ? &prev->vm_next : &mm->mmap);
2478         vma->vm_prev = NULL;
2479         do {
2480                 vma_rb_erase(vma, &mm->mm_rb);
2481                 mm->map_count--;
2482                 tail_vma = vma;
2483                 vma = vma->vm_next;
2484         } while (vma && vma->vm_start < end);
2485         *insertion_point = vma;
2486         if (vma) {
2487                 vma->vm_prev = prev;
2488                 vma_gap_update(vma);
2489         } else
2490                 mm->highest_vm_end = prev ? prev->vm_end : 0;
2491         tail_vma->vm_next = NULL;
2492 
2493         /* Kill the cache */
2494         vmacache_invalidate(mm);
2495 }
2496 
2497 /*
2498  * __split_vma() bypasses sysctl_max_map_count checking.  We use this on the
2499  * munmap path where it doesn't make sense to fail.
2500  */
2501 static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2502               unsigned long addr, int new_below)
2503 {
2504         struct vm_area_struct *new;
2505         int err;
2506 
2507         if (is_vm_hugetlb_page(vma) && (addr &
2508                                         ~(huge_page_mask(hstate_vma(vma)))))
2509                 return -EINVAL;
2510 
2511         new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2512         if (!new)
2513                 return -ENOMEM;
2514 
2515         /* most fields are the same, copy all, and then fixup */
2516         *new = *vma;
2517 
2518         INIT_LIST_HEAD(&new->anon_vma_chain);
2519 
2520         if (new_below)
2521                 new->vm_end = addr;
2522         else {
2523                 new->vm_start = addr;
2524                 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
2525         }
2526 
2527         err = vma_dup_policy(vma, new);
2528         if (err)
2529                 goto out_free_vma;
2530 
2531         err = anon_vma_clone(new, vma);
2532         if (err)
2533                 goto out_free_mpol;
2534 
2535         if (new->vm_file)
2536                 get_file(new->vm_file);
2537 
2538         if (new->vm_ops && new->vm_ops->open)
2539                 new->vm_ops->open(new);
2540 
2541         if (new_below)
2542                 err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
2543                         ((addr - new->vm_start) >> PAGE_SHIFT), new);
2544         else
2545                 err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
2546 
2547         /* Success. */
2548         if (!err)
2549                 return 0;
2550 
2551         /* Clean everything up if vma_adjust failed. */
2552         if (new->vm_ops && new->vm_ops->close)
2553                 new->vm_ops->close(new);
2554         if (new->vm_file)
2555                 fput(new->vm_file);
2556         unlink_anon_vmas(new);
2557  out_free_mpol:
2558         mpol_put(vma_policy(new));
2559  out_free_vma:
2560         kmem_cache_free(vm_area_cachep, new);
2561         return err;
2562 }
2563 
2564 /*
2565  * Split a vma into two pieces at address 'addr', a new vma is allocated
2566  * either for the first part or the tail.
2567  */
2568 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2569               unsigned long addr, int new_below)
2570 {
2571         if (mm->map_count >= sysctl_max_map_count)
2572                 return -ENOMEM;
2573 
2574         return __split_vma(mm, vma, addr, new_below);
2575 }
2576 
2577 /* Munmap is split into 2 main parts -- this part which finds
2578  * what needs doing, and the areas themselves, which do the
2579  * work.  This now handles partial unmappings.
2580  * Jeremy Fitzhardinge <jeremy@goop.org>
2581  */
2582 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
2583 {
2584         unsigned long end;
2585         struct vm_area_struct *vma, *prev, *last;
2586 
2587         if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
2588                 return -EINVAL;
2589 
2590         len = PAGE_ALIGN(len);
2591         if (len == 0)
2592                 return -EINVAL;
2593 
2594         /* Find the first overlapping VMA */
2595         vma = find_vma(mm, start);
2596         if (!vma)
2597                 return 0;
2598         prev = vma->vm_prev;
2599         /* we have  start < vma->vm_end  */
2600 
2601         /* if it doesn't overlap, we have nothing.. */
2602         end = start + len;
2603         if (vma->vm_start >= end)
2604                 return 0;
2605 
2606         /*
2607          * If we need to split any vma, do it now to save pain later.
2608          *
2609          * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2610          * unmapped vm_area_struct will remain in use: so lower split_vma
2611          * places tmp vma above, and higher split_vma places tmp vma below.
2612          */
2613         if (start > vma->vm_start) {
2614                 int error;
2615 
2616                 /*
2617                  * Make sure that map_count on return from munmap() will
2618                  * not exceed its limit; but let map_count go just above
2619                  * its limit temporarily, to help free resources as expected.
2620                  */
2621                 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
2622                         return -ENOMEM;
2623 
2624                 error = __split_vma(mm, vma, start, 0);
2625                 if (error)
2626                         return error;
2627                 prev = vma;
2628         }
2629 
2630         /* Does it split the last one? */
2631         last = find_vma(mm, end);
2632         if (last && end > last->vm_start) {
2633                 int error = __split_vma(mm, last, end, 1);
2634                 if (error)
2635                         return error;
2636         }
2637         vma = prev ? prev->vm_next : mm->mmap;
2638 
2639         /*
2640          * unlock any mlock()ed ranges before detaching vmas
2641          */
2642         if (mm->locked_vm) {
2643                 struct vm_area_struct *tmp = vma;
2644                 while (tmp && tmp->vm_start < end) {
2645                         if (tmp->vm_flags & VM_LOCKED) {
2646                                 mm->locked_vm -= vma_pages(tmp);
2647                                 munlock_vma_pages_all(tmp);
2648                         }
2649                         tmp = tmp->vm_next;
2650                 }
2651         }
2652 
2653         /*
2654          * Remove the vma's, and unmap the actual pages
2655          */
2656         detach_vmas_to_be_unmapped(mm, vma, prev, end);
2657         unmap_region(mm, vma, prev, start, end);
2658 
2659         arch_unmap(mm, vma, start, end);
2660 
2661         /* Fix up all other VM information */
2662         remove_vma_list(mm, vma);
2663 
2664         return 0;
2665 }
2666 
2667 int vm_munmap(unsigned long start, size_t len)
2668 {
2669         int ret;
2670         struct mm_struct *mm = current->mm;
2671 
2672         if (down_write_killable(&mm->mmap_sem))
2673                 return -EINTR;
2674 
2675         ret = do_munmap(mm, start, len);
2676         up_write(&mm->mmap_sem);
2677         return ret;
2678 }
2679 EXPORT_SYMBOL(vm_munmap);
2680 
2681 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
2682 {
2683         int ret;
2684         struct mm_struct *mm = current->mm;
2685 
2686         profile_munmap(addr);
2687         if (down_write_killable(&mm->mmap_sem))
2688                 return -EINTR;
2689         ret = do_munmap(mm, addr, len);
2690         up_write(&mm->mmap_sem);
2691         return ret;
2692 }
2693 
2694 
2695 /*
2696  * Emulation of deprecated remap_file_pages() syscall.
2697  */
2698 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
2699                 unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
2700 {
2701 
2702         struct mm_struct *mm = current->mm;
2703         struct vm_area_struct *vma;
2704         unsigned long populate = 0;
2705         unsigned long ret = -EINVAL;
2706         struct file *file;
2707 
2708         pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n",
2709                      current->comm, current->pid);
2710 
2711         if (prot)
2712                 return ret;
2713         start = start & PAGE_MASK;
2714         size = size & PAGE_MASK;
2715 
2716         if (start + size <= start)
2717                 return ret;
2718 
2719         /* Does pgoff wrap? */
2720         if (pgoff + (size >> PAGE_SHIFT) < pgoff)
2721                 return ret;
2722 
2723         if (down_write_killable(&mm->mmap_sem))
2724                 return -EINTR;
2725 
2726         vma = find_vma(mm, start);
2727 
2728         if (!vma || !(vma->vm_flags & VM_SHARED))
2729                 goto out;
2730 
2731         if (start < vma->vm_start)
2732                 goto out;
2733 
2734         if (start + size > vma->vm_end) {
2735                 struct vm_area_struct *next;
2736 
2737                 for (next = vma->vm_next; next; next = next->vm_next) {
2738                         /* hole between vmas ? */
2739                         if (next->vm_start != next->vm_prev->vm_end)
2740                                 goto out;
2741 
2742                         if (next->vm_file != vma->vm_file)
2743                                 goto out;
2744 
2745                         if (next->vm_flags != vma->vm_flags)
2746                                 goto out;
2747 
2748                         if (start + size <= next->vm_end)
2749                                 break;
2750                 }
2751 
2752                 if (!next)
2753                         goto out;
2754         }
2755 
2756         prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
2757         prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
2758         prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
2759 
2760         flags &= MAP_NONBLOCK;
2761         flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
2762         if (vma->vm_flags & VM_LOCKED) {
2763                 struct vm_area_struct *tmp;
2764                 flags |= MAP_LOCKED;
2765 
2766                 /* drop PG_Mlocked flag for over-mapped range */
2767                 for (tmp = vma; tmp->vm_start >= start + size;
2768                                 tmp = tmp->vm_next) {
2769                         /*
2770                          * Split pmd and munlock page on the border
2771                          * of the range.
2772                          */
2773                         vma_adjust_trans_huge(tmp, start, start + size, 0);
2774 
2775                         munlock_vma_pages_range(tmp,
2776                                         max(tmp->vm_start, start),
2777                                         min(tmp->vm_end, start + size));
2778                 }
2779         }
2780 
2781         file = get_file(vma->vm_file);
2782         ret = do_mmap_pgoff(vma->vm_file, start, size,
2783                         prot, flags, pgoff, &populate);
2784         fput(file);
2785 out:
2786         up_write(&mm->mmap_sem);
2787         if (populate)
2788                 mm_populate(ret, populate);
2789         if (!IS_ERR_VALUE(ret))
2790                 ret = 0;
2791         return ret;
2792 }
2793 
2794 static inline void verify_mm_writelocked(struct mm_struct *mm)
2795 {
2796 #ifdef CONFIG_DEBUG_VM
2797         if (unlikely(down_read_trylock(&mm->mmap_sem))) {
2798                 WARN_ON(1);
2799                 up_read(&mm->mmap_sem);
2800         }
2801 #endif
2802 }
2803 
2804 /*
2805  *  this is really a simplified "do_mmap".  it only handles
2806  *  anonymous maps.  eventually we may be able to do some
2807  *  brk-specific accounting here.
2808  */
2809 static int do_brk(unsigned long addr, unsigned long request)
2810 {
2811         struct mm_struct *mm = current->mm;
2812         struct vm_area_struct *vma, *prev;
2813         unsigned long flags, len;
2814         struct rb_node **rb_link, *rb_parent;
2815         pgoff_t pgoff = addr >> PAGE_SHIFT;
2816         int error;
2817 
2818         len = PAGE_ALIGN(request);
2819         if (len < request)
2820                 return -ENOMEM;
2821         if (!len)
2822                 return 0;
2823 
2824         flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
2825 
2826         error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2827         if (offset_in_page(error))
2828                 return error;
2829 
2830         error = mlock_future_check(mm, mm->def_flags, len);
2831         if (error)
2832                 return error;
2833 
2834         /*
2835          * mm->mmap_sem is required to protect against another thread
2836          * changing the mappings in case we sleep.
2837          */
2838         verify_mm_writelocked(mm);
2839 
2840         /*
2841          * Clear old maps.  this also does some error checking for us
2842          */
2843         while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
2844                               &rb_parent)) {
2845                 if (do_munmap(mm, addr, len))
2846                         return -ENOMEM;
2847         }
2848 
2849         /* Check against address space limits *after* clearing old maps... */
2850         if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
2851                 return -ENOMEM;
2852 
2853         if (mm->map_count > sysctl_max_map_count)
2854                 return -ENOMEM;
2855 
2856         if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
2857                 return -ENOMEM;
2858 
2859         /* Can we just expand an old private anonymous mapping? */
2860         vma = vma_merge(mm, prev, addr, addr + len, flags,
2861                         NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
2862         if (vma)
2863                 goto out;
2864 
2865         /*
2866          * create a vma struct for an anonymous mapping
2867          */
2868         vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2869         if (!vma) {
2870                 vm_unacct_memory(len >> PAGE_SHIFT);
2871                 return -ENOMEM;
2872         }
2873 
2874         INIT_LIST_HEAD(&vma->anon_vma_chain);
2875         vma->vm_mm = mm;
2876         vma->vm_start = addr;
2877         vma->vm_end = addr + len;
2878         vma->vm_pgoff = pgoff;
2879         vma->vm_flags = flags;
2880         vma->vm_page_prot = vm_get_page_prot(flags);
2881         vma_link(mm, vma, prev, rb_link, rb_parent);
2882 out:
2883         perf_event_mmap(vma);
2884         mm->total_vm += len >> PAGE_SHIFT;
2885         mm->data_vm += len >> PAGE_SHIFT;
2886         if (flags & VM_LOCKED)
2887                 mm->locked_vm += (len >> PAGE_SHIFT);
2888         vma->vm_flags |= VM_SOFTDIRTY;
2889         return 0;
2890 }
2891 
2892 int vm_brk(unsigned long addr, unsigned long len)
2893 {
2894         struct mm_struct *mm = current->mm;
2895         int ret;
2896         bool populate;
2897 
2898         if (down_write_killable(&mm->mmap_sem))
2899                 return -EINTR;
2900 
2901         ret = do_brk(addr, len);
2902         populate = ((mm->def_flags & VM_LOCKED) != 0);
2903         up_write(&mm->mmap_sem);
2904         if (populate && !ret)
2905                 mm_populate(addr, len);
2906         return ret;
2907 }
2908 EXPORT_SYMBOL(vm_brk);
2909 
2910 /* Release all mmaps. */
2911 void exit_mmap(struct mm_struct *mm)
2912 {
2913         struct mmu_gather tlb;
2914         struct vm_area_struct *vma;
2915         unsigned long nr_accounted = 0;
2916 
2917         /* mm's last user has gone, and its about to be pulled down */
2918         mmu_notifier_release(mm);
2919 
2920         if (mm->locked_vm) {
2921                 vma = mm->mmap;
2922                 while (vma) {
2923                         if (vma->vm_flags & VM_LOCKED)
2924                                 munlock_vma_pages_all(vma);
2925                         vma = vma->vm_next;
2926                 }
2927         }
2928 
2929         arch_exit_mmap(mm);
2930 
2931         vma = mm->mmap;
2932         if (!vma)       /* Can happen if dup_mmap() received an OOM */
2933                 return;
2934 
2935         lru_add_drain();
2936         flush_cache_mm(mm);
2937         tlb_gather_mmu(&tlb, mm, 0, -1);
2938         /* update_hiwater_rss(mm) here? but nobody should be looking */
2939         /* Use -1 here to ensure all VMAs in the mm are unmapped */
2940         unmap_vmas(&tlb, vma, 0, -1);
2941 
2942         free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
2943         tlb_finish_mmu(&tlb, 0, -1);
2944 
2945         /*
2946          * Walk the list again, actually closing and freeing it,
2947          * with preemption enabled, without holding any MM locks.
2948          */
2949         while (vma) {
2950                 if (vma->vm_flags & VM_ACCOUNT)
2951                         nr_accounted += vma_pages(vma);
2952                 vma = remove_vma(vma);
2953         }
2954         vm_unacct_memory(nr_accounted);
2955 }
2956 
2957 /* Insert vm structure into process list sorted by address
2958  * and into the inode's i_mmap tree.  If vm_file is non-NULL
2959  * then i_mmap_rwsem is taken here.
2960  */
2961 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
2962 {
2963         struct vm_area_struct *prev;
2964         struct rb_node **rb_link, *rb_parent;
2965 
2966         if (find_vma_links(mm, vma->vm_start, vma->vm_end,
2967                            &prev, &rb_link, &rb_parent))
2968                 return -ENOMEM;
2969         if ((vma->vm_flags & VM_ACCOUNT) &&
2970              security_vm_enough_memory_mm(mm, vma_pages(vma)))
2971                 return -ENOMEM;
2972 
2973         /*
2974          * The vm_pgoff of a purely anonymous vma should be irrelevant
2975          * until its first write fault, when page's anon_vma and index
2976          * are set.  But now set the vm_pgoff it will almost certainly
2977          * end up with (unless mremap moves it elsewhere before that
2978          * first wfault), so /proc/pid/maps tells a consistent story.
2979          *
2980          * By setting it to reflect the virtual start address of the
2981          * vma, merges and splits can happen in a seamless way, just
2982          * using the existing file pgoff checks and manipulations.
2983          * Similarly in do_mmap_pgoff and in do_brk.
2984          */
2985         if (vma_is_anonymous(vma)) {
2986                 BUG_ON(vma->anon_vma);
2987                 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2988         }
2989 
2990         vma_link(mm, vma, prev, rb_link, rb_parent);
2991         return 0;
2992 }
2993 
2994 /*
2995  * Copy the vma structure to a new location in the same mm,
2996  * prior to moving page table entries, to effect an mremap move.
2997  */
2998 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
2999         unsigned long addr, unsigned long len, pgoff_t pgoff,
3000         bool *need_rmap_locks)
3001 {
3002         struct vm_area_struct *vma = *vmap;
3003         unsigned long vma_start = vma->vm_start;
3004         struct mm_struct *mm = vma->vm_mm;
3005         struct vm_area_struct *new_vma, *prev;
3006         struct rb_node **rb_link, *rb_parent;
3007         bool faulted_in_anon_vma = true;
3008 
3009         /*
3010          * If anonymous vma has not yet been faulted, update new pgoff
3011          * to match new location, to increase its chance of merging.
3012          */
3013         if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
3014                 pgoff = addr >> PAGE_SHIFT;
3015                 faulted_in_anon_vma = false;
3016         }
3017 
3018         if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
3019                 return NULL;    /* should never get here */
3020         new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
3021                             vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
3022                             vma->vm_userfaultfd_ctx);
3023         if (new_vma) {
3024                 /*
3025                  * Source vma may have been merged into new_vma
3026                  */
3027                 if (unlikely(vma_start >= new_vma->vm_start &&
3028                              vma_start < new_vma->vm_end)) {
3029                         /*
3030                          * The only way we can get a vma_merge with
3031                          * self during an mremap is if the vma hasn't
3032                          * been faulted in yet and we were allowed to
3033                          * reset the dst vma->vm_pgoff to the
3034                          * destination address of the mremap to allow
3035                          * the merge to happen. mremap must change the
3036                          * vm_pgoff linearity between src and dst vmas
3037                          * (in turn preventing a vma_merge) to be
3038                          * safe. It is only safe to keep the vm_pgoff
3039                          * linear if there are no pages mapped yet.
3040                          */
3041                         VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
3042                         *vmap = vma = new_vma;
3043                 }
3044                 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
3045         } else {
3046                 new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
3047                 if (!new_vma)
3048                         goto out;
3049                 *new_vma = *vma;
3050                 new_vma->vm_start = addr;
3051                 new_vma->vm_end = addr + len;
3052                 new_vma->vm_pgoff = pgoff;
3053                 if (vma_dup_policy(vma, new_vma))
3054                         goto out_free_vma;
3055                 INIT_LIST_HEAD(&new_vma->anon_vma_chain);
3056                 if (anon_vma_clone(new_vma, vma))
3057                         goto out_free_mempol;
3058                 if (new_vma->vm_file)
3059                         get_file(new_vma->vm_file);
3060                 if (new_vma->vm_ops && new_vma->vm_ops->open)
3061                         new_vma->vm_ops->open(new_vma);
3062                 vma_link(mm, new_vma, prev, rb_link, rb_parent);
3063                 *need_rmap_locks = false;
3064         }
3065         return new_vma;
3066 
3067 out_free_mempol:
3068         mpol_put(vma_policy(new_vma));
3069 out_free_vma:
3070         kmem_cache_free(vm_area_cachep, new_vma);
3071 out:
3072         return NULL;
3073 }
3074 
3075 /*
3076  * Return true if the calling process may expand its vm space by the passed
3077  * number of pages
3078  */
3079 bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
3080 {
3081         if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
3082                 return false;
3083 
3084         if (is_data_mapping(flags) &&
3085             mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
3086                 /* Workaround for Valgrind */
3087                 if (rlimit(RLIMIT_DATA) == 0 &&
3088                     mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
3089                         return true;
3090                 if (!ignore_rlimit_data) {
3091                         pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits or use boot option ignore_rlimit_data.\n",
3092                                      current->comm, current->pid,
3093                                      (mm->data_vm + npages) << PAGE_SHIFT,
3094                                      rlimit(RLIMIT_DATA));
3095                         return false;
3096                 }
3097         }
3098 
3099         return true;
3100 }
3101 
3102 void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
3103 {
3104         mm->total_vm += npages;
3105 
3106         if (is_exec_mapping(flags))
3107                 mm->exec_vm += npages;
3108         else if (is_stack_mapping(flags))
3109                 mm->stack_vm += npages;
3110         else if (is_data_mapping(flags))
3111                 mm->data_vm += npages;
3112 }
3113 
3114 static int special_mapping_fault(struct vm_area_struct *vma,
3115                                  struct vm_fault *vmf);
3116 
3117 /*
3118  * Having a close hook prevents vma merging regardless of flags.
3119  */
3120 static void special_mapping_close(struct vm_area_struct *vma)
3121 {
3122 }
3123 
3124 static const char *special_mapping_name(struct vm_area_struct *vma)
3125 {
3126         return ((struct vm_special_mapping *)vma->vm_private_data)->name;
3127 }
3128 
3129 static int special_mapping_mremap(struct vm_area_struct *new_vma)
3130 {
3131         struct vm_special_mapping *sm = new_vma->vm_private_data;
3132 
3133         if (sm->mremap)
3134                 return sm->mremap(sm, new_vma);
3135         return 0;
3136 }
3137 
3138 static const struct vm_operations_struct special_mapping_vmops = {
3139         .close = special_mapping_close,
3140         .fault = special_mapping_fault,
3141         .mremap = special_mapping_mremap,
3142         .name = special_mapping_name,
3143 };
3144 
3145 static const struct vm_operations_struct legacy_special_mapping_vmops = {
3146         .close = special_mapping_close,
3147         .fault = special_mapping_fault,
3148 };
3149 
3150 static int special_mapping_fault(struct vm_area_struct *vma,
3151                                 struct vm_fault *vmf)
3152 {
3153         pgoff_t pgoff;
3154         struct page **pages;
3155 
3156         if (vma->vm_ops == &legacy_special_mapping_vmops) {
3157                 pages = vma->vm_private_data;
3158         } else {
3159                 struct vm_special_mapping *sm = vma->vm_private_data;
3160 
3161                 if (sm->fault)
3162                         return sm->fault(sm, vma, vmf);
3163 
3164                 pages = sm->pages;
3165         }
3166 
3167         for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
3168                 pgoff--;
3169 
3170         if (*pages) {
3171                 struct page *page = *pages;
3172                 get_page(page);
3173                 vmf->page = page;
3174                 return 0;
3175         }
3176 
3177         return VM_FAULT_SIGBUS;
3178 }
3179 
3180 static struct vm_area_struct *__install_special_mapping(
3181         struct mm_struct *mm,
3182         unsigned long addr, unsigned long len,
3183         unsigned long vm_flags, void *priv,
3184         const struct vm_operations_struct *ops)
3185 {
3186         int ret;
3187         struct vm_area_struct *vma;
3188 
3189         vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
3190         if (unlikely(vma == NULL))
3191                 return ERR_PTR(-ENOMEM);
3192 
3193         INIT_LIST_HEAD(&vma->anon_vma_chain);
3194         vma->vm_mm = mm;
3195         vma->vm_start = addr;
3196         vma->vm_end = addr + len;
3197 
3198         vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
3199         vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
3200 
3201         vma->vm_ops = ops;
3202         vma->vm_private_data = priv;
3203 
3204         ret = insert_vm_struct(mm, vma);
3205         if (ret)
3206                 goto out;
3207 
3208         vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
3209 
3210         perf_event_mmap(vma);
3211 
3212         return vma;
3213 
3214 out:
3215         kmem_cache_free(vm_area_cachep, vma);
3216         return ERR_PTR(ret);
3217 }
3218 
3219 bool vma_is_special_mapping(const struct vm_area_struct *vma,
3220         const struct vm_special_mapping *sm)
3221 {
3222         return vma->vm_private_data == sm &&
3223                 (vma->vm_ops == &special_mapping_vmops ||
3224                  vma->vm_ops == &legacy_special_mapping_vmops);
3225 }
3226 
3227 /*
3228  * Called with mm->mmap_sem held for writing.
3229  * Insert a new vma covering the given region, with the given flags.
3230  * Its pages are supplied by the given array of struct page *.
3231  * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
3232  * The region past the last page supplied will always produce SIGBUS.
3233  * The array pointer and the pages it points to are assumed to stay alive
3234  * for as long as this mapping might exist.
3235  */
3236 struct vm_area_struct *_install_special_mapping(
3237         struct mm_struct *mm,
3238         unsigned long addr, unsigned long len,
3239         unsigned long vm_flags, const struct vm_special_mapping *spec)
3240 {
3241         return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
3242                                         &special_mapping_vmops);
3243 }
3244 
3245 int install_special_mapping(struct mm_struct *mm,
3246                             unsigned long addr, unsigned long len,
3247                             unsigned long vm_flags, struct page **pages)
3248 {
3249         struct vm_area_struct *vma = __install_special_mapping(
3250                 mm, addr, len, vm_flags, (void *)pages,
3251                 &legacy_special_mapping_vmops);
3252 
3253         return PTR_ERR_OR_ZERO(vma);
3254 }
3255 
3256 static DEFINE_MUTEX(mm_all_locks_mutex);
3257 
3258 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
3259 {
3260         if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3261                 /*
3262                  * The LSB of head.next can't change from under us
3263                  * because we hold the mm_all_locks_mutex.
3264                  */
3265                 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
3266                 /*
3267                  * We can safely modify head.next after taking the
3268                  * anon_vma->root->rwsem. If some other vma in this mm shares
3269                  * the same anon_vma we won't take it again.
3270                  *
3271                  * No need of atomic instructions here, head.next
3272                  * can't change from under us thanks to the
3273                  * anon_vma->root->rwsem.
3274                  */
3275                 if (__test_and_set_bit(0, (unsigned long *)
3276                                        &anon_vma->root->rb_root.rb_node))
3277                         BUG();
3278         }
3279 }
3280 
3281 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
3282 {
3283         if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3284                 /*
3285                  * AS_MM_ALL_LOCKS can't change from under us because
3286                  * we hold the mm_all_locks_mutex.
3287                  *
3288                  * Operations on ->flags have to be atomic because
3289                  * even if AS_MM_ALL_LOCKS is stable thanks to the
3290                  * mm_all_locks_mutex, there may be other cpus
3291                  * changing other bitflags in parallel to us.
3292                  */
3293                 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
3294                         BUG();
3295                 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem);
3296         }
3297 }
3298 
3299 /*
3300  * This operation locks against the VM for all pte/vma/mm related
3301  * operations that could ever happen on a certain mm. This includes
3302  * vmtruncate, try_to_unmap, and all page faults.
3303  *
3304  * The caller must take the mmap_sem in write mode before calling
3305  * mm_take_all_locks(). The caller isn't allowed to release the
3306  * mmap_sem until mm_drop_all_locks() returns.
3307  *
3308  * mmap_sem in write mode is required in order to block all operations
3309  * that could modify pagetables and free pages without need of
3310  * altering the vma layout. It's also needed in write mode to avoid new
3311  * anon_vmas to be associated with existing vmas.
3312  *
3313  * A single task can't take more than one mm_take_all_locks() in a row
3314  * or it would deadlock.
3315  *
3316  * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3317  * mapping->flags avoid to take the same lock twice, if more than one
3318  * vma in this mm is backed by the same anon_vma or address_space.
3319  *
3320  * We take locks in following order, accordingly to comment at beginning
3321  * of mm/rmap.c:
3322  *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
3323  *     hugetlb mapping);
3324  *   - all i_mmap_rwsem locks;
3325  *   - all anon_vma->rwseml
3326  *
3327  * We can take all locks within these types randomly because the VM code
3328  * doesn't nest them and we protected from parallel mm_take_all_locks() by
3329  * mm_all_locks_mutex.
3330  *
3331  * mm_take_all_locks() and mm_drop_all_locks are expensive operations
3332  * that may have to take thousand of locks.
3333  *
3334  * mm_take_all_locks() can fail if it's interrupted by signals.
3335  */
3336 int mm_take_all_locks(struct mm_struct *mm)
3337 {
3338         struct vm_area_struct *vma;
3339         struct anon_vma_chain *avc;
3340 
3341         BUG_ON(down_read_trylock(&mm->mmap_sem));
3342 
3343         mutex_lock(&mm_all_locks_mutex);
3344 
3345         for (vma = mm->mmap; vma; vma = vma->vm_next) {
3346                 if (signal_pending(current))
3347                         goto out_unlock;
3348                 if (vma->vm_file && vma->vm_file->f_mapping &&
3349                                 is_vm_hugetlb_page(vma))
3350                         vm_lock_mapping(mm, vma->vm_file->f_mapping);
3351         }
3352 
3353         for (vma = mm->mmap; vma; vma = vma->vm_next) {
3354                 if (signal_pending(current))
3355                         goto out_unlock;
3356                 if (vma->vm_file && vma->vm_file->f_mapping &&
3357                                 !is_vm_hugetlb_page(vma))
3358                         vm_lock_mapping(mm, vma->vm_file->f_mapping);
3359         }
3360 
3361         for (vma = mm->mmap; vma; vma = vma->vm_next) {
3362                 if (signal_pending(current))
3363                         goto out_unlock;
3364                 if (vma->anon_vma)
3365                         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3366                                 vm_lock_anon_vma(mm, avc->anon_vma);
3367         }
3368 
3369         return 0;
3370 
3371 out_unlock:
3372         mm_drop_all_locks(mm);
3373         return -EINTR;
3374 }
3375 
3376 static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
3377 {
3378         if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3379                 /*
3380                  * The LSB of head.next can't change to 0 from under
3381                  * us because we hold the mm_all_locks_mutex.
3382                  *
3383                  * We must however clear the bitflag before unlocking
3384                  * the vma so the users using the anon_vma->rb_root will
3385                  * never see our bitflag.
3386                  *
3387                  * No need of atomic instructions here, head.next
3388                  * can't change from under us until we release the
3389                  * anon_vma->root->rwsem.
3390                  */
3391                 if (!__test_and_clear_bit(0, (unsigned long *)
3392                                           &anon_vma->root->rb_root.rb_node))
3393                         BUG();
3394                 anon_vma_unlock_write(anon_vma);
3395         }
3396 }
3397 
3398 static void vm_unlock_mapping(struct address_space *mapping)
3399 {
3400         if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3401                 /*
3402                  * AS_MM_ALL_LOCKS can't change to 0 from under us
3403                  * because we hold the mm_all_locks_mutex.
3404                  */
3405                 i_mmap_unlock_write(mapping);
3406                 if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
3407                                         &mapping->flags))
3408                         BUG();
3409         }
3410 }
3411 
3412 /*
3413  * The mmap_sem cannot be released by the caller until
3414  * mm_drop_all_locks() returns.
3415  */
3416 void mm_drop_all_locks(struct mm_struct *mm)
3417 {
3418         struct vm_area_struct *vma;
3419         struct anon_vma_chain *avc;
3420 
3421         BUG_ON(down_read_trylock(&mm->mmap_sem));
3422         BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
3423 
3424         for (vma = mm->mmap; vma; vma = vma->vm_next) {
3425                 if (vma->anon_vma)
3426                         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3427                                 vm_unlock_anon_vma(avc->anon_vma);
3428                 if (vma->vm_file && vma->vm_file->f_mapping)
3429                         vm_unlock_mapping(vma->vm_file->f_mapping);
3430         }
3431 
3432         mutex_unlock(&mm_all_locks_mutex);
3433 }
3434 
3435 /*
3436  * initialise the VMA slab
3437  */
3438 void __init mmap_init(void)
3439 {
3440         int ret;
3441 
3442         ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
3443         VM_BUG_ON(ret);
3444 }
3445 
3446 /*
3447  * Initialise sysctl_user_reserve_kbytes.
3448  *
3449  * This is intended to prevent a user from starting a single memory hogging
3450  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3451  * mode.
3452  *
3453  * The default value is min(3% of free memory, 128MB)
3454  * 128MB is enough to recover with sshd/login, bash, and top/kill.
3455  */
3456 static int init_user_reserve(void)
3457 {
3458         unsigned long free_kbytes;
3459 
3460         free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3461 
3462         sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
3463         return 0;
3464 }
3465 subsys_initcall(init_user_reserve);
3466 
3467 /*
3468  * Initialise sysctl_admin_reserve_kbytes.
3469  *
3470  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3471  * to log in and kill a memory hogging process.
3472  *
3473  * Systems with more than 256MB will reserve 8MB, enough to recover
3474  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3475  * only reserve 3% of free pages by default.
3476  */
3477 static int init_admin_reserve(void)
3478 {
3479         unsigned long free_kbytes;
3480 
3481         free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3482 
3483         sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
3484         return 0;
3485 }
3486 subsys_initcall(init_admin_reserve);
3487 
3488 /*
3489  * Reinititalise user and admin reserves if memory is added or removed.
3490  *
3491  * The default user reserve max is 128MB, and the default max for the
3492  * admin reserve is 8MB. These are usually, but not always, enough to
3493  * enable recovery from a memory hogging process using login/sshd, a shell,
3494  * and tools like top. It may make sense to increase or even disable the
3495  * reserve depending on the existence of swap or variations in the recovery
3496  * tools. So, the admin may have changed them.
3497  *
3498  * If memory is added and the reserves have been eliminated or increased above
3499  * the default max, then we'll trust the admin.
3500  *
3501  * If memory is removed and there isn't enough free memory, then we
3502  * need to reset the reserves.
3503  *
3504  * Otherwise keep the reserve set by the admin.
3505  */
3506 static int reserve_mem_notifier(struct notifier_block *nb,
3507                              unsigned long action, void *data)
3508 {
3509         unsigned long tmp, free_kbytes;
3510 
3511         switch (action) {
3512         case MEM_ONLINE:
3513                 /* Default max is 128MB. Leave alone if modified by operator. */
3514                 tmp = sysctl_user_reserve_kbytes;
3515                 if (0 < tmp && tmp < (1UL << 17))
3516                         init_user_reserve();
3517 
3518                 /* Default max is 8MB.  Leave alone if modified by operator. */
3519                 tmp = sysctl_admin_reserve_kbytes;
3520                 if (0 < tmp && tmp < (1UL << 13))
3521                         init_admin_reserve();
3522 
3523                 break;
3524         case MEM_OFFLINE:
3525                 free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3526 
3527                 if (sysctl_user_reserve_kbytes > free_kbytes) {
3528                         init_user_reserve();
3529                         pr_info("vm.user_reserve_kbytes reset to %lu\n",
3530                                 sysctl_user_reserve_kbytes);
3531                 }
3532 
3533                 if (sysctl_admin_reserve_kbytes > free_kbytes) {
3534                         init_admin_reserve();
3535                         pr_info("vm.admin_reserve_kbytes reset to %lu\n",
3536                                 sysctl_admin_reserve_kbytes);
3537                 }
3538                 break;
3539         default:
3540                 break;
3541         }
3542         return NOTIFY_OK;
3543 }
3544 
3545 static struct notifier_block reserve_mem_nb = {
3546         .notifier_call = reserve_mem_notifier,
3547 };
3548 
3549 static int __meminit init_reserve_notifier(void)
3550 {
3551         if (register_hotmemory_notifier(&reserve_mem_nb))
3552                 pr_err("Failed registering memory add/remove notifier for admin reserve\n");
3553 
3554         return 0;
3555 }
3556 subsys_initcall(init_reserve_notifier);
3557 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us