Version:  2.0.40 2.2.26 2.4.37 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9

Linux/kernel/fork.c

  1 /*
  2  *  linux/kernel/fork.c
  3  *
  4  *  Copyright (C) 1991, 1992  Linus Torvalds
  5  */
  6 
  7 /*
  8  *  'fork.c' contains the help-routines for the 'fork' system call
  9  * (see also entry.S and others).
 10  * Fork is rather simple, once you get the hang of it, but the memory
 11  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
 12  */
 13 
 14 #include <linux/slab.h>
 15 #include <linux/init.h>
 16 #include <linux/unistd.h>
 17 #include <linux/module.h>
 18 #include <linux/vmalloc.h>
 19 #include <linux/completion.h>
 20 #include <linux/personality.h>
 21 #include <linux/mempolicy.h>
 22 #include <linux/sem.h>
 23 #include <linux/file.h>
 24 #include <linux/fdtable.h>
 25 #include <linux/iocontext.h>
 26 #include <linux/key.h>
 27 #include <linux/binfmts.h>
 28 #include <linux/mman.h>
 29 #include <linux/mmu_notifier.h>
 30 #include <linux/fs.h>
 31 #include <linux/mm.h>
 32 #include <linux/vmacache.h>
 33 #include <linux/nsproxy.h>
 34 #include <linux/capability.h>
 35 #include <linux/cpu.h>
 36 #include <linux/cgroup.h>
 37 #include <linux/security.h>
 38 #include <linux/hugetlb.h>
 39 #include <linux/seccomp.h>
 40 #include <linux/swap.h>
 41 #include <linux/syscalls.h>
 42 #include <linux/jiffies.h>
 43 #include <linux/futex.h>
 44 #include <linux/compat.h>
 45 #include <linux/kthread.h>
 46 #include <linux/task_io_accounting_ops.h>
 47 #include <linux/rcupdate.h>
 48 #include <linux/ptrace.h>
 49 #include <linux/mount.h>
 50 #include <linux/audit.h>
 51 #include <linux/memcontrol.h>
 52 #include <linux/ftrace.h>
 53 #include <linux/proc_fs.h>
 54 #include <linux/profile.h>
 55 #include <linux/rmap.h>
 56 #include <linux/ksm.h>
 57 #include <linux/acct.h>
 58 #include <linux/tsacct_kern.h>
 59 #include <linux/cn_proc.h>
 60 #include <linux/freezer.h>
 61 #include <linux/delayacct.h>
 62 #include <linux/taskstats_kern.h>
 63 #include <linux/random.h>
 64 #include <linux/tty.h>
 65 #include <linux/blkdev.h>
 66 #include <linux/fs_struct.h>
 67 #include <linux/magic.h>
 68 #include <linux/perf_event.h>
 69 #include <linux/posix-timers.h>
 70 #include <linux/user-return-notifier.h>
 71 #include <linux/oom.h>
 72 #include <linux/khugepaged.h>
 73 #include <linux/signalfd.h>
 74 #include <linux/uprobes.h>
 75 #include <linux/aio.h>
 76 #include <linux/compiler.h>
 77 #include <linux/sysctl.h>
 78 #include <linux/kcov.h>
 79 
 80 #include <asm/pgtable.h>
 81 #include <asm/pgalloc.h>
 82 #include <asm/uaccess.h>
 83 #include <asm/mmu_context.h>
 84 #include <asm/cacheflush.h>
 85 #include <asm/tlbflush.h>
 86 
 87 #include <trace/events/sched.h>
 88 
 89 #define CREATE_TRACE_POINTS
 90 #include <trace/events/task.h>
 91 
 92 /*
 93  * Minimum number of threads to boot the kernel
 94  */
 95 #define MIN_THREADS 20
 96 
 97 /*
 98  * Maximum number of threads
 99  */
100 #define MAX_THREADS FUTEX_TID_MASK
101 
102 /*
103  * Protected counters by write_lock_irq(&tasklist_lock)
104  */
105 unsigned long total_forks;      /* Handle normal Linux uptimes. */
106 int nr_threads;                 /* The idle threads do not count.. */
107 
108 int max_threads;                /* tunable limit on nr_threads */
109 
110 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
111 
112 __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
113 
114 #ifdef CONFIG_PROVE_RCU
115 int lockdep_tasklist_lock_is_held(void)
116 {
117         return lockdep_is_held(&tasklist_lock);
118 }
119 EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
120 #endif /* #ifdef CONFIG_PROVE_RCU */
121 
122 int nr_processes(void)
123 {
124         int cpu;
125         int total = 0;
126 
127         for_each_possible_cpu(cpu)
128                 total += per_cpu(process_counts, cpu);
129 
130         return total;
131 }
132 
133 void __weak arch_release_task_struct(struct task_struct *tsk)
134 {
135 }
136 
137 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
138 static struct kmem_cache *task_struct_cachep;
139 
140 static inline struct task_struct *alloc_task_struct_node(int node)
141 {
142         return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
143 }
144 
145 static inline void free_task_struct(struct task_struct *tsk)
146 {
147         kmem_cache_free(task_struct_cachep, tsk);
148 }
149 #endif
150 
151 void __weak arch_release_thread_stack(unsigned long *stack)
152 {
153 }
154 
155 #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
156 
157 /*
158  * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
159  * kmemcache based allocator.
160  */
161 # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
162 
163 #ifdef CONFIG_VMAP_STACK
164 /*
165  * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
166  * flush.  Try to minimize the number of calls by caching stacks.
167  */
168 #define NR_CACHED_STACKS 2
169 static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
170 #endif
171 
172 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
173 {
174 #ifdef CONFIG_VMAP_STACK
175         void *stack;
176         int i;
177 
178         local_irq_disable();
179         for (i = 0; i < NR_CACHED_STACKS; i++) {
180                 struct vm_struct *s = this_cpu_read(cached_stacks[i]);
181 
182                 if (!s)
183                         continue;
184                 this_cpu_write(cached_stacks[i], NULL);
185 
186                 tsk->stack_vm_area = s;
187                 local_irq_enable();
188                 return s->addr;
189         }
190         local_irq_enable();
191 
192         stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
193                                      VMALLOC_START, VMALLOC_END,
194                                      THREADINFO_GFP | __GFP_HIGHMEM,
195                                      PAGE_KERNEL,
196                                      0, node, __builtin_return_address(0));
197 
198         /*
199          * We can't call find_vm_area() in interrupt context, and
200          * free_thread_stack() can be called in interrupt context,
201          * so cache the vm_struct.
202          */
203         if (stack)
204                 tsk->stack_vm_area = find_vm_area(stack);
205         return stack;
206 #else
207         struct page *page = alloc_pages_node(node, THREADINFO_GFP,
208                                              THREAD_SIZE_ORDER);
209 
210         return page ? page_address(page) : NULL;
211 #endif
212 }
213 
214 static inline void free_thread_stack(struct task_struct *tsk)
215 {
216 #ifdef CONFIG_VMAP_STACK
217         if (task_stack_vm_area(tsk)) {
218                 unsigned long flags;
219                 int i;
220 
221                 local_irq_save(flags);
222                 for (i = 0; i < NR_CACHED_STACKS; i++) {
223                         if (this_cpu_read(cached_stacks[i]))
224                                 continue;
225 
226                         this_cpu_write(cached_stacks[i], tsk->stack_vm_area);
227                         local_irq_restore(flags);
228                         return;
229                 }
230                 local_irq_restore(flags);
231 
232                 vfree(tsk->stack);
233                 return;
234         }
235 #endif
236 
237         __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
238 }
239 # else
240 static struct kmem_cache *thread_stack_cache;
241 
242 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
243                                                   int node)
244 {
245         return kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
246 }
247 
248 static void free_thread_stack(struct task_struct *tsk)
249 {
250         kmem_cache_free(thread_stack_cache, tsk->stack);
251 }
252 
253 void thread_stack_cache_init(void)
254 {
255         thread_stack_cache = kmem_cache_create("thread_stack", THREAD_SIZE,
256                                               THREAD_SIZE, 0, NULL);
257         BUG_ON(thread_stack_cache == NULL);
258 }
259 # endif
260 #endif
261 
262 /* SLAB cache for signal_struct structures (tsk->signal) */
263 static struct kmem_cache *signal_cachep;
264 
265 /* SLAB cache for sighand_struct structures (tsk->sighand) */
266 struct kmem_cache *sighand_cachep;
267 
268 /* SLAB cache for files_struct structures (tsk->files) */
269 struct kmem_cache *files_cachep;
270 
271 /* SLAB cache for fs_struct structures (tsk->fs) */
272 struct kmem_cache *fs_cachep;
273 
274 /* SLAB cache for vm_area_struct structures */
275 struct kmem_cache *vm_area_cachep;
276 
277 /* SLAB cache for mm_struct structures (tsk->mm) */
278 static struct kmem_cache *mm_cachep;
279 
280 static void account_kernel_stack(struct task_struct *tsk, int account)
281 {
282         void *stack = task_stack_page(tsk);
283         struct vm_struct *vm = task_stack_vm_area(tsk);
284 
285         BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
286 
287         if (vm) {
288                 int i;
289 
290                 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
291 
292                 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
293                         mod_zone_page_state(page_zone(vm->pages[i]),
294                                             NR_KERNEL_STACK_KB,
295                                             PAGE_SIZE / 1024 * account);
296                 }
297 
298                 /* All stack pages belong to the same memcg. */
299                 memcg_kmem_update_page_stat(vm->pages[0], MEMCG_KERNEL_STACK_KB,
300                                             account * (THREAD_SIZE / 1024));
301         } else {
302                 /*
303                  * All stack pages are in the same zone and belong to the
304                  * same memcg.
305                  */
306                 struct page *first_page = virt_to_page(stack);
307 
308                 mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB,
309                                     THREAD_SIZE / 1024 * account);
310 
311                 memcg_kmem_update_page_stat(first_page, MEMCG_KERNEL_STACK_KB,
312                                             account * (THREAD_SIZE / 1024));
313         }
314 }
315 
316 static void release_task_stack(struct task_struct *tsk)
317 {
318         if (WARN_ON(tsk->state != TASK_DEAD))
319                 return;  /* Better to leak the stack than to free prematurely */
320 
321         account_kernel_stack(tsk, -1);
322         arch_release_thread_stack(tsk->stack);
323         free_thread_stack(tsk);
324         tsk->stack = NULL;
325 #ifdef CONFIG_VMAP_STACK
326         tsk->stack_vm_area = NULL;
327 #endif
328 }
329 
330 #ifdef CONFIG_THREAD_INFO_IN_TASK
331 void put_task_stack(struct task_struct *tsk)
332 {
333         if (atomic_dec_and_test(&tsk->stack_refcount))
334                 release_task_stack(tsk);
335 }
336 #endif
337 
338 void free_task(struct task_struct *tsk)
339 {
340 #ifndef CONFIG_THREAD_INFO_IN_TASK
341         /*
342          * The task is finally done with both the stack and thread_info,
343          * so free both.
344          */
345         release_task_stack(tsk);
346 #else
347         /*
348          * If the task had a separate stack allocation, it should be gone
349          * by now.
350          */
351         WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0);
352 #endif
353         rt_mutex_debug_task_free(tsk);
354         ftrace_graph_exit_task(tsk);
355         put_seccomp_filter(tsk);
356         arch_release_task_struct(tsk);
357         free_task_struct(tsk);
358 }
359 EXPORT_SYMBOL(free_task);
360 
361 static inline void free_signal_struct(struct signal_struct *sig)
362 {
363         taskstats_tgid_free(sig);
364         sched_autogroup_exit(sig);
365         /*
366          * __mmdrop is not safe to call from softirq context on x86 due to
367          * pgd_dtor so postpone it to the async context
368          */
369         if (sig->oom_mm)
370                 mmdrop_async(sig->oom_mm);
371         kmem_cache_free(signal_cachep, sig);
372 }
373 
374 static inline void put_signal_struct(struct signal_struct *sig)
375 {
376         if (atomic_dec_and_test(&sig->sigcnt))
377                 free_signal_struct(sig);
378 }
379 
380 void __put_task_struct(struct task_struct *tsk)
381 {
382         WARN_ON(!tsk->exit_state);
383         WARN_ON(atomic_read(&tsk->usage));
384         WARN_ON(tsk == current);
385 
386         cgroup_free(tsk);
387         task_numa_free(tsk);
388         security_task_free(tsk);
389         exit_creds(tsk);
390         delayacct_tsk_free(tsk);
391         put_signal_struct(tsk->signal);
392 
393         if (!profile_handoff_task(tsk))
394                 free_task(tsk);
395 }
396 EXPORT_SYMBOL_GPL(__put_task_struct);
397 
398 void __init __weak arch_task_cache_init(void) { }
399 
400 /*
401  * set_max_threads
402  */
403 static void set_max_threads(unsigned int max_threads_suggested)
404 {
405         u64 threads;
406 
407         /*
408          * The number of threads shall be limited such that the thread
409          * structures may only consume a small part of the available memory.
410          */
411         if (fls64(totalram_pages) + fls64(PAGE_SIZE) > 64)
412                 threads = MAX_THREADS;
413         else
414                 threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE,
415                                     (u64) THREAD_SIZE * 8UL);
416 
417         if (threads > max_threads_suggested)
418                 threads = max_threads_suggested;
419 
420         max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
421 }
422 
423 #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
424 /* Initialized by the architecture: */
425 int arch_task_struct_size __read_mostly;
426 #endif
427 
428 void __init fork_init(void)
429 {
430         int i;
431 #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
432 #ifndef ARCH_MIN_TASKALIGN
433 #define ARCH_MIN_TASKALIGN      L1_CACHE_BYTES
434 #endif
435         /* create a slab on which task_structs can be allocated */
436         task_struct_cachep = kmem_cache_create("task_struct",
437                         arch_task_struct_size, ARCH_MIN_TASKALIGN,
438                         SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT, NULL);
439 #endif
440 
441         /* do the arch specific task caches init */
442         arch_task_cache_init();
443 
444         set_max_threads(MAX_THREADS);
445 
446         init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
447         init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
448         init_task.signal->rlim[RLIMIT_SIGPENDING] =
449                 init_task.signal->rlim[RLIMIT_NPROC];
450 
451         for (i = 0; i < UCOUNT_COUNTS; i++) {
452                 init_user_ns.ucount_max[i] = max_threads/2;
453         }
454 }
455 
456 int __weak arch_dup_task_struct(struct task_struct *dst,
457                                                struct task_struct *src)
458 {
459         *dst = *src;
460         return 0;
461 }
462 
463 void set_task_stack_end_magic(struct task_struct *tsk)
464 {
465         unsigned long *stackend;
466 
467         stackend = end_of_stack(tsk);
468         *stackend = STACK_END_MAGIC;    /* for overflow detection */
469 }
470 
471 static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
472 {
473         struct task_struct *tsk;
474         unsigned long *stack;
475         struct vm_struct *stack_vm_area;
476         int err;
477 
478         if (node == NUMA_NO_NODE)
479                 node = tsk_fork_get_node(orig);
480         tsk = alloc_task_struct_node(node);
481         if (!tsk)
482                 return NULL;
483 
484         stack = alloc_thread_stack_node(tsk, node);
485         if (!stack)
486                 goto free_tsk;
487 
488         stack_vm_area = task_stack_vm_area(tsk);
489 
490         err = arch_dup_task_struct(tsk, orig);
491 
492         /*
493          * arch_dup_task_struct() clobbers the stack-related fields.  Make
494          * sure they're properly initialized before using any stack-related
495          * functions again.
496          */
497         tsk->stack = stack;
498 #ifdef CONFIG_VMAP_STACK
499         tsk->stack_vm_area = stack_vm_area;
500 #endif
501 #ifdef CONFIG_THREAD_INFO_IN_TASK
502         atomic_set(&tsk->stack_refcount, 1);
503 #endif
504 
505         if (err)
506                 goto free_stack;
507 
508 #ifdef CONFIG_SECCOMP
509         /*
510          * We must handle setting up seccomp filters once we're under
511          * the sighand lock in case orig has changed between now and
512          * then. Until then, filter must be NULL to avoid messing up
513          * the usage counts on the error path calling free_task.
514          */
515         tsk->seccomp.filter = NULL;
516 #endif
517 
518         setup_thread_stack(tsk, orig);
519         clear_user_return_notifier(tsk);
520         clear_tsk_need_resched(tsk);
521         set_task_stack_end_magic(tsk);
522 
523 #ifdef CONFIG_CC_STACKPROTECTOR
524         tsk->stack_canary = get_random_int();
525 #endif
526 
527         /*
528          * One for us, one for whoever does the "release_task()" (usually
529          * parent)
530          */
531         atomic_set(&tsk->usage, 2);
532 #ifdef CONFIG_BLK_DEV_IO_TRACE
533         tsk->btrace_seq = 0;
534 #endif
535         tsk->splice_pipe = NULL;
536         tsk->task_frag.page = NULL;
537         tsk->wake_q.next = NULL;
538 
539         account_kernel_stack(tsk, 1);
540 
541         kcov_task_init(tsk);
542 
543         return tsk;
544 
545 free_stack:
546         free_thread_stack(tsk);
547 free_tsk:
548         free_task_struct(tsk);
549         return NULL;
550 }
551 
552 #ifdef CONFIG_MMU
553 static __latent_entropy int dup_mmap(struct mm_struct *mm,
554                                         struct mm_struct *oldmm)
555 {
556         struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
557         struct rb_node **rb_link, *rb_parent;
558         int retval;
559         unsigned long charge;
560 
561         uprobe_start_dup_mmap();
562         if (down_write_killable(&oldmm->mmap_sem)) {
563                 retval = -EINTR;
564                 goto fail_uprobe_end;
565         }
566         flush_cache_dup_mm(oldmm);
567         uprobe_dup_mmap(oldmm, mm);
568         /*
569          * Not linked in yet - no deadlock potential:
570          */
571         down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
572 
573         /* No ordering required: file already has been exposed. */
574         RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
575 
576         mm->total_vm = oldmm->total_vm;
577         mm->data_vm = oldmm->data_vm;
578         mm->exec_vm = oldmm->exec_vm;
579         mm->stack_vm = oldmm->stack_vm;
580 
581         rb_link = &mm->mm_rb.rb_node;
582         rb_parent = NULL;
583         pprev = &mm->mmap;
584         retval = ksm_fork(mm, oldmm);
585         if (retval)
586                 goto out;
587         retval = khugepaged_fork(mm, oldmm);
588         if (retval)
589                 goto out;
590 
591         prev = NULL;
592         for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
593                 struct file *file;
594 
595                 if (mpnt->vm_flags & VM_DONTCOPY) {
596                         vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
597                         continue;
598                 }
599                 charge = 0;
600                 if (mpnt->vm_flags & VM_ACCOUNT) {
601                         unsigned long len = vma_pages(mpnt);
602 
603                         if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
604                                 goto fail_nomem;
605                         charge = len;
606                 }
607                 tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
608                 if (!tmp)
609                         goto fail_nomem;
610                 *tmp = *mpnt;
611                 INIT_LIST_HEAD(&tmp->anon_vma_chain);
612                 retval = vma_dup_policy(mpnt, tmp);
613                 if (retval)
614                         goto fail_nomem_policy;
615                 tmp->vm_mm = mm;
616                 if (anon_vma_fork(tmp, mpnt))
617                         goto fail_nomem_anon_vma_fork;
618                 tmp->vm_flags &=
619                         ~(VM_LOCKED|VM_LOCKONFAULT|VM_UFFD_MISSING|VM_UFFD_WP);
620                 tmp->vm_next = tmp->vm_prev = NULL;
621                 tmp->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
622                 file = tmp->vm_file;
623                 if (file) {
624                         struct inode *inode = file_inode(file);
625                         struct address_space *mapping = file->f_mapping;
626 
627                         get_file(file);
628                         if (tmp->vm_flags & VM_DENYWRITE)
629                                 atomic_dec(&inode->i_writecount);
630                         i_mmap_lock_write(mapping);
631                         if (tmp->vm_flags & VM_SHARED)
632                                 atomic_inc(&mapping->i_mmap_writable);
633                         flush_dcache_mmap_lock(mapping);
634                         /* insert tmp into the share list, just after mpnt */
635                         vma_interval_tree_insert_after(tmp, mpnt,
636                                         &mapping->i_mmap);
637                         flush_dcache_mmap_unlock(mapping);
638                         i_mmap_unlock_write(mapping);
639                 }
640 
641                 /*
642                  * Clear hugetlb-related page reserves for children. This only
643                  * affects MAP_PRIVATE mappings. Faults generated by the child
644                  * are not guaranteed to succeed, even if read-only
645                  */
646                 if (is_vm_hugetlb_page(tmp))
647                         reset_vma_resv_huge_pages(tmp);
648 
649                 /*
650                  * Link in the new vma and copy the page table entries.
651                  */
652                 *pprev = tmp;
653                 pprev = &tmp->vm_next;
654                 tmp->vm_prev = prev;
655                 prev = tmp;
656 
657                 __vma_link_rb(mm, tmp, rb_link, rb_parent);
658                 rb_link = &tmp->vm_rb.rb_right;
659                 rb_parent = &tmp->vm_rb;
660 
661                 mm->map_count++;
662                 retval = copy_page_range(mm, oldmm, mpnt);
663 
664                 if (tmp->vm_ops && tmp->vm_ops->open)
665                         tmp->vm_ops->open(tmp);
666 
667                 if (retval)
668                         goto out;
669         }
670         /* a new mm has just been created */
671         arch_dup_mmap(oldmm, mm);
672         retval = 0;
673 out:
674         up_write(&mm->mmap_sem);
675         flush_tlb_mm(oldmm);
676         up_write(&oldmm->mmap_sem);
677 fail_uprobe_end:
678         uprobe_end_dup_mmap();
679         return retval;
680 fail_nomem_anon_vma_fork:
681         mpol_put(vma_policy(tmp));
682 fail_nomem_policy:
683         kmem_cache_free(vm_area_cachep, tmp);
684 fail_nomem:
685         retval = -ENOMEM;
686         vm_unacct_memory(charge);
687         goto out;
688 }
689 
690 static inline int mm_alloc_pgd(struct mm_struct *mm)
691 {
692         mm->pgd = pgd_alloc(mm);
693         if (unlikely(!mm->pgd))
694                 return -ENOMEM;
695         return 0;
696 }
697 
698 static inline void mm_free_pgd(struct mm_struct *mm)
699 {
700         pgd_free(mm, mm->pgd);
701 }
702 #else
703 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
704 {
705         down_write(&oldmm->mmap_sem);
706         RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
707         up_write(&oldmm->mmap_sem);
708         return 0;
709 }
710 #define mm_alloc_pgd(mm)        (0)
711 #define mm_free_pgd(mm)
712 #endif /* CONFIG_MMU */
713 
714 __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
715 
716 #define allocate_mm()   (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
717 #define free_mm(mm)     (kmem_cache_free(mm_cachep, (mm)))
718 
719 static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
720 
721 static int __init coredump_filter_setup(char *s)
722 {
723         default_dump_filter =
724                 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
725                 MMF_DUMP_FILTER_MASK;
726         return 1;
727 }
728 
729 __setup("coredump_filter=", coredump_filter_setup);
730 
731 #include <linux/init_task.h>
732 
733 static void mm_init_aio(struct mm_struct *mm)
734 {
735 #ifdef CONFIG_AIO
736         spin_lock_init(&mm->ioctx_lock);
737         mm->ioctx_table = NULL;
738 #endif
739 }
740 
741 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
742 {
743 #ifdef CONFIG_MEMCG
744         mm->owner = p;
745 #endif
746 }
747 
748 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
749 {
750         mm->mmap = NULL;
751         mm->mm_rb = RB_ROOT;
752         mm->vmacache_seqnum = 0;
753         atomic_set(&mm->mm_users, 1);
754         atomic_set(&mm->mm_count, 1);
755         init_rwsem(&mm->mmap_sem);
756         INIT_LIST_HEAD(&mm->mmlist);
757         mm->core_state = NULL;
758         atomic_long_set(&mm->nr_ptes, 0);
759         mm_nr_pmds_init(mm);
760         mm->map_count = 0;
761         mm->locked_vm = 0;
762         mm->pinned_vm = 0;
763         memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
764         spin_lock_init(&mm->page_table_lock);
765         mm_init_cpumask(mm);
766         mm_init_aio(mm);
767         mm_init_owner(mm, p);
768         mmu_notifier_mm_init(mm);
769         clear_tlb_flush_pending(mm);
770 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
771         mm->pmd_huge_pte = NULL;
772 #endif
773 
774         if (current->mm) {
775                 mm->flags = current->mm->flags & MMF_INIT_MASK;
776                 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
777         } else {
778                 mm->flags = default_dump_filter;
779                 mm->def_flags = 0;
780         }
781 
782         if (mm_alloc_pgd(mm))
783                 goto fail_nopgd;
784 
785         if (init_new_context(p, mm))
786                 goto fail_nocontext;
787 
788         return mm;
789 
790 fail_nocontext:
791         mm_free_pgd(mm);
792 fail_nopgd:
793         free_mm(mm);
794         return NULL;
795 }
796 
797 static void check_mm(struct mm_struct *mm)
798 {
799         int i;
800 
801         for (i = 0; i < NR_MM_COUNTERS; i++) {
802                 long x = atomic_long_read(&mm->rss_stat.count[i]);
803 
804                 if (unlikely(x))
805                         printk(KERN_ALERT "BUG: Bad rss-counter state "
806                                           "mm:%p idx:%d val:%ld\n", mm, i, x);
807         }
808 
809         if (atomic_long_read(&mm->nr_ptes))
810                 pr_alert("BUG: non-zero nr_ptes on freeing mm: %ld\n",
811                                 atomic_long_read(&mm->nr_ptes));
812         if (mm_nr_pmds(mm))
813                 pr_alert("BUG: non-zero nr_pmds on freeing mm: %ld\n",
814                                 mm_nr_pmds(mm));
815 
816 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
817         VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
818 #endif
819 }
820 
821 /*
822  * Allocate and initialize an mm_struct.
823  */
824 struct mm_struct *mm_alloc(void)
825 {
826         struct mm_struct *mm;
827 
828         mm = allocate_mm();
829         if (!mm)
830                 return NULL;
831 
832         memset(mm, 0, sizeof(*mm));
833         return mm_init(mm, current);
834 }
835 
836 /*
837  * Called when the last reference to the mm
838  * is dropped: either by a lazy thread or by
839  * mmput. Free the page directory and the mm.
840  */
841 void __mmdrop(struct mm_struct *mm)
842 {
843         BUG_ON(mm == &init_mm);
844         mm_free_pgd(mm);
845         destroy_context(mm);
846         mmu_notifier_mm_destroy(mm);
847         check_mm(mm);
848         free_mm(mm);
849 }
850 EXPORT_SYMBOL_GPL(__mmdrop);
851 
852 static inline void __mmput(struct mm_struct *mm)
853 {
854         VM_BUG_ON(atomic_read(&mm->mm_users));
855 
856         uprobe_clear_state(mm);
857         exit_aio(mm);
858         ksm_exit(mm);
859         khugepaged_exit(mm); /* must run before exit_mmap */
860         exit_mmap(mm);
861         mm_put_huge_zero_page(mm);
862         set_mm_exe_file(mm, NULL);
863         if (!list_empty(&mm->mmlist)) {
864                 spin_lock(&mmlist_lock);
865                 list_del(&mm->mmlist);
866                 spin_unlock(&mmlist_lock);
867         }
868         if (mm->binfmt)
869                 module_put(mm->binfmt->module);
870         set_bit(MMF_OOM_SKIP, &mm->flags);
871         mmdrop(mm);
872 }
873 
874 /*
875  * Decrement the use count and release all resources for an mm.
876  */
877 void mmput(struct mm_struct *mm)
878 {
879         might_sleep();
880 
881         if (atomic_dec_and_test(&mm->mm_users))
882                 __mmput(mm);
883 }
884 EXPORT_SYMBOL_GPL(mmput);
885 
886 #ifdef CONFIG_MMU
887 static void mmput_async_fn(struct work_struct *work)
888 {
889         struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
890         __mmput(mm);
891 }
892 
893 void mmput_async(struct mm_struct *mm)
894 {
895         if (atomic_dec_and_test(&mm->mm_users)) {
896                 INIT_WORK(&mm->async_put_work, mmput_async_fn);
897                 schedule_work(&mm->async_put_work);
898         }
899 }
900 #endif
901 
902 /**
903  * set_mm_exe_file - change a reference to the mm's executable file
904  *
905  * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
906  *
907  * Main users are mmput() and sys_execve(). Callers prevent concurrent
908  * invocations: in mmput() nobody alive left, in execve task is single
909  * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the
910  * mm->exe_file, but does so without using set_mm_exe_file() in order
911  * to do avoid the need for any locks.
912  */
913 void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
914 {
915         struct file *old_exe_file;
916 
917         /*
918          * It is safe to dereference the exe_file without RCU as
919          * this function is only called if nobody else can access
920          * this mm -- see comment above for justification.
921          */
922         old_exe_file = rcu_dereference_raw(mm->exe_file);
923 
924         if (new_exe_file)
925                 get_file(new_exe_file);
926         rcu_assign_pointer(mm->exe_file, new_exe_file);
927         if (old_exe_file)
928                 fput(old_exe_file);
929 }
930 
931 /**
932  * get_mm_exe_file - acquire a reference to the mm's executable file
933  *
934  * Returns %NULL if mm has no associated executable file.
935  * User must release file via fput().
936  */
937 struct file *get_mm_exe_file(struct mm_struct *mm)
938 {
939         struct file *exe_file;
940 
941         rcu_read_lock();
942         exe_file = rcu_dereference(mm->exe_file);
943         if (exe_file && !get_file_rcu(exe_file))
944                 exe_file = NULL;
945         rcu_read_unlock();
946         return exe_file;
947 }
948 EXPORT_SYMBOL(get_mm_exe_file);
949 
950 /**
951  * get_task_exe_file - acquire a reference to the task's executable file
952  *
953  * Returns %NULL if task's mm (if any) has no associated executable file or
954  * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
955  * User must release file via fput().
956  */
957 struct file *get_task_exe_file(struct task_struct *task)
958 {
959         struct file *exe_file = NULL;
960         struct mm_struct *mm;
961 
962         task_lock(task);
963         mm = task->mm;
964         if (mm) {
965                 if (!(task->flags & PF_KTHREAD))
966                         exe_file = get_mm_exe_file(mm);
967         }
968         task_unlock(task);
969         return exe_file;
970 }
971 EXPORT_SYMBOL(get_task_exe_file);
972 
973 /**
974  * get_task_mm - acquire a reference to the task's mm
975  *
976  * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
977  * this kernel workthread has transiently adopted a user mm with use_mm,
978  * to do its AIO) is not set and if so returns a reference to it, after
979  * bumping up the use count.  User must release the mm via mmput()
980  * after use.  Typically used by /proc and ptrace.
981  */
982 struct mm_struct *get_task_mm(struct task_struct *task)
983 {
984         struct mm_struct *mm;
985 
986         task_lock(task);
987         mm = task->mm;
988         if (mm) {
989                 if (task->flags & PF_KTHREAD)
990                         mm = NULL;
991                 else
992                         atomic_inc(&mm->mm_users);
993         }
994         task_unlock(task);
995         return mm;
996 }
997 EXPORT_SYMBOL_GPL(get_task_mm);
998 
999 struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
1000 {
1001         struct mm_struct *mm;
1002         int err;
1003 
1004         err =  mutex_lock_killable(&task->signal->cred_guard_mutex);
1005         if (err)
1006                 return ERR_PTR(err);
1007 
1008         mm = get_task_mm(task);
1009         if (mm && mm != current->mm &&
1010                         !ptrace_may_access(task, mode)) {
1011                 mmput(mm);
1012                 mm = ERR_PTR(-EACCES);
1013         }
1014         mutex_unlock(&task->signal->cred_guard_mutex);
1015 
1016         return mm;
1017 }
1018 
1019 static void complete_vfork_done(struct task_struct *tsk)
1020 {
1021         struct completion *vfork;
1022 
1023         task_lock(tsk);
1024         vfork = tsk->vfork_done;
1025         if (likely(vfork)) {
1026                 tsk->vfork_done = NULL;
1027                 complete(vfork);
1028         }
1029         task_unlock(tsk);
1030 }
1031 
1032 static int wait_for_vfork_done(struct task_struct *child,
1033                                 struct completion *vfork)
1034 {
1035         int killed;
1036 
1037         freezer_do_not_count();
1038         killed = wait_for_completion_killable(vfork);
1039         freezer_count();
1040 
1041         if (killed) {
1042                 task_lock(child);
1043                 child->vfork_done = NULL;
1044                 task_unlock(child);
1045         }
1046 
1047         put_task_struct(child);
1048         return killed;
1049 }
1050 
1051 /* Please note the differences between mmput and mm_release.
1052  * mmput is called whenever we stop holding onto a mm_struct,
1053  * error success whatever.
1054  *
1055  * mm_release is called after a mm_struct has been removed
1056  * from the current process.
1057  *
1058  * This difference is important for error handling, when we
1059  * only half set up a mm_struct for a new process and need to restore
1060  * the old one.  Because we mmput the new mm_struct before
1061  * restoring the old one. . .
1062  * Eric Biederman 10 January 1998
1063  */
1064 void mm_release(struct task_struct *tsk, struct mm_struct *mm)
1065 {
1066         /* Get rid of any futexes when releasing the mm */
1067 #ifdef CONFIG_FUTEX
1068         if (unlikely(tsk->robust_list)) {
1069                 exit_robust_list(tsk);
1070                 tsk->robust_list = NULL;
1071         }
1072 #ifdef CONFIG_COMPAT
1073         if (unlikely(tsk->compat_robust_list)) {
1074                 compat_exit_robust_list(tsk);
1075                 tsk->compat_robust_list = NULL;
1076         }
1077 #endif
1078         if (unlikely(!list_empty(&tsk->pi_state_list)))
1079                 exit_pi_state_list(tsk);
1080 #endif
1081 
1082         uprobe_free_utask(tsk);
1083 
1084         /* Get rid of any cached register state */
1085         deactivate_mm(tsk, mm);
1086 
1087         /*
1088          * Signal userspace if we're not exiting with a core dump
1089          * because we want to leave the value intact for debugging
1090          * purposes.
1091          */
1092         if (tsk->clear_child_tid) {
1093                 if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) &&
1094                     atomic_read(&mm->mm_users) > 1) {
1095                         /*
1096                          * We don't check the error code - if userspace has
1097                          * not set up a proper pointer then tough luck.
1098                          */
1099                         put_user(0, tsk->clear_child_tid);
1100                         sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
1101                                         1, NULL, NULL, 0);
1102                 }
1103                 tsk->clear_child_tid = NULL;
1104         }
1105 
1106         /*
1107          * All done, finally we can wake up parent and return this mm to him.
1108          * Also kthread_stop() uses this completion for synchronization.
1109          */
1110         if (tsk->vfork_done)
1111                 complete_vfork_done(tsk);
1112 }
1113 
1114 /*
1115  * Allocate a new mm structure and copy contents from the
1116  * mm structure of the passed in task structure.
1117  */
1118 static struct mm_struct *dup_mm(struct task_struct *tsk)
1119 {
1120         struct mm_struct *mm, *oldmm = current->mm;
1121         int err;
1122 
1123         mm = allocate_mm();
1124         if (!mm)
1125                 goto fail_nomem;
1126 
1127         memcpy(mm, oldmm, sizeof(*mm));
1128 
1129         if (!mm_init(mm, tsk))
1130                 goto fail_nomem;
1131 
1132         err = dup_mmap(mm, oldmm);
1133         if (err)
1134                 goto free_pt;
1135 
1136         mm->hiwater_rss = get_mm_rss(mm);
1137         mm->hiwater_vm = mm->total_vm;
1138 
1139         if (mm->binfmt && !try_module_get(mm->binfmt->module))
1140                 goto free_pt;
1141 
1142         return mm;
1143 
1144 free_pt:
1145         /* don't put binfmt in mmput, we haven't got module yet */
1146         mm->binfmt = NULL;
1147         mmput(mm);
1148 
1149 fail_nomem:
1150         return NULL;
1151 }
1152 
1153 static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
1154 {
1155         struct mm_struct *mm, *oldmm;
1156         int retval;
1157 
1158         tsk->min_flt = tsk->maj_flt = 0;
1159         tsk->nvcsw = tsk->nivcsw = 0;
1160 #ifdef CONFIG_DETECT_HUNG_TASK
1161         tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1162 #endif
1163 
1164         tsk->mm = NULL;
1165         tsk->active_mm = NULL;
1166 
1167         /*
1168          * Are we cloning a kernel thread?
1169          *
1170          * We need to steal a active VM for that..
1171          */
1172         oldmm = current->mm;
1173         if (!oldmm)
1174                 return 0;
1175 
1176         /* initialize the new vmacache entries */
1177         vmacache_flush(tsk);
1178 
1179         if (clone_flags & CLONE_VM) {
1180                 atomic_inc(&oldmm->mm_users);
1181                 mm = oldmm;
1182                 goto good_mm;
1183         }
1184 
1185         retval = -ENOMEM;
1186         mm = dup_mm(tsk);
1187         if (!mm)
1188                 goto fail_nomem;
1189 
1190 good_mm:
1191         tsk->mm = mm;
1192         tsk->active_mm = mm;
1193         return 0;
1194 
1195 fail_nomem:
1196         return retval;
1197 }
1198 
1199 static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
1200 {
1201         struct fs_struct *fs = current->fs;
1202         if (clone_flags & CLONE_FS) {
1203                 /* tsk->fs is already what we want */
1204                 spin_lock(&fs->lock);
1205                 if (fs->in_exec) {
1206                         spin_unlock(&fs->lock);
1207                         return -EAGAIN;
1208                 }
1209                 fs->users++;
1210                 spin_unlock(&fs->lock);
1211                 return 0;
1212         }
1213         tsk->fs = copy_fs_struct(fs);
1214         if (!tsk->fs)
1215                 return -ENOMEM;
1216         return 0;
1217 }
1218 
1219 static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
1220 {
1221         struct files_struct *oldf, *newf;
1222         int error = 0;
1223 
1224         /*
1225          * A background process may not have any files ...
1226          */
1227         oldf = current->files;
1228         if (!oldf)
1229                 goto out;
1230 
1231         if (clone_flags & CLONE_FILES) {
1232                 atomic_inc(&oldf->count);
1233                 goto out;
1234         }
1235 
1236         newf = dup_fd(oldf, &error);
1237         if (!newf)
1238                 goto out;
1239 
1240         tsk->files = newf;
1241         error = 0;
1242 out:
1243         return error;
1244 }
1245 
1246 static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
1247 {
1248 #ifdef CONFIG_BLOCK
1249         struct io_context *ioc = current->io_context;
1250         struct io_context *new_ioc;
1251 
1252         if (!ioc)
1253                 return 0;
1254         /*
1255          * Share io context with parent, if CLONE_IO is set
1256          */
1257         if (clone_flags & CLONE_IO) {
1258                 ioc_task_link(ioc);
1259                 tsk->io_context = ioc;
1260         } else if (ioprio_valid(ioc->ioprio)) {
1261                 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
1262                 if (unlikely(!new_ioc))
1263                         return -ENOMEM;
1264 
1265                 new_ioc->ioprio = ioc->ioprio;
1266                 put_io_context(new_ioc);
1267         }
1268 #endif
1269         return 0;
1270 }
1271 
1272 static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
1273 {
1274         struct sighand_struct *sig;
1275 
1276         if (clone_flags & CLONE_SIGHAND) {
1277                 atomic_inc(&current->sighand->count);
1278                 return 0;
1279         }
1280         sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1281         rcu_assign_pointer(tsk->sighand, sig);
1282         if (!sig)
1283                 return -ENOMEM;
1284 
1285         atomic_set(&sig->count, 1);
1286         memcpy(sig->action, current->sighand->action, sizeof(sig->action));
1287         return 0;
1288 }
1289 
1290 void __cleanup_sighand(struct sighand_struct *sighand)
1291 {
1292         if (atomic_dec_and_test(&sighand->count)) {
1293                 signalfd_cleanup(sighand);
1294                 /*
1295                  * sighand_cachep is SLAB_DESTROY_BY_RCU so we can free it
1296                  * without an RCU grace period, see __lock_task_sighand().
1297                  */
1298                 kmem_cache_free(sighand_cachep, sighand);
1299         }
1300 }
1301 
1302 /*
1303  * Initialize POSIX timer handling for a thread group.
1304  */
1305 static void posix_cpu_timers_init_group(struct signal_struct *sig)
1306 {
1307         unsigned long cpu_limit;
1308 
1309         cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1310         if (cpu_limit != RLIM_INFINITY) {
1311                 sig->cputime_expires.prof_exp = secs_to_cputime(cpu_limit);
1312                 sig->cputimer.running = true;
1313         }
1314 
1315         /* The timer lists. */
1316         INIT_LIST_HEAD(&sig->cpu_timers[0]);
1317         INIT_LIST_HEAD(&sig->cpu_timers[1]);
1318         INIT_LIST_HEAD(&sig->cpu_timers[2]);
1319 }
1320 
1321 static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
1322 {
1323         struct signal_struct *sig;
1324 
1325         if (clone_flags & CLONE_THREAD)
1326                 return 0;
1327 
1328         sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
1329         tsk->signal = sig;
1330         if (!sig)
1331                 return -ENOMEM;
1332 
1333         sig->nr_threads = 1;
1334         atomic_set(&sig->live, 1);
1335         atomic_set(&sig->sigcnt, 1);
1336 
1337         /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
1338         sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
1339         tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
1340 
1341         init_waitqueue_head(&sig->wait_chldexit);
1342         sig->curr_target = tsk;
1343         init_sigpending(&sig->shared_pending);
1344         INIT_LIST_HEAD(&sig->posix_timers);
1345         seqlock_init(&sig->stats_lock);
1346         prev_cputime_init(&sig->prev_cputime);
1347 
1348         hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1349         sig->real_timer.function = it_real_fn;
1350 
1351         task_lock(current->group_leader);
1352         memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1353         task_unlock(current->group_leader);
1354 
1355         posix_cpu_timers_init_group(sig);
1356 
1357         tty_audit_fork(sig);
1358         sched_autogroup_fork(sig);
1359 
1360         sig->oom_score_adj = current->signal->oom_score_adj;
1361         sig->oom_score_adj_min = current->signal->oom_score_adj_min;
1362 
1363         sig->has_child_subreaper = current->signal->has_child_subreaper ||
1364                                    current->signal->is_child_subreaper;
1365 
1366         mutex_init(&sig->cred_guard_mutex);
1367 
1368         return 0;
1369 }
1370 
1371 static void copy_seccomp(struct task_struct *p)
1372 {
1373 #ifdef CONFIG_SECCOMP
1374         /*
1375          * Must be called with sighand->lock held, which is common to
1376          * all threads in the group. Holding cred_guard_mutex is not
1377          * needed because this new task is not yet running and cannot
1378          * be racing exec.
1379          */
1380         assert_spin_locked(&current->sighand->siglock);
1381 
1382         /* Ref-count the new filter user, and assign it. */
1383         get_seccomp_filter(current);
1384         p->seccomp = current->seccomp;
1385 
1386         /*
1387          * Explicitly enable no_new_privs here in case it got set
1388          * between the task_struct being duplicated and holding the
1389          * sighand lock. The seccomp state and nnp must be in sync.
1390          */
1391         if (task_no_new_privs(current))
1392                 task_set_no_new_privs(p);
1393 
1394         /*
1395          * If the parent gained a seccomp mode after copying thread
1396          * flags and between before we held the sighand lock, we have
1397          * to manually enable the seccomp thread flag here.
1398          */
1399         if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1400                 set_tsk_thread_flag(p, TIF_SECCOMP);
1401 #endif
1402 }
1403 
1404 SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
1405 {
1406         current->clear_child_tid = tidptr;
1407 
1408         return task_pid_vnr(current);
1409 }
1410 
1411 static void rt_mutex_init_task(struct task_struct *p)
1412 {
1413         raw_spin_lock_init(&p->pi_lock);
1414 #ifdef CONFIG_RT_MUTEXES
1415         p->pi_waiters = RB_ROOT;
1416         p->pi_waiters_leftmost = NULL;
1417         p->pi_blocked_on = NULL;
1418 #endif
1419 }
1420 
1421 /*
1422  * Initialize POSIX timer handling for a single task.
1423  */
1424 static void posix_cpu_timers_init(struct task_struct *tsk)
1425 {
1426         tsk->cputime_expires.prof_exp = 0;
1427         tsk->cputime_expires.virt_exp = 0;
1428         tsk->cputime_expires.sched_exp = 0;
1429         INIT_LIST_HEAD(&tsk->cpu_timers[0]);
1430         INIT_LIST_HEAD(&tsk->cpu_timers[1]);
1431         INIT_LIST_HEAD(&tsk->cpu_timers[2]);
1432 }
1433 
1434 static inline void
1435 init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
1436 {
1437          task->pids[type].pid = pid;
1438 }
1439 
1440 /*
1441  * This creates a new process as a copy of the old one,
1442  * but does not actually start it yet.
1443  *
1444  * It copies the registers, and all the appropriate
1445  * parts of the process environment (as per the clone
1446  * flags). The actual kick-off is left to the caller.
1447  */
1448 static __latent_entropy struct task_struct *copy_process(
1449                                         unsigned long clone_flags,
1450                                         unsigned long stack_start,
1451                                         unsigned long stack_size,
1452                                         int __user *child_tidptr,
1453                                         struct pid *pid,
1454                                         int trace,
1455                                         unsigned long tls,
1456                                         int node)
1457 {
1458         int retval;
1459         struct task_struct *p;
1460 
1461         if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
1462                 return ERR_PTR(-EINVAL);
1463 
1464         if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
1465                 return ERR_PTR(-EINVAL);
1466 
1467         /*
1468          * Thread groups must share signals as well, and detached threads
1469          * can only be started up within the thread group.
1470          */
1471         if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
1472                 return ERR_PTR(-EINVAL);
1473 
1474         /*
1475          * Shared signal handlers imply shared VM. By way of the above,
1476          * thread groups also imply shared VM. Blocking this case allows
1477          * for various simplifications in other code.
1478          */
1479         if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
1480                 return ERR_PTR(-EINVAL);
1481 
1482         /*
1483          * Siblings of global init remain as zombies on exit since they are
1484          * not reaped by their parent (swapper). To solve this and to avoid
1485          * multi-rooted process trees, prevent global and container-inits
1486          * from creating siblings.
1487          */
1488         if ((clone_flags & CLONE_PARENT) &&
1489                                 current->signal->flags & SIGNAL_UNKILLABLE)
1490                 return ERR_PTR(-EINVAL);
1491 
1492         /*
1493          * If the new process will be in a different pid or user namespace
1494          * do not allow it to share a thread group with the forking task.
1495          */
1496         if (clone_flags & CLONE_THREAD) {
1497                 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
1498                     (task_active_pid_ns(current) !=
1499                                 current->nsproxy->pid_ns_for_children))
1500                         return ERR_PTR(-EINVAL);
1501         }
1502 
1503         retval = security_task_create(clone_flags);
1504         if (retval)
1505                 goto fork_out;
1506 
1507         retval = -ENOMEM;
1508         p = dup_task_struct(current, node);
1509         if (!p)
1510                 goto fork_out;
1511 
1512         ftrace_graph_init_task(p);
1513 
1514         rt_mutex_init_task(p);
1515 
1516 #ifdef CONFIG_PROVE_LOCKING
1517         DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
1518         DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1519 #endif
1520         retval = -EAGAIN;
1521         if (atomic_read(&p->real_cred->user->processes) >=
1522                         task_rlimit(p, RLIMIT_NPROC)) {
1523                 if (p->real_cred->user != INIT_USER &&
1524                     !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
1525                         goto bad_fork_free;
1526         }
1527         current->flags &= ~PF_NPROC_EXCEEDED;
1528 
1529         retval = copy_creds(p, clone_flags);
1530         if (retval < 0)
1531                 goto bad_fork_free;
1532 
1533         /*
1534          * If multiple threads are within copy_process(), then this check
1535          * triggers too late. This doesn't hurt, the check is only there
1536          * to stop root fork bombs.
1537          */
1538         retval = -EAGAIN;
1539         if (nr_threads >= max_threads)
1540                 goto bad_fork_cleanup_count;
1541 
1542         delayacct_tsk_init(p);  /* Must remain after dup_task_struct() */
1543         p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
1544         p->flags |= PF_FORKNOEXEC;
1545         INIT_LIST_HEAD(&p->children);
1546         INIT_LIST_HEAD(&p->sibling);
1547         rcu_copy_process(p);
1548         p->vfork_done = NULL;
1549         spin_lock_init(&p->alloc_lock);
1550 
1551         init_sigpending(&p->pending);
1552 
1553         p->utime = p->stime = p->gtime = 0;
1554         p->utimescaled = p->stimescaled = 0;
1555         prev_cputime_init(&p->prev_cputime);
1556 
1557 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1558         seqcount_init(&p->vtime_seqcount);
1559         p->vtime_snap = 0;
1560         p->vtime_snap_whence = VTIME_INACTIVE;
1561 #endif
1562 
1563 #if defined(SPLIT_RSS_COUNTING)
1564         memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1565 #endif
1566 
1567         p->default_timer_slack_ns = current->timer_slack_ns;
1568 
1569         task_io_accounting_init(&p->ioac);
1570         acct_clear_integrals(p);
1571 
1572         posix_cpu_timers_init(p);
1573 
1574         p->start_time = ktime_get_ns();
1575         p->real_start_time = ktime_get_boot_ns();
1576         p->io_context = NULL;
1577         p->audit_context = NULL;
1578         cgroup_fork(p);
1579 #ifdef CONFIG_NUMA
1580         p->mempolicy = mpol_dup(p->mempolicy);
1581         if (IS_ERR(p->mempolicy)) {
1582                 retval = PTR_ERR(p->mempolicy);
1583                 p->mempolicy = NULL;
1584                 goto bad_fork_cleanup_threadgroup_lock;
1585         }
1586 #endif
1587 #ifdef CONFIG_CPUSETS
1588         p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
1589         p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
1590         seqcount_init(&p->mems_allowed_seq);
1591 #endif
1592 #ifdef CONFIG_TRACE_IRQFLAGS
1593         p->irq_events = 0;
1594         p->hardirqs_enabled = 0;
1595         p->hardirq_enable_ip = 0;
1596         p->hardirq_enable_event = 0;
1597         p->hardirq_disable_ip = _THIS_IP_;
1598         p->hardirq_disable_event = 0;
1599         p->softirqs_enabled = 1;
1600         p->softirq_enable_ip = _THIS_IP_;
1601         p->softirq_enable_event = 0;
1602         p->softirq_disable_ip = 0;
1603         p->softirq_disable_event = 0;
1604         p->hardirq_context = 0;
1605         p->softirq_context = 0;
1606 #endif
1607 
1608         p->pagefault_disabled = 0;
1609 
1610 #ifdef CONFIG_LOCKDEP
1611         p->lockdep_depth = 0; /* no locks held yet */
1612         p->curr_chain_key = 0;
1613         p->lockdep_recursion = 0;
1614 #endif
1615 
1616 #ifdef CONFIG_DEBUG_MUTEXES
1617         p->blocked_on = NULL; /* not blocked yet */
1618 #endif
1619 #ifdef CONFIG_BCACHE
1620         p->sequential_io        = 0;
1621         p->sequential_io_avg    = 0;
1622 #endif
1623 
1624         /* Perform scheduler related setup. Assign this task to a CPU. */
1625         retval = sched_fork(clone_flags, p);
1626         if (retval)
1627                 goto bad_fork_cleanup_policy;
1628 
1629         retval = perf_event_init_task(p);
1630         if (retval)
1631                 goto bad_fork_cleanup_policy;
1632         retval = audit_alloc(p);
1633         if (retval)
1634                 goto bad_fork_cleanup_perf;
1635         /* copy all the process information */
1636         shm_init_task(p);
1637         retval = copy_semundo(clone_flags, p);
1638         if (retval)
1639                 goto bad_fork_cleanup_audit;
1640         retval = copy_files(clone_flags, p);
1641         if (retval)
1642                 goto bad_fork_cleanup_semundo;
1643         retval = copy_fs(clone_flags, p);
1644         if (retval)
1645                 goto bad_fork_cleanup_files;
1646         retval = copy_sighand(clone_flags, p);
1647         if (retval)
1648                 goto bad_fork_cleanup_fs;
1649         retval = copy_signal(clone_flags, p);
1650         if (retval)
1651                 goto bad_fork_cleanup_sighand;
1652         retval = copy_mm(clone_flags, p);
1653         if (retval)
1654                 goto bad_fork_cleanup_signal;
1655         retval = copy_namespaces(clone_flags, p);
1656         if (retval)
1657                 goto bad_fork_cleanup_mm;
1658         retval = copy_io(clone_flags, p);
1659         if (retval)
1660                 goto bad_fork_cleanup_namespaces;
1661         retval = copy_thread_tls(clone_flags, stack_start, stack_size, p, tls);
1662         if (retval)
1663                 goto bad_fork_cleanup_io;
1664 
1665         if (pid != &init_struct_pid) {
1666                 pid = alloc_pid(p->nsproxy->pid_ns_for_children);
1667                 if (IS_ERR(pid)) {
1668                         retval = PTR_ERR(pid);
1669                         goto bad_fork_cleanup_thread;
1670                 }
1671         }
1672 
1673         p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
1674         /*
1675          * Clear TID on mm_release()?
1676          */
1677         p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr : NULL;
1678 #ifdef CONFIG_BLOCK
1679         p->plug = NULL;
1680 #endif
1681 #ifdef CONFIG_FUTEX
1682         p->robust_list = NULL;
1683 #ifdef CONFIG_COMPAT
1684         p->compat_robust_list = NULL;
1685 #endif
1686         INIT_LIST_HEAD(&p->pi_state_list);
1687         p->pi_state_cache = NULL;
1688 #endif
1689         /*
1690          * sigaltstack should be cleared when sharing the same VM
1691          */
1692         if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
1693                 sas_ss_reset(p);
1694 
1695         /*
1696          * Syscall tracing and stepping should be turned off in the
1697          * child regardless of CLONE_PTRACE.
1698          */
1699         user_disable_single_step(p);
1700         clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
1701 #ifdef TIF_SYSCALL_EMU
1702         clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
1703 #endif
1704         clear_all_latency_tracing(p);
1705 
1706         /* ok, now we should be set up.. */
1707         p->pid = pid_nr(pid);
1708         if (clone_flags & CLONE_THREAD) {
1709                 p->exit_signal = -1;
1710                 p->group_leader = current->group_leader;
1711                 p->tgid = current->tgid;
1712         } else {
1713                 if (clone_flags & CLONE_PARENT)
1714                         p->exit_signal = current->group_leader->exit_signal;
1715                 else
1716                         p->exit_signal = (clone_flags & CSIGNAL);
1717                 p->group_leader = p;
1718                 p->tgid = p->pid;
1719         }
1720 
1721         p->nr_dirtied = 0;
1722         p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
1723         p->dirty_paused_when = 0;
1724 
1725         p->pdeath_signal = 0;
1726         INIT_LIST_HEAD(&p->thread_group);
1727         p->task_works = NULL;
1728 
1729         threadgroup_change_begin(current);
1730         /*
1731          * Ensure that the cgroup subsystem policies allow the new process to be
1732          * forked. It should be noted the the new process's css_set can be changed
1733          * between here and cgroup_post_fork() if an organisation operation is in
1734          * progress.
1735          */
1736         retval = cgroup_can_fork(p);
1737         if (retval)
1738                 goto bad_fork_free_pid;
1739 
1740         /*
1741          * Make it visible to the rest of the system, but dont wake it up yet.
1742          * Need tasklist lock for parent etc handling!
1743          */
1744         write_lock_irq(&tasklist_lock);
1745 
1746         /* CLONE_PARENT re-uses the old parent */
1747         if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
1748                 p->real_parent = current->real_parent;
1749                 p->parent_exec_id = current->parent_exec_id;
1750         } else {
1751                 p->real_parent = current;
1752                 p->parent_exec_id = current->self_exec_id;
1753         }
1754 
1755         spin_lock(&current->sighand->siglock);
1756 
1757         /*
1758          * Copy seccomp details explicitly here, in case they were changed
1759          * before holding sighand lock.
1760          */
1761         copy_seccomp(p);
1762 
1763         /*
1764          * Process group and session signals need to be delivered to just the
1765          * parent before the fork or both the parent and the child after the
1766          * fork. Restart if a signal comes in before we add the new process to
1767          * it's process group.
1768          * A fatal signal pending means that current will exit, so the new
1769          * thread can't slip out of an OOM kill (or normal SIGKILL).
1770         */
1771         recalc_sigpending();
1772         if (signal_pending(current)) {
1773                 spin_unlock(&current->sighand->siglock);
1774                 write_unlock_irq(&tasklist_lock);
1775                 retval = -ERESTARTNOINTR;
1776                 goto bad_fork_cancel_cgroup;
1777         }
1778 
1779         if (likely(p->pid)) {
1780                 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
1781 
1782                 init_task_pid(p, PIDTYPE_PID, pid);
1783                 if (thread_group_leader(p)) {
1784                         init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
1785                         init_task_pid(p, PIDTYPE_SID, task_session(current));
1786 
1787                         if (is_child_reaper(pid)) {
1788                                 ns_of_pid(pid)->child_reaper = p;
1789                                 p->signal->flags |= SIGNAL_UNKILLABLE;
1790                         }
1791 
1792                         p->signal->leader_pid = pid;
1793                         p->signal->tty = tty_kref_get(current->signal->tty);
1794                         list_add_tail(&p->sibling, &p->real_parent->children);
1795                         list_add_tail_rcu(&p->tasks, &init_task.tasks);
1796                         attach_pid(p, PIDTYPE_PGID);
1797                         attach_pid(p, PIDTYPE_SID);
1798                         __this_cpu_inc(process_counts);
1799                 } else {
1800                         current->signal->nr_threads++;
1801                         atomic_inc(&current->signal->live);
1802                         atomic_inc(&current->signal->sigcnt);
1803                         list_add_tail_rcu(&p->thread_group,
1804                                           &p->group_leader->thread_group);
1805                         list_add_tail_rcu(&p->thread_node,
1806                                           &p->signal->thread_head);
1807                 }
1808                 attach_pid(p, PIDTYPE_PID);
1809                 nr_threads++;
1810         }
1811 
1812         total_forks++;
1813         spin_unlock(&current->sighand->siglock);
1814         syscall_tracepoint_update(p);
1815         write_unlock_irq(&tasklist_lock);
1816 
1817         proc_fork_connector(p);
1818         cgroup_post_fork(p);
1819         threadgroup_change_end(current);
1820         perf_event_fork(p);
1821 
1822         trace_task_newtask(p, clone_flags);
1823         uprobe_copy_process(p, clone_flags);
1824 
1825         return p;
1826 
1827 bad_fork_cancel_cgroup:
1828         cgroup_cancel_fork(p);
1829 bad_fork_free_pid:
1830         threadgroup_change_end(current);
1831         if (pid != &init_struct_pid)
1832                 free_pid(pid);
1833 bad_fork_cleanup_thread:
1834         exit_thread(p);
1835 bad_fork_cleanup_io:
1836         if (p->io_context)
1837                 exit_io_context(p);
1838 bad_fork_cleanup_namespaces:
1839         exit_task_namespaces(p);
1840 bad_fork_cleanup_mm:
1841         if (p->mm)
1842                 mmput(p->mm);
1843 bad_fork_cleanup_signal:
1844         if (!(clone_flags & CLONE_THREAD))
1845                 free_signal_struct(p->signal);
1846 bad_fork_cleanup_sighand:
1847         __cleanup_sighand(p->sighand);
1848 bad_fork_cleanup_fs:
1849         exit_fs(p); /* blocking */
1850 bad_fork_cleanup_files:
1851         exit_files(p); /* blocking */
1852 bad_fork_cleanup_semundo:
1853         exit_sem(p);
1854 bad_fork_cleanup_audit:
1855         audit_free(p);
1856 bad_fork_cleanup_perf:
1857         perf_event_free_task(p);
1858 bad_fork_cleanup_policy:
1859 #ifdef CONFIG_NUMA
1860         mpol_put(p->mempolicy);
1861 bad_fork_cleanup_threadgroup_lock:
1862 #endif
1863         delayacct_tsk_free(p);
1864 bad_fork_cleanup_count:
1865         atomic_dec(&p->cred->user->processes);
1866         exit_creds(p);
1867 bad_fork_free:
1868         p->state = TASK_DEAD;
1869         put_task_stack(p);
1870         free_task(p);
1871 fork_out:
1872         return ERR_PTR(retval);
1873 }
1874 
1875 static inline void init_idle_pids(struct pid_link *links)
1876 {
1877         enum pid_type type;
1878 
1879         for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
1880                 INIT_HLIST_NODE(&links[type].node); /* not really needed */
1881                 links[type].pid = &init_struct_pid;
1882         }
1883 }
1884 
1885 struct task_struct *fork_idle(int cpu)
1886 {
1887         struct task_struct *task;
1888         task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0, 0,
1889                             cpu_to_node(cpu));
1890         if (!IS_ERR(task)) {
1891                 init_idle_pids(task->pids);
1892                 init_idle(task, cpu);
1893         }
1894 
1895         return task;
1896 }
1897 
1898 /*
1899  *  Ok, this is the main fork-routine.
1900  *
1901  * It copies the process, and if successful kick-starts
1902  * it and waits for it to finish using the VM if required.
1903  */
1904 long _do_fork(unsigned long clone_flags,
1905               unsigned long stack_start,
1906               unsigned long stack_size,
1907               int __user *parent_tidptr,
1908               int __user *child_tidptr,
1909               unsigned long tls)
1910 {
1911         struct task_struct *p;
1912         int trace = 0;
1913         long nr;
1914 
1915         /*
1916          * Determine whether and which event to report to ptracer.  When
1917          * called from kernel_thread or CLONE_UNTRACED is explicitly
1918          * requested, no event is reported; otherwise, report if the event
1919          * for the type of forking is enabled.
1920          */
1921         if (!(clone_flags & CLONE_UNTRACED)) {
1922                 if (clone_flags & CLONE_VFORK)
1923                         trace = PTRACE_EVENT_VFORK;
1924                 else if ((clone_flags & CSIGNAL) != SIGCHLD)
1925                         trace = PTRACE_EVENT_CLONE;
1926                 else
1927                         trace = PTRACE_EVENT_FORK;
1928 
1929                 if (likely(!ptrace_event_enabled(current, trace)))
1930                         trace = 0;
1931         }
1932 
1933         p = copy_process(clone_flags, stack_start, stack_size,
1934                          child_tidptr, NULL, trace, tls, NUMA_NO_NODE);
1935         add_latent_entropy();
1936         /*
1937          * Do this prior waking up the new thread - the thread pointer
1938          * might get invalid after that point, if the thread exits quickly.
1939          */
1940         if (!IS_ERR(p)) {
1941                 struct completion vfork;
1942                 struct pid *pid;
1943 
1944                 trace_sched_process_fork(current, p);
1945 
1946                 pid = get_task_pid(p, PIDTYPE_PID);
1947                 nr = pid_vnr(pid);
1948 
1949                 if (clone_flags & CLONE_PARENT_SETTID)
1950                         put_user(nr, parent_tidptr);
1951 
1952                 if (clone_flags & CLONE_VFORK) {
1953                         p->vfork_done = &vfork;
1954                         init_completion(&vfork);
1955                         get_task_struct(p);
1956                 }
1957 
1958                 wake_up_new_task(p);
1959 
1960                 /* forking complete and child started to run, tell ptracer */
1961                 if (unlikely(trace))
1962                         ptrace_event_pid(trace, pid);
1963 
1964                 if (clone_flags & CLONE_VFORK) {
1965                         if (!wait_for_vfork_done(p, &vfork))
1966                                 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
1967                 }
1968 
1969                 put_pid(pid);
1970         } else {
1971                 nr = PTR_ERR(p);
1972         }
1973         return nr;
1974 }
1975 
1976 #ifndef CONFIG_HAVE_COPY_THREAD_TLS
1977 /* For compatibility with architectures that call do_fork directly rather than
1978  * using the syscall entry points below. */
1979 long do_fork(unsigned long clone_flags,
1980               unsigned long stack_start,
1981               unsigned long stack_size,
1982               int __user *parent_tidptr,
1983               int __user *child_tidptr)
1984 {
1985         return _do_fork(clone_flags, stack_start, stack_size,
1986                         parent_tidptr, child_tidptr, 0);
1987 }
1988 #endif
1989 
1990 /*
1991  * Create a kernel thread.
1992  */
1993 pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
1994 {
1995         return _do_fork(flags|CLONE_VM|CLONE_UNTRACED, (unsigned long)fn,
1996                 (unsigned long)arg, NULL, NULL, 0);
1997 }
1998 
1999 #ifdef __ARCH_WANT_SYS_FORK
2000 SYSCALL_DEFINE0(fork)
2001 {
2002 #ifdef CONFIG_MMU
2003         return _do_fork(SIGCHLD, 0, 0, NULL, NULL, 0);
2004 #else
2005         /* can not support in nommu mode */
2006         return -EINVAL;
2007 #endif
2008 }
2009 #endif
2010 
2011 #ifdef __ARCH_WANT_SYS_VFORK
2012 SYSCALL_DEFINE0(vfork)
2013 {
2014         return _do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, 0,
2015                         0, NULL, NULL, 0);
2016 }
2017 #endif
2018 
2019 #ifdef __ARCH_WANT_SYS_CLONE
2020 #ifdef CONFIG_CLONE_BACKWARDS
2021 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2022                  int __user *, parent_tidptr,
2023                  unsigned long, tls,
2024                  int __user *, child_tidptr)
2025 #elif defined(CONFIG_CLONE_BACKWARDS2)
2026 SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
2027                  int __user *, parent_tidptr,
2028                  int __user *, child_tidptr,
2029                  unsigned long, tls)
2030 #elif defined(CONFIG_CLONE_BACKWARDS3)
2031 SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
2032                 int, stack_size,
2033                 int __user *, parent_tidptr,
2034                 int __user *, child_tidptr,
2035                 unsigned long, tls)
2036 #else
2037 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2038                  int __user *, parent_tidptr,
2039                  int __user *, child_tidptr,
2040                  unsigned long, tls)
2041 #endif
2042 {
2043         return _do_fork(clone_flags, newsp, 0, parent_tidptr, child_tidptr, tls);
2044 }
2045 #endif
2046 
2047 #ifndef ARCH_MIN_MMSTRUCT_ALIGN
2048 #define ARCH_MIN_MMSTRUCT_ALIGN 0
2049 #endif
2050 
2051 static void sighand_ctor(void *data)
2052 {
2053         struct sighand_struct *sighand = data;
2054 
2055         spin_lock_init(&sighand->siglock);
2056         init_waitqueue_head(&sighand->signalfd_wqh);
2057 }
2058 
2059 void __init proc_caches_init(void)
2060 {
2061         sighand_cachep = kmem_cache_create("sighand_cache",
2062                         sizeof(struct sighand_struct), 0,
2063                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_DESTROY_BY_RCU|
2064                         SLAB_NOTRACK|SLAB_ACCOUNT, sighand_ctor);
2065         signal_cachep = kmem_cache_create("signal_cache",
2066                         sizeof(struct signal_struct), 0,
2067                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,
2068                         NULL);
2069         files_cachep = kmem_cache_create("files_cache",
2070                         sizeof(struct files_struct), 0,
2071                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,
2072                         NULL);
2073         fs_cachep = kmem_cache_create("fs_cache",
2074                         sizeof(struct fs_struct), 0,
2075                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,
2076                         NULL);
2077         /*
2078          * FIXME! The "sizeof(struct mm_struct)" currently includes the
2079          * whole struct cpumask for the OFFSTACK case. We could change
2080          * this to *only* allocate as much of it as required by the
2081          * maximum number of CPU's we can ever have.  The cpumask_allocation
2082          * is at the end of the structure, exactly for that reason.
2083          */
2084         mm_cachep = kmem_cache_create("mm_struct",
2085                         sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
2086                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK|SLAB_ACCOUNT,
2087                         NULL);
2088         vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT);
2089         mmap_init();
2090         nsproxy_cache_init();
2091 }
2092 
2093 /*
2094  * Check constraints on flags passed to the unshare system call.
2095  */
2096 static int check_unshare_flags(unsigned long unshare_flags)
2097 {
2098         if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
2099                                 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
2100                                 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
2101                                 CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP))
2102                 return -EINVAL;
2103         /*
2104          * Not implemented, but pretend it works if there is nothing
2105          * to unshare.  Note that unsharing the address space or the
2106          * signal handlers also need to unshare the signal queues (aka
2107          * CLONE_THREAD).
2108          */
2109         if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
2110                 if (!thread_group_empty(current))
2111                         return -EINVAL;
2112         }
2113         if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
2114                 if (atomic_read(&current->sighand->count) > 1)
2115                         return -EINVAL;
2116         }
2117         if (unshare_flags & CLONE_VM) {
2118                 if (!current_is_single_threaded())
2119                         return -EINVAL;
2120         }
2121 
2122         return 0;
2123 }
2124 
2125 /*
2126  * Unshare the filesystem structure if it is being shared
2127  */
2128 static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
2129 {
2130         struct fs_struct *fs = current->fs;
2131 
2132         if (!(unshare_flags & CLONE_FS) || !fs)
2133                 return 0;
2134 
2135         /* don't need lock here; in the worst case we'll do useless copy */
2136         if (fs->users == 1)
2137                 return 0;
2138 
2139         *new_fsp = copy_fs_struct(fs);
2140         if (!*new_fsp)
2141                 return -ENOMEM;
2142 
2143         return 0;
2144 }
2145 
2146 /*
2147  * Unshare file descriptor table if it is being shared
2148  */
2149 static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
2150 {
2151         struct files_struct *fd = current->files;
2152         int error = 0;
2153 
2154         if ((unshare_flags & CLONE_FILES) &&
2155             (fd && atomic_read(&fd->count) > 1)) {
2156                 *new_fdp = dup_fd(fd, &error);
2157                 if (!*new_fdp)
2158                         return error;
2159         }
2160 
2161         return 0;
2162 }
2163 
2164 /*
2165  * unshare allows a process to 'unshare' part of the process
2166  * context which was originally shared using clone.  copy_*
2167  * functions used by do_fork() cannot be used here directly
2168  * because they modify an inactive task_struct that is being
2169  * constructed. Here we are modifying the current, active,
2170  * task_struct.
2171  */
2172 SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
2173 {
2174         struct fs_struct *fs, *new_fs = NULL;
2175         struct files_struct *fd, *new_fd = NULL;
2176         struct cred *new_cred = NULL;
2177         struct nsproxy *new_nsproxy = NULL;
2178         int do_sysvsem = 0;
2179         int err;
2180 
2181         /*
2182          * If unsharing a user namespace must also unshare the thread group
2183          * and unshare the filesystem root and working directories.
2184          */
2185         if (unshare_flags & CLONE_NEWUSER)
2186                 unshare_flags |= CLONE_THREAD | CLONE_FS;
2187         /*
2188          * If unsharing vm, must also unshare signal handlers.
2189          */
2190         if (unshare_flags & CLONE_VM)
2191                 unshare_flags |= CLONE_SIGHAND;
2192         /*
2193          * If unsharing a signal handlers, must also unshare the signal queues.
2194          */
2195         if (unshare_flags & CLONE_SIGHAND)
2196                 unshare_flags |= CLONE_THREAD;
2197         /*
2198          * If unsharing namespace, must also unshare filesystem information.
2199          */
2200         if (unshare_flags & CLONE_NEWNS)
2201                 unshare_flags |= CLONE_FS;
2202 
2203         err = check_unshare_flags(unshare_flags);
2204         if (err)
2205                 goto bad_unshare_out;
2206         /*
2207          * CLONE_NEWIPC must also detach from the undolist: after switching
2208          * to a new ipc namespace, the semaphore arrays from the old
2209          * namespace are unreachable.
2210          */
2211         if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
2212                 do_sysvsem = 1;
2213         err = unshare_fs(unshare_flags, &new_fs);
2214         if (err)
2215                 goto bad_unshare_out;
2216         err = unshare_fd(unshare_flags, &new_fd);
2217         if (err)
2218                 goto bad_unshare_cleanup_fs;
2219         err = unshare_userns(unshare_flags, &new_cred);
2220         if (err)
2221                 goto bad_unshare_cleanup_fd;
2222         err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
2223                                          new_cred, new_fs);
2224         if (err)
2225                 goto bad_unshare_cleanup_cred;
2226 
2227         if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
2228                 if (do_sysvsem) {
2229                         /*
2230                          * CLONE_SYSVSEM is equivalent to sys_exit().
2231                          */
2232                         exit_sem(current);
2233                 }
2234                 if (unshare_flags & CLONE_NEWIPC) {
2235                         /* Orphan segments in old ns (see sem above). */
2236                         exit_shm(current);
2237                         shm_init_task(current);
2238                 }
2239 
2240                 if (new_nsproxy)
2241                         switch_task_namespaces(current, new_nsproxy);
2242 
2243                 task_lock(current);
2244 
2245                 if (new_fs) {
2246                         fs = current->fs;
2247                         spin_lock(&fs->lock);
2248                         current->fs = new_fs;
2249                         if (--fs->users)
2250                                 new_fs = NULL;
2251                         else
2252                                 new_fs = fs;
2253                         spin_unlock(&fs->lock);
2254                 }
2255 
2256                 if (new_fd) {
2257                         fd = current->files;
2258                         current->files = new_fd;
2259                         new_fd = fd;
2260                 }
2261 
2262                 task_unlock(current);
2263 
2264                 if (new_cred) {
2265                         /* Install the new user namespace */
2266                         commit_creds(new_cred);
2267                         new_cred = NULL;
2268                 }
2269         }
2270 
2271 bad_unshare_cleanup_cred:
2272         if (new_cred)
2273                 put_cred(new_cred);
2274 bad_unshare_cleanup_fd:
2275         if (new_fd)
2276                 put_files_struct(new_fd);
2277 
2278 bad_unshare_cleanup_fs:
2279         if (new_fs)
2280                 free_fs_struct(new_fs);
2281 
2282 bad_unshare_out:
2283         return err;
2284 }
2285 
2286 /*
2287  *      Helper to unshare the files of the current task.
2288  *      We don't want to expose copy_files internals to
2289  *      the exec layer of the kernel.
2290  */
2291 
2292 int unshare_files(struct files_struct **displaced)
2293 {
2294         struct task_struct *task = current;
2295         struct files_struct *copy = NULL;
2296         int error;
2297 
2298         error = unshare_fd(CLONE_FILES, &copy);
2299         if (error || !copy) {
2300                 *displaced = NULL;
2301                 return error;
2302         }
2303         *displaced = task->files;
2304         task_lock(task);
2305         task->files = copy;
2306         task_unlock(task);
2307         return 0;
2308 }
2309 
2310 int sysctl_max_threads(struct ctl_table *table, int write,
2311                        void __user *buffer, size_t *lenp, loff_t *ppos)
2312 {
2313         struct ctl_table t;
2314         int ret;
2315         int threads = max_threads;
2316         int min = MIN_THREADS;
2317         int max = MAX_THREADS;
2318 
2319         t = *table;
2320         t.data = &threads;
2321         t.extra1 = &min;
2322         t.extra2 = &max;
2323 
2324         ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2325         if (ret || !write)
2326                 return ret;
2327 
2328         set_max_threads(threads);
2329 
2330         return 0;
2331 }
2332 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us