Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/kernel/cgroup.c

  1 /*
  2  *  Generic process-grouping system.
  3  *
  4  *  Based originally on the cpuset system, extracted by Paul Menage
  5  *  Copyright (C) 2006 Google, Inc
  6  *
  7  *  Notifications support
  8  *  Copyright (C) 2009 Nokia Corporation
  9  *  Author: Kirill A. Shutemov
 10  *
 11  *  Copyright notices from the original cpuset code:
 12  *  --------------------------------------------------
 13  *  Copyright (C) 2003 BULL SA.
 14  *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 15  *
 16  *  Portions derived from Patrick Mochel's sysfs code.
 17  *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 18  *
 19  *  2003-10-10 Written by Simon Derr.
 20  *  2003-10-22 Updates by Stephen Hemminger.
 21  *  2004 May-July Rework by Paul Jackson.
 22  *  ---------------------------------------------------
 23  *
 24  *  This file is subject to the terms and conditions of the GNU General Public
 25  *  License.  See the file COPYING in the main directory of the Linux
 26  *  distribution for more details.
 27  */
 28 
 29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 30 
 31 #include <linux/cgroup.h>
 32 #include <linux/cred.h>
 33 #include <linux/ctype.h>
 34 #include <linux/errno.h>
 35 #include <linux/init_task.h>
 36 #include <linux/kernel.h>
 37 #include <linux/list.h>
 38 #include <linux/magic.h>
 39 #include <linux/mm.h>
 40 #include <linux/mutex.h>
 41 #include <linux/mount.h>
 42 #include <linux/pagemap.h>
 43 #include <linux/proc_fs.h>
 44 #include <linux/rcupdate.h>
 45 #include <linux/sched.h>
 46 #include <linux/slab.h>
 47 #include <linux/spinlock.h>
 48 #include <linux/rwsem.h>
 49 #include <linux/string.h>
 50 #include <linux/sort.h>
 51 #include <linux/kmod.h>
 52 #include <linux/delayacct.h>
 53 #include <linux/cgroupstats.h>
 54 #include <linux/hashtable.h>
 55 #include <linux/pid_namespace.h>
 56 #include <linux/idr.h>
 57 #include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
 58 #include <linux/kthread.h>
 59 #include <linux/delay.h>
 60 
 61 #include <linux/atomic.h>
 62 
 63 /*
 64  * pidlists linger the following amount before being destroyed.  The goal
 65  * is avoiding frequent destruction in the middle of consecutive read calls
 66  * Expiring in the middle is a performance problem not a correctness one.
 67  * 1 sec should be enough.
 68  */
 69 #define CGROUP_PIDLIST_DESTROY_DELAY    HZ
 70 
 71 #define CGROUP_FILE_NAME_MAX            (MAX_CGROUP_TYPE_NAMELEN +      \
 72                                          MAX_CFTYPE_NAME + 2)
 73 
 74 /*
 75  * cgroup_mutex is the master lock.  Any modification to cgroup or its
 76  * hierarchy must be performed while holding it.
 77  *
 78  * css_set_rwsem protects task->cgroups pointer, the list of css_set
 79  * objects, and the chain of tasks off each css_set.
 80  *
 81  * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 82  * cgroup.h can use them for lockdep annotations.
 83  */
 84 #ifdef CONFIG_PROVE_RCU
 85 DEFINE_MUTEX(cgroup_mutex);
 86 DECLARE_RWSEM(css_set_rwsem);
 87 EXPORT_SYMBOL_GPL(cgroup_mutex);
 88 EXPORT_SYMBOL_GPL(css_set_rwsem);
 89 #else
 90 static DEFINE_MUTEX(cgroup_mutex);
 91 static DECLARE_RWSEM(css_set_rwsem);
 92 #endif
 93 
 94 /*
 95  * Protects cgroup_idr and css_idr so that IDs can be released without
 96  * grabbing cgroup_mutex.
 97  */
 98 static DEFINE_SPINLOCK(cgroup_idr_lock);
 99 
100 /*
101  * Protects cgroup_subsys->release_agent_path.  Modifying it also requires
102  * cgroup_mutex.  Reading requires either cgroup_mutex or this spinlock.
103  */
104 static DEFINE_SPINLOCK(release_agent_path_lock);
105 
106 #define cgroup_assert_mutex_or_rcu_locked()                             \
107         rcu_lockdep_assert(rcu_read_lock_held() ||                      \
108                            lockdep_is_held(&cgroup_mutex),              \
109                            "cgroup_mutex or RCU read lock required");
110 
111 /*
112  * cgroup destruction makes heavy use of work items and there can be a lot
113  * of concurrent destructions.  Use a separate workqueue so that cgroup
114  * destruction work items don't end up filling up max_active of system_wq
115  * which may lead to deadlock.
116  */
117 static struct workqueue_struct *cgroup_destroy_wq;
118 
119 /*
120  * pidlist destructions need to be flushed on cgroup destruction.  Use a
121  * separate workqueue as flush domain.
122  */
123 static struct workqueue_struct *cgroup_pidlist_destroy_wq;
124 
125 /* generate an array of cgroup subsystem pointers */
126 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
127 static struct cgroup_subsys *cgroup_subsys[] = {
128 #include <linux/cgroup_subsys.h>
129 };
130 #undef SUBSYS
131 
132 /* array of cgroup subsystem names */
133 #define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
134 static const char *cgroup_subsys_name[] = {
135 #include <linux/cgroup_subsys.h>
136 };
137 #undef SUBSYS
138 
139 /*
140  * The default hierarchy, reserved for the subsystems that are otherwise
141  * unattached - it never has more than a single cgroup, and all tasks are
142  * part of that cgroup.
143  */
144 struct cgroup_root cgrp_dfl_root;
145 
146 /*
147  * The default hierarchy always exists but is hidden until mounted for the
148  * first time.  This is for backward compatibility.
149  */
150 static bool cgrp_dfl_root_visible;
151 
152 /*
153  * Set by the boot param of the same name and makes subsystems with NULL
154  * ->dfl_files to use ->legacy_files on the default hierarchy.
155  */
156 static bool cgroup_legacy_files_on_dfl;
157 
158 /* some controllers are not supported in the default hierarchy */
159 static unsigned int cgrp_dfl_root_inhibit_ss_mask;
160 
161 /* The list of hierarchy roots */
162 
163 static LIST_HEAD(cgroup_roots);
164 static int cgroup_root_count;
165 
166 /* hierarchy ID allocation and mapping, protected by cgroup_mutex */
167 static DEFINE_IDR(cgroup_hierarchy_idr);
168 
169 /*
170  * Assign a monotonically increasing serial number to csses.  It guarantees
171  * cgroups with bigger numbers are newer than those with smaller numbers.
172  * Also, as csses are always appended to the parent's ->children list, it
173  * guarantees that sibling csses are always sorted in the ascending serial
174  * number order on the list.  Protected by cgroup_mutex.
175  */
176 static u64 css_serial_nr_next = 1;
177 
178 /* This flag indicates whether tasks in the fork and exit paths should
179  * check for fork/exit handlers to call. This avoids us having to do
180  * extra work in the fork/exit path if none of the subsystems need to
181  * be called.
182  */
183 static int need_forkexit_callback __read_mostly;
184 
185 static struct cftype cgroup_dfl_base_files[];
186 static struct cftype cgroup_legacy_base_files[];
187 
188 static void cgroup_put(struct cgroup *cgrp);
189 static int rebind_subsystems(struct cgroup_root *dst_root,
190                              unsigned int ss_mask);
191 static int cgroup_destroy_locked(struct cgroup *cgrp);
192 static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
193                       bool visible);
194 static void css_release(struct percpu_ref *ref);
195 static void kill_css(struct cgroup_subsys_state *css);
196 static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
197                               bool is_add);
198 static void cgroup_pidlist_destroy_all(struct cgroup *cgrp);
199 
200 /* IDR wrappers which synchronize using cgroup_idr_lock */
201 static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
202                             gfp_t gfp_mask)
203 {
204         int ret;
205 
206         idr_preload(gfp_mask);
207         spin_lock_bh(&cgroup_idr_lock);
208         ret = idr_alloc(idr, ptr, start, end, gfp_mask);
209         spin_unlock_bh(&cgroup_idr_lock);
210         idr_preload_end();
211         return ret;
212 }
213 
214 static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
215 {
216         void *ret;
217 
218         spin_lock_bh(&cgroup_idr_lock);
219         ret = idr_replace(idr, ptr, id);
220         spin_unlock_bh(&cgroup_idr_lock);
221         return ret;
222 }
223 
224 static void cgroup_idr_remove(struct idr *idr, int id)
225 {
226         spin_lock_bh(&cgroup_idr_lock);
227         idr_remove(idr, id);
228         spin_unlock_bh(&cgroup_idr_lock);
229 }
230 
231 static struct cgroup *cgroup_parent(struct cgroup *cgrp)
232 {
233         struct cgroup_subsys_state *parent_css = cgrp->self.parent;
234 
235         if (parent_css)
236                 return container_of(parent_css, struct cgroup, self);
237         return NULL;
238 }
239 
240 /**
241  * cgroup_css - obtain a cgroup's css for the specified subsystem
242  * @cgrp: the cgroup of interest
243  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
244  *
245  * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
246  * function must be called either under cgroup_mutex or rcu_read_lock() and
247  * the caller is responsible for pinning the returned css if it wants to
248  * keep accessing it outside the said locks.  This function may return
249  * %NULL if @cgrp doesn't have @subsys_id enabled.
250  */
251 static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
252                                               struct cgroup_subsys *ss)
253 {
254         if (ss)
255                 return rcu_dereference_check(cgrp->subsys[ss->id],
256                                         lockdep_is_held(&cgroup_mutex));
257         else
258                 return &cgrp->self;
259 }
260 
261 /**
262  * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
263  * @cgrp: the cgroup of interest
264  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
265  *
266  * Similar to cgroup_css() but returns the effctive css, which is defined
267  * as the matching css of the nearest ancestor including self which has @ss
268  * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
269  * function is guaranteed to return non-NULL css.
270  */
271 static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
272                                                 struct cgroup_subsys *ss)
273 {
274         lockdep_assert_held(&cgroup_mutex);
275 
276         if (!ss)
277                 return &cgrp->self;
278 
279         if (!(cgrp->root->subsys_mask & (1 << ss->id)))
280                 return NULL;
281 
282         while (cgroup_parent(cgrp) &&
283                !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id)))
284                 cgrp = cgroup_parent(cgrp);
285 
286         return cgroup_css(cgrp, ss);
287 }
288 
289 /* convenient tests for these bits */
290 static inline bool cgroup_is_dead(const struct cgroup *cgrp)
291 {
292         return !(cgrp->self.flags & CSS_ONLINE);
293 }
294 
295 struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
296 {
297         struct cgroup *cgrp = of->kn->parent->priv;
298         struct cftype *cft = of_cft(of);
299 
300         /*
301          * This is open and unprotected implementation of cgroup_css().
302          * seq_css() is only called from a kernfs file operation which has
303          * an active reference on the file.  Because all the subsystem
304          * files are drained before a css is disassociated with a cgroup,
305          * the matching css from the cgroup's subsys table is guaranteed to
306          * be and stay valid until the enclosing operation is complete.
307          */
308         if (cft->ss)
309                 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
310         else
311                 return &cgrp->self;
312 }
313 EXPORT_SYMBOL_GPL(of_css);
314 
315 /**
316  * cgroup_is_descendant - test ancestry
317  * @cgrp: the cgroup to be tested
318  * @ancestor: possible ancestor of @cgrp
319  *
320  * Test whether @cgrp is a descendant of @ancestor.  It also returns %true
321  * if @cgrp == @ancestor.  This function is safe to call as long as @cgrp
322  * and @ancestor are accessible.
323  */
324 bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
325 {
326         while (cgrp) {
327                 if (cgrp == ancestor)
328                         return true;
329                 cgrp = cgroup_parent(cgrp);
330         }
331         return false;
332 }
333 
334 static int cgroup_is_releasable(const struct cgroup *cgrp)
335 {
336         const int bits =
337                 (1 << CGRP_RELEASABLE) |
338                 (1 << CGRP_NOTIFY_ON_RELEASE);
339         return (cgrp->flags & bits) == bits;
340 }
341 
342 static int notify_on_release(const struct cgroup *cgrp)
343 {
344         return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
345 }
346 
347 /**
348  * for_each_css - iterate all css's of a cgroup
349  * @css: the iteration cursor
350  * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
351  * @cgrp: the target cgroup to iterate css's of
352  *
353  * Should be called under cgroup_[tree_]mutex.
354  */
355 #define for_each_css(css, ssid, cgrp)                                   \
356         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)        \
357                 if (!((css) = rcu_dereference_check(                    \
358                                 (cgrp)->subsys[(ssid)],                 \
359                                 lockdep_is_held(&cgroup_mutex)))) { }   \
360                 else
361 
362 /**
363  * for_each_e_css - iterate all effective css's of a cgroup
364  * @css: the iteration cursor
365  * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
366  * @cgrp: the target cgroup to iterate css's of
367  *
368  * Should be called under cgroup_[tree_]mutex.
369  */
370 #define for_each_e_css(css, ssid, cgrp)                                 \
371         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)        \
372                 if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
373                         ;                                               \
374                 else
375 
376 /**
377  * for_each_subsys - iterate all enabled cgroup subsystems
378  * @ss: the iteration cursor
379  * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
380  */
381 #define for_each_subsys(ss, ssid)                                       \
382         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT &&                \
383              (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
384 
385 /* iterate across the hierarchies */
386 #define for_each_root(root)                                             \
387         list_for_each_entry((root), &cgroup_roots, root_list)
388 
389 /* iterate over child cgrps, lock should be held throughout iteration */
390 #define cgroup_for_each_live_child(child, cgrp)                         \
391         list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
392                 if (({ lockdep_assert_held(&cgroup_mutex);              \
393                        cgroup_is_dead(child); }))                       \
394                         ;                                               \
395                 else
396 
397 /* the list of cgroups eligible for automatic release. Protected by
398  * release_list_lock */
399 static LIST_HEAD(release_list);
400 static DEFINE_RAW_SPINLOCK(release_list_lock);
401 static void cgroup_release_agent(struct work_struct *work);
402 static DECLARE_WORK(release_agent_work, cgroup_release_agent);
403 static void check_for_release(struct cgroup *cgrp);
404 
405 /*
406  * A cgroup can be associated with multiple css_sets as different tasks may
407  * belong to different cgroups on different hierarchies.  In the other
408  * direction, a css_set is naturally associated with multiple cgroups.
409  * This M:N relationship is represented by the following link structure
410  * which exists for each association and allows traversing the associations
411  * from both sides.
412  */
413 struct cgrp_cset_link {
414         /* the cgroup and css_set this link associates */
415         struct cgroup           *cgrp;
416         struct css_set          *cset;
417 
418         /* list of cgrp_cset_links anchored at cgrp->cset_links */
419         struct list_head        cset_link;
420 
421         /* list of cgrp_cset_links anchored at css_set->cgrp_links */
422         struct list_head        cgrp_link;
423 };
424 
425 /*
426  * The default css_set - used by init and its children prior to any
427  * hierarchies being mounted. It contains a pointer to the root state
428  * for each subsystem. Also used to anchor the list of css_sets. Not
429  * reference-counted, to improve performance when child cgroups
430  * haven't been created.
431  */
432 struct css_set init_css_set = {
433         .refcount               = ATOMIC_INIT(1),
434         .cgrp_links             = LIST_HEAD_INIT(init_css_set.cgrp_links),
435         .tasks                  = LIST_HEAD_INIT(init_css_set.tasks),
436         .mg_tasks               = LIST_HEAD_INIT(init_css_set.mg_tasks),
437         .mg_preload_node        = LIST_HEAD_INIT(init_css_set.mg_preload_node),
438         .mg_node                = LIST_HEAD_INIT(init_css_set.mg_node),
439 };
440 
441 static int css_set_count        = 1;    /* 1 for init_css_set */
442 
443 /**
444  * cgroup_update_populated - updated populated count of a cgroup
445  * @cgrp: the target cgroup
446  * @populated: inc or dec populated count
447  *
448  * @cgrp is either getting the first task (css_set) or losing the last.
449  * Update @cgrp->populated_cnt accordingly.  The count is propagated
450  * towards root so that a given cgroup's populated_cnt is zero iff the
451  * cgroup and all its descendants are empty.
452  *
453  * @cgrp's interface file "cgroup.populated" is zero if
454  * @cgrp->populated_cnt is zero and 1 otherwise.  When @cgrp->populated_cnt
455  * changes from or to zero, userland is notified that the content of the
456  * interface file has changed.  This can be used to detect when @cgrp and
457  * its descendants become populated or empty.
458  */
459 static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
460 {
461         lockdep_assert_held(&css_set_rwsem);
462 
463         do {
464                 bool trigger;
465 
466                 if (populated)
467                         trigger = !cgrp->populated_cnt++;
468                 else
469                         trigger = !--cgrp->populated_cnt;
470 
471                 if (!trigger)
472                         break;
473 
474                 if (cgrp->populated_kn)
475                         kernfs_notify(cgrp->populated_kn);
476                 cgrp = cgroup_parent(cgrp);
477         } while (cgrp);
478 }
479 
480 /*
481  * hash table for cgroup groups. This improves the performance to find
482  * an existing css_set. This hash doesn't (currently) take into
483  * account cgroups in empty hierarchies.
484  */
485 #define CSS_SET_HASH_BITS       7
486 static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
487 
488 static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
489 {
490         unsigned long key = 0UL;
491         struct cgroup_subsys *ss;
492         int i;
493 
494         for_each_subsys(ss, i)
495                 key += (unsigned long)css[i];
496         key = (key >> 16) ^ key;
497 
498         return key;
499 }
500 
501 static void put_css_set_locked(struct css_set *cset, bool taskexit)
502 {
503         struct cgrp_cset_link *link, *tmp_link;
504         struct cgroup_subsys *ss;
505         int ssid;
506 
507         lockdep_assert_held(&css_set_rwsem);
508 
509         if (!atomic_dec_and_test(&cset->refcount))
510                 return;
511 
512         /* This css_set is dead. unlink it and release cgroup refcounts */
513         for_each_subsys(ss, ssid)
514                 list_del(&cset->e_cset_node[ssid]);
515         hash_del(&cset->hlist);
516         css_set_count--;
517 
518         list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
519                 struct cgroup *cgrp = link->cgrp;
520 
521                 list_del(&link->cset_link);
522                 list_del(&link->cgrp_link);
523 
524                 /* @cgrp can't go away while we're holding css_set_rwsem */
525                 if (list_empty(&cgrp->cset_links)) {
526                         cgroup_update_populated(cgrp, false);
527                         if (notify_on_release(cgrp)) {
528                                 if (taskexit)
529                                         set_bit(CGRP_RELEASABLE, &cgrp->flags);
530                                 check_for_release(cgrp);
531                         }
532                 }
533 
534                 kfree(link);
535         }
536 
537         kfree_rcu(cset, rcu_head);
538 }
539 
540 static void put_css_set(struct css_set *cset, bool taskexit)
541 {
542         /*
543          * Ensure that the refcount doesn't hit zero while any readers
544          * can see it. Similar to atomic_dec_and_lock(), but for an
545          * rwlock
546          */
547         if (atomic_add_unless(&cset->refcount, -1, 1))
548                 return;
549 
550         down_write(&css_set_rwsem);
551         put_css_set_locked(cset, taskexit);
552         up_write(&css_set_rwsem);
553 }
554 
555 /*
556  * refcounted get/put for css_set objects
557  */
558 static inline void get_css_set(struct css_set *cset)
559 {
560         atomic_inc(&cset->refcount);
561 }
562 
563 /**
564  * compare_css_sets - helper function for find_existing_css_set().
565  * @cset: candidate css_set being tested
566  * @old_cset: existing css_set for a task
567  * @new_cgrp: cgroup that's being entered by the task
568  * @template: desired set of css pointers in css_set (pre-calculated)
569  *
570  * Returns true if "cset" matches "old_cset" except for the hierarchy
571  * which "new_cgrp" belongs to, for which it should match "new_cgrp".
572  */
573 static bool compare_css_sets(struct css_set *cset,
574                              struct css_set *old_cset,
575                              struct cgroup *new_cgrp,
576                              struct cgroup_subsys_state *template[])
577 {
578         struct list_head *l1, *l2;
579 
580         /*
581          * On the default hierarchy, there can be csets which are
582          * associated with the same set of cgroups but different csses.
583          * Let's first ensure that csses match.
584          */
585         if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
586                 return false;
587 
588         /*
589          * Compare cgroup pointers in order to distinguish between
590          * different cgroups in hierarchies.  As different cgroups may
591          * share the same effective css, this comparison is always
592          * necessary.
593          */
594         l1 = &cset->cgrp_links;
595         l2 = &old_cset->cgrp_links;
596         while (1) {
597                 struct cgrp_cset_link *link1, *link2;
598                 struct cgroup *cgrp1, *cgrp2;
599 
600                 l1 = l1->next;
601                 l2 = l2->next;
602                 /* See if we reached the end - both lists are equal length. */
603                 if (l1 == &cset->cgrp_links) {
604                         BUG_ON(l2 != &old_cset->cgrp_links);
605                         break;
606                 } else {
607                         BUG_ON(l2 == &old_cset->cgrp_links);
608                 }
609                 /* Locate the cgroups associated with these links. */
610                 link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
611                 link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
612                 cgrp1 = link1->cgrp;
613                 cgrp2 = link2->cgrp;
614                 /* Hierarchies should be linked in the same order. */
615                 BUG_ON(cgrp1->root != cgrp2->root);
616 
617                 /*
618                  * If this hierarchy is the hierarchy of the cgroup
619                  * that's changing, then we need to check that this
620                  * css_set points to the new cgroup; if it's any other
621                  * hierarchy, then this css_set should point to the
622                  * same cgroup as the old css_set.
623                  */
624                 if (cgrp1->root == new_cgrp->root) {
625                         if (cgrp1 != new_cgrp)
626                                 return false;
627                 } else {
628                         if (cgrp1 != cgrp2)
629                                 return false;
630                 }
631         }
632         return true;
633 }
634 
635 /**
636  * find_existing_css_set - init css array and find the matching css_set
637  * @old_cset: the css_set that we're using before the cgroup transition
638  * @cgrp: the cgroup that we're moving into
639  * @template: out param for the new set of csses, should be clear on entry
640  */
641 static struct css_set *find_existing_css_set(struct css_set *old_cset,
642                                         struct cgroup *cgrp,
643                                         struct cgroup_subsys_state *template[])
644 {
645         struct cgroup_root *root = cgrp->root;
646         struct cgroup_subsys *ss;
647         struct css_set *cset;
648         unsigned long key;
649         int i;
650 
651         /*
652          * Build the set of subsystem state objects that we want to see in the
653          * new css_set. while subsystems can change globally, the entries here
654          * won't change, so no need for locking.
655          */
656         for_each_subsys(ss, i) {
657                 if (root->subsys_mask & (1UL << i)) {
658                         /*
659                          * @ss is in this hierarchy, so we want the
660                          * effective css from @cgrp.
661                          */
662                         template[i] = cgroup_e_css(cgrp, ss);
663                 } else {
664                         /*
665                          * @ss is not in this hierarchy, so we don't want
666                          * to change the css.
667                          */
668                         template[i] = old_cset->subsys[i];
669                 }
670         }
671 
672         key = css_set_hash(template);
673         hash_for_each_possible(css_set_table, cset, hlist, key) {
674                 if (!compare_css_sets(cset, old_cset, cgrp, template))
675                         continue;
676 
677                 /* This css_set matches what we need */
678                 return cset;
679         }
680 
681         /* No existing cgroup group matched */
682         return NULL;
683 }
684 
685 static void free_cgrp_cset_links(struct list_head *links_to_free)
686 {
687         struct cgrp_cset_link *link, *tmp_link;
688 
689         list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
690                 list_del(&link->cset_link);
691                 kfree(link);
692         }
693 }
694 
695 /**
696  * allocate_cgrp_cset_links - allocate cgrp_cset_links
697  * @count: the number of links to allocate
698  * @tmp_links: list_head the allocated links are put on
699  *
700  * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
701  * through ->cset_link.  Returns 0 on success or -errno.
702  */
703 static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
704 {
705         struct cgrp_cset_link *link;
706         int i;
707 
708         INIT_LIST_HEAD(tmp_links);
709 
710         for (i = 0; i < count; i++) {
711                 link = kzalloc(sizeof(*link), GFP_KERNEL);
712                 if (!link) {
713                         free_cgrp_cset_links(tmp_links);
714                         return -ENOMEM;
715                 }
716                 list_add(&link->cset_link, tmp_links);
717         }
718         return 0;
719 }
720 
721 /**
722  * link_css_set - a helper function to link a css_set to a cgroup
723  * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
724  * @cset: the css_set to be linked
725  * @cgrp: the destination cgroup
726  */
727 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
728                          struct cgroup *cgrp)
729 {
730         struct cgrp_cset_link *link;
731 
732         BUG_ON(list_empty(tmp_links));
733 
734         if (cgroup_on_dfl(cgrp))
735                 cset->dfl_cgrp = cgrp;
736 
737         link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
738         link->cset = cset;
739         link->cgrp = cgrp;
740 
741         if (list_empty(&cgrp->cset_links))
742                 cgroup_update_populated(cgrp, true);
743         list_move(&link->cset_link, &cgrp->cset_links);
744 
745         /*
746          * Always add links to the tail of the list so that the list
747          * is sorted by order of hierarchy creation
748          */
749         list_add_tail(&link->cgrp_link, &cset->cgrp_links);
750 }
751 
752 /**
753  * find_css_set - return a new css_set with one cgroup updated
754  * @old_cset: the baseline css_set
755  * @cgrp: the cgroup to be updated
756  *
757  * Return a new css_set that's equivalent to @old_cset, but with @cgrp
758  * substituted into the appropriate hierarchy.
759  */
760 static struct css_set *find_css_set(struct css_set *old_cset,
761                                     struct cgroup *cgrp)
762 {
763         struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
764         struct css_set *cset;
765         struct list_head tmp_links;
766         struct cgrp_cset_link *link;
767         struct cgroup_subsys *ss;
768         unsigned long key;
769         int ssid;
770 
771         lockdep_assert_held(&cgroup_mutex);
772 
773         /* First see if we already have a cgroup group that matches
774          * the desired set */
775         down_read(&css_set_rwsem);
776         cset = find_existing_css_set(old_cset, cgrp, template);
777         if (cset)
778                 get_css_set(cset);
779         up_read(&css_set_rwsem);
780 
781         if (cset)
782                 return cset;
783 
784         cset = kzalloc(sizeof(*cset), GFP_KERNEL);
785         if (!cset)
786                 return NULL;
787 
788         /* Allocate all the cgrp_cset_link objects that we'll need */
789         if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
790                 kfree(cset);
791                 return NULL;
792         }
793 
794         atomic_set(&cset->refcount, 1);
795         INIT_LIST_HEAD(&cset->cgrp_links);
796         INIT_LIST_HEAD(&cset->tasks);
797         INIT_LIST_HEAD(&cset->mg_tasks);
798         INIT_LIST_HEAD(&cset->mg_preload_node);
799         INIT_LIST_HEAD(&cset->mg_node);
800         INIT_HLIST_NODE(&cset->hlist);
801 
802         /* Copy the set of subsystem state objects generated in
803          * find_existing_css_set() */
804         memcpy(cset->subsys, template, sizeof(cset->subsys));
805 
806         down_write(&css_set_rwsem);
807         /* Add reference counts and links from the new css_set. */
808         list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
809                 struct cgroup *c = link->cgrp;
810 
811                 if (c->root == cgrp->root)
812                         c = cgrp;
813                 link_css_set(&tmp_links, cset, c);
814         }
815 
816         BUG_ON(!list_empty(&tmp_links));
817 
818         css_set_count++;
819 
820         /* Add @cset to the hash table */
821         key = css_set_hash(cset->subsys);
822         hash_add(css_set_table, &cset->hlist, key);
823 
824         for_each_subsys(ss, ssid)
825                 list_add_tail(&cset->e_cset_node[ssid],
826                               &cset->subsys[ssid]->cgroup->e_csets[ssid]);
827 
828         up_write(&css_set_rwsem);
829 
830         return cset;
831 }
832 
833 static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
834 {
835         struct cgroup *root_cgrp = kf_root->kn->priv;
836 
837         return root_cgrp->root;
838 }
839 
840 static int cgroup_init_root_id(struct cgroup_root *root)
841 {
842         int id;
843 
844         lockdep_assert_held(&cgroup_mutex);
845 
846         id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
847         if (id < 0)
848                 return id;
849 
850         root->hierarchy_id = id;
851         return 0;
852 }
853 
854 static void cgroup_exit_root_id(struct cgroup_root *root)
855 {
856         lockdep_assert_held(&cgroup_mutex);
857 
858         if (root->hierarchy_id) {
859                 idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
860                 root->hierarchy_id = 0;
861         }
862 }
863 
864 static void cgroup_free_root(struct cgroup_root *root)
865 {
866         if (root) {
867                 /* hierarhcy ID shoulid already have been released */
868                 WARN_ON_ONCE(root->hierarchy_id);
869 
870                 idr_destroy(&root->cgroup_idr);
871                 kfree(root);
872         }
873 }
874 
875 static void cgroup_destroy_root(struct cgroup_root *root)
876 {
877         struct cgroup *cgrp = &root->cgrp;
878         struct cgrp_cset_link *link, *tmp_link;
879 
880         mutex_lock(&cgroup_mutex);
881 
882         BUG_ON(atomic_read(&root->nr_cgrps));
883         BUG_ON(!list_empty(&cgrp->self.children));
884 
885         /* Rebind all subsystems back to the default hierarchy */
886         rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
887 
888         /*
889          * Release all the links from cset_links to this hierarchy's
890          * root cgroup
891          */
892         down_write(&css_set_rwsem);
893 
894         list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
895                 list_del(&link->cset_link);
896                 list_del(&link->cgrp_link);
897                 kfree(link);
898         }
899         up_write(&css_set_rwsem);
900 
901         if (!list_empty(&root->root_list)) {
902                 list_del(&root->root_list);
903                 cgroup_root_count--;
904         }
905 
906         cgroup_exit_root_id(root);
907 
908         mutex_unlock(&cgroup_mutex);
909 
910         kernfs_destroy_root(root->kf_root);
911         cgroup_free_root(root);
912 }
913 
914 /* look up cgroup associated with given css_set on the specified hierarchy */
915 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
916                                             struct cgroup_root *root)
917 {
918         struct cgroup *res = NULL;
919 
920         lockdep_assert_held(&cgroup_mutex);
921         lockdep_assert_held(&css_set_rwsem);
922 
923         if (cset == &init_css_set) {
924                 res = &root->cgrp;
925         } else {
926                 struct cgrp_cset_link *link;
927 
928                 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
929                         struct cgroup *c = link->cgrp;
930 
931                         if (c->root == root) {
932                                 res = c;
933                                 break;
934                         }
935                 }
936         }
937 
938         BUG_ON(!res);
939         return res;
940 }
941 
942 /*
943  * Return the cgroup for "task" from the given hierarchy. Must be
944  * called with cgroup_mutex and css_set_rwsem held.
945  */
946 static struct cgroup *task_cgroup_from_root(struct task_struct *task,
947                                             struct cgroup_root *root)
948 {
949         /*
950          * No need to lock the task - since we hold cgroup_mutex the
951          * task can't change groups, so the only thing that can happen
952          * is that it exits and its css is set back to init_css_set.
953          */
954         return cset_cgroup_from_root(task_css_set(task), root);
955 }
956 
957 /*
958  * A task must hold cgroup_mutex to modify cgroups.
959  *
960  * Any task can increment and decrement the count field without lock.
961  * So in general, code holding cgroup_mutex can't rely on the count
962  * field not changing.  However, if the count goes to zero, then only
963  * cgroup_attach_task() can increment it again.  Because a count of zero
964  * means that no tasks are currently attached, therefore there is no
965  * way a task attached to that cgroup can fork (the other way to
966  * increment the count).  So code holding cgroup_mutex can safely
967  * assume that if the count is zero, it will stay zero. Similarly, if
968  * a task holds cgroup_mutex on a cgroup with zero count, it
969  * knows that the cgroup won't be removed, as cgroup_rmdir()
970  * needs that mutex.
971  *
972  * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
973  * (usually) take cgroup_mutex.  These are the two most performance
974  * critical pieces of code here.  The exception occurs on cgroup_exit(),
975  * when a task in a notify_on_release cgroup exits.  Then cgroup_mutex
976  * is taken, and if the cgroup count is zero, a usermode call made
977  * to the release agent with the name of the cgroup (path relative to
978  * the root of cgroup file system) as the argument.
979  *
980  * A cgroup can only be deleted if both its 'count' of using tasks
981  * is zero, and its list of 'children' cgroups is empty.  Since all
982  * tasks in the system use _some_ cgroup, and since there is always at
983  * least one task in the system (init, pid == 1), therefore, root cgroup
984  * always has either children cgroups and/or using tasks.  So we don't
985  * need a special hack to ensure that root cgroup cannot be deleted.
986  *
987  * P.S.  One more locking exception.  RCU is used to guard the
988  * update of a tasks cgroup pointer by cgroup_attach_task()
989  */
990 
991 static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask);
992 static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
993 static const struct file_operations proc_cgroupstats_operations;
994 
995 static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
996                               char *buf)
997 {
998         if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
999             !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
1000                 snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
1001                          cft->ss->name, cft->name);
1002         else
1003                 strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
1004         return buf;
1005 }
1006 
1007 /**
1008  * cgroup_file_mode - deduce file mode of a control file
1009  * @cft: the control file in question
1010  *
1011  * returns cft->mode if ->mode is not 0
1012  * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
1013  * returns S_IRUGO if it has only a read handler
1014  * returns S_IWUSR if it has only a write hander
1015  */
1016 static umode_t cgroup_file_mode(const struct cftype *cft)
1017 {
1018         umode_t mode = 0;
1019 
1020         if (cft->mode)
1021                 return cft->mode;
1022 
1023         if (cft->read_u64 || cft->read_s64 || cft->seq_show)
1024                 mode |= S_IRUGO;
1025 
1026         if (cft->write_u64 || cft->write_s64 || cft->write)
1027                 mode |= S_IWUSR;
1028 
1029         return mode;
1030 }
1031 
1032 static void cgroup_get(struct cgroup *cgrp)
1033 {
1034         WARN_ON_ONCE(cgroup_is_dead(cgrp));
1035         css_get(&cgrp->self);
1036 }
1037 
1038 static bool cgroup_tryget(struct cgroup *cgrp)
1039 {
1040         return css_tryget(&cgrp->self);
1041 }
1042 
1043 static void cgroup_put(struct cgroup *cgrp)
1044 {
1045         css_put(&cgrp->self);
1046 }
1047 
1048 /**
1049  * cgroup_refresh_child_subsys_mask - update child_subsys_mask
1050  * @cgrp: the target cgroup
1051  *
1052  * On the default hierarchy, a subsystem may request other subsystems to be
1053  * enabled together through its ->depends_on mask.  In such cases, more
1054  * subsystems than specified in "cgroup.subtree_control" may be enabled.
1055  *
1056  * This function determines which subsystems need to be enabled given the
1057  * current @cgrp->subtree_control and records it in
1058  * @cgrp->child_subsys_mask.  The resulting mask is always a superset of
1059  * @cgrp->subtree_control and follows the usual hierarchy rules.
1060  */
1061 static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp)
1062 {
1063         struct cgroup *parent = cgroup_parent(cgrp);
1064         unsigned int cur_ss_mask = cgrp->subtree_control;
1065         struct cgroup_subsys *ss;
1066         int ssid;
1067 
1068         lockdep_assert_held(&cgroup_mutex);
1069 
1070         if (!cgroup_on_dfl(cgrp)) {
1071                 cgrp->child_subsys_mask = cur_ss_mask;
1072                 return;
1073         }
1074 
1075         while (true) {
1076                 unsigned int new_ss_mask = cur_ss_mask;
1077 
1078                 for_each_subsys(ss, ssid)
1079                         if (cur_ss_mask & (1 << ssid))
1080                                 new_ss_mask |= ss->depends_on;
1081 
1082                 /*
1083                  * Mask out subsystems which aren't available.  This can
1084                  * happen only if some depended-upon subsystems were bound
1085                  * to non-default hierarchies.
1086                  */
1087                 if (parent)
1088                         new_ss_mask &= parent->child_subsys_mask;
1089                 else
1090                         new_ss_mask &= cgrp->root->subsys_mask;
1091 
1092                 if (new_ss_mask == cur_ss_mask)
1093                         break;
1094                 cur_ss_mask = new_ss_mask;
1095         }
1096 
1097         cgrp->child_subsys_mask = cur_ss_mask;
1098 }
1099 
1100 /**
1101  * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1102  * @kn: the kernfs_node being serviced
1103  *
1104  * This helper undoes cgroup_kn_lock_live() and should be invoked before
1105  * the method finishes if locking succeeded.  Note that once this function
1106  * returns the cgroup returned by cgroup_kn_lock_live() may become
1107  * inaccessible any time.  If the caller intends to continue to access the
1108  * cgroup, it should pin it before invoking this function.
1109  */
1110 static void cgroup_kn_unlock(struct kernfs_node *kn)
1111 {
1112         struct cgroup *cgrp;
1113 
1114         if (kernfs_type(kn) == KERNFS_DIR)
1115                 cgrp = kn->priv;
1116         else
1117                 cgrp = kn->parent->priv;
1118 
1119         mutex_unlock(&cgroup_mutex);
1120 
1121         kernfs_unbreak_active_protection(kn);
1122         cgroup_put(cgrp);
1123 }
1124 
1125 /**
1126  * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1127  * @kn: the kernfs_node being serviced
1128  *
1129  * This helper is to be used by a cgroup kernfs method currently servicing
1130  * @kn.  It breaks the active protection, performs cgroup locking and
1131  * verifies that the associated cgroup is alive.  Returns the cgroup if
1132  * alive; otherwise, %NULL.  A successful return should be undone by a
1133  * matching cgroup_kn_unlock() invocation.
1134  *
1135  * Any cgroup kernfs method implementation which requires locking the
1136  * associated cgroup should use this helper.  It avoids nesting cgroup
1137  * locking under kernfs active protection and allows all kernfs operations
1138  * including self-removal.
1139  */
1140 static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn)
1141 {
1142         struct cgroup *cgrp;
1143 
1144         if (kernfs_type(kn) == KERNFS_DIR)
1145                 cgrp = kn->priv;
1146         else
1147                 cgrp = kn->parent->priv;
1148 
1149         /*
1150          * We're gonna grab cgroup_mutex which nests outside kernfs
1151          * active_ref.  cgroup liveliness check alone provides enough
1152          * protection against removal.  Ensure @cgrp stays accessible and
1153          * break the active_ref protection.
1154          */
1155         if (!cgroup_tryget(cgrp))
1156                 return NULL;
1157         kernfs_break_active_protection(kn);
1158 
1159         mutex_lock(&cgroup_mutex);
1160 
1161         if (!cgroup_is_dead(cgrp))
1162                 return cgrp;
1163 
1164         cgroup_kn_unlock(kn);
1165         return NULL;
1166 }
1167 
1168 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
1169 {
1170         char name[CGROUP_FILE_NAME_MAX];
1171 
1172         lockdep_assert_held(&cgroup_mutex);
1173         kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
1174 }
1175 
1176 /**
1177  * cgroup_clear_dir - remove subsys files in a cgroup directory
1178  * @cgrp: target cgroup
1179  * @subsys_mask: mask of the subsystem ids whose files should be removed
1180  */
1181 static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask)
1182 {
1183         struct cgroup_subsys *ss;
1184         int i;
1185 
1186         for_each_subsys(ss, i) {
1187                 struct cftype *cfts;
1188 
1189                 if (!(subsys_mask & (1 << i)))
1190                         continue;
1191                 list_for_each_entry(cfts, &ss->cfts, node)
1192                         cgroup_addrm_files(cgrp, cfts, false);
1193         }
1194 }
1195 
1196 static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask)
1197 {
1198         struct cgroup_subsys *ss;
1199         unsigned int tmp_ss_mask;
1200         int ssid, i, ret;
1201 
1202         lockdep_assert_held(&cgroup_mutex);
1203 
1204         for_each_subsys(ss, ssid) {
1205                 if (!(ss_mask & (1 << ssid)))
1206                         continue;
1207 
1208                 /* if @ss has non-root csses attached to it, can't move */
1209                 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)))
1210                         return -EBUSY;
1211 
1212                 /* can't move between two non-dummy roots either */
1213                 if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1214                         return -EBUSY;
1215         }
1216 
1217         /* skip creating root files on dfl_root for inhibited subsystems */
1218         tmp_ss_mask = ss_mask;
1219         if (dst_root == &cgrp_dfl_root)
1220                 tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask;
1221 
1222         ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask);
1223         if (ret) {
1224                 if (dst_root != &cgrp_dfl_root)
1225                         return ret;
1226 
1227                 /*
1228                  * Rebinding back to the default root is not allowed to
1229                  * fail.  Using both default and non-default roots should
1230                  * be rare.  Moving subsystems back and forth even more so.
1231                  * Just warn about it and continue.
1232                  */
1233                 if (cgrp_dfl_root_visible) {
1234                         pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n",
1235                                 ret, ss_mask);
1236                         pr_warn("you may retry by moving them to a different hierarchy and unbinding\n");
1237                 }
1238         }
1239 
1240         /*
1241          * Nothing can fail from this point on.  Remove files for the
1242          * removed subsystems and rebind each subsystem.
1243          */
1244         for_each_subsys(ss, ssid)
1245                 if (ss_mask & (1 << ssid))
1246                         cgroup_clear_dir(&ss->root->cgrp, 1 << ssid);
1247 
1248         for_each_subsys(ss, ssid) {
1249                 struct cgroup_root *src_root;
1250                 struct cgroup_subsys_state *css;
1251                 struct css_set *cset;
1252 
1253                 if (!(ss_mask & (1 << ssid)))
1254                         continue;
1255 
1256                 src_root = ss->root;
1257                 css = cgroup_css(&src_root->cgrp, ss);
1258 
1259                 WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss));
1260 
1261                 RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL);
1262                 rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css);
1263                 ss->root = dst_root;
1264                 css->cgroup = &dst_root->cgrp;
1265 
1266                 down_write(&css_set_rwsem);
1267                 hash_for_each(css_set_table, i, cset, hlist)
1268                         list_move_tail(&cset->e_cset_node[ss->id],
1269                                        &dst_root->cgrp.e_csets[ss->id]);
1270                 up_write(&css_set_rwsem);
1271 
1272                 src_root->subsys_mask &= ~(1 << ssid);
1273                 src_root->cgrp.subtree_control &= ~(1 << ssid);
1274                 cgroup_refresh_child_subsys_mask(&src_root->cgrp);
1275 
1276                 /* default hierarchy doesn't enable controllers by default */
1277                 dst_root->subsys_mask |= 1 << ssid;
1278                 if (dst_root != &cgrp_dfl_root) {
1279                         dst_root->cgrp.subtree_control |= 1 << ssid;
1280                         cgroup_refresh_child_subsys_mask(&dst_root->cgrp);
1281                 }
1282 
1283                 if (ss->bind)
1284                         ss->bind(css);
1285         }
1286 
1287         kernfs_activate(dst_root->cgrp.kn);
1288         return 0;
1289 }
1290 
1291 static int cgroup_show_options(struct seq_file *seq,
1292                                struct kernfs_root *kf_root)
1293 {
1294         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1295         struct cgroup_subsys *ss;
1296         int ssid;
1297 
1298         for_each_subsys(ss, ssid)
1299                 if (root->subsys_mask & (1 << ssid))
1300                         seq_printf(seq, ",%s", ss->name);
1301         if (root->flags & CGRP_ROOT_NOPREFIX)
1302                 seq_puts(seq, ",noprefix");
1303         if (root->flags & CGRP_ROOT_XATTR)
1304                 seq_puts(seq, ",xattr");
1305 
1306         spin_lock(&release_agent_path_lock);
1307         if (strlen(root->release_agent_path))
1308                 seq_printf(seq, ",release_agent=%s", root->release_agent_path);
1309         spin_unlock(&release_agent_path_lock);
1310 
1311         if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
1312                 seq_puts(seq, ",clone_children");
1313         if (strlen(root->name))
1314                 seq_printf(seq, ",name=%s", root->name);
1315         return 0;
1316 }
1317 
1318 struct cgroup_sb_opts {
1319         unsigned int subsys_mask;
1320         unsigned int flags;
1321         char *release_agent;
1322         bool cpuset_clone_children;
1323         char *name;
1324         /* User explicitly requested empty subsystem */
1325         bool none;
1326 };
1327 
1328 static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1329 {
1330         char *token, *o = data;
1331         bool all_ss = false, one_ss = false;
1332         unsigned int mask = -1U;
1333         struct cgroup_subsys *ss;
1334         int nr_opts = 0;
1335         int i;
1336 
1337 #ifdef CONFIG_CPUSETS
1338         mask = ~(1U << cpuset_cgrp_id);
1339 #endif
1340 
1341         memset(opts, 0, sizeof(*opts));
1342 
1343         while ((token = strsep(&o, ",")) != NULL) {
1344                 nr_opts++;
1345 
1346                 if (!*token)
1347                         return -EINVAL;
1348                 if (!strcmp(token, "none")) {
1349                         /* Explicitly have no subsystems */
1350                         opts->none = true;
1351                         continue;
1352                 }
1353                 if (!strcmp(token, "all")) {
1354                         /* Mutually exclusive option 'all' + subsystem name */
1355                         if (one_ss)
1356                                 return -EINVAL;
1357                         all_ss = true;
1358                         continue;
1359                 }
1360                 if (!strcmp(token, "__DEVEL__sane_behavior")) {
1361                         opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
1362                         continue;
1363                 }
1364                 if (!strcmp(token, "noprefix")) {
1365                         opts->flags |= CGRP_ROOT_NOPREFIX;
1366                         continue;
1367                 }
1368                 if (!strcmp(token, "clone_children")) {
1369                         opts->cpuset_clone_children = true;
1370                         continue;
1371                 }
1372                 if (!strcmp(token, "xattr")) {
1373                         opts->flags |= CGRP_ROOT_XATTR;
1374                         continue;
1375                 }
1376                 if (!strncmp(token, "release_agent=", 14)) {
1377                         /* Specifying two release agents is forbidden */
1378                         if (opts->release_agent)
1379                                 return -EINVAL;
1380                         opts->release_agent =
1381                                 kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1382                         if (!opts->release_agent)
1383                                 return -ENOMEM;
1384                         continue;
1385                 }
1386                 if (!strncmp(token, "name=", 5)) {
1387                         const char *name = token + 5;
1388                         /* Can't specify an empty name */
1389                         if (!strlen(name))
1390                                 return -EINVAL;
1391                         /* Must match [\w.-]+ */
1392                         for (i = 0; i < strlen(name); i++) {
1393                                 char c = name[i];
1394                                 if (isalnum(c))
1395                                         continue;
1396                                 if ((c == '.') || (c == '-') || (c == '_'))
1397                                         continue;
1398                                 return -EINVAL;
1399                         }
1400                         /* Specifying two names is forbidden */
1401                         if (opts->name)
1402                                 return -EINVAL;
1403                         opts->name = kstrndup(name,
1404                                               MAX_CGROUP_ROOT_NAMELEN - 1,
1405                                               GFP_KERNEL);
1406                         if (!opts->name)
1407                                 return -ENOMEM;
1408 
1409                         continue;
1410                 }
1411 
1412                 for_each_subsys(ss, i) {
1413                         if (strcmp(token, ss->name))
1414                                 continue;
1415                         if (ss->disabled)
1416                                 continue;
1417 
1418                         /* Mutually exclusive option 'all' + subsystem name */
1419                         if (all_ss)
1420                                 return -EINVAL;
1421                         opts->subsys_mask |= (1 << i);
1422                         one_ss = true;
1423 
1424                         break;
1425                 }
1426                 if (i == CGROUP_SUBSYS_COUNT)
1427                         return -ENOENT;
1428         }
1429 
1430         if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1431                 pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");
1432                 if (nr_opts != 1) {
1433                         pr_err("sane_behavior: no other mount options allowed\n");
1434                         return -EINVAL;
1435                 }
1436                 return 0;
1437         }
1438 
1439         /*
1440          * If the 'all' option was specified select all the subsystems,
1441          * otherwise if 'none', 'name=' and a subsystem name options were
1442          * not specified, let's default to 'all'
1443          */
1444         if (all_ss || (!one_ss && !opts->none && !opts->name))
1445                 for_each_subsys(ss, i)
1446                         if (!ss->disabled)
1447                                 opts->subsys_mask |= (1 << i);
1448 
1449         /*
1450          * We either have to specify by name or by subsystems. (So all
1451          * empty hierarchies must have a name).
1452          */
1453         if (!opts->subsys_mask && !opts->name)
1454                 return -EINVAL;
1455 
1456         /*
1457          * Option noprefix was introduced just for backward compatibility
1458          * with the old cpuset, so we allow noprefix only if mounting just
1459          * the cpuset subsystem.
1460          */
1461         if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1462                 return -EINVAL;
1463 
1464         /* Can't specify "none" and some subsystems */
1465         if (opts->subsys_mask && opts->none)
1466                 return -EINVAL;
1467 
1468         return 0;
1469 }
1470 
1471 static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1472 {
1473         int ret = 0;
1474         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1475         struct cgroup_sb_opts opts;
1476         unsigned int added_mask, removed_mask;
1477 
1478         if (root == &cgrp_dfl_root) {
1479                 pr_err("remount is not allowed\n");
1480                 return -EINVAL;
1481         }
1482 
1483         mutex_lock(&cgroup_mutex);
1484 
1485         /* See what subsystems are wanted */
1486         ret = parse_cgroupfs_options(data, &opts);
1487         if (ret)
1488                 goto out_unlock;
1489 
1490         if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1491                 pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
1492                         task_tgid_nr(current), current->comm);
1493 
1494         added_mask = opts.subsys_mask & ~root->subsys_mask;
1495         removed_mask = root->subsys_mask & ~opts.subsys_mask;
1496 
1497         /* Don't allow flags or name to change at remount */
1498         if ((opts.flags ^ root->flags) ||
1499             (opts.name && strcmp(opts.name, root->name))) {
1500                 pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
1501                        opts.flags, opts.name ?: "", root->flags, root->name);
1502                 ret = -EINVAL;
1503                 goto out_unlock;
1504         }
1505 
1506         /* remounting is not allowed for populated hierarchies */
1507         if (!list_empty(&root->cgrp.self.children)) {
1508                 ret = -EBUSY;
1509                 goto out_unlock;
1510         }
1511 
1512         ret = rebind_subsystems(root, added_mask);
1513         if (ret)
1514                 goto out_unlock;
1515 
1516         rebind_subsystems(&cgrp_dfl_root, removed_mask);
1517 
1518         if (opts.release_agent) {
1519                 spin_lock(&release_agent_path_lock);
1520                 strcpy(root->release_agent_path, opts.release_agent);
1521                 spin_unlock(&release_agent_path_lock);
1522         }
1523  out_unlock:
1524         kfree(opts.release_agent);
1525         kfree(opts.name);
1526         mutex_unlock(&cgroup_mutex);
1527         return ret;
1528 }
1529 
1530 /*
1531  * To reduce the fork() overhead for systems that are not actually using
1532  * their cgroups capability, we don't maintain the lists running through
1533  * each css_set to its tasks until we see the list actually used - in other
1534  * words after the first mount.
1535  */
1536 static bool use_task_css_set_links __read_mostly;
1537 
1538 static void cgroup_enable_task_cg_lists(void)
1539 {
1540         struct task_struct *p, *g;
1541 
1542         down_write(&css_set_rwsem);
1543 
1544         if (use_task_css_set_links)
1545                 goto out_unlock;
1546 
1547         use_task_css_set_links = true;
1548 
1549         /*
1550          * We need tasklist_lock because RCU is not safe against
1551          * while_each_thread(). Besides, a forking task that has passed
1552          * cgroup_post_fork() without seeing use_task_css_set_links = 1
1553          * is not guaranteed to have its child immediately visible in the
1554          * tasklist if we walk through it with RCU.
1555          */
1556         read_lock(&tasklist_lock);
1557         do_each_thread(g, p) {
1558                 WARN_ON_ONCE(!list_empty(&p->cg_list) ||
1559                              task_css_set(p) != &init_css_set);
1560 
1561                 /*
1562                  * We should check if the process is exiting, otherwise
1563                  * it will race with cgroup_exit() in that the list
1564                  * entry won't be deleted though the process has exited.
1565                  * Do it while holding siglock so that we don't end up
1566                  * racing against cgroup_exit().
1567                  */
1568                 spin_lock_irq(&p->sighand->siglock);
1569                 if (!(p->flags & PF_EXITING)) {
1570                         struct css_set *cset = task_css_set(p);
1571 
1572                         list_add(&p->cg_list, &cset->tasks);
1573                         get_css_set(cset);
1574                 }
1575                 spin_unlock_irq(&p->sighand->siglock);
1576         } while_each_thread(g, p);
1577         read_unlock(&tasklist_lock);
1578 out_unlock:
1579         up_write(&css_set_rwsem);
1580 }
1581 
1582 static void init_cgroup_housekeeping(struct cgroup *cgrp)
1583 {
1584         struct cgroup_subsys *ss;
1585         int ssid;
1586 
1587         INIT_LIST_HEAD(&cgrp->self.sibling);
1588         INIT_LIST_HEAD(&cgrp->self.children);
1589         INIT_LIST_HEAD(&cgrp->cset_links);
1590         INIT_LIST_HEAD(&cgrp->release_list);
1591         INIT_LIST_HEAD(&cgrp->pidlists);
1592         mutex_init(&cgrp->pidlist_mutex);
1593         cgrp->self.cgroup = cgrp;
1594         cgrp->self.flags |= CSS_ONLINE;
1595 
1596         for_each_subsys(ss, ssid)
1597                 INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1598 
1599         init_waitqueue_head(&cgrp->offline_waitq);
1600 }
1601 
1602 static void init_cgroup_root(struct cgroup_root *root,
1603                              struct cgroup_sb_opts *opts)
1604 {
1605         struct cgroup *cgrp = &root->cgrp;
1606 
1607         INIT_LIST_HEAD(&root->root_list);
1608         atomic_set(&root->nr_cgrps, 1);
1609         cgrp->root = root;
1610         init_cgroup_housekeeping(cgrp);
1611         idr_init(&root->cgroup_idr);
1612 
1613         root->flags = opts->flags;
1614         if (opts->release_agent)
1615                 strcpy(root->release_agent_path, opts->release_agent);
1616         if (opts->name)
1617                 strcpy(root->name, opts->name);
1618         if (opts->cpuset_clone_children)
1619                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1620 }
1621 
1622 static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask)
1623 {
1624         LIST_HEAD(tmp_links);
1625         struct cgroup *root_cgrp = &root->cgrp;
1626         struct cftype *base_files;
1627         struct css_set *cset;
1628         int i, ret;
1629 
1630         lockdep_assert_held(&cgroup_mutex);
1631 
1632         ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT);
1633         if (ret < 0)
1634                 goto out;
1635         root_cgrp->id = ret;
1636 
1637         ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release);
1638         if (ret)
1639                 goto out;
1640 
1641         /*
1642          * We're accessing css_set_count without locking css_set_rwsem here,
1643          * but that's OK - it can only be increased by someone holding
1644          * cgroup_lock, and that's us. The worst that can happen is that we
1645          * have some link structures left over
1646          */
1647         ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
1648         if (ret)
1649                 goto cancel_ref;
1650 
1651         ret = cgroup_init_root_id(root);
1652         if (ret)
1653                 goto cancel_ref;
1654 
1655         root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
1656                                            KERNFS_ROOT_CREATE_DEACTIVATED,
1657                                            root_cgrp);
1658         if (IS_ERR(root->kf_root)) {
1659                 ret = PTR_ERR(root->kf_root);
1660                 goto exit_root_id;
1661         }
1662         root_cgrp->kn = root->kf_root->kn;
1663 
1664         if (root == &cgrp_dfl_root)
1665                 base_files = cgroup_dfl_base_files;
1666         else
1667                 base_files = cgroup_legacy_base_files;
1668 
1669         ret = cgroup_addrm_files(root_cgrp, base_files, true);
1670         if (ret)
1671                 goto destroy_root;
1672 
1673         ret = rebind_subsystems(root, ss_mask);
1674         if (ret)
1675                 goto destroy_root;
1676 
1677         /*
1678          * There must be no failure case after here, since rebinding takes
1679          * care of subsystems' refcounts, which are explicitly dropped in
1680          * the failure exit path.
1681          */
1682         list_add(&root->root_list, &cgroup_roots);
1683         cgroup_root_count++;
1684 
1685         /*
1686          * Link the root cgroup in this hierarchy into all the css_set
1687          * objects.
1688          */
1689         down_write(&css_set_rwsem);
1690         hash_for_each(css_set_table, i, cset, hlist)
1691                 link_css_set(&tmp_links, cset, root_cgrp);
1692         up_write(&css_set_rwsem);
1693 
1694         BUG_ON(!list_empty(&root_cgrp->self.children));
1695         BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1696 
1697         kernfs_activate(root_cgrp->kn);
1698         ret = 0;
1699         goto out;
1700 
1701 destroy_root:
1702         kernfs_destroy_root(root->kf_root);
1703         root->kf_root = NULL;
1704 exit_root_id:
1705         cgroup_exit_root_id(root);
1706 cancel_ref:
1707         percpu_ref_exit(&root_cgrp->self.refcnt);
1708 out:
1709         free_cgrp_cset_links(&tmp_links);
1710         return ret;
1711 }
1712 
1713 static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1714                          int flags, const char *unused_dev_name,
1715                          void *data)
1716 {
1717         struct super_block *pinned_sb = NULL;
1718         struct cgroup_subsys *ss;
1719         struct cgroup_root *root;
1720         struct cgroup_sb_opts opts;
1721         struct dentry *dentry;
1722         int ret;
1723         int i;
1724         bool new_sb;
1725 
1726         /*
1727          * The first time anyone tries to mount a cgroup, enable the list
1728          * linking each css_set to its tasks and fix up all existing tasks.
1729          */
1730         if (!use_task_css_set_links)
1731                 cgroup_enable_task_cg_lists();
1732 
1733         mutex_lock(&cgroup_mutex);
1734 
1735         /* First find the desired set of subsystems */
1736         ret = parse_cgroupfs_options(data, &opts);
1737         if (ret)
1738                 goto out_unlock;
1739 
1740         /* look for a matching existing root */
1741         if (opts.flags & CGRP_ROOT_SANE_BEHAVIOR) {
1742                 cgrp_dfl_root_visible = true;
1743                 root = &cgrp_dfl_root;
1744                 cgroup_get(&root->cgrp);
1745                 ret = 0;
1746                 goto out_unlock;
1747         }
1748 
1749         /*
1750          * Destruction of cgroup root is asynchronous, so subsystems may
1751          * still be dying after the previous unmount.  Let's drain the
1752          * dying subsystems.  We just need to ensure that the ones
1753          * unmounted previously finish dying and don't care about new ones
1754          * starting.  Testing ref liveliness is good enough.
1755          */
1756         for_each_subsys(ss, i) {
1757                 if (!(opts.subsys_mask & (1 << i)) ||
1758                     ss->root == &cgrp_dfl_root)
1759                         continue;
1760 
1761                 if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) {
1762                         mutex_unlock(&cgroup_mutex);
1763                         msleep(10);
1764                         ret = restart_syscall();
1765                         goto out_free;
1766                 }
1767                 cgroup_put(&ss->root->cgrp);
1768         }
1769 
1770         for_each_root(root) {
1771                 bool name_match = false;
1772 
1773                 if (root == &cgrp_dfl_root)
1774                         continue;
1775 
1776                 /*
1777                  * If we asked for a name then it must match.  Also, if
1778                  * name matches but sybsys_mask doesn't, we should fail.
1779                  * Remember whether name matched.
1780                  */
1781                 if (opts.name) {
1782                         if (strcmp(opts.name, root->name))
1783                                 continue;
1784                         name_match = true;
1785                 }
1786 
1787                 /*
1788                  * If we asked for subsystems (or explicitly for no
1789                  * subsystems) then they must match.
1790                  */
1791                 if ((opts.subsys_mask || opts.none) &&
1792                     (opts.subsys_mask != root->subsys_mask)) {
1793                         if (!name_match)
1794                                 continue;
1795                         ret = -EBUSY;
1796                         goto out_unlock;
1797                 }
1798 
1799                 if (root->flags ^ opts.flags)
1800                         pr_warn("new mount options do not match the existing superblock, will be ignored\n");
1801 
1802                 /*
1803                  * We want to reuse @root whose lifetime is governed by its
1804                  * ->cgrp.  Let's check whether @root is alive and keep it
1805                  * that way.  As cgroup_kill_sb() can happen anytime, we
1806                  * want to block it by pinning the sb so that @root doesn't
1807                  * get killed before mount is complete.
1808                  *
1809                  * With the sb pinned, tryget_live can reliably indicate
1810                  * whether @root can be reused.  If it's being killed,
1811                  * drain it.  We can use wait_queue for the wait but this
1812                  * path is super cold.  Let's just sleep a bit and retry.
1813                  */
1814                 pinned_sb = kernfs_pin_sb(root->kf_root, NULL);
1815                 if (IS_ERR(pinned_sb) ||
1816                     !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
1817                         mutex_unlock(&cgroup_mutex);
1818                         if (!IS_ERR_OR_NULL(pinned_sb))
1819                                 deactivate_super(pinned_sb);
1820                         msleep(10);
1821                         ret = restart_syscall();
1822                         goto out_free;
1823                 }
1824 
1825                 ret = 0;
1826                 goto out_unlock;
1827         }
1828 
1829         /*
1830          * No such thing, create a new one.  name= matching without subsys
1831          * specification is allowed for already existing hierarchies but we
1832          * can't create new one without subsys specification.
1833          */
1834         if (!opts.subsys_mask && !opts.none) {
1835                 ret = -EINVAL;
1836                 goto out_unlock;
1837         }
1838 
1839         root = kzalloc(sizeof(*root), GFP_KERNEL);
1840         if (!root) {
1841                 ret = -ENOMEM;
1842                 goto out_unlock;
1843         }
1844 
1845         init_cgroup_root(root, &opts);
1846 
1847         ret = cgroup_setup_root(root, opts.subsys_mask);
1848         if (ret)
1849                 cgroup_free_root(root);
1850 
1851 out_unlock:
1852         mutex_unlock(&cgroup_mutex);
1853 out_free:
1854         kfree(opts.release_agent);
1855         kfree(opts.name);
1856 
1857         if (ret)
1858                 return ERR_PTR(ret);
1859 
1860         dentry = kernfs_mount(fs_type, flags, root->kf_root,
1861                                 CGROUP_SUPER_MAGIC, &new_sb);
1862         if (IS_ERR(dentry) || !new_sb)
1863                 cgroup_put(&root->cgrp);
1864 
1865         /*
1866          * If @pinned_sb, we're reusing an existing root and holding an
1867          * extra ref on its sb.  Mount is complete.  Put the extra ref.
1868          */
1869         if (pinned_sb) {
1870                 WARN_ON(new_sb);
1871                 deactivate_super(pinned_sb);
1872         }
1873 
1874         return dentry;
1875 }
1876 
1877 static void cgroup_kill_sb(struct super_block *sb)
1878 {
1879         struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
1880         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1881 
1882         /*
1883          * If @root doesn't have any mounts or children, start killing it.
1884          * This prevents new mounts by disabling percpu_ref_tryget_live().
1885          * cgroup_mount() may wait for @root's release.
1886          *
1887          * And don't kill the default root.
1888          */
1889         if (css_has_online_children(&root->cgrp.self) ||
1890             root == &cgrp_dfl_root)
1891                 cgroup_put(&root->cgrp);
1892         else
1893                 percpu_ref_kill(&root->cgrp.self.refcnt);
1894 
1895         kernfs_kill_sb(sb);
1896 }
1897 
1898 static struct file_system_type cgroup_fs_type = {
1899         .name = "cgroup",
1900         .mount = cgroup_mount,
1901         .kill_sb = cgroup_kill_sb,
1902 };
1903 
1904 static struct kobject *cgroup_kobj;
1905 
1906 /**
1907  * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
1908  * @task: target task
1909  * @buf: the buffer to write the path into
1910  * @buflen: the length of the buffer
1911  *
1912  * Determine @task's cgroup on the first (the one with the lowest non-zero
1913  * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
1914  * function grabs cgroup_mutex and shouldn't be used inside locks used by
1915  * cgroup controller callbacks.
1916  *
1917  * Return value is the same as kernfs_path().
1918  */
1919 char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
1920 {
1921         struct cgroup_root *root;
1922         struct cgroup *cgrp;
1923         int hierarchy_id = 1;
1924         char *path = NULL;
1925 
1926         mutex_lock(&cgroup_mutex);
1927         down_read(&css_set_rwsem);
1928 
1929         root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
1930 
1931         if (root) {
1932                 cgrp = task_cgroup_from_root(task, root);
1933                 path = cgroup_path(cgrp, buf, buflen);
1934         } else {
1935                 /* if no hierarchy exists, everyone is in "/" */
1936                 if (strlcpy(buf, "/", buflen) < buflen)
1937                         path = buf;
1938         }
1939 
1940         up_read(&css_set_rwsem);
1941         mutex_unlock(&cgroup_mutex);
1942         return path;
1943 }
1944 EXPORT_SYMBOL_GPL(task_cgroup_path);
1945 
1946 /* used to track tasks and other necessary states during migration */
1947 struct cgroup_taskset {
1948         /* the src and dst cset list running through cset->mg_node */
1949         struct list_head        src_csets;
1950         struct list_head        dst_csets;
1951 
1952         /*
1953          * Fields for cgroup_taskset_*() iteration.
1954          *
1955          * Before migration is committed, the target migration tasks are on
1956          * ->mg_tasks of the csets on ->src_csets.  After, on ->mg_tasks of
1957          * the csets on ->dst_csets.  ->csets point to either ->src_csets
1958          * or ->dst_csets depending on whether migration is committed.
1959          *
1960          * ->cur_csets and ->cur_task point to the current task position
1961          * during iteration.
1962          */
1963         struct list_head        *csets;
1964         struct css_set          *cur_cset;
1965         struct task_struct      *cur_task;
1966 };
1967 
1968 /**
1969  * cgroup_taskset_first - reset taskset and return the first task
1970  * @tset: taskset of interest
1971  *
1972  * @tset iteration is initialized and the first task is returned.
1973  */
1974 struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
1975 {
1976         tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
1977         tset->cur_task = NULL;
1978 
1979         return cgroup_taskset_next(tset);
1980 }
1981 
1982 /**
1983  * cgroup_taskset_next - iterate to the next task in taskset
1984  * @tset: taskset of interest
1985  *
1986  * Return the next task in @tset.  Iteration must have been initialized
1987  * with cgroup_taskset_first().
1988  */
1989 struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
1990 {
1991         struct css_set *cset = tset->cur_cset;
1992         struct task_struct *task = tset->cur_task;
1993 
1994         while (&cset->mg_node != tset->csets) {
1995                 if (!task)
1996                         task = list_first_entry(&cset->mg_tasks,
1997                                                 struct task_struct, cg_list);
1998                 else
1999                         task = list_next_entry(task, cg_list);
2000 
2001                 if (&task->cg_list != &cset->mg_tasks) {
2002                         tset->cur_cset = cset;
2003                         tset->cur_task = task;
2004                         return task;
2005                 }
2006 
2007                 cset = list_next_entry(cset, mg_node);
2008                 task = NULL;
2009         }
2010 
2011         return NULL;
2012 }
2013 
2014 /**
2015  * cgroup_task_migrate - move a task from one cgroup to another.
2016  * @old_cgrp: the cgroup @tsk is being migrated from
2017  * @tsk: the task being migrated
2018  * @new_cset: the new css_set @tsk is being attached to
2019  *
2020  * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked.
2021  */
2022 static void cgroup_task_migrate(struct cgroup *old_cgrp,
2023                                 struct task_struct *tsk,
2024                                 struct css_set *new_cset)
2025 {
2026         struct css_set *old_cset;
2027 
2028         lockdep_assert_held(&cgroup_mutex);
2029         lockdep_assert_held(&css_set_rwsem);
2030 
2031         /*
2032          * We are synchronized through threadgroup_lock() against PF_EXITING
2033          * setting such that we can't race against cgroup_exit() changing the
2034          * css_set to init_css_set and dropping the old one.
2035          */
2036         WARN_ON_ONCE(tsk->flags & PF_EXITING);
2037         old_cset = task_css_set(tsk);
2038 
2039         get_css_set(new_cset);
2040         rcu_assign_pointer(tsk->cgroups, new_cset);
2041 
2042         /*
2043          * Use move_tail so that cgroup_taskset_first() still returns the
2044          * leader after migration.  This works because cgroup_migrate()
2045          * ensures that the dst_cset of the leader is the first on the
2046          * tset's dst_csets list.
2047          */
2048         list_move_tail(&tsk->cg_list, &new_cset->mg_tasks);
2049 
2050         /*
2051          * We just gained a reference on old_cset by taking it from the
2052          * task. As trading it for new_cset is protected by cgroup_mutex,
2053          * we're safe to drop it here; it will be freed under RCU.
2054          */
2055         set_bit(CGRP_RELEASABLE, &old_cgrp->flags);
2056         put_css_set_locked(old_cset, false);
2057 }
2058 
2059 /**
2060  * cgroup_migrate_finish - cleanup after attach
2061  * @preloaded_csets: list of preloaded css_sets
2062  *
2063  * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
2064  * those functions for details.
2065  */
2066 static void cgroup_migrate_finish(struct list_head *preloaded_csets)
2067 {
2068         struct css_set *cset, *tmp_cset;
2069 
2070         lockdep_assert_held(&cgroup_mutex);
2071 
2072         down_write(&css_set_rwsem);
2073         list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
2074                 cset->mg_src_cgrp = NULL;
2075                 cset->mg_dst_cset = NULL;
2076                 list_del_init(&cset->mg_preload_node);
2077                 put_css_set_locked(cset, false);
2078         }
2079         up_write(&css_set_rwsem);
2080 }
2081 
2082 /**
2083  * cgroup_migrate_add_src - add a migration source css_set
2084  * @src_cset: the source css_set to add
2085  * @dst_cgrp: the destination cgroup
2086  * @preloaded_csets: list of preloaded css_sets
2087  *
2088  * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
2089  * @src_cset and add it to @preloaded_csets, which should later be cleaned
2090  * up by cgroup_migrate_finish().
2091  *
2092  * This function may be called without holding threadgroup_lock even if the
2093  * target is a process.  Threads may be created and destroyed but as long
2094  * as cgroup_mutex is not dropped, no new css_set can be put into play and
2095  * the preloaded css_sets are guaranteed to cover all migrations.
2096  */
2097 static void cgroup_migrate_add_src(struct css_set *src_cset,
2098                                    struct cgroup *dst_cgrp,
2099                                    struct list_head *preloaded_csets)
2100 {
2101         struct cgroup *src_cgrp;
2102 
2103         lockdep_assert_held(&cgroup_mutex);
2104         lockdep_assert_held(&css_set_rwsem);
2105 
2106         src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);
2107 
2108         if (!list_empty(&src_cset->mg_preload_node))
2109                 return;
2110 
2111         WARN_ON(src_cset->mg_src_cgrp);
2112         WARN_ON(!list_empty(&src_cset->mg_tasks));
2113         WARN_ON(!list_empty(&src_cset->mg_node));
2114 
2115         src_cset->mg_src_cgrp = src_cgrp;
2116         get_css_set(src_cset);
2117         list_add(&src_cset->mg_preload_node, preloaded_csets);
2118 }
2119 
2120 /**
2121  * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2122  * @dst_cgrp: the destination cgroup (may be %NULL)
2123  * @preloaded_csets: list of preloaded source css_sets
2124  *
2125  * Tasks are about to be moved to @dst_cgrp and all the source css_sets
2126  * have been preloaded to @preloaded_csets.  This function looks up and
2127  * pins all destination css_sets, links each to its source, and append them
2128  * to @preloaded_csets.  If @dst_cgrp is %NULL, the destination of each
2129  * source css_set is assumed to be its cgroup on the default hierarchy.
2130  *
2131  * This function must be called after cgroup_migrate_add_src() has been
2132  * called on each migration source css_set.  After migration is performed
2133  * using cgroup_migrate(), cgroup_migrate_finish() must be called on
2134  * @preloaded_csets.
2135  */
2136 static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
2137                                       struct list_head *preloaded_csets)
2138 {
2139         LIST_HEAD(csets);
2140         struct css_set *src_cset, *tmp_cset;
2141 
2142         lockdep_assert_held(&cgroup_mutex);
2143 
2144         /*
2145          * Except for the root, child_subsys_mask must be zero for a cgroup
2146          * with tasks so that child cgroups don't compete against tasks.
2147          */
2148         if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) &&
2149             dst_cgrp->child_subsys_mask)
2150                 return -EBUSY;
2151 
2152         /* look up the dst cset for each src cset and link it to src */
2153         list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
2154                 struct css_set *dst_cset;
2155 
2156                 dst_cset = find_css_set(src_cset,
2157                                         dst_cgrp ?: src_cset->dfl_cgrp);
2158                 if (!dst_cset)
2159                         goto err;
2160 
2161                 WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
2162 
2163                 /*
2164                  * If src cset equals dst, it's noop.  Drop the src.
2165                  * cgroup_migrate() will skip the cset too.  Note that we
2166                  * can't handle src == dst as some nodes are used by both.
2167                  */
2168                 if (src_cset == dst_cset) {
2169                         src_cset->mg_src_cgrp = NULL;
2170                         list_del_init(&src_cset->mg_preload_node);
2171                         put_css_set(src_cset, false);
2172                         put_css_set(dst_cset, false);
2173                         continue;
2174                 }
2175 
2176                 src_cset->mg_dst_cset = dst_cset;
2177 
2178                 if (list_empty(&dst_cset->mg_preload_node))
2179                         list_add(&dst_cset->mg_preload_node, &csets);
2180                 else
2181                         put_css_set(dst_cset, false);
2182         }
2183 
2184         list_splice_tail(&csets, preloaded_csets);
2185         return 0;
2186 err:
2187         cgroup_migrate_finish(&csets);
2188         return -ENOMEM;
2189 }
2190 
2191 /**
2192  * cgroup_migrate - migrate a process or task to a cgroup
2193  * @cgrp: the destination cgroup
2194  * @leader: the leader of the process or the task to migrate
2195  * @threadgroup: whether @leader points to the whole process or a single task
2196  *
2197  * Migrate a process or task denoted by @leader to @cgrp.  If migrating a
2198  * process, the caller must be holding threadgroup_lock of @leader.  The
2199  * caller is also responsible for invoking cgroup_migrate_add_src() and
2200  * cgroup_migrate_prepare_dst() on the targets before invoking this
2201  * function and following up with cgroup_migrate_finish().
2202  *
2203  * As long as a controller's ->can_attach() doesn't fail, this function is
2204  * guaranteed to succeed.  This means that, excluding ->can_attach()
2205  * failure, when migrating multiple targets, the success or failure can be
2206  * decided for all targets by invoking group_migrate_prepare_dst() before
2207  * actually starting migrating.
2208  */
2209 static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader,
2210                           bool threadgroup)
2211 {
2212         struct cgroup_taskset tset = {
2213                 .src_csets      = LIST_HEAD_INIT(tset.src_csets),
2214                 .dst_csets      = LIST_HEAD_INIT(tset.dst_csets),
2215                 .csets          = &tset.src_csets,
2216         };
2217         struct cgroup_subsys_state *css, *failed_css = NULL;
2218         struct css_set *cset, *tmp_cset;
2219         struct task_struct *task, *tmp_task;
2220         int i, ret;
2221 
2222         /*
2223          * Prevent freeing of tasks while we take a snapshot. Tasks that are
2224          * already PF_EXITING could be freed from underneath us unless we
2225          * take an rcu_read_lock.
2226          */
2227         down_write(&css_set_rwsem);
2228         rcu_read_lock();
2229         task = leader;
2230         do {
2231                 /* @task either already exited or can't exit until the end */
2232                 if (task->flags & PF_EXITING)
2233                         goto next;
2234 
2235                 /* leave @task alone if post_fork() hasn't linked it yet */
2236                 if (list_empty(&task->cg_list))
2237                         goto next;
2238 
2239                 cset = task_css_set(task);
2240                 if (!cset->mg_src_cgrp)
2241                         goto next;
2242 
2243                 /*
2244                  * cgroup_taskset_first() must always return the leader.
2245                  * Take care to avoid disturbing the ordering.
2246                  */
2247                 list_move_tail(&task->cg_list, &cset->mg_tasks);
2248                 if (list_empty(&cset->mg_node))
2249                         list_add_tail(&cset->mg_node, &tset.src_csets);
2250                 if (list_empty(&cset->mg_dst_cset->mg_node))
2251                         list_move_tail(&cset->mg_dst_cset->mg_node,
2252                                        &tset.dst_csets);
2253         next:
2254                 if (!threadgroup)
2255                         break;
2256         } while_each_thread(leader, task);
2257         rcu_read_unlock();
2258         up_write(&css_set_rwsem);
2259 
2260         /* methods shouldn't be called if no task is actually migrating */
2261         if (list_empty(&tset.src_csets))
2262                 return 0;
2263 
2264         /* check that we can legitimately attach to the cgroup */
2265         for_each_e_css(css, i, cgrp) {
2266                 if (css->ss->can_attach) {
2267                         ret = css->ss->can_attach(css, &tset);
2268                         if (ret) {
2269                                 failed_css = css;
2270                                 goto out_cancel_attach;
2271                         }
2272                 }
2273         }
2274 
2275         /*
2276          * Now that we're guaranteed success, proceed to move all tasks to
2277          * the new cgroup.  There are no failure cases after here, so this
2278          * is the commit point.
2279          */
2280         down_write(&css_set_rwsem);
2281         list_for_each_entry(cset, &tset.src_csets, mg_node) {
2282                 list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list)
2283                         cgroup_task_migrate(cset->mg_src_cgrp, task,
2284                                             cset->mg_dst_cset);
2285         }
2286         up_write(&css_set_rwsem);
2287 
2288         /*
2289          * Migration is committed, all target tasks are now on dst_csets.
2290          * Nothing is sensitive to fork() after this point.  Notify
2291          * controllers that migration is complete.
2292          */
2293         tset.csets = &tset.dst_csets;
2294 
2295         for_each_e_css(css, i, cgrp)
2296                 if (css->ss->attach)
2297                         css->ss->attach(css, &tset);
2298 
2299         ret = 0;
2300         goto out_release_tset;
2301 
2302 out_cancel_attach:
2303         for_each_e_css(css, i, cgrp) {
2304                 if (css == failed_css)
2305                         break;
2306                 if (css->ss->cancel_attach)
2307                         css->ss->cancel_attach(css, &tset);
2308         }
2309 out_release_tset:
2310         down_write(&css_set_rwsem);
2311         list_splice_init(&tset.dst_csets, &tset.src_csets);
2312         list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) {
2313                 list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2314                 list_del_init(&cset->mg_node);
2315         }
2316         up_write(&css_set_rwsem);
2317         return ret;
2318 }
2319 
2320 /**
2321  * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2322  * @dst_cgrp: the cgroup to attach to
2323  * @leader: the task or the leader of the threadgroup to be attached
2324  * @threadgroup: attach the whole threadgroup?
2325  *
2326  * Call holding cgroup_mutex and threadgroup_lock of @leader.
2327  */
2328 static int cgroup_attach_task(struct cgroup *dst_cgrp,
2329                               struct task_struct *leader, bool threadgroup)
2330 {
2331         LIST_HEAD(preloaded_csets);
2332         struct task_struct *task;
2333         int ret;
2334 
2335         /* look up all src csets */
2336         down_read(&css_set_rwsem);
2337         rcu_read_lock();
2338         task = leader;
2339         do {
2340                 cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
2341                                        &preloaded_csets);
2342                 if (!threadgroup)
2343                         break;
2344         } while_each_thread(leader, task);
2345         rcu_read_unlock();
2346         up_read(&css_set_rwsem);
2347 
2348         /* prepare dst csets and commit */
2349         ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
2350         if (!ret)
2351                 ret = cgroup_migrate(dst_cgrp, leader, threadgroup);
2352 
2353         cgroup_migrate_finish(&preloaded_csets);
2354         return ret;
2355 }
2356 
2357 /*
2358  * Find the task_struct of the task to attach by vpid and pass it along to the
2359  * function to attach either it or all tasks in its threadgroup. Will lock
2360  * cgroup_mutex and threadgroup.
2361  */
2362 static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
2363                                     size_t nbytes, loff_t off, bool threadgroup)
2364 {
2365         struct task_struct *tsk;
2366         const struct cred *cred = current_cred(), *tcred;
2367         struct cgroup *cgrp;
2368         pid_t pid;
2369         int ret;
2370 
2371         if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
2372                 return -EINVAL;
2373 
2374         cgrp = cgroup_kn_lock_live(of->kn);
2375         if (!cgrp)
2376                 return -ENODEV;
2377 
2378 retry_find_task:
2379         rcu_read_lock();
2380         if (pid) {
2381                 tsk = find_task_by_vpid(pid);
2382                 if (!tsk) {
2383                         rcu_read_unlock();
2384                         ret = -ESRCH;
2385                         goto out_unlock_cgroup;
2386                 }
2387                 /*
2388                  * even if we're attaching all tasks in the thread group, we
2389                  * only need to check permissions on one of them.
2390                  */
2391                 tcred = __task_cred(tsk);
2392                 if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
2393                     !uid_eq(cred->euid, tcred->uid) &&
2394                     !uid_eq(cred->euid, tcred->suid)) {
2395                         rcu_read_unlock();
2396                         ret = -EACCES;
2397                         goto out_unlock_cgroup;
2398                 }
2399         } else
2400                 tsk = current;
2401 
2402         if (threadgroup)
2403                 tsk = tsk->group_leader;
2404 
2405         /*
2406          * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2407          * trapped in a cpuset, or RT worker may be born in a cgroup
2408          * with no rt_runtime allocated.  Just say no.
2409          */
2410         if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2411                 ret = -EINVAL;
2412                 rcu_read_unlock();
2413                 goto out_unlock_cgroup;
2414         }
2415 
2416         get_task_struct(tsk);
2417         rcu_read_unlock();
2418 
2419         threadgroup_lock(tsk);
2420         if (threadgroup) {
2421                 if (!thread_group_leader(tsk)) {
2422                         /*
2423                          * a race with de_thread from another thread's exec()
2424                          * may strip us of our leadership, if this happens,
2425                          * there is no choice but to throw this task away and
2426                          * try again; this is
2427                          * "double-double-toil-and-trouble-check locking".
2428                          */
2429                         threadgroup_unlock(tsk);
2430                         put_task_struct(tsk);
2431                         goto retry_find_task;
2432                 }
2433         }
2434 
2435         ret = cgroup_attach_task(cgrp, tsk, threadgroup);
2436 
2437         threadgroup_unlock(tsk);
2438 
2439         put_task_struct(tsk);
2440 out_unlock_cgroup:
2441         cgroup_kn_unlock(of->kn);
2442         return ret ?: nbytes;
2443 }
2444 
2445 /**
2446  * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
2447  * @from: attach to all cgroups of a given task
2448  * @tsk: the task to be attached
2449  */
2450 int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
2451 {
2452         struct cgroup_root *root;
2453         int retval = 0;
2454 
2455         mutex_lock(&cgroup_mutex);
2456         for_each_root(root) {
2457                 struct cgroup *from_cgrp;
2458 
2459                 if (root == &cgrp_dfl_root)
2460                         continue;
2461 
2462                 down_read(&css_set_rwsem);
2463                 from_cgrp = task_cgroup_from_root(from, root);
2464                 up_read(&css_set_rwsem);
2465 
2466                 retval = cgroup_attach_task(from_cgrp, tsk, false);
2467                 if (retval)
2468                         break;
2469         }
2470         mutex_unlock(&cgroup_mutex);
2471 
2472         return retval;
2473 }
2474 EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
2475 
2476 static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
2477                                   char *buf, size_t nbytes, loff_t off)
2478 {
2479         return __cgroup_procs_write(of, buf, nbytes, off, false);
2480 }
2481 
2482 static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
2483                                   char *buf, size_t nbytes, loff_t off)
2484 {
2485         return __cgroup_procs_write(of, buf, nbytes, off, true);
2486 }
2487 
2488 static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
2489                                           char *buf, size_t nbytes, loff_t off)
2490 {
2491         struct cgroup *cgrp;
2492 
2493         BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
2494 
2495         cgrp = cgroup_kn_lock_live(of->kn);
2496         if (!cgrp)
2497                 return -ENODEV;
2498         spin_lock(&release_agent_path_lock);
2499         strlcpy(cgrp->root->release_agent_path, strstrip(buf),
2500                 sizeof(cgrp->root->release_agent_path));
2501         spin_unlock(&release_agent_path_lock);
2502         cgroup_kn_unlock(of->kn);
2503         return nbytes;
2504 }
2505 
2506 static int cgroup_release_agent_show(struct seq_file *seq, void *v)
2507 {
2508         struct cgroup *cgrp = seq_css(seq)->cgroup;
2509 
2510         spin_lock(&release_agent_path_lock);
2511         seq_puts(seq, cgrp->root->release_agent_path);
2512         spin_unlock(&release_agent_path_lock);
2513         seq_putc(seq, '\n');
2514         return 0;
2515 }
2516 
2517 static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
2518 {
2519         seq_puts(seq, "\n");
2520         return 0;
2521 }
2522 
2523 static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask)
2524 {
2525         struct cgroup_subsys *ss;
2526         bool printed = false;
2527         int ssid;
2528 
2529         for_each_subsys(ss, ssid) {
2530                 if (ss_mask & (1 << ssid)) {
2531                         if (printed)
2532                                 seq_putc(seq, ' ');
2533                         seq_printf(seq, "%s", ss->name);
2534                         printed = true;
2535                 }
2536         }
2537         if (printed)
2538                 seq_putc(seq, '\n');
2539 }
2540 
2541 /* show controllers which are currently attached to the default hierarchy */
2542 static int cgroup_root_controllers_show(struct seq_file *seq, void *v)
2543 {
2544         struct cgroup *cgrp = seq_css(seq)->cgroup;
2545 
2546         cgroup_print_ss_mask(seq, cgrp->root->subsys_mask &
2547                              ~cgrp_dfl_root_inhibit_ss_mask);
2548         return 0;
2549 }
2550 
2551 /* show controllers which are enabled from the parent */
2552 static int cgroup_controllers_show(struct seq_file *seq, void *v)
2553 {
2554         struct cgroup *cgrp = seq_css(seq)->cgroup;
2555 
2556         cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control);
2557         return 0;
2558 }
2559 
2560 /* show controllers which are enabled for a given cgroup's children */
2561 static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
2562 {
2563         struct cgroup *cgrp = seq_css(seq)->cgroup;
2564 
2565         cgroup_print_ss_mask(seq, cgrp->subtree_control);
2566         return 0;
2567 }
2568 
2569 /**
2570  * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
2571  * @cgrp: root of the subtree to update csses for
2572  *
2573  * @cgrp's child_subsys_mask has changed and its subtree's (self excluded)
2574  * css associations need to be updated accordingly.  This function looks up
2575  * all css_sets which are attached to the subtree, creates the matching
2576  * updated css_sets and migrates the tasks to the new ones.
2577  */
2578 static int cgroup_update_dfl_csses(struct cgroup *cgrp)
2579 {
2580         LIST_HEAD(preloaded_csets);
2581         struct cgroup_subsys_state *css;
2582         struct css_set *src_cset;
2583         int ret;
2584 
2585         lockdep_assert_held(&cgroup_mutex);
2586 
2587         /* look up all csses currently attached to @cgrp's subtree */
2588         down_read(&css_set_rwsem);
2589         css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
2590                 struct cgrp_cset_link *link;
2591 
2592                 /* self is not affected by child_subsys_mask change */
2593                 if (css->cgroup == cgrp)
2594                         continue;
2595 
2596                 list_for_each_entry(link, &css->cgroup->cset_links, cset_link)
2597                         cgroup_migrate_add_src(link->cset, cgrp,
2598                                                &preloaded_csets);
2599         }
2600         up_read(&css_set_rwsem);
2601 
2602         /* NULL dst indicates self on default hierarchy */
2603         ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets);
2604         if (ret)
2605                 goto out_finish;
2606 
2607         list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
2608                 struct task_struct *last_task = NULL, *task;
2609 
2610                 /* src_csets precede dst_csets, break on the first dst_cset */
2611                 if (!src_cset->mg_src_cgrp)
2612                         break;
2613 
2614                 /*
2615                  * All tasks in src_cset need to be migrated to the
2616                  * matching dst_cset.  Empty it process by process.  We
2617                  * walk tasks but migrate processes.  The leader might even
2618                  * belong to a different cset but such src_cset would also
2619                  * be among the target src_csets because the default
2620                  * hierarchy enforces per-process membership.
2621                  */
2622                 while (true) {
2623                         down_read(&css_set_rwsem);
2624                         task = list_first_entry_or_null(&src_cset->tasks,
2625                                                 struct task_struct, cg_list);
2626                         if (task) {
2627                                 task = task->group_leader;
2628                                 WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp);
2629                                 get_task_struct(task);
2630                         }
2631                         up_read(&css_set_rwsem);
2632 
2633                         if (!task)
2634                                 break;
2635 
2636                         /* guard against possible infinite loop */
2637                         if (WARN(last_task == task,
2638                                  "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n"))
2639                                 goto out_finish;
2640                         last_task = task;
2641 
2642                         threadgroup_lock(task);
2643                         /* raced against de_thread() from another thread? */
2644                         if (!thread_group_leader(task)) {
2645                                 threadgroup_unlock(task);
2646                                 put_task_struct(task);
2647                                 continue;
2648                         }
2649 
2650                         ret = cgroup_migrate(src_cset->dfl_cgrp, task, true);
2651 
2652                         threadgroup_unlock(task);
2653                         put_task_struct(task);
2654 
2655                         if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret))
2656                                 goto out_finish;
2657                 }
2658         }
2659 
2660 out_finish:
2661         cgroup_migrate_finish(&preloaded_csets);
2662         return ret;
2663 }
2664 
2665 /* change the enabled child controllers for a cgroup in the default hierarchy */
2666 static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
2667                                             char *buf, size_t nbytes,
2668                                             loff_t off)
2669 {
2670         unsigned int enable = 0, disable = 0;
2671         unsigned int css_enable, css_disable, old_ctrl, new_ctrl;
2672         struct cgroup *cgrp, *child;
2673         struct cgroup_subsys *ss;
2674         char *tok;
2675         int ssid, ret;
2676 
2677         /*
2678          * Parse input - space separated list of subsystem names prefixed
2679          * with either + or -.
2680          */
2681         buf = strstrip(buf);
2682         while ((tok = strsep(&buf, " "))) {
2683                 if (tok[0] == '\0')
2684                         continue;
2685                 for_each_subsys(ss, ssid) {
2686                         if (ss->disabled || strcmp(tok + 1, ss->name) ||
2687                             ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask))
2688                                 continue;
2689 
2690                         if (*tok == '+') {
2691                                 enable |= 1 << ssid;
2692                                 disable &= ~(1 << ssid);
2693                         } else if (*tok == '-') {
2694                                 disable |= 1 << ssid;
2695                                 enable &= ~(1 << ssid);
2696                         } else {
2697                                 return -EINVAL;
2698                         }
2699                         break;
2700                 }
2701                 if (ssid == CGROUP_SUBSYS_COUNT)
2702                         return -EINVAL;
2703         }
2704 
2705         cgrp = cgroup_kn_lock_live(of->kn);
2706         if (!cgrp)
2707                 return -ENODEV;
2708 
2709         for_each_subsys(ss, ssid) {
2710                 if (enable & (1 << ssid)) {
2711                         if (cgrp->subtree_control & (1 << ssid)) {
2712                                 enable &= ~(1 << ssid);
2713                                 continue;
2714                         }
2715 
2716                         /* unavailable or not enabled on the parent? */
2717                         if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) ||
2718                             (cgroup_parent(cgrp) &&
2719                              !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) {
2720                                 ret = -ENOENT;
2721                                 goto out_unlock;
2722                         }
2723 
2724                         /*
2725                          * @ss is already enabled through dependency and
2726                          * we'll just make it visible.  Skip draining.
2727                          */
2728                         if (cgrp->child_subsys_mask & (1 << ssid))
2729                                 continue;
2730 
2731                         /*
2732                          * Because css offlining is asynchronous, userland
2733                          * might try to re-enable the same controller while
2734                          * the previous instance is still around.  In such
2735                          * cases, wait till it's gone using offline_waitq.
2736                          */
2737                         cgroup_for_each_live_child(child, cgrp) {
2738                                 DEFINE_WAIT(wait);
2739 
2740                                 if (!cgroup_css(child, ss))
2741                                         continue;
2742 
2743                                 cgroup_get(child);
2744                                 prepare_to_wait(&child->offline_waitq, &wait,
2745                                                 TASK_UNINTERRUPTIBLE);
2746                                 cgroup_kn_unlock(of->kn);
2747                                 schedule();
2748                                 finish_wait(&child->offline_waitq, &wait);
2749                                 cgroup_put(child);
2750 
2751                                 return restart_syscall();
2752                         }
2753                 } else if (disable & (1 << ssid)) {
2754                         if (!(cgrp->subtree_control & (1 << ssid))) {
2755                                 disable &= ~(1 << ssid);
2756                                 continue;
2757                         }
2758 
2759                         /* a child has it enabled? */
2760                         cgroup_for_each_live_child(child, cgrp) {
2761                                 if (child->subtree_control & (1 << ssid)) {
2762                                         ret = -EBUSY;
2763                                         goto out_unlock;
2764                                 }
2765                         }
2766                 }
2767         }
2768 
2769         if (!enable && !disable) {
2770                 ret = 0;
2771                 goto out_unlock;
2772         }
2773 
2774         /*
2775          * Except for the root, subtree_control must be zero for a cgroup
2776          * with tasks so that child cgroups don't compete against tasks.
2777          */
2778         if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) {
2779                 ret = -EBUSY;
2780                 goto out_unlock;
2781         }
2782 
2783         /*
2784          * Update subsys masks and calculate what needs to be done.  More
2785          * subsystems than specified may need to be enabled or disabled
2786          * depending on subsystem dependencies.
2787          */
2788         cgrp->subtree_control |= enable;
2789         cgrp->subtree_control &= ~disable;
2790 
2791         old_ctrl = cgrp->child_subsys_mask;
2792         cgroup_refresh_child_subsys_mask(cgrp);
2793         new_ctrl = cgrp->child_subsys_mask;
2794 
2795         css_enable = ~old_ctrl & new_ctrl;
2796         css_disable = old_ctrl & ~new_ctrl;
2797         enable |= css_enable;
2798         disable |= css_disable;
2799 
2800         /*
2801          * Create new csses or make the existing ones visible.  A css is
2802          * created invisible if it's being implicitly enabled through
2803          * dependency.  An invisible css is made visible when the userland
2804          * explicitly enables it.
2805          */
2806         for_each_subsys(ss, ssid) {
2807                 if (!(enable & (1 << ssid)))
2808                         continue;
2809 
2810                 cgroup_for_each_live_child(child, cgrp) {
2811                         if (css_enable & (1 << ssid))
2812                                 ret = create_css(child, ss,
2813                                         cgrp->subtree_control & (1 << ssid));
2814                         else
2815                                 ret = cgroup_populate_dir(child, 1 << ssid);
2816                         if (ret)
2817                                 goto err_undo_css;
2818                 }
2819         }
2820 
2821         /*
2822          * At this point, cgroup_e_css() results reflect the new csses
2823          * making the following cgroup_update_dfl_csses() properly update
2824          * css associations of all tasks in the subtree.
2825          */
2826         ret = cgroup_update_dfl_csses(cgrp);
2827         if (ret)
2828                 goto err_undo_css;
2829 
2830         /*
2831          * All tasks are migrated out of disabled csses.  Kill or hide
2832          * them.  A css is hidden when the userland requests it to be
2833          * disabled while other subsystems are still depending on it.  The
2834          * css must not actively control resources and be in the vanilla
2835          * state if it's made visible again later.  Controllers which may
2836          * be depended upon should provide ->css_reset() for this purpose.
2837          */
2838         for_each_subsys(ss, ssid) {
2839                 if (!(disable & (1 << ssid)))
2840                         continue;
2841 
2842                 cgroup_for_each_live_child(child, cgrp) {
2843                         struct cgroup_subsys_state *css = cgroup_css(child, ss);
2844 
2845                         if (css_disable & (1 << ssid)) {
2846                                 kill_css(css);
2847                         } else {
2848                                 cgroup_clear_dir(child, 1 << ssid);
2849                                 if (ss->css_reset)
2850                                         ss->css_reset(css);
2851                         }
2852                 }
2853         }
2854 
2855         kernfs_activate(cgrp->kn);
2856         ret = 0;
2857 out_unlock:
2858         cgroup_kn_unlock(of->kn);
2859         return ret ?: nbytes;
2860 
2861 err_undo_css:
2862         cgrp->subtree_control &= ~enable;
2863         cgrp->subtree_control |= disable;
2864         cgroup_refresh_child_subsys_mask(cgrp);
2865 
2866         for_each_subsys(ss, ssid) {
2867                 if (!(enable & (1 << ssid)))
2868                         continue;
2869 
2870                 cgroup_for_each_live_child(child, cgrp) {
2871                         struct cgroup_subsys_state *css = cgroup_css(child, ss);
2872 
2873                         if (!css)
2874                                 continue;
2875 
2876                         if (css_enable & (1 << ssid))
2877                                 kill_css(css);
2878                         else
2879                                 cgroup_clear_dir(child, 1 << ssid);
2880                 }
2881         }
2882         goto out_unlock;
2883 }
2884 
2885 static int cgroup_populated_show(struct seq_file *seq, void *v)
2886 {
2887         seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt);
2888         return 0;
2889 }
2890 
2891 static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
2892                                  size_t nbytes, loff_t off)
2893 {
2894         struct cgroup *cgrp = of->kn->parent->priv;
2895         struct cftype *cft = of->kn->priv;
2896         struct cgroup_subsys_state *css;
2897         int ret;
2898 
2899         if (cft->write)
2900                 return cft->write(of, buf, nbytes, off);
2901 
2902         /*
2903          * kernfs guarantees that a file isn't deleted with operations in
2904          * flight, which means that the matching css is and stays alive and
2905          * doesn't need to be pinned.  The RCU locking is not necessary
2906          * either.  It's just for the convenience of using cgroup_css().
2907          */
2908         rcu_read_lock();
2909         css = cgroup_css(cgrp, cft->ss);
2910         rcu_read_unlock();
2911 
2912         if (cft->write_u64) {
2913                 unsigned long long v;
2914                 ret = kstrtoull(buf, 0, &v);
2915                 if (!ret)
2916                         ret = cft->write_u64(css, cft, v);
2917         } else if (cft->write_s64) {
2918                 long long v;
2919                 ret = kstrtoll(buf, 0, &v);
2920                 if (!ret)
2921                         ret = cft->write_s64(css, cft, v);
2922         } else {
2923                 ret = -EINVAL;
2924         }
2925 
2926         return ret ?: nbytes;
2927 }
2928 
2929 static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
2930 {
2931         return seq_cft(seq)->seq_start(seq, ppos);
2932 }
2933 
2934 static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
2935 {
2936         return seq_cft(seq)->seq_next(seq, v, ppos);
2937 }
2938 
2939 static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
2940 {
2941         seq_cft(seq)->seq_stop(seq, v);
2942 }
2943 
2944 static int cgroup_seqfile_show(struct seq_file *m, void *arg)
2945 {
2946         struct cftype *cft = seq_cft(m);
2947         struct cgroup_subsys_state *css = seq_css(m);
2948 
2949         if (cft->seq_show)
2950                 return cft->seq_show(m, arg);
2951 
2952         if (cft->read_u64)
2953                 seq_printf(m, "%llu\n", cft->read_u64(css, cft));
2954         else if (cft->read_s64)
2955                 seq_printf(m, "%lld\n", cft->read_s64(css, cft));
2956         else
2957                 return -EINVAL;
2958         return 0;
2959 }
2960 
2961 static struct kernfs_ops cgroup_kf_single_ops = {
2962         .atomic_write_len       = PAGE_SIZE,
2963         .write                  = cgroup_file_write,
2964         .seq_show               = cgroup_seqfile_show,
2965 };
2966 
2967 static struct kernfs_ops cgroup_kf_ops = {
2968         .atomic_write_len       = PAGE_SIZE,
2969         .write                  = cgroup_file_write,
2970         .seq_start              = cgroup_seqfile_start,
2971         .seq_next               = cgroup_seqfile_next,
2972         .seq_stop               = cgroup_seqfile_stop,
2973         .seq_show               = cgroup_seqfile_show,
2974 };
2975 
2976 /*
2977  * cgroup_rename - Only allow simple rename of directories in place.
2978  */
2979 static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
2980                          const char *new_name_str)
2981 {
2982         struct cgroup *cgrp = kn->priv;
2983         int ret;
2984 
2985         if (kernfs_type(kn) != KERNFS_DIR)
2986                 return -ENOTDIR;
2987         if (kn->parent != new_parent)
2988                 return -EIO;
2989 
2990         /*
2991          * This isn't a proper migration and its usefulness is very
2992          * limited.  Disallow on the default hierarchy.
2993          */
2994         if (cgroup_on_dfl(cgrp))
2995                 return -EPERM;
2996 
2997         /*
2998          * We're gonna grab cgroup_mutex which nests outside kernfs
2999          * active_ref.  kernfs_rename() doesn't require active_ref
3000          * protection.  Break them before grabbing cgroup_mutex.
3001          */
3002         kernfs_break_active_protection(new_parent);
3003         kernfs_break_active_protection(kn);
3004 
3005         mutex_lock(&cgroup_mutex);
3006 
3007         ret = kernfs_rename(kn, new_parent, new_name_str);
3008 
3009         mutex_unlock(&cgroup_mutex);
3010 
3011         kernfs_unbreak_active_protection(kn);
3012         kernfs_unbreak_active_protection(new_parent);
3013         return ret;
3014 }
3015 
3016 /* set uid and gid of cgroup dirs and files to that of the creator */
3017 static int cgroup_kn_set_ugid(struct kernfs_node *kn)
3018 {
3019         struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
3020                                .ia_uid = current_fsuid(),
3021                                .ia_gid = current_fsgid(), };
3022 
3023         if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
3024             gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
3025                 return 0;
3026 
3027         return kernfs_setattr(kn, &iattr);
3028 }
3029 
3030 static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
3031 {
3032         char name[CGROUP_FILE_NAME_MAX];
3033         struct kernfs_node *kn;
3034         struct lock_class_key *key = NULL;
3035         int ret;
3036 
3037 #ifdef CONFIG_DEBUG_LOCK_ALLOC
3038         key = &cft->lockdep_key;
3039 #endif
3040         kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
3041                                   cgroup_file_mode(cft), 0, cft->kf_ops, cft,
3042                                   NULL, false, key);
3043         if (IS_ERR(kn))
3044                 return PTR_ERR(kn);
3045 
3046         ret = cgroup_kn_set_ugid(kn);
3047         if (ret) {
3048                 kernfs_remove(kn);
3049                 return ret;
3050         }
3051 
3052         if (cft->seq_show == cgroup_populated_show)
3053                 cgrp->populated_kn = kn;
3054         return 0;
3055 }
3056 
3057 /**
3058  * cgroup_addrm_files - add or remove files to a cgroup directory
3059  * @cgrp: the target cgroup
3060  * @cfts: array of cftypes to be added
3061  * @is_add: whether to add or remove
3062  *
3063  * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3064  * For removals, this function never fails.  If addition fails, this
3065  * function doesn't remove files already added.  The caller is responsible
3066  * for cleaning up.
3067  */
3068 static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
3069                               bool is_add)
3070 {
3071         struct cftype *cft;
3072         int ret;
3073 
3074         lockdep_assert_held(&cgroup_mutex);
3075 
3076         for (cft = cfts; cft->name[0] != '\0'; cft++) {
3077                 /* does cft->flags tell us to skip this file on @cgrp? */
3078                 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
3079                         continue;
3080                 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))
3081                         continue;
3082                 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
3083                         continue;
3084                 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
3085                         continue;
3086 
3087                 if (is_add) {
3088                         ret = cgroup_add_file(cgrp, cft);
3089                         if (ret) {
3090                                 pr_warn("%s: failed to add %s, err=%d\n",
3091                                         __func__, cft->name, ret);
3092                                 return ret;
3093                         }
3094                 } else {
3095                         cgroup_rm_file(cgrp, cft);
3096                 }
3097         }
3098         return 0;
3099 }
3100 
3101 static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
3102 {
3103         LIST_HEAD(pending);
3104         struct cgroup_subsys *ss = cfts[0].ss;
3105         struct cgroup *root = &ss->root->cgrp;
3106         struct cgroup_subsys_state *css;
3107         int ret = 0;
3108 
3109         lockdep_assert_held(&cgroup_mutex);
3110 
3111         /* add/rm files for all cgroups created before */
3112         css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3113                 struct cgroup *cgrp = css->cgroup;
3114 
3115                 if (cgroup_is_dead(cgrp))
3116                         continue;
3117 
3118                 ret = cgroup_addrm_files(cgrp, cfts, is_add);
3119                 if (ret)
3120                         break;
3121         }
3122 
3123         if (is_add && !ret)
3124                 kernfs_activate(root->kn);
3125         return ret;
3126 }
3127 
3128 static void cgroup_exit_cftypes(struct cftype *cfts)
3129 {
3130         struct cftype *cft;
3131 
3132         for (cft = cfts; cft->name[0] != '\0'; cft++) {
3133                 /* free copy for custom atomic_write_len, see init_cftypes() */
3134                 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
3135                         kfree(cft->kf_ops);
3136                 cft->kf_ops = NULL;
3137                 cft->ss = NULL;
3138 
3139                 /* revert flags set by cgroup core while adding @cfts */
3140                 cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL);
3141         }
3142 }
3143 
3144 static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3145 {
3146         struct cftype *cft;
3147 
3148         for (cft = cfts; cft->name[0] != '\0'; cft++) {
3149                 struct kernfs_ops *kf_ops;
3150 
3151                 WARN_ON(cft->ss || cft->kf_ops);
3152 
3153                 if (cft->seq_start)
3154                         kf_ops = &cgroup_kf_ops;
3155                 else
3156                         kf_ops = &cgroup_kf_single_ops;
3157 
3158                 /*
3159                  * Ugh... if @cft wants a custom max_write_len, we need to
3160                  * make a copy of kf_ops to set its atomic_write_len.
3161                  */
3162                 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
3163                         kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
3164                         if (!kf_ops) {
3165                                 cgroup_exit_cftypes(cfts);
3166                                 return -ENOMEM;
3167                         }
3168                         kf_ops->atomic_write_len = cft->max_write_len;
3169                 }
3170 
3171                 cft->kf_ops = kf_ops;
3172                 cft->ss = ss;
3173         }
3174 
3175         return 0;
3176 }
3177 
3178 static int cgroup_rm_cftypes_locked(struct cftype *cfts)
3179 {
3180         lockdep_assert_held(&cgroup_mutex);
3181 
3182         if (!cfts || !cfts[0].ss)
3183                 return -ENOENT;
3184 
3185         list_del(&cfts->node);
3186         cgroup_apply_cftypes(cfts, false);
3187         cgroup_exit_cftypes(cfts);
3188         return 0;
3189 }
3190 
3191 /**
3192  * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
3193  * @cfts: zero-length name terminated array of cftypes
3194  *
3195  * Unregister @cfts.  Files described by @cfts are removed from all
3196  * existing cgroups and all future cgroups won't have them either.  This
3197  * function can be called anytime whether @cfts' subsys is attached or not.
3198  *
3199  * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3200  * registered.
3201  */
3202 int cgroup_rm_cftypes(struct cftype *cfts)
3203 {
3204         int ret;
3205 
3206         mutex_lock(&cgroup_mutex);
3207         ret = cgroup_rm_cftypes_locked(cfts);
3208         mutex_unlock(&cgroup_mutex);
3209         return ret;
3210 }
3211 
3212 /**
3213  * cgroup_add_cftypes - add an array of cftypes to a subsystem
3214  * @ss: target cgroup subsystem
3215  * @cfts: zero-length name terminated array of cftypes
3216  *
3217  * Register @cfts to @ss.  Files described by @cfts are created for all
3218  * existing cgroups to which @ss is attached and all future cgroups will
3219  * have them too.  This function can be called anytime whether @ss is
3220  * attached or not.
3221  *
3222  * Returns 0 on successful registration, -errno on failure.  Note that this
3223  * function currently returns 0 as long as @cfts registration is successful
3224  * even if some file creation attempts on existing cgroups fail.
3225  */
3226 static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3227 {
3228         int ret;
3229 
3230         if (ss->disabled)
3231                 return 0;
3232 
3233         if (!cfts || cfts[0].name[0] == '\0')
3234                 return 0;
3235 
3236         ret = cgroup_init_cftypes(ss, cfts);
3237         if (ret)
3238                 return ret;
3239 
3240         mutex_lock(&cgroup_mutex);
3241 
3242         list_add_tail(&cfts->node, &ss->cfts);
3243         ret = cgroup_apply_cftypes(cfts, true);
3244         if (ret)
3245                 cgroup_rm_cftypes_locked(cfts);
3246 
3247         mutex_unlock(&cgroup_mutex);
3248         return ret;
3249 }
3250 
3251 /**
3252  * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
3253  * @ss: target cgroup subsystem
3254  * @cfts: zero-length name terminated array of cftypes
3255  *
3256  * Similar to cgroup_add_cftypes() but the added files are only used for
3257  * the default hierarchy.
3258  */
3259 int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3260 {
3261         struct cftype *cft;
3262 
3263         for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3264                 cft->flags |= __CFTYPE_ONLY_ON_DFL;
3265         return cgroup_add_cftypes(ss, cfts);
3266 }
3267 
3268 /**
3269  * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
3270  * @ss: target cgroup subsystem
3271  * @cfts: zero-length name terminated array of cftypes
3272  *
3273  * Similar to cgroup_add_cftypes() but the added files are only used for
3274  * the legacy hierarchies.
3275  */
3276 int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3277 {
3278         struct cftype *cft;
3279 
3280         /*
3281          * If legacy_flies_on_dfl, we want to show the legacy files on the
3282          * dfl hierarchy but iff the target subsystem hasn't been updated
3283          * for the dfl hierarchy yet.
3284          */
3285         if (!cgroup_legacy_files_on_dfl ||
3286             ss->dfl_cftypes != ss->legacy_cftypes) {
3287                 for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3288                         cft->flags |= __CFTYPE_NOT_ON_DFL;
3289         }
3290 
3291         return cgroup_add_cftypes(ss, cfts);
3292 }
3293 
3294 /**
3295  * cgroup_task_count - count the number of tasks in a cgroup.
3296  * @cgrp: the cgroup in question
3297  *
3298  * Return the number of tasks in the cgroup.
3299  */
3300 static int cgroup_task_count(const struct cgroup *cgrp)
3301 {
3302         int count = 0;
3303         struct cgrp_cset_link *link;
3304 
3305         down_read(&css_set_rwsem);
3306         list_for_each_entry(link, &cgrp->cset_links, cset_link)
3307                 count += atomic_read(&link->cset->refcount);
3308         up_read(&css_set_rwsem);
3309         return count;
3310 }
3311 
3312 /**
3313  * css_next_child - find the next child of a given css
3314  * @pos: the current position (%NULL to initiate traversal)
3315  * @parent: css whose children to walk
3316  *
3317  * This function returns the next child of @parent and should be called
3318  * under either cgroup_mutex or RCU read lock.  The only requirement is
3319  * that @parent and @pos are accessible.  The next sibling is guaranteed to
3320  * be returned regardless of their states.
3321  *
3322  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3323  * css which finished ->css_online() is guaranteed to be visible in the
3324  * future iterations and will stay visible until the last reference is put.
3325  * A css which hasn't finished ->css_online() or already finished
3326  * ->css_offline() may show up during traversal.  It's each subsystem's
3327  * responsibility to synchronize against on/offlining.
3328  */
3329 struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
3330                                            struct cgroup_subsys_state *parent)
3331 {
3332         struct cgroup_subsys_state *next;
3333 
3334         cgroup_assert_mutex_or_rcu_locked();
3335 
3336         /*
3337          * @pos could already have been unlinked from the sibling list.
3338          * Once a cgroup is removed, its ->sibling.next is no longer
3339          * updated when its next sibling changes.  CSS_RELEASED is set when
3340          * @pos is taken off list, at which time its next pointer is valid,
3341          * and, as releases are serialized, the one pointed to by the next
3342          * pointer is guaranteed to not have started release yet.  This
3343          * implies that if we observe !CSS_RELEASED on @pos in this RCU
3344          * critical section, the one pointed to by its next pointer is
3345          * guaranteed to not have finished its RCU grace period even if we
3346          * have dropped rcu_read_lock() inbetween iterations.
3347          *
3348          * If @pos has CSS_RELEASED set, its next pointer can't be
3349          * dereferenced; however, as each css is given a monotonically
3350          * increasing unique serial number and always appended to the
3351          * sibling list, the next one can be found by walking the parent's
3352          * children until the first css with higher serial number than
3353          * @pos's.  While this path can be slower, it happens iff iteration
3354          * races against release and the race window is very small.
3355          */
3356         if (!pos) {
3357                 next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
3358         } else if (likely(!(pos->flags & CSS_RELEASED))) {
3359                 next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3360         } else {
3361                 list_for_each_entry_rcu(next, &parent->children, sibling)
3362                         if (next->serial_nr > pos->serial_nr)
3363                                 break;
3364         }
3365 
3366         /*
3367          * @next, if not pointing to the head, can be dereferenced and is
3368          * the next sibling.
3369          */
3370         if (&next->sibling != &parent->children)
3371                 return next;
3372         return NULL;
3373 }
3374 
3375 /**
3376  * css_next_descendant_pre - find the next descendant for pre-order walk
3377  * @pos: the current position (%NULL to initiate traversal)
3378  * @root: css whose descendants to walk
3379  *
3380  * To be used by css_for_each_descendant_pre().  Find the next descendant
3381  * to visit for pre-order traversal of @root's descendants.  @root is
3382  * included in the iteration and the first node to be visited.
3383  *
3384  * While this function requires cgroup_mutex or RCU read locking, it
3385  * doesn't require the whole traversal to be contained in a single critical
3386  * section.  This function will return the correct next descendant as long
3387  * as both @pos and @root are accessible and @pos is a descendant of @root.
3388  *
3389  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3390  * css which finished ->css_online() is guaranteed to be visible in the
3391  * future iterations and will stay visible until the last reference is put.
3392  * A css which hasn't finished ->css_online() or already finished
3393  * ->css_offline() may show up during traversal.  It's each subsystem's
3394  * responsibility to synchronize against on/offlining.
3395  */
3396 struct cgroup_subsys_state *
3397 css_next_descendant_pre(struct cgroup_subsys_state *pos,
3398                         struct cgroup_subsys_state *root)
3399 {
3400         struct cgroup_subsys_state *next;
3401 
3402         cgroup_assert_mutex_or_rcu_locked();
3403 
3404         /* if first iteration, visit @root */
3405         if (!pos)
3406                 return root;
3407 
3408         /* visit the first child if exists */
3409         next = css_next_child(NULL, pos);
3410         if (next)
3411                 return next;
3412 
3413         /* no child, visit my or the closest ancestor's next sibling */
3414         while (pos != root) {
3415                 next = css_next_child(pos, pos->parent);
3416                 if (next)
3417                         return next;
3418                 pos = pos->parent;
3419         }
3420 
3421         return NULL;
3422 }
3423 
3424 /**
3425  * css_rightmost_descendant - return the rightmost descendant of a css
3426  * @pos: css of interest
3427  *
3428  * Return the rightmost descendant of @pos.  If there's no descendant, @pos
3429  * is returned.  This can be used during pre-order traversal to skip
3430  * subtree of @pos.
3431  *
3432  * While this function requires cgroup_mutex or RCU read locking, it
3433  * doesn't require the whole traversal to be contained in a single critical
3434  * section.  This function will return the correct rightmost descendant as
3435  * long as @pos is accessible.
3436  */
3437 struct cgroup_subsys_state *
3438 css_rightmost_descendant(struct cgroup_subsys_state *pos)
3439 {
3440         struct cgroup_subsys_state *last, *tmp;
3441 
3442         cgroup_assert_mutex_or_rcu_locked();
3443 
3444         do {
3445                 last = pos;
3446                 /* ->prev isn't RCU safe, walk ->next till the end */
3447                 pos = NULL;
3448                 css_for_each_child(tmp, last)
3449                         pos = tmp;
3450         } while (pos);
3451 
3452         return last;
3453 }
3454 
3455 static struct cgroup_subsys_state *
3456 css_leftmost_descendant(struct cgroup_subsys_state *pos)
3457 {
3458         struct cgroup_subsys_state *last;
3459 
3460         do {
3461                 last = pos;
3462                 pos = css_next_child(NULL, pos);
3463         } while (pos);
3464 
3465         return last;
3466 }
3467 
3468 /**
3469  * css_next_descendant_post - find the next descendant for post-order walk
3470  * @pos: the current position (%NULL to initiate traversal)
3471  * @root: css whose descendants to walk
3472  *
3473  * To be used by css_for_each_descendant_post().  Find the next descendant
3474  * to visit for post-order traversal of @root's descendants.  @root is
3475  * included in the iteration and the last node to be visited.
3476  *
3477  * While this function requires cgroup_mutex or RCU read locking, it
3478  * doesn't require the whole traversal to be contained in a single critical
3479  * section.  This function will return the correct next descendant as long
3480  * as both @pos and @cgroup are accessible and @pos is a descendant of
3481  * @cgroup.
3482  *
3483  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3484  * css which finished ->css_online() is guaranteed to be visible in the
3485  * future iterations and will stay visible until the last reference is put.
3486  * A css which hasn't finished ->css_online() or already finished
3487  * ->css_offline() may show up during traversal.  It's each subsystem's
3488  * responsibility to synchronize against on/offlining.
3489  */
3490 struct cgroup_subsys_state *
3491 css_next_descendant_post(struct cgroup_subsys_state *pos,
3492                          struct cgroup_subsys_state *root)
3493 {
3494         struct cgroup_subsys_state *next;
3495 
3496         cgroup_assert_mutex_or_rcu_locked();
3497 
3498         /* if first iteration, visit leftmost descendant which may be @root */
3499         if (!pos)
3500                 return css_leftmost_descendant(root);
3501 
3502         /* if we visited @root, we're done */
3503         if (pos == root)
3504                 return NULL;
3505 
3506         /* if there's an unvisited sibling, visit its leftmost descendant */
3507         next = css_next_child(pos, pos->parent);
3508         if (next)
3509                 return css_leftmost_descendant(next);
3510 
3511         /* no sibling left, visit parent */
3512         return pos->parent;
3513 }
3514 
3515 /**
3516  * css_has_online_children - does a css have online children
3517  * @css: the target css
3518  *
3519  * Returns %true if @css has any online children; otherwise, %false.  This
3520  * function can be called from any context but the caller is responsible
3521  * for synchronizing against on/offlining as necessary.
3522  */
3523 bool css_has_online_children(struct cgroup_subsys_state *css)
3524 {
3525         struct cgroup_subsys_state *child;
3526         bool ret = false;
3527 
3528         rcu_read_lock();
3529         css_for_each_child(child, css) {
3530                 if (child->flags & CSS_ONLINE) {
3531                         ret = true;
3532                         break;
3533                 }
3534         }
3535         rcu_read_unlock();
3536         return ret;
3537 }
3538 
3539 /**
3540  * css_advance_task_iter - advance a task itererator to the next css_set
3541  * @it: the iterator to advance
3542  *
3543  * Advance @it to the next css_set to walk.
3544  */
3545 static void css_advance_task_iter(struct css_task_iter *it)
3546 {
3547         struct list_head *l = it->cset_pos;
3548         struct cgrp_cset_link *link;
3549         struct css_set *cset;
3550 
3551         /* Advance to the next non-empty css_set */
3552         do {
3553                 l = l->next;
3554                 if (l == it->cset_head) {
3555                         it->cset_pos = NULL;
3556                         return;
3557                 }
3558 
3559                 if (it->ss) {
3560                         cset = container_of(l, struct css_set,
3561                                             e_cset_node[it->ss->id]);
3562                 } else {
3563                         link = list_entry(l, struct cgrp_cset_link, cset_link);
3564                         cset = link->cset;
3565                 }
3566         } while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks));
3567 
3568         it->cset_pos = l;
3569 
3570         if (!list_empty(&cset->tasks))
3571                 it->task_pos = cset->tasks.next;
3572         else
3573                 it->task_pos = cset->mg_tasks.next;
3574 
3575         it->tasks_head = &cset->tasks;
3576         it->mg_tasks_head = &cset->mg_tasks;
3577 }
3578 
3579 /**
3580  * css_task_iter_start - initiate task iteration
3581  * @css: the css to walk tasks of
3582  * @it: the task iterator to use
3583  *
3584  * Initiate iteration through the tasks of @css.  The caller can call
3585  * css_task_iter_next() to walk through the tasks until the function
3586  * returns NULL.  On completion of iteration, css_task_iter_end() must be
3587  * called.
3588  *
3589  * Note that this function acquires a lock which is released when the
3590  * iteration finishes.  The caller can't sleep while iteration is in
3591  * progress.
3592  */
3593 void css_task_iter_start(struct cgroup_subsys_state *css,
3594                          struct css_task_iter *it)
3595         __acquires(css_set_rwsem)
3596 {
3597         /* no one should try to iterate before mounting cgroups */
3598         WARN_ON_ONCE(!use_task_css_set_links);
3599 
3600         down_read(&css_set_rwsem);
3601 
3602         it->ss = css->ss;
3603 
3604         if (it->ss)
3605                 it->cset_pos = &css->cgroup->e_csets[css->ss->id];
3606         else
3607                 it->cset_pos = &css->cgroup->cset_links;
3608 
3609         it->cset_head = it->cset_pos;
3610 
3611         css_advance_task_iter(it);
3612 }
3613 
3614 /**
3615  * css_task_iter_next - return the next task for the iterator
3616  * @it: the task iterator being iterated
3617  *
3618  * The "next" function for task iteration.  @it should have been
3619  * initialized via css_task_iter_start().  Returns NULL when the iteration
3620  * reaches the end.
3621  */
3622 struct task_struct *css_task_iter_next(struct css_task_iter *it)
3623 {
3624         struct task_struct *res;
3625         struct list_head *l = it->task_pos;
3626 
3627         /* If the iterator cg is NULL, we have no tasks */
3628         if (!it->cset_pos)
3629                 return NULL;
3630         res = list_entry(l, struct task_struct, cg_list);
3631 
3632         /*
3633          * Advance iterator to find next entry.  cset->tasks is consumed
3634          * first and then ->mg_tasks.  After ->mg_tasks, we move onto the
3635          * next cset.
3636          */
3637         l = l->next;
3638 
3639         if (l == it->tasks_head)
3640                 l = it->mg_tasks_head->next;
3641 
3642         if (l == it->mg_tasks_head)
3643                 css_advance_task_iter(it);
3644         else
3645                 it->task_pos = l;
3646 
3647         return res;
3648 }
3649 
3650 /**
3651  * css_task_iter_end - finish task iteration
3652  * @it: the task iterator to finish
3653  *
3654  * Finish task iteration started by css_task_iter_start().
3655  */
3656 void css_task_iter_end(struct css_task_iter *it)
3657         __releases(css_set_rwsem)
3658 {
3659         up_read(&css_set_rwsem);
3660 }
3661 
3662 /**
3663  * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
3664  * @to: cgroup to which the tasks will be moved
3665  * @from: cgroup in which the tasks currently reside
3666  *
3667  * Locking rules between cgroup_post_fork() and the migration path
3668  * guarantee that, if a task is forking while being migrated, the new child
3669  * is guaranteed to be either visible in the source cgroup after the
3670  * parent's migration is complete or put into the target cgroup.  No task
3671  * can slip out of migration through forking.
3672  */
3673 int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
3674 {
3675         LIST_HEAD(preloaded_csets);
3676         struct cgrp_cset_link *link;
3677         struct css_task_iter it;
3678         struct task_struct *task;
3679         int ret;
3680 
3681         mutex_lock(&cgroup_mutex);
3682 
3683         /* all tasks in @from are being moved, all csets are source */
3684         down_read(&css_set_rwsem);
3685         list_for_each_entry(link, &from->cset_links, cset_link)
3686                 cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
3687         up_read(&css_set_rwsem);
3688 
3689         ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
3690         if (ret)
3691                 goto out_err;
3692 
3693         /*
3694          * Migrate tasks one-by-one until @form is empty.  This fails iff
3695          * ->can_attach() fails.
3696          */
3697         do {
3698                 css_task_iter_start(&from->self, &it);
3699                 task = css_task_iter_next(&it);
3700                 if (task)
3701                         get_task_struct(task);
3702                 css_task_iter_end(&it);
3703 
3704                 if (task) {
3705                         ret = cgroup_migrate(to, task, false);
3706                         put_task_struct(task);
3707                 }
3708         } while (task && !ret);
3709 out_err:
3710         cgroup_migrate_finish(&preloaded_csets);
3711         mutex_unlock(&cgroup_mutex);
3712         return ret;
3713 }
3714 
3715 /*
3716  * Stuff for reading the 'tasks'/'procs' files.
3717  *
3718  * Reading this file can return large amounts of data if a cgroup has
3719  * *lots* of attached tasks. So it may need several calls to read(),
3720  * but we cannot guarantee that the information we produce is correct
3721  * unless we produce it entirely atomically.
3722  *
3723  */
3724 
3725 /* which pidlist file are we talking about? */
3726 enum cgroup_filetype {
3727         CGROUP_FILE_PROCS,
3728         CGROUP_FILE_TASKS,
3729 };
3730 
3731 /*
3732  * A pidlist is a list of pids that virtually represents the contents of one
3733  * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
3734  * a pair (one each for procs, tasks) for each pid namespace that's relevant
3735  * to the cgroup.
3736  */
3737 struct cgroup_pidlist {
3738         /*
3739          * used to find which pidlist is wanted. doesn't change as long as
3740          * this particular list stays in the list.
3741         */
3742         struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
3743         /* array of xids */
3744         pid_t *list;
3745         /* how many elements the above list has */
3746         int length;
3747         /* each of these stored in a list by its cgroup */
3748         struct list_head links;
3749         /* pointer to the cgroup we belong to, for list removal purposes */
3750         struct cgroup *owner;
3751         /* for delayed destruction */
3752         struct delayed_work destroy_dwork;
3753 };
3754 
3755 /*
3756  * The following two functions "fix" the issue where there are more pids
3757  * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
3758  * TODO: replace with a kernel-wide solution to this problem
3759  */
3760 #define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
3761 static void *pidlist_allocate(int count)
3762 {
3763         if (PIDLIST_TOO_LARGE(count))
3764                 return vmalloc(count * sizeof(pid_t));
3765         else
3766                 return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
3767 }
3768 
3769 static void pidlist_free(void *p)
3770 {
3771         if (is_vmalloc_addr(p))
3772                 vfree(p);
3773         else
3774                 kfree(p);
3775 }
3776 
3777 /*
3778  * Used to destroy all pidlists lingering waiting for destroy timer.  None
3779  * should be left afterwards.
3780  */
3781 static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
3782 {
3783         struct cgroup_pidlist *l, *tmp_l;
3784 
3785         mutex_lock(&cgrp->pidlist_mutex);
3786         list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
3787                 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
3788         mutex_unlock(&cgrp->pidlist_mutex);
3789 
3790         flush_workqueue(cgroup_pidlist_destroy_wq);
3791         BUG_ON(!list_empty(&cgrp->pidlists));
3792 }
3793 
3794 static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
3795 {
3796         struct delayed_work *dwork = to_delayed_work(work);
3797         struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
3798                                                 destroy_dwork);
3799         struct cgroup_pidlist *tofree = NULL;
3800 
3801         mutex_lock(&l->owner->pidlist_mutex);
3802 
3803         /*
3804          * Destroy iff we didn't get queued again.  The state won't change
3805          * as destroy_dwork can only be queued while locked.
3806          */
3807         if (!delayed_work_pending(dwork)) {
3808                 list_del(&l->links);
3809                 pidlist_free(l->list);
3810                 put_pid_ns(l->key.ns);
3811                 tofree = l;
3812         }
3813 
3814         mutex_unlock(&l->owner->pidlist_mutex);
3815         kfree(tofree);
3816 }
3817 
3818 /*
3819  * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
3820  * Returns the number of unique elements.
3821  */
3822 static int pidlist_uniq(pid_t *list, int length)
3823 {
3824         int src, dest = 1;
3825 
3826         /*
3827          * we presume the 0th element is unique, so i starts at 1. trivial
3828          * edge cases first; no work needs to be done for either
3829          */
3830         if (length == 0 || length == 1)
3831                 return length;
3832         /* src and dest walk down the list; dest counts unique elements */
3833         for (src = 1; src < length; src++) {
3834                 /* find next unique element */
3835                 while (list[src] == list[src-1]) {
3836                         src++;
3837                         if (src == length)
3838                                 goto after;
3839                 }
3840                 /* dest always points to where the next unique element goes */
3841                 list[dest] = list[src];
3842                 dest++;
3843         }
3844 after:
3845         return dest;
3846 }
3847 
3848 /*
3849  * The two pid files - task and cgroup.procs - guaranteed that the result
3850  * is sorted, which forced this whole pidlist fiasco.  As pid order is
3851  * different per namespace, each namespace needs differently sorted list,
3852  * making it impossible to use, for example, single rbtree of member tasks
3853  * sorted by task pointer.  As pidlists can be fairly large, allocating one
3854  * per open file is dangerous, so cgroup had to implement shared pool of
3855  * pidlists keyed by cgroup and namespace.
3856  *
3857  * All this extra complexity was caused by the original implementation
3858  * committing to an entirely unnecessary property.  In the long term, we
3859  * want to do away with it.  Explicitly scramble sort order if on the
3860  * default hierarchy so that no such expectation exists in the new
3861  * interface.
3862  *
3863  * Scrambling is done by swapping every two consecutive bits, which is
3864  * non-identity one-to-one mapping which disturbs sort order sufficiently.
3865  */
3866 static pid_t pid_fry(pid_t pid)
3867 {
3868         unsigned a = pid & 0x55555555;
3869         unsigned b = pid & 0xAAAAAAAA;
3870 
3871         return (a << 1) | (b >> 1);
3872 }
3873 
3874 static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
3875 {
3876         if (cgroup_on_dfl(cgrp))
3877                 return pid_fry(pid);
3878         else
3879                 return pid;
3880 }
3881 
3882 static int cmppid(const void *a, const void *b)
3883 {
3884         return *(pid_t *)a - *(pid_t *)b;
3885 }
3886 
3887 static int fried_cmppid(const void *a, const void *b)
3888 {
3889         return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
3890 }
3891 
3892 static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
3893                                                   enum cgroup_filetype type)
3894 {
3895         struct cgroup_pidlist *l;
3896         /* don't need task_nsproxy() if we're looking at ourself */
3897         struct pid_namespace *ns = task_active_pid_ns(current);
3898 
3899         lockdep_assert_held(&cgrp->pidlist_mutex);
3900 
3901         list_for_each_entry(l, &cgrp->pidlists, links)
3902                 if (l->key.type == type && l->key.ns == ns)
3903                         return l;
3904         return NULL;
3905 }
3906 
3907 /*
3908  * find the appropriate pidlist for our purpose (given procs vs tasks)
3909  * returns with the lock on that pidlist already held, and takes care
3910  * of the use count, or returns NULL with no locks held if we're out of
3911  * memory.
3912  */
3913 static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
3914                                                 enum cgroup_filetype type)
3915 {
3916         struct cgroup_pidlist *l;
3917 
3918         lockdep_assert_held(&cgrp->pidlist_mutex);
3919 
3920         l = cgroup_pidlist_find(cgrp, type);
3921         if (l)
3922                 return l;
3923 
3924         /* entry not found; create a new one */
3925         l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
3926         if (!l)
3927                 return l;
3928 
3929         INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
3930         l->key.type = type;
3931         /* don't need task_nsproxy() if we're looking at ourself */
3932         l->key.ns = get_pid_ns(task_active_pid_ns(current));
3933         l->owner = cgrp;
3934         list_add(&l->links, &cgrp->pidlists);
3935         return l;
3936 }
3937 
3938 /*
3939  * Load a cgroup's pidarray with either procs' tgids or tasks' pids
3940  */
3941 static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
3942                               struct cgroup_pidlist **lp)
3943 {
3944         pid_t *array;
3945         int length;
3946         int pid, n = 0; /* used for populating the array */
3947         struct css_task_iter it;
3948         struct task_struct *tsk;
3949         struct cgroup_pidlist *l;
3950 
3951         lockdep_assert_held(&cgrp->pidlist_mutex);
3952 
3953         /*
3954          * If cgroup gets more users after we read count, we won't have
3955          * enough space - tough.  This race is indistinguishable to the
3956          * caller from the case that the additional cgroup users didn't
3957          * show up until sometime later on.
3958          */
3959         length = cgroup_task_count(cgrp);
3960         array = pidlist_allocate(length);
3961         if (!array)
3962                 return -ENOMEM;
3963         /* now, populate the array */
3964         css_task_iter_start(&cgrp->self, &it);
3965         while ((tsk = css_task_iter_next(&it))) {
3966                 if (unlikely(n == length))
3967                         break;
3968                 /* get tgid or pid for procs or tasks file respectively */
3969                 if (type == CGROUP_FILE_PROCS)
3970                         pid = task_tgid_vnr(tsk);
3971                 else
3972                         pid = task_pid_vnr(tsk);
3973                 if (pid > 0) /* make sure to only use valid results */
3974                         array[n++] = pid;
3975         }
3976         css_task_iter_end(&it);
3977         length = n;
3978         /* now sort & (if procs) strip out duplicates */
3979         if (cgroup_on_dfl(cgrp))
3980                 sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
3981         else
3982                 sort(array, length, sizeof(pid_t), cmppid, NULL);
3983         if (type == CGROUP_FILE_PROCS)
3984                 length = pidlist_uniq(array, length);
3985 
3986         l = cgroup_pidlist_find_create(cgrp, type);
3987         if (!l) {
3988                 pidlist_free(array);
3989                 return -ENOMEM;
3990         }
3991 
3992         /* store array, freeing old if necessary */
3993         pidlist_free(l->list);
3994         l->list = array;
3995         l->length = length;
3996         *lp = l;
3997         return 0;
3998 }
3999 
4000 /**
4001  * cgroupstats_build - build and fill cgroupstats
4002  * @stats: cgroupstats to fill information into
4003  * @dentry: A dentry entry belonging to the cgroup for which stats have
4004  * been requested.
4005  *
4006  * Build and fill cgroupstats so that taskstats can export it to user
4007  * space.
4008  */
4009 int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
4010 {
4011         struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
4012         struct cgroup *cgrp;
4013         struct css_task_iter it;
4014         struct task_struct *tsk;
4015 
4016         /* it should be kernfs_node belonging to cgroupfs and is a directory */
4017         if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
4018             kernfs_type(kn) != KERNFS_DIR)
4019                 return -EINVAL;
4020 
4021         mutex_lock(&cgroup_mutex);
4022 
4023         /*
4024          * We aren't being called from kernfs and there's no guarantee on
4025          * @kn->priv's validity.  For this and css_tryget_online_from_dir(),
4026          * @kn->priv is RCU safe.  Let's do the RCU dancing.
4027          */
4028         rcu_read_lock();
4029         cgrp = rcu_dereference(kn->priv);
4030         if (!cgrp || cgroup_is_dead(cgrp)) {
4031                 rcu_read_unlock();
4032                 mutex_unlock(&cgroup_mutex);
4033                 return -ENOENT;
4034         }
4035         rcu_read_unlock();
4036 
4037         css_task_iter_start(&cgrp->self, &it);
4038         while ((tsk = css_task_iter_next(&it))) {
4039                 switch (tsk->state) {
4040                 case TASK_RUNNING:
4041                         stats->nr_running++;
4042                         break;
4043                 case TASK_INTERRUPTIBLE:
4044                         stats->nr_sleeping++;
4045                         break;
4046                 case TASK_UNINTERRUPTIBLE:
4047                         stats->nr_uninterruptible++;
4048                         break;
4049                 case TASK_STOPPED:
4050                         stats->nr_stopped++;
4051                         break;
4052                 default:
4053                         if (delayacct_is_task_waiting_on_io(tsk))
4054                                 stats->nr_io_wait++;
4055                         break;
4056                 }
4057         }
4058         css_task_iter_end(&it);
4059 
4060         mutex_unlock(&cgroup_mutex);
4061         return 0;
4062 }
4063 
4064 
4065 /*
4066  * seq_file methods for the tasks/procs files. The seq_file position is the
4067  * next pid to display; the seq_file iterator is a pointer to the pid
4068  * in the cgroup->l->list array.
4069  */
4070 
4071 static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
4072 {
4073         /*
4074          * Initially we receive a position value that corresponds to
4075          * one more than the last pid shown (or 0 on the first call or
4076          * after a seek to the start). Use a binary-search to find the
4077          * next pid to display, if any
4078          */
4079         struct kernfs_open_file *of = s->private;
4080         struct cgroup *cgrp = seq_css(s)->cgroup;
4081         struct cgroup_pidlist *l;
4082         enum cgroup_filetype type = seq_cft(s)->private;
4083         int index = 0, pid = *pos;
4084         int *iter, ret;
4085 
4086         mutex_lock(&cgrp->pidlist_mutex);
4087 
4088         /*
4089          * !NULL @of->priv indicates that this isn't the first start()
4090          * after open.  If the matching pidlist is around, we can use that.
4091          * Look for it.  Note that @of->priv can't be used directly.  It
4092          * could already have been destroyed.
4093          */
4094         if (of->priv)
4095                 of->priv = cgroup_pidlist_find(cgrp, type);
4096 
4097         /*
4098          * Either this is the first start() after open or the matching
4099          * pidlist has been destroyed inbetween.  Create a new one.
4100          */
4101         if (!of->priv) {
4102                 ret = pidlist_array_load(cgrp, type,
4103                                          (struct cgroup_pidlist **)&of->priv);
4104                 if (ret)
4105                         return ERR_PTR(ret);
4106         }
4107         l = of->priv;
4108 
4109         if (pid) {
4110                 int end = l->length;
4111 
4112                 while (index < end) {
4113                         int mid = (index + end) / 2;
4114                         if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
4115                                 index = mid;
4116                                 break;
4117                         } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
4118                                 index = mid + 1;
4119                         else
4120                                 end = mid;
4121                 }
4122         }
4123         /* If we're off the end of the array, we're done */
4124         if (index >= l->length)
4125                 return NULL;
4126         /* Update the abstract position to be the actual pid that we found */
4127         iter = l->list + index;
4128         *pos = cgroup_pid_fry(cgrp, *iter);
4129         return iter;
4130 }
4131 
4132 static void cgroup_pidlist_stop(struct seq_file *s, void *v)
4133 {
4134         struct kernfs_open_file *of = s->private;
4135         struct cgroup_pidlist *l = of->priv;
4136 
4137         if (l)
4138                 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
4139                                  CGROUP_PIDLIST_DESTROY_DELAY);
4140         mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
4141 }
4142 
4143 static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
4144 {
4145         struct kernfs_open_file *of = s->private;
4146         struct cgroup_pidlist *l = of->priv;
4147         pid_t *p = v;
4148         pid_t *end = l->list + l->length;
4149         /*
4150          * Advance to the next pid in the array. If this goes off the
4151          * end, we're done
4152          */
4153         p++;
4154         if (p >= end) {
4155                 return NULL;
4156         } else {
4157                 *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
4158                 return p;
4159         }
4160 }
4161 
4162 static int cgroup_pidlist_show(struct seq_file *s, void *v)
4163 {
4164         return seq_printf(s, "%d\n", *(int *)v);
4165 }
4166 
4167 static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
4168                                          struct cftype *cft)
4169 {
4170         return notify_on_release(css->cgroup);
4171 }
4172 
4173 static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
4174                                           struct cftype *cft, u64 val)
4175 {
4176         clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
4177         if (val)
4178                 set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4179         else
4180                 clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4181         return 0;
4182 }
4183 
4184 static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
4185                                       struct cftype *cft)
4186 {
4187         return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4188 }
4189 
4190 static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
4191                                        struct cftype *cft, u64 val)
4192 {
4193         if (val)
4194                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4195         else
4196                 clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4197         return 0;
4198 }
4199 
4200 /* cgroup core interface files for the default hierarchy */
4201 static struct cftype cgroup_dfl_base_files[] = {
4202         {
4203                 .name = "cgroup.procs",
4204                 .seq_start = cgroup_pidlist_start,
4205                 .seq_next = cgroup_pidlist_next,
4206                 .seq_stop = cgroup_pidlist_stop,
4207                 .seq_show = cgroup_pidlist_show,
4208                 .private = CGROUP_FILE_PROCS,
4209                 .write = cgroup_procs_write,
4210                 .mode = S_IRUGO | S_IWUSR,
4211         },
4212         {
4213                 .name = "cgroup.controllers",
4214                 .flags = CFTYPE_ONLY_ON_ROOT,
4215                 .seq_show = cgroup_root_controllers_show,
4216         },
4217         {
4218                 .name = "cgroup.controllers",
4219                 .flags = CFTYPE_NOT_ON_ROOT,
4220                 .seq_show = cgroup_controllers_show,
4221         },
4222         {
4223                 .name = "cgroup.subtree_control",
4224                 .seq_show = cgroup_subtree_control_show,
4225                 .write = cgroup_subtree_control_write,
4226         },
4227         {
4228                 .name = "cgroup.populated",
4229                 .flags = CFTYPE_NOT_ON_ROOT,
4230                 .seq_show = cgroup_populated_show,
4231         },
4232         { }     /* terminate */
4233 };
4234 
4235 /* cgroup core interface files for the legacy hierarchies */
4236 static struct cftype cgroup_legacy_base_files[] = {
4237         {
4238                 .name = "cgroup.procs",
4239                 .seq_start = cgroup_pidlist_start,
4240                 .seq_next = cgroup_pidlist_next,
4241                 .seq_stop = cgroup_pidlist_stop,
4242                 .seq_show = cgroup_pidlist_show,
4243                 .private = CGROUP_FILE_PROCS,
4244                 .write = cgroup_procs_write,
4245                 .mode = S_IRUGO | S_IWUSR,
4246         },
4247         {
4248                 .name = "cgroup.clone_children",
4249                 .read_u64 = cgroup_clone_children_read,
4250                 .write_u64 = cgroup_clone_children_write,
4251         },
4252         {
4253                 .name = "cgroup.sane_behavior",
4254                 .flags = CFTYPE_ONLY_ON_ROOT,
4255                 .seq_show = cgroup_sane_behavior_show,
4256         },
4257         {
4258                 .name = "tasks",
4259                 .seq_start = cgroup_pidlist_start,
4260                 .seq_next = cgroup_pidlist_next,
4261                 .seq_stop = cgroup_pidlist_stop,
4262                 .seq_show = cgroup_pidlist_show,
4263                 .private = CGROUP_FILE_TASKS,
4264                 .write = cgroup_tasks_write,
4265                 .mode = S_IRUGO | S_IWUSR,
4266         },
4267         {
4268                 .name = "notify_on_release",
4269                 .read_u64 = cgroup_read_notify_on_release,
4270                 .write_u64 = cgroup_write_notify_on_release,
4271         },
4272         {
4273                 .name = "release_agent",
4274                 .flags = CFTYPE_ONLY_ON_ROOT,
4275                 .seq_show = cgroup_release_agent_show,
4276                 .write = cgroup_release_agent_write,
4277                 .max_write_len = PATH_MAX - 1,
4278         },
4279         { }     /* terminate */
4280 };
4281 
4282 /**
4283  * cgroup_populate_dir - create subsys files in a cgroup directory
4284  * @cgrp: target cgroup
4285  * @subsys_mask: mask of the subsystem ids whose files should be added
4286  *
4287  * On failure, no file is added.
4288  */
4289 static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask)
4290 {
4291         struct cgroup_subsys *ss;
4292         int i, ret = 0;
4293 
4294         /* process cftsets of each subsystem */
4295         for_each_subsys(ss, i) {
4296                 struct cftype *cfts;
4297 
4298                 if (!(subsys_mask & (1 << i)))
4299                         continue;
4300 
4301                 list_for_each_entry(cfts, &ss->cfts, node) {
4302                         ret = cgroup_addrm_files(cgrp, cfts, true);
4303                         if (ret < 0)
4304                                 goto err;
4305                 }
4306         }
4307         return 0;
4308 err:
4309         cgroup_clear_dir(cgrp, subsys_mask);
4310         return ret;
4311 }
4312 
4313 /*
4314  * css destruction is four-stage process.
4315  *
4316  * 1. Destruction starts.  Killing of the percpu_ref is initiated.
4317  *    Implemented in kill_css().
4318  *
4319  * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4320  *    and thus css_tryget_online() is guaranteed to fail, the css can be
4321  *    offlined by invoking offline_css().  After offlining, the base ref is
4322  *    put.  Implemented in css_killed_work_fn().
4323  *
4324  * 3. When the percpu_ref reaches zero, the only possible remaining
4325  *    accessors are inside RCU read sections.  css_release() schedules the
4326  *    RCU callback.
4327  *
4328  * 4. After the grace period, the css can be freed.  Implemented in
4329  *    css_free_work_fn().
4330  *
4331  * It is actually hairier because both step 2 and 4 require process context
4332  * and thus involve punting to css->destroy_work adding two additional
4333  * steps to the already complex sequence.
4334  */
4335 static void css_free_work_fn(struct work_struct *work)
4336 {
4337         struct cgroup_subsys_state *css =
4338                 container_of(work, struct cgroup_subsys_state, destroy_work);
4339         struct cgroup *cgrp = css->cgroup;
4340 
4341         percpu_ref_exit(&css->refcnt);
4342 
4343         if (css->ss) {
4344                 /* css free path */
4345                 if (css->parent)
4346                         css_put(css->parent);
4347 
4348                 css->ss->css_free(css);
4349                 cgroup_put(cgrp);
4350         } else {
4351                 /* cgroup free path */
4352                 atomic_dec(&cgrp->root->nr_cgrps);
4353                 cgroup_pidlist_destroy_all(cgrp);
4354 
4355                 if (cgroup_parent(cgrp)) {
4356                         /*
4357                          * We get a ref to the parent, and put the ref when
4358                          * this cgroup is being freed, so it's guaranteed
4359                          * that the parent won't be destroyed before its
4360                          * children.
4361                          */
4362                         cgroup_put(cgroup_parent(cgrp));
4363                         kernfs_put(cgrp->kn);
4364                         kfree(cgrp);
4365                 } else {
4366                         /*
4367                          * This is root cgroup's refcnt reaching zero,
4368                          * which indicates that the root should be
4369                          * released.
4370                          */
4371                         cgroup_destroy_root(cgrp->root);
4372                 }
4373         }
4374 }
4375 
4376 static void css_free_rcu_fn(struct rcu_head *rcu_head)
4377 {
4378         struct cgroup_subsys_state *css =
4379                 container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4380 
4381         INIT_WORK(&css->destroy_work, css_free_work_fn);
4382         queue_work(cgroup_destroy_wq, &css->destroy_work);
4383 }
4384 
4385 static void css_release_work_fn(struct work_struct *work)
4386 {
4387         struct cgroup_subsys_state *css =
4388                 container_of(work, struct cgroup_subsys_state, destroy_work);
4389         struct cgroup_subsys *ss = css->ss;
4390         struct cgroup *cgrp = css->cgroup;
4391 
4392         mutex_lock(&cgroup_mutex);
4393 
4394         css->flags |= CSS_RELEASED;
4395         list_del_rcu(&css->sibling);
4396 
4397         if (ss) {
4398                 /* css release path */
4399                 cgroup_idr_remove(&ss->css_idr, css->id);
4400         } else {
4401                 /* cgroup release path */
4402                 cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
4403                 cgrp->id = -1;
4404 
4405                 /*
4406                  * There are two control paths which try to determine
4407                  * cgroup from dentry without going through kernfs -
4408                  * cgroupstats_build() and css_tryget_online_from_dir().
4409                  * Those are supported by RCU protecting clearing of
4410                  * cgrp->kn->priv backpointer.
4411                  */
4412                 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, NULL);
4413         }
4414 
4415         mutex_unlock(&cgroup_mutex);
4416 
4417         call_rcu(&css->rcu_head, css_free_rcu_fn);
4418 }
4419 
4420 static void css_release(struct percpu_ref *ref)
4421 {
4422         struct cgroup_subsys_state *css =
4423                 container_of(ref, struct cgroup_subsys_state, refcnt);
4424 
4425         INIT_WORK(&css->destroy_work, css_release_work_fn);
4426         queue_work(cgroup_destroy_wq, &css->destroy_work);
4427 }
4428 
4429 static void init_and_link_css(struct cgroup_subsys_state *css,
4430                               struct cgroup_subsys *ss, struct cgroup *cgrp)
4431 {
4432         lockdep_assert_held(&cgroup_mutex);
4433 
4434         cgroup_get(cgrp);
4435 
4436         memset(css, 0, sizeof(*css));
4437         css->cgroup = cgrp;
4438         css->ss = ss;
4439         INIT_LIST_HEAD(&css->sibling);
4440         INIT_LIST_HEAD(&css->children);
4441         css->serial_nr = css_serial_nr_next++;
4442 
4443         if (cgroup_parent(cgrp)) {
4444                 css->parent = cgroup_css(cgroup_parent(cgrp), ss);
4445                 css_get(css->parent);
4446         }
4447 
4448         BUG_ON(cgroup_css(cgrp, ss));
4449 }
4450 
4451 /* invoke ->css_online() on a new CSS and mark it online if successful */
4452 static int online_css(struct cgroup_subsys_state *css)
4453 {
4454         struct cgroup_subsys *ss = css->ss;
4455         int ret = 0;
4456 
4457         lockdep_assert_held(&cgroup_mutex);
4458 
4459         if (ss->css_online)
4460                 ret = ss->css_online(css);
4461         if (!ret) {
4462                 css->flags |= CSS_ONLINE;
4463                 rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
4464         }
4465         return ret;
4466 }
4467 
4468 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4469 static void offline_css(struct cgroup_subsys_state *css)
4470 {
4471         struct cgroup_subsys *ss = css->ss;
4472 
4473         lockdep_assert_held(&cgroup_mutex);
4474 
4475         if (!(css->flags & CSS_ONLINE))
4476                 return;
4477 
4478         if (ss->css_offline)
4479                 ss->css_offline(css);
4480 
4481         css->flags &= ~CSS_ONLINE;
4482         RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
4483 
4484         wake_up_all(&css->cgroup->offline_waitq);
4485 }
4486 
4487 /**
4488  * create_css - create a cgroup_subsys_state
4489  * @cgrp: the cgroup new css will be associated with
4490  * @ss: the subsys of new css
4491  * @visible: whether to create control knobs for the new css or not
4492  *
4493  * Create a new css associated with @cgrp - @ss pair.  On success, the new
4494  * css is online and installed in @cgrp with all interface files created if
4495  * @visible.  Returns 0 on success, -errno on failure.
4496  */
4497 static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
4498                       bool visible)
4499 {
4500         struct cgroup *parent = cgroup_parent(cgrp);
4501         struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
4502         struct cgroup_subsys_state *css;
4503         int err;
4504 
4505         lockdep_assert_held(&cgroup_mutex);
4506 
4507         css = ss->css_alloc(parent_css);
4508         if (IS_ERR(css))
4509                 return PTR_ERR(css);
4510 
4511         init_and_link_css(css, ss, cgrp);
4512 
4513         err = percpu_ref_init(&css->refcnt, css_release);
4514         if (err)
4515                 goto err_free_css;
4516 
4517         err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT);
4518         if (err < 0)
4519                 goto err_free_percpu_ref;
4520         css->id = err;
4521 
4522         if (visible) {
4523                 err = cgroup_populate_dir(cgrp, 1 << ss->id);
4524                 if (err)
4525                         goto err_free_id;
4526         }
4527 
4528         /* @css is ready to be brought online now, make it visible */
4529         list_add_tail_rcu(&css->sibling, &parent_css->children);
4530         cgroup_idr_replace(&ss->css_idr, css, css->id);
4531 
4532         err = online_css(css);
4533         if (err)
4534                 goto err_list_del;
4535 
4536         if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
4537             cgroup_parent(parent)) {
4538                 pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4539                         current->comm, current->pid, ss->name);
4540                 if (!strcmp(ss->name, "memory"))
4541                         pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
4542                 ss->warned_broken_hierarchy = true;
4543         }
4544 
4545         return 0;
4546 
4547 err_list_del:
4548         list_del_rcu(&css->sibling);
4549         cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4550 err_free_id:
4551         cgroup_idr_remove(&ss->css_idr, css->id);
4552 err_free_percpu_ref:
4553         percpu_ref_exit(&css->refcnt);
4554 err_free_css:
4555         call_rcu(&css->rcu_head, css_free_rcu_fn);
4556         return err;
4557 }
4558 
4559 static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
4560                         umode_t mode)
4561 {
4562         struct cgroup *parent, *cgrp;
4563         struct cgroup_root *root;
4564         struct cgroup_subsys *ss;
4565         struct kernfs_node *kn;
4566         struct cftype *base_files;
4567         int ssid, ret;
4568 
4569         /* Do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable.
4570          */
4571         if (strchr(name, '\n'))
4572                 return -EINVAL;
4573 
4574         parent = cgroup_kn_lock_live(parent_kn);
4575         if (!parent)
4576                 return -ENODEV;
4577         root = parent->root;
4578 
4579         /* allocate the cgroup and its ID, 0 is reserved for the root */
4580         cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
4581         if (!cgrp) {
4582                 ret = -ENOMEM;
4583                 goto out_unlock;
4584         }
4585 
4586         ret = percpu_ref_init(&cgrp->self.refcnt, css_release);
4587         if (ret)
4588                 goto out_free_cgrp;
4589 
4590         /*
4591          * Temporarily set the pointer to NULL, so idr_find() won't return
4592          * a half-baked cgroup.
4593          */
4594         cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT);
4595         if (cgrp->id < 0) {
4596                 ret = -ENOMEM;
4597                 goto out_cancel_ref;
4598         }
4599 
4600         init_cgroup_housekeeping(cgrp);
4601 
4602         cgrp->self.parent = &parent->self;
4603         cgrp->root = root;
4604 
4605         if (notify_on_release(parent))
4606                 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
4607 
4608         if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
4609                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4610 
4611         /* create the directory */
4612         kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
4613         if (IS_ERR(kn)) {
4614                 ret = PTR_ERR(kn);
4615                 goto out_free_id;
4616         }
4617         cgrp->kn = kn;
4618 
4619         /*
4620          * This extra ref will be put in cgroup_free_fn() and guarantees
4621          * that @cgrp->kn is always accessible.
4622          */
4623         kernfs_get(kn);
4624 
4625         cgrp->self.serial_nr = css_serial_nr_next++;
4626 
4627         /* allocation complete, commit to creation */
4628         list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
4629         atomic_inc(&root->nr_cgrps);
4630         cgroup_get(parent);
4631 
4632         /*
4633          * @cgrp is now fully operational.  If something fails after this
4634          * point, it'll be released via the normal destruction path.
4635          */
4636         cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4637 
4638         ret = cgroup_kn_set_ugid(kn);
4639         if (ret)
4640                 goto out_destroy;
4641 
4642         if (cgroup_on_dfl(cgrp))
4643                 base_files = cgroup_dfl_base_files;
4644         else
4645                 base_files = cgroup_legacy_base_files;
4646 
4647         ret = cgroup_addrm_files(cgrp, base_files, true);
4648         if (ret)
4649                 goto out_destroy;
4650 
4651         /* let's create and online css's */
4652         for_each_subsys(ss, ssid) {
4653                 if (parent->child_subsys_mask & (1 << ssid)) {
4654                         ret = create_css(cgrp, ss,
4655                                          parent->subtree_control & (1 << ssid));
4656                         if (ret)
4657                                 goto out_destroy;
4658                 }
4659         }
4660 
4661         /*
4662          * On the default hierarchy, a child doesn't automatically inherit
4663          * subtree_control from the parent.  Each is configured manually.
4664          */
4665         if (!cgroup_on_dfl(cgrp)) {
4666                 cgrp->subtree_control = parent->subtree_control;
4667                 cgroup_refresh_child_subsys_mask(cgrp);
4668         }
4669 
4670         kernfs_activate(kn);
4671 
4672         ret = 0;
4673         goto out_unlock;
4674 
4675 out_free_id:
4676         cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
4677 out_cancel_ref:
4678         percpu_ref_exit(&cgrp->self.refcnt);
4679 out_free_cgrp:
4680         kfree(cgrp);
4681 out_unlock:
4682         cgroup_kn_unlock(parent_kn);
4683         return ret;
4684 
4685 out_destroy:
4686         cgroup_destroy_locked(cgrp);
4687         goto out_unlock;
4688 }
4689 
4690 /*
4691  * This is called when the refcnt of a css is confirmed to be killed.
4692  * css_tryget_online() is now guaranteed to fail.  Tell the subsystem to
4693  * initate destruction and put the css ref from kill_css().
4694  */
4695 static void css_killed_work_fn(struct work_struct *work)
4696 {
4697         struct cgroup_subsys_state *css =
4698                 container_of(work, struct cgroup_subsys_state, destroy_work);
4699 
4700         mutex_lock(&cgroup_mutex);
4701         offline_css(css);
4702         mutex_unlock(&cgroup_mutex);
4703 
4704         css_put(css);
4705 }
4706 
4707 /* css kill confirmation processing requires process context, bounce */
4708 static void css_killed_ref_fn(struct percpu_ref *ref)
4709 {
4710         struct cgroup_subsys_state *css =
4711                 container_of(ref, struct cgroup_subsys_state, refcnt);
4712 
4713         INIT_WORK(&css->destroy_work, css_killed_work_fn);
4714         queue_work(cgroup_destroy_wq, &css->destroy_work);
4715 }
4716 
4717 /**
4718  * kill_css - destroy a css
4719  * @css: css to destroy
4720  *
4721  * This function initiates destruction of @css by removing cgroup interface
4722  * files and putting its base reference.  ->css_offline() will be invoked
4723  * asynchronously once css_tryget_online() is guaranteed to fail and when
4724  * the reference count reaches zero, @css will be released.
4725  */
4726 static void kill_css(struct cgroup_subsys_state *css)
4727 {
4728         lockdep_assert_held(&cgroup_mutex);
4729 
4730         /*
4731          * This must happen before css is disassociated with its cgroup.
4732          * See seq_css() for details.
4733          */
4734         cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4735 
4736         /*
4737          * Killing would put the base ref, but we need to keep it alive
4738          * until after ->css_offline().
4739          */
4740         css_get(css);
4741 
4742         /*
4743          * cgroup core guarantees that, by the time ->css_offline() is
4744          * invoked, no new css reference will be given out via
4745          * css_tryget_online().  We can't simply call percpu_ref_kill() and
4746          * proceed to offlining css's because percpu_ref_kill() doesn't
4747          * guarantee that the ref is seen as killed on all CPUs on return.
4748          *
4749          * Use percpu_ref_kill_and_confirm() to get notifications as each
4750          * css is confirmed to be seen as killed on all CPUs.
4751          */
4752         percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4753 }
4754 
4755 /**
4756  * cgroup_destroy_locked - the first stage of cgroup destruction
4757  * @cgrp: cgroup to be destroyed
4758  *
4759  * css's make use of percpu refcnts whose killing latency shouldn't be
4760  * exposed to userland and are RCU protected.  Also, cgroup core needs to
4761  * guarantee that css_tryget_online() won't succeed by the time
4762  * ->css_offline() is invoked.  To satisfy all the requirements,
4763  * destruction is implemented in the following two steps.
4764  *
4765  * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
4766  *     userland visible parts and start killing the percpu refcnts of
4767  *     css's.  Set up so that the next stage will be kicked off once all
4768  *     the percpu refcnts are confirmed to be killed.
4769  *
4770  * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
4771  *     rest of destruction.  Once all cgroup references are gone, the
4772  *     cgroup is RCU-freed.
4773  *
4774  * This function implements s1.  After this step, @cgrp is gone as far as
4775  * the userland is concerned and a new cgroup with the same name may be
4776  * created.  As cgroup doesn't care about the names internally, this
4777  * doesn't cause any problem.
4778  */
4779 static int cgroup_destroy_locked(struct cgroup *cgrp)
4780         __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4781 {
4782         struct cgroup_subsys_state *css;
4783         bool empty;
4784         int ssid;
4785 
4786         lockdep_assert_held(&cgroup_mutex);
4787 
4788         /*
4789          * css_set_rwsem synchronizes access to ->cset_links and prevents
4790          * @cgrp from being removed while put_css_set() is in progress.
4791          */
4792         down_read(&css_set_rwsem);
4793         empty = list_empty(&cgrp->cset_links);
4794         up_read(&css_set_rwsem);
4795         if (!empty)
4796                 return -EBUSY;
4797 
4798         /*
4799          * Make sure there's no live children.  We can't test emptiness of
4800          * ->self.children as dead children linger on it while being
4801          * drained; otherwise, "rmdir parent/child parent" may fail.
4802          */
4803         if (css_has_online_children(&cgrp->self))
4804                 return -EBUSY;
4805 
4806         /*
4807          * Mark @cgrp dead.  This prevents further task migration and child
4808          * creation by disabling cgroup_lock_live_group().
4809          */
4810         cgrp->self.flags &= ~CSS_ONLINE;
4811 
4812         /* initiate massacre of all css's */
4813         for_each_css(css, ssid, cgrp)
4814                 kill_css(css);
4815 
4816         /* CSS_ONLINE is clear, remove from ->release_list for the last time */
4817         raw_spin_lock(&release_list_lock);
4818         if (!list_empty(&cgrp->release_list))
4819                 list_del_init(&cgrp->release_list);
4820         raw_spin_unlock(&release_list_lock);
4821 
4822         /*
4823          * Remove @cgrp directory along with the base files.  @cgrp has an
4824          * extra ref on its kn.
4825          */
4826         kernfs_remove(cgrp->kn);
4827 
4828         set_bit(CGRP_RELEASABLE, &cgroup_parent(cgrp)->flags);
4829         check_for_release(cgroup_parent(cgrp));
4830 
4831         /* put the base reference */
4832         percpu_ref_kill(&cgrp->self.refcnt);
4833 
4834         return 0;
4835 };
4836 
4837 static int cgroup_rmdir(struct kernfs_node *kn)
4838 {
4839         struct cgroup *cgrp;
4840         int ret = 0;
4841 
4842         cgrp = cgroup_kn_lock_live(kn);
4843         if (!cgrp)
4844                 return 0;
4845         cgroup_get(cgrp);       /* for @kn->priv clearing */
4846 
4847         ret = cgroup_destroy_locked(cgrp);
4848 
4849         cgroup_kn_unlock(kn);
4850 
4851         cgroup_put(cgrp);
4852         return ret;
4853 }
4854 
4855 static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
4856         .remount_fs             = cgroup_remount,
4857         .show_options           = cgroup_show_options,
4858         .mkdir                  = cgroup_mkdir,
4859         .rmdir                  = cgroup_rmdir,
4860         .rename                 = cgroup_rename,
4861 };
4862 
4863 static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
4864 {
4865         struct cgroup_subsys_state *css;
4866 
4867         printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4868 
4869         mutex_lock(&cgroup_mutex);
4870 
4871         idr_init(&ss->css_idr);
4872         INIT_LIST_HEAD(&ss->cfts);
4873 
4874         /* Create the root cgroup state for this subsystem */
4875         ss->root = &cgrp_dfl_root;
4876         css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
4877         /* We don't handle early failures gracefully */
4878         BUG_ON(IS_ERR(css));
4879         init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
4880 
4881         /*
4882          * Root csses are never destroyed and we can't initialize
4883          * percpu_ref during early init.  Disable refcnting.
4884          */
4885         css->flags |= CSS_NO_REF;
4886 
4887         if (early) {
4888                 /* allocation can't be done safely during early init */
4889                 css->id = 1;
4890         } else {
4891                 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
4892                 BUG_ON(css->id < 0);
4893         }
4894 
4895         /* Update the init_css_set to contain a subsys
4896          * pointer to this state - since the subsystem is
4897          * newly registered, all tasks and hence the
4898          * init_css_set is in the subsystem's root cgroup. */
4899         init_css_set.subsys[ss->id] = css;
4900 
4901         need_forkexit_callback |= ss->fork || ss->exit;
4902 
4903         /* At system boot, before all subsystems have been
4904          * registered, no tasks have been forked, so we don't
4905          * need to invoke fork callbacks here. */
4906         BUG_ON(!list_empty(&init_task.tasks));
4907 
4908         BUG_ON(online_css(css));
4909 
4910         mutex_unlock(&cgroup_mutex);
4911 }
4912 
4913 /**
4914  * cgroup_init_early - cgroup initialization at system boot
4915  *
4916  * Initialize cgroups at system boot, and initialize any
4917  * subsystems that request early init.
4918  */
4919 int __init cgroup_init_early(void)
4920 {
4921         static struct cgroup_sb_opts __initdata opts;
4922         struct cgroup_subsys *ss;
4923         int i;
4924 
4925         init_cgroup_root(&cgrp_dfl_root, &opts);
4926         cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;
4927 
4928         RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
4929 
4930         for_each_subsys(ss, i) {
4931                 WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
4932                      "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n",
4933                      i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
4934                      ss->id, ss->name);
4935                 WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
4936                      "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);
4937 
4938                 ss->id = i;
4939                 ss->name = cgroup_subsys_name[i];
4940 
4941                 if (ss->early_init)
4942                         cgroup_init_subsys(ss, true);
4943         }
4944         return 0;
4945 }
4946 
4947 /**
4948  * cgroup_init - cgroup initialization
4949  *
4950  * Register cgroup filesystem and /proc file, and initialize
4951  * any subsystems that didn't request early init.
4952  */
4953 int __init cgroup_init(void)
4954 {
4955         struct cgroup_subsys *ss;
4956         unsigned long key;
4957         int ssid, err;
4958 
4959         BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files));
4960         BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files));
4961 
4962         mutex_lock(&cgroup_mutex);
4963 
4964         /* Add init_css_set to the hash table */
4965         key = css_set_hash(init_css_set.subsys);
4966         hash_add(css_set_table, &init_css_set.hlist, key);
4967 
4968         BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
4969 
4970         mutex_unlock(&cgroup_mutex);
4971 
4972         for_each_subsys(ss, ssid) {
4973                 if (ss->early_init) {
4974                         struct cgroup_subsys_state *css =
4975                                 init_css_set.subsys[ss->id];
4976 
4977                         css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
4978                                                    GFP_KERNEL);
4979                         BUG_ON(css->id < 0);
4980                 } else {
4981                         cgroup_init_subsys(ss, false);
4982                 }
4983 
4984                 list_add_tail(&init_css_set.e_cset_node[ssid],
4985                               &cgrp_dfl_root.cgrp.e_csets[ssid]);
4986 
4987                 /*
4988                  * Setting dfl_root subsys_mask needs to consider the
4989                  * disabled flag and cftype registration needs kmalloc,
4990                  * both of which aren't available during early_init.
4991                  */
4992                 if (ss->disabled)
4993                         continue;
4994 
4995                 cgrp_dfl_root.subsys_mask |= 1 << ss->id;
4996 
4997                 if (cgroup_legacy_files_on_dfl && !ss->dfl_cftypes)
4998                         ss->dfl_cftypes = ss->legacy_cftypes;
4999 
5000                 if (!ss->dfl_cftypes)
5001                         cgrp_dfl_root_inhibit_ss_mask |= 1 << ss->id;
5002 
5003                 if (ss->dfl_cftypes == ss->legacy_cftypes) {
5004                         WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
5005                 } else {
5006                         WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
5007                         WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
5008                 }
5009         }
5010 
5011         cgroup_kobj = kobject_create_and_add("cgroup", fs_kobj);
5012         if (!cgroup_kobj)
5013                 return -ENOMEM;
5014 
5015         err = register_filesystem(&cgroup_fs_type);
5016         if (err < 0) {
5017                 kobject_put(cgroup_kobj);
5018                 return err;
5019         }
5020 
5021         proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
5022         return 0;
5023 }
5024 
5025 static int __init cgroup_wq_init(void)
5026 {
5027         /*
5028          * There isn't much point in executing destruction path in
5029          * parallel.  Good chunk is serialized with cgroup_mutex anyway.
5030          * Use 1 for @max_active.
5031          *
5032          * We would prefer to do this in cgroup_init() above, but that
5033          * is called before init_workqueues(): so leave this until after.
5034          */
5035         cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
5036         BUG_ON(!cgroup_destroy_wq);
5037 
5038         /*
5039          * Used to destroy pidlists and separate to serve as flush domain.
5040          * Cap @max_active to 1 too.
5041          */
5042         cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
5043                                                     0, 1);
5044         BUG_ON(!cgroup_pidlist_destroy_wq);
5045 
5046         return 0;
5047 }
5048 core_initcall(cgroup_wq_init);
5049 
5050 /*
5051  * proc_cgroup_show()
5052  *  - Print task's cgroup paths into seq_file, one line for each hierarchy
5053  *  - Used for /proc/<pid>/cgroup.
5054  */
5055 
5056 /* TODO: Use a proper seq_file iterator */
5057 int proc_cgroup_show(struct seq_file *m, void *v)
5058 {
5059         struct pid *pid;
5060         struct task_struct *tsk;
5061         char *buf, *path;
5062         int retval;
5063         struct cgroup_root *root;
5064 
5065         retval = -ENOMEM;
5066         buf = kmalloc(PATH_MAX, GFP_KERNEL);
5067         if (!buf)
5068                 goto out;
5069 
5070         retval = -ESRCH;
5071         pid = m->private;
5072         tsk = get_pid_task(pid, PIDTYPE_PID);
5073         if (!tsk)
5074                 goto out_free;
5075 
5076         retval = 0;
5077 
5078         mutex_lock(&cgroup_mutex);
5079         down_read(&css_set_rwsem);
5080 
5081         for_each_root(root) {
5082                 struct cgroup_subsys *ss;
5083                 struct cgroup *cgrp;
5084                 int ssid, count = 0;
5085 
5086                 if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible)
5087                         continue;
5088 
5089                 seq_printf(m, "%d:", root->hierarchy_id);
5090                 for_each_subsys(ss, ssid)
5091                         if (root->subsys_mask & (1 << ssid))
5092                                 seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
5093                 if (strlen(root->name))
5094                         seq_printf(m, "%sname=%s", count ? "," : "",
5095                                    root->name);
5096                 seq_putc(m, ':');
5097                 cgrp = task_cgroup_from_root(tsk, root);
5098                 path = cgroup_path(cgrp, buf, PATH_MAX);
5099                 if (!path) {
5100                         retval = -ENAMETOOLONG;
5101                         goto out_unlock;
5102                 }
5103                 seq_puts(m, path);
5104                 seq_putc(m, '\n');
5105         }
5106 
5107 out_unlock:
5108         up_read(&css_set_rwsem);
5109         mutex_unlock(&cgroup_mutex);
5110         put_task_struct(tsk);
5111 out_free:
5112         kfree(buf);
5113 out:
5114         return retval;
5115 }
5116 
5117 /* Display information about each subsystem and each hierarchy */
5118 static int proc_cgroupstats_show(struct seq_file *m, void *v)
5119 {
5120         struct cgroup_subsys *ss;
5121         int i;
5122 
5123         seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
5124         /*
5125          * ideally we don't want subsystems moving around while we do this.
5126          * cgroup_mutex is also necessary to guarantee an atomic snapshot of
5127          * subsys/hierarchy state.
5128          */
5129         mutex_lock(&cgroup_mutex);
5130 
5131         for_each_subsys(ss, i)
5132                 seq_printf(m, "%s\t%d\t%d\t%d\n",
5133                            ss->name, ss->root->hierarchy_id,
5134                            atomic_read(&ss->root->nr_cgrps), !ss->disabled);
5135 
5136         mutex_unlock(&cgroup_mutex);
5137         return 0;
5138 }
5139 
5140 static int cgroupstats_open(struct inode *inode, struct file *file)
5141 {
5142         return single_open(file, proc_cgroupstats_show, NULL);
5143 }
5144 
5145 static const struct file_operations proc_cgroupstats_operations = {
5146         .open = cgroupstats_open,
5147         .read = seq_read,
5148         .llseek = seq_lseek,
5149         .release = single_release,
5150 };
5151 
5152 /**
5153  * cgroup_fork - initialize cgroup related fields during copy_process()
5154  * @child: pointer to task_struct of forking parent process.
5155  *
5156  * A task is associated with the init_css_set until cgroup_post_fork()
5157  * attaches it to the parent's css_set.  Empty cg_list indicates that
5158  * @child isn't holding reference to its css_set.
5159  */
5160 void cgroup_fork(struct task_struct *child)
5161 {
5162         RCU_INIT_POINTER(child->cgroups, &init_css_set);
5163         INIT_LIST_HEAD(&child->cg_list);
5164 }
5165 
5166 /**
5167  * cgroup_post_fork - called on a new task after adding it to the task list
5168  * @child: the task in question
5169  *
5170  * Adds the task to the list running through its css_set if necessary and
5171  * call the subsystem fork() callbacks.  Has to be after the task is
5172  * visible on the task list in case we race with the first call to
5173  * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5174  * list.
5175  */
5176 void cgroup_post_fork(struct task_struct *child)
5177 {
5178         struct cgroup_subsys *ss;
5179         int i;
5180 
5181         /*
5182          * This may race against cgroup_enable_task_cg_links().  As that
5183          * function sets use_task_css_set_links before grabbing
5184          * tasklist_lock and we just went through tasklist_lock to add
5185          * @child, it's guaranteed that either we see the set
5186          * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
5187          * @child during its iteration.
5188          *
5189          * If we won the race, @child is associated with %current's
5190          * css_set.  Grabbing css_set_rwsem guarantees both that the
5191          * association is stable, and, on completion of the parent's
5192          * migration, @child is visible in the source of migration or
5193          * already in the destination cgroup.  This guarantee is necessary
5194          * when implementing operations which need to migrate all tasks of
5195          * a cgroup to another.
5196          *
5197          * Note that if we lose to cgroup_enable_task_cg_links(), @child
5198          * will remain in init_css_set.  This is safe because all tasks are
5199          * in the init_css_set before cg_links is enabled and there's no
5200          * operation which transfers all tasks out of init_css_set.
5201          */
5202         if (use_task_css_set_links) {
5203                 struct css_set *cset;
5204 
5205                 down_write(&css_set_rwsem);
5206                 cset = task_css_set(current);
5207                 if (list_empty(&child->cg_list)) {
5208                         rcu_assign_pointer(child->cgroups, cset);
5209                         list_add(&child->cg_list, &cset->tasks);
5210                         get_css_set(cset);
5211                 }
5212                 up_write(&css_set_rwsem);
5213         }
5214 
5215         /*
5216          * Call ss->fork().  This must happen after @child is linked on
5217          * css_set; otherwise, @child might change state between ->fork()
5218          * and addition to css_set.
5219          */
5220         if (need_forkexit_callback) {
5221                 for_each_subsys(ss, i)
5222                         if (ss->fork)
5223                                 ss->fork(child);
5224         }
5225 }
5226 
5227 /**
5228  * cgroup_exit - detach cgroup from exiting task
5229  * @tsk: pointer to task_struct of exiting process
5230  *
5231  * Description: Detach cgroup from @tsk and release it.
5232  *
5233  * Note that cgroups marked notify_on_release force every task in
5234  * them to take the global cgroup_mutex mutex when exiting.
5235  * This could impact scaling on very large systems.  Be reluctant to
5236  * use notify_on_release cgroups where very high task exit scaling
5237  * is required on large systems.
5238  *
5239  * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
5240  * call cgroup_exit() while the task is still competent to handle
5241  * notify_on_release(), then leave the task attached to the root cgroup in
5242  * each hierarchy for the remainder of its exit.  No need to bother with
5243  * init_css_set refcnting.  init_css_set never goes away and we can't race
5244  * with migration path - PF_EXITING is visible to migration path.
5245  */
5246 void cgroup_exit(struct task_struct *tsk)
5247 {
5248         struct cgroup_subsys *ss;
5249         struct css_set *cset;
5250         bool put_cset = false;
5251         int i;
5252 
5253         /*
5254          * Unlink from @tsk from its css_set.  As migration path can't race
5255          * with us, we can check cg_list without grabbing css_set_rwsem.
5256          */
5257         if (!list_empty(&tsk->cg_list)) {
5258                 down_write(&css_set_rwsem);
5259                 list_del_init(&tsk->cg_list);
5260                 up_write(&css_set_rwsem);
5261                 put_cset = true;
5262         }
5263 
5264         /* Reassign the task to the init_css_set. */
5265         cset = task_css_set(tsk);
5266         RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
5267 
5268         if (need_forkexit_callback) {
5269                 /* see cgroup_post_fork() for details */
5270                 for_each_subsys(ss, i) {
5271                         if (ss->exit) {
5272                                 struct cgroup_subsys_state *old_css = cset->subsys[i];
5273                                 struct cgroup_subsys_state *css = task_css(tsk, i);
5274 
5275                                 ss->exit(css, old_css, tsk);
5276                         }
5277                 }
5278         }
5279 
5280         if (put_cset)
5281                 put_css_set(cset, true);
5282 }
5283 
5284 static void check_for_release(struct cgroup *cgrp)
5285 {
5286         if (cgroup_is_releasable(cgrp) && list_empty(&cgrp->cset_links) &&
5287             !css_has_online_children(&cgrp->self)) {
5288                 /*
5289                  * Control Group is currently removeable. If it's not
5290                  * already queued for a userspace notification, queue
5291                  * it now
5292                  */
5293                 int need_schedule_work = 0;
5294 
5295                 raw_spin_lock(&release_list_lock);
5296                 if (!cgroup_is_dead(cgrp) &&
5297                     list_empty(&cgrp->release_list)) {
5298                         list_add(&cgrp->release_list, &release_list);
5299                         need_schedule_work = 1;
5300                 }
5301                 raw_spin_unlock(&release_list_lock);
5302                 if (need_schedule_work)
5303                         schedule_work(&release_agent_work);
5304         }
5305 }
5306 
5307 /*
5308  * Notify userspace when a cgroup is released, by running the
5309  * configured release agent with the name of the cgroup (path
5310  * relative to the root of cgroup file system) as the argument.
5311  *
5312  * Most likely, this user command will try to rmdir this cgroup.
5313  *
5314  * This races with the possibility that some other task will be
5315  * attached to this cgroup before it is removed, or that some other
5316  * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
5317  * The presumed 'rmdir' will fail quietly if this cgroup is no longer
5318  * unused, and this cgroup will be reprieved from its death sentence,
5319  * to continue to serve a useful existence.  Next time it's released,
5320  * we will get notified again, if it still has 'notify_on_release' set.
5321  *
5322  * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
5323  * means only wait until the task is successfully execve()'d.  The
5324  * separate release agent task is forked by call_usermodehelper(),
5325  * then control in this thread returns here, without waiting for the
5326  * release agent task.  We don't bother to wait because the caller of
5327  * this routine has no use for the exit status of the release agent
5328  * task, so no sense holding our caller up for that.
5329  */
5330 static void cgroup_release_agent(struct work_struct *work)
5331 {
5332         BUG_ON(work != &release_agent_work);
5333         mutex_lock(&cgroup_mutex);
5334         raw_spin_lock(&release_list_lock);
5335         while (!list_empty(&release_list)) {
5336                 char *argv[3], *envp[3];
5337                 int i;
5338                 char *pathbuf = NULL, *agentbuf = NULL, *path;
5339                 struct cgroup *cgrp = list_entry(release_list.next,
5340                                                     struct cgroup,
5341                                                     release_list);
5342                 list_del_init(&cgrp->release_list);
5343                 raw_spin_unlock(&release_list_lock);
5344                 pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
5345                 if (!pathbuf)
5346                         goto continue_free;
5347                 path = cgroup_path(cgrp, pathbuf, PATH_MAX);
5348                 if (!path)
5349                         goto continue_free;
5350                 agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
5351                 if (!agentbuf)
5352                         goto continue_free;
5353 
5354                 i = 0;
5355                 argv[i++] = agentbuf;
5356                 argv[i++] = path;
5357                 argv[i] = NULL;
5358 
5359                 i = 0;
5360                 /* minimal command environment */
5361                 envp[i++] = "HOME=/";
5362                 envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
5363                 envp[i] = NULL;
5364 
5365                 /* Drop the lock while we invoke the usermode helper,
5366                  * since the exec could involve hitting disk and hence
5367                  * be a slow process */
5368                 mutex_unlock(&cgroup_mutex);
5369                 call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
5370                 mutex_lock(&cgroup_mutex);
5371  continue_free:
5372                 kfree(pathbuf);
5373                 kfree(agentbuf);
5374                 raw_spin_lock(&release_list_lock);
5375         }
5376         raw_spin_unlock(&release_list_lock);
5377         mutex_unlock(&cgroup_mutex);
5378 }
5379 
5380 static int __init cgroup_disable(char *str)
5381 {
5382         struct cgroup_subsys *ss;
5383         char *token;
5384         int i;
5385 
5386         while ((token = strsep(&str, ",")) != NULL) {
5387                 if (!*token)
5388                         continue;
5389 
5390                 for_each_subsys(ss, i) {
5391                         if (!strcmp(token, ss->name)) {
5392                                 ss->disabled = 1;
5393                                 printk(KERN_INFO "Disabling %s control group"
5394                                         " subsystem\n", ss->name);
5395                                 break;
5396                         }
5397                 }
5398         }
5399         return 1;
5400 }
5401 __setup("cgroup_disable=", cgroup_disable);
5402 
5403 static int __init cgroup_set_legacy_files_on_dfl(char *str)
5404 {
5405         printk("cgroup: using legacy files on the default hierarchy\n");
5406         cgroup_legacy_files_on_dfl = true;
5407         return 0;
5408 }
5409 __setup("cgroup__DEVEL__legacy_files_on_dfl", cgroup_set_legacy_files_on_dfl);
5410 
5411 /**
5412  * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
5413  * @dentry: directory dentry of interest
5414  * @ss: subsystem of interest
5415  *
5416  * If @dentry is a directory for a cgroup which has @ss enabled on it, try
5417  * to get the corresponding css and return it.  If such css doesn't exist
5418  * or can't be pinned, an ERR_PTR value is returned.
5419  */
5420 struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
5421                                                        struct cgroup_subsys *ss)
5422 {
5423         struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
5424         struct cgroup_subsys_state *css = NULL;
5425         struct cgroup *cgrp;
5426 
5427         /* is @dentry a cgroup dir? */
5428         if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
5429             kernfs_type(kn) != KERNFS_DIR)
5430                 return ERR_PTR(-EBADF);
5431 
5432         rcu_read_lock();
5433 
5434         /*
5435          * This path doesn't originate from kernfs and @kn could already
5436          * have been or be removed at any point.  @kn->priv is RCU
5437          * protected for this access.  See css_release_work_fn() for details.
5438          */
5439         cgrp = rcu_dereference(kn->priv);
5440         if (cgrp)
5441                 css = cgroup_css(cgrp, ss);
5442 
5443         if (!css || !css_tryget_online(css))
5444                 css = ERR_PTR(-ENOENT);
5445 
5446         rcu_read_unlock();
5447         return css;
5448 }
5449 
5450 /**
5451  * css_from_id - lookup css by id
5452  * @id: the cgroup id
5453  * @ss: cgroup subsys to be looked into
5454  *
5455  * Returns the css if there's valid one with @id, otherwise returns NULL.
5456  * Should be called under rcu_read_lock().
5457  */
5458 struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
5459 {
5460         WARN_ON_ONCE(!rcu_read_lock_held());
5461         return idr_find(&ss->css_idr, id);
5462 }
5463 
5464 #ifdef CONFIG_CGROUP_DEBUG
5465 static struct cgroup_subsys_state *
5466 debug_css_alloc(struct cgroup_subsys_state *parent_css)
5467 {
5468         struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
5469 
5470         if (!css)
5471                 return ERR_PTR(-ENOMEM);
5472 
5473         return css;
5474 }
5475 
5476 static void debug_css_free(struct cgroup_subsys_state *css)
5477 {
5478         kfree(css);
5479 }
5480 
5481 static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
5482                                 struct cftype *cft)
5483 {
5484         return cgroup_task_count(css->cgroup);
5485 }
5486 
5487 static u64 current_css_set_read(struct cgroup_subsys_state *css,
5488                                 struct cftype *cft)
5489 {
5490         return (u64)(unsigned long)current->cgroups;
5491 }
5492 
5493 static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
5494                                          struct cftype *cft)
5495 {
5496         u64 count;
5497 
5498         rcu_read_lock();
5499         count = atomic_read(&task_css_set(current)->refcount);
5500         rcu_read_unlock();
5501         return count;
5502 }
5503 
5504 static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
5505 {
5506         struct cgrp_cset_link *link;
5507         struct css_set *cset;
5508         char *name_buf;
5509 
5510         name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
5511         if (!name_buf)
5512                 return -ENOMEM;
5513 
5514         down_read(&css_set_rwsem);
5515         rcu_read_lock();
5516         cset = rcu_dereference(current->cgroups);
5517         list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
5518                 struct cgroup *c = link->cgrp;
5519 
5520                 cgroup_name(c, name_buf, NAME_MAX + 1);
5521                 seq_printf(seq, "Root %d group %s\n",
5522                            c->root->hierarchy_id, name_buf);
5523         }
5524         rcu_read_unlock();
5525         up_read(&css_set_rwsem);
5526         kfree(name_buf);
5527         return 0;
5528 }
5529 
5530 #define MAX_TASKS_SHOWN_PER_CSS 25
5531 static int cgroup_css_links_read(struct seq_file *seq, void *v)
5532 {
5533         struct cgroup_subsys_state *css = seq_css(seq);
5534         struct cgrp_cset_link *link;
5535 
5536         down_read(&css_set_rwsem);
5537         list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
5538                 struct css_set *cset = link->cset;
5539                 struct task_struct *task;
5540                 int count = 0;
5541 
5542                 seq_printf(seq, "css_set %p\n", cset);
5543 
5544                 list_for_each_entry(task, &cset->tasks, cg_list) {
5545                         if (count++ > MAX_TASKS_SHOWN_PER_CSS)
5546                                 goto overflow;
5547                         seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5548                 }
5549 
5550                 list_for_each_entry(task, &cset->mg_tasks, cg_list) {
5551                         if (count++ > MAX_TASKS_SHOWN_PER_CSS)
5552                                 goto overflow;
5553                         seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5554                 }
5555                 continue;
5556         overflow:
5557                 seq_puts(seq, "  ...\n");
5558         }
5559         up_read(&css_set_rwsem);
5560         return 0;
5561 }
5562 
5563 static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
5564 {
5565         return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
5566 }
5567 
5568 static struct cftype debug_files[] =  {
5569         {
5570                 .name = "taskcount",
5571                 .read_u64 = debug_taskcount_read,
5572         },
5573 
5574         {
5575                 .name = "current_css_set",
5576                 .read_u64 = current_css_set_read,
5577         },
5578 
5579         {
5580                 .name = "current_css_set_refcount",
5581                 .read_u64 = current_css_set_refcount_read,
5582         },
5583 
5584         {
5585                 .name = "current_css_set_cg_links",
5586                 .seq_show = current_css_set_cg_links_read,
5587         },
5588 
5589         {
5590                 .name = "cgroup_css_links",
5591                 .seq_show = cgroup_css_links_read,
5592         },
5593 
5594         {
5595                 .name = "releasable",
5596                 .read_u64 = releasable_read,
5597         },
5598 
5599         { }     /* terminate */
5600 };
5601 
5602 struct cgroup_subsys debug_cgrp_subsys = {
5603         .css_alloc = debug_css_alloc,
5604         .css_free = debug_css_free,
5605         .legacy_cftypes = debug_files,
5606 };
5607 #endif /* CONFIG_CGROUP_DEBUG */
5608 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us