Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/kernel/cgroup.c

  1 /*
  2  *  Generic process-grouping system.
  3  *
  4  *  Based originally on the cpuset system, extracted by Paul Menage
  5  *  Copyright (C) 2006 Google, Inc
  6  *
  7  *  Notifications support
  8  *  Copyright (C) 2009 Nokia Corporation
  9  *  Author: Kirill A. Shutemov
 10  *
 11  *  Copyright notices from the original cpuset code:
 12  *  --------------------------------------------------
 13  *  Copyright (C) 2003 BULL SA.
 14  *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 15  *
 16  *  Portions derived from Patrick Mochel's sysfs code.
 17  *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 18  *
 19  *  2003-10-10 Written by Simon Derr.
 20  *  2003-10-22 Updates by Stephen Hemminger.
 21  *  2004 May-July Rework by Paul Jackson.
 22  *  ---------------------------------------------------
 23  *
 24  *  This file is subject to the terms and conditions of the GNU General Public
 25  *  License.  See the file COPYING in the main directory of the Linux
 26  *  distribution for more details.
 27  */
 28 
 29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 30 
 31 #include <linux/cgroup.h>
 32 #include <linux/cred.h>
 33 #include <linux/ctype.h>
 34 #include <linux/errno.h>
 35 #include <linux/init_task.h>
 36 #include <linux/kernel.h>
 37 #include <linux/list.h>
 38 #include <linux/magic.h>
 39 #include <linux/mm.h>
 40 #include <linux/mutex.h>
 41 #include <linux/mount.h>
 42 #include <linux/pagemap.h>
 43 #include <linux/proc_fs.h>
 44 #include <linux/rcupdate.h>
 45 #include <linux/sched.h>
 46 #include <linux/slab.h>
 47 #include <linux/spinlock.h>
 48 #include <linux/rwsem.h>
 49 #include <linux/string.h>
 50 #include <linux/sort.h>
 51 #include <linux/kmod.h>
 52 #include <linux/delayacct.h>
 53 #include <linux/cgroupstats.h>
 54 #include <linux/hashtable.h>
 55 #include <linux/pid_namespace.h>
 56 #include <linux/idr.h>
 57 #include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
 58 #include <linux/kthread.h>
 59 #include <linux/delay.h>
 60 
 61 #include <linux/atomic.h>
 62 
 63 /*
 64  * pidlists linger the following amount before being destroyed.  The goal
 65  * is avoiding frequent destruction in the middle of consecutive read calls
 66  * Expiring in the middle is a performance problem not a correctness one.
 67  * 1 sec should be enough.
 68  */
 69 #define CGROUP_PIDLIST_DESTROY_DELAY    HZ
 70 
 71 #define CGROUP_FILE_NAME_MAX            (MAX_CGROUP_TYPE_NAMELEN +      \
 72                                          MAX_CFTYPE_NAME + 2)
 73 
 74 /*
 75  * cgroup_mutex is the master lock.  Any modification to cgroup or its
 76  * hierarchy must be performed while holding it.
 77  *
 78  * css_set_rwsem protects task->cgroups pointer, the list of css_set
 79  * objects, and the chain of tasks off each css_set.
 80  *
 81  * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 82  * cgroup.h can use them for lockdep annotations.
 83  */
 84 #ifdef CONFIG_PROVE_RCU
 85 DEFINE_MUTEX(cgroup_mutex);
 86 DECLARE_RWSEM(css_set_rwsem);
 87 EXPORT_SYMBOL_GPL(cgroup_mutex);
 88 EXPORT_SYMBOL_GPL(css_set_rwsem);
 89 #else
 90 static DEFINE_MUTEX(cgroup_mutex);
 91 static DECLARE_RWSEM(css_set_rwsem);
 92 #endif
 93 
 94 /*
 95  * Protects cgroup_idr and css_idr so that IDs can be released without
 96  * grabbing cgroup_mutex.
 97  */
 98 static DEFINE_SPINLOCK(cgroup_idr_lock);
 99 
100 /*
101  * Protects cgroup_subsys->release_agent_path.  Modifying it also requires
102  * cgroup_mutex.  Reading requires either cgroup_mutex or this spinlock.
103  */
104 static DEFINE_SPINLOCK(release_agent_path_lock);
105 
106 #define cgroup_assert_mutex_or_rcu_locked()                             \
107         rcu_lockdep_assert(rcu_read_lock_held() ||                      \
108                            lockdep_is_held(&cgroup_mutex),              \
109                            "cgroup_mutex or RCU read lock required");
110 
111 /*
112  * cgroup destruction makes heavy use of work items and there can be a lot
113  * of concurrent destructions.  Use a separate workqueue so that cgroup
114  * destruction work items don't end up filling up max_active of system_wq
115  * which may lead to deadlock.
116  */
117 static struct workqueue_struct *cgroup_destroy_wq;
118 
119 /*
120  * pidlist destructions need to be flushed on cgroup destruction.  Use a
121  * separate workqueue as flush domain.
122  */
123 static struct workqueue_struct *cgroup_pidlist_destroy_wq;
124 
125 /* generate an array of cgroup subsystem pointers */
126 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
127 static struct cgroup_subsys *cgroup_subsys[] = {
128 #include <linux/cgroup_subsys.h>
129 };
130 #undef SUBSYS
131 
132 /* array of cgroup subsystem names */
133 #define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
134 static const char *cgroup_subsys_name[] = {
135 #include <linux/cgroup_subsys.h>
136 };
137 #undef SUBSYS
138 
139 /*
140  * The default hierarchy, reserved for the subsystems that are otherwise
141  * unattached - it never has more than a single cgroup, and all tasks are
142  * part of that cgroup.
143  */
144 struct cgroup_root cgrp_dfl_root;
145 
146 /*
147  * The default hierarchy always exists but is hidden until mounted for the
148  * first time.  This is for backward compatibility.
149  */
150 static bool cgrp_dfl_root_visible;
151 
152 /* some controllers are not supported in the default hierarchy */
153 static const unsigned int cgrp_dfl_root_inhibit_ss_mask = 0
154 #ifdef CONFIG_CGROUP_DEBUG
155         | (1 << debug_cgrp_id)
156 #endif
157         ;
158 
159 /* The list of hierarchy roots */
160 
161 static LIST_HEAD(cgroup_roots);
162 static int cgroup_root_count;
163 
164 /* hierarchy ID allocation and mapping, protected by cgroup_mutex */
165 static DEFINE_IDR(cgroup_hierarchy_idr);
166 
167 /*
168  * Assign a monotonically increasing serial number to csses.  It guarantees
169  * cgroups with bigger numbers are newer than those with smaller numbers.
170  * Also, as csses are always appended to the parent's ->children list, it
171  * guarantees that sibling csses are always sorted in the ascending serial
172  * number order on the list.  Protected by cgroup_mutex.
173  */
174 static u64 css_serial_nr_next = 1;
175 
176 /* This flag indicates whether tasks in the fork and exit paths should
177  * check for fork/exit handlers to call. This avoids us having to do
178  * extra work in the fork/exit path if none of the subsystems need to
179  * be called.
180  */
181 static int need_forkexit_callback __read_mostly;
182 
183 static struct cftype cgroup_base_files[];
184 
185 static void cgroup_put(struct cgroup *cgrp);
186 static int rebind_subsystems(struct cgroup_root *dst_root,
187                              unsigned int ss_mask);
188 static int cgroup_destroy_locked(struct cgroup *cgrp);
189 static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss);
190 static void css_release(struct percpu_ref *ref);
191 static void kill_css(struct cgroup_subsys_state *css);
192 static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
193                               bool is_add);
194 static void cgroup_pidlist_destroy_all(struct cgroup *cgrp);
195 
196 /* IDR wrappers which synchronize using cgroup_idr_lock */
197 static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
198                             gfp_t gfp_mask)
199 {
200         int ret;
201 
202         idr_preload(gfp_mask);
203         spin_lock_bh(&cgroup_idr_lock);
204         ret = idr_alloc(idr, ptr, start, end, gfp_mask);
205         spin_unlock_bh(&cgroup_idr_lock);
206         idr_preload_end();
207         return ret;
208 }
209 
210 static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
211 {
212         void *ret;
213 
214         spin_lock_bh(&cgroup_idr_lock);
215         ret = idr_replace(idr, ptr, id);
216         spin_unlock_bh(&cgroup_idr_lock);
217         return ret;
218 }
219 
220 static void cgroup_idr_remove(struct idr *idr, int id)
221 {
222         spin_lock_bh(&cgroup_idr_lock);
223         idr_remove(idr, id);
224         spin_unlock_bh(&cgroup_idr_lock);
225 }
226 
227 static struct cgroup *cgroup_parent(struct cgroup *cgrp)
228 {
229         struct cgroup_subsys_state *parent_css = cgrp->self.parent;
230 
231         if (parent_css)
232                 return container_of(parent_css, struct cgroup, self);
233         return NULL;
234 }
235 
236 /**
237  * cgroup_css - obtain a cgroup's css for the specified subsystem
238  * @cgrp: the cgroup of interest
239  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
240  *
241  * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
242  * function must be called either under cgroup_mutex or rcu_read_lock() and
243  * the caller is responsible for pinning the returned css if it wants to
244  * keep accessing it outside the said locks.  This function may return
245  * %NULL if @cgrp doesn't have @subsys_id enabled.
246  */
247 static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
248                                               struct cgroup_subsys *ss)
249 {
250         if (ss)
251                 return rcu_dereference_check(cgrp->subsys[ss->id],
252                                         lockdep_is_held(&cgroup_mutex));
253         else
254                 return &cgrp->self;
255 }
256 
257 /**
258  * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
259  * @cgrp: the cgroup of interest
260  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
261  *
262  * Similar to cgroup_css() but returns the effctive css, which is defined
263  * as the matching css of the nearest ancestor including self which has @ss
264  * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
265  * function is guaranteed to return non-NULL css.
266  */
267 static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
268                                                 struct cgroup_subsys *ss)
269 {
270         lockdep_assert_held(&cgroup_mutex);
271 
272         if (!ss)
273                 return &cgrp->self;
274 
275         if (!(cgrp->root->subsys_mask & (1 << ss->id)))
276                 return NULL;
277 
278         while (cgroup_parent(cgrp) &&
279                !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id)))
280                 cgrp = cgroup_parent(cgrp);
281 
282         return cgroup_css(cgrp, ss);
283 }
284 
285 /* convenient tests for these bits */
286 static inline bool cgroup_is_dead(const struct cgroup *cgrp)
287 {
288         return !(cgrp->self.flags & CSS_ONLINE);
289 }
290 
291 struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
292 {
293         struct cgroup *cgrp = of->kn->parent->priv;
294         struct cftype *cft = of_cft(of);
295 
296         /*
297          * This is open and unprotected implementation of cgroup_css().
298          * seq_css() is only called from a kernfs file operation which has
299          * an active reference on the file.  Because all the subsystem
300          * files are drained before a css is disassociated with a cgroup,
301          * the matching css from the cgroup's subsys table is guaranteed to
302          * be and stay valid until the enclosing operation is complete.
303          */
304         if (cft->ss)
305                 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
306         else
307                 return &cgrp->self;
308 }
309 EXPORT_SYMBOL_GPL(of_css);
310 
311 /**
312  * cgroup_is_descendant - test ancestry
313  * @cgrp: the cgroup to be tested
314  * @ancestor: possible ancestor of @cgrp
315  *
316  * Test whether @cgrp is a descendant of @ancestor.  It also returns %true
317  * if @cgrp == @ancestor.  This function is safe to call as long as @cgrp
318  * and @ancestor are accessible.
319  */
320 bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
321 {
322         while (cgrp) {
323                 if (cgrp == ancestor)
324                         return true;
325                 cgrp = cgroup_parent(cgrp);
326         }
327         return false;
328 }
329 
330 static int cgroup_is_releasable(const struct cgroup *cgrp)
331 {
332         const int bits =
333                 (1 << CGRP_RELEASABLE) |
334                 (1 << CGRP_NOTIFY_ON_RELEASE);
335         return (cgrp->flags & bits) == bits;
336 }
337 
338 static int notify_on_release(const struct cgroup *cgrp)
339 {
340         return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
341 }
342 
343 /**
344  * for_each_css - iterate all css's of a cgroup
345  * @css: the iteration cursor
346  * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
347  * @cgrp: the target cgroup to iterate css's of
348  *
349  * Should be called under cgroup_[tree_]mutex.
350  */
351 #define for_each_css(css, ssid, cgrp)                                   \
352         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)        \
353                 if (!((css) = rcu_dereference_check(                    \
354                                 (cgrp)->subsys[(ssid)],                 \
355                                 lockdep_is_held(&cgroup_mutex)))) { }   \
356                 else
357 
358 /**
359  * for_each_e_css - iterate all effective css's of a cgroup
360  * @css: the iteration cursor
361  * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
362  * @cgrp: the target cgroup to iterate css's of
363  *
364  * Should be called under cgroup_[tree_]mutex.
365  */
366 #define for_each_e_css(css, ssid, cgrp)                                 \
367         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)        \
368                 if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
369                         ;                                               \
370                 else
371 
372 /**
373  * for_each_subsys - iterate all enabled cgroup subsystems
374  * @ss: the iteration cursor
375  * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
376  */
377 #define for_each_subsys(ss, ssid)                                       \
378         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT &&                \
379              (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
380 
381 /* iterate across the hierarchies */
382 #define for_each_root(root)                                             \
383         list_for_each_entry((root), &cgroup_roots, root_list)
384 
385 /* iterate over child cgrps, lock should be held throughout iteration */
386 #define cgroup_for_each_live_child(child, cgrp)                         \
387         list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
388                 if (({ lockdep_assert_held(&cgroup_mutex);              \
389                        cgroup_is_dead(child); }))                       \
390                         ;                                               \
391                 else
392 
393 /* the list of cgroups eligible for automatic release. Protected by
394  * release_list_lock */
395 static LIST_HEAD(release_list);
396 static DEFINE_RAW_SPINLOCK(release_list_lock);
397 static void cgroup_release_agent(struct work_struct *work);
398 static DECLARE_WORK(release_agent_work, cgroup_release_agent);
399 static void check_for_release(struct cgroup *cgrp);
400 
401 /*
402  * A cgroup can be associated with multiple css_sets as different tasks may
403  * belong to different cgroups on different hierarchies.  In the other
404  * direction, a css_set is naturally associated with multiple cgroups.
405  * This M:N relationship is represented by the following link structure
406  * which exists for each association and allows traversing the associations
407  * from both sides.
408  */
409 struct cgrp_cset_link {
410         /* the cgroup and css_set this link associates */
411         struct cgroup           *cgrp;
412         struct css_set          *cset;
413 
414         /* list of cgrp_cset_links anchored at cgrp->cset_links */
415         struct list_head        cset_link;
416 
417         /* list of cgrp_cset_links anchored at css_set->cgrp_links */
418         struct list_head        cgrp_link;
419 };
420 
421 /*
422  * The default css_set - used by init and its children prior to any
423  * hierarchies being mounted. It contains a pointer to the root state
424  * for each subsystem. Also used to anchor the list of css_sets. Not
425  * reference-counted, to improve performance when child cgroups
426  * haven't been created.
427  */
428 struct css_set init_css_set = {
429         .refcount               = ATOMIC_INIT(1),
430         .cgrp_links             = LIST_HEAD_INIT(init_css_set.cgrp_links),
431         .tasks                  = LIST_HEAD_INIT(init_css_set.tasks),
432         .mg_tasks               = LIST_HEAD_INIT(init_css_set.mg_tasks),
433         .mg_preload_node        = LIST_HEAD_INIT(init_css_set.mg_preload_node),
434         .mg_node                = LIST_HEAD_INIT(init_css_set.mg_node),
435 };
436 
437 static int css_set_count        = 1;    /* 1 for init_css_set */
438 
439 /**
440  * cgroup_update_populated - updated populated count of a cgroup
441  * @cgrp: the target cgroup
442  * @populated: inc or dec populated count
443  *
444  * @cgrp is either getting the first task (css_set) or losing the last.
445  * Update @cgrp->populated_cnt accordingly.  The count is propagated
446  * towards root so that a given cgroup's populated_cnt is zero iff the
447  * cgroup and all its descendants are empty.
448  *
449  * @cgrp's interface file "cgroup.populated" is zero if
450  * @cgrp->populated_cnt is zero and 1 otherwise.  When @cgrp->populated_cnt
451  * changes from or to zero, userland is notified that the content of the
452  * interface file has changed.  This can be used to detect when @cgrp and
453  * its descendants become populated or empty.
454  */
455 static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
456 {
457         lockdep_assert_held(&css_set_rwsem);
458 
459         do {
460                 bool trigger;
461 
462                 if (populated)
463                         trigger = !cgrp->populated_cnt++;
464                 else
465                         trigger = !--cgrp->populated_cnt;
466 
467                 if (!trigger)
468                         break;
469 
470                 if (cgrp->populated_kn)
471                         kernfs_notify(cgrp->populated_kn);
472                 cgrp = cgroup_parent(cgrp);
473         } while (cgrp);
474 }
475 
476 /*
477  * hash table for cgroup groups. This improves the performance to find
478  * an existing css_set. This hash doesn't (currently) take into
479  * account cgroups in empty hierarchies.
480  */
481 #define CSS_SET_HASH_BITS       7
482 static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
483 
484 static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
485 {
486         unsigned long key = 0UL;
487         struct cgroup_subsys *ss;
488         int i;
489 
490         for_each_subsys(ss, i)
491                 key += (unsigned long)css[i];
492         key = (key >> 16) ^ key;
493 
494         return key;
495 }
496 
497 static void put_css_set_locked(struct css_set *cset, bool taskexit)
498 {
499         struct cgrp_cset_link *link, *tmp_link;
500         struct cgroup_subsys *ss;
501         int ssid;
502 
503         lockdep_assert_held(&css_set_rwsem);
504 
505         if (!atomic_dec_and_test(&cset->refcount))
506                 return;
507 
508         /* This css_set is dead. unlink it and release cgroup refcounts */
509         for_each_subsys(ss, ssid)
510                 list_del(&cset->e_cset_node[ssid]);
511         hash_del(&cset->hlist);
512         css_set_count--;
513 
514         list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
515                 struct cgroup *cgrp = link->cgrp;
516 
517                 list_del(&link->cset_link);
518                 list_del(&link->cgrp_link);
519 
520                 /* @cgrp can't go away while we're holding css_set_rwsem */
521                 if (list_empty(&cgrp->cset_links)) {
522                         cgroup_update_populated(cgrp, false);
523                         if (notify_on_release(cgrp)) {
524                                 if (taskexit)
525                                         set_bit(CGRP_RELEASABLE, &cgrp->flags);
526                                 check_for_release(cgrp);
527                         }
528                 }
529 
530                 kfree(link);
531         }
532 
533         kfree_rcu(cset, rcu_head);
534 }
535 
536 static void put_css_set(struct css_set *cset, bool taskexit)
537 {
538         /*
539          * Ensure that the refcount doesn't hit zero while any readers
540          * can see it. Similar to atomic_dec_and_lock(), but for an
541          * rwlock
542          */
543         if (atomic_add_unless(&cset->refcount, -1, 1))
544                 return;
545 
546         down_write(&css_set_rwsem);
547         put_css_set_locked(cset, taskexit);
548         up_write(&css_set_rwsem);
549 }
550 
551 /*
552  * refcounted get/put for css_set objects
553  */
554 static inline void get_css_set(struct css_set *cset)
555 {
556         atomic_inc(&cset->refcount);
557 }
558 
559 /**
560  * compare_css_sets - helper function for find_existing_css_set().
561  * @cset: candidate css_set being tested
562  * @old_cset: existing css_set for a task
563  * @new_cgrp: cgroup that's being entered by the task
564  * @template: desired set of css pointers in css_set (pre-calculated)
565  *
566  * Returns true if "cset" matches "old_cset" except for the hierarchy
567  * which "new_cgrp" belongs to, for which it should match "new_cgrp".
568  */
569 static bool compare_css_sets(struct css_set *cset,
570                              struct css_set *old_cset,
571                              struct cgroup *new_cgrp,
572                              struct cgroup_subsys_state *template[])
573 {
574         struct list_head *l1, *l2;
575 
576         /*
577          * On the default hierarchy, there can be csets which are
578          * associated with the same set of cgroups but different csses.
579          * Let's first ensure that csses match.
580          */
581         if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
582                 return false;
583 
584         /*
585          * Compare cgroup pointers in order to distinguish between
586          * different cgroups in hierarchies.  As different cgroups may
587          * share the same effective css, this comparison is always
588          * necessary.
589          */
590         l1 = &cset->cgrp_links;
591         l2 = &old_cset->cgrp_links;
592         while (1) {
593                 struct cgrp_cset_link *link1, *link2;
594                 struct cgroup *cgrp1, *cgrp2;
595 
596                 l1 = l1->next;
597                 l2 = l2->next;
598                 /* See if we reached the end - both lists are equal length. */
599                 if (l1 == &cset->cgrp_links) {
600                         BUG_ON(l2 != &old_cset->cgrp_links);
601                         break;
602                 } else {
603                         BUG_ON(l2 == &old_cset->cgrp_links);
604                 }
605                 /* Locate the cgroups associated with these links. */
606                 link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
607                 link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
608                 cgrp1 = link1->cgrp;
609                 cgrp2 = link2->cgrp;
610                 /* Hierarchies should be linked in the same order. */
611                 BUG_ON(cgrp1->root != cgrp2->root);
612 
613                 /*
614                  * If this hierarchy is the hierarchy of the cgroup
615                  * that's changing, then we need to check that this
616                  * css_set points to the new cgroup; if it's any other
617                  * hierarchy, then this css_set should point to the
618                  * same cgroup as the old css_set.
619                  */
620                 if (cgrp1->root == new_cgrp->root) {
621                         if (cgrp1 != new_cgrp)
622                                 return false;
623                 } else {
624                         if (cgrp1 != cgrp2)
625                                 return false;
626                 }
627         }
628         return true;
629 }
630 
631 /**
632  * find_existing_css_set - init css array and find the matching css_set
633  * @old_cset: the css_set that we're using before the cgroup transition
634  * @cgrp: the cgroup that we're moving into
635  * @template: out param for the new set of csses, should be clear on entry
636  */
637 static struct css_set *find_existing_css_set(struct css_set *old_cset,
638                                         struct cgroup *cgrp,
639                                         struct cgroup_subsys_state *template[])
640 {
641         struct cgroup_root *root = cgrp->root;
642         struct cgroup_subsys *ss;
643         struct css_set *cset;
644         unsigned long key;
645         int i;
646 
647         /*
648          * Build the set of subsystem state objects that we want to see in the
649          * new css_set. while subsystems can change globally, the entries here
650          * won't change, so no need for locking.
651          */
652         for_each_subsys(ss, i) {
653                 if (root->subsys_mask & (1UL << i)) {
654                         /*
655                          * @ss is in this hierarchy, so we want the
656                          * effective css from @cgrp.
657                          */
658                         template[i] = cgroup_e_css(cgrp, ss);
659                 } else {
660                         /*
661                          * @ss is not in this hierarchy, so we don't want
662                          * to change the css.
663                          */
664                         template[i] = old_cset->subsys[i];
665                 }
666         }
667 
668         key = css_set_hash(template);
669         hash_for_each_possible(css_set_table, cset, hlist, key) {
670                 if (!compare_css_sets(cset, old_cset, cgrp, template))
671                         continue;
672 
673                 /* This css_set matches what we need */
674                 return cset;
675         }
676 
677         /* No existing cgroup group matched */
678         return NULL;
679 }
680 
681 static void free_cgrp_cset_links(struct list_head *links_to_free)
682 {
683         struct cgrp_cset_link *link, *tmp_link;
684 
685         list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
686                 list_del(&link->cset_link);
687                 kfree(link);
688         }
689 }
690 
691 /**
692  * allocate_cgrp_cset_links - allocate cgrp_cset_links
693  * @count: the number of links to allocate
694  * @tmp_links: list_head the allocated links are put on
695  *
696  * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
697  * through ->cset_link.  Returns 0 on success or -errno.
698  */
699 static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
700 {
701         struct cgrp_cset_link *link;
702         int i;
703 
704         INIT_LIST_HEAD(tmp_links);
705 
706         for (i = 0; i < count; i++) {
707                 link = kzalloc(sizeof(*link), GFP_KERNEL);
708                 if (!link) {
709                         free_cgrp_cset_links(tmp_links);
710                         return -ENOMEM;
711                 }
712                 list_add(&link->cset_link, tmp_links);
713         }
714         return 0;
715 }
716 
717 /**
718  * link_css_set - a helper function to link a css_set to a cgroup
719  * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
720  * @cset: the css_set to be linked
721  * @cgrp: the destination cgroup
722  */
723 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
724                          struct cgroup *cgrp)
725 {
726         struct cgrp_cset_link *link;
727 
728         BUG_ON(list_empty(tmp_links));
729 
730         if (cgroup_on_dfl(cgrp))
731                 cset->dfl_cgrp = cgrp;
732 
733         link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
734         link->cset = cset;
735         link->cgrp = cgrp;
736 
737         if (list_empty(&cgrp->cset_links))
738                 cgroup_update_populated(cgrp, true);
739         list_move(&link->cset_link, &cgrp->cset_links);
740 
741         /*
742          * Always add links to the tail of the list so that the list
743          * is sorted by order of hierarchy creation
744          */
745         list_add_tail(&link->cgrp_link, &cset->cgrp_links);
746 }
747 
748 /**
749  * find_css_set - return a new css_set with one cgroup updated
750  * @old_cset: the baseline css_set
751  * @cgrp: the cgroup to be updated
752  *
753  * Return a new css_set that's equivalent to @old_cset, but with @cgrp
754  * substituted into the appropriate hierarchy.
755  */
756 static struct css_set *find_css_set(struct css_set *old_cset,
757                                     struct cgroup *cgrp)
758 {
759         struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
760         struct css_set *cset;
761         struct list_head tmp_links;
762         struct cgrp_cset_link *link;
763         struct cgroup_subsys *ss;
764         unsigned long key;
765         int ssid;
766 
767         lockdep_assert_held(&cgroup_mutex);
768 
769         /* First see if we already have a cgroup group that matches
770          * the desired set */
771         down_read(&css_set_rwsem);
772         cset = find_existing_css_set(old_cset, cgrp, template);
773         if (cset)
774                 get_css_set(cset);
775         up_read(&css_set_rwsem);
776 
777         if (cset)
778                 return cset;
779 
780         cset = kzalloc(sizeof(*cset), GFP_KERNEL);
781         if (!cset)
782                 return NULL;
783 
784         /* Allocate all the cgrp_cset_link objects that we'll need */
785         if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
786                 kfree(cset);
787                 return NULL;
788         }
789 
790         atomic_set(&cset->refcount, 1);
791         INIT_LIST_HEAD(&cset->cgrp_links);
792         INIT_LIST_HEAD(&cset->tasks);
793         INIT_LIST_HEAD(&cset->mg_tasks);
794         INIT_LIST_HEAD(&cset->mg_preload_node);
795         INIT_LIST_HEAD(&cset->mg_node);
796         INIT_HLIST_NODE(&cset->hlist);
797 
798         /* Copy the set of subsystem state objects generated in
799          * find_existing_css_set() */
800         memcpy(cset->subsys, template, sizeof(cset->subsys));
801 
802         down_write(&css_set_rwsem);
803         /* Add reference counts and links from the new css_set. */
804         list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
805                 struct cgroup *c = link->cgrp;
806 
807                 if (c->root == cgrp->root)
808                         c = cgrp;
809                 link_css_set(&tmp_links, cset, c);
810         }
811 
812         BUG_ON(!list_empty(&tmp_links));
813 
814         css_set_count++;
815 
816         /* Add @cset to the hash table */
817         key = css_set_hash(cset->subsys);
818         hash_add(css_set_table, &cset->hlist, key);
819 
820         for_each_subsys(ss, ssid)
821                 list_add_tail(&cset->e_cset_node[ssid],
822                               &cset->subsys[ssid]->cgroup->e_csets[ssid]);
823 
824         up_write(&css_set_rwsem);
825 
826         return cset;
827 }
828 
829 static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
830 {
831         struct cgroup *root_cgrp = kf_root->kn->priv;
832 
833         return root_cgrp->root;
834 }
835 
836 static int cgroup_init_root_id(struct cgroup_root *root)
837 {
838         int id;
839 
840         lockdep_assert_held(&cgroup_mutex);
841 
842         id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
843         if (id < 0)
844                 return id;
845 
846         root->hierarchy_id = id;
847         return 0;
848 }
849 
850 static void cgroup_exit_root_id(struct cgroup_root *root)
851 {
852         lockdep_assert_held(&cgroup_mutex);
853 
854         if (root->hierarchy_id) {
855                 idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
856                 root->hierarchy_id = 0;
857         }
858 }
859 
860 static void cgroup_free_root(struct cgroup_root *root)
861 {
862         if (root) {
863                 /* hierarhcy ID shoulid already have been released */
864                 WARN_ON_ONCE(root->hierarchy_id);
865 
866                 idr_destroy(&root->cgroup_idr);
867                 kfree(root);
868         }
869 }
870 
871 static void cgroup_destroy_root(struct cgroup_root *root)
872 {
873         struct cgroup *cgrp = &root->cgrp;
874         struct cgrp_cset_link *link, *tmp_link;
875 
876         mutex_lock(&cgroup_mutex);
877 
878         BUG_ON(atomic_read(&root->nr_cgrps));
879         BUG_ON(!list_empty(&cgrp->self.children));
880 
881         /* Rebind all subsystems back to the default hierarchy */
882         rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
883 
884         /*
885          * Release all the links from cset_links to this hierarchy's
886          * root cgroup
887          */
888         down_write(&css_set_rwsem);
889 
890         list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
891                 list_del(&link->cset_link);
892                 list_del(&link->cgrp_link);
893                 kfree(link);
894         }
895         up_write(&css_set_rwsem);
896 
897         if (!list_empty(&root->root_list)) {
898                 list_del(&root->root_list);
899                 cgroup_root_count--;
900         }
901 
902         cgroup_exit_root_id(root);
903 
904         mutex_unlock(&cgroup_mutex);
905 
906         kernfs_destroy_root(root->kf_root);
907         cgroup_free_root(root);
908 }
909 
910 /* look up cgroup associated with given css_set on the specified hierarchy */
911 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
912                                             struct cgroup_root *root)
913 {
914         struct cgroup *res = NULL;
915 
916         lockdep_assert_held(&cgroup_mutex);
917         lockdep_assert_held(&css_set_rwsem);
918 
919         if (cset == &init_css_set) {
920                 res = &root->cgrp;
921         } else {
922                 struct cgrp_cset_link *link;
923 
924                 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
925                         struct cgroup *c = link->cgrp;
926 
927                         if (c->root == root) {
928                                 res = c;
929                                 break;
930                         }
931                 }
932         }
933 
934         BUG_ON(!res);
935         return res;
936 }
937 
938 /*
939  * Return the cgroup for "task" from the given hierarchy. Must be
940  * called with cgroup_mutex and css_set_rwsem held.
941  */
942 static struct cgroup *task_cgroup_from_root(struct task_struct *task,
943                                             struct cgroup_root *root)
944 {
945         /*
946          * No need to lock the task - since we hold cgroup_mutex the
947          * task can't change groups, so the only thing that can happen
948          * is that it exits and its css is set back to init_css_set.
949          */
950         return cset_cgroup_from_root(task_css_set(task), root);
951 }
952 
953 /*
954  * A task must hold cgroup_mutex to modify cgroups.
955  *
956  * Any task can increment and decrement the count field without lock.
957  * So in general, code holding cgroup_mutex can't rely on the count
958  * field not changing.  However, if the count goes to zero, then only
959  * cgroup_attach_task() can increment it again.  Because a count of zero
960  * means that no tasks are currently attached, therefore there is no
961  * way a task attached to that cgroup can fork (the other way to
962  * increment the count).  So code holding cgroup_mutex can safely
963  * assume that if the count is zero, it will stay zero. Similarly, if
964  * a task holds cgroup_mutex on a cgroup with zero count, it
965  * knows that the cgroup won't be removed, as cgroup_rmdir()
966  * needs that mutex.
967  *
968  * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
969  * (usually) take cgroup_mutex.  These are the two most performance
970  * critical pieces of code here.  The exception occurs on cgroup_exit(),
971  * when a task in a notify_on_release cgroup exits.  Then cgroup_mutex
972  * is taken, and if the cgroup count is zero, a usermode call made
973  * to the release agent with the name of the cgroup (path relative to
974  * the root of cgroup file system) as the argument.
975  *
976  * A cgroup can only be deleted if both its 'count' of using tasks
977  * is zero, and its list of 'children' cgroups is empty.  Since all
978  * tasks in the system use _some_ cgroup, and since there is always at
979  * least one task in the system (init, pid == 1), therefore, root cgroup
980  * always has either children cgroups and/or using tasks.  So we don't
981  * need a special hack to ensure that root cgroup cannot be deleted.
982  *
983  * P.S.  One more locking exception.  RCU is used to guard the
984  * update of a tasks cgroup pointer by cgroup_attach_task()
985  */
986 
987 static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask);
988 static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
989 static const struct file_operations proc_cgroupstats_operations;
990 
991 static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
992                               char *buf)
993 {
994         if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
995             !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
996                 snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
997                          cft->ss->name, cft->name);
998         else
999                 strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
1000         return buf;
1001 }
1002 
1003 /**
1004  * cgroup_file_mode - deduce file mode of a control file
1005  * @cft: the control file in question
1006  *
1007  * returns cft->mode if ->mode is not 0
1008  * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
1009  * returns S_IRUGO if it has only a read handler
1010  * returns S_IWUSR if it has only a write hander
1011  */
1012 static umode_t cgroup_file_mode(const struct cftype *cft)
1013 {
1014         umode_t mode = 0;
1015 
1016         if (cft->mode)
1017                 return cft->mode;
1018 
1019         if (cft->read_u64 || cft->read_s64 || cft->seq_show)
1020                 mode |= S_IRUGO;
1021 
1022         if (cft->write_u64 || cft->write_s64 || cft->write)
1023                 mode |= S_IWUSR;
1024 
1025         return mode;
1026 }
1027 
1028 static void cgroup_get(struct cgroup *cgrp)
1029 {
1030         WARN_ON_ONCE(cgroup_is_dead(cgrp));
1031         css_get(&cgrp->self);
1032 }
1033 
1034 static void cgroup_put(struct cgroup *cgrp)
1035 {
1036         css_put(&cgrp->self);
1037 }
1038 
1039 /**
1040  * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1041  * @kn: the kernfs_node being serviced
1042  *
1043  * This helper undoes cgroup_kn_lock_live() and should be invoked before
1044  * the method finishes if locking succeeded.  Note that once this function
1045  * returns the cgroup returned by cgroup_kn_lock_live() may become
1046  * inaccessible any time.  If the caller intends to continue to access the
1047  * cgroup, it should pin it before invoking this function.
1048  */
1049 static void cgroup_kn_unlock(struct kernfs_node *kn)
1050 {
1051         struct cgroup *cgrp;
1052 
1053         if (kernfs_type(kn) == KERNFS_DIR)
1054                 cgrp = kn->priv;
1055         else
1056                 cgrp = kn->parent->priv;
1057 
1058         mutex_unlock(&cgroup_mutex);
1059 
1060         kernfs_unbreak_active_protection(kn);
1061         cgroup_put(cgrp);
1062 }
1063 
1064 /**
1065  * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1066  * @kn: the kernfs_node being serviced
1067  *
1068  * This helper is to be used by a cgroup kernfs method currently servicing
1069  * @kn.  It breaks the active protection, performs cgroup locking and
1070  * verifies that the associated cgroup is alive.  Returns the cgroup if
1071  * alive; otherwise, %NULL.  A successful return should be undone by a
1072  * matching cgroup_kn_unlock() invocation.
1073  *
1074  * Any cgroup kernfs method implementation which requires locking the
1075  * associated cgroup should use this helper.  It avoids nesting cgroup
1076  * locking under kernfs active protection and allows all kernfs operations
1077  * including self-removal.
1078  */
1079 static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn)
1080 {
1081         struct cgroup *cgrp;
1082 
1083         if (kernfs_type(kn) == KERNFS_DIR)
1084                 cgrp = kn->priv;
1085         else
1086                 cgrp = kn->parent->priv;
1087 
1088         /*
1089          * We're gonna grab cgroup_mutex which nests outside kernfs
1090          * active_ref.  cgroup liveliness check alone provides enough
1091          * protection against removal.  Ensure @cgrp stays accessible and
1092          * break the active_ref protection.
1093          */
1094         cgroup_get(cgrp);
1095         kernfs_break_active_protection(kn);
1096 
1097         mutex_lock(&cgroup_mutex);
1098 
1099         if (!cgroup_is_dead(cgrp))
1100                 return cgrp;
1101 
1102         cgroup_kn_unlock(kn);
1103         return NULL;
1104 }
1105 
1106 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
1107 {
1108         char name[CGROUP_FILE_NAME_MAX];
1109 
1110         lockdep_assert_held(&cgroup_mutex);
1111         kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
1112 }
1113 
1114 /**
1115  * cgroup_clear_dir - remove subsys files in a cgroup directory
1116  * @cgrp: target cgroup
1117  * @subsys_mask: mask of the subsystem ids whose files should be removed
1118  */
1119 static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask)
1120 {
1121         struct cgroup_subsys *ss;
1122         int i;
1123 
1124         for_each_subsys(ss, i) {
1125                 struct cftype *cfts;
1126 
1127                 if (!(subsys_mask & (1 << i)))
1128                         continue;
1129                 list_for_each_entry(cfts, &ss->cfts, node)
1130                         cgroup_addrm_files(cgrp, cfts, false);
1131         }
1132 }
1133 
1134 static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask)
1135 {
1136         struct cgroup_subsys *ss;
1137         unsigned int tmp_ss_mask;
1138         int ssid, i, ret;
1139 
1140         lockdep_assert_held(&cgroup_mutex);
1141 
1142         for_each_subsys(ss, ssid) {
1143                 if (!(ss_mask & (1 << ssid)))
1144                         continue;
1145 
1146                 /* if @ss has non-root csses attached to it, can't move */
1147                 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)))
1148                         return -EBUSY;
1149 
1150                 /* can't move between two non-dummy roots either */
1151                 if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1152                         return -EBUSY;
1153         }
1154 
1155         /* skip creating root files on dfl_root for inhibited subsystems */
1156         tmp_ss_mask = ss_mask;
1157         if (dst_root == &cgrp_dfl_root)
1158                 tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask;
1159 
1160         ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask);
1161         if (ret) {
1162                 if (dst_root != &cgrp_dfl_root)
1163                         return ret;
1164 
1165                 /*
1166                  * Rebinding back to the default root is not allowed to
1167                  * fail.  Using both default and non-default roots should
1168                  * be rare.  Moving subsystems back and forth even more so.
1169                  * Just warn about it and continue.
1170                  */
1171                 if (cgrp_dfl_root_visible) {
1172                         pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n",
1173                                 ret, ss_mask);
1174                         pr_warn("you may retry by moving them to a different hierarchy and unbinding\n");
1175                 }
1176         }
1177 
1178         /*
1179          * Nothing can fail from this point on.  Remove files for the
1180          * removed subsystems and rebind each subsystem.
1181          */
1182         for_each_subsys(ss, ssid)
1183                 if (ss_mask & (1 << ssid))
1184                         cgroup_clear_dir(&ss->root->cgrp, 1 << ssid);
1185 
1186         for_each_subsys(ss, ssid) {
1187                 struct cgroup_root *src_root;
1188                 struct cgroup_subsys_state *css;
1189                 struct css_set *cset;
1190 
1191                 if (!(ss_mask & (1 << ssid)))
1192                         continue;
1193 
1194                 src_root = ss->root;
1195                 css = cgroup_css(&src_root->cgrp, ss);
1196 
1197                 WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss));
1198 
1199                 RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL);
1200                 rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css);
1201                 ss->root = dst_root;
1202                 css->cgroup = &dst_root->cgrp;
1203 
1204                 down_write(&css_set_rwsem);
1205                 hash_for_each(css_set_table, i, cset, hlist)
1206                         list_move_tail(&cset->e_cset_node[ss->id],
1207                                        &dst_root->cgrp.e_csets[ss->id]);
1208                 up_write(&css_set_rwsem);
1209 
1210                 src_root->subsys_mask &= ~(1 << ssid);
1211                 src_root->cgrp.child_subsys_mask &= ~(1 << ssid);
1212 
1213                 /* default hierarchy doesn't enable controllers by default */
1214                 dst_root->subsys_mask |= 1 << ssid;
1215                 if (dst_root != &cgrp_dfl_root)
1216                         dst_root->cgrp.child_subsys_mask |= 1 << ssid;
1217 
1218                 if (ss->bind)
1219                         ss->bind(css);
1220         }
1221 
1222         kernfs_activate(dst_root->cgrp.kn);
1223         return 0;
1224 }
1225 
1226 static int cgroup_show_options(struct seq_file *seq,
1227                                struct kernfs_root *kf_root)
1228 {
1229         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1230         struct cgroup_subsys *ss;
1231         int ssid;
1232 
1233         for_each_subsys(ss, ssid)
1234                 if (root->subsys_mask & (1 << ssid))
1235                         seq_printf(seq, ",%s", ss->name);
1236         if (root->flags & CGRP_ROOT_SANE_BEHAVIOR)
1237                 seq_puts(seq, ",sane_behavior");
1238         if (root->flags & CGRP_ROOT_NOPREFIX)
1239                 seq_puts(seq, ",noprefix");
1240         if (root->flags & CGRP_ROOT_XATTR)
1241                 seq_puts(seq, ",xattr");
1242 
1243         spin_lock(&release_agent_path_lock);
1244         if (strlen(root->release_agent_path))
1245                 seq_printf(seq, ",release_agent=%s", root->release_agent_path);
1246         spin_unlock(&release_agent_path_lock);
1247 
1248         if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
1249                 seq_puts(seq, ",clone_children");
1250         if (strlen(root->name))
1251                 seq_printf(seq, ",name=%s", root->name);
1252         return 0;
1253 }
1254 
1255 struct cgroup_sb_opts {
1256         unsigned int subsys_mask;
1257         unsigned int flags;
1258         char *release_agent;
1259         bool cpuset_clone_children;
1260         char *name;
1261         /* User explicitly requested empty subsystem */
1262         bool none;
1263 };
1264 
1265 static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1266 {
1267         char *token, *o = data;
1268         bool all_ss = false, one_ss = false;
1269         unsigned int mask = -1U;
1270         struct cgroup_subsys *ss;
1271         int i;
1272 
1273 #ifdef CONFIG_CPUSETS
1274         mask = ~(1U << cpuset_cgrp_id);
1275 #endif
1276 
1277         memset(opts, 0, sizeof(*opts));
1278 
1279         while ((token = strsep(&o, ",")) != NULL) {
1280                 if (!*token)
1281                         return -EINVAL;
1282                 if (!strcmp(token, "none")) {
1283                         /* Explicitly have no subsystems */
1284                         opts->none = true;
1285                         continue;
1286                 }
1287                 if (!strcmp(token, "all")) {
1288                         /* Mutually exclusive option 'all' + subsystem name */
1289                         if (one_ss)
1290                                 return -EINVAL;
1291                         all_ss = true;
1292                         continue;
1293                 }
1294                 if (!strcmp(token, "__DEVEL__sane_behavior")) {
1295                         opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
1296                         continue;
1297                 }
1298                 if (!strcmp(token, "noprefix")) {
1299                         opts->flags |= CGRP_ROOT_NOPREFIX;
1300                         continue;
1301                 }
1302                 if (!strcmp(token, "clone_children")) {
1303                         opts->cpuset_clone_children = true;
1304                         continue;
1305                 }
1306                 if (!strcmp(token, "xattr")) {
1307                         opts->flags |= CGRP_ROOT_XATTR;
1308                         continue;
1309                 }
1310                 if (!strncmp(token, "release_agent=", 14)) {
1311                         /* Specifying two release agents is forbidden */
1312                         if (opts->release_agent)
1313                                 return -EINVAL;
1314                         opts->release_agent =
1315                                 kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1316                         if (!opts->release_agent)
1317                                 return -ENOMEM;
1318                         continue;
1319                 }
1320                 if (!strncmp(token, "name=", 5)) {
1321                         const char *name = token + 5;
1322                         /* Can't specify an empty name */
1323                         if (!strlen(name))
1324                                 return -EINVAL;
1325                         /* Must match [\w.-]+ */
1326                         for (i = 0; i < strlen(name); i++) {
1327                                 char c = name[i];
1328                                 if (isalnum(c))
1329                                         continue;
1330                                 if ((c == '.') || (c == '-') || (c == '_'))
1331                                         continue;
1332                                 return -EINVAL;
1333                         }
1334                         /* Specifying two names is forbidden */
1335                         if (opts->name)
1336                                 return -EINVAL;
1337                         opts->name = kstrndup(name,
1338                                               MAX_CGROUP_ROOT_NAMELEN - 1,
1339                                               GFP_KERNEL);
1340                         if (!opts->name)
1341                                 return -ENOMEM;
1342 
1343                         continue;
1344                 }
1345 
1346                 for_each_subsys(ss, i) {
1347                         if (strcmp(token, ss->name))
1348                                 continue;
1349                         if (ss->disabled)
1350                                 continue;
1351 
1352                         /* Mutually exclusive option 'all' + subsystem name */
1353                         if (all_ss)
1354                                 return -EINVAL;
1355                         opts->subsys_mask |= (1 << i);
1356                         one_ss = true;
1357 
1358                         break;
1359                 }
1360                 if (i == CGROUP_SUBSYS_COUNT)
1361                         return -ENOENT;
1362         }
1363 
1364         /* Consistency checks */
1365 
1366         if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1367                 pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");
1368 
1369                 if ((opts->flags & (CGRP_ROOT_NOPREFIX | CGRP_ROOT_XATTR)) ||
1370                     opts->cpuset_clone_children || opts->release_agent ||
1371                     opts->name) {
1372                         pr_err("sane_behavior: noprefix, xattr, clone_children, release_agent and name are not allowed\n");
1373                         return -EINVAL;
1374                 }
1375         } else {
1376                 /*
1377                  * If the 'all' option was specified select all the
1378                  * subsystems, otherwise if 'none', 'name=' and a subsystem
1379                  * name options were not specified, let's default to 'all'
1380                  */
1381                 if (all_ss || (!one_ss && !opts->none && !opts->name))
1382                         for_each_subsys(ss, i)
1383                                 if (!ss->disabled)
1384                                         opts->subsys_mask |= (1 << i);
1385 
1386                 /*
1387                  * We either have to specify by name or by subsystems. (So
1388                  * all empty hierarchies must have a name).
1389                  */
1390                 if (!opts->subsys_mask && !opts->name)
1391                         return -EINVAL;
1392         }
1393 
1394         /*
1395          * Option noprefix was introduced just for backward compatibility
1396          * with the old cpuset, so we allow noprefix only if mounting just
1397          * the cpuset subsystem.
1398          */
1399         if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1400                 return -EINVAL;
1401 
1402 
1403         /* Can't specify "none" and some subsystems */
1404         if (opts->subsys_mask && opts->none)
1405                 return -EINVAL;
1406 
1407         return 0;
1408 }
1409 
1410 static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1411 {
1412         int ret = 0;
1413         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1414         struct cgroup_sb_opts opts;
1415         unsigned int added_mask, removed_mask;
1416 
1417         if (root->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1418                 pr_err("sane_behavior: remount is not allowed\n");
1419                 return -EINVAL;
1420         }
1421 
1422         mutex_lock(&cgroup_mutex);
1423 
1424         /* See what subsystems are wanted */
1425         ret = parse_cgroupfs_options(data, &opts);
1426         if (ret)
1427                 goto out_unlock;
1428 
1429         if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1430                 pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
1431                         task_tgid_nr(current), current->comm);
1432 
1433         added_mask = opts.subsys_mask & ~root->subsys_mask;
1434         removed_mask = root->subsys_mask & ~opts.subsys_mask;
1435 
1436         /* Don't allow flags or name to change at remount */
1437         if (((opts.flags ^ root->flags) & CGRP_ROOT_OPTION_MASK) ||
1438             (opts.name && strcmp(opts.name, root->name))) {
1439                 pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
1440                        opts.flags & CGRP_ROOT_OPTION_MASK, opts.name ?: "",
1441                        root->flags & CGRP_ROOT_OPTION_MASK, root->name);
1442                 ret = -EINVAL;
1443                 goto out_unlock;
1444         }
1445 
1446         /* remounting is not allowed for populated hierarchies */
1447         if (!list_empty(&root->cgrp.self.children)) {
1448                 ret = -EBUSY;
1449                 goto out_unlock;
1450         }
1451 
1452         ret = rebind_subsystems(root, added_mask);
1453         if (ret)
1454                 goto out_unlock;
1455 
1456         rebind_subsystems(&cgrp_dfl_root, removed_mask);
1457 
1458         if (opts.release_agent) {
1459                 spin_lock(&release_agent_path_lock);
1460                 strcpy(root->release_agent_path, opts.release_agent);
1461                 spin_unlock(&release_agent_path_lock);
1462         }
1463  out_unlock:
1464         kfree(opts.release_agent);
1465         kfree(opts.name);
1466         mutex_unlock(&cgroup_mutex);
1467         return ret;
1468 }
1469 
1470 /*
1471  * To reduce the fork() overhead for systems that are not actually using
1472  * their cgroups capability, we don't maintain the lists running through
1473  * each css_set to its tasks until we see the list actually used - in other
1474  * words after the first mount.
1475  */
1476 static bool use_task_css_set_links __read_mostly;
1477 
1478 static void cgroup_enable_task_cg_lists(void)
1479 {
1480         struct task_struct *p, *g;
1481 
1482         down_write(&css_set_rwsem);
1483 
1484         if (use_task_css_set_links)
1485                 goto out_unlock;
1486 
1487         use_task_css_set_links = true;
1488 
1489         /*
1490          * We need tasklist_lock because RCU is not safe against
1491          * while_each_thread(). Besides, a forking task that has passed
1492          * cgroup_post_fork() without seeing use_task_css_set_links = 1
1493          * is not guaranteed to have its child immediately visible in the
1494          * tasklist if we walk through it with RCU.
1495          */
1496         read_lock(&tasklist_lock);
1497         do_each_thread(g, p) {
1498                 WARN_ON_ONCE(!list_empty(&p->cg_list) ||
1499                              task_css_set(p) != &init_css_set);
1500 
1501                 /*
1502                  * We should check if the process is exiting, otherwise
1503                  * it will race with cgroup_exit() in that the list
1504                  * entry won't be deleted though the process has exited.
1505                  * Do it while holding siglock so that we don't end up
1506                  * racing against cgroup_exit().
1507                  */
1508                 spin_lock_irq(&p->sighand->siglock);
1509                 if (!(p->flags & PF_EXITING)) {
1510                         struct css_set *cset = task_css_set(p);
1511 
1512                         list_add(&p->cg_list, &cset->tasks);
1513                         get_css_set(cset);
1514                 }
1515                 spin_unlock_irq(&p->sighand->siglock);
1516         } while_each_thread(g, p);
1517         read_unlock(&tasklist_lock);
1518 out_unlock:
1519         up_write(&css_set_rwsem);
1520 }
1521 
1522 static void init_cgroup_housekeeping(struct cgroup *cgrp)
1523 {
1524         struct cgroup_subsys *ss;
1525         int ssid;
1526 
1527         INIT_LIST_HEAD(&cgrp->self.sibling);
1528         INIT_LIST_HEAD(&cgrp->self.children);
1529         INIT_LIST_HEAD(&cgrp->cset_links);
1530         INIT_LIST_HEAD(&cgrp->release_list);
1531         INIT_LIST_HEAD(&cgrp->pidlists);
1532         mutex_init(&cgrp->pidlist_mutex);
1533         cgrp->self.cgroup = cgrp;
1534         cgrp->self.flags |= CSS_ONLINE;
1535 
1536         for_each_subsys(ss, ssid)
1537                 INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1538 
1539         init_waitqueue_head(&cgrp->offline_waitq);
1540 }
1541 
1542 static void init_cgroup_root(struct cgroup_root *root,
1543                              struct cgroup_sb_opts *opts)
1544 {
1545         struct cgroup *cgrp = &root->cgrp;
1546 
1547         INIT_LIST_HEAD(&root->root_list);
1548         atomic_set(&root->nr_cgrps, 1);
1549         cgrp->root = root;
1550         init_cgroup_housekeeping(cgrp);
1551         idr_init(&root->cgroup_idr);
1552 
1553         root->flags = opts->flags;
1554         if (opts->release_agent)
1555                 strcpy(root->release_agent_path, opts->release_agent);
1556         if (opts->name)
1557                 strcpy(root->name, opts->name);
1558         if (opts->cpuset_clone_children)
1559                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1560 }
1561 
1562 static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask)
1563 {
1564         LIST_HEAD(tmp_links);
1565         struct cgroup *root_cgrp = &root->cgrp;
1566         struct css_set *cset;
1567         int i, ret;
1568 
1569         lockdep_assert_held(&cgroup_mutex);
1570 
1571         ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT);
1572         if (ret < 0)
1573                 goto out;
1574         root_cgrp->id = ret;
1575 
1576         ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release);
1577         if (ret)
1578                 goto out;
1579 
1580         /*
1581          * We're accessing css_set_count without locking css_set_rwsem here,
1582          * but that's OK - it can only be increased by someone holding
1583          * cgroup_lock, and that's us. The worst that can happen is that we
1584          * have some link structures left over
1585          */
1586         ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
1587         if (ret)
1588                 goto cancel_ref;
1589 
1590         ret = cgroup_init_root_id(root);
1591         if (ret)
1592                 goto cancel_ref;
1593 
1594         root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
1595                                            KERNFS_ROOT_CREATE_DEACTIVATED,
1596                                            root_cgrp);
1597         if (IS_ERR(root->kf_root)) {
1598                 ret = PTR_ERR(root->kf_root);
1599                 goto exit_root_id;
1600         }
1601         root_cgrp->kn = root->kf_root->kn;
1602 
1603         ret = cgroup_addrm_files(root_cgrp, cgroup_base_files, true);
1604         if (ret)
1605                 goto destroy_root;
1606 
1607         ret = rebind_subsystems(root, ss_mask);
1608         if (ret)
1609                 goto destroy_root;
1610 
1611         /*
1612          * There must be no failure case after here, since rebinding takes
1613          * care of subsystems' refcounts, which are explicitly dropped in
1614          * the failure exit path.
1615          */
1616         list_add(&root->root_list, &cgroup_roots);
1617         cgroup_root_count++;
1618 
1619         /*
1620          * Link the root cgroup in this hierarchy into all the css_set
1621          * objects.
1622          */
1623         down_write(&css_set_rwsem);
1624         hash_for_each(css_set_table, i, cset, hlist)
1625                 link_css_set(&tmp_links, cset, root_cgrp);
1626         up_write(&css_set_rwsem);
1627 
1628         BUG_ON(!list_empty(&root_cgrp->self.children));
1629         BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1630 
1631         kernfs_activate(root_cgrp->kn);
1632         ret = 0;
1633         goto out;
1634 
1635 destroy_root:
1636         kernfs_destroy_root(root->kf_root);
1637         root->kf_root = NULL;
1638 exit_root_id:
1639         cgroup_exit_root_id(root);
1640 cancel_ref:
1641         percpu_ref_cancel_init(&root_cgrp->self.refcnt);
1642 out:
1643         free_cgrp_cset_links(&tmp_links);
1644         return ret;
1645 }
1646 
1647 static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1648                          int flags, const char *unused_dev_name,
1649                          void *data)
1650 {
1651         struct super_block *pinned_sb = NULL;
1652         struct cgroup_subsys *ss;
1653         struct cgroup_root *root;
1654         struct cgroup_sb_opts opts;
1655         struct dentry *dentry;
1656         int ret;
1657         int i;
1658         bool new_sb;
1659 
1660         /*
1661          * The first time anyone tries to mount a cgroup, enable the list
1662          * linking each css_set to its tasks and fix up all existing tasks.
1663          */
1664         if (!use_task_css_set_links)
1665                 cgroup_enable_task_cg_lists();
1666 
1667         mutex_lock(&cgroup_mutex);
1668 
1669         /* First find the desired set of subsystems */
1670         ret = parse_cgroupfs_options(data, &opts);
1671         if (ret)
1672                 goto out_unlock;
1673 
1674         /* look for a matching existing root */
1675         if (!opts.subsys_mask && !opts.none && !opts.name) {
1676                 cgrp_dfl_root_visible = true;
1677                 root = &cgrp_dfl_root;
1678                 cgroup_get(&root->cgrp);
1679                 ret = 0;
1680                 goto out_unlock;
1681         }
1682 
1683         /*
1684          * Destruction of cgroup root is asynchronous, so subsystems may
1685          * still be dying after the previous unmount.  Let's drain the
1686          * dying subsystems.  We just need to ensure that the ones
1687          * unmounted previously finish dying and don't care about new ones
1688          * starting.  Testing ref liveliness is good enough.
1689          */
1690         for_each_subsys(ss, i) {
1691                 if (!(opts.subsys_mask & (1 << i)) ||
1692                     ss->root == &cgrp_dfl_root)
1693                         continue;
1694 
1695                 if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) {
1696                         mutex_unlock(&cgroup_mutex);
1697                         msleep(10);
1698                         ret = restart_syscall();
1699                         goto out_free;
1700                 }
1701                 cgroup_put(&ss->root->cgrp);
1702         }
1703 
1704         for_each_root(root) {
1705                 bool name_match = false;
1706 
1707                 if (root == &cgrp_dfl_root)
1708                         continue;
1709 
1710                 /*
1711                  * If we asked for a name then it must match.  Also, if
1712                  * name matches but sybsys_mask doesn't, we should fail.
1713                  * Remember whether name matched.
1714                  */
1715                 if (opts.name) {
1716                         if (strcmp(opts.name, root->name))
1717                                 continue;
1718                         name_match = true;
1719                 }
1720 
1721                 /*
1722                  * If we asked for subsystems (or explicitly for no
1723                  * subsystems) then they must match.
1724                  */
1725                 if ((opts.subsys_mask || opts.none) &&
1726                     (opts.subsys_mask != root->subsys_mask)) {
1727                         if (!name_match)
1728                                 continue;
1729                         ret = -EBUSY;
1730                         goto out_unlock;
1731                 }
1732 
1733                 if ((root->flags ^ opts.flags) & CGRP_ROOT_OPTION_MASK) {
1734                         if ((root->flags | opts.flags) & CGRP_ROOT_SANE_BEHAVIOR) {
1735                                 pr_err("sane_behavior: new mount options should match the existing superblock\n");
1736                                 ret = -EINVAL;
1737                                 goto out_unlock;
1738                         } else {
1739                                 pr_warn("new mount options do not match the existing superblock, will be ignored\n");
1740                         }
1741                 }
1742 
1743                 /*
1744                  * We want to reuse @root whose lifetime is governed by its
1745                  * ->cgrp.  Let's check whether @root is alive and keep it
1746                  * that way.  As cgroup_kill_sb() can happen anytime, we
1747                  * want to block it by pinning the sb so that @root doesn't
1748                  * get killed before mount is complete.
1749                  *
1750                  * With the sb pinned, tryget_live can reliably indicate
1751                  * whether @root can be reused.  If it's being killed,
1752                  * drain it.  We can use wait_queue for the wait but this
1753                  * path is super cold.  Let's just sleep a bit and retry.
1754                  */
1755                 pinned_sb = kernfs_pin_sb(root->kf_root, NULL);
1756                 if (IS_ERR(pinned_sb) ||
1757                     !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
1758                         mutex_unlock(&cgroup_mutex);
1759                         if (!IS_ERR_OR_NULL(pinned_sb))
1760                                 deactivate_super(pinned_sb);
1761                         msleep(10);
1762                         ret = restart_syscall();
1763                         goto out_free;
1764                 }
1765 
1766                 ret = 0;
1767                 goto out_unlock;
1768         }
1769 
1770         /*
1771          * No such thing, create a new one.  name= matching without subsys
1772          * specification is allowed for already existing hierarchies but we
1773          * can't create new one without subsys specification.
1774          */
1775         if (!opts.subsys_mask && !opts.none) {
1776                 ret = -EINVAL;
1777                 goto out_unlock;
1778         }
1779 
1780         root = kzalloc(sizeof(*root), GFP_KERNEL);
1781         if (!root) {
1782                 ret = -ENOMEM;
1783                 goto out_unlock;
1784         }
1785 
1786         init_cgroup_root(root, &opts);
1787 
1788         ret = cgroup_setup_root(root, opts.subsys_mask);
1789         if (ret)
1790                 cgroup_free_root(root);
1791 
1792 out_unlock:
1793         mutex_unlock(&cgroup_mutex);
1794 out_free:
1795         kfree(opts.release_agent);
1796         kfree(opts.name);
1797 
1798         if (ret)
1799                 return ERR_PTR(ret);
1800 
1801         dentry = kernfs_mount(fs_type, flags, root->kf_root,
1802                                 CGROUP_SUPER_MAGIC, &new_sb);
1803         if (IS_ERR(dentry) || !new_sb)
1804                 cgroup_put(&root->cgrp);
1805 
1806         /*
1807          * If @pinned_sb, we're reusing an existing root and holding an
1808          * extra ref on its sb.  Mount is complete.  Put the extra ref.
1809          */
1810         if (pinned_sb) {
1811                 WARN_ON(new_sb);
1812                 deactivate_super(pinned_sb);
1813         }
1814 
1815         return dentry;
1816 }
1817 
1818 static void cgroup_kill_sb(struct super_block *sb)
1819 {
1820         struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
1821         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1822 
1823         /*
1824          * If @root doesn't have any mounts or children, start killing it.
1825          * This prevents new mounts by disabling percpu_ref_tryget_live().
1826          * cgroup_mount() may wait for @root's release.
1827          *
1828          * And don't kill the default root.
1829          */
1830         if (css_has_online_children(&root->cgrp.self) ||
1831             root == &cgrp_dfl_root)
1832                 cgroup_put(&root->cgrp);
1833         else
1834                 percpu_ref_kill(&root->cgrp.self.refcnt);
1835 
1836         kernfs_kill_sb(sb);
1837 }
1838 
1839 static struct file_system_type cgroup_fs_type = {
1840         .name = "cgroup",
1841         .mount = cgroup_mount,
1842         .kill_sb = cgroup_kill_sb,
1843 };
1844 
1845 static struct kobject *cgroup_kobj;
1846 
1847 /**
1848  * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
1849  * @task: target task
1850  * @buf: the buffer to write the path into
1851  * @buflen: the length of the buffer
1852  *
1853  * Determine @task's cgroup on the first (the one with the lowest non-zero
1854  * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
1855  * function grabs cgroup_mutex and shouldn't be used inside locks used by
1856  * cgroup controller callbacks.
1857  *
1858  * Return value is the same as kernfs_path().
1859  */
1860 char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
1861 {
1862         struct cgroup_root *root;
1863         struct cgroup *cgrp;
1864         int hierarchy_id = 1;
1865         char *path = NULL;
1866 
1867         mutex_lock(&cgroup_mutex);
1868         down_read(&css_set_rwsem);
1869 
1870         root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
1871 
1872         if (root) {
1873                 cgrp = task_cgroup_from_root(task, root);
1874                 path = cgroup_path(cgrp, buf, buflen);
1875         } else {
1876                 /* if no hierarchy exists, everyone is in "/" */
1877                 if (strlcpy(buf, "/", buflen) < buflen)
1878                         path = buf;
1879         }
1880 
1881         up_read(&css_set_rwsem);
1882         mutex_unlock(&cgroup_mutex);
1883         return path;
1884 }
1885 EXPORT_SYMBOL_GPL(task_cgroup_path);
1886 
1887 /* used to track tasks and other necessary states during migration */
1888 struct cgroup_taskset {
1889         /* the src and dst cset list running through cset->mg_node */
1890         struct list_head        src_csets;
1891         struct list_head        dst_csets;
1892 
1893         /*
1894          * Fields for cgroup_taskset_*() iteration.
1895          *
1896          * Before migration is committed, the target migration tasks are on
1897          * ->mg_tasks of the csets on ->src_csets.  After, on ->mg_tasks of
1898          * the csets on ->dst_csets.  ->csets point to either ->src_csets
1899          * or ->dst_csets depending on whether migration is committed.
1900          *
1901          * ->cur_csets and ->cur_task point to the current task position
1902          * during iteration.
1903          */
1904         struct list_head        *csets;
1905         struct css_set          *cur_cset;
1906         struct task_struct      *cur_task;
1907 };
1908 
1909 /**
1910  * cgroup_taskset_first - reset taskset and return the first task
1911  * @tset: taskset of interest
1912  *
1913  * @tset iteration is initialized and the first task is returned.
1914  */
1915 struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
1916 {
1917         tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
1918         tset->cur_task = NULL;
1919 
1920         return cgroup_taskset_next(tset);
1921 }
1922 
1923 /**
1924  * cgroup_taskset_next - iterate to the next task in taskset
1925  * @tset: taskset of interest
1926  *
1927  * Return the next task in @tset.  Iteration must have been initialized
1928  * with cgroup_taskset_first().
1929  */
1930 struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
1931 {
1932         struct css_set *cset = tset->cur_cset;
1933         struct task_struct *task = tset->cur_task;
1934 
1935         while (&cset->mg_node != tset->csets) {
1936                 if (!task)
1937                         task = list_first_entry(&cset->mg_tasks,
1938                                                 struct task_struct, cg_list);
1939                 else
1940                         task = list_next_entry(task, cg_list);
1941 
1942                 if (&task->cg_list != &cset->mg_tasks) {
1943                         tset->cur_cset = cset;
1944                         tset->cur_task = task;
1945                         return task;
1946                 }
1947 
1948                 cset = list_next_entry(cset, mg_node);
1949                 task = NULL;
1950         }
1951 
1952         return NULL;
1953 }
1954 
1955 /**
1956  * cgroup_task_migrate - move a task from one cgroup to another.
1957  * @old_cgrp: the cgroup @tsk is being migrated from
1958  * @tsk: the task being migrated
1959  * @new_cset: the new css_set @tsk is being attached to
1960  *
1961  * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked.
1962  */
1963 static void cgroup_task_migrate(struct cgroup *old_cgrp,
1964                                 struct task_struct *tsk,
1965                                 struct css_set *new_cset)
1966 {
1967         struct css_set *old_cset;
1968 
1969         lockdep_assert_held(&cgroup_mutex);
1970         lockdep_assert_held(&css_set_rwsem);
1971 
1972         /*
1973          * We are synchronized through threadgroup_lock() against PF_EXITING
1974          * setting such that we can't race against cgroup_exit() changing the
1975          * css_set to init_css_set and dropping the old one.
1976          */
1977         WARN_ON_ONCE(tsk->flags & PF_EXITING);
1978         old_cset = task_css_set(tsk);
1979 
1980         get_css_set(new_cset);
1981         rcu_assign_pointer(tsk->cgroups, new_cset);
1982 
1983         /*
1984          * Use move_tail so that cgroup_taskset_first() still returns the
1985          * leader after migration.  This works because cgroup_migrate()
1986          * ensures that the dst_cset of the leader is the first on the
1987          * tset's dst_csets list.
1988          */
1989         list_move_tail(&tsk->cg_list, &new_cset->mg_tasks);
1990 
1991         /*
1992          * We just gained a reference on old_cset by taking it from the
1993          * task. As trading it for new_cset is protected by cgroup_mutex,
1994          * we're safe to drop it here; it will be freed under RCU.
1995          */
1996         set_bit(CGRP_RELEASABLE, &old_cgrp->flags);
1997         put_css_set_locked(old_cset, false);
1998 }
1999 
2000 /**
2001  * cgroup_migrate_finish - cleanup after attach
2002  * @preloaded_csets: list of preloaded css_sets
2003  *
2004  * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
2005  * those functions for details.
2006  */
2007 static void cgroup_migrate_finish(struct list_head *preloaded_csets)
2008 {
2009         struct css_set *cset, *tmp_cset;
2010 
2011         lockdep_assert_held(&cgroup_mutex);
2012 
2013         down_write(&css_set_rwsem);
2014         list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
2015                 cset->mg_src_cgrp = NULL;
2016                 cset->mg_dst_cset = NULL;
2017                 list_del_init(&cset->mg_preload_node);
2018                 put_css_set_locked(cset, false);
2019         }
2020         up_write(&css_set_rwsem);
2021 }
2022 
2023 /**
2024  * cgroup_migrate_add_src - add a migration source css_set
2025  * @src_cset: the source css_set to add
2026  * @dst_cgrp: the destination cgroup
2027  * @preloaded_csets: list of preloaded css_sets
2028  *
2029  * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
2030  * @src_cset and add it to @preloaded_csets, which should later be cleaned
2031  * up by cgroup_migrate_finish().
2032  *
2033  * This function may be called without holding threadgroup_lock even if the
2034  * target is a process.  Threads may be created and destroyed but as long
2035  * as cgroup_mutex is not dropped, no new css_set can be put into play and
2036  * the preloaded css_sets are guaranteed to cover all migrations.
2037  */
2038 static void cgroup_migrate_add_src(struct css_set *src_cset,
2039                                    struct cgroup *dst_cgrp,
2040                                    struct list_head *preloaded_csets)
2041 {
2042         struct cgroup *src_cgrp;
2043 
2044         lockdep_assert_held(&cgroup_mutex);
2045         lockdep_assert_held(&css_set_rwsem);
2046 
2047         src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);
2048 
2049         if (!list_empty(&src_cset->mg_preload_node))
2050                 return;
2051 
2052         WARN_ON(src_cset->mg_src_cgrp);
2053         WARN_ON(!list_empty(&src_cset->mg_tasks));
2054         WARN_ON(!list_empty(&src_cset->mg_node));
2055 
2056         src_cset->mg_src_cgrp = src_cgrp;
2057         get_css_set(src_cset);
2058         list_add(&src_cset->mg_preload_node, preloaded_csets);
2059 }
2060 
2061 /**
2062  * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2063  * @dst_cgrp: the destination cgroup (may be %NULL)
2064  * @preloaded_csets: list of preloaded source css_sets
2065  *
2066  * Tasks are about to be moved to @dst_cgrp and all the source css_sets
2067  * have been preloaded to @preloaded_csets.  This function looks up and
2068  * pins all destination css_sets, links each to its source, and append them
2069  * to @preloaded_csets.  If @dst_cgrp is %NULL, the destination of each
2070  * source css_set is assumed to be its cgroup on the default hierarchy.
2071  *
2072  * This function must be called after cgroup_migrate_add_src() has been
2073  * called on each migration source css_set.  After migration is performed
2074  * using cgroup_migrate(), cgroup_migrate_finish() must be called on
2075  * @preloaded_csets.
2076  */
2077 static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
2078                                       struct list_head *preloaded_csets)
2079 {
2080         LIST_HEAD(csets);
2081         struct css_set *src_cset, *tmp_cset;
2082 
2083         lockdep_assert_held(&cgroup_mutex);
2084 
2085         /*
2086          * Except for the root, child_subsys_mask must be zero for a cgroup
2087          * with tasks so that child cgroups don't compete against tasks.
2088          */
2089         if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) &&
2090             dst_cgrp->child_subsys_mask)
2091                 return -EBUSY;
2092 
2093         /* look up the dst cset for each src cset and link it to src */
2094         list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
2095                 struct css_set *dst_cset;
2096 
2097                 dst_cset = find_css_set(src_cset,
2098                                         dst_cgrp ?: src_cset->dfl_cgrp);
2099                 if (!dst_cset)
2100                         goto err;
2101 
2102                 WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
2103 
2104                 /*
2105                  * If src cset equals dst, it's noop.  Drop the src.
2106                  * cgroup_migrate() will skip the cset too.  Note that we
2107                  * can't handle src == dst as some nodes are used by both.
2108                  */
2109                 if (src_cset == dst_cset) {
2110                         src_cset->mg_src_cgrp = NULL;
2111                         list_del_init(&src_cset->mg_preload_node);
2112                         put_css_set(src_cset, false);
2113                         put_css_set(dst_cset, false);
2114                         continue;
2115                 }
2116 
2117                 src_cset->mg_dst_cset = dst_cset;
2118 
2119                 if (list_empty(&dst_cset->mg_preload_node))
2120                         list_add(&dst_cset->mg_preload_node, &csets);
2121                 else
2122                         put_css_set(dst_cset, false);
2123         }
2124 
2125         list_splice_tail(&csets, preloaded_csets);
2126         return 0;
2127 err:
2128         cgroup_migrate_finish(&csets);
2129         return -ENOMEM;
2130 }
2131 
2132 /**
2133  * cgroup_migrate - migrate a process or task to a cgroup
2134  * @cgrp: the destination cgroup
2135  * @leader: the leader of the process or the task to migrate
2136  * @threadgroup: whether @leader points to the whole process or a single task
2137  *
2138  * Migrate a process or task denoted by @leader to @cgrp.  If migrating a
2139  * process, the caller must be holding threadgroup_lock of @leader.  The
2140  * caller is also responsible for invoking cgroup_migrate_add_src() and
2141  * cgroup_migrate_prepare_dst() on the targets before invoking this
2142  * function and following up with cgroup_migrate_finish().
2143  *
2144  * As long as a controller's ->can_attach() doesn't fail, this function is
2145  * guaranteed to succeed.  This means that, excluding ->can_attach()
2146  * failure, when migrating multiple targets, the success or failure can be
2147  * decided for all targets by invoking group_migrate_prepare_dst() before
2148  * actually starting migrating.
2149  */
2150 static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader,
2151                           bool threadgroup)
2152 {
2153         struct cgroup_taskset tset = {
2154                 .src_csets      = LIST_HEAD_INIT(tset.src_csets),
2155                 .dst_csets      = LIST_HEAD_INIT(tset.dst_csets),
2156                 .csets          = &tset.src_csets,
2157         };
2158         struct cgroup_subsys_state *css, *failed_css = NULL;
2159         struct css_set *cset, *tmp_cset;
2160         struct task_struct *task, *tmp_task;
2161         int i, ret;
2162 
2163         /*
2164          * Prevent freeing of tasks while we take a snapshot. Tasks that are
2165          * already PF_EXITING could be freed from underneath us unless we
2166          * take an rcu_read_lock.
2167          */
2168         down_write(&css_set_rwsem);
2169         rcu_read_lock();
2170         task = leader;
2171         do {
2172                 /* @task either already exited or can't exit until the end */
2173                 if (task->flags & PF_EXITING)
2174                         goto next;
2175 
2176                 /* leave @task alone if post_fork() hasn't linked it yet */
2177                 if (list_empty(&task->cg_list))
2178                         goto next;
2179 
2180                 cset = task_css_set(task);
2181                 if (!cset->mg_src_cgrp)
2182                         goto next;
2183 
2184                 /*
2185                  * cgroup_taskset_first() must always return the leader.
2186                  * Take care to avoid disturbing the ordering.
2187                  */
2188                 list_move_tail(&task->cg_list, &cset->mg_tasks);
2189                 if (list_empty(&cset->mg_node))
2190                         list_add_tail(&cset->mg_node, &tset.src_csets);
2191                 if (list_empty(&cset->mg_dst_cset->mg_node))
2192                         list_move_tail(&cset->mg_dst_cset->mg_node,
2193                                        &tset.dst_csets);
2194         next:
2195                 if (!threadgroup)
2196                         break;
2197         } while_each_thread(leader, task);
2198         rcu_read_unlock();
2199         up_write(&css_set_rwsem);
2200 
2201         /* methods shouldn't be called if no task is actually migrating */
2202         if (list_empty(&tset.src_csets))
2203                 return 0;
2204 
2205         /* check that we can legitimately attach to the cgroup */
2206         for_each_e_css(css, i, cgrp) {
2207                 if (css->ss->can_attach) {
2208                         ret = css->ss->can_attach(css, &tset);
2209                         if (ret) {
2210                                 failed_css = css;
2211                                 goto out_cancel_attach;
2212                         }
2213                 }
2214         }
2215 
2216         /*
2217          * Now that we're guaranteed success, proceed to move all tasks to
2218          * the new cgroup.  There are no failure cases after here, so this
2219          * is the commit point.
2220          */
2221         down_write(&css_set_rwsem);
2222         list_for_each_entry(cset, &tset.src_csets, mg_node) {
2223                 list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list)
2224                         cgroup_task_migrate(cset->mg_src_cgrp, task,
2225                                             cset->mg_dst_cset);
2226         }
2227         up_write(&css_set_rwsem);
2228 
2229         /*
2230          * Migration is committed, all target tasks are now on dst_csets.
2231          * Nothing is sensitive to fork() after this point.  Notify
2232          * controllers that migration is complete.
2233          */
2234         tset.csets = &tset.dst_csets;
2235 
2236         for_each_e_css(css, i, cgrp)
2237                 if (css->ss->attach)
2238                         css->ss->attach(css, &tset);
2239 
2240         ret = 0;
2241         goto out_release_tset;
2242 
2243 out_cancel_attach:
2244         for_each_e_css(css, i, cgrp) {
2245                 if (css == failed_css)
2246                         break;
2247                 if (css->ss->cancel_attach)
2248                         css->ss->cancel_attach(css, &tset);
2249         }
2250 out_release_tset:
2251         down_write(&css_set_rwsem);
2252         list_splice_init(&tset.dst_csets, &tset.src_csets);
2253         list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) {
2254                 list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2255                 list_del_init(&cset->mg_node);
2256         }
2257         up_write(&css_set_rwsem);
2258         return ret;
2259 }
2260 
2261 /**
2262  * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2263  * @dst_cgrp: the cgroup to attach to
2264  * @leader: the task or the leader of the threadgroup to be attached
2265  * @threadgroup: attach the whole threadgroup?
2266  *
2267  * Call holding cgroup_mutex and threadgroup_lock of @leader.
2268  */
2269 static int cgroup_attach_task(struct cgroup *dst_cgrp,
2270                               struct task_struct *leader, bool threadgroup)
2271 {
2272         LIST_HEAD(preloaded_csets);
2273         struct task_struct *task;
2274         int ret;
2275 
2276         /* look up all src csets */
2277         down_read(&css_set_rwsem);
2278         rcu_read_lock();
2279         task = leader;
2280         do {
2281                 cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
2282                                        &preloaded_csets);
2283                 if (!threadgroup)
2284                         break;
2285         } while_each_thread(leader, task);
2286         rcu_read_unlock();
2287         up_read(&css_set_rwsem);
2288 
2289         /* prepare dst csets and commit */
2290         ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
2291         if (!ret)
2292                 ret = cgroup_migrate(dst_cgrp, leader, threadgroup);
2293 
2294         cgroup_migrate_finish(&preloaded_csets);
2295         return ret;
2296 }
2297 
2298 /*
2299  * Find the task_struct of the task to attach by vpid and pass it along to the
2300  * function to attach either it or all tasks in its threadgroup. Will lock
2301  * cgroup_mutex and threadgroup.
2302  */
2303 static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
2304                                     size_t nbytes, loff_t off, bool threadgroup)
2305 {
2306         struct task_struct *tsk;
2307         const struct cred *cred = current_cred(), *tcred;
2308         struct cgroup *cgrp;
2309         pid_t pid;
2310         int ret;
2311 
2312         if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
2313                 return -EINVAL;
2314 
2315         cgrp = cgroup_kn_lock_live(of->kn);
2316         if (!cgrp)
2317                 return -ENODEV;
2318 
2319 retry_find_task:
2320         rcu_read_lock();
2321         if (pid) {
2322                 tsk = find_task_by_vpid(pid);
2323                 if (!tsk) {
2324                         rcu_read_unlock();
2325                         ret = -ESRCH;
2326                         goto out_unlock_cgroup;
2327                 }
2328                 /*
2329                  * even if we're attaching all tasks in the thread group, we
2330                  * only need to check permissions on one of them.
2331                  */
2332                 tcred = __task_cred(tsk);
2333                 if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
2334                     !uid_eq(cred->euid, tcred->uid) &&
2335                     !uid_eq(cred->euid, tcred->suid)) {
2336                         rcu_read_unlock();
2337                         ret = -EACCES;
2338                         goto out_unlock_cgroup;
2339                 }
2340         } else
2341                 tsk = current;
2342 
2343         if (threadgroup)
2344                 tsk = tsk->group_leader;
2345 
2346         /*
2347          * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2348          * trapped in a cpuset, or RT worker may be born in a cgroup
2349          * with no rt_runtime allocated.  Just say no.
2350          */
2351         if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2352                 ret = -EINVAL;
2353                 rcu_read_unlock();
2354                 goto out_unlock_cgroup;
2355         }
2356 
2357         get_task_struct(tsk);
2358         rcu_read_unlock();
2359 
2360         threadgroup_lock(tsk);
2361         if (threadgroup) {
2362                 if (!thread_group_leader(tsk)) {
2363                         /*
2364                          * a race with de_thread from another thread's exec()
2365                          * may strip us of our leadership, if this happens,
2366                          * there is no choice but to throw this task away and
2367                          * try again; this is
2368                          * "double-double-toil-and-trouble-check locking".
2369                          */
2370                         threadgroup_unlock(tsk);
2371                         put_task_struct(tsk);
2372                         goto retry_find_task;
2373                 }
2374         }
2375 
2376         ret = cgroup_attach_task(cgrp, tsk, threadgroup);
2377 
2378         threadgroup_unlock(tsk);
2379 
2380         put_task_struct(tsk);
2381 out_unlock_cgroup:
2382         cgroup_kn_unlock(of->kn);
2383         return ret ?: nbytes;
2384 }
2385 
2386 /**
2387  * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
2388  * @from: attach to all cgroups of a given task
2389  * @tsk: the task to be attached
2390  */
2391 int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
2392 {
2393         struct cgroup_root *root;
2394         int retval = 0;
2395 
2396         mutex_lock(&cgroup_mutex);
2397         for_each_root(root) {
2398                 struct cgroup *from_cgrp;
2399 
2400                 if (root == &cgrp_dfl_root)
2401                         continue;
2402 
2403                 down_read(&css_set_rwsem);
2404                 from_cgrp = task_cgroup_from_root(from, root);
2405                 up_read(&css_set_rwsem);
2406 
2407                 retval = cgroup_attach_task(from_cgrp, tsk, false);
2408                 if (retval)
2409                         break;
2410         }
2411         mutex_unlock(&cgroup_mutex);
2412 
2413         return retval;
2414 }
2415 EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
2416 
2417 static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
2418                                   char *buf, size_t nbytes, loff_t off)
2419 {
2420         return __cgroup_procs_write(of, buf, nbytes, off, false);
2421 }
2422 
2423 static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
2424                                   char *buf, size_t nbytes, loff_t off)
2425 {
2426         return __cgroup_procs_write(of, buf, nbytes, off, true);
2427 }
2428 
2429 static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
2430                                           char *buf, size_t nbytes, loff_t off)
2431 {
2432         struct cgroup *cgrp;
2433 
2434         BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
2435 
2436         cgrp = cgroup_kn_lock_live(of->kn);
2437         if (!cgrp)
2438                 return -ENODEV;
2439         spin_lock(&release_agent_path_lock);
2440         strlcpy(cgrp->root->release_agent_path, strstrip(buf),
2441                 sizeof(cgrp->root->release_agent_path));
2442         spin_unlock(&release_agent_path_lock);
2443         cgroup_kn_unlock(of->kn);
2444         return nbytes;
2445 }
2446 
2447 static int cgroup_release_agent_show(struct seq_file *seq, void *v)
2448 {
2449         struct cgroup *cgrp = seq_css(seq)->cgroup;
2450 
2451         spin_lock(&release_agent_path_lock);
2452         seq_puts(seq, cgrp->root->release_agent_path);
2453         spin_unlock(&release_agent_path_lock);
2454         seq_putc(seq, '\n');
2455         return 0;
2456 }
2457 
2458 static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
2459 {
2460         struct cgroup *cgrp = seq_css(seq)->cgroup;
2461 
2462         seq_printf(seq, "%d\n", cgroup_sane_behavior(cgrp));
2463         return 0;
2464 }
2465 
2466 static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask)
2467 {
2468         struct cgroup_subsys *ss;
2469         bool printed = false;
2470         int ssid;
2471 
2472         for_each_subsys(ss, ssid) {
2473                 if (ss_mask & (1 << ssid)) {
2474                         if (printed)
2475                                 seq_putc(seq, ' ');
2476                         seq_printf(seq, "%s", ss->name);
2477                         printed = true;
2478                 }
2479         }
2480         if (printed)
2481                 seq_putc(seq, '\n');
2482 }
2483 
2484 /* show controllers which are currently attached to the default hierarchy */
2485 static int cgroup_root_controllers_show(struct seq_file *seq, void *v)
2486 {
2487         struct cgroup *cgrp = seq_css(seq)->cgroup;
2488 
2489         cgroup_print_ss_mask(seq, cgrp->root->subsys_mask &
2490                              ~cgrp_dfl_root_inhibit_ss_mask);
2491         return 0;
2492 }
2493 
2494 /* show controllers which are enabled from the parent */
2495 static int cgroup_controllers_show(struct seq_file *seq, void *v)
2496 {
2497         struct cgroup *cgrp = seq_css(seq)->cgroup;
2498 
2499         cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->child_subsys_mask);
2500         return 0;
2501 }
2502 
2503 /* show controllers which are enabled for a given cgroup's children */
2504 static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
2505 {
2506         struct cgroup *cgrp = seq_css(seq)->cgroup;
2507 
2508         cgroup_print_ss_mask(seq, cgrp->child_subsys_mask);
2509         return 0;
2510 }
2511 
2512 /**
2513  * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
2514  * @cgrp: root of the subtree to update csses for
2515  *
2516  * @cgrp's child_subsys_mask has changed and its subtree's (self excluded)
2517  * css associations need to be updated accordingly.  This function looks up
2518  * all css_sets which are attached to the subtree, creates the matching
2519  * updated css_sets and migrates the tasks to the new ones.
2520  */
2521 static int cgroup_update_dfl_csses(struct cgroup *cgrp)
2522 {
2523         LIST_HEAD(preloaded_csets);
2524         struct cgroup_subsys_state *css;
2525         struct css_set *src_cset;
2526         int ret;
2527 
2528         lockdep_assert_held(&cgroup_mutex);
2529 
2530         /* look up all csses currently attached to @cgrp's subtree */
2531         down_read(&css_set_rwsem);
2532         css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
2533                 struct cgrp_cset_link *link;
2534 
2535                 /* self is not affected by child_subsys_mask change */
2536                 if (css->cgroup == cgrp)
2537                         continue;
2538 
2539                 list_for_each_entry(link, &css->cgroup->cset_links, cset_link)
2540                         cgroup_migrate_add_src(link->cset, cgrp,
2541                                                &preloaded_csets);
2542         }
2543         up_read(&css_set_rwsem);
2544 
2545         /* NULL dst indicates self on default hierarchy */
2546         ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets);
2547         if (ret)
2548                 goto out_finish;
2549 
2550         list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
2551                 struct task_struct *last_task = NULL, *task;
2552 
2553                 /* src_csets precede dst_csets, break on the first dst_cset */
2554                 if (!src_cset->mg_src_cgrp)
2555                         break;
2556 
2557                 /*
2558                  * All tasks in src_cset need to be migrated to the
2559                  * matching dst_cset.  Empty it process by process.  We
2560                  * walk tasks but migrate processes.  The leader might even
2561                  * belong to a different cset but such src_cset would also
2562                  * be among the target src_csets because the default
2563                  * hierarchy enforces per-process membership.
2564                  */
2565                 while (true) {
2566                         down_read(&css_set_rwsem);
2567                         task = list_first_entry_or_null(&src_cset->tasks,
2568                                                 struct task_struct, cg_list);
2569                         if (task) {
2570                                 task = task->group_leader;
2571                                 WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp);
2572                                 get_task_struct(task);
2573                         }
2574                         up_read(&css_set_rwsem);
2575 
2576                         if (!task)
2577                                 break;
2578 
2579                         /* guard against possible infinite loop */
2580                         if (WARN(last_task == task,
2581                                  "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n"))
2582                                 goto out_finish;
2583                         last_task = task;
2584 
2585                         threadgroup_lock(task);
2586                         /* raced against de_thread() from another thread? */
2587                         if (!thread_group_leader(task)) {
2588                                 threadgroup_unlock(task);
2589                                 put_task_struct(task);
2590                                 continue;
2591                         }
2592 
2593                         ret = cgroup_migrate(src_cset->dfl_cgrp, task, true);
2594 
2595                         threadgroup_unlock(task);
2596                         put_task_struct(task);
2597 
2598                         if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret))
2599                                 goto out_finish;
2600                 }
2601         }
2602 
2603 out_finish:
2604         cgroup_migrate_finish(&preloaded_csets);
2605         return ret;
2606 }
2607 
2608 /* change the enabled child controllers for a cgroup in the default hierarchy */
2609 static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
2610                                             char *buf, size_t nbytes,
2611                                             loff_t off)
2612 {
2613         unsigned int enable = 0, disable = 0;
2614         struct cgroup *cgrp, *child;
2615         struct cgroup_subsys *ss;
2616         char *tok;
2617         int ssid, ret;
2618 
2619         /*
2620          * Parse input - space separated list of subsystem names prefixed
2621          * with either + or -.
2622          */
2623         buf = strstrip(buf);
2624         while ((tok = strsep(&buf, " "))) {
2625                 if (tok[0] == '\0')
2626                         continue;
2627                 for_each_subsys(ss, ssid) {
2628                         if (ss->disabled || strcmp(tok + 1, ss->name) ||
2629                             ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask))
2630                                 continue;
2631 
2632                         if (*tok == '+') {
2633                                 enable |= 1 << ssid;
2634                                 disable &= ~(1 << ssid);
2635                         } else if (*tok == '-') {
2636                                 disable |= 1 << ssid;
2637                                 enable &= ~(1 << ssid);
2638                         } else {
2639                                 return -EINVAL;
2640                         }
2641                         break;
2642                 }
2643                 if (ssid == CGROUP_SUBSYS_COUNT)
2644                         return -EINVAL;
2645         }
2646 
2647         cgrp = cgroup_kn_lock_live(of->kn);
2648         if (!cgrp)
2649                 return -ENODEV;
2650 
2651         for_each_subsys(ss, ssid) {
2652                 if (enable & (1 << ssid)) {
2653                         if (cgrp->child_subsys_mask & (1 << ssid)) {
2654                                 enable &= ~(1 << ssid);
2655                                 continue;
2656                         }
2657 
2658                         /*
2659                          * Because css offlining is asynchronous, userland
2660                          * might try to re-enable the same controller while
2661                          * the previous instance is still around.  In such
2662                          * cases, wait till it's gone using offline_waitq.
2663                          */
2664                         cgroup_for_each_live_child(child, cgrp) {
2665                                 DEFINE_WAIT(wait);
2666 
2667                                 if (!cgroup_css(child, ss))
2668                                         continue;
2669 
2670                                 cgroup_get(child);
2671                                 prepare_to_wait(&child->offline_waitq, &wait,
2672                                                 TASK_UNINTERRUPTIBLE);
2673                                 cgroup_kn_unlock(of->kn);
2674                                 schedule();
2675                                 finish_wait(&child->offline_waitq, &wait);
2676                                 cgroup_put(child);
2677 
2678                                 return restart_syscall();
2679                         }
2680 
2681                         /* unavailable or not enabled on the parent? */
2682                         if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) ||
2683                             (cgroup_parent(cgrp) &&
2684                              !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ssid)))) {
2685                                 ret = -ENOENT;
2686                                 goto out_unlock;
2687                         }
2688                 } else if (disable & (1 << ssid)) {
2689                         if (!(cgrp->child_subsys_mask & (1 << ssid))) {
2690                                 disable &= ~(1 << ssid);
2691                                 continue;
2692                         }
2693 
2694                         /* a child has it enabled? */
2695                         cgroup_for_each_live_child(child, cgrp) {
2696                                 if (child->child_subsys_mask & (1 << ssid)) {
2697                                         ret = -EBUSY;
2698                                         goto out_unlock;
2699                                 }
2700                         }
2701                 }
2702         }
2703 
2704         if (!enable && !disable) {
2705                 ret = 0;
2706                 goto out_unlock;
2707         }
2708 
2709         /*
2710          * Except for the root, child_subsys_mask must be zero for a cgroup
2711          * with tasks so that child cgroups don't compete against tasks.
2712          */
2713         if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) {
2714                 ret = -EBUSY;
2715                 goto out_unlock;
2716         }
2717 
2718         /*
2719          * Create csses for enables and update child_subsys_mask.  This
2720          * changes cgroup_e_css() results which in turn makes the
2721          * subsequent cgroup_update_dfl_csses() associate all tasks in the
2722          * subtree to the updated csses.
2723          */
2724         for_each_subsys(ss, ssid) {
2725                 if (!(enable & (1 << ssid)))
2726                         continue;
2727 
2728                 cgroup_for_each_live_child(child, cgrp) {
2729                         ret = create_css(child, ss);
2730                         if (ret)
2731                                 goto err_undo_css;
2732                 }
2733         }
2734 
2735         cgrp->child_subsys_mask |= enable;
2736         cgrp->child_subsys_mask &= ~disable;
2737 
2738         ret = cgroup_update_dfl_csses(cgrp);
2739         if (ret)
2740                 goto err_undo_css;
2741 
2742         /* all tasks are now migrated away from the old csses, kill them */
2743         for_each_subsys(ss, ssid) {
2744                 if (!(disable & (1 << ssid)))
2745                         continue;
2746 
2747                 cgroup_for_each_live_child(child, cgrp)
2748                         kill_css(cgroup_css(child, ss));
2749         }
2750 
2751         kernfs_activate(cgrp->kn);
2752         ret = 0;
2753 out_unlock:
2754         cgroup_kn_unlock(of->kn);
2755         return ret ?: nbytes;
2756 
2757 err_undo_css:
2758         cgrp->child_subsys_mask &= ~enable;
2759         cgrp->child_subsys_mask |= disable;
2760 
2761         for_each_subsys(ss, ssid) {
2762                 if (!(enable & (1 << ssid)))
2763                         continue;
2764 
2765                 cgroup_for_each_live_child(child, cgrp) {
2766                         struct cgroup_subsys_state *css = cgroup_css(child, ss);
2767                         if (css)
2768                                 kill_css(css);
2769                 }
2770         }
2771         goto out_unlock;
2772 }
2773 
2774 static int cgroup_populated_show(struct seq_file *seq, void *v)
2775 {
2776         seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt);
2777         return 0;
2778 }
2779 
2780 static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
2781                                  size_t nbytes, loff_t off)
2782 {
2783         struct cgroup *cgrp = of->kn->parent->priv;
2784         struct cftype *cft = of->kn->priv;
2785         struct cgroup_subsys_state *css;
2786         int ret;
2787 
2788         if (cft->write)
2789                 return cft->write(of, buf, nbytes, off);
2790 
2791         /*
2792          * kernfs guarantees that a file isn't deleted with operations in
2793          * flight, which means that the matching css is and stays alive and
2794          * doesn't need to be pinned.  The RCU locking is not necessary
2795          * either.  It's just for the convenience of using cgroup_css().
2796          */
2797         rcu_read_lock();
2798         css = cgroup_css(cgrp, cft->ss);
2799         rcu_read_unlock();
2800 
2801         if (cft->write_u64) {
2802                 unsigned long long v;
2803                 ret = kstrtoull(buf, 0, &v);
2804                 if (!ret)
2805                         ret = cft->write_u64(css, cft, v);
2806         } else if (cft->write_s64) {
2807                 long long v;
2808                 ret = kstrtoll(buf, 0, &v);
2809                 if (!ret)
2810                         ret = cft->write_s64(css, cft, v);
2811         } else {
2812                 ret = -EINVAL;
2813         }
2814 
2815         return ret ?: nbytes;
2816 }
2817 
2818 static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
2819 {
2820         return seq_cft(seq)->seq_start(seq, ppos);
2821 }
2822 
2823 static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
2824 {
2825         return seq_cft(seq)->seq_next(seq, v, ppos);
2826 }
2827 
2828 static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
2829 {
2830         seq_cft(seq)->seq_stop(seq, v);
2831 }
2832 
2833 static int cgroup_seqfile_show(struct seq_file *m, void *arg)
2834 {
2835         struct cftype *cft = seq_cft(m);
2836         struct cgroup_subsys_state *css = seq_css(m);
2837 
2838         if (cft->seq_show)
2839                 return cft->seq_show(m, arg);
2840 
2841         if (cft->read_u64)
2842                 seq_printf(m, "%llu\n", cft->read_u64(css, cft));
2843         else if (cft->read_s64)
2844                 seq_printf(m, "%lld\n", cft->read_s64(css, cft));
2845         else
2846                 return -EINVAL;
2847         return 0;
2848 }
2849 
2850 static struct kernfs_ops cgroup_kf_single_ops = {
2851         .atomic_write_len       = PAGE_SIZE,
2852         .write                  = cgroup_file_write,
2853         .seq_show               = cgroup_seqfile_show,
2854 };
2855 
2856 static struct kernfs_ops cgroup_kf_ops = {
2857         .atomic_write_len       = PAGE_SIZE,
2858         .write                  = cgroup_file_write,
2859         .seq_start              = cgroup_seqfile_start,
2860         .seq_next               = cgroup_seqfile_next,
2861         .seq_stop               = cgroup_seqfile_stop,
2862         .seq_show               = cgroup_seqfile_show,
2863 };
2864 
2865 /*
2866  * cgroup_rename - Only allow simple rename of directories in place.
2867  */
2868 static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
2869                          const char *new_name_str)
2870 {
2871         struct cgroup *cgrp = kn->priv;
2872         int ret;
2873 
2874         if (kernfs_type(kn) != KERNFS_DIR)
2875                 return -ENOTDIR;
2876         if (kn->parent != new_parent)
2877                 return -EIO;
2878 
2879         /*
2880          * This isn't a proper migration and its usefulness is very
2881          * limited.  Disallow if sane_behavior.
2882          */
2883         if (cgroup_sane_behavior(cgrp))
2884                 return -EPERM;
2885 
2886         /*
2887          * We're gonna grab cgroup_mutex which nests outside kernfs
2888          * active_ref.  kernfs_rename() doesn't require active_ref
2889          * protection.  Break them before grabbing cgroup_mutex.
2890          */
2891         kernfs_break_active_protection(new_parent);
2892         kernfs_break_active_protection(kn);
2893 
2894         mutex_lock(&cgroup_mutex);
2895 
2896         ret = kernfs_rename(kn, new_parent, new_name_str);
2897 
2898         mutex_unlock(&cgroup_mutex);
2899 
2900         kernfs_unbreak_active_protection(kn);
2901         kernfs_unbreak_active_protection(new_parent);
2902         return ret;
2903 }
2904 
2905 /* set uid and gid of cgroup dirs and files to that of the creator */
2906 static int cgroup_kn_set_ugid(struct kernfs_node *kn)
2907 {
2908         struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
2909                                .ia_uid = current_fsuid(),
2910                                .ia_gid = current_fsgid(), };
2911 
2912         if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
2913             gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
2914                 return 0;
2915 
2916         return kernfs_setattr(kn, &iattr);
2917 }
2918 
2919 static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
2920 {
2921         char name[CGROUP_FILE_NAME_MAX];
2922         struct kernfs_node *kn;
2923         struct lock_class_key *key = NULL;
2924         int ret;
2925 
2926 #ifdef CONFIG_DEBUG_LOCK_ALLOC
2927         key = &cft->lockdep_key;
2928 #endif
2929         kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
2930                                   cgroup_file_mode(cft), 0, cft->kf_ops, cft,
2931                                   NULL, false, key);
2932         if (IS_ERR(kn))
2933                 return PTR_ERR(kn);
2934 
2935         ret = cgroup_kn_set_ugid(kn);
2936         if (ret) {
2937                 kernfs_remove(kn);
2938                 return ret;
2939         }
2940 
2941         if (cft->seq_show == cgroup_populated_show)
2942                 cgrp->populated_kn = kn;
2943         return 0;
2944 }
2945 
2946 /**
2947  * cgroup_addrm_files - add or remove files to a cgroup directory
2948  * @cgrp: the target cgroup
2949  * @cfts: array of cftypes to be added
2950  * @is_add: whether to add or remove
2951  *
2952  * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
2953  * For removals, this function never fails.  If addition fails, this
2954  * function doesn't remove files already added.  The caller is responsible
2955  * for cleaning up.
2956  */
2957 static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
2958                               bool is_add)
2959 {
2960         struct cftype *cft;
2961         int ret;
2962 
2963         lockdep_assert_held(&cgroup_mutex);
2964 
2965         for (cft = cfts; cft->name[0] != '\0'; cft++) {
2966                 /* does cft->flags tell us to skip this file on @cgrp? */
2967                 if ((cft->flags & CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
2968                         continue;
2969                 if ((cft->flags & CFTYPE_INSANE) && cgroup_sane_behavior(cgrp))
2970                         continue;
2971                 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
2972                         continue;
2973                 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
2974                         continue;
2975 
2976                 if (is_add) {
2977                         ret = cgroup_add_file(cgrp, cft);
2978                         if (ret) {
2979                                 pr_warn("%s: failed to add %s, err=%d\n",
2980                                         __func__, cft->name, ret);
2981                                 return ret;
2982                         }
2983                 } else {
2984                         cgroup_rm_file(cgrp, cft);
2985                 }
2986         }
2987         return 0;
2988 }
2989 
2990 static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
2991 {
2992         LIST_HEAD(pending);
2993         struct cgroup_subsys *ss = cfts[0].ss;
2994         struct cgroup *root = &ss->root->cgrp;
2995         struct cgroup_subsys_state *css;
2996         int ret = 0;
2997 
2998         lockdep_assert_held(&cgroup_mutex);
2999 
3000         /* add/rm files for all cgroups created before */
3001         css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3002                 struct cgroup *cgrp = css->cgroup;
3003 
3004                 if (cgroup_is_dead(cgrp))
3005                         continue;
3006 
3007                 ret = cgroup_addrm_files(cgrp, cfts, is_add);
3008                 if (ret)
3009                         break;
3010         }
3011 
3012         if (is_add && !ret)
3013                 kernfs_activate(root->kn);
3014         return ret;
3015 }
3016 
3017 static void cgroup_exit_cftypes(struct cftype *cfts)
3018 {
3019         struct cftype *cft;
3020 
3021         for (cft = cfts; cft->name[0] != '\0'; cft++) {
3022                 /* free copy for custom atomic_write_len, see init_cftypes() */
3023                 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
3024                         kfree(cft->kf_ops);
3025                 cft->kf_ops = NULL;
3026                 cft->ss = NULL;
3027         }
3028 }
3029 
3030 static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3031 {
3032         struct cftype *cft;
3033 
3034         for (cft = cfts; cft->name[0] != '\0'; cft++) {
3035                 struct kernfs_ops *kf_ops;
3036 
3037                 WARN_ON(cft->ss || cft->kf_ops);
3038 
3039                 if (cft->seq_start)
3040                         kf_ops = &cgroup_kf_ops;
3041                 else
3042                         kf_ops = &cgroup_kf_single_ops;
3043 
3044                 /*
3045                  * Ugh... if @cft wants a custom max_write_len, we need to
3046                  * make a copy of kf_ops to set its atomic_write_len.
3047                  */
3048                 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
3049                         kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
3050                         if (!kf_ops) {
3051                                 cgroup_exit_cftypes(cfts);
3052                                 return -ENOMEM;
3053                         }
3054                         kf_ops->atomic_write_len = cft->max_write_len;
3055                 }
3056 
3057                 cft->kf_ops = kf_ops;
3058                 cft->ss = ss;
3059         }
3060 
3061         return 0;
3062 }
3063 
3064 static int cgroup_rm_cftypes_locked(struct cftype *cfts)
3065 {
3066         lockdep_assert_held(&cgroup_mutex);
3067 
3068         if (!cfts || !cfts[0].ss)
3069                 return -ENOENT;
3070 
3071         list_del(&cfts->node);
3072         cgroup_apply_cftypes(cfts, false);
3073         cgroup_exit_cftypes(cfts);
3074         return 0;
3075 }
3076 
3077 /**
3078  * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
3079  * @cfts: zero-length name terminated array of cftypes
3080  *
3081  * Unregister @cfts.  Files described by @cfts are removed from all
3082  * existing cgroups and all future cgroups won't have them either.  This
3083  * function can be called anytime whether @cfts' subsys is attached or not.
3084  *
3085  * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3086  * registered.
3087  */
3088 int cgroup_rm_cftypes(struct cftype *cfts)
3089 {
3090         int ret;
3091 
3092         mutex_lock(&cgroup_mutex);
3093         ret = cgroup_rm_cftypes_locked(cfts);
3094         mutex_unlock(&cgroup_mutex);
3095         return ret;
3096 }
3097 
3098 /**
3099  * cgroup_add_cftypes - add an array of cftypes to a subsystem
3100  * @ss: target cgroup subsystem
3101  * @cfts: zero-length name terminated array of cftypes
3102  *
3103  * Register @cfts to @ss.  Files described by @cfts are created for all
3104  * existing cgroups to which @ss is attached and all future cgroups will
3105  * have them too.  This function can be called anytime whether @ss is
3106  * attached or not.
3107  *
3108  * Returns 0 on successful registration, -errno on failure.  Note that this
3109  * function currently returns 0 as long as @cfts registration is successful
3110  * even if some file creation attempts on existing cgroups fail.
3111  */
3112 int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3113 {
3114         int ret;
3115 
3116         if (ss->disabled)
3117                 return 0;
3118 
3119         if (!cfts || cfts[0].name[0] == '\0')
3120                 return 0;
3121 
3122         ret = cgroup_init_cftypes(ss, cfts);
3123         if (ret)
3124                 return ret;
3125 
3126         mutex_lock(&cgroup_mutex);
3127 
3128         list_add_tail(&cfts->node, &ss->cfts);
3129         ret = cgroup_apply_cftypes(cfts, true);
3130         if (ret)
3131                 cgroup_rm_cftypes_locked(cfts);
3132 
3133         mutex_unlock(&cgroup_mutex);
3134         return ret;
3135 }
3136 
3137 /**
3138  * cgroup_task_count - count the number of tasks in a cgroup.
3139  * @cgrp: the cgroup in question
3140  *
3141  * Return the number of tasks in the cgroup.
3142  */
3143 static int cgroup_task_count(const struct cgroup *cgrp)
3144 {
3145         int count = 0;
3146         struct cgrp_cset_link *link;
3147 
3148         down_read(&css_set_rwsem);
3149         list_for_each_entry(link, &cgrp->cset_links, cset_link)
3150                 count += atomic_read(&link->cset->refcount);
3151         up_read(&css_set_rwsem);
3152         return count;
3153 }
3154 
3155 /**
3156  * css_next_child - find the next child of a given css
3157  * @pos: the current position (%NULL to initiate traversal)
3158  * @parent: css whose children to walk
3159  *
3160  * This function returns the next child of @parent and should be called
3161  * under either cgroup_mutex or RCU read lock.  The only requirement is
3162  * that @parent and @pos are accessible.  The next sibling is guaranteed to
3163  * be returned regardless of their states.
3164  *
3165  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3166  * css which finished ->css_online() is guaranteed to be visible in the
3167  * future iterations and will stay visible until the last reference is put.
3168  * A css which hasn't finished ->css_online() or already finished
3169  * ->css_offline() may show up during traversal.  It's each subsystem's
3170  * responsibility to synchronize against on/offlining.
3171  */
3172 struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
3173                                            struct cgroup_subsys_state *parent)
3174 {
3175         struct cgroup_subsys_state *next;
3176 
3177         cgroup_assert_mutex_or_rcu_locked();
3178 
3179         /*
3180          * @pos could already have been unlinked from the sibling list.
3181          * Once a cgroup is removed, its ->sibling.next is no longer
3182          * updated when its next sibling changes.  CSS_RELEASED is set when
3183          * @pos is taken off list, at which time its next pointer is valid,
3184          * and, as releases are serialized, the one pointed to by the next
3185          * pointer is guaranteed to not have started release yet.  This
3186          * implies that if we observe !CSS_RELEASED on @pos in this RCU
3187          * critical section, the one pointed to by its next pointer is
3188          * guaranteed to not have finished its RCU grace period even if we
3189          * have dropped rcu_read_lock() inbetween iterations.
3190          *
3191          * If @pos has CSS_RELEASED set, its next pointer can't be
3192          * dereferenced; however, as each css is given a monotonically
3193          * increasing unique serial number and always appended to the
3194          * sibling list, the next one can be found by walking the parent's
3195          * children until the first css with higher serial number than
3196          * @pos's.  While this path can be slower, it happens iff iteration
3197          * races against release and the race window is very small.
3198          */
3199         if (!pos) {
3200                 next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
3201         } else if (likely(!(pos->flags & CSS_RELEASED))) {
3202                 next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3203         } else {
3204                 list_for_each_entry_rcu(next, &parent->children, sibling)
3205                         if (next->serial_nr > pos->serial_nr)
3206                                 break;
3207         }
3208 
3209         /*
3210          * @next, if not pointing to the head, can be dereferenced and is
3211          * the next sibling.
3212          */
3213         if (&next->sibling != &parent->children)
3214                 return next;
3215         return NULL;
3216 }
3217 
3218 /**
3219  * css_next_descendant_pre - find the next descendant for pre-order walk
3220  * @pos: the current position (%NULL to initiate traversal)
3221  * @root: css whose descendants to walk
3222  *
3223  * To be used by css_for_each_descendant_pre().  Find the next descendant
3224  * to visit for pre-order traversal of @root's descendants.  @root is
3225  * included in the iteration and the first node to be visited.
3226  *
3227  * While this function requires cgroup_mutex or RCU read locking, it
3228  * doesn't require the whole traversal to be contained in a single critical
3229  * section.  This function will return the correct next descendant as long
3230  * as both @pos and @root are accessible and @pos is a descendant of @root.
3231  *
3232  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3233  * css which finished ->css_online() is guaranteed to be visible in the
3234  * future iterations and will stay visible until the last reference is put.
3235  * A css which hasn't finished ->css_online() or already finished
3236  * ->css_offline() may show up during traversal.  It's each subsystem's
3237  * responsibility to synchronize against on/offlining.
3238  */
3239 struct cgroup_subsys_state *
3240 css_next_descendant_pre(struct cgroup_subsys_state *pos,
3241                         struct cgroup_subsys_state *root)
3242 {
3243         struct cgroup_subsys_state *next;
3244 
3245         cgroup_assert_mutex_or_rcu_locked();
3246 
3247         /* if first iteration, visit @root */
3248         if (!pos)
3249                 return root;
3250 
3251         /* visit the first child if exists */
3252         next = css_next_child(NULL, pos);
3253         if (next)
3254                 return next;
3255 
3256         /* no child, visit my or the closest ancestor's next sibling */
3257         while (pos != root) {
3258                 next = css_next_child(pos, pos->parent);
3259                 if (next)
3260                         return next;
3261                 pos = pos->parent;
3262         }
3263 
3264         return NULL;
3265 }
3266 
3267 /**
3268  * css_rightmost_descendant - return the rightmost descendant of a css
3269  * @pos: css of interest
3270  *
3271  * Return the rightmost descendant of @pos.  If there's no descendant, @pos
3272  * is returned.  This can be used during pre-order traversal to skip
3273  * subtree of @pos.
3274  *
3275  * While this function requires cgroup_mutex or RCU read locking, it
3276  * doesn't require the whole traversal to be contained in a single critical
3277  * section.  This function will return the correct rightmost descendant as
3278  * long as @pos is accessible.
3279  */
3280 struct cgroup_subsys_state *
3281 css_rightmost_descendant(struct cgroup_subsys_state *pos)
3282 {
3283         struct cgroup_subsys_state *last, *tmp;
3284 
3285         cgroup_assert_mutex_or_rcu_locked();
3286 
3287         do {
3288                 last = pos;
3289                 /* ->prev isn't RCU safe, walk ->next till the end */
3290                 pos = NULL;
3291                 css_for_each_child(tmp, last)
3292                         pos = tmp;
3293         } while (pos);
3294 
3295         return last;
3296 }
3297 
3298 static struct cgroup_subsys_state *
3299 css_leftmost_descendant(struct cgroup_subsys_state *pos)
3300 {
3301         struct cgroup_subsys_state *last;
3302 
3303         do {
3304                 last = pos;
3305                 pos = css_next_child(NULL, pos);
3306         } while (pos);
3307 
3308         return last;
3309 }
3310 
3311 /**
3312  * css_next_descendant_post - find the next descendant for post-order walk
3313  * @pos: the current position (%NULL to initiate traversal)
3314  * @root: css whose descendants to walk
3315  *
3316  * To be used by css_for_each_descendant_post().  Find the next descendant
3317  * to visit for post-order traversal of @root's descendants.  @root is
3318  * included in the iteration and the last node to be visited.
3319  *
3320  * While this function requires cgroup_mutex or RCU read locking, it
3321  * doesn't require the whole traversal to be contained in a single critical
3322  * section.  This function will return the correct next descendant as long
3323  * as both @pos and @cgroup are accessible and @pos is a descendant of
3324  * @cgroup.
3325  *
3326  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3327  * css which finished ->css_online() is guaranteed to be visible in the
3328  * future iterations and will stay visible until the last reference is put.
3329  * A css which hasn't finished ->css_online() or already finished
3330  * ->css_offline() may show up during traversal.  It's each subsystem's
3331  * responsibility to synchronize against on/offlining.
3332  */
3333 struct cgroup_subsys_state *
3334 css_next_descendant_post(struct cgroup_subsys_state *pos,
3335                          struct cgroup_subsys_state *root)
3336 {
3337         struct cgroup_subsys_state *next;
3338 
3339         cgroup_assert_mutex_or_rcu_locked();
3340 
3341         /* if first iteration, visit leftmost descendant which may be @root */
3342         if (!pos)
3343                 return css_leftmost_descendant(root);
3344 
3345         /* if we visited @root, we're done */
3346         if (pos == root)
3347                 return NULL;
3348 
3349         /* if there's an unvisited sibling, visit its leftmost descendant */
3350         next = css_next_child(pos, pos->parent);
3351         if (next)
3352                 return css_leftmost_descendant(next);
3353 
3354         /* no sibling left, visit parent */
3355         return pos->parent;
3356 }
3357 
3358 /**
3359  * css_has_online_children - does a css have online children
3360  * @css: the target css
3361  *
3362  * Returns %true if @css has any online children; otherwise, %false.  This
3363  * function can be called from any context but the caller is responsible
3364  * for synchronizing against on/offlining as necessary.
3365  */
3366 bool css_has_online_children(struct cgroup_subsys_state *css)
3367 {
3368         struct cgroup_subsys_state *child;
3369         bool ret = false;
3370 
3371         rcu_read_lock();
3372         css_for_each_child(child, css) {
3373                 if (child->flags & CSS_ONLINE) {
3374                         ret = true;
3375                         break;
3376                 }
3377         }
3378         rcu_read_unlock();
3379         return ret;
3380 }
3381 
3382 /**
3383  * css_advance_task_iter - advance a task itererator to the next css_set
3384  * @it: the iterator to advance
3385  *
3386  * Advance @it to the next css_set to walk.
3387  */
3388 static void css_advance_task_iter(struct css_task_iter *it)
3389 {
3390         struct list_head *l = it->cset_pos;
3391         struct cgrp_cset_link *link;
3392         struct css_set *cset;
3393 
3394         /* Advance to the next non-empty css_set */
3395         do {
3396                 l = l->next;
3397                 if (l == it->cset_head) {
3398                         it->cset_pos = NULL;
3399                         return;
3400                 }
3401 
3402                 if (it->ss) {
3403                         cset = container_of(l, struct css_set,
3404                                             e_cset_node[it->ss->id]);
3405                 } else {
3406                         link = list_entry(l, struct cgrp_cset_link, cset_link);
3407                         cset = link->cset;
3408                 }
3409         } while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks));
3410 
3411         it->cset_pos = l;
3412 
3413         if (!list_empty(&cset->tasks))
3414                 it->task_pos = cset->tasks.next;
3415         else
3416                 it->task_pos = cset->mg_tasks.next;
3417 
3418         it->tasks_head = &cset->tasks;
3419         it->mg_tasks_head = &cset->mg_tasks;
3420 }
3421 
3422 /**
3423  * css_task_iter_start - initiate task iteration
3424  * @css: the css to walk tasks of
3425  * @it: the task iterator to use
3426  *
3427  * Initiate iteration through the tasks of @css.  The caller can call
3428  * css_task_iter_next() to walk through the tasks until the function
3429  * returns NULL.  On completion of iteration, css_task_iter_end() must be
3430  * called.
3431  *
3432  * Note that this function acquires a lock which is released when the
3433  * iteration finishes.  The caller can't sleep while iteration is in
3434  * progress.
3435  */
3436 void css_task_iter_start(struct cgroup_subsys_state *css,
3437                          struct css_task_iter *it)
3438         __acquires(css_set_rwsem)
3439 {
3440         /* no one should try to iterate before mounting cgroups */
3441         WARN_ON_ONCE(!use_task_css_set_links);
3442 
3443         down_read(&css_set_rwsem);
3444 
3445         it->ss = css->ss;
3446 
3447         if (it->ss)
3448                 it->cset_pos = &css->cgroup->e_csets[css->ss->id];
3449         else
3450                 it->cset_pos = &css->cgroup->cset_links;
3451 
3452         it->cset_head = it->cset_pos;
3453 
3454         css_advance_task_iter(it);
3455 }
3456 
3457 /**
3458  * css_task_iter_next - return the next task for the iterator
3459  * @it: the task iterator being iterated
3460  *
3461  * The "next" function for task iteration.  @it should have been
3462  * initialized via css_task_iter_start().  Returns NULL when the iteration
3463  * reaches the end.
3464  */
3465 struct task_struct *css_task_iter_next(struct css_task_iter *it)
3466 {
3467         struct task_struct *res;
3468         struct list_head *l = it->task_pos;
3469 
3470         /* If the iterator cg is NULL, we have no tasks */
3471         if (!it->cset_pos)
3472                 return NULL;
3473         res = list_entry(l, struct task_struct, cg_list);
3474 
3475         /*
3476          * Advance iterator to find next entry.  cset->tasks is consumed
3477          * first and then ->mg_tasks.  After ->mg_tasks, we move onto the
3478          * next cset.
3479          */
3480         l = l->next;
3481 
3482         if (l == it->tasks_head)
3483                 l = it->mg_tasks_head->next;
3484 
3485         if (l == it->mg_tasks_head)
3486                 css_advance_task_iter(it);
3487         else
3488                 it->task_pos = l;
3489 
3490         return res;
3491 }
3492 
3493 /**
3494  * css_task_iter_end - finish task iteration
3495  * @it: the task iterator to finish
3496  *
3497  * Finish task iteration started by css_task_iter_start().
3498  */
3499 void css_task_iter_end(struct css_task_iter *it)
3500         __releases(css_set_rwsem)
3501 {
3502         up_read(&css_set_rwsem);
3503 }
3504 
3505 /**
3506  * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
3507  * @to: cgroup to which the tasks will be moved
3508  * @from: cgroup in which the tasks currently reside
3509  *
3510  * Locking rules between cgroup_post_fork() and the migration path
3511  * guarantee that, if a task is forking while being migrated, the new child
3512  * is guaranteed to be either visible in the source cgroup after the
3513  * parent's migration is complete or put into the target cgroup.  No task
3514  * can slip out of migration through forking.
3515  */
3516 int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
3517 {
3518         LIST_HEAD(preloaded_csets);
3519         struct cgrp_cset_link *link;
3520         struct css_task_iter it;
3521         struct task_struct *task;
3522         int ret;
3523 
3524         mutex_lock(&cgroup_mutex);
3525 
3526         /* all tasks in @from are being moved, all csets are source */
3527         down_read(&css_set_rwsem);
3528         list_for_each_entry(link, &from->cset_links, cset_link)
3529                 cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
3530         up_read(&css_set_rwsem);
3531 
3532         ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
3533         if (ret)
3534                 goto out_err;
3535 
3536         /*
3537          * Migrate tasks one-by-one until @form is empty.  This fails iff
3538          * ->can_attach() fails.
3539          */
3540         do {
3541                 css_task_iter_start(&from->self, &it);
3542                 task = css_task_iter_next(&it);
3543                 if (task)
3544                         get_task_struct(task);
3545                 css_task_iter_end(&it);
3546 
3547                 if (task) {
3548                         ret = cgroup_migrate(to, task, false);
3549                         put_task_struct(task);
3550                 }
3551         } while (task && !ret);
3552 out_err:
3553         cgroup_migrate_finish(&preloaded_csets);
3554         mutex_unlock(&cgroup_mutex);
3555         return ret;
3556 }
3557 
3558 /*
3559  * Stuff for reading the 'tasks'/'procs' files.
3560  *
3561  * Reading this file can return large amounts of data if a cgroup has
3562  * *lots* of attached tasks. So it may need several calls to read(),
3563  * but we cannot guarantee that the information we produce is correct
3564  * unless we produce it entirely atomically.
3565  *
3566  */
3567 
3568 /* which pidlist file are we talking about? */
3569 enum cgroup_filetype {
3570         CGROUP_FILE_PROCS,
3571         CGROUP_FILE_TASKS,
3572 };
3573 
3574 /*
3575  * A pidlist is a list of pids that virtually represents the contents of one
3576  * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
3577  * a pair (one each for procs, tasks) for each pid namespace that's relevant
3578  * to the cgroup.
3579  */
3580 struct cgroup_pidlist {
3581         /*
3582          * used to find which pidlist is wanted. doesn't change as long as
3583          * this particular list stays in the list.
3584         */
3585         struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
3586         /* array of xids */
3587         pid_t *list;
3588         /* how many elements the above list has */
3589         int length;
3590         /* each of these stored in a list by its cgroup */
3591         struct list_head links;
3592         /* pointer to the cgroup we belong to, for list removal purposes */
3593         struct cgroup *owner;
3594         /* for delayed destruction */
3595         struct delayed_work destroy_dwork;
3596 };
3597 
3598 /*
3599  * The following two functions "fix" the issue where there are more pids
3600  * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
3601  * TODO: replace with a kernel-wide solution to this problem
3602  */
3603 #define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
3604 static void *pidlist_allocate(int count)
3605 {
3606         if (PIDLIST_TOO_LARGE(count))
3607                 return vmalloc(count * sizeof(pid_t));
3608         else
3609                 return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
3610 }
3611 
3612 static void pidlist_free(void *p)
3613 {
3614         if (is_vmalloc_addr(p))
3615                 vfree(p);
3616         else
3617                 kfree(p);
3618 }
3619 
3620 /*
3621  * Used to destroy all pidlists lingering waiting for destroy timer.  None
3622  * should be left afterwards.
3623  */
3624 static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
3625 {
3626         struct cgroup_pidlist *l, *tmp_l;
3627 
3628         mutex_lock(&cgrp->pidlist_mutex);
3629         list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
3630                 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
3631         mutex_unlock(&cgrp->pidlist_mutex);
3632 
3633         flush_workqueue(cgroup_pidlist_destroy_wq);
3634         BUG_ON(!list_empty(&cgrp->pidlists));
3635 }
3636 
3637 static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
3638 {
3639         struct delayed_work *dwork = to_delayed_work(work);
3640         struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
3641                                                 destroy_dwork);
3642         struct cgroup_pidlist *tofree = NULL;
3643 
3644         mutex_lock(&l->owner->pidlist_mutex);
3645 
3646         /*
3647          * Destroy iff we didn't get queued again.  The state won't change
3648          * as destroy_dwork can only be queued while locked.
3649          */
3650         if (!delayed_work_pending(dwork)) {
3651                 list_del(&l->links);
3652                 pidlist_free(l->list);
3653                 put_pid_ns(l->key.ns);
3654                 tofree = l;
3655         }
3656 
3657         mutex_unlock(&l->owner->pidlist_mutex);
3658         kfree(tofree);
3659 }
3660 
3661 /*
3662  * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
3663  * Returns the number of unique elements.
3664  */
3665 static int pidlist_uniq(pid_t *list, int length)
3666 {
3667         int src, dest = 1;
3668 
3669         /*
3670          * we presume the 0th element is unique, so i starts at 1. trivial
3671          * edge cases first; no work needs to be done for either
3672          */
3673         if (length == 0 || length == 1)
3674                 return length;
3675         /* src and dest walk down the list; dest counts unique elements */
3676         for (src = 1; src < length; src++) {
3677                 /* find next unique element */
3678                 while (list[src] == list[src-1]) {
3679                         src++;
3680                         if (src == length)
3681                                 goto after;
3682                 }
3683                 /* dest always points to where the next unique element goes */
3684                 list[dest] = list[src];
3685                 dest++;
3686         }
3687 after:
3688         return dest;
3689 }
3690 
3691 /*
3692  * The two pid files - task and cgroup.procs - guaranteed that the result
3693  * is sorted, which forced this whole pidlist fiasco.  As pid order is
3694  * different per namespace, each namespace needs differently sorted list,
3695  * making it impossible to use, for example, single rbtree of member tasks
3696  * sorted by task pointer.  As pidlists can be fairly large, allocating one
3697  * per open file is dangerous, so cgroup had to implement shared pool of
3698  * pidlists keyed by cgroup and namespace.
3699  *
3700  * All this extra complexity was caused by the original implementation
3701  * committing to an entirely unnecessary property.  In the long term, we
3702  * want to do away with it.  Explicitly scramble sort order if
3703  * sane_behavior so that no such expectation exists in the new interface.
3704  *
3705  * Scrambling is done by swapping every two consecutive bits, which is
3706  * non-identity one-to-one mapping which disturbs sort order sufficiently.
3707  */
3708 static pid_t pid_fry(pid_t pid)
3709 {
3710         unsigned a = pid & 0x55555555;
3711         unsigned b = pid & 0xAAAAAAAA;
3712 
3713         return (a << 1) | (b >> 1);
3714 }
3715 
3716 static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
3717 {
3718         if (cgroup_sane_behavior(cgrp))
3719                 return pid_fry(pid);
3720         else
3721                 return pid;
3722 }
3723 
3724 static int cmppid(const void *a, const void *b)
3725 {
3726         return *(pid_t *)a - *(pid_t *)b;
3727 }
3728 
3729 static int fried_cmppid(const void *a, const void *b)
3730 {
3731         return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
3732 }
3733 
3734 static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
3735                                                   enum cgroup_filetype type)
3736 {
3737         struct cgroup_pidlist *l;
3738         /* don't need task_nsproxy() if we're looking at ourself */
3739         struct pid_namespace *ns = task_active_pid_ns(current);
3740 
3741         lockdep_assert_held(&cgrp->pidlist_mutex);
3742 
3743         list_for_each_entry(l, &cgrp->pidlists, links)
3744                 if (l->key.type == type && l->key.ns == ns)
3745                         return l;
3746         return NULL;
3747 }
3748 
3749 /*
3750  * find the appropriate pidlist for our purpose (given procs vs tasks)
3751  * returns with the lock on that pidlist already held, and takes care
3752  * of the use count, or returns NULL with no locks held if we're out of
3753  * memory.
3754  */
3755 static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
3756                                                 enum cgroup_filetype type)
3757 {
3758         struct cgroup_pidlist *l;
3759 
3760         lockdep_assert_held(&cgrp->pidlist_mutex);
3761 
3762         l = cgroup_pidlist_find(cgrp, type);
3763         if (l)
3764                 return l;
3765 
3766         /* entry not found; create a new one */
3767         l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
3768         if (!l)
3769                 return l;
3770 
3771         INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
3772         l->key.type = type;
3773         /* don't need task_nsproxy() if we're looking at ourself */
3774         l->key.ns = get_pid_ns(task_active_pid_ns(current));
3775         l->owner = cgrp;
3776         list_add(&l->links, &cgrp->pidlists);
3777         return l;
3778 }
3779 
3780 /*
3781  * Load a cgroup's pidarray with either procs' tgids or tasks' pids
3782  */
3783 static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
3784                               struct cgroup_pidlist **lp)
3785 {
3786         pid_t *array;
3787         int length;
3788         int pid, n = 0; /* used for populating the array */
3789         struct css_task_iter it;
3790         struct task_struct *tsk;
3791         struct cgroup_pidlist *l;
3792 
3793         lockdep_assert_held(&cgrp->pidlist_mutex);
3794 
3795         /*
3796          * If cgroup gets more users after we read count, we won't have
3797          * enough space - tough.  This race is indistinguishable to the
3798          * caller from the case that the additional cgroup users didn't
3799          * show up until sometime later on.
3800          */
3801         length = cgroup_task_count(cgrp);
3802         array = pidlist_allocate(length);
3803         if (!array)
3804                 return -ENOMEM;
3805         /* now, populate the array */
3806         css_task_iter_start(&cgrp->self, &it);
3807         while ((tsk = css_task_iter_next(&it))) {
3808                 if (unlikely(n == length))
3809                         break;
3810                 /* get tgid or pid for procs or tasks file respectively */
3811                 if (type == CGROUP_FILE_PROCS)
3812                         pid = task_tgid_vnr(tsk);
3813                 else
3814                         pid = task_pid_vnr(tsk);
3815                 if (pid > 0) /* make sure to only use valid results */
3816                         array[n++] = pid;
3817         }
3818         css_task_iter_end(&it);
3819         length = n;
3820         /* now sort & (if procs) strip out duplicates */
3821         if (cgroup_sane_behavior(cgrp))
3822                 sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
3823         else
3824                 sort(array, length, sizeof(pid_t), cmppid, NULL);
3825         if (type == CGROUP_FILE_PROCS)
3826                 length = pidlist_uniq(array, length);
3827 
3828         l = cgroup_pidlist_find_create(cgrp, type);
3829         if (!l) {
3830                 mutex_unlock(&cgrp->pidlist_mutex);
3831                 pidlist_free(array);
3832                 return -ENOMEM;
3833         }
3834 
3835         /* store array, freeing old if necessary */
3836         pidlist_free(l->list);
3837         l->list = array;
3838         l->length = length;
3839         *lp = l;
3840         return 0;
3841 }
3842 
3843 /**
3844  * cgroupstats_build - build and fill cgroupstats
3845  * @stats: cgroupstats to fill information into
3846  * @dentry: A dentry entry belonging to the cgroup for which stats have
3847  * been requested.
3848  *
3849  * Build and fill cgroupstats so that taskstats can export it to user
3850  * space.
3851  */
3852 int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
3853 {
3854         struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
3855         struct cgroup *cgrp;
3856         struct css_task_iter it;
3857         struct task_struct *tsk;
3858 
3859         /* it should be kernfs_node belonging to cgroupfs and is a directory */
3860         if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
3861             kernfs_type(kn) != KERNFS_DIR)
3862                 return -EINVAL;
3863 
3864         mutex_lock(&cgroup_mutex);
3865 
3866         /*
3867          * We aren't being called from kernfs and there's no guarantee on
3868          * @kn->priv's validity.  For this and css_tryget_online_from_dir(),
3869          * @kn->priv is RCU safe.  Let's do the RCU dancing.
3870          */
3871         rcu_read_lock();
3872         cgrp = rcu_dereference(kn->priv);
3873         if (!cgrp || cgroup_is_dead(cgrp)) {
3874                 rcu_read_unlock();
3875                 mutex_unlock(&cgroup_mutex);
3876                 return -ENOENT;
3877         }
3878         rcu_read_unlock();
3879 
3880         css_task_iter_start(&cgrp->self, &it);
3881         while ((tsk = css_task_iter_next(&it))) {
3882                 switch (tsk->state) {
3883                 case TASK_RUNNING:
3884                         stats->nr_running++;
3885                         break;
3886                 case TASK_INTERRUPTIBLE:
3887                         stats->nr_sleeping++;
3888                         break;
3889                 case TASK_UNINTERRUPTIBLE:
3890                         stats->nr_uninterruptible++;
3891                         break;
3892                 case TASK_STOPPED:
3893                         stats->nr_stopped++;
3894                         break;
3895                 default:
3896                         if (delayacct_is_task_waiting_on_io(tsk))
3897                                 stats->nr_io_wait++;
3898                         break;
3899                 }
3900         }
3901         css_task_iter_end(&it);
3902 
3903         mutex_unlock(&cgroup_mutex);
3904         return 0;
3905 }
3906 
3907 
3908 /*
3909  * seq_file methods for the tasks/procs files. The seq_file position is the
3910  * next pid to display; the seq_file iterator is a pointer to the pid
3911  * in the cgroup->l->list array.
3912  */
3913 
3914 static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
3915 {
3916         /*
3917          * Initially we receive a position value that corresponds to
3918          * one more than the last pid shown (or 0 on the first call or
3919          * after a seek to the start). Use a binary-search to find the
3920          * next pid to display, if any
3921          */
3922         struct kernfs_open_file *of = s->private;
3923         struct cgroup *cgrp = seq_css(s)->cgroup;
3924         struct cgroup_pidlist *l;
3925         enum cgroup_filetype type = seq_cft(s)->private;
3926         int index = 0, pid = *pos;
3927         int *iter, ret;
3928 
3929         mutex_lock(&cgrp->pidlist_mutex);
3930 
3931         /*
3932          * !NULL @of->priv indicates that this isn't the first start()
3933          * after open.  If the matching pidlist is around, we can use that.
3934          * Look for it.  Note that @of->priv can't be used directly.  It
3935          * could already have been destroyed.
3936          */
3937         if (of->priv)
3938                 of->priv = cgroup_pidlist_find(cgrp, type);
3939 
3940         /*
3941          * Either this is the first start() after open or the matching
3942          * pidlist has been destroyed inbetween.  Create a new one.
3943          */
3944         if (!of->priv) {
3945                 ret = pidlist_array_load(cgrp, type,
3946                                          (struct cgroup_pidlist **)&of->priv);
3947                 if (ret)
3948                         return ERR_PTR(ret);
3949         }
3950         l = of->priv;
3951 
3952         if (pid) {
3953                 int end = l->length;
3954 
3955                 while (index < end) {
3956                         int mid = (index + end) / 2;
3957                         if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
3958                                 index = mid;
3959                                 break;
3960                         } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
3961                                 index = mid + 1;
3962                         else
3963                                 end = mid;
3964                 }
3965         }
3966         /* If we're off the end of the array, we're done */
3967         if (index >= l->length)
3968                 return NULL;
3969         /* Update the abstract position to be the actual pid that we found */
3970         iter = l->list + index;
3971         *pos = cgroup_pid_fry(cgrp, *iter);
3972         return iter;
3973 }
3974 
3975 static void cgroup_pidlist_stop(struct seq_file *s, void *v)
3976 {
3977         struct kernfs_open_file *of = s->private;
3978         struct cgroup_pidlist *l = of->priv;
3979 
3980         if (l)
3981                 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
3982                                  CGROUP_PIDLIST_DESTROY_DELAY);
3983         mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
3984 }
3985 
3986 static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
3987 {
3988         struct kernfs_open_file *of = s->private;
3989         struct cgroup_pidlist *l = of->priv;
3990         pid_t *p = v;
3991         pid_t *end = l->list + l->length;
3992         /*
3993          * Advance to the next pid in the array. If this goes off the
3994          * end, we're done
3995          */
3996         p++;
3997         if (p >= end) {
3998                 return NULL;
3999         } else {
4000                 *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
4001                 return p;
4002         }
4003 }
4004 
4005 static int cgroup_pidlist_show(struct seq_file *s, void *v)
4006 {
4007         return seq_printf(s, "%d\n", *(int *)v);
4008 }
4009 
4010 static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
4011                                          struct cftype *cft)
4012 {
4013         return notify_on_release(css->cgroup);
4014 }
4015 
4016 static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
4017                                           struct cftype *cft, u64 val)
4018 {
4019         clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
4020         if (val)
4021                 set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4022         else
4023                 clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4024         return 0;
4025 }
4026 
4027 static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
4028                                       struct cftype *cft)
4029 {
4030         return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4031 }
4032 
4033 static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
4034                                        struct cftype *cft, u64 val)
4035 {
4036         if (val)
4037                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4038         else
4039                 clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4040         return 0;
4041 }
4042 
4043 static struct cftype cgroup_base_files[] = {
4044         {
4045                 .name = "cgroup.procs",
4046                 .seq_start = cgroup_pidlist_start,
4047                 .seq_next = cgroup_pidlist_next,
4048                 .seq_stop = cgroup_pidlist_stop,
4049                 .seq_show = cgroup_pidlist_show,
4050                 .private = CGROUP_FILE_PROCS,
4051                 .write = cgroup_procs_write,
4052                 .mode = S_IRUGO | S_IWUSR,
4053         },
4054         {
4055                 .name = "cgroup.clone_children",
4056                 .flags = CFTYPE_INSANE,
4057                 .read_u64 = cgroup_clone_children_read,
4058                 .write_u64 = cgroup_clone_children_write,
4059         },
4060         {
4061                 .name = "cgroup.sane_behavior",
4062                 .flags = CFTYPE_ONLY_ON_ROOT,
4063                 .seq_show = cgroup_sane_behavior_show,
4064         },
4065         {
4066                 .name = "cgroup.controllers",
4067                 .flags = CFTYPE_ONLY_ON_DFL | CFTYPE_ONLY_ON_ROOT,
4068                 .seq_show = cgroup_root_controllers_show,
4069         },
4070         {
4071                 .name = "cgroup.controllers",
4072                 .flags = CFTYPE_ONLY_ON_DFL | CFTYPE_NOT_ON_ROOT,
4073                 .seq_show = cgroup_controllers_show,
4074         },
4075         {
4076                 .name = "cgroup.subtree_control",
4077                 .flags = CFTYPE_ONLY_ON_DFL,
4078                 .seq_show = cgroup_subtree_control_show,
4079                 .write = cgroup_subtree_control_write,
4080         },
4081         {
4082                 .name = "cgroup.populated",
4083                 .flags = CFTYPE_ONLY_ON_DFL | CFTYPE_NOT_ON_ROOT,
4084                 .seq_show = cgroup_populated_show,
4085         },
4086 
4087         /*
4088          * Historical crazy stuff.  These don't have "cgroup."  prefix and
4089          * don't exist if sane_behavior.  If you're depending on these, be
4090          * prepared to be burned.
4091          */
4092         {
4093                 .name = "tasks",
4094                 .flags = CFTYPE_INSANE,         /* use "procs" instead */
4095                 .seq_start = cgroup_pidlist_start,
4096                 .seq_next = cgroup_pidlist_next,
4097                 .seq_stop = cgroup_pidlist_stop,
4098                 .seq_show = cgroup_pidlist_show,
4099                 .private = CGROUP_FILE_TASKS,
4100                 .write = cgroup_tasks_write,
4101                 .mode = S_IRUGO | S_IWUSR,
4102         },
4103         {
4104                 .name = "notify_on_release",
4105                 .flags = CFTYPE_INSANE,
4106                 .read_u64 = cgroup_read_notify_on_release,
4107                 .write_u64 = cgroup_write_notify_on_release,
4108         },
4109         {
4110                 .name = "release_agent",
4111                 .flags = CFTYPE_INSANE | CFTYPE_ONLY_ON_ROOT,
4112                 .seq_show = cgroup_release_agent_show,
4113                 .write = cgroup_release_agent_write,
4114                 .max_write_len = PATH_MAX - 1,
4115         },
4116         { }     /* terminate */
4117 };
4118 
4119 /**
4120  * cgroup_populate_dir - create subsys files in a cgroup directory
4121  * @cgrp: target cgroup
4122  * @subsys_mask: mask of the subsystem ids whose files should be added
4123  *
4124  * On failure, no file is added.
4125  */
4126 static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask)
4127 {
4128         struct cgroup_subsys *ss;
4129         int i, ret = 0;
4130 
4131         /* process cftsets of each subsystem */
4132         for_each_subsys(ss, i) {
4133                 struct cftype *cfts;
4134 
4135                 if (!(subsys_mask & (1 << i)))
4136                         continue;
4137 
4138                 list_for_each_entry(cfts, &ss->cfts, node) {
4139                         ret = cgroup_addrm_files(cgrp, cfts, true);
4140                         if (ret < 0)
4141                                 goto err;
4142                 }
4143         }
4144         return 0;
4145 err:
4146         cgroup_clear_dir(cgrp, subsys_mask);
4147         return ret;
4148 }
4149 
4150 /*
4151  * css destruction is four-stage process.
4152  *
4153  * 1. Destruction starts.  Killing of the percpu_ref is initiated.
4154  *    Implemented in kill_css().
4155  *
4156  * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4157  *    and thus css_tryget_online() is guaranteed to fail, the css can be
4158  *    offlined by invoking offline_css().  After offlining, the base ref is
4159  *    put.  Implemented in css_killed_work_fn().
4160  *
4161  * 3. When the percpu_ref reaches zero, the only possible remaining
4162  *    accessors are inside RCU read sections.  css_release() schedules the
4163  *    RCU callback.
4164  *
4165  * 4. After the grace period, the css can be freed.  Implemented in
4166  *    css_free_work_fn().
4167  *
4168  * It is actually hairier because both step 2 and 4 require process context
4169  * and thus involve punting to css->destroy_work adding two additional
4170  * steps to the already complex sequence.
4171  */
4172 static void css_free_work_fn(struct work_struct *work)
4173 {
4174         struct cgroup_subsys_state *css =
4175                 container_of(work, struct cgroup_subsys_state, destroy_work);
4176         struct cgroup *cgrp = css->cgroup;
4177 
4178         if (css->ss) {
4179                 /* css free path */
4180                 if (css->parent)
4181                         css_put(css->parent);
4182 
4183                 css->ss->css_free(css);
4184                 cgroup_put(cgrp);
4185         } else {
4186                 /* cgroup free path */
4187                 atomic_dec(&cgrp->root->nr_cgrps);
4188                 cgroup_pidlist_destroy_all(cgrp);
4189 
4190                 if (cgroup_parent(cgrp)) {
4191                         /*
4192                          * We get a ref to the parent, and put the ref when
4193                          * this cgroup is being freed, so it's guaranteed
4194                          * that the parent won't be destroyed before its
4195                          * children.
4196                          */
4197                         cgroup_put(cgroup_parent(cgrp));
4198                         kernfs_put(cgrp->kn);
4199                         kfree(cgrp);
4200                 } else {
4201                         /*
4202                          * This is root cgroup's refcnt reaching zero,
4203                          * which indicates that the root should be
4204                          * released.
4205                          */
4206                         cgroup_destroy_root(cgrp->root);
4207                 }
4208         }
4209 }
4210 
4211 static void css_free_rcu_fn(struct rcu_head *rcu_head)
4212 {
4213         struct cgroup_subsys_state *css =
4214                 container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4215 
4216         INIT_WORK(&css->destroy_work, css_free_work_fn);
4217         queue_work(cgroup_destroy_wq, &css->destroy_work);
4218 }
4219 
4220 static void css_release_work_fn(struct work_struct *work)
4221 {
4222         struct cgroup_subsys_state *css =
4223                 container_of(work, struct cgroup_subsys_state, destroy_work);
4224         struct cgroup_subsys *ss = css->ss;
4225         struct cgroup *cgrp = css->cgroup;
4226 
4227         mutex_lock(&cgroup_mutex);
4228 
4229         css->flags |= CSS_RELEASED;
4230         list_del_rcu(&css->sibling);
4231 
4232         if (ss) {
4233                 /* css release path */
4234                 cgroup_idr_remove(&ss->css_idr, css->id);
4235         } else {
4236                 /* cgroup release path */
4237                 cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
4238                 cgrp->id = -1;
4239         }
4240 
4241         mutex_unlock(&cgroup_mutex);
4242 
4243         call_rcu(&css->rcu_head, css_free_rcu_fn);
4244 }
4245 
4246 static void css_release(struct percpu_ref *ref)
4247 {
4248         struct cgroup_subsys_state *css =
4249                 container_of(ref, struct cgroup_subsys_state, refcnt);
4250 
4251         INIT_WORK(&css->destroy_work, css_release_work_fn);
4252         queue_work(cgroup_destroy_wq, &css->destroy_work);
4253 }
4254 
4255 static void init_and_link_css(struct cgroup_subsys_state *css,
4256                               struct cgroup_subsys *ss, struct cgroup *cgrp)
4257 {
4258         lockdep_assert_held(&cgroup_mutex);
4259 
4260         cgroup_get(cgrp);
4261 
4262         memset(css, 0, sizeof(*css));
4263         css->cgroup = cgrp;
4264         css->ss = ss;
4265         INIT_LIST_HEAD(&css->sibling);
4266         INIT_LIST_HEAD(&css->children);
4267         css->serial_nr = css_serial_nr_next++;
4268 
4269         if (cgroup_parent(cgrp)) {
4270                 css->parent = cgroup_css(cgroup_parent(cgrp), ss);
4271                 css_get(css->parent);
4272         }
4273 
4274         BUG_ON(cgroup_css(cgrp, ss));
4275 }
4276 
4277 /* invoke ->css_online() on a new CSS and mark it online if successful */
4278 static int online_css(struct cgroup_subsys_state *css)
4279 {
4280         struct cgroup_subsys *ss = css->ss;
4281         int ret = 0;
4282 
4283         lockdep_assert_held(&cgroup_mutex);
4284 
4285         if (ss->css_online)
4286                 ret = ss->css_online(css);
4287         if (!ret) {
4288                 css->flags |= CSS_ONLINE;
4289                 rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
4290         }
4291         return ret;
4292 }
4293 
4294 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4295 static void offline_css(struct cgroup_subsys_state *css)
4296 {
4297         struct cgroup_subsys *ss = css->ss;
4298 
4299         lockdep_assert_held(&cgroup_mutex);
4300 
4301         if (!(css->flags & CSS_ONLINE))
4302                 return;
4303 
4304         if (ss->css_offline)
4305                 ss->css_offline(css);
4306 
4307         css->flags &= ~CSS_ONLINE;
4308         RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
4309 
4310         wake_up_all(&css->cgroup->offline_waitq);
4311 }
4312 
4313 /**
4314  * create_css - create a cgroup_subsys_state
4315  * @cgrp: the cgroup new css will be associated with
4316  * @ss: the subsys of new css
4317  *
4318  * Create a new css associated with @cgrp - @ss pair.  On success, the new
4319  * css is online and installed in @cgrp with all interface files created.
4320  * Returns 0 on success, -errno on failure.
4321  */
4322 static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss)
4323 {
4324         struct cgroup *parent = cgroup_parent(cgrp);
4325         struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
4326         struct cgroup_subsys_state *css;
4327         int err;
4328 
4329         lockdep_assert_held(&cgroup_mutex);
4330 
4331         css = ss->css_alloc(parent_css);
4332         if (IS_ERR(css))
4333                 return PTR_ERR(css);
4334 
4335         init_and_link_css(css, ss, cgrp);
4336 
4337         err = percpu_ref_init(&css->refcnt, css_release);
4338         if (err)
4339                 goto err_free_css;
4340 
4341         err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT);
4342         if (err < 0)
4343                 goto err_free_percpu_ref;
4344         css->id = err;
4345 
4346         err = cgroup_populate_dir(cgrp, 1 << ss->id);
4347         if (err)
4348                 goto err_free_id;
4349 
4350         /* @css is ready to be brought online now, make it visible */
4351         list_add_tail_rcu(&css->sibling, &parent_css->children);
4352         cgroup_idr_replace(&ss->css_idr, css, css->id);
4353 
4354         err = online_css(css);
4355         if (err)
4356                 goto err_list_del;
4357 
4358         if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
4359             cgroup_parent(parent)) {
4360                 pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4361                         current->comm, current->pid, ss->name);
4362                 if (!strcmp(ss->name, "memory"))
4363                         pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
4364                 ss->warned_broken_hierarchy = true;
4365         }
4366 
4367         return 0;
4368 
4369 err_list_del:
4370         list_del_rcu(&css->sibling);
4371         cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4372 err_free_id:
4373         cgroup_idr_remove(&ss->css_idr, css->id);
4374 err_free_percpu_ref:
4375         percpu_ref_cancel_init(&css->refcnt);
4376 err_free_css:
4377         call_rcu(&css->rcu_head, css_free_rcu_fn);
4378         return err;
4379 }
4380 
4381 static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
4382                         umode_t mode)
4383 {
4384         struct cgroup *parent, *cgrp;
4385         struct cgroup_root *root;
4386         struct cgroup_subsys *ss;
4387         struct kernfs_node *kn;
4388         int ssid, ret;
4389 
4390         parent = cgroup_kn_lock_live(parent_kn);
4391         if (!parent)
4392                 return -ENODEV;
4393         root = parent->root;
4394 
4395         /* allocate the cgroup and its ID, 0 is reserved for the root */
4396         cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
4397         if (!cgrp) {
4398                 ret = -ENOMEM;
4399                 goto out_unlock;
4400         }
4401 
4402         ret = percpu_ref_init(&cgrp->self.refcnt, css_release);
4403         if (ret)
4404                 goto out_free_cgrp;
4405 
4406         /*
4407          * Temporarily set the pointer to NULL, so idr_find() won't return
4408          * a half-baked cgroup.
4409          */
4410         cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT);
4411         if (cgrp->id < 0) {
4412                 ret = -ENOMEM;
4413                 goto out_cancel_ref;
4414         }
4415 
4416         init_cgroup_housekeeping(cgrp);
4417 
4418         cgrp->self.parent = &parent->self;
4419         cgrp->root = root;
4420 
4421         if (notify_on_release(parent))
4422                 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
4423 
4424         if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
4425                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4426 
4427         /* create the directory */
4428         kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
4429         if (IS_ERR(kn)) {
4430                 ret = PTR_ERR(kn);
4431                 goto out_free_id;
4432         }
4433         cgrp->kn = kn;
4434 
4435         /*
4436          * This extra ref will be put in cgroup_free_fn() and guarantees
4437          * that @cgrp->kn is always accessible.
4438          */
4439         kernfs_get(kn);
4440 
4441         cgrp->self.serial_nr = css_serial_nr_next++;
4442 
4443         /* allocation complete, commit to creation */
4444         list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
4445         atomic_inc(&root->nr_cgrps);
4446         cgroup_get(parent);
4447 
4448         /*
4449          * @cgrp is now fully operational.  If something fails after this
4450          * point, it'll be released via the normal destruction path.
4451          */
4452         cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4453 
4454         ret = cgroup_kn_set_ugid(kn);
4455         if (ret)
4456                 goto out_destroy;
4457 
4458         ret = cgroup_addrm_files(cgrp, cgroup_base_files, true);
4459         if (ret)
4460                 goto out_destroy;
4461 
4462         /* let's create and online css's */
4463         for_each_subsys(ss, ssid) {
4464                 if (parent->child_subsys_mask & (1 << ssid)) {
4465                         ret = create_css(cgrp, ss);
4466                         if (ret)
4467                                 goto out_destroy;
4468                 }
4469         }
4470 
4471         /*
4472          * On the default hierarchy, a child doesn't automatically inherit
4473          * child_subsys_mask from the parent.  Each is configured manually.
4474          */
4475         if (!cgroup_on_dfl(cgrp))
4476                 cgrp->child_subsys_mask = parent->child_subsys_mask;
4477 
4478         kernfs_activate(kn);
4479 
4480         ret = 0;
4481         goto out_unlock;
4482 
4483 out_free_id:
4484         cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
4485 out_cancel_ref:
4486         percpu_ref_cancel_init(&cgrp->self.refcnt);
4487 out_free_cgrp:
4488         kfree(cgrp);
4489 out_unlock:
4490         cgroup_kn_unlock(parent_kn);
4491         return ret;
4492 
4493 out_destroy:
4494         cgroup_destroy_locked(cgrp);
4495         goto out_unlock;
4496 }
4497 
4498 /*
4499  * This is called when the refcnt of a css is confirmed to be killed.
4500  * css_tryget_online() is now guaranteed to fail.  Tell the subsystem to
4501  * initate destruction and put the css ref from kill_css().
4502  */
4503 static void css_killed_work_fn(struct work_struct *work)
4504 {
4505         struct cgroup_subsys_state *css =
4506                 container_of(work, struct cgroup_subsys_state, destroy_work);
4507 
4508         mutex_lock(&cgroup_mutex);
4509         offline_css(css);
4510         mutex_unlock(&cgroup_mutex);
4511 
4512         css_put(css);
4513 }
4514 
4515 /* css kill confirmation processing requires process context, bounce */
4516 static void css_killed_ref_fn(struct percpu_ref *ref)
4517 {
4518         struct cgroup_subsys_state *css =
4519                 container_of(ref, struct cgroup_subsys_state, refcnt);
4520 
4521         INIT_WORK(&css->destroy_work, css_killed_work_fn);
4522         queue_work(cgroup_destroy_wq, &css->destroy_work);
4523 }
4524 
4525 /**
4526  * kill_css - destroy a css
4527  * @css: css to destroy
4528  *
4529  * This function initiates destruction of @css by removing cgroup interface
4530  * files and putting its base reference.  ->css_offline() will be invoked
4531  * asynchronously once css_tryget_online() is guaranteed to fail and when
4532  * the reference count reaches zero, @css will be released.
4533  */
4534 static void kill_css(struct cgroup_subsys_state *css)
4535 {
4536         lockdep_assert_held(&cgroup_mutex);
4537 
4538         /*
4539          * This must happen before css is disassociated with its cgroup.
4540          * See seq_css() for details.
4541          */
4542         cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4543 
4544         /*
4545          * Killing would put the base ref, but we need to keep it alive
4546          * until after ->css_offline().
4547          */
4548         css_get(css);
4549 
4550         /*
4551          * cgroup core guarantees that, by the time ->css_offline() is
4552          * invoked, no new css reference will be given out via
4553          * css_tryget_online().  We can't simply call percpu_ref_kill() and
4554          * proceed to offlining css's because percpu_ref_kill() doesn't
4555          * guarantee that the ref is seen as killed on all CPUs on return.
4556          *
4557          * Use percpu_ref_kill_and_confirm() to get notifications as each
4558          * css is confirmed to be seen as killed on all CPUs.
4559          */
4560         percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4561 }
4562 
4563 /**
4564  * cgroup_destroy_locked - the first stage of cgroup destruction
4565  * @cgrp: cgroup to be destroyed
4566  *
4567  * css's make use of percpu refcnts whose killing latency shouldn't be
4568  * exposed to userland and are RCU protected.  Also, cgroup core needs to
4569  * guarantee that css_tryget_online() won't succeed by the time
4570  * ->css_offline() is invoked.  To satisfy all the requirements,
4571  * destruction is implemented in the following two steps.
4572  *
4573  * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
4574  *     userland visible parts and start killing the percpu refcnts of
4575  *     css's.  Set up so that the next stage will be kicked off once all
4576  *     the percpu refcnts are confirmed to be killed.
4577  *
4578  * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
4579  *     rest of destruction.  Once all cgroup references are gone, the
4580  *     cgroup is RCU-freed.
4581  *
4582  * This function implements s1.  After this step, @cgrp is gone as far as
4583  * the userland is concerned and a new cgroup with the same name may be
4584  * created.  As cgroup doesn't care about the names internally, this
4585  * doesn't cause any problem.
4586  */
4587 static int cgroup_destroy_locked(struct cgroup *cgrp)
4588         __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4589 {
4590         struct cgroup_subsys_state *css;
4591         bool empty;
4592         int ssid;
4593 
4594         lockdep_assert_held(&cgroup_mutex);
4595 
4596         /*
4597          * css_set_rwsem synchronizes access to ->cset_links and prevents
4598          * @cgrp from being removed while put_css_set() is in progress.
4599          */
4600         down_read(&css_set_rwsem);
4601         empty = list_empty(&cgrp->cset_links);
4602         up_read(&css_set_rwsem);
4603         if (!empty)
4604                 return -EBUSY;
4605 
4606         /*
4607          * Make sure there's no live children.  We can't test emptiness of
4608          * ->self.children as dead children linger on it while being
4609          * drained; otherwise, "rmdir parent/child parent" may fail.
4610          */
4611         if (css_has_online_children(&cgrp->self))
4612                 return -EBUSY;
4613 
4614         /*
4615          * Mark @cgrp dead.  This prevents further task migration and child
4616          * creation by disabling cgroup_lock_live_group().
4617          */
4618         cgrp->self.flags &= ~CSS_ONLINE;
4619 
4620         /* initiate massacre of all css's */
4621         for_each_css(css, ssid, cgrp)
4622                 kill_css(css);
4623 
4624         /* CSS_ONLINE is clear, remove from ->release_list for the last time */
4625         raw_spin_lock(&release_list_lock);
4626         if (!list_empty(&cgrp->release_list))
4627                 list_del_init(&cgrp->release_list);
4628         raw_spin_unlock(&release_list_lock);
4629 
4630         /*
4631          * Remove @cgrp directory along with the base files.  @cgrp has an
4632          * extra ref on its kn.
4633          */
4634         kernfs_remove(cgrp->kn);
4635 
4636         set_bit(CGRP_RELEASABLE, &cgroup_parent(cgrp)->flags);
4637         check_for_release(cgroup_parent(cgrp));
4638 
4639         /* put the base reference */
4640         percpu_ref_kill(&cgrp->self.refcnt);
4641 
4642         return 0;
4643 };
4644 
4645 static int cgroup_rmdir(struct kernfs_node *kn)
4646 {
4647         struct cgroup *cgrp;
4648         int ret = 0;
4649 
4650         cgrp = cgroup_kn_lock_live(kn);
4651         if (!cgrp)
4652                 return 0;
4653         cgroup_get(cgrp);       /* for @kn->priv clearing */
4654 
4655         ret = cgroup_destroy_locked(cgrp);
4656 
4657         cgroup_kn_unlock(kn);
4658 
4659         /*
4660          * There are two control paths which try to determine cgroup from
4661          * dentry without going through kernfs - cgroupstats_build() and
4662          * css_tryget_online_from_dir().  Those are supported by RCU
4663          * protecting clearing of cgrp->kn->priv backpointer, which should
4664          * happen after all files under it have been removed.
4665          */
4666         if (!ret)
4667                 RCU_INIT_POINTER(*(void __rcu __force **)&kn->priv, NULL);
4668 
4669         cgroup_put(cgrp);
4670         return ret;
4671 }
4672 
4673 static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
4674         .remount_fs             = cgroup_remount,
4675         .show_options           = cgroup_show_options,
4676         .mkdir                  = cgroup_mkdir,
4677         .rmdir                  = cgroup_rmdir,
4678         .rename                 = cgroup_rename,
4679 };
4680 
4681 static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
4682 {
4683         struct cgroup_subsys_state *css;
4684 
4685         printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4686 
4687         mutex_lock(&cgroup_mutex);
4688 
4689         idr_init(&ss->css_idr);
4690         INIT_LIST_HEAD(&ss->cfts);
4691 
4692         /* Create the root cgroup state for this subsystem */
4693         ss->root = &cgrp_dfl_root;
4694         css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
4695         /* We don't handle early failures gracefully */
4696         BUG_ON(IS_ERR(css));
4697         init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
4698 
4699         /*
4700          * Root csses are never destroyed and we can't initialize
4701          * percpu_ref during early init.  Disable refcnting.
4702          */
4703         css->flags |= CSS_NO_REF;
4704 
4705         if (early) {
4706                 /* allocation can't be done safely during early init */
4707                 css->id = 1;
4708         } else {
4709                 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
4710                 BUG_ON(css->id < 0);
4711         }
4712 
4713         /* Update the init_css_set to contain a subsys
4714          * pointer to this state - since the subsystem is
4715          * newly registered, all tasks and hence the
4716          * init_css_set is in the subsystem's root cgroup. */
4717         init_css_set.subsys[ss->id] = css;
4718 
4719         need_forkexit_callback |= ss->fork || ss->exit;
4720 
4721         /* At system boot, before all subsystems have been
4722          * registered, no tasks have been forked, so we don't
4723          * need to invoke fork callbacks here. */
4724         BUG_ON(!list_empty(&init_task.tasks));
4725 
4726         BUG_ON(online_css(css));
4727 
4728         mutex_unlock(&cgroup_mutex);
4729 }
4730 
4731 /**
4732  * cgroup_init_early - cgroup initialization at system boot
4733  *
4734  * Initialize cgroups at system boot, and initialize any
4735  * subsystems that request early init.
4736  */
4737 int __init cgroup_init_early(void)
4738 {
4739         static struct cgroup_sb_opts __initdata opts =
4740                 { .flags = CGRP_ROOT_SANE_BEHAVIOR };
4741         struct cgroup_subsys *ss;
4742         int i;
4743 
4744         init_cgroup_root(&cgrp_dfl_root, &opts);
4745         cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;
4746 
4747         RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
4748 
4749         for_each_subsys(ss, i) {
4750                 WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
4751                      "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n",
4752                      i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
4753                      ss->id, ss->name);
4754                 WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
4755                      "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);
4756 
4757                 ss->id = i;
4758                 ss->name = cgroup_subsys_name[i];
4759 
4760                 if (ss->early_init)
4761                         cgroup_init_subsys(ss, true);
4762         }
4763         return 0;
4764 }
4765 
4766 /**
4767  * cgroup_init - cgroup initialization
4768  *
4769  * Register cgroup filesystem and /proc file, and initialize
4770  * any subsystems that didn't request early init.
4771  */
4772 int __init cgroup_init(void)
4773 {
4774         struct cgroup_subsys *ss;
4775         unsigned long key;
4776         int ssid, err;
4777 
4778         BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files));
4779 
4780         mutex_lock(&cgroup_mutex);
4781 
4782         /* Add init_css_set to the hash table */
4783         key = css_set_hash(init_css_set.subsys);
4784         hash_add(css_set_table, &init_css_set.hlist, key);
4785 
4786         BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
4787 
4788         mutex_unlock(&cgroup_mutex);
4789 
4790         for_each_subsys(ss, ssid) {
4791                 if (ss->early_init) {
4792                         struct cgroup_subsys_state *css =
4793                                 init_css_set.subsys[ss->id];
4794 
4795                         css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
4796                                                    GFP_KERNEL);
4797                         BUG_ON(css->id < 0);
4798                 } else {
4799                         cgroup_init_subsys(ss, false);
4800                 }
4801 
4802                 list_add_tail(&init_css_set.e_cset_node[ssid],
4803                               &cgrp_dfl_root.cgrp.e_csets[ssid]);
4804 
4805                 /*
4806                  * Setting dfl_root subsys_mask needs to consider the
4807                  * disabled flag and cftype registration needs kmalloc,
4808                  * both of which aren't available during early_init.
4809                  */
4810                 if (!ss->disabled) {
4811                         cgrp_dfl_root.subsys_mask |= 1 << ss->id;
4812                         WARN_ON(cgroup_add_cftypes(ss, ss->base_cftypes));
4813                 }
4814         }
4815 
4816         cgroup_kobj = kobject_create_and_add("cgroup", fs_kobj);
4817         if (!cgroup_kobj)
4818                 return -ENOMEM;
4819 
4820         err = register_filesystem(&cgroup_fs_type);
4821         if (err < 0) {
4822                 kobject_put(cgroup_kobj);
4823                 return err;
4824         }
4825 
4826         proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
4827         return 0;
4828 }
4829 
4830 static int __init cgroup_wq_init(void)
4831 {
4832         /*
4833          * There isn't much point in executing destruction path in
4834          * parallel.  Good chunk is serialized with cgroup_mutex anyway.
4835          * Use 1 for @max_active.
4836          *
4837          * We would prefer to do this in cgroup_init() above, but that
4838          * is called before init_workqueues(): so leave this until after.
4839          */
4840         cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
4841         BUG_ON(!cgroup_destroy_wq);
4842 
4843         /*
4844          * Used to destroy pidlists and separate to serve as flush domain.
4845          * Cap @max_active to 1 too.
4846          */
4847         cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
4848                                                     0, 1);
4849         BUG_ON(!cgroup_pidlist_destroy_wq);
4850 
4851         return 0;
4852 }
4853 core_initcall(cgroup_wq_init);
4854 
4855 /*
4856  * proc_cgroup_show()
4857  *  - Print task's cgroup paths into seq_file, one line for each hierarchy
4858  *  - Used for /proc/<pid>/cgroup.
4859  */
4860 
4861 /* TODO: Use a proper seq_file iterator */
4862 int proc_cgroup_show(struct seq_file *m, void *v)
4863 {
4864         struct pid *pid;
4865         struct task_struct *tsk;
4866         char *buf, *path;
4867         int retval;
4868         struct cgroup_root *root;
4869 
4870         retval = -ENOMEM;
4871         buf = kmalloc(PATH_MAX, GFP_KERNEL);
4872         if (!buf)
4873                 goto out;
4874 
4875         retval = -ESRCH;
4876         pid = m->private;
4877         tsk = get_pid_task(pid, PIDTYPE_PID);
4878         if (!tsk)
4879                 goto out_free;
4880 
4881         retval = 0;
4882 
4883         mutex_lock(&cgroup_mutex);
4884         down_read(&css_set_rwsem);
4885 
4886         for_each_root(root) {
4887                 struct cgroup_subsys *ss;
4888                 struct cgroup *cgrp;
4889                 int ssid, count = 0;
4890 
4891                 if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible)
4892                         continue;
4893 
4894                 seq_printf(m, "%d:", root->hierarchy_id);
4895                 for_each_subsys(ss, ssid)
4896                         if (root->subsys_mask & (1 << ssid))
4897                                 seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
4898                 if (strlen(root->name))
4899                         seq_printf(m, "%sname=%s", count ? "," : "",
4900                                    root->name);
4901                 seq_putc(m, ':');
4902                 cgrp = task_cgroup_from_root(tsk, root);
4903                 path = cgroup_path(cgrp, buf, PATH_MAX);
4904                 if (!path) {
4905                         retval = -ENAMETOOLONG;
4906                         goto out_unlock;
4907                 }
4908                 seq_puts(m, path);
4909                 seq_putc(m, '\n');
4910         }
4911 
4912 out_unlock:
4913         up_read(&css_set_rwsem);
4914         mutex_unlock(&cgroup_mutex);
4915         put_task_struct(tsk);
4916 out_free:
4917         kfree(buf);
4918 out:
4919         return retval;
4920 }
4921 
4922 /* Display information about each subsystem and each hierarchy */
4923 static int proc_cgroupstats_show(struct seq_file *m, void *v)
4924 {
4925         struct cgroup_subsys *ss;
4926         int i;
4927 
4928         seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
4929         /*
4930          * ideally we don't want subsystems moving around while we do this.
4931          * cgroup_mutex is also necessary to guarantee an atomic snapshot of
4932          * subsys/hierarchy state.
4933          */
4934         mutex_lock(&cgroup_mutex);
4935 
4936         for_each_subsys(ss, i)
4937                 seq_printf(m, "%s\t%d\t%d\t%d\n",
4938                            ss->name, ss->root->hierarchy_id,
4939                            atomic_read(&ss->root->nr_cgrps), !ss->disabled);
4940 
4941         mutex_unlock(&cgroup_mutex);
4942         return 0;
4943 }
4944 
4945 static int cgroupstats_open(struct inode *inode, struct file *file)
4946 {
4947         return single_open(file, proc_cgroupstats_show, NULL);
4948 }
4949 
4950 static const struct file_operations proc_cgroupstats_operations = {
4951         .open = cgroupstats_open,
4952         .read = seq_read,
4953         .llseek = seq_lseek,
4954         .release = single_release,
4955 };
4956 
4957 /**
4958  * cgroup_fork - initialize cgroup related fields during copy_process()
4959  * @child: pointer to task_struct of forking parent process.
4960  *
4961  * A task is associated with the init_css_set until cgroup_post_fork()
4962  * attaches it to the parent's css_set.  Empty cg_list indicates that
4963  * @child isn't holding reference to its css_set.
4964  */
4965 void cgroup_fork(struct task_struct *child)
4966 {
4967         RCU_INIT_POINTER(child->cgroups, &init_css_set);
4968         INIT_LIST_HEAD(&child->cg_list);
4969 }
4970 
4971 /**
4972  * cgroup_post_fork - called on a new task after adding it to the task list
4973  * @child: the task in question
4974  *
4975  * Adds the task to the list running through its css_set if necessary and
4976  * call the subsystem fork() callbacks.  Has to be after the task is
4977  * visible on the task list in case we race with the first call to
4978  * cgroup_task_iter_start() - to guarantee that the new task ends up on its
4979  * list.
4980  */
4981 void cgroup_post_fork(struct task_struct *child)
4982 {
4983         struct cgroup_subsys *ss;
4984         int i;
4985 
4986         /*
4987          * This may race against cgroup_enable_task_cg_links().  As that
4988          * function sets use_task_css_set_links before grabbing
4989          * tasklist_lock and we just went through tasklist_lock to add
4990          * @child, it's guaranteed that either we see the set
4991          * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
4992          * @child during its iteration.
4993          *
4994          * If we won the race, @child is associated with %current's
4995          * css_set.  Grabbing css_set_rwsem guarantees both that the
4996          * association is stable, and, on completion of the parent's
4997          * migration, @child is visible in the source of migration or
4998          * already in the destination cgroup.  This guarantee is necessary
4999          * when implementing operations which need to migrate all tasks of
5000          * a cgroup to another.
5001          *
5002          * Note that if we lose to cgroup_enable_task_cg_links(), @child
5003          * will remain in init_css_set.  This is safe because all tasks are
5004          * in the init_css_set before cg_links is enabled and there's no
5005          * operation which transfers all tasks out of init_css_set.
5006          */
5007         if (use_task_css_set_links) {
5008                 struct css_set *cset;
5009 
5010                 down_write(&css_set_rwsem);
5011                 cset = task_css_set(current);
5012                 if (list_empty(&child->cg_list)) {
5013                         rcu_assign_pointer(child->cgroups, cset);
5014                         list_add(&child->cg_list, &cset->tasks);
5015                         get_css_set(cset);
5016                 }
5017                 up_write(&css_set_rwsem);
5018         }
5019 
5020         /*
5021          * Call ss->fork().  This must happen after @child is linked on
5022          * css_set; otherwise, @child might change state between ->fork()
5023          * and addition to css_set.
5024          */
5025         if (need_forkexit_callback) {
5026                 for_each_subsys(ss, i)
5027                         if (ss->fork)
5028                                 ss->fork(child);
5029         }
5030 }
5031 
5032 /**
5033  * cgroup_exit - detach cgroup from exiting task
5034  * @tsk: pointer to task_struct of exiting process
5035  *
5036  * Description: Detach cgroup from @tsk and release it.
5037  *
5038  * Note that cgroups marked notify_on_release force every task in
5039  * them to take the global cgroup_mutex mutex when exiting.
5040  * This could impact scaling on very large systems.  Be reluctant to
5041  * use notify_on_release cgroups where very high task exit scaling
5042  * is required on large systems.
5043  *
5044  * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
5045  * call cgroup_exit() while the task is still competent to handle
5046  * notify_on_release(), then leave the task attached to the root cgroup in
5047  * each hierarchy for the remainder of its exit.  No need to bother with
5048  * init_css_set refcnting.  init_css_set never goes away and we can't race
5049  * with migration path - PF_EXITING is visible to migration path.
5050  */
5051 void cgroup_exit(struct task_struct *tsk)
5052 {
5053         struct cgroup_subsys *ss;
5054         struct css_set *cset;
5055         bool put_cset = false;
5056         int i;
5057 
5058         /*
5059          * Unlink from @tsk from its css_set.  As migration path can't race
5060          * with us, we can check cg_list without grabbing css_set_rwsem.
5061          */
5062         if (!list_empty(&tsk->cg_list)) {
5063                 down_write(&css_set_rwsem);
5064                 list_del_init(&tsk->cg_list);
5065                 up_write(&css_set_rwsem);
5066                 put_cset = true;
5067         }
5068 
5069         /* Reassign the task to the init_css_set. */
5070         cset = task_css_set(tsk);
5071         RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
5072 
5073         if (need_forkexit_callback) {
5074                 /* see cgroup_post_fork() for details */
5075                 for_each_subsys(ss, i) {
5076                         if (ss->exit) {
5077                                 struct cgroup_subsys_state *old_css = cset->subsys[i];
5078                                 struct cgroup_subsys_state *css = task_css(tsk, i);
5079 
5080                                 ss->exit(css, old_css, tsk);
5081                         }
5082                 }
5083         }
5084 
5085         if (put_cset)
5086                 put_css_set(cset, true);
5087 }
5088 
5089 static void check_for_release(struct cgroup *cgrp)
5090 {
5091         if (cgroup_is_releasable(cgrp) && list_empty(&cgrp->cset_links) &&
5092             !css_has_online_children(&cgrp->self)) {
5093                 /*
5094                  * Control Group is currently removeable. If it's not
5095                  * already queued for a userspace notification, queue
5096                  * it now
5097                  */
5098                 int need_schedule_work = 0;
5099 
5100                 raw_spin_lock(&release_list_lock);
5101                 if (!cgroup_is_dead(cgrp) &&
5102                     list_empty(&cgrp->release_list)) {
5103                         list_add(&cgrp->release_list, &release_list);
5104                         need_schedule_work = 1;
5105                 }
5106                 raw_spin_unlock(&release_list_lock);
5107                 if (need_schedule_work)
5108                         schedule_work(&release_agent_work);
5109         }
5110 }
5111 
5112 /*
5113  * Notify userspace when a cgroup is released, by running the
5114  * configured release agent with the name of the cgroup (path
5115  * relative to the root of cgroup file system) as the argument.
5116  *
5117  * Most likely, this user command will try to rmdir this cgroup.
5118  *
5119  * This races with the possibility that some other task will be
5120  * attached to this cgroup before it is removed, or that some other
5121  * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
5122  * The presumed 'rmdir' will fail quietly if this cgroup is no longer
5123  * unused, and this cgroup will be reprieved from its death sentence,
5124  * to continue to serve a useful existence.  Next time it's released,
5125  * we will get notified again, if it still has 'notify_on_release' set.
5126  *
5127  * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
5128  * means only wait until the task is successfully execve()'d.  The
5129  * separate release agent task is forked by call_usermodehelper(),
5130  * then control in this thread returns here, without waiting for the
5131  * release agent task.  We don't bother to wait because the caller of
5132  * this routine has no use for the exit status of the release agent
5133  * task, so no sense holding our caller up for that.
5134  */
5135 static void cgroup_release_agent(struct work_struct *work)
5136 {
5137         BUG_ON(work != &release_agent_work);
5138         mutex_lock(&cgroup_mutex);
5139         raw_spin_lock(&release_list_lock);
5140         while (!list_empty(&release_list)) {
5141                 char *argv[3], *envp[3];
5142                 int i;
5143                 char *pathbuf = NULL, *agentbuf = NULL, *path;
5144                 struct cgroup *cgrp = list_entry(release_list.next,
5145                                                     struct cgroup,
5146                                                     release_list);
5147                 list_del_init(&cgrp->release_list);
5148                 raw_spin_unlock(&release_list_lock);
5149                 pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
5150                 if (!pathbuf)
5151                         goto continue_free;
5152                 path = cgroup_path(cgrp, pathbuf, PATH_MAX);
5153                 if (!path)
5154                         goto continue_free;
5155                 agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
5156                 if (!agentbuf)
5157                         goto continue_free;
5158 
5159                 i = 0;
5160                 argv[i++] = agentbuf;
5161                 argv[i++] = path;
5162                 argv[i] = NULL;
5163 
5164                 i = 0;
5165                 /* minimal command environment */
5166                 envp[i++] = "HOME=/";
5167                 envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
5168                 envp[i] = NULL;
5169 
5170                 /* Drop the lock while we invoke the usermode helper,
5171                  * since the exec could involve hitting disk and hence
5172                  * be a slow process */
5173                 mutex_unlock(&cgroup_mutex);
5174                 call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
5175                 mutex_lock(&cgroup_mutex);
5176  continue_free:
5177                 kfree(pathbuf);
5178                 kfree(agentbuf);
5179                 raw_spin_lock(&release_list_lock);
5180         }
5181         raw_spin_unlock(&release_list_lock);
5182         mutex_unlock(&cgroup_mutex);
5183 }
5184 
5185 static int __init cgroup_disable(char *str)
5186 {
5187         struct cgroup_subsys *ss;
5188         char *token;
5189         int i;
5190 
5191         while ((token = strsep(&str, ",")) != NULL) {
5192                 if (!*token)
5193                         continue;
5194 
5195                 for_each_subsys(ss, i) {
5196                         if (!strcmp(token, ss->name)) {
5197                                 ss->disabled = 1;
5198                                 printk(KERN_INFO "Disabling %s control group"
5199                                         " subsystem\n", ss->name);
5200                                 break;
5201                         }
5202                 }
5203         }
5204         return 1;
5205 }
5206 __setup("cgroup_disable=", cgroup_disable);
5207 
5208 /**
5209  * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
5210  * @dentry: directory dentry of interest
5211  * @ss: subsystem of interest
5212  *
5213  * If @dentry is a directory for a cgroup which has @ss enabled on it, try
5214  * to get the corresponding css and return it.  If such css doesn't exist
5215  * or can't be pinned, an ERR_PTR value is returned.
5216  */
5217 struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
5218                                                        struct cgroup_subsys *ss)
5219 {
5220         struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
5221         struct cgroup_subsys_state *css = NULL;
5222         struct cgroup *cgrp;
5223 
5224         /* is @dentry a cgroup dir? */
5225         if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
5226             kernfs_type(kn) != KERNFS_DIR)
5227                 return ERR_PTR(-EBADF);
5228 
5229         rcu_read_lock();
5230 
5231         /*
5232          * This path doesn't originate from kernfs and @kn could already
5233          * have been or be removed at any point.  @kn->priv is RCU
5234          * protected for this access.  See cgroup_rmdir() for details.
5235          */
5236         cgrp = rcu_dereference(kn->priv);
5237         if (cgrp)
5238                 css = cgroup_css(cgrp, ss);
5239 
5240         if (!css || !css_tryget_online(css))
5241                 css = ERR_PTR(-ENOENT);
5242 
5243         rcu_read_unlock();
5244         return css;
5245 }
5246 
5247 /**
5248  * css_from_id - lookup css by id
5249  * @id: the cgroup id
5250  * @ss: cgroup subsys to be looked into
5251  *
5252  * Returns the css if there's valid one with @id, otherwise returns NULL.
5253  * Should be called under rcu_read_lock().
5254  */
5255 struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
5256 {
5257         WARN_ON_ONCE(!rcu_read_lock_held());
5258         return idr_find(&ss->css_idr, id);
5259 }
5260 
5261 #ifdef CONFIG_CGROUP_DEBUG
5262 static struct cgroup_subsys_state *
5263 debug_css_alloc(struct cgroup_subsys_state *parent_css)
5264 {
5265         struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
5266 
5267         if (!css)
5268                 return ERR_PTR(-ENOMEM);
5269 
5270         return css;
5271 }
5272 
5273 static void debug_css_free(struct cgroup_subsys_state *css)
5274 {
5275         kfree(css);
5276 }
5277 
5278 static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
5279                                 struct cftype *cft)
5280 {
5281         return cgroup_task_count(css->cgroup);
5282 }
5283 
5284 static u64 current_css_set_read(struct cgroup_subsys_state *css,
5285                                 struct cftype *cft)
5286 {
5287         return (u64)(unsigned long)current->cgroups;
5288 }
5289 
5290 static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
5291                                          struct cftype *cft)
5292 {
5293         u64 count;
5294 
5295         rcu_read_lock();
5296         count = atomic_read(&task_css_set(current)->refcount);
5297         rcu_read_unlock();
5298         return count;
5299 }
5300 
5301 static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
5302 {
5303         struct cgrp_cset_link *link;
5304         struct css_set *cset;
5305         char *name_buf;
5306 
5307         name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
5308         if (!name_buf)
5309                 return -ENOMEM;
5310 
5311         down_read(&css_set_rwsem);
5312         rcu_read_lock();
5313         cset = rcu_dereference(current->cgroups);
5314         list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
5315                 struct cgroup *c = link->cgrp;
5316 
5317                 cgroup_name(c, name_buf, NAME_MAX + 1);
5318                 seq_printf(seq, "Root %d group %s\n",
5319                            c->root->hierarchy_id, name_buf);
5320         }
5321         rcu_read_unlock();
5322         up_read(&css_set_rwsem);
5323         kfree(name_buf);
5324         return 0;
5325 }
5326 
5327 #define MAX_TASKS_SHOWN_PER_CSS 25
5328 static int cgroup_css_links_read(struct seq_file *seq, void *v)
5329 {
5330         struct cgroup_subsys_state *css = seq_css(seq);
5331         struct cgrp_cset_link *link;
5332 
5333         down_read(&css_set_rwsem);
5334         list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
5335                 struct css_set *cset = link->cset;
5336                 struct task_struct *task;
5337                 int count = 0;
5338 
5339                 seq_printf(seq, "css_set %p\n", cset);
5340 
5341                 list_for_each_entry(task, &cset->tasks, cg_list) {
5342                         if (count++ > MAX_TASKS_SHOWN_PER_CSS)
5343                                 goto overflow;
5344                         seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5345                 }
5346 
5347                 list_for_each_entry(task, &cset->mg_tasks, cg_list) {
5348                         if (count++ > MAX_TASKS_SHOWN_PER_CSS)
5349                                 goto overflow;
5350                         seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5351                 }
5352                 continue;
5353         overflow:
5354                 seq_puts(seq, "  ...\n");
5355         }
5356         up_read(&css_set_rwsem);
5357         return 0;
5358 }
5359 
5360 static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
5361 {
5362         return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
5363 }
5364 
5365 static struct cftype debug_files[] =  {
5366         {
5367                 .name = "taskcount",
5368                 .read_u64 = debug_taskcount_read,
5369         },
5370 
5371         {
5372                 .name = "current_css_set",
5373                 .read_u64 = current_css_set_read,
5374         },
5375 
5376         {
5377                 .name = "current_css_set_refcount",
5378                 .read_u64 = current_css_set_refcount_read,
5379         },
5380 
5381         {
5382                 .name = "current_css_set_cg_links",
5383                 .seq_show = current_css_set_cg_links_read,
5384         },
5385 
5386         {
5387                 .name = "cgroup_css_links",
5388                 .seq_show = cgroup_css_links_read,
5389         },
5390 
5391         {
5392                 .name = "releasable",
5393                 .read_u64 = releasable_read,
5394         },
5395 
5396         { }     /* terminate */
5397 };
5398 
5399 struct cgroup_subsys debug_cgrp_subsys = {
5400         .css_alloc = debug_css_alloc,
5401         .css_free = debug_css_free,
5402         .base_cftypes = debug_files,
5403 };
5404 #endif /* CONFIG_CGROUP_DEBUG */
5405 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us