Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/kernel/cgroup.c

  1 /*
  2  *  Generic process-grouping system.
  3  *
  4  *  Based originally on the cpuset system, extracted by Paul Menage
  5  *  Copyright (C) 2006 Google, Inc
  6  *
  7  *  Notifications support
  8  *  Copyright (C) 2009 Nokia Corporation
  9  *  Author: Kirill A. Shutemov
 10  *
 11  *  Copyright notices from the original cpuset code:
 12  *  --------------------------------------------------
 13  *  Copyright (C) 2003 BULL SA.
 14  *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 15  *
 16  *  Portions derived from Patrick Mochel's sysfs code.
 17  *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 18  *
 19  *  2003-10-10 Written by Simon Derr.
 20  *  2003-10-22 Updates by Stephen Hemminger.
 21  *  2004 May-July Rework by Paul Jackson.
 22  *  ---------------------------------------------------
 23  *
 24  *  This file is subject to the terms and conditions of the GNU General Public
 25  *  License.  See the file COPYING in the main directory of the Linux
 26  *  distribution for more details.
 27  */
 28 
 29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 30 
 31 #include <linux/cgroup.h>
 32 #include <linux/cred.h>
 33 #include <linux/ctype.h>
 34 #include <linux/errno.h>
 35 #include <linux/init_task.h>
 36 #include <linux/kernel.h>
 37 #include <linux/list.h>
 38 #include <linux/magic.h>
 39 #include <linux/mm.h>
 40 #include <linux/mutex.h>
 41 #include <linux/mount.h>
 42 #include <linux/pagemap.h>
 43 #include <linux/proc_fs.h>
 44 #include <linux/rcupdate.h>
 45 #include <linux/sched.h>
 46 #include <linux/slab.h>
 47 #include <linux/spinlock.h>
 48 #include <linux/percpu-rwsem.h>
 49 #include <linux/string.h>
 50 #include <linux/sort.h>
 51 #include <linux/kmod.h>
 52 #include <linux/delayacct.h>
 53 #include <linux/cgroupstats.h>
 54 #include <linux/hashtable.h>
 55 #include <linux/pid_namespace.h>
 56 #include <linux/idr.h>
 57 #include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
 58 #include <linux/kthread.h>
 59 #include <linux/delay.h>
 60 #include <linux/atomic.h>
 61 #include <linux/cpuset.h>
 62 #include <net/sock.h>
 63 
 64 /*
 65  * pidlists linger the following amount before being destroyed.  The goal
 66  * is avoiding frequent destruction in the middle of consecutive read calls
 67  * Expiring in the middle is a performance problem not a correctness one.
 68  * 1 sec should be enough.
 69  */
 70 #define CGROUP_PIDLIST_DESTROY_DELAY    HZ
 71 
 72 #define CGROUP_FILE_NAME_MAX            (MAX_CGROUP_TYPE_NAMELEN +      \
 73                                          MAX_CFTYPE_NAME + 2)
 74 
 75 /*
 76  * cgroup_mutex is the master lock.  Any modification to cgroup or its
 77  * hierarchy must be performed while holding it.
 78  *
 79  * css_set_lock protects task->cgroups pointer, the list of css_set
 80  * objects, and the chain of tasks off each css_set.
 81  *
 82  * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 83  * cgroup.h can use them for lockdep annotations.
 84  */
 85 #ifdef CONFIG_PROVE_RCU
 86 DEFINE_MUTEX(cgroup_mutex);
 87 DEFINE_SPINLOCK(css_set_lock);
 88 EXPORT_SYMBOL_GPL(cgroup_mutex);
 89 EXPORT_SYMBOL_GPL(css_set_lock);
 90 #else
 91 static DEFINE_MUTEX(cgroup_mutex);
 92 static DEFINE_SPINLOCK(css_set_lock);
 93 #endif
 94 
 95 /*
 96  * Protects cgroup_idr and css_idr so that IDs can be released without
 97  * grabbing cgroup_mutex.
 98  */
 99 static DEFINE_SPINLOCK(cgroup_idr_lock);
100 
101 /*
102  * Protects cgroup_file->kn for !self csses.  It synchronizes notifications
103  * against file removal/re-creation across css hiding.
104  */
105 static DEFINE_SPINLOCK(cgroup_file_kn_lock);
106 
107 /*
108  * Protects cgroup_subsys->release_agent_path.  Modifying it also requires
109  * cgroup_mutex.  Reading requires either cgroup_mutex or this spinlock.
110  */
111 static DEFINE_SPINLOCK(release_agent_path_lock);
112 
113 struct percpu_rw_semaphore cgroup_threadgroup_rwsem;
114 
115 #define cgroup_assert_mutex_or_rcu_locked()                             \
116         RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&                       \
117                            !lockdep_is_held(&cgroup_mutex),             \
118                            "cgroup_mutex or RCU read lock required");
119 
120 /*
121  * cgroup destruction makes heavy use of work items and there can be a lot
122  * of concurrent destructions.  Use a separate workqueue so that cgroup
123  * destruction work items don't end up filling up max_active of system_wq
124  * which may lead to deadlock.
125  */
126 static struct workqueue_struct *cgroup_destroy_wq;
127 
128 /*
129  * pidlist destructions need to be flushed on cgroup destruction.  Use a
130  * separate workqueue as flush domain.
131  */
132 static struct workqueue_struct *cgroup_pidlist_destroy_wq;
133 
134 /* generate an array of cgroup subsystem pointers */
135 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
136 static struct cgroup_subsys *cgroup_subsys[] = {
137 #include <linux/cgroup_subsys.h>
138 };
139 #undef SUBSYS
140 
141 /* array of cgroup subsystem names */
142 #define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
143 static const char *cgroup_subsys_name[] = {
144 #include <linux/cgroup_subsys.h>
145 };
146 #undef SUBSYS
147 
148 /* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */
149 #define SUBSYS(_x)                                                              \
150         DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key);                 \
151         DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key);                  \
152         EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key);                      \
153         EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key);
154 #include <linux/cgroup_subsys.h>
155 #undef SUBSYS
156 
157 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key,
158 static struct static_key_true *cgroup_subsys_enabled_key[] = {
159 #include <linux/cgroup_subsys.h>
160 };
161 #undef SUBSYS
162 
163 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key,
164 static struct static_key_true *cgroup_subsys_on_dfl_key[] = {
165 #include <linux/cgroup_subsys.h>
166 };
167 #undef SUBSYS
168 
169 /*
170  * The default hierarchy, reserved for the subsystems that are otherwise
171  * unattached - it never has more than a single cgroup, and all tasks are
172  * part of that cgroup.
173  */
174 struct cgroup_root cgrp_dfl_root;
175 EXPORT_SYMBOL_GPL(cgrp_dfl_root);
176 
177 /*
178  * The default hierarchy always exists but is hidden until mounted for the
179  * first time.  This is for backward compatibility.
180  */
181 static bool cgrp_dfl_root_visible;
182 
183 /* some controllers are not supported in the default hierarchy */
184 static unsigned long cgrp_dfl_root_inhibit_ss_mask;
185 
186 /* The list of hierarchy roots */
187 
188 static LIST_HEAD(cgroup_roots);
189 static int cgroup_root_count;
190 
191 /* hierarchy ID allocation and mapping, protected by cgroup_mutex */
192 static DEFINE_IDR(cgroup_hierarchy_idr);
193 
194 /*
195  * Assign a monotonically increasing serial number to csses.  It guarantees
196  * cgroups with bigger numbers are newer than those with smaller numbers.
197  * Also, as csses are always appended to the parent's ->children list, it
198  * guarantees that sibling csses are always sorted in the ascending serial
199  * number order on the list.  Protected by cgroup_mutex.
200  */
201 static u64 css_serial_nr_next = 1;
202 
203 /*
204  * These bitmask flags indicate whether tasks in the fork and exit paths have
205  * fork/exit handlers to call. This avoids us having to do extra work in the
206  * fork/exit path to check which subsystems have fork/exit callbacks.
207  */
208 static unsigned long have_fork_callback __read_mostly;
209 static unsigned long have_exit_callback __read_mostly;
210 static unsigned long have_free_callback __read_mostly;
211 
212 /* Ditto for the can_fork callback. */
213 static unsigned long have_canfork_callback __read_mostly;
214 
215 static struct file_system_type cgroup2_fs_type;
216 static struct cftype cgroup_dfl_base_files[];
217 static struct cftype cgroup_legacy_base_files[];
218 
219 static int rebind_subsystems(struct cgroup_root *dst_root,
220                              unsigned long ss_mask);
221 static void css_task_iter_advance(struct css_task_iter *it);
222 static int cgroup_destroy_locked(struct cgroup *cgrp);
223 static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
224                       bool visible);
225 static void css_release(struct percpu_ref *ref);
226 static void kill_css(struct cgroup_subsys_state *css);
227 static int cgroup_addrm_files(struct cgroup_subsys_state *css,
228                               struct cgroup *cgrp, struct cftype cfts[],
229                               bool is_add);
230 
231 /**
232  * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
233  * @ssid: subsys ID of interest
234  *
235  * cgroup_subsys_enabled() can only be used with literal subsys names which
236  * is fine for individual subsystems but unsuitable for cgroup core.  This
237  * is slower static_key_enabled() based test indexed by @ssid.
238  */
239 static bool cgroup_ssid_enabled(int ssid)
240 {
241         return static_key_enabled(cgroup_subsys_enabled_key[ssid]);
242 }
243 
244 /**
245  * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
246  * @cgrp: the cgroup of interest
247  *
248  * The default hierarchy is the v2 interface of cgroup and this function
249  * can be used to test whether a cgroup is on the default hierarchy for
250  * cases where a subsystem should behave differnetly depending on the
251  * interface version.
252  *
253  * The set of behaviors which change on the default hierarchy are still
254  * being determined and the mount option is prefixed with __DEVEL__.
255  *
256  * List of changed behaviors:
257  *
258  * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
259  *   and "name" are disallowed.
260  *
261  * - When mounting an existing superblock, mount options should match.
262  *
263  * - Remount is disallowed.
264  *
265  * - rename(2) is disallowed.
266  *
267  * - "tasks" is removed.  Everything should be at process granularity.  Use
268  *   "cgroup.procs" instead.
269  *
270  * - "cgroup.procs" is not sorted.  pids will be unique unless they got
271  *   recycled inbetween reads.
272  *
273  * - "release_agent" and "notify_on_release" are removed.  Replacement
274  *   notification mechanism will be implemented.
275  *
276  * - "cgroup.clone_children" is removed.
277  *
278  * - "cgroup.subtree_populated" is available.  Its value is 0 if the cgroup
279  *   and its descendants contain no task; otherwise, 1.  The file also
280  *   generates kernfs notification which can be monitored through poll and
281  *   [di]notify when the value of the file changes.
282  *
283  * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
284  *   take masks of ancestors with non-empty cpus/mems, instead of being
285  *   moved to an ancestor.
286  *
287  * - cpuset: a task can be moved into an empty cpuset, and again it takes
288  *   masks of ancestors.
289  *
290  * - memcg: use_hierarchy is on by default and the cgroup file for the flag
291  *   is not created.
292  *
293  * - blkcg: blk-throttle becomes properly hierarchical.
294  *
295  * - debug: disallowed on the default hierarchy.
296  */
297 static bool cgroup_on_dfl(const struct cgroup *cgrp)
298 {
299         return cgrp->root == &cgrp_dfl_root;
300 }
301 
302 /* IDR wrappers which synchronize using cgroup_idr_lock */
303 static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
304                             gfp_t gfp_mask)
305 {
306         int ret;
307 
308         idr_preload(gfp_mask);
309         spin_lock_bh(&cgroup_idr_lock);
310         ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM);
311         spin_unlock_bh(&cgroup_idr_lock);
312         idr_preload_end();
313         return ret;
314 }
315 
316 static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
317 {
318         void *ret;
319 
320         spin_lock_bh(&cgroup_idr_lock);
321         ret = idr_replace(idr, ptr, id);
322         spin_unlock_bh(&cgroup_idr_lock);
323         return ret;
324 }
325 
326 static void cgroup_idr_remove(struct idr *idr, int id)
327 {
328         spin_lock_bh(&cgroup_idr_lock);
329         idr_remove(idr, id);
330         spin_unlock_bh(&cgroup_idr_lock);
331 }
332 
333 static struct cgroup *cgroup_parent(struct cgroup *cgrp)
334 {
335         struct cgroup_subsys_state *parent_css = cgrp->self.parent;
336 
337         if (parent_css)
338                 return container_of(parent_css, struct cgroup, self);
339         return NULL;
340 }
341 
342 /**
343  * cgroup_css - obtain a cgroup's css for the specified subsystem
344  * @cgrp: the cgroup of interest
345  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
346  *
347  * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
348  * function must be called either under cgroup_mutex or rcu_read_lock() and
349  * the caller is responsible for pinning the returned css if it wants to
350  * keep accessing it outside the said locks.  This function may return
351  * %NULL if @cgrp doesn't have @subsys_id enabled.
352  */
353 static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
354                                               struct cgroup_subsys *ss)
355 {
356         if (ss)
357                 return rcu_dereference_check(cgrp->subsys[ss->id],
358                                         lockdep_is_held(&cgroup_mutex));
359         else
360                 return &cgrp->self;
361 }
362 
363 /**
364  * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
365  * @cgrp: the cgroup of interest
366  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
367  *
368  * Similar to cgroup_css() but returns the effective css, which is defined
369  * as the matching css of the nearest ancestor including self which has @ss
370  * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
371  * function is guaranteed to return non-NULL css.
372  */
373 static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
374                                                 struct cgroup_subsys *ss)
375 {
376         lockdep_assert_held(&cgroup_mutex);
377 
378         if (!ss)
379                 return &cgrp->self;
380 
381         if (!(cgrp->root->subsys_mask & (1 << ss->id)))
382                 return NULL;
383 
384         /*
385          * This function is used while updating css associations and thus
386          * can't test the csses directly.  Use ->child_subsys_mask.
387          */
388         while (cgroup_parent(cgrp) &&
389                !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id)))
390                 cgrp = cgroup_parent(cgrp);
391 
392         return cgroup_css(cgrp, ss);
393 }
394 
395 /**
396  * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
397  * @cgrp: the cgroup of interest
398  * @ss: the subsystem of interest
399  *
400  * Find and get the effective css of @cgrp for @ss.  The effective css is
401  * defined as the matching css of the nearest ancestor including self which
402  * has @ss enabled.  If @ss is not mounted on the hierarchy @cgrp is on,
403  * the root css is returned, so this function always returns a valid css.
404  * The returned css must be put using css_put().
405  */
406 struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp,
407                                              struct cgroup_subsys *ss)
408 {
409         struct cgroup_subsys_state *css;
410 
411         rcu_read_lock();
412 
413         do {
414                 css = cgroup_css(cgrp, ss);
415 
416                 if (css && css_tryget_online(css))
417                         goto out_unlock;
418                 cgrp = cgroup_parent(cgrp);
419         } while (cgrp);
420 
421         css = init_css_set.subsys[ss->id];
422         css_get(css);
423 out_unlock:
424         rcu_read_unlock();
425         return css;
426 }
427 
428 /* convenient tests for these bits */
429 static inline bool cgroup_is_dead(const struct cgroup *cgrp)
430 {
431         return !(cgrp->self.flags & CSS_ONLINE);
432 }
433 
434 static void cgroup_get(struct cgroup *cgrp)
435 {
436         WARN_ON_ONCE(cgroup_is_dead(cgrp));
437         css_get(&cgrp->self);
438 }
439 
440 static bool cgroup_tryget(struct cgroup *cgrp)
441 {
442         return css_tryget(&cgrp->self);
443 }
444 
445 struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
446 {
447         struct cgroup *cgrp = of->kn->parent->priv;
448         struct cftype *cft = of_cft(of);
449 
450         /*
451          * This is open and unprotected implementation of cgroup_css().
452          * seq_css() is only called from a kernfs file operation which has
453          * an active reference on the file.  Because all the subsystem
454          * files are drained before a css is disassociated with a cgroup,
455          * the matching css from the cgroup's subsys table is guaranteed to
456          * be and stay valid until the enclosing operation is complete.
457          */
458         if (cft->ss)
459                 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
460         else
461                 return &cgrp->self;
462 }
463 EXPORT_SYMBOL_GPL(of_css);
464 
465 static int notify_on_release(const struct cgroup *cgrp)
466 {
467         return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
468 }
469 
470 /**
471  * for_each_css - iterate all css's of a cgroup
472  * @css: the iteration cursor
473  * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
474  * @cgrp: the target cgroup to iterate css's of
475  *
476  * Should be called under cgroup_[tree_]mutex.
477  */
478 #define for_each_css(css, ssid, cgrp)                                   \
479         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)        \
480                 if (!((css) = rcu_dereference_check(                    \
481                                 (cgrp)->subsys[(ssid)],                 \
482                                 lockdep_is_held(&cgroup_mutex)))) { }   \
483                 else
484 
485 /**
486  * for_each_e_css - iterate all effective css's of a cgroup
487  * @css: the iteration cursor
488  * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
489  * @cgrp: the target cgroup to iterate css's of
490  *
491  * Should be called under cgroup_[tree_]mutex.
492  */
493 #define for_each_e_css(css, ssid, cgrp)                                 \
494         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)        \
495                 if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
496                         ;                                               \
497                 else
498 
499 /**
500  * for_each_subsys - iterate all enabled cgroup subsystems
501  * @ss: the iteration cursor
502  * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
503  */
504 #define for_each_subsys(ss, ssid)                                       \
505         for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT &&                \
506              (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
507 
508 /**
509  * for_each_subsys_which - filter for_each_subsys with a bitmask
510  * @ss: the iteration cursor
511  * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
512  * @ss_maskp: a pointer to the bitmask
513  *
514  * The block will only run for cases where the ssid-th bit (1 << ssid) of
515  * mask is set to 1.
516  */
517 #define for_each_subsys_which(ss, ssid, ss_maskp)                       \
518         if (!CGROUP_SUBSYS_COUNT) /* to avoid spurious gcc warning */   \
519                 (ssid) = 0;                                             \
520         else                                                            \
521                 for_each_set_bit(ssid, ss_maskp, CGROUP_SUBSYS_COUNT)   \
522                         if (((ss) = cgroup_subsys[ssid]) && false)      \
523                                 break;                                  \
524                         else
525 
526 /* iterate across the hierarchies */
527 #define for_each_root(root)                                             \
528         list_for_each_entry((root), &cgroup_roots, root_list)
529 
530 /* iterate over child cgrps, lock should be held throughout iteration */
531 #define cgroup_for_each_live_child(child, cgrp)                         \
532         list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
533                 if (({ lockdep_assert_held(&cgroup_mutex);              \
534                        cgroup_is_dead(child); }))                       \
535                         ;                                               \
536                 else
537 
538 static void cgroup_release_agent(struct work_struct *work);
539 static void check_for_release(struct cgroup *cgrp);
540 
541 /*
542  * A cgroup can be associated with multiple css_sets as different tasks may
543  * belong to different cgroups on different hierarchies.  In the other
544  * direction, a css_set is naturally associated with multiple cgroups.
545  * This M:N relationship is represented by the following link structure
546  * which exists for each association and allows traversing the associations
547  * from both sides.
548  */
549 struct cgrp_cset_link {
550         /* the cgroup and css_set this link associates */
551         struct cgroup           *cgrp;
552         struct css_set          *cset;
553 
554         /* list of cgrp_cset_links anchored at cgrp->cset_links */
555         struct list_head        cset_link;
556 
557         /* list of cgrp_cset_links anchored at css_set->cgrp_links */
558         struct list_head        cgrp_link;
559 };
560 
561 /*
562  * The default css_set - used by init and its children prior to any
563  * hierarchies being mounted. It contains a pointer to the root state
564  * for each subsystem. Also used to anchor the list of css_sets. Not
565  * reference-counted, to improve performance when child cgroups
566  * haven't been created.
567  */
568 struct css_set init_css_set = {
569         .refcount               = ATOMIC_INIT(1),
570         .cgrp_links             = LIST_HEAD_INIT(init_css_set.cgrp_links),
571         .tasks                  = LIST_HEAD_INIT(init_css_set.tasks),
572         .mg_tasks               = LIST_HEAD_INIT(init_css_set.mg_tasks),
573         .mg_preload_node        = LIST_HEAD_INIT(init_css_set.mg_preload_node),
574         .mg_node                = LIST_HEAD_INIT(init_css_set.mg_node),
575         .task_iters             = LIST_HEAD_INIT(init_css_set.task_iters),
576 };
577 
578 static int css_set_count        = 1;    /* 1 for init_css_set */
579 
580 /**
581  * css_set_populated - does a css_set contain any tasks?
582  * @cset: target css_set
583  */
584 static bool css_set_populated(struct css_set *cset)
585 {
586         lockdep_assert_held(&css_set_lock);
587 
588         return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);
589 }
590 
591 /**
592  * cgroup_update_populated - updated populated count of a cgroup
593  * @cgrp: the target cgroup
594  * @populated: inc or dec populated count
595  *
596  * One of the css_sets associated with @cgrp is either getting its first
597  * task or losing the last.  Update @cgrp->populated_cnt accordingly.  The
598  * count is propagated towards root so that a given cgroup's populated_cnt
599  * is zero iff the cgroup and all its descendants don't contain any tasks.
600  *
601  * @cgrp's interface file "cgroup.populated" is zero if
602  * @cgrp->populated_cnt is zero and 1 otherwise.  When @cgrp->populated_cnt
603  * changes from or to zero, userland is notified that the content of the
604  * interface file has changed.  This can be used to detect when @cgrp and
605  * its descendants become populated or empty.
606  */
607 static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
608 {
609         lockdep_assert_held(&css_set_lock);
610 
611         do {
612                 bool trigger;
613 
614                 if (populated)
615                         trigger = !cgrp->populated_cnt++;
616                 else
617                         trigger = !--cgrp->populated_cnt;
618 
619                 if (!trigger)
620                         break;
621 
622                 check_for_release(cgrp);
623                 cgroup_file_notify(&cgrp->events_file);
624 
625                 cgrp = cgroup_parent(cgrp);
626         } while (cgrp);
627 }
628 
629 /**
630  * css_set_update_populated - update populated state of a css_set
631  * @cset: target css_set
632  * @populated: whether @cset is populated or depopulated
633  *
634  * @cset is either getting the first task or losing the last.  Update the
635  * ->populated_cnt of all associated cgroups accordingly.
636  */
637 static void css_set_update_populated(struct css_set *cset, bool populated)
638 {
639         struct cgrp_cset_link *link;
640 
641         lockdep_assert_held(&css_set_lock);
642 
643         list_for_each_entry(link, &cset->cgrp_links, cgrp_link)
644                 cgroup_update_populated(link->cgrp, populated);
645 }
646 
647 /**
648  * css_set_move_task - move a task from one css_set to another
649  * @task: task being moved
650  * @from_cset: css_set @task currently belongs to (may be NULL)
651  * @to_cset: new css_set @task is being moved to (may be NULL)
652  * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
653  *
654  * Move @task from @from_cset to @to_cset.  If @task didn't belong to any
655  * css_set, @from_cset can be NULL.  If @task is being disassociated
656  * instead of moved, @to_cset can be NULL.
657  *
658  * This function automatically handles populated_cnt updates and
659  * css_task_iter adjustments but the caller is responsible for managing
660  * @from_cset and @to_cset's reference counts.
661  */
662 static void css_set_move_task(struct task_struct *task,
663                               struct css_set *from_cset, struct css_set *to_cset,
664                               bool use_mg_tasks)
665 {
666         lockdep_assert_held(&css_set_lock);
667 
668         if (from_cset) {
669                 struct css_task_iter *it, *pos;
670 
671                 WARN_ON_ONCE(list_empty(&task->cg_list));
672 
673                 /*
674                  * @task is leaving, advance task iterators which are
675                  * pointing to it so that they can resume at the next
676                  * position.  Advancing an iterator might remove it from
677                  * the list, use safe walk.  See css_task_iter_advance*()
678                  * for details.
679                  */
680                 list_for_each_entry_safe(it, pos, &from_cset->task_iters,
681                                          iters_node)
682                         if (it->task_pos == &task->cg_list)
683                                 css_task_iter_advance(it);
684 
685                 list_del_init(&task->cg_list);
686                 if (!css_set_populated(from_cset))
687                         css_set_update_populated(from_cset, false);
688         } else {
689                 WARN_ON_ONCE(!list_empty(&task->cg_list));
690         }
691 
692         if (to_cset) {
693                 /*
694                  * We are synchronized through cgroup_threadgroup_rwsem
695                  * against PF_EXITING setting such that we can't race
696                  * against cgroup_exit() changing the css_set to
697                  * init_css_set and dropping the old one.
698                  */
699                 WARN_ON_ONCE(task->flags & PF_EXITING);
700 
701                 if (!css_set_populated(to_cset))
702                         css_set_update_populated(to_cset, true);
703                 rcu_assign_pointer(task->cgroups, to_cset);
704                 list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks :
705                                                              &to_cset->tasks);
706         }
707 }
708 
709 /*
710  * hash table for cgroup groups. This improves the performance to find
711  * an existing css_set. This hash doesn't (currently) take into
712  * account cgroups in empty hierarchies.
713  */
714 #define CSS_SET_HASH_BITS       7
715 static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
716 
717 static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
718 {
719         unsigned long key = 0UL;
720         struct cgroup_subsys *ss;
721         int i;
722 
723         for_each_subsys(ss, i)
724                 key += (unsigned long)css[i];
725         key = (key >> 16) ^ key;
726 
727         return key;
728 }
729 
730 static void put_css_set_locked(struct css_set *cset)
731 {
732         struct cgrp_cset_link *link, *tmp_link;
733         struct cgroup_subsys *ss;
734         int ssid;
735 
736         lockdep_assert_held(&css_set_lock);
737 
738         if (!atomic_dec_and_test(&cset->refcount))
739                 return;
740 
741         /* This css_set is dead. unlink it and release cgroup and css refs */
742         for_each_subsys(ss, ssid) {
743                 list_del(&cset->e_cset_node[ssid]);
744                 css_put(cset->subsys[ssid]);
745         }
746         hash_del(&cset->hlist);
747         css_set_count--;
748 
749         list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
750                 list_del(&link->cset_link);
751                 list_del(&link->cgrp_link);
752                 if (cgroup_parent(link->cgrp))
753                         cgroup_put(link->cgrp);
754                 kfree(link);
755         }
756 
757         kfree_rcu(cset, rcu_head);
758 }
759 
760 static void put_css_set(struct css_set *cset)
761 {
762         /*
763          * Ensure that the refcount doesn't hit zero while any readers
764          * can see it. Similar to atomic_dec_and_lock(), but for an
765          * rwlock
766          */
767         if (atomic_add_unless(&cset->refcount, -1, 1))
768                 return;
769 
770         spin_lock_bh(&css_set_lock);
771         put_css_set_locked(cset);
772         spin_unlock_bh(&css_set_lock);
773 }
774 
775 /*
776  * refcounted get/put for css_set objects
777  */
778 static inline void get_css_set(struct css_set *cset)
779 {
780         atomic_inc(&cset->refcount);
781 }
782 
783 /**
784  * compare_css_sets - helper function for find_existing_css_set().
785  * @cset: candidate css_set being tested
786  * @old_cset: existing css_set for a task
787  * @new_cgrp: cgroup that's being entered by the task
788  * @template: desired set of css pointers in css_set (pre-calculated)
789  *
790  * Returns true if "cset" matches "old_cset" except for the hierarchy
791  * which "new_cgrp" belongs to, for which it should match "new_cgrp".
792  */
793 static bool compare_css_sets(struct css_set *cset,
794                              struct css_set *old_cset,
795                              struct cgroup *new_cgrp,
796                              struct cgroup_subsys_state *template[])
797 {
798         struct list_head *l1, *l2;
799 
800         /*
801          * On the default hierarchy, there can be csets which are
802          * associated with the same set of cgroups but different csses.
803          * Let's first ensure that csses match.
804          */
805         if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
806                 return false;
807 
808         /*
809          * Compare cgroup pointers in order to distinguish between
810          * different cgroups in hierarchies.  As different cgroups may
811          * share the same effective css, this comparison is always
812          * necessary.
813          */
814         l1 = &cset->cgrp_links;
815         l2 = &old_cset->cgrp_links;
816         while (1) {
817                 struct cgrp_cset_link *link1, *link2;
818                 struct cgroup *cgrp1, *cgrp2;
819 
820                 l1 = l1->next;
821                 l2 = l2->next;
822                 /* See if we reached the end - both lists are equal length. */
823                 if (l1 == &cset->cgrp_links) {
824                         BUG_ON(l2 != &old_cset->cgrp_links);
825                         break;
826                 } else {
827                         BUG_ON(l2 == &old_cset->cgrp_links);
828                 }
829                 /* Locate the cgroups associated with these links. */
830                 link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
831                 link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
832                 cgrp1 = link1->cgrp;
833                 cgrp2 = link2->cgrp;
834                 /* Hierarchies should be linked in the same order. */
835                 BUG_ON(cgrp1->root != cgrp2->root);
836 
837                 /*
838                  * If this hierarchy is the hierarchy of the cgroup
839                  * that's changing, then we need to check that this
840                  * css_set points to the new cgroup; if it's any other
841                  * hierarchy, then this css_set should point to the
842                  * same cgroup as the old css_set.
843                  */
844                 if (cgrp1->root == new_cgrp->root) {
845                         if (cgrp1 != new_cgrp)
846                                 return false;
847                 } else {
848                         if (cgrp1 != cgrp2)
849                                 return false;
850                 }
851         }
852         return true;
853 }
854 
855 /**
856  * find_existing_css_set - init css array and find the matching css_set
857  * @old_cset: the css_set that we're using before the cgroup transition
858  * @cgrp: the cgroup that we're moving into
859  * @template: out param for the new set of csses, should be clear on entry
860  */
861 static struct css_set *find_existing_css_set(struct css_set *old_cset,
862                                         struct cgroup *cgrp,
863                                         struct cgroup_subsys_state *template[])
864 {
865         struct cgroup_root *root = cgrp->root;
866         struct cgroup_subsys *ss;
867         struct css_set *cset;
868         unsigned long key;
869         int i;
870 
871         /*
872          * Build the set of subsystem state objects that we want to see in the
873          * new css_set. while subsystems can change globally, the entries here
874          * won't change, so no need for locking.
875          */
876         for_each_subsys(ss, i) {
877                 if (root->subsys_mask & (1UL << i)) {
878                         /*
879                          * @ss is in this hierarchy, so we want the
880                          * effective css from @cgrp.
881                          */
882                         template[i] = cgroup_e_css(cgrp, ss);
883                 } else {
884                         /*
885                          * @ss is not in this hierarchy, so we don't want
886                          * to change the css.
887                          */
888                         template[i] = old_cset->subsys[i];
889                 }
890         }
891 
892         key = css_set_hash(template);
893         hash_for_each_possible(css_set_table, cset, hlist, key) {
894                 if (!compare_css_sets(cset, old_cset, cgrp, template))
895                         continue;
896 
897                 /* This css_set matches what we need */
898                 return cset;
899         }
900 
901         /* No existing cgroup group matched */
902         return NULL;
903 }
904 
905 static void free_cgrp_cset_links(struct list_head *links_to_free)
906 {
907         struct cgrp_cset_link *link, *tmp_link;
908 
909         list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
910                 list_del(&link->cset_link);
911                 kfree(link);
912         }
913 }
914 
915 /**
916  * allocate_cgrp_cset_links - allocate cgrp_cset_links
917  * @count: the number of links to allocate
918  * @tmp_links: list_head the allocated links are put on
919  *
920  * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
921  * through ->cset_link.  Returns 0 on success or -errno.
922  */
923 static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
924 {
925         struct cgrp_cset_link *link;
926         int i;
927 
928         INIT_LIST_HEAD(tmp_links);
929 
930         for (i = 0; i < count; i++) {
931                 link = kzalloc(sizeof(*link), GFP_KERNEL);
932                 if (!link) {
933                         free_cgrp_cset_links(tmp_links);
934                         return -ENOMEM;
935                 }
936                 list_add(&link->cset_link, tmp_links);
937         }
938         return 0;
939 }
940 
941 /**
942  * link_css_set - a helper function to link a css_set to a cgroup
943  * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
944  * @cset: the css_set to be linked
945  * @cgrp: the destination cgroup
946  */
947 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
948                          struct cgroup *cgrp)
949 {
950         struct cgrp_cset_link *link;
951 
952         BUG_ON(list_empty(tmp_links));
953 
954         if (cgroup_on_dfl(cgrp))
955                 cset->dfl_cgrp = cgrp;
956 
957         link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
958         link->cset = cset;
959         link->cgrp = cgrp;
960 
961         /*
962          * Always add links to the tail of the lists so that the lists are
963          * in choronological order.
964          */
965         list_move_tail(&link->cset_link, &cgrp->cset_links);
966         list_add_tail(&link->cgrp_link, &cset->cgrp_links);
967 
968         if (cgroup_parent(cgrp))
969                 cgroup_get(cgrp);
970 }
971 
972 /**
973  * find_css_set - return a new css_set with one cgroup updated
974  * @old_cset: the baseline css_set
975  * @cgrp: the cgroup to be updated
976  *
977  * Return a new css_set that's equivalent to @old_cset, but with @cgrp
978  * substituted into the appropriate hierarchy.
979  */
980 static struct css_set *find_css_set(struct css_set *old_cset,
981                                     struct cgroup *cgrp)
982 {
983         struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
984         struct css_set *cset;
985         struct list_head tmp_links;
986         struct cgrp_cset_link *link;
987         struct cgroup_subsys *ss;
988         unsigned long key;
989         int ssid;
990 
991         lockdep_assert_held(&cgroup_mutex);
992 
993         /* First see if we already have a cgroup group that matches
994          * the desired set */
995         spin_lock_bh(&css_set_lock);
996         cset = find_existing_css_set(old_cset, cgrp, template);
997         if (cset)
998                 get_css_set(cset);
999         spin_unlock_bh(&css_set_lock);
1000 
1001         if (cset)
1002                 return cset;
1003 
1004         cset = kzalloc(sizeof(*cset), GFP_KERNEL);
1005         if (!cset)
1006                 return NULL;
1007 
1008         /* Allocate all the cgrp_cset_link objects that we'll need */
1009         if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
1010                 kfree(cset);
1011                 return NULL;
1012         }
1013 
1014         atomic_set(&cset->refcount, 1);
1015         INIT_LIST_HEAD(&cset->cgrp_links);
1016         INIT_LIST_HEAD(&cset->tasks);
1017         INIT_LIST_HEAD(&cset->mg_tasks);
1018         INIT_LIST_HEAD(&cset->mg_preload_node);
1019         INIT_LIST_HEAD(&cset->mg_node);
1020         INIT_LIST_HEAD(&cset->task_iters);
1021         INIT_HLIST_NODE(&cset->hlist);
1022 
1023         /* Copy the set of subsystem state objects generated in
1024          * find_existing_css_set() */
1025         memcpy(cset->subsys, template, sizeof(cset->subsys));
1026 
1027         spin_lock_bh(&css_set_lock);
1028         /* Add reference counts and links from the new css_set. */
1029         list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
1030                 struct cgroup *c = link->cgrp;
1031 
1032                 if (c->root == cgrp->root)
1033                         c = cgrp;
1034                 link_css_set(&tmp_links, cset, c);
1035         }
1036 
1037         BUG_ON(!list_empty(&tmp_links));
1038 
1039         css_set_count++;
1040 
1041         /* Add @cset to the hash table */
1042         key = css_set_hash(cset->subsys);
1043         hash_add(css_set_table, &cset->hlist, key);
1044 
1045         for_each_subsys(ss, ssid) {
1046                 struct cgroup_subsys_state *css = cset->subsys[ssid];
1047 
1048                 list_add_tail(&cset->e_cset_node[ssid],
1049                               &css->cgroup->e_csets[ssid]);
1050                 css_get(css);
1051         }
1052 
1053         spin_unlock_bh(&css_set_lock);
1054 
1055         return cset;
1056 }
1057 
1058 static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
1059 {
1060         struct cgroup *root_cgrp = kf_root->kn->priv;
1061 
1062         return root_cgrp->root;
1063 }
1064 
1065 static int cgroup_init_root_id(struct cgroup_root *root)
1066 {
1067         int id;
1068 
1069         lockdep_assert_held(&cgroup_mutex);
1070 
1071         id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
1072         if (id < 0)
1073                 return id;
1074 
1075         root->hierarchy_id = id;
1076         return 0;
1077 }
1078 
1079 static void cgroup_exit_root_id(struct cgroup_root *root)
1080 {
1081         lockdep_assert_held(&cgroup_mutex);
1082 
1083         if (root->hierarchy_id) {
1084                 idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
1085                 root->hierarchy_id = 0;
1086         }
1087 }
1088 
1089 static void cgroup_free_root(struct cgroup_root *root)
1090 {
1091         if (root) {
1092                 /* hierarchy ID should already have been released */
1093                 WARN_ON_ONCE(root->hierarchy_id);
1094 
1095                 idr_destroy(&root->cgroup_idr);
1096                 kfree(root);
1097         }
1098 }
1099 
1100 static void cgroup_destroy_root(struct cgroup_root *root)
1101 {
1102         struct cgroup *cgrp = &root->cgrp;
1103         struct cgrp_cset_link *link, *tmp_link;
1104 
1105         mutex_lock(&cgroup_mutex);
1106 
1107         BUG_ON(atomic_read(&root->nr_cgrps));
1108         BUG_ON(!list_empty(&cgrp->self.children));
1109 
1110         /* Rebind all subsystems back to the default hierarchy */
1111         rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
1112 
1113         /*
1114          * Release all the links from cset_links to this hierarchy's
1115          * root cgroup
1116          */
1117         spin_lock_bh(&css_set_lock);
1118 
1119         list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
1120                 list_del(&link->cset_link);
1121                 list_del(&link->cgrp_link);
1122                 kfree(link);
1123         }
1124 
1125         spin_unlock_bh(&css_set_lock);
1126 
1127         if (!list_empty(&root->root_list)) {
1128                 list_del(&root->root_list);
1129                 cgroup_root_count--;
1130         }
1131 
1132         cgroup_exit_root_id(root);
1133 
1134         mutex_unlock(&cgroup_mutex);
1135 
1136         kernfs_destroy_root(root->kf_root);
1137         cgroup_free_root(root);
1138 }
1139 
1140 /* look up cgroup associated with given css_set on the specified hierarchy */
1141 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
1142                                             struct cgroup_root *root)
1143 {
1144         struct cgroup *res = NULL;
1145 
1146         lockdep_assert_held(&cgroup_mutex);
1147         lockdep_assert_held(&css_set_lock);
1148 
1149         if (cset == &init_css_set) {
1150                 res = &root->cgrp;
1151         } else {
1152                 struct cgrp_cset_link *link;
1153 
1154                 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
1155                         struct cgroup *c = link->cgrp;
1156 
1157                         if (c->root == root) {
1158                                 res = c;
1159                                 break;
1160                         }
1161                 }
1162         }
1163 
1164         BUG_ON(!res);
1165         return res;
1166 }
1167 
1168 /*
1169  * Return the cgroup for "task" from the given hierarchy. Must be
1170  * called with cgroup_mutex and css_set_lock held.
1171  */
1172 static struct cgroup *task_cgroup_from_root(struct task_struct *task,
1173                                             struct cgroup_root *root)
1174 {
1175         /*
1176          * No need to lock the task - since we hold cgroup_mutex the
1177          * task can't change groups, so the only thing that can happen
1178          * is that it exits and its css is set back to init_css_set.
1179          */
1180         return cset_cgroup_from_root(task_css_set(task), root);
1181 }
1182 
1183 /*
1184  * A task must hold cgroup_mutex to modify cgroups.
1185  *
1186  * Any task can increment and decrement the count field without lock.
1187  * So in general, code holding cgroup_mutex can't rely on the count
1188  * field not changing.  However, if the count goes to zero, then only
1189  * cgroup_attach_task() can increment it again.  Because a count of zero
1190  * means that no tasks are currently attached, therefore there is no
1191  * way a task attached to that cgroup can fork (the other way to
1192  * increment the count).  So code holding cgroup_mutex can safely
1193  * assume that if the count is zero, it will stay zero. Similarly, if
1194  * a task holds cgroup_mutex on a cgroup with zero count, it
1195  * knows that the cgroup won't be removed, as cgroup_rmdir()
1196  * needs that mutex.
1197  *
1198  * A cgroup can only be deleted if both its 'count' of using tasks
1199  * is zero, and its list of 'children' cgroups is empty.  Since all
1200  * tasks in the system use _some_ cgroup, and since there is always at
1201  * least one task in the system (init, pid == 1), therefore, root cgroup
1202  * always has either children cgroups and/or using tasks.  So we don't
1203  * need a special hack to ensure that root cgroup cannot be deleted.
1204  *
1205  * P.S.  One more locking exception.  RCU is used to guard the
1206  * update of a tasks cgroup pointer by cgroup_attach_task()
1207  */
1208 
1209 static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
1210 static const struct file_operations proc_cgroupstats_operations;
1211 
1212 static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
1213                               char *buf)
1214 {
1215         struct cgroup_subsys *ss = cft->ss;
1216 
1217         if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
1218             !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
1219                 snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
1220                          cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name,
1221                          cft->name);
1222         else
1223                 strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
1224         return buf;
1225 }
1226 
1227 /**
1228  * cgroup_file_mode - deduce file mode of a control file
1229  * @cft: the control file in question
1230  *
1231  * S_IRUGO for read, S_IWUSR for write.
1232  */
1233 static umode_t cgroup_file_mode(const struct cftype *cft)
1234 {
1235         umode_t mode = 0;
1236 
1237         if (cft->read_u64 || cft->read_s64 || cft->seq_show)
1238                 mode |= S_IRUGO;
1239 
1240         if (cft->write_u64 || cft->write_s64 || cft->write) {
1241                 if (cft->flags & CFTYPE_WORLD_WRITABLE)
1242                         mode |= S_IWUGO;
1243                 else
1244                         mode |= S_IWUSR;
1245         }
1246 
1247         return mode;
1248 }
1249 
1250 /**
1251  * cgroup_calc_child_subsys_mask - calculate child_subsys_mask
1252  * @cgrp: the target cgroup
1253  * @subtree_control: the new subtree_control mask to consider
1254  *
1255  * On the default hierarchy, a subsystem may request other subsystems to be
1256  * enabled together through its ->depends_on mask.  In such cases, more
1257  * subsystems than specified in "cgroup.subtree_control" may be enabled.
1258  *
1259  * This function calculates which subsystems need to be enabled if
1260  * @subtree_control is to be applied to @cgrp.  The returned mask is always
1261  * a superset of @subtree_control and follows the usual hierarchy rules.
1262  */
1263 static unsigned long cgroup_calc_child_subsys_mask(struct cgroup *cgrp,
1264                                                   unsigned long subtree_control)
1265 {
1266         struct cgroup *parent = cgroup_parent(cgrp);
1267         unsigned long cur_ss_mask = subtree_control;
1268         struct cgroup_subsys *ss;
1269         int ssid;
1270 
1271         lockdep_assert_held(&cgroup_mutex);
1272 
1273         if (!cgroup_on_dfl(cgrp))
1274                 return cur_ss_mask;
1275 
1276         while (true) {
1277                 unsigned long new_ss_mask = cur_ss_mask;
1278 
1279                 for_each_subsys_which(ss, ssid, &cur_ss_mask)
1280                         new_ss_mask |= ss->depends_on;
1281 
1282                 /*
1283                  * Mask out subsystems which aren't available.  This can
1284                  * happen only if some depended-upon subsystems were bound
1285                  * to non-default hierarchies.
1286                  */
1287                 if (parent)
1288                         new_ss_mask &= parent->child_subsys_mask;
1289                 else
1290                         new_ss_mask &= cgrp->root->subsys_mask;
1291 
1292                 if (new_ss_mask == cur_ss_mask)
1293                         break;
1294                 cur_ss_mask = new_ss_mask;
1295         }
1296 
1297         return cur_ss_mask;
1298 }
1299 
1300 /**
1301  * cgroup_refresh_child_subsys_mask - update child_subsys_mask
1302  * @cgrp: the target cgroup
1303  *
1304  * Update @cgrp->child_subsys_mask according to the current
1305  * @cgrp->subtree_control using cgroup_calc_child_subsys_mask().
1306  */
1307 static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp)
1308 {
1309         cgrp->child_subsys_mask =
1310                 cgroup_calc_child_subsys_mask(cgrp, cgrp->subtree_control);
1311 }
1312 
1313 /**
1314  * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1315  * @kn: the kernfs_node being serviced
1316  *
1317  * This helper undoes cgroup_kn_lock_live() and should be invoked before
1318  * the method finishes if locking succeeded.  Note that once this function
1319  * returns the cgroup returned by cgroup_kn_lock_live() may become
1320  * inaccessible any time.  If the caller intends to continue to access the
1321  * cgroup, it should pin it before invoking this function.
1322  */
1323 static void cgroup_kn_unlock(struct kernfs_node *kn)
1324 {
1325         struct cgroup *cgrp;
1326 
1327         if (kernfs_type(kn) == KERNFS_DIR)
1328                 cgrp = kn->priv;
1329         else
1330                 cgrp = kn->parent->priv;
1331 
1332         mutex_unlock(&cgroup_mutex);
1333 
1334         kernfs_unbreak_active_protection(kn);
1335         cgroup_put(cgrp);
1336 }
1337 
1338 /**
1339  * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1340  * @kn: the kernfs_node being serviced
1341  *
1342  * This helper is to be used by a cgroup kernfs method currently servicing
1343  * @kn.  It breaks the active protection, performs cgroup locking and
1344  * verifies that the associated cgroup is alive.  Returns the cgroup if
1345  * alive; otherwise, %NULL.  A successful return should be undone by a
1346  * matching cgroup_kn_unlock() invocation.
1347  *
1348  * Any cgroup kernfs method implementation which requires locking the
1349  * associated cgroup should use this helper.  It avoids nesting cgroup
1350  * locking under kernfs active protection and allows all kernfs operations
1351  * including self-removal.
1352  */
1353 static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn)
1354 {
1355         struct cgroup *cgrp;
1356 
1357         if (kernfs_type(kn) == KERNFS_DIR)
1358                 cgrp = kn->priv;
1359         else
1360                 cgrp = kn->parent->priv;
1361 
1362         /*
1363          * We're gonna grab cgroup_mutex which nests outside kernfs
1364          * active_ref.  cgroup liveliness check alone provides enough
1365          * protection against removal.  Ensure @cgrp stays accessible and
1366          * break the active_ref protection.
1367          */
1368         if (!cgroup_tryget(cgrp))
1369                 return NULL;
1370         kernfs_break_active_protection(kn);
1371 
1372         mutex_lock(&cgroup_mutex);
1373 
1374         if (!cgroup_is_dead(cgrp))
1375                 return cgrp;
1376 
1377         cgroup_kn_unlock(kn);
1378         return NULL;
1379 }
1380 
1381 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
1382 {
1383         char name[CGROUP_FILE_NAME_MAX];
1384 
1385         lockdep_assert_held(&cgroup_mutex);
1386 
1387         if (cft->file_offset) {
1388                 struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);
1389                 struct cgroup_file *cfile = (void *)css + cft->file_offset;
1390 
1391                 spin_lock_irq(&cgroup_file_kn_lock);
1392                 cfile->kn = NULL;
1393                 spin_unlock_irq(&cgroup_file_kn_lock);
1394         }
1395 
1396         kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
1397 }
1398 
1399 /**
1400  * css_clear_dir - remove subsys files in a cgroup directory
1401  * @css: taget css
1402  * @cgrp_override: specify if target cgroup is different from css->cgroup
1403  */
1404 static void css_clear_dir(struct cgroup_subsys_state *css,
1405                           struct cgroup *cgrp_override)
1406 {
1407         struct cgroup *cgrp = cgrp_override ?: css->cgroup;
1408         struct cftype *cfts;
1409 
1410         list_for_each_entry(cfts, &css->ss->cfts, node)
1411                 cgroup_addrm_files(css, cgrp, cfts, false);
1412 }
1413 
1414 /**
1415  * css_populate_dir - create subsys files in a cgroup directory
1416  * @css: target css
1417  * @cgrp_overried: specify if target cgroup is different from css->cgroup
1418  *
1419  * On failure, no file is added.
1420  */
1421 static int css_populate_dir(struct cgroup_subsys_state *css,
1422                             struct cgroup *cgrp_override)
1423 {
1424         struct cgroup *cgrp = cgrp_override ?: css->cgroup;
1425         struct cftype *cfts, *failed_cfts;
1426         int ret;
1427 
1428         if (!css->ss) {
1429                 if (cgroup_on_dfl(cgrp))
1430                         cfts = cgroup_dfl_base_files;
1431                 else
1432                         cfts = cgroup_legacy_base_files;
1433 
1434                 return cgroup_addrm_files(&cgrp->self, cgrp, cfts, true);
1435         }
1436 
1437         list_for_each_entry(cfts, &css->ss->cfts, node) {
1438                 ret = cgroup_addrm_files(css, cgrp, cfts, true);
1439                 if (ret < 0) {
1440                         failed_cfts = cfts;
1441                         goto err;
1442                 }
1443         }
1444         return 0;
1445 err:
1446         list_for_each_entry(cfts, &css->ss->cfts, node) {
1447                 if (cfts == failed_cfts)
1448                         break;
1449                 cgroup_addrm_files(css, cgrp, cfts, false);
1450         }
1451         return ret;
1452 }
1453 
1454 static int rebind_subsystems(struct cgroup_root *dst_root,
1455                              unsigned long ss_mask)
1456 {
1457         struct cgroup *dcgrp = &dst_root->cgrp;
1458         struct cgroup_subsys *ss;
1459         unsigned long tmp_ss_mask;
1460         int ssid, i, ret;
1461 
1462         lockdep_assert_held(&cgroup_mutex);
1463 
1464         for_each_subsys_which(ss, ssid, &ss_mask) {
1465                 /* if @ss has non-root csses attached to it, can't move */
1466                 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)))
1467                         return -EBUSY;
1468 
1469                 /* can't move between two non-dummy roots either */
1470                 if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1471                         return -EBUSY;
1472         }
1473 
1474         /* skip creating root files on dfl_root for inhibited subsystems */
1475         tmp_ss_mask = ss_mask;
1476         if (dst_root == &cgrp_dfl_root)
1477                 tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask;
1478 
1479         for_each_subsys_which(ss, ssid, &tmp_ss_mask) {
1480                 struct cgroup *scgrp = &ss->root->cgrp;
1481                 int tssid;
1482 
1483                 ret = css_populate_dir(cgroup_css(scgrp, ss), dcgrp);
1484                 if (!ret)
1485                         continue;
1486 
1487                 /*
1488                  * Rebinding back to the default root is not allowed to
1489                  * fail.  Using both default and non-default roots should
1490                  * be rare.  Moving subsystems back and forth even more so.
1491                  * Just warn about it and continue.
1492                  */
1493                 if (dst_root == &cgrp_dfl_root) {
1494                         if (cgrp_dfl_root_visible) {
1495                                 pr_warn("failed to create files (%d) while rebinding 0x%lx to default root\n",
1496                                         ret, ss_mask);
1497                                 pr_warn("you may retry by moving them to a different hierarchy and unbinding\n");
1498                         }
1499                         continue;
1500                 }
1501 
1502                 for_each_subsys_which(ss, tssid, &tmp_ss_mask) {
1503                         if (tssid == ssid)
1504                                 break;
1505                         css_clear_dir(cgroup_css(scgrp, ss), dcgrp);
1506                 }
1507                 return ret;
1508         }
1509 
1510         /*
1511          * Nothing can fail from this point on.  Remove files for the
1512          * removed subsystems and rebind each subsystem.
1513          */
1514         for_each_subsys_which(ss, ssid, &ss_mask) {
1515                 struct cgroup_root *src_root = ss->root;
1516                 struct cgroup *scgrp = &src_root->cgrp;
1517                 struct cgroup_subsys_state *css = cgroup_css(scgrp, ss);
1518                 struct css_set *cset;
1519 
1520                 WARN_ON(!css || cgroup_css(dcgrp, ss));
1521 
1522                 css_clear_dir(css, NULL);
1523 
1524                 RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);
1525                 rcu_assign_pointer(dcgrp->subsys[ssid], css);
1526                 ss->root = dst_root;
1527                 css->cgroup = dcgrp;
1528 
1529                 spin_lock_bh(&css_set_lock);
1530                 hash_for_each(css_set_table, i, cset, hlist)
1531                         list_move_tail(&cset->e_cset_node[ss->id],
1532                                        &dcgrp->e_csets[ss->id]);
1533                 spin_unlock_bh(&css_set_lock);
1534 
1535                 src_root->subsys_mask &= ~(1 << ssid);
1536                 scgrp->subtree_control &= ~(1 << ssid);
1537                 cgroup_refresh_child_subsys_mask(scgrp);
1538 
1539                 /* default hierarchy doesn't enable controllers by default */
1540                 dst_root->subsys_mask |= 1 << ssid;
1541                 if (dst_root == &cgrp_dfl_root) {
1542                         static_branch_enable(cgroup_subsys_on_dfl_key[ssid]);
1543                 } else {
1544                         dcgrp->subtree_control |= 1 << ssid;
1545                         cgroup_refresh_child_subsys_mask(dcgrp);
1546                         static_branch_disable(cgroup_subsys_on_dfl_key[ssid]);
1547                 }
1548 
1549                 if (ss->bind)
1550                         ss->bind(css);
1551         }
1552 
1553         kernfs_activate(dcgrp->kn);
1554         return 0;
1555 }
1556 
1557 static int cgroup_show_options(struct seq_file *seq,
1558                                struct kernfs_root *kf_root)
1559 {
1560         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1561         struct cgroup_subsys *ss;
1562         int ssid;
1563 
1564         if (root != &cgrp_dfl_root)
1565                 for_each_subsys(ss, ssid)
1566                         if (root->subsys_mask & (1 << ssid))
1567                                 seq_show_option(seq, ss->legacy_name, NULL);
1568         if (root->flags & CGRP_ROOT_NOPREFIX)
1569                 seq_puts(seq, ",noprefix");
1570         if (root->flags & CGRP_ROOT_XATTR)
1571                 seq_puts(seq, ",xattr");
1572 
1573         spin_lock(&release_agent_path_lock);
1574         if (strlen(root->release_agent_path))
1575                 seq_show_option(seq, "release_agent",
1576                                 root->release_agent_path);
1577         spin_unlock(&release_agent_path_lock);
1578 
1579         if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
1580                 seq_puts(seq, ",clone_children");
1581         if (strlen(root->name))
1582                 seq_show_option(seq, "name", root->name);
1583         return 0;
1584 }
1585 
1586 struct cgroup_sb_opts {
1587         unsigned long subsys_mask;
1588         unsigned int flags;
1589         char *release_agent;
1590         bool cpuset_clone_children;
1591         char *name;
1592         /* User explicitly requested empty subsystem */
1593         bool none;
1594 };
1595 
1596 static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1597 {
1598         char *token, *o = data;
1599         bool all_ss = false, one_ss = false;
1600         unsigned long mask = -1UL;
1601         struct cgroup_subsys *ss;
1602         int nr_opts = 0;
1603         int i;
1604 
1605 #ifdef CONFIG_CPUSETS
1606         mask = ~(1U << cpuset_cgrp_id);
1607 #endif
1608 
1609         memset(opts, 0, sizeof(*opts));
1610 
1611         while ((token = strsep(&o, ",")) != NULL) {
1612                 nr_opts++;
1613 
1614                 if (!*token)
1615                         return -EINVAL;
1616                 if (!strcmp(token, "none")) {
1617                         /* Explicitly have no subsystems */
1618                         opts->none = true;
1619                         continue;
1620                 }
1621                 if (!strcmp(token, "all")) {
1622                         /* Mutually exclusive option 'all' + subsystem name */
1623                         if (one_ss)
1624                                 return -EINVAL;
1625                         all_ss = true;
1626                         continue;
1627                 }
1628                 if (!strcmp(token, "noprefix")) {
1629                         opts->flags |= CGRP_ROOT_NOPREFIX;
1630                         continue;
1631                 }
1632                 if (!strcmp(token, "clone_children")) {
1633                         opts->cpuset_clone_children = true;
1634                         continue;
1635                 }
1636                 if (!strcmp(token, "xattr")) {
1637                         opts->flags |= CGRP_ROOT_XATTR;
1638                         continue;
1639                 }
1640                 if (!strncmp(token, "release_agent=", 14)) {
1641                         /* Specifying two release agents is forbidden */
1642                         if (opts->release_agent)
1643                                 return -EINVAL;
1644                         opts->release_agent =
1645                                 kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1646                         if (!opts->release_agent)
1647                                 return -ENOMEM;
1648                         continue;
1649                 }
1650                 if (!strncmp(token, "name=", 5)) {
1651                         const char *name = token + 5;
1652                         /* Can't specify an empty name */
1653                         if (!strlen(name))
1654                                 return -EINVAL;
1655                         /* Must match [\w.-]+ */
1656                         for (i = 0; i < strlen(name); i++) {
1657                                 char c = name[i];
1658                                 if (isalnum(c))
1659                                         continue;
1660                                 if ((c == '.') || (c == '-') || (c == '_'))
1661                                         continue;
1662                                 return -EINVAL;
1663                         }
1664                         /* Specifying two names is forbidden */
1665                         if (opts->name)
1666                                 return -EINVAL;
1667                         opts->name = kstrndup(name,
1668                                               MAX_CGROUP_ROOT_NAMELEN - 1,
1669                                               GFP_KERNEL);
1670                         if (!opts->name)
1671                                 return -ENOMEM;
1672 
1673                         continue;
1674                 }
1675 
1676                 for_each_subsys(ss, i) {
1677                         if (strcmp(token, ss->legacy_name))
1678                                 continue;
1679                         if (!cgroup_ssid_enabled(i))
1680                                 continue;
1681 
1682                         /* Mutually exclusive option 'all' + subsystem name */
1683                         if (all_ss)
1684                                 return -EINVAL;
1685                         opts->subsys_mask |= (1 << i);
1686                         one_ss = true;
1687 
1688                         break;
1689                 }
1690                 if (i == CGROUP_SUBSYS_COUNT)
1691                         return -ENOENT;
1692         }
1693 
1694         /*
1695          * If the 'all' option was specified select all the subsystems,
1696          * otherwise if 'none', 'name=' and a subsystem name options were
1697          * not specified, let's default to 'all'
1698          */
1699         if (all_ss || (!one_ss && !opts->none && !opts->name))
1700                 for_each_subsys(ss, i)
1701                         if (cgroup_ssid_enabled(i))
1702                                 opts->subsys_mask |= (1 << i);
1703 
1704         /*
1705          * We either have to specify by name or by subsystems. (So all
1706          * empty hierarchies must have a name).
1707          */
1708         if (!opts->subsys_mask && !opts->name)
1709                 return -EINVAL;
1710 
1711         /*
1712          * Option noprefix was introduced just for backward compatibility
1713          * with the old cpuset, so we allow noprefix only if mounting just
1714          * the cpuset subsystem.
1715          */
1716         if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1717                 return -EINVAL;
1718 
1719         /* Can't specify "none" and some subsystems */
1720         if (opts->subsys_mask && opts->none)
1721                 return -EINVAL;
1722 
1723         return 0;
1724 }
1725 
1726 static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1727 {
1728         int ret = 0;
1729         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1730         struct cgroup_sb_opts opts;
1731         unsigned long added_mask, removed_mask;
1732 
1733         if (root == &cgrp_dfl_root) {
1734                 pr_err("remount is not allowed\n");
1735                 return -EINVAL;
1736         }
1737 
1738         mutex_lock(&cgroup_mutex);
1739 
1740         /* See what subsystems are wanted */
1741         ret = parse_cgroupfs_options(data, &opts);
1742         if (ret)
1743                 goto out_unlock;
1744 
1745         if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1746                 pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
1747                         task_tgid_nr(current), current->comm);
1748 
1749         added_mask = opts.subsys_mask & ~root->subsys_mask;
1750         removed_mask = root->subsys_mask & ~opts.subsys_mask;
1751 
1752         /* Don't allow flags or name to change at remount */
1753         if ((opts.flags ^ root->flags) ||
1754             (opts.name && strcmp(opts.name, root->name))) {
1755                 pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
1756                        opts.flags, opts.name ?: "", root->flags, root->name);
1757                 ret = -EINVAL;
1758                 goto out_unlock;
1759         }
1760 
1761         /* remounting is not allowed for populated hierarchies */
1762         if (!list_empty(&root->cgrp.self.children)) {
1763                 ret = -EBUSY;
1764                 goto out_unlock;
1765         }
1766 
1767         ret = rebind_subsystems(root, added_mask);
1768         if (ret)
1769                 goto out_unlock;
1770 
1771         rebind_subsystems(&cgrp_dfl_root, removed_mask);
1772 
1773         if (opts.release_agent) {
1774                 spin_lock(&release_agent_path_lock);
1775                 strcpy(root->release_agent_path, opts.release_agent);
1776                 spin_unlock(&release_agent_path_lock);
1777         }
1778  out_unlock:
1779         kfree(opts.release_agent);
1780         kfree(opts.name);
1781         mutex_unlock(&cgroup_mutex);
1782         return ret;
1783 }
1784 
1785 /*
1786  * To reduce the fork() overhead for systems that are not actually using
1787  * their cgroups capability, we don't maintain the lists running through
1788  * each css_set to its tasks until we see the list actually used - in other
1789  * words after the first mount.
1790  */
1791 static bool use_task_css_set_links __read_mostly;
1792 
1793 static void cgroup_enable_task_cg_lists(void)
1794 {
1795         struct task_struct *p, *g;
1796 
1797         spin_lock_bh(&css_set_lock);
1798 
1799         if (use_task_css_set_links)
1800                 goto out_unlock;
1801 
1802         use_task_css_set_links = true;
1803 
1804         /*
1805          * We need tasklist_lock because RCU is not safe against
1806          * while_each_thread(). Besides, a forking task that has passed
1807          * cgroup_post_fork() without seeing use_task_css_set_links = 1
1808          * is not guaranteed to have its child immediately visible in the
1809          * tasklist if we walk through it with RCU.
1810          */
1811         read_lock(&tasklist_lock);
1812         do_each_thread(g, p) {
1813                 WARN_ON_ONCE(!list_empty(&p->cg_list) ||
1814                              task_css_set(p) != &init_css_set);
1815 
1816                 /*
1817                  * We should check if the process is exiting, otherwise
1818                  * it will race with cgroup_exit() in that the list
1819                  * entry won't be deleted though the process has exited.
1820                  * Do it while holding siglock so that we don't end up
1821                  * racing against cgroup_exit().
1822                  */
1823                 spin_lock_irq(&p->sighand->siglock);
1824                 if (!(p->flags & PF_EXITING)) {
1825                         struct css_set *cset = task_css_set(p);
1826 
1827                         if (!css_set_populated(cset))
1828                                 css_set_update_populated(cset, true);
1829                         list_add_tail(&p->cg_list, &cset->tasks);
1830                         get_css_set(cset);
1831                 }
1832                 spin_unlock_irq(&p->sighand->siglock);
1833         } while_each_thread(g, p);
1834         read_unlock(&tasklist_lock);
1835 out_unlock:
1836         spin_unlock_bh(&css_set_lock);
1837 }
1838 
1839 static void init_cgroup_housekeeping(struct cgroup *cgrp)
1840 {
1841         struct cgroup_subsys *ss;
1842         int ssid;
1843 
1844         INIT_LIST_HEAD(&cgrp->self.sibling);
1845         INIT_LIST_HEAD(&cgrp->self.children);
1846         INIT_LIST_HEAD(&cgrp->cset_links);
1847         INIT_LIST_HEAD(&cgrp->pidlists);
1848         mutex_init(&cgrp->pidlist_mutex);
1849         cgrp->self.cgroup = cgrp;
1850         cgrp->self.flags |= CSS_ONLINE;
1851 
1852         for_each_subsys(ss, ssid)
1853                 INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1854 
1855         init_waitqueue_head(&cgrp->offline_waitq);
1856         INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent);
1857 }
1858 
1859 static void init_cgroup_root(struct cgroup_root *root,
1860                              struct cgroup_sb_opts *opts)
1861 {
1862         struct cgroup *cgrp = &root->cgrp;
1863 
1864         INIT_LIST_HEAD(&root->root_list);
1865         atomic_set(&root->nr_cgrps, 1);
1866         cgrp->root = root;
1867         init_cgroup_housekeeping(cgrp);
1868         idr_init(&root->cgroup_idr);
1869 
1870         root->flags = opts->flags;
1871         if (opts->release_agent)
1872                 strcpy(root->release_agent_path, opts->release_agent);
1873         if (opts->name)
1874                 strcpy(root->name, opts->name);
1875         if (opts->cpuset_clone_children)
1876                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1877 }
1878 
1879 static int cgroup_setup_root(struct cgroup_root *root, unsigned long ss_mask)
1880 {
1881         LIST_HEAD(tmp_links);
1882         struct cgroup *root_cgrp = &root->cgrp;
1883         struct css_set *cset;
1884         int i, ret;
1885 
1886         lockdep_assert_held(&cgroup_mutex);
1887 
1888         ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL);
1889         if (ret < 0)
1890                 goto out;
1891         root_cgrp->id = ret;
1892         root_cgrp->ancestor_ids[0] = ret;
1893 
1894         ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0,
1895                               GFP_KERNEL);
1896         if (ret)
1897                 goto out;
1898 
1899         /*
1900          * We're accessing css_set_count without locking css_set_lock here,
1901          * but that's OK - it can only be increased by someone holding
1902          * cgroup_lock, and that's us. The worst that can happen is that we
1903          * have some link structures left over
1904          */
1905         ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
1906         if (ret)
1907                 goto cancel_ref;
1908 
1909         ret = cgroup_init_root_id(root);
1910         if (ret)
1911                 goto cancel_ref;
1912 
1913         root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
1914                                            KERNFS_ROOT_CREATE_DEACTIVATED,
1915                                            root_cgrp);
1916         if (IS_ERR(root->kf_root)) {
1917                 ret = PTR_ERR(root->kf_root);
1918                 goto exit_root_id;
1919         }
1920         root_cgrp->kn = root->kf_root->kn;
1921 
1922         ret = css_populate_dir(&root_cgrp->self, NULL);
1923         if (ret)
1924                 goto destroy_root;
1925 
1926         ret = rebind_subsystems(root, ss_mask);
1927         if (ret)
1928                 goto destroy_root;
1929 
1930         /*
1931          * There must be no failure case after here, since rebinding takes
1932          * care of subsystems' refcounts, which are explicitly dropped in
1933          * the failure exit path.
1934          */
1935         list_add(&root->root_list, &cgroup_roots);
1936         cgroup_root_count++;
1937 
1938         /*
1939          * Link the root cgroup in this hierarchy into all the css_set
1940          * objects.
1941          */
1942         spin_lock_bh(&css_set_lock);
1943         hash_for_each(css_set_table, i, cset, hlist) {
1944                 link_css_set(&tmp_links, cset, root_cgrp);
1945                 if (css_set_populated(cset))
1946                         cgroup_update_populated(root_cgrp, true);
1947         }
1948         spin_unlock_bh(&css_set_lock);
1949 
1950         BUG_ON(!list_empty(&root_cgrp->self.children));
1951         BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1952 
1953         kernfs_activate(root_cgrp->kn);
1954         ret = 0;
1955         goto out;
1956 
1957 destroy_root:
1958         kernfs_destroy_root(root->kf_root);
1959         root->kf_root = NULL;
1960 exit_root_id:
1961         cgroup_exit_root_id(root);
1962 cancel_ref:
1963         percpu_ref_exit(&root_cgrp->self.refcnt);
1964 out:
1965         free_cgrp_cset_links(&tmp_links);
1966         return ret;
1967 }
1968 
1969 static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1970                          int flags, const char *unused_dev_name,
1971                          void *data)
1972 {
1973         bool is_v2 = fs_type == &cgroup2_fs_type;
1974         struct super_block *pinned_sb = NULL;
1975         struct cgroup_subsys *ss;
1976         struct cgroup_root *root;
1977         struct cgroup_sb_opts opts;
1978         struct dentry *dentry;
1979         int ret;
1980         int i;
1981         bool new_sb;
1982 
1983         /*
1984          * The first time anyone tries to mount a cgroup, enable the list
1985          * linking each css_set to its tasks and fix up all existing tasks.
1986          */
1987         if (!use_task_css_set_links)
1988                 cgroup_enable_task_cg_lists();
1989 
1990         if (is_v2) {
1991                 if (data) {
1992                         pr_err("cgroup2: unknown option \"%s\"\n", (char *)data);
1993                         return ERR_PTR(-EINVAL);
1994                 }
1995                 cgrp_dfl_root_visible = true;
1996                 root = &cgrp_dfl_root;
1997                 cgroup_get(&root->cgrp);
1998                 goto out_mount;
1999         }
2000 
2001         mutex_lock(&cgroup_mutex);
2002 
2003         /* First find the desired set of subsystems */
2004         ret = parse_cgroupfs_options(data, &opts);
2005         if (ret)
2006                 goto out_unlock;
2007 
2008         /*
2009          * Destruction of cgroup root is asynchronous, so subsystems may
2010          * still be dying after the previous unmount.  Let's drain the
2011          * dying subsystems.  We just need to ensure that the ones
2012          * unmounted previously finish dying and don't care about new ones
2013          * starting.  Testing ref liveliness is good enough.
2014          */
2015         for_each_subsys(ss, i) {
2016                 if (!(opts.subsys_mask & (1 << i)) ||
2017                     ss->root == &cgrp_dfl_root)
2018                         continue;
2019 
2020                 if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) {
2021                         mutex_unlock(&cgroup_mutex);
2022                         msleep(10);
2023                         ret = restart_syscall();
2024                         goto out_free;
2025                 }
2026                 cgroup_put(&ss->root->cgrp);
2027         }
2028 
2029         for_each_root(root) {
2030                 bool name_match = false;
2031 
2032                 if (root == &cgrp_dfl_root)
2033                         continue;
2034 
2035                 /*
2036                  * If we asked for a name then it must match.  Also, if
2037                  * name matches but sybsys_mask doesn't, we should fail.
2038                  * Remember whether name matched.
2039                  */
2040                 if (opts.name) {
2041                         if (strcmp(opts.name, root->name))
2042                                 continue;
2043                         name_match = true;
2044                 }
2045 
2046                 /*
2047                  * If we asked for subsystems (or explicitly for no
2048                  * subsystems) then they must match.
2049                  */
2050                 if ((opts.subsys_mask || opts.none) &&
2051                     (opts.subsys_mask != root->subsys_mask)) {
2052                         if (!name_match)
2053                                 continue;
2054                         ret = -EBUSY;
2055                         goto out_unlock;
2056                 }
2057 
2058                 if (root->flags ^ opts.flags)
2059                         pr_warn("new mount options do not match the existing superblock, will be ignored\n");
2060 
2061                 /*
2062                  * We want to reuse @root whose lifetime is governed by its
2063                  * ->cgrp.  Let's check whether @root is alive and keep it
2064                  * that way.  As cgroup_kill_sb() can happen anytime, we
2065                  * want to block it by pinning the sb so that @root doesn't
2066                  * get killed before mount is complete.
2067                  *
2068                  * With the sb pinned, tryget_live can reliably indicate
2069                  * whether @root can be reused.  If it's being killed,
2070                  * drain it.  We can use wait_queue for the wait but this
2071                  * path is super cold.  Let's just sleep a bit and retry.
2072                  */
2073                 pinned_sb = kernfs_pin_sb(root->kf_root, NULL);
2074                 if (IS_ERR(pinned_sb) ||
2075                     !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
2076                         mutex_unlock(&cgroup_mutex);
2077                         if (!IS_ERR_OR_NULL(pinned_sb))
2078                                 deactivate_super(pinned_sb);
2079                         msleep(10);
2080                         ret = restart_syscall();
2081                         goto out_free;
2082                 }
2083 
2084                 ret = 0;
2085                 goto out_unlock;
2086         }
2087 
2088         /*
2089          * No such thing, create a new one.  name= matching without subsys
2090          * specification is allowed for already existing hierarchies but we
2091          * can't create new one without subsys specification.
2092          */
2093         if (!opts.subsys_mask && !opts.none) {
2094                 ret = -EINVAL;
2095                 goto out_unlock;
2096         }
2097 
2098         root = kzalloc(sizeof(*root), GFP_KERNEL);
2099         if (!root) {
2100                 ret = -ENOMEM;
2101                 goto out_unlock;
2102         }
2103 
2104         init_cgroup_root(root, &opts);
2105 
2106         ret = cgroup_setup_root(root, opts.subsys_mask);
2107         if (ret)
2108                 cgroup_free_root(root);
2109 
2110 out_unlock:
2111         mutex_unlock(&cgroup_mutex);
2112 out_free:
2113         kfree(opts.release_agent);
2114         kfree(opts.name);
2115 
2116         if (ret)
2117                 return ERR_PTR(ret);
2118 out_mount:
2119         dentry = kernfs_mount(fs_type, flags, root->kf_root,
2120                               is_v2 ? CGROUP2_SUPER_MAGIC : CGROUP_SUPER_MAGIC,
2121                               &new_sb);
2122         if (IS_ERR(dentry) || !new_sb)
2123                 cgroup_put(&root->cgrp);
2124 
2125         /*
2126          * If @pinned_sb, we're reusing an existing root and holding an
2127          * extra ref on its sb.  Mount is complete.  Put the extra ref.
2128          */
2129         if (pinned_sb) {
2130                 WARN_ON(new_sb);
2131                 deactivate_super(pinned_sb);
2132         }
2133 
2134         return dentry;
2135 }
2136 
2137 static void cgroup_kill_sb(struct super_block *sb)
2138 {
2139         struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
2140         struct cgroup_root *root = cgroup_root_from_kf(kf_root);
2141 
2142         /*
2143          * If @root doesn't have any mounts or children, start killing it.
2144          * This prevents new mounts by disabling percpu_ref_tryget_live().
2145          * cgroup_mount() may wait for @root's release.
2146          *
2147          * And don't kill the default root.
2148          */
2149         if (!list_empty(&root->cgrp.self.children) ||
2150             root == &cgrp_dfl_root)
2151                 cgroup_put(&root->cgrp);
2152         else
2153                 percpu_ref_kill(&root->cgrp.self.refcnt);
2154 
2155         kernfs_kill_sb(sb);
2156 }
2157 
2158 static struct file_system_type cgroup_fs_type = {
2159         .name = "cgroup",
2160         .mount = cgroup_mount,
2161         .kill_sb = cgroup_kill_sb,
2162 };
2163 
2164 static struct file_system_type cgroup2_fs_type = {
2165         .name = "cgroup2",
2166         .mount = cgroup_mount,
2167         .kill_sb = cgroup_kill_sb,
2168 };
2169 
2170 /**
2171  * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
2172  * @task: target task
2173  * @buf: the buffer to write the path into
2174  * @buflen: the length of the buffer
2175  *
2176  * Determine @task's cgroup on the first (the one with the lowest non-zero
2177  * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
2178  * function grabs cgroup_mutex and shouldn't be used inside locks used by
2179  * cgroup controller callbacks.
2180  *
2181  * Return value is the same as kernfs_path().
2182  */
2183 char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
2184 {
2185         struct cgroup_root *root;
2186         struct cgroup *cgrp;
2187         int hierarchy_id = 1;
2188         char *path = NULL;
2189 
2190         mutex_lock(&cgroup_mutex);
2191         spin_lock_bh(&css_set_lock);
2192 
2193         root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
2194 
2195         if (root) {
2196                 cgrp = task_cgroup_from_root(task, root);
2197                 path = cgroup_path(cgrp, buf, buflen);
2198         } else {
2199                 /* if no hierarchy exists, everyone is in "/" */
2200                 if (strlcpy(buf, "/", buflen) < buflen)
2201                         path = buf;
2202         }
2203 
2204         spin_unlock_bh(&css_set_lock);
2205         mutex_unlock(&cgroup_mutex);
2206         return path;
2207 }
2208 EXPORT_SYMBOL_GPL(task_cgroup_path);
2209 
2210 /* used to track tasks and other necessary states during migration */
2211 struct cgroup_taskset {
2212         /* the src and dst cset list running through cset->mg_node */
2213         struct list_head        src_csets;
2214         struct list_head        dst_csets;
2215 
2216         /* the subsys currently being processed */
2217         int                     ssid;
2218 
2219         /*
2220          * Fields for cgroup_taskset_*() iteration.
2221          *
2222          * Before migration is committed, the target migration tasks are on
2223          * ->mg_tasks of the csets on ->src_csets.  After, on ->mg_tasks of
2224          * the csets on ->dst_csets.  ->csets point to either ->src_csets
2225          * or ->dst_csets depending on whether migration is committed.
2226          *
2227          * ->cur_csets and ->cur_task point to the current task position
2228          * during iteration.
2229          */
2230         struct list_head        *csets;
2231         struct css_set          *cur_cset;
2232         struct task_struct      *cur_task;
2233 };
2234 
2235 #define CGROUP_TASKSET_INIT(tset)       (struct cgroup_taskset){        \
2236         .src_csets              = LIST_HEAD_INIT(tset.src_csets),       \
2237         .dst_csets              = LIST_HEAD_INIT(tset.dst_csets),       \
2238         .csets                  = &tset.src_csets,                      \
2239 }
2240 
2241 /**
2242  * cgroup_taskset_add - try to add a migration target task to a taskset
2243  * @task: target task
2244  * @tset: target taskset
2245  *
2246  * Add @task, which is a migration target, to @tset.  This function becomes
2247  * noop if @task doesn't need to be migrated.  @task's css_set should have
2248  * been added as a migration source and @task->cg_list will be moved from
2249  * the css_set's tasks list to mg_tasks one.
2250  */
2251 static void cgroup_taskset_add(struct task_struct *task,
2252                                struct cgroup_taskset *tset)
2253 {
2254         struct css_set *cset;
2255 
2256         lockdep_assert_held(&css_set_lock);
2257 
2258         /* @task either already exited or can't exit until the end */
2259         if (task->flags & PF_EXITING)
2260                 return;
2261 
2262         /* leave @task alone if post_fork() hasn't linked it yet */
2263         if (list_empty(&task->cg_list))
2264                 return;
2265 
2266         cset = task_css_set(task);
2267         if (!cset->mg_src_cgrp)
2268                 return;
2269 
2270         list_move_tail(&task->cg_list, &cset->mg_tasks);
2271         if (list_empty(&cset->mg_node))
2272                 list_add_tail(&cset->mg_node, &tset->src_csets);
2273         if (list_empty(&cset->mg_dst_cset->mg_node))
2274                 list_move_tail(&cset->mg_dst_cset->mg_node,
2275                                &tset->dst_csets);
2276 }
2277 
2278 /**
2279  * cgroup_taskset_first - reset taskset and return the first task
2280  * @tset: taskset of interest
2281  * @dst_cssp: output variable for the destination css
2282  *
2283  * @tset iteration is initialized and the first task is returned.
2284  */
2285 struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
2286                                          struct cgroup_subsys_state **dst_cssp)
2287 {
2288         tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
2289         tset->cur_task = NULL;
2290 
2291         return cgroup_taskset_next(tset, dst_cssp);
2292 }
2293 
2294 /**
2295  * cgroup_taskset_next - iterate to the next task in taskset
2296  * @tset: taskset of interest
2297  * @dst_cssp: output variable for the destination css
2298  *
2299  * Return the next task in @tset.  Iteration must have been initialized
2300  * with cgroup_taskset_first().
2301  */
2302 struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
2303                                         struct cgroup_subsys_state **dst_cssp)
2304 {
2305         struct css_set *cset = tset->cur_cset;
2306         struct task_struct *task = tset->cur_task;
2307 
2308         while (&cset->mg_node != tset->csets) {
2309                 if (!task)
2310                         task = list_first_entry(&cset->mg_tasks,
2311                                                 struct task_struct, cg_list);
2312                 else
2313                         task = list_next_entry(task, cg_list);
2314 
2315                 if (&task->cg_list != &cset->mg_tasks) {
2316                         tset->cur_cset = cset;
2317                         tset->cur_task = task;
2318 
2319                         /*
2320                          * This function may be called both before and
2321                          * after cgroup_taskset_migrate().  The two cases
2322                          * can be distinguished by looking at whether @cset
2323                          * has its ->mg_dst_cset set.
2324                          */
2325                         if (cset->mg_dst_cset)
2326                                 *dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];
2327                         else
2328                                 *dst_cssp = cset->subsys[tset->ssid];
2329 
2330                         return task;
2331                 }
2332 
2333                 cset = list_next_entry(cset, mg_node);
2334                 task = NULL;
2335         }
2336 
2337         return NULL;
2338 }
2339 
2340 /**
2341  * cgroup_taskset_migrate - migrate a taskset to a cgroup
2342  * @tset: taget taskset
2343  * @dst_cgrp: destination cgroup
2344  *
2345  * Migrate tasks in @tset to @dst_cgrp.  This function fails iff one of the
2346  * ->can_attach callbacks fails and guarantees that either all or none of
2347  * the tasks in @tset are migrated.  @tset is consumed regardless of
2348  * success.
2349  */
2350 static int cgroup_taskset_migrate(struct cgroup_taskset *tset,
2351                                   struct cgroup *dst_cgrp)
2352 {
2353         struct cgroup_subsys_state *css, *failed_css = NULL;
2354         struct task_struct *task, *tmp_task;
2355         struct css_set *cset, *tmp_cset;
2356         int i, ret;
2357 
2358         /* methods shouldn't be called if no task is actually migrating */
2359         if (list_empty(&tset->src_csets))
2360                 return 0;
2361 
2362         /* check that we can legitimately attach to the cgroup */
2363         for_each_e_css(css, i, dst_cgrp) {
2364                 if (css->ss->can_attach) {
2365                         tset->ssid = i;
2366                         ret = css->ss->can_attach(tset);
2367                         if (ret) {
2368                                 failed_css = css;
2369                                 goto out_cancel_attach;
2370                         }
2371                 }
2372         }
2373 
2374         /*
2375          * Now that we're guaranteed success, proceed to move all tasks to
2376          * the new cgroup.  There are no failure cases after here, so this
2377          * is the commit point.
2378          */
2379         spin_lock_bh(&css_set_lock);
2380         list_for_each_entry(cset, &tset->src_csets, mg_node) {
2381                 list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {
2382                         struct css_set *from_cset = task_css_set(task);
2383                         struct css_set *to_cset = cset->mg_dst_cset;
2384 
2385                         get_css_set(to_cset);
2386                         css_set_move_task(task, from_cset, to_cset, true);
2387                         put_css_set_locked(from_cset);
2388                 }
2389         }
2390         spin_unlock_bh(&css_set_lock);
2391 
2392         /*
2393          * Migration is committed, all target tasks are now on dst_csets.
2394          * Nothing is sensitive to fork() after this point.  Notify
2395          * controllers that migration is complete.
2396          */
2397         tset->csets = &tset->dst_csets;
2398 
2399         for_each_e_css(css, i, dst_cgrp) {
2400                 if (css->ss->attach) {
2401                         tset->ssid = i;
2402                         css->ss->attach(tset);
2403                 }
2404         }
2405 
2406         ret = 0;
2407         goto out_release_tset;
2408 
2409 out_cancel_attach:
2410         for_each_e_css(css, i, dst_cgrp) {
2411                 if (css == failed_css)
2412                         break;
2413                 if (css->ss->cancel_attach) {
2414                         tset->ssid = i;
2415                         css->ss->cancel_attach(tset);
2416                 }
2417         }
2418 out_release_tset:
2419         spin_lock_bh(&css_set_lock);
2420         list_splice_init(&tset->dst_csets, &tset->src_csets);
2421         list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {
2422                 list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2423                 list_del_init(&cset->mg_node);
2424         }
2425         spin_unlock_bh(&css_set_lock);
2426         return ret;
2427 }
2428 
2429 /**
2430  * cgroup_migrate_finish - cleanup after attach
2431  * @preloaded_csets: list of preloaded css_sets
2432  *
2433  * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
2434  * those functions for details.
2435  */
2436 static void cgroup_migrate_finish(struct list_head *preloaded_csets)
2437 {
2438         struct css_set *cset, *tmp_cset;
2439 
2440         lockdep_assert_held(&cgroup_mutex);
2441 
2442         spin_lock_bh(&css_set_lock);
2443         list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
2444                 cset->mg_src_cgrp = NULL;
2445                 cset->mg_dst_cset = NULL;
2446                 list_del_init(&cset->mg_preload_node);
2447                 put_css_set_locked(cset);
2448         }
2449         spin_unlock_bh(&css_set_lock);
2450 }
2451 
2452 /**
2453  * cgroup_migrate_add_src - add a migration source css_set
2454  * @src_cset: the source css_set to add
2455  * @dst_cgrp: the destination cgroup
2456  * @preloaded_csets: list of preloaded css_sets
2457  *
2458  * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
2459  * @src_cset and add it to @preloaded_csets, which should later be cleaned
2460  * up by cgroup_migrate_finish().
2461  *
2462  * This function may be called without holding cgroup_threadgroup_rwsem
2463  * even if the target is a process.  Threads may be created and destroyed
2464  * but as long as cgroup_mutex is not dropped, no new css_set can be put
2465  * into play and the preloaded css_sets are guaranteed to cover all
2466  * migrations.
2467  */
2468 static void cgroup_migrate_add_src(struct css_set *src_cset,
2469                                    struct cgroup *dst_cgrp,
2470                                    struct list_head *preloaded_csets)
2471 {
2472         struct cgroup *src_cgrp;
2473 
2474         lockdep_assert_held(&cgroup_mutex);
2475         lockdep_assert_held(&css_set_lock);
2476 
2477         src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);
2478 
2479         if (!list_empty(&src_cset->mg_preload_node))
2480                 return;
2481 
2482         WARN_ON(src_cset->mg_src_cgrp);
2483         WARN_ON(!list_empty(&src_cset->mg_tasks));
2484         WARN_ON(!list_empty(&src_cset->mg_node));
2485 
2486         src_cset->mg_src_cgrp = src_cgrp;
2487         get_css_set(src_cset);
2488         list_add(&src_cset->mg_preload_node, preloaded_csets);
2489 }
2490 
2491 /**
2492  * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2493  * @dst_cgrp: the destination cgroup (may be %NULL)
2494  * @preloaded_csets: list of preloaded source css_sets
2495  *
2496  * Tasks are about to be moved to @dst_cgrp and all the source css_sets
2497  * have been preloaded to @preloaded_csets.  This function looks up and
2498  * pins all destination css_sets, links each to its source, and append them
2499  * to @preloaded_csets.  If @dst_cgrp is %NULL, the destination of each
2500  * source css_set is assumed to be its cgroup on the default hierarchy.
2501  *
2502  * This function must be called after cgroup_migrate_add_src() has been
2503  * called on each migration source css_set.  After migration is performed
2504  * using cgroup_migrate(), cgroup_migrate_finish() must be called on
2505  * @preloaded_csets.
2506  */
2507 static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
2508                                       struct list_head *preloaded_csets)
2509 {
2510         LIST_HEAD(csets);
2511         struct css_set *src_cset, *tmp_cset;
2512 
2513         lockdep_assert_held(&cgroup_mutex);
2514 
2515         /*
2516          * Except for the root, child_subsys_mask must be zero for a cgroup
2517          * with tasks so that child cgroups don't compete against tasks.
2518          */
2519         if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) &&
2520             dst_cgrp->child_subsys_mask)
2521                 return -EBUSY;
2522 
2523         /* look up the dst cset for each src cset and link it to src */
2524         list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
2525                 struct css_set *dst_cset;
2526 
2527                 dst_cset = find_css_set(src_cset,
2528                                         dst_cgrp ?: src_cset->dfl_cgrp);
2529                 if (!dst_cset)
2530                         goto err;
2531 
2532                 WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
2533 
2534                 /*
2535                  * If src cset equals dst, it's noop.  Drop the src.
2536                  * cgroup_migrate() will skip the cset too.  Note that we
2537                  * can't handle src == dst as some nodes are used by both.
2538                  */
2539                 if (src_cset == dst_cset) {
2540                         src_cset->mg_src_cgrp = NULL;
2541                         list_del_init(&src_cset->mg_preload_node);
2542                         put_css_set(src_cset);
2543                         put_css_set(dst_cset);
2544                         continue;
2545                 }
2546 
2547                 src_cset->mg_dst_cset = dst_cset;
2548 
2549                 if (list_empty(&dst_cset->mg_preload_node))
2550                         list_add(&dst_cset->mg_preload_node, &csets);
2551                 else
2552                         put_css_set(dst_cset);
2553         }
2554 
2555         list_splice_tail(&csets, preloaded_csets);
2556         return 0;
2557 err:
2558         cgroup_migrate_finish(&csets);
2559         return -ENOMEM;
2560 }
2561 
2562 /**
2563  * cgroup_migrate - migrate a process or task to a cgroup
2564  * @leader: the leader of the process or the task to migrate
2565  * @threadgroup: whether @leader points to the whole process or a single task
2566  * @cgrp: the destination cgroup
2567  *
2568  * Migrate a process or task denoted by @leader to @cgrp.  If migrating a
2569  * process, the caller must be holding cgroup_threadgroup_rwsem.  The
2570  * caller is also responsible for invoking cgroup_migrate_add_src() and
2571  * cgroup_migrate_prepare_dst() on the targets before invoking this
2572  * function and following up with cgroup_migrate_finish().
2573  *
2574  * As long as a controller's ->can_attach() doesn't fail, this function is
2575  * guaranteed to succeed.  This means that, excluding ->can_attach()
2576  * failure, when migrating multiple targets, the success or failure can be
2577  * decided for all targets by invoking group_migrate_prepare_dst() before
2578  * actually starting migrating.
2579  */
2580 static int cgroup_migrate(struct task_struct *leader, bool threadgroup,
2581                           struct cgroup *cgrp)
2582 {
2583         struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset);
2584         struct task_struct *task;
2585 
2586         /*
2587          * Prevent freeing of tasks while we take a snapshot. Tasks that are
2588          * already PF_EXITING could be freed from underneath us unless we
2589          * take an rcu_read_lock.
2590          */
2591         spin_lock_bh(&css_set_lock);
2592         rcu_read_lock();
2593         task = leader;
2594         do {
2595                 cgroup_taskset_add(task, &tset);
2596                 if (!threadgroup)
2597                         break;
2598         } while_each_thread(leader, task);
2599         rcu_read_unlock();
2600         spin_unlock_bh(&css_set_lock);
2601 
2602         return cgroup_taskset_migrate(&tset, cgrp);
2603 }
2604 
2605 /**
2606  * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2607  * @dst_cgrp: the cgroup to attach to
2608  * @leader: the task or the leader of the threadgroup to be attached
2609  * @threadgroup: attach the whole threadgroup?
2610  *
2611  * Call holding cgroup_mutex and cgroup_threadgroup_rwsem.
2612  */
2613 static int cgroup_attach_task(struct cgroup *dst_cgrp,
2614                               struct task_struct *leader, bool threadgroup)
2615 {
2616         LIST_HEAD(preloaded_csets);
2617         struct task_struct *task;
2618         int ret;
2619 
2620         /* look up all src csets */
2621         spin_lock_bh(&css_set_lock);
2622         rcu_read_lock();
2623         task = leader;
2624         do {
2625                 cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
2626                                        &preloaded_csets);
2627                 if (!threadgroup)
2628                         break;
2629         } while_each_thread(leader, task);
2630         rcu_read_unlock();
2631         spin_unlock_bh(&css_set_lock);
2632 
2633         /* prepare dst csets and commit */
2634         ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
2635         if (!ret)
2636                 ret = cgroup_migrate(leader, threadgroup, dst_cgrp);
2637 
2638         cgroup_migrate_finish(&preloaded_csets);
2639         return ret;
2640 }
2641 
2642 static int cgroup_procs_write_permission(struct task_struct *task,
2643                                          struct cgroup *dst_cgrp,
2644                                          struct kernfs_open_file *of)
2645 {
2646         const struct cred *cred = current_cred();
2647         const struct cred *tcred = get_task_cred(task);
2648         int ret = 0;
2649 
2650         /*
2651          * even if we're attaching all tasks in the thread group, we only
2652          * need to check permissions on one of them.
2653          */
2654         if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
2655             !uid_eq(cred->euid, tcred->uid) &&
2656             !uid_eq(cred->euid, tcred->suid))
2657                 ret = -EACCES;
2658 
2659         if (!ret && cgroup_on_dfl(dst_cgrp)) {
2660                 struct super_block *sb = of->file->f_path.dentry->d_sb;
2661                 struct cgroup *cgrp;
2662                 struct inode *inode;
2663 
2664                 spin_lock_bh(&css_set_lock);
2665                 cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
2666                 spin_unlock_bh(&css_set_lock);
2667 
2668                 while (!cgroup_is_descendant(dst_cgrp, cgrp))
2669                         cgrp = cgroup_parent(cgrp);
2670 
2671                 ret = -ENOMEM;
2672                 inode = kernfs_get_inode(sb, cgrp->procs_file.kn);
2673                 if (inode) {
2674                         ret = inode_permission(inode, MAY_WRITE);
2675                         iput(inode);
2676                 }
2677         }
2678 
2679         put_cred(tcred);
2680         return ret;
2681 }
2682 
2683 /*
2684  * Find the task_struct of the task to attach by vpid and pass it along to the
2685  * function to attach either it or all tasks in its threadgroup. Will lock
2686  * cgroup_mutex and threadgroup.
2687  */
2688 static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
2689                                     size_t nbytes, loff_t off, bool threadgroup)
2690 {
2691         struct task_struct *tsk;
2692         struct cgroup *cgrp;
2693         pid_t pid;
2694         int ret;
2695 
2696         if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
2697                 return -EINVAL;
2698 
2699         cgrp = cgroup_kn_lock_live(of->kn);
2700         if (!cgrp)
2701                 return -ENODEV;
2702 
2703         percpu_down_write(&cgroup_threadgroup_rwsem);
2704         rcu_read_lock();
2705         if (pid) {
2706                 tsk = find_task_by_vpid(pid);
2707                 if (!tsk) {
2708                         ret = -ESRCH;
2709                         goto out_unlock_rcu;
2710                 }
2711         } else {
2712                 tsk = current;
2713         }
2714 
2715         if (threadgroup)
2716                 tsk = tsk->group_leader;
2717 
2718         /*
2719          * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2720          * trapped in a cpuset, or RT worker may be born in a cgroup
2721          * with no rt_runtime allocated.  Just say no.
2722          */
2723         if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2724                 ret = -EINVAL;
2725                 goto out_unlock_rcu;
2726         }
2727 
2728         get_task_struct(tsk);
2729         rcu_read_unlock();
2730 
2731         ret = cgroup_procs_write_permission(tsk, cgrp, of);
2732         if (!ret)
2733                 ret = cgroup_attach_task(cgrp, tsk, threadgroup);
2734 
2735         put_task_struct(tsk);
2736         goto out_unlock_threadgroup;
2737 
2738 out_unlock_rcu:
2739         rcu_read_unlock();
2740 out_unlock_threadgroup:
2741         percpu_up_write(&cgroup_threadgroup_rwsem);
2742         cgroup_kn_unlock(of->kn);
2743         cpuset_post_attach_flush();
2744         return ret ?: nbytes;
2745 }
2746 
2747 /**
2748  * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
2749  * @from: attach to all cgroups of a given task
2750  * @tsk: the task to be attached
2751  */
2752 int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
2753 {
2754         struct cgroup_root *root;
2755         int retval = 0;
2756 
2757         mutex_lock(&cgroup_mutex);
2758         for_each_root(root) {
2759                 struct cgroup *from_cgrp;
2760 
2761                 if (root == &cgrp_dfl_root)
2762                         continue;
2763 
2764                 spin_lock_bh(&css_set_lock);
2765                 from_cgrp = task_cgroup_from_root(from, root);
2766                 spin_unlock_bh(&css_set_lock);
2767 
2768                 retval = cgroup_attach_task(from_cgrp, tsk, false);
2769                 if (retval)
2770                         break;
2771         }
2772         mutex_unlock(&cgroup_mutex);
2773 
2774         return retval;
2775 }
2776 EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
2777 
2778 static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
2779                                   char *buf, size_t nbytes, loff_t off)
2780 {
2781         return __cgroup_procs_write(of, buf, nbytes, off, false);
2782 }
2783 
2784 static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
2785                                   char *buf, size_t nbytes, loff_t off)
2786 {
2787         return __cgroup_procs_write(of, buf, nbytes, off, true);
2788 }
2789 
2790 static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
2791                                           char *buf, size_t nbytes, loff_t off)
2792 {
2793         struct cgroup *cgrp;
2794 
2795         BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
2796 
2797         cgrp = cgroup_kn_lock_live(of->kn);
2798         if (!cgrp)
2799                 return -ENODEV;
2800         spin_lock(&release_agent_path_lock);
2801         strlcpy(cgrp->root->release_agent_path, strstrip(buf),
2802                 sizeof(cgrp->root->release_agent_path));
2803         spin_unlock(&release_agent_path_lock);
2804         cgroup_kn_unlock(of->kn);
2805         return nbytes;
2806 }
2807 
2808 static int cgroup_release_agent_show(struct seq_file *seq, void *v)
2809 {
2810         struct cgroup *cgrp = seq_css(seq)->cgroup;
2811 
2812         spin_lock(&release_agent_path_lock);
2813         seq_puts(seq, cgrp->root->release_agent_path);
2814         spin_unlock(&release_agent_path_lock);
2815         seq_putc(seq, '\n');
2816         return 0;
2817 }
2818 
2819 static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
2820 {
2821         seq_puts(seq, "\n");
2822         return 0;
2823 }
2824 
2825 static void cgroup_print_ss_mask(struct seq_file *seq, unsigned long ss_mask)
2826 {
2827         struct cgroup_subsys *ss;
2828         bool printed = false;
2829         int ssid;
2830 
2831         for_each_subsys_which(ss, ssid, &ss_mask) {
2832                 if (printed)
2833                         seq_putc(seq, ' ');
2834                 seq_printf(seq, "%s", ss->name);
2835                 printed = true;
2836         }
2837         if (printed)
2838                 seq_putc(seq, '\n');
2839 }
2840 
2841 /* show controllers which are currently attached to the default hierarchy */
2842 static int cgroup_root_controllers_show(struct seq_file *seq, void *v)
2843 {
2844         struct cgroup *cgrp = seq_css(seq)->cgroup;
2845 
2846         cgroup_print_ss_mask(seq, cgrp->root->subsys_mask &
2847                              ~cgrp_dfl_root_inhibit_ss_mask);
2848         return 0;
2849 }
2850 
2851 /* show controllers which are enabled from the parent */
2852 static int cgroup_controllers_show(struct seq_file *seq, void *v)
2853 {
2854         struct cgroup *cgrp = seq_css(seq)->cgroup;
2855 
2856         cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control);
2857         return 0;
2858 }
2859 
2860 /* show controllers which are enabled for a given cgroup's children */
2861 static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
2862 {
2863         struct cgroup *cgrp = seq_css(seq)->cgroup;
2864 
2865         cgroup_print_ss_mask(seq, cgrp->subtree_control);
2866         return 0;
2867 }
2868 
2869 /**
2870  * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
2871  * @cgrp: root of the subtree to update csses for
2872  *
2873  * @cgrp's child_subsys_mask has changed and its subtree's (self excluded)
2874  * css associations need to be updated accordingly.  This function looks up
2875  * all css_sets which are attached to the subtree, creates the matching
2876  * updated css_sets and migrates the tasks to the new ones.
2877  */
2878 static int cgroup_update_dfl_csses(struct cgroup *cgrp)
2879 {
2880         LIST_HEAD(preloaded_csets);
2881         struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset);
2882         struct cgroup_subsys_state *css;
2883         struct css_set *src_cset;
2884         int ret;
2885 
2886         lockdep_assert_held(&cgroup_mutex);
2887 
2888         percpu_down_write(&cgroup_threadgroup_rwsem);
2889 
2890         /* look up all csses currently attached to @cgrp's subtree */
2891         spin_lock_bh(&css_set_lock);
2892         css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
2893                 struct cgrp_cset_link *link;
2894 
2895                 /* self is not affected by child_subsys_mask change */
2896                 if (css->cgroup == cgrp)
2897                         continue;
2898 
2899                 list_for_each_entry(link, &css->cgroup->cset_links, cset_link)
2900                         cgroup_migrate_add_src(link->cset, cgrp,
2901                                                &preloaded_csets);
2902         }
2903         spin_unlock_bh(&css_set_lock);
2904 
2905         /* NULL dst indicates self on default hierarchy */
2906         ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets);
2907         if (ret)
2908                 goto out_finish;
2909 
2910         spin_lock_bh(&css_set_lock);
2911         list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
2912                 struct task_struct *task, *ntask;
2913 
2914                 /* src_csets precede dst_csets, break on the first dst_cset */
2915                 if (!src_cset->mg_src_cgrp)
2916                         break;
2917 
2918                 /* all tasks in src_csets need to be migrated */
2919                 list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list)
2920                         cgroup_taskset_add(task, &tset);
2921         }
2922         spin_unlock_bh(&css_set_lock);
2923 
2924         ret = cgroup_taskset_migrate(&tset, cgrp);
2925 out_finish:
2926         cgroup_migrate_finish(&preloaded_csets);
2927         percpu_up_write(&cgroup_threadgroup_rwsem);
2928         return ret;
2929 }
2930 
2931 /* change the enabled child controllers for a cgroup in the default hierarchy */
2932 static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
2933                                             char *buf, size_t nbytes,
2934                                             loff_t off)
2935 {
2936         unsigned long enable = 0, disable = 0;
2937         unsigned long css_enable, css_disable, old_sc, new_sc, old_ss, new_ss;
2938         struct cgroup *cgrp, *child;
2939         struct cgroup_subsys *ss;
2940         char *tok;
2941         int ssid, ret;
2942 
2943         /*
2944          * Parse input - space separated list of subsystem names prefixed
2945          * with either + or -.
2946          */
2947         buf = strstrip(buf);
2948         while ((tok = strsep(&buf, " "))) {
2949                 unsigned long tmp_ss_mask = ~cgrp_dfl_root_inhibit_ss_mask;
2950 
2951                 if (tok[0] == '\0')
2952                         continue;
2953                 for_each_subsys_which(ss, ssid, &tmp_ss_mask) {
2954                         if (!cgroup_ssid_enabled(ssid) ||
2955                             strcmp(tok + 1, ss->name))
2956                                 continue;
2957 
2958                         if (*tok == '+') {
2959                                 enable |= 1 << ssid;
2960                                 disable &= ~(1 << ssid);
2961                         } else if (*tok == '-') {
2962                                 disable |= 1 << ssid;
2963                                 enable &= ~(1 << ssid);
2964                         } else {
2965                                 return -EINVAL;
2966                         }
2967                         break;
2968                 }
2969                 if (ssid == CGROUP_SUBSYS_COUNT)
2970                         return -EINVAL;
2971         }
2972 
2973         cgrp = cgroup_kn_lock_live(of->kn);
2974         if (!cgrp)
2975                 return -ENODEV;
2976 
2977         for_each_subsys(ss, ssid) {
2978                 if (enable & (1 << ssid)) {
2979                         if (cgrp->subtree_control & (1 << ssid)) {
2980                                 enable &= ~(1 << ssid);
2981                                 continue;
2982                         }
2983 
2984                         /* unavailable or not enabled on the parent? */
2985                         if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) ||
2986                             (cgroup_parent(cgrp) &&
2987                              !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) {
2988                                 ret = -ENOENT;
2989                                 goto out_unlock;
2990                         }
2991                 } else if (disable & (1 << ssid)) {
2992                         if (!(cgrp->subtree_control & (1 << ssid))) {
2993                                 disable &= ~(1 << ssid);
2994                                 continue;
2995                         }
2996 
2997                         /* a child has it enabled? */
2998                         cgroup_for_each_live_child(child, cgrp) {
2999                                 if (child->subtree_control & (1 << ssid)) {
3000                                         ret = -EBUSY;
3001                                         goto out_unlock;
3002                                 }
3003                         }
3004                 }
3005         }
3006 
3007         if (!enable && !disable) {
3008                 ret = 0;
3009                 goto out_unlock;
3010         }
3011 
3012         /*
3013          * Except for the root, subtree_control must be zero for a cgroup
3014          * with tasks so that child cgroups don't compete against tasks.
3015          */
3016         if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) {
3017                 ret = -EBUSY;
3018                 goto out_unlock;
3019         }
3020 
3021         /*
3022          * Update subsys masks and calculate what needs to be done.  More
3023          * subsystems than specified may need to be enabled or disabled
3024          * depending on subsystem dependencies.
3025          */
3026         old_sc = cgrp->subtree_control;
3027         old_ss = cgrp->child_subsys_mask;
3028         new_sc = (old_sc | enable) & ~disable;
3029         new_ss = cgroup_calc_child_subsys_mask(cgrp, new_sc);
3030 
3031         css_enable = ~old_ss & new_ss;
3032         css_disable = old_ss & ~new_ss;
3033         enable |= css_enable;
3034         disable |= css_disable;
3035 
3036         /*
3037          * Because css offlining is asynchronous, userland might try to
3038          * re-enable the same controller while the previous instance is
3039          * still around.  In such cases, wait till it's gone using
3040          * offline_waitq.
3041          */
3042         for_each_subsys_which(ss, ssid, &css_enable) {
3043                 cgroup_for_each_live_child(child, cgrp) {
3044                         DEFINE_WAIT(wait);
3045 
3046                         if (!cgroup_css(child, ss))
3047                                 continue;
3048 
3049                         cgroup_get(child);
3050                         prepare_to_wait(&child->offline_waitq, &wait,
3051                                         TASK_UNINTERRUPTIBLE);
3052                         cgroup_kn_unlock(of->kn);
3053                         schedule();
3054                         finish_wait(&child->offline_waitq, &wait);
3055                         cgroup_put(child);
3056 
3057                         return restart_syscall();
3058                 }
3059         }
3060 
3061         cgrp->subtree_control = new_sc;
3062         cgrp->child_subsys_mask = new_ss;
3063 
3064         /*
3065          * Create new csses or make the existing ones visible.  A css is
3066          * created invisible if it's being implicitly enabled through
3067          * dependency.  An invisible css is made visible when the userland
3068          * explicitly enables it.
3069          */
3070         for_each_subsys(ss, ssid) {
3071                 if (!(enable & (1 << ssid)))
3072                         continue;
3073 
3074                 cgroup_for_each_live_child(child, cgrp) {
3075                         if (css_enable & (1 << ssid))
3076                                 ret = create_css(child, ss,
3077                                         cgrp->subtree_control & (1 << ssid));
3078                         else
3079                                 ret = css_populate_dir(cgroup_css(child, ss),
3080                                                        NULL);
3081                         if (ret)
3082                                 goto err_undo_css;
3083                 }
3084         }
3085 
3086         /*
3087          * At this point, cgroup_e_css() results reflect the new csses
3088          * making the following cgroup_update_dfl_csses() properly update
3089          * css associations of all tasks in the subtree.
3090          */
3091         ret = cgroup_update_dfl_csses(cgrp);
3092         if (ret)
3093                 goto err_undo_css;
3094 
3095         /*
3096          * All tasks are migrated out of disabled csses.  Kill or hide
3097          * them.  A css is hidden when the userland requests it to be
3098          * disabled while other subsystems are still depending on it.  The
3099          * css must not actively control resources and be in the vanilla
3100          * state if it's made visible again later.  Controllers which may
3101          * be depended upon should provide ->css_reset() for this purpose.
3102          */
3103         for_each_subsys(ss, ssid) {
3104                 if (!(disable & (1 << ssid)))
3105                         continue;
3106 
3107                 cgroup_for_each_live_child(child, cgrp) {
3108                         struct cgroup_subsys_state *css = cgroup_css(child, ss);
3109 
3110                         if (css_disable & (1 << ssid)) {
3111                                 kill_css(css);
3112                         } else {
3113                                 css_clear_dir(css, NULL);
3114                                 if (ss->css_reset)
3115                                         ss->css_reset(css);
3116                         }
3117                 }
3118         }
3119 
3120         /*
3121          * The effective csses of all the descendants (excluding @cgrp) may
3122          * have changed.  Subsystems can optionally subscribe to this event
3123          * by implementing ->css_e_css_changed() which is invoked if any of
3124          * the effective csses seen from the css's cgroup may have changed.
3125          */
3126         for_each_subsys(ss, ssid) {
3127                 struct cgroup_subsys_state *this_css = cgroup_css(cgrp, ss);
3128                 struct cgroup_subsys_state *css;
3129 
3130                 if (!ss->css_e_css_changed || !this_css)
3131                         continue;
3132 
3133                 css_for_each_descendant_pre(css, this_css)
3134                         if (css != this_css)
3135                                 ss->css_e_css_changed(css);
3136         }
3137 
3138         kernfs_activate(cgrp->kn);
3139         ret = 0;
3140 out_unlock:
3141         cgroup_kn_unlock(of->kn);
3142         return ret ?: nbytes;
3143 
3144 err_undo_css:
3145         cgrp->subtree_control = old_sc;
3146         cgrp->child_subsys_mask = old_ss;
3147 
3148         for_each_subsys(ss, ssid) {
3149                 if (!(enable & (1 << ssid)))
3150                         continue;
3151 
3152                 cgroup_for_each_live_child(child, cgrp) {
3153                         struct cgroup_subsys_state *css = cgroup_css(child, ss);
3154 
3155                         if (!css)
3156                                 continue;
3157 
3158                         if (css_enable & (1 << ssid))
3159                                 kill_css(css);
3160                         else
3161                                 css_clear_dir(css, NULL);
3162                 }
3163         }
3164         goto out_unlock;
3165 }
3166 
3167 static int cgroup_events_show(struct seq_file *seq, void *v)
3168 {
3169         seq_printf(seq, "populated %d\n",
3170                    cgroup_is_populated(seq_css(seq)->cgroup));
3171         return 0;
3172 }
3173 
3174 static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
3175                                  size_t nbytes, loff_t off)
3176 {
3177         struct cgroup *cgrp = of->kn->parent->priv;
3178         struct cftype *cft = of->kn->priv;
3179         struct cgroup_subsys_state *css;
3180         int ret;
3181 
3182         if (cft->write)
3183                 return cft->write(of, buf, nbytes, off);
3184 
3185         /*
3186          * kernfs guarantees that a file isn't deleted with operations in
3187          * flight, which means that the matching css is and stays alive and
3188          * doesn't need to be pinned.  The RCU locking is not necessary
3189          * either.  It's just for the convenience of using cgroup_css().
3190          */
3191         rcu_read_lock();
3192         css = cgroup_css(cgrp, cft->ss);
3193         rcu_read_unlock();
3194 
3195         if (cft->write_u64) {
3196                 unsigned long long v;
3197                 ret = kstrtoull(buf, 0, &v);
3198                 if (!ret)
3199                         ret = cft->write_u64(css, cft, v);
3200         } else if (cft->write_s64) {
3201                 long long v;
3202                 ret = kstrtoll(buf, 0, &v);
3203                 if (!ret)
3204                         ret = cft->write_s64(css, cft, v);
3205         } else {
3206                 ret = -EINVAL;
3207         }
3208 
3209         return ret ?: nbytes;
3210 }
3211 
3212 static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
3213 {
3214         return seq_cft(seq)->seq_start(seq, ppos);
3215 }
3216 
3217 static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
3218 {
3219         return seq_cft(seq)->seq_next(seq, v, ppos);
3220 }
3221 
3222 static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
3223 {
3224         seq_cft(seq)->seq_stop(seq, v);
3225 }
3226 
3227 static int cgroup_seqfile_show(struct seq_file *m, void *arg)
3228 {
3229         struct cftype *cft = seq_cft(m);
3230         struct cgroup_subsys_state *css = seq_css(m);
3231 
3232         if (cft->seq_show)
3233                 return cft->seq_show(m, arg);
3234 
3235         if (cft->read_u64)
3236                 seq_printf(m, "%llu\n", cft->read_u64(css, cft));
3237         else if (cft->read_s64)
3238                 seq_printf(m, "%lld\n", cft->read_s64(css, cft));
3239         else
3240                 return -EINVAL;
3241         return 0;
3242 }
3243 
3244 static struct kernfs_ops cgroup_kf_single_ops = {
3245         .atomic_write_len       = PAGE_SIZE,
3246         .write                  = cgroup_file_write,
3247         .seq_show               = cgroup_seqfile_show,
3248 };
3249 
3250 static struct kernfs_ops cgroup_kf_ops = {
3251         .atomic_write_len       = PAGE_SIZE,
3252         .write                  = cgroup_file_write,
3253         .seq_start              = cgroup_seqfile_start,
3254         .seq_next               = cgroup_seqfile_next,
3255         .seq_stop               = cgroup_seqfile_stop,
3256         .seq_show               = cgroup_seqfile_show,
3257 };
3258 
3259 /*
3260  * cgroup_rename - Only allow simple rename of directories in place.
3261  */
3262 static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
3263                          const char *new_name_str)
3264 {
3265         struct cgroup *cgrp = kn->priv;
3266         int ret;
3267 
3268         if (kernfs_type(kn) != KERNFS_DIR)
3269                 return -ENOTDIR;
3270         if (kn->parent != new_parent)
3271                 return -EIO;
3272 
3273         /*
3274          * This isn't a proper migration and its usefulness is very
3275          * limited.  Disallow on the default hierarchy.
3276          */
3277         if (cgroup_on_dfl(cgrp))
3278                 return -EPERM;
3279 
3280         /*
3281          * We're gonna grab cgroup_mutex which nests outside kernfs
3282          * active_ref.  kernfs_rename() doesn't require active_ref
3283          * protection.  Break them before grabbing cgroup_mutex.
3284          */
3285         kernfs_break_active_protection(new_parent);
3286         kernfs_break_active_protection(kn);
3287 
3288         mutex_lock(&cgroup_mutex);
3289 
3290         ret = kernfs_rename(kn, new_parent, new_name_str);
3291 
3292         mutex_unlock(&cgroup_mutex);
3293 
3294         kernfs_unbreak_active_protection(kn);
3295         kernfs_unbreak_active_protection(new_parent);
3296         return ret;
3297 }
3298 
3299 /* set uid and gid of cgroup dirs and files to that of the creator */
3300 static int cgroup_kn_set_ugid(struct kernfs_node *kn)
3301 {
3302         struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
3303                                .ia_uid = current_fsuid(),
3304                                .ia_gid = current_fsgid(), };
3305 
3306         if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
3307             gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
3308                 return 0;
3309 
3310         return kernfs_setattr(kn, &iattr);
3311 }
3312 
3313 static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,
3314                            struct cftype *cft)
3315 {
3316         char name[CGROUP_FILE_NAME_MAX];
3317         struct kernfs_node *kn;
3318         struct lock_class_key *key = NULL;
3319         int ret;
3320 
3321 #ifdef CONFIG_DEBUG_LOCK_ALLOC
3322         key = &cft->lockdep_key;
3323 #endif
3324         kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
3325                                   cgroup_file_mode(cft), 0, cft->kf_ops, cft,
3326                                   NULL, key);
3327         if (IS_ERR(kn))
3328                 return PTR_ERR(kn);
3329 
3330         ret = cgroup_kn_set_ugid(kn);
3331         if (ret) {
3332                 kernfs_remove(kn);
3333                 return ret;
3334         }
3335 
3336         if (cft->file_offset) {
3337                 struct cgroup_file *cfile = (void *)css + cft->file_offset;
3338 
3339                 spin_lock_irq(&cgroup_file_kn_lock);
3340                 cfile->kn = kn;
3341                 spin_unlock_irq(&cgroup_file_kn_lock);
3342         }
3343 
3344         return 0;
3345 }
3346 
3347 /**
3348  * cgroup_addrm_files - add or remove files to a cgroup directory
3349  * @css: the target css
3350  * @cgrp: the target cgroup (usually css->cgroup)
3351  * @cfts: array of cftypes to be added
3352  * @is_add: whether to add or remove
3353  *
3354  * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3355  * For removals, this function never fails.
3356  */
3357 static int cgroup_addrm_files(struct cgroup_subsys_state *css,
3358                               struct cgroup *cgrp, struct cftype cfts[],
3359                               bool is_add)
3360 {
3361         struct cftype *cft, *cft_end = NULL;
3362         int ret;
3363 
3364         lockdep_assert_held(&cgroup_mutex);
3365 
3366 restart:
3367         for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) {
3368                 /* does cft->flags tell us to skip this file on @cgrp? */
3369                 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
3370                         continue;
3371                 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))
3372                         continue;
3373                 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
3374                         continue;
3375                 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
3376                         continue;
3377 
3378                 if (is_add) {
3379                         ret = cgroup_add_file(css, cgrp, cft);
3380                         if (ret) {
3381                                 pr_warn("%s: failed to add %s, err=%d\n",
3382                                         __func__, cft->name, ret);
3383                                 cft_end = cft;
3384                                 is_add = false;
3385                                 goto restart;
3386                         }
3387                 } else {
3388                         cgroup_rm_file(cgrp, cft);
3389                 }
3390         }
3391         return 0;
3392 }
3393 
3394 static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
3395 {
3396         LIST_HEAD(pending);
3397         struct cgroup_subsys *ss = cfts[0].ss;
3398         struct cgroup *root = &ss->root->cgrp;
3399         struct cgroup_subsys_state *css;
3400         int ret = 0;
3401 
3402         lockdep_assert_held(&cgroup_mutex);
3403 
3404         /* add/rm files for all cgroups created before */
3405         css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3406                 struct cgroup *cgrp = css->cgroup;
3407 
3408                 if (cgroup_is_dead(cgrp))
3409                         continue;
3410 
3411                 ret = cgroup_addrm_files(css, cgrp, cfts, is_add);
3412                 if (ret)
3413                         break;
3414         }
3415 
3416         if (is_add && !ret)
3417                 kernfs_activate(root->kn);
3418         return ret;
3419 }
3420 
3421 static void cgroup_exit_cftypes(struct cftype *cfts)
3422 {
3423         struct cftype *cft;
3424 
3425         for (cft = cfts; cft->name[0] != '\0'; cft++) {
3426                 /* free copy for custom atomic_write_len, see init_cftypes() */
3427                 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
3428                         kfree(cft->kf_ops);
3429                 cft->kf_ops = NULL;
3430                 cft->ss = NULL;
3431 
3432                 /* revert flags set by cgroup core while adding @cfts */
3433                 cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL);
3434         }
3435 }
3436 
3437 static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3438 {
3439         struct cftype *cft;
3440 
3441         for (cft = cfts; cft->name[0] != '\0'; cft++) {
3442                 struct kernfs_ops *kf_ops;
3443 
3444                 WARN_ON(cft->ss || cft->kf_ops);
3445 
3446                 if (cft->seq_start)
3447                         kf_ops = &cgroup_kf_ops;
3448                 else
3449                         kf_ops = &cgroup_kf_single_ops;
3450 
3451                 /*
3452                  * Ugh... if @cft wants a custom max_write_len, we need to
3453                  * make a copy of kf_ops to set its atomic_write_len.
3454                  */
3455                 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
3456                         kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
3457                         if (!kf_ops) {
3458                                 cgroup_exit_cftypes(cfts);
3459                                 return -ENOMEM;
3460                         }
3461                         kf_ops->atomic_write_len = cft->max_write_len;
3462                 }
3463 
3464                 cft->kf_ops = kf_ops;
3465                 cft->ss = ss;
3466         }
3467 
3468         return 0;
3469 }
3470 
3471 static int cgroup_rm_cftypes_locked(struct cftype *cfts)
3472 {
3473         lockdep_assert_held(&cgroup_mutex);
3474 
3475         if (!cfts || !cfts[0].ss)
3476                 return -ENOENT;
3477 
3478         list_del(&cfts->node);
3479         cgroup_apply_cftypes(cfts, false);
3480         cgroup_exit_cftypes(cfts);
3481         return 0;
3482 }
3483 
3484 /**
3485  * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
3486  * @cfts: zero-length name terminated array of cftypes
3487  *
3488  * Unregister @cfts.  Files described by @cfts are removed from all
3489  * existing cgroups and all future cgroups won't have them either.  This
3490  * function can be called anytime whether @cfts' subsys is attached or not.
3491  *
3492  * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3493  * registered.
3494  */
3495 int cgroup_rm_cftypes(struct cftype *cfts)
3496 {
3497         int ret;
3498 
3499         mutex_lock(&cgroup_mutex);
3500         ret = cgroup_rm_cftypes_locked(cfts);
3501         mutex_unlock(&cgroup_mutex);
3502         return ret;
3503 }
3504 
3505 /**
3506  * cgroup_add_cftypes - add an array of cftypes to a subsystem
3507  * @ss: target cgroup subsystem
3508  * @cfts: zero-length name terminated array of cftypes
3509  *
3510  * Register @cfts to @ss.  Files described by @cfts are created for all
3511  * existing cgroups to which @ss is attached and all future cgroups will
3512  * have them too.  This function can be called anytime whether @ss is
3513  * attached or not.
3514  *
3515  * Returns 0 on successful registration, -errno on failure.  Note that this
3516  * function currently returns 0 as long as @cfts registration is successful
3517  * even if some file creation attempts on existing cgroups fail.
3518  */
3519 static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3520 {
3521         int ret;
3522 
3523         if (!cgroup_ssid_enabled(ss->id))
3524                 return 0;
3525 
3526         if (!cfts || cfts[0].name[0] == '\0')
3527                 return 0;
3528 
3529         ret = cgroup_init_cftypes(ss, cfts);
3530         if (ret)
3531                 return ret;
3532 
3533         mutex_lock(&cgroup_mutex);
3534 
3535         list_add_tail(&cfts->node, &ss->cfts);
3536         ret = cgroup_apply_cftypes(cfts, true);
3537         if (ret)
3538                 cgroup_rm_cftypes_locked(cfts);
3539 
3540         mutex_unlock(&cgroup_mutex);
3541         return ret;
3542 }
3543 
3544 /**
3545  * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
3546  * @ss: target cgroup subsystem
3547  * @cfts: zero-length name terminated array of cftypes
3548  *
3549  * Similar to cgroup_add_cftypes() but the added files are only used for
3550  * the default hierarchy.
3551  */
3552 int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3553 {
3554         struct cftype *cft;
3555 
3556         for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3557                 cft->flags |= __CFTYPE_ONLY_ON_DFL;
3558         return cgroup_add_cftypes(ss, cfts);
3559 }
3560 
3561 /**
3562  * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
3563  * @ss: target cgroup subsystem
3564  * @cfts: zero-length name terminated array of cftypes
3565  *
3566  * Similar to cgroup_add_cftypes() but the added files are only used for
3567  * the legacy hierarchies.
3568  */
3569 int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3570 {
3571         struct cftype *cft;
3572 
3573         for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3574                 cft->flags |= __CFTYPE_NOT_ON_DFL;
3575         return cgroup_add_cftypes(ss, cfts);
3576 }
3577 
3578 /**
3579  * cgroup_file_notify - generate a file modified event for a cgroup_file
3580  * @cfile: target cgroup_file
3581  *
3582  * @cfile must have been obtained by setting cftype->file_offset.
3583  */
3584 void cgroup_file_notify(struct cgroup_file *cfile)
3585 {
3586         unsigned long flags;
3587 
3588         spin_lock_irqsave(&cgroup_file_kn_lock, flags);
3589         if (cfile->kn)
3590                 kernfs_notify(cfile->kn);
3591         spin_unlock_irqrestore(&cgroup_file_kn_lock, flags);
3592 }
3593 
3594 /**
3595  * cgroup_task_count - count the number of tasks in a cgroup.
3596  * @cgrp: the cgroup in question
3597  *
3598  * Return the number of tasks in the cgroup.
3599  */
3600 static int cgroup_task_count(const struct cgroup *cgrp)
3601 {
3602         int count = 0;
3603         struct cgrp_cset_link *link;
3604 
3605         spin_lock_bh(&css_set_lock);
3606         list_for_each_entry(link, &cgrp->cset_links, cset_link)
3607                 count += atomic_read(&link->cset->refcount);
3608         spin_unlock_bh(&css_set_lock);
3609         return count;
3610 }
3611 
3612 /**
3613  * css_next_child - find the next child of a given css
3614  * @pos: the current position (%NULL to initiate traversal)
3615  * @parent: css whose children to walk
3616  *
3617  * This function returns the next child of @parent and should be called
3618  * under either cgroup_mutex or RCU read lock.  The only requirement is
3619  * that @parent and @pos are accessible.  The next sibling is guaranteed to
3620  * be returned regardless of their states.
3621  *
3622  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3623  * css which finished ->css_online() is guaranteed to be visible in the
3624  * future iterations and will stay visible until the last reference is put.
3625  * A css which hasn't finished ->css_online() or already finished
3626  * ->css_offline() may show up during traversal.  It's each subsystem's
3627  * responsibility to synchronize against on/offlining.
3628  */
3629 struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
3630                                            struct cgroup_subsys_state *parent)
3631 {
3632         struct cgroup_subsys_state *next;
3633 
3634         cgroup_assert_mutex_or_rcu_locked();
3635 
3636         /*
3637          * @pos could already have been unlinked from the sibling list.
3638          * Once a cgroup is removed, its ->sibling.next is no longer
3639          * updated when its next sibling changes.  CSS_RELEASED is set when
3640          * @pos is taken off list, at which time its next pointer is valid,
3641          * and, as releases are serialized, the one pointed to by the next
3642          * pointer is guaranteed to not have started release yet.  This
3643          * implies that if we observe !CSS_RELEASED on @pos in this RCU
3644          * critical section, the one pointed to by its next pointer is
3645          * guaranteed to not have finished its RCU grace period even if we
3646          * have dropped rcu_read_lock() inbetween iterations.
3647          *
3648          * If @pos has CSS_RELEASED set, its next pointer can't be
3649          * dereferenced; however, as each css is given a monotonically
3650          * increasing unique serial number and always appended to the
3651          * sibling list, the next one can be found by walking the parent's
3652          * children until the first css with higher serial number than
3653          * @pos's.  While this path can be slower, it happens iff iteration
3654          * races against release and the race window is very small.
3655          */
3656         if (!pos) {
3657                 next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
3658         } else if (likely(!(pos->flags & CSS_RELEASED))) {
3659                 next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3660         } else {
3661                 list_for_each_entry_rcu(next, &parent->children, sibling)
3662                         if (next->serial_nr > pos->serial_nr)
3663                                 break;
3664         }
3665 
3666         /*
3667          * @next, if not pointing to the head, can be dereferenced and is
3668          * the next sibling.
3669          */
3670         if (&next->sibling != &parent->children)
3671                 return next;
3672         return NULL;
3673 }
3674 
3675 /**
3676  * css_next_descendant_pre - find the next descendant for pre-order walk
3677  * @pos: the current position (%NULL to initiate traversal)
3678  * @root: css whose descendants to walk
3679  *
3680  * To be used by css_for_each_descendant_pre().  Find the next descendant
3681  * to visit for pre-order traversal of @root's descendants.  @root is
3682  * included in the iteration and the first node to be visited.
3683  *
3684  * While this function requires cgroup_mutex or RCU read locking, it
3685  * doesn't require the whole traversal to be contained in a single critical
3686  * section.  This function will return the correct next descendant as long
3687  * as both @pos and @root are accessible and @pos is a descendant of @root.
3688  *
3689  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3690  * css which finished ->css_online() is guaranteed to be visible in the
3691  * future iterations and will stay visible until the last reference is put.
3692  * A css which hasn't finished ->css_online() or already finished
3693  * ->css_offline() may show up during traversal.  It's each subsystem's
3694  * responsibility to synchronize against on/offlining.
3695  */
3696 struct cgroup_subsys_state *
3697 css_next_descendant_pre(struct cgroup_subsys_state *pos,
3698                         struct cgroup_subsys_state *root)
3699 {
3700         struct cgroup_subsys_state *next;
3701 
3702         cgroup_assert_mutex_or_rcu_locked();
3703 
3704         /* if first iteration, visit @root */
3705         if (!pos)
3706                 return root;
3707 
3708         /* visit the first child if exists */
3709         next = css_next_child(NULL, pos);
3710         if (next)
3711                 return next;
3712 
3713         /* no child, visit my or the closest ancestor's next sibling */
3714         while (pos != root) {
3715                 next = css_next_child(pos, pos->parent);
3716                 if (next)
3717                         return next;
3718                 pos = pos->parent;
3719         }
3720 
3721         return NULL;
3722 }
3723 
3724 /**
3725  * css_rightmost_descendant - return the rightmost descendant of a css
3726  * @pos: css of interest
3727  *
3728  * Return the rightmost descendant of @pos.  If there's no descendant, @pos
3729  * is returned.  This can be used during pre-order traversal to skip
3730  * subtree of @pos.
3731  *
3732  * While this function requires cgroup_mutex or RCU read locking, it
3733  * doesn't require the whole traversal to be contained in a single critical
3734  * section.  This function will return the correct rightmost descendant as
3735  * long as @pos is accessible.
3736  */
3737 struct cgroup_subsys_state *
3738 css_rightmost_descendant(struct cgroup_subsys_state *pos)
3739 {
3740         struct cgroup_subsys_state *last, *tmp;
3741 
3742         cgroup_assert_mutex_or_rcu_locked();
3743 
3744         do {
3745                 last = pos;
3746                 /* ->prev isn't RCU safe, walk ->next till the end */
3747                 pos = NULL;
3748                 css_for_each_child(tmp, last)
3749                         pos = tmp;
3750         } while (pos);
3751 
3752         return last;
3753 }
3754 
3755 static struct cgroup_subsys_state *
3756 css_leftmost_descendant(struct cgroup_subsys_state *pos)
3757 {
3758         struct cgroup_subsys_state *last;
3759 
3760         do {
3761                 last = pos;
3762                 pos = css_next_child(NULL, pos);
3763         } while (pos);
3764 
3765         return last;
3766 }
3767 
3768 /**
3769  * css_next_descendant_post - find the next descendant for post-order walk
3770  * @pos: the current position (%NULL to initiate traversal)
3771  * @root: css whose descendants to walk
3772  *
3773  * To be used by css_for_each_descendant_post().  Find the next descendant
3774  * to visit for post-order traversal of @root's descendants.  @root is
3775  * included in the iteration and the last node to be visited.
3776  *
3777  * While this function requires cgroup_mutex or RCU read locking, it
3778  * doesn't require the whole traversal to be contained in a single critical
3779  * section.  This function will return the correct next descendant as long
3780  * as both @pos and @cgroup are accessible and @pos is a descendant of
3781  * @cgroup.
3782  *
3783  * If a subsystem synchronizes ->css_online() and the start of iteration, a
3784  * css which finished ->css_online() is guaranteed to be visible in the
3785  * future iterations and will stay visible until the last reference is put.
3786  * A css which hasn't finished ->css_online() or already finished
3787  * ->css_offline() may show up during traversal.  It's each subsystem's
3788  * responsibility to synchronize against on/offlining.
3789  */
3790 struct cgroup_subsys_state *
3791 css_next_descendant_post(struct cgroup_subsys_state *pos,
3792                          struct cgroup_subsys_state *root)
3793 {
3794         struct cgroup_subsys_state *next;
3795 
3796         cgroup_assert_mutex_or_rcu_locked();
3797 
3798         /* if first iteration, visit leftmost descendant which may be @root */
3799         if (!pos)
3800                 return css_leftmost_descendant(root);
3801 
3802         /* if we visited @root, we're done */
3803         if (pos == root)
3804                 return NULL;
3805 
3806         /* if there's an unvisited sibling, visit its leftmost descendant */
3807         next = css_next_child(pos, pos->parent);
3808         if (next)
3809                 return css_leftmost_descendant(next);
3810 
3811         /* no sibling left, visit parent */
3812         return pos->parent;
3813 }
3814 
3815 /**
3816  * css_has_online_children - does a css have online children
3817  * @css: the target css
3818  *
3819  * Returns %true if @css has any online children; otherwise, %false.  This
3820  * function can be called from any context but the caller is responsible
3821  * for synchronizing against on/offlining as necessary.
3822  */
3823 bool css_has_online_children(struct cgroup_subsys_state *css)
3824 {
3825         struct cgroup_subsys_state *child;
3826         bool ret = false;
3827 
3828         rcu_read_lock();
3829         css_for_each_child(child, css) {
3830                 if (child->flags & CSS_ONLINE) {
3831                         ret = true;
3832                         break;
3833                 }
3834         }
3835         rcu_read_unlock();
3836         return ret;
3837 }
3838 
3839 /**
3840  * css_task_iter_advance_css_set - advance a task itererator to the next css_set
3841  * @it: the iterator to advance
3842  *
3843  * Advance @it to the next css_set to walk.
3844  */
3845 static void css_task_iter_advance_css_set(struct css_task_iter *it)
3846 {
3847         struct list_head *l = it->cset_pos;
3848         struct cgrp_cset_link *link;
3849         struct css_set *cset;
3850 
3851         lockdep_assert_held(&css_set_lock);
3852 
3853         /* Advance to the next non-empty css_set */
3854         do {
3855                 l = l->next;
3856                 if (l == it->cset_head) {
3857                         it->cset_pos = NULL;
3858                         it->task_pos = NULL;
3859                         return;
3860                 }
3861 
3862                 if (it->ss) {
3863                         cset = container_of(l, struct css_set,
3864                                             e_cset_node[it->ss->id]);
3865                 } else {
3866                         link = list_entry(l, struct cgrp_cset_link, cset_link);
3867                         cset = link->cset;
3868                 }
3869         } while (!css_set_populated(cset));
3870 
3871         it->cset_pos = l;
3872 
3873         if (!list_empty(&cset->tasks))
3874                 it->task_pos = cset->tasks.next;
3875         else
3876                 it->task_pos = cset->mg_tasks.next;
3877 
3878         it->tasks_head = &cset->tasks;
3879         it->mg_tasks_head = &cset->mg_tasks;
3880 
3881         /*
3882          * We don't keep css_sets locked across iteration steps and thus
3883          * need to take steps to ensure that iteration can be resumed after
3884          * the lock is re-acquired.  Iteration is performed at two levels -
3885          * css_sets and tasks in them.
3886          *
3887          * Once created, a css_set never leaves its cgroup lists, so a
3888          * pinned css_set is guaranteed to stay put and we can resume
3889          * iteration afterwards.
3890          *
3891          * Tasks may leave @cset across iteration steps.  This is resolved
3892          * by registering each iterator with the css_set currently being
3893          * walked and making css_set_move_task() advance iterators whose
3894          * next task is leaving.
3895          */
3896         if (it->cur_cset) {
3897                 list_del(&it->iters_node);
3898                 put_css_set_locked(it->cur_cset);
3899         }
3900         get_css_set(cset);
3901         it->cur_cset = cset;
3902         list_add(&it->iters_node, &cset->task_iters);
3903 }
3904 
3905 static void css_task_iter_advance(struct css_task_iter *it)
3906 {
3907         struct list_head *l = it->task_pos;
3908 
3909         lockdep_assert_held(&css_set_lock);
3910         WARN_ON_ONCE(!l);
3911 
3912         /*
3913          * Advance iterator to find next entry.  cset->tasks is consumed
3914          * first and then ->mg_tasks.  After ->mg_tasks, we move onto the
3915          * next cset.
3916          */
3917         l = l->next;
3918 
3919         if (l == it->tasks_head)
3920                 l = it->mg_tasks_head->next;
3921 
3922         if (l == it->mg_tasks_head)
3923                 css_task_iter_advance_css_set(it);
3924         else
3925                 it->task_pos = l;
3926 }
3927 
3928 /**
3929  * css_task_iter_start - initiate task iteration
3930  * @css: the css to walk tasks of
3931  * @it: the task iterator to use
3932  *
3933  * Initiate iteration through the tasks of @css.  The caller can call
3934  * css_task_iter_next() to walk through the tasks until the function
3935  * returns NULL.  On completion of iteration, css_task_iter_end() must be
3936  * called.
3937  */
3938 void css_task_iter_start(struct cgroup_subsys_state *css,
3939                          struct css_task_iter *it)
3940 {
3941         /* no one should try to iterate before mounting cgroups */
3942         WARN_ON_ONCE(!use_task_css_set_links);
3943 
3944         memset(it, 0, sizeof(*it));
3945 
3946         spin_lock_bh(&css_set_lock);
3947 
3948         it->ss = css->ss;
3949 
3950         if (it->ss)
3951                 it->cset_pos = &css->cgroup->e_csets[css->ss->id];
3952         else
3953                 it->cset_pos = &css->cgroup->cset_links;
3954 
3955         it->cset_head = it->cset_pos;
3956 
3957         css_task_iter_advance_css_set(it);
3958 
3959         spin_unlock_bh(&css_set_lock);
3960 }
3961 
3962 /**
3963  * css_task_iter_next - return the next task for the iterator
3964  * @it: the task iterator being iterated
3965  *
3966  * The "next" function for task iteration.  @it should have been
3967  * initialized via css_task_iter_start().  Returns NULL when the iteration
3968  * reaches the end.
3969  */
3970 struct task_struct *css_task_iter_next(struct css_task_iter *it)
3971 {
3972         if (it->cur_task) {
3973                 put_task_struct(it->cur_task);
3974                 it->cur_task = NULL;
3975         }
3976 
3977         spin_lock_bh(&css_set_lock);
3978 
3979         if (it->task_pos) {
3980                 it->cur_task = list_entry(it->task_pos, struct task_struct,
3981                                           cg_list);
3982                 get_task_struct(it->cur_task);
3983                 css_task_iter_advance(it);
3984         }
3985 
3986         spin_unlock_bh(&css_set_lock);
3987 
3988         return it->cur_task;
3989 }
3990 
3991 /**
3992  * css_task_iter_end - finish task iteration
3993  * @it: the task iterator to finish
3994  *
3995  * Finish task iteration started by css_task_iter_start().
3996  */
3997 void css_task_iter_end(struct css_task_iter *it)
3998 {
3999         if (it->cur_cset) {
4000                 spin_lock_bh(&css_set_lock);
4001                 list_del(&it->iters_node);
4002                 put_css_set_locked(it->cur_cset);
4003                 spin_unlock_bh(&css_set_lock);
4004         }
4005 
4006         if (it->cur_task)
4007                 put_task_struct(it->cur_task);
4008 }
4009 
4010 /**
4011  * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
4012  * @to: cgroup to which the tasks will be moved
4013  * @from: cgroup in which the tasks currently reside
4014  *
4015  * Locking rules between cgroup_post_fork() and the migration path
4016  * guarantee that, if a task is forking while being migrated, the new child
4017  * is guaranteed to be either visible in the source cgroup after the
4018  * parent's migration is complete or put into the target cgroup.  No task
4019  * can slip out of migration through forking.
4020  */
4021 int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
4022 {
4023         LIST_HEAD(preloaded_csets);
4024         struct cgrp_cset_link *link;
4025         struct css_task_iter it;
4026         struct task_struct *task;
4027         int ret;
4028 
4029         mutex_lock(&cgroup_mutex);
4030 
4031         /* all tasks in @from are being moved, all csets are source */
4032         spin_lock_bh(&css_set_lock);
4033         list_for_each_entry(link, &from->cset_links, cset_link)
4034                 cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
4035         spin_unlock_bh(&css_set_lock);
4036 
4037         ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
4038         if (ret)
4039                 goto out_err;
4040 
4041         /*
4042          * Migrate tasks one-by-one until @from is empty.  This fails iff
4043          * ->can_attach() fails.
4044          */
4045         do {
4046                 css_task_iter_start(&from->self, &it);
4047                 task = css_task_iter_next(&it);
4048                 if (task)
4049                         get_task_struct(task);
4050                 css_task_iter_end(&it);
4051 
4052                 if (task) {
4053                         ret = cgroup_migrate(task, false, to);
4054                         put_task_struct(task);
4055                 }
4056         } while (task && !ret);
4057 out_err:
4058         cgroup_migrate_finish(&preloaded_csets);
4059         mutex_unlock(&cgroup_mutex);
4060         return ret;
4061 }
4062 
4063 /*
4064  * Stuff for reading the 'tasks'/'procs' files.
4065  *
4066  * Reading this file can return large amounts of data if a cgroup has
4067  * *lots* of attached tasks. So it may need several calls to read(),
4068  * but we cannot guarantee that the information we produce is correct
4069  * unless we produce it entirely atomically.
4070  *
4071  */
4072 
4073 /* which pidlist file are we talking about? */
4074 enum cgroup_filetype {
4075         CGROUP_FILE_PROCS,
4076         CGROUP_FILE_TASKS,
4077 };
4078 
4079 /*
4080  * A pidlist is a list of pids that virtually represents the contents of one
4081  * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
4082  * a pair (one each for procs, tasks) for each pid namespace that's relevant
4083  * to the cgroup.
4084  */
4085 struct cgroup_pidlist {
4086         /*
4087          * used to find which pidlist is wanted. doesn't change as long as
4088          * this particular list stays in the list.
4089         */
4090         struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
4091         /* array of xids */
4092         pid_t *list;
4093         /* how many elements the above list has */
4094         int length;
4095         /* each of these stored in a list by its cgroup */
4096         struct list_head links;
4097         /* pointer to the cgroup we belong to, for list removal purposes */
4098         struct cgroup *owner;
4099         /* for delayed destruction */
4100         struct delayed_work destroy_dwork;
4101 };
4102 
4103 /*
4104  * The following two functions "fix" the issue where there are more pids
4105  * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
4106  * TODO: replace with a kernel-wide solution to this problem
4107  */
4108 #define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
4109 static void *pidlist_allocate(int count)
4110 {
4111         if (PIDLIST_TOO_LARGE(count))
4112                 return vmalloc(count * sizeof(pid_t));
4113         else
4114                 return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
4115 }
4116 
4117 static void pidlist_free(void *p)
4118 {
4119         kvfree(p);
4120 }
4121 
4122 /*
4123  * Used to destroy all pidlists lingering waiting for destroy timer.  None
4124  * should be left afterwards.
4125  */
4126 static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
4127 {
4128         struct cgroup_pidlist *l, *tmp_l;
4129 
4130         mutex_lock(&cgrp->pidlist_mutex);
4131         list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
4132                 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
4133         mutex_unlock(&cgrp->pidlist_mutex);
4134 
4135         flush_workqueue(cgroup_pidlist_destroy_wq);
4136         BUG_ON(!list_empty(&cgrp->pidlists));
4137 }
4138 
4139 static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
4140 {
4141         struct delayed_work *dwork = to_delayed_work(work);
4142         struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
4143                                                 destroy_dwork);
4144         struct cgroup_pidlist *tofree = NULL;
4145 
4146         mutex_lock(&l->owner->pidlist_mutex);
4147 
4148         /*
4149          * Destroy iff we didn't get queued again.  The state won't change
4150          * as destroy_dwork can only be queued while locked.
4151          */
4152         if (!delayed_work_pending(dwork)) {
4153                 list_del(&l->links);
4154                 pidlist_free(l->list);
4155                 put_pid_ns(l->key.ns);
4156                 tofree = l;
4157         }
4158 
4159         mutex_unlock(&l->owner->pidlist_mutex);
4160         kfree(tofree);
4161 }
4162 
4163 /*
4164  * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
4165  * Returns the number of unique elements.
4166  */
4167 static int pidlist_uniq(pid_t *list, int length)
4168 {
4169         int src, dest = 1;
4170 
4171         /*
4172          * we presume the 0th element is unique, so i starts at 1. trivial
4173          * edge cases first; no work needs to be done for either
4174          */
4175         if (length == 0 || length == 1)
4176                 return length;
4177         /* src and dest walk down the list; dest counts unique elements */
4178         for (src = 1; src < length; src++) {
4179                 /* find next unique element */
4180                 while (list[src] == list[src-1]) {
4181                         src++;
4182                         if (src == length)
4183                                 goto after;
4184                 }
4185                 /* dest always points to where the next unique element goes */
4186                 list[dest] = list[src];
4187                 dest++;
4188         }
4189 after:
4190         return dest;
4191 }
4192 
4193 /*
4194  * The two pid files - task and cgroup.procs - guaranteed that the result
4195  * is sorted, which forced this whole pidlist fiasco.  As pid order is
4196  * different per namespace, each namespace needs differently sorted list,
4197  * making it impossible to use, for example, single rbtree of member tasks
4198  * sorted by task pointer.  As pidlists can be fairly large, allocating one
4199  * per open file is dangerous, so cgroup had to implement shared pool of
4200  * pidlists keyed by cgroup and namespace.
4201  *
4202  * All this extra complexity was caused by the original implementation
4203  * committing to an entirely unnecessary property.  In the long term, we
4204  * want to do away with it.  Explicitly scramble sort order if on the
4205  * default hierarchy so that no such expectation exists in the new
4206  * interface.
4207  *
4208  * Scrambling is done by swapping every two consecutive bits, which is
4209  * non-identity one-to-one mapping which disturbs sort order sufficiently.
4210  */
4211 static pid_t pid_fry(pid_t pid)
4212 {
4213         unsigned a = pid & 0x55555555;
4214         unsigned b = pid & 0xAAAAAAAA;
4215 
4216         return (a << 1) | (b >> 1);
4217 }
4218 
4219 static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
4220 {
4221         if (cgroup_on_dfl(cgrp))
4222                 return pid_fry(pid);
4223         else
4224                 return pid;
4225 }
4226 
4227 static int cmppid(const void *a, const void *b)
4228 {
4229         return *(pid_t *)a - *(pid_t *)b;
4230 }
4231 
4232 static int fried_cmppid(const void *a, const void *b)
4233 {
4234         return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
4235 }
4236 
4237 static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
4238                                                   enum cgroup_filetype type)
4239 {
4240         struct cgroup_pidlist *l;
4241         /* don't need task_nsproxy() if we're looking at ourself */
4242         struct pid_namespace *ns = task_active_pid_ns(current);
4243 
4244         lockdep_assert_held(&cgrp->pidlist_mutex);
4245 
4246         list_for_each_entry(l, &cgrp->pidlists, links)
4247                 if (l->key.type == type && l->key.ns == ns)
4248                         return l;
4249         return NULL;
4250 }
4251 
4252 /*
4253  * find the appropriate pidlist for our purpose (given procs vs tasks)
4254  * returns with the lock on that pidlist already held, and takes care
4255  * of the use count, or returns NULL with no locks held if we're out of
4256  * memory.
4257  */
4258 static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
4259                                                 enum cgroup_filetype type)
4260 {
4261         struct cgroup_pidlist *l;
4262 
4263         lockdep_assert_held(&cgrp->pidlist_mutex);
4264 
4265         l = cgroup_pidlist_find(cgrp, type);
4266         if (l)
4267                 return l;
4268 
4269         /* entry not found; create a new one */
4270         l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
4271         if (!l)
4272                 return l;
4273 
4274         INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
4275         l->key.type = type;
4276         /* don't need task_nsproxy() if we're looking at ourself */
4277         l->key.ns = get_pid_ns(task_active_pid_ns(current));
4278         l->owner = cgrp;
4279         list_add(&l->links, &cgrp->pidlists);
4280         return l;
4281 }
4282 
4283 /*
4284  * Load a cgroup's pidarray with either procs' tgids or tasks' pids
4285  */
4286 static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
4287                               struct cgroup_pidlist **lp)
4288 {
4289         pid_t *array;
4290         int length;
4291         int pid, n = 0; /* used for populating the array */
4292         struct css_task_iter it;
4293         struct task_struct *tsk;
4294         struct cgroup_pidlist *l;
4295 
4296         lockdep_assert_held(&cgrp->pidlist_mutex);
4297 
4298         /*
4299          * If cgroup gets more users after we read count, we won't have
4300          * enough space - tough.  This race is indistinguishable to the
4301          * caller from the case that the additional cgroup users didn't
4302          * show up until sometime later on.
4303          */
4304         length = cgroup_task_count(cgrp);
4305         array = pidlist_allocate(length);
4306         if (!array)
4307                 return -ENOMEM;
4308         /* now, populate the array */
4309         css_task_iter_start(&cgrp->self, &it);
4310         while ((tsk = css_task_iter_next(&it))) {
4311                 if (unlikely(n == length))
4312                         break;
4313                 /* get tgid or pid for procs or tasks file respectively */
4314                 if (type == CGROUP_FILE_PROCS)
4315                         pid = task_tgid_vnr(tsk);
4316                 else
4317                         pid = task_pid_vnr(tsk);
4318                 if (pid > 0) /* make sure to only use valid results */
4319                         array[n++] = pid;
4320         }
4321         css_task_iter_end(&it);
4322         length = n;
4323         /* now sort & (if procs) strip out duplicates */
4324         if (cgroup_on_dfl(cgrp))
4325                 sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
4326         else
4327                 sort(array, length, sizeof(pid_t), cmppid, NULL);
4328         if (type == CGROUP_FILE_PROCS)
4329                 length = pidlist_uniq(array, length);
4330 
4331         l = cgroup_pidlist_find_create(cgrp, type);
4332         if (!l) {
4333                 pidlist_free(array);
4334                 return -ENOMEM;
4335         }
4336 
4337         /* store array, freeing old if necessary */
4338         pidlist_free(l->list);
4339         l->list = array;
4340         l->length = length;
4341         *lp = l;
4342         return 0;
4343 }
4344 
4345 /**
4346  * cgroupstats_build - build and fill cgroupstats
4347  * @stats: cgroupstats to fill information into
4348  * @dentry: A dentry entry belonging to the cgroup for which stats have
4349  * been requested.
4350  *
4351  * Build and fill cgroupstats so that taskstats can export it to user
4352  * space.
4353  */
4354 int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
4355 {
4356         struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
4357         struct cgroup *cgrp;
4358         struct css_task_iter it;
4359         struct task_struct *tsk;
4360 
4361         /* it should be kernfs_node belonging to cgroupfs and is a directory */
4362         if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
4363             kernfs_type(kn) != KERNFS_DIR)
4364                 return -EINVAL;
4365 
4366         mutex_lock(&cgroup_mutex);
4367 
4368         /*
4369          * We aren't being called from kernfs and there's no guarantee on
4370          * @kn->priv's validity.  For this and css_tryget_online_from_dir(),
4371          * @kn->priv is RCU safe.  Let's do the RCU dancing.
4372          */
4373         rcu_read_lock();
4374         cgrp = rcu_dereference(kn->priv);
4375         if (!cgrp || cgroup_is_dead(cgrp)) {
4376                 rcu_read_unlock();
4377                 mutex_unlock(&cgroup_mutex);
4378                 return -ENOENT;
4379         }
4380         rcu_read_unlock();
4381 
4382         css_task_iter_start(&cgrp->self, &it);
4383         while ((tsk = css_task_iter_next(&it))) {
4384                 switch (tsk->state) {
4385                 case TASK_RUNNING:
4386                         stats->nr_running++;
4387                         break;
4388                 case TASK_INTERRUPTIBLE:
4389                         stats->nr_sleeping++;
4390                         break;
4391                 case TASK_UNINTERRUPTIBLE:
4392                         stats->nr_uninterruptible++;
4393                         break;
4394                 case TASK_STOPPED:
4395                         stats->nr_stopped++;
4396                         break;
4397                 default:
4398                         if (delayacct_is_task_waiting_on_io(tsk))
4399                                 stats->nr_io_wait++;
4400                         break;
4401                 }
4402         }
4403         css_task_iter_end(&it);
4404 
4405         mutex_unlock(&cgroup_mutex);
4406         return 0;
4407 }
4408 
4409 
4410 /*
4411  * seq_file methods for the tasks/procs files. The seq_file position is the
4412  * next pid to display; the seq_file iterator is a pointer to the pid
4413  * in the cgroup->l->list array.
4414  */
4415 
4416 static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
4417 {
4418         /*
4419          * Initially we receive a position value that corresponds to
4420          * one more than the last pid shown (or 0 on the first call or
4421          * after a seek to the start). Use a binary-search to find the
4422          * next pid to display, if any
4423          */
4424         struct kernfs_open_file *of = s->private;
4425         struct cgroup *cgrp = seq_css(s)->cgroup;
4426         struct cgroup_pidlist *l;
4427         enum cgroup_filetype type = seq_cft(s)->private;
4428         int index = 0, pid = *pos;
4429         int *iter, ret;
4430 
4431         mutex_lock(&cgrp->pidlist_mutex);
4432 
4433         /*
4434          * !NULL @of->priv indicates that this isn't the first start()
4435          * after open.  If the matching pidlist is around, we can use that.
4436          * Look for it.  Note that @of->priv can't be used directly.  It
4437          * could already have been destroyed.
4438          */
4439         if (of->priv)
4440                 of->priv = cgroup_pidlist_find(cgrp, type);
4441 
4442         /*
4443          * Either this is the first start() after open or the matching
4444          * pidlist has been destroyed inbetween.  Create a new one.
4445          */
4446         if (!of->priv) {
4447                 ret = pidlist_array_load(cgrp, type,
4448                                          (struct cgroup_pidlist **)&of->priv);
4449                 if (ret)
4450                         return ERR_PTR(ret);
4451         }
4452         l = of->priv;
4453 
4454         if (pid) {
4455                 int end = l->length;
4456 
4457                 while (index < end) {
4458                         int mid = (index + end) / 2;
4459                         if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
4460                                 index = mid;
4461                                 break;
4462                         } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
4463                                 index = mid + 1;
4464                         else
4465                                 end = mid;
4466                 }
4467         }
4468         /* If we're off the end of the array, we're done */
4469         if (index >= l->length)
4470                 return NULL;
4471         /* Update the abstract position to be the actual pid that we found */
4472         iter = l->list + index;
4473         *pos = cgroup_pid_fry(cgrp, *iter);
4474         return iter;
4475 }
4476 
4477 static void cgroup_pidlist_stop(struct seq_file *s, void *v)
4478 {
4479         struct kernfs_open_file *of = s->private;
4480         struct cgroup_pidlist *l = of->priv;
4481 
4482         if (l)
4483                 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
4484                                  CGROUP_PIDLIST_DESTROY_DELAY);
4485         mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
4486 }
4487 
4488 static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
4489 {
4490         struct kernfs_open_file *of = s->private;
4491         struct cgroup_pidlist *l = of->priv;
4492         pid_t *p = v;
4493         pid_t *end = l->list + l->length;
4494         /*
4495          * Advance to the next pid in the array. If this goes off the
4496          * end, we're done
4497          */
4498         p++;
4499         if (p >= end) {
4500                 return NULL;
4501         } else {
4502                 *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
4503                 return p;
4504         }
4505 }
4506 
4507 static int cgroup_pidlist_show(struct seq_file *s, void *v)
4508 {
4509         seq_printf(s, "%d\n", *(int *)v);
4510 
4511         return 0;
4512 }
4513 
4514 static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
4515                                          struct cftype *cft)
4516 {
4517         return notify_on_release(css->cgroup);
4518 }
4519 
4520 static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
4521                                           struct cftype *cft, u64 val)
4522 {
4523         if (val)
4524                 set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4525         else
4526                 clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4527         return 0;
4528 }
4529 
4530 static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
4531                                       struct cftype *cft)
4532 {
4533         return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4534 }
4535 
4536 static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
4537                                        struct cftype *cft, u64 val)
4538 {
4539         if (val)
4540                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4541         else
4542                 clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4543         return 0;
4544 }
4545 
4546 /* cgroup core interface files for the default hierarchy */
4547 static struct cftype cgroup_dfl_base_files[] = {
4548         {
4549                 .name = "cgroup.procs",
4550                 .file_offset = offsetof(struct cgroup, procs_file),
4551                 .seq_start = cgroup_pidlist_start,
4552                 .seq_next = cgroup_pidlist_next,
4553                 .seq_stop = cgroup_pidlist_stop,
4554                 .seq_show = cgroup_pidlist_show,
4555                 .private = CGROUP_FILE_PROCS,
4556                 .write = cgroup_procs_write,
4557         },
4558         {
4559                 .name = "cgroup.controllers",
4560                 .flags = CFTYPE_ONLY_ON_ROOT,
4561                 .seq_show = cgroup_root_controllers_show,
4562         },
4563         {
4564                 .name = "cgroup.controllers",
4565                 .flags = CFTYPE_NOT_ON_ROOT,
4566                 .seq_show = cgroup_controllers_show,
4567         },
4568         {
4569                 .name = "cgroup.subtree_control",
4570                 .seq_show = cgroup_subtree_control_show,
4571                 .write = cgroup_subtree_control_write,
4572         },
4573         {
4574                 .name = "cgroup.events",
4575                 .flags = CFTYPE_NOT_ON_ROOT,
4576                 .file_offset = offsetof(struct cgroup, events_file),
4577                 .seq_show = cgroup_events_show,
4578         },
4579         { }     /* terminate */
4580 };
4581 
4582 /* cgroup core interface files for the legacy hierarchies */
4583 static struct cftype cgroup_legacy_base_files[] = {
4584         {
4585                 .name = "cgroup.procs",
4586                 .seq_start = cgroup_pidlist_start,
4587                 .seq_next = cgroup_pidlist_next,
4588                 .seq_stop = cgroup_pidlist_stop,
4589                 .seq_show = cgroup_pidlist_show,
4590                 .private = CGROUP_FILE_PROCS,
4591                 .write = cgroup_procs_write,
4592         },
4593         {
4594                 .name = "cgroup.clone_children",
4595                 .read_u64 = cgroup_clone_children_read,
4596                 .write_u64 = cgroup_clone_children_write,
4597         },
4598         {
4599                 .name = "cgroup.sane_behavior",
4600                 .flags = CFTYPE_ONLY_ON_ROOT,
4601                 .seq_show = cgroup_sane_behavior_show,
4602         },
4603         {
4604                 .name = "tasks",
4605                 .seq_start = cgroup_pidlist_start,
4606                 .seq_next = cgroup_pidlist_next,
4607                 .seq_stop = cgroup_pidlist_stop,
4608                 .seq_show = cgroup_pidlist_show,
4609                 .private = CGROUP_FILE_TASKS,
4610                 .write = cgroup_tasks_write,
4611         },
4612         {
4613                 .name = "notify_on_release",
4614                 .read_u64 = cgroup_read_notify_on_release,
4615                 .write_u64 = cgroup_write_notify_on_release,
4616         },
4617         {
4618                 .name = "release_agent",
4619                 .flags = CFTYPE_ONLY_ON_ROOT,
4620                 .seq_show = cgroup_release_agent_show,
4621                 .write = cgroup_release_agent_write,
4622                 .max_write_len = PATH_MAX - 1,
4623         },
4624         { }     /* terminate */
4625 };
4626 
4627 /*
4628  * css destruction is four-stage process.
4629  *
4630  * 1. Destruction starts.  Killing of the percpu_ref is initiated.
4631  *    Implemented in kill_css().
4632  *
4633  * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4634  *    and thus css_tryget_online() is guaranteed to fail, the css can be
4635  *    offlined by invoking offline_css().  After offlining, the base ref is
4636  *    put.  Implemented in css_killed_work_fn().
4637  *
4638  * 3. When the percpu_ref reaches zero, the only possible remaining
4639  *    accessors are inside RCU read sections.  css_release() schedules the
4640  *    RCU callback.
4641  *
4642  * 4. After the grace period, the css can be freed.  Implemented in
4643  *    css_free_work_fn().
4644  *
4645  * It is actually hairier because both step 2 and 4 require process context
4646  * and thus involve punting to css->destroy_work adding two additional
4647  * steps to the already complex sequence.
4648  */
4649 static void css_free_work_fn(struct work_struct *work)
4650 {
4651         struct cgroup_subsys_state *css =
4652                 container_of(work, struct cgroup_subsys_state, destroy_work);
4653         struct cgroup_subsys *ss = css->ss;
4654         struct cgroup *cgrp = css->cgroup;
4655 
4656         percpu_ref_exit(&css->refcnt);
4657 
4658         if (ss) {
4659                 /* css free path */
4660                 struct cgroup_subsys_state *parent = css->parent;
4661                 int id = css->id;
4662 
4663                 ss->css_free(css);
4664                 cgroup_idr_remove(&ss->css_idr, id);
4665                 cgroup_put(cgrp);
4666 
4667                 if (parent)
4668                         css_put(parent);
4669         } else {
4670                 /* cgroup free path */
4671                 atomic_dec(&cgrp->root->nr_cgrps);
4672                 cgroup_pidlist_destroy_all(cgrp);
4673                 cancel_work_sync(&cgrp->release_agent_work);
4674 
4675                 if (cgroup_parent(cgrp)) {
4676                         /*
4677                          * We get a ref to the parent, and put the ref when
4678                          * this cgroup is being freed, so it's guaranteed
4679                          * that the parent won't be destroyed before its
4680                          * children.
4681                          */
4682                         cgroup_put(cgroup_parent(cgrp));
4683                         kernfs_put(cgrp->kn);
4684                         kfree(cgrp);
4685                 } else {
4686                         /*
4687                          * This is root cgroup's refcnt reaching zero,
4688                          * which indicates that the root should be
4689                          * released.
4690                          */
4691                         cgroup_destroy_root(cgrp->root);
4692                 }
4693         }
4694 }
4695 
4696 static void css_free_rcu_fn(struct rcu_head *rcu_head)
4697 {
4698         struct cgroup_subsys_state *css =
4699                 container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4700 
4701         INIT_WORK(&css->destroy_work, css_free_work_fn);
4702         queue_work(cgroup_destroy_wq, &css->destroy_work);
4703 }
4704 
4705 static void css_release_work_fn(struct work_struct *work)
4706 {
4707         struct cgroup_subsys_state *css =
4708                 container_of(work, struct cgroup_subsys_state, destroy_work);
4709         struct cgroup_subsys *ss = css->ss;
4710         struct cgroup *cgrp = css->cgroup;
4711 
4712         mutex_lock(&cgroup_mutex);
4713 
4714         css->flags |= CSS_RELEASED;
4715         list_del_rcu(&css->sibling);
4716 
4717         if (ss) {
4718                 /* css release path */
4719                 cgroup_idr_replace(&ss->css_idr, NULL, css->id);
4720                 if (ss->css_released)
4721                         ss->css_released(css);
4722         } else {
4723                 /* cgroup release path */
4724                 cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
4725                 cgrp->id = -1;
4726 
4727                 /*
4728                  * There are two control paths which try to determine
4729                  * cgroup from dentry without going through kernfs -
4730                  * cgroupstats_build() and css_tryget_online_from_dir().
4731                  * Those are supported by RCU protecting clearing of
4732                  * cgrp->kn->priv backpointer.
4733                  */
4734                 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, NULL);
4735         }
4736 
4737         mutex_unlock(&cgroup_mutex);
4738 
4739         call_rcu(&css->rcu_head, css_free_rcu_fn);
4740 }
4741 
4742 static void css_release(struct percpu_ref *ref)
4743 {
4744         struct cgroup_subsys_state *css =
4745                 container_of(ref, struct cgroup_subsys_state, refcnt);
4746 
4747         INIT_WORK(&css->destroy_work, css_release_work_fn);
4748         queue_work(cgroup_destroy_wq, &css->destroy_work);
4749 }
4750 
4751 static void init_and_link_css(struct cgroup_subsys_state *css,
4752                               struct cgroup_subsys *ss, struct cgroup *cgrp)
4753 {
4754         lockdep_assert_held(&cgroup_mutex);
4755 
4756         cgroup_get(cgrp);
4757 
4758         memset(css, 0, sizeof(*css));
4759         css->cgroup = cgrp;
4760         css->ss = ss;
4761         INIT_LIST_HEAD(&css->sibling);
4762         INIT_LIST_HEAD(&css->children);
4763         css->serial_nr = css_serial_nr_next++;
4764         atomic_set(&css->online_cnt, 0);
4765 
4766         if (cgroup_parent(cgrp)) {
4767                 css->parent = cgroup_css(cgroup_parent(cgrp), ss);
4768                 css_get(css->parent);
4769         }
4770 
4771         BUG_ON(cgroup_css(cgrp, ss));
4772 }
4773 
4774 /* invoke ->css_online() on a new CSS and mark it online if successful */
4775 static int online_css(struct cgroup_subsys_state *css)
4776 {
4777         struct cgroup_subsys *ss = css->ss;
4778         int ret = 0;
4779 
4780         lockdep_assert_held(&cgroup_mutex);
4781 
4782         if (ss->css_online)
4783                 ret = ss->css_online(css);
4784         if (!ret) {
4785                 css->flags |= CSS_ONLINE;
4786                 rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
4787 
4788                 atomic_inc(&css->online_cnt);
4789                 if (css->parent)
4790                         atomic_inc(&css->parent->online_cnt);
4791         }
4792         return ret;
4793 }
4794 
4795 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4796 static void offline_css(struct cgroup_subsys_state *css)
4797 {
4798         struct cgroup_subsys *ss = css->ss;
4799 
4800         lockdep_assert_held(&cgroup_mutex);
4801 
4802         if (!(css->flags & CSS_ONLINE))
4803                 return;
4804 
4805         if (ss->css_offline)
4806                 ss->css_offline(css);
4807 
4808         css->flags &= ~CSS_ONLINE;
4809         RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
4810 
4811         wake_up_all(&css->cgroup->offline_waitq);
4812 }
4813 
4814 /**
4815  * create_css - create a cgroup_subsys_state
4816  * @cgrp: the cgroup new css will be associated with
4817  * @ss: the subsys of new css
4818  * @visible: whether to create control knobs for the new css or not
4819  *
4820  * Create a new css associated with @cgrp - @ss pair.  On success, the new
4821  * css is online and installed in @cgrp with all interface files created if
4822  * @visible.  Returns 0 on success, -errno on failure.
4823  */
4824 static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
4825                       bool visible)
4826 {
4827         struct cgroup *parent = cgroup_parent(cgrp);
4828         struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
4829         struct cgroup_subsys_state *css;
4830         int err;
4831 
4832         lockdep_assert_held(&cgroup_mutex);
4833 
4834         css = ss->css_alloc(parent_css);
4835         if (IS_ERR(css))
4836                 return PTR_ERR(css);
4837 
4838         init_and_link_css(css, ss, cgrp);
4839 
4840         err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);
4841         if (err)
4842                 goto err_free_css;
4843 
4844         err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL);
4845         if (err < 0)
4846                 goto err_free_percpu_ref;
4847         css->id = err;
4848 
4849         if (visible) {
4850                 err = css_populate_dir(css, NULL);
4851                 if (err)
4852                         goto err_free_id;
4853         }
4854 
4855         /* @css is ready to be brought online now, make it visible */
4856         list_add_tail_rcu(&css->sibling, &parent_css->children);
4857         cgroup_idr_replace(&ss->css_idr, css, css->id);
4858 
4859         err = online_css(css);
4860         if (err)
4861                 goto err_list_del;
4862 
4863         if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
4864             cgroup_parent(parent)) {
4865                 pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4866                         current->comm, current->pid, ss->name);
4867                 if (!strcmp(ss->name, "memory"))
4868                         pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
4869                 ss->warned_broken_hierarchy = true;
4870         }
4871 
4872         return 0;
4873 
4874 err_list_del:
4875         list_del_rcu(&css->sibling);
4876         css_clear_dir(css, NULL);
4877 err_free_id:
4878         cgroup_idr_remove(&ss->css_idr, css->id);
4879 err_free_percpu_ref:
4880         percpu_ref_exit(&css->refcnt);
4881 err_free_css:
4882         call_rcu(&css->rcu_head, css_free_rcu_fn);
4883         return err;
4884 }
4885 
4886 static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
4887                         umode_t mode)
4888 {
4889         struct cgroup *parent, *cgrp, *tcgrp;
4890         struct cgroup_root *root;
4891         struct cgroup_subsys *ss;
4892         struct kernfs_node *kn;
4893         int level, ssid, ret;
4894 
4895         /* Do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable.
4896          */
4897         if (strchr(name, '\n'))
4898                 return -EINVAL;
4899 
4900         parent = cgroup_kn_lock_live(parent_kn);
4901         if (!parent)
4902                 return -ENODEV;
4903         root = parent->root;
4904         level = parent->level + 1;
4905 
4906         /* allocate the cgroup and its ID, 0 is reserved for the root */
4907         cgrp = kzalloc(sizeof(*cgrp) +
4908                        sizeof(cgrp->ancestor_ids[0]) * (level + 1), GFP_KERNEL);
4909         if (!cgrp) {
4910                 ret = -ENOMEM;
4911                 goto out_unlock;
4912         }
4913 
4914         ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);
4915         if (ret)
4916                 goto out_free_cgrp;
4917 
4918         /*
4919          * Temporarily set the pointer to NULL, so idr_find() won't return
4920          * a half-baked cgroup.
4921          */
4922         cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL);
4923         if (cgrp->id < 0) {
4924                 ret = -ENOMEM;
4925                 goto out_cancel_ref;
4926         }
4927 
4928         init_cgroup_housekeeping(cgrp);
4929 
4930         cgrp->self.parent = &parent->self;
4931         cgrp->root = root;
4932         cgrp->level = level;
4933 
4934         for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp))
4935                 cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
4936 
4937         if (notify_on_release(parent))
4938                 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
4939 
4940         if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
4941                 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4942 
4943         /* create the directory */
4944         kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
4945         if (IS_ERR(kn)) {
4946                 ret = PTR_ERR(kn);
4947                 goto out_free_id;
4948         }
4949         cgrp->kn = kn;
4950 
4951         /*
4952          * This extra ref will be put in cgroup_free_fn() and guarantees
4953          * that @cgrp->kn is always accessible.
4954          */
4955         kernfs_get(kn);
4956 
4957         cgrp->self.serial_nr = css_serial_nr_next++;
4958 
4959         /* allocation complete, commit to creation */
4960         list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
4961         atomic_inc(&root->nr_cgrps);
4962         cgroup_get(parent);
4963 
4964         /*
4965          * @cgrp is now fully operational.  If something fails after this
4966          * point, it'll be released via the normal destruction path.
4967          */
4968         cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4969 
4970         ret = cgroup_kn_set_ugid(kn);
4971         if (ret)
4972                 goto out_destroy;
4973 
4974         ret = css_populate_dir(&cgrp->self, NULL);
4975         if (ret)
4976                 goto out_destroy;
4977 
4978         /* let's create and online css's */
4979         for_each_subsys(ss, ssid) {
4980                 if (parent->child_subsys_mask & (1 << ssid)) {
4981                         ret = create_css(cgrp, ss,
4982                                          parent->subtree_control & (1 << ssid));
4983                         if (ret)
4984                                 goto out_destroy;
4985                 }
4986         }
4987 
4988         /*
4989          * On the default hierarchy, a child doesn't automatically inherit
4990          * subtree_control from the parent.  Each is configured manually.
4991          */
4992         if (!cgroup_on_dfl(cgrp)) {
4993                 cgrp->subtree_control = parent->subtree_control;
4994                 cgroup_refresh_child_subsys_mask(cgrp);
4995         }
4996 
4997         kernfs_activate(kn);
4998 
4999         ret = 0;
5000         goto out_unlock;
5001 
5002 out_free_id:
5003         cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
5004 out_cancel_ref:
5005         percpu_ref_exit(&cgrp->self.refcnt);
5006 out_free_cgrp:
5007         kfree(cgrp);
5008 out_unlock:
5009         cgroup_kn_unlock(parent_kn);
5010         return ret;
5011 
5012 out_destroy:
5013         cgroup_destroy_locked(cgrp);
5014         goto out_unlock;
5015 }
5016 
5017 /*
5018  * This is called when the refcnt of a css is confirmed to be killed.
5019  * css_tryget_online() is now guaranteed to fail.  Tell the subsystem to
5020  * initate destruction and put the css ref from kill_css().
5021  */
5022 static void css_killed_work_fn(struct work_struct *work)
5023 {
5024         struct cgroup_subsys_state *css =
5025                 container_of(work, struct cgroup_subsys_state, destroy_work);
5026 
5027         mutex_lock(&cgroup_mutex);
5028 
5029         do {
5030                 offline_css(css);
5031                 css_put(css);
5032                 /* @css can't go away while we're holding cgroup_mutex */
5033                 css = css->parent;
5034         } while (css && atomic_dec_and_test(&css->online_cnt));
5035 
5036         mutex_unlock(&cgroup_mutex);
5037 }
5038 
5039 /* css kill confirmation processing requires process context, bounce */
5040 static void css_killed_ref_fn(struct percpu_ref *ref)
5041 {
5042         struct cgroup_subsys_state *css =
5043                 container_of(ref, struct cgroup_subsys_state, refcnt);
5044 
5045         if (atomic_dec_and_test(&css->online_cnt)) {
5046                 INIT_WORK(&css->destroy_work, css_killed_work_fn);
5047                 queue_work(cgroup_destroy_wq, &css->destroy_work);
5048         }
5049 }
5050 
5051 /**
5052  * kill_css - destroy a css
5053  * @css: css to destroy
5054  *
5055  * This function initiates destruction of @css by removing cgroup interface
5056  * files and putting its base reference.  ->css_offline() will be invoked
5057  * asynchronously once css_tryget_online() is guaranteed to fail and when
5058  * the reference count reaches zero, @css will be released.
5059  */
5060 static void kill_css(struct cgroup_subsys_state *css)
5061 {
5062         lockdep_assert_held(&cgroup_mutex);
5063 
5064         /*
5065          * This must happen before css is disassociated with its cgroup.
5066          * See seq_css() for details.
5067          */
5068         css_clear_dir(css, NULL);
5069 
5070         /*
5071          * Killing would put the base ref, but we need to keep it alive
5072          * until after ->css_offline().
5073          */
5074         css_get(css);
5075 
5076         /*
5077          * cgroup core guarantees that, by the time ->css_offline() is
5078          * invoked, no new css reference will be given out via
5079          * css_tryget_online().  We can't simply call percpu_ref_kill() and
5080          * proceed to offlining css's because percpu_ref_kill() doesn't
5081          * guarantee that the ref is seen as killed on all CPUs on return.
5082          *
5083          * Use percpu_ref_kill_and_confirm() to get notifications as each
5084          * css is confirmed to be seen as killed on all CPUs.
5085          */
5086         percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
5087 }
5088 
5089 /**
5090  * cgroup_destroy_locked - the first stage of cgroup destruction
5091  * @cgrp: cgroup to be destroyed
5092  *
5093  * css's make use of percpu refcnts whose killing latency shouldn't be
5094  * exposed to userland and are RCU protected.  Also, cgroup core needs to
5095  * guarantee that css_tryget_online() won't succeed by the time
5096  * ->css_offline() is invoked.  To satisfy all the requirements,
5097  * destruction is implemented in the following two steps.
5098  *
5099  * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
5100  *     userland visible parts and start killing the percpu refcnts of
5101  *     css's.  Set up so that the next stage will be kicked off once all
5102  *     the percpu refcnts are confirmed to be killed.
5103  *
5104  * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
5105  *     rest of destruction.  Once all cgroup references are gone, the
5106  *     cgroup is RCU-freed.
5107  *
5108  * This function implements s1.  After this step, @cgrp is gone as far as
5109  * the userland is concerned and a new cgroup with the same name may be
5110  * created.  As cgroup doesn't care about the names internally, this
5111  * doesn't cause any problem.
5112  */
5113 static int cgroup_destroy_locked(struct cgroup *cgrp)
5114         __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
5115 {
5116         struct cgroup_subsys_state *css;
5117         int ssid;
5118 
5119         lockdep_assert_held(&cgroup_mutex);
5120 
5121         /*
5122          * Only migration can raise populated from zero and we're already
5123          * holding cgroup_mutex.
5124          */
5125         if (cgroup_is_populated(cgrp))
5126                 return -EBUSY;
5127 
5128         /*
5129          * Make sure there's no live children.  We can't test emptiness of
5130          * ->self.children as dead children linger on it while being
5131          * drained; otherwise, "rmdir parent/child parent" may fail.
5132          */
5133         if (css_has_online_children(&cgrp->self))
5134                 return -EBUSY;
5135 
5136         /*
5137          * Mark @cgrp dead.  This prevents further task migration and child
5138          * creation by disabling cgroup_lock_live_group().
5139          */
5140         cgrp->self.flags &= ~CSS_ONLINE;
5141 
5142         /* initiate massacre of all css's */
5143         for_each_css(css, ssid, cgrp)
5144                 kill_css(css);
5145 
5146         /*
5147          * Remove @cgrp directory along with the base files.  @cgrp has an
5148          * extra ref on its kn.
5149          */
5150         kernfs_remove(cgrp->kn);
5151 
5152         check_for_release(cgroup_parent(cgrp));
5153 
5154         /* put the base reference */
5155         percpu_ref_kill(&cgrp->self.refcnt);
5156 
5157         return 0;
5158 };
5159 
5160 static int cgroup_rmdir(struct kernfs_node *kn)
5161 {
5162         struct cgroup *cgrp;
5163         int ret = 0;
5164 
5165         cgrp = cgroup_kn_lock_live(kn);
5166         if (!cgrp)
5167                 return 0;
5168 
5169         ret = cgroup_destroy_locked(cgrp);
5170 
5171         cgroup_kn_unlock(kn);
5172         return ret;
5173 }
5174 
5175 static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
5176         .remount_fs             = cgroup_remount,
5177         .show_options           = cgroup_show_options,
5178         .mkdir                  = cgroup_mkdir,
5179         .rmdir                  = cgroup_rmdir,
5180         .rename                 = cgroup_rename,
5181 };
5182 
5183 static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
5184 {
5185         struct cgroup_subsys_state *css;
5186 
5187         pr_debug("Initializing cgroup subsys %s\n", ss->name);
5188 
5189         mutex_lock(&cgroup_mutex);
5190 
5191         idr_init(&ss->css_idr);
5192         INIT_LIST_HEAD(&ss->cfts);
5193 
5194         /* Create the root cgroup state for this subsystem */
5195         ss->root = &cgrp_dfl_root;
5196         css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
5197         /* We don't handle early failures gracefully */
5198         BUG_ON(IS_ERR(css));
5199         init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
5200 
5201         /*
5202          * Root csses are never destroyed and we can't initialize
5203          * percpu_ref during early init.  Disable refcnting.
5204          */
5205         css->flags |= CSS_NO_REF;
5206 
5207         if (early) {
5208                 /* allocation can't be done safely during early init */
5209                 css->id = 1;
5210         } else {
5211                 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
5212                 BUG_ON(css->id < 0);
5213         }
5214 
5215         /* Update the init_css_set to contain a subsys
5216          * pointer to this state - since the subsystem is
5217          * newly registered, all tasks and hence the
5218          * init_css_set is in the subsystem's root cgroup. */
5219         init_css_set.subsys[ss->id] = css;
5220 
5221         have_fork_callback |= (bool)ss->fork << ss->id;
5222         have_exit_callback |= (bool)ss->exit << ss->id;
5223         have_free_callback |= (bool)ss->free << ss->id;
5224         have_canfork_callback |= (bool)ss->can_fork << ss->id;
5225 
5226         /* At system boot, before all subsystems have been
5227          * registered, no tasks have been forked, so we don't
5228          * need to invoke fork callbacks here. */
5229         BUG_ON(!list_empty(&init_task.tasks));
5230 
5231         BUG_ON(online_css(css));
5232 
5233         mutex_unlock(&cgroup_mutex);
5234 }
5235 
5236 /**
5237  * cgroup_init_early - cgroup initialization at system boot
5238  *
5239  * Initialize cgroups at system boot, and initialize any
5240  * subsystems that request early init.
5241  */
5242 int __init cgroup_init_early(void)
5243 {
5244         static struct cgroup_sb_opts __initdata opts;
5245         struct cgroup_subsys *ss;
5246         int i;
5247 
5248         init_cgroup_root(&cgrp_dfl_root, &opts);
5249         cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;
5250 
5251         RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
5252 
5253         for_each_subsys(ss, i) {
5254                 WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
5255                      "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n",
5256                      i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
5257                      ss->id, ss->name);
5258                 WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
5259                      "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);
5260 
5261                 ss->id = i;
5262                 ss->name = cgroup_subsys_name[i];
5263                 if (!ss->legacy_name)
5264                         ss->legacy_name = cgroup_subsys_name[i];
5265 
5266                 if (ss->early_init)
5267                         cgroup_init_subsys(ss, true);
5268         }
5269         return 0;
5270 }
5271 
5272 static unsigned long cgroup_disable_mask __initdata;
5273 
5274 /**
5275  * cgroup_init - cgroup initialization
5276  *
5277  * Register cgroup filesystem and /proc file, and initialize
5278  * any subsystems that didn't request early init.
5279  */
5280 int __init cgroup_init(void)
5281 {
5282         struct cgroup_subsys *ss;
5283         unsigned long key;
5284         int ssid;
5285 
5286         BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem));
5287         BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files));
5288         BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files));
5289 
5290         mutex_lock(&cgroup_mutex);
5291 
5292         /* Add init_css_set to the hash table */
5293         key = css_set_hash(init_css_set.subsys);
5294         hash_add(css_set_table, &init_css_set.hlist, key);
5295 
5296         BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
5297 
5298         mutex_unlock(&cgroup_mutex);
5299 
5300         for_each_subsys(ss, ssid) {
5301                 if (ss->early_init) {
5302                         struct cgroup_subsys_state *css =
5303                                 init_css_set.subsys[ss->id];
5304 
5305                         css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
5306                                                    GFP_KERNEL);
5307                         BUG_ON(css->id < 0);
5308                 } else {
5309                         cgroup_init_subsys(ss, false);
5310                 }
5311 
5312                 list_add_tail(&init_css_set.e_cset_node[ssid],
5313                               &cgrp_dfl_root.cgrp.e_csets[ssid]);
5314 
5315                 /*
5316                  * Setting dfl_root subsys_mask needs to consider the
5317                  * disabled flag and cftype registration needs kmalloc,
5318                  * both of which aren't available during early_init.
5319                  */
5320                 if (cgroup_disable_mask & (1 << ssid)) {
5321                         static_branch_disable(cgroup_subsys_enabled_key[ssid]);
5322                         printk(KERN_INFO "Disabling %s control group subsystem\n",
5323                                ss->name);
5324                         continue;
5325                 }
5326 
5327                 cgrp_dfl_root.subsys_mask |= 1 << ss->id;
5328 
5329                 if (!ss->dfl_cftypes)
5330                         cgrp_dfl_root_inhibit_ss_mask |= 1 << ss->id;
5331 
5332                 if (ss->dfl_cftypes == ss->legacy_cftypes) {
5333                         WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
5334                 } else {
5335                         WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
5336                         WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
5337                 }
5338 
5339                 if (ss->bind)
5340                         ss->bind(init_css_set.subsys[ssid]);
5341         }
5342 
5343         WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup"));
5344         WARN_ON(register_filesystem(&cgroup_fs_type));
5345         WARN_ON(register_filesystem(&cgroup2_fs_type));
5346         WARN_ON(!proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations));
5347 
5348         return 0;
5349 }
5350 
5351 static int __init cgroup_wq_init(void)
5352 {
5353         /*
5354          * There isn't much point in executing destruction path in
5355          * parallel.  Good chunk is serialized with cgroup_mutex anyway.
5356          * Use 1 for @max_active.
5357          *
5358          * We would prefer to do this in cgroup_init() above, but that
5359          * is called before init_workqueues(): so leave this until after.
5360          */
5361         cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
5362         BUG_ON(!cgroup_destroy_wq);
5363 
5364         /*
5365          * Used to destroy pidlists and separate to serve as flush domain.
5366          * Cap @max_active to 1 too.
5367          */
5368         cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
5369                                                     0, 1);
5370         BUG_ON(!cgroup_pidlist_destroy_wq);
5371 
5372         return 0;
5373 }
5374 core_initcall(cgroup_wq_init);
5375 
5376 /*
5377  * proc_cgroup_show()
5378  *  - Print task's cgroup paths into seq_file, one line for each hierarchy
5379  *  - Used for /proc/<pid>/cgroup.
5380  */
5381 int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
5382                      struct pid *pid, struct task_struct *tsk)
5383 {
5384         char *buf, *path;
5385         int retval;
5386         struct cgroup_root *root;
5387 
5388         retval = -ENOMEM;
5389         buf = kmalloc(PATH_MAX, GFP_KERNEL);
5390         if (!buf)
5391                 goto out;
5392 
5393         mutex_lock(&cgroup_mutex);
5394         spin_lock_bh(&css_set_lock);
5395 
5396         for_each_root(root) {
5397                 struct cgroup_subsys *ss;
5398                 struct cgroup *cgrp;
5399                 int ssid, count = 0;
5400 
5401                 if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible)
5402                         continue;
5403 
5404                 seq_printf(m, "%d:", root->hierarchy_id);
5405                 if (root != &cgrp_dfl_root)
5406                         for_each_subsys(ss, ssid)
5407                                 if (root->subsys_mask & (1 << ssid))
5408                                         seq_printf(m, "%s%s", count++ ? "," : "",
5409                                                    ss->legacy_name);
5410                 if (strlen(root->name))
5411                         seq_printf(m, "%sname=%s", count ? "," : "",
5412                                    root->name);
5413                 seq_putc(m, ':');
5414 
5415                 cgrp = task_cgroup_from_root(tsk, root);
5416 
5417                 /*
5418                  * On traditional hierarchies, all zombie tasks show up as
5419                  * belonging to the root cgroup.  On the default hierarchy,
5420                  * while a zombie doesn't show up in "cgroup.procs" and
5421                  * thus can't be migrated, its /proc/PID/cgroup keeps
5422                  * reporting the cgroup it belonged to before exiting.  If
5423                  * the cgroup is removed before the zombie is reaped,
5424                  * " (deleted)" is appended to the cgroup path.
5425                  */
5426                 if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) {
5427                         path = cgroup_path(cgrp, buf, PATH_MAX);
5428                         if (!path) {
5429                                 retval = -ENAMETOOLONG;
5430                                 goto out_unlock;
5431                         }
5432                 } else {
5433                         path = "/";
5434                 }
5435 
5436                 seq_puts(m, path);
5437 
5438                 if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp))
5439                         seq_puts(m, " (deleted)\n");
5440                 else
5441                         seq_putc(m, '\n');
5442         }
5443 
5444         retval = 0;
5445 out_unlock:
5446         spin_unlock_bh(&css_set_lock);
5447         mutex_unlock(&cgroup_mutex);
5448         kfree(buf);
5449 out:
5450         return retval;
5451 }
5452 
5453 /* Display information about each subsystem and each hierarchy */
5454 static int proc_cgroupstats_show(struct seq_file *m, void *v)
5455 {
5456         struct cgroup_subsys *ss;
5457         int i;
5458 
5459         seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
5460         /*
5461          * ideally we don't want subsystems moving around while we do this.
5462          * cgroup_mutex is also necessary to guarantee an atomic snapshot of
5463          * subsys/hierarchy state.
5464          */
5465         mutex_lock(&cgroup_mutex);
5466 
5467         for_each_subsys(ss, i)
5468                 seq_printf(m, "%s\t%d\t%d\t%d\n",
5469                            ss->legacy_name, ss->root->hierarchy_id,
5470                            atomic_read(&ss->root->nr_cgrps),
5471                            cgroup_ssid_enabled(i));
5472 
5473         mutex_unlock(&cgroup_mutex);
5474         return 0;
5475 }
5476 
5477 static int cgroupstats_open(struct inode *inode, struct file *file)
5478 {
5479         return single_open(file, proc_cgroupstats_show, NULL);
5480 }
5481 
5482 static const struct file_operations proc_cgroupstats_operations = {
5483         .open = cgroupstats_open,
5484         .read = seq_read,
5485         .llseek = seq_lseek,
5486         .release = single_release,
5487 };
5488 
5489 /**
5490  * cgroup_fork - initialize cgroup related fields during copy_process()
5491  * @child: pointer to task_struct of forking parent process.
5492  *
5493  * A task is associated with the init_css_set until cgroup_post_fork()
5494  * attaches it to the parent's css_set.  Empty cg_list indicates that
5495  * @child isn't holding reference to its css_set.
5496  */
5497 void cgroup_fork(struct task_struct *child)
5498 {
5499         RCU_INIT_POINTER(child->cgroups, &init_css_set);
5500         INIT_LIST_HEAD(&child->cg_list);
5501 }
5502 
5503 /**
5504  * cgroup_can_fork - called on a new task before the process is exposed
5505  * @child: the task in question.
5506  *
5507  * This calls the subsystem can_fork() callbacks. If the can_fork() callback
5508  * returns an error, the fork aborts with that error code. This allows for
5509  * a cgroup subsystem to conditionally allow or deny new forks.
5510  */
5511 int cgroup_can_fork(struct task_struct *child)
5512 {
5513         struct cgroup_subsys *ss;
5514         int i, j, ret;
5515 
5516         for_each_subsys_which(ss, i, &have_canfork_callback) {
5517                 ret = ss->can_fork(child);
5518                 if (ret)
5519                         goto out_revert;
5520         }
5521 
5522         return 0;
5523 
5524 out_revert:
5525         for_each_subsys(ss, j) {
5526                 if (j >= i)
5527                         break;
5528                 if (ss->cancel_fork)
5529                         ss->cancel_fork(child);
5530         }
5531 
5532         return ret;
5533 }
5534 
5535 /**
5536  * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
5537  * @child: the task in question
5538  *
5539  * This calls the cancel_fork() callbacks if a fork failed *after*
5540  * cgroup_can_fork() succeded.
5541  */
5542 void cgroup_cancel_fork(struct task_struct *child)
5543 {
5544         struct cgroup_subsys *ss;
5545         int i;
5546 
5547         for_each_subsys(ss, i)
5548                 if (ss->cancel_fork)
5549                         ss->cancel_fork(child);
5550 }
5551 
5552 /**
5553  * cgroup_post_fork - called on a new task after adding it to the task list
5554  * @child: the task in question
5555  *
5556  * Adds the task to the list running through its css_set if necessary and
5557  * call the subsystem fork() callbacks.  Has to be after the task is
5558  * visible on the task list in case we race with the first call to
5559  * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5560  * list.
5561  */
5562 void cgroup_post_fork(struct task_struct *child)
5563 {
5564         struct cgroup_subsys *ss;
5565         int i;
5566 
5567         /*
5568          * This may race against cgroup_enable_task_cg_lists().  As that
5569          * function sets use_task_css_set_links before grabbing
5570          * tasklist_lock and we just went through tasklist_lock to add
5571          * @child, it's guaranteed that either we see the set
5572          * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
5573          * @child during its iteration.
5574          *
5575          * If we won the race, @child is associated with %current's
5576          * css_set.  Grabbing css_set_lock guarantees both that the
5577          * association is stable, and, on completion of the parent's
5578          * migration, @child is visible in the source of migration or
5579          * already in the destination cgroup.  This guarantee is necessary
5580          * when implementing operations which need to migrate all tasks of
5581          * a cgroup to another.
5582          *
5583          * Note that if we lose to cgroup_enable_task_cg_lists(), @child
5584          * will remain in init_css_set.  This is safe because all tasks are
5585          * in the init_css_set before cg_links is enabled and there's no
5586          * operation which transfers all tasks out of init_css_set.
5587          */
5588         if (use_task_css_set_links) {
5589                 struct css_set *cset;
5590 
5591                 spin_lock_bh(&css_set_lock);
5592                 cset = task_css_set(current);
5593                 if (list_empty(&child->cg_list)) {
5594                         get_css_set(cset);
5595                         css_set_move_task(child, NULL, cset, false);
5596                 }
5597                 spin_unlock_bh(&css_set_lock);
5598         }
5599 
5600         /*
5601          * Call ss->fork().  This must happen after @child is linked on
5602          * css_set; otherwise, @child might change state between ->fork()
5603          * and addition to css_set.
5604          */
5605         for_each_subsys_which(ss, i, &have_fork_callback)
5606                 ss->fork(child);
5607 }
5608 
5609 /**
5610  * cgroup_exit - detach cgroup from exiting task
5611  * @tsk: pointer to task_struct of exiting process
5612  *
5613  * Description: Detach cgroup from @tsk and release it.
5614  *
5615  * Note that cgroups marked notify_on_release force every task in
5616  * them to take the global cgroup_mutex mutex when exiting.
5617  * This could impact scaling on very large systems.  Be reluctant to
5618  * use notify_on_release cgroups where very high task exit scaling
5619  * is required on large systems.
5620  *
5621  * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
5622  * call cgroup_exit() while the task is still competent to handle
5623  * notify_on_release(), then leave the task attached to the root cgroup in
5624  * each hierarchy for the remainder of its exit.  No need to bother with
5625  * init_css_set refcnting.  init_css_set never goes away and we can't race
5626  * with migration path - PF_EXITING is visible to migration path.
5627  */
5628 void cgroup_exit(struct task_struct *tsk)
5629 {
5630         struct cgroup_subsys *ss;
5631         struct css_set *cset;
5632         int i;
5633 
5634         /*
5635          * Unlink from @tsk from its css_set.  As migration path can't race
5636          * with us, we can check css_set and cg_list without synchronization.
5637          */
5638         cset = task_css_set(tsk);
5639 
5640         if (!list_empty(&tsk->cg_list)) {
5641                 spin_lock_bh(&css_set_lock);
5642                 css_set_move_task(tsk, cset, NULL, false);
5643                 spin_unlock_bh(&css_set_lock);
5644         } else {
5645                 get_css_set(cset);
5646         }
5647 
5648         /* see cgroup_post_fork() for details */
5649         for_each_subsys_which(ss, i, &have_exit_callback)
5650                 ss->exit(tsk);
5651 }
5652 
5653 void cgroup_free(struct task_struct *task)
5654 {
5655         struct css_set *cset = task_css_set(task);
5656         struct cgroup_subsys *ss;
5657         int ssid;
5658 
5659         for_each_subsys_which(ss, ssid, &have_free_callback)
5660                 ss->free(task);
5661 
5662         put_css_set(cset);
5663 }
5664 
5665 static void check_for_release(struct cgroup *cgrp)
5666 {
5667         if (notify_on_release(cgrp) && !cgroup_is_populated(cgrp) &&
5668             !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp))
5669                 schedule_work(&cgrp->release_agent_work);
5670 }
5671 
5672 /*
5673  * Notify userspace when a cgroup is released, by running the
5674  * configured release agent with the name of the cgroup (path
5675  * relative to the root of cgroup file system) as the argument.
5676  *
5677  * Most likely, this user command will try to rmdir this cgroup.
5678  *
5679  * This races with the possibility that some other task will be
5680  * attached to this cgroup before it is removed, or that some other
5681  * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
5682  * The presumed 'rmdir' will fail quietly if this cgroup is no longer
5683  * unused, and this cgroup will be reprieved from its death sentence,
5684  * to continue to serve a useful existence.  Next time it's released,
5685  * we will get notified again, if it still has 'notify_on_release' set.
5686  *
5687  * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
5688  * means only wait until the task is successfully execve()'d.  The
5689  * separate release agent task is forked by call_usermodehelper(),
5690  * then control in this thread returns here, without waiting for the
5691  * release agent task.  We don't bother to wait because the caller of
5692  * this routine has no use for the exit status of the release agent
5693  * task, so no sense holding our caller up for that.
5694  */
5695 static void cgroup_release_agent(struct work_struct *work)
5696 {
5697         struct cgroup *cgrp =
5698                 container_of(work, struct cgroup, release_agent_work);
5699         char *pathbuf = NULL, *agentbuf = NULL, *path;
5700         char *argv[3], *envp[3];
5701 
5702         mutex_lock(&cgroup_mutex);
5703 
5704         pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
5705         agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
5706         if (!pathbuf || !agentbuf)
5707                 goto out;
5708 
5709         path = cgroup_path(cgrp, pathbuf, PATH_MAX);
5710         if (!path)
5711                 goto out;
5712 
5713         argv[0] = agentbuf;
5714         argv[1] = path;
5715         argv[2] = NULL;
5716 
5717         /* minimal command environment */
5718         envp[0] = "HOME=/";
5719         envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
5720         envp[2] = NULL;
5721 
5722         mutex_unlock(&cgroup_mutex);
5723         call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
5724         goto out_free;
5725 out:
5726         mutex_unlock(&cgroup_mutex);
5727 out_free:
5728         kfree(agentbuf);
5729         kfree(pathbuf);
5730 }
5731 
5732 static int __init cgroup_disable(char *str)
5733 {
5734         struct cgroup_subsys *ss;
5735         char *token;
5736         int i;
5737 
5738         while ((token = strsep(&str, ",")) != NULL) {
5739                 if (!*token)
5740                         continue;
5741 
5742                 for_each_subsys(ss, i) {
5743                         if (strcmp(token, ss->name) &&
5744                             strcmp(token, ss->legacy_name))
5745                                 continue;
5746                         cgroup_disable_mask |= 1 << i;
5747                 }
5748         }
5749         return 1;
5750 }
5751 __setup("cgroup_disable=", cgroup_disable);
5752 
5753 /**
5754  * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
5755  * @dentry: directory dentry of interest
5756  * @ss: subsystem of interest
5757  *
5758  * If @dentry is a directory for a cgroup which has @ss enabled on it, try
5759  * to get the corresponding css and return it.  If such css doesn't exist
5760  * or can't be pinned, an ERR_PTR value is returned.
5761  */
5762 struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
5763                                                        struct cgroup_subsys *ss)
5764 {
5765         struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
5766         struct cgroup_subsys_state *css = NULL;
5767         struct cgroup *cgrp;
5768 
5769         /* is @dentry a cgroup dir? */
5770         if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
5771             kernfs_type(kn) != KERNFS_DIR)
5772                 return ERR_PTR(-EBADF);
5773 
5774         rcu_read_lock();
5775 
5776         /*
5777          * This path doesn't originate from kernfs and @kn could already
5778          * have been or be removed at any point.  @kn->priv is RCU
5779          * protected for this access.  See css_release_work_fn() for details.
5780          */
5781         cgrp = rcu_dereference(kn->priv);
5782         if (cgrp)
5783                 css = cgroup_css(cgrp, ss);
5784 
5785         if (!css || !css_tryget_online(css))
5786                 css = ERR_PTR(-ENOENT);
5787 
5788         rcu_read_unlock();
5789         return css;
5790 }
5791 
5792 /**
5793  * css_from_id - lookup css by id
5794  * @id: the cgroup id
5795  * @ss: cgroup subsys to be looked into
5796  *
5797  * Returns the css if there's valid one with @id, otherwise returns NULL.
5798  * Should be called under rcu_read_lock().
5799  */
5800 struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
5801 {
5802         WARN_ON_ONCE(!rcu_read_lock_held());
5803         return id > 0 ? idr_find(&ss->css_idr, id) : NULL;
5804 }
5805 
5806 /**
5807  * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
5808  * @path: path on the default hierarchy
5809  *
5810  * Find the cgroup at @path on the default hierarchy, increment its
5811  * reference count and return it.  Returns pointer to the found cgroup on
5812  * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR)
5813  * if @path points to a non-directory.
5814  */
5815 struct cgroup *cgroup_get_from_path(const char *path)
5816 {
5817         struct kernfs_node *kn;
5818         struct cgroup *cgrp;
5819 
5820         mutex_lock(&cgroup_mutex);
5821 
5822         kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path);
5823         if (kn) {
5824                 if (kernfs_type(kn) == KERNFS_DIR) {
5825                         cgrp = kn->priv;
5826                         cgroup_get(cgrp);
5827                 } else {
5828                         cgrp = ERR_PTR(-ENOTDIR);
5829                 }
5830                 kernfs_put(kn);
5831         } else {
5832                 cgrp = ERR_PTR(-ENOENT);
5833         }
5834 
5835         mutex_unlock(&cgroup_mutex);
5836         return cgrp;
5837 }
5838 EXPORT_SYMBOL_GPL(cgroup_get_from_path);
5839 
5840 /*
5841  * sock->sk_cgrp_data handling.  For more info, see sock_cgroup_data
5842  * definition in cgroup-defs.h.
5843  */
5844 #ifdef CONFIG_SOCK_CGROUP_DATA
5845 
5846 #if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID)
5847 
5848 DEFINE_SPINLOCK(cgroup_sk_update_lock);
5849 static bool cgroup_sk_alloc_disabled __read_mostly;
5850 
5851 void cgroup_sk_alloc_disable(void)
5852 {
5853         if (cgroup_sk_alloc_disabled)
5854                 return;
5855         pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n");
5856         cgroup_sk_alloc_disabled = true;
5857 }
5858 
5859 #else
5860 
5861 #define cgroup_sk_alloc_disabled        false
5862 
5863 #endif
5864 
5865 void cgroup_sk_alloc(struct sock_cgroup_data *skcd)
5866 {
5867         if (cgroup_sk_alloc_disabled)
5868                 return;
5869 
5870         rcu_read_lock();
5871 
5872         while (true) {
5873                 struct css_set *cset;
5874 
5875                 cset = task_css_set(current);
5876                 if (likely(cgroup_tryget(cset->dfl_cgrp))) {
5877                         skcd->val = (unsigned long)cset->dfl_cgrp;
5878                         break;
5879                 }
5880                 cpu_relax();
5881         }
5882 
5883         rcu_read_unlock();
5884 }
5885 
5886 void cgroup_sk_free(struct sock_cgroup_data *skcd)
5887 {
5888         cgroup_put(sock_cgroup_ptr(skcd));
5889 }
5890 
5891 #endif  /* CONFIG_SOCK_CGROUP_DATA */
5892 
5893 #ifdef CONFIG_CGROUP_DEBUG
5894 static struct cgroup_subsys_state *
5895 debug_css_alloc(struct cgroup_subsys_state *parent_css)
5896 {
5897         struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
5898 
5899         if (!css)
5900                 return ERR_PTR(-ENOMEM);
5901 
5902         return css;
5903 }
5904 
5905 static void debug_css_free(struct cgroup_subsys_state *css)
5906 {
5907         kfree(css);
5908 }
5909 
5910 static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
5911                                 struct cftype *cft)
5912 {
5913         return cgroup_task_count(css->cgroup);
5914 }
5915 
5916 static u64 current_css_set_read(struct cgroup_subsys_state *css,
5917                                 struct cftype *cft)
5918 {
5919         return (u64)(unsigned long)current->cgroups;
5920 }
5921 
5922 static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
5923                                          struct cftype *cft)
5924 {
5925         u64 count;
5926 
5927         rcu_read_lock();
5928         count = atomic_read(&task_css_set(current)->refcount);
5929         rcu_read_unlock();
5930         return count;
5931 }
5932 
5933 static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
5934 {
5935         struct cgrp_cset_link *link;
5936         struct css_set *cset;
5937         char *name_buf;
5938 
5939         name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
5940         if (!name_buf)
5941                 return -ENOMEM;
5942 
5943         spin_lock_bh(&css_set_lock);
5944         rcu_read_lock();
5945         cset = rcu_dereference(current->cgroups);
5946         list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
5947                 struct cgroup *c = link->cgrp;
5948 
5949                 cgroup_name(c, name_buf, NAME_MAX + 1);
5950                 seq_printf(seq, "Root %d group %s\n",
5951                            c->root->hierarchy_id, name_buf);
5952         }
5953         rcu_read_unlock();
5954         spin_unlock_bh(&css_set_lock);
5955         kfree(name_buf);
5956         return 0;
5957 }
5958 
5959 #define MAX_TASKS_SHOWN_PER_CSS 25
5960 static int cgroup_css_links_read(struct seq_file *seq, void *v)
5961 {
5962         struct cgroup_subsys_state *css = seq_css(seq);
5963         struct cgrp_cset_link *link;
5964 
5965         spin_lock_bh(&css_set_lock);
5966         list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
5967                 struct css_set *cset = link->cset;
5968                 struct task_struct *task;
5969                 int count = 0;
5970 
5971                 seq_printf(seq, "css_set %p\n", cset);
5972 
5973                 list_for_each_entry(task, &cset->tasks, cg_list) {
5974                         if (count++ > MAX_TASKS_SHOWN_PER_CSS)
5975                                 goto overflow;
5976                         seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5977                 }
5978 
5979                 list_for_each_entry(task, &cset->mg_tasks, cg_list) {
5980                         if (count++ > MAX_TASKS_SHOWN_PER_CSS)
5981                                 goto overflow;
5982                         seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5983                 }
5984                 continue;
5985         overflow:
5986                 seq_puts(seq, "  ...\n");
5987         }
5988         spin_unlock_bh(&css_set_lock);
5989         return 0;
5990 }
5991 
5992 static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
5993 {
5994         return (!cgroup_is_populated(css->cgroup) &&
5995                 !css_has_online_children(&css->cgroup->self));
5996 }
5997 
5998 static struct cftype debug_files[] =  {
5999         {
6000                 .name = "taskcount",
6001                 .read_u64 = debug_taskcount_read,
6002         },
6003 
6004         {
6005                 .name = "current_css_set",
6006                 .read_u64 = current_css_set_read,
6007         },
6008 
6009         {
6010                 .name = "current_css_set_refcount",
6011                 .read_u64 = current_css_set_refcount_read,
6012         },
6013 
6014         {
6015                 .name = "current_css_set_cg_links",
6016                 .seq_show = current_css_set_cg_links_read,
6017         },
6018 
6019         {
6020                 .name = "cgroup_css_links",
6021                 .seq_show = cgroup_css_links_read,
6022         },
6023 
6024         {
6025                 .name = "releasable",
6026                 .read_u64 = releasable_read,
6027         },
6028 
6029         { }     /* terminate */
6030 };
6031 
6032 struct cgroup_subsys debug_cgrp_subsys = {
6033         .css_alloc = debug_css_alloc,
6034         .css_free = debug_css_free,
6035         .legacy_cftypes = debug_files,
6036 };
6037 #endif /* CONFIG_CGROUP_DEBUG */
6038 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us