Version:  2.0.40 2.2.26 2.4.37 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16

Linux/kernel/cpuset.c

  1 /*
  2  *  kernel/cpuset.c
  3  *
  4  *  Processor and Memory placement constraints for sets of tasks.
  5  *
  6  *  Copyright (C) 2003 BULL SA.
  7  *  Copyright (C) 2004-2007 Silicon Graphics, Inc.
  8  *  Copyright (C) 2006 Google, Inc
  9  *
 10  *  Portions derived from Patrick Mochel's sysfs code.
 11  *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 12  *
 13  *  2003-10-10 Written by Simon Derr.
 14  *  2003-10-22 Updates by Stephen Hemminger.
 15  *  2004 May-July Rework by Paul Jackson.
 16  *  2006 Rework by Paul Menage to use generic cgroups
 17  *  2008 Rework of the scheduler domains and CPU hotplug handling
 18  *       by Max Krasnyansky
 19  *
 20  *  This file is subject to the terms and conditions of the GNU General Public
 21  *  License.  See the file COPYING in the main directory of the Linux
 22  *  distribution for more details.
 23  */
 24 
 25 #include <linux/cpu.h>
 26 #include <linux/cpumask.h>
 27 #include <linux/cpuset.h>
 28 #include <linux/err.h>
 29 #include <linux/errno.h>
 30 #include <linux/file.h>
 31 #include <linux/fs.h>
 32 #include <linux/init.h>
 33 #include <linux/interrupt.h>
 34 #include <linux/kernel.h>
 35 #include <linux/kmod.h>
 36 #include <linux/list.h>
 37 #include <linux/mempolicy.h>
 38 #include <linux/mm.h>
 39 #include <linux/memory.h>
 40 #include <linux/export.h>
 41 #include <linux/mount.h>
 42 #include <linux/namei.h>
 43 #include <linux/pagemap.h>
 44 #include <linux/proc_fs.h>
 45 #include <linux/rcupdate.h>
 46 #include <linux/sched.h>
 47 #include <linux/seq_file.h>
 48 #include <linux/security.h>
 49 #include <linux/slab.h>
 50 #include <linux/spinlock.h>
 51 #include <linux/stat.h>
 52 #include <linux/string.h>
 53 #include <linux/time.h>
 54 #include <linux/backing-dev.h>
 55 #include <linux/sort.h>
 56 
 57 #include <asm/uaccess.h>
 58 #include <linux/atomic.h>
 59 #include <linux/mutex.h>
 60 #include <linux/workqueue.h>
 61 #include <linux/cgroup.h>
 62 #include <linux/wait.h>
 63 
 64 struct static_key cpusets_enabled_key __read_mostly = STATIC_KEY_INIT_FALSE;
 65 
 66 /* See "Frequency meter" comments, below. */
 67 
 68 struct fmeter {
 69         int cnt;                /* unprocessed events count */
 70         int val;                /* most recent output value */
 71         time_t time;            /* clock (secs) when val computed */
 72         spinlock_t lock;        /* guards read or write of above */
 73 };
 74 
 75 struct cpuset {
 76         struct cgroup_subsys_state css;
 77 
 78         unsigned long flags;            /* "unsigned long" so bitops work */
 79         cpumask_var_t cpus_allowed;     /* CPUs allowed to tasks in cpuset */
 80         nodemask_t mems_allowed;        /* Memory Nodes allowed to tasks */
 81 
 82         /*
 83          * This is old Memory Nodes tasks took on.
 84          *
 85          * - top_cpuset.old_mems_allowed is initialized to mems_allowed.
 86          * - A new cpuset's old_mems_allowed is initialized when some
 87          *   task is moved into it.
 88          * - old_mems_allowed is used in cpuset_migrate_mm() when we change
 89          *   cpuset.mems_allowed and have tasks' nodemask updated, and
 90          *   then old_mems_allowed is updated to mems_allowed.
 91          */
 92         nodemask_t old_mems_allowed;
 93 
 94         struct fmeter fmeter;           /* memory_pressure filter */
 95 
 96         /*
 97          * Tasks are being attached to this cpuset.  Used to prevent
 98          * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
 99          */
100         int attach_in_progress;
101 
102         /* partition number for rebuild_sched_domains() */
103         int pn;
104 
105         /* for custom sched domain */
106         int relax_domain_level;
107 };
108 
109 static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
110 {
111         return css ? container_of(css, struct cpuset, css) : NULL;
112 }
113 
114 /* Retrieve the cpuset for a task */
115 static inline struct cpuset *task_cs(struct task_struct *task)
116 {
117         return css_cs(task_css(task, cpuset_cgrp_id));
118 }
119 
120 static inline struct cpuset *parent_cs(struct cpuset *cs)
121 {
122         return css_cs(cs->css.parent);
123 }
124 
125 #ifdef CONFIG_NUMA
126 static inline bool task_has_mempolicy(struct task_struct *task)
127 {
128         return task->mempolicy;
129 }
130 #else
131 static inline bool task_has_mempolicy(struct task_struct *task)
132 {
133         return false;
134 }
135 #endif
136 
137 
138 /* bits in struct cpuset flags field */
139 typedef enum {
140         CS_ONLINE,
141         CS_CPU_EXCLUSIVE,
142         CS_MEM_EXCLUSIVE,
143         CS_MEM_HARDWALL,
144         CS_MEMORY_MIGRATE,
145         CS_SCHED_LOAD_BALANCE,
146         CS_SPREAD_PAGE,
147         CS_SPREAD_SLAB,
148 } cpuset_flagbits_t;
149 
150 /* convenient tests for these bits */
151 static inline bool is_cpuset_online(const struct cpuset *cs)
152 {
153         return test_bit(CS_ONLINE, &cs->flags);
154 }
155 
156 static inline int is_cpu_exclusive(const struct cpuset *cs)
157 {
158         return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
159 }
160 
161 static inline int is_mem_exclusive(const struct cpuset *cs)
162 {
163         return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
164 }
165 
166 static inline int is_mem_hardwall(const struct cpuset *cs)
167 {
168         return test_bit(CS_MEM_HARDWALL, &cs->flags);
169 }
170 
171 static inline int is_sched_load_balance(const struct cpuset *cs)
172 {
173         return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
174 }
175 
176 static inline int is_memory_migrate(const struct cpuset *cs)
177 {
178         return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
179 }
180 
181 static inline int is_spread_page(const struct cpuset *cs)
182 {
183         return test_bit(CS_SPREAD_PAGE, &cs->flags);
184 }
185 
186 static inline int is_spread_slab(const struct cpuset *cs)
187 {
188         return test_bit(CS_SPREAD_SLAB, &cs->flags);
189 }
190 
191 static struct cpuset top_cpuset = {
192         .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |
193                   (1 << CS_MEM_EXCLUSIVE)),
194 };
195 
196 /**
197  * cpuset_for_each_child - traverse online children of a cpuset
198  * @child_cs: loop cursor pointing to the current child
199  * @pos_css: used for iteration
200  * @parent_cs: target cpuset to walk children of
201  *
202  * Walk @child_cs through the online children of @parent_cs.  Must be used
203  * with RCU read locked.
204  */
205 #define cpuset_for_each_child(child_cs, pos_css, parent_cs)             \
206         css_for_each_child((pos_css), &(parent_cs)->css)                \
207                 if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
208 
209 /**
210  * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
211  * @des_cs: loop cursor pointing to the current descendant
212  * @pos_css: used for iteration
213  * @root_cs: target cpuset to walk ancestor of
214  *
215  * Walk @des_cs through the online descendants of @root_cs.  Must be used
216  * with RCU read locked.  The caller may modify @pos_css by calling
217  * css_rightmost_descendant() to skip subtree.  @root_cs is included in the
218  * iteration and the first node to be visited.
219  */
220 #define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs)        \
221         css_for_each_descendant_pre((pos_css), &(root_cs)->css)         \
222                 if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
223 
224 /*
225  * There are two global mutexes guarding cpuset structures - cpuset_mutex
226  * and callback_mutex.  The latter may nest inside the former.  We also
227  * require taking task_lock() when dereferencing a task's cpuset pointer.
228  * See "The task_lock() exception", at the end of this comment.
229  *
230  * A task must hold both mutexes to modify cpusets.  If a task holds
231  * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it
232  * is the only task able to also acquire callback_mutex and be able to
233  * modify cpusets.  It can perform various checks on the cpuset structure
234  * first, knowing nothing will change.  It can also allocate memory while
235  * just holding cpuset_mutex.  While it is performing these checks, various
236  * callback routines can briefly acquire callback_mutex to query cpusets.
237  * Once it is ready to make the changes, it takes callback_mutex, blocking
238  * everyone else.
239  *
240  * Calls to the kernel memory allocator can not be made while holding
241  * callback_mutex, as that would risk double tripping on callback_mutex
242  * from one of the callbacks into the cpuset code from within
243  * __alloc_pages().
244  *
245  * If a task is only holding callback_mutex, then it has read-only
246  * access to cpusets.
247  *
248  * Now, the task_struct fields mems_allowed and mempolicy may be changed
249  * by other task, we use alloc_lock in the task_struct fields to protect
250  * them.
251  *
252  * The cpuset_common_file_read() handlers only hold callback_mutex across
253  * small pieces of code, such as when reading out possibly multi-word
254  * cpumasks and nodemasks.
255  *
256  * Accessing a task's cpuset should be done in accordance with the
257  * guidelines for accessing subsystem state in kernel/cgroup.c
258  */
259 
260 static DEFINE_MUTEX(cpuset_mutex);
261 static DEFINE_MUTEX(callback_mutex);
262 
263 /*
264  * CPU / memory hotplug is handled asynchronously.
265  */
266 static void cpuset_hotplug_workfn(struct work_struct *work);
267 static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn);
268 
269 static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);
270 
271 /*
272  * This is ugly, but preserves the userspace API for existing cpuset
273  * users. If someone tries to mount the "cpuset" filesystem, we
274  * silently switch it to mount "cgroup" instead
275  */
276 static struct dentry *cpuset_mount(struct file_system_type *fs_type,
277                          int flags, const char *unused_dev_name, void *data)
278 {
279         struct file_system_type *cgroup_fs = get_fs_type("cgroup");
280         struct dentry *ret = ERR_PTR(-ENODEV);
281         if (cgroup_fs) {
282                 char mountopts[] =
283                         "cpuset,noprefix,"
284                         "release_agent=/sbin/cpuset_release_agent";
285                 ret = cgroup_fs->mount(cgroup_fs, flags,
286                                            unused_dev_name, mountopts);
287                 put_filesystem(cgroup_fs);
288         }
289         return ret;
290 }
291 
292 static struct file_system_type cpuset_fs_type = {
293         .name = "cpuset",
294         .mount = cpuset_mount,
295 };
296 
297 /*
298  * Return in pmask the portion of a cpusets's cpus_allowed that
299  * are online.  If none are online, walk up the cpuset hierarchy
300  * until we find one that does have some online cpus.  The top
301  * cpuset always has some cpus online.
302  *
303  * One way or another, we guarantee to return some non-empty subset
304  * of cpu_online_mask.
305  *
306  * Call with callback_mutex held.
307  */
308 static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask)
309 {
310         while (!cpumask_intersects(cs->cpus_allowed, cpu_online_mask))
311                 cs = parent_cs(cs);
312         cpumask_and(pmask, cs->cpus_allowed, cpu_online_mask);
313 }
314 
315 /*
316  * Return in *pmask the portion of a cpusets's mems_allowed that
317  * are online, with memory.  If none are online with memory, walk
318  * up the cpuset hierarchy until we find one that does have some
319  * online mems.  The top cpuset always has some mems online.
320  *
321  * One way or another, we guarantee to return some non-empty subset
322  * of node_states[N_MEMORY].
323  *
324  * Call with callback_mutex held.
325  */
326 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
327 {
328         while (!nodes_intersects(cs->mems_allowed, node_states[N_MEMORY]))
329                 cs = parent_cs(cs);
330         nodes_and(*pmask, cs->mems_allowed, node_states[N_MEMORY]);
331 }
332 
333 /*
334  * update task's spread flag if cpuset's page/slab spread flag is set
335  *
336  * Called with callback_mutex/cpuset_mutex held
337  */
338 static void cpuset_update_task_spread_flag(struct cpuset *cs,
339                                         struct task_struct *tsk)
340 {
341         if (is_spread_page(cs))
342                 tsk->flags |= PF_SPREAD_PAGE;
343         else
344                 tsk->flags &= ~PF_SPREAD_PAGE;
345         if (is_spread_slab(cs))
346                 tsk->flags |= PF_SPREAD_SLAB;
347         else
348                 tsk->flags &= ~PF_SPREAD_SLAB;
349 }
350 
351 /*
352  * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
353  *
354  * One cpuset is a subset of another if all its allowed CPUs and
355  * Memory Nodes are a subset of the other, and its exclusive flags
356  * are only set if the other's are set.  Call holding cpuset_mutex.
357  */
358 
359 static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
360 {
361         return  cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&
362                 nodes_subset(p->mems_allowed, q->mems_allowed) &&
363                 is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
364                 is_mem_exclusive(p) <= is_mem_exclusive(q);
365 }
366 
367 /**
368  * alloc_trial_cpuset - allocate a trial cpuset
369  * @cs: the cpuset that the trial cpuset duplicates
370  */
371 static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
372 {
373         struct cpuset *trial;
374 
375         trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
376         if (!trial)
377                 return NULL;
378 
379         if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) {
380                 kfree(trial);
381                 return NULL;
382         }
383         cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
384 
385         return trial;
386 }
387 
388 /**
389  * free_trial_cpuset - free the trial cpuset
390  * @trial: the trial cpuset to be freed
391  */
392 static void free_trial_cpuset(struct cpuset *trial)
393 {
394         free_cpumask_var(trial->cpus_allowed);
395         kfree(trial);
396 }
397 
398 /*
399  * validate_change() - Used to validate that any proposed cpuset change
400  *                     follows the structural rules for cpusets.
401  *
402  * If we replaced the flag and mask values of the current cpuset
403  * (cur) with those values in the trial cpuset (trial), would
404  * our various subset and exclusive rules still be valid?  Presumes
405  * cpuset_mutex held.
406  *
407  * 'cur' is the address of an actual, in-use cpuset.  Operations
408  * such as list traversal that depend on the actual address of the
409  * cpuset in the list must use cur below, not trial.
410  *
411  * 'trial' is the address of bulk structure copy of cur, with
412  * perhaps one or more of the fields cpus_allowed, mems_allowed,
413  * or flags changed to new, trial values.
414  *
415  * Return 0 if valid, -errno if not.
416  */
417 
418 static int validate_change(struct cpuset *cur, struct cpuset *trial)
419 {
420         struct cgroup_subsys_state *css;
421         struct cpuset *c, *par;
422         int ret;
423 
424         rcu_read_lock();
425 
426         /* Each of our child cpusets must be a subset of us */
427         ret = -EBUSY;
428         cpuset_for_each_child(c, css, cur)
429                 if (!is_cpuset_subset(c, trial))
430                         goto out;
431 
432         /* Remaining checks don't apply to root cpuset */
433         ret = 0;
434         if (cur == &top_cpuset)
435                 goto out;
436 
437         par = parent_cs(cur);
438 
439         /* We must be a subset of our parent cpuset */
440         ret = -EACCES;
441         if (!is_cpuset_subset(trial, par))
442                 goto out;
443 
444         /*
445          * If either I or some sibling (!= me) is exclusive, we can't
446          * overlap
447          */
448         ret = -EINVAL;
449         cpuset_for_each_child(c, css, par) {
450                 if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
451                     c != cur &&
452                     cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
453                         goto out;
454                 if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
455                     c != cur &&
456                     nodes_intersects(trial->mems_allowed, c->mems_allowed))
457                         goto out;
458         }
459 
460         /*
461          * Cpusets with tasks - existing or newly being attached - can't
462          * be changed to have empty cpus_allowed or mems_allowed.
463          */
464         ret = -ENOSPC;
465         if ((cgroup_has_tasks(cur->css.cgroup) || cur->attach_in_progress)) {
466                 if (!cpumask_empty(cur->cpus_allowed) &&
467                     cpumask_empty(trial->cpus_allowed))
468                         goto out;
469                 if (!nodes_empty(cur->mems_allowed) &&
470                     nodes_empty(trial->mems_allowed))
471                         goto out;
472         }
473 
474         ret = 0;
475 out:
476         rcu_read_unlock();
477         return ret;
478 }
479 
480 #ifdef CONFIG_SMP
481 /*
482  * Helper routine for generate_sched_domains().
483  * Do cpusets a, b have overlapping cpus_allowed masks?
484  */
485 static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
486 {
487         return cpumask_intersects(a->cpus_allowed, b->cpus_allowed);
488 }
489 
490 static void
491 update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
492 {
493         if (dattr->relax_domain_level < c->relax_domain_level)
494                 dattr->relax_domain_level = c->relax_domain_level;
495         return;
496 }
497 
498 static void update_domain_attr_tree(struct sched_domain_attr *dattr,
499                                     struct cpuset *root_cs)
500 {
501         struct cpuset *cp;
502         struct cgroup_subsys_state *pos_css;
503 
504         rcu_read_lock();
505         cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
506                 if (cp == root_cs)
507                         continue;
508 
509                 /* skip the whole subtree if @cp doesn't have any CPU */
510                 if (cpumask_empty(cp->cpus_allowed)) {
511                         pos_css = css_rightmost_descendant(pos_css);
512                         continue;
513                 }
514 
515                 if (is_sched_load_balance(cp))
516                         update_domain_attr(dattr, cp);
517         }
518         rcu_read_unlock();
519 }
520 
521 /*
522  * generate_sched_domains()
523  *
524  * This function builds a partial partition of the systems CPUs
525  * A 'partial partition' is a set of non-overlapping subsets whose
526  * union is a subset of that set.
527  * The output of this function needs to be passed to kernel/sched/core.c
528  * partition_sched_domains() routine, which will rebuild the scheduler's
529  * load balancing domains (sched domains) as specified by that partial
530  * partition.
531  *
532  * See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt
533  * for a background explanation of this.
534  *
535  * Does not return errors, on the theory that the callers of this
536  * routine would rather not worry about failures to rebuild sched
537  * domains when operating in the severe memory shortage situations
538  * that could cause allocation failures below.
539  *
540  * Must be called with cpuset_mutex held.
541  *
542  * The three key local variables below are:
543  *    q  - a linked-list queue of cpuset pointers, used to implement a
544  *         top-down scan of all cpusets.  This scan loads a pointer
545  *         to each cpuset marked is_sched_load_balance into the
546  *         array 'csa'.  For our purposes, rebuilding the schedulers
547  *         sched domains, we can ignore !is_sched_load_balance cpusets.
548  *  csa  - (for CpuSet Array) Array of pointers to all the cpusets
549  *         that need to be load balanced, for convenient iterative
550  *         access by the subsequent code that finds the best partition,
551  *         i.e the set of domains (subsets) of CPUs such that the
552  *         cpus_allowed of every cpuset marked is_sched_load_balance
553  *         is a subset of one of these domains, while there are as
554  *         many such domains as possible, each as small as possible.
555  * doms  - Conversion of 'csa' to an array of cpumasks, for passing to
556  *         the kernel/sched/core.c routine partition_sched_domains() in a
557  *         convenient format, that can be easily compared to the prior
558  *         value to determine what partition elements (sched domains)
559  *         were changed (added or removed.)
560  *
561  * Finding the best partition (set of domains):
562  *      The triple nested loops below over i, j, k scan over the
563  *      load balanced cpusets (using the array of cpuset pointers in
564  *      csa[]) looking for pairs of cpusets that have overlapping
565  *      cpus_allowed, but which don't have the same 'pn' partition
566  *      number and gives them in the same partition number.  It keeps
567  *      looping on the 'restart' label until it can no longer find
568  *      any such pairs.
569  *
570  *      The union of the cpus_allowed masks from the set of
571  *      all cpusets having the same 'pn' value then form the one
572  *      element of the partition (one sched domain) to be passed to
573  *      partition_sched_domains().
574  */
575 static int generate_sched_domains(cpumask_var_t **domains,
576                         struct sched_domain_attr **attributes)
577 {
578         struct cpuset *cp;      /* scans q */
579         struct cpuset **csa;    /* array of all cpuset ptrs */
580         int csn;                /* how many cpuset ptrs in csa so far */
581         int i, j, k;            /* indices for partition finding loops */
582         cpumask_var_t *doms;    /* resulting partition; i.e. sched domains */
583         struct sched_domain_attr *dattr;  /* attributes for custom domains */
584         int ndoms = 0;          /* number of sched domains in result */
585         int nslot;              /* next empty doms[] struct cpumask slot */
586         struct cgroup_subsys_state *pos_css;
587 
588         doms = NULL;
589         dattr = NULL;
590         csa = NULL;
591 
592         /* Special case for the 99% of systems with one, full, sched domain */
593         if (is_sched_load_balance(&top_cpuset)) {
594                 ndoms = 1;
595                 doms = alloc_sched_domains(ndoms);
596                 if (!doms)
597                         goto done;
598 
599                 dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
600                 if (dattr) {
601                         *dattr = SD_ATTR_INIT;
602                         update_domain_attr_tree(dattr, &top_cpuset);
603                 }
604                 cpumask_copy(doms[0], top_cpuset.cpus_allowed);
605 
606                 goto done;
607         }
608 
609         csa = kmalloc(nr_cpusets() * sizeof(cp), GFP_KERNEL);
610         if (!csa)
611                 goto done;
612         csn = 0;
613 
614         rcu_read_lock();
615         cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
616                 if (cp == &top_cpuset)
617                         continue;
618                 /*
619                  * Continue traversing beyond @cp iff @cp has some CPUs and
620                  * isn't load balancing.  The former is obvious.  The
621                  * latter: All child cpusets contain a subset of the
622                  * parent's cpus, so just skip them, and then we call
623                  * update_domain_attr_tree() to calc relax_domain_level of
624                  * the corresponding sched domain.
625                  */
626                 if (!cpumask_empty(cp->cpus_allowed) &&
627                     !is_sched_load_balance(cp))
628                         continue;
629 
630                 if (is_sched_load_balance(cp))
631                         csa[csn++] = cp;
632 
633                 /* skip @cp's subtree */
634                 pos_css = css_rightmost_descendant(pos_css);
635         }
636         rcu_read_unlock();
637 
638         for (i = 0; i < csn; i++)
639                 csa[i]->pn = i;
640         ndoms = csn;
641 
642 restart:
643         /* Find the best partition (set of sched domains) */
644         for (i = 0; i < csn; i++) {
645                 struct cpuset *a = csa[i];
646                 int apn = a->pn;
647 
648                 for (j = 0; j < csn; j++) {
649                         struct cpuset *b = csa[j];
650                         int bpn = b->pn;
651 
652                         if (apn != bpn && cpusets_overlap(a, b)) {
653                                 for (k = 0; k < csn; k++) {
654                                         struct cpuset *c = csa[k];
655 
656                                         if (c->pn == bpn)
657                                                 c->pn = apn;
658                                 }
659                                 ndoms--;        /* one less element */
660                                 goto restart;
661                         }
662                 }
663         }
664 
665         /*
666          * Now we know how many domains to create.
667          * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
668          */
669         doms = alloc_sched_domains(ndoms);
670         if (!doms)
671                 goto done;
672 
673         /*
674          * The rest of the code, including the scheduler, can deal with
675          * dattr==NULL case. No need to abort if alloc fails.
676          */
677         dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);
678 
679         for (nslot = 0, i = 0; i < csn; i++) {
680                 struct cpuset *a = csa[i];
681                 struct cpumask *dp;
682                 int apn = a->pn;
683 
684                 if (apn < 0) {
685                         /* Skip completed partitions */
686                         continue;
687                 }
688 
689                 dp = doms[nslot];
690 
691                 if (nslot == ndoms) {
692                         static int warnings = 10;
693                         if (warnings) {
694                                 pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n",
695                                         nslot, ndoms, csn, i, apn);
696                                 warnings--;
697                         }
698                         continue;
699                 }
700 
701                 cpumask_clear(dp);
702                 if (dattr)
703                         *(dattr + nslot) = SD_ATTR_INIT;
704                 for (j = i; j < csn; j++) {
705                         struct cpuset *b = csa[j];
706 
707                         if (apn == b->pn) {
708                                 cpumask_or(dp, dp, b->cpus_allowed);
709                                 if (dattr)
710                                         update_domain_attr_tree(dattr + nslot, b);
711 
712                                 /* Done with this partition */
713                                 b->pn = -1;
714                         }
715                 }
716                 nslot++;
717         }
718         BUG_ON(nslot != ndoms);
719 
720 done:
721         kfree(csa);
722 
723         /*
724          * Fallback to the default domain if kmalloc() failed.
725          * See comments in partition_sched_domains().
726          */
727         if (doms == NULL)
728                 ndoms = 1;
729 
730         *domains    = doms;
731         *attributes = dattr;
732         return ndoms;
733 }
734 
735 /*
736  * Rebuild scheduler domains.
737  *
738  * If the flag 'sched_load_balance' of any cpuset with non-empty
739  * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
740  * which has that flag enabled, or if any cpuset with a non-empty
741  * 'cpus' is removed, then call this routine to rebuild the
742  * scheduler's dynamic sched domains.
743  *
744  * Call with cpuset_mutex held.  Takes get_online_cpus().
745  */
746 static void rebuild_sched_domains_locked(void)
747 {
748         struct sched_domain_attr *attr;
749         cpumask_var_t *doms;
750         int ndoms;
751 
752         lockdep_assert_held(&cpuset_mutex);
753         get_online_cpus();
754 
755         /*
756          * We have raced with CPU hotplug. Don't do anything to avoid
757          * passing doms with offlined cpu to partition_sched_domains().
758          * Anyways, hotplug work item will rebuild sched domains.
759          */
760         if (!cpumask_equal(top_cpuset.cpus_allowed, cpu_active_mask))
761                 goto out;
762 
763         /* Generate domain masks and attrs */
764         ndoms = generate_sched_domains(&doms, &attr);
765 
766         /* Have scheduler rebuild the domains */
767         partition_sched_domains(ndoms, doms, attr);
768 out:
769         put_online_cpus();
770 }
771 #else /* !CONFIG_SMP */
772 static void rebuild_sched_domains_locked(void)
773 {
774 }
775 #endif /* CONFIG_SMP */
776 
777 void rebuild_sched_domains(void)
778 {
779         mutex_lock(&cpuset_mutex);
780         rebuild_sched_domains_locked();
781         mutex_unlock(&cpuset_mutex);
782 }
783 
784 /*
785  * effective_cpumask_cpuset - return nearest ancestor with non-empty cpus
786  * @cs: the cpuset in interest
787  *
788  * A cpuset's effective cpumask is the cpumask of the nearest ancestor
789  * with non-empty cpus. We use effective cpumask whenever:
790  * - we update tasks' cpus_allowed. (they take on the ancestor's cpumask
791  *   if the cpuset they reside in has no cpus)
792  * - we want to retrieve task_cs(tsk)'s cpus_allowed.
793  *
794  * Called with cpuset_mutex held. cpuset_cpus_allowed_fallback() is an
795  * exception. See comments there.
796  */
797 static struct cpuset *effective_cpumask_cpuset(struct cpuset *cs)
798 {
799         while (cpumask_empty(cs->cpus_allowed))
800                 cs = parent_cs(cs);
801         return cs;
802 }
803 
804 /*
805  * effective_nodemask_cpuset - return nearest ancestor with non-empty mems
806  * @cs: the cpuset in interest
807  *
808  * A cpuset's effective nodemask is the nodemask of the nearest ancestor
809  * with non-empty memss. We use effective nodemask whenever:
810  * - we update tasks' mems_allowed. (they take on the ancestor's nodemask
811  *   if the cpuset they reside in has no mems)
812  * - we want to retrieve task_cs(tsk)'s mems_allowed.
813  *
814  * Called with cpuset_mutex held.
815  */
816 static struct cpuset *effective_nodemask_cpuset(struct cpuset *cs)
817 {
818         while (nodes_empty(cs->mems_allowed))
819                 cs = parent_cs(cs);
820         return cs;
821 }
822 
823 /**
824  * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
825  * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
826  *
827  * Iterate through each task of @cs updating its cpus_allowed to the
828  * effective cpuset's.  As this function is called with cpuset_mutex held,
829  * cpuset membership stays stable.
830  */
831 static void update_tasks_cpumask(struct cpuset *cs)
832 {
833         struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
834         struct css_task_iter it;
835         struct task_struct *task;
836 
837         css_task_iter_start(&cs->css, &it);
838         while ((task = css_task_iter_next(&it)))
839                 set_cpus_allowed_ptr(task, cpus_cs->cpus_allowed);
840         css_task_iter_end(&it);
841 }
842 
843 /*
844  * update_tasks_cpumask_hier - Update the cpumasks of tasks in the hierarchy.
845  * @root_cs: the root cpuset of the hierarchy
846  * @update_root: update root cpuset or not?
847  *
848  * This will update cpumasks of tasks in @root_cs and all other empty cpusets
849  * which take on cpumask of @root_cs.
850  *
851  * Called with cpuset_mutex held
852  */
853 static void update_tasks_cpumask_hier(struct cpuset *root_cs, bool update_root)
854 {
855         struct cpuset *cp;
856         struct cgroup_subsys_state *pos_css;
857 
858         rcu_read_lock();
859         cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
860                 if (cp == root_cs) {
861                         if (!update_root)
862                                 continue;
863                 } else {
864                         /* skip the whole subtree if @cp have some CPU */
865                         if (!cpumask_empty(cp->cpus_allowed)) {
866                                 pos_css = css_rightmost_descendant(pos_css);
867                                 continue;
868                         }
869                 }
870                 if (!css_tryget_online(&cp->css))
871                         continue;
872                 rcu_read_unlock();
873 
874                 update_tasks_cpumask(cp);
875 
876                 rcu_read_lock();
877                 css_put(&cp->css);
878         }
879         rcu_read_unlock();
880 }
881 
882 /**
883  * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
884  * @cs: the cpuset to consider
885  * @trialcs: trial cpuset
886  * @buf: buffer of cpu numbers written to this cpuset
887  */
888 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
889                           const char *buf)
890 {
891         int retval;
892         int is_load_balanced;
893 
894         /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
895         if (cs == &top_cpuset)
896                 return -EACCES;
897 
898         /*
899          * An empty cpus_allowed is ok only if the cpuset has no tasks.
900          * Since cpulist_parse() fails on an empty mask, we special case
901          * that parsing.  The validate_change() call ensures that cpusets
902          * with tasks have cpus.
903          */
904         if (!*buf) {
905                 cpumask_clear(trialcs->cpus_allowed);
906         } else {
907                 retval = cpulist_parse(buf, trialcs->cpus_allowed);
908                 if (retval < 0)
909                         return retval;
910 
911                 if (!cpumask_subset(trialcs->cpus_allowed, cpu_active_mask))
912                         return -EINVAL;
913         }
914 
915         /* Nothing to do if the cpus didn't change */
916         if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
917                 return 0;
918 
919         retval = validate_change(cs, trialcs);
920         if (retval < 0)
921                 return retval;
922 
923         is_load_balanced = is_sched_load_balance(trialcs);
924 
925         mutex_lock(&callback_mutex);
926         cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
927         mutex_unlock(&callback_mutex);
928 
929         update_tasks_cpumask_hier(cs, true);
930 
931         if (is_load_balanced)
932                 rebuild_sched_domains_locked();
933         return 0;
934 }
935 
936 /*
937  * cpuset_migrate_mm
938  *
939  *    Migrate memory region from one set of nodes to another.
940  *
941  *    Temporarilly set tasks mems_allowed to target nodes of migration,
942  *    so that the migration code can allocate pages on these nodes.
943  *
944  *    While the mm_struct we are migrating is typically from some
945  *    other task, the task_struct mems_allowed that we are hacking
946  *    is for our current task, which must allocate new pages for that
947  *    migrating memory region.
948  */
949 
950 static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
951                                                         const nodemask_t *to)
952 {
953         struct task_struct *tsk = current;
954         struct cpuset *mems_cs;
955 
956         tsk->mems_allowed = *to;
957 
958         do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);
959 
960         rcu_read_lock();
961         mems_cs = effective_nodemask_cpuset(task_cs(tsk));
962         guarantee_online_mems(mems_cs, &tsk->mems_allowed);
963         rcu_read_unlock();
964 }
965 
966 /*
967  * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
968  * @tsk: the task to change
969  * @newmems: new nodes that the task will be set
970  *
971  * In order to avoid seeing no nodes if the old and new nodes are disjoint,
972  * we structure updates as setting all new allowed nodes, then clearing newly
973  * disallowed ones.
974  */
975 static void cpuset_change_task_nodemask(struct task_struct *tsk,
976                                         nodemask_t *newmems)
977 {
978         bool need_loop;
979 
980         /*
981          * Allow tasks that have access to memory reserves because they have
982          * been OOM killed to get memory anywhere.
983          */
984         if (unlikely(test_thread_flag(TIF_MEMDIE)))
985                 return;
986         if (current->flags & PF_EXITING) /* Let dying task have memory */
987                 return;
988 
989         task_lock(tsk);
990         /*
991          * Determine if a loop is necessary if another thread is doing
992          * read_mems_allowed_begin().  If at least one node remains unchanged and
993          * tsk does not have a mempolicy, then an empty nodemask will not be
994          * possible when mems_allowed is larger than a word.
995          */
996         need_loop = task_has_mempolicy(tsk) ||
997                         !nodes_intersects(*newmems, tsk->mems_allowed);
998 
999         if (need_loop) {
1000                 local_irq_disable();
1001                 write_seqcount_begin(&tsk->mems_allowed_seq);
1002         }
1003 
1004         nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
1005         mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1);
1006 
1007         mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2);
1008         tsk->mems_allowed = *newmems;
1009 
1010         if (need_loop) {
1011                 write_seqcount_end(&tsk->mems_allowed_seq);
1012                 local_irq_enable();
1013         }
1014 
1015         task_unlock(tsk);
1016 }
1017 
1018 static void *cpuset_being_rebound;
1019 
1020 /**
1021  * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
1022  * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
1023  *
1024  * Iterate through each task of @cs updating its mems_allowed to the
1025  * effective cpuset's.  As this function is called with cpuset_mutex held,
1026  * cpuset membership stays stable.
1027  */
1028 static void update_tasks_nodemask(struct cpuset *cs)
1029 {
1030         static nodemask_t newmems;      /* protected by cpuset_mutex */
1031         struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
1032         struct css_task_iter it;
1033         struct task_struct *task;
1034 
1035         cpuset_being_rebound = cs;              /* causes mpol_dup() rebind */
1036 
1037         guarantee_online_mems(mems_cs, &newmems);
1038 
1039         /*
1040          * The mpol_rebind_mm() call takes mmap_sem, which we couldn't
1041          * take while holding tasklist_lock.  Forks can happen - the
1042          * mpol_dup() cpuset_being_rebound check will catch such forks,
1043          * and rebind their vma mempolicies too.  Because we still hold
1044          * the global cpuset_mutex, we know that no other rebind effort
1045          * will be contending for the global variable cpuset_being_rebound.
1046          * It's ok if we rebind the same mm twice; mpol_rebind_mm()
1047          * is idempotent.  Also migrate pages in each mm to new nodes.
1048          */
1049         css_task_iter_start(&cs->css, &it);
1050         while ((task = css_task_iter_next(&it))) {
1051                 struct mm_struct *mm;
1052                 bool migrate;
1053 
1054                 cpuset_change_task_nodemask(task, &newmems);
1055 
1056                 mm = get_task_mm(task);
1057                 if (!mm)
1058                         continue;
1059 
1060                 migrate = is_memory_migrate(cs);
1061 
1062                 mpol_rebind_mm(mm, &cs->mems_allowed);
1063                 if (migrate)
1064                         cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
1065                 mmput(mm);
1066         }
1067         css_task_iter_end(&it);
1068 
1069         /*
1070          * All the tasks' nodemasks have been updated, update
1071          * cs->old_mems_allowed.
1072          */
1073         cs->old_mems_allowed = newmems;
1074 
1075         /* We're done rebinding vmas to this cpuset's new mems_allowed. */
1076         cpuset_being_rebound = NULL;
1077 }
1078 
1079 /*
1080  * update_tasks_nodemask_hier - Update the nodemasks of tasks in the hierarchy.
1081  * @cs: the root cpuset of the hierarchy
1082  * @update_root: update the root cpuset or not?
1083  *
1084  * This will update nodemasks of tasks in @root_cs and all other empty cpusets
1085  * which take on nodemask of @root_cs.
1086  *
1087  * Called with cpuset_mutex held
1088  */
1089 static void update_tasks_nodemask_hier(struct cpuset *root_cs, bool update_root)
1090 {
1091         struct cpuset *cp;
1092         struct cgroup_subsys_state *pos_css;
1093 
1094         rcu_read_lock();
1095         cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
1096                 if (cp == root_cs) {
1097                         if (!update_root)
1098                                 continue;
1099                 } else {
1100                         /* skip the whole subtree if @cp have some CPU */
1101                         if (!nodes_empty(cp->mems_allowed)) {
1102                                 pos_css = css_rightmost_descendant(pos_css);
1103                                 continue;
1104                         }
1105                 }
1106                 if (!css_tryget_online(&cp->css))
1107                         continue;
1108                 rcu_read_unlock();
1109 
1110                 update_tasks_nodemask(cp);
1111 
1112                 rcu_read_lock();
1113                 css_put(&cp->css);
1114         }
1115         rcu_read_unlock();
1116 }
1117 
1118 /*
1119  * Handle user request to change the 'mems' memory placement
1120  * of a cpuset.  Needs to validate the request, update the
1121  * cpusets mems_allowed, and for each task in the cpuset,
1122  * update mems_allowed and rebind task's mempolicy and any vma
1123  * mempolicies and if the cpuset is marked 'memory_migrate',
1124  * migrate the tasks pages to the new memory.
1125  *
1126  * Call with cpuset_mutex held.  May take callback_mutex during call.
1127  * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
1128  * lock each such tasks mm->mmap_sem, scan its vma's and rebind
1129  * their mempolicies to the cpusets new mems_allowed.
1130  */
1131 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
1132                            const char *buf)
1133 {
1134         int retval;
1135 
1136         /*
1137          * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
1138          * it's read-only
1139          */
1140         if (cs == &top_cpuset) {
1141                 retval = -EACCES;
1142                 goto done;
1143         }
1144 
1145         /*
1146          * An empty mems_allowed is ok iff there are no tasks in the cpuset.
1147          * Since nodelist_parse() fails on an empty mask, we special case
1148          * that parsing.  The validate_change() call ensures that cpusets
1149          * with tasks have memory.
1150          */
1151         if (!*buf) {
1152                 nodes_clear(trialcs->mems_allowed);
1153         } else {
1154                 retval = nodelist_parse(buf, trialcs->mems_allowed);
1155                 if (retval < 0)
1156                         goto done;
1157 
1158                 if (!nodes_subset(trialcs->mems_allowed,
1159                                 node_states[N_MEMORY])) {
1160                         retval =  -EINVAL;
1161                         goto done;
1162                 }
1163         }
1164 
1165         if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) {
1166                 retval = 0;             /* Too easy - nothing to do */
1167                 goto done;
1168         }
1169         retval = validate_change(cs, trialcs);
1170         if (retval < 0)
1171                 goto done;
1172 
1173         mutex_lock(&callback_mutex);
1174         cs->mems_allowed = trialcs->mems_allowed;
1175         mutex_unlock(&callback_mutex);
1176 
1177         update_tasks_nodemask_hier(cs, true);
1178 done:
1179         return retval;
1180 }
1181 
1182 int current_cpuset_is_being_rebound(void)
1183 {
1184         int ret;
1185 
1186         rcu_read_lock();
1187         ret = task_cs(current) == cpuset_being_rebound;
1188         rcu_read_unlock();
1189 
1190         return ret;
1191 }
1192 
1193 static int update_relax_domain_level(struct cpuset *cs, s64 val)
1194 {
1195 #ifdef CONFIG_SMP
1196         if (val < -1 || val >= sched_domain_level_max)
1197                 return -EINVAL;
1198 #endif
1199 
1200         if (val != cs->relax_domain_level) {
1201                 cs->relax_domain_level = val;
1202                 if (!cpumask_empty(cs->cpus_allowed) &&
1203                     is_sched_load_balance(cs))
1204                         rebuild_sched_domains_locked();
1205         }
1206 
1207         return 0;
1208 }
1209 
1210 /**
1211  * update_tasks_flags - update the spread flags of tasks in the cpuset.
1212  * @cs: the cpuset in which each task's spread flags needs to be changed
1213  *
1214  * Iterate through each task of @cs updating its spread flags.  As this
1215  * function is called with cpuset_mutex held, cpuset membership stays
1216  * stable.
1217  */
1218 static void update_tasks_flags(struct cpuset *cs)
1219 {
1220         struct css_task_iter it;
1221         struct task_struct *task;
1222 
1223         css_task_iter_start(&cs->css, &it);
1224         while ((task = css_task_iter_next(&it)))
1225                 cpuset_update_task_spread_flag(cs, task);
1226         css_task_iter_end(&it);
1227 }
1228 
1229 /*
1230  * update_flag - read a 0 or a 1 in a file and update associated flag
1231  * bit:         the bit to update (see cpuset_flagbits_t)
1232  * cs:          the cpuset to update
1233  * turning_on:  whether the flag is being set or cleared
1234  *
1235  * Call with cpuset_mutex held.
1236  */
1237 
1238 static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1239                        int turning_on)
1240 {
1241         struct cpuset *trialcs;
1242         int balance_flag_changed;
1243         int spread_flag_changed;
1244         int err;
1245 
1246         trialcs = alloc_trial_cpuset(cs);
1247         if (!trialcs)
1248                 return -ENOMEM;
1249 
1250         if (turning_on)
1251                 set_bit(bit, &trialcs->flags);
1252         else
1253                 clear_bit(bit, &trialcs->flags);
1254 
1255         err = validate_change(cs, trialcs);
1256         if (err < 0)
1257                 goto out;
1258 
1259         balance_flag_changed = (is_sched_load_balance(cs) !=
1260                                 is_sched_load_balance(trialcs));
1261 
1262         spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
1263                         || (is_spread_page(cs) != is_spread_page(trialcs)));
1264 
1265         mutex_lock(&callback_mutex);
1266         cs->flags = trialcs->flags;
1267         mutex_unlock(&callback_mutex);
1268 
1269         if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
1270                 rebuild_sched_domains_locked();
1271 
1272         if (spread_flag_changed)
1273                 update_tasks_flags(cs);
1274 out:
1275         free_trial_cpuset(trialcs);
1276         return err;
1277 }
1278 
1279 /*
1280  * Frequency meter - How fast is some event occurring?
1281  *
1282  * These routines manage a digitally filtered, constant time based,
1283  * event frequency meter.  There are four routines:
1284  *   fmeter_init() - initialize a frequency meter.
1285  *   fmeter_markevent() - called each time the event happens.
1286  *   fmeter_getrate() - returns the recent rate of such events.
1287  *   fmeter_update() - internal routine used to update fmeter.
1288  *
1289  * A common data structure is passed to each of these routines,
1290  * which is used to keep track of the state required to manage the
1291  * frequency meter and its digital filter.
1292  *
1293  * The filter works on the number of events marked per unit time.
1294  * The filter is single-pole low-pass recursive (IIR).  The time unit
1295  * is 1 second.  Arithmetic is done using 32-bit integers scaled to
1296  * simulate 3 decimal digits of precision (multiplied by 1000).
1297  *
1298  * With an FM_COEF of 933, and a time base of 1 second, the filter
1299  * has a half-life of 10 seconds, meaning that if the events quit
1300  * happening, then the rate returned from the fmeter_getrate()
1301  * will be cut in half each 10 seconds, until it converges to zero.
1302  *
1303  * It is not worth doing a real infinitely recursive filter.  If more
1304  * than FM_MAXTICKS ticks have elapsed since the last filter event,
1305  * just compute FM_MAXTICKS ticks worth, by which point the level
1306  * will be stable.
1307  *
1308  * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
1309  * arithmetic overflow in the fmeter_update() routine.
1310  *
1311  * Given the simple 32 bit integer arithmetic used, this meter works
1312  * best for reporting rates between one per millisecond (msec) and
1313  * one per 32 (approx) seconds.  At constant rates faster than one
1314  * per msec it maxes out at values just under 1,000,000.  At constant
1315  * rates between one per msec, and one per second it will stabilize
1316  * to a value N*1000, where N is the rate of events per second.
1317  * At constant rates between one per second and one per 32 seconds,
1318  * it will be choppy, moving up on the seconds that have an event,
1319  * and then decaying until the next event.  At rates slower than
1320  * about one in 32 seconds, it decays all the way back to zero between
1321  * each event.
1322  */
1323 
1324 #define FM_COEF 933             /* coefficient for half-life of 10 secs */
1325 #define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */
1326 #define FM_MAXCNT 1000000       /* limit cnt to avoid overflow */
1327 #define FM_SCALE 1000           /* faux fixed point scale */
1328 
1329 /* Initialize a frequency meter */
1330 static void fmeter_init(struct fmeter *fmp)
1331 {
1332         fmp->cnt = 0;
1333         fmp->val = 0;
1334         fmp->time = 0;
1335         spin_lock_init(&fmp->lock);
1336 }
1337 
1338 /* Internal meter update - process cnt events and update value */
1339 static void fmeter_update(struct fmeter *fmp)
1340 {
1341         time_t now = get_seconds();
1342         time_t ticks = now - fmp->time;
1343 
1344         if (ticks == 0)
1345                 return;
1346 
1347         ticks = min(FM_MAXTICKS, ticks);
1348         while (ticks-- > 0)
1349                 fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
1350         fmp->time = now;
1351 
1352         fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
1353         fmp->cnt = 0;
1354 }
1355 
1356 /* Process any previous ticks, then bump cnt by one (times scale). */
1357 static void fmeter_markevent(struct fmeter *fmp)
1358 {
1359         spin_lock(&fmp->lock);
1360         fmeter_update(fmp);
1361         fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
1362         spin_unlock(&fmp->lock);
1363 }
1364 
1365 /* Process any previous ticks, then return current value. */
1366 static int fmeter_getrate(struct fmeter *fmp)
1367 {
1368         int val;
1369 
1370         spin_lock(&fmp->lock);
1371         fmeter_update(fmp);
1372         val = fmp->val;
1373         spin_unlock(&fmp->lock);
1374         return val;
1375 }
1376 
1377 static struct cpuset *cpuset_attach_old_cs;
1378 
1379 /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
1380 static int cpuset_can_attach(struct cgroup_subsys_state *css,
1381                              struct cgroup_taskset *tset)
1382 {
1383         struct cpuset *cs = css_cs(css);
1384         struct task_struct *task;
1385         int ret;
1386 
1387         /* used later by cpuset_attach() */
1388         cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset));
1389 
1390         mutex_lock(&cpuset_mutex);
1391 
1392         /*
1393          * We allow to move tasks into an empty cpuset if sane_behavior
1394          * flag is set.
1395          */
1396         ret = -ENOSPC;
1397         if (!cgroup_sane_behavior(css->cgroup) &&
1398             (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)))
1399                 goto out_unlock;
1400 
1401         cgroup_taskset_for_each(task, tset) {
1402                 /*
1403                  * Kthreads which disallow setaffinity shouldn't be moved
1404                  * to a new cpuset; we don't want to change their cpu
1405                  * affinity and isolating such threads by their set of
1406                  * allowed nodes is unnecessary.  Thus, cpusets are not
1407                  * applicable for such threads.  This prevents checking for
1408                  * success of set_cpus_allowed_ptr() on all attached tasks
1409                  * before cpus_allowed may be changed.
1410                  */
1411                 ret = -EINVAL;
1412                 if (task->flags & PF_NO_SETAFFINITY)
1413                         goto out_unlock;
1414                 ret = security_task_setscheduler(task);
1415                 if (ret)
1416                         goto out_unlock;
1417         }
1418 
1419         /*
1420          * Mark attach is in progress.  This makes validate_change() fail
1421          * changes which zero cpus/mems_allowed.
1422          */
1423         cs->attach_in_progress++;
1424         ret = 0;
1425 out_unlock:
1426         mutex_unlock(&cpuset_mutex);
1427         return ret;
1428 }
1429 
1430 static void cpuset_cancel_attach(struct cgroup_subsys_state *css,
1431                                  struct cgroup_taskset *tset)
1432 {
1433         mutex_lock(&cpuset_mutex);
1434         css_cs(css)->attach_in_progress--;
1435         mutex_unlock(&cpuset_mutex);
1436 }
1437 
1438 /*
1439  * Protected by cpuset_mutex.  cpus_attach is used only by cpuset_attach()
1440  * but we can't allocate it dynamically there.  Define it global and
1441  * allocate from cpuset_init().
1442  */
1443 static cpumask_var_t cpus_attach;
1444 
1445 static void cpuset_attach(struct cgroup_subsys_state *css,
1446                           struct cgroup_taskset *tset)
1447 {
1448         /* static buf protected by cpuset_mutex */
1449         static nodemask_t cpuset_attach_nodemask_to;
1450         struct mm_struct *mm;
1451         struct task_struct *task;
1452         struct task_struct *leader = cgroup_taskset_first(tset);
1453         struct cpuset *cs = css_cs(css);
1454         struct cpuset *oldcs = cpuset_attach_old_cs;
1455         struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
1456         struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
1457 
1458         mutex_lock(&cpuset_mutex);
1459 
1460         /* prepare for attach */
1461         if (cs == &top_cpuset)
1462                 cpumask_copy(cpus_attach, cpu_possible_mask);
1463         else
1464                 guarantee_online_cpus(cpus_cs, cpus_attach);
1465 
1466         guarantee_online_mems(mems_cs, &cpuset_attach_nodemask_to);
1467 
1468         cgroup_taskset_for_each(task, tset) {
1469                 /*
1470                  * can_attach beforehand should guarantee that this doesn't
1471                  * fail.  TODO: have a better way to handle failure here
1472                  */
1473                 WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));
1474 
1475                 cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
1476                 cpuset_update_task_spread_flag(cs, task);
1477         }
1478 
1479         /*
1480          * Change mm, possibly for multiple threads in a threadgroup. This is
1481          * expensive and may sleep.
1482          */
1483         cpuset_attach_nodemask_to = cs->mems_allowed;
1484         mm = get_task_mm(leader);
1485         if (mm) {
1486                 struct cpuset *mems_oldcs = effective_nodemask_cpuset(oldcs);
1487 
1488                 mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
1489 
1490                 /*
1491                  * old_mems_allowed is the same with mems_allowed here, except
1492                  * if this task is being moved automatically due to hotplug.
1493                  * In that case @mems_allowed has been updated and is empty,
1494                  * so @old_mems_allowed is the right nodesets that we migrate
1495                  * mm from.
1496                  */
1497                 if (is_memory_migrate(cs)) {
1498                         cpuset_migrate_mm(mm, &mems_oldcs->old_mems_allowed,
1499                                           &cpuset_attach_nodemask_to);
1500                 }
1501                 mmput(mm);
1502         }
1503 
1504         cs->old_mems_allowed = cpuset_attach_nodemask_to;
1505 
1506         cs->attach_in_progress--;
1507         if (!cs->attach_in_progress)
1508                 wake_up(&cpuset_attach_wq);
1509 
1510         mutex_unlock(&cpuset_mutex);
1511 }
1512 
1513 /* The various types of files and directories in a cpuset file system */
1514 
1515 typedef enum {
1516         FILE_MEMORY_MIGRATE,
1517         FILE_CPULIST,
1518         FILE_MEMLIST,
1519         FILE_CPU_EXCLUSIVE,
1520         FILE_MEM_EXCLUSIVE,
1521         FILE_MEM_HARDWALL,
1522         FILE_SCHED_LOAD_BALANCE,
1523         FILE_SCHED_RELAX_DOMAIN_LEVEL,
1524         FILE_MEMORY_PRESSURE_ENABLED,
1525         FILE_MEMORY_PRESSURE,
1526         FILE_SPREAD_PAGE,
1527         FILE_SPREAD_SLAB,
1528 } cpuset_filetype_t;
1529 
1530 static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
1531                             u64 val)
1532 {
1533         struct cpuset *cs = css_cs(css);
1534         cpuset_filetype_t type = cft->private;
1535         int retval = 0;
1536 
1537         mutex_lock(&cpuset_mutex);
1538         if (!is_cpuset_online(cs)) {
1539                 retval = -ENODEV;
1540                 goto out_unlock;
1541         }
1542 
1543         switch (type) {
1544         case FILE_CPU_EXCLUSIVE:
1545                 retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
1546                 break;
1547         case FILE_MEM_EXCLUSIVE:
1548                 retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
1549                 break;
1550         case FILE_MEM_HARDWALL:
1551                 retval = update_flag(CS_MEM_HARDWALL, cs, val);
1552                 break;
1553         case FILE_SCHED_LOAD_BALANCE:
1554                 retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
1555                 break;
1556         case FILE_MEMORY_MIGRATE:
1557                 retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
1558                 break;
1559         case FILE_MEMORY_PRESSURE_ENABLED:
1560                 cpuset_memory_pressure_enabled = !!val;
1561                 break;
1562         case FILE_MEMORY_PRESSURE:
1563                 retval = -EACCES;
1564                 break;
1565         case FILE_SPREAD_PAGE:
1566                 retval = update_flag(CS_SPREAD_PAGE, cs, val);
1567                 break;
1568         case FILE_SPREAD_SLAB:
1569                 retval = update_flag(CS_SPREAD_SLAB, cs, val);
1570                 break;
1571         default:
1572                 retval = -EINVAL;
1573                 break;
1574         }
1575 out_unlock:
1576         mutex_unlock(&cpuset_mutex);
1577         return retval;
1578 }
1579 
1580 static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
1581                             s64 val)
1582 {
1583         struct cpuset *cs = css_cs(css);
1584         cpuset_filetype_t type = cft->private;
1585         int retval = -ENODEV;
1586 
1587         mutex_lock(&cpuset_mutex);
1588         if (!is_cpuset_online(cs))
1589                 goto out_unlock;
1590 
1591         switch (type) {
1592         case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1593                 retval = update_relax_domain_level(cs, val);
1594                 break;
1595         default:
1596                 retval = -EINVAL;
1597                 break;
1598         }
1599 out_unlock:
1600         mutex_unlock(&cpuset_mutex);
1601         return retval;
1602 }
1603 
1604 /*
1605  * Common handling for a write to a "cpus" or "mems" file.
1606  */
1607 static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
1608                                     char *buf, size_t nbytes, loff_t off)
1609 {
1610         struct cpuset *cs = css_cs(of_css(of));
1611         struct cpuset *trialcs;
1612         int retval = -ENODEV;
1613 
1614         buf = strstrip(buf);
1615 
1616         /*
1617          * CPU or memory hotunplug may leave @cs w/o any execution
1618          * resources, in which case the hotplug code asynchronously updates
1619          * configuration and transfers all tasks to the nearest ancestor
1620          * which can execute.
1621          *
1622          * As writes to "cpus" or "mems" may restore @cs's execution
1623          * resources, wait for the previously scheduled operations before
1624          * proceeding, so that we don't end up keep removing tasks added
1625          * after execution capability is restored.
1626          *
1627          * cpuset_hotplug_work calls back into cgroup core via
1628          * cgroup_transfer_tasks() and waiting for it from a cgroupfs
1629          * operation like this one can lead to a deadlock through kernfs
1630          * active_ref protection.  Let's break the protection.  Losing the
1631          * protection is okay as we check whether @cs is online after
1632          * grabbing cpuset_mutex anyway.  This only happens on the legacy
1633          * hierarchies.
1634          */
1635         css_get(&cs->css);
1636         kernfs_break_active_protection(of->kn);
1637         flush_work(&cpuset_hotplug_work);
1638 
1639         mutex_lock(&cpuset_mutex);
1640         if (!is_cpuset_online(cs))
1641                 goto out_unlock;
1642 
1643         trialcs = alloc_trial_cpuset(cs);
1644         if (!trialcs) {
1645                 retval = -ENOMEM;
1646                 goto out_unlock;
1647         }
1648 
1649         switch (of_cft(of)->private) {
1650         case FILE_CPULIST:
1651                 retval = update_cpumask(cs, trialcs, buf);
1652                 break;
1653         case FILE_MEMLIST:
1654                 retval = update_nodemask(cs, trialcs, buf);
1655                 break;
1656         default:
1657                 retval = -EINVAL;
1658                 break;
1659         }
1660 
1661         free_trial_cpuset(trialcs);
1662 out_unlock:
1663         mutex_unlock(&cpuset_mutex);
1664         kernfs_unbreak_active_protection(of->kn);
1665         css_put(&cs->css);
1666         return retval ?: nbytes;
1667 }
1668 
1669 /*
1670  * These ascii lists should be read in a single call, by using a user
1671  * buffer large enough to hold the entire map.  If read in smaller
1672  * chunks, there is no guarantee of atomicity.  Since the display format
1673  * used, list of ranges of sequential numbers, is variable length,
1674  * and since these maps can change value dynamically, one could read
1675  * gibberish by doing partial reads while a list was changing.
1676  */
1677 static int cpuset_common_seq_show(struct seq_file *sf, void *v)
1678 {
1679         struct cpuset *cs = css_cs(seq_css(sf));
1680         cpuset_filetype_t type = seq_cft(sf)->private;
1681         ssize_t count;
1682         char *buf, *s;
1683         int ret = 0;
1684 
1685         count = seq_get_buf(sf, &buf);
1686         s = buf;
1687 
1688         mutex_lock(&callback_mutex);
1689 
1690         switch (type) {
1691         case FILE_CPULIST:
1692                 s += cpulist_scnprintf(s, count, cs->cpus_allowed);
1693                 break;
1694         case FILE_MEMLIST:
1695                 s += nodelist_scnprintf(s, count, cs->mems_allowed);
1696                 break;
1697         default:
1698                 ret = -EINVAL;
1699                 goto out_unlock;
1700         }
1701 
1702         if (s < buf + count - 1) {
1703                 *s++ = '\n';
1704                 seq_commit(sf, s - buf);
1705         } else {
1706                 seq_commit(sf, -1);
1707         }
1708 out_unlock:
1709         mutex_unlock(&callback_mutex);
1710         return ret;
1711 }
1712 
1713 static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
1714 {
1715         struct cpuset *cs = css_cs(css);
1716         cpuset_filetype_t type = cft->private;
1717         switch (type) {
1718         case FILE_CPU_EXCLUSIVE:
1719                 return is_cpu_exclusive(cs);
1720         case FILE_MEM_EXCLUSIVE:
1721                 return is_mem_exclusive(cs);
1722         case FILE_MEM_HARDWALL:
1723                 return is_mem_hardwall(cs);
1724         case FILE_SCHED_LOAD_BALANCE:
1725                 return is_sched_load_balance(cs);
1726         case FILE_MEMORY_MIGRATE:
1727                 return is_memory_migrate(cs);
1728         case FILE_MEMORY_PRESSURE_ENABLED:
1729                 return cpuset_memory_pressure_enabled;
1730         case FILE_MEMORY_PRESSURE:
1731                 return fmeter_getrate(&cs->fmeter);
1732         case FILE_SPREAD_PAGE:
1733                 return is_spread_page(cs);
1734         case FILE_SPREAD_SLAB:
1735                 return is_spread_slab(cs);
1736         default:
1737                 BUG();
1738         }
1739 
1740         /* Unreachable but makes gcc happy */
1741         return 0;
1742 }
1743 
1744 static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
1745 {
1746         struct cpuset *cs = css_cs(css);
1747         cpuset_filetype_t type = cft->private;
1748         switch (type) {
1749         case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1750                 return cs->relax_domain_level;
1751         default:
1752                 BUG();
1753         }
1754 
1755         /* Unrechable but makes gcc happy */
1756         return 0;
1757 }
1758 
1759 
1760 /*
1761  * for the common functions, 'private' gives the type of file
1762  */
1763 
1764 static struct cftype files[] = {
1765         {
1766                 .name = "cpus",
1767                 .seq_show = cpuset_common_seq_show,
1768                 .write = cpuset_write_resmask,
1769                 .max_write_len = (100U + 6 * NR_CPUS),
1770                 .private = FILE_CPULIST,
1771         },
1772 
1773         {
1774                 .name = "mems",
1775                 .seq_show = cpuset_common_seq_show,
1776                 .write = cpuset_write_resmask,
1777                 .max_write_len = (100U + 6 * MAX_NUMNODES),
1778                 .private = FILE_MEMLIST,
1779         },
1780 
1781         {
1782                 .name = "cpu_exclusive",
1783                 .read_u64 = cpuset_read_u64,
1784                 .write_u64 = cpuset_write_u64,
1785                 .private = FILE_CPU_EXCLUSIVE,
1786         },
1787 
1788         {
1789                 .name = "mem_exclusive",
1790                 .read_u64 = cpuset_read_u64,
1791                 .write_u64 = cpuset_write_u64,
1792                 .private = FILE_MEM_EXCLUSIVE,
1793         },
1794 
1795         {
1796                 .name = "mem_hardwall",
1797                 .read_u64 = cpuset_read_u64,
1798                 .write_u64 = cpuset_write_u64,
1799                 .private = FILE_MEM_HARDWALL,
1800         },
1801 
1802         {
1803                 .name = "sched_load_balance",
1804                 .read_u64 = cpuset_read_u64,
1805                 .write_u64 = cpuset_write_u64,
1806                 .private = FILE_SCHED_LOAD_BALANCE,
1807         },
1808 
1809         {
1810                 .name = "sched_relax_domain_level",
1811                 .read_s64 = cpuset_read_s64,
1812                 .write_s64 = cpuset_write_s64,
1813                 .private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
1814         },
1815 
1816         {
1817                 .name = "memory_migrate",
1818                 .read_u64 = cpuset_read_u64,
1819                 .write_u64 = cpuset_write_u64,
1820                 .private = FILE_MEMORY_MIGRATE,
1821         },
1822 
1823         {
1824                 .name = "memory_pressure",
1825                 .read_u64 = cpuset_read_u64,
1826                 .write_u64 = cpuset_write_u64,
1827                 .private = FILE_MEMORY_PRESSURE,
1828                 .mode = S_IRUGO,
1829         },
1830 
1831         {
1832                 .name = "memory_spread_page",
1833                 .read_u64 = cpuset_read_u64,
1834                 .write_u64 = cpuset_write_u64,
1835                 .private = FILE_SPREAD_PAGE,
1836         },
1837 
1838         {
1839                 .name = "memory_spread_slab",
1840                 .read_u64 = cpuset_read_u64,
1841                 .write_u64 = cpuset_write_u64,
1842                 .private = FILE_SPREAD_SLAB,
1843         },
1844 
1845         {
1846                 .name = "memory_pressure_enabled",
1847                 .flags = CFTYPE_ONLY_ON_ROOT,
1848                 .read_u64 = cpuset_read_u64,
1849                 .write_u64 = cpuset_write_u64,
1850                 .private = FILE_MEMORY_PRESSURE_ENABLED,
1851         },
1852 
1853         { }     /* terminate */
1854 };
1855 
1856 /*
1857  *      cpuset_css_alloc - allocate a cpuset css
1858  *      cgrp:   control group that the new cpuset will be part of
1859  */
1860 
1861 static struct cgroup_subsys_state *
1862 cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
1863 {
1864         struct cpuset *cs;
1865 
1866         if (!parent_css)
1867                 return &top_cpuset.css;
1868 
1869         cs = kzalloc(sizeof(*cs), GFP_KERNEL);
1870         if (!cs)
1871                 return ERR_PTR(-ENOMEM);
1872         if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) {
1873                 kfree(cs);
1874                 return ERR_PTR(-ENOMEM);
1875         }
1876 
1877         set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
1878         cpumask_clear(cs->cpus_allowed);
1879         nodes_clear(cs->mems_allowed);
1880         fmeter_init(&cs->fmeter);
1881         cs->relax_domain_level = -1;
1882 
1883         return &cs->css;
1884 }
1885 
1886 static int cpuset_css_online(struct cgroup_subsys_state *css)
1887 {
1888         struct cpuset *cs = css_cs(css);
1889         struct cpuset *parent = parent_cs(cs);
1890         struct cpuset *tmp_cs;
1891         struct cgroup_subsys_state *pos_css;
1892 
1893         if (!parent)
1894                 return 0;
1895 
1896         mutex_lock(&cpuset_mutex);
1897 
1898         set_bit(CS_ONLINE, &cs->flags);
1899         if (is_spread_page(parent))
1900                 set_bit(CS_SPREAD_PAGE, &cs->flags);
1901         if (is_spread_slab(parent))
1902                 set_bit(CS_SPREAD_SLAB, &cs->flags);
1903 
1904         cpuset_inc();
1905 
1906         if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
1907                 goto out_unlock;
1908 
1909         /*
1910          * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
1911          * set.  This flag handling is implemented in cgroup core for
1912          * histrical reasons - the flag may be specified during mount.
1913          *
1914          * Currently, if any sibling cpusets have exclusive cpus or mem, we
1915          * refuse to clone the configuration - thereby refusing the task to
1916          * be entered, and as a result refusing the sys_unshare() or
1917          * clone() which initiated it.  If this becomes a problem for some
1918          * users who wish to allow that scenario, then this could be
1919          * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
1920          * (and likewise for mems) to the new cgroup.
1921          */
1922         rcu_read_lock();
1923         cpuset_for_each_child(tmp_cs, pos_css, parent) {
1924                 if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
1925                         rcu_read_unlock();
1926                         goto out_unlock;
1927                 }
1928         }
1929         rcu_read_unlock();
1930 
1931         mutex_lock(&callback_mutex);
1932         cs->mems_allowed = parent->mems_allowed;
1933         cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
1934         mutex_unlock(&callback_mutex);
1935 out_unlock:
1936         mutex_unlock(&cpuset_mutex);
1937         return 0;
1938 }
1939 
1940 /*
1941  * If the cpuset being removed has its flag 'sched_load_balance'
1942  * enabled, then simulate turning sched_load_balance off, which
1943  * will call rebuild_sched_domains_locked().
1944  */
1945 
1946 static void cpuset_css_offline(struct cgroup_subsys_state *css)
1947 {
1948         struct cpuset *cs = css_cs(css);
1949 
1950         mutex_lock(&cpuset_mutex);
1951 
1952         if (is_sched_load_balance(cs))
1953                 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
1954 
1955         cpuset_dec();
1956         clear_bit(CS_ONLINE, &cs->flags);
1957 
1958         mutex_unlock(&cpuset_mutex);
1959 }
1960 
1961 static void cpuset_css_free(struct cgroup_subsys_state *css)
1962 {
1963         struct cpuset *cs = css_cs(css);
1964 
1965         free_cpumask_var(cs->cpus_allowed);
1966         kfree(cs);
1967 }
1968 
1969 struct cgroup_subsys cpuset_cgrp_subsys = {
1970         .css_alloc = cpuset_css_alloc,
1971         .css_online = cpuset_css_online,
1972         .css_offline = cpuset_css_offline,
1973         .css_free = cpuset_css_free,
1974         .can_attach = cpuset_can_attach,
1975         .cancel_attach = cpuset_cancel_attach,
1976         .attach = cpuset_attach,
1977         .base_cftypes = files,
1978         .early_init = 1,
1979 };
1980 
1981 /**
1982  * cpuset_init - initialize cpusets at system boot
1983  *
1984  * Description: Initialize top_cpuset and the cpuset internal file system,
1985  **/
1986 
1987 int __init cpuset_init(void)
1988 {
1989         int err = 0;
1990 
1991         if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL))
1992                 BUG();
1993 
1994         cpumask_setall(top_cpuset.cpus_allowed);
1995         nodes_setall(top_cpuset.mems_allowed);
1996 
1997         fmeter_init(&top_cpuset.fmeter);
1998         set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
1999         top_cpuset.relax_domain_level = -1;
2000 
2001         err = register_filesystem(&cpuset_fs_type);
2002         if (err < 0)
2003                 return err;
2004 
2005         if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
2006                 BUG();
2007 
2008         return 0;
2009 }
2010 
2011 /*
2012  * If CPU and/or memory hotplug handlers, below, unplug any CPUs
2013  * or memory nodes, we need to walk over the cpuset hierarchy,
2014  * removing that CPU or node from all cpusets.  If this removes the
2015  * last CPU or node from a cpuset, then move the tasks in the empty
2016  * cpuset to its next-highest non-empty parent.
2017  */
2018 static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
2019 {
2020         struct cpuset *parent;
2021 
2022         /*
2023          * Find its next-highest non-empty parent, (top cpuset
2024          * has online cpus, so can't be empty).
2025          */
2026         parent = parent_cs(cs);
2027         while (cpumask_empty(parent->cpus_allowed) ||
2028                         nodes_empty(parent->mems_allowed))
2029                 parent = parent_cs(parent);
2030 
2031         if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {
2032                 pr_err("cpuset: failed to transfer tasks out of empty cpuset ");
2033                 pr_cont_cgroup_name(cs->css.cgroup);
2034                 pr_cont("\n");
2035         }
2036 }
2037 
2038 /**
2039  * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
2040  * @cs: cpuset in interest
2041  *
2042  * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
2043  * offline, update @cs accordingly.  If @cs ends up with no CPU or memory,
2044  * all its tasks are moved to the nearest ancestor with both resources.
2045  */
2046 static void cpuset_hotplug_update_tasks(struct cpuset *cs)
2047 {
2048         static cpumask_t off_cpus;
2049         static nodemask_t off_mems;
2050         bool is_empty;
2051         bool sane = cgroup_sane_behavior(cs->css.cgroup);
2052 
2053 retry:
2054         wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
2055 
2056         mutex_lock(&cpuset_mutex);
2057 
2058         /*
2059          * We have raced with task attaching. We wait until attaching
2060          * is finished, so we won't attach a task to an empty cpuset.
2061          */
2062         if (cs->attach_in_progress) {
2063                 mutex_unlock(&cpuset_mutex);
2064                 goto retry;
2065         }
2066 
2067         cpumask_andnot(&off_cpus, cs->cpus_allowed, top_cpuset.cpus_allowed);
2068         nodes_andnot(off_mems, cs->mems_allowed, top_cpuset.mems_allowed);
2069 
2070         mutex_lock(&callback_mutex);
2071         cpumask_andnot(cs->cpus_allowed, cs->cpus_allowed, &off_cpus);
2072         mutex_unlock(&callback_mutex);
2073 
2074         /*
2075          * If sane_behavior flag is set, we need to update tasks' cpumask
2076          * for empty cpuset to take on ancestor's cpumask. Otherwise, don't
2077          * call update_tasks_cpumask() if the cpuset becomes empty, as
2078          * the tasks in it will be migrated to an ancestor.
2079          */
2080         if ((sane && cpumask_empty(cs->cpus_allowed)) ||
2081             (!cpumask_empty(&off_cpus) && !cpumask_empty(cs->cpus_allowed)))
2082                 update_tasks_cpumask(cs);
2083 
2084         mutex_lock(&callback_mutex);
2085         nodes_andnot(cs->mems_allowed, cs->mems_allowed, off_mems);
2086         mutex_unlock(&callback_mutex);
2087 
2088         /*
2089          * If sane_behavior flag is set, we need to update tasks' nodemask
2090          * for empty cpuset to take on ancestor's nodemask. Otherwise, don't
2091          * call update_tasks_nodemask() if the cpuset becomes empty, as
2092          * the tasks in it will be migratd to an ancestor.
2093          */
2094         if ((sane && nodes_empty(cs->mems_allowed)) ||
2095             (!nodes_empty(off_mems) && !nodes_empty(cs->mems_allowed)))
2096                 update_tasks_nodemask(cs);
2097 
2098         is_empty = cpumask_empty(cs->cpus_allowed) ||
2099                 nodes_empty(cs->mems_allowed);
2100 
2101         mutex_unlock(&cpuset_mutex);
2102 
2103         /*
2104          * If sane_behavior flag is set, we'll keep tasks in empty cpusets.
2105          *
2106          * Otherwise move tasks to the nearest ancestor with execution
2107          * resources.  This is full cgroup operation which will
2108          * also call back into cpuset.  Should be done outside any lock.
2109          */
2110         if (!sane && is_empty)
2111                 remove_tasks_in_empty_cpuset(cs);
2112 }
2113 
2114 /**
2115  * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
2116  *
2117  * This function is called after either CPU or memory configuration has
2118  * changed and updates cpuset accordingly.  The top_cpuset is always
2119  * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
2120  * order to make cpusets transparent (of no affect) on systems that are
2121  * actively using CPU hotplug but making no active use of cpusets.
2122  *
2123  * Non-root cpusets are only affected by offlining.  If any CPUs or memory
2124  * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
2125  * all descendants.
2126  *
2127  * Note that CPU offlining during suspend is ignored.  We don't modify
2128  * cpusets across suspend/resume cycles at all.
2129  */
2130 static void cpuset_hotplug_workfn(struct work_struct *work)
2131 {
2132         static cpumask_t new_cpus;
2133         static nodemask_t new_mems;
2134         bool cpus_updated, mems_updated;
2135 
2136         mutex_lock(&cpuset_mutex);
2137 
2138         /* fetch the available cpus/mems and find out which changed how */
2139         cpumask_copy(&new_cpus, cpu_active_mask);
2140         new_mems = node_states[N_MEMORY];
2141 
2142         cpus_updated = !cpumask_equal(top_cpuset.cpus_allowed, &new_cpus);
2143         mems_updated = !nodes_equal(top_cpuset.mems_allowed, new_mems);
2144 
2145         /* synchronize cpus_allowed to cpu_active_mask */
2146         if (cpus_updated) {
2147                 mutex_lock(&callback_mutex);
2148                 cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
2149                 mutex_unlock(&callback_mutex);
2150                 /* we don't mess with cpumasks of tasks in top_cpuset */
2151         }
2152 
2153         /* synchronize mems_allowed to N_MEMORY */
2154         if (mems_updated) {
2155                 mutex_lock(&callback_mutex);
2156                 top_cpuset.mems_allowed = new_mems;
2157                 mutex_unlock(&callback_mutex);
2158                 update_tasks_nodemask(&top_cpuset);
2159         }
2160 
2161         mutex_unlock(&cpuset_mutex);
2162 
2163         /* if cpus or mems changed, we need to propagate to descendants */
2164         if (cpus_updated || mems_updated) {
2165                 struct cpuset *cs;
2166                 struct cgroup_subsys_state *pos_css;
2167 
2168                 rcu_read_lock();
2169                 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
2170                         if (cs == &top_cpuset || !css_tryget_online(&cs->css))
2171                                 continue;
2172                         rcu_read_unlock();
2173 
2174                         cpuset_hotplug_update_tasks(cs);
2175 
2176                         rcu_read_lock();
2177                         css_put(&cs->css);
2178                 }
2179                 rcu_read_unlock();
2180         }
2181 
2182         /* rebuild sched domains if cpus_allowed has changed */
2183         if (cpus_updated)
2184                 rebuild_sched_domains();
2185 }
2186 
2187 void cpuset_update_active_cpus(bool cpu_online)
2188 {
2189         /*
2190          * We're inside cpu hotplug critical region which usually nests
2191          * inside cgroup synchronization.  Bounce actual hotplug processing
2192          * to a work item to avoid reverse locking order.
2193          *
2194          * We still need to do partition_sched_domains() synchronously;
2195          * otherwise, the scheduler will get confused and put tasks to the
2196          * dead CPU.  Fall back to the default single domain.
2197          * cpuset_hotplug_workfn() will rebuild it as necessary.
2198          */
2199         partition_sched_domains(1, NULL, NULL);
2200         schedule_work(&cpuset_hotplug_work);
2201 }
2202 
2203 /*
2204  * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
2205  * Call this routine anytime after node_states[N_MEMORY] changes.
2206  * See cpuset_update_active_cpus() for CPU hotplug handling.
2207  */
2208 static int cpuset_track_online_nodes(struct notifier_block *self,
2209                                 unsigned long action, void *arg)
2210 {
2211         schedule_work(&cpuset_hotplug_work);
2212         return NOTIFY_OK;
2213 }
2214 
2215 static struct notifier_block cpuset_track_online_nodes_nb = {
2216         .notifier_call = cpuset_track_online_nodes,
2217         .priority = 10,         /* ??! */
2218 };
2219 
2220 /**
2221  * cpuset_init_smp - initialize cpus_allowed
2222  *
2223  * Description: Finish top cpuset after cpu, node maps are initialized
2224  */
2225 void __init cpuset_init_smp(void)
2226 {
2227         cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
2228         top_cpuset.mems_allowed = node_states[N_MEMORY];
2229         top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
2230 
2231         register_hotmemory_notifier(&cpuset_track_online_nodes_nb);
2232 }
2233 
2234 /**
2235  * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
2236  * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
2237  * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
2238  *
2239  * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
2240  * attached to the specified @tsk.  Guaranteed to return some non-empty
2241  * subset of cpu_online_mask, even if this means going outside the
2242  * tasks cpuset.
2243  **/
2244 
2245 void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
2246 {
2247         struct cpuset *cpus_cs;
2248 
2249         mutex_lock(&callback_mutex);
2250         rcu_read_lock();
2251         cpus_cs = effective_cpumask_cpuset(task_cs(tsk));
2252         guarantee_online_cpus(cpus_cs, pmask);
2253         rcu_read_unlock();
2254         mutex_unlock(&callback_mutex);
2255 }
2256 
2257 void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
2258 {
2259         struct cpuset *cpus_cs;
2260 
2261         rcu_read_lock();
2262         cpus_cs = effective_cpumask_cpuset(task_cs(tsk));
2263         do_set_cpus_allowed(tsk, cpus_cs->cpus_allowed);
2264         rcu_read_unlock();
2265 
2266         /*
2267          * We own tsk->cpus_allowed, nobody can change it under us.
2268          *
2269          * But we used cs && cs->cpus_allowed lockless and thus can
2270          * race with cgroup_attach_task() or update_cpumask() and get
2271          * the wrong tsk->cpus_allowed. However, both cases imply the
2272          * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
2273          * which takes task_rq_lock().
2274          *
2275          * If we are called after it dropped the lock we must see all
2276          * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
2277          * set any mask even if it is not right from task_cs() pov,
2278          * the pending set_cpus_allowed_ptr() will fix things.
2279          *
2280          * select_fallback_rq() will fix things ups and set cpu_possible_mask
2281          * if required.
2282          */
2283 }
2284 
2285 void cpuset_init_current_mems_allowed(void)
2286 {
2287         nodes_setall(current->mems_allowed);
2288 }
2289 
2290 /**
2291  * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
2292  * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
2293  *
2294  * Description: Returns the nodemask_t mems_allowed of the cpuset
2295  * attached to the specified @tsk.  Guaranteed to return some non-empty
2296  * subset of node_states[N_MEMORY], even if this means going outside the
2297  * tasks cpuset.
2298  **/
2299 
2300 nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
2301 {
2302         struct cpuset *mems_cs;
2303         nodemask_t mask;
2304 
2305         mutex_lock(&callback_mutex);
2306         rcu_read_lock();
2307         mems_cs = effective_nodemask_cpuset(task_cs(tsk));
2308         guarantee_online_mems(mems_cs, &mask);
2309         rcu_read_unlock();
2310         mutex_unlock(&callback_mutex);
2311 
2312         return mask;
2313 }
2314 
2315 /**
2316  * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
2317  * @nodemask: the nodemask to be checked
2318  *
2319  * Are any of the nodes in the nodemask allowed in current->mems_allowed?
2320  */
2321 int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
2322 {
2323         return nodes_intersects(*nodemask, current->mems_allowed);
2324 }
2325 
2326 /*
2327  * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
2328  * mem_hardwall ancestor to the specified cpuset.  Call holding
2329  * callback_mutex.  If no ancestor is mem_exclusive or mem_hardwall
2330  * (an unusual configuration), then returns the root cpuset.
2331  */
2332 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
2333 {
2334         while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
2335                 cs = parent_cs(cs);
2336         return cs;
2337 }
2338 
2339 /**
2340  * cpuset_node_allowed_softwall - Can we allocate on a memory node?
2341  * @node: is this an allowed node?
2342  * @gfp_mask: memory allocation flags
2343  *
2344  * If we're in interrupt, yes, we can always allocate.  If __GFP_THISNODE is
2345  * set, yes, we can always allocate.  If node is in our task's mems_allowed,
2346  * yes.  If it's not a __GFP_HARDWALL request and this node is in the nearest
2347  * hardwalled cpuset ancestor to this task's cpuset, yes.  If the task has been
2348  * OOM killed and has access to memory reserves as specified by the TIF_MEMDIE
2349  * flag, yes.
2350  * Otherwise, no.
2351  *
2352  * If __GFP_HARDWALL is set, cpuset_node_allowed_softwall() reduces to
2353  * cpuset_node_allowed_hardwall().  Otherwise, cpuset_node_allowed_softwall()
2354  * might sleep, and might allow a node from an enclosing cpuset.
2355  *
2356  * cpuset_node_allowed_hardwall() only handles the simpler case of hardwall
2357  * cpusets, and never sleeps.
2358  *
2359  * The __GFP_THISNODE placement logic is really handled elsewhere,
2360  * by forcibly using a zonelist starting at a specified node, and by
2361  * (in get_page_from_freelist()) refusing to consider the zones for
2362  * any node on the zonelist except the first.  By the time any such
2363  * calls get to this routine, we should just shut up and say 'yes'.
2364  *
2365  * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
2366  * and do not allow allocations outside the current tasks cpuset
2367  * unless the task has been OOM killed as is marked TIF_MEMDIE.
2368  * GFP_KERNEL allocations are not so marked, so can escape to the
2369  * nearest enclosing hardwalled ancestor cpuset.
2370  *
2371  * Scanning up parent cpusets requires callback_mutex.  The
2372  * __alloc_pages() routine only calls here with __GFP_HARDWALL bit
2373  * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
2374  * current tasks mems_allowed came up empty on the first pass over
2375  * the zonelist.  So only GFP_KERNEL allocations, if all nodes in the
2376  * cpuset are short of memory, might require taking the callback_mutex
2377  * mutex.
2378  *
2379  * The first call here from mm/page_alloc:get_page_from_freelist()
2380  * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
2381  * so no allocation on a node outside the cpuset is allowed (unless
2382  * in interrupt, of course).
2383  *
2384  * The second pass through get_page_from_freelist() doesn't even call
2385  * here for GFP_ATOMIC calls.  For those calls, the __alloc_pages()
2386  * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
2387  * in alloc_flags.  That logic and the checks below have the combined
2388  * affect that:
2389  *      in_interrupt - any node ok (current task context irrelevant)
2390  *      GFP_ATOMIC   - any node ok
2391  *      TIF_MEMDIE   - any node ok
2392  *      GFP_KERNEL   - any node in enclosing hardwalled cpuset ok
2393  *      GFP_USER     - only nodes in current tasks mems allowed ok.
2394  *
2395  * Rule:
2396  *    Don't call cpuset_node_allowed_softwall if you can't sleep, unless you
2397  *    pass in the __GFP_HARDWALL flag set in gfp_flag, which disables
2398  *    the code that might scan up ancestor cpusets and sleep.
2399  */
2400 int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
2401 {
2402         struct cpuset *cs;              /* current cpuset ancestors */
2403         int allowed;                    /* is allocation in zone z allowed? */
2404 
2405         if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
2406                 return 1;
2407         might_sleep_if(!(gfp_mask & __GFP_HARDWALL));
2408         if (node_isset(node, current->mems_allowed))
2409                 return 1;
2410         /*
2411          * Allow tasks that have access to memory reserves because they have
2412          * been OOM killed to get memory anywhere.
2413          */
2414         if (unlikely(test_thread_flag(TIF_MEMDIE)))
2415                 return 1;
2416         if (gfp_mask & __GFP_HARDWALL)  /* If hardwall request, stop here */
2417                 return 0;
2418 
2419         if (current->flags & PF_EXITING) /* Let dying task have memory */
2420                 return 1;
2421 
2422         /* Not hardwall and node outside mems_allowed: scan up cpusets */
2423         mutex_lock(&callback_mutex);
2424 
2425         rcu_read_lock();
2426         cs = nearest_hardwall_ancestor(task_cs(current));
2427         allowed = node_isset(node, cs->mems_allowed);
2428         rcu_read_unlock();
2429 
2430         mutex_unlock(&callback_mutex);
2431         return allowed;
2432 }
2433 
2434 /*
2435  * cpuset_node_allowed_hardwall - Can we allocate on a memory node?
2436  * @node: is this an allowed node?
2437  * @gfp_mask: memory allocation flags
2438  *
2439  * If we're in interrupt, yes, we can always allocate.  If __GFP_THISNODE is
2440  * set, yes, we can always allocate.  If node is in our task's mems_allowed,
2441  * yes.  If the task has been OOM killed and has access to memory reserves as
2442  * specified by the TIF_MEMDIE flag, yes.
2443  * Otherwise, no.
2444  *
2445  * The __GFP_THISNODE placement logic is really handled elsewhere,
2446  * by forcibly using a zonelist starting at a specified node, and by
2447  * (in get_page_from_freelist()) refusing to consider the zones for
2448  * any node on the zonelist except the first.  By the time any such
2449  * calls get to this routine, we should just shut up and say 'yes'.
2450  *
2451  * Unlike the cpuset_node_allowed_softwall() variant, above,
2452  * this variant requires that the node be in the current task's
2453  * mems_allowed or that we're in interrupt.  It does not scan up the
2454  * cpuset hierarchy for the nearest enclosing mem_exclusive cpuset.
2455  * It never sleeps.
2456  */
2457 int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
2458 {
2459         if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
2460                 return 1;
2461         if (node_isset(node, current->mems_allowed))
2462                 return 1;
2463         /*
2464          * Allow tasks that have access to memory reserves because they have
2465          * been OOM killed to get memory anywhere.
2466          */
2467         if (unlikely(test_thread_flag(TIF_MEMDIE)))
2468                 return 1;
2469         return 0;
2470 }
2471 
2472 /**
2473  * cpuset_mem_spread_node() - On which node to begin search for a file page
2474  * cpuset_slab_spread_node() - On which node to begin search for a slab page
2475  *
2476  * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
2477  * tasks in a cpuset with is_spread_page or is_spread_slab set),
2478  * and if the memory allocation used cpuset_mem_spread_node()
2479  * to determine on which node to start looking, as it will for
2480  * certain page cache or slab cache pages such as used for file
2481  * system buffers and inode caches, then instead of starting on the
2482  * local node to look for a free page, rather spread the starting
2483  * node around the tasks mems_allowed nodes.
2484  *
2485  * We don't have to worry about the returned node being offline
2486  * because "it can't happen", and even if it did, it would be ok.
2487  *
2488  * The routines calling guarantee_online_mems() are careful to
2489  * only set nodes in task->mems_allowed that are online.  So it
2490  * should not be possible for the following code to return an
2491  * offline node.  But if it did, that would be ok, as this routine
2492  * is not returning the node where the allocation must be, only
2493  * the node where the search should start.  The zonelist passed to
2494  * __alloc_pages() will include all nodes.  If the slab allocator
2495  * is passed an offline node, it will fall back to the local node.
2496  * See kmem_cache_alloc_node().
2497  */
2498 
2499 static int cpuset_spread_node(int *rotor)
2500 {
2501         int node;
2502 
2503         node = next_node(*rotor, current->mems_allowed);
2504         if (node == MAX_NUMNODES)
2505                 node = first_node(current->mems_allowed);
2506         *rotor = node;
2507         return node;
2508 }
2509 
2510 int cpuset_mem_spread_node(void)
2511 {
2512         if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
2513                 current->cpuset_mem_spread_rotor =
2514                         node_random(&current->mems_allowed);
2515 
2516         return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
2517 }
2518 
2519 int cpuset_slab_spread_node(void)
2520 {
2521         if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE)
2522                 current->cpuset_slab_spread_rotor =
2523                         node_random(&current->mems_allowed);
2524 
2525         return cpuset_spread_node(&current->cpuset_slab_spread_rotor);
2526 }
2527 
2528 EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);
2529 
2530 /**
2531  * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
2532  * @tsk1: pointer to task_struct of some task.
2533  * @tsk2: pointer to task_struct of some other task.
2534  *
2535  * Description: Return true if @tsk1's mems_allowed intersects the
2536  * mems_allowed of @tsk2.  Used by the OOM killer to determine if
2537  * one of the task's memory usage might impact the memory available
2538  * to the other.
2539  **/
2540 
2541 int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
2542                                    const struct task_struct *tsk2)
2543 {
2544         return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
2545 }
2546 
2547 #define CPUSET_NODELIST_LEN     (256)
2548 
2549 /**
2550  * cpuset_print_task_mems_allowed - prints task's cpuset and mems_allowed
2551  * @tsk: pointer to task_struct of some task.
2552  *
2553  * Description: Prints @task's name, cpuset name, and cached copy of its
2554  * mems_allowed to the kernel log.
2555  */
2556 void cpuset_print_task_mems_allowed(struct task_struct *tsk)
2557 {
2558          /* Statically allocated to prevent using excess stack. */
2559         static char cpuset_nodelist[CPUSET_NODELIST_LEN];
2560         static DEFINE_SPINLOCK(cpuset_buffer_lock);
2561         struct cgroup *cgrp;
2562 
2563         spin_lock(&cpuset_buffer_lock);
2564         rcu_read_lock();
2565 
2566         cgrp = task_cs(tsk)->css.cgroup;
2567         nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,
2568                            tsk->mems_allowed);
2569         pr_info("%s cpuset=", tsk->comm);
2570         pr_cont_cgroup_name(cgrp);
2571         pr_cont(" mems_allowed=%s\n", cpuset_nodelist);
2572 
2573         rcu_read_unlock();
2574         spin_unlock(&cpuset_buffer_lock);
2575 }
2576 
2577 /*
2578  * Collection of memory_pressure is suppressed unless
2579  * this flag is enabled by writing "1" to the special
2580  * cpuset file 'memory_pressure_enabled' in the root cpuset.
2581  */
2582 
2583 int cpuset_memory_pressure_enabled __read_mostly;
2584 
2585 /**
2586  * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
2587  *
2588  * Keep a running average of the rate of synchronous (direct)
2589  * page reclaim efforts initiated by tasks in each cpuset.
2590  *
2591  * This represents the rate at which some task in the cpuset
2592  * ran low on memory on all nodes it was allowed to use, and
2593  * had to enter the kernels page reclaim code in an effort to
2594  * create more free memory by tossing clean pages or swapping
2595  * or writing dirty pages.
2596  *
2597  * Display to user space in the per-cpuset read-only file
2598  * "memory_pressure".  Value displayed is an integer
2599  * representing the recent rate of entry into the synchronous
2600  * (direct) page reclaim by any task attached to the cpuset.
2601  **/
2602 
2603 void __cpuset_memory_pressure_bump(void)
2604 {
2605         rcu_read_lock();
2606         fmeter_markevent(&task_cs(current)->fmeter);
2607         rcu_read_unlock();
2608 }
2609 
2610 #ifdef CONFIG_PROC_PID_CPUSET
2611 /*
2612  * proc_cpuset_show()
2613  *  - Print tasks cpuset path into seq_file.
2614  *  - Used for /proc/<pid>/cpuset.
2615  *  - No need to task_lock(tsk) on this tsk->cpuset reference, as it
2616  *    doesn't really matter if tsk->cpuset changes after we read it,
2617  *    and we take cpuset_mutex, keeping cpuset_attach() from changing it
2618  *    anyway.
2619  */
2620 int proc_cpuset_show(struct seq_file *m, void *unused_v)
2621 {
2622         struct pid *pid;
2623         struct task_struct *tsk;
2624         char *buf, *p;
2625         struct cgroup_subsys_state *css;
2626         int retval;
2627 
2628         retval = -ENOMEM;
2629         buf = kmalloc(PATH_MAX, GFP_KERNEL);
2630         if (!buf)
2631                 goto out;
2632 
2633         retval = -ESRCH;
2634         pid = m->private;
2635         tsk = get_pid_task(pid, PIDTYPE_PID);
2636         if (!tsk)
2637                 goto out_free;
2638 
2639         retval = -ENAMETOOLONG;
2640         rcu_read_lock();
2641         css = task_css(tsk, cpuset_cgrp_id);
2642         p = cgroup_path(css->cgroup, buf, PATH_MAX);
2643         rcu_read_unlock();
2644         if (!p)
2645                 goto out_put_task;
2646         seq_puts(m, p);
2647         seq_putc(m, '\n');
2648         retval = 0;
2649 out_put_task:
2650         put_task_struct(tsk);
2651 out_free:
2652         kfree(buf);
2653 out:
2654         return retval;
2655 }
2656 #endif /* CONFIG_PROC_PID_CPUSET */
2657 
2658 /* Display task mems_allowed in /proc/<pid>/status file. */
2659 void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
2660 {
2661         seq_puts(m, "Mems_allowed:\t");
2662         seq_nodemask(m, &task->mems_allowed);
2663         seq_puts(m, "\n");
2664         seq_puts(m, "Mems_allowed_list:\t");
2665         seq_nodemask_list(m, &task->mems_allowed);
2666         seq_puts(m, "\n");
2667 }
2668 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us