Version:  2.0.40 2.2.26 2.4.37 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7

Linux/kernel/sched/fair.c

  1 /*
  2  * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
  3  *
  4  *  Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  5  *
  6  *  Interactivity improvements by Mike Galbraith
  7  *  (C) 2007 Mike Galbraith <efault@gmx.de>
  8  *
  9  *  Various enhancements by Dmitry Adamushko.
 10  *  (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
 11  *
 12  *  Group scheduling enhancements by Srivatsa Vaddagiri
 13  *  Copyright IBM Corporation, 2007
 14  *  Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
 15  *
 16  *  Scaled math optimizations by Thomas Gleixner
 17  *  Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
 18  *
 19  *  Adaptive scheduling granularity, math enhancements by Peter Zijlstra
 20  *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
 21  */
 22 
 23 #include <linux/sched.h>
 24 #include <linux/latencytop.h>
 25 #include <linux/cpumask.h>
 26 #include <linux/cpuidle.h>
 27 #include <linux/slab.h>
 28 #include <linux/profile.h>
 29 #include <linux/interrupt.h>
 30 #include <linux/mempolicy.h>
 31 #include <linux/migrate.h>
 32 #include <linux/task_work.h>
 33 
 34 #include <trace/events/sched.h>
 35 
 36 #include "sched.h"
 37 
 38 /*
 39  * Targeted preemption latency for CPU-bound tasks:
 40  * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
 41  *
 42  * NOTE: this latency value is not the same as the concept of
 43  * 'timeslice length' - timeslices in CFS are of variable length
 44  * and have no persistent notion like in traditional, time-slice
 45  * based scheduling concepts.
 46  *
 47  * (to see the precise effective timeslice length of your workload,
 48  *  run vmstat and monitor the context-switches (cs) field)
 49  */
 50 unsigned int sysctl_sched_latency = 6000000ULL;
 51 unsigned int normalized_sysctl_sched_latency = 6000000ULL;
 52 
 53 /*
 54  * The initial- and re-scaling of tunables is configurable
 55  * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
 56  *
 57  * Options are:
 58  * SCHED_TUNABLESCALING_NONE - unscaled, always *1
 59  * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
 60  * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
 61  */
 62 enum sched_tunable_scaling sysctl_sched_tunable_scaling
 63         = SCHED_TUNABLESCALING_LOG;
 64 
 65 /*
 66  * Minimal preemption granularity for CPU-bound tasks:
 67  * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
 68  */
 69 unsigned int sysctl_sched_min_granularity = 750000ULL;
 70 unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;
 71 
 72 /*
 73  * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
 74  */
 75 static unsigned int sched_nr_latency = 8;
 76 
 77 /*
 78  * After fork, child runs first. If set to 0 (default) then
 79  * parent will (try to) run first.
 80  */
 81 unsigned int sysctl_sched_child_runs_first __read_mostly;
 82 
 83 /*
 84  * SCHED_OTHER wake-up granularity.
 85  * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
 86  *
 87  * This option delays the preemption effects of decoupled workloads
 88  * and reduces their over-scheduling. Synchronous workloads will still
 89  * have immediate wakeup/sleep latencies.
 90  */
 91 unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
 92 unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
 93 
 94 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 95 
 96 /*
 97  * The exponential sliding  window over which load is averaged for shares
 98  * distribution.
 99  * (default: 10msec)
100  */
101 unsigned int __read_mostly sysctl_sched_shares_window = 10000000UL;
102 
103 #ifdef CONFIG_CFS_BANDWIDTH
104 /*
105  * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
106  * each time a cfs_rq requests quota.
107  *
108  * Note: in the case that the slice exceeds the runtime remaining (either due
109  * to consumption or the quota being specified to be smaller than the slice)
110  * we will always only issue the remaining available time.
111  *
112  * default: 5 msec, units: microseconds
113   */
114 unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
115 #endif
116 
117 static inline void update_load_add(struct load_weight *lw, unsigned long inc)
118 {
119         lw->weight += inc;
120         lw->inv_weight = 0;
121 }
122 
123 static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
124 {
125         lw->weight -= dec;
126         lw->inv_weight = 0;
127 }
128 
129 static inline void update_load_set(struct load_weight *lw, unsigned long w)
130 {
131         lw->weight = w;
132         lw->inv_weight = 0;
133 }
134 
135 /*
136  * Increase the granularity value when there are more CPUs,
137  * because with more CPUs the 'effective latency' as visible
138  * to users decreases. But the relationship is not linear,
139  * so pick a second-best guess by going with the log2 of the
140  * number of CPUs.
141  *
142  * This idea comes from the SD scheduler of Con Kolivas:
143  */
144 static unsigned int get_update_sysctl_factor(void)
145 {
146         unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
147         unsigned int factor;
148 
149         switch (sysctl_sched_tunable_scaling) {
150         case SCHED_TUNABLESCALING_NONE:
151                 factor = 1;
152                 break;
153         case SCHED_TUNABLESCALING_LINEAR:
154                 factor = cpus;
155                 break;
156         case SCHED_TUNABLESCALING_LOG:
157         default:
158                 factor = 1 + ilog2(cpus);
159                 break;
160         }
161 
162         return factor;
163 }
164 
165 static void update_sysctl(void)
166 {
167         unsigned int factor = get_update_sysctl_factor();
168 
169 #define SET_SYSCTL(name) \
170         (sysctl_##name = (factor) * normalized_sysctl_##name)
171         SET_SYSCTL(sched_min_granularity);
172         SET_SYSCTL(sched_latency);
173         SET_SYSCTL(sched_wakeup_granularity);
174 #undef SET_SYSCTL
175 }
176 
177 void sched_init_granularity(void)
178 {
179         update_sysctl();
180 }
181 
182 #define WMULT_CONST     (~0U)
183 #define WMULT_SHIFT     32
184 
185 static void __update_inv_weight(struct load_weight *lw)
186 {
187         unsigned long w;
188 
189         if (likely(lw->inv_weight))
190                 return;
191 
192         w = scale_load_down(lw->weight);
193 
194         if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
195                 lw->inv_weight = 1;
196         else if (unlikely(!w))
197                 lw->inv_weight = WMULT_CONST;
198         else
199                 lw->inv_weight = WMULT_CONST / w;
200 }
201 
202 /*
203  * delta_exec * weight / lw.weight
204  *   OR
205  * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
206  *
207  * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case
208  * we're guaranteed shift stays positive because inv_weight is guaranteed to
209  * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
210  *
211  * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
212  * weight/lw.weight <= 1, and therefore our shift will also be positive.
213  */
214 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
215 {
216         u64 fact = scale_load_down(weight);
217         int shift = WMULT_SHIFT;
218 
219         __update_inv_weight(lw);
220 
221         if (unlikely(fact >> 32)) {
222                 while (fact >> 32) {
223                         fact >>= 1;
224                         shift--;
225                 }
226         }
227 
228         /* hint to use a 32x32->64 mul */
229         fact = (u64)(u32)fact * lw->inv_weight;
230 
231         while (fact >> 32) {
232                 fact >>= 1;
233                 shift--;
234         }
235 
236         return mul_u64_u32_shr(delta_exec, fact, shift);
237 }
238 
239 
240 const struct sched_class fair_sched_class;
241 
242 /**************************************************************
243  * CFS operations on generic schedulable entities:
244  */
245 
246 #ifdef CONFIG_FAIR_GROUP_SCHED
247 
248 /* cpu runqueue to which this cfs_rq is attached */
249 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
250 {
251         return cfs_rq->rq;
252 }
253 
254 /* An entity is a task if it doesn't "own" a runqueue */
255 #define entity_is_task(se)      (!se->my_q)
256 
257 static inline struct task_struct *task_of(struct sched_entity *se)
258 {
259 #ifdef CONFIG_SCHED_DEBUG
260         WARN_ON_ONCE(!entity_is_task(se));
261 #endif
262         return container_of(se, struct task_struct, se);
263 }
264 
265 /* Walk up scheduling entities hierarchy */
266 #define for_each_sched_entity(se) \
267                 for (; se; se = se->parent)
268 
269 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
270 {
271         return p->se.cfs_rq;
272 }
273 
274 /* runqueue on which this entity is (to be) queued */
275 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
276 {
277         return se->cfs_rq;
278 }
279 
280 /* runqueue "owned" by this group */
281 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
282 {
283         return grp->my_q;
284 }
285 
286 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
287 {
288         if (!cfs_rq->on_list) {
289                 /*
290                  * Ensure we either appear before our parent (if already
291                  * enqueued) or force our parent to appear after us when it is
292                  * enqueued.  The fact that we always enqueue bottom-up
293                  * reduces this to two cases.
294                  */
295                 if (cfs_rq->tg->parent &&
296                     cfs_rq->tg->parent->cfs_rq[cpu_of(rq_of(cfs_rq))]->on_list) {
297                         list_add_rcu(&cfs_rq->leaf_cfs_rq_list,
298                                 &rq_of(cfs_rq)->leaf_cfs_rq_list);
299                 } else {
300                         list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
301                                 &rq_of(cfs_rq)->leaf_cfs_rq_list);
302                 }
303 
304                 cfs_rq->on_list = 1;
305         }
306 }
307 
308 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
309 {
310         if (cfs_rq->on_list) {
311                 list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
312                 cfs_rq->on_list = 0;
313         }
314 }
315 
316 /* Iterate thr' all leaf cfs_rq's on a runqueue */
317 #define for_each_leaf_cfs_rq(rq, cfs_rq) \
318         list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list)
319 
320 /* Do the two (enqueued) entities belong to the same group ? */
321 static inline struct cfs_rq *
322 is_same_group(struct sched_entity *se, struct sched_entity *pse)
323 {
324         if (se->cfs_rq == pse->cfs_rq)
325                 return se->cfs_rq;
326 
327         return NULL;
328 }
329 
330 static inline struct sched_entity *parent_entity(struct sched_entity *se)
331 {
332         return se->parent;
333 }
334 
335 static void
336 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
337 {
338         int se_depth, pse_depth;
339 
340         /*
341          * preemption test can be made between sibling entities who are in the
342          * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
343          * both tasks until we find their ancestors who are siblings of common
344          * parent.
345          */
346 
347         /* First walk up until both entities are at same depth */
348         se_depth = (*se)->depth;
349         pse_depth = (*pse)->depth;
350 
351         while (se_depth > pse_depth) {
352                 se_depth--;
353                 *se = parent_entity(*se);
354         }
355 
356         while (pse_depth > se_depth) {
357                 pse_depth--;
358                 *pse = parent_entity(*pse);
359         }
360 
361         while (!is_same_group(*se, *pse)) {
362                 *se = parent_entity(*se);
363                 *pse = parent_entity(*pse);
364         }
365 }
366 
367 #else   /* !CONFIG_FAIR_GROUP_SCHED */
368 
369 static inline struct task_struct *task_of(struct sched_entity *se)
370 {
371         return container_of(se, struct task_struct, se);
372 }
373 
374 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
375 {
376         return container_of(cfs_rq, struct rq, cfs);
377 }
378 
379 #define entity_is_task(se)      1
380 
381 #define for_each_sched_entity(se) \
382                 for (; se; se = NULL)
383 
384 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p)
385 {
386         return &task_rq(p)->cfs;
387 }
388 
389 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se)
390 {
391         struct task_struct *p = task_of(se);
392         struct rq *rq = task_rq(p);
393 
394         return &rq->cfs;
395 }
396 
397 /* runqueue "owned" by this group */
398 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp)
399 {
400         return NULL;
401 }
402 
403 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
404 {
405 }
406 
407 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
408 {
409 }
410 
411 #define for_each_leaf_cfs_rq(rq, cfs_rq) \
412                 for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL)
413 
414 static inline struct sched_entity *parent_entity(struct sched_entity *se)
415 {
416         return NULL;
417 }
418 
419 static inline void
420 find_matching_se(struct sched_entity **se, struct sched_entity **pse)
421 {
422 }
423 
424 #endif  /* CONFIG_FAIR_GROUP_SCHED */
425 
426 static __always_inline
427 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
428 
429 /**************************************************************
430  * Scheduling class tree data structure manipulation methods:
431  */
432 
433 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
434 {
435         s64 delta = (s64)(vruntime - max_vruntime);
436         if (delta > 0)
437                 max_vruntime = vruntime;
438 
439         return max_vruntime;
440 }
441 
442 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
443 {
444         s64 delta = (s64)(vruntime - min_vruntime);
445         if (delta < 0)
446                 min_vruntime = vruntime;
447 
448         return min_vruntime;
449 }
450 
451 static inline int entity_before(struct sched_entity *a,
452                                 struct sched_entity *b)
453 {
454         return (s64)(a->vruntime - b->vruntime) < 0;
455 }
456 
457 static void update_min_vruntime(struct cfs_rq *cfs_rq)
458 {
459         u64 vruntime = cfs_rq->min_vruntime;
460 
461         if (cfs_rq->curr)
462                 vruntime = cfs_rq->curr->vruntime;
463 
464         if (cfs_rq->rb_leftmost) {
465                 struct sched_entity *se = rb_entry(cfs_rq->rb_leftmost,
466                                                    struct sched_entity,
467                                                    run_node);
468 
469                 if (!cfs_rq->curr)
470                         vruntime = se->vruntime;
471                 else
472                         vruntime = min_vruntime(vruntime, se->vruntime);
473         }
474 
475         /* ensure we never gain time by being placed backwards. */
476         cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
477 #ifndef CONFIG_64BIT
478         smp_wmb();
479         cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
480 #endif
481 }
482 
483 /*
484  * Enqueue an entity into the rb-tree:
485  */
486 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
487 {
488         struct rb_node **link = &cfs_rq->tasks_timeline.rb_node;
489         struct rb_node *parent = NULL;
490         struct sched_entity *entry;
491         int leftmost = 1;
492 
493         /*
494          * Find the right place in the rbtree:
495          */
496         while (*link) {
497                 parent = *link;
498                 entry = rb_entry(parent, struct sched_entity, run_node);
499                 /*
500                  * We dont care about collisions. Nodes with
501                  * the same key stay together.
502                  */
503                 if (entity_before(se, entry)) {
504                         link = &parent->rb_left;
505                 } else {
506                         link = &parent->rb_right;
507                         leftmost = 0;
508                 }
509         }
510 
511         /*
512          * Maintain a cache of leftmost tree entries (it is frequently
513          * used):
514          */
515         if (leftmost)
516                 cfs_rq->rb_leftmost = &se->run_node;
517 
518         rb_link_node(&se->run_node, parent, link);
519         rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline);
520 }
521 
522 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
523 {
524         if (cfs_rq->rb_leftmost == &se->run_node) {
525                 struct rb_node *next_node;
526 
527                 next_node = rb_next(&se->run_node);
528                 cfs_rq->rb_leftmost = next_node;
529         }
530 
531         rb_erase(&se->run_node, &cfs_rq->tasks_timeline);
532 }
533 
534 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
535 {
536         struct rb_node *left = cfs_rq->rb_leftmost;
537 
538         if (!left)
539                 return NULL;
540 
541         return rb_entry(left, struct sched_entity, run_node);
542 }
543 
544 static struct sched_entity *__pick_next_entity(struct sched_entity *se)
545 {
546         struct rb_node *next = rb_next(&se->run_node);
547 
548         if (!next)
549                 return NULL;
550 
551         return rb_entry(next, struct sched_entity, run_node);
552 }
553 
554 #ifdef CONFIG_SCHED_DEBUG
555 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
556 {
557         struct rb_node *last = rb_last(&cfs_rq->tasks_timeline);
558 
559         if (!last)
560                 return NULL;
561 
562         return rb_entry(last, struct sched_entity, run_node);
563 }
564 
565 /**************************************************************
566  * Scheduling class statistics methods:
567  */
568 
569 int sched_proc_update_handler(struct ctl_table *table, int write,
570                 void __user *buffer, size_t *lenp,
571                 loff_t *ppos)
572 {
573         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
574         unsigned int factor = get_update_sysctl_factor();
575 
576         if (ret || !write)
577                 return ret;
578 
579         sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
580                                         sysctl_sched_min_granularity);
581 
582 #define WRT_SYSCTL(name) \
583         (normalized_sysctl_##name = sysctl_##name / (factor))
584         WRT_SYSCTL(sched_min_granularity);
585         WRT_SYSCTL(sched_latency);
586         WRT_SYSCTL(sched_wakeup_granularity);
587 #undef WRT_SYSCTL
588 
589         return 0;
590 }
591 #endif
592 
593 /*
594  * delta /= w
595  */
596 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
597 {
598         if (unlikely(se->load.weight != NICE_0_LOAD))
599                 delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
600 
601         return delta;
602 }
603 
604 /*
605  * The idea is to set a period in which each task runs once.
606  *
607  * When there are too many tasks (sched_nr_latency) we have to stretch
608  * this period because otherwise the slices get too small.
609  *
610  * p = (nr <= nl) ? l : l*nr/nl
611  */
612 static u64 __sched_period(unsigned long nr_running)
613 {
614         if (unlikely(nr_running > sched_nr_latency))
615                 return nr_running * sysctl_sched_min_granularity;
616         else
617                 return sysctl_sched_latency;
618 }
619 
620 /*
621  * We calculate the wall-time slice from the period by taking a part
622  * proportional to the weight.
623  *
624  * s = p*P[w/rw]
625  */
626 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
627 {
628         u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq);
629 
630         for_each_sched_entity(se) {
631                 struct load_weight *load;
632                 struct load_weight lw;
633 
634                 cfs_rq = cfs_rq_of(se);
635                 load = &cfs_rq->load;
636 
637                 if (unlikely(!se->on_rq)) {
638                         lw = cfs_rq->load;
639 
640                         update_load_add(&lw, se->load.weight);
641                         load = &lw;
642                 }
643                 slice = __calc_delta(slice, se->load.weight, load);
644         }
645         return slice;
646 }
647 
648 /*
649  * We calculate the vruntime slice of a to-be-inserted task.
650  *
651  * vs = s/w
652  */
653 static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
654 {
655         return calc_delta_fair(sched_slice(cfs_rq, se), se);
656 }
657 
658 #ifdef CONFIG_SMP
659 static int select_idle_sibling(struct task_struct *p, int cpu);
660 static unsigned long task_h_load(struct task_struct *p);
661 
662 /*
663  * We choose a half-life close to 1 scheduling period.
664  * Note: The tables runnable_avg_yN_inv and runnable_avg_yN_sum are
665  * dependent on this value.
666  */
667 #define LOAD_AVG_PERIOD 32
668 #define LOAD_AVG_MAX 47742 /* maximum possible load avg */
669 #define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_AVG_MAX */
670 
671 /* Give new sched_entity start runnable values to heavy its load in infant time */
672 void init_entity_runnable_average(struct sched_entity *se)
673 {
674         struct sched_avg *sa = &se->avg;
675 
676         sa->last_update_time = 0;
677         /*
678          * sched_avg's period_contrib should be strictly less then 1024, so
679          * we give it 1023 to make sure it is almost a period (1024us), and
680          * will definitely be update (after enqueue).
681          */
682         sa->period_contrib = 1023;
683         sa->load_avg = scale_load_down(se->load.weight);
684         sa->load_sum = sa->load_avg * LOAD_AVG_MAX;
685         /*
686          * At this point, util_avg won't be used in select_task_rq_fair anyway
687          */
688         sa->util_avg = 0;
689         sa->util_sum = 0;
690         /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
691 }
692 
693 /*
694  * With new tasks being created, their initial util_avgs are extrapolated
695  * based on the cfs_rq's current util_avg:
696  *
697  *   util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight
698  *
699  * However, in many cases, the above util_avg does not give a desired
700  * value. Moreover, the sum of the util_avgs may be divergent, such
701  * as when the series is a harmonic series.
702  *
703  * To solve this problem, we also cap the util_avg of successive tasks to
704  * only 1/2 of the left utilization budget:
705  *
706  *   util_avg_cap = (1024 - cfs_rq->avg.util_avg) / 2^n
707  *
708  * where n denotes the nth task.
709  *
710  * For example, a simplest series from the beginning would be like:
711  *
712  *  task  util_avg: 512, 256, 128,  64,  32,   16,    8, ...
713  * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ...
714  *
715  * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap)
716  * if util_avg > util_avg_cap.
717  */
718 void post_init_entity_util_avg(struct sched_entity *se)
719 {
720         struct cfs_rq *cfs_rq = cfs_rq_of(se);
721         struct sched_avg *sa = &se->avg;
722         long cap = (long)(SCHED_CAPACITY_SCALE - cfs_rq->avg.util_avg) / 2;
723 
724         if (cap > 0) {
725                 if (cfs_rq->avg.util_avg != 0) {
726                         sa->util_avg  = cfs_rq->avg.util_avg * se->load.weight;
727                         sa->util_avg /= (cfs_rq->avg.load_avg + 1);
728 
729                         if (sa->util_avg > cap)
730                                 sa->util_avg = cap;
731                 } else {
732                         sa->util_avg = cap;
733                 }
734                 sa->util_sum = sa->util_avg * LOAD_AVG_MAX;
735         }
736 }
737 
738 #else
739 void init_entity_runnable_average(struct sched_entity *se)
740 {
741 }
742 void post_init_entity_util_avg(struct sched_entity *se)
743 {
744 }
745 #endif
746 
747 /*
748  * Update the current task's runtime statistics.
749  */
750 static void update_curr(struct cfs_rq *cfs_rq)
751 {
752         struct sched_entity *curr = cfs_rq->curr;
753         u64 now = rq_clock_task(rq_of(cfs_rq));
754         u64 delta_exec;
755 
756         if (unlikely(!curr))
757                 return;
758 
759         delta_exec = now - curr->exec_start;
760         if (unlikely((s64)delta_exec <= 0))
761                 return;
762 
763         curr->exec_start = now;
764 
765         schedstat_set(curr->statistics.exec_max,
766                       max(delta_exec, curr->statistics.exec_max));
767 
768         curr->sum_exec_runtime += delta_exec;
769         schedstat_add(cfs_rq, exec_clock, delta_exec);
770 
771         curr->vruntime += calc_delta_fair(delta_exec, curr);
772         update_min_vruntime(cfs_rq);
773 
774         if (entity_is_task(curr)) {
775                 struct task_struct *curtask = task_of(curr);
776 
777                 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
778                 cpuacct_charge(curtask, delta_exec);
779                 account_group_exec_runtime(curtask, delta_exec);
780         }
781 
782         account_cfs_rq_runtime(cfs_rq, delta_exec);
783 }
784 
785 static void update_curr_fair(struct rq *rq)
786 {
787         update_curr(cfs_rq_of(&rq->curr->se));
788 }
789 
790 #ifdef CONFIG_SCHEDSTATS
791 static inline void
792 update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
793 {
794         u64 wait_start = rq_clock(rq_of(cfs_rq));
795 
796         if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) &&
797             likely(wait_start > se->statistics.wait_start))
798                 wait_start -= se->statistics.wait_start;
799 
800         se->statistics.wait_start = wait_start;
801 }
802 
803 static void
804 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
805 {
806         struct task_struct *p;
807         u64 delta;
808 
809         delta = rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start;
810 
811         if (entity_is_task(se)) {
812                 p = task_of(se);
813                 if (task_on_rq_migrating(p)) {
814                         /*
815                          * Preserve migrating task's wait time so wait_start
816                          * time stamp can be adjusted to accumulate wait time
817                          * prior to migration.
818                          */
819                         se->statistics.wait_start = delta;
820                         return;
821                 }
822                 trace_sched_stat_wait(p, delta);
823         }
824 
825         se->statistics.wait_max = max(se->statistics.wait_max, delta);
826         se->statistics.wait_count++;
827         se->statistics.wait_sum += delta;
828         se->statistics.wait_start = 0;
829 }
830 
831 /*
832  * Task is being enqueued - update stats:
833  */
834 static inline void
835 update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
836 {
837         /*
838          * Are we enqueueing a waiting task? (for current tasks
839          * a dequeue/enqueue event is a NOP)
840          */
841         if (se != cfs_rq->curr)
842                 update_stats_wait_start(cfs_rq, se);
843 }
844 
845 static inline void
846 update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
847 {
848         /*
849          * Mark the end of the wait period if dequeueing a
850          * waiting task:
851          */
852         if (se != cfs_rq->curr)
853                 update_stats_wait_end(cfs_rq, se);
854 
855         if (flags & DEQUEUE_SLEEP) {
856                 if (entity_is_task(se)) {
857                         struct task_struct *tsk = task_of(se);
858 
859                         if (tsk->state & TASK_INTERRUPTIBLE)
860                                 se->statistics.sleep_start = rq_clock(rq_of(cfs_rq));
861                         if (tsk->state & TASK_UNINTERRUPTIBLE)
862                                 se->statistics.block_start = rq_clock(rq_of(cfs_rq));
863                 }
864         }
865 
866 }
867 #else
868 static inline void
869 update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
870 {
871 }
872 
873 static inline void
874 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
875 {
876 }
877 
878 static inline void
879 update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
880 {
881 }
882 
883 static inline void
884 update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
885 {
886 }
887 #endif
888 
889 /*
890  * We are picking a new current task - update its stats:
891  */
892 static inline void
893 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
894 {
895         /*
896          * We are starting a new run period:
897          */
898         se->exec_start = rq_clock_task(rq_of(cfs_rq));
899 }
900 
901 /**************************************************
902  * Scheduling class queueing methods:
903  */
904 
905 #ifdef CONFIG_NUMA_BALANCING
906 /*
907  * Approximate time to scan a full NUMA task in ms. The task scan period is
908  * calculated based on the tasks virtual memory size and
909  * numa_balancing_scan_size.
910  */
911 unsigned int sysctl_numa_balancing_scan_period_min = 1000;
912 unsigned int sysctl_numa_balancing_scan_period_max = 60000;
913 
914 /* Portion of address space to scan in MB */
915 unsigned int sysctl_numa_balancing_scan_size = 256;
916 
917 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
918 unsigned int sysctl_numa_balancing_scan_delay = 1000;
919 
920 static unsigned int task_nr_scan_windows(struct task_struct *p)
921 {
922         unsigned long rss = 0;
923         unsigned long nr_scan_pages;
924 
925         /*
926          * Calculations based on RSS as non-present and empty pages are skipped
927          * by the PTE scanner and NUMA hinting faults should be trapped based
928          * on resident pages
929          */
930         nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
931         rss = get_mm_rss(p->mm);
932         if (!rss)
933                 rss = nr_scan_pages;
934 
935         rss = round_up(rss, nr_scan_pages);
936         return rss / nr_scan_pages;
937 }
938 
939 /* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
940 #define MAX_SCAN_WINDOW 2560
941 
942 static unsigned int task_scan_min(struct task_struct *p)
943 {
944         unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
945         unsigned int scan, floor;
946         unsigned int windows = 1;
947 
948         if (scan_size < MAX_SCAN_WINDOW)
949                 windows = MAX_SCAN_WINDOW / scan_size;
950         floor = 1000 / windows;
951 
952         scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
953         return max_t(unsigned int, floor, scan);
954 }
955 
956 static unsigned int task_scan_max(struct task_struct *p)
957 {
958         unsigned int smin = task_scan_min(p);
959         unsigned int smax;
960 
961         /* Watch for min being lower than max due to floor calculations */
962         smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
963         return max(smin, smax);
964 }
965 
966 static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
967 {
968         rq->nr_numa_running += (p->numa_preferred_nid != -1);
969         rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
970 }
971 
972 static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
973 {
974         rq->nr_numa_running -= (p->numa_preferred_nid != -1);
975         rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
976 }
977 
978 struct numa_group {
979         atomic_t refcount;
980 
981         spinlock_t lock; /* nr_tasks, tasks */
982         int nr_tasks;
983         pid_t gid;
984         int active_nodes;
985 
986         struct rcu_head rcu;
987         unsigned long total_faults;
988         unsigned long max_faults_cpu;
989         /*
990          * Faults_cpu is used to decide whether memory should move
991          * towards the CPU. As a consequence, these stats are weighted
992          * more by CPU use than by memory faults.
993          */
994         unsigned long *faults_cpu;
995         unsigned long faults[0];
996 };
997 
998 /* Shared or private faults. */
999 #define NR_NUMA_HINT_FAULT_TYPES 2
1000 
1001 /* Memory and CPU locality */
1002 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
1003 
1004 /* Averaged statistics, and temporary buffers. */
1005 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
1006 
1007 pid_t task_numa_group_id(struct task_struct *p)
1008 {
1009         return p->numa_group ? p->numa_group->gid : 0;
1010 }
1011 
1012 /*
1013  * The averaged statistics, shared & private, memory & cpu,
1014  * occupy the first half of the array. The second half of the
1015  * array is for current counters, which are averaged into the
1016  * first set by task_numa_placement.
1017  */
1018 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
1019 {
1020         return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
1021 }
1022 
1023 static inline unsigned long task_faults(struct task_struct *p, int nid)
1024 {
1025         if (!p->numa_faults)
1026                 return 0;
1027 
1028         return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1029                 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];
1030 }
1031 
1032 static inline unsigned long group_faults(struct task_struct *p, int nid)
1033 {
1034         if (!p->numa_group)
1035                 return 0;
1036 
1037         return p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1038                 p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 1)];
1039 }
1040 
1041 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
1042 {
1043         return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] +
1044                 group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)];
1045 }
1046 
1047 /*
1048  * A node triggering more than 1/3 as many NUMA faults as the maximum is
1049  * considered part of a numa group's pseudo-interleaving set. Migrations
1050  * between these nodes are slowed down, to allow things to settle down.
1051  */
1052 #define ACTIVE_NODE_FRACTION 3
1053 
1054 static bool numa_is_active_node(int nid, struct numa_group *ng)
1055 {
1056         return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
1057 }
1058 
1059 /* Handle placement on systems where not all nodes are directly connected. */
1060 static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
1061                                         int maxdist, bool task)
1062 {
1063         unsigned long score = 0;
1064         int node;
1065 
1066         /*
1067          * All nodes are directly connected, and the same distance
1068          * from each other. No need for fancy placement algorithms.
1069          */
1070         if (sched_numa_topology_type == NUMA_DIRECT)
1071                 return 0;
1072 
1073         /*
1074          * This code is called for each node, introducing N^2 complexity,
1075          * which should be ok given the number of nodes rarely exceeds 8.
1076          */
1077         for_each_online_node(node) {
1078                 unsigned long faults;
1079                 int dist = node_distance(nid, node);
1080 
1081                 /*
1082                  * The furthest away nodes in the system are not interesting
1083                  * for placement; nid was already counted.
1084                  */
1085                 if (dist == sched_max_numa_distance || node == nid)
1086                         continue;
1087 
1088                 /*
1089                  * On systems with a backplane NUMA topology, compare groups
1090                  * of nodes, and move tasks towards the group with the most
1091                  * memory accesses. When comparing two nodes at distance
1092                  * "hoplimit", only nodes closer by than "hoplimit" are part
1093                  * of each group. Skip other nodes.
1094                  */
1095                 if (sched_numa_topology_type == NUMA_BACKPLANE &&
1096                                         dist > maxdist)
1097                         continue;
1098 
1099                 /* Add up the faults from nearby nodes. */
1100                 if (task)
1101                         faults = task_faults(p, node);
1102                 else
1103                         faults = group_faults(p, node);
1104 
1105                 /*
1106                  * On systems with a glueless mesh NUMA topology, there are
1107                  * no fixed "groups of nodes". Instead, nodes that are not
1108                  * directly connected bounce traffic through intermediate
1109                  * nodes; a numa_group can occupy any set of nodes.
1110                  * The further away a node is, the less the faults count.
1111                  * This seems to result in good task placement.
1112                  */
1113                 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
1114                         faults *= (sched_max_numa_distance - dist);
1115                         faults /= (sched_max_numa_distance - LOCAL_DISTANCE);
1116                 }
1117 
1118                 score += faults;
1119         }
1120 
1121         return score;
1122 }
1123 
1124 /*
1125  * These return the fraction of accesses done by a particular task, or
1126  * task group, on a particular numa node.  The group weight is given a
1127  * larger multiplier, in order to group tasks together that are almost
1128  * evenly spread out between numa nodes.
1129  */
1130 static inline unsigned long task_weight(struct task_struct *p, int nid,
1131                                         int dist)
1132 {
1133         unsigned long faults, total_faults;
1134 
1135         if (!p->numa_faults)
1136                 return 0;
1137 
1138         total_faults = p->total_numa_faults;
1139 
1140         if (!total_faults)
1141                 return 0;
1142 
1143         faults = task_faults(p, nid);
1144         faults += score_nearby_nodes(p, nid, dist, true);
1145 
1146         return 1000 * faults / total_faults;
1147 }
1148 
1149 static inline unsigned long group_weight(struct task_struct *p, int nid,
1150                                          int dist)
1151 {
1152         unsigned long faults, total_faults;
1153 
1154         if (!p->numa_group)
1155                 return 0;
1156 
1157         total_faults = p->numa_group->total_faults;
1158 
1159         if (!total_faults)
1160                 return 0;
1161 
1162         faults = group_faults(p, nid);
1163         faults += score_nearby_nodes(p, nid, dist, false);
1164 
1165         return 1000 * faults / total_faults;
1166 }
1167 
1168 bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
1169                                 int src_nid, int dst_cpu)
1170 {
1171         struct numa_group *ng = p->numa_group;
1172         int dst_nid = cpu_to_node(dst_cpu);
1173         int last_cpupid, this_cpupid;
1174 
1175         this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
1176 
1177         /*
1178          * Multi-stage node selection is used in conjunction with a periodic
1179          * migration fault to build a temporal task<->page relation. By using
1180          * a two-stage filter we remove short/unlikely relations.
1181          *
1182          * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
1183          * a task's usage of a particular page (n_p) per total usage of this
1184          * page (n_t) (in a given time-span) to a probability.
1185          *
1186          * Our periodic faults will sample this probability and getting the
1187          * same result twice in a row, given these samples are fully
1188          * independent, is then given by P(n)^2, provided our sample period
1189          * is sufficiently short compared to the usage pattern.
1190          *
1191          * This quadric squishes small probabilities, making it less likely we
1192          * act on an unlikely task<->page relation.
1193          */
1194         last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
1195         if (!cpupid_pid_unset(last_cpupid) &&
1196                                 cpupid_to_nid(last_cpupid) != dst_nid)
1197                 return false;
1198 
1199         /* Always allow migrate on private faults */
1200         if (cpupid_match_pid(p, last_cpupid))
1201                 return true;
1202 
1203         /* A shared fault, but p->numa_group has not been set up yet. */
1204         if (!ng)
1205                 return true;
1206 
1207         /*
1208          * Destination node is much more heavily used than the source
1209          * node? Allow migration.
1210          */
1211         if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) *
1212                                         ACTIVE_NODE_FRACTION)
1213                 return true;
1214 
1215         /*
1216          * Distribute memory according to CPU & memory use on each node,
1217          * with 3/4 hysteresis to avoid unnecessary memory migrations:
1218          *
1219          * faults_cpu(dst)   3   faults_cpu(src)
1220          * --------------- * - > ---------------
1221          * faults_mem(dst)   4   faults_mem(src)
1222          */
1223         return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 >
1224                group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4;
1225 }
1226 
1227 static unsigned long weighted_cpuload(const int cpu);
1228 static unsigned long source_load(int cpu, int type);
1229 static unsigned long target_load(int cpu, int type);
1230 static unsigned long capacity_of(int cpu);
1231 static long effective_load(struct task_group *tg, int cpu, long wl, long wg);
1232 
1233 /* Cached statistics for all CPUs within a node */
1234 struct numa_stats {
1235         unsigned long nr_running;
1236         unsigned long load;
1237 
1238         /* Total compute capacity of CPUs on a node */
1239         unsigned long compute_capacity;
1240 
1241         /* Approximate capacity in terms of runnable tasks on a node */
1242         unsigned long task_capacity;
1243         int has_free_capacity;
1244 };
1245 
1246 /*
1247  * XXX borrowed from update_sg_lb_stats
1248  */
1249 static void update_numa_stats(struct numa_stats *ns, int nid)
1250 {
1251         int smt, cpu, cpus = 0;
1252         unsigned long capacity;
1253 
1254         memset(ns, 0, sizeof(*ns));
1255         for_each_cpu(cpu, cpumask_of_node(nid)) {
1256                 struct rq *rq = cpu_rq(cpu);
1257 
1258                 ns->nr_running += rq->nr_running;
1259                 ns->load += weighted_cpuload(cpu);
1260                 ns->compute_capacity += capacity_of(cpu);
1261 
1262                 cpus++;
1263         }
1264 
1265         /*
1266          * If we raced with hotplug and there are no CPUs left in our mask
1267          * the @ns structure is NULL'ed and task_numa_compare() will
1268          * not find this node attractive.
1269          *
1270          * We'll either bail at !has_free_capacity, or we'll detect a huge
1271          * imbalance and bail there.
1272          */
1273         if (!cpus)
1274                 return;
1275 
1276         /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */
1277         smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity);
1278         capacity = cpus / smt; /* cores */
1279 
1280         ns->task_capacity = min_t(unsigned, capacity,
1281                 DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE));
1282         ns->has_free_capacity = (ns->nr_running < ns->task_capacity);
1283 }
1284 
1285 struct task_numa_env {
1286         struct task_struct *p;
1287 
1288         int src_cpu, src_nid;
1289         int dst_cpu, dst_nid;
1290 
1291         struct numa_stats src_stats, dst_stats;
1292 
1293         int imbalance_pct;
1294         int dist;
1295 
1296         struct task_struct *best_task;
1297         long best_imp;
1298         int best_cpu;
1299 };
1300 
1301 static void task_numa_assign(struct task_numa_env *env,
1302                              struct task_struct *p, long imp)
1303 {
1304         if (env->best_task)
1305                 put_task_struct(env->best_task);
1306 
1307         env->best_task = p;
1308         env->best_imp = imp;
1309         env->best_cpu = env->dst_cpu;
1310 }
1311 
1312 static bool load_too_imbalanced(long src_load, long dst_load,
1313                                 struct task_numa_env *env)
1314 {
1315         long imb, old_imb;
1316         long orig_src_load, orig_dst_load;
1317         long src_capacity, dst_capacity;
1318 
1319         /*
1320          * The load is corrected for the CPU capacity available on each node.
1321          *
1322          * src_load        dst_load
1323          * ------------ vs ---------
1324          * src_capacity    dst_capacity
1325          */
1326         src_capacity = env->src_stats.compute_capacity;
1327         dst_capacity = env->dst_stats.compute_capacity;
1328 
1329         /* We care about the slope of the imbalance, not the direction. */
1330         if (dst_load < src_load)
1331                 swap(dst_load, src_load);
1332 
1333         /* Is the difference below the threshold? */
1334         imb = dst_load * src_capacity * 100 -
1335               src_load * dst_capacity * env->imbalance_pct;
1336         if (imb <= 0)
1337                 return false;
1338 
1339         /*
1340          * The imbalance is above the allowed threshold.
1341          * Compare it with the old imbalance.
1342          */
1343         orig_src_load = env->src_stats.load;
1344         orig_dst_load = env->dst_stats.load;
1345 
1346         if (orig_dst_load < orig_src_load)
1347                 swap(orig_dst_load, orig_src_load);
1348 
1349         old_imb = orig_dst_load * src_capacity * 100 -
1350                   orig_src_load * dst_capacity * env->imbalance_pct;
1351 
1352         /* Would this change make things worse? */
1353         return (imb > old_imb);
1354 }
1355 
1356 /*
1357  * This checks if the overall compute and NUMA accesses of the system would
1358  * be improved if the source tasks was migrated to the target dst_cpu taking
1359  * into account that it might be best if task running on the dst_cpu should
1360  * be exchanged with the source task
1361  */
1362 static void task_numa_compare(struct task_numa_env *env,
1363                               long taskimp, long groupimp)
1364 {
1365         struct rq *src_rq = cpu_rq(env->src_cpu);
1366         struct rq *dst_rq = cpu_rq(env->dst_cpu);
1367         struct task_struct *cur;
1368         long src_load, dst_load;
1369         long load;
1370         long imp = env->p->numa_group ? groupimp : taskimp;
1371         long moveimp = imp;
1372         int dist = env->dist;
1373         bool assigned = false;
1374 
1375         rcu_read_lock();
1376 
1377         raw_spin_lock_irq(&dst_rq->lock);
1378         cur = dst_rq->curr;
1379         /*
1380          * No need to move the exiting task or idle task.
1381          */
1382         if ((cur->flags & PF_EXITING) || is_idle_task(cur))
1383                 cur = NULL;
1384         else {
1385                 /*
1386                  * The task_struct must be protected here to protect the
1387                  * p->numa_faults access in the task_weight since the
1388                  * numa_faults could already be freed in the following path:
1389                  * finish_task_switch()
1390                  *     --> put_task_struct()
1391                  *         --> __put_task_struct()
1392                  *             --> task_numa_free()
1393                  */
1394                 get_task_struct(cur);
1395         }
1396 
1397         raw_spin_unlock_irq(&dst_rq->lock);
1398 
1399         /*
1400          * Because we have preemption enabled we can get migrated around and
1401          * end try selecting ourselves (current == env->p) as a swap candidate.
1402          */
1403         if (cur == env->p)
1404                 goto unlock;
1405 
1406         /*
1407          * "imp" is the fault differential for the source task between the
1408          * source and destination node. Calculate the total differential for
1409          * the source task and potential destination task. The more negative
1410          * the value is, the more rmeote accesses that would be expected to
1411          * be incurred if the tasks were swapped.
1412          */
1413         if (cur) {
1414                 /* Skip this swap candidate if cannot move to the source cpu */
1415                 if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur)))
1416                         goto unlock;
1417 
1418                 /*
1419                  * If dst and source tasks are in the same NUMA group, or not
1420                  * in any group then look only at task weights.
1421                  */
1422                 if (cur->numa_group == env->p->numa_group) {
1423                         imp = taskimp + task_weight(cur, env->src_nid, dist) -
1424                               task_weight(cur, env->dst_nid, dist);
1425                         /*
1426                          * Add some hysteresis to prevent swapping the
1427                          * tasks within a group over tiny differences.
1428                          */
1429                         if (cur->numa_group)
1430                                 imp -= imp/16;
1431                 } else {
1432                         /*
1433                          * Compare the group weights. If a task is all by
1434                          * itself (not part of a group), use the task weight
1435                          * instead.
1436                          */
1437                         if (cur->numa_group)
1438                                 imp += group_weight(cur, env->src_nid, dist) -
1439                                        group_weight(cur, env->dst_nid, dist);
1440                         else
1441                                 imp += task_weight(cur, env->src_nid, dist) -
1442                                        task_weight(cur, env->dst_nid, dist);
1443                 }
1444         }
1445 
1446         if (imp <= env->best_imp && moveimp <= env->best_imp)
1447                 goto unlock;
1448 
1449         if (!cur) {
1450                 /* Is there capacity at our destination? */
1451                 if (env->src_stats.nr_running <= env->src_stats.task_capacity &&
1452                     !env->dst_stats.has_free_capacity)
1453                         goto unlock;
1454 
1455                 goto balance;
1456         }
1457 
1458         /* Balance doesn't matter much if we're running a task per cpu */
1459         if (imp > env->best_imp && src_rq->nr_running == 1 &&
1460                         dst_rq->nr_running == 1)
1461                 goto assign;
1462 
1463         /*
1464          * In the overloaded case, try and keep the load balanced.
1465          */
1466 balance:
1467         load = task_h_load(env->p);
1468         dst_load = env->dst_stats.load + load;
1469         src_load = env->src_stats.load - load;
1470 
1471         if (moveimp > imp && moveimp > env->best_imp) {
1472                 /*
1473                  * If the improvement from just moving env->p direction is
1474                  * better than swapping tasks around, check if a move is
1475                  * possible. Store a slightly smaller score than moveimp,
1476                  * so an actually idle CPU will win.
1477                  */
1478                 if (!load_too_imbalanced(src_load, dst_load, env)) {
1479                         imp = moveimp - 1;
1480                         put_task_struct(cur);
1481                         cur = NULL;
1482                         goto assign;
1483                 }
1484         }
1485 
1486         if (imp <= env->best_imp)
1487                 goto unlock;
1488 
1489         if (cur) {
1490                 load = task_h_load(cur);
1491                 dst_load -= load;
1492                 src_load += load;
1493         }
1494 
1495         if (load_too_imbalanced(src_load, dst_load, env))
1496                 goto unlock;
1497 
1498         /*
1499          * One idle CPU per node is evaluated for a task numa move.
1500          * Call select_idle_sibling to maybe find a better one.
1501          */
1502         if (!cur)
1503                 env->dst_cpu = select_idle_sibling(env->p, env->dst_cpu);
1504 
1505 assign:
1506         assigned = true;
1507         task_numa_assign(env, cur, imp);
1508 unlock:
1509         rcu_read_unlock();
1510         /*
1511          * The dst_rq->curr isn't assigned. The protection for task_struct is
1512          * finished.
1513          */
1514         if (cur && !assigned)
1515                 put_task_struct(cur);
1516 }
1517 
1518 static void task_numa_find_cpu(struct task_numa_env *env,
1519                                 long taskimp, long groupimp)
1520 {
1521         int cpu;
1522 
1523         for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
1524                 /* Skip this CPU if the source task cannot migrate */
1525                 if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p)))
1526                         continue;
1527 
1528                 env->dst_cpu = cpu;
1529                 task_numa_compare(env, taskimp, groupimp);
1530         }
1531 }
1532 
1533 /* Only move tasks to a NUMA node less busy than the current node. */
1534 static bool numa_has_capacity(struct task_numa_env *env)
1535 {
1536         struct numa_stats *src = &env->src_stats;
1537         struct numa_stats *dst = &env->dst_stats;
1538 
1539         if (src->has_free_capacity && !dst->has_free_capacity)
1540                 return false;
1541 
1542         /*
1543          * Only consider a task move if the source has a higher load
1544          * than the destination, corrected for CPU capacity on each node.
1545          *
1546          *      src->load                dst->load
1547          * --------------------- vs ---------------------
1548          * src->compute_capacity    dst->compute_capacity
1549          */
1550         if (src->load * dst->compute_capacity * env->imbalance_pct >
1551 
1552             dst->load * src->compute_capacity * 100)
1553                 return true;
1554 
1555         return false;
1556 }
1557 
1558 static int task_numa_migrate(struct task_struct *p)
1559 {
1560         struct task_numa_env env = {
1561                 .p = p,
1562 
1563                 .src_cpu = task_cpu(p),
1564                 .src_nid = task_node(p),
1565 
1566                 .imbalance_pct = 112,
1567 
1568                 .best_task = NULL,
1569                 .best_imp = 0,
1570                 .best_cpu = -1,
1571         };
1572         struct sched_domain *sd;
1573         unsigned long taskweight, groupweight;
1574         int nid, ret, dist;
1575         long taskimp, groupimp;
1576 
1577         /*
1578          * Pick the lowest SD_NUMA domain, as that would have the smallest
1579          * imbalance and would be the first to start moving tasks about.
1580          *
1581          * And we want to avoid any moving of tasks about, as that would create
1582          * random movement of tasks -- counter the numa conditions we're trying
1583          * to satisfy here.
1584          */
1585         rcu_read_lock();
1586         sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
1587         if (sd)
1588                 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
1589         rcu_read_unlock();
1590 
1591         /*
1592          * Cpusets can break the scheduler domain tree into smaller
1593          * balance domains, some of which do not cross NUMA boundaries.
1594          * Tasks that are "trapped" in such domains cannot be migrated
1595          * elsewhere, so there is no point in (re)trying.
1596          */
1597         if (unlikely(!sd)) {
1598                 p->numa_preferred_nid = task_node(p);
1599                 return -EINVAL;
1600         }
1601 
1602         env.dst_nid = p->numa_preferred_nid;
1603         dist = env.dist = node_distance(env.src_nid, env.dst_nid);
1604         taskweight = task_weight(p, env.src_nid, dist);
1605         groupweight = group_weight(p, env.src_nid, dist);
1606         update_numa_stats(&env.src_stats, env.src_nid);
1607         taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
1608         groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
1609         update_numa_stats(&env.dst_stats, env.dst_nid);
1610 
1611         /* Try to find a spot on the preferred nid. */
1612         if (numa_has_capacity(&env))
1613                 task_numa_find_cpu(&env, taskimp, groupimp);
1614 
1615         /*
1616          * Look at other nodes in these cases:
1617          * - there is no space available on the preferred_nid
1618          * - the task is part of a numa_group that is interleaved across
1619          *   multiple NUMA nodes; in order to better consolidate the group,
1620          *   we need to check other locations.
1621          */
1622         if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) {
1623                 for_each_online_node(nid) {
1624                         if (nid == env.src_nid || nid == p->numa_preferred_nid)
1625                                 continue;
1626 
1627                         dist = node_distance(env.src_nid, env.dst_nid);
1628                         if (sched_numa_topology_type == NUMA_BACKPLANE &&
1629                                                 dist != env.dist) {
1630                                 taskweight = task_weight(p, env.src_nid, dist);
1631                                 groupweight = group_weight(p, env.src_nid, dist);
1632                         }
1633 
1634                         /* Only consider nodes where both task and groups benefit */
1635                         taskimp = task_weight(p, nid, dist) - taskweight;
1636                         groupimp = group_weight(p, nid, dist) - groupweight;
1637                         if (taskimp < 0 && groupimp < 0)
1638                                 continue;
1639 
1640                         env.dist = dist;
1641                         env.dst_nid = nid;
1642                         update_numa_stats(&env.dst_stats, env.dst_nid);
1643                         if (numa_has_capacity(&env))
1644                                 task_numa_find_cpu(&env, taskimp, groupimp);
1645                 }
1646         }
1647 
1648         /*
1649          * If the task is part of a workload that spans multiple NUMA nodes,
1650          * and is migrating into one of the workload's active nodes, remember
1651          * this node as the task's preferred numa node, so the workload can
1652          * settle down.
1653          * A task that migrated to a second choice node will be better off
1654          * trying for a better one later. Do not set the preferred node here.
1655          */
1656         if (p->numa_group) {
1657                 struct numa_group *ng = p->numa_group;
1658 
1659                 if (env.best_cpu == -1)
1660                         nid = env.src_nid;
1661                 else
1662                         nid = env.dst_nid;
1663 
1664                 if (ng->active_nodes > 1 && numa_is_active_node(env.dst_nid, ng))
1665                         sched_setnuma(p, env.dst_nid);
1666         }
1667 
1668         /* No better CPU than the current one was found. */
1669         if (env.best_cpu == -1)
1670                 return -EAGAIN;
1671 
1672         /*
1673          * Reset the scan period if the task is being rescheduled on an
1674          * alternative node to recheck if the tasks is now properly placed.
1675          */
1676         p->numa_scan_period = task_scan_min(p);
1677 
1678         if (env.best_task == NULL) {
1679                 ret = migrate_task_to(p, env.best_cpu);
1680                 if (ret != 0)
1681                         trace_sched_stick_numa(p, env.src_cpu, env.best_cpu);
1682                 return ret;
1683         }
1684 
1685         ret = migrate_swap(p, env.best_task);
1686         if (ret != 0)
1687                 trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task));
1688         put_task_struct(env.best_task);
1689         return ret;
1690 }
1691 
1692 /* Attempt to migrate a task to a CPU on the preferred node. */
1693 static void numa_migrate_preferred(struct task_struct *p)
1694 {
1695         unsigned long interval = HZ;
1696 
1697         /* This task has no NUMA fault statistics yet */
1698         if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults))
1699                 return;
1700 
1701         /* Periodically retry migrating the task to the preferred node */
1702         interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
1703         p->numa_migrate_retry = jiffies + interval;
1704 
1705         /* Success if task is already running on preferred CPU */
1706         if (task_node(p) == p->numa_preferred_nid)
1707                 return;
1708 
1709         /* Otherwise, try migrate to a CPU on the preferred node */
1710         task_numa_migrate(p);
1711 }
1712 
1713 /*
1714  * Find out how many nodes on the workload is actively running on. Do this by
1715  * tracking the nodes from which NUMA hinting faults are triggered. This can
1716  * be different from the set of nodes where the workload's memory is currently
1717  * located.
1718  */
1719 static void numa_group_count_active_nodes(struct numa_group *numa_group)
1720 {
1721         unsigned long faults, max_faults = 0;
1722         int nid, active_nodes = 0;
1723 
1724         for_each_online_node(nid) {
1725                 faults = group_faults_cpu(numa_group, nid);
1726                 if (faults > max_faults)
1727                         max_faults = faults;
1728         }
1729 
1730         for_each_online_node(nid) {
1731                 faults = group_faults_cpu(numa_group, nid);
1732                 if (faults * ACTIVE_NODE_FRACTION > max_faults)
1733                         active_nodes++;
1734         }
1735 
1736         numa_group->max_faults_cpu = max_faults;
1737         numa_group->active_nodes = active_nodes;
1738 }
1739 
1740 /*
1741  * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
1742  * increments. The more local the fault statistics are, the higher the scan
1743  * period will be for the next scan window. If local/(local+remote) ratio is
1744  * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS)
1745  * the scan period will decrease. Aim for 70% local accesses.
1746  */
1747 #define NUMA_PERIOD_SLOTS 10
1748 #define NUMA_PERIOD_THRESHOLD 7
1749 
1750 /*
1751  * Increase the scan period (slow down scanning) if the majority of
1752  * our memory is already on our local node, or if the majority of
1753  * the page accesses are shared with other processes.
1754  * Otherwise, decrease the scan period.
1755  */
1756 static void update_task_scan_period(struct task_struct *p,
1757                         unsigned long shared, unsigned long private)
1758 {
1759         unsigned int period_slot;
1760         int ratio;
1761         int diff;
1762 
1763         unsigned long remote = p->numa_faults_locality[0];
1764         unsigned long local = p->numa_faults_locality[1];
1765 
1766         /*
1767          * If there were no record hinting faults then either the task is
1768          * completely idle or all activity is areas that are not of interest
1769          * to automatic numa balancing. Related to that, if there were failed
1770          * migration then it implies we are migrating too quickly or the local
1771          * node is overloaded. In either case, scan slower
1772          */
1773         if (local + shared == 0 || p->numa_faults_locality[2]) {
1774                 p->numa_scan_period = min(p->numa_scan_period_max,
1775                         p->numa_scan_period << 1);
1776 
1777                 p->mm->numa_next_scan = jiffies +
1778                         msecs_to_jiffies(p->numa_scan_period);
1779 
1780                 return;
1781         }
1782 
1783         /*
1784          * Prepare to scale scan period relative to the current period.
1785          *       == NUMA_PERIOD_THRESHOLD scan period stays the same
1786          *       <  NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
1787          *       >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
1788          */
1789         period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
1790         ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
1791         if (ratio >= NUMA_PERIOD_THRESHOLD) {
1792                 int slot = ratio - NUMA_PERIOD_THRESHOLD;
1793                 if (!slot)
1794                         slot = 1;
1795                 diff = slot * period_slot;
1796         } else {
1797                 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
1798 
1799                 /*
1800                  * Scale scan rate increases based on sharing. There is an
1801                  * inverse relationship between the degree of sharing and
1802                  * the adjustment made to the scanning period. Broadly
1803                  * speaking the intent is that there is little point
1804                  * scanning faster if shared accesses dominate as it may
1805                  * simply bounce migrations uselessly
1806                  */
1807                 ratio = DIV_ROUND_UP(private * NUMA_PERIOD_SLOTS, (private + shared + 1));
1808                 diff = (diff * ratio) / NUMA_PERIOD_SLOTS;
1809         }
1810 
1811         p->numa_scan_period = clamp(p->numa_scan_period + diff,
1812                         task_scan_min(p), task_scan_max(p));
1813         memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
1814 }
1815 
1816 /*
1817  * Get the fraction of time the task has been running since the last
1818  * NUMA placement cycle. The scheduler keeps similar statistics, but
1819  * decays those on a 32ms period, which is orders of magnitude off
1820  * from the dozens-of-seconds NUMA balancing period. Use the scheduler
1821  * stats only if the task is so new there are no NUMA statistics yet.
1822  */
1823 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
1824 {
1825         u64 runtime, delta, now;
1826         /* Use the start of this time slice to avoid calculations. */
1827         now = p->se.exec_start;
1828         runtime = p->se.sum_exec_runtime;
1829 
1830         if (p->last_task_numa_placement) {
1831                 delta = runtime - p->last_sum_exec_runtime;
1832                 *period = now - p->last_task_numa_placement;
1833         } else {
1834                 delta = p->se.avg.load_sum / p->se.load.weight;
1835                 *period = LOAD_AVG_MAX;
1836         }
1837 
1838         p->last_sum_exec_runtime = runtime;
1839         p->last_task_numa_placement = now;
1840 
1841         return delta;
1842 }
1843 
1844 /*
1845  * Determine the preferred nid for a task in a numa_group. This needs to
1846  * be done in a way that produces consistent results with group_weight,
1847  * otherwise workloads might not converge.
1848  */
1849 static int preferred_group_nid(struct task_struct *p, int nid)
1850 {
1851         nodemask_t nodes;
1852         int dist;
1853 
1854         /* Direct connections between all NUMA nodes. */
1855         if (sched_numa_topology_type == NUMA_DIRECT)
1856                 return nid;
1857 
1858         /*
1859          * On a system with glueless mesh NUMA topology, group_weight
1860          * scores nodes according to the number of NUMA hinting faults on
1861          * both the node itself, and on nearby nodes.
1862          */
1863         if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
1864                 unsigned long score, max_score = 0;
1865                 int node, max_node = nid;
1866 
1867                 dist = sched_max_numa_distance;
1868 
1869                 for_each_online_node(node) {
1870                         score = group_weight(p, node, dist);
1871                         if (score > max_score) {
1872                                 max_score = score;
1873                                 max_node = node;
1874                         }
1875                 }
1876                 return max_node;
1877         }
1878 
1879         /*
1880          * Finding the preferred nid in a system with NUMA backplane
1881          * interconnect topology is more involved. The goal is to locate
1882          * tasks from numa_groups near each other in the system, and
1883          * untangle workloads from different sides of the system. This requires
1884          * searching down the hierarchy of node groups, recursively searching
1885          * inside the highest scoring group of nodes. The nodemask tricks
1886          * keep the complexity of the search down.
1887          */
1888         nodes = node_online_map;
1889         for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
1890                 unsigned long max_faults = 0;
1891                 nodemask_t max_group = NODE_MASK_NONE;
1892                 int a, b;
1893 
1894                 /* Are there nodes at this distance from each other? */
1895                 if (!find_numa_distance(dist))
1896                         continue;
1897 
1898                 for_each_node_mask(a, nodes) {
1899                         unsigned long faults = 0;
1900                         nodemask_t this_group;
1901                         nodes_clear(this_group);
1902 
1903                         /* Sum group's NUMA faults; includes a==b case. */
1904                         for_each_node_mask(b, nodes) {
1905                                 if (node_distance(a, b) < dist) {
1906                                         faults += group_faults(p, b);
1907                                         node_set(b, this_group);
1908                                         node_clear(b, nodes);
1909                                 }
1910                         }
1911 
1912                         /* Remember the top group. */
1913                         if (faults > max_faults) {
1914                                 max_faults = faults;
1915                                 max_group = this_group;
1916                                 /*
1917                                  * subtle: at the smallest distance there is
1918                                  * just one node left in each "group", the
1919                                  * winner is the preferred nid.
1920                                  */
1921                                 nid = a;
1922                         }
1923                 }
1924                 /* Next round, evaluate the nodes within max_group. */
1925                 if (!max_faults)
1926                         break;
1927                 nodes = max_group;
1928         }
1929         return nid;
1930 }
1931 
1932 static void task_numa_placement(struct task_struct *p)
1933 {
1934         int seq, nid, max_nid = -1, max_group_nid = -1;
1935         unsigned long max_faults = 0, max_group_faults = 0;
1936         unsigned long fault_types[2] = { 0, 0 };
1937         unsigned long total_faults;
1938         u64 runtime, period;
1939         spinlock_t *group_lock = NULL;
1940 
1941         /*
1942          * The p->mm->numa_scan_seq field gets updated without
1943          * exclusive access. Use READ_ONCE() here to ensure
1944          * that the field is read in a single access:
1945          */
1946         seq = READ_ONCE(p->mm->numa_scan_seq);
1947         if (p->numa_scan_seq == seq)
1948                 return;
1949         p->numa_scan_seq = seq;
1950         p->numa_scan_period_max = task_scan_max(p);
1951 
1952         total_faults = p->numa_faults_locality[0] +
1953                        p->numa_faults_locality[1];
1954         runtime = numa_get_avg_runtime(p, &period);
1955 
1956         /* If the task is part of a group prevent parallel updates to group stats */
1957         if (p->numa_group) {
1958                 group_lock = &p->numa_group->lock;
1959                 spin_lock_irq(group_lock);
1960         }
1961 
1962         /* Find the node with the highest number of faults */
1963         for_each_online_node(nid) {
1964                 /* Keep track of the offsets in numa_faults array */
1965                 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx;
1966                 unsigned long faults = 0, group_faults = 0;
1967                 int priv;
1968 
1969                 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
1970                         long diff, f_diff, f_weight;
1971 
1972                         mem_idx = task_faults_idx(NUMA_MEM, nid, priv);
1973                         membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);
1974                         cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);
1975                         cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);
1976 
1977                         /* Decay existing window, copy faults since last scan */
1978                         diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2;
1979                         fault_types[priv] += p->numa_faults[membuf_idx];
1980                         p->numa_faults[membuf_idx] = 0;
1981 
1982                         /*
1983                          * Normalize the faults_from, so all tasks in a group
1984                          * count according to CPU use, instead of by the raw
1985                          * number of faults. Tasks with little runtime have
1986                          * little over-all impact on throughput, and thus their
1987                          * faults are less important.
1988                          */
1989                         f_weight = div64_u64(runtime << 16, period + 1);
1990                         f_weight = (f_weight * p->numa_faults[cpubuf_idx]) /
1991                                    (total_faults + 1);
1992                         f_diff = f_weight - p->numa_faults[cpu_idx] / 2;
1993                         p->numa_faults[cpubuf_idx] = 0;
1994 
1995                         p->numa_faults[mem_idx] += diff;
1996                         p->numa_faults[cpu_idx] += f_diff;
1997                         faults += p->numa_faults[mem_idx];
1998                         p->total_numa_faults += diff;
1999                         if (p->numa_group) {
2000                                 /*
2001                                  * safe because we can only change our own group
2002                                  *
2003                                  * mem_idx represents the offset for a given
2004                                  * nid and priv in a specific region because it
2005                                  * is at the beginning of the numa_faults array.
2006                                  */
2007                                 p->numa_group->faults[mem_idx] += diff;
2008                                 p->numa_group->faults_cpu[mem_idx] += f_diff;
2009                                 p->numa_group->total_faults += diff;
2010                                 group_faults += p->numa_group->faults[mem_idx];
2011                         }
2012                 }
2013 
2014                 if (faults > max_faults) {
2015                         max_faults = faults;
2016                         max_nid = nid;
2017                 }
2018 
2019                 if (group_faults > max_group_faults) {
2020                         max_group_faults = group_faults;
2021                         max_group_nid = nid;
2022                 }
2023         }
2024 
2025         update_task_scan_period(p, fault_types[0], fault_types[1]);
2026 
2027         if (p->numa_group) {
2028                 numa_group_count_active_nodes(p->numa_group);
2029                 spin_unlock_irq(group_lock);
2030                 max_nid = preferred_group_nid(p, max_group_nid);
2031         }
2032 
2033         if (max_faults) {
2034                 /* Set the new preferred node */
2035                 if (max_nid != p->numa_preferred_nid)
2036                         sched_setnuma(p, max_nid);
2037 
2038                 if (task_node(p) != p->numa_preferred_nid)
2039                         numa_migrate_preferred(p);
2040         }
2041 }
2042 
2043 static inline int get_numa_group(struct numa_group *grp)
2044 {
2045         return atomic_inc_not_zero(&grp->refcount);
2046 }
2047 
2048 static inline void put_numa_group(struct numa_group *grp)
2049 {
2050         if (atomic_dec_and_test(&grp->refcount))
2051                 kfree_rcu(grp, rcu);
2052 }
2053 
2054 static void task_numa_group(struct task_struct *p, int cpupid, int flags,
2055                         int *priv)
2056 {
2057         struct numa_group *grp, *my_grp;
2058         struct task_struct *tsk;
2059         bool join = false;
2060         int cpu = cpupid_to_cpu(cpupid);
2061         int i;
2062 
2063         if (unlikely(!p->numa_group)) {
2064                 unsigned int size = sizeof(struct numa_group) +
2065                                     4*nr_node_ids*sizeof(unsigned long);
2066 
2067                 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
2068                 if (!grp)
2069                         return;
2070 
2071                 atomic_set(&grp->refcount, 1);
2072                 grp->active_nodes = 1;
2073                 grp->max_faults_cpu = 0;
2074                 spin_lock_init(&grp->lock);
2075                 grp->gid = p->pid;
2076                 /* Second half of the array tracks nids where faults happen */
2077                 grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES *
2078                                                 nr_node_ids;
2079 
2080                 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2081                         grp->faults[i] = p->numa_faults[i];
2082 
2083                 grp->total_faults = p->total_numa_faults;
2084 
2085                 grp->nr_tasks++;
2086                 rcu_assign_pointer(p->numa_group, grp);
2087         }
2088 
2089         rcu_read_lock();
2090         tsk = READ_ONCE(cpu_rq(cpu)->curr);
2091 
2092         if (!cpupid_match_pid(tsk, cpupid))
2093                 goto no_join;
2094 
2095         grp = rcu_dereference(tsk->numa_group);
2096         if (!grp)
2097                 goto no_join;
2098 
2099         my_grp = p->numa_group;
2100         if (grp == my_grp)
2101                 goto no_join;
2102 
2103         /*
2104          * Only join the other group if its bigger; if we're the bigger group,
2105          * the other task will join us.
2106          */
2107         if (my_grp->nr_tasks > grp->nr_tasks)
2108                 goto no_join;
2109 
2110         /*
2111          * Tie-break on the grp address.
2112          */
2113         if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
2114                 goto no_join;
2115 
2116         /* Always join threads in the same process. */
2117         if (tsk->mm == current->mm)
2118                 join = true;
2119 
2120         /* Simple filter to avoid false positives due to PID collisions */
2121         if (flags & TNF_SHARED)
2122                 join = true;
2123 
2124         /* Update priv based on whether false sharing was detected */
2125         *priv = !join;
2126 
2127         if (join && !get_numa_group(grp))
2128                 goto no_join;
2129 
2130         rcu_read_unlock();
2131 
2132         if (!join)
2133                 return;
2134 
2135         BUG_ON(irqs_disabled());
2136         double_lock_irq(&my_grp->lock, &grp->lock);
2137 
2138         for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
2139                 my_grp->faults[i] -= p->numa_faults[i];
2140                 grp->faults[i] += p->numa_faults[i];
2141         }
2142         my_grp->total_faults -= p->total_numa_faults;
2143         grp->total_faults += p->total_numa_faults;
2144 
2145         my_grp->nr_tasks--;
2146         grp->nr_tasks++;
2147 
2148         spin_unlock(&my_grp->lock);
2149         spin_unlock_irq(&grp->lock);
2150 
2151         rcu_assign_pointer(p->numa_group, grp);
2152 
2153         put_numa_group(my_grp);
2154         return;
2155 
2156 no_join:
2157         rcu_read_unlock();
2158         return;
2159 }
2160 
2161 void task_numa_free(struct task_struct *p)
2162 {
2163         struct numa_group *grp = p->numa_group;
2164         void *numa_faults = p->numa_faults;
2165         unsigned long flags;
2166         int i;
2167 
2168         if (grp) {
2169                 spin_lock_irqsave(&grp->lock, flags);
2170                 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2171                         grp->faults[i] -= p->numa_faults[i];
2172                 grp->total_faults -= p->total_numa_faults;
2173 
2174                 grp->nr_tasks--;
2175                 spin_unlock_irqrestore(&grp->lock, flags);
2176                 RCU_INIT_POINTER(p->numa_group, NULL);
2177                 put_numa_group(grp);
2178         }
2179 
2180         p->numa_faults = NULL;
2181         kfree(numa_faults);
2182 }
2183 
2184 /*
2185  * Got a PROT_NONE fault for a page on @node.
2186  */
2187 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
2188 {
2189         struct task_struct *p = current;
2190         bool migrated = flags & TNF_MIGRATED;
2191         int cpu_node = task_node(current);
2192         int local = !!(flags & TNF_FAULT_LOCAL);
2193         struct numa_group *ng;
2194         int priv;
2195 
2196         if (!static_branch_likely(&sched_numa_balancing))
2197                 return;
2198 
2199         /* for example, ksmd faulting in a user's mm */
2200         if (!p->mm)
2201                 return;
2202 
2203         /* Allocate buffer to track faults on a per-node basis */
2204         if (unlikely(!p->numa_faults)) {
2205                 int size = sizeof(*p->numa_faults) *
2206                            NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
2207 
2208                 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN);
2209                 if (!p->numa_faults)
2210                         return;
2211 
2212                 p->total_numa_faults = 0;
2213                 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2214         }
2215 
2216         /*
2217          * First accesses are treated as private, otherwise consider accesses
2218          * to be private if the accessing pid has not changed
2219          */
2220         if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
2221                 priv = 1;
2222         } else {
2223                 priv = cpupid_match_pid(p, last_cpupid);
2224                 if (!priv && !(flags & TNF_NO_GROUP))
2225                         task_numa_group(p, last_cpupid, flags, &priv);
2226         }
2227 
2228         /*
2229          * If a workload spans multiple NUMA nodes, a shared fault that
2230          * occurs wholly within the set of nodes that the workload is
2231          * actively using should be counted as local. This allows the
2232          * scan rate to slow down when a workload has settled down.
2233          */
2234         ng = p->numa_group;
2235         if (!priv && !local && ng && ng->active_nodes > 1 &&
2236                                 numa_is_active_node(cpu_node, ng) &&
2237                                 numa_is_active_node(mem_node, ng))
2238                 local = 1;
2239 
2240         task_numa_placement(p);
2241 
2242         /*
2243          * Retry task to preferred node migration periodically, in case it
2244          * case it previously failed, or the scheduler moved us.
2245          */
2246         if (time_after(jiffies, p->numa_migrate_retry))
2247                 numa_migrate_preferred(p);
2248 
2249         if (migrated)
2250                 p->numa_pages_migrated += pages;
2251         if (flags & TNF_MIGRATE_FAIL)
2252                 p->numa_faults_locality[2] += pages;
2253 
2254         p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages;
2255         p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages;
2256         p->numa_faults_locality[local] += pages;
2257 }
2258 
2259 static void reset_ptenuma_scan(struct task_struct *p)
2260 {
2261         /*
2262          * We only did a read acquisition of the mmap sem, so
2263          * p->mm->numa_scan_seq is written to without exclusive access
2264          * and the update is not guaranteed to be atomic. That's not
2265          * much of an issue though, since this is just used for
2266          * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
2267          * expensive, to avoid any form of compiler optimizations:
2268          */
2269         WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
2270         p->mm->numa_scan_offset = 0;
2271 }
2272 
2273 /*
2274  * The expensive part of numa migration is done from task_work context.
2275  * Triggered from task_tick_numa().
2276  */
2277 void task_numa_work(struct callback_head *work)
2278 {
2279         unsigned long migrate, next_scan, now = jiffies;
2280         struct task_struct *p = current;
2281         struct mm_struct *mm = p->mm;
2282         u64 runtime = p->se.sum_exec_runtime;
2283         struct vm_area_struct *vma;
2284         unsigned long start, end;
2285         unsigned long nr_pte_updates = 0;
2286         long pages, virtpages;
2287 
2288         WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
2289 
2290         work->next = work; /* protect against double add */
2291         /*
2292          * Who cares about NUMA placement when they're dying.
2293          *
2294          * NOTE: make sure not to dereference p->mm before this check,
2295          * exit_task_work() happens _after_ exit_mm() so we could be called
2296          * without p->mm even though we still had it when we enqueued this
2297          * work.
2298          */
2299         if (p->flags & PF_EXITING)
2300                 return;
2301 
2302         if (!mm->numa_next_scan) {
2303                 mm->numa_next_scan = now +
2304                         msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2305         }
2306 
2307         /*
2308          * Enforce maximal scan/migration frequency..
2309          */
2310         migrate = mm->numa_next_scan;
2311         if (time_before(now, migrate))
2312                 return;
2313 
2314         if (p->numa_scan_period == 0) {
2315                 p->numa_scan_period_max = task_scan_max(p);
2316                 p->numa_scan_period = task_scan_min(p);
2317         }
2318 
2319         next_scan = now + msecs_to_jiffies(p->numa_scan_period);
2320         if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
2321                 return;
2322 
2323         /*
2324          * Delay this task enough that another task of this mm will likely win
2325          * the next time around.
2326          */
2327         p->node_stamp += 2 * TICK_NSEC;
2328 
2329         start = mm->numa_scan_offset;
2330         pages = sysctl_numa_balancing_scan_size;
2331         pages <<= 20 - PAGE_SHIFT; /* MB in pages */
2332         virtpages = pages * 8;     /* Scan up to this much virtual space */
2333         if (!pages)
2334                 return;
2335 
2336 
2337         down_read(&mm->mmap_sem);
2338         vma = find_vma(mm, start);
2339         if (!vma) {
2340                 reset_ptenuma_scan(p);
2341                 start = 0;
2342                 vma = mm->mmap;
2343         }
2344         for (; vma; vma = vma->vm_next) {
2345                 if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
2346                         is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
2347                         continue;
2348                 }
2349 
2350                 /*
2351                  * Shared library pages mapped by multiple processes are not
2352                  * migrated as it is expected they are cache replicated. Avoid
2353                  * hinting faults in read-only file-backed mappings or the vdso
2354                  * as migrating the pages will be of marginal benefit.
2355                  */
2356                 if (!vma->vm_mm ||
2357                     (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
2358                         continue;
2359 
2360                 /*
2361                  * Skip inaccessible VMAs to avoid any confusion between
2362                  * PROT_NONE and NUMA hinting ptes
2363                  */
2364                 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
2365                         continue;
2366 
2367                 do {
2368                         start = max(start, vma->vm_start);
2369                         end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
2370                         end = min(end, vma->vm_end);
2371                         nr_pte_updates = change_prot_numa(vma, start, end);
2372 
2373                         /*
2374                          * Try to scan sysctl_numa_balancing_size worth of
2375                          * hpages that have at least one present PTE that
2376                          * is not already pte-numa. If the VMA contains
2377                          * areas that are unused or already full of prot_numa
2378                          * PTEs, scan up to virtpages, to skip through those
2379                          * areas faster.
2380                          */
2381                         if (nr_pte_updates)
2382                                 pages -= (end - start) >> PAGE_SHIFT;
2383                         virtpages -= (end - start) >> PAGE_SHIFT;
2384 
2385                         start = end;
2386                         if (pages <= 0 || virtpages <= 0)
2387                                 goto out;
2388 
2389                         cond_resched();
2390                 } while (end != vma->vm_end);
2391         }
2392 
2393 out:
2394         /*
2395          * It is possible to reach the end of the VMA list but the last few
2396          * VMAs are not guaranteed to the vma_migratable. If they are not, we
2397          * would find the !migratable VMA on the next scan but not reset the
2398          * scanner to the start so check it now.
2399          */
2400         if (vma)
2401                 mm->numa_scan_offset = start;
2402         else
2403                 reset_ptenuma_scan(p);
2404         up_read(&mm->mmap_sem);
2405 
2406         /*
2407          * Make sure tasks use at least 32x as much time to run other code
2408          * than they used here, to limit NUMA PTE scanning overhead to 3% max.
2409          * Usually update_task_scan_period slows down scanning enough; on an
2410          * overloaded system we need to limit overhead on a per task basis.
2411          */
2412         if (unlikely(p->se.sum_exec_runtime != runtime)) {
2413                 u64 diff = p->se.sum_exec_runtime - runtime;
2414                 p->node_stamp += 32 * diff;
2415         }
2416 }
2417 
2418 /*
2419  * Drive the periodic memory faults..
2420  */
2421 void task_tick_numa(struct rq *rq, struct task_struct *curr)
2422 {
2423         struct callback_head *work = &curr->numa_work;
2424         u64 period, now;
2425 
2426         /*
2427          * We don't care about NUMA placement if we don't have memory.
2428          */
2429         if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work)
2430                 return;
2431 
2432         /*
2433          * Using runtime rather than walltime has the dual advantage that
2434          * we (mostly) drive the selection from busy threads and that the
2435          * task needs to have done some actual work before we bother with
2436          * NUMA placement.
2437          */
2438         now = curr->se.sum_exec_runtime;
2439         period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
2440 
2441         if (now > curr->node_stamp + period) {
2442                 if (!curr->node_stamp)
2443                         curr->numa_scan_period = task_scan_min(curr);
2444                 curr->node_stamp += period;
2445 
2446                 if (!time_before(jiffies, curr->mm->numa_next_scan)) {
2447                         init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */
2448                         task_work_add(curr, work, true);
2449                 }
2450         }
2451 }
2452 #else
2453 static void task_tick_numa(struct rq *rq, struct task_struct *curr)
2454 {
2455 }
2456 
2457 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
2458 {
2459 }
2460 
2461 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
2462 {
2463 }
2464 #endif /* CONFIG_NUMA_BALANCING */
2465 
2466 static void
2467 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2468 {
2469         update_load_add(&cfs_rq->load, se->load.weight);
2470         if (!parent_entity(se))
2471                 update_load_add(&rq_of(cfs_rq)->load, se->load.weight);
2472 #ifdef CONFIG_SMP
2473         if (entity_is_task(se)) {
2474                 struct rq *rq = rq_of(cfs_rq);
2475 
2476                 account_numa_enqueue(rq, task_of(se));
2477                 list_add(&se->group_node, &rq->cfs_tasks);
2478         }
2479 #endif
2480         cfs_rq->nr_running++;
2481 }
2482 
2483 static void
2484 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
2485 {
2486         update_load_sub(&cfs_rq->load, se->load.weight);
2487         if (!parent_entity(se))
2488                 update_load_sub(&rq_of(cfs_rq)->load, se->load.weight);
2489 #ifdef CONFIG_SMP
2490         if (entity_is_task(se)) {
2491                 account_numa_dequeue(rq_of(cfs_rq), task_of(se));
2492                 list_del_init(&se->group_node);
2493         }
2494 #endif
2495         cfs_rq->nr_running--;
2496 }
2497 
2498 #ifdef CONFIG_FAIR_GROUP_SCHED
2499 # ifdef CONFIG_SMP
2500 static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
2501 {
2502         long tg_weight, load, shares;
2503 
2504         /*
2505          * This really should be: cfs_rq->avg.load_avg, but instead we use
2506          * cfs_rq->load.weight, which is its upper bound. This helps ramp up
2507          * the shares for small weight interactive tasks.
2508          */
2509         load = scale_load_down(cfs_rq->load.weight);
2510 
2511         tg_weight = atomic_long_read(&tg->load_avg);
2512 
2513         /* Ensure tg_weight >= load */
2514         tg_weight -= cfs_rq->tg_load_avg_contrib;
2515         tg_weight += load;
2516 
2517         shares = (tg->shares * load);
2518         if (tg_weight)
2519                 shares /= tg_weight;
2520 
2521         if (shares < MIN_SHARES)
2522                 shares = MIN_SHARES;
2523         if (shares > tg->shares)
2524                 shares = tg->shares;
2525 
2526         return shares;
2527 }
2528 # else /* CONFIG_SMP */
2529 static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg)
2530 {
2531         return tg->shares;
2532 }
2533 # endif /* CONFIG_SMP */
2534 
2535 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
2536                             unsigned long weight)
2537 {
2538         if (se->on_rq) {
2539                 /* commit outstanding execution time */
2540                 if (cfs_rq->curr == se)
2541                         update_curr(cfs_rq);
2542                 account_entity_dequeue(cfs_rq, se);
2543         }
2544 
2545         update_load_set(&se->load, weight);
2546 
2547         if (se->on_rq)
2548                 account_entity_enqueue(cfs_rq, se);
2549 }
2550 
2551 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
2552 
2553 static void update_cfs_shares(struct cfs_rq *cfs_rq)
2554 {
2555         struct task_group *tg;
2556         struct sched_entity *se;
2557         long shares;
2558 
2559         tg = cfs_rq->tg;
2560         se = tg->se[cpu_of(rq_of(cfs_rq))];
2561         if (!se || throttled_hierarchy(cfs_rq))
2562                 return;
2563 #ifndef CONFIG_SMP
2564         if (likely(se->load.weight == tg->shares))
2565                 return;
2566 #endif
2567         shares = calc_cfs_shares(cfs_rq, tg);
2568 
2569         reweight_entity(cfs_rq_of(se), se, shares);
2570 }
2571 #else /* CONFIG_FAIR_GROUP_SCHED */
2572 static inline void update_cfs_shares(struct cfs_rq *cfs_rq)
2573 {
2574 }
2575 #endif /* CONFIG_FAIR_GROUP_SCHED */
2576 
2577 #ifdef CONFIG_SMP
2578 /* Precomputed fixed inverse multiplies for multiplication by y^n */
2579 static const u32 runnable_avg_yN_inv[] = {
2580         0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6,
2581         0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85,
2582         0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581,
2583         0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9,
2584         0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80,
2585         0x85aac367, 0x82cd8698,
2586 };
2587 
2588 /*
2589  * Precomputed \Sum y^k { 1<=k<=n }.  These are floor(true_value) to prevent
2590  * over-estimates when re-combining.
2591  */
2592 static const u32 runnable_avg_yN_sum[] = {
2593             0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103,
2594          9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082,
2595         17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371,
2596 };
2597 
2598 /*
2599  * Precomputed \Sum y^k { 1<=k<=n, where n%32=0). Values are rolled down to
2600  * lower integers. See Documentation/scheduler/sched-avg.txt how these
2601  * were generated:
2602  */
2603 static const u32 __accumulated_sum_N32[] = {
2604             0, 23371, 35056, 40899, 43820, 45281,
2605         46011, 46376, 46559, 46650, 46696, 46719,
2606 };
2607 
2608 /*
2609  * Approximate:
2610  *   val * y^n,    where y^32 ~= 0.5 (~1 scheduling period)
2611  */
2612 static __always_inline u64 decay_load(u64 val, u64 n)
2613 {
2614         unsigned int local_n;
2615 
2616         if (!n)
2617                 return val;
2618         else if (unlikely(n > LOAD_AVG_PERIOD * 63))
2619                 return 0;
2620 
2621         /* after bounds checking we can collapse to 32-bit */
2622         local_n = n;
2623 
2624         /*
2625          * As y^PERIOD = 1/2, we can combine
2626          *    y^n = 1/2^(n/PERIOD) * y^(n%PERIOD)
2627          * With a look-up table which covers y^n (n<PERIOD)
2628          *
2629          * To achieve constant time decay_load.
2630          */
2631         if (unlikely(local_n >= LOAD_AVG_PERIOD)) {
2632                 val >>= local_n / LOAD_AVG_PERIOD;
2633                 local_n %= LOAD_AVG_PERIOD;
2634         }
2635 
2636         val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32);
2637         return val;
2638 }
2639 
2640 /*
2641  * For updates fully spanning n periods, the contribution to runnable
2642  * average will be: \Sum 1024*y^n
2643  *
2644  * We can compute this reasonably efficiently by combining:
2645  *   y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for  n <PERIOD}
2646  */
2647 static u32 __compute_runnable_contrib(u64 n)
2648 {
2649         u32 contrib = 0;
2650 
2651         if (likely(n <= LOAD_AVG_PERIOD))
2652                 return runnable_avg_yN_sum[n];
2653         else if (unlikely(n >= LOAD_AVG_MAX_N))
2654                 return LOAD_AVG_MAX;
2655 
2656         /* Since n < LOAD_AVG_MAX_N, n/LOAD_AVG_PERIOD < 11 */
2657         contrib = __accumulated_sum_N32[n/LOAD_AVG_PERIOD];
2658         n %= LOAD_AVG_PERIOD;
2659         contrib = decay_load(contrib, n);
2660         return contrib + runnable_avg_yN_sum[n];
2661 }
2662 
2663 #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT)
2664 
2665 /*
2666  * We can represent the historical contribution to runnable average as the
2667  * coefficients of a geometric series.  To do this we sub-divide our runnable
2668  * history into segments of approximately 1ms (1024us); label the segment that
2669  * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g.
2670  *
2671  * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ...
2672  *      p0            p1           p2
2673  *     (now)       (~1ms ago)  (~2ms ago)
2674  *
2675  * Let u_i denote the fraction of p_i that the entity was runnable.
2676  *
2677  * We then designate the fractions u_i as our co-efficients, yielding the
2678  * following representation of historical load:
2679  *   u_0 + u_1*y + u_2*y^2 + u_3*y^3 + ...
2680  *
2681  * We choose y based on the with of a reasonably scheduling period, fixing:
2682  *   y^32 = 0.5
2683  *
2684  * This means that the contribution to load ~32ms ago (u_32) will be weighted
2685  * approximately half as much as the contribution to load within the last ms
2686  * (u_0).
2687  *
2688  * When a period "rolls over" and we have new u_0`, multiplying the previous
2689  * sum again by y is sufficient to update:
2690  *   load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... )
2691  *            = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}]
2692  */
2693 static __always_inline int
2694 __update_load_avg(u64 now, int cpu, struct sched_avg *sa,
2695                   unsigned long weight, int running, struct cfs_rq *cfs_rq)
2696 {
2697         u64 delta, scaled_delta, periods;
2698         u32 contrib;
2699         unsigned int delta_w, scaled_delta_w, decayed = 0;
2700         unsigned long scale_freq, scale_cpu;
2701 
2702         delta = now - sa->last_update_time;
2703         /*
2704          * This should only happen when time goes backwards, which it
2705          * unfortunately does during sched clock init when we swap over to TSC.
2706          */
2707         if ((s64)delta < 0) {
2708                 sa->last_update_time = now;
2709                 return 0;
2710         }
2711 
2712         /*
2713          * Use 1024ns as the unit of measurement since it's a reasonable
2714          * approximation of 1us and fast to compute.
2715          */
2716         delta >>= 10;
2717         if (!delta)
2718                 return 0;
2719         sa->last_update_time = now;
2720 
2721         scale_freq = arch_scale_freq_capacity(NULL, cpu);
2722         scale_cpu = arch_scale_cpu_capacity(NULL, cpu);
2723 
2724         /* delta_w is the amount already accumulated against our next period */
2725         delta_w = sa->period_contrib;
2726         if (delta + delta_w >= 1024) {
2727                 decayed = 1;
2728 
2729                 /* how much left for next period will start over, we don't know yet */
2730                 sa->period_contrib = 0;
2731 
2732                 /*
2733                  * Now that we know we're crossing a period boundary, figure
2734                  * out how much from delta we need to complete the current
2735                  * period and accrue it.
2736                  */
2737                 delta_w = 1024 - delta_w;
2738                 scaled_delta_w = cap_scale(delta_w, scale_freq);
2739                 if (weight) {
2740                         sa->load_sum += weight * scaled_delta_w;
2741                         if (cfs_rq) {
2742                                 cfs_rq->runnable_load_sum +=
2743                                                 weight * scaled_delta_w;
2744                         }
2745                 }
2746                 if (running)
2747                         sa->util_sum += scaled_delta_w * scale_cpu;
2748 
2749                 delta -= delta_w;
2750 
2751                 /* Figure out how many additional periods this update spans */
2752                 periods = delta / 1024;
2753                 delta %= 1024;
2754 
2755                 sa->load_sum = decay_load(sa->load_sum, periods + 1);
2756                 if (cfs_rq) {
2757                         cfs_rq->runnable_load_sum =
2758                                 decay_load(cfs_rq->runnable_load_sum, periods + 1);
2759                 }
2760                 sa->util_sum = decay_load((u64)(sa->util_sum), periods + 1);
2761 
2762                 /* Efficiently calculate \sum (1..n_period) 1024*y^i */
2763                 contrib = __compute_runnable_contrib(periods);
2764                 contrib = cap_scale(contrib, scale_freq);
2765                 if (weight) {
2766                         sa->load_sum += weight * contrib;
2767                         if (cfs_rq)
2768                                 cfs_rq->runnable_load_sum += weight * contrib;
2769                 }
2770                 if (running)
2771                         sa->util_sum += contrib * scale_cpu;
2772         }
2773 
2774         /* Remainder of delta accrued against u_0` */
2775         scaled_delta = cap_scale(delta, scale_freq);
2776         if (weight) {
2777                 sa->load_sum += weight * scaled_delta;
2778                 if (cfs_rq)
2779                         cfs_rq->runnable_load_sum += weight * scaled_delta;
2780         }
2781         if (running)
2782                 sa->util_sum += scaled_delta * scale_cpu;
2783 
2784         sa->period_contrib += delta;
2785 
2786         if (decayed) {
2787                 sa->load_avg = div_u64(sa->load_sum, LOAD_AVG_MAX);
2788                 if (cfs_rq) {
2789                         cfs_rq->runnable_load_avg =
2790                                 div_u64(cfs_rq->runnable_load_sum, LOAD_AVG_MAX);
2791                 }
2792                 sa->util_avg = sa->util_sum / LOAD_AVG_MAX;
2793         }
2794 
2795         return decayed;
2796 }
2797 
2798 #ifdef CONFIG_FAIR_GROUP_SCHED
2799 /*
2800  * Updating tg's load_avg is necessary before update_cfs_share (which is done)
2801  * and effective_load (which is not done because it is too costly).
2802  */
2803 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force)
2804 {
2805         long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
2806 
2807         /*
2808          * No need to update load_avg for root_task_group as it is not used.
2809          */
2810         if (cfs_rq->tg == &root_task_group)
2811                 return;
2812 
2813         if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
2814                 atomic_long_add(delta, &cfs_rq->tg->load_avg);
2815                 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
2816         }
2817 }
2818 
2819 /*
2820  * Called within set_task_rq() right before setting a task's cpu. The
2821  * caller only guarantees p->pi_lock is held; no other assumptions,
2822  * including the state of rq->lock, should be made.
2823  */
2824 void set_task_rq_fair(struct sched_entity *se,
2825                       struct cfs_rq *prev, struct cfs_rq *next)
2826 {
2827         if (!sched_feat(ATTACH_AGE_LOAD))
2828                 return;
2829 
2830         /*
2831          * We are supposed to update the task to "current" time, then its up to
2832          * date and ready to go to new CPU/cfs_rq. But we have difficulty in
2833          * getting what current time is, so simply throw away the out-of-date
2834          * time. This will result in the wakee task is less decayed, but giving
2835          * the wakee more load sounds not bad.
2836          */
2837         if (se->avg.last_update_time && prev) {
2838                 u64 p_last_update_time;
2839                 u64 n_last_update_time;
2840 
2841 #ifndef CONFIG_64BIT
2842                 u64 p_last_update_time_copy;
2843                 u64 n_last_update_time_copy;
2844 
2845                 do {
2846                         p_last_update_time_copy = prev->load_last_update_time_copy;
2847                         n_last_update_time_copy = next->load_last_update_time_copy;
2848 
2849                         smp_rmb();
2850 
2851                         p_last_update_time = prev->avg.last_update_time;
2852                         n_last_update_time = next->avg.last_update_time;
2853 
2854                 } while (p_last_update_time != p_last_update_time_copy ||
2855                          n_last_update_time != n_last_update_time_copy);
2856 #else
2857                 p_last_update_time = prev->avg.last_update_time;
2858                 n_last_update_time = next->avg.last_update_time;
2859 #endif
2860                 __update_load_avg(p_last_update_time, cpu_of(rq_of(prev)),
2861                                   &se->avg, 0, 0, NULL);
2862                 se->avg.last_update_time = n_last_update_time;
2863         }
2864 }
2865 #else /* CONFIG_FAIR_GROUP_SCHED */
2866 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {}
2867 #endif /* CONFIG_FAIR_GROUP_SCHED */
2868 
2869 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq);
2870 
2871 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq)
2872 {
2873         struct rq *rq = rq_of(cfs_rq);
2874         int cpu = cpu_of(rq);
2875 
2876         if (cpu == smp_processor_id() && &rq->cfs == cfs_rq) {
2877                 unsigned long max = rq->cpu_capacity_orig;
2878 
2879                 /*
2880                  * There are a few boundary cases this might miss but it should
2881                  * get called often enough that that should (hopefully) not be
2882                  * a real problem -- added to that it only calls on the local
2883                  * CPU, so if we enqueue remotely we'll miss an update, but
2884                  * the next tick/schedule should update.
2885                  *
2886                  * It will not get called when we go idle, because the idle
2887                  * thread is a different class (!fair), nor will the utilization
2888                  * number include things like RT tasks.
2889                  *
2890                  * As is, the util number is not freq-invariant (we'd have to
2891                  * implement arch_scale_freq_capacity() for that).
2892                  *
2893                  * See cpu_util().
2894                  */
2895                 cpufreq_update_util(rq_clock(rq),
2896                                     min(cfs_rq->avg.util_avg, max), max);
2897         }
2898 }
2899 
2900 /*
2901  * Unsigned subtract and clamp on underflow.
2902  *
2903  * Explicitly do a load-store to ensure the intermediate value never hits
2904  * memory. This allows lockless observations without ever seeing the negative
2905  * values.
2906  */
2907 #define sub_positive(_ptr, _val) do {                           \
2908         typeof(_ptr) ptr = (_ptr);                              \
2909         typeof(*ptr) val = (_val);                              \
2910         typeof(*ptr) res, var = READ_ONCE(*ptr);                \
2911         res = var - val;                                        \
2912         if (res > var)                                          \
2913                 res = 0;                                        \
2914         WRITE_ONCE(*ptr, res);                                  \
2915 } while (0)
2916 
2917 /* Group cfs_rq's load_avg is used for task_h_load and update_cfs_share */
2918 static inline int
2919 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq, bool update_freq)
2920 {
2921         struct sched_avg *sa = &cfs_rq->avg;
2922         int decayed, removed_load = 0, removed_util = 0;
2923 
2924         if (atomic_long_read(&cfs_rq->removed_load_avg)) {
2925                 s64 r = atomic_long_xchg(&cfs_rq->removed_load_avg, 0);
2926                 sub_positive(&sa->load_avg, r);
2927                 sub_positive(&sa->load_sum, r * LOAD_AVG_MAX);
2928                 removed_load = 1;
2929         }
2930 
2931         if (atomic_long_read(&cfs_rq->removed_util_avg)) {
2932                 long r = atomic_long_xchg(&cfs_rq->removed_util_avg, 0);
2933                 sub_positive(&sa->util_avg, r);
2934                 sub_positive(&sa->util_sum, r * LOAD_AVG_MAX);
2935                 removed_util = 1;
2936         }
2937 
2938         decayed = __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa,
2939                 scale_load_down(cfs_rq->load.weight), cfs_rq->curr != NULL, cfs_rq);
2940 
2941 #ifndef CONFIG_64BIT
2942         smp_wmb();
2943         cfs_rq->load_last_update_time_copy = sa->last_update_time;
2944 #endif
2945 
2946         if (update_freq && (decayed || removed_util))
2947                 cfs_rq_util_change(cfs_rq);
2948 
2949         return decayed || removed_load;
2950 }
2951 
2952 /* Update task and its cfs_rq load average */
2953 static inline void update_load_avg(struct sched_entity *se, int update_tg)
2954 {
2955         struct cfs_rq *cfs_rq = cfs_rq_of(se);
2956         u64 now = cfs_rq_clock_task(cfs_rq);
2957         struct rq *rq = rq_of(cfs_rq);
2958         int cpu = cpu_of(rq);
2959 
2960         /*
2961          * Track task load average for carrying it to new CPU after migrated, and
2962          * track group sched_entity load average for task_h_load calc in migration
2963          */
2964         __update_load_avg(now, cpu, &se->avg,
2965                           se->on_rq * scale_load_down(se->load.weight),
2966                           cfs_rq->curr == se, NULL);
2967 
2968         if (update_cfs_rq_load_avg(now, cfs_rq, true) && update_tg)
2969                 update_tg_load_avg(cfs_rq, 0);
2970 }
2971 
2972 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2973 {
2974         if (!sched_feat(ATTACH_AGE_LOAD))
2975                 goto skip_aging;
2976 
2977         /*
2978          * If we got migrated (either between CPUs or between cgroups) we'll
2979          * have aged the average right before clearing @last_update_time.
2980          */
2981         if (se->avg.last_update_time) {
2982                 __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)),
2983                                   &se->avg, 0, 0, NULL);
2984 
2985                 /*
2986                  * XXX: we could have just aged the entire load away if we've been
2987                  * absent from the fair class for too long.
2988                  */
2989         }
2990 
2991 skip_aging:
2992         se->avg.last_update_time = cfs_rq->avg.last_update_time;
2993         cfs_rq->avg.load_avg += se->avg.load_avg;
2994         cfs_rq->avg.load_sum += se->avg.load_sum;
2995         cfs_rq->avg.util_avg += se->avg.util_avg;
2996         cfs_rq->avg.util_sum += se->avg.util_sum;
2997 
2998         cfs_rq_util_change(cfs_rq);
2999 }
3000 
3001 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3002 {
3003         __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)),
3004                           &se->avg, se->on_rq * scale_load_down(se->load.weight),
3005                           cfs_rq->curr == se, NULL);
3006 
3007         sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
3008         sub_positive(&cfs_rq->avg.load_sum, se->avg.load_sum);
3009         sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
3010         sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);
3011 
3012         cfs_rq_util_change(cfs_rq);
3013 }
3014 
3015 /* Add the load generated by se into cfs_rq's load average */
3016 static inline void
3017 enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3018 {
3019         struct sched_avg *sa = &se->avg;
3020         u64 now = cfs_rq_clock_task(cfs_rq);
3021         int migrated, decayed;
3022 
3023         migrated = !sa->last_update_time;
3024         if (!migrated) {
3025                 __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa,
3026                         se->on_rq * scale_load_down(se->load.weight),
3027                         cfs_rq->curr == se, NULL);
3028         }
3029 
3030         decayed = update_cfs_rq_load_avg(now, cfs_rq, !migrated);
3031 
3032         cfs_rq->runnable_load_avg += sa->load_avg;
3033         cfs_rq->runnable_load_sum += sa->load_sum;
3034 
3035         if (migrated)
3036                 attach_entity_load_avg(cfs_rq, se);
3037 
3038         if (decayed || migrated)
3039                 update_tg_load_avg(cfs_rq, 0);
3040 }
3041 
3042 /* Remove the runnable load generated by se from cfs_rq's runnable load average */
3043 static inline void
3044 dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3045 {
3046         update_load_avg(se, 1);
3047 
3048         cfs_rq->runnable_load_avg =
3049                 max_t(long, cfs_rq->runnable_load_avg - se->avg.load_avg, 0);
3050         cfs_rq->runnable_load_sum =
3051                 max_t(s64,  cfs_rq->runnable_load_sum - se->avg.load_sum, 0);
3052 }
3053 
3054 #ifndef CONFIG_64BIT
3055 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3056 {
3057         u64 last_update_time_copy;
3058         u64 last_update_time;
3059 
3060         do {
3061                 last_update_time_copy = cfs_rq->load_last_update_time_copy;
3062                 smp_rmb();
3063                 last_update_time = cfs_rq->avg.last_update_time;
3064         } while (last_update_time != last_update_time_copy);
3065 
3066         return last_update_time;
3067 }
3068 #else
3069 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3070 {
3071         return cfs_rq->avg.last_update_time;
3072 }
3073 #endif
3074 
3075 /*
3076  * Task first catches up with cfs_rq, and then subtract
3077  * itself from the cfs_rq (task must be off the queue now).
3078  */
3079 void remove_entity_load_avg(struct sched_entity *se)
3080 {
3081         struct cfs_rq *cfs_rq = cfs_rq_of(se);
3082         u64 last_update_time;
3083 
3084         /*
3085          * Newly created task or never used group entity should not be removed
3086          * from its (source) cfs_rq
3087          */
3088         if (se->avg.last_update_time == 0)
3089                 return;
3090 
3091         last_update_time = cfs_rq_last_update_time(cfs_rq);
3092 
3093         __update_load_avg(last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, NULL);
3094         atomic_long_add(se->avg.load_avg, &cfs_rq->removed_load_avg);
3095         atomic_long_add(se->avg.util_avg, &cfs_rq->removed_util_avg);
3096 }
3097 
3098 static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq)
3099 {
3100         return cfs_rq->runnable_load_avg;
3101 }
3102 
3103 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
3104 {
3105         return cfs_rq->avg.load_avg;
3106 }
3107 
3108 static int idle_balance(struct rq *this_rq);
3109 
3110 #else /* CONFIG_SMP */
3111 
3112 static inline void update_load_avg(struct sched_entity *se, int not_used)
3113 {
3114         struct cfs_rq *cfs_rq = cfs_rq_of(se);
3115         struct rq *rq = rq_of(cfs_rq);
3116 
3117         cpufreq_trigger_update(rq_clock(rq));
3118 }
3119 
3120 static inline void
3121 enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
3122 static inline void
3123 dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
3124 static inline void remove_entity_load_avg(struct sched_entity *se) {}
3125 
3126 static inline void
3127 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
3128 static inline void
3129 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
3130 
3131 static inline int idle_balance(struct rq *rq)
3132 {
3133         return 0;
3134 }
3135 
3136 #endif /* CONFIG_SMP */
3137 
3138 static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
3139 {
3140 #ifdef CONFIG_SCHEDSTATS
3141         struct task_struct *tsk = NULL;
3142 
3143         if (entity_is_task(se))
3144                 tsk = task_of(se);
3145 
3146         if (se->statistics.sleep_start) {
3147                 u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.sleep_start;
3148 
3149                 if ((s64)delta < 0)
3150                         delta = 0;
3151 
3152                 if (unlikely(delta > se->statistics.sleep_max))
3153                         se->statistics.sleep_max = delta;
3154 
3155                 se->statistics.sleep_start = 0;
3156                 se->statistics.sum_sleep_runtime += delta;
3157 
3158                 if (tsk) {
3159                         account_scheduler_latency(tsk, delta >> 10, 1);
3160                         trace_sched_stat_sleep(tsk, delta);
3161                 }
3162         }
3163         if (se->statistics.block_start) {
3164                 u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.block_start;
3165 
3166                 if ((s64)delta < 0)
3167                         delta = 0;
3168 
3169                 if (unlikely(delta > se->statistics.block_max))
3170                         se->statistics.block_max = delta;
3171 
3172                 se->statistics.block_start = 0;
3173                 se->statistics.sum_sleep_runtime += delta;
3174 
3175                 if (tsk) {
3176                         if (tsk->in_iowait) {
3177                                 se->statistics.iowait_sum += delta;
3178                                 se->statistics.iowait_count++;
3179                                 trace_sched_stat_iowait(tsk, delta);
3180                         }
3181 
3182                         trace_sched_stat_blocked(tsk, delta);
3183 
3184                         /*
3185                          * Blocking time is in units of nanosecs, so shift by
3186                          * 20 to get a milliseconds-range estimation of the
3187                          * amount of time that the task spent sleeping:
3188                          */
3189                         if (unlikely(prof_on == SLEEP_PROFILING)) {
3190                                 profile_hits(SLEEP_PROFILING,
3191                                                 (void *)get_wchan(tsk),
3192                                                 delta >> 20);
3193                         }
3194                         account_scheduler_latency(tsk, delta >> 10, 0);
3195                 }
3196         }
3197 #endif
3198 }
3199 
3200 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
3201 {
3202 #ifdef CONFIG_SCHED_DEBUG
3203         s64 d = se->vruntime - cfs_rq->min_vruntime;
3204 
3205         if (d < 0)
3206                 d = -d;
3207 
3208         if (d > 3*sysctl_sched_latency)
3209                 schedstat_inc(cfs_rq, nr_spread_over);
3210 #endif
3211 }
3212 
3213 static void
3214 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
3215 {
3216         u64 vruntime = cfs_rq->min_vruntime;
3217 
3218         /*
3219          * The 'current' period is already promised to the current tasks,
3220          * however the extra weight of the new task will slow them down a
3221          * little, place the new task so that it fits in the slot that
3222          * stays open at the end.
3223          */
3224         if (initial && sched_feat(START_DEBIT))
3225                 vruntime += sched_vslice(cfs_rq, se);
3226 
3227         /* sleeps up to a single latency don't count. */
3228         if (!initial) {
3229                 unsigned long thresh = sysctl_sched_latency;
3230 
3231                 /*
3232                  * Halve their sleep time's effect, to allow
3233                  * for a gentler effect of sleepers:
3234                  */
3235                 if (sched_feat(GENTLE_FAIR_SLEEPERS))
3236                         thresh >>= 1;
3237 
3238                 vruntime -= thresh;
3239         }
3240 
3241         /* ensure we never gain time by being placed backwards. */
3242         se->vruntime = max_vruntime(se->vruntime, vruntime);
3243 }
3244 
3245 static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
3246 
3247 static inline void check_schedstat_required(void)
3248 {
3249 #ifdef CONFIG_SCHEDSTATS
3250         if (schedstat_enabled())
3251                 return;
3252 
3253         /* Force schedstat enabled if a dependent tracepoint is active */
3254         if (trace_sched_stat_wait_enabled()    ||
3255                         trace_sched_stat_sleep_enabled()   ||
3256                         trace_sched_stat_iowait_enabled()  ||
3257                         trace_sched_stat_blocked_enabled() ||
3258                         trace_sched_stat_runtime_enabled())  {
3259                 printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, "
3260                              "stat_blocked and stat_runtime require the "
3261                              "kernel parameter schedstats=enabled or "
3262                              "kernel.sched_schedstats=1\n");
3263         }
3264 #endif
3265 }
3266 
3267 
3268 /*
3269  * MIGRATION
3270  *
3271  *      dequeue
3272  *        update_curr()
3273  *          update_min_vruntime()
3274  *        vruntime -= min_vruntime
3275  *
3276  *      enqueue
3277  *        update_curr()
3278  *          update_min_vruntime()
3279  *        vruntime += min_vruntime
3280  *
3281  * this way the vruntime transition between RQs is done when both
3282  * min_vruntime are up-to-date.
3283  *
3284  * WAKEUP (remote)
3285  *
3286  *      ->migrate_task_rq_fair() (p->state == TASK_WAKING)
3287  *        vruntime -= min_vruntime
3288  *
3289  *      enqueue
3290  *        update_curr()
3291  *          update_min_vruntime()
3292  *        vruntime += min_vruntime
3293  *
3294  * this way we don't have the most up-to-date min_vruntime on the originating
3295  * CPU and an up-to-date min_vruntime on the destination CPU.
3296  */
3297 
3298 static void
3299 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
3300 {
3301         bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED);
3302         bool curr = cfs_rq->curr == se;
3303 
3304         /*
3305          * If we're the current task, we must renormalise before calling
3306          * update_curr().
3307          */
3308         if (renorm && curr)
3309                 se->vruntime += cfs_rq->min_vruntime;
3310 
3311         update_curr(cfs_rq);
3312 
3313         /*
3314          * Otherwise, renormalise after, such that we're placed at the current
3315          * moment in time, instead of some random moment in the past. Being
3316          * placed in the past could significantly boost this task to the
3317          * fairness detriment of existing tasks.
3318          */
3319         if (renorm && !curr)
3320                 se->vruntime += cfs_rq->min_vruntime;
3321 
3322         enqueue_entity_load_avg(cfs_rq, se);
3323         account_entity_enqueue(cfs_rq, se);
3324         update_cfs_shares(cfs_rq);
3325 
3326         if (flags & ENQUEUE_WAKEUP) {
3327                 place_entity(cfs_rq, se, 0);
3328                 if (schedstat_enabled())
3329                         enqueue_sleeper(cfs_rq, se);
3330         }
3331 
3332         check_schedstat_required();
3333         if (schedstat_enabled()) {
3334                 update_stats_enqueue(cfs_rq, se);
3335                 check_spread(cfs_rq, se);
3336         }
3337         if (!curr)
3338                 __enqueue_entity(cfs_rq, se);
3339         se->on_rq = 1;
3340 
3341         if (cfs_rq->nr_running == 1) {
3342                 list_add_leaf_cfs_rq(cfs_rq);
3343                 check_enqueue_throttle(cfs_rq);
3344         }
3345 }
3346 
3347 static void __clear_buddies_last(struct sched_entity *se)
3348 {
3349         for_each_sched_entity(se) {
3350                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
3351                 if (cfs_rq->last != se)
3352                         break;
3353 
3354                 cfs_rq->last = NULL;
3355         }
3356 }
3357 
3358 static void __clear_buddies_next(struct sched_entity *se)
3359 {
3360         for_each_sched_entity(se) {
3361                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
3362                 if (cfs_rq->next != se)
3363                         break;
3364 
3365                 cfs_rq->next = NULL;
3366         }
3367 }
3368 
3369 static void __clear_buddies_skip(struct sched_entity *se)
3370 {
3371         for_each_sched_entity(se) {
3372                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
3373                 if (cfs_rq->skip != se)
3374                         break;
3375 
3376                 cfs_rq->skip = NULL;
3377         }
3378 }
3379 
3380 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
3381 {
3382         if (cfs_rq->last == se)
3383                 __clear_buddies_last(se);
3384 
3385         if (cfs_rq->next == se)
3386                 __clear_buddies_next(se);
3387 
3388         if (cfs_rq->skip == se)
3389                 __clear_buddies_skip(se);
3390 }
3391 
3392 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
3393 
3394 static void
3395 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
3396 {
3397         /*
3398          * Update run-time statistics of the 'current'.
3399          */
3400         update_curr(cfs_rq);
3401         dequeue_entity_load_avg(cfs_rq, se);
3402 
3403         if (schedstat_enabled())
3404                 update_stats_dequeue(cfs_rq, se, flags);
3405 
3406         clear_buddies(cfs_rq, se);
3407 
3408         if (se != cfs_rq->curr)
3409                 __dequeue_entity(cfs_rq, se);
3410         se->on_rq = 0;
3411         account_entity_dequeue(cfs_rq, se);
3412 
3413         /*
3414          * Normalize the entity after updating the min_vruntime because the
3415          * update can refer to the ->curr item and we need to reflect this
3416          * movement in our normalized position.
3417          */
3418         if (!(flags & DEQUEUE_SLEEP))
3419                 se->vruntime -= cfs_rq->min_vruntime;
3420 
3421         /* return excess runtime on last dequeue */
3422         return_cfs_rq_runtime(cfs_rq);
3423 
3424         update_min_vruntime(cfs_rq);
3425         update_cfs_shares(cfs_rq);
3426 }
3427 
3428 /*
3429  * Preempt the current task with a newly woken task if needed:
3430  */
3431 static void
3432 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
3433 {
3434         unsigned long ideal_runtime, delta_exec;
3435         struct sched_entity *se;
3436         s64 delta;
3437 
3438         ideal_runtime = sched_slice(cfs_rq, curr);
3439         delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
3440         if (delta_exec > ideal_runtime) {
3441                 resched_curr(rq_of(cfs_rq));
3442                 /*
3443                  * The current task ran long enough, ensure it doesn't get
3444                  * re-elected due to buddy favours.
3445                  */
3446                 clear_buddies(cfs_rq, curr);
3447                 return;
3448         }
3449 
3450         /*
3451          * Ensure that a task that missed wakeup preemption by a
3452          * narrow margin doesn't have to wait for a full slice.
3453          * This also mitigates buddy induced latencies under load.
3454          */
3455         if (delta_exec < sysctl_sched_min_granularity)
3456                 return;
3457 
3458         se = __pick_first_entity(cfs_rq);
3459         delta = curr->vruntime - se->vruntime;
3460 
3461         if (delta < 0)
3462                 return;
3463 
3464         if (delta > ideal_runtime)
3465                 resched_curr(rq_of(cfs_rq));
3466 }
3467 
3468 static void
3469 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
3470 {
3471         /* 'current' is not kept within the tree. */
3472         if (se->on_rq) {
3473                 /*
3474                  * Any task has to be enqueued before it get to execute on
3475                  * a CPU. So account for the time it spent waiting on the
3476                  * runqueue.
3477                  */
3478                 if (schedstat_enabled())
3479                         update_stats_wait_end(cfs_rq, se);
3480                 __dequeue_entity(cfs_rq, se);
3481                 update_load_avg(se, 1);
3482         }
3483 
3484         update_stats_curr_start(cfs_rq, se);
3485         cfs_rq->curr = se;
3486 #ifdef CONFIG_SCHEDSTATS
3487         /*
3488          * Track our maximum slice length, if the CPU's load is at
3489          * least twice that of our own weight (i.e. dont track it
3490          * when there are only lesser-weight tasks around):
3491          */
3492         if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
3493                 se->statistics.slice_max = max(se->statistics.slice_max,
3494                         se->sum_exec_runtime - se->prev_sum_exec_runtime);
3495         }
3496 #endif
3497         se->prev_sum_exec_runtime = se->sum_exec_runtime;
3498 }
3499 
3500 static int
3501 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
3502 
3503 /*
3504  * Pick the next process, keeping these things in mind, in this order:
3505  * 1) keep things fair between processes/task groups
3506  * 2) pick the "next" process, since someone really wants that to run
3507  * 3) pick the "last" process, for cache locality
3508  * 4) do not run the "skip" process, if something else is available
3509  */
3510 static struct sched_entity *
3511 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
3512 {
3513         struct sched_entity *left = __pick_first_entity(cfs_rq);
3514         struct sched_entity *se;
3515 
3516         /*
3517          * If curr is set we have to see if its left of the leftmost entity
3518          * still in the tree, provided there was anything in the tree at all.
3519          */
3520         if (!left || (curr && entity_before(curr, left)))
3521                 left = curr;
3522 
3523         se = left; /* ideally we run the leftmost entity */
3524 
3525         /*
3526          * Avoid running the skip buddy, if running something else can
3527          * be done without getting too unfair.
3528          */
3529         if (cfs_rq->skip == se) {
3530                 struct sched_entity *second;
3531 
3532                 if (se == curr) {
3533                         second = __pick_first_entity(cfs_rq);
3534                 } else {
3535                         second = __pick_next_entity(se);
3536                         if (!second || (curr && entity_before(curr, second)))
3537                                 second = curr;
3538                 }
3539 
3540                 if (second && wakeup_preempt_entity(second, left) < 1)
3541                         se = second;
3542         }
3543 
3544         /*
3545          * Prefer last buddy, try to return the CPU to a preempted task.
3546          */
3547         if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
3548                 se = cfs_rq->last;
3549 
3550         /*
3551          * Someone really wants this to run. If it's not unfair, run it.
3552          */
3553         if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
3554                 se = cfs_rq->next;
3555 
3556         clear_buddies(cfs_rq, se);
3557 
3558         return se;
3559 }
3560 
3561 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
3562 
3563 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
3564 {
3565         /*
3566          * If still on the runqueue then deactivate_task()
3567          * was not called and update_curr() has to be done:
3568          */
3569         if (prev->on_rq)
3570                 update_curr(cfs_rq);
3571 
3572         /* throttle cfs_rqs exceeding runtime */
3573         check_cfs_rq_runtime(cfs_rq);
3574 
3575         if (schedstat_enabled()) {
3576                 check_spread(cfs_rq, prev);
3577                 if (prev->on_rq)
3578                         update_stats_wait_start(cfs_rq, prev);
3579         }
3580 
3581         if (prev->on_rq) {
3582                 /* Put 'current' back into the tree. */
3583                 __enqueue_entity(cfs_rq, prev);
3584                 /* in !on_rq case, update occurred at dequeue */
3585                 update_load_avg(prev, 0);
3586         }
3587         cfs_rq->curr = NULL;
3588 }
3589 
3590 static void
3591 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
3592 {
3593         /*
3594          * Update run-time statistics of the 'current'.
3595          */
3596         update_curr(cfs_rq);
3597 
3598         /*
3599          * Ensure that runnable average is periodically updated.
3600          */
3601         update_load_avg(curr, 1);
3602         update_cfs_shares(cfs_rq);
3603 
3604 #ifdef CONFIG_SCHED_HRTICK
3605         /*
3606          * queued ticks are scheduled to match the slice, so don't bother
3607          * validating it and just reschedule.
3608          */
3609         if (queued) {
3610                 resched_curr(rq_of(cfs_rq));
3611                 return;
3612         }
3613         /*
3614          * don't let the period tick interfere with the hrtick preemption
3615          */
3616         if (!sched_feat(DOUBLE_TICK) &&
3617                         hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
3618                 return;
3619 #endif
3620 
3621         if (cfs_rq->nr_running > 1)
3622                 check_preempt_tick(cfs_rq, curr);
3623 }
3624 
3625 
3626 /**************************************************
3627  * CFS bandwidth control machinery
3628  */
3629 
3630 #ifdef CONFIG_CFS_BANDWIDTH
3631 
3632 #ifdef HAVE_JUMP_LABEL
3633 static struct static_key __cfs_bandwidth_used;
3634 
3635 static inline bool cfs_bandwidth_used(void)
3636 {
3637         return static_key_false(&__cfs_bandwidth_used);
3638 }
3639 
3640 void cfs_bandwidth_usage_inc(void)
3641 {
3642         static_key_slow_inc(&__cfs_bandwidth_used);
3643 }
3644 
3645 void cfs_bandwidth_usage_dec(void)
3646 {
3647         static_key_slow_dec(&__cfs_bandwidth_used);
3648 }
3649 #else /* HAVE_JUMP_LABEL */
3650 static bool cfs_bandwidth_used(void)
3651 {
3652         return true;
3653 }
3654 
3655 void cfs_bandwidth_usage_inc(void) {}
3656 void cfs_bandwidth_usage_dec(void) {}
3657 #endif /* HAVE_JUMP_LABEL */
3658 
3659 /*
3660  * default period for cfs group bandwidth.
3661  * default: 0.1s, units: nanoseconds
3662  */
3663 static inline u64 default_cfs_period(void)
3664 {
3665         return 100000000ULL;
3666 }
3667 
3668 static inline u64 sched_cfs_bandwidth_slice(void)
3669 {
3670         return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
3671 }
3672 
3673 /*
3674  * Replenish runtime according to assigned quota and update expiration time.
3675  * We use sched_clock_cpu directly instead of rq->clock to avoid adding
3676  * additional synchronization around rq->lock.
3677  *
3678  * requires cfs_b->lock
3679  */
3680 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
3681 {
3682         u64 now;
3683 
3684         if (cfs_b->quota == RUNTIME_INF)
3685                 return;
3686 
3687         now = sched_clock_cpu(smp_processor_id());
3688         cfs_b->runtime = cfs_b->quota;
3689         cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
3690 }
3691 
3692 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
3693 {
3694         return &tg->cfs_bandwidth;
3695 }
3696 
3697 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
3698 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
3699 {
3700         if (unlikely(cfs_rq->throttle_count))
3701                 return cfs_rq->throttled_clock_task;
3702 
3703         return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
3704 }
3705 
3706 /* returns 0 on failure to allocate runtime */
3707 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
3708 {
3709         struct task_group *tg = cfs_rq->tg;
3710         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
3711         u64 amount = 0, min_amount, expires;
3712 
3713         /* note: this is a positive sum as runtime_remaining <= 0 */
3714         min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
3715 
3716         raw_spin_lock(&cfs_b->lock);
3717         if (cfs_b->quota == RUNTIME_INF)
3718                 amount = min_amount;
3719         else {
3720                 start_cfs_bandwidth(cfs_b);
3721 
3722                 if (cfs_b->runtime > 0) {
3723                         amount = min(cfs_b->runtime, min_amount);
3724                         cfs_b->runtime -= amount;
3725                         cfs_b->idle = 0;
3726                 }
3727         }
3728         expires = cfs_b->runtime_expires;
3729         raw_spin_unlock(&cfs_b->lock);
3730 
3731         cfs_rq->runtime_remaining += amount;
3732         /*
3733          * we may have advanced our local expiration to account for allowed
3734          * spread between our sched_clock and the one on which runtime was
3735          * issued.
3736          */
3737         if ((s64)(expires - cfs_rq->runtime_expires) > 0)
3738                 cfs_rq->runtime_expires = expires;
3739 
3740         return cfs_rq->runtime_remaining > 0;
3741 }
3742 
3743 /*
3744  * Note: This depends on the synchronization provided by sched_clock and the
3745  * fact that rq->clock snapshots this value.
3746  */
3747 static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq)
3748 {
3749         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
3750 
3751         /* if the deadline is ahead of our clock, nothing to do */
3752         if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0))
3753                 return;
3754 
3755         if (cfs_rq->runtime_remaining < 0)
3756                 return;
3757 
3758         /*
3759          * If the local deadline has passed we have to consider the
3760          * possibility that our sched_clock is 'fast' and the global deadline
3761          * has not truly expired.
3762          *
3763          * Fortunately we can check determine whether this the case by checking
3764          * whether the global deadline has advanced. It is valid to compare
3765          * cfs_b->runtime_expires without any locks since we only care about
3766          * exact equality, so a partial write will still work.
3767          */
3768 
3769         if (cfs_rq->runtime_expires != cfs_b->runtime_expires) {
3770                 /* extend local deadline, drift is bounded above by 2 ticks */
3771                 cfs_rq->runtime_expires += TICK_NSEC;
3772         } else {
3773                 /* global deadline is ahead, expiration has passed */
3774                 cfs_rq->runtime_remaining = 0;
3775         }
3776 }
3777 
3778 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
3779 {
3780         /* dock delta_exec before expiring quota (as it could span periods) */
3781         cfs_rq->runtime_remaining -= delta_exec;
3782         expire_cfs_rq_runtime(cfs_rq);
3783 
3784         if (likely(cfs_rq->runtime_remaining > 0))
3785                 return;
3786 
3787         /*
3788          * if we're unable to extend our runtime we resched so that the active
3789          * hierarchy can be throttled
3790          */
3791         if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
3792                 resched_curr(rq_of(cfs_rq));
3793 }
3794 
3795 static __always_inline
3796 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
3797 {
3798         if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
3799                 return;
3800 
3801         __account_cfs_rq_runtime(cfs_rq, delta_exec);
3802 }
3803 
3804 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
3805 {
3806         return cfs_bandwidth_used() && cfs_rq->throttled;
3807 }
3808 
3809 /* check whether cfs_rq, or any parent, is throttled */
3810 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
3811 {
3812         return cfs_bandwidth_used() && cfs_rq->throttle_count;
3813 }
3814 
3815 /*
3816  * Ensure that neither of the group entities corresponding to src_cpu or
3817  * dest_cpu are members of a throttled hierarchy when performing group
3818  * load-balance operations.
3819  */
3820 static inline int throttled_lb_pair(struct task_group *tg,
3821                                     int src_cpu, int dest_cpu)
3822 {
3823         struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
3824 
3825         src_cfs_rq = tg->cfs_rq[src_cpu];
3826         dest_cfs_rq = tg->cfs_rq[dest_cpu];
3827 
3828         return throttled_hierarchy(src_cfs_rq) ||
3829                throttled_hierarchy(dest_cfs_rq);
3830 }
3831 
3832 /* updated child weight may affect parent so we have to do this bottom up */
3833 static int tg_unthrottle_up(struct task_group *tg, void *data)
3834 {
3835         struct rq *rq = data;
3836         struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
3837 
3838         cfs_rq->throttle_count--;
3839 #ifdef CONFIG_SMP
3840         if (!cfs_rq->throttle_count) {
3841                 /* adjust cfs_rq_clock_task() */
3842                 cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
3843                                              cfs_rq->throttled_clock_task;
3844         }
3845 #endif
3846 
3847         return 0;
3848 }
3849 
3850 static int tg_throttle_down(struct task_group *tg, void *data)
3851 {
3852         struct rq *rq = data;
3853         struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
3854 
3855         /* group is entering throttled state, stop time */
3856         if (!cfs_rq->throttle_count)
3857                 cfs_rq->throttled_clock_task = rq_clock_task(rq);
3858         cfs_rq->throttle_count++;
3859 
3860         return 0;
3861 }
3862 
3863 static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
3864 {
3865         struct rq *rq = rq_of(cfs_rq);
3866         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
3867         struct sched_entity *se;
3868         long task_delta, dequeue = 1;
3869         bool empty;
3870 
3871         se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
3872 
3873         /* freeze hierarchy runnable averages while throttled */
3874         rcu_read_lock();
3875         walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
3876         rcu_read_unlock();
3877 
3878         task_delta = cfs_rq->h_nr_running;
3879         for_each_sched_entity(se) {
3880                 struct cfs_rq *qcfs_rq = cfs_rq_of(se);
3881                 /* throttled entity or throttle-on-deactivate */
3882                 if (!se->on_rq)
3883                         break;
3884 
3885                 if (dequeue)
3886                         dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
3887                 qcfs_rq->h_nr_running -= task_delta;
3888 
3889                 if (qcfs_rq->load.weight)
3890                         dequeue = 0;
3891         }
3892 
3893         if (!se)
3894                 sub_nr_running(rq, task_delta);
3895 
3896         cfs_rq->throttled = 1;
3897         cfs_rq->throttled_clock = rq_clock(rq);
3898         raw_spin_lock(&cfs_b->lock);
3899         empty = list_empty(&cfs_b->throttled_cfs_rq);
3900 
3901         /*
3902          * Add to the _head_ of the list, so that an already-started
3903          * distribute_cfs_runtime will not see us
3904          */
3905         list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
3906 
3907         /*
3908          * If we're the first throttled task, make sure the bandwidth
3909          * timer is running.
3910          */
3911         if (empty)
3912                 start_cfs_bandwidth(cfs_b);
3913 
3914         raw_spin_unlock(&cfs_b->lock);
3915 }
3916 
3917 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
3918 {
3919         struct rq *rq = rq_of(cfs_rq);
3920         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
3921         struct sched_entity *se;
3922         int enqueue = 1;
3923         long task_delta;
3924 
3925         se = cfs_rq->tg->se[cpu_of(rq)];
3926 
3927         cfs_rq->throttled = 0;
3928 
3929         update_rq_clock(rq);
3930 
3931         raw_spin_lock(&cfs_b->lock);
3932         cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
3933         list_del_rcu(&cfs_rq->throttled_list);
3934         raw_spin_unlock(&cfs_b->lock);
3935 
3936         /* update hierarchical throttle state */
3937         walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
3938 
3939         if (!cfs_rq->load.weight)
3940                 return;
3941 
3942         task_delta = cfs_rq->h_nr_running;
3943         for_each_sched_entity(se) {
3944                 if (se->on_rq)
3945                         enqueue = 0;
3946 
3947                 cfs_rq = cfs_rq_of(se);
3948                 if (enqueue)
3949                         enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
3950                 cfs_rq->h_nr_running += task_delta;
3951 
3952                 if (cfs_rq_throttled(cfs_rq))
3953                         break;
3954         }
3955 
3956         if (!se)
3957                 add_nr_running(rq, task_delta);
3958 
3959         /* determine whether we need to wake up potentially idle cpu */
3960         if (rq->curr == rq->idle && rq->cfs.nr_running)
3961                 resched_curr(rq);
3962 }
3963 
3964 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,
3965                 u64 remaining, u64 expires)
3966 {
3967         struct cfs_rq *cfs_rq;
3968         u64 runtime;
3969         u64 starting_runtime = remaining;
3970 
3971         rcu_read_lock();
3972         list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
3973                                 throttled_list) {
3974                 struct rq *rq = rq_of(cfs_rq);
3975 
3976                 raw_spin_lock(&rq->lock);
3977                 if (!cfs_rq_throttled(cfs_rq))
3978                         goto next;
3979 
3980                 runtime = -cfs_rq->runtime_remaining + 1;
3981                 if (runtime > remaining)
3982                         runtime = remaining;
3983                 remaining -= runtime;
3984 
3985                 cfs_rq->runtime_remaining += runtime;
3986                 cfs_rq->runtime_expires = expires;
3987 
3988                 /* we check whether we're throttled above */
3989                 if (cfs_rq->runtime_remaining > 0)
3990                         unthrottle_cfs_rq(cfs_rq);
3991 
3992 next:
3993                 raw_spin_unlock(&rq->lock);
3994 
3995                 if (!remaining)
3996                         break;
3997         }
3998         rcu_read_unlock();
3999 
4000         return starting_runtime - remaining;
4001 }
4002 
4003 /*
4004  * Responsible for refilling a task_group's bandwidth and unthrottling its
4005  * cfs_rqs as appropriate. If there has been no activity within the last
4006  * period the timer is deactivated until scheduling resumes; cfs_b->idle is
4007  * used to track this state.
4008  */
4009 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
4010 {
4011         u64 runtime, runtime_expires;
4012         int throttled;
4013 
4014         /* no need to continue the timer with no bandwidth constraint */
4015         if (cfs_b->quota == RUNTIME_INF)
4016                 goto out_deactivate;
4017 
4018         throttled = !list_empty(&cfs_b->throttled_cfs_rq);
4019         cfs_b->nr_periods += overrun;
4020 
4021         /*
4022          * idle depends on !throttled (for the case of a large deficit), and if
4023          * we're going inactive then everything else can be deferred
4024          */
4025         if (cfs_b->idle && !throttled)
4026                 goto out_deactivate;
4027 
4028         __refill_cfs_bandwidth_runtime(cfs_b);
4029 
4030         if (!throttled) {
4031                 /* mark as potentially idle for the upcoming period */
4032                 cfs_b->idle = 1;
4033                 return 0;
4034         }
4035 
4036         /* account preceding periods in which throttling occurred */
4037         cfs_b->nr_throttled += overrun;
4038 
4039         runtime_expires = cfs_b->runtime_expires;
4040 
4041         /*
4042          * This check is repeated as we are holding onto the new bandwidth while
4043          * we unthrottle. This can potentially race with an unthrottled group
4044          * trying to acquire new bandwidth from the global pool. This can result
4045          * in us over-using our runtime if it is all used during this loop, but
4046          * only by limited amounts in that extreme case.
4047          */
4048         while (throttled && cfs_b->runtime > 0) {
4049                 runtime = cfs_b->runtime;
4050                 raw_spin_unlock(&cfs_b->lock);
4051                 /* we can't nest cfs_b->lock while distributing bandwidth */
4052                 runtime = distribute_cfs_runtime(cfs_b, runtime,
4053                                                  runtime_expires);
4054                 raw_spin_lock(&cfs_b->lock);
4055 
4056                 throttled = !list_empty(&cfs_b->throttled_cfs_rq);
4057 
4058                 cfs_b->runtime -= min(runtime, cfs_b->runtime);
4059         }
4060 
4061         /*
4062          * While we are ensured activity in the period following an
4063          * unthrottle, this also covers the case in which the new bandwidth is
4064          * insufficient to cover the existing bandwidth deficit.  (Forcing the
4065          * timer to remain active while there are any throttled entities.)
4066          */
4067         cfs_b->idle = 0;
4068 
4069         return 0;
4070 
4071 out_deactivate:
4072         return 1;
4073 }
4074 
4075 /* a cfs_rq won't donate quota below this amount */
4076 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
4077 /* minimum remaining period time to redistribute slack quota */
4078 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
4079 /* how long we wait to gather additional slack before distributing */
4080 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
4081 
4082 /*
4083  * Are we near the end of the current quota period?
4084  *
4085  * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
4086  * hrtimer base being cleared by hrtimer_start. In the case of
4087  * migrate_hrtimers, base is never cleared, so we are fine.
4088  */
4089 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
4090 {
4091         struct hrtimer *refresh_timer = &cfs_b->period_timer;
4092         u64 remaining;
4093 
4094         /* if the call-back is running a quota refresh is already occurring */
4095         if (hrtimer_callback_running(refresh_timer))
4096                 return 1;
4097 
4098         /* is a quota refresh about to occur? */
4099         remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
4100         if (remaining < min_expire)
4101                 return 1;
4102 
4103         return 0;
4104 }
4105 
4106 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
4107 {
4108         u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
4109 
4110         /* if there's a quota refresh soon don't bother with slack */
4111         if (runtime_refresh_within(cfs_b, min_left))
4112                 return;
4113 
4114         hrtimer_start(&cfs_b->slack_timer,
4115                         ns_to_ktime(cfs_bandwidth_slack_period),
4116                         HRTIMER_MODE_REL);
4117 }
4118 
4119 /* we know any runtime found here is valid as update_curr() precedes return */
4120 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4121 {
4122         struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4123         s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
4124 
4125         if (slack_runtime <= 0)
4126                 return;
4127 
4128         raw_spin_lock(&cfs_b->lock);
4129         if (cfs_b->quota != RUNTIME_INF &&
4130             cfs_rq->runtime_expires == cfs_b->runtime_expires) {
4131                 cfs_b->runtime += slack_runtime;
4132 
4133                 /* we are under rq->lock, defer unthrottling using a timer */
4134                 if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
4135                     !list_empty(&cfs_b->throttled_cfs_rq))
4136                         start_cfs_slack_bandwidth(cfs_b);
4137         }
4138         raw_spin_unlock(&cfs_b->lock);
4139 
4140         /* even if it's not valid for return we don't want to try again */
4141         cfs_rq->runtime_remaining -= slack_runtime;
4142 }
4143 
4144 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4145 {
4146         if (!cfs_bandwidth_used())
4147                 return;
4148 
4149         if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
4150                 return;
4151 
4152         __return_cfs_rq_runtime(cfs_rq);
4153 }
4154 
4155 /*
4156  * This is done with a timer (instead of inline with bandwidth return) since
4157  * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
4158  */
4159 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
4160 {
4161         u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
4162         u64 expires;
4163 
4164         /* confirm we're still not at a refresh boundary */
4165         raw_spin_lock(&cfs_b->lock);
4166         if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
4167                 raw_spin_unlock(&cfs_b->lock);
4168                 return;
4169         }
4170 
4171         if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
4172                 runtime = cfs_b->runtime;
4173 
4174         expires = cfs_b->runtime_expires;
4175         raw_spin_unlock(&cfs_b->lock);
4176 
4177         if (!runtime)
4178                 return;
4179 
4180         runtime = distribute_cfs_runtime(cfs_b, runtime, expires);
4181 
4182         raw_spin_lock(&cfs_b->lock);
4183         if (expires == cfs_b->runtime_expires)
4184                 cfs_b->runtime -= min(runtime, cfs_b->runtime);
4185         raw_spin_unlock(&cfs_b->lock);
4186 }
4187 
4188 /*
4189  * When a group wakes up we want to make sure that its quota is not already
4190  * expired/exceeded, otherwise it may be allowed to steal additional ticks of
4191  * runtime as update_curr() throttling can not not trigger until it's on-rq.
4192  */
4193 static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
4194 {
4195         if (!cfs_bandwidth_used())
4196                 return;
4197 
4198         /* Synchronize hierarchical throttle counter: */
4199         if (unlikely(!cfs_rq->throttle_uptodate)) {
4200                 struct rq *rq = rq_of(cfs_rq);
4201                 struct cfs_rq *pcfs_rq;
4202                 struct task_group *tg;
4203 
4204                 cfs_rq->throttle_uptodate = 1;
4205 
4206                 /* Get closest up-to-date node, because leaves go first: */
4207                 for (tg = cfs_rq->tg->parent; tg; tg = tg->parent) {
4208                         pcfs_rq = tg->cfs_rq[cpu_of(rq)];
4209                         if (pcfs_rq->throttle_uptodate)
4210                                 break;
4211                 }
4212                 if (tg) {
4213                         cfs_rq->throttle_count = pcfs_rq->throttle_count;
4214                         cfs_rq->throttled_clock_task = rq_clock_task(rq);
4215                 }
4216         }
4217 
4218         /* an active group must be handled by the update_curr()->put() path */
4219         if (!cfs_rq->runtime_enabled || cfs_rq->curr)
4220                 return;
4221 
4222         /* ensure the group is not already throttled */
4223         if (cfs_rq_throttled(cfs_rq))
4224                 return;
4225 
4226         /* update runtime allocation */
4227         account_cfs_rq_runtime(cfs_rq, 0);
4228         if (cfs_rq->runtime_remaining <= 0)
4229                 throttle_cfs_rq(cfs_rq);
4230 }
4231 
4232 /* conditionally throttle active cfs_rq's from put_prev_entity() */
4233 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4234 {
4235         if (!cfs_bandwidth_used())
4236                 return false;
4237 
4238         if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
4239                 return false;
4240 
4241         /*
4242          * it's possible for a throttled entity to be forced into a running
4243          * state (e.g. set_curr_task), in this case we're finished.
4244          */
4245         if (cfs_rq_throttled(cfs_rq))
4246                 return true;
4247 
4248         throttle_cfs_rq(cfs_rq);
4249         return true;
4250 }
4251 
4252 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
4253 {
4254         struct cfs_bandwidth *cfs_b =
4255                 container_of(timer, struct cfs_bandwidth, slack_timer);
4256 
4257         do_sched_cfs_slack_timer(cfs_b);
4258 
4259         return HRTIMER_NORESTART;
4260 }
4261 
4262 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
4263 {
4264         struct cfs_bandwidth *cfs_b =
4265                 container_of(timer, struct cfs_bandwidth, period_timer);
4266         int overrun;
4267         int idle = 0;
4268 
4269         raw_spin_lock(&cfs_b->lock);
4270         for (;;) {
4271                 overrun = hrtimer_forward_now(timer, cfs_b->period);
4272                 if (!overrun)
4273                         break;
4274 
4275                 idle = do_sched_cfs_period_timer(cfs_b, overrun);
4276         }
4277         if (idle)
4278                 cfs_b->period_active = 0;
4279         raw_spin_unlock(&cfs_b->lock);
4280 
4281         return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
4282 }
4283 
4284 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
4285 {
4286         raw_spin_lock_init(&cfs_b->lock);
4287         cfs_b->runtime = 0;
4288         cfs_b->quota = RUNTIME_INF;
4289         cfs_b->period = ns_to_ktime(default_cfs_period());
4290 
4291         INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
4292         hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
4293         cfs_b->period_timer.function = sched_cfs_period_timer;
4294         hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
4295         cfs_b->slack_timer.function = sched_cfs_slack_timer;
4296 }
4297 
4298 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4299 {
4300         cfs_rq->runtime_enabled = 0;
4301         INIT_LIST_HEAD(&cfs_rq->throttled_list);
4302 }
4303 
4304 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
4305 {
4306         lockdep_assert_held(&cfs_b->lock);
4307 
4308         if (!cfs_b->period_active) {
4309                 cfs_b->period_active = 1;
4310                 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
4311                 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);
4312         }
4313 }
4314 
4315 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
4316 {
4317         /* init_cfs_bandwidth() was not called */
4318         if (!cfs_b->throttled_cfs_rq.next)
4319                 return;
4320 
4321         hrtimer_cancel(&cfs_b->period_timer);
4322         hrtimer_cancel(&cfs_b->slack_timer);
4323 }
4324 
4325 static void __maybe_unused update_runtime_enabled(struct rq *rq)
4326 {
4327         struct cfs_rq *cfs_rq;
4328 
4329         for_each_leaf_cfs_rq(rq, cfs_rq) {
4330                 struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth;
4331 
4332                 raw_spin_lock(&cfs_b->lock);
4333                 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
4334                 raw_spin_unlock(&cfs_b->lock);
4335         }
4336 }
4337 
4338 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
4339 {
4340         struct cfs_rq *cfs_rq;
4341 
4342         for_each_leaf_cfs_rq(rq, cfs_rq) {
4343                 if (!cfs_rq->runtime_enabled)
4344                         continue;
4345 
4346                 /*
4347                  * clock_task is not advancing so we just need to make sure
4348                  * there's some valid quota amount
4349                  */
4350                 cfs_rq->runtime_remaining = 1;
4351                 /*
4352                  * Offline rq is schedulable till cpu is completely disabled
4353                  * in take_cpu_down(), so we prevent new cfs throttling here.
4354                  */
4355                 cfs_rq->runtime_enabled = 0;
4356 
4357                 if (cfs_rq_throttled(cfs_rq))
4358                         unthrottle_cfs_rq(cfs_rq);
4359         }
4360 }
4361 
4362 #else /* CONFIG_CFS_BANDWIDTH */
4363 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
4364 {
4365         return rq_clock_task(rq_of(cfs_rq));
4366 }
4367 
4368 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
4369 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
4370 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
4371 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
4372 
4373 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
4374 {
4375         return 0;
4376 }
4377 
4378 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
4379 {
4380         return 0;
4381 }
4382 
4383 static inline int throttled_lb_pair(struct task_group *tg,
4384                                     int src_cpu, int dest_cpu)
4385 {
4386         return 0;
4387 }
4388 
4389 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
4390 
4391 #ifdef CONFIG_FAIR_GROUP_SCHED
4392 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
4393 #endif
4394 
4395 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
4396 {
4397         return NULL;
4398 }
4399 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
4400 static inline void update_runtime_enabled(struct rq *rq) {}
4401 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
4402 
4403 #endif /* CONFIG_CFS_BANDWIDTH */
4404 
4405 /**************************************************
4406  * CFS operations on tasks:
4407  */
4408 
4409 #ifdef CONFIG_SCHED_HRTICK
4410 static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
4411 {
4412         struct sched_entity *se = &p->se;
4413         struct cfs_rq *cfs_rq = cfs_rq_of(se);
4414 
4415         WARN_ON(task_rq(p) != rq);
4416 
4417         if (cfs_rq->nr_running > 1) {
4418                 u64 slice = sched_slice(cfs_rq, se);
4419                 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
4420                 s64 delta = slice - ran;
4421 
4422                 if (delta < 0) {
4423                         if (rq->curr == p)
4424                                 resched_curr(rq);
4425                         return;
4426                 }
4427                 hrtick_start(rq, delta);
4428         }
4429 }
4430 
4431 /*
4432  * called from enqueue/dequeue and updates the hrtick when the
4433  * current task is from our class and nr_running is low enough
4434  * to matter.
4435  */
4436 static void hrtick_update(struct rq *rq)
4437 {
4438         struct task_struct *curr = rq->curr;
4439 
4440         if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class)
4441                 return;
4442 
4443         if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
4444                 hrtick_start_fair(rq, curr);
4445 }
4446 #else /* !CONFIG_SCHED_HRTICK */
4447 static inline void
4448 hrtick_start_fair(struct rq *rq, struct task_struct *p)
4449 {
4450 }
4451 
4452 static inline void hrtick_update(struct rq *rq)
4453 {
4454 }
4455 #endif
4456 
4457 /*
4458  * The enqueue_task method is called before nr_running is
4459  * increased. Here we update the fair scheduling stats and
4460  * then put the task into the rbtree:
4461  */
4462 static void
4463 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
4464 {
4465         struct cfs_rq *cfs_rq;
4466         struct sched_entity *se = &p->se;
4467 
4468         for_each_sched_entity(se) {
4469                 if (se->on_rq)
4470                         break;
4471                 cfs_rq = cfs_rq_of(se);
4472                 enqueue_entity(cfs_rq, se, flags);
4473 
4474                 /*
4475                  * end evaluation on encountering a throttled cfs_rq
4476                  *
4477                  * note: in the case of encountering a throttled cfs_rq we will
4478                  * post the final h_nr_running increment below.
4479                 */
4480                 if (cfs_rq_throttled(cfs_rq))
4481                         break;
4482                 cfs_rq->h_nr_running++;
4483 
4484                 flags = ENQUEUE_WAKEUP;
4485         }
4486 
4487         for_each_sched_entity(se) {
4488                 cfs_rq = cfs_rq_of(se);
4489                 cfs_rq->h_nr_running++;
4490 
4491                 if (cfs_rq_throttled(cfs_rq))
4492                         break;
4493 
4494                 update_load_avg(se, 1);
4495                 update_cfs_shares(cfs_rq);
4496         }
4497 
4498         if (!se)
4499                 add_nr_running(rq, 1);
4500 
4501         hrtick_update(rq);
4502 }
4503 
4504 static void set_next_buddy(struct sched_entity *se);
4505 
4506 /*
4507  * The dequeue_task method is called before nr_running is
4508  * decreased. We remove the task from the rbtree and
4509  * update the fair scheduling stats:
4510  */
4511 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
4512 {
4513         struct cfs_rq *cfs_rq;
4514         struct sched_entity *se = &p->se;
4515         int task_sleep = flags & DEQUEUE_SLEEP;
4516 
4517         for_each_sched_entity(se) {
4518                 cfs_rq = cfs_rq_of(se);
4519                 dequeue_entity(cfs_rq, se, flags);
4520 
4521                 /*
4522                  * end evaluation on encountering a throttled cfs_rq
4523                  *
4524                  * note: in the case of encountering a throttled cfs_rq we will
4525                  * post the final h_nr_running decrement below.
4526                 */
4527                 if (cfs_rq_throttled(cfs_rq))
4528                         break;
4529                 cfs_rq->h_nr_running--;
4530 
4531                 /* Don't dequeue parent if it has other entities besides us */
4532                 if (cfs_rq->load.weight) {
4533                         /* Avoid re-evaluating load for this entity: */
4534                         se = parent_entity(se);
4535                         /*
4536                          * Bias pick_next to pick a task from this cfs_rq, as
4537                          * p is sleeping when it is within its sched_slice.
4538                          */
4539                         if (task_sleep && se && !throttled_hierarchy(cfs_rq))
4540                                 set_next_buddy(se);
4541                         break;
4542                 }
4543                 flags |= DEQUEUE_SLEEP;
4544         }
4545 
4546         for_each_sched_entity(se) {
4547                 cfs_rq = cfs_rq_of(se);
4548                 cfs_rq->h_nr_running--;
4549 
4550                 if (cfs_rq_throttled(cfs_rq))
4551                         break;
4552 
4553                 update_load_avg(se, 1);
4554                 update_cfs_shares(cfs_rq);
4555         }
4556 
4557         if (!se)
4558                 sub_nr_running(rq, 1);
4559 
4560         hrtick_update(rq);
4561 }
4562 
4563 #ifdef CONFIG_SMP
4564 #ifdef CONFIG_NO_HZ_COMMON
4565 /*
4566  * per rq 'load' arrray crap; XXX kill this.
4567  */
4568 
4569 /*
4570  * The exact cpuload calculated at every tick would be:
4571  *
4572  *   load' = (1 - 1/2^i) * load + (1/2^i) * cur_load
4573  *
4574  * If a cpu misses updates for n ticks (as it was idle) and update gets
4575  * called on the n+1-th tick when cpu may be busy, then we have:
4576  *
4577  *   load_n   = (1 - 1/2^i)^n * load_0
4578  *   load_n+1 = (1 - 1/2^i)   * load_n + (1/2^i) * cur_load
4579  *
4580  * decay_load_missed() below does efficient calculation of
4581  *
4582  *   load' = (1 - 1/2^i)^n * load
4583  *
4584  * Because x^(n+m) := x^n * x^m we can decompose any x^n in power-of-2 factors.
4585  * This allows us to precompute the above in said factors, thereby allowing the
4586  * reduction of an arbitrary n in O(log_2 n) steps. (See also
4587  * fixed_power_int())
4588  *
4589  * The calculation is approximated on a 128 point scale.
4590  */
4591 #define DEGRADE_SHIFT           7
4592 
4593 static const u8 degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
4594 static const u8 degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
4595         {   0,   0,  0,  0,  0,  0, 0, 0 },
4596         {  64,  32,  8,  0,  0,  0, 0, 0 },
4597         {  96,  72, 40, 12,  1,  0, 0, 0 },
4598         { 112,  98, 75, 43, 15,  1, 0, 0 },
4599         { 120, 112, 98, 76, 45, 16, 2, 0 }
4600 };
4601 
4602 /*
4603  * Update cpu_load for any missed ticks, due to tickless idle. The backlog
4604  * would be when CPU is idle and so we just decay the old load without
4605  * adding any new load.
4606  */
4607 static unsigned long
4608 decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
4609 {
4610         int j = 0;
4611 
4612         if (!missed_updates)
4613                 return load;
4614 
4615         if (missed_updates >= degrade_zero_ticks[idx])
4616                 return 0;
4617 
4618         if (idx == 1)
4619                 return load >> missed_updates;
4620 
4621         while (missed_updates) {
4622                 if (missed_updates % 2)
4623                         load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;
4624 
4625                 missed_updates >>= 1;
4626                 j++;
4627         }
4628         return load;
4629 }
4630 #endif /* CONFIG_NO_HZ_COMMON */
4631 
4632 /**
4633  * __cpu_load_update - update the rq->cpu_load[] statistics
4634  * @this_rq: The rq to update statistics for
4635  * @this_load: The current load
4636  * @pending_updates: The number of missed updates
4637  *
4638  * Update rq->cpu_load[] statistics. This function is usually called every
4639  * scheduler tick (TICK_NSEC).
4640  *
4641  * This function computes a decaying average:
4642  *
4643  *   load[i]' = (1 - 1/2^i) * load[i] + (1/2^i) * load
4644  *
4645  * Because of NOHZ it might not get called on every tick which gives need for
4646  * the @pending_updates argument.
4647  *
4648  *   load[i]_n = (1 - 1/2^i) * load[i]_n-1 + (1/2^i) * load_n-1
4649  *             = A * load[i]_n-1 + B ; A := (1 - 1/2^i), B := (1/2^i) * load
4650  *             = A * (A * load[i]_n-2 + B) + B
4651  *             = A * (A * (A * load[i]_n-3 + B) + B) + B
4652  *             = A^3 * load[i]_n-3 + (A^2 + A + 1) * B
4653  *             = A^n * load[i]_0 + (A^(n-1) + A^(n-2) + ... + 1) * B
4654  *             = A^n * load[i]_0 + ((1 - A^n) / (1 - A)) * B
4655  *             = (1 - 1/2^i)^n * (load[i]_0 - load) + load
4656  *
4657  * In the above we've assumed load_n := load, which is true for NOHZ_FULL as
4658  * any change in load would have resulted in the tick being turned back on.
4659  *
4660  * For regular NOHZ, this reduces to:
4661  *
4662  *   load[i]_n = (1 - 1/2^i)^n * load[i]_0
4663  *
4664  * see decay_load_misses(). For NOHZ_FULL we get to subtract and add the extra
4665  * term.
4666  */
4667 static void cpu_load_update(struct rq *this_rq, unsigned long this_load,
4668                             unsigned long pending_updates)
4669 {
4670         unsigned long __maybe_unused tickless_load = this_rq->cpu_load[0];
4671         int i, scale;
4672 
4673         this_rq->nr_load_updates++;
4674 
4675         /* Update our load: */
4676         this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
4677         for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
4678                 unsigned long old_load, new_load;
4679 
4680                 /* scale is effectively 1 << i now, and >> i divides by scale */
4681 
4682                 old_load = this_rq->cpu_load[i];
4683 #ifdef CONFIG_NO_HZ_COMMON
4684                 old_load = decay_load_missed(old_load, pending_updates - 1, i);
4685                 if (tickless_load) {
4686                         old_load -= decay_load_missed(tickless_load, pending_updates - 1, i);
4687                         /*
4688                          * old_load can never be a negative value because a
4689                          * decayed tickless_load cannot be greater than the
4690                          * original tickless_load.
4691                          */
4692                         old_load += tickless_load;
4693                 }
4694 #endif
4695                 new_load = this_load;
4696                 /*
4697                  * Round up the averaging division if load is increasing. This
4698                  * prevents us from getting stuck on 9 if the load is 10, for
4699                  * example.
4700                  */
4701                 if (new_load > old_load)
4702                         new_load += scale - 1;
4703 
4704                 this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
4705         }
4706 
4707         sched_avg_update(this_rq);
4708 }
4709 
4710 /* Used instead of source_load when we know the type == 0 */
4711 static unsigned long weighted_cpuload(const int cpu)
4712 {
4713         return cfs_rq_runnable_load_avg(&cpu_rq(cpu)->cfs);
4714 }
4715 
4716 #ifdef CONFIG_NO_HZ_COMMON
4717 /*
4718  * There is no sane way to deal with nohz on smp when using jiffies because the
4719  * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading
4720  * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}.
4721  *
4722  * Therefore we need to avoid the delta approach from the regular tick when
4723  * possible since that would seriously skew the load calculation. This is why we
4724  * use cpu_load_update_periodic() for CPUs out of nohz. However we'll rely on
4725  * jiffies deltas for updates happening while in nohz mode (idle ticks, idle
4726  * loop exit, nohz_idle_balance, nohz full exit...)
4727  *
4728  * This means we might still be one tick off for nohz periods.
4729  */
4730 
4731 static void cpu_load_update_nohz(struct rq *this_rq,
4732                                  unsigned long curr_jiffies,
4733                                  unsigned long load)
4734 {
4735         unsigned long pending_updates;
4736 
4737         pending_updates = curr_jiffies - this_rq->last_load_update_tick;
4738         if (pending_updates) {
4739                 this_rq->last_load_update_tick = curr_jiffies;
4740                 /*
4741                  * In the regular NOHZ case, we were idle, this means load 0.
4742                  * In the NOHZ_FULL case, we were non-idle, we should consider
4743                  * its weighted load.
4744                  */
4745                 cpu_load_update(this_rq, load, pending_updates);
4746         }
4747 }
4748 
4749 /*
4750  * Called from nohz_idle_balance() to update the load ratings before doing the
4751  * idle balance.
4752  */
4753 static void cpu_load_update_idle(struct rq *this_rq)
4754 {
4755         /*
4756          * bail if there's load or we're actually up-to-date.
4757          */
4758         if (weighted_cpuload(cpu_of(this_rq)))
4759                 return;
4760 
4761         cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), 0);
4762 }
4763 
4764 /*
4765  * Record CPU load on nohz entry so we know the tickless load to account
4766  * on nohz exit. cpu_load[0] happens then to be updated more frequently
4767  * than other cpu_load[idx] but it should be fine as cpu_load readers
4768  * shouldn't rely into synchronized cpu_load[*] updates.
4769  */
4770 void cpu_load_update_nohz_start(void)
4771 {
4772         struct rq *this_rq = this_rq();
4773 
4774         /*
4775          * This is all lockless but should be fine. If weighted_cpuload changes
4776          * concurrently we'll exit nohz. And cpu_load write can race with
4777          * cpu_load_update_idle() but both updater would be writing the same.
4778          */
4779         this_rq->cpu_load[0] = weighted_cpuload(cpu_of(this_rq));
4780 }
4781 
4782 /*
4783  * Account the tickless load in the end of a nohz frame.
4784  */
4785 void cpu_load_update_nohz_stop(void)
4786 {
4787         unsigned long curr_jiffies = READ_ONCE(jiffies);
4788         struct rq *this_rq = this_rq();
4789         unsigned long load;
4790 
4791         if (curr_jiffies == this_rq->last_load_update_tick)
4792                 return;
4793 
4794         load = weighted_cpuload(cpu_of(this_rq));
4795         raw_spin_lock(&this_rq->lock);
4796         update_rq_clock(this_rq);
4797         cpu_load_update_nohz(this_rq, curr_jiffies, load);
4798         raw_spin_unlock(&this_rq->lock);
4799 }
4800 #else /* !CONFIG_NO_HZ_COMMON */
4801 static inline void cpu_load_update_nohz(struct rq *this_rq,
4802                                         unsigned long curr_jiffies,
4803                                         unsigned long load) { }
4804 #endif /* CONFIG_NO_HZ_COMMON */
4805 
4806 static void cpu_load_update_periodic(struct rq *this_rq, unsigned long load)
4807 {
4808 #ifdef CONFIG_NO_HZ_COMMON
4809         /* See the mess around cpu_load_update_nohz(). */
4810         this_rq->last_load_update_tick = READ_ONCE(jiffies);
4811 #endif
4812         cpu_load_update(this_rq, load, 1);
4813 }
4814 
4815 /*
4816  * Called from scheduler_tick()
4817  */
4818 void cpu_load_update_active(struct rq *this_rq)
4819 {
4820         unsigned long load = weighted_cpuload(cpu_of(this_rq));
4821 
4822         if (tick_nohz_tick_stopped())
4823                 cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), load);
4824         else
4825                 cpu_load_update_periodic(this_rq, load);
4826 }
4827 
4828 /*
4829  * Return a low guess at the load of a migration-source cpu weighted
4830  * according to the scheduling class and "nice" value.
4831  *
4832  * We want to under-estimate the load of migration sources, to
4833  * balance conservatively.
4834  */
4835 static unsigned long source_load(int cpu, int type)
4836 {
4837         struct rq *rq = cpu_rq(cpu);
4838         unsigned long total = weighted_cpuload(cpu);
4839 
4840         if (type == 0 || !sched_feat(LB_BIAS))
4841                 return total;
4842 
4843         return min(rq->cpu_load[type-1], total);
4844 }
4845 
4846 /*
4847  * Return a high guess at the load of a migration-target cpu weighted
4848  * according to the scheduling class and "nice" value.
4849  */
4850 static unsigned long target_load(int cpu, int type)
4851 {
4852         struct rq *rq = cpu_rq(cpu);
4853         unsigned long total = weighted_cpuload(cpu);
4854 
4855         if (type == 0 || !sched_feat(LB_BIAS))
4856                 return total;
4857 
4858         return max(rq->cpu_load[type-1], total);
4859 }
4860 
4861 static unsigned long capacity_of(int cpu)
4862 {
4863         return cpu_rq(cpu)->cpu_capacity;
4864 }
4865 
4866 static unsigned long capacity_orig_of(int cpu)
4867 {
4868         return cpu_rq(cpu)->cpu_capacity_orig;
4869 }
4870 
4871 static unsigned long cpu_avg_load_per_task(int cpu)
4872 {
4873         struct rq *rq = cpu_rq(cpu);
4874         unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running);
4875         unsigned long load_avg = weighted_cpuload(cpu);
4876 
4877         if (nr_running)
4878                 return load_avg / nr_running;
4879 
4880         return 0;
4881 }
4882 
4883 #ifdef CONFIG_FAIR_GROUP_SCHED
4884 /*
4885  * effective_load() calculates the load change as seen from the root_task_group
4886  *
4887  * Adding load to a group doesn't make a group heavier, but can cause movement
4888  * of group shares between cpus. Assuming the shares were perfectly aligned one
4889  * can calculate the shift in shares.
4890  *
4891  * Calculate the effective load difference if @wl is added (subtracted) to @tg
4892  * on this @cpu and results in a total addition (subtraction) of @wg to the
4893  * total group weight.
4894  *
4895  * Given a runqueue weight distribution (rw_i) we can compute a shares
4896  * distribution (s_i) using:
4897  *
4898  *   s_i = rw_i / \Sum rw_j                                             (1)
4899  *
4900  * Suppose we have 4 CPUs and our @tg is a direct child of the root group and
4901  * has 7 equal weight tasks, distributed as below (rw_i), with the resulting
4902  * shares distribution (s_i):
4903  *
4904  *   rw_i = {   2,   4,   1,   0 }
4905  *   s_i  = { 2/7, 4/7, 1/7,   0 }
4906  *
4907  * As per wake_affine() we're interested in the load of two CPUs (the CPU the
4908  * task used to run on and the CPU the waker is running on), we need to
4909  * compute the effect of waking a task on either CPU and, in case of a sync
4910  * wakeup, compute the effect of the current task going to sleep.
4911  *
4912  * So for a change of @wl to the local @cpu with an overall group weight change
4913  * of @wl we can compute the new shares distribution (s'_i) using:
4914  *
4915  *   s'_i = (rw_i + @wl) / (@wg + \Sum rw_j)                            (2)
4916  *
4917  * Suppose we're interested in CPUs 0 and 1, and want to compute the load
4918  * differences in waking a task to CPU 0. The additional task changes the
4919  * weight and shares distributions like:
4920  *
4921  *   rw'_i = {   3,   4,   1,   0 }
4922  *   s'_i  = { 3/8, 4/8, 1/8,   0 }
4923  *
4924  * We can then compute the difference in effective weight by using:
4925  *
4926  *   dw_i = S * (s'_i - s_i)                                            (3)
4927  *
4928  * Where 'S' is the group weight as seen by its parent.
4929  *
4930  * Therefore the effective change in loads on CPU 0 would be 5/56 (3/8 - 2/7)
4931  * times the weight of the group. The effect on CPU 1 would be -4/56 (4/8 -
4932  * 4/7) times the weight of the group.
4933  */
4934 static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
4935 {
4936         struct sched_entity *se = tg->se[cpu];
4937 
4938         if (!tg->parent)        /* the trivial, non-cgroup case */
4939                 return wl;
4940 
4941         for_each_sched_entity(se) {
4942                 struct cfs_rq *cfs_rq = se->my_q;
4943                 long W, w = cfs_rq_load_avg(cfs_rq);
4944 
4945                 tg = cfs_rq->tg;
4946 
4947                 /*
4948                  * W = @wg + \Sum rw_j
4949                  */
4950                 W = wg + atomic_long_read(&tg->load_avg);
4951 
4952                 /* Ensure \Sum rw_j >= rw_i */
4953                 W -= cfs_rq->tg_load_avg_contrib;
4954                 W += w;
4955 
4956                 /*
4957                  * w = rw_i + @wl
4958                  */
4959                 w += wl;
4960 
4961                 /*
4962                  * wl = S * s'_i; see (2)
4963                  */
4964                 if (W > 0 && w < W)
4965                         wl = (w * (long)tg->shares) / W;
4966                 else
4967                         wl = tg->shares;
4968 
4969                 /*
4970                  * Per the above, wl is the new se->load.weight value; since
4971                  * those are clipped to [MIN_SHARES, ...) do so now. See
4972                  * calc_cfs_shares().
4973                  */
4974                 if (wl < MIN_SHARES)
4975                         wl = MIN_SHARES;
4976 
4977                 /*
4978                  * wl = dw_i = S * (s'_i - s_i); see (3)
4979                  */
4980                 wl -= se->avg.load_avg;
4981 
4982                 /*
4983                  * Recursively apply this logic to all parent groups to compute
4984                  * the final effective load change on the root group. Since
4985                  * only the @tg group gets extra weight, all parent groups can
4986                  * only redistribute existing shares. @wl is the shift in shares
4987                  * resulting from this level per the above.
4988                  */
4989                 wg = 0;
4990         }
4991 
4992         return wl;
4993 }
4994 #else
4995 
4996 static long effective_load(struct task_group *tg, int cpu, long wl, long wg)
4997 {
4998         return wl;
4999 }
5000 
5001 #endif
5002 
5003 static void record_wakee(struct task_struct *p)
5004 {
5005         /*
5006          * Only decay a single time; tasks that have less then 1 wakeup per
5007          * jiffy will not have built up many flips.
5008          */
5009         if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) {
5010                 current->wakee_flips >>= 1;
5011                 current->wakee_flip_decay_ts = jiffies;
5012         }
5013 
5014         if (current->last_wakee != p) {
5015                 current->last_wakee = p;
5016                 current->wakee_flips++;
5017         }
5018 }
5019 
5020 /*
5021  * Detect M:N waker/wakee relationships via a switching-frequency heuristic.
5022  *
5023  * A waker of many should wake a different task than the one last awakened
5024  * at a frequency roughly N times higher than one of its wakees.
5025  *
5026  * In order to determine whether we should let the load spread vs consolidating
5027  * to shared cache, we look for a minimum 'flip' frequency of llc_size in one
5028  * partner, and a factor of lls_size higher frequency in the other.
5029  *
5030  * With both conditions met, we can be relatively sure that the relationship is
5031  * non-monogamous, with partner count exceeding socket size.
5032  *
5033  * Waker/wakee being client/server, worker/dispatcher, interrupt source or
5034  * whatever is irrelevant, spread criteria is apparent partner count exceeds
5035  * socket size.
5036  */
5037 static int wake_wide(struct task_struct *p)
5038 {
5039         unsigned int master = current->wakee_flips;
5040         unsigned int slave = p->wakee_flips;
5041         int factor = this_cpu_read(sd_llc_size);
5042 
5043         if (master < slave)
5044                 swap(master, slave);
5045         if (slave < factor || master < slave * factor)
5046                 return 0;
5047         return 1;
5048 }
5049 
5050 static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
5051 {
5052         s64 this_load, load;
5053         s64 this_eff_load, prev_eff_load;
5054         int idx, this_cpu, prev_cpu;
5055         struct task_group *tg;
5056         unsigned long weight;
5057         int balanced;
5058 
5059         idx       = sd->wake_idx;
5060         this_cpu  = smp_processor_id();
5061         prev_cpu  = task_cpu(p);
5062         load      = source_load(prev_cpu, idx);
5063         this_load = target_load(this_cpu, idx);
5064 
5065         /*
5066          * If sync wakeup then subtract the (maximum possible)
5067          * effect of the currently running task from the load
5068          * of the current CPU:
5069          */
5070         if (sync) {
5071                 tg = task_group(current);
5072                 weight = current->se.avg.load_avg;
5073 
5074                 this_load += effective_load(tg, this_cpu, -weight, -weight);
5075                 load += effective_load(tg, prev_cpu, 0, -weight);
5076         }
5077 
5078         tg = task_group(p);
5079         weight = p->se.avg.load_avg;
5080 
5081         /*
5082          * In low-load situations, where prev_cpu is idle and this_cpu is idle
5083          * due to the sync cause above having dropped this_load to 0, we'll
5084          * always have an imbalance, but there's really nothing you can do
5085          * about that, so that's good too.
5086          *
5087          * Otherwise check if either cpus are near enough in load to allow this
5088          * task to be woken on this_cpu.
5089          */
5090         this_eff_load = 100;
5091         this_eff_load *= capacity_of(prev_cpu);
5092 
5093         prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2;
5094         prev_eff_load *= capacity_of(this_cpu);
5095 
5096         if (this_load > 0) {
5097                 this_eff_load *= this_load +
5098                         effective_load(tg, this_cpu, weight, weight);
5099 
5100                 prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight);
5101         }
5102 
5103         balanced = this_eff_load <= prev_eff_load;
5104 
5105         schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts);
5106 
5107         if (!balanced)
5108                 return 0;
5109 
5110         schedstat_inc(sd, ttwu_move_affine);
5111         schedstat_inc(p, se.statistics.nr_wakeups_affine);
5112 
5113         return 1;
5114 }
5115 
5116 /*
5117  * find_idlest_group finds and returns the least busy CPU group within the
5118  * domain.
5119  */
5120 static struct sched_group *
5121 find_idlest_group(struct sched_domain *sd, struct task_struct *p,
5122                   int this_cpu, int sd_flag)
5123 {
5124         struct sched_group *idlest = NULL, *group = sd->groups;
5125         unsigned long min_load = ULONG_MAX, this_load = 0;
5126         int load_idx = sd->forkexec_idx;
5127         int imbalance = 100 + (sd->imbalance_pct-100)/2;
5128 
5129         if (sd_flag & SD_BALANCE_WAKE)
5130                 load_idx = sd->wake_idx;
5131 
5132         do {
5133                 unsigned long load, avg_load;
5134                 int local_group;
5135                 int i;
5136 
5137                 /* Skip over this group if it has no CPUs allowed */
5138                 if (!cpumask_intersects(sched_group_cpus(group),
5139                                         tsk_cpus_allowed(p)))
5140                         continue;
5141 
5142                 local_group = cpumask_test_cpu(this_cpu,
5143                                                sched_group_cpus(group));
5144 
5145                 /* Tally up the load of all CPUs in the group */
5146                 avg_load = 0;
5147 
5148                 for_each_cpu(i, sched_group_cpus(group)) {
5149                         /* Bias balancing toward cpus of our domain */
5150                         if (local_group)
5151                                 load = source_load(i, load_idx);
5152                         else
5153                                 load = target_load(i, load_idx);
5154 
5155                         avg_load += load;
5156                 }
5157 
5158                 /* Adjust by relative CPU capacity of the group */
5159                 avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgc->capacity;
5160 
5161                 if (local_group) {
5162                         this_load = avg_load;
5163                 } else if (avg_load < min_load) {
5164                         min_load = avg_load;
5165                         idlest = group;
5166                 }
5167         } while (group = group->next, group != sd->groups);
5168 
5169         if (!idlest || 100*this_load < imbalance*min_load)
5170                 return NULL;
5171         return idlest;
5172 }
5173 
5174 /*
5175  * find_idlest_cpu - find the idlest cpu among the cpus in group.
5176  */
5177 static int
5178 find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
5179 {
5180         unsigned long load, min_load = ULONG_MAX;
5181         unsigned int min_exit_latency = UINT_MAX;
5182         u64 latest_idle_timestamp = 0;
5183         int least_loaded_cpu = this_cpu;
5184         int shallowest_idle_cpu = -1;
5185         int i;
5186 
5187         /* Traverse only the allowed CPUs */
5188         for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) {
5189                 if (idle_cpu(i)) {
5190                         struct rq *rq = cpu_rq(i);
5191                         struct cpuidle_state *idle = idle_get_state(rq);
5192                         if (idle && idle->exit_latency < min_exit_latency) {
5193                                 /*
5194                                  * We give priority to a CPU whose idle state
5195                                  * has the smallest exit latency irrespective
5196                                  * of any idle timestamp.
5197                                  */
5198                                 min_exit_latency = idle->exit_latency;
5199                                 latest_idle_timestamp = rq->idle_stamp;
5200                                 shallowest_idle_cpu = i;
5201                         } else if ((!idle || idle->exit_latency == min_exit_latency) &&
5202                                    rq->idle_stamp > latest_idle_timestamp) {
5203                                 /*
5204                                  * If equal or no active idle state, then
5205                                  * the most recently idled CPU might have
5206                                  * a warmer cache.
5207                                  */
5208                                 latest_idle_timestamp = rq->idle_stamp;
5209                                 shallowest_idle_cpu = i;
5210                         }
5211                 } else if (shallowest_idle_cpu == -1) {
5212                         load = weighted_cpuload(i);
5213                         if (load < min_load || (load == min_load && i == this_cpu)) {
5214                                 min_load = load;
5215                                 least_loaded_cpu = i;
5216                         }
5217                 }
5218         }
5219 
5220         return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
5221 }
5222 
5223 /*
5224  * Try and locate an idle CPU in the sched_domain.
5225  */
5226 static int select_idle_sibling(struct task_struct *p, int target)
5227 {
5228         struct sched_domain *sd;
5229         struct sched_group *sg;
5230         int i = task_cpu(p);
5231 
5232         if (idle_cpu(target))
5233                 return target;
5234 
5235         /*
5236          * If the prevous cpu is cache affine and idle, don't be stupid.
5237          */
5238         if (i != target && cpus_share_cache(i, target) && idle_cpu(i))
5239                 return i;
5240 
5241         /*
5242          * Otherwise, iterate the domains and find an eligible idle cpu.
5243          *
5244          * A completely idle sched group at higher domains is more
5245          * desirable than an idle group at a lower level, because lower
5246          * domains have smaller groups and usually share hardware
5247          * resources which causes tasks to contend on them, e.g. x86
5248          * hyperthread siblings in the lowest domain (SMT) can contend
5249          * on the shared cpu pipeline.
5250          *
5251          * However, while we prefer idle groups at higher domains
5252          * finding an idle cpu at the lowest domain is still better than
5253          * returning 'target', which we've already established, isn't
5254          * idle.
5255          */
5256         sd = rcu_dereference(per_cpu(sd_llc, target));
5257         for_each_lower_domain(sd) {
5258                 sg = sd->groups;
5259                 do {
5260                         if (!cpumask_intersects(sched_group_cpus(sg),
5261                                                 tsk_cpus_allowed(p)))
5262                                 goto next;
5263 
5264                         /* Ensure the entire group is idle */
5265                         for_each_cpu(i, sched_group_cpus(sg)) {
5266                                 if (i == target || !idle_cpu(i))
5267                                         goto next;
5268                         }
5269 
5270                         /*
5271                          * It doesn't matter which cpu we pick, the
5272                          * whole group is idle.
5273                          */
5274                         target = cpumask_first_and(sched_group_cpus(sg),
5275                                         tsk_cpus_allowed(p));
5276                         goto done;
5277 next:
5278                         sg = sg->next;
5279                 } while (sg != sd->groups);
5280         }
5281 done:
5282         return target;
5283 }
5284 
5285 /*
5286  * cpu_util returns the amount of capacity of a CPU that is used by CFS
5287  * tasks. The unit of the return value must be the one of capacity so we can
5288  * compare the utilization with the capacity of the CPU that is available for
5289  * CFS task (ie cpu_capacity).
5290  *
5291  * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the
5292  * recent utilization of currently non-runnable tasks on a CPU. It represents
5293  * the amount of utilization of a CPU in the range [0..capacity_orig] where
5294  * capacity_orig is the cpu_capacity available at the highest frequency
5295  * (arch_scale_freq_capacity()).
5296  * The utilization of a CPU converges towards a sum equal to or less than the
5297  * current capacity (capacity_curr <= capacity_orig) of the CPU because it is
5298  * the running time on this CPU scaled by capacity_curr.
5299  *
5300  * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even
5301  * higher than capacity_orig because of unfortunate rounding in
5302  * cfs.avg.util_avg or just after migrating tasks and new task wakeups until
5303  * the average stabilizes with the new running time. We need to check that the
5304  * utilization stays within the range of [0..capacity_orig] and cap it if
5305  * necessary. Without utilization capping, a group could be seen as overloaded
5306  * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of
5307  * available capacity. We allow utilization to overshoot capacity_curr (but not
5308  * capacity_orig) as it useful for predicting the capacity required after task
5309  * migrations (scheduler-driven DVFS).
5310  */
5311 static int cpu_util(int cpu)
5312 {
5313         unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg;
5314         unsigned long capacity = capacity_orig_of(cpu);
5315 
5316         return (util >= capacity) ? capacity : util;
5317 }
5318 
5319 /*
5320  * select_task_rq_fair: Select target runqueue for the waking task in domains
5321  * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE,
5322  * SD_BALANCE_FORK, or SD_BALANCE_EXEC.
5323  *
5324  * Balances load by selecting the idlest cpu in the idlest group, or under
5325  * certain conditions an idle sibling cpu if the domain has SD_WAKE_AFFINE set.
5326  *
5327  * Returns the target cpu number.
5328  *
5329  * preempt must be disabled.
5330  */
5331 static int
5332 select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags)
5333 {
5334         struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
5335         int cpu = smp_processor_id();
5336         int new_cpu = prev_cpu;
5337         int want_affine = 0;
5338         int sync = wake_flags & WF_SYNC;
5339 
5340         if (sd_flag & SD_BALANCE_WAKE) {
5341                 record_wakee(p);
5342                 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, tsk_cpus_allowed(p));
5343         }
5344 
5345         rcu_read_lock();
5346         for_each_domain(cpu, tmp) {
5347                 if (!(tmp->flags & SD_LOAD_BALANCE))
5348                         break;
5349 
5350                 /*
5351                  * If both cpu and prev_cpu are part of this domain,
5352                  * cpu is a valid SD_WAKE_AFFINE target.
5353                  */
5354                 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
5355                     cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
5356                         affine_sd = tmp;
5357                         break;
5358                 }
5359 
5360                 if (tmp->flags & sd_flag)
5361                         sd = tmp;
5362                 else if (!want_affine)
5363                         break;
5364         }
5365 
5366         if (affine_sd) {
5367                 sd = NULL; /* Prefer wake_affine over balance flags */
5368                 if (cpu != prev_cpu && wake_affine(affine_sd, p, sync))
5369                         new_cpu = cpu;
5370         }
5371 
5372         if (!sd) {
5373                 if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */
5374                         new_cpu = select_idle_sibling(p, new_cpu);
5375 
5376         } else while (sd) {
5377                 struct sched_group *group;
5378                 int weight;
5379 
5380                 if (!(sd->flags & sd_flag)) {
5381                         sd = sd->child;
5382                         continue;
5383                 }
5384 
5385                 group = find_idlest_group(sd, p, cpu, sd_flag);
5386                 if (!group) {
5387                         sd = sd->child;
5388                         continue;
5389                 }
5390 
5391                 new_cpu = find_idlest_cpu(group, p, cpu);
5392                 if (new_cpu == -1 || new_cpu == cpu) {
5393                         /* Now try balancing at a lower domain level of cpu */
5394                         sd = sd->child;
5395                         continue;
5396                 }
5397 
5398                 /* Now try balancing at a lower domain level of new_cpu */
5399                 cpu = new_cpu;
5400                 weight = sd->span_weight;
5401                 sd = NULL;
5402                 for_each_domain(cpu, tmp) {
5403                         if (weight <= tmp->span_weight)
5404                                 break;
5405                         if (tmp->flags & sd_flag)
5406                                 sd = tmp;
5407                 }
5408                 /* while loop will break here if sd == NULL */
5409         }
5410         rcu_read_unlock();
5411 
5412         return new_cpu;
5413 }
5414 
5415 /*
5416  * Called immediately before a task is migrated to a new cpu; task_cpu(p) and
5417  * cfs_rq_of(p) references at time of call are still valid and identify the
5418  * previous cpu. The caller guarantees p->pi_lock or task_rq(p)->lock is held.
5419  */
5420 static void migrate_task_rq_fair(struct task_struct *p)
5421 {
5422         /*
5423          * As blocked tasks retain absolute vruntime the migration needs to
5424          * deal with this by subtracting the old and adding the new
5425          * min_vruntime -- the latter is done by enqueue_entity() when placing
5426          * the task on the new runqueue.
5427          */
5428         if (p->state == TASK_WAKING) {
5429                 struct sched_entity *se = &p->se;
5430                 struct cfs_rq *cfs_rq = cfs_rq_of(se);
5431                 u64 min_vruntime;
5432 
5433 #ifndef CONFIG_64BIT
5434                 u64 min_vruntime_copy;
5435 
5436                 do {
5437                         min_vruntime_copy = cfs_rq->min_vruntime_copy;
5438                         smp_rmb();
5439                         min_vruntime = cfs_rq->min_vruntime;
5440                 } while (min_vruntime != min_vruntime_copy);
5441 #else
5442                 min_vruntime = cfs_rq->min_vruntime;
5443 #endif
5444 
5445                 se->vruntime -= min_vruntime;
5446         }
5447 
5448         /*
5449          * We are supposed to update the task to "current" time, then its up to date
5450          * and ready to go to new CPU/cfs_rq. But we have difficulty in getting
5451          * what current time is, so simply throw away the out-of-date time. This
5452          * will result in the wakee task is less decayed, but giving the wakee more
5453          * load sounds not bad.
5454          */
5455         remove_entity_load_avg(&p->se);
5456 
5457         /* Tell new CPU we are migrated */
5458         p->se.avg.last_update_time = 0;
5459 
5460         /* We have migrated, no longer consider this task hot */
5461         p->se.exec_start = 0;
5462 }
5463 
5464 static void task_dead_fair(struct task_struct *p)
5465 {
5466         remove_entity_load_avg(&p->se);
5467 }
5468 #endif /* CONFIG_SMP */
5469 
5470 static unsigned long
5471 wakeup_gran(struct sched_entity *curr, struct sched_entity *se)
5472 {
5473         unsigned long gran = sysctl_sched_wakeup_granularity;
5474 
5475         /*
5476          * Since its curr running now, convert the gran from real-time
5477          * to virtual-time in his units.
5478          *
5479          * By using 'se' instead of 'curr' we penalize light tasks, so
5480          * they get preempted easier. That is, if 'se' < 'curr' then
5481          * the resulting gran will be larger, therefore penalizing the
5482          * lighter, if otoh 'se' > 'curr' then the resulting gran will
5483          * be smaller, again penalizing the lighter task.
5484          *
5485          * This is especially important for buddies when the leftmost
5486          * task is higher priority than the buddy.
5487          */
5488         return calc_delta_fair(gran, se);
5489 }
5490 
5491 /*
5492  * Should 'se' preempt 'curr'.
5493  *
5494  *             |s1
5495  *        |s2
5496  *   |s3
5497  *         g
5498  *      |<--->|c
5499  *
5500  *  w(c, s1) = -1
5501  *  w(c, s2) =  0
5502  *  w(c, s3) =  1
5503  *
5504  */
5505 static int
5506 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
5507 {
5508         s64 gran, vdiff = curr->vruntime - se->vruntime;
5509 
5510         if (vdiff <= 0)
5511                 return -1;
5512 
5513         gran = wakeup_gran(curr, se);
5514         if (vdiff > gran)
5515                 return 1;
5516 
5517         return 0;
5518 }
5519 
5520 static void set_last_buddy(struct sched_entity *se)
5521 {
5522         if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
5523                 return;
5524 
5525         for_each_sched_entity(se)
5526                 cfs_rq_of(se)->last = se;
5527 }
5528 
5529 static void set_next_buddy(struct sched_entity *se)
5530 {
5531         if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
5532                 return;
5533 
5534         for_each_sched_entity(se)
5535                 cfs_rq_of(se)->next = se;
5536 }
5537 
5538 static void set_skip_buddy(struct sched_entity *se)
5539 {
5540         for_each_sched_entity(se)
5541                 cfs_rq_of(se)->skip = se;
5542 }
5543 
5544 /*
5545  * Preempt the current task with a newly woken task if needed:
5546  */
5547 static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
5548 {
5549         struct task_struct *curr = rq->curr;
5550         struct sched_entity *se = &curr->se, *pse = &p->se;
5551         struct cfs_rq *cfs_rq = task_cfs_rq(curr);
5552         int scale = cfs_rq->nr_running >= sched_nr_latency;
5553         int next_buddy_marked = 0;
5554 
5555         if (unlikely(se == pse))
5556                 return;
5557 
5558         /*
5559          * This is possible from callers such as attach_tasks(), in which we
5560          * unconditionally check_prempt_curr() after an enqueue (which may have
5561          * lead to a throttle).  This both saves work and prevents false
5562          * next-buddy nomination below.
5563          */
5564         if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
5565                 return;
5566 
5567         if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
5568                 set_next_buddy(pse);
5569                 next_buddy_marked = 1;
5570         }
5571 
5572         /*
5573          * We can come here with TIF_NEED_RESCHED already set from new task
5574          * wake up path.
5575          *
5576          * Note: this also catches the edge-case of curr being in a throttled
5577          * group (e.g. via set_curr_task), since update_curr() (in the
5578          * enqueue of curr) will have resulted in resched being set.  This
5579          * prevents us from potentially nominating it as a false LAST_BUDDY
5580          * below.
5581          */
5582         if (test_tsk_need_resched(curr))
5583                 return;
5584 
5585         /* Idle tasks are by definition preempted by non-idle tasks. */
5586         if (unlikely(curr->policy == SCHED_IDLE) &&
5587             likely(p->policy != SCHED_IDLE))
5588                 goto preempt;
5589 
5590         /*
5591          * Batch and idle tasks do not preempt non-idle tasks (their preemption
5592          * is driven by the tick):
5593          */
5594         if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
5595                 return;
5596 
5597         find_matching_se(&se, &pse);
5598         update_curr(cfs_rq_of(se));
5599         BUG_ON(!pse);
5600         if (wakeup_preempt_entity(se, pse) == 1) {
5601                 /*
5602                  * Bias pick_next to pick the sched entity that is
5603                  * triggering this preemption.
5604                  */
5605                 if (!next_buddy_marked)
5606                         set_next_buddy(pse);
5607                 goto preempt;
5608         }
5609 
5610         return;
5611 
5612 preempt:
5613         resched_curr(rq);
5614         /*
5615          * Only set the backward buddy when the current task is still
5616          * on the rq. This can happen when a wakeup gets interleaved
5617          * with schedule on the ->pre_schedule() or idle_balance()
5618          * point, either of which can * drop the rq lock.
5619          *
5620          * Also, during early boot the idle thread is in the fair class,
5621          * for obvious reasons its a bad idea to schedule back to it.
5622          */
5623         if (unlikely(!se->on_rq || curr == rq->idle))
5624                 return;
5625 
5626         if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
5627                 set_last_buddy(se);
5628 }
5629 
5630 static struct task_struct *
5631 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct pin_cookie cookie)
5632 {
5633         struct cfs_rq *cfs_rq = &rq->cfs;
5634         struct sched_entity *se;
5635         struct task_struct *p;
5636         int new_tasks;
5637 
5638 again:
5639 #ifdef CONFIG_FAIR_GROUP_SCHED
5640         if (!cfs_rq->nr_running)
5641                 goto idle;
5642 
5643         if (prev->sched_class != &fair_sched_class)
5644                 goto simple;
5645 
5646         /*
5647          * Because of the set_next_buddy() in dequeue_task_fair() it is rather
5648          * likely that a next task is from the same cgroup as the current.
5649          *
5650          * Therefore attempt to avoid putting and setting the entire cgroup
5651          * hierarchy, only change the part that actually changes.
5652          */
5653 
5654         do {
5655                 struct sched_entity *curr = cfs_rq->curr;
5656 
5657                 /*
5658                  * Since we got here without doing put_prev_entity() we also
5659                  * have to consider cfs_rq->curr. If it is still a runnable
5660                  * entity, update_curr() will update its vruntime, otherwise
5661                  * forget we've ever seen it.
5662                  */
5663                 if (curr) {
5664                         if (curr->on_rq)
5665                                 update_curr(cfs_rq);
5666                         else
5667                                 curr = NULL;
5668 
5669                         /*
5670                          * This call to check_cfs_rq_runtime() will do the
5671                          * throttle and dequeue its entity in the parent(s).
5672                          * Therefore the 'simple' nr_running test will indeed
5673                          * be correct.
5674                          */
5675                         if (unlikely(check_cfs_rq_runtime(cfs_rq)))
5676                                 goto simple;
5677                 }
5678 
5679                 se = pick_next_entity(cfs_rq, curr);
5680                 cfs_rq = group_cfs_rq(se);
5681         } while (cfs_rq);
5682 
5683         p = task_of(se);
5684 
5685         /*
5686          * Since we haven't yet done put_prev_entity and if the selected task
5687          * is a different task than we started out with, try and touch the
5688          * least amount of cfs_rqs.
5689          */
5690         if (prev != p) {
5691                 struct sched_entity *pse = &prev->se;
5692 
5693                 while (!(cfs_rq = is_same_group(se, pse))) {
5694                         int se_depth = se->depth;
5695                         int pse_depth = pse->depth;
5696 
5697                         if (se_depth <= pse_depth) {
5698                                 put_prev_entity(cfs_rq_of(pse), pse);
5699                                 pse = parent_entity(pse);
5700                         }
5701                         if (se_depth >= pse_depth) {
5702                                 set_next_entity(cfs_rq_of(se), se);
5703                                 se = parent_entity(se);
5704                         }
5705                 }
5706 
5707                 put_prev_entity(cfs_rq, pse);
5708                 set_next_entity(cfs_rq, se);
5709         }
5710 
5711         if (hrtick_enabled(rq))
5712                 hrtick_start_fair(rq, p);
5713 
5714         return p;
5715 simple:
5716         cfs_rq = &rq->cfs;
5717 #endif
5718 
5719         if (!cfs_rq->nr_running)
5720                 goto idle;
5721 
5722         put_prev_task(rq, prev);
5723 
5724         do {
5725                 se = pick_next_entity(cfs_rq, NULL);
5726                 set_next_entity(cfs_rq, se);
5727                 cfs_rq = group_cfs_rq(se);
5728         } while (cfs_rq);
5729 
5730         p = task_of(se);
5731 
5732         if (hrtick_enabled(rq))
5733                 hrtick_start_fair(rq, p);
5734 
5735         return p;
5736 
5737 idle:
5738         /*
5739          * This is OK, because current is on_cpu, which avoids it being picked
5740          * for load-balance and preemption/IRQs are still disabled avoiding
5741          * further scheduler activity on it and we're being very careful to
5742          * re-start the picking loop.
5743          */
5744         lockdep_unpin_lock(&rq->lock, cookie);
5745         new_tasks = idle_balance(rq);
5746         lockdep_repin_lock(&rq->lock, cookie);
5747         /*
5748          * Because idle_balance() releases (and re-acquires) rq->lock, it is
5749          * possible for any higher priority task to appear. In that case we
5750          * must re-start the pick_next_entity() loop.
5751          */
5752         if (new_tasks < 0)
5753                 return RETRY_TASK;
5754 
5755         if (new_tasks > 0)
5756                 goto again;
5757 
5758         return NULL;
5759 }
5760 
5761 /*
5762  * Account for a descheduled task:
5763  */
5764 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
5765 {
5766         struct sched_entity *se = &prev->se;
5767         struct cfs_rq *cfs_rq;
5768 
5769         for_each_sched_entity(se) {
5770                 cfs_rq = cfs_rq_of(se);
5771                 put_prev_entity(cfs_rq, se);
5772         }
5773 }
5774 
5775 /*
5776  * sched_yield() is very simple
5777  *
5778  * The magic of dealing with the ->skip buddy is in pick_next_entity.
5779  */
5780 static void yield_task_fair(struct rq *rq)
5781 {
5782         struct task_struct *curr = rq->curr;
5783         struct cfs_rq *cfs_rq = task_cfs_rq(curr);
5784         struct sched_entity *se = &curr->se;
5785 
5786         /*
5787          * Are we the only task in the tree?
5788          */
5789         if (unlikely(rq->nr_running == 1))
5790                 return;
5791 
5792         clear_buddies(cfs_rq, se);
5793 
5794         if (curr->policy != SCHED_BATCH) {
5795                 update_rq_clock(rq);
5796                 /*
5797                  * Update run-time statistics of the 'current'.
5798                  */
5799                 update_curr(cfs_rq);
5800                 /*
5801                  * Tell update_rq_clock() that we've just updated,
5802                  * so we don't do microscopic update in schedule()
5803                  * and double the fastpath cost.
5804                  */
5805                 rq_clock_skip_update(rq, true);
5806         }
5807 
5808         set_skip_buddy(se);
5809 }
5810 
5811 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preempt)
5812 {
5813         struct sched_entity *se = &p->se;
5814 
5815         /* throttled hierarchies are not runnable */
5816         if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se)))
5817                 return false;
5818 
5819         /* Tell the scheduler that we'd really like pse to run next. */
5820         set_next_buddy(se);
5821 
5822         yield_task_fair(rq);
5823 
5824         return true;
5825 }
5826 
5827 #ifdef CONFIG_SMP
5828 /**************************************************
5829  * Fair scheduling class load-balancing methods.
5830  *
5831  * BASICS
5832  *
5833  * The purpose of load-balancing is to achieve the same basic fairness the
5834  * per-cpu scheduler provides, namely provide a proportional amount of compute
5835  * time to each task. This is expressed in the following equation:
5836  *
5837  *   W_i,n/P_i == W_j,n/P_j for all i,j                               (1)
5838  *
5839  * Where W_i,n is the n-th weight average for cpu i. The instantaneous weight
5840  * W_i,0 is defined as:
5841  *
5842  *   W_i,0 = \Sum_j w_i,j                                             (2)
5843  *
5844  * Where w_i,j is the weight of the j-th runnable task on cpu i. This weight
5845  * is derived from the nice value as per sched_prio_to_weight[].
5846  *
5847  * The weight average is an exponential decay average of the instantaneous
5848  * weight:
5849  *
5850  *   W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0               (3)
5851  *
5852  * C_i is the compute capacity of cpu i, typically it is the
5853  * fraction of 'recent' time available for SCHED_OTHER task execution. But it
5854  * can also include other factors [XXX].
5855  *
5856  * To achieve this balance we define a measure of imbalance which follows
5857  * directly from (1):
5858  *
5859  *   imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j }    (4)
5860  *
5861  * We them move tasks around to minimize the imbalance. In the continuous
5862  * function space it is obvious this converges, in the discrete case we get
5863  * a few fun cases generally called infeasible weight scenarios.
5864  *
5865  * [XXX expand on:
5866  *     - infeasible weights;
5867  *     - local vs global optima in the discrete case. ]
5868  *
5869  *
5870  * SCHED DOMAINS
5871  *
5872  * In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
5873  * for all i,j solution, we create a tree of cpus that follows the hardware
5874  * topology where each level pairs two lower groups (or better). This results
5875  * in O(log n) layers. Furthermore we reduce the number of cpus going up the
5876  * tree to only the first of the previous level and we decrease the frequency
5877  * of load-balance at each level inv. proportional to the number of cpus in
5878  * the groups.
5879  *
5880  * This yields:
5881  *
5882  *     log_2 n     1     n
5883  *   \Sum       { --- * --- * 2^i } = O(n)                            (5)
5884  *     i = 0      2^i   2^i
5885  *                               `- size of each group
5886  *         |         |     `- number of cpus doing load-balance
5887  *         |         `- freq
5888  *         `- sum over all levels
5889  *
5890  * Coupled with a limit on how many tasks we can migrate every balance pass,
5891  * this makes (5) the runtime complexity of the balancer.
5892  *
5893  * An important property here is that each CPU is still (indirectly) connected
5894  * to every other cpu in at most O(log n) steps:
5895  *
5896  * The adjacency matrix of the resulting graph is given by:
5897  *
5898  *             log_2 n     
5899  *   A_i,j = \Union     (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1)  (6)
5900  *             k = 0
5901  *
5902  * And you'll find that:
5903  *
5904  *   A^(log_2 n)_i,j != 0  for all i,j                                (7)
5905  *
5906  * Showing there's indeed a path between every cpu in at most O(log n) steps.
5907  * The task movement gives a factor of O(m), giving a convergence complexity
5908  * of:
5909  *
5910  *   O(nm log n),  n := nr_cpus, m := nr_tasks                        (8)
5911  *
5912  *
5913  * WORK CONSERVING
5914  *
5915  * In order to avoid CPUs going idle while there's still work to do, new idle
5916  * balancing is more aggressive and has the newly idle cpu iterate up the domain
5917  * tree itself instead of relying on other CPUs to bring it work.
5918  *
5919  * This adds some complexity to both (5) and (8) but it reduces the total idle
5920  * time.
5921  *
5922  * [XXX more?]
5923  *
5924  *
5925  * CGROUPS
5926  *
5927  * Cgroups make a horror show out of (2), instead of a simple sum we get:
5928  *
5929  *                                s_k,i
5930  *   W_i,0 = \Sum_j \Prod_k w_k * -----                               (9)
5931  *                                 S_k
5932  *
5933  * Where
5934  *
5935  *   s_k,i = \Sum_j w_i,j,k  and  S_k = \Sum_i s_k,i                 (10)
5936  *
5937  * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on cpu i.
5938  *
5939  * The big problem is S_k, its a global sum needed to compute a local (W_i)
5940  * property.
5941  *
5942  * [XXX write more on how we solve this.. _after_ merging pjt's patches that
5943  *      rewrite all of this once again.]
5944  */ 
5945 
5946 static unsigned long __read_mostly max_load_balance_interval = HZ/10;
5947 
5948 enum fbq_type { regular, remote, all };
5949 
5950 #define LBF_ALL_PINNED  0x01
5951 #define LBF_NEED_BREAK  0x02
5952 #define LBF_DST_PINNED  0x04
5953 #define LBF_SOME_PINNED 0x08
5954 
5955 struct lb_env {
5956         struct sched_domain     *sd;
5957 
5958         struct rq               *src_rq;
5959         int                     src_cpu;
5960 
5961         int                     dst_cpu;
5962         struct rq               *dst_rq;
5963 
5964         struct cpumask          *dst_grpmask;
5965         int                     new_dst_cpu;
5966         enum cpu_idle_type      idle;
5967         long                    imbalance;
5968         /* The set of CPUs under consideration for load-balancing */
5969         struct cpumask          *cpus;
5970 
5971         unsigned int            flags;
5972 
5973         unsigned int            loop;
5974         unsigned int            loop_break;
5975         unsigned int            loop_max;
5976 
5977         enum fbq_type           fbq_type;
5978         struct list_head        tasks;
5979 };
5980 
5981 /*
5982  * Is this task likely cache-hot:
5983  */
5984 static int task_hot(struct task_struct *p, struct lb_env *env)
5985 {
5986         s64 delta;
5987 
5988         lockdep_assert_held(&env->src_rq->lock);
5989 
5990         if (p->sched_class != &fair_sched_class)
5991                 return 0;
5992 
5993         if (unlikely(p->policy == SCHED_IDLE))
5994                 return 0;
5995 
5996         /*
5997          * Buddy candidates are cache hot:
5998          */
5999         if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
6000                         (&p->se == cfs_rq_of(&p->se)->next ||
6001                          &p->se == cfs_rq_of(&p->se)->last))
6002                 return 1;
6003 
6004         if (sysctl_sched_migration_cost == -1)
6005                 return 1;
6006         if (sysctl_sched_migration_cost == 0)
6007                 return 0;
6008 
6009         delta = rq_clock_task(env->src_rq) - p->se.exec_start;
6010 
6011         return delta < (s64)sysctl_sched_migration_cost;
6012 }
6013 
6014 #ifdef CONFIG_NUMA_BALANCING
6015 /*
6016  * Returns 1, if task migration degrades locality
6017  * Returns 0, if task migration improves locality i.e migration preferred.
6018  * Returns -1, if task migration is not affected by locality.
6019  */
6020 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
6021 {
6022         struct numa_group *numa_group = rcu_dereference(p->numa_group);
6023         unsigned long src_faults, dst_faults;
6024         int src_nid, dst_nid;
6025 
6026         if (!static_branch_likely(&sched_numa_balancing))
6027                 return -1;
6028 
6029         if (!p->numa_faults || !(env->sd->flags & SD_NUMA))
6030                 return -1;
6031 
6032         src_nid = cpu_to_node(env->src_cpu);
6033         dst_nid = cpu_to_node(env->dst_cpu);
6034 
6035         if (src_nid == dst_nid)
6036                 return -1;
6037 
6038         /* Migrating away from the preferred node is always bad. */
6039         if (src_nid == p->numa_preferred_nid) {
6040                 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
6041                         return 1;
6042                 else
6043                         return -1;
6044         }
6045 
6046         /* Encourage migration to the preferred node. */
6047         if (dst_nid == p->numa_preferred_nid)
6048                 return 0;
6049 
6050         if (numa_group) {
6051                 src_faults = group_faults(p, src_nid);
6052                 dst_faults = group_faults(p, dst_nid);
6053         } else {
6054                 src_faults = task_faults(p, src_nid);
6055                 dst_faults = task_faults(p, dst_nid);
6056         }
6057 
6058         return dst_faults < src_faults;
6059 }
6060 
6061 #else
6062 static inline int migrate_degrades_locality(struct task_struct *p,
6063                                              struct lb_env *env)
6064 {
6065         return -1;
6066 }
6067 #endif
6068 
6069 /*
6070  * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
6071  */
6072 static
6073 int can_migrate_task(struct task_struct *p, struct lb_env *env)
6074 {
6075         int tsk_cache_hot;
6076 
6077         lockdep_assert_held(&env->src_rq->lock);
6078 
6079         /*
6080          * We do not migrate tasks that are:
6081          * 1) throttled_lb_pair, or
6082          * 2) cannot be migrated to this CPU due to cpus_allowed, or
6083          * 3) running (obviously), or
6084          * 4) are cache-hot on their current CPU.
6085          */
6086         if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
6087                 return 0;
6088 
6089         if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) {
6090                 int cpu;
6091 
6092                 schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
6093 
6094                 env->flags |= LBF_SOME_PINNED;
6095 
6096                 /*
6097                  * Remember if this task can be migrated to any other cpu in
6098                  * our sched_group. We may want to revisit it if we couldn't
6099                  * meet load balance goals by pulling other tasks on src_cpu.
6100                  *
6101                  * Also avoid computing new_dst_cpu if we have already computed
6102                  * one in current iteration.
6103                  */
6104                 if (!env->dst_grpmask || (env->flags & LBF_DST_PINNED))
6105                         return 0;
6106 
6107                 /* Prevent to re-select dst_cpu via env's cpus */
6108                 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
6109                         if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) {
6110                                 env->flags |= LBF_DST_PINNED;
6111                                 env->new_dst_cpu = cpu;
6112                                 break;
6113                         }
6114                 }
6115 
6116                 return 0;
6117         }
6118 
6119         /* Record that we found atleast one task that could run on dst_cpu */
6120         env->flags &= ~LBF_ALL_PINNED;
6121 
6122         if (task_running(env->src_rq, p)) {
6123                 schedstat_inc(p, se.statistics.nr_failed_migrations_running);
6124                 return 0;
6125         }
6126 
6127         /*
6128          * Aggressive migration if:
6129          * 1) destination numa is preferred
6130          * 2) task is cache cold, or
6131          * 3) too many balance attempts have failed.
6132          */
6133         tsk_cache_hot = migrate_degrades_locality(p, env);
6134         if (tsk_cache_hot == -1)
6135                 tsk_cache_hot = task_hot(p, env);
6136 
6137         if (tsk_cache_hot <= 0 ||
6138             env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
6139                 if (tsk_cache_hot == 1) {
6140                         schedstat_inc(env->sd, lb_hot_gained[env->idle]);
6141                         schedstat_inc(p, se.statistics.nr_forced_migrations);
6142                 }
6143                 return 1;
6144         }
6145 
6146         schedstat_inc(p, se.statistics.nr_failed_migrations_hot);
6147         return 0;
6148 }
6149 
6150 /*
6151  * detach_task() -- detach the task for the migration specified in env
6152  */
6153 static void detach_task(struct task_struct *p, struct lb_env *env)
6154 {
6155         lockdep_assert_held(&env->src_rq->lock);
6156 
6157         p->on_rq = TASK_ON_RQ_MIGRATING;
6158         deactivate_task(env->src_rq, p, 0);
6159         set_task_cpu(p, env->dst_cpu);
6160 }
6161 
6162 /*
6163  * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as
6164  * part of active balancing operations within "domain".
6165  *
6166  * Returns a task if successful and NULL otherwise.
6167  */
6168 static struct task_struct *detach_one_task(struct lb_env *env)
6169 {
6170         struct task_struct *p, *n;
6171 
6172         lockdep_assert_held(&env->src_rq->lock);
6173 
6174         list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) {
6175                 if (!can_migrate_task(p, env))
6176                         continue;
6177 
6178                 detach_task(p, env);
6179 
6180                 /*
6181                  * Right now, this is only the second place where
6182                  * lb_gained[env->idle] is updated (other is detach_tasks)
6183                  * so we can safely collect stats here rather than
6184                  * inside detach_tasks().
6185                  */
6186                 schedstat_inc(env->sd, lb_gained[env->idle]);
6187                 return p;
6188         }
6189         return NULL;
6190 }
6191 
6192 static const unsigned int sched_nr_migrate_break = 32;
6193 
6194 /*
6195  * detach_tasks() -- tries to detach up to imbalance weighted load from
6196  * busiest_rq, as part of a balancing operation within domain "sd".
6197  *
6198  * Returns number of detached tasks if successful and 0 otherwise.
6199  */
6200 static int detach_tasks(struct lb_env *env)
6201 {
6202         struct list_head *tasks = &env->src_rq->cfs_tasks;
6203         struct task_struct *p;
6204         unsigned long load;
6205         int detached = 0;
6206 
6207         lockdep_assert_held(&env->src_rq->lock);
6208 
6209         if (env->imbalance <= 0)
6210                 return 0;
6211 
6212         while (!list_empty(tasks)) {
6213                 /*
6214                  * We don't want to steal all, otherwise we may be treated likewise,
6215                  * which could at worst lead to a livelock crash.
6216                  */
6217                 if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
6218                         break;
6219 
6220                 p = list_first_entry(tasks, struct task_struct, se.group_node);
6221 
6222                 env->loop++;
6223                 /* We've more or less seen every task there is, call it quits */
6224                 if (env->loop > env->loop_max)
6225                         break;
6226 
6227                 /* take a breather every nr_migrate tasks */
6228                 if (env->loop > env->loop_break) {
6229                         env->loop_break += sched_nr_migrate_break;
6230                         env->flags |= LBF_NEED_BREAK;
6231                         break;
6232                 }
6233 
6234                 if (!can_migrate_task(p, env))
6235                         goto next;
6236 
6237                 load = task_h_load(p);
6238 
6239                 if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed)
6240                         goto next;
6241 
6242                 if ((load / 2) > env->imbalance)
6243                         goto next;
6244 
6245                 detach_task(p, env);
6246                 list_add(&p->se.group_node, &env->tasks);
6247 
6248                 detached++;
6249                 env->imbalance -= load;
6250 
6251 #ifdef CONFIG_PREEMPT
6252                 /*
6253                  * NEWIDLE balancing is a source of latency, so preemptible
6254                  * kernels will stop after the first task is detached to minimize
6255                  * the critical section.
6256                  */
6257                 if (env->idle == CPU_NEWLY_IDLE)
6258                         break;
6259 #endif
6260 
6261                 /*
6262                  * We only want to steal up to the prescribed amount of
6263                  * weighted load.
6264                  */
6265                 if (env->imbalance <= 0)
6266                         break;
6267 
6268                 continue;
6269 next:
6270                 list_move_tail(&p->se.group_node, tasks);
6271         }
6272 
6273         /*
6274          * Right now, this is one of only two places we collect this stat
6275          * so we can safely collect detach_one_task() stats here rather
6276          * than inside detach_one_task().
6277          */
6278         schedstat_add(env->sd, lb_gained[env->idle], detached);
6279 
6280         return detached;
6281 }
6282 
6283 /*
6284  * attach_task() -- attach the task detached by detach_task() to its new rq.
6285  */
6286 static void attach_task(struct rq *rq, struct task_struct *p)
6287 {
6288         lockdep_assert_held(&rq->lock);
6289 
6290         BUG_ON(task_rq(p) != rq);
6291         activate_task(rq, p, 0);
6292         p->on_rq = TASK_ON_RQ_QUEUED;
6293         check_preempt_curr(rq, p, 0);
6294 }
6295 
6296 /*
6297  * attach_one_task() -- attaches the task returned from detach_one_task() to
6298  * its new rq.
6299  */
6300 static void attach_one_task(struct rq *rq, struct task_struct *p)
6301 {
6302         raw_spin_lock(&rq->lock);
6303         attach_task(rq, p);
6304         raw_spin_unlock(&rq->lock);
6305 }
6306 
6307 /*
6308  * attach_tasks() -- attaches all tasks detached by detach_tasks() to their
6309  * new rq.
6310  */
6311 static void attach_tasks(struct lb_env *env)
6312 {
6313         struct list_head *tasks = &env->tasks;
6314         struct task_struct *p;
6315 
6316         raw_spin_lock(&env->dst_rq->lock);
6317 
6318         while (!list_empty(tasks)) {
6319                 p = list_first_entry(tasks, struct task_struct, se.group_node);
6320                 list_del_init(&p->se.group_node);
6321 
6322                 attach_task(env->dst_rq, p);
6323         }
6324 
6325         raw_spin_unlock(&env->dst_rq->lock);
6326 }
6327 
6328 #ifdef CONFIG_FAIR_GROUP_SCHED
6329 static void update_blocked_averages(int cpu)
6330 {
6331         struct rq *rq = cpu_rq(cpu);
6332         struct cfs_rq *cfs_rq;
6333         unsigned long flags;
6334 
6335         raw_spin_lock_irqsave(&rq->lock, flags);
6336         update_rq_clock(rq);
6337 
6338         /*
6339          * Iterates the task_group tree in a bottom up fashion, see
6340          * list_add_leaf_cfs_rq() for details.
6341          */
6342         for_each_leaf_cfs_rq(rq, cfs_rq) {
6343                 /* throttled entities do not contribute to load */
6344                 if (throttled_hierarchy(cfs_rq))
6345                         continue;
6346 
6347                 if (update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq, true))
6348                         update_tg_load_avg(cfs_rq, 0);
6349         }
6350         raw_spin_unlock_irqrestore(&rq->lock, flags);
6351 }
6352 
6353 /*
6354  * Compute the hierarchical load factor for cfs_rq and all its ascendants.
6355  * This needs to be done in a top-down fashion because the load of a child
6356  * group is a fraction of its parents load.
6357  */
6358 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
6359 {
6360         struct rq *rq = rq_of(cfs_rq);
6361         struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
6362         unsigned long now = jiffies;
6363         unsigned long load;
6364 
6365         if (cfs_rq->last_h_load_update == now)
6366                 return;
6367 
6368         cfs_rq->h_load_next = NULL;
6369         for_each_sched_entity(se) {
6370                 cfs_rq = cfs_rq_of(se);
6371                 cfs_rq->h_load_next = se;
6372                 if (cfs_rq->last_h_load_update == now)
6373                         break;
6374         }
6375 
6376         if (!se) {
6377                 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq);
6378                 cfs_rq->last_h_load_update = now;
6379         }
6380 
6381         while ((se = cfs_rq->h_load_next) != NULL) {
6382                 load = cfs_rq->h_load;
6383                 load = div64_ul(load * se->avg.load_avg,
6384                         cfs_rq_load_avg(cfs_rq) + 1);
6385                 cfs_rq = group_cfs_rq(se);
6386                 cfs_rq->h_load = load;
6387                 cfs_rq->last_h_load_update = now;
6388         }
6389 }
6390 
6391 static unsigned long task_h_load(struct task_struct *p)
6392 {
6393         struct cfs_rq *cfs_rq = task_cfs_rq(p);
6394 
6395         update_cfs_rq_h_load(cfs_rq);
6396         return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,
6397                         cfs_rq_load_avg(cfs_rq) + 1);
6398 }
6399 #else
6400 static inline void update_blocked_averages(int cpu)
6401 {
6402         struct rq *rq = cpu_rq(cpu);
6403         struct cfs_rq *cfs_rq = &rq->cfs;
6404         unsigned long flags;
6405 
6406         raw_spin_lock_irqsave(&rq->lock, flags);
6407         update_rq_clock(rq);
6408         update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq, true);
6409         raw_spin_unlock_irqrestore(&rq->lock, flags);
6410 }
6411 
6412 static unsigned long task_h_load(struct task_struct *p)
6413 {
6414         return p->se.avg.load_avg;
6415 }
6416 #endif
6417 
6418 /********** Helpers for find_busiest_group ************************/
6419 
6420 enum group_type {
6421         group_other = 0,
6422         group_imbalanced,
6423         group_overloaded,
6424 };
6425 
6426 /*
6427  * sg_lb_stats - stats of a sched_group required for load_balancing
6428  */
6429 struct sg_lb_stats {
6430         unsigned long avg_load; /*Avg load across the CPUs of the group */
6431         unsigned long group_load; /* Total load over the CPUs of the group */
6432         unsigned long sum_weighted_load; /* Weighted load of group's tasks */
6433         unsigned long load_per_task;
6434         unsigned long group_capacity;
6435         unsigned long group_util; /* Total utilization of the group */
6436         unsigned int sum_nr_running; /* Nr tasks running in the group */
6437         unsigned int idle_cpus;
6438         unsigned int group_weight;
6439         enum group_type group_type;
6440         int group_no_capacity;
6441 #ifdef CONFIG_NUMA_BALANCING
6442         unsigned int nr_numa_running;
6443         unsigned int nr_preferred_running;
6444 #endif
6445 };
6446 
6447 /*
6448  * sd_lb_stats - Structure to store the statistics of a sched_domain
6449  *               during load balancing.
6450  */
6451 struct sd_lb_stats {
6452         struct sched_group *busiest;    /* Busiest group in this sd */
6453         struct sched_group *local;      /* Local group in this sd */
6454         unsigned long total_load;       /* Total load of all groups in sd */
6455         unsigned long total_capacity;   /* Total capacity of all groups in sd */
6456         unsigned long avg_load; /* Average load across all groups in sd */
6457 
6458         struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
6459         struct sg_lb_stats local_stat;  /* Statistics of the local group */
6460 };
6461 
6462 static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
6463 {
6464         /*
6465          * Skimp on the clearing to avoid duplicate work. We can avoid clearing
6466          * local_stat because update_sg_lb_stats() does a full clear/assignment.
6467          * We must however clear busiest_stat::avg_load because
6468          * update_sd_pick_busiest() reads this before assignment.
6469          */
6470         *sds = (struct sd_lb_stats){
6471                 .busiest = NULL,
6472                 .local = NULL,
6473                 .total_load = 0UL,
6474                 .total_capacity = 0UL,
6475                 .busiest_stat = {
6476                         .avg_load = 0UL,
6477                         .sum_nr_running = 0,
6478                         .group_type = group_other,
6479                 },
6480         };
6481 }
6482 
6483 /**
6484  * get_sd_load_idx - Obtain the load index for a given sched domain.
6485  * @sd: The sched_domain whose load_idx is to be obtained.
6486  * @idle: The idle status of the CPU for whose sd load_idx is obtained.
6487  *
6488  * Return: The load index.
6489  */
6490 static inline int get_sd_load_idx(struct sched_domain *sd,
6491                                         enum cpu_idle_type idle)
6492 {
6493         int load_idx;
6494 
6495         switch (idle) {
6496         case CPU_NOT_IDLE:
6497                 load_idx = sd->busy_idx;
6498                 break;
6499 
6500         case CPU_NEWLY_IDLE:
6501                 load_idx = sd->newidle_idx;
6502                 break;
6503         default:
6504                 load_idx = sd->idle_idx;
6505                 break;
6506         }
6507 
6508         return load_idx;
6509 }
6510 
6511 static unsigned long scale_rt_capacity(int cpu)
6512 {
6513         struct rq *rq = cpu_rq(cpu);
6514         u64 total, used, age_stamp, avg;
6515         s64 delta;
6516 
6517         /*
6518          * Since we're reading these variables without serialization make sure
6519          * we read them once before doing sanity checks on them.
6520          */
6521         age_stamp = READ_ONCE(rq->age_stamp);
6522         avg = READ_ONCE(rq->rt_avg);
6523         delta = __rq_clock_broken(rq) - age_stamp;
6524 
6525         if (unlikely(delta < 0))
6526                 delta = 0;
6527 
6528         total = sched_avg_period() + delta;
6529 
6530         used = div_u64(avg, total);
6531 
6532         if (likely(used < SCHED_CAPACITY_SCALE))
6533                 return SCHED_CAPACITY_SCALE - used;
6534 
6535         return 1;
6536 }
6537 
6538 static void update_cpu_capacity(struct sched_domain *sd, int cpu)
6539 {
6540         unsigned long capacity = arch_scale_cpu_capacity(sd, cpu);
6541         struct sched_group *sdg = sd->groups;
6542 
6543         cpu_rq(cpu)->cpu_capacity_orig = capacity;
6544 
6545         capacity *= scale_rt_capacity(cpu);
6546         capacity >>= SCHED_CAPACITY_SHIFT;
6547 
6548         if (!capacity)
6549                 capacity = 1;
6550 
6551         cpu_rq(cpu)->cpu_capacity = capacity;
6552         sdg->sgc->capacity = capacity;
6553 }
6554 
6555 void update_group_capacity(struct sched_domain *sd, int cpu)
6556 {
6557         struct sched_domain *child = sd->child;
6558         struct sched_group *group, *sdg = sd->groups;
6559         unsigned long capacity;
6560         unsigned long interval;
6561 
6562         interval = msecs_to_jiffies(sd->balance_interval);
6563         interval = clamp(interval, 1UL, max_load_balance_interval);
6564         sdg->sgc->next_update = jiffies + interval;
6565 
6566         if (!child) {
6567                 update_cpu_capacity(sd, cpu);
6568                 return;
6569         }
6570 
6571         capacity = 0;
6572 
6573         if (child->flags & SD_OVERLAP) {
6574                 /*
6575                  * SD_OVERLAP domains cannot assume that child groups
6576                  * span the current group.
6577                  */
6578 
6579                 for_each_cpu(cpu, sched_group_cpus(sdg)) {
6580                         struct sched_group_capacity *sgc;
6581                         struct rq *rq = cpu_rq(cpu);
6582 
6583                         /*
6584                          * build_sched_domains() -> init_sched_groups_capacity()
6585                          * gets here before we've attached the domains to the
6586                          * runqueues.
6587                          *
6588                          * Use capacity_of(), which is set irrespective of domains
6589                          * in update_cpu_capacity().
6590                          *
6591                          * This avoids capacity from being 0 and
6592                          * causing divide-by-zero issues on boot.
6593                          */
6594                         if (unlikely(!rq->sd)) {
6595                                 capacity += capacity_of(cpu);
6596                                 continue;
6597                         }
6598 
6599                         sgc = rq->sd->groups->sgc;
6600                         capacity += sgc->capacity;
6601                 }
6602         } else  {
6603                 /*
6604                  * !SD_OVERLAP domains can assume that child groups
6605                  * span the current group.
6606                  */ 
6607 
6608                 group = child->groups;
6609                 do {
6610                         capacity += group->sgc->capacity;
6611                         group = group->next;
6612                 } while (group != child->groups);
6613         }
6614 
6615         sdg->sgc->capacity = capacity;
6616 }
6617 
6618 /*
6619  * Check whether the capacity of the rq has been noticeably reduced by side
6620  * activity. The imbalance_pct is used for the threshold.
6621  * Return true is the capacity is reduced
6622  */
6623 static inline int
6624 check_cpu_capacity(struct rq *rq, struct sched_domain *sd)
6625 {
6626         return ((rq->cpu_capacity * sd->imbalance_pct) <
6627                                 (rq->cpu_capacity_orig * 100));
6628 }
6629 
6630 /*
6631  * Group imbalance indicates (and tries to solve) the problem where balancing
6632  * groups is inadequate due to tsk_cpus_allowed() constraints.
6633  *
6634  * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a
6635  * cpumask covering 1 cpu of the first group and 3 cpus of the second group.
6636  * Something like:
6637  *
6638  *      { 0 1 2 3 } { 4 5 6 7 }
6639  *              *     * * *
6640  *
6641  * If we were to balance group-wise we'd place two tasks in the first group and
6642  * two tasks in the second group. Clearly this is undesired as it will overload
6643  * cpu 3 and leave one of the cpus in the second group unused.
6644  *
6645  * The current solution to this issue is detecting the skew in the first group
6646  * by noticing the lower domain failed to reach balance and had difficulty
6647  * moving tasks due to affinity constraints.
6648  *
6649  * When this is so detected; this group becomes a candidate for busiest; see
6650  * update_sd_pick_busiest(). And calculate_imbalance() and
6651  * find_busiest_group() avoid some of the usual balance conditions to allow it
6652  * to create an effective group imbalance.
6653  *
6654  * This is a somewhat tricky proposition since the next run might not find the
6655  * group imbalance and decide the groups need to be balanced again. A most
6656  * subtle and fragile situation.
6657  */
6658 
6659 static inline int sg_imbalanced(struct sched_group *group)
6660 {
6661         return group->sgc->imbalance;
6662 }
6663 
6664 /*
6665  * group_has_capacity returns true if the group has spare capacity that could
6666  * be used by some tasks.
6667  * We consider that a group has spare capacity if the  * number of task is
6668  * smaller than the number of CPUs or if the utilization is lower than the
6669  * available capacity for CFS tasks.
6670  * For the latter, we use a threshold to stabilize the state, to take into
6671  * account the variance of the tasks' load and to return true if the available
6672  * capacity in meaningful for the load balancer.
6673  * As an example, an available capacity of 1% can appear but it doesn't make
6674  * any benefit for the load balance.
6675  */
6676 static inline bool
6677 group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs)
6678 {
6679         if (sgs->sum_nr_running < sgs->group_weight)
6680                 return true;
6681 
6682         if ((sgs->group_capacity * 100) >
6683                         (sgs->group_util * env->sd->imbalance_pct))
6684                 return true;
6685 
6686         return false;
6687 }
6688 
6689 /*
6690  *  group_is_overloaded returns true if the group has more tasks than it can
6691  *  handle.
6692  *  group_is_overloaded is not equals to !group_has_capacity because a group
6693  *  with the exact right number of tasks, has no more spare capacity but is not
6694  *  overloaded so both group_has_capacity and group_is_overloaded return
6695  *  false.
6696  */
6697 static inline bool
6698 group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs)
6699 {
6700         if (sgs->sum_nr_running <= sgs->group_weight)
6701                 return false;
6702 
6703         if ((sgs->group_capacity * 100) <
6704                         (sgs->group_util * env->sd->imbalance_pct))
6705                 return true;
6706 
6707         return false;
6708 }
6709 
6710 static inline enum
6711 group_type group_classify(struct sched_group *group,
6712                           struct sg_lb_stats *sgs)
6713 {
6714         if (sgs->group_no_capacity)
6715                 return group_overloaded;
6716 
6717         if (sg_imbalanced(group))
6718                 return group_imbalanced;
6719 
6720         return group_other;
6721 }
6722 
6723 /**
6724  * update_sg_lb_stats - Update sched_group's statistics for load balancing.
6725  * @env: The load balancing environment.
6726  * @group: sched_group whose statistics are to be updated.
6727  * @load_idx: Load index of sched_domain of this_cpu for load calc.
6728  * @local_group: Does group contain this_cpu.
6729  * @sgs: variable to hold the statistics for this group.
6730  * @overload: Indicate more than one runnable task for any CPU.
6731  */
6732 static inline void update_sg_lb_stats(struct lb_env *env,
6733                         struct sched_group *group, int load_idx,
6734                         int local_group, struct sg_lb_stats *sgs,
6735                         bool *overload)
6736 {
6737         unsigned long load;
6738         int i, nr_running;
6739 
6740         memset(sgs, 0, sizeof(*sgs));
6741 
6742         for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
6743                 struct rq *rq = cpu_rq(i);
6744 
6745                 /* Bias balancing toward cpus of our domain */
6746                 if (local_group)
6747                         load = target_load(i, load_idx);
6748                 else
6749                         load = source_load(i, load_idx);
6750 
6751                 sgs->group_load += load;
6752                 sgs->group_util += cpu_util(i);
6753                 sgs->sum_nr_running += rq->cfs.h_nr_running;
6754 
6755                 nr_running = rq->nr_running;
6756                 if (nr_running > 1)
6757                         *overload = true;
6758 
6759 #ifdef CONFIG_NUMA_BALANCING
6760                 sgs->nr_numa_running += rq->nr_numa_running;
6761                 sgs->nr_preferred_running += rq->nr_preferred_running;
6762 #endif
6763                 sgs->sum_weighted_load += weighted_cpuload(i);
6764                 /*
6765                  * No need to call idle_cpu() if nr_running is not 0
6766                  */
6767                 if (!nr_running && idle_cpu(i))
6768                         sgs->idle_cpus++;
6769         }
6770 
6771         /* Adjust by relative CPU capacity of the group */
6772         sgs->group_capacity = group->sgc->capacity;
6773         sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity;
6774 
6775         if (sgs->sum_nr_running)
6776                 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
6777 
6778         sgs->group_weight = group->group_weight;
6779 
6780         sgs->group_no_capacity = group_is_overloaded(env, sgs);
6781         sgs->group_type = group_classify(group, sgs);
6782 }
6783 
6784 /**
6785  * update_sd_pick_busiest - return 1 on busiest group
6786  * @env: The load balancing environment.
6787  * @sds: sched_domain statistics
6788  * @sg: sched_group candidate to be checked for being the busiest
6789  * @sgs: sched_group statistics
6790  *
6791  * Determine if @sg is a busier group than the previously selected
6792  * busiest group.
6793  *
6794  * Return: %true if @sg is a busier group than the previously selected
6795  * busiest group. %false otherwise.
6796  */
6797 static bool update_sd_pick_busiest(struct lb_env *env,
6798                                    struct sd_lb_stats *sds,
6799                                    struct sched_group *sg,
6800                                    struct sg_lb_stats *sgs)
6801 {
6802         struct sg_lb_stats *busiest = &sds->busiest_stat;
6803 
6804         if (sgs->group_type > busiest->group_type)
6805                 return true;
6806 
6807         if (sgs->group_type < busiest->group_type)
6808                 return false;
6809 
6810         if (sgs->avg_load <= busiest->avg_load)
6811                 return false;
6812 
6813         /* This is the busiest node in its class. */
6814         if (!(env->sd->flags & SD_ASYM_PACKING))
6815                 return true;
6816 
6817         /* No ASYM_PACKING if target cpu is already busy */
6818         if (env->idle == CPU_NOT_IDLE)
6819                 return true;
6820         /*
6821          * ASYM_PACKING needs to move all the work to the lowest
6822          * numbered CPUs in the group, therefore mark all groups
6823          * higher than ourself as busy.
6824          */
6825         if (sgs->sum_nr_running && env->dst_cpu < group_first_cpu(sg)) {
6826                 if (!sds->busiest)
6827                         return true;
6828 
6829                 /* Prefer to move from highest possible cpu's work */
6830                 if (group_first_cpu(sds->busiest) < group_first_cpu(sg))
6831                         return true;
6832         }
6833 
6834         return false;
6835 }
6836 
6837 #ifdef CONFIG_NUMA_BALANCING
6838 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
6839 {
6840         if (sgs->sum_nr_running > sgs->nr_numa_running)
6841                 return regular;
6842         if (sgs->sum_nr_running > sgs->nr_preferred_running)
6843                 return remote;
6844         return all;
6845 }
6846 
6847 static inline enum fbq_type fbq_classify_rq(struct rq *rq)
6848 {
6849         if (rq->nr_running > rq->nr_numa_running)
6850                 return regular;
6851         if (rq->nr_running > rq->nr_preferred_running)
6852                 return remote;
6853         return all;
6854 }
6855 #else
6856 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
6857 {
6858         return all;
6859 }
6860 
6861 static inline enum fbq_type fbq_classify_rq(struct rq *rq)
6862 {
6863         return regular;
6864 }
6865 #endif /* CONFIG_NUMA_BALANCING */
6866 
6867 /**
6868  * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
6869  * @env: The load balancing environment.
6870  * @sds: variable to hold the statistics for this sched_domain.
6871  */
6872 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
6873 {
6874         struct sched_domain *child = env->sd->child;
6875         struct sched_group *sg = env->sd->groups;
6876         struct sg_lb_stats tmp_sgs;
6877         int load_idx, prefer_sibling = 0;
6878         bool overload = false;
6879 
6880         if (child && child->flags & SD_PREFER_SIBLING)
6881                 prefer_sibling = 1;
6882 
6883         load_idx = get_sd_load_idx(env->sd, env->idle);
6884 
6885         do {
6886                 struct sg_lb_stats *sgs = &tmp_sgs;
6887                 int local_group;
6888 
6889                 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg));
6890                 if (local_group) {
6891                         sds->local = sg;
6892                         sgs = &sds->local_stat;
6893 
6894                         if (env->idle != CPU_NEWLY_IDLE ||
6895                             time_after_eq(jiffies, sg->sgc->next_update))
6896                                 update_group_capacity(env->sd, env->dst_cpu);
6897                 }
6898 
6899                 update_sg_lb_stats(env, sg, load_idx, local_group, sgs,
6900                                                 &overload);
6901 
6902                 if (local_group)
6903                         goto next_group;
6904 
6905                 /*
6906                  * In case the child domain prefers tasks go to siblings
6907                  * first, lower the sg capacity so that we'll try
6908                  * and move all the excess tasks away. We lower the capacity
6909                  * of a group only if the local group has the capacity to fit
6910                  * these excess tasks. The extra check prevents the case where
6911                  * you always pull from the heaviest group when it is already
6912                  * under-utilized (possible with a large weight task outweighs
6913                  * the tasks on the system).
6914                  */
6915                 if (prefer_sibling && sds->local &&
6916                     group_has_capacity(env, &sds->local_stat) &&
6917                     (sgs->sum_nr_running > 1)) {
6918                         sgs->group_no_capacity = 1;
6919                         sgs->group_type = group_classify(sg, sgs);
6920                 }
6921 
6922                 if (update_sd_pick_busiest(env, sds, sg, sgs)) {
6923                         sds->busiest = sg;
6924                         sds->busiest_stat = *sgs;
6925                 }
6926 
6927 next_group:
6928                 /* Now, start updating sd_lb_stats */
6929                 sds->total_load += sgs->group_load;
6930                 sds->total_capacity += sgs->group_capacity;
6931 
6932                 sg = sg->next;
6933         } while (sg != env->sd->groups);
6934 
6935         if (env->sd->flags & SD_NUMA)
6936                 env->fbq_type = fbq_classify_group(&sds->busiest_stat);
6937 
6938         if (!env->sd->parent) {
6939                 /* update overload indicator if we are at root domain */
6940                 if (env->dst_rq->rd->overload != overload)
6941                         env->dst_rq->rd->overload = overload;
6942         }
6943 
6944 }
6945 
6946 /**
6947  * check_asym_packing - Check to see if the group is packed into the
6948  *                      sched doman.
6949  *
6950  * This is primarily intended to used at the sibling level.  Some
6951  * cores like POWER7 prefer to use lower numbered SMT threads.  In the
6952  * case of POWER7, it can move to lower SMT modes only when higher
6953  * threads are idle.  When in lower SMT modes, the threads will
6954  * perform better since they share less core resources.  Hence when we
6955  * have idle threads, we want them to be the higher ones.
6956  *
6957  * This packing function is run on idle threads.  It checks to see if
6958  * the busiest CPU in this domain (core in the P7 case) has a higher
6959  * CPU number than the packing function is being run on.  Here we are
6960  * assuming lower CPU number will be equivalent to lower a SMT thread
6961  * number.
6962  *
6963  * Return: 1 when packing is required and a task should be moved to
6964  * this CPU.  The amount of the imbalance is returned in *imbalance.
6965  *
6966  * @env: The load balancing environment.
6967  * @sds: Statistics of the sched_domain which is to be packed
6968  */
6969 static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)
6970 {
6971         int busiest_cpu;
6972 
6973         if (!(env->sd->flags & SD_ASYM_PACKING))
6974                 return 0;
6975 
6976         if (env->idle == CPU_NOT_IDLE)
6977                 return 0;
6978 
6979         if (!sds->busiest)
6980                 return 0;
6981 
6982         busiest_cpu = group_first_cpu(sds->busiest);
6983         if (env->dst_cpu > busiest_cpu)
6984                 return 0;
6985 
6986         env->imbalance = DIV_ROUND_CLOSEST(
6987                 sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity,
6988                 SCHED_CAPACITY_SCALE);
6989 
6990         return 1;
6991 }
6992 
6993 /**
6994  * fix_small_imbalance - Calculate the minor imbalance that exists
6995  *                      amongst the groups of a sched_domain, during
6996  *                      load balancing.
6997  * @env: The load balancing environment.
6998  * @sds: Statistics of the sched_domain whose imbalance is to be calculated.
6999  */
7000 static inline
7001 void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
7002 {
7003         unsigned long tmp, capa_now = 0, capa_move = 0;
7004         unsigned int imbn = 2;
7005         unsigned long scaled_busy_load_per_task;
7006         struct sg_lb_stats *local, *busiest;
7007 
7008         local = &sds->local_stat;
7009         busiest = &sds->busiest_stat;
7010 
7011         if (!local->sum_nr_running)
7012                 local->load_per_task = cpu_avg_load_per_task(env->dst_cpu);
7013         else if (busiest->load_per_task > local->load_per_task)
7014                 imbn = 1;
7015 
7016         scaled_busy_load_per_task =
7017                 (busiest->load_per_task * SCHED_CAPACITY_SCALE) /
7018                 busiest->group_capacity;
7019 
7020         if (busiest->avg_load + scaled_busy_load_per_task >=
7021             local->avg_load + (scaled_busy_load_per_task * imbn)) {
7022                 env->imbalance = busiest->load_per_task;
7023                 return;
7024         }
7025 
7026         /*
7027          * OK, we don't have enough imbalance to justify moving tasks,
7028          * however we may be able to increase total CPU capacity used by
7029          * moving them.
7030          */
7031 
7032         capa_now += busiest->group_capacity *
7033                         min(busiest->load_per_task, busiest->avg_load);
7034         capa_now += local->group_capacity *
7035                         min(local->load_per_task, local->avg_load);
7036         capa_now /= SCHED_CAPACITY_SCALE;
7037 
7038         /* Amount of load we'd subtract */
7039         if (busiest->avg_load > scaled_busy_load_per_task) {
7040                 capa_move += busiest->group_capacity *
7041                             min(busiest->load_per_task,
7042                                 busiest->avg_load - scaled_busy_load_per_task);
7043         }
7044 
7045         /* Amount of load we'd add */
7046         if (busiest->avg_load * busiest->group_capacity <
7047             busiest->load_per_task * SCHED_CAPACITY_SCALE) {
7048                 tmp = (busiest->avg_load * busiest->group_capacity) /
7049                       local->group_capacity;
7050         } else {
7051                 tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) /
7052                       local->group_capacity;
7053         }
7054         capa_move += local->group_capacity *
7055                     min(local->load_per_task, local->avg_load + tmp);
7056         capa_move /= SCHED_CAPACITY_SCALE;
7057 
7058         /* Move if we gain throughput */
7059         if (capa_move > capa_now)
7060                 env->imbalance = busiest->load_per_task;
7061 }
7062 
7063 /**
7064  * calculate_imbalance - Calculate the amount of imbalance present within the
7065  *                       groups of a given sched_domain during load balance.
7066  * @env: load balance environment
7067  * @sds: statistics of the sched_domain whose imbalance is to be calculated.
7068  */
7069 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
7070 {
7071         unsigned long max_pull, load_above_capacity = ~0UL;
7072         struct sg_lb_stats *local, *busiest;
7073 
7074         local = &sds->local_stat;
7075         busiest = &sds->busiest_stat;
7076 
7077         if (busiest->group_type == group_imbalanced) {
7078                 /*
7079                  * In the group_imb case we cannot rely on group-wide averages
7080                  * to ensure cpu-load equilibrium, look at wider averages. XXX
7081                  */
7082                 busiest->load_per_task =
7083                         min(busiest->load_per_task, sds->avg_load);
7084         }
7085 
7086         /*
7087          * Avg load of busiest sg can be less and avg load of local sg can
7088          * be greater than avg load across all sgs of sd because avg load
7089          * factors in sg capacity and sgs with smaller group_type are
7090          * skipped when updating the busiest sg:
7091          */
7092         if (busiest->avg_load <= sds->avg_load ||
7093             local->avg_load >= sds->avg_load) {
7094                 env->imbalance = 0;
7095                 return fix_small_imbalance(env, sds);
7096         }
7097 
7098         /*
7099          * If there aren't any idle cpus, avoid creating some.
7100          */
7101         if (busiest->group_type == group_overloaded &&
7102             local->group_type   == group_overloaded) {
7103                 load_above_capacity = busiest->sum_nr_running * SCHED_CAPACITY_SCALE;
7104                 if (load_above_capacity > busiest->group_capacity) {
7105                         load_above_capacity -= busiest->group_capacity;
7106                         load_above_capacity *= NICE_0_LOAD;
7107                         load_above_capacity /= busiest->group_capacity;
7108                 } else
7109                         load_above_capacity = ~0UL;
7110         }
7111 
7112         /*
7113          * We're trying to get all the cpus to the average_load, so we don't
7114          * want to push ourselves above the average load, nor do we wish to
7115          * reduce the max loaded cpu below the average load. At the same time,
7116          * we also don't want to reduce the group load below the group
7117          * capacity. Thus we look for the minimum possible imbalance.
7118          */
7119         max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity);
7120 
7121         /* How much load to actually move to equalise the imbalance */
7122         env->imbalance = min(
7123                 max_pull * busiest->group_capacity,
7124                 (sds->avg_load - local->avg_load) * local->group_capacity
7125         ) / SCHED_CAPACITY_SCALE;
7126 
7127         /*
7128          * if *imbalance is less than the average load per runnable task
7129          * there is no guarantee that any tasks will be moved so we'll have
7130          * a think about bumping its value to force at least one task to be
7131          * moved
7132          */
7133         if (env->imbalance < busiest->load_per_task)
7134                 return fix_small_imbalance(env, sds);
7135 }
7136 
7137 /******* find_busiest_group() helpers end here *********************/
7138 
7139 /**
7140  * find_busiest_group - Returns the busiest group within the sched_domain
7141  * if there is an imbalance.
7142  *
7143  * Also calculates the amount of weighted load which should be moved
7144  * to restore balance.
7145  *
7146  * @env: The load balancing environment.
7147  *
7148  * Return:      - The busiest group if imbalance exists.
7149  */
7150 static struct sched_group *find_busiest_group(struct lb_env *env)
7151 {
7152         struct sg_lb_stats *local, *busiest;
7153         struct sd_lb_stats sds;
7154 
7155         init_sd_lb_stats(&sds);
7156 
7157         /*
7158          * Compute the various statistics relavent for load balancing at
7159          * this level.
7160          */
7161         update_sd_lb_stats(env, &sds);
7162         local = &sds.local_stat;
7163         busiest = &sds.busiest_stat;
7164 
7165         /* ASYM feature bypasses nice load balance check */
7166         if (check_asym_packing(env, &sds))
7167                 return sds.busiest;
7168 
7169         /* There is no busy sibling group to pull tasks from */
7170         if (!sds.busiest || busiest->sum_nr_running == 0)
7171                 goto out_balanced;
7172 
7173         sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load)
7174                                                 / sds.total_capacity;
7175 
7176         /*
7177          * If the busiest group is imbalanced the below checks don't
7178          * work because they assume all things are equal, which typically
7179          * isn't true due to cpus_allowed constraints and the like.
7180          */
7181         if (busiest->group_type == group_imbalanced)
7182                 goto force_balance;
7183 
7184         /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */
7185         if (env->idle == CPU_NEWLY_IDLE && group_has_capacity(env, local) &&
7186             busiest->group_no_capacity)
7187                 goto force_balance;
7188 
7189         /*
7190          * If the local group is busier than the selected busiest group
7191          * don't try and pull any tasks.
7192          */
7193         if (local->avg_load >= busiest->avg_load)
7194                 goto out_balanced;
7195 
7196         /*
7197          * Don't pull any tasks if this group is already above the domain
7198          * average load.
7199          */
7200         if (local->avg_load >= sds.avg_load)
7201                 goto out_balanced;
7202 
7203         if (env->idle == CPU_IDLE) {
7204                 /*
7205                  * This cpu is idle. If the busiest group is not overloaded
7206                  * and there is no imbalance between this and busiest group
7207                  * wrt idle cpus, it is balanced. The imbalance becomes
7208                  * significant if the diff is greater than 1 otherwise we
7209                  * might end up to just move the imbalance on another group
7210                  */
7211                 if ((busiest->group_type != group_overloaded) &&
7212                                 (local->idle_cpus <= (busiest->idle_cpus + 1)))
7213                         goto out_balanced;
7214         } else {
7215                 /*
7216                  * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use
7217                  * imbalance_pct to be conservative.
7218                  */
7219                 if (100 * busiest->avg_load <=
7220                                 env->sd->imbalance_pct * local->avg_load)
7221                         goto out_balanced;
7222         }
7223 
7224 force_balance:
7225         /* Looks like there is an imbalance. Compute it */
7226         calculate_imbalance(env, &sds);
7227         return sds.busiest;
7228 
7229 out_balanced:
7230         env->imbalance = 0;
7231         return NULL;
7232 }
7233 
7234 /*
7235  * find_busiest_queue - find the busiest runqueue among the cpus in group.
7236  */
7237 static struct rq *find_busiest_queue(struct lb_env *env,
7238                                      struct sched_group *group)
7239 {
7240         struct rq *busiest = NULL, *rq;
7241         unsigned long busiest_load = 0, busiest_capacity = 1;
7242         int i;
7243 
7244         for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {
7245                 unsigned long capacity, wl;
7246                 enum fbq_type rt;
7247 
7248                 rq = cpu_rq(i);
7249                 rt = fbq_classify_rq(rq);
7250 
7251                 /*
7252                  * We classify groups/runqueues into three groups:
7253                  *  - regular: there are !numa tasks
7254                  *  - remote:  there are numa tasks that run on the 'wrong' node
7255                  *  - all:     there is no distinction
7256                  *
7257                  * In order to avoid migrating ideally placed numa tasks,
7258                  * ignore those when there's better options.
7259                  *
7260                  * If we ignore the actual busiest queue to migrate another
7261                  * task, the next balance pass can still reduce the busiest
7262                  * queue by moving tasks around inside the node.
7263                  *
7264                  * If we cannot move enough load due to this classification
7265                  * the next pass will adjust the group classification and
7266                  * allow migration of more tasks.
7267                  *
7268                  * Both cases only affect the total convergence complexity.
7269                  */
7270                 if (rt > env->fbq_type)
7271                         continue;
7272 
7273                 capacity = capacity_of(i);
7274 
7275                 wl = weighted_cpuload(i);
7276 
7277                 /*
7278                  * When comparing with imbalance, use weighted_cpuload()
7279                  * which is not scaled with the cpu capacity.
7280                  */
7281 
7282                 if (rq->nr_running == 1 && wl > env->imbalance &&
7283                     !check_cpu_capacity(rq, env->sd))
7284                         continue;
7285 
7286                 /*
7287                  * For the load comparisons with the other cpu's, consider
7288                  * the weighted_cpuload() scaled with the cpu capacity, so
7289                  * that the load can be moved away from the cpu that is
7290                  * potentially running at a lower capacity.
7291                  *
7292                  * Thus we're looking for max(wl_i / capacity_i), crosswise
7293                  * multiplication to rid ourselves of the division works out
7294                  * to: wl_i * capacity_j > wl_j * capacity_i;  where j is
7295                  * our previous maximum.
7296                  */
7297                 if (wl * busiest_capacity > busiest_load * capacity) {
7298                         busiest_load = wl;
7299                         busiest_capacity = capacity;
7300                         busiest = rq;
7301                 }
7302         }
7303 
7304         return busiest;
7305 }
7306 
7307 /*
7308  * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
7309  * so long as it is large enough.
7310  */
7311 #define MAX_PINNED_INTERVAL     512
7312 
7313 /* Working cpumask for load_balance and load_balance_newidle. */
7314 DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
7315 
7316 static int need_active_balance(struct lb_env *env)
7317 {
7318         struct sched_domain *sd = env->sd;
7319 
7320         if (env->idle == CPU_NEWLY_IDLE) {
7321 
7322                 /*
7323                  * ASYM_PACKING needs to force migrate tasks from busy but
7324                  * higher numbered CPUs in order to pack all tasks in the
7325                  * lowest numbered CPUs.
7326                  */
7327                 if ((sd->flags & SD_ASYM_PACKING) && env->src_cpu > env->dst_cpu)
7328                         return 1;
7329         }
7330 
7331         /*
7332          * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task.
7333          * It's worth migrating the task if the src_cpu's capacity is reduced
7334          * because of other sched_class or IRQs if more capacity stays
7335          * available on dst_cpu.
7336          */
7337         if ((env->idle != CPU_NOT_IDLE) &&
7338             (env->src_rq->cfs.h_nr_running == 1)) {
7339                 if ((check_cpu_capacity(env->src_rq, sd)) &&
7340                     (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))
7341                         return 1;
7342         }
7343 
7344         return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2);
7345 }
7346 
7347 static int active_load_balance_cpu_stop(void *data);
7348 
7349 static int should_we_balance(struct lb_env *env)
7350 {
7351         struct sched_group *sg = env->sd->groups;
7352         struct cpumask *sg_cpus, *sg_mask;
7353         int cpu, balance_cpu = -1;
7354 
7355         /*
7356          * In the newly idle case, we will allow all the cpu's
7357          * to do the newly idle load balance.
7358          */
7359         if (env->idle == CPU_NEWLY_IDLE)
7360                 return 1;
7361 
7362         sg_cpus = sched_group_cpus(sg);
7363         sg_mask = sched_group_mask(sg);
7364         /* Try to find first idle cpu */
7365         for_each_cpu_and(cpu, sg_cpus, env->cpus) {
7366                 if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu))
7367                         continue;
7368 
7369                 balance_cpu = cpu;
7370                 break;
7371         }
7372 
7373         if (balance_cpu == -1)
7374                 balance_cpu = group_balance_cpu(sg);
7375 
7376         /*
7377          * First idle cpu or the first cpu(busiest) in this sched group
7378          * is eligible for doing load balancing at this and above domains.
7379          */
7380         return balance_cpu == env->dst_cpu;
7381 }
7382 
7383 /*
7384  * Check this_cpu to ensure it is balanced within domain. Attempt to move
7385  * tasks if there is an imbalance.
7386  */
7387 static int load_balance(int this_cpu, struct rq *this_rq,
7388                         struct sched_domain *sd, enum cpu_idle_type idle,
7389                         int *continue_balancing)
7390 {
7391         int ld_moved, cur_ld_moved, active_balance = 0;
7392         struct sched_domain *sd_parent = sd->parent;
7393         struct sched_group *group;
7394         struct rq *busiest;
7395         unsigned long flags;
7396         struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);
7397 
7398         struct lb_env env = {
7399                 .sd             = sd,
7400                 .dst_cpu        = this_cpu,
7401                 .dst_rq         = this_rq,
7402                 .dst_grpmask    = sched_group_cpus(sd->groups),
7403                 .idle           = idle,
7404                 .loop_break     = sched_nr_migrate_break,
7405                 .cpus           = cpus,
7406                 .fbq_type       = all,
7407                 .tasks          = LIST_HEAD_INIT(env.tasks),
7408         };
7409 
7410         /*
7411          * For NEWLY_IDLE load_balancing, we don't need to consider
7412          * other cpus in our group
7413          */
7414         if (idle == CPU_NEWLY_IDLE)
7415                 env.dst_grpmask = NULL;
7416 
7417         cpumask_copy(cpus, cpu_active_mask);
7418 
7419         schedstat_inc(sd, lb_count[idle]);
7420 
7421 redo:
7422         if (!should_we_balance(&env)) {
7423                 *continue_balancing = 0;
7424                 goto out_balanced;
7425         }
7426 
7427         group = find_busiest_group(&env);
7428         if (!group) {
7429                 schedstat_inc(sd, lb_nobusyg[idle]);
7430                 goto out_balanced;
7431         }
7432 
7433         busiest = find_busiest_queue(&env, group);
7434         if (!busiest) {
7435                 schedstat_inc(sd, lb_nobusyq[idle]);
7436                 goto out_balanced;
7437         }
7438 
7439         BUG_ON(busiest == env.dst_rq);
7440 
7441         schedstat_add(sd, lb_imbalance[idle], env.imbalance);
7442 
7443         env.src_cpu = busiest->cpu;
7444         env.src_rq = busiest;
7445 
7446         ld_moved = 0;
7447         if (busiest->nr_running > 1) {
7448                 /*
7449                  * Attempt to move tasks. If find_busiest_group has found
7450                  * an imbalance but busiest->nr_running <= 1, the group is
7451                  * still unbalanced. ld_moved simply stays zero, so it is
7452                  * correctly treated as an imbalance.
7453                  */
7454                 env.flags |= LBF_ALL_PINNED;
7455                 env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);
7456 
7457 more_balance:
7458                 raw_spin_lock_irqsave(&busiest->lock, flags);
7459 
7460                 /*
7461                  * cur_ld_moved - load moved in current iteration
7462                  * ld_moved     - cumulative load moved across iterations
7463                  */
7464                 cur_ld_moved = detach_tasks(&env);
7465 
7466                 /*
7467                  * We've detached some tasks from busiest_rq. Every
7468                  * task is masked "TASK_ON_RQ_MIGRATING", so we can safely
7469                  * unlock busiest->lock, and we are able to be sure
7470                  * that nobody can manipulate the tasks in parallel.
7471                  * See task_rq_lock() family for the details.
7472                  */
7473 
7474                 raw_spin_unlock(&busiest->lock);
7475 
7476                 if (cur_ld_moved) {
7477                         attach_tasks(&env);
7478                         ld_moved += cur_ld_moved;
7479                 }
7480 
7481                 local_irq_restore(flags);
7482 
7483                 if (env.flags & LBF_NEED_BREAK) {
7484                         env.flags &= ~LBF_NEED_BREAK;
7485                         goto more_balance;
7486                 }
7487 
7488                 /*
7489                  * Revisit (affine) tasks on src_cpu that couldn't be moved to
7490                  * us and move them to an alternate dst_cpu in our sched_group
7491                  * where they can run. The upper limit on how many times we
7492                  * iterate on same src_cpu is dependent on number of cpus in our
7493                  * sched_group.
7494                  *
7495                  * This changes load balance semantics a bit on who can move
7496                  * load to a given_cpu. In addition to the given_cpu itself
7497                  * (or a ilb_cpu acting on its behalf where given_cpu is
7498                  * nohz-idle), we now have balance_cpu in a position to move
7499                  * load to given_cpu. In rare situations, this may cause
7500                  * conflicts (balance_cpu and given_cpu/ilb_cpu deciding
7501                  * _independently_ and at _same_ time to move some load to
7502                  * given_cpu) causing exceess load to be moved to given_cpu.
7503                  * This however should not happen so much in practice and
7504                  * moreover subsequent load balance cycles should correct the
7505                  * excess load moved.
7506                  */
7507                 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
7508 
7509                         /* Prevent to re-select dst_cpu via env's cpus */
7510                         cpumask_clear_cpu(env.dst_cpu, env.cpus);
7511 
7512                         env.dst_rq       = cpu_rq(env.new_dst_cpu);
7513                         env.dst_cpu      = env.new_dst_cpu;
7514                         env.flags       &= ~LBF_DST_PINNED;
7515                         env.loop         = 0;
7516                         env.loop_break   = sched_nr_migrate_break;
7517 
7518                         /*
7519                          * Go back to "more_balance" rather than "redo" since we
7520                          * need to continue with same src_cpu.
7521                          */
7522                         goto more_balance;
7523                 }
7524 
7525                 /*
7526                  * We failed to reach balance because of affinity.
7527                  */
7528                 if (sd_parent) {
7529                         int *group_imbalance = &sd_parent->groups->sgc->imbalance;
7530 
7531                         if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)
7532                                 *group_imbalance = 1;
7533                 }
7534 
7535                 /* All tasks on this runqueue were pinned by CPU affinity */
7536                 if (unlikely(env.flags & LBF_ALL_PINNED)) {
7537                         cpumask_clear_cpu(cpu_of(busiest), cpus);
7538                         if (!cpumask_empty(cpus)) {
7539                                 env.loop = 0;
7540                                 env.loop_break = sched_nr_migrate_break;
7541                                 goto redo;
7542                         }
7543                         goto out_all_pinned;
7544                 }
7545         }
7546 
7547         if (!ld_moved) {
7548                 schedstat_inc(sd, lb_failed[idle]);
7549                 /*
7550                  * Increment the failure counter only on periodic balance.
7551                  * We do not want newidle balance, which can be very
7552                  * frequent, pollute the failure counter causing
7553                  * excessive cache_hot migrations and active balances.
7554                  */
7555                 if (idle != CPU_NEWLY_IDLE)
7556                         sd->nr_balance_failed++;
7557 
7558                 if (need_active_balance(&env)) {
7559                         raw_spin_lock_irqsave(&busiest->lock, flags);
7560 
7561                         /* don't kick the active_load_balance_cpu_stop,
7562                          * if the curr task on busiest cpu can't be
7563                          * moved to this_cpu
7564                          */
7565                         if (!cpumask_test_cpu(this_cpu,
7566                                         tsk_cpus_allowed(busiest->curr))) {
7567                                 raw_spin_unlock_irqrestore(&busiest->lock,
7568                                                             flags);
7569                                 env.flags |= LBF_ALL_PINNED;
7570                                 goto out_one_pinned;
7571                         }
7572 
7573                         /*
7574                          * ->active_balance synchronizes accesses to
7575                          * ->active_balance_work.  Once set, it's cleared
7576                          * only after active load balance is finished.
7577                          */
7578                         if (!busiest->active_balance) {
7579                                 busiest->active_balance = 1;
7580                                 busiest->push_cpu = this_cpu;
7581                                 active_balance = 1;
7582                         }
7583                         raw_spin_unlock_irqrestore(&busiest->lock, flags);
7584 
7585                         if (active_balance) {
7586                                 stop_one_cpu_nowait(cpu_of(busiest),
7587                                         active_load_balance_cpu_stop, busiest,
7588                                         &busiest->active_balance_work);
7589                         }
7590 
7591                         /* We've kicked active balancing, force task migration. */
7592                         sd->nr_balance_failed = sd->cache_nice_tries+1;
7593                 }
7594         } else
7595                 sd->nr_balance_failed = 0;
7596 
7597         if (likely(!active_balance)) {
7598                 /* We were unbalanced, so reset the balancing interval */
7599                 sd->balance_interval = sd->min_interval;
7600         } else {
7601                 /*
7602                  * If we've begun active balancing, start to back off. This
7603                  * case may not be covered by the all_pinned logic if there
7604                  * is only 1 task on the busy runqueue (because we don't call
7605                  * detach_tasks).
7606                  */
7607                 if (sd->balance_interval < sd->max_interval)
7608                         sd->balance_interval *= 2;
7609         }
7610 
7611         goto out;
7612 
7613 out_balanced:
7614         /*
7615          * We reach balance although we may have faced some affinity
7616          * constraints. Clear the imbalance flag if it was set.
7617          */
7618         if (sd_parent) {
7619                 int *group_imbalance = &sd_parent->groups->sgc->imbalance;
7620 
7621                 if (*group_imbalance)
7622                         *group_imbalance = 0;
7623         }
7624 
7625 out_all_pinned:
7626         /*
7627          * We reach balance because all tasks are pinned at this level so
7628          * we can't migrate them. Let the imbalance flag set so parent level
7629          * can try to migrate them.
7630          */
7631         schedstat_inc(sd, lb_balanced[idle]);
7632 
7633         sd->nr_balance_failed = 0;
7634 
7635 out_one_pinned:
7636         /* tune up the balancing interval */
7637         if (((env.flags & LBF_ALL_PINNED) &&
7638                         sd->balance_interval < MAX_PINNED_INTERVAL) ||
7639                         (sd->balance_interval < sd->max_interval))
7640                 sd->balance_interval *= 2;
7641 
7642         ld_moved = 0;
7643 out:
7644         return ld_moved;
7645 }
7646 
7647 static inline unsigned long
7648 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)
7649 {
7650         unsigned long interval = sd->balance_interval;
7651 
7652         if (cpu_busy)
7653                 interval *= sd->busy_factor;
7654 
7655         /* scale ms to jiffies */
7656         interval = msecs_to_jiffies(interval);
7657         interval = clamp(interval, 1UL, max_load_balance_interval);
7658 
7659         return interval;
7660 }
7661 
7662 static inline void
7663 update_next_balance(struct sched_domain *sd, int cpu_busy, unsigned long *next_balance)
7664 {
7665         unsigned long interval, next;
7666 
7667         interval = get_sd_balance_interval(sd, cpu_busy);
7668         next = sd->last_balance + interval;
7669 
7670         if (time_after(*next_balance, next))
7671                 *next_balance = next;
7672 }
7673 
7674 /*
7675  * idle_balance is called by schedule() if this_cpu is about to become
7676  * idle. Attempts to pull tasks from other CPUs.
7677  */
7678 static int idle_balance(struct rq *this_rq)
7679 {
7680         unsigned long next_balance = jiffies + HZ;
7681         int this_cpu = this_rq->cpu;
7682         struct sched_domain *sd;
7683         int pulled_task = 0;
7684         u64 curr_cost = 0;
7685 
7686         /*
7687          * We must set idle_stamp _before_ calling idle_balance(), such that we
7688          * measure the duration of idle_balance() as idle time.
7689          */
7690         this_rq->idle_stamp = rq_clock(this_rq);
7691 
7692         if (this_rq->avg_idle < sysctl_sched_migration_cost ||
7693             !this_rq->rd->overload) {
7694                 rcu_read_lock();
7695                 sd = rcu_dereference_check_sched_domain(this_rq->sd);
7696                 if (sd)
7697                         update_next_balance(sd, 0, &next_balance);
7698                 rcu_read_unlock();
7699 
7700                 goto out;
7701         }
7702 
7703         raw_spin_unlock(&this_rq->lock);
7704 
7705         update_blocked_averages(this_cpu);
7706         rcu_read_lock();
7707         for_each_domain(this_cpu, sd) {
7708                 int continue_balancing = 1;
7709                 u64 t0, domain_cost;
7710 
7711                 if (!(sd->flags & SD_LOAD_BALANCE))
7712                         continue;
7713 
7714                 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) {
7715                         update_next_balance(sd, 0, &next_balance);
7716                         break;
7717                 }
7718 
7719                 if (sd->flags & SD_BALANCE_NEWIDLE) {
7720                         t0 = sched_clock_cpu(this_cpu);
7721 
7722                         pulled_task = load_balance(this_cpu, this_rq,
7723                                                    sd, CPU_NEWLY_IDLE,
7724                                                    &continue_balancing);
7725 
7726                         domain_cost = sched_clock_cpu(this_cpu) - t0;
7727                         if (domain_cost > sd->max_newidle_lb_cost)
7728                                 sd->max_newidle_lb_cost = domain_cost;
7729 
7730                         curr_cost += domain_cost;
7731                 }
7732 
7733                 update_next_balance(sd, 0, &next_balance);
7734 
7735                 /*
7736                  * Stop searching for tasks to pull if there are
7737                  * now runnable tasks on this rq.
7738                  */
7739                 if (pulled_task || this_rq->nr_running > 0)
7740                         break;
7741         }
7742         rcu_read_unlock();
7743 
7744         raw_spin_lock(&this_rq->lock);
7745 
7746         if (curr_cost > this_rq->max_idle_balance_cost)
7747                 this_rq->max_idle_balance_cost = curr_cost;
7748 
7749         /*
7750          * While browsing the domains, we released the rq lock, a task could
7751          * have been enqueued in the meantime. Since we're not going idle,
7752          * pretend we pulled a task.
7753          */
7754         if (this_rq->cfs.h_nr_running && !pulled_task)
7755                 pulled_task = 1;
7756 
7757 out:
7758         /* Move the next balance forward */
7759         if (time_after(this_rq->next_balance, next_balance))
7760                 this_rq->next_balance = next_balance;
7761 
7762         /* Is there a task of a high priority class? */
7763         if (this_rq->nr_running != this_rq->cfs.h_nr_running)
7764                 pulled_task = -1;
7765 
7766         if (pulled_task)
7767                 this_rq->idle_stamp = 0;
7768 
7769         return pulled_task;
7770 }
7771 
7772 /*
7773  * active_load_balance_cpu_stop is run by cpu stopper. It pushes
7774  * running tasks off the busiest CPU onto idle CPUs. It requires at
7775  * least 1 task to be running on each physical CPU where possible, and
7776  * avoids physical / logical imbalances.
7777  */
7778 static int active_load_balance_cpu_stop(void *data)
7779 {
7780         struct rq *busiest_rq = data;
7781         int busiest_cpu = cpu_of(busiest_rq);
7782         int target_cpu = busiest_rq->push_cpu;
7783         struct rq *target_rq = cpu_rq(target_cpu);
7784         struct sched_domain *sd;
7785         struct task_struct *p = NULL;
7786 
7787         raw_spin_lock_irq(&busiest_rq->lock);
7788 
7789         /* make sure the requested cpu hasn't gone down in the meantime */
7790         if (unlikely(busiest_cpu != smp_processor_id() ||
7791                      !busiest_rq->active_balance))
7792                 goto out_unlock;
7793 
7794         /* Is there any task to move? */
7795         if (busiest_rq->nr_running <= 1)
7796                 goto out_unlock;
7797 
7798         /*
7799          * This condition is "impossible", if it occurs
7800          * we need to fix it. Originally reported by
7801          * Bjorn Helgaas on a 128-cpu setup.
7802          */
7803         BUG_ON(busiest_rq == target_rq);
7804 
7805         /* Search for an sd spanning us and the target CPU. */
7806         rcu_read_lock();
7807         for_each_domain(target_cpu, sd) {
7808                 if ((sd->flags & SD_LOAD_BALANCE) &&
7809                     cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
7810                                 break;
7811         }
7812 
7813         if (likely(sd)) {
7814                 struct lb_env env = {
7815                         .sd             = sd,
7816                         .dst_cpu        = target_cpu,
7817                         .dst_rq         = target_rq,
7818                         .src_cpu        = busiest_rq->cpu,
7819                         .src_rq         = busiest_rq,
7820                         .idle           = CPU_IDLE,
7821                 };
7822 
7823                 schedstat_inc(sd, alb_count);
7824 
7825                 p = detach_one_task(&env);
7826                 if (p) {
7827                         schedstat_inc(sd, alb_pushed);
7828                         /* Active balancing done, reset the failure counter. */
7829                         sd->nr_balance_failed = 0;
7830                 } else {
7831                         schedstat_inc(sd, alb_failed);
7832                 }
7833         }
7834         rcu_read_unlock();
7835 out_unlock:
7836         busiest_rq->active_balance = 0;
7837         raw_spin_unlock(&busiest_rq->lock);
7838 
7839         if (p)
7840                 attach_one_task(target_rq, p);
7841 
7842         local_irq_enable();
7843 
7844         return 0;
7845 }
7846 
7847 static inline int on_null_domain(struct rq *rq)
7848 {
7849         return unlikely(!rcu_dereference_sched(rq->sd));
7850 }
7851 
7852 #ifdef CONFIG_NO_HZ_COMMON
7853 /*
7854  * idle load balancing details
7855  * - When one of the busy CPUs notice that there may be an idle rebalancing
7856  *   needed, they will kick the idle load balancer, which then does idle
7857  *   load balancing for all the idle CPUs.
7858  */
7859 static struct {
7860         cpumask_var_t idle_cpus_mask;
7861         atomic_t nr_cpus;
7862         unsigned long next_balance;     /* in jiffy units */
7863 } nohz ____cacheline_aligned;
7864 
7865 static inline int find_new_ilb(void)
7866 {
7867         int ilb = cpumask_first(nohz.idle_cpus_mask);
7868 
7869         if (ilb < nr_cpu_ids && idle_cpu(ilb))
7870                 return ilb;
7871 
7872         return nr_cpu_ids;
7873 }
7874 
7875 /*
7876  * Kick a CPU to do the nohz balancing, if it is time for it. We pick the
7877  * nohz_load_balancer CPU (if there is one) otherwise fallback to any idle
7878  * CPU (if there is one).
7879  */
7880 static void nohz_balancer_kick(void)
7881 {
7882         int ilb_cpu;
7883 
7884         nohz.next_balance++;
7885 
7886         ilb_cpu = find_new_ilb();
7887 
7888         if (ilb_cpu >= nr_cpu_ids)
7889                 return;
7890 
7891         if (test_and_set_bit(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu)))
7892                 return;
7893         /*
7894          * Use smp_send_reschedule() instead of resched_cpu().
7895          * This way we generate a sched IPI on the target cpu which
7896          * is idle. And the softirq performing nohz idle load balance
7897          * will be run before returning from the IPI.
7898          */
7899         smp_send_reschedule(ilb_cpu);
7900         return;
7901 }
7902 
7903 void nohz_balance_exit_idle(unsigned int cpu)
7904 {
7905         if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) {
7906                 /*
7907                  * Completely isolated CPUs don't ever set, so we must test.
7908                  */
7909                 if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) {
7910                         cpumask_clear_cpu(cpu, nohz.idle_cpus_mask);
7911                         atomic_dec(&nohz.nr_cpus);
7912                 }
7913                 clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
7914         }
7915 }
7916 
7917 static inline void set_cpu_sd_state_busy(void)
7918 {
7919         struct sched_domain *sd;
7920         int cpu = smp_processor_id();
7921 
7922         rcu_read_lock();
7923         sd = rcu_dereference(per_cpu(sd_busy, cpu));
7924 
7925         if (!sd || !sd->nohz_idle)
7926                 goto unlock;
7927         sd->nohz_idle = 0;
7928 
7929         atomic_inc(&sd->groups->sgc->nr_busy_cpus);
7930 unlock:
7931         rcu_read_unlock();
7932 }
7933 
7934 void set_cpu_sd_state_idle(void)
7935 {
7936         struct sched_domain *sd;
7937         int cpu = smp_processor_id();
7938 
7939         rcu_read_lock();
7940         sd = rcu_dereference(per_cpu(sd_busy, cpu));
7941 
7942         if (!sd || sd->nohz_idle)
7943                 goto unlock;
7944         sd->nohz_idle = 1;
7945 
7946         atomic_dec(&sd->groups->sgc->nr_busy_cpus);
7947 unlock:
7948         rcu_read_unlock();
7949 }
7950 
7951 /*
7952  * This routine will record that the cpu is going idle with tick stopped.
7953  * This info will be used in performing idle load balancing in the future.
7954  */
7955 void nohz_balance_enter_idle(int cpu)
7956 {
7957         /*
7958          * If this cpu is going down, then nothing needs to be done.
7959          */
7960         if (!cpu_active(cpu))
7961                 return;
7962 
7963         if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))
7964                 return;
7965 
7966         /*
7967          * If we're a completely isolated CPU, we don't play.
7968          */
7969         if (on_null_domain(cpu_rq(cpu)))
7970                 return;
7971 
7972         cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
7973         atomic_inc(&nohz.nr_cpus);
7974         set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu));
7975 }
7976 #endif
7977 
7978 static DEFINE_SPINLOCK(balancing);
7979 
7980 /*
7981  * Scale the max load_balance interval with the number of CPUs in the system.
7982  * This trades load-balance latency on larger machines for less cross talk.
7983  */
7984 void update_max_interval(void)
7985 {
7986         max_load_balance_interval = HZ*num_online_cpus()/10;
7987 }
7988 
7989 /*
7990  * It checks each scheduling domain to see if it is due to be balanced,
7991  * and initiates a balancing operation if so.
7992  *
7993  * Balancing parameters are set up in init_sched_domains.
7994  */
7995 static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
7996 {
7997         int continue_balancing = 1;
7998         int cpu = rq->cpu;
7999         unsigned long interval;
8000         struct sched_domain *sd;
8001         /* Earliest time when we have to do rebalance again */
8002         unsigned long next_balance = jiffies + 60*HZ;
8003         int update_next_balance = 0;
8004         int need_serialize, need_decay = 0;
8005         u64 max_cost = 0;
8006 
8007         update_blocked_averages(cpu);
8008 
8009         rcu_read_lock();
8010         for_each_domain(cpu, sd) {
8011                 /*
8012                  * Decay the newidle max times here because this is a regular
8013                  * visit to all the domains. Decay ~1% per second.
8014                  */
8015                 if (time_after(jiffies, sd->next_decay_max_lb_cost)) {
8016                         sd->max_newidle_lb_cost =
8017                                 (sd->max_newidle_lb_cost * 253) / 256;
8018                         sd->next_decay_max_lb_cost = jiffies + HZ;
8019                         need_decay = 1;
8020                 }
8021                 max_cost += sd->max_newidle_lb_cost;
8022 
8023                 if (!(sd->flags & SD_LOAD_BALANCE))
8024                         continue;
8025 
8026                 /*
8027                  * Stop the load balance at this level. There is another
8028                  * CPU in our sched group which is doing load balancing more
8029                  * actively.
8030                  */
8031                 if (!continue_balancing) {
8032                         if (need_decay)
8033                                 continue;
8034                         break;
8035                 }
8036 
8037                 interval = get_sd_balance_interval(sd, idle != CPU_IDLE);
8038 
8039                 need_serialize = sd->flags & SD_SERIALIZE;
8040                 if (need_serialize) {
8041                         if (!spin_trylock(&balancing))
8042                                 goto out;
8043                 }
8044 
8045                 if (time_after_eq(jiffies, sd->last_balance + interval)) {
8046                         if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
8047                                 /*
8048                                  * The LBF_DST_PINNED logic could have changed
8049                                  * env->dst_cpu, so we can't know our idle
8050                                  * state even if we migrated tasks. Update it.
8051                                  */
8052                                 idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE;
8053                         }
8054                         sd->last_balance = jiffies;
8055                         interval = get_sd_balance_interval(sd, idle != CPU_IDLE);
8056                 }
8057                 if (need_serialize)
8058                         spin_unlock(&balancing);
8059 out:
8060                 if (time_after(next_balance, sd->last_balance + interval)) {
8061                         next_balance = sd->last_balance + interval;
8062                         update_next_balance = 1;
8063                 }
8064         }
8065         if (need_decay) {
8066                 /*
8067                  * Ensure the rq-wide value also decays but keep it at a
8068                  * reasonable floor to avoid funnies with rq->avg_idle.
8069                  */
8070                 rq->max_idle_balance_cost =
8071                         max((u64)sysctl_sched_migration_cost, max_cost);
8072         }
8073         rcu_read_unlock();
8074 
8075         /*
8076          * next_balance will be updated only when there is a need.
8077          * When the cpu is attached to null domain for ex, it will not be
8078          * updated.
8079          */
8080         if (likely(update_next_balance)) {
8081                 rq->next_balance = next_balance;
8082 
8083 #ifdef CONFIG_NO_HZ_COMMON
8084                 /*
8085                  * If this CPU has been elected to perform the nohz idle
8086                  * balance. Other idle CPUs have already rebalanced with
8087                  * nohz_idle_balance() and nohz.next_balance has been
8088                  * updated accordingly. This CPU is now running the idle load
8089                  * balance for itself and we need to update the
8090                  * nohz.next_balance accordingly.
8091                  */
8092                 if ((idle == CPU_IDLE) && time_after(nohz.next_balance, rq->next_balance))
8093                         nohz.next_balance = rq->next_balance;
8094 #endif
8095         }
8096 }
8097 
8098 #ifdef CONFIG_NO_HZ_COMMON
8099 /*
8100  * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
8101  * rebalancing for all the cpus for whom scheduler ticks are stopped.
8102  */
8103 static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
8104 {
8105         int this_cpu = this_rq->cpu;
8106         struct rq *rq;
8107         int balance_cpu;
8108         /* Earliest time when we have to do rebalance again */
8109         unsigned long next_balance = jiffies + 60*HZ;
8110         int update_next_balance = 0;
8111 
8112         if (idle != CPU_IDLE ||
8113             !test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)))
8114                 goto end;
8115 
8116         for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
8117                 if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
8118                         continue;
8119 
8120                 /*
8121                  * If this cpu gets work to do, stop the load balancing
8122                  * work being done for other cpus. Next load
8123                  * balancing owner will pick it up.
8124                  */
8125                 if (need_resched())
8126                         break;
8127 
8128                 rq = cpu_rq(balance_cpu);
8129 
8130                 /*
8131                  * If time for next balance is due,
8132                  * do the balance.
8133                  */
8134                 if (time_after_eq(jiffies, rq->next_balance)) {
8135                         raw_spin_lock_irq(&rq->lock);
8136                         update_rq_clock(rq);
8137                         cpu_load_update_idle(rq);
8138                         raw_spin_unlock_irq(&rq->lock);
8139                         rebalance_domains(rq, CPU_IDLE);
8140                 }
8141 
8142                 if (time_after(next_balance, rq->next_balance)) {
8143                         next_balance = rq->next_balance;
8144                         update_next_balance = 1;
8145                 }
8146         }
8147 
8148         /*
8149          * next_balance will be updated only when there is a need.
8150          * When the CPU is attached to null domain for ex, it will not be
8151          * updated.
8152          */
8153         if (likely(update_next_balance))
8154                 nohz.next_balance = next_balance;
8155 end:
8156         clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));
8157 }
8158 
8159 /*
8160  * Current heuristic for kicking the idle load balancer in the presence
8161  * of an idle cpu in the system.
8162  *   - This rq has more than one task.
8163  *   - This rq has at least one CFS task and the capacity of the CPU is
8164  *     significantly reduced because of RT tasks or IRQs.
8165  *   - At parent of LLC scheduler domain level, this cpu's scheduler group has
8166  *     multiple busy cpu.
8167  *   - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler
8168  *     domain span are idle.
8169  */
8170 static inline bool nohz_kick_needed(struct rq *rq)
8171 {
8172         unsigned long now = jiffies;
8173         struct sched_domain *sd;
8174         struct sched_group_capacity *sgc;
8175         int nr_busy, cpu = rq->cpu;
8176         bool kick = false;
8177 
8178         if (unlikely(rq->idle_balance))
8179                 return false;
8180 
8181        /*
8182         * We may be recently in ticked or tickless idle mode. At the first
8183         * busy tick after returning from idle, we will update the busy stats.
8184         */
8185         set_cpu_sd_state_busy();
8186         nohz_balance_exit_idle(cpu);
8187 
8188         /*
8189          * None are in tickless mode and hence no need for NOHZ idle load
8190          * balancing.
8191          */
8192         if (likely(!atomic_read(&nohz.nr_cpus)))
8193                 return false;
8194 
8195         if (time_before(now, nohz.next_balance))
8196                 return false;
8197 
8198         if (rq->nr_running >= 2)
8199                 return true;
8200 
8201         rcu_read_lock();
8202         sd = rcu_dereference(per_cpu(sd_busy, cpu));
8203         if (sd) {
8204                 sgc = sd->groups->sgc;
8205                 nr_busy = atomic_read(&sgc->nr_busy_cpus);
8206 
8207                 if (nr_busy > 1) {
8208                         kick = true;
8209                         goto unlock;
8210                 }
8211 
8212         }
8213 
8214         sd = rcu_dereference(rq->sd);
8215         if (sd) {
8216                 if ((rq->cfs.h_nr_running >= 1) &&
8217                                 check_cpu_capacity(rq, sd)) {
8218                         kick = true;
8219                         goto unlock;
8220                 }
8221         }
8222 
8223         sd = rcu_dereference(per_cpu(sd_asym, cpu));
8224         if (sd && (cpumask_first_and(nohz.idle_cpus_mask,
8225                                   sched_domain_span(sd)) < cpu)) {
8226                 kick = true;
8227                 goto unlock;
8228         }
8229 
8230 unlock:
8231         rcu_read_unlock();
8232         return kick;
8233 }
8234 #else
8235 static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { }
8236 #endif
8237 
8238 /*
8239  * run_rebalance_domains is triggered when needed from the scheduler tick.
8240  * Also triggered for nohz idle balancing (with nohz_balancing_kick set).
8241  */
8242 static void run_rebalance_domains(struct softirq_action *h)
8243 {
8244         struct rq *this_rq = this_rq();
8245         enum cpu_idle_type idle = this_rq->idle_balance ?
8246                                                 CPU_IDLE : CPU_NOT_IDLE;
8247 
8248         /*
8249          * If this cpu has a pending nohz_balance_kick, then do the
8250          * balancing on behalf of the other idle cpus whose ticks are
8251          * stopped. Do nohz_idle_balance *before* rebalance_domains to
8252          * give the idle cpus a chance to load balance. Else we may
8253          * load balance only within the local sched_domain hierarchy
8254          * and abort nohz_idle_balance altogether if we pull some load.
8255          */
8256         nohz_idle_balance(this_rq, idle);
8257         rebalance_domains(this_rq, idle);
8258 }
8259 
8260 /*
8261  * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
8262  */
8263 void trigger_load_balance(struct rq *rq)
8264 {
8265         /* Don't need to rebalance while attached to NULL domain */
8266         if (unlikely(on_null_domain(rq)))
8267                 return;
8268 
8269         if (time_after_eq(jiffies, rq->next_balance))
8270                 raise_softirq(SCHED_SOFTIRQ);
8271 #ifdef CONFIG_NO_HZ_COMMON
8272         if (nohz_kick_needed(rq))
8273                 nohz_balancer_kick();
8274 #endif
8275 }
8276 
8277 static void rq_online_fair(struct rq *rq)
8278 {
8279         update_sysctl();
8280 
8281         update_runtime_enabled(rq);
8282 }
8283 
8284 static void rq_offline_fair(struct rq *rq)
8285 {
8286         update_sysctl();
8287 
8288         /* Ensure any throttled groups are reachable by pick_next_task */
8289         unthrottle_offline_cfs_rqs(rq);
8290 }
8291 
8292 #endif /* CONFIG_SMP */
8293 
8294 /*
8295  * scheduler tick hitting a task of our scheduling class:
8296  */
8297 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
8298 {
8299         struct cfs_rq *cfs_rq;
8300         struct sched_entity *se = &curr->se;
8301 
8302         for_each_sched_entity(se) {
8303                 cfs_rq = cfs_rq_of(se);
8304                 entity_tick(cfs_rq, se, queued);
8305         }
8306 
8307         if (static_branch_unlikely(&sched_numa_balancing))
8308                 task_tick_numa(rq, curr);
8309 }
8310 
8311 /*
8312  * called on fork with the child task as argument from the parent's context
8313  *  - child not yet on the tasklist
8314  *  - preemption disabled
8315  */
8316 static void task_fork_fair(struct task_struct *p)
8317 {
8318