Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/kernel/sched/core.c

  1 /*
  2  *  kernel/sched/core.c
  3  *
  4  *  Kernel scheduler and related syscalls
  5  *
  6  *  Copyright (C) 1991-2002  Linus Torvalds
  7  *
  8  *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
  9  *              make semaphores SMP safe
 10  *  1998-11-19  Implemented schedule_timeout() and related stuff
 11  *              by Andrea Arcangeli
 12  *  2002-01-04  New ultra-scalable O(1) scheduler by Ingo Molnar:
 13  *              hybrid priority-list and round-robin design with
 14  *              an array-switch method of distributing timeslices
 15  *              and per-CPU runqueues.  Cleanups and useful suggestions
 16  *              by Davide Libenzi, preemptible kernel bits by Robert Love.
 17  *  2003-09-03  Interactivity tuning by Con Kolivas.
 18  *  2004-04-02  Scheduler domains code by Nick Piggin
 19  *  2007-04-15  Work begun on replacing all interactivity tuning with a
 20  *              fair scheduling design by Con Kolivas.
 21  *  2007-05-05  Load balancing (smp-nice) and other improvements
 22  *              by Peter Williams
 23  *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 24  *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
 25  *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 26  *              Thomas Gleixner, Mike Kravetz
 27  */
 28 
 29 #include <linux/kasan.h>
 30 #include <linux/mm.h>
 31 #include <linux/module.h>
 32 #include <linux/nmi.h>
 33 #include <linux/init.h>
 34 #include <linux/uaccess.h>
 35 #include <linux/highmem.h>
 36 #include <linux/mmu_context.h>
 37 #include <linux/interrupt.h>
 38 #include <linux/capability.h>
 39 #include <linux/completion.h>
 40 #include <linux/kernel_stat.h>
 41 #include <linux/debug_locks.h>
 42 #include <linux/perf_event.h>
 43 #include <linux/security.h>
 44 #include <linux/notifier.h>
 45 #include <linux/profile.h>
 46 #include <linux/freezer.h>
 47 #include <linux/vmalloc.h>
 48 #include <linux/blkdev.h>
 49 #include <linux/delay.h>
 50 #include <linux/pid_namespace.h>
 51 #include <linux/smp.h>
 52 #include <linux/threads.h>
 53 #include <linux/timer.h>
 54 #include <linux/rcupdate.h>
 55 #include <linux/cpu.h>
 56 #include <linux/cpuset.h>
 57 #include <linux/percpu.h>
 58 #include <linux/proc_fs.h>
 59 #include <linux/seq_file.h>
 60 #include <linux/sysctl.h>
 61 #include <linux/syscalls.h>
 62 #include <linux/times.h>
 63 #include <linux/tsacct_kern.h>
 64 #include <linux/kprobes.h>
 65 #include <linux/delayacct.h>
 66 #include <linux/unistd.h>
 67 #include <linux/pagemap.h>
 68 #include <linux/hrtimer.h>
 69 #include <linux/tick.h>
 70 #include <linux/ctype.h>
 71 #include <linux/ftrace.h>
 72 #include <linux/slab.h>
 73 #include <linux/init_task.h>
 74 #include <linux/context_tracking.h>
 75 #include <linux/compiler.h>
 76 #include <linux/frame.h>
 77 #include <linux/prefetch.h>
 78 #include <linux/mutex.h>
 79 
 80 #include <asm/switch_to.h>
 81 #include <asm/tlb.h>
 82 #include <asm/irq_regs.h>
 83 #ifdef CONFIG_PARAVIRT
 84 #include <asm/paravirt.h>
 85 #endif
 86 
 87 #include "sched.h"
 88 #include "../workqueue_internal.h"
 89 #include "../smpboot.h"
 90 
 91 #define CREATE_TRACE_POINTS
 92 #include <trace/events/sched.h>
 93 
 94 DEFINE_MUTEX(sched_domains_mutex);
 95 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 96 
 97 static void update_rq_clock_task(struct rq *rq, s64 delta);
 98 
 99 void update_rq_clock(struct rq *rq)
100 {
101         s64 delta;
102 
103         lockdep_assert_held(&rq->lock);
104 
105         if (rq->clock_skip_update & RQCF_ACT_SKIP)
106                 return;
107 
108         delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
109         if (delta < 0)
110                 return;
111         rq->clock += delta;
112         update_rq_clock_task(rq, delta);
113 }
114 
115 /*
116  * Debugging: various feature bits
117  */
118 
119 #define SCHED_FEAT(name, enabled)       \
120         (1UL << __SCHED_FEAT_##name) * enabled |
121 
122 const_debug unsigned int sysctl_sched_features =
123 #include "features.h"
124         0;
125 
126 #undef SCHED_FEAT
127 
128 /*
129  * Number of tasks to iterate in a single balance run.
130  * Limited because this is done with IRQs disabled.
131  */
132 const_debug unsigned int sysctl_sched_nr_migrate = 32;
133 
134 /*
135  * period over which we average the RT time consumption, measured
136  * in ms.
137  *
138  * default: 1s
139  */
140 const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
141 
142 /*
143  * period over which we measure -rt task cpu usage in us.
144  * default: 1s
145  */
146 unsigned int sysctl_sched_rt_period = 1000000;
147 
148 __read_mostly int scheduler_running;
149 
150 /*
151  * part of the period that we allow rt tasks to run in us.
152  * default: 0.95s
153  */
154 int sysctl_sched_rt_runtime = 950000;
155 
156 /* cpus with isolated domains */
157 cpumask_var_t cpu_isolated_map;
158 
159 /*
160  * this_rq_lock - lock this runqueue and disable interrupts.
161  */
162 static struct rq *this_rq_lock(void)
163         __acquires(rq->lock)
164 {
165         struct rq *rq;
166 
167         local_irq_disable();
168         rq = this_rq();
169         raw_spin_lock(&rq->lock);
170 
171         return rq;
172 }
173 
174 /*
175  * __task_rq_lock - lock the rq @p resides on.
176  */
177 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
178         __acquires(rq->lock)
179 {
180         struct rq *rq;
181 
182         lockdep_assert_held(&p->pi_lock);
183 
184         for (;;) {
185                 rq = task_rq(p);
186                 raw_spin_lock(&rq->lock);
187                 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
188                         rf->cookie = lockdep_pin_lock(&rq->lock);
189                         return rq;
190                 }
191                 raw_spin_unlock(&rq->lock);
192 
193                 while (unlikely(task_on_rq_migrating(p)))
194                         cpu_relax();
195         }
196 }
197 
198 /*
199  * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
200  */
201 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
202         __acquires(p->pi_lock)
203         __acquires(rq->lock)
204 {
205         struct rq *rq;
206 
207         for (;;) {
208                 raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
209                 rq = task_rq(p);
210                 raw_spin_lock(&rq->lock);
211                 /*
212                  *      move_queued_task()              task_rq_lock()
213                  *
214                  *      ACQUIRE (rq->lock)
215                  *      [S] ->on_rq = MIGRATING         [L] rq = task_rq()
216                  *      WMB (__set_task_cpu())          ACQUIRE (rq->lock);
217                  *      [S] ->cpu = new_cpu             [L] task_rq()
218                  *                                      [L] ->on_rq
219                  *      RELEASE (rq->lock)
220                  *
221                  * If we observe the old cpu in task_rq_lock, the acquire of
222                  * the old rq->lock will fully serialize against the stores.
223                  *
224                  * If we observe the new cpu in task_rq_lock, the acquire will
225                  * pair with the WMB to ensure we must then also see migrating.
226                  */
227                 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
228                         rf->cookie = lockdep_pin_lock(&rq->lock);
229                         return rq;
230                 }
231                 raw_spin_unlock(&rq->lock);
232                 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
233 
234                 while (unlikely(task_on_rq_migrating(p)))
235                         cpu_relax();
236         }
237 }
238 
239 #ifdef CONFIG_SCHED_HRTICK
240 /*
241  * Use HR-timers to deliver accurate preemption points.
242  */
243 
244 static void hrtick_clear(struct rq *rq)
245 {
246         if (hrtimer_active(&rq->hrtick_timer))
247                 hrtimer_cancel(&rq->hrtick_timer);
248 }
249 
250 /*
251  * High-resolution timer tick.
252  * Runs from hardirq context with interrupts disabled.
253  */
254 static enum hrtimer_restart hrtick(struct hrtimer *timer)
255 {
256         struct rq *rq = container_of(timer, struct rq, hrtick_timer);
257 
258         WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
259 
260         raw_spin_lock(&rq->lock);
261         update_rq_clock(rq);
262         rq->curr->sched_class->task_tick(rq, rq->curr, 1);
263         raw_spin_unlock(&rq->lock);
264 
265         return HRTIMER_NORESTART;
266 }
267 
268 #ifdef CONFIG_SMP
269 
270 static void __hrtick_restart(struct rq *rq)
271 {
272         struct hrtimer *timer = &rq->hrtick_timer;
273 
274         hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
275 }
276 
277 /*
278  * called from hardirq (IPI) context
279  */
280 static void __hrtick_start(void *arg)
281 {
282         struct rq *rq = arg;
283 
284         raw_spin_lock(&rq->lock);
285         __hrtick_restart(rq);
286         rq->hrtick_csd_pending = 0;
287         raw_spin_unlock(&rq->lock);
288 }
289 
290 /*
291  * Called to set the hrtick timer state.
292  *
293  * called with rq->lock held and irqs disabled
294  */
295 void hrtick_start(struct rq *rq, u64 delay)
296 {
297         struct hrtimer *timer = &rq->hrtick_timer;
298         ktime_t time;
299         s64 delta;
300 
301         /*
302          * Don't schedule slices shorter than 10000ns, that just
303          * doesn't make sense and can cause timer DoS.
304          */
305         delta = max_t(s64, delay, 10000LL);
306         time = ktime_add_ns(timer->base->get_time(), delta);
307 
308         hrtimer_set_expires(timer, time);
309 
310         if (rq == this_rq()) {
311                 __hrtick_restart(rq);
312         } else if (!rq->hrtick_csd_pending) {
313                 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
314                 rq->hrtick_csd_pending = 1;
315         }
316 }
317 
318 #else
319 /*
320  * Called to set the hrtick timer state.
321  *
322  * called with rq->lock held and irqs disabled
323  */
324 void hrtick_start(struct rq *rq, u64 delay)
325 {
326         /*
327          * Don't schedule slices shorter than 10000ns, that just
328          * doesn't make sense. Rely on vruntime for fairness.
329          */
330         delay = max_t(u64, delay, 10000LL);
331         hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
332                       HRTIMER_MODE_REL_PINNED);
333 }
334 #endif /* CONFIG_SMP */
335 
336 static void init_rq_hrtick(struct rq *rq)
337 {
338 #ifdef CONFIG_SMP
339         rq->hrtick_csd_pending = 0;
340 
341         rq->hrtick_csd.flags = 0;
342         rq->hrtick_csd.func = __hrtick_start;
343         rq->hrtick_csd.info = rq;
344 #endif
345 
346         hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
347         rq->hrtick_timer.function = hrtick;
348 }
349 #else   /* CONFIG_SCHED_HRTICK */
350 static inline void hrtick_clear(struct rq *rq)
351 {
352 }
353 
354 static inline void init_rq_hrtick(struct rq *rq)
355 {
356 }
357 #endif  /* CONFIG_SCHED_HRTICK */
358 
359 /*
360  * cmpxchg based fetch_or, macro so it works for different integer types
361  */
362 #define fetch_or(ptr, mask)                                             \
363         ({                                                              \
364                 typeof(ptr) _ptr = (ptr);                               \
365                 typeof(mask) _mask = (mask);                            \
366                 typeof(*_ptr) _old, _val = *_ptr;                       \
367                                                                         \
368                 for (;;) {                                              \
369                         _old = cmpxchg(_ptr, _val, _val | _mask);       \
370                         if (_old == _val)                               \
371                                 break;                                  \
372                         _val = _old;                                    \
373                 }                                                       \
374         _old;                                                           \
375 })
376 
377 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
378 /*
379  * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
380  * this avoids any races wrt polling state changes and thereby avoids
381  * spurious IPIs.
382  */
383 static bool set_nr_and_not_polling(struct task_struct *p)
384 {
385         struct thread_info *ti = task_thread_info(p);
386         return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
387 }
388 
389 /*
390  * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
391  *
392  * If this returns true, then the idle task promises to call
393  * sched_ttwu_pending() and reschedule soon.
394  */
395 static bool set_nr_if_polling(struct task_struct *p)
396 {
397         struct thread_info *ti = task_thread_info(p);
398         typeof(ti->flags) old, val = READ_ONCE(ti->flags);
399 
400         for (;;) {
401                 if (!(val & _TIF_POLLING_NRFLAG))
402                         return false;
403                 if (val & _TIF_NEED_RESCHED)
404                         return true;
405                 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
406                 if (old == val)
407                         break;
408                 val = old;
409         }
410         return true;
411 }
412 
413 #else
414 static bool set_nr_and_not_polling(struct task_struct *p)
415 {
416         set_tsk_need_resched(p);
417         return true;
418 }
419 
420 #ifdef CONFIG_SMP
421 static bool set_nr_if_polling(struct task_struct *p)
422 {
423         return false;
424 }
425 #endif
426 #endif
427 
428 void wake_q_add(struct wake_q_head *head, struct task_struct *task)
429 {
430         struct wake_q_node *node = &task->wake_q;
431 
432         /*
433          * Atomically grab the task, if ->wake_q is !nil already it means
434          * its already queued (either by us or someone else) and will get the
435          * wakeup due to that.
436          *
437          * This cmpxchg() implies a full barrier, which pairs with the write
438          * barrier implied by the wakeup in wake_up_q().
439          */
440         if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL))
441                 return;
442 
443         get_task_struct(task);
444 
445         /*
446          * The head is context local, there can be no concurrency.
447          */
448         *head->lastp = node;
449         head->lastp = &node->next;
450 }
451 
452 void wake_up_q(struct wake_q_head *head)
453 {
454         struct wake_q_node *node = head->first;
455 
456         while (node != WAKE_Q_TAIL) {
457                 struct task_struct *task;
458 
459                 task = container_of(node, struct task_struct, wake_q);
460                 BUG_ON(!task);
461                 /* task can safely be re-inserted now */
462                 node = node->next;
463                 task->wake_q.next = NULL;
464 
465                 /*
466                  * wake_up_process() implies a wmb() to pair with the queueing
467                  * in wake_q_add() so as not to miss wakeups.
468                  */
469                 wake_up_process(task);
470                 put_task_struct(task);
471         }
472 }
473 
474 /*
475  * resched_curr - mark rq's current task 'to be rescheduled now'.
476  *
477  * On UP this means the setting of the need_resched flag, on SMP it
478  * might also involve a cross-CPU call to trigger the scheduler on
479  * the target CPU.
480  */
481 void resched_curr(struct rq *rq)
482 {
483         struct task_struct *curr = rq->curr;
484         int cpu;
485 
486         lockdep_assert_held(&rq->lock);
487 
488         if (test_tsk_need_resched(curr))
489                 return;
490 
491         cpu = cpu_of(rq);
492 
493         if (cpu == smp_processor_id()) {
494                 set_tsk_need_resched(curr);
495                 set_preempt_need_resched();
496                 return;
497         }
498 
499         if (set_nr_and_not_polling(curr))
500                 smp_send_reschedule(cpu);
501         else
502                 trace_sched_wake_idle_without_ipi(cpu);
503 }
504 
505 void resched_cpu(int cpu)
506 {
507         struct rq *rq = cpu_rq(cpu);
508         unsigned long flags;
509 
510         if (!raw_spin_trylock_irqsave(&rq->lock, flags))
511                 return;
512         resched_curr(rq);
513         raw_spin_unlock_irqrestore(&rq->lock, flags);
514 }
515 
516 #ifdef CONFIG_SMP
517 #ifdef CONFIG_NO_HZ_COMMON
518 /*
519  * In the semi idle case, use the nearest busy cpu for migrating timers
520  * from an idle cpu.  This is good for power-savings.
521  *
522  * We don't do similar optimization for completely idle system, as
523  * selecting an idle cpu will add more delays to the timers than intended
524  * (as that cpu's timer base may not be uptodate wrt jiffies etc).
525  */
526 int get_nohz_timer_target(void)
527 {
528         int i, cpu = smp_processor_id();
529         struct sched_domain *sd;
530 
531         if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
532                 return cpu;
533 
534         rcu_read_lock();
535         for_each_domain(cpu, sd) {
536                 for_each_cpu(i, sched_domain_span(sd)) {
537                         if (cpu == i)
538                                 continue;
539 
540                         if (!idle_cpu(i) && is_housekeeping_cpu(i)) {
541                                 cpu = i;
542                                 goto unlock;
543                         }
544                 }
545         }
546 
547         if (!is_housekeeping_cpu(cpu))
548                 cpu = housekeeping_any_cpu();
549 unlock:
550         rcu_read_unlock();
551         return cpu;
552 }
553 /*
554  * When add_timer_on() enqueues a timer into the timer wheel of an
555  * idle CPU then this timer might expire before the next timer event
556  * which is scheduled to wake up that CPU. In case of a completely
557  * idle system the next event might even be infinite time into the
558  * future. wake_up_idle_cpu() ensures that the CPU is woken up and
559  * leaves the inner idle loop so the newly added timer is taken into
560  * account when the CPU goes back to idle and evaluates the timer
561  * wheel for the next timer event.
562  */
563 static void wake_up_idle_cpu(int cpu)
564 {
565         struct rq *rq = cpu_rq(cpu);
566 
567         if (cpu == smp_processor_id())
568                 return;
569 
570         if (set_nr_and_not_polling(rq->idle))
571                 smp_send_reschedule(cpu);
572         else
573                 trace_sched_wake_idle_without_ipi(cpu);
574 }
575 
576 static bool wake_up_full_nohz_cpu(int cpu)
577 {
578         /*
579          * We just need the target to call irq_exit() and re-evaluate
580          * the next tick. The nohz full kick at least implies that.
581          * If needed we can still optimize that later with an
582          * empty IRQ.
583          */
584         if (cpu_is_offline(cpu))
585                 return true;  /* Don't try to wake offline CPUs. */
586         if (tick_nohz_full_cpu(cpu)) {
587                 if (cpu != smp_processor_id() ||
588                     tick_nohz_tick_stopped())
589                         tick_nohz_full_kick_cpu(cpu);
590                 return true;
591         }
592 
593         return false;
594 }
595 
596 /*
597  * Wake up the specified CPU.  If the CPU is going offline, it is the
598  * caller's responsibility to deal with the lost wakeup, for example,
599  * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
600  */
601 void wake_up_nohz_cpu(int cpu)
602 {
603         if (!wake_up_full_nohz_cpu(cpu))
604                 wake_up_idle_cpu(cpu);
605 }
606 
607 static inline bool got_nohz_idle_kick(void)
608 {
609         int cpu = smp_processor_id();
610 
611         if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
612                 return false;
613 
614         if (idle_cpu(cpu) && !need_resched())
615                 return true;
616 
617         /*
618          * We can't run Idle Load Balance on this CPU for this time so we
619          * cancel it and clear NOHZ_BALANCE_KICK
620          */
621         clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
622         return false;
623 }
624 
625 #else /* CONFIG_NO_HZ_COMMON */
626 
627 static inline bool got_nohz_idle_kick(void)
628 {
629         return false;
630 }
631 
632 #endif /* CONFIG_NO_HZ_COMMON */
633 
634 #ifdef CONFIG_NO_HZ_FULL
635 bool sched_can_stop_tick(struct rq *rq)
636 {
637         int fifo_nr_running;
638 
639         /* Deadline tasks, even if single, need the tick */
640         if (rq->dl.dl_nr_running)
641                 return false;
642 
643         /*
644          * If there are more than one RR tasks, we need the tick to effect the
645          * actual RR behaviour.
646          */
647         if (rq->rt.rr_nr_running) {
648                 if (rq->rt.rr_nr_running == 1)
649                         return true;
650                 else
651                         return false;
652         }
653 
654         /*
655          * If there's no RR tasks, but FIFO tasks, we can skip the tick, no
656          * forced preemption between FIFO tasks.
657          */
658         fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
659         if (fifo_nr_running)
660                 return true;
661 
662         /*
663          * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
664          * if there's more than one we need the tick for involuntary
665          * preemption.
666          */
667         if (rq->nr_running > 1)
668                 return false;
669 
670         return true;
671 }
672 #endif /* CONFIG_NO_HZ_FULL */
673 
674 void sched_avg_update(struct rq *rq)
675 {
676         s64 period = sched_avg_period();
677 
678         while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
679                 /*
680                  * Inline assembly required to prevent the compiler
681                  * optimising this loop into a divmod call.
682                  * See __iter_div_u64_rem() for another example of this.
683                  */
684                 asm("" : "+rm" (rq->age_stamp));
685                 rq->age_stamp += period;
686                 rq->rt_avg /= 2;
687         }
688 }
689 
690 #endif /* CONFIG_SMP */
691 
692 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
693                         (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
694 /*
695  * Iterate task_group tree rooted at *from, calling @down when first entering a
696  * node and @up when leaving it for the final time.
697  *
698  * Caller must hold rcu_lock or sufficient equivalent.
699  */
700 int walk_tg_tree_from(struct task_group *from,
701                              tg_visitor down, tg_visitor up, void *data)
702 {
703         struct task_group *parent, *child;
704         int ret;
705 
706         parent = from;
707 
708 down:
709         ret = (*down)(parent, data);
710         if (ret)
711                 goto out;
712         list_for_each_entry_rcu(child, &parent->children, siblings) {
713                 parent = child;
714                 goto down;
715 
716 up:
717                 continue;
718         }
719         ret = (*up)(parent, data);
720         if (ret || parent == from)
721                 goto out;
722 
723         child = parent;
724         parent = parent->parent;
725         if (parent)
726                 goto up;
727 out:
728         return ret;
729 }
730 
731 int tg_nop(struct task_group *tg, void *data)
732 {
733         return 0;
734 }
735 #endif
736 
737 static void set_load_weight(struct task_struct *p)
738 {
739         int prio = p->static_prio - MAX_RT_PRIO;
740         struct load_weight *load = &p->se.load;
741 
742         /*
743          * SCHED_IDLE tasks get minimal weight:
744          */
745         if (idle_policy(p->policy)) {
746                 load->weight = scale_load(WEIGHT_IDLEPRIO);
747                 load->inv_weight = WMULT_IDLEPRIO;
748                 return;
749         }
750 
751         load->weight = scale_load(sched_prio_to_weight[prio]);
752         load->inv_weight = sched_prio_to_wmult[prio];
753 }
754 
755 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
756 {
757         update_rq_clock(rq);
758         if (!(flags & ENQUEUE_RESTORE))
759                 sched_info_queued(rq, p);
760         p->sched_class->enqueue_task(rq, p, flags);
761 }
762 
763 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
764 {
765         update_rq_clock(rq);
766         if (!(flags & DEQUEUE_SAVE))
767                 sched_info_dequeued(rq, p);
768         p->sched_class->dequeue_task(rq, p, flags);
769 }
770 
771 void activate_task(struct rq *rq, struct task_struct *p, int flags)
772 {
773         if (task_contributes_to_load(p))
774                 rq->nr_uninterruptible--;
775 
776         enqueue_task(rq, p, flags);
777 }
778 
779 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
780 {
781         if (task_contributes_to_load(p))
782                 rq->nr_uninterruptible++;
783 
784         dequeue_task(rq, p, flags);
785 }
786 
787 static void update_rq_clock_task(struct rq *rq, s64 delta)
788 {
789 /*
790  * In theory, the compile should just see 0 here, and optimize out the call
791  * to sched_rt_avg_update. But I don't trust it...
792  */
793 #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
794         s64 steal = 0, irq_delta = 0;
795 #endif
796 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
797         irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
798 
799         /*
800          * Since irq_time is only updated on {soft,}irq_exit, we might run into
801          * this case when a previous update_rq_clock() happened inside a
802          * {soft,}irq region.
803          *
804          * When this happens, we stop ->clock_task and only update the
805          * prev_irq_time stamp to account for the part that fit, so that a next
806          * update will consume the rest. This ensures ->clock_task is
807          * monotonic.
808          *
809          * It does however cause some slight miss-attribution of {soft,}irq
810          * time, a more accurate solution would be to update the irq_time using
811          * the current rq->clock timestamp, except that would require using
812          * atomic ops.
813          */
814         if (irq_delta > delta)
815                 irq_delta = delta;
816 
817         rq->prev_irq_time += irq_delta;
818         delta -= irq_delta;
819 #endif
820 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
821         if (static_key_false((&paravirt_steal_rq_enabled))) {
822                 steal = paravirt_steal_clock(cpu_of(rq));
823                 steal -= rq->prev_steal_time_rq;
824 
825                 if (unlikely(steal > delta))
826                         steal = delta;
827 
828                 rq->prev_steal_time_rq += steal;
829                 delta -= steal;
830         }
831 #endif
832 
833         rq->clock_task += delta;
834 
835 #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
836         if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
837                 sched_rt_avg_update(rq, irq_delta + steal);
838 #endif
839 }
840 
841 void sched_set_stop_task(int cpu, struct task_struct *stop)
842 {
843         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
844         struct task_struct *old_stop = cpu_rq(cpu)->stop;
845 
846         if (stop) {
847                 /*
848                  * Make it appear like a SCHED_FIFO task, its something
849                  * userspace knows about and won't get confused about.
850                  *
851                  * Also, it will make PI more or less work without too
852                  * much confusion -- but then, stop work should not
853                  * rely on PI working anyway.
854                  */
855                 sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
856 
857                 stop->sched_class = &stop_sched_class;
858         }
859 
860         cpu_rq(cpu)->stop = stop;
861 
862         if (old_stop) {
863                 /*
864                  * Reset it back to a normal scheduling class so that
865                  * it can die in pieces.
866                  */
867                 old_stop->sched_class = &rt_sched_class;
868         }
869 }
870 
871 /*
872  * __normal_prio - return the priority that is based on the static prio
873  */
874 static inline int __normal_prio(struct task_struct *p)
875 {
876         return p->static_prio;
877 }
878 
879 /*
880  * Calculate the expected normal priority: i.e. priority
881  * without taking RT-inheritance into account. Might be
882  * boosted by interactivity modifiers. Changes upon fork,
883  * setprio syscalls, and whenever the interactivity
884  * estimator recalculates.
885  */
886 static inline int normal_prio(struct task_struct *p)
887 {
888         int prio;
889 
890         if (task_has_dl_policy(p))
891                 prio = MAX_DL_PRIO-1;
892         else if (task_has_rt_policy(p))
893                 prio = MAX_RT_PRIO-1 - p->rt_priority;
894         else
895                 prio = __normal_prio(p);
896         return prio;
897 }
898 
899 /*
900  * Calculate the current priority, i.e. the priority
901  * taken into account by the scheduler. This value might
902  * be boosted by RT tasks, or might be boosted by
903  * interactivity modifiers. Will be RT if the task got
904  * RT-boosted. If not then it returns p->normal_prio.
905  */
906 static int effective_prio(struct task_struct *p)
907 {
908         p->normal_prio = normal_prio(p);
909         /*
910          * If we are RT tasks or we were boosted to RT priority,
911          * keep the priority unchanged. Otherwise, update priority
912          * to the normal priority:
913          */
914         if (!rt_prio(p->prio))
915                 return p->normal_prio;
916         return p->prio;
917 }
918 
919 /**
920  * task_curr - is this task currently executing on a CPU?
921  * @p: the task in question.
922  *
923  * Return: 1 if the task is currently executing. 0 otherwise.
924  */
925 inline int task_curr(const struct task_struct *p)
926 {
927         return cpu_curr(task_cpu(p)) == p;
928 }
929 
930 /*
931  * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
932  * use the balance_callback list if you want balancing.
933  *
934  * this means any call to check_class_changed() must be followed by a call to
935  * balance_callback().
936  */
937 static inline void check_class_changed(struct rq *rq, struct task_struct *p,
938                                        const struct sched_class *prev_class,
939                                        int oldprio)
940 {
941         if (prev_class != p->sched_class) {
942                 if (prev_class->switched_from)
943                         prev_class->switched_from(rq, p);
944 
945                 p->sched_class->switched_to(rq, p);
946         } else if (oldprio != p->prio || dl_task(p))
947                 p->sched_class->prio_changed(rq, p, oldprio);
948 }
949 
950 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
951 {
952         const struct sched_class *class;
953 
954         if (p->sched_class == rq->curr->sched_class) {
955                 rq->curr->sched_class->check_preempt_curr(rq, p, flags);
956         } else {
957                 for_each_class(class) {
958                         if (class == rq->curr->sched_class)
959                                 break;
960                         if (class == p->sched_class) {
961                                 resched_curr(rq);
962                                 break;
963                         }
964                 }
965         }
966 
967         /*
968          * A queue event has occurred, and we're going to schedule.  In
969          * this case, we can save a useless back to back clock update.
970          */
971         if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
972                 rq_clock_skip_update(rq, true);
973 }
974 
975 #ifdef CONFIG_SMP
976 /*
977  * This is how migration works:
978  *
979  * 1) we invoke migration_cpu_stop() on the target CPU using
980  *    stop_one_cpu().
981  * 2) stopper starts to run (implicitly forcing the migrated thread
982  *    off the CPU)
983  * 3) it checks whether the migrated task is still in the wrong runqueue.
984  * 4) if it's in the wrong runqueue then the migration thread removes
985  *    it and puts it into the right queue.
986  * 5) stopper completes and stop_one_cpu() returns and the migration
987  *    is done.
988  */
989 
990 /*
991  * move_queued_task - move a queued task to new rq.
992  *
993  * Returns (locked) new rq. Old rq's lock is released.
994  */
995 static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new_cpu)
996 {
997         lockdep_assert_held(&rq->lock);
998 
999         p->on_rq = TASK_ON_RQ_MIGRATING;
1000         dequeue_task(rq, p, 0);
1001         set_task_cpu(p, new_cpu);
1002         raw_spin_unlock(&rq->lock);
1003 
1004         rq = cpu_rq(new_cpu);
1005 
1006         raw_spin_lock(&rq->lock);
1007         BUG_ON(task_cpu(p) != new_cpu);
1008         enqueue_task(rq, p, 0);
1009         p->on_rq = TASK_ON_RQ_QUEUED;
1010         check_preempt_curr(rq, p, 0);
1011 
1012         return rq;
1013 }
1014 
1015 struct migration_arg {
1016         struct task_struct *task;
1017         int dest_cpu;
1018 };
1019 
1020 /*
1021  * Move (not current) task off this cpu, onto dest cpu. We're doing
1022  * this because either it can't run here any more (set_cpus_allowed()
1023  * away from this CPU, or CPU going down), or because we're
1024  * attempting to rebalance this task on exec (sched_exec).
1025  *
1026  * So we race with normal scheduler movements, but that's OK, as long
1027  * as the task is no longer on this CPU.
1028  */
1029 static struct rq *__migrate_task(struct rq *rq, struct task_struct *p, int dest_cpu)
1030 {
1031         if (unlikely(!cpu_active(dest_cpu)))
1032                 return rq;
1033 
1034         /* Affinity changed (again). */
1035         if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
1036                 return rq;
1037 
1038         rq = move_queued_task(rq, p, dest_cpu);
1039 
1040         return rq;
1041 }
1042 
1043 /*
1044  * migration_cpu_stop - this will be executed by a highprio stopper thread
1045  * and performs thread migration by bumping thread off CPU then
1046  * 'pushing' onto another runqueue.
1047  */
1048 static int migration_cpu_stop(void *data)
1049 {
1050         struct migration_arg *arg = data;
1051         struct task_struct *p = arg->task;
1052         struct rq *rq = this_rq();
1053 
1054         /*
1055          * The original target cpu might have gone down and we might
1056          * be on another cpu but it doesn't matter.
1057          */
1058         local_irq_disable();
1059         /*
1060          * We need to explicitly wake pending tasks before running
1061          * __migrate_task() such that we will not miss enforcing cpus_allowed
1062          * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
1063          */
1064         sched_ttwu_pending();
1065 
1066         raw_spin_lock(&p->pi_lock);
1067         raw_spin_lock(&rq->lock);
1068         /*
1069          * If task_rq(p) != rq, it cannot be migrated here, because we're
1070          * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1071          * we're holding p->pi_lock.
1072          */
1073         if (task_rq(p) == rq) {
1074                 if (task_on_rq_queued(p))
1075                         rq = __migrate_task(rq, p, arg->dest_cpu);
1076                 else
1077                         p->wake_cpu = arg->dest_cpu;
1078         }
1079         raw_spin_unlock(&rq->lock);
1080         raw_spin_unlock(&p->pi_lock);
1081 
1082         local_irq_enable();
1083         return 0;
1084 }
1085 
1086 /*
1087  * sched_class::set_cpus_allowed must do the below, but is not required to
1088  * actually call this function.
1089  */
1090 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
1091 {
1092         cpumask_copy(&p->cpus_allowed, new_mask);
1093         p->nr_cpus_allowed = cpumask_weight(new_mask);
1094 }
1095 
1096 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1097 {
1098         struct rq *rq = task_rq(p);
1099         bool queued, running;
1100 
1101         lockdep_assert_held(&p->pi_lock);
1102 
1103         queued = task_on_rq_queued(p);
1104         running = task_current(rq, p);
1105 
1106         if (queued) {
1107                 /*
1108                  * Because __kthread_bind() calls this on blocked tasks without
1109                  * holding rq->lock.
1110                  */
1111                 lockdep_assert_held(&rq->lock);
1112                 dequeue_task(rq, p, DEQUEUE_SAVE);
1113         }
1114         if (running)
1115                 put_prev_task(rq, p);
1116 
1117         p->sched_class->set_cpus_allowed(p, new_mask);
1118 
1119         if (queued)
1120                 enqueue_task(rq, p, ENQUEUE_RESTORE);
1121         if (running)
1122                 set_curr_task(rq, p);
1123 }
1124 
1125 /*
1126  * Change a given task's CPU affinity. Migrate the thread to a
1127  * proper CPU and schedule it away if the CPU it's executing on
1128  * is removed from the allowed bitmask.
1129  *
1130  * NOTE: the caller must have a valid reference to the task, the
1131  * task must not exit() & deallocate itself prematurely. The
1132  * call is not atomic; no spinlocks may be held.
1133  */
1134 static int __set_cpus_allowed_ptr(struct task_struct *p,
1135                                   const struct cpumask *new_mask, bool check)
1136 {
1137         const struct cpumask *cpu_valid_mask = cpu_active_mask;
1138         unsigned int dest_cpu;
1139         struct rq_flags rf;
1140         struct rq *rq;
1141         int ret = 0;
1142 
1143         rq = task_rq_lock(p, &rf);
1144 
1145         if (p->flags & PF_KTHREAD) {
1146                 /*
1147                  * Kernel threads are allowed on online && !active CPUs
1148                  */
1149                 cpu_valid_mask = cpu_online_mask;
1150         }
1151 
1152         /*
1153          * Must re-check here, to close a race against __kthread_bind(),
1154          * sched_setaffinity() is not guaranteed to observe the flag.
1155          */
1156         if (check && (p->flags & PF_NO_SETAFFINITY)) {
1157                 ret = -EINVAL;
1158                 goto out;
1159         }
1160 
1161         if (cpumask_equal(&p->cpus_allowed, new_mask))
1162                 goto out;
1163 
1164         if (!cpumask_intersects(new_mask, cpu_valid_mask)) {
1165                 ret = -EINVAL;
1166                 goto out;
1167         }
1168 
1169         do_set_cpus_allowed(p, new_mask);
1170 
1171         if (p->flags & PF_KTHREAD) {
1172                 /*
1173                  * For kernel threads that do indeed end up on online &&
1174                  * !active we want to ensure they are strict per-cpu threads.
1175                  */
1176                 WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) &&
1177                         !cpumask_intersects(new_mask, cpu_active_mask) &&
1178                         p->nr_cpus_allowed != 1);
1179         }
1180 
1181         /* Can the task run on the task's current CPU? If so, we're done */
1182         if (cpumask_test_cpu(task_cpu(p), new_mask))
1183                 goto out;
1184 
1185         dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
1186         if (task_running(rq, p) || p->state == TASK_WAKING) {
1187                 struct migration_arg arg = { p, dest_cpu };
1188                 /* Need help from migration thread: drop lock and wait. */
1189                 task_rq_unlock(rq, p, &rf);
1190                 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1191                 tlb_migrate_finish(p->mm);
1192                 return 0;
1193         } else if (task_on_rq_queued(p)) {
1194                 /*
1195                  * OK, since we're going to drop the lock immediately
1196                  * afterwards anyway.
1197                  */
1198                 lockdep_unpin_lock(&rq->lock, rf.cookie);
1199                 rq = move_queued_task(rq, p, dest_cpu);
1200                 lockdep_repin_lock(&rq->lock, rf.cookie);
1201         }
1202 out:
1203         task_rq_unlock(rq, p, &rf);
1204 
1205         return ret;
1206 }
1207 
1208 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1209 {
1210         return __set_cpus_allowed_ptr(p, new_mask, false);
1211 }
1212 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1213 
1214 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1215 {
1216 #ifdef CONFIG_SCHED_DEBUG
1217         /*
1218          * We should never call set_task_cpu() on a blocked task,
1219          * ttwu() will sort out the placement.
1220          */
1221         WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
1222                         !p->on_rq);
1223 
1224         /*
1225          * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
1226          * because schedstat_wait_{start,end} rebase migrating task's wait_start
1227          * time relying on p->on_rq.
1228          */
1229         WARN_ON_ONCE(p->state == TASK_RUNNING &&
1230                      p->sched_class == &fair_sched_class &&
1231                      (p->on_rq && !task_on_rq_migrating(p)));
1232 
1233 #ifdef CONFIG_LOCKDEP
1234         /*
1235          * The caller should hold either p->pi_lock or rq->lock, when changing
1236          * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
1237          *
1238          * sched_move_task() holds both and thus holding either pins the cgroup,
1239          * see task_group().
1240          *
1241          * Furthermore, all task_rq users should acquire both locks, see
1242          * task_rq_lock().
1243          */
1244         WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
1245                                       lockdep_is_held(&task_rq(p)->lock)));
1246 #endif
1247 #endif
1248 
1249         trace_sched_migrate_task(p, new_cpu);
1250 
1251         if (task_cpu(p) != new_cpu) {
1252                 if (p->sched_class->migrate_task_rq)
1253                         p->sched_class->migrate_task_rq(p);
1254                 p->se.nr_migrations++;
1255                 perf_event_task_migrate(p);
1256         }
1257 
1258         __set_task_cpu(p, new_cpu);
1259 }
1260 
1261 static void __migrate_swap_task(struct task_struct *p, int cpu)
1262 {
1263         if (task_on_rq_queued(p)) {
1264                 struct rq *src_rq, *dst_rq;
1265 
1266                 src_rq = task_rq(p);
1267                 dst_rq = cpu_rq(cpu);
1268 
1269                 p->on_rq = TASK_ON_RQ_MIGRATING;
1270                 deactivate_task(src_rq, p, 0);
1271                 set_task_cpu(p, cpu);
1272                 activate_task(dst_rq, p, 0);
1273                 p->on_rq = TASK_ON_RQ_QUEUED;
1274                 check_preempt_curr(dst_rq, p, 0);
1275         } else {
1276                 /*
1277                  * Task isn't running anymore; make it appear like we migrated
1278                  * it before it went to sleep. This means on wakeup we make the
1279                  * previous cpu our target instead of where it really is.
1280                  */
1281                 p->wake_cpu = cpu;
1282         }
1283 }
1284 
1285 struct migration_swap_arg {
1286         struct task_struct *src_task, *dst_task;
1287         int src_cpu, dst_cpu;
1288 };
1289 
1290 static int migrate_swap_stop(void *data)
1291 {
1292         struct migration_swap_arg *arg = data;
1293         struct rq *src_rq, *dst_rq;
1294         int ret = -EAGAIN;
1295 
1296         if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
1297                 return -EAGAIN;
1298 
1299         src_rq = cpu_rq(arg->src_cpu);
1300         dst_rq = cpu_rq(arg->dst_cpu);
1301 
1302         double_raw_lock(&arg->src_task->pi_lock,
1303                         &arg->dst_task->pi_lock);
1304         double_rq_lock(src_rq, dst_rq);
1305 
1306         if (task_cpu(arg->dst_task) != arg->dst_cpu)
1307                 goto unlock;
1308 
1309         if (task_cpu(arg->src_task) != arg->src_cpu)
1310                 goto unlock;
1311 
1312         if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
1313                 goto unlock;
1314 
1315         if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
1316                 goto unlock;
1317 
1318         __migrate_swap_task(arg->src_task, arg->dst_cpu);
1319         __migrate_swap_task(arg->dst_task, arg->src_cpu);
1320 
1321         ret = 0;
1322 
1323 unlock:
1324         double_rq_unlock(src_rq, dst_rq);
1325         raw_spin_unlock(&arg->dst_task->pi_lock);
1326         raw_spin_unlock(&arg->src_task->pi_lock);
1327 
1328         return ret;
1329 }
1330 
1331 /*
1332  * Cross migrate two tasks
1333  */
1334 int migrate_swap(struct task_struct *cur, struct task_struct *p)
1335 {
1336         struct migration_swap_arg arg;
1337         int ret = -EINVAL;
1338 
1339         arg = (struct migration_swap_arg){
1340                 .src_task = cur,
1341                 .src_cpu = task_cpu(cur),
1342                 .dst_task = p,
1343                 .dst_cpu = task_cpu(p),
1344         };
1345 
1346         if (arg.src_cpu == arg.dst_cpu)
1347                 goto out;
1348 
1349         /*
1350          * These three tests are all lockless; this is OK since all of them
1351          * will be re-checked with proper locks held further down the line.
1352          */
1353         if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
1354                 goto out;
1355 
1356         if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
1357                 goto out;
1358 
1359         if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
1360                 goto out;
1361 
1362         trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
1363         ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
1364 
1365 out:
1366         return ret;
1367 }
1368 
1369 /*
1370  * wait_task_inactive - wait for a thread to unschedule.
1371  *
1372  * If @match_state is nonzero, it's the @p->state value just checked and
1373  * not expected to change.  If it changes, i.e. @p might have woken up,
1374  * then return zero.  When we succeed in waiting for @p to be off its CPU,
1375  * we return a positive number (its total switch count).  If a second call
1376  * a short while later returns the same number, the caller can be sure that
1377  * @p has remained unscheduled the whole time.
1378  *
1379  * The caller must ensure that the task *will* unschedule sometime soon,
1380  * else this function might spin for a *long* time. This function can't
1381  * be called with interrupts off, or it may introduce deadlock with
1382  * smp_call_function() if an IPI is sent by the same process we are
1383  * waiting to become inactive.
1384  */
1385 unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1386 {
1387         int running, queued;
1388         struct rq_flags rf;
1389         unsigned long ncsw;
1390         struct rq *rq;
1391 
1392         for (;;) {
1393                 /*
1394                  * We do the initial early heuristics without holding
1395                  * any task-queue locks at all. We'll only try to get
1396                  * the runqueue lock when things look like they will
1397                  * work out!
1398                  */
1399                 rq = task_rq(p);
1400 
1401                 /*
1402                  * If the task is actively running on another CPU
1403                  * still, just relax and busy-wait without holding
1404                  * any locks.
1405                  *
1406                  * NOTE! Since we don't hold any locks, it's not
1407                  * even sure that "rq" stays as the right runqueue!
1408                  * But we don't care, since "task_running()" will
1409                  * return false if the runqueue has changed and p
1410                  * is actually now running somewhere else!
1411                  */
1412                 while (task_running(rq, p)) {
1413                         if (match_state && unlikely(p->state != match_state))
1414                                 return 0;
1415                         cpu_relax();
1416                 }
1417 
1418                 /*
1419                  * Ok, time to look more closely! We need the rq
1420                  * lock now, to be *sure*. If we're wrong, we'll
1421                  * just go back and repeat.
1422                  */
1423                 rq = task_rq_lock(p, &rf);
1424                 trace_sched_wait_task(p);
1425                 running = task_running(rq, p);
1426                 queued = task_on_rq_queued(p);
1427                 ncsw = 0;
1428                 if (!match_state || p->state == match_state)
1429                         ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1430                 task_rq_unlock(rq, p, &rf);
1431 
1432                 /*
1433                  * If it changed from the expected state, bail out now.
1434                  */
1435                 if (unlikely(!ncsw))
1436                         break;
1437 
1438                 /*
1439                  * Was it really running after all now that we
1440                  * checked with the proper locks actually held?
1441                  *
1442                  * Oops. Go back and try again..
1443                  */
1444                 if (unlikely(running)) {
1445                         cpu_relax();
1446                         continue;
1447                 }
1448 
1449                 /*
1450                  * It's not enough that it's not actively running,
1451                  * it must be off the runqueue _entirely_, and not
1452                  * preempted!
1453                  *
1454                  * So if it was still runnable (but just not actively
1455                  * running right now), it's preempted, and we should
1456                  * yield - it could be a while.
1457                  */
1458                 if (unlikely(queued)) {
1459                         ktime_t to = NSEC_PER_SEC / HZ;
1460 
1461                         set_current_state(TASK_UNINTERRUPTIBLE);
1462                         schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1463                         continue;
1464                 }
1465 
1466                 /*
1467                  * Ahh, all good. It wasn't running, and it wasn't
1468                  * runnable, which means that it will never become
1469                  * running in the future either. We're all done!
1470                  */
1471                 break;
1472         }
1473 
1474         return ncsw;
1475 }
1476 
1477 /***
1478  * kick_process - kick a running thread to enter/exit the kernel
1479  * @p: the to-be-kicked thread
1480  *
1481  * Cause a process which is running on another CPU to enter
1482  * kernel-mode, without any delay. (to get signals handled.)
1483  *
1484  * NOTE: this function doesn't have to take the runqueue lock,
1485  * because all it wants to ensure is that the remote task enters
1486  * the kernel. If the IPI races and the task has been migrated
1487  * to another CPU then no harm is done and the purpose has been
1488  * achieved as well.
1489  */
1490 void kick_process(struct task_struct *p)
1491 {
1492         int cpu;
1493 
1494         preempt_disable();
1495         cpu = task_cpu(p);
1496         if ((cpu != smp_processor_id()) && task_curr(p))
1497                 smp_send_reschedule(cpu);
1498         preempt_enable();
1499 }
1500 EXPORT_SYMBOL_GPL(kick_process);
1501 
1502 /*
1503  * ->cpus_allowed is protected by both rq->lock and p->pi_lock
1504  *
1505  * A few notes on cpu_active vs cpu_online:
1506  *
1507  *  - cpu_active must be a subset of cpu_online
1508  *
1509  *  - on cpu-up we allow per-cpu kthreads on the online && !active cpu,
1510  *    see __set_cpus_allowed_ptr(). At this point the newly online
1511  *    cpu isn't yet part of the sched domains, and balancing will not
1512  *    see it.
1513  *
1514  *  - on cpu-down we clear cpu_active() to mask the sched domains and
1515  *    avoid the load balancer to place new tasks on the to be removed
1516  *    cpu. Existing tasks will remain running there and will be taken
1517  *    off.
1518  *
1519  * This means that fallback selection must not select !active CPUs.
1520  * And can assume that any active CPU must be online. Conversely
1521  * select_task_rq() below may allow selection of !active CPUs in order
1522  * to satisfy the above rules.
1523  */
1524 static int select_fallback_rq(int cpu, struct task_struct *p)
1525 {
1526         int nid = cpu_to_node(cpu);
1527         const struct cpumask *nodemask = NULL;
1528         enum { cpuset, possible, fail } state = cpuset;
1529         int dest_cpu;
1530 
1531         /*
1532          * If the node that the cpu is on has been offlined, cpu_to_node()
1533          * will return -1. There is no cpu on the node, and we should
1534          * select the cpu on the other node.
1535          */
1536         if (nid != -1) {
1537                 nodemask = cpumask_of_node(nid);
1538 
1539                 /* Look for allowed, online CPU in same node. */
1540                 for_each_cpu(dest_cpu, nodemask) {
1541                         if (!cpu_active(dest_cpu))
1542                                 continue;
1543                         if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
1544                                 return dest_cpu;
1545                 }
1546         }
1547 
1548         for (;;) {
1549                 /* Any allowed, online CPU? */
1550                 for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
1551                         if (!(p->flags & PF_KTHREAD) && !cpu_active(dest_cpu))
1552                                 continue;
1553                         if (!cpu_online(dest_cpu))
1554                                 continue;
1555                         goto out;
1556                 }
1557 
1558                 /* No more Mr. Nice Guy. */
1559                 switch (state) {
1560                 case cpuset:
1561                         if (IS_ENABLED(CONFIG_CPUSETS)) {
1562                                 cpuset_cpus_allowed_fallback(p);
1563                                 state = possible;
1564                                 break;
1565                         }
1566                         /* fall-through */
1567                 case possible:
1568                         do_set_cpus_allowed(p, cpu_possible_mask);
1569                         state = fail;
1570                         break;
1571 
1572                 case fail:
1573                         BUG();
1574                         break;
1575                 }
1576         }
1577 
1578 out:
1579         if (state != cpuset) {
1580                 /*
1581                  * Don't tell them about moving exiting tasks or
1582                  * kernel threads (both mm NULL), since they never
1583                  * leave kernel.
1584                  */
1585                 if (p->mm && printk_ratelimit()) {
1586                         printk_deferred("process %d (%s) no longer affine to cpu%d\n",
1587                                         task_pid_nr(p), p->comm, cpu);
1588                 }
1589         }
1590 
1591         return dest_cpu;
1592 }
1593 
1594 /*
1595  * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
1596  */
1597 static inline
1598 int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
1599 {
1600         lockdep_assert_held(&p->pi_lock);
1601 
1602         if (tsk_nr_cpus_allowed(p) > 1)
1603                 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
1604         else
1605                 cpu = cpumask_any(tsk_cpus_allowed(p));
1606 
1607         /*
1608          * In order not to call set_task_cpu() on a blocking task we need
1609          * to rely on ttwu() to place the task on a valid ->cpus_allowed
1610          * cpu.
1611          *
1612          * Since this is common to all placement strategies, this lives here.
1613          *
1614          * [ this allows ->select_task() to simply return task_cpu(p) and
1615          *   not worry about this generic constraint ]
1616          */
1617         if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
1618                      !cpu_online(cpu)))
1619                 cpu = select_fallback_rq(task_cpu(p), p);
1620 
1621         return cpu;
1622 }
1623 
1624 static void update_avg(u64 *avg, u64 sample)
1625 {
1626         s64 diff = sample - *avg;
1627         *avg += diff >> 3;
1628 }
1629 
1630 #else
1631 
1632 static inline int __set_cpus_allowed_ptr(struct task_struct *p,
1633                                          const struct cpumask *new_mask, bool check)
1634 {
1635         return set_cpus_allowed_ptr(p, new_mask);
1636 }
1637 
1638 #endif /* CONFIG_SMP */
1639 
1640 static void
1641 ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
1642 {
1643         struct rq *rq;
1644 
1645         if (!schedstat_enabled())
1646                 return;
1647 
1648         rq = this_rq();
1649 
1650 #ifdef CONFIG_SMP
1651         if (cpu == rq->cpu) {
1652                 schedstat_inc(rq->ttwu_local);
1653                 schedstat_inc(p->se.statistics.nr_wakeups_local);
1654         } else {
1655                 struct sched_domain *sd;
1656 
1657                 schedstat_inc(p->se.statistics.nr_wakeups_remote);
1658                 rcu_read_lock();
1659                 for_each_domain(rq->cpu, sd) {
1660                         if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
1661                                 schedstat_inc(sd->ttwu_wake_remote);
1662                                 break;
1663                         }
1664                 }
1665                 rcu_read_unlock();
1666         }
1667 
1668         if (wake_flags & WF_MIGRATED)
1669                 schedstat_inc(p->se.statistics.nr_wakeups_migrate);
1670 #endif /* CONFIG_SMP */
1671 
1672         schedstat_inc(rq->ttwu_count);
1673         schedstat_inc(p->se.statistics.nr_wakeups);
1674 
1675         if (wake_flags & WF_SYNC)
1676                 schedstat_inc(p->se.statistics.nr_wakeups_sync);
1677 }
1678 
1679 static inline void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
1680 {
1681         activate_task(rq, p, en_flags);
1682         p->on_rq = TASK_ON_RQ_QUEUED;
1683 
1684         /* if a worker is waking up, notify workqueue */
1685         if (p->flags & PF_WQ_WORKER)
1686                 wq_worker_waking_up(p, cpu_of(rq));
1687 }
1688 
1689 /*
1690  * Mark the task runnable and perform wakeup-preemption.
1691  */
1692 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
1693                            struct pin_cookie cookie)
1694 {
1695         check_preempt_curr(rq, p, wake_flags);
1696         p->state = TASK_RUNNING;
1697         trace_sched_wakeup(p);
1698 
1699 #ifdef CONFIG_SMP
1700         if (p->sched_class->task_woken) {
1701                 /*
1702                  * Our task @p is fully woken up and running; so its safe to
1703                  * drop the rq->lock, hereafter rq is only used for statistics.
1704                  */
1705                 lockdep_unpin_lock(&rq->lock, cookie);
1706                 p->sched_class->task_woken(rq, p);
1707                 lockdep_repin_lock(&rq->lock, cookie);
1708         }
1709 
1710         if (rq->idle_stamp) {
1711                 u64 delta = rq_clock(rq) - rq->idle_stamp;
1712                 u64 max = 2*rq->max_idle_balance_cost;
1713 
1714                 update_avg(&rq->avg_idle, delta);
1715 
1716                 if (rq->avg_idle > max)
1717                         rq->avg_idle = max;
1718 
1719                 rq->idle_stamp = 0;
1720         }
1721 #endif
1722 }
1723 
1724 static void
1725 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
1726                  struct pin_cookie cookie)
1727 {
1728         int en_flags = ENQUEUE_WAKEUP;
1729 
1730         lockdep_assert_held(&rq->lock);
1731 
1732 #ifdef CONFIG_SMP
1733         if (p->sched_contributes_to_load)
1734                 rq->nr_uninterruptible--;
1735 
1736         if (wake_flags & WF_MIGRATED)
1737                 en_flags |= ENQUEUE_MIGRATED;
1738 #endif
1739 
1740         ttwu_activate(rq, p, en_flags);
1741         ttwu_do_wakeup(rq, p, wake_flags, cookie);
1742 }
1743 
1744 /*
1745  * Called in case the task @p isn't fully descheduled from its runqueue,
1746  * in this case we must do a remote wakeup. Its a 'light' wakeup though,
1747  * since all we need to do is flip p->state to TASK_RUNNING, since
1748  * the task is still ->on_rq.
1749  */
1750 static int ttwu_remote(struct task_struct *p, int wake_flags)
1751 {
1752         struct rq_flags rf;
1753         struct rq *rq;
1754         int ret = 0;
1755 
1756         rq = __task_rq_lock(p, &rf);
1757         if (task_on_rq_queued(p)) {
1758                 /* check_preempt_curr() may use rq clock */
1759                 update_rq_clock(rq);
1760                 ttwu_do_wakeup(rq, p, wake_flags, rf.cookie);
1761                 ret = 1;
1762         }
1763         __task_rq_unlock(rq, &rf);
1764 
1765         return ret;
1766 }
1767 
1768 #ifdef CONFIG_SMP
1769 void sched_ttwu_pending(void)
1770 {
1771         struct rq *rq = this_rq();
1772         struct llist_node *llist = llist_del_all(&rq->wake_list);
1773         struct pin_cookie cookie;
1774         struct task_struct *p;
1775         unsigned long flags;
1776 
1777         if (!llist)
1778                 return;
1779 
1780         raw_spin_lock_irqsave(&rq->lock, flags);
1781         cookie = lockdep_pin_lock(&rq->lock);
1782 
1783         while (llist) {
1784                 int wake_flags = 0;
1785 
1786                 p = llist_entry(llist, struct task_struct, wake_entry);
1787                 llist = llist_next(llist);
1788 
1789                 if (p->sched_remote_wakeup)
1790                         wake_flags = WF_MIGRATED;
1791 
1792                 ttwu_do_activate(rq, p, wake_flags, cookie);
1793         }
1794 
1795         lockdep_unpin_lock(&rq->lock, cookie);
1796         raw_spin_unlock_irqrestore(&rq->lock, flags);
1797 }
1798 
1799 void scheduler_ipi(void)
1800 {
1801         /*
1802          * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1803          * TIF_NEED_RESCHED remotely (for the first time) will also send
1804          * this IPI.
1805          */
1806         preempt_fold_need_resched();
1807 
1808         if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
1809                 return;
1810 
1811         /*
1812          * Not all reschedule IPI handlers call irq_enter/irq_exit, since
1813          * traditionally all their work was done from the interrupt return
1814          * path. Now that we actually do some work, we need to make sure
1815          * we do call them.
1816          *
1817          * Some archs already do call them, luckily irq_enter/exit nest
1818          * properly.
1819          *
1820          * Arguably we should visit all archs and update all handlers,
1821          * however a fair share of IPIs are still resched only so this would
1822          * somewhat pessimize the simple resched case.
1823          */
1824         irq_enter();
1825         sched_ttwu_pending();
1826 
1827         /*
1828          * Check if someone kicked us for doing the nohz idle load balance.
1829          */
1830         if (unlikely(got_nohz_idle_kick())) {
1831                 this_rq()->idle_balance = 1;
1832                 raise_softirq_irqoff(SCHED_SOFTIRQ);
1833         }
1834         irq_exit();
1835 }
1836 
1837 static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
1838 {
1839         struct rq *rq = cpu_rq(cpu);
1840 
1841         p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
1842 
1843         if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
1844                 if (!set_nr_if_polling(rq->idle))
1845                         smp_send_reschedule(cpu);
1846                 else
1847                         trace_sched_wake_idle_without_ipi(cpu);
1848         }
1849 }
1850 
1851 void wake_up_if_idle(int cpu)
1852 {
1853         struct rq *rq = cpu_rq(cpu);
1854         unsigned long flags;
1855 
1856         rcu_read_lock();
1857 
1858         if (!is_idle_task(rcu_dereference(rq->curr)))
1859                 goto out;
1860 
1861         if (set_nr_if_polling(rq->idle)) {
1862                 trace_sched_wake_idle_without_ipi(cpu);
1863         } else {
1864                 raw_spin_lock_irqsave(&rq->lock, flags);
1865                 if (is_idle_task(rq->curr))
1866                         smp_send_reschedule(cpu);
1867                 /* Else cpu is not in idle, do nothing here */
1868                 raw_spin_unlock_irqrestore(&rq->lock, flags);
1869         }
1870 
1871 out:
1872         rcu_read_unlock();
1873 }
1874 
1875 bool cpus_share_cache(int this_cpu, int that_cpu)
1876 {
1877         return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
1878 }
1879 #endif /* CONFIG_SMP */
1880 
1881 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
1882 {
1883         struct rq *rq = cpu_rq(cpu);
1884         struct pin_cookie cookie;
1885 
1886 #if defined(CONFIG_SMP)
1887         if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
1888                 sched_clock_cpu(cpu); /* sync clocks x-cpu */
1889                 ttwu_queue_remote(p, cpu, wake_flags);
1890                 return;
1891         }
1892 #endif
1893 
1894         raw_spin_lock(&rq->lock);
1895         cookie = lockdep_pin_lock(&rq->lock);
1896         ttwu_do_activate(rq, p, wake_flags, cookie);
1897         lockdep_unpin_lock(&rq->lock, cookie);
1898         raw_spin_unlock(&rq->lock);
1899 }
1900 
1901 /*
1902  * Notes on Program-Order guarantees on SMP systems.
1903  *
1904  *  MIGRATION
1905  *
1906  * The basic program-order guarantee on SMP systems is that when a task [t]
1907  * migrates, all its activity on its old cpu [c0] happens-before any subsequent
1908  * execution on its new cpu [c1].
1909  *
1910  * For migration (of runnable tasks) this is provided by the following means:
1911  *
1912  *  A) UNLOCK of the rq(c0)->lock scheduling out task t
1913  *  B) migration for t is required to synchronize *both* rq(c0)->lock and
1914  *     rq(c1)->lock (if not at the same time, then in that order).
1915  *  C) LOCK of the rq(c1)->lock scheduling in task
1916  *
1917  * Transitivity guarantees that B happens after A and C after B.
1918  * Note: we only require RCpc transitivity.
1919  * Note: the cpu doing B need not be c0 or c1
1920  *
1921  * Example:
1922  *
1923  *   CPU0            CPU1            CPU2
1924  *
1925  *   LOCK rq(0)->lock
1926  *   sched-out X
1927  *   sched-in Y
1928  *   UNLOCK rq(0)->lock
1929  *
1930  *                                   LOCK rq(0)->lock // orders against CPU0
1931  *                                   dequeue X
1932  *                                   UNLOCK rq(0)->lock
1933  *
1934  *                                   LOCK rq(1)->lock
1935  *                                   enqueue X
1936  *                                   UNLOCK rq(1)->lock
1937  *
1938  *                   LOCK rq(1)->lock // orders against CPU2
1939  *                   sched-out Z
1940  *                   sched-in X
1941  *                   UNLOCK rq(1)->lock
1942  *
1943  *
1944  *  BLOCKING -- aka. SLEEP + WAKEUP
1945  *
1946  * For blocking we (obviously) need to provide the same guarantee as for
1947  * migration. However the means are completely different as there is no lock
1948  * chain to provide order. Instead we do:
1949  *
1950  *   1) smp_store_release(X->on_cpu, 0)
1951  *   2) smp_cond_load_acquire(!X->on_cpu)
1952  *
1953  * Example:
1954  *
1955  *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
1956  *
1957  *   LOCK rq(0)->lock LOCK X->pi_lock
1958  *   dequeue X
1959  *   sched-out X
1960  *   smp_store_release(X->on_cpu, 0);
1961  *
1962  *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
1963  *                    X->state = WAKING
1964  *                    set_task_cpu(X,2)
1965  *
1966  *                    LOCK rq(2)->lock
1967  *                    enqueue X
1968  *                    X->state = RUNNING
1969  *                    UNLOCK rq(2)->lock
1970  *
1971  *                                          LOCK rq(2)->lock // orders against CPU1
1972  *                                          sched-out Z
1973  *                                          sched-in X
1974  *                                          UNLOCK rq(2)->lock
1975  *
1976  *                    UNLOCK X->pi_lock
1977  *   UNLOCK rq(0)->lock
1978  *
1979  *
1980  * However; for wakeups there is a second guarantee we must provide, namely we
1981  * must observe the state that lead to our wakeup. That is, not only must our
1982  * task observe its own prior state, it must also observe the stores prior to
1983  * its wakeup.
1984  *
1985  * This means that any means of doing remote wakeups must order the CPU doing
1986  * the wakeup against the CPU the task is going to end up running on. This,
1987  * however, is already required for the regular Program-Order guarantee above,
1988  * since the waking CPU is the one issueing the ACQUIRE (smp_cond_load_acquire).
1989  *
1990  */
1991 
1992 /**
1993  * try_to_wake_up - wake up a thread
1994  * @p: the thread to be awakened
1995  * @state: the mask of task states that can be woken
1996  * @wake_flags: wake modifier flags (WF_*)
1997  *
1998  * If (@state & @p->state) @p->state = TASK_RUNNING.
1999  *
2000  * If the task was not queued/runnable, also place it back on a runqueue.
2001  *
2002  * Atomic against schedule() which would dequeue a task, also see
2003  * set_current_state().
2004  *
2005  * Return: %true if @p->state changes (an actual wakeup was done),
2006  *         %false otherwise.
2007  */
2008 static int
2009 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
2010 {
2011         unsigned long flags;
2012         int cpu, success = 0;
2013 
2014         /*
2015          * If we are going to wake up a thread waiting for CONDITION we
2016          * need to ensure that CONDITION=1 done by the caller can not be
2017          * reordered with p->state check below. This pairs with mb() in
2018          * set_current_state() the waiting thread does.
2019          */
2020         smp_mb__before_spinlock();
2021         raw_spin_lock_irqsave(&p->pi_lock, flags);
2022         if (!(p->state & state))
2023                 goto out;
2024 
2025         trace_sched_waking(p);
2026 
2027         success = 1; /* we're going to change ->state */
2028         cpu = task_cpu(p);
2029 
2030         /*
2031          * Ensure we load p->on_rq _after_ p->state, otherwise it would
2032          * be possible to, falsely, observe p->on_rq == 0 and get stuck
2033          * in smp_cond_load_acquire() below.
2034          *
2035          * sched_ttwu_pending()                 try_to_wake_up()
2036          *   [S] p->on_rq = 1;                  [L] P->state
2037          *       UNLOCK rq->lock  -----.
2038          *                              \
2039          *                               +---   RMB
2040          * schedule()                   /
2041          *       LOCK rq->lock    -----'
2042          *       UNLOCK rq->lock
2043          *
2044          * [task p]
2045          *   [S] p->state = UNINTERRUPTIBLE     [L] p->on_rq
2046          *
2047          * Pairs with the UNLOCK+LOCK on rq->lock from the
2048          * last wakeup of our task and the schedule that got our task
2049          * current.
2050          */
2051         smp_rmb();
2052         if (p->on_rq && ttwu_remote(p, wake_flags))
2053                 goto stat;
2054 
2055 #ifdef CONFIG_SMP
2056         /*
2057          * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
2058          * possible to, falsely, observe p->on_cpu == 0.
2059          *
2060          * One must be running (->on_cpu == 1) in order to remove oneself
2061          * from the runqueue.
2062          *
2063          *  [S] ->on_cpu = 1;   [L] ->on_rq
2064          *      UNLOCK rq->lock
2065          *                      RMB
2066          *      LOCK   rq->lock
2067          *  [S] ->on_rq = 0;    [L] ->on_cpu
2068          *
2069          * Pairs with the full barrier implied in the UNLOCK+LOCK on rq->lock
2070          * from the consecutive calls to schedule(); the first switching to our
2071          * task, the second putting it to sleep.
2072          */
2073         smp_rmb();
2074 
2075         /*
2076          * If the owning (remote) cpu is still in the middle of schedule() with
2077          * this task as prev, wait until its done referencing the task.
2078          *
2079          * Pairs with the smp_store_release() in finish_lock_switch().
2080          *
2081          * This ensures that tasks getting woken will be fully ordered against
2082          * their previous state and preserve Program Order.
2083          */
2084         smp_cond_load_acquire(&p->on_cpu, !VAL);
2085 
2086         p->sched_contributes_to_load = !!task_contributes_to_load(p);
2087         p->state = TASK_WAKING;
2088 
2089         cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
2090         if (task_cpu(p) != cpu) {
2091                 wake_flags |= WF_MIGRATED;
2092                 set_task_cpu(p, cpu);
2093         }
2094 #endif /* CONFIG_SMP */
2095 
2096         ttwu_queue(p, cpu, wake_flags);
2097 stat:
2098         ttwu_stat(p, cpu, wake_flags);
2099 out:
2100         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2101 
2102         return success;
2103 }
2104 
2105 /**
2106  * try_to_wake_up_local - try to wake up a local task with rq lock held
2107  * @p: the thread to be awakened
2108  * @cookie: context's cookie for pinning
2109  *
2110  * Put @p on the run-queue if it's not already there. The caller must
2111  * ensure that this_rq() is locked, @p is bound to this_rq() and not
2112  * the current task.
2113  */
2114 static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie)
2115 {
2116         struct rq *rq = task_rq(p);
2117 
2118         if (WARN_ON_ONCE(rq != this_rq()) ||
2119             WARN_ON_ONCE(p == current))
2120                 return;
2121 
2122         lockdep_assert_held(&rq->lock);
2123 
2124         if (!raw_spin_trylock(&p->pi_lock)) {
2125                 /*
2126                  * This is OK, because current is on_cpu, which avoids it being
2127                  * picked for load-balance and preemption/IRQs are still
2128                  * disabled avoiding further scheduler activity on it and we've
2129                  * not yet picked a replacement task.
2130                  */
2131                 lockdep_unpin_lock(&rq->lock, cookie);
2132                 raw_spin_unlock(&rq->lock);
2133                 raw_spin_lock(&p->pi_lock);
2134                 raw_spin_lock(&rq->lock);
2135                 lockdep_repin_lock(&rq->lock, cookie);
2136         }
2137 
2138         if (!(p->state & TASK_NORMAL))
2139                 goto out;
2140 
2141         trace_sched_waking(p);
2142 
2143         if (!task_on_rq_queued(p))
2144                 ttwu_activate(rq, p, ENQUEUE_WAKEUP);
2145 
2146         ttwu_do_wakeup(rq, p, 0, cookie);
2147         ttwu_stat(p, smp_processor_id(), 0);
2148 out:
2149         raw_spin_unlock(&p->pi_lock);
2150 }
2151 
2152 /**
2153  * wake_up_process - Wake up a specific process
2154  * @p: The process to be woken up.
2155  *
2156  * Attempt to wake up the nominated process and move it to the set of runnable
2157  * processes.
2158  *
2159  * Return: 1 if the process was woken up, 0 if it was already running.
2160  *
2161  * It may be assumed that this function implies a write memory barrier before
2162  * changing the task state if and only if any tasks are woken up.
2163  */
2164 int wake_up_process(struct task_struct *p)
2165 {
2166         return try_to_wake_up(p, TASK_NORMAL, 0);
2167 }
2168 EXPORT_SYMBOL(wake_up_process);
2169 
2170 int wake_up_state(struct task_struct *p, unsigned int state)
2171 {
2172         return try_to_wake_up(p, state, 0);
2173 }
2174 
2175 /*
2176  * This function clears the sched_dl_entity static params.
2177  */
2178 void __dl_clear_params(struct task_struct *p)
2179 {
2180         struct sched_dl_entity *dl_se = &p->dl;
2181 
2182         dl_se->dl_runtime = 0;
2183         dl_se->dl_deadline = 0;
2184         dl_se->dl_period = 0;
2185         dl_se->flags = 0;
2186         dl_se->dl_bw = 0;
2187 
2188         dl_se->dl_throttled = 0;
2189         dl_se->dl_yielded = 0;
2190 }
2191 
2192 /*
2193  * Perform scheduler related setup for a newly forked process p.
2194  * p is forked by current.
2195  *
2196  * __sched_fork() is basic setup used by init_idle() too:
2197  */
2198 static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
2199 {
2200         p->on_rq                        = 0;
2201 
2202         p->se.on_rq                     = 0;
2203         p->se.exec_start                = 0;
2204         p->se.sum_exec_runtime          = 0;
2205         p->se.prev_sum_exec_runtime     = 0;
2206         p->se.nr_migrations             = 0;
2207         p->se.vruntime                  = 0;
2208         INIT_LIST_HEAD(&p->se.group_node);
2209 
2210 #ifdef CONFIG_FAIR_GROUP_SCHED
2211         p->se.cfs_rq                    = NULL;
2212 #endif
2213 
2214 #ifdef CONFIG_SCHEDSTATS
2215         /* Even if schedstat is disabled, there should not be garbage */
2216         memset(&p->se.statistics, 0, sizeof(p->se.statistics));
2217 #endif
2218 
2219         RB_CLEAR_NODE(&p->dl.rb_node);
2220         init_dl_task_timer(&p->dl);
2221         __dl_clear_params(p);
2222 
2223         INIT_LIST_HEAD(&p->rt.run_list);
2224         p->rt.timeout           = 0;
2225         p->rt.time_slice        = sched_rr_timeslice;
2226         p->rt.on_rq             = 0;
2227         p->rt.on_list           = 0;
2228 
2229 #ifdef CONFIG_PREEMPT_NOTIFIERS
2230         INIT_HLIST_HEAD(&p->preempt_notifiers);
2231 #endif
2232 
2233 #ifdef CONFIG_NUMA_BALANCING
2234         if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
2235                 p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2236                 p->mm->numa_scan_seq = 0;
2237         }
2238 
2239         if (clone_flags & CLONE_VM)
2240                 p->numa_preferred_nid = current->numa_preferred_nid;
2241         else
2242                 p->numa_preferred_nid = -1;
2243 
2244         p->node_stamp = 0ULL;
2245         p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
2246         p->numa_scan_period = sysctl_numa_balancing_scan_delay;
2247         p->numa_work.next = &p->numa_work;
2248         p->numa_faults = NULL;
2249         p->last_task_numa_placement = 0;
2250         p->last_sum_exec_runtime = 0;
2251 
2252         p->numa_group = NULL;
2253 #endif /* CONFIG_NUMA_BALANCING */
2254 }
2255 
2256 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
2257 
2258 #ifdef CONFIG_NUMA_BALANCING
2259 
2260 void set_numabalancing_state(bool enabled)
2261 {
2262         if (enabled)
2263                 static_branch_enable(&sched_numa_balancing);
2264         else
2265                 static_branch_disable(&sched_numa_balancing);
2266 }
2267 
2268 #ifdef CONFIG_PROC_SYSCTL
2269 int sysctl_numa_balancing(struct ctl_table *table, int write,
2270                          void __user *buffer, size_t *lenp, loff_t *ppos)
2271 {
2272         struct ctl_table t;
2273         int err;
2274         int state = static_branch_likely(&sched_numa_balancing);
2275 
2276         if (write && !capable(CAP_SYS_ADMIN))
2277                 return -EPERM;
2278 
2279         t = *table;
2280         t.data = &state;
2281         err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2282         if (err < 0)
2283                 return err;
2284         if (write)
2285                 set_numabalancing_state(state);
2286         return err;
2287 }
2288 #endif
2289 #endif
2290 
2291 #ifdef CONFIG_SCHEDSTATS
2292 
2293 DEFINE_STATIC_KEY_FALSE(sched_schedstats);
2294 static bool __initdata __sched_schedstats = false;
2295 
2296 static void set_schedstats(bool enabled)
2297 {
2298         if (enabled)
2299                 static_branch_enable(&sched_schedstats);
2300         else
2301                 static_branch_disable(&sched_schedstats);
2302 }
2303 
2304 void force_schedstat_enabled(void)
2305 {
2306         if (!schedstat_enabled()) {
2307                 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
2308                 static_branch_enable(&sched_schedstats);
2309         }
2310 }
2311 
2312 static int __init setup_schedstats(char *str)
2313 {
2314         int ret = 0;
2315         if (!str)
2316                 goto out;
2317 
2318         /*
2319          * This code is called before jump labels have been set up, so we can't
2320          * change the static branch directly just yet.  Instead set a temporary
2321          * variable so init_schedstats() can do it later.
2322          */
2323         if (!strcmp(str, "enable")) {
2324                 __sched_schedstats = true;
2325                 ret = 1;
2326         } else if (!strcmp(str, "disable")) {
2327                 __sched_schedstats = false;
2328                 ret = 1;
2329         }
2330 out:
2331         if (!ret)
2332                 pr_warn("Unable to parse schedstats=\n");
2333 
2334         return ret;
2335 }
2336 __setup("schedstats=", setup_schedstats);
2337 
2338 static void __init init_schedstats(void)
2339 {
2340         set_schedstats(__sched_schedstats);
2341 }
2342 
2343 #ifdef CONFIG_PROC_SYSCTL
2344 int sysctl_schedstats(struct ctl_table *table, int write,
2345                          void __user *buffer, size_t *lenp, loff_t *ppos)
2346 {
2347         struct ctl_table t;
2348         int err;
2349         int state = static_branch_likely(&sched_schedstats);
2350 
2351         if (write && !capable(CAP_SYS_ADMIN))
2352                 return -EPERM;
2353 
2354         t = *table;
2355         t.data = &state;
2356         err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2357         if (err < 0)
2358                 return err;
2359         if (write)
2360                 set_schedstats(state);
2361         return err;
2362 }
2363 #endif /* CONFIG_PROC_SYSCTL */
2364 #else  /* !CONFIG_SCHEDSTATS */
2365 static inline void init_schedstats(void) {}
2366 #endif /* CONFIG_SCHEDSTATS */
2367 
2368 /*
2369  * fork()/clone()-time setup:
2370  */
2371 int sched_fork(unsigned long clone_flags, struct task_struct *p)
2372 {
2373         unsigned long flags;
2374         int cpu = get_cpu();
2375 
2376         __sched_fork(clone_flags, p);
2377         /*
2378          * We mark the process as NEW here. This guarantees that
2379          * nobody will actually run it, and a signal or other external
2380          * event cannot wake it up and insert it on the runqueue either.
2381          */
2382         p->state = TASK_NEW;
2383 
2384         /*
2385          * Make sure we do not leak PI boosting priority to the child.
2386          */
2387         p->prio = current->normal_prio;
2388 
2389         /*
2390          * Revert to default priority/policy on fork if requested.
2391          */
2392         if (unlikely(p->sched_reset_on_fork)) {
2393                 if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
2394                         p->policy = SCHED_NORMAL;
2395                         p->static_prio = NICE_TO_PRIO(0);
2396                         p->rt_priority = 0;
2397                 } else if (PRIO_TO_NICE(p->static_prio) < 0)
2398                         p->static_prio = NICE_TO_PRIO(0);
2399 
2400                 p->prio = p->normal_prio = __normal_prio(p);
2401                 set_load_weight(p);
2402 
2403                 /*
2404                  * We don't need the reset flag anymore after the fork. It has
2405                  * fulfilled its duty:
2406                  */
2407                 p->sched_reset_on_fork = 0;
2408         }
2409 
2410         if (dl_prio(p->prio)) {
2411                 put_cpu();
2412                 return -EAGAIN;
2413         } else if (rt_prio(p->prio)) {
2414                 p->sched_class = &rt_sched_class;
2415         } else {
2416                 p->sched_class = &fair_sched_class;
2417         }
2418 
2419         init_entity_runnable_average(&p->se);
2420 
2421         /*
2422          * The child is not yet in the pid-hash so no cgroup attach races,
2423          * and the cgroup is pinned to this child due to cgroup_fork()
2424          * is ran before sched_fork().
2425          *
2426          * Silence PROVE_RCU.
2427          */
2428         raw_spin_lock_irqsave(&p->pi_lock, flags);
2429         /*
2430          * We're setting the cpu for the first time, we don't migrate,
2431          * so use __set_task_cpu().
2432          */
2433         __set_task_cpu(p, cpu);
2434         if (p->sched_class->task_fork)
2435                 p->sched_class->task_fork(p);
2436         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2437 
2438 #ifdef CONFIG_SCHED_INFO
2439         if (likely(sched_info_on()))
2440                 memset(&p->sched_info, 0, sizeof(p->sched_info));
2441 #endif
2442 #if defined(CONFIG_SMP)
2443         p->on_cpu = 0;
2444 #endif
2445         init_task_preempt_count(p);
2446 #ifdef CONFIG_SMP
2447         plist_node_init(&p->pushable_tasks, MAX_PRIO);
2448         RB_CLEAR_NODE(&p->pushable_dl_tasks);
2449 #endif
2450 
2451         put_cpu();
2452         return 0;
2453 }
2454 
2455 unsigned long to_ratio(u64 period, u64 runtime)
2456 {
2457         if (runtime == RUNTIME_INF)
2458                 return 1ULL << 20;
2459 
2460         /*
2461          * Doing this here saves a lot of checks in all
2462          * the calling paths, and returning zero seems
2463          * safe for them anyway.
2464          */
2465         if (period == 0)
2466                 return 0;
2467 
2468         return div64_u64(runtime << 20, period);
2469 }
2470 
2471 #ifdef CONFIG_SMP
2472 inline struct dl_bw *dl_bw_of(int i)
2473 {
2474         RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
2475                          "sched RCU must be held");
2476         return &cpu_rq(i)->rd->dl_bw;
2477 }
2478 
2479 static inline int dl_bw_cpus(int i)
2480 {
2481         struct root_domain *rd = cpu_rq(i)->rd;
2482         int cpus = 0;
2483 
2484         RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
2485                          "sched RCU must be held");
2486         for_each_cpu_and(i, rd->span, cpu_active_mask)
2487                 cpus++;
2488 
2489         return cpus;
2490 }
2491 #else
2492 inline struct dl_bw *dl_bw_of(int i)
2493 {
2494         return &cpu_rq(i)->dl.dl_bw;
2495 }
2496 
2497 static inline int dl_bw_cpus(int i)
2498 {
2499         return 1;
2500 }
2501 #endif
2502 
2503 /*
2504  * We must be sure that accepting a new task (or allowing changing the
2505  * parameters of an existing one) is consistent with the bandwidth
2506  * constraints. If yes, this function also accordingly updates the currently
2507  * allocated bandwidth to reflect the new situation.
2508  *
2509  * This function is called while holding p's rq->lock.
2510  *
2511  * XXX we should delay bw change until the task's 0-lag point, see
2512  * __setparam_dl().
2513  */
2514 static int dl_overflow(struct task_struct *p, int policy,
2515                        const struct sched_attr *attr)
2516 {
2517 
2518         struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
2519         u64 period = attr->sched_period ?: attr->sched_deadline;
2520         u64 runtime = attr->sched_runtime;
2521         u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
2522         int cpus, err = -1;
2523 
2524         /* !deadline task may carry old deadline bandwidth */
2525         if (new_bw == p->dl.dl_bw && task_has_dl_policy(p))
2526                 return 0;
2527 
2528         /*
2529          * Either if a task, enters, leave, or stays -deadline but changes
2530          * its parameters, we may need to update accordingly the total
2531          * allocated bandwidth of the container.
2532          */
2533         raw_spin_lock(&dl_b->lock);
2534         cpus = dl_bw_cpus(task_cpu(p));
2535         if (dl_policy(policy) && !task_has_dl_policy(p) &&
2536             !__dl_overflow(dl_b, cpus, 0, new_bw)) {
2537                 __dl_add(dl_b, new_bw);
2538                 err = 0;
2539         } else if (dl_policy(policy) && task_has_dl_policy(p) &&
2540                    !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
2541                 __dl_clear(dl_b, p->dl.dl_bw);
2542                 __dl_add(dl_b, new_bw);
2543                 err = 0;
2544         } else if (!dl_policy(policy) && task_has_dl_policy(p)) {
2545                 __dl_clear(dl_b, p->dl.dl_bw);
2546                 err = 0;
2547         }
2548         raw_spin_unlock(&dl_b->lock);
2549 
2550         return err;
2551 }
2552 
2553 extern void init_dl_bw(struct dl_bw *dl_b);
2554 
2555 /*
2556  * wake_up_new_task - wake up a newly created task for the first time.
2557  *
2558  * This function will do some initial scheduler statistics housekeeping
2559  * that must be done for every newly created context, then puts the task
2560  * on the runqueue and wakes it.
2561  */
2562 void wake_up_new_task(struct task_struct *p)
2563 {
2564         struct rq_flags rf;
2565         struct rq *rq;
2566 
2567         raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
2568         p->state = TASK_RUNNING;
2569 #ifdef CONFIG_SMP
2570         /*
2571          * Fork balancing, do it here and not earlier because:
2572          *  - cpus_allowed can change in the fork path
2573          *  - any previously selected cpu might disappear through hotplug
2574          *
2575          * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
2576          * as we're not fully set-up yet.
2577          */
2578         __set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
2579 #endif
2580         rq = __task_rq_lock(p, &rf);
2581         post_init_entity_util_avg(&p->se);
2582 
2583         activate_task(rq, p, 0);
2584         p->on_rq = TASK_ON_RQ_QUEUED;
2585         trace_sched_wakeup_new(p);
2586         check_preempt_curr(rq, p, WF_FORK);
2587 #ifdef CONFIG_SMP
2588         if (p->sched_class->task_woken) {
2589                 /*
2590                  * Nothing relies on rq->lock after this, so its fine to
2591                  * drop it.
2592                  */
2593                 lockdep_unpin_lock(&rq->lock, rf.cookie);
2594                 p->sched_class->task_woken(rq, p);
2595                 lockdep_repin_lock(&rq->lock, rf.cookie);
2596         }
2597 #endif
2598         task_rq_unlock(rq, p, &rf);
2599 }
2600 
2601 #ifdef CONFIG_PREEMPT_NOTIFIERS
2602 
2603 static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE;
2604 
2605 void preempt_notifier_inc(void)
2606 {
2607         static_key_slow_inc(&preempt_notifier_key);
2608 }
2609 EXPORT_SYMBOL_GPL(preempt_notifier_inc);
2610 
2611 void preempt_notifier_dec(void)
2612 {
2613         static_key_slow_dec(&preempt_notifier_key);
2614 }
2615 EXPORT_SYMBOL_GPL(preempt_notifier_dec);
2616 
2617 /**
2618  * preempt_notifier_register - tell me when current is being preempted & rescheduled
2619  * @notifier: notifier struct to register
2620  */
2621 void preempt_notifier_register(struct preempt_notifier *notifier)
2622 {
2623         if (!static_key_false(&preempt_notifier_key))
2624                 WARN(1, "registering preempt_notifier while notifiers disabled\n");
2625 
2626         hlist_add_head(&notifier->link, &current->preempt_notifiers);
2627 }
2628 EXPORT_SYMBOL_GPL(preempt_notifier_register);
2629 
2630 /**
2631  * preempt_notifier_unregister - no longer interested in preemption notifications
2632  * @notifier: notifier struct to unregister
2633  *
2634  * This is *not* safe to call from within a preemption notifier.
2635  */
2636 void preempt_notifier_unregister(struct preempt_notifier *notifier)
2637 {
2638         hlist_del(&notifier->link);
2639 }
2640 EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2641 
2642 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
2643 {
2644         struct preempt_notifier *notifier;
2645 
2646         hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2647                 notifier->ops->sched_in(notifier, raw_smp_processor_id());
2648 }
2649 
2650 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2651 {
2652         if (static_key_false(&preempt_notifier_key))
2653                 __fire_sched_in_preempt_notifiers(curr);
2654 }
2655 
2656 static void
2657 __fire_sched_out_preempt_notifiers(struct task_struct *curr,
2658                                    struct task_struct *next)
2659 {
2660         struct preempt_notifier *notifier;
2661 
2662         hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2663                 notifier->ops->sched_out(notifier, next);
2664 }
2665 
2666 static __always_inline void
2667 fire_sched_out_preempt_notifiers(struct task_struct *curr,
2668                                  struct task_struct *next)
2669 {
2670         if (static_key_false(&preempt_notifier_key))
2671                 __fire_sched_out_preempt_notifiers(curr, next);
2672 }
2673 
2674 #else /* !CONFIG_PREEMPT_NOTIFIERS */
2675 
2676 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2677 {
2678 }
2679 
2680 static inline void
2681 fire_sched_out_preempt_notifiers(struct task_struct *curr,
2682                                  struct task_struct *next)
2683 {
2684 }
2685 
2686 #endif /* CONFIG_PREEMPT_NOTIFIERS */
2687 
2688 /**
2689  * prepare_task_switch - prepare to switch tasks
2690  * @rq: the runqueue preparing to switch
2691  * @prev: the current task that is being switched out
2692  * @next: the task we are going to switch to.
2693  *
2694  * This is called with the rq lock held and interrupts off. It must
2695  * be paired with a subsequent finish_task_switch after the context
2696  * switch.
2697  *
2698  * prepare_task_switch sets up locking and calls architecture specific
2699  * hooks.
2700  */
2701 static inline void
2702 prepare_task_switch(struct rq *rq, struct task_struct *prev,
2703                     struct task_struct *next)
2704 {
2705         sched_info_switch(rq, prev, next);
2706         perf_event_task_sched_out(prev, next);
2707         fire_sched_out_preempt_notifiers(prev, next);
2708         prepare_lock_switch(rq, next);
2709         prepare_arch_switch(next);
2710 }
2711 
2712 /**
2713  * finish_task_switch - clean up after a task-switch
2714  * @prev: the thread we just switched away from.
2715  *
2716  * finish_task_switch must be called after the context switch, paired
2717  * with a prepare_task_switch call before the context switch.
2718  * finish_task_switch will reconcile locking set up by prepare_task_switch,
2719  * and do any other architecture-specific cleanup actions.
2720  *
2721  * Note that we may have delayed dropping an mm in context_switch(). If
2722  * so, we finish that here outside of the runqueue lock. (Doing it
2723  * with the lock held can cause deadlocks; see schedule() for
2724  * details.)
2725  *
2726  * The context switch have flipped the stack from under us and restored the
2727  * local variables which were saved when this task called schedule() in the
2728  * past. prev == current is still correct but we need to recalculate this_rq
2729  * because prev may have moved to another CPU.
2730  */
2731 static struct rq *finish_task_switch(struct task_struct *prev)
2732         __releases(rq->lock)
2733 {
2734         struct rq *rq = this_rq();
2735         struct mm_struct *mm = rq->prev_mm;
2736         long prev_state;
2737 
2738         /*
2739          * The previous task will have left us with a preempt_count of 2
2740          * because it left us after:
2741          *
2742          *      schedule()
2743          *        preempt_disable();                    // 1
2744          *        __schedule()
2745          *          raw_spin_lock_irq(&rq->lock)        // 2
2746          *
2747          * Also, see FORK_PREEMPT_COUNT.
2748          */
2749         if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
2750                       "corrupted preempt_count: %s/%d/0x%x\n",
2751                       current->comm, current->pid, preempt_count()))
2752                 preempt_count_set(FORK_PREEMPT_COUNT);
2753 
2754         rq->prev_mm = NULL;
2755 
2756         /*
2757          * A task struct has one reference for the use as "current".
2758          * If a task dies, then it sets TASK_DEAD in tsk->state and calls
2759          * schedule one last time. The schedule call will never return, and
2760          * the scheduled task must drop that reference.
2761          *
2762          * We must observe prev->state before clearing prev->on_cpu (in
2763          * finish_lock_switch), otherwise a concurrent wakeup can get prev
2764          * running on another CPU and we could rave with its RUNNING -> DEAD
2765          * transition, resulting in a double drop.
2766          */
2767         prev_state = prev->state;
2768         vtime_task_switch(prev);
2769         perf_event_task_sched_in(prev, current);
2770         finish_lock_switch(rq, prev);
2771         finish_arch_post_lock_switch();
2772 
2773         fire_sched_in_preempt_notifiers(current);
2774         if (mm)
2775                 mmdrop(mm);
2776         if (unlikely(prev_state == TASK_DEAD)) {
2777                 if (prev->sched_class->task_dead)
2778                         prev->sched_class->task_dead(prev);
2779 
2780                 /*
2781                  * Remove function-return probe instances associated with this
2782                  * task and put them back on the free list.
2783                  */
2784                 kprobe_flush_task(prev);
2785 
2786                 /* Task is done with its stack. */
2787                 put_task_stack(prev);
2788 
2789                 put_task_struct(prev);
2790         }
2791 
2792         tick_nohz_task_switch();
2793         return rq;
2794 }
2795 
2796 #ifdef CONFIG_SMP
2797 
2798 /* rq->lock is NOT held, but preemption is disabled */
2799 static void __balance_callback(struct rq *rq)
2800 {
2801         struct callback_head *head, *next;
2802         void (*func)(struct rq *rq);
2803         unsigned long flags;
2804 
2805         raw_spin_lock_irqsave(&rq->lock, flags);
2806         head = rq->balance_callback;
2807         rq->balance_callback = NULL;
2808         while (head) {
2809                 func = (void (*)(struct rq *))head->func;
2810                 next = head->next;
2811                 head->next = NULL;
2812                 head = next;
2813 
2814                 func(rq);
2815         }
2816         raw_spin_unlock_irqrestore(&rq->lock, flags);
2817 }
2818 
2819 static inline void balance_callback(struct rq *rq)
2820 {
2821         if (unlikely(rq->balance_callback))
2822                 __balance_callback(rq);
2823 }
2824 
2825 #else
2826 
2827 static inline void balance_callback(struct rq *rq)
2828 {
2829 }
2830 
2831 #endif
2832 
2833 /**
2834  * schedule_tail - first thing a freshly forked thread must call.
2835  * @prev: the thread we just switched away from.
2836  */
2837 asmlinkage __visible void schedule_tail(struct task_struct *prev)
2838         __releases(rq->lock)
2839 {
2840         struct rq *rq;
2841 
2842         /*
2843          * New tasks start with FORK_PREEMPT_COUNT, see there and
2844          * finish_task_switch() for details.
2845          *
2846          * finish_task_switch() will drop rq->lock() and lower preempt_count
2847          * and the preempt_enable() will end up enabling preemption (on
2848          * PREEMPT_COUNT kernels).
2849          */
2850 
2851         rq = finish_task_switch(prev);
2852         balance_callback(rq);
2853         preempt_enable();
2854 
2855         if (current->set_child_tid)
2856                 put_user(task_pid_vnr(current), current->set_child_tid);
2857 }
2858 
2859 /*
2860  * context_switch - switch to the new MM and the new thread's register state.
2861  */
2862 static __always_inline struct rq *
2863 context_switch(struct rq *rq, struct task_struct *prev,
2864                struct task_struct *next, struct pin_cookie cookie)
2865 {
2866         struct mm_struct *mm, *oldmm;
2867 
2868         prepare_task_switch(rq, prev, next);
2869 
2870         mm = next->mm;
2871         oldmm = prev->active_mm;
2872         /*
2873          * For paravirt, this is coupled with an exit in switch_to to
2874          * combine the page table reload and the switch backend into
2875          * one hypercall.
2876          */
2877         arch_start_context_switch(prev);
2878 
2879         if (!mm) {
2880                 next->active_mm = oldmm;
2881                 atomic_inc(&oldmm->mm_count);
2882                 enter_lazy_tlb(oldmm, next);
2883         } else
2884                 switch_mm_irqs_off(oldmm, mm, next);
2885 
2886         if (!prev->mm) {
2887                 prev->active_mm = NULL;
2888                 rq->prev_mm = oldmm;
2889         }
2890         /*
2891          * Since the runqueue lock will be released by the next
2892          * task (which is an invalid locking op but in the case
2893          * of the scheduler it's an obvious special-case), so we
2894          * do an early lockdep release here:
2895          */
2896         lockdep_unpin_lock(&rq->lock, cookie);
2897         spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2898 
2899         /* Here we just switch the register state and the stack. */
2900         switch_to(prev, next, prev);
2901         barrier();
2902 
2903         return finish_task_switch(prev);
2904 }
2905 
2906 /*
2907  * nr_running and nr_context_switches:
2908  *
2909  * externally visible scheduler statistics: current number of runnable
2910  * threads, total number of context switches performed since bootup.
2911  */
2912 unsigned long nr_running(void)
2913 {
2914         unsigned long i, sum = 0;
2915 
2916         for_each_online_cpu(i)
2917                 sum += cpu_rq(i)->nr_running;
2918 
2919         return sum;
2920 }
2921 
2922 /*
2923  * Check if only the current task is running on the cpu.
2924  *
2925  * Caution: this function does not check that the caller has disabled
2926  * preemption, thus the result might have a time-of-check-to-time-of-use
2927  * race.  The caller is responsible to use it correctly, for example:
2928  *
2929  * - from a non-preemptable section (of course)
2930  *
2931  * - from a thread that is bound to a single CPU
2932  *
2933  * - in a loop with very short iterations (e.g. a polling loop)
2934  */
2935 bool single_task_running(void)
2936 {
2937         return raw_rq()->nr_running == 1;
2938 }
2939 EXPORT_SYMBOL(single_task_running);
2940 
2941 unsigned long long nr_context_switches(void)
2942 {
2943         int i;
2944         unsigned long long sum = 0;
2945 
2946         for_each_possible_cpu(i)
2947                 sum += cpu_rq(i)->nr_switches;
2948 
2949         return sum;
2950 }
2951 
2952 unsigned long nr_iowait(void)
2953 {
2954         unsigned long i, sum = 0;
2955 
2956         for_each_possible_cpu(i)
2957                 sum += atomic_read(&cpu_rq(i)->nr_iowait);
2958 
2959         return sum;
2960 }
2961 
2962 unsigned long nr_iowait_cpu(int cpu)
2963 {
2964         struct rq *this = cpu_rq(cpu);
2965         return atomic_read(&this->nr_iowait);
2966 }
2967 
2968 void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
2969 {
2970         struct rq *rq = this_rq();
2971         *nr_waiters = atomic_read(&rq->nr_iowait);
2972         *load = rq->load.weight;
2973 }
2974 
2975 #ifdef CONFIG_SMP
2976 
2977 /*
2978  * sched_exec - execve() is a valuable balancing opportunity, because at
2979  * this point the task has the smallest effective memory and cache footprint.
2980  */
2981 void sched_exec(void)
2982 {
2983         struct task_struct *p = current;
2984         unsigned long flags;
2985         int dest_cpu;
2986 
2987         raw_spin_lock_irqsave(&p->pi_lock, flags);
2988         dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
2989         if (dest_cpu == smp_processor_id())
2990                 goto unlock;
2991 
2992         if (likely(cpu_active(dest_cpu))) {
2993                 struct migration_arg arg = { p, dest_cpu };
2994 
2995                 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2996                 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
2997                 return;
2998         }
2999 unlock:
3000         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3001 }
3002 
3003 #endif
3004 
3005 DEFINE_PER_CPU(struct kernel_stat, kstat);
3006 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
3007 
3008 EXPORT_PER_CPU_SYMBOL(kstat);
3009 EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
3010 
3011 /*
3012  * The function fair_sched_class.update_curr accesses the struct curr
3013  * and its field curr->exec_start; when called from task_sched_runtime(),
3014  * we observe a high rate of cache misses in practice.
3015  * Prefetching this data results in improved performance.
3016  */
3017 static inline void prefetch_curr_exec_start(struct task_struct *p)
3018 {
3019 #ifdef CONFIG_FAIR_GROUP_SCHED
3020         struct sched_entity *curr = (&p->se)->cfs_rq->curr;
3021 #else
3022         struct sched_entity *curr = (&task_rq(p)->cfs)->curr;
3023 #endif
3024         prefetch(curr);
3025         prefetch(&curr->exec_start);
3026 }
3027 
3028 /*
3029  * Return accounted runtime for the task.
3030  * In case the task is currently running, return the runtime plus current's
3031  * pending runtime that have not been accounted yet.
3032  */
3033 unsigned long long task_sched_runtime(struct task_struct *p)
3034 {
3035         struct rq_flags rf;
3036         struct rq *rq;
3037         u64 ns;
3038 
3039 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
3040         /*
3041          * 64-bit doesn't need locks to atomically read a 64bit value.
3042          * So we have a optimization chance when the task's delta_exec is 0.
3043          * Reading ->on_cpu is racy, but this is ok.
3044          *
3045          * If we race with it leaving cpu, we'll take a lock. So we're correct.
3046          * If we race with it entering cpu, unaccounted time is 0. This is
3047          * indistinguishable from the read occurring a few cycles earlier.
3048          * If we see ->on_cpu without ->on_rq, the task is leaving, and has
3049          * been accounted, so we're correct here as well.
3050          */
3051         if (!p->on_cpu || !task_on_rq_queued(p))
3052                 return p->se.sum_exec_runtime;
3053 #endif
3054 
3055         rq = task_rq_lock(p, &rf);
3056         /*
3057          * Must be ->curr _and_ ->on_rq.  If dequeued, we would
3058          * project cycles that may never be accounted to this
3059          * thread, breaking clock_gettime().
3060          */
3061         if (task_current(rq, p) && task_on_rq_queued(p)) {
3062                 prefetch_curr_exec_start(p);
3063                 update_rq_clock(rq);
3064                 p->sched_class->update_curr(rq);
3065         }
3066         ns = p->se.sum_exec_runtime;
3067         task_rq_unlock(rq, p, &rf);
3068 
3069         return ns;
3070 }
3071 
3072 /*
3073  * This function gets called by the timer code, with HZ frequency.
3074  * We call it with interrupts disabled.
3075  */
3076 void scheduler_tick(void)
3077 {
3078         int cpu = smp_processor_id();
3079         struct rq *rq = cpu_rq(cpu);
3080         struct task_struct *curr = rq->curr;
3081 
3082         sched_clock_tick();
3083 
3084         raw_spin_lock(&rq->lock);
3085         update_rq_clock(rq);
3086         curr->sched_class->task_tick(rq, curr, 0);
3087         cpu_load_update_active(rq);
3088         calc_global_load_tick(rq);
3089         raw_spin_unlock(&rq->lock);
3090 
3091         perf_event_task_tick();
3092 
3093 #ifdef CONFIG_SMP
3094         rq->idle_balance = idle_cpu(cpu);
3095         trigger_load_balance(rq);
3096 #endif
3097         rq_last_tick_reset(rq);
3098 }
3099 
3100 #ifdef CONFIG_NO_HZ_FULL
3101 /**
3102  * scheduler_tick_max_deferment
3103  *
3104  * Keep at least one tick per second when a single
3105  * active task is running because the scheduler doesn't
3106  * yet completely support full dynticks environment.
3107  *
3108  * This makes sure that uptime, CFS vruntime, load
3109  * balancing, etc... continue to move forward, even
3110  * with a very low granularity.
3111  *
3112  * Return: Maximum deferment in nanoseconds.
3113  */
3114 u64 scheduler_tick_max_deferment(void)
3115 {
3116         struct rq *rq = this_rq();
3117         unsigned long next, now = READ_ONCE(jiffies);
3118 
3119         next = rq->last_sched_tick + HZ;
3120 
3121         if (time_before_eq(next, now))
3122                 return 0;
3123 
3124         return jiffies_to_nsecs(next - now);
3125 }
3126 #endif
3127 
3128 #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
3129                                 defined(CONFIG_PREEMPT_TRACER))
3130 /*
3131  * If the value passed in is equal to the current preempt count
3132  * then we just disabled preemption. Start timing the latency.
3133  */
3134 static inline void preempt_latency_start(int val)
3135 {
3136         if (preempt_count() == val) {
3137                 unsigned long ip = get_lock_parent_ip();
3138 #ifdef CONFIG_DEBUG_PREEMPT
3139                 current->preempt_disable_ip = ip;
3140 #endif
3141                 trace_preempt_off(CALLER_ADDR0, ip);
3142         }
3143 }
3144 
3145 void preempt_count_add(int val)
3146 {
3147 #ifdef CONFIG_DEBUG_PREEMPT
3148         /*
3149          * Underflow?
3150          */
3151         if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
3152                 return;
3153 #endif
3154         __preempt_count_add(val);
3155 #ifdef CONFIG_DEBUG_PREEMPT
3156         /*
3157          * Spinlock count overflowing soon?
3158          */
3159         DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
3160                                 PREEMPT_MASK - 10);
3161 #endif
3162         preempt_latency_start(val);
3163 }
3164 EXPORT_SYMBOL(preempt_count_add);
3165 NOKPROBE_SYMBOL(preempt_count_add);
3166 
3167 /*
3168  * If the value passed in equals to the current preempt count
3169  * then we just enabled preemption. Stop timing the latency.
3170  */
3171 static inline void preempt_latency_stop(int val)
3172 {
3173         if (preempt_count() == val)
3174                 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
3175 }
3176 
3177 void preempt_count_sub(int val)
3178 {
3179 #ifdef CONFIG_DEBUG_PREEMPT
3180         /*
3181          * Underflow?
3182          */
3183         if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
3184                 return;
3185         /*
3186          * Is the spinlock portion underflowing?
3187          */
3188         if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
3189                         !(preempt_count() & PREEMPT_MASK)))
3190                 return;
3191 #endif
3192 
3193         preempt_latency_stop(val);
3194         __preempt_count_sub(val);
3195 }
3196 EXPORT_SYMBOL(preempt_count_sub);
3197 NOKPROBE_SYMBOL(preempt_count_sub);
3198 
3199 #else
3200 static inline void preempt_latency_start(int val) { }
3201 static inline void preempt_latency_stop(int val) { }
3202 #endif
3203 
3204 /*
3205  * Print scheduling while atomic bug:
3206  */
3207 static noinline void __schedule_bug(struct task_struct *prev)
3208 {
3209         /* Save this before calling printk(), since that will clobber it */
3210         unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
3211 
3212         if (oops_in_progress)
3213                 return;
3214 
3215         printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
3216                 prev->comm, prev->pid, preempt_count());
3217 
3218         debug_show_held_locks(prev);
3219         print_modules();
3220         if (irqs_disabled())
3221                 print_irqtrace_events(prev);
3222         if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
3223             && in_atomic_preempt_off()) {
3224                 pr_err("Preemption disabled at:");
3225                 print_ip_sym(preempt_disable_ip);
3226                 pr_cont("\n");
3227         }
3228         if (panic_on_warn)
3229                 panic("scheduling while atomic\n");
3230 
3231         dump_stack();
3232         add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
3233 }
3234 
3235 /*
3236  * Various schedule()-time debugging checks and statistics:
3237  */
3238 static inline void schedule_debug(struct task_struct *prev)
3239 {
3240 #ifdef CONFIG_SCHED_STACK_END_CHECK
3241         if (task_stack_end_corrupted(prev))
3242                 panic("corrupted stack end detected inside scheduler\n");
3243 #endif
3244 
3245         if (unlikely(in_atomic_preempt_off())) {
3246                 __schedule_bug(prev);
3247                 preempt_count_set(PREEMPT_DISABLED);
3248         }
3249         rcu_sleep_check();
3250 
3251         profile_hit(SCHED_PROFILING, __builtin_return_address(0));
3252 
3253         schedstat_inc(this_rq()->sched_count);
3254 }
3255 
3256 /*
3257  * Pick up the highest-prio task:
3258  */
3259 static inline struct task_struct *
3260 pick_next_task(struct rq *rq, struct task_struct *prev, struct pin_cookie cookie)
3261 {
3262         const struct sched_class *class = &fair_sched_class;
3263         struct task_struct *p;
3264 
3265         /*
3266          * Optimization: we know that if all tasks are in
3267          * the fair class we can call that function directly:
3268          */
3269         if (likely(prev->sched_class == class &&
3270                    rq->nr_running == rq->cfs.h_nr_running)) {
3271                 p = fair_sched_class.pick_next_task(rq, prev, cookie);
3272                 if (unlikely(p == RETRY_TASK))
3273                         goto again;
3274 
3275                 /* assumes fair_sched_class->next == idle_sched_class */
3276                 if (unlikely(!p))
3277                         p = idle_sched_class.pick_next_task(rq, prev, cookie);
3278 
3279                 return p;
3280         }
3281 
3282 again:
3283         for_each_class(class) {
3284                 p = class->pick_next_task(rq, prev, cookie);
3285                 if (p) {
3286                         if (unlikely(p == RETRY_TASK))
3287                                 goto again;
3288                         return p;
3289                 }
3290         }
3291 
3292         BUG(); /* the idle class will always have a runnable task */
3293 }
3294 
3295 /*
3296  * __schedule() is the main scheduler function.
3297  *
3298  * The main means of driving the scheduler and thus entering this function are:
3299  *
3300  *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
3301  *
3302  *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
3303  *      paths. For example, see arch/x86/entry_64.S.
3304  *
3305  *      To drive preemption between tasks, the scheduler sets the flag in timer
3306  *      interrupt handler scheduler_tick().
3307  *
3308  *   3. Wakeups don't really cause entry into schedule(). They add a
3309  *      task to the run-queue and that's it.
3310  *
3311  *      Now, if the new task added to the run-queue preempts the current
3312  *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
3313  *      called on the nearest possible occasion:
3314  *
3315  *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
3316  *
3317  *         - in syscall or exception context, at the next outmost
3318  *           preempt_enable(). (this might be as soon as the wake_up()'s
3319  *           spin_unlock()!)
3320  *
3321  *         - in IRQ context, return from interrupt-handler to
3322  *           preemptible context
3323  *
3324  *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
3325  *         then at the next:
3326  *
3327  *          - cond_resched() call
3328  *          - explicit schedule() call
3329  *          - return from syscall or exception to user-space
3330  *          - return from interrupt-handler to user-space
3331  *
3332  * WARNING: must be called with preemption disabled!
3333  */
3334 static void __sched notrace __schedule(bool preempt)
3335 {
3336         struct task_struct *prev, *next;
3337         unsigned long *switch_count;
3338         struct pin_cookie cookie;
3339         struct rq *rq;
3340         int cpu;
3341 
3342         cpu = smp_processor_id();
3343         rq = cpu_rq(cpu);
3344         prev = rq->curr;
3345 
3346         schedule_debug(prev);
3347 
3348         if (sched_feat(HRTICK))
3349                 hrtick_clear(rq);
3350 
3351         local_irq_disable();
3352         rcu_note_context_switch();
3353 
3354         /*
3355          * Make sure that signal_pending_state()->signal_pending() below
3356          * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
3357          * done by the caller to avoid the race with signal_wake_up().
3358          */
3359         smp_mb__before_spinlock();
3360         raw_spin_lock(&rq->lock);
3361         cookie = lockdep_pin_lock(&rq->lock);
3362 
3363         rq->clock_skip_update <<= 1; /* promote REQ to ACT */
3364 
3365         switch_count = &prev->nivcsw;
3366         if (!preempt && prev->state) {
3367                 if (unlikely(signal_pending_state(prev->state, prev))) {
3368                         prev->state = TASK_RUNNING;
3369                 } else {
3370                         deactivate_task(rq, prev, DEQUEUE_SLEEP);
3371                         prev->on_rq = 0;
3372 
3373                         /*
3374                          * If a worker went to sleep, notify and ask workqueue
3375                          * whether it wants to wake up a task to maintain
3376                          * concurrency.
3377                          */
3378                         if (prev->flags & PF_WQ_WORKER) {
3379                                 struct task_struct *to_wakeup;
3380 
3381                                 to_wakeup = wq_worker_sleeping(prev);
3382                                 if (to_wakeup)
3383                                         try_to_wake_up_local(to_wakeup, cookie);
3384                         }
3385                 }
3386                 switch_count = &prev->nvcsw;
3387         }
3388 
3389         if (task_on_rq_queued(prev))
3390                 update_rq_clock(rq);
3391 
3392         next = pick_next_task(rq, prev, cookie);
3393         clear_tsk_need_resched(prev);
3394         clear_preempt_need_resched();
3395         rq->clock_skip_update = 0;
3396 
3397         if (likely(prev != next)) {
3398                 rq->nr_switches++;
3399                 rq->curr = next;
3400                 ++*switch_count;
3401 
3402                 trace_sched_switch(preempt, prev, next);
3403                 rq = context_switch(rq, prev, next, cookie); /* unlocks the rq */
3404         } else {
3405                 lockdep_unpin_lock(&rq->lock, cookie);
3406                 raw_spin_unlock_irq(&rq->lock);
3407         }
3408 
3409         balance_callback(rq);
3410 }
3411 
3412 void __noreturn do_task_dead(void)
3413 {
3414         /*
3415          * The setting of TASK_RUNNING by try_to_wake_up() may be delayed
3416          * when the following two conditions become true.
3417          *   - There is race condition of mmap_sem (It is acquired by
3418          *     exit_mm()), and
3419          *   - SMI occurs before setting TASK_RUNINNG.
3420          *     (or hypervisor of virtual machine switches to other guest)
3421          *  As a result, we may become TASK_RUNNING after becoming TASK_DEAD
3422          *
3423          * To avoid it, we have to wait for releasing tsk->pi_lock which
3424          * is held by try_to_wake_up()
3425          */
3426         smp_mb();
3427         raw_spin_unlock_wait(&current->pi_lock);
3428 
3429         /* causes final put_task_struct in finish_task_switch(). */
3430         __set_current_state(TASK_DEAD);
3431         current->flags |= PF_NOFREEZE;  /* tell freezer to ignore us */
3432         __schedule(false);
3433         BUG();
3434         /* Avoid "noreturn function does return".  */
3435         for (;;)
3436                 cpu_relax();    /* For when BUG is null */
3437 }
3438 
3439 static inline void sched_submit_work(struct task_struct *tsk)
3440 {
3441         if (!tsk->state || tsk_is_pi_blocked(tsk))
3442                 return;
3443         /*
3444          * If we are going to sleep and we have plugged IO queued,
3445          * make sure to submit it to avoid deadlocks.
3446          */
3447         if (blk_needs_flush_plug(tsk))
3448                 blk_schedule_flush_plug(tsk);
3449 }
3450 
3451 asmlinkage __visible void __sched schedule(void)
3452 {
3453         struct task_struct *tsk = current;
3454 
3455         sched_submit_work(tsk);
3456         do {
3457                 preempt_disable();
3458                 __schedule(false);
3459                 sched_preempt_enable_no_resched();
3460         } while (need_resched());
3461 }
3462 EXPORT_SYMBOL(schedule);
3463 
3464 #ifdef CONFIG_CONTEXT_TRACKING
3465 asmlinkage __visible void __sched schedule_user(void)
3466 {
3467         /*
3468          * If we come here after a random call to set_need_resched(),
3469          * or we have been woken up remotely but the IPI has not yet arrived,
3470          * we haven't yet exited the RCU idle mode. Do it here manually until
3471          * we find a better solution.
3472          *
3473          * NB: There are buggy callers of this function.  Ideally we
3474          * should warn if prev_state != CONTEXT_USER, but that will trigger
3475          * too frequently to make sense yet.
3476          */
3477         enum ctx_state prev_state = exception_enter();
3478         schedule();
3479         exception_exit(prev_state);
3480 }
3481 #endif
3482 
3483 /**
3484  * schedule_preempt_disabled - called with preemption disabled
3485  *
3486  * Returns with preemption disabled. Note: preempt_count must be 1
3487  */
3488 void __sched schedule_preempt_disabled(void)
3489 {
3490         sched_preempt_enable_no_resched();
3491         schedule();
3492         preempt_disable();
3493 }
3494 
3495 static void __sched notrace preempt_schedule_common(void)
3496 {
3497         do {
3498                 /*
3499                  * Because the function tracer can trace preempt_count_sub()
3500                  * and it also uses preempt_enable/disable_notrace(), if
3501                  * NEED_RESCHED is set, the preempt_enable_notrace() called
3502                  * by the function tracer will call this function again and
3503                  * cause infinite recursion.
3504                  *
3505                  * Preemption must be disabled here before the function
3506                  * tracer can trace. Break up preempt_disable() into two
3507                  * calls. One to disable preemption without fear of being
3508                  * traced. The other to still record the preemption latency,
3509                  * which can also be traced by the function tracer.
3510                  */
3511                 preempt_disable_notrace();
3512                 preempt_latency_start(1);
3513                 __schedule(true);
3514                 preempt_latency_stop(1);
3515                 preempt_enable_no_resched_notrace();
3516 
3517                 /*
3518                  * Check again in case we missed a preemption opportunity
3519                  * between schedule and now.
3520                  */
3521         } while (need_resched());
3522 }
3523 
3524 #ifdef CONFIG_PREEMPT
3525 /*
3526  * this is the entry point to schedule() from in-kernel preemption
3527  * off of preempt_enable. Kernel preemptions off return from interrupt
3528  * occur there and call schedule directly.
3529  */
3530 asmlinkage __visible void __sched notrace preempt_schedule(void)
3531 {
3532         /*
3533          * If there is a non-zero preempt_count or interrupts are disabled,
3534          * we do not want to preempt the current task. Just return..
3535          */
3536         if (likely(!preemptible()))
3537                 return;
3538 
3539         preempt_schedule_common();
3540 }
3541 NOKPROBE_SYMBOL(preempt_schedule);
3542 EXPORT_SYMBOL(preempt_schedule);
3543 
3544 /**
3545  * preempt_schedule_notrace - preempt_schedule called by tracing
3546  *
3547  * The tracing infrastructure uses preempt_enable_notrace to prevent
3548  * recursion and tracing preempt enabling caused by the tracing
3549  * infrastructure itself. But as tracing can happen in areas coming
3550  * from userspace or just about to enter userspace, a preempt enable
3551  * can occur before user_exit() is called. This will cause the scheduler
3552  * to be called when the system is still in usermode.
3553  *
3554  * To prevent this, the preempt_enable_notrace will use this function
3555  * instead of preempt_schedule() to exit user context if needed before
3556  * calling the scheduler.
3557  */
3558 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
3559 {
3560         enum ctx_state prev_ctx;
3561 
3562         if (likely(!preemptible()))
3563                 return;
3564 
3565         do {
3566                 /*
3567                  * Because the function tracer can trace preempt_count_sub()
3568                  * and it also uses preempt_enable/disable_notrace(), if
3569                  * NEED_RESCHED is set, the preempt_enable_notrace() called
3570                  * by the function tracer will call this function again and
3571                  * cause infinite recursion.
3572                  *
3573                  * Preemption must be disabled here before the function
3574                  * tracer can trace. Break up preempt_disable() into two
3575                  * calls. One to disable preemption without fear of being
3576                  * traced. The other to still record the preemption latency,
3577                  * which can also be traced by the function tracer.
3578                  */
3579                 preempt_disable_notrace();
3580                 preempt_latency_start(1);
3581                 /*
3582                  * Needs preempt disabled in case user_exit() is traced
3583                  * and the tracer calls preempt_enable_notrace() causing
3584                  * an infinite recursion.
3585                  */
3586                 prev_ctx = exception_enter();
3587                 __schedule(true);
3588                 exception_exit(prev_ctx);
3589 
3590                 preempt_latency_stop(1);
3591                 preempt_enable_no_resched_notrace();
3592         } while (need_resched());
3593 }
3594 EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
3595 
3596 #endif /* CONFIG_PREEMPT */
3597 
3598 /*
3599  * this is the entry point to schedule() from kernel preemption
3600  * off of irq context.
3601  * Note, that this is called and return with irqs disabled. This will
3602  * protect us against recursive calling from irq.
3603  */
3604 asmlinkage __visible void __sched preempt_schedule_irq(void)
3605 {
3606         enum ctx_state prev_state;
3607 
3608         /* Catch callers which need to be fixed */
3609         BUG_ON(preempt_count() || !irqs_disabled());
3610 
3611         prev_state = exception_enter();
3612 
3613         do {
3614                 preempt_disable();
3615                 local_irq_enable();
3616                 __schedule(true);
3617                 local_irq_disable();
3618                 sched_preempt_enable_no_resched();
3619         } while (need_resched());
3620 
3621         exception_exit(prev_state);
3622 }
3623 
3624 int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
3625                           void *key)
3626 {
3627         return try_to_wake_up(curr->private, mode, wake_flags);
3628 }
3629 EXPORT_SYMBOL(default_wake_function);
3630 
3631 #ifdef CONFIG_RT_MUTEXES
3632 
3633 /*
3634  * rt_mutex_setprio - set the current priority of a task
3635  * @p: task
3636  * @prio: prio value (kernel-internal form)
3637  *
3638  * This function changes the 'effective' priority of a task. It does
3639  * not touch ->normal_prio like __setscheduler().
3640  *
3641  * Used by the rt_mutex code to implement priority inheritance
3642  * logic. Call site only calls if the priority of the task changed.
3643  */
3644 void rt_mutex_setprio(struct task_struct *p, int prio)
3645 {
3646         int oldprio, queued, running, queue_flag = DEQUEUE_SAVE | DEQUEUE_MOVE;
3647         const struct sched_class *prev_class;
3648         struct rq_flags rf;
3649         struct rq *rq;
3650 
3651         BUG_ON(prio > MAX_PRIO);
3652 
3653         rq = __task_rq_lock(p, &rf);
3654 
3655         /*
3656          * Idle task boosting is a nono in general. There is one
3657          * exception, when PREEMPT_RT and NOHZ is active:
3658          *
3659          * The idle task calls get_next_timer_interrupt() and holds
3660          * the timer wheel base->lock on the CPU and another CPU wants
3661          * to access the timer (probably to cancel it). We can safely
3662          * ignore the boosting request, as the idle CPU runs this code
3663          * with interrupts disabled and will complete the lock
3664          * protected section without being interrupted. So there is no
3665          * real need to boost.
3666          */
3667         if (unlikely(p == rq->idle)) {
3668                 WARN_ON(p != rq->curr);
3669                 WARN_ON(p->pi_blocked_on);
3670                 goto out_unlock;
3671         }
3672 
3673         trace_sched_pi_setprio(p, prio);
3674         oldprio = p->prio;
3675 
3676         if (oldprio == prio)
3677                 queue_flag &= ~DEQUEUE_MOVE;
3678 
3679         prev_class = p->sched_class;
3680         queued = task_on_rq_queued(p);
3681         running = task_current(rq, p);
3682         if (queued)
3683                 dequeue_task(rq, p, queue_flag);
3684         if (running)
3685                 put_prev_task(rq, p);
3686 
3687         /*
3688          * Boosting condition are:
3689          * 1. -rt task is running and holds mutex A
3690          *      --> -dl task blocks on mutex A
3691          *
3692          * 2. -dl task is running and holds mutex A
3693          *      --> -dl task blocks on mutex A and could preempt the
3694          *          running task
3695          */
3696         if (dl_prio(prio)) {
3697                 struct task_struct *pi_task = rt_mutex_get_top_task(p);
3698                 if (!dl_prio(p->normal_prio) ||
3699                     (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
3700                         p->dl.dl_boosted = 1;
3701                         queue_flag |= ENQUEUE_REPLENISH;
3702                 } else
3703                         p->dl.dl_boosted = 0;
3704                 p->sched_class = &dl_sched_class;
3705         } else if (rt_prio(prio)) {
3706                 if (dl_prio(oldprio))
3707                         p->dl.dl_boosted = 0;
3708                 if (oldprio < prio)
3709                         queue_flag |= ENQUEUE_HEAD;
3710                 p->sched_class = &rt_sched_class;
3711         } else {
3712                 if (dl_prio(oldprio))
3713                         p->dl.dl_boosted = 0;
3714                 if (rt_prio(oldprio))
3715                         p->rt.timeout = 0;
3716                 p->sched_class = &fair_sched_class;
3717         }
3718 
3719         p->prio = prio;
3720 
3721         if (queued)
3722                 enqueue_task(rq, p, queue_flag);
3723         if (running)
3724                 set_curr_task(rq, p);
3725 
3726         check_class_changed(rq, p, prev_class, oldprio);
3727 out_unlock:
3728         preempt_disable(); /* avoid rq from going away on us */
3729         __task_rq_unlock(rq, &rf);
3730 
3731         balance_callback(rq);
3732         preempt_enable();
3733 }
3734 #endif
3735 
3736 void set_user_nice(struct task_struct *p, long nice)
3737 {
3738         bool queued, running;
3739         int old_prio, delta;
3740         struct rq_flags rf;
3741         struct rq *rq;
3742 
3743         if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
3744                 return;
3745         /*
3746          * We have to be careful, if called from sys_setpriority(),
3747          * the task might be in the middle of scheduling on another CPU.
3748          */
3749         rq = task_rq_lock(p, &rf);
3750         /*
3751          * The RT priorities are set via sched_setscheduler(), but we still
3752          * allow the 'normal' nice value to be set - but as expected
3753          * it wont have any effect on scheduling until the task is
3754          * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
3755          */
3756         if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
3757                 p->static_prio = NICE_TO_PRIO(nice);
3758                 goto out_unlock;
3759         }
3760         queued = task_on_rq_queued(p);
3761         running = task_current(rq, p);
3762         if (queued)
3763                 dequeue_task(rq, p, DEQUEUE_SAVE);
3764         if (running)
3765                 put_prev_task(rq, p);
3766 
3767         p->static_prio = NICE_TO_PRIO(nice);
3768         set_load_weight(p);
3769         old_prio = p->prio;
3770         p->prio = effective_prio(p);
3771         delta = p->prio - old_prio;
3772 
3773         if (queued) {
3774                 enqueue_task(rq, p, ENQUEUE_RESTORE);
3775                 /*
3776                  * If the task increased its priority or is running and
3777                  * lowered its priority, then reschedule its CPU:
3778                  */
3779                 if (delta < 0 || (delta > 0 && task_running(rq, p)))
3780                         resched_curr(rq);
3781         }
3782         if (running)
3783                 set_curr_task(rq, p);
3784 out_unlock:
3785         task_rq_unlock(rq, p, &rf);
3786 }
3787 EXPORT_SYMBOL(set_user_nice);
3788 
3789 /*
3790  * can_nice - check if a task can reduce its nice value
3791  * @p: task
3792  * @nice: nice value
3793  */
3794 int can_nice(const struct task_struct *p, const int nice)
3795 {
3796         /* convert nice value [19,-20] to rlimit style value [1,40] */
3797         int nice_rlim = nice_to_rlimit(nice);
3798 
3799         return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
3800                 capable(CAP_SYS_NICE));
3801 }
3802 
3803 #ifdef __ARCH_WANT_SYS_NICE
3804 
3805 /*
3806  * sys_nice - change the priority of the current process.
3807  * @increment: priority increment
3808  *
3809  * sys_setpriority is a more generic, but much slower function that
3810  * does similar things.
3811  */
3812 SYSCALL_DEFINE1(nice, int, increment)
3813 {
3814         long nice, retval;
3815 
3816         /*
3817          * Setpriority might change our priority at the same moment.
3818          * We don't have to worry. Conceptually one call occurs first
3819          * and we have a single winner.
3820          */
3821         increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
3822         nice = task_nice(current) + increment;
3823 
3824         nice = clamp_val(nice, MIN_NICE, MAX_NICE);
3825         if (increment < 0 && !can_nice(current, nice))
3826                 return -EPERM;
3827 
3828         retval = security_task_setnice(current, nice);
3829         if (retval)
3830                 return retval;
3831 
3832         set_user_nice(current, nice);
3833         return 0;
3834 }
3835 
3836 #endif
3837 
3838 /**
3839  * task_prio - return the priority value of a given task.
3840  * @p: the task in question.
3841  *
3842  * Return: The priority value as seen by users in /proc.
3843  * RT tasks are offset by -200. Normal tasks are centered
3844  * around 0, value goes from -16 to +15.
3845  */
3846 int task_prio(const struct task_struct *p)
3847 {
3848         return p->prio - MAX_RT_PRIO;
3849 }
3850 
3851 /**
3852  * idle_cpu - is a given cpu idle currently?
3853  * @cpu: the processor in question.
3854  *
3855  * Return: 1 if the CPU is currently idle. 0 otherwise.
3856  */
3857 int idle_cpu(int cpu)
3858 {
3859         struct rq *rq = cpu_rq(cpu);
3860 
3861         if (rq->curr != rq->idle)
3862                 return 0;
3863 
3864         if (rq->nr_running)
3865                 return 0;
3866 
3867 #ifdef CONFIG_SMP
3868         if (!llist_empty(&rq->wake_list))
3869                 return 0;
3870 #endif
3871 
3872         return 1;
3873 }
3874 
3875 /**
3876  * idle_task - return the idle task for a given cpu.
3877  * @cpu: the processor in question.
3878  *
3879  * Return: The idle task for the cpu @cpu.
3880  */
3881 struct task_struct *idle_task(int cpu)
3882 {
3883         return cpu_rq(cpu)->idle;
3884 }
3885 
3886 /**
3887  * find_process_by_pid - find a process with a matching PID value.
3888  * @pid: the pid in question.
3889  *
3890  * The task of @pid, if found. %NULL otherwise.
3891  */
3892 static struct task_struct *find_process_by_pid(pid_t pid)
3893 {
3894         return pid ? find_task_by_vpid(pid) : current;
3895 }
3896 
3897 /*
3898  * This function initializes the sched_dl_entity of a newly becoming
3899  * SCHED_DEADLINE task.
3900  *
3901  * Only the static values are considered here, the actual runtime and the
3902  * absolute deadline will be properly calculated when the task is enqueued
3903  * for the first time with its new policy.
3904  */
3905 static void
3906 __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
3907 {
3908         struct sched_dl_entity *dl_se = &p->dl;
3909 
3910         dl_se->dl_runtime = attr->sched_runtime;
3911         dl_se->dl_deadline = attr->sched_deadline;
3912         dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
3913         dl_se->flags = attr->sched_flags;
3914         dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
3915 
3916         /*
3917          * Changing the parameters of a task is 'tricky' and we're not doing
3918          * the correct thing -- also see task_dead_dl() and switched_from_dl().
3919          *
3920          * What we SHOULD do is delay the bandwidth release until the 0-lag
3921          * point. This would include retaining the task_struct until that time
3922          * and change dl_overflow() to not immediately decrement the current
3923          * amount.
3924          *
3925          * Instead we retain the current runtime/deadline and let the new
3926          * parameters take effect after the current reservation period lapses.
3927          * This is safe (albeit pessimistic) because the 0-lag point is always
3928          * before the current scheduling deadline.
3929          *
3930          * We can still have temporary overloads because we do not delay the
3931          * change in bandwidth until that time; so admission control is
3932          * not on the safe side. It does however guarantee tasks will never
3933          * consume more than promised.
3934          */
3935 }
3936 
3937 /*
3938  * sched_setparam() passes in -1 for its policy, to let the functions
3939  * it calls know not to change it.
3940  */
3941 #define SETPARAM_POLICY -1
3942 
3943 static void __setscheduler_params(struct task_struct *p,
3944                 const struct sched_attr *attr)
3945 {
3946         int policy = attr->sched_policy;
3947 
3948         if (policy == SETPARAM_POLICY)
3949                 policy = p->policy;
3950 
3951         p->policy = policy;
3952 
3953         if (dl_policy(policy))
3954                 __setparam_dl(p, attr);
3955         else if (fair_policy(policy))
3956                 p->static_prio = NICE_TO_PRIO(attr->sched_nice);
3957 
3958         /*
3959          * __sched_setscheduler() ensures attr->sched_priority == 0 when
3960          * !rt_policy. Always setting this ensures that things like
3961          * getparam()/getattr() don't report silly values for !rt tasks.
3962          */
3963         p->rt_priority = attr->sched_priority;
3964         p->normal_prio = normal_prio(p);
3965         set_load_weight(p);
3966 }
3967 
3968 /* Actually do priority change: must hold pi & rq lock. */
3969 static void __setscheduler(struct rq *rq, struct task_struct *p,
3970                            const struct sched_attr *attr, bool keep_boost)
3971 {
3972         __setscheduler_params(p, attr);
3973 
3974         /*
3975          * Keep a potential priority boosting if called from
3976          * sched_setscheduler().
3977          */
3978         if (keep_boost)
3979                 p->prio = rt_mutex_get_effective_prio(p, normal_prio(p));
3980         else
3981                 p->prio = normal_prio(p);
3982 
3983         if (dl_prio(p->prio))
3984                 p->sched_class = &dl_sched_class;
3985         else if (rt_prio(p->prio))
3986                 p->sched_class = &rt_sched_class;
3987         else
3988                 p->sched_class = &fair_sched_class;
3989 }
3990 
3991 static void
3992 __getparam_dl(struct task_struct *p, struct sched_attr *attr)
3993 {
3994         struct sched_dl_entity *dl_se = &p->dl;
3995 
3996         attr->sched_priority = p->rt_priority;
3997         attr->sched_runtime = dl_se->dl_runtime;
3998         attr->sched_deadline = dl_se->dl_deadline;
3999         attr->sched_period = dl_se->dl_period;
4000         attr->sched_flags = dl_se->flags;
4001 }
4002 
4003 /*
4004  * This function validates the new parameters of a -deadline task.
4005  * We ask for the deadline not being zero, and greater or equal
4006  * than the runtime, as well as the period of being zero or
4007  * greater than deadline. Furthermore, we have to be sure that
4008  * user parameters are above the internal resolution of 1us (we
4009  * check sched_runtime only since it is always the smaller one) and
4010  * below 2^63 ns (we have to check both sched_deadline and
4011  * sched_period, as the latter can be zero).
4012  */
4013 static bool
4014 __checkparam_dl(const struct sched_attr *attr)
4015 {
4016         /* deadline != 0 */
4017         if (attr->sched_deadline == 0)
4018                 return false;
4019 
4020         /*
4021          * Since we truncate DL_SCALE bits, make sure we're at least
4022          * that big.
4023          */
4024         if (attr->sched_runtime < (1ULL << DL_SCALE))
4025                 return false;
4026 
4027         /*
4028          * Since we use the MSB for wrap-around and sign issues, make
4029          * sure it's not set (mind that period can be equal to zero).
4030          */
4031         if (attr->sched_deadline & (1ULL << 63) ||
4032             attr->sched_period & (1ULL << 63))
4033                 return false;
4034 
4035         /* runtime <= deadline <= period (if period != 0) */
4036         if ((attr->sched_period != 0 &&
4037              attr->sched_period < attr->sched_deadline) ||
4038             attr->sched_deadline < attr->sched_runtime)
4039                 return false;
4040 
4041         return true;
4042 }
4043 
4044 /*
4045  * check the target process has a UID that matches the current process's
4046  */
4047 static bool check_same_owner(struct task_struct *p)
4048 {
4049         const struct cred *cred = current_cred(), *pcred;
4050         bool match;
4051 
4052         rcu_read_lock();
4053         pcred = __task_cred(p);
4054         match = (uid_eq(cred->euid, pcred->euid) ||
4055                  uid_eq(cred->euid, pcred->uid));
4056         rcu_read_unlock();
4057         return match;
4058 }
4059 
4060 static bool dl_param_changed(struct task_struct *p,
4061                 const struct sched_attr *attr)
4062 {
4063         struct sched_dl_entity *dl_se = &p->dl;
4064 
4065         if (dl_se->dl_runtime != attr->sched_runtime ||
4066                 dl_se->dl_deadline != attr->sched_deadline ||
4067                 dl_se->dl_period != attr->sched_period ||
4068                 dl_se->flags != attr->sched_flags)
4069                 return true;
4070 
4071         return false;
4072 }
4073 
4074 static int __sched_setscheduler(struct task_struct *p,
4075                                 const struct sched_attr *attr,
4076                                 bool user, bool pi)
4077 {
4078         int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
4079                       MAX_RT_PRIO - 1 - attr->sched_priority;
4080         int retval, oldprio, oldpolicy = -1, queued, running;
4081         int new_effective_prio, policy = attr->sched_policy;
4082         const struct sched_class *prev_class;
4083         struct rq_flags rf;
4084         int reset_on_fork;
4085         int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
4086         struct rq *rq;
4087 
4088         /* may grab non-irq protected spin_locks */
4089         BUG_ON(in_interrupt());
4090 recheck:
4091         /* double check policy once rq lock held */
4092         if (policy < 0) {
4093                 reset_on_fork = p->sched_reset_on_fork;
4094                 policy = oldpolicy = p->policy;
4095         } else {
4096                 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
4097 
4098                 if (!valid_policy(policy))
4099                         return -EINVAL;
4100         }
4101 
4102         if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
4103                 return -EINVAL;
4104 
4105         /*
4106          * Valid priorities for SCHED_FIFO and SCHED_RR are
4107          * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
4108          * SCHED_BATCH and SCHED_IDLE is 0.
4109          */
4110         if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
4111             (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
4112                 return -EINVAL;
4113         if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
4114             (rt_policy(policy) != (attr->sched_priority != 0)))
4115                 return -EINVAL;
4116 
4117         /*
4118          * Allow unprivileged RT tasks to decrease priority:
4119          */
4120         if (user && !capable(CAP_SYS_NICE)) {
4121                 if (fair_policy(policy)) {
4122                         if (attr->sched_nice < task_nice(p) &&
4123                             !can_nice(p, attr->sched_nice))
4124                                 return -EPERM;
4125                 }
4126 
4127                 if (rt_policy(policy)) {
4128                         unsigned long rlim_rtprio =
4129                                         task_rlimit(p, RLIMIT_RTPRIO);
4130 
4131                         /* can't set/change the rt policy */
4132                         if (policy != p->policy && !rlim_rtprio)
4133                                 return -EPERM;
4134 
4135                         /* can't increase priority */
4136                         if (attr->sched_priority > p->rt_priority &&
4137                             attr->sched_priority > rlim_rtprio)
4138                                 return -EPERM;
4139                 }
4140 
4141                  /*
4142                   * Can't set/change SCHED_DEADLINE policy at all for now
4143                   * (safest behavior); in the future we would like to allow
4144                   * unprivileged DL tasks to increase their relative deadline
4145                   * or reduce their runtime (both ways reducing utilization)
4146                   */
4147                 if (dl_policy(policy))
4148                         return -EPERM;
4149 
4150                 /*
4151                  * Treat SCHED_IDLE as nice 20. Only allow a switch to
4152                  * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
4153                  */
4154                 if (idle_policy(p->policy) && !idle_policy(policy)) {
4155                         if (!can_nice(p, task_nice(p)))
4156                                 return -EPERM;
4157                 }
4158 
4159                 /* can't change other user's priorities */
4160                 if (!check_same_owner(p))
4161                         return -EPERM;
4162 
4163                 /* Normal users shall not reset the sched_reset_on_fork flag */
4164                 if (p->sched_reset_on_fork && !reset_on_fork)
4165                         return -EPERM;
4166         }
4167 
4168         if (user) {
4169                 retval = security_task_setscheduler(p);
4170                 if (retval)
4171                         return retval;
4172         }
4173 
4174         /*
4175          * make sure no PI-waiters arrive (or leave) while we are
4176          * changing the priority of the task:
4177          *
4178          * To be able to change p->policy safely, the appropriate
4179          * runqueue lock must be held.
4180          */
4181         rq = task_rq_lock(p, &rf);
4182 
4183         /*
4184          * Changing the policy of the stop threads its a very bad idea
4185          */
4186         if (p == rq->stop) {
4187                 task_rq_unlock(rq, p, &rf);
4188                 return -EINVAL;
4189         }
4190 
4191         /*
4192          * If not changing anything there's no need to proceed further,
4193          * but store a possible modification of reset_on_fork.
4194          */
4195         if (unlikely(policy == p->policy)) {
4196                 if (fair_policy(policy) && attr->sched_nice != task_nice(p))
4197                         goto change;
4198                 if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
4199                         goto change;
4200                 if (dl_policy(policy) && dl_param_changed(p, attr))
4201                         goto change;
4202 
4203                 p->sched_reset_on_fork = reset_on_fork;
4204                 task_rq_unlock(rq, p, &rf);
4205                 return 0;
4206         }
4207 change:
4208 
4209         if (user) {
4210 #ifdef CONFIG_RT_GROUP_SCHED
4211                 /*
4212                  * Do not allow realtime tasks into groups that have no runtime
4213                  * assigned.
4214                  */
4215                 if (rt_bandwidth_enabled() && rt_policy(policy) &&
4216                                 task_group(p)->rt_bandwidth.rt_runtime == 0 &&
4217                                 !task_group_is_autogroup(task_group(p))) {
4218                         task_rq_unlock(rq, p, &rf);
4219                         return -EPERM;
4220                 }
4221 #endif
4222 #ifdef CONFIG_SMP
4223                 if (dl_bandwidth_enabled() && dl_policy(policy)) {
4224                         cpumask_t *span = rq->rd->span;
4225 
4226                         /*
4227                          * Don't allow tasks with an affinity mask smaller than
4228                          * the entire root_domain to become SCHED_DEADLINE. We
4229                          * will also fail if there's no bandwidth available.
4230                          */
4231                         if (!cpumask_subset(span, &p->cpus_allowed) ||
4232                             rq->rd->dl_bw.bw == 0) {
4233                                 task_rq_unlock(rq, p, &rf);
4234                                 return -EPERM;
4235                         }
4236                 }
4237 #endif
4238         }
4239 
4240         /* recheck policy now with rq lock held */
4241         if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
4242                 policy = oldpolicy = -1;
4243                 task_rq_unlock(rq, p, &rf);
4244                 goto recheck;
4245         }
4246 
4247         /*
4248          * If setscheduling to SCHED_DEADLINE (or changing the parameters
4249          * of a SCHED_DEADLINE task) we need to check if enough bandwidth
4250          * is available.
4251          */
4252         if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
4253                 task_rq_unlock(rq, p, &rf);
4254                 return -EBUSY;
4255         }
4256 
4257         p->sched_reset_on_fork = reset_on_fork;
4258         oldprio = p->prio;
4259 
4260         if (pi) {
4261                 /*
4262                  * Take priority boosted tasks into account. If the new
4263                  * effective priority is unchanged, we just store the new
4264                  * normal parameters and do not touch the scheduler class and
4265                  * the runqueue. This will be done when the task deboost
4266                  * itself.
4267                  */
4268                 new_effective_prio = rt_mutex_get_effective_prio(p, newprio);
4269                 if (new_effective_prio == oldprio)
4270                         queue_flags &= ~DEQUEUE_MOVE;
4271         }
4272 
4273         queued = task_on_rq_queued(p);
4274         running = task_current(rq, p);
4275         if (queued)
4276                 dequeue_task(rq, p, queue_flags);
4277         if (running)
4278                 put_prev_task(rq, p);
4279 
4280         prev_class = p->sched_class;
4281         __setscheduler(rq, p, attr, pi);
4282 
4283         if (queued) {
4284                 /*
4285                  * We enqueue to tail when the priority of a task is
4286                  * increased (user space view).
4287                  */
4288                 if (oldprio < p->prio)
4289                         queue_flags |= ENQUEUE_HEAD;
4290 
4291                 enqueue_task(rq, p, queue_flags);
4292         }
4293         if (running)
4294                 set_curr_task(rq, p);
4295 
4296         check_class_changed(rq, p, prev_class, oldprio);
4297         preempt_disable(); /* avoid rq from going away on us */
4298         task_rq_unlock(rq, p, &rf);
4299 
4300         if (pi)
4301                 rt_mutex_adjust_pi(p);
4302 
4303         /*
4304          * Run balance callbacks after we've adjusted the PI chain.
4305          */
4306         balance_callback(rq);
4307         preempt_enable();
4308 
4309         return 0;
4310 }
4311 
4312 static int _sched_setscheduler(struct task_struct *p, int policy,
4313                                const struct sched_param *param, bool check)
4314 {
4315         struct sched_attr attr = {
4316                 .sched_policy   = policy,
4317                 .sched_priority = param->sched_priority,
4318                 .sched_nice     = PRIO_TO_NICE(p->static_prio),
4319         };
4320 
4321         /* Fixup the legacy SCHED_RESET_ON_FORK hack. */
4322         if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
4323                 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
4324                 policy &= ~SCHED_RESET_ON_FORK;
4325                 attr.sched_policy = policy;
4326         }
4327 
4328         return __sched_setscheduler(p, &attr, check, true);
4329 }
4330 /**
4331  * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
4332  * @p: the task in question.
4333  * @policy: new policy.
4334  * @param: structure containing the new RT priority.
4335  *
4336  * Return: 0 on success. An error code otherwise.
4337  *
4338  * NOTE that the task may be already dead.
4339  */
4340 int sched_setscheduler(struct task_struct *p, int policy,
4341                        const struct sched_param *param)
4342 {
4343         return _sched_setscheduler(p, policy, param, true);
4344 }
4345 EXPORT_SYMBOL_GPL(sched_setscheduler);
4346 
4347 int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
4348 {
4349         return __sched_setscheduler(p, attr, true, true);
4350 }
4351 EXPORT_SYMBOL_GPL(sched_setattr);
4352 
4353 /**
4354  * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
4355  * @p: the task in question.
4356  * @policy: new policy.
4357  * @param: structure containing the new RT priority.
4358  *
4359  * Just like sched_setscheduler, only don't bother checking if the
4360  * current context has permission.  For example, this is needed in
4361  * stop_machine(): we create temporary high priority worker threads,
4362  * but our caller might not have that capability.
4363  *
4364  * Return: 0 on success. An error code otherwise.
4365  */
4366 int sched_setscheduler_nocheck(struct task_struct *p, int policy,
4367                                const struct sched_param *param)
4368 {
4369         return _sched_setscheduler(p, policy, param, false);
4370 }
4371 EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck);
4372 
4373 static int
4374 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
4375 {
4376         struct sched_param lparam;
4377         struct task_struct *p;
4378         int retval;
4379 
4380         if (!param || pid < 0)
4381                 return -EINVAL;
4382         if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
4383                 return -EFAULT;
4384 
4385         rcu_read_lock();
4386         retval = -ESRCH;
4387         p = find_process_by_pid(pid);
4388         if (p != NULL)
4389                 retval = sched_setscheduler(p, policy, &lparam);
4390         rcu_read_unlock();
4391 
4392         return retval;
4393 }
4394 
4395 /*
4396  * Mimics kernel/events/core.c perf_copy_attr().
4397  */
4398 static int sched_copy_attr(struct sched_attr __user *uattr,
4399                            struct sched_attr *attr)
4400 {
4401         u32 size;
4402         int ret;
4403 
4404         if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
4405                 return -EFAULT;
4406 
4407         /*
4408          * zero the full structure, so that a short copy will be nice.
4409          */
4410         memset(attr, 0, sizeof(*attr));
4411 
4412         ret = get_user(size, &uattr->size);
4413         if (ret)
4414                 return ret;
4415 
4416         if (size > PAGE_SIZE)   /* silly large */
4417                 goto err_size;
4418 
4419         if (!size)              /* abi compat */
4420                 size = SCHED_ATTR_SIZE_VER0;
4421 
4422         if (size < SCHED_ATTR_SIZE_VER0)
4423                 goto err_size;
4424 
4425         /*
4426          * If we're handed a bigger struct than we know of,
4427          * ensure all the unknown bits are 0 - i.e. new
4428          * user-space does not rely on any kernel feature
4429          * extensions we dont know about yet.
4430          */
4431         if (size > sizeof(*attr)) {
4432                 unsigned char __user *addr;
4433                 unsigned char __user *end;
4434                 unsigned char val;
4435 
4436                 addr = (void __user *)uattr + sizeof(*attr);
4437                 end  = (void __user *)uattr + size;
4438 
4439                 for (; addr < end; addr++) {
4440                         ret = get_user(val, addr);
4441                         if (ret)
4442                                 return ret;
4443                         if (val)
4444                                 goto err_size;
4445                 }
4446                 size = sizeof(*attr);
4447         }
4448 
4449         ret = copy_from_user(attr, uattr, size);
4450         if (ret)
4451                 return -EFAULT;
4452 
4453         /*
4454          * XXX: do we want to be lenient like existing syscalls; or do we want
4455          * to be strict and return an error on out-of-bounds values?
4456          */
4457         attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
4458 
4459         return 0;
4460 
4461 err_size:
4462         put_user(sizeof(*attr), &uattr->size);
4463         return -E2BIG;
4464 }
4465 
4466 /**
4467  * sys_sched_setscheduler - set/change the scheduler policy and RT priority
4468  * @pid: the pid in question.
4469  * @policy: new policy.
4470  * @param: structure containing the new RT priority.
4471  *
4472  * Return: 0 on success. An error code otherwise.
4473  */
4474 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
4475                 struct sched_param __user *, param)
4476 {
4477         /* negative values for policy are not valid */
4478         if (policy < 0)
4479                 return -EINVAL;
4480 
4481         return do_sched_setscheduler(pid, policy, param);
4482 }
4483 
4484 /**
4485  * sys_sched_setparam - set/change the RT priority of a thread
4486  * @pid: the pid in question.
4487  * @param: structure containing the new RT priority.
4488  *
4489  * Return: 0 on success. An error code otherwise.
4490  */
4491 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
4492 {
4493         return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
4494 }
4495 
4496 /**
4497  * sys_sched_setattr - same as above, but with extended sched_attr
4498  * @pid: the pid in question.
4499  * @uattr: structure containing the extended parameters.
4500  * @flags: for future extension.
4501  */
4502 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
4503                                unsigned int, flags)
4504 {
4505         struct sched_attr attr;
4506         struct task_struct *p;
4507         int retval;
4508 
4509         if (!uattr || pid < 0 || flags)
4510                 return -EINVAL;
4511 
4512         retval = sched_copy_attr(uattr, &attr);
4513         if (retval)
4514                 return retval;
4515 
4516         if ((int)attr.sched_policy < 0)
4517                 return -EINVAL;
4518 
4519         rcu_read_lock();
4520         retval = -ESRCH;
4521         p = find_process_by_pid(pid);
4522         if (p != NULL)
4523                 retval = sched_setattr(p, &attr);
4524         rcu_read_unlock();
4525 
4526         return retval;
4527 }
4528 
4529 /**
4530  * sys_sched_getscheduler - get the policy (scheduling class) of a thread
4531  * @pid: the pid in question.
4532  *
4533  * Return: On success, the policy of the thread. Otherwise, a negative error
4534  * code.
4535  */
4536 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
4537 {
4538         struct task_struct *p;
4539         int retval;
4540 
4541         if (pid < 0)
4542                 return -EINVAL;
4543 
4544         retval = -ESRCH;
4545         rcu_read_lock();
4546         p = find_process_by_pid(pid);
4547         if (p) {
4548                 retval = security_task_getscheduler(p);
4549                 if (!retval)
4550                         retval = p->policy
4551                                 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
4552         }
4553         rcu_read_unlock();
4554         return retval;
4555 }
4556 
4557 /**
4558  * sys_sched_getparam - get the RT priority of a thread
4559  * @pid: the pid in question.
4560  * @param: structure containing the RT priority.
4561  *
4562  * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
4563  * code.
4564  */
4565 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
4566 {
4567         struct sched_param lp = { .sched_priority = 0 };
4568         struct task_struct *p;
4569         int retval;
4570 
4571         if (!param || pid < 0)
4572                 return -EINVAL;
4573 
4574         rcu_read_lock();
4575         p = find_process_by_pid(pid);
4576         retval = -ESRCH;
4577         if (!p)
4578                 goto out_unlock;
4579 
4580         retval = security_task_getscheduler(p);
4581         if (retval)
4582                 goto out_unlock;
4583 
4584         if (task_has_rt_policy(p))
4585                 lp.sched_priority = p->rt_priority;
4586         rcu_read_unlock();
4587 
4588         /*
4589          * This one might sleep, we cannot do it with a spinlock held ...
4590          */
4591         retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
4592 
4593         return retval;
4594 
4595 out_unlock:
4596         rcu_read_unlock();
4597         return retval;
4598 }
4599 
4600 static int sched_read_attr(struct sched_attr __user *uattr,
4601                            struct sched_attr *attr,
4602                            unsigned int usize)
4603 {
4604         int ret;
4605 
4606         if (!access_ok(VERIFY_WRITE, uattr, usize))
4607                 return -EFAULT;
4608 
4609         /*
4610          * If we're handed a smaller struct than we know of,
4611          * ensure all the unknown bits are 0 - i.e. old
4612          * user-space does not get uncomplete information.
4613          */
4614         if (usize < sizeof(*attr)) {
4615                 unsigned char *addr;
4616                 unsigned char *end;
4617 
4618                 addr = (void *)attr + usize;
4619                 end  = (void *)attr + sizeof(*attr);
4620 
4621                 for (; addr < end; addr++) {
4622                         if (*addr)
4623                                 return -EFBIG;
4624                 }
4625 
4626                 attr->size = usize;
4627         }
4628 
4629         ret = copy_to_user(uattr, attr, attr->size);
4630         if (ret)
4631                 return -EFAULT;
4632 
4633         return 0;
4634 }
4635 
4636 /**
4637  * sys_sched_getattr - similar to sched_getparam, but with sched_attr
4638  * @pid: the pid in question.
4639  * @uattr: structure containing the extended parameters.
4640  * @size: sizeof(attr) for fwd/bwd comp.
4641  * @flags: for future extension.
4642  */
4643 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
4644                 unsigned int, size, unsigned int, flags)
4645 {
4646         struct sched_attr attr = {
4647                 .size = sizeof(struct sched_attr),
4648         };
4649         struct task_struct *p;
4650         int retval;
4651 
4652         if (!uattr || pid < 0 || size > PAGE_SIZE ||
4653             size < SCHED_ATTR_SIZE_VER0 || flags)
4654                 return -EINVAL;
4655 
4656         rcu_read_lock();
4657         p = find_process_by_pid(pid);
4658         retval = -ESRCH;
4659         if (!p)
4660                 goto out_unlock;
4661 
4662         retval = security_task_getscheduler(p);
4663         if (retval)
4664                 goto out_unlock;
4665 
4666         attr.sched_policy = p->policy;
4667         if (p->sched_reset_on_fork)
4668                 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
4669         if (task_has_dl_policy(p))
4670                 __getparam_dl(p, &attr);
4671         else if (task_has_rt_policy(p))
4672                 attr.sched_priority = p->rt_priority;
4673         else
4674                 attr.sched_nice = task_nice(p);
4675 
4676         rcu_read_unlock();
4677 
4678         retval = sched_read_attr(uattr, &attr, size);
4679         return retval;
4680 
4681 out_unlock:
4682         rcu_read_unlock();
4683         return retval;
4684 }
4685 
4686 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
4687 {
4688         cpumask_var_t cpus_allowed, new_mask;
4689         struct task_struct *p;
4690         int retval;
4691 
4692         rcu_read_lock();
4693 
4694         p = find_process_by_pid(pid);
4695         if (!p) {
4696                 rcu_read_unlock();
4697                 return -ESRCH;
4698         }
4699 
4700         /* Prevent p going away */
4701         get_task_struct(p);
4702         rcu_read_unlock();
4703 
4704         if (p->flags & PF_NO_SETAFFINITY) {
4705                 retval = -EINVAL;
4706                 goto out_put_task;
4707         }
4708         if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
4709                 retval = -ENOMEM;
4710                 goto out_put_task;
4711         }
4712         if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
4713                 retval = -ENOMEM;
4714                 goto out_free_cpus_allowed;
4715         }
4716         retval = -EPERM;
4717         if (!check_same_owner(p)) {
4718                 rcu_read_lock();
4719                 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
4720                         rcu_read_unlock();
4721                         goto out_free_new_mask;
4722                 }
4723                 rcu_read_unlock();
4724         }
4725 
4726         retval = security_task_setscheduler(p);
4727         if (retval)
4728                 goto out_free_new_mask;
4729 
4730 
4731         cpuset_cpus_allowed(p, cpus_allowed);
4732         cpumask_and(new_mask, in_mask, cpus_allowed);
4733 
4734         /*
4735          * Since bandwidth control happens on root_domain basis,
4736          * if admission test is enabled, we only admit -deadline
4737          * tasks allowed to run on all the CPUs in the task's
4738          * root_domain.
4739          */
4740 #ifdef CONFIG_SMP
4741         if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
4742                 rcu_read_lock();
4743                 if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
4744                         retval = -EBUSY;
4745                         rcu_read_unlock();
4746                         goto out_free_new_mask;
4747                 }
4748                 rcu_read_unlock();
4749         }
4750 #endif
4751 again:
4752         retval = __set_cpus_allowed_ptr(p, new_mask, true);
4753 
4754         if (!retval) {
4755                 cpuset_cpus_allowed(p, cpus_allowed);
4756                 if (!cpumask_subset(new_mask, cpus_allowed)) {
4757                         /*
4758                          * We must have raced with a concurrent cpuset
4759                          * update. Just reset the cpus_allowed to the
4760                          * cpuset's cpus_allowed
4761                          */
4762                         cpumask_copy(new_mask, cpus_allowed);
4763                         goto again;
4764                 }
4765         }
4766 out_free_new_mask:
4767         free_cpumask_var(new_mask);
4768 out_free_cpus_allowed:
4769         free_cpumask_var(cpus_allowed);
4770 out_put_task:
4771         put_task_struct(p);
4772         return retval;
4773 }
4774 
4775 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
4776                              struct cpumask *new_mask)
4777 {
4778         if (len < cpumask_size())
4779                 cpumask_clear(new_mask);
4780         else if (len > cpumask_size())
4781                 len = cpumask_size();
4782 
4783         return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
4784 }
4785 
4786 /**
4787  * sys_sched_setaffinity - set the cpu affinity of a process
4788  * @pid: pid of the process
4789  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4790  * @user_mask_ptr: user-space pointer to the new cpu mask
4791  *
4792  * Return: 0 on success. An error code otherwise.
4793  */
4794 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
4795                 unsigned long __user *, user_mask_ptr)
4796 {
4797         cpumask_var_t new_mask;
4798         int retval;
4799 
4800         if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
4801                 return -ENOMEM;
4802 
4803         retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
4804         if (retval == 0)
4805                 retval = sched_setaffinity(pid, new_mask);
4806         free_cpumask_var(new_mask);
4807         return retval;
4808 }
4809 
4810 long sched_getaffinity(pid_t pid, struct cpumask *mask)
4811 {
4812         struct task_struct *p;
4813         unsigned long flags;
4814         int retval;
4815 
4816         rcu_read_lock();
4817 
4818         retval = -ESRCH;
4819         p = find_process_by_pid(pid);
4820         if (!p)
4821                 goto out_unlock;
4822 
4823         retval = security_task_getscheduler(p);
4824         if (retval)
4825                 goto out_unlock;
4826 
4827         raw_spin_lock_irqsave(&p->pi_lock, flags);
4828         cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
4829         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4830 
4831 out_unlock:
4832         rcu_read_unlock();
4833 
4834         return retval;
4835 }
4836 
4837 /**
4838  * sys_sched_getaffinity - get the cpu affinity of a process
4839  * @pid: pid of the process
4840  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4841  * @user_mask_ptr: user-space pointer to hold the current cpu mask
4842  *
4843  * Return: size of CPU mask copied to user_mask_ptr on success. An
4844  * error code otherwise.
4845  */
4846 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
4847                 unsigned long __user *, user_mask_ptr)
4848 {
4849         int ret;
4850         cpumask_var_t mask;
4851 
4852         if ((len * BITS_PER_BYTE) < nr_cpu_ids)
4853                 return -EINVAL;
4854         if (len & (sizeof(unsigned long)-1))
4855                 return -EINVAL;
4856 
4857         if (!alloc_cpumask_var(&mask, GFP_KERNEL))
4858                 return -ENOMEM;
4859 
4860         ret = sched_getaffinity(pid, mask);
4861         if (ret == 0) {
4862                 size_t retlen = min_t(size_t, len, cpumask_size());
4863 
4864                 if (copy_to_user(user_mask_ptr, mask, retlen))
4865                         ret = -EFAULT;
4866                 else
4867                         ret = retlen;
4868         }
4869         free_cpumask_var(mask);
4870 
4871         return ret;
4872 }
4873 
4874 /**
4875  * sys_sched_yield - yield the current processor to other threads.
4876  *
4877  * This function yields the current CPU to other tasks. If there are no
4878  * other threads running on this CPU then this function will return.
4879  *
4880  * Return: 0.
4881  */
4882 SYSCALL_DEFINE0(sched_yield)
4883 {
4884         struct rq *rq = this_rq_lock();
4885 
4886         schedstat_inc(rq->yld_count);
4887         current->sched_class->yield_task(rq);
4888 
4889         /*
4890          * Since we are going to call schedule() anyway, there's
4891          * no need to preempt or enable interrupts:
4892          */
4893         __release(rq->lock);
4894         spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
4895         do_raw_spin_unlock(&rq->lock);
4896         sched_preempt_enable_no_resched();
4897 
4898         schedule();
4899 
4900         return 0;
4901 }
4902 
4903 #ifndef CONFIG_PREEMPT
4904 int __sched _cond_resched(void)
4905 {
4906         if (should_resched(0)) {
4907                 preempt_schedule_common();
4908                 return 1;
4909         }
4910         return 0;
4911 }
4912 EXPORT_SYMBOL(_cond_resched);
4913 #endif
4914 
4915 /*
4916  * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
4917  * call schedule, and on return reacquire the lock.
4918  *
4919  * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
4920  * operations here to prevent schedule() from being called twice (once via
4921  * spin_unlock(), once by hand).
4922  */
4923 int __cond_resched_lock(spinlock_t *lock)
4924 {
4925         int resched = should_resched(PREEMPT_LOCK_OFFSET);
4926         int ret = 0;
4927 
4928         lockdep_assert_held(lock);
4929 
4930         if (spin_needbreak(lock) || resched) {
4931                 spin_unlock(lock);
4932                 if (resched)
4933                         preempt_schedule_common();
4934                 else
4935                         cpu_relax();
4936                 ret = 1;
4937                 spin_lock(lock);
4938         }
4939         return ret;
4940 }
4941 EXPORT_SYMBOL(__cond_resched_lock);
4942 
4943 int __sched __cond_resched_softirq(void)
4944 {
4945         BUG_ON(!in_softirq());
4946 
4947         if (should_resched(SOFTIRQ_DISABLE_OFFSET)) {
4948                 local_bh_enable();
4949                 preempt_schedule_common();
4950                 local_bh_disable();
4951                 return 1;
4952         }
4953         return 0;
4954 }
4955 EXPORT_SYMBOL(__cond_resched_softirq);
4956 
4957 /**
4958  * yield - yield the current processor to other threads.
4959  *
4960  * Do not ever use this function, there's a 99% chance you're doing it wrong.
4961  *
4962  * The scheduler is at all times free to pick the calling task as the most
4963  * eligible task to run, if removing the yield() call from your code breaks
4964  * it, its already broken.
4965  *
4966  * Typical broken usage is:
4967  *
4968  * while (!event)
4969  *      yield();
4970  *
4971  * where one assumes that yield() will let 'the other' process run that will
4972  * make event true. If the current task is a SCHED_FIFO task that will never
4973  * happen. Never use yield() as a progress guarantee!!
4974  *
4975  * If you want to use yield() to wait for something, use wait_event().
4976  * If you want to use yield() to be 'nice' for others, use cond_resched().
4977  * If you still want to use yield(), do not!
4978  */
4979 void __sched yield(void)
4980 {
4981         set_current_state(TASK_RUNNING);
4982         sys_sched_yield();
4983 }
4984 EXPORT_SYMBOL(yield);
4985 
4986 /**
4987  * yield_to - yield the current processor to another thread in
4988  * your thread group, or accelerate that thread toward the
4989  * processor it's on.
4990  * @p: target task
4991  * @preempt: whether task preemption is allowed or not
4992  *
4993  * It's the caller's job to ensure that the target task struct
4994  * can't go away on us before we can do any checks.
4995  *
4996  * Return:
4997  *      true (>0) if we indeed boosted the target task.
4998  *      false (0) if we failed to boost the target.
4999  *      -ESRCH if there's no task to yield to.
5000  */
5001 int __sched yield_to(struct task_struct *p, bool preempt)
5002 {
5003         struct task_struct *curr = current;
5004         struct rq *rq, *p_rq;
5005         unsigned long flags;
5006         int yielded = 0;
5007 
5008         local_irq_save(flags);
5009         rq = this_rq();
5010 
5011 again:
5012         p_rq = task_rq(p);
5013         /*
5014          * If we're the only runnable task on the rq and target rq also
5015          * has only one task, there's absolutely no point in yielding.
5016          */
5017         if (rq->nr_running == 1 && p_rq->nr_running == 1) {
5018                 yielded = -ESRCH;
5019                 goto out_irq;
5020         }
5021 
5022         double_rq_lock(rq, p_rq);
5023         if (task_rq(p) != p_rq) {
5024                 double_rq_unlock(rq, p_rq);
5025                 goto again;
5026         }
5027 
5028         if (!curr->sched_class->yield_to_task)
5029                 goto out_unlock;
5030 
5031         if (curr->sched_class != p->sched_class)
5032                 goto out_unlock;
5033 
5034         if (task_running(p_rq, p) || p->state)
5035                 goto out_unlock;
5036 
5037         yielded = curr->sched_class->yield_to_task(rq, p, preempt);
5038         if (yielded) {
5039                 schedstat_inc(rq->yld_count);
5040                 /*
5041                  * Make p's CPU reschedule; pick_next_entity takes care of
5042                  * fairness.
5043                  */
5044                 if (preempt && rq != p_rq)
5045                         resched_curr(p_rq);
5046         }
5047 
5048 out_unlock:
5049         double_rq_unlock(rq, p_rq);
5050 out_irq:
5051         local_irq_restore(flags);
5052 
5053         if (yielded > 0)
5054                 schedule();
5055 
5056         return yielded;
5057 }
5058 EXPORT_SYMBOL_GPL(yield_to);
5059 
5060 /*
5061  * This task is about to go to sleep on IO. Increment rq->nr_iowait so
5062  * that process accounting knows that this is a task in IO wait state.
5063  */
5064 long __sched io_schedule_timeout(long timeout)
5065 {
5066         int old_iowait = current->in_iowait;
5067         struct rq *rq;
5068         long ret;
5069 
5070         current->in_iowait = 1;
5071         blk_schedule_flush_plug(current);
5072 
5073         delayacct_blkio_start();
5074         rq = raw_rq();
5075         atomic_inc(&rq->nr_iowait);
5076         ret = schedule_timeout(timeout);
5077         current->in_iowait = old_iowait;
5078         atomic_dec(&rq->nr_iowait);
5079         delayacct_blkio_end();
5080 
5081         return ret;
5082 }
5083 EXPORT_SYMBOL(io_schedule_timeout);
5084 
5085 /**
5086  * sys_sched_get_priority_max - return maximum RT priority.
5087  * @policy: scheduling class.
5088  *
5089  * Return: On success, this syscall returns the maximum
5090  * rt_priority that can be used by a given scheduling class.
5091  * On failure, a negative error code is returned.
5092  */
5093 SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
5094 {
5095         int ret = -EINVAL;
5096 
5097         switch (policy) {
5098         case SCHED_FIFO:
5099         case SCHED_RR:
5100                 ret = MAX_USER_RT_PRIO-1;
5101                 break;
5102         case SCHED_DEADLINE:
5103         case SCHED_NORMAL:
5104         case SCHED_BATCH:
5105         case SCHED_IDLE:
5106                 ret = 0;
5107                 break;
5108         }
5109         return ret;
5110 }
5111 
5112 /**
5113  * sys_sched_get_priority_min - return minimum RT priority.
5114  * @policy: scheduling class.
5115  *
5116  * Return: On success, this syscall returns the minimum
5117  * rt_priority that can be used by a given scheduling class.
5118  * On failure, a negative error code is returned.
5119  */
5120 SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
5121 {
5122         int ret = -EINVAL;
5123 
5124         switch (policy) {
5125         case SCHED_FIFO:
5126         case SCHED_RR:
5127                 ret = 1;
5128                 break;
5129         case SCHED_DEADLINE:
5130         case SCHED_NORMAL:
5131         case SCHED_BATCH:
5132         case SCHED_IDLE:
5133                 ret = 0;
5134         }
5135         return ret;
5136 }
5137 
5138 /**
5139  * sys_sched_rr_get_interval - return the default timeslice of a process.
5140  * @pid: pid of the process.
5141  * @interval: userspace pointer to the timeslice value.
5142  *
5143  * this syscall writes the default timeslice value of a given process
5144  * into the user-space timespec buffer. A value of '' means infinity.
5145  *
5146  * Return: On success, 0 and the timeslice is in @interval. Otherwise,
5147  * an error code.
5148  */
5149 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
5150                 struct timespec __user *, interval)
5151 {
5152         struct task_struct *p;
5153         unsigned int time_slice;
5154         struct rq_flags rf;
5155         struct timespec t;
5156         struct rq *rq;
5157         int retval;
5158 
5159         if (pid < 0)
5160                 return -EINVAL;
5161 
5162         retval = -ESRCH;
5163         rcu_read_lock();
5164         p = find_process_by_pid(pid);
5165         if (!p)
5166                 goto out_unlock;
5167 
5168         retval = security_task_getscheduler(p);
5169         if (retval)
5170                 goto out_unlock;
5171 
5172         rq = task_rq_lock(p, &rf);
5173         time_slice = 0;
5174         if (p->sched_class->get_rr_interval)
5175                 time_slice = p->sched_class->get_rr_interval(rq, p);
5176         task_rq_unlock(rq, p, &rf);
5177 
5178         rcu_read_unlock();
5179         jiffies_to_timespec(time_slice, &t);
5180         retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
5181         return retval;
5182 
5183 out_unlock:
5184         rcu_read_unlock();
5185         return retval;
5186 }
5187 
5188 static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5189 
5190 void sched_show_task(struct task_struct *p)
5191 {
5192         unsigned long free = 0;
5193         int ppid;
5194         unsigned long state = p->state;
5195 
5196         if (!try_get_task_stack(p))
5197                 return;
5198         if (state)
5199                 state = __ffs(state) + 1;
5200         printk(KERN_INFO "%-15.15s %c", p->comm,
5201                 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5202         if (state == TASK_RUNNING)
5203                 printk(KERN_CONT "  running task    ");
5204 #ifdef CONFIG_DEBUG_STACK_USAGE
5205         free = stack_not_used(p);
5206 #endif
5207         ppid = 0;
5208         rcu_read_lock();
5209         if (pid_alive(p))
5210                 ppid = task_pid_nr(rcu_dereference(p->real_parent));
5211         rcu_read_unlock();
5212         printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
5213                 task_pid_nr(p), ppid,
5214                 (unsigned long)task_thread_info(p)->flags);
5215 
5216         print_worker_info(KERN_INFO, p);
5217         show_stack(p, NULL);
5218         put_task_stack(p);
5219 }
5220 
5221 void show_state_filter(unsigned long state_filter)
5222 {
5223         struct task_struct *g, *p;
5224 
5225 #if BITS_PER_LONG == 32
5226         printk(KERN_INFO
5227                 "  task                PC stack   pid father\n");
5228 #else
5229         printk(KERN_INFO
5230                 "  task                        PC stack   pid father\n");
5231 #endif
5232         rcu_read_lock();
5233         for_each_process_thread(g, p) {
5234                 /*
5235                  * reset the NMI-timeout, listing all files on a slow
5236                  * console might take a lot of time:
5237                  * Also, reset softlockup watchdogs on all CPUs, because
5238                  * another CPU might be blocked waiting for us to process
5239                  * an IPI.
5240                  */
5241                 touch_nmi_watchdog();
5242                 touch_all_softlockup_watchdogs();
5243                 if (!state_filter || (p->state & state_filter))
5244                         sched_show_task(p);
5245         }
5246 
5247 #ifdef CONFIG_SCHED_DEBUG
5248         if (!state_filter)
5249                 sysrq_sched_debug_show();
5250 #endif
5251         rcu_read_unlock();
5252         /*
5253          * Only show locks if all tasks are dumped:
5254          */
5255         if (!state_filter)
5256                 debug_show_all_locks();
5257 }
5258 
5259 void init_idle_bootup_task(struct task_struct *idle)
5260 {
5261         idle->sched_class = &idle_sched_class;
5262 }
5263 
5264 /**
5265  * init_idle - set up an idle thread for a given CPU
5266  * @idle: task in question
5267  * @cpu: cpu the idle task belongs to
5268  *
5269  * NOTE: this function does not set the idle thread's NEED_RESCHED
5270  * flag, to make booting more robust.
5271  */
5272 void init_idle(struct task_struct *idle, int cpu)
5273 {
5274         struct rq *rq = cpu_rq(cpu);
5275         unsigned long flags;
5276 
5277         raw_spin_lock_irqsave(&idle->pi_lock, flags);
5278         raw_spin_lock(&rq->lock);
5279 
5280         __sched_fork(0, idle);
5281         idle->state = TASK_RUNNING;
5282         idle->se.exec_start = sched_clock();
5283         idle->flags |= PF_IDLE;
5284 
5285         kasan_unpoison_task_stack(idle);
5286 
5287 #ifdef CONFIG_SMP
5288         /*
5289          * Its possible that init_idle() gets called multiple times on a task,
5290          * in that case do_set_cpus_allowed() will not do the right thing.
5291          *
5292          * And since this is boot we can forgo the serialization.
5293          */
5294         set_cpus_allowed_common(idle, cpumask_of(cpu));
5295 #endif
5296         /*
5297          * We're having a chicken and egg problem, even though we are
5298          * holding rq->lock, the cpu isn't yet set to this cpu so the
5299          * lockdep check in task_group() will fail.
5300          *
5301          * Similar case to sched_fork(). / Alternatively we could
5302          * use task_rq_lock() here and obtain the other rq->lock.
5303          *
5304          * Silence PROVE_RCU
5305          */
5306         rcu_read_lock();
5307         __set_task_cpu(idle, cpu);
5308         rcu_read_unlock();
5309 
5310         rq->curr = rq->idle = idle;
5311         idle->on_rq = TASK_ON_RQ_QUEUED;
5312 #ifdef CONFIG_SMP
5313         idle->on_cpu = 1;
5314 #endif
5315         raw_spin_unlock(&rq->lock);
5316         raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
5317 
5318         /* Set the preempt count _outside_ the spinlocks! */
5319         init_idle_preempt_count(idle, cpu);
5320 
5321         /*
5322          * The idle tasks have their own, simple scheduling class:
5323          */
5324         idle->sched_class = &idle_sched_class;
5325         ftrace_graph_init_idle_task(idle, cpu);
5326         vtime_init_idle(idle, cpu);
5327 #ifdef CONFIG_SMP
5328         sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
5329 #endif
5330 }
5331 
5332 int cpuset_cpumask_can_shrink(const struct cpumask *cur,
5333                               const struct cpumask *trial)
5334 {
5335         int ret = 1, trial_cpus;
5336         struct dl_bw *cur_dl_b;
5337         unsigned long flags;
5338 
5339         if (!cpumask_weight(cur))
5340                 return ret;
5341 
5342         rcu_read_lock_sched();
5343         cur_dl_b = dl_bw_of(cpumask_any(cur));
5344         trial_cpus = cpumask_weight(trial);
5345 
5346         raw_spin_lock_irqsave(&cur_dl_b->lock, flags);
5347         if (cur_dl_b->bw != -1 &&
5348             cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw)
5349                 ret = 0;
5350         raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags);
5351         rcu_read_unlock_sched();
5352 
5353         return ret;
5354 }
5355 
5356 int task_can_attach(struct task_struct *p,
5357                     const struct cpumask *cs_cpus_allowed)
5358 {
5359         int ret = 0;
5360 
5361         /*
5362          * Kthreads which disallow setaffinity shouldn't be moved
5363          * to a new cpuset; we don't want to change their cpu
5364          * affinity and isolating such threads by their set of
5365          * allowed nodes is unnecessary.  Thus, cpusets are not
5366          * applicable for such threads.  This prevents checking for
5367          * success of set_cpus_allowed_ptr() on all attached tasks
5368          * before cpus_allowed may be changed.
5369          */
5370         if (p->flags & PF_NO_SETAFFINITY) {
5371                 ret = -EINVAL;
5372                 goto out;
5373         }
5374 
5375 #ifdef CONFIG_SMP
5376         if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
5377                                               cs_cpus_allowed)) {
5378                 unsigned int dest_cpu = cpumask_any_and(cpu_active_mask,
5379                                                         cs_cpus_allowed);
5380                 struct dl_bw *dl_b;
5381                 bool overflow;
5382                 int cpus;
5383                 unsigned long flags;
5384 
5385                 rcu_read_lock_sched();
5386                 dl_b = dl_bw_of(dest_cpu);
5387                 raw_spin_lock_irqsave(&dl_b->lock, flags);
5388                 cpus = dl_bw_cpus(dest_cpu);
5389                 overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw);
5390                 if (overflow)
5391                         ret = -EBUSY;
5392                 else {
5393                         /*
5394                          * We reserve space for this task in the destination
5395                          * root_domain, as we can't fail after this point.
5396                          * We will free resources in the source root_domain
5397                          * later on (see set_cpus_allowed_dl()).
5398                          */
5399                         __dl_add(dl_b, p->dl.dl_bw);
5400                 }
5401                 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
5402                 rcu_read_unlock_sched();
5403 
5404         }
5405 #endif
5406 out:
5407         return ret;
5408 }
5409 
5410 #ifdef CONFIG_SMP
5411 
5412 static bool sched_smp_initialized __read_mostly;
5413 
5414 #ifdef CONFIG_NUMA_BALANCING
5415 /* Migrate current task p to target_cpu */
5416 int migrate_task_to(struct task_struct *p, int target_cpu)
5417 {
5418         struct migration_arg arg = { p, target_cpu };
5419         int curr_cpu = task_cpu(p);
5420 
5421         if (curr_cpu == target_cpu)
5422                 return 0;
5423 
5424         if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
5425                 return -EINVAL;
5426 
5427         /* TODO: This is not properly updating schedstats */
5428 
5429         trace_sched_move_numa(p, curr_cpu, target_cpu);
5430         return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
5431 }
5432 
5433 /*
5434  * Requeue a task on a given node and accurately track the number of NUMA
5435  * tasks on the runqueues
5436  */
5437 void sched_setnuma(struct task_struct *p, int nid)
5438 {
5439         bool queued, running;
5440         struct rq_flags rf;
5441         struct rq *rq;
5442 
5443         rq = task_rq_lock(p, &rf);
5444         queued = task_on_rq_queued(p);
5445         running = task_current(rq, p);
5446 
5447         if (queued)
5448                 dequeue_task(rq, p, DEQUEUE_SAVE);
5449         if (running)
5450                 put_prev_task(rq, p);
5451 
5452         p->numa_preferred_nid = nid;
5453 
5454         if (queued)
5455                 enqueue_task(rq, p, ENQUEUE_RESTORE);
5456         if (running)
5457                 set_curr_task(rq, p);
5458         task_rq_unlock(rq, p, &rf);
5459 }
5460 #endif /* CONFIG_NUMA_BALANCING */
5461 
5462 #ifdef CONFIG_HOTPLUG_CPU
5463 /*
5464  * Ensures that the idle task is using init_mm right before its cpu goes
5465  * offline.
5466  */
5467 void idle_task_exit(void)
5468 {
5469         struct mm_struct *mm = current->active_mm;
5470 
5471         BUG_ON(cpu_online(smp_processor_id()));
5472 
5473         if (mm != &init_mm) {
5474                 switch_mm_irqs_off(mm, &init_mm, current);
5475                 finish_arch_post_lock_switch();
5476         }
5477         mmdrop(mm);
5478 }
5479 
5480 /*
5481  * Since this CPU is going 'away' for a while, fold any nr_active delta
5482  * we might have. Assumes we're called after migrate_tasks() so that the
5483  * nr_active count is stable. We need to take the teardown thread which
5484  * is calling this into account, so we hand in adjust = 1 to the load
5485  * calculation.
5486  *
5487  * Also see the comment "Global load-average calculations".
5488  */
5489 static void calc_load_migrate(struct rq *rq)
5490 {
5491         long delta = calc_load_fold_active(rq, 1);
5492         if (delta)
5493                 atomic_long_add(delta, &calc_load_tasks);
5494 }
5495 
5496 static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
5497 {
5498 }
5499 
5500 static const struct sched_class fake_sched_class = {
5501         .put_prev_task = put_prev_task_fake,
5502 };
5503 
5504 static struct task_struct fake_task = {
5505         /*
5506          * Avoid pull_{rt,dl}_task()
5507          */
5508         .prio = MAX_PRIO + 1,
5509         .sched_class = &fake_sched_class,
5510 };
5511 
5512 /*
5513  * Migrate all tasks from the rq, sleeping tasks will be migrated by
5514  * try_to_wake_up()->select_task_rq().
5515  *
5516  * Called with rq->lock held even though we'er in stop_machine() and
5517  * there's no concurrency possible, we hold the required locks anyway
5518  * because of lock validation efforts.
5519  */
5520 static void migrate_tasks(struct rq *dead_rq)
5521 {
5522         struct rq *rq = dead_rq;
5523         struct task_struct *next, *stop = rq->stop;
5524         struct pin_cookie cookie;
5525         int dest_cpu;
5526 
5527         /*
5528          * Fudge the rq selection such that the below task selection loop
5529          * doesn't get stuck on the currently eligible stop task.
5530          *
5531          * We're currently inside stop_machine() and the rq is either stuck
5532          * in the stop_machine_cpu_stop() loop, or we're executing this code,
5533          * either way we should never end up calling schedule() until we're
5534          * done here.
5535          */
5536         rq->stop = NULL;
5537 
5538         /*
5539          * put_prev_task() and pick_next_task() sched
5540          * class method both need to have an up-to-date
5541          * value of rq->clock[_task]
5542          */
5543         update_rq_clock(rq);
5544 
5545         for (;;) {
5546                 /*
5547                  * There's this thread running, bail when that's the only
5548                  * remaining thread.
5549                  */
5550                 if (rq->nr_running == 1)
5551                         break;
5552 
5553                 /*
5554                  * pick_next_task assumes pinned rq->lock.
5555                  */
5556                 cookie = lockdep_pin_lock(&rq->lock);
5557                 next = pick_next_task(rq, &fake_task, cookie);
5558                 BUG_ON(!next);
5559                 next->sched_class->put_prev_task(rq, next);
5560 
5561                 /*
5562                  * Rules for changing task_struct::cpus_allowed are holding
5563                  * both pi_lock and rq->lock, such that holding either
5564                  * stabilizes the mask.
5565                  *
5566                  * Drop rq->lock is not quite as disastrous as it usually is
5567                  * because !cpu_active at this point, which means load-balance
5568                  * will not interfere. Also, stop-machine.
5569                  */
5570                 lockdep_unpin_lock(&rq->lock, cookie);
5571                 raw_spin_unlock(&rq->lock);
5572                 raw_spin_lock(&next->pi_lock);
5573                 raw_spin_lock(&rq->lock);
5574 
5575                 /*
5576                  * Since we're inside stop-machine, _nothing_ should have
5577                  * changed the task, WARN if weird stuff happened, because in
5578                  * that case the above rq->lock drop is a fail too.
5579                  */
5580                 if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
5581                         raw_spin_unlock(&next->pi_lock);
5582                         continue;
5583                 }
5584 
5585                 /* Find suitable destination for @next, with force if needed. */
5586                 dest_cpu = select_fallback_rq(dead_rq->cpu, next);
5587 
5588                 rq = __migrate_task(rq, next, dest_cpu);
5589                 if (rq != dead_rq) {
5590                         raw_spin_unlock(&rq->lock);
5591                         rq = dead_rq;
5592                         raw_spin_lock(&rq->lock);
5593                 }
5594                 raw_spin_unlock(&next->pi_lock);
5595         }
5596 
5597         rq->stop = stop;
5598 }
5599 #endif /* CONFIG_HOTPLUG_CPU */
5600 
5601 static void set_rq_online(struct rq *rq)
5602 {
5603         if (!rq->online) {
5604                 const struct sched_class *class;
5605 
5606                 cpumask_set_cpu(rq->cpu, rq->rd->online);
5607                 rq->online = 1;
5608 
5609                 for_each_class(class) {
5610                         if (class->rq_online)
5611                                 class->rq_online(rq);
5612                 }
5613         }
5614 }
5615 
5616 static void set_rq_offline(struct rq *rq)
5617 {
5618         if (rq->online) {
5619                 const struct sched_class *class;
5620 
5621                 for_each_class(class) {
5622                         if (class->rq_offline)
5623                                 class->rq_offline(rq);
5624                 }
5625 
5626                 cpumask_clear_cpu(rq->cpu, rq->rd->online);
5627                 rq->online = 0;
5628         }
5629 }
5630 
5631 static void set_cpu_rq_start_time(unsigned int cpu)
5632 {
5633         struct rq *rq = cpu_rq(cpu);
5634 
5635         rq->age_stamp = sched_clock_cpu(cpu);
5636 }
5637 
5638 static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
5639 
5640 #ifdef CONFIG_SCHED_DEBUG
5641 
5642 static __read_mostly int sched_debug_enabled;
5643 
5644 static int __init sched_debug_setup(char *str)
5645 {
5646         sched_debug_enabled = 1;
5647 
5648         return 0;
5649 }
5650 early_param("sched_debug", sched_debug_setup);
5651 
5652 static inline bool sched_debug(void)
5653 {
5654         return sched_debug_enabled;
5655 }
5656 
5657 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5658                                   struct cpumask *groupmask)
5659 {
5660         struct sched_group *group = sd->groups;
5661 
5662         cpumask_clear(groupmask);
5663 
5664         printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
5665 
5666         if (!(sd->flags & SD_LOAD_BALANCE)) {
5667                 printk("does not load-balance\n");
5668                 if (sd->parent)
5669                         printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5670                                         " has parent");
5671                 return -1;
5672         }
5673 
5674         printk(KERN_CONT "span %*pbl level %s\n",
5675                cpumask_pr_args(sched_domain_span(sd)), sd->name);
5676 
5677         if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
5678                 printk(KERN_ERR "ERROR: domain->span does not contain "
5679                                 "CPU%d\n", cpu);
5680         }
5681         if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
5682                 printk(KERN_ERR "ERROR: domain->groups does not contain"
5683                                 " CPU%d\n", cpu);
5684         }
5685 
5686         printk(KERN_DEBUG "%*s groups:", level + 1, "");
5687         do {
5688                 if (!group) {
5689                         printk("\n");
5690                         printk(KERN_ERR "ERROR: group is NULL\n");
5691                         break;
5692                 }
5693 
5694                 if (!cpumask_weight(sched_group_cpus(group))) {
5695                         printk(KERN_CONT "\n");
5696                         printk(KERN_ERR "ERROR: empty group\n");
5697                         break;
5698                 }
5699 
5700                 if (!(sd->flags & SD_OVERLAP) &&
5701                     cpumask_intersects(groupmask, sched_group_cpus(group))) {
5702                         printk(KERN_CONT "\n");
5703                         printk(KERN_ERR "ERROR: repeated CPUs\n");
5704                         break;
5705                 }
5706 
5707                 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
5708 
5709                 printk(KERN_CONT " %*pbl",
5710                        cpumask_pr_args(sched_group_cpus(group)));
5711                 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
5712                         printk(KERN_CONT " (cpu_capacity = %lu)",
5713                                 group->sgc->capacity);
5714                 }
5715 
5716                 group = group->next;
5717         } while (group != sd->groups);
5718         printk(KERN_CONT "\n");
5719 
5720         if (!cpumask_equal(sched_domain_span(sd), groupmask))
5721                 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
5722 
5723         if (sd->parent &&
5724             !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
5725                 printk(KERN_ERR "ERROR: parent span is not a superset "
5726                         "of domain->span\n");
5727         return 0;
5728 }
5729 
5730 static void sched_domain_debug(struct sched_domain *sd, int cpu)
5731 {
5732         int level = 0;
5733 
5734         if (!sched_debug_enabled)
5735                 return;
5736 
5737         if (!sd) {
5738                 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
5739                 return;
5740         }
5741 
5742         printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
5743 
5744         for (;;) {
5745                 if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
5746                         break;
5747                 level++;
5748                 sd = sd->parent;
5749                 if (!sd)
5750                         break;
5751         }
5752 }
5753 #else /* !CONFIG_SCHED_DEBUG */
5754 
5755 # define sched_debug_enabled 0
5756 # define sched_domain_debug(sd, cpu) do { } while (0)
5757 static inline bool sched_debug(void)
5758 {
5759         return false;
5760 }
5761 #endif /* CONFIG_SCHED_DEBUG */
5762 
5763 static int sd_degenerate(struct sched_domain *sd)
5764 {
5765         if (cpumask_weight(sched_domain_span(sd)) == 1)
5766                 return 1;
5767 
5768         /* Following flags need at least 2 groups */
5769         if (sd->flags & (SD_LOAD_BALANCE |
5770                          SD_BALANCE_NEWIDLE |
5771                          SD_BALANCE_FORK |
5772                          SD_BALANCE_EXEC |
5773                          SD_SHARE_CPUCAPACITY |
5774                          SD_ASYM_CPUCAPACITY |
5775                          SD_SHARE_PKG_RESOURCES |
5776                          SD_SHARE_POWERDOMAIN)) {
5777                 if (sd->groups != sd->groups->next)
5778                         return 0;
5779         }
5780 
5781         /* Following flags don't use groups */
5782         if (sd->flags & (SD_WAKE_AFFINE))
5783                 return 0;
5784 
5785         return 1;
5786 }
5787 
5788 static int
5789 sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5790 {
5791         unsigned long cflags = sd->flags, pflags = parent->flags;
5792 
5793         if (sd_degenerate(parent))
5794                 return 1;
5795 
5796         if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
5797                 return 0;
5798 
5799         /* Flags needing groups don't count if only 1 group in parent */
5800         if (parent->groups == parent->groups->next) {
5801                 pflags &= ~(SD_LOAD_BALANCE |
5802                                 SD_BALANCE_NEWIDLE |
5803                                 SD_BALANCE_FORK |
5804                                 SD_BALANCE_EXEC |
5805                                 SD_ASYM_CPUCAPACITY |
5806                                 SD_SHARE_CPUCAPACITY |
5807                                 SD_SHARE_PKG_RESOURCES |
5808                                 SD_PREFER_SIBLING |
5809                                 SD_SHARE_POWERDOMAIN);
5810                 if (nr_node_ids == 1)
5811                         pflags &= ~SD_SERIALIZE;
5812         }
5813         if (~cflags & pflags)
5814                 return 0;
5815 
5816         return 1;
5817 }
5818 
5819 static void free_rootdomain(struct rcu_head *rcu)
5820 {
5821         struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
5822 
5823         cpupri_cleanup(&rd->cpupri);
5824         cpudl_cleanup(&rd->cpudl);
5825         free_cpumask_var(rd->dlo_mask);
5826         free_cpumask_var(rd->rto_mask);
5827         free_cpumask_var(rd->online);
5828         free_cpumask_var(rd->span);
5829         kfree(rd);
5830 }
5831 
5832 static void rq_attach_root(struct rq *rq, struct root_domain *rd)
5833 {
5834         struct root_domain *old_rd = NULL;
5835         unsigned long flags;
5836 
5837         raw_spin_lock_irqsave(&rq->lock, flags);
5838 
5839         if (rq->rd) {
5840                 old_rd = rq->rd;
5841 
5842                 if (cpumask_test_cpu(rq->cpu, old_rd->online))
5843                         set_rq_offline(rq);
5844 
5845                 cpumask_clear_cpu(rq->cpu, old_rd->span);
5846 
5847                 /*
5848                  * If we dont want to free the old_rd yet then
5849                  * set old_rd to NULL to skip the freeing later
5850                  * in this function:
5851                  */
5852                 if (!atomic_dec_and_test(&old_rd->refcount))
5853                         old_rd = NULL;
5854         }
5855 
5856         atomic_inc(&rd->refcount);
5857         rq->rd = rd;
5858 
5859         cpumask_set_cpu(rq->cpu, rd->span);
5860         if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
5861                 set_rq_online(rq);
5862 
5863         raw_spin_unlock_irqrestore(&rq->lock, flags);
5864 
5865         if (old_rd)
5866                 call_rcu_sched(&old_rd->rcu, free_rootdomain);
5867 }
5868 
5869 static int init_rootdomain(struct root_domain *rd)
5870 {
5871         memset(rd, 0, sizeof(*rd));
5872 
5873         if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL))
5874                 goto out;
5875         if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL))
5876                 goto free_span;
5877         if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
5878                 goto free_online;
5879         if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
5880                 goto free_dlo_mask;
5881 
5882         init_dl_bw(&rd->dl_bw);
5883         if (cpudl_init(&rd->cpudl) != 0)
5884                 goto free_dlo_mask;
5885 
5886         if (cpupri_init(&rd->cpupri) != 0)
5887                 goto free_rto_mask;
5888         return 0;
5889 
5890 free_rto_mask:
5891         free_cpumask_var(rd->rto_mask);
5892 free_dlo_mask:
5893         free_cpumask_var(rd->dlo_mask);
5894 free_online:
5895         free_cpumask_var(rd->online);
5896 free_span:
5897         free_cpumask_var(rd->span);
5898 out:
5899         return -ENOMEM;
5900 }
5901 
5902 /*
5903  * By default the system creates a single root-domain with all cpus as
5904  * members (mimicking the global state we have today).
5905  */
5906 struct root_domain def_root_domain;
5907 
5908 static void init_defrootdomain(void)
5909 {
5910         init_rootdomain(&def_root_domain);
5911 
5912         atomic_set(&def_root_domain.refcount, 1);
5913 }
5914 
5915 static struct root_domain *alloc_rootdomain(void)
5916 {
5917         struct root_domain *rd;
5918 
5919         rd = kmalloc(sizeof(*rd), GFP_KERNEL);
5920         if (!rd)
5921                 return NULL;
5922 
5923         if (init_rootdomain(rd) != 0) {
5924                 kfree(rd);
5925                 return NULL;
5926         }
5927 
5928         return rd;
5929 }
5930 
5931 static void free_sched_groups(struct sched_group *sg, int free_sgc)
5932 {
5933         struct sched_group *tmp, *first;
5934 
5935         if (!sg)
5936                 return;
5937 
5938         first = sg;
5939         do {
5940                 tmp = sg->next;
5941 
5942                 if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
5943                         kfree(sg->sgc);
5944 
5945                 kfree(sg);
5946                 sg = tmp;
5947         } while (sg != first);
5948 }
5949 
5950 static void destroy_sched_domain(struct sched_domain *sd)
5951 {
5952         /*
5953          * If its an overlapping domain it has private groups, iterate and
5954          * nuke them all.
5955          */
5956         if (sd->flags & SD_OVERLAP) {
5957                 free_sched_groups(sd->groups, 1);
5958         } else if (atomic_dec_and_test(&sd->groups->ref)) {
5959                 kfree(sd->groups->sgc);
5960                 kfree(sd->groups);
5961         }
5962         if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
5963                 kfree(sd->shared);
5964         kfree(sd);
5965 }
5966 
5967 static void destroy_sched_domains_rcu(struct rcu_head *rcu)
5968 {
5969         struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
5970 
5971         while (sd) {
5972                 struct sched_domain *parent = sd->parent;
5973                 destroy_sched_domain(sd);
5974                 sd = parent;
5975         }
5976 }
5977 
5978 static void destroy_sched_domains(struct sched_domain *sd)
5979 {
5980         if (sd)
5981                 call_rcu(&sd->rcu, destroy_sched_domains_rcu);
5982 }
5983 
5984 /*
5985  * Keep a special pointer to the highest sched_domain that has
5986  * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
5987  * allows us to avoid some pointer chasing select_idle_sibling().
5988  *
5989  * Also keep a unique ID per domain (we use the first cpu number in
5990  * the cpumask of the domain), this allows us to quickly tell if
5991  * two cpus are in the same cache domain, see cpus_share_cache().
5992  */
5993 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
5994 DEFINE_PER_CPU(int, sd_llc_size);
5995 DEFINE_PER_CPU(int, sd_llc_id);
5996 DEFINE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
5997 DEFINE_PER_CPU(struct sched_domain *, sd_numa);
5998 DEFINE_PER_CPU(struct sched_domain *, sd_asym);
5999 
6000 static void update_top_cache_domain(int cpu)
6001 {
6002         struct sched_domain_shared *sds = NULL;
6003         struct sched_domain *sd;
6004         int id = cpu;
6005         int size = 1;
6006 
6007         sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
6008         if (sd) {
6009                 id = cpumask_first(sched_domain_span(sd));
6010                 size = cpumask_weight(sched_domain_span(sd));
6011                 sds = sd->shared;
6012         }
6013 
6014         rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
6015         per_cpu(sd_llc_size, cpu) = size;
6016         per_cpu(sd_llc_id, cpu) = id;
6017         rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
6018 
6019         sd = lowest_flag_domain(cpu, SD_NUMA);
6020         rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
6021 
6022         sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
6023         rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
6024 }
6025 
6026 /*
6027  * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
6028  * hold the hotplug lock.
6029  */
6030 static void
6031 cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6032 {
6033         struct rq *rq = cpu_rq(cpu);
6034         struct sched_domain *tmp;
6035 
6036         /* Remove the sched domains which do not contribute to scheduling. */
6037         for (tmp = sd; tmp; ) {
6038                 struct sched_domain *parent = tmp->parent;
6039                 if (!parent)
6040                         break;
6041 
6042                 if (sd_parent_degenerate(tmp, parent)) {
6043                         tmp->parent = parent->parent;
6044                         if (parent->parent)
6045                                 parent->parent->child = tmp;
6046                         /*
6047                          * Transfer SD_PREFER_SIBLING down in case of a
6048                          * degenerate parent; the spans match for this
6049                          * so the property transfers.
6050                          */
6051                         if (parent->flags & SD_PREFER_SIBLING)
6052                                 tmp->flags |= SD_PREFER_SIBLING;
6053                         destroy_sched_domain(parent);
6054                 } else
6055                         tmp = tmp->parent;
6056         }
6057 
6058         if (sd && sd_degenerate(sd)) {
6059                 tmp = sd;
6060                 sd = sd->parent;
6061                 destroy_sched_domain(tmp);
6062                 if (sd)
6063                         sd->child = NULL;
6064         }
6065 
6066         sched_domain_debug(sd, cpu);
6067 
6068         rq_attach_root(rq, rd);
6069         tmp = rq->sd;
6070         rcu_assign_pointer(rq->sd, sd);
6071         destroy_sched_domains(tmp);
6072 
6073         update_top_cache_domain(cpu);
6074 }
6075 
6076 /* Setup the mask of cpus configured for isolated domains */
6077 static int __init isolated_cpu_setup(char *str)
6078 {
6079         int ret;
6080 
6081         alloc_bootmem_cpumask_var(&cpu_isolated_map);
6082         ret = cpulist_parse(str, cpu_isolated_map);
6083         if (ret) {
6084                 pr_err("sched: Error, all isolcpus= values must be between 0 and %d\n", nr_cpu_ids);
6085                 return 0;
6086         }
6087         return 1;
6088 }
6089 __setup("isolcpus=", isolated_cpu_setup);
6090 
6091 struct s_data {
6092         struct sched_domain ** __percpu sd;
6093         struct root_domain      *rd;
6094 };
6095 
6096 enum s_alloc {
6097         sa_rootdomain,
6098         sa_sd,
6099         sa_sd_storage,
6100         sa_none,
6101 };
6102 
6103 /*
6104  * Build an iteration mask that can exclude certain CPUs from the upwards
6105  * domain traversal.
6106  *
6107  * Asymmetric node setups can result in situations where the domain tree is of
6108  * unequal depth, make sure to skip domains that already cover the entire
6109  * range.
6110  *
6111  * In that case build_sched_domains() will have terminated the iteration early
6112  * and our sibling sd spans will be empty. Domains should always include the
6113  * cpu they're built on, so check that.
6114  *
6115  */
6116 static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
6117 {
6118         const struct cpumask *span = sched_domain_span(sd);
6119         struct sd_data *sdd = sd->private;
6120         struct sched_domain *sibling;
6121         int i;
6122 
6123         for_each_cpu(i, span) {
6124                 sibling = *per_cpu_ptr(sdd->sd, i);
6125                 if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
6126                         continue;
6127 
6128                 cpumask_set_cpu(i, sched_group_mask(sg));
6129         }
6130 }
6131 
6132 /*
6133  * Return the canonical balance cpu for this group, this is the first cpu
6134  * of this group that's also in the iteration mask.
6135  */
6136 int group_balance_cpu(struct sched_group *sg)
6137 {
6138         return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
6139 }
6140 
6141 static int
6142 build_overlap_sched_groups(struct sched_domain *sd, int cpu)
6143 {
6144         struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
6145         const struct cpumask *span = sched_domain_span(sd);
6146         struct cpumask *covered = sched_domains_tmpmask;
6147         struct sd_data *sdd = sd->private;
6148         struct sched_domain *sibling;
6149         int i;
6150 
6151         cpumask_clear(covered);
6152 
6153         for_each_cpu(i, span) {
6154                 struct cpumask *sg_span;
6155 
6156                 if (cpumask_test_cpu(i, covered))
6157                         continue;
6158 
6159                 sibling = *per_cpu_ptr(sdd->sd, i);
6160 
6161                 /* See the comment near build_group_mask(). */
6162                 if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
6163                         continue;
6164 
6165                 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
6166                                 GFP_KERNEL, cpu_to_node(cpu));
6167 
6168                 if (!sg)
6169                         goto fail;
6170 
6171                 sg_span = sched_group_cpus(sg);
6172                 if (sibling->child)
6173                         cpumask_copy(sg_span, sched_domain_span(sibling->child));
6174                 else
6175                         cpumask_set_cpu(i, sg_span);
6176 
6177                 cpumask_or(covered, covered, sg_span);
6178 
6179                 sg->sgc = *per_cpu_ptr(sdd->sgc, i);
6180                 if (atomic_inc_return(&sg->sgc->ref) == 1)
6181                         build_group_mask(sd, sg);
6182 
6183                 /*
6184                  * Initialize sgc->capacity such that even if we mess up the
6185                  * domains and no possible iteration will get us here, we won't
6186                  * die on a /0 trap.
6187                  */
6188                 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
6189                 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
6190 
6191                 /*
6192                  * Make sure the first group of this domain contains the
6193                  * canonical balance cpu. Otherwise the sched_domain iteration
6194                  * breaks. See update_sg_lb_stats().
6195                  */
6196                 if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
6197                     group_balance_cpu(sg) == cpu)
6198                         groups = sg;
6199 
6200                 if (!first)
6201                         first = sg;
6202                 if (last)
6203                         last->next = sg;
6204                 last = sg;
6205                 last->next = first;
6206         }
6207         sd->groups = groups;
6208 
6209         return 0;
6210 
6211 fail:
6212         free_sched_groups(first, 0);
6213 
6214         return -ENOMEM;
6215 }
6216 
6217 static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
6218 {
6219         struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
6220         struct sched_domain *child = sd->child;
6221 
6222         if (child)
6223                 cpu = cpumask_first(sched_domain_span(child));
6224 
6225         if (sg) {
6226                 *sg = *per_cpu_ptr(sdd->sg, cpu);
6227                 (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu);
6228                 atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */
6229         }
6230 
6231         return cpu;
6232 }
6233 
6234 /*
6235  * build_sched_groups will build a circular linked list of the groups
6236  * covered by the given span, and will set each group's ->cpumask correctly,
6237  * and ->cpu_capacity to 0.
6238  *
6239  * Assumes the sched_domain tree is fully constructed
6240  */
6241 static int
6242 build_sched_groups(struct sched_domain *sd, int cpu)
6243 {
6244         struct sched_group *first = NULL, *last = NULL;
6245         struct sd_data *sdd = sd->private;
6246         const struct cpumask *span = sched_domain_span(sd);
6247         struct cpumask *covered;
6248         int i;
6249 
6250         get_group(cpu, sdd, &sd->groups);
6251         atomic_inc(&sd->groups->ref);
6252 
6253         if (cpu != cpumask_first(span))
6254                 return 0;
6255 
6256         lockdep_assert_held(&sched_domains_mutex);
6257         covered = sched_domains_tmpmask;
6258 
6259         cpumask_clear(covered);
6260 
6261         for_each_cpu(i, span) {
6262                 struct sched_group *sg;
6263                 int group, j;
6264 
6265                 if (cpumask_test_cpu(i, covered))
6266                         continue;
6267 
6268                 group = get_group(i, sdd, &sg);
6269                 cpumask_setall(sched_group_mask(sg));
6270 
6271                 for_each_cpu(j, span) {
6272                         if (get_group(j, sdd, NULL) != group)
6273                                 continue;
6274 
6275                         cpumask_set_cpu(j, covered);
6276                         cpumask_set_cpu(j, sched_group_cpus(sg));
6277                 }
6278 
6279                 if (!first)
6280                         first = sg;
6281                 if (last)
6282                         last->next = sg;
6283                 last = sg;
6284         }
6285         last->next = first;
6286 
6287         return 0;
6288 }
6289 
6290 /*
6291  * Initialize sched groups cpu_capacity.
6292  *
6293  * cpu_capacity indicates the capacity of sched group, which is used while
6294  * distributing the load between different sched groups in a sched domain.
6295  * Typically cpu_capacity for all the groups in a sched domain will be same
6296  * unless there are asymmetries in the topology. If there are asymmetries,
6297  * group having more cpu_capacity will pickup more load compared to the
6298  * group having less cpu_capacity.
6299  */
6300 static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
6301 {
6302         struct sched_group *sg = sd->groups;
6303 
6304         WARN_ON(!sg);
6305 
6306         do {
6307                 int cpu, max_cpu = -1;
6308 
6309                 sg->group_weight = cpumask_weight(sched_group_cpus(sg));
6310 
6311                 if (!(sd->flags & SD_ASYM_PACKING))
6312                         goto next;
6313 
6314                 for_each_cpu(cpu, sched_group_cpus(sg)) {
6315                         if (max_cpu < 0)
6316                                 max_cpu = cpu;
6317                         else if (sched_asym_prefer(cpu, max_cpu))
6318                                 max_cpu = cpu;
6319                 }
6320                 sg->asym_prefer_cpu = max_cpu;
6321 
6322 next:
6323                 sg = sg->next;
6324         } while (sg != sd->groups);
6325 
6326         if (cpu != group_balance_cpu(sg))
6327                 return;
6328 
6329         update_group_capacity(sd, cpu);
6330 }
6331 
6332 /*
6333  * Initializers for schedule domains
6334  * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
6335  */
6336 
6337 static int default_relax_domain_level = -1;
6338 int sched_domain_level_max;
6339 
6340 static int __init setup_relax_domain_level(char *str)
6341 {
6342         if (kstrtoint(str, 0, &default_relax_domain_level))
6343                 pr_warn("Unable to set relax_domain_level\n");
6344 
6345         return 1;
6346 }
6347 __setup("relax_domain_level=", setup_relax_domain_level);
6348 
6349 static void set_domain_attribute(struct sched_domain *sd,
6350                                  struct sched_domain_attr *attr)
6351 {
6352         int request;
6353 
6354         if (!attr || attr->relax_domain_level < 0) {
6355                 if (default_relax_domain_level < 0)
6356                         return;
6357                 else
6358                         request = default_relax_domain_level;
6359         } else
6360                 request = attr->relax_domain_level;
6361         if (request < sd->level) {
6362                 /* turn off idle balance on this domain */
6363                 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
6364         } else {
6365                 /* turn on idle balance on this domain */
6366                 sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
6367         }
6368 }
6369 
6370 static void __sdt_free(const struct cpumask *cpu_map);
6371 static int __sdt_alloc(const struct cpumask *cpu_map);
6372 
6373 static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
6374                                  const struct cpumask *cpu_map)
6375 {
6376         switch (what) {
6377         case sa_rootdomain:
6378                 if (!atomic_read(&d->rd->refcount))
6379                         free_rootdomain(&d->rd->rcu); /* fall through */
6380         case sa_sd:
6381                 free_percpu(d->sd); /* fall through */
6382         case sa_sd_storage:
6383                 __sdt_free(cpu_map); /* fall through */
6384         case sa_none:
6385                 break;
6386         }
6387 }
6388 
6389 static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
6390                                                    const struct cpumask *cpu_map)
6391 {
6392         memset(d, 0, sizeof(*d));
6393 
6394         if (__sdt_alloc(cpu_map))
6395                 return sa_sd_storage;
6396         d->sd = alloc_percpu(struct sched_domain *);
6397         if (!d->sd)
6398                 return sa_sd_storage;
6399         d->rd = alloc_rootdomain();
6400         if (!d->rd)
6401                 return sa_sd;
6402         return sa_rootdomain;
6403 }
6404 
6405 /*
6406  * NULL the sd_data elements we've used to build the sched_domain and
6407  * sched_group structure so that the subsequent __free_domain_allocs()
6408  * will not free the data we're using.
6409  */
6410 static void claim_allocations(int cpu, struct sched_domain *sd)
6411 {
6412         struct sd_data *sdd = sd->private;
6413 
6414         WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
6415         *per_cpu_ptr(sdd->sd, cpu) = NULL;
6416 
6417         if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref))
6418                 *per_cpu_ptr(sdd->sds, cpu) = NULL;
6419 
6420         if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
6421                 *per_cpu_ptr(sdd->sg, cpu) = NULL;
6422 
6423         if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
6424                 *per_cpu_ptr(sdd->sgc, cpu) = NULL;
6425 }
6426 
6427 #ifdef CONFIG_NUMA
6428 static int sched_domains_numa_levels;
6429 enum numa_topology_type sched_numa_topology_type;
6430 static int *sched_domains_numa_distance;
6431 int sched_max_numa_distance;
6432 static struct cpumask ***sched_domains_numa_masks;
6433 static int sched_domains_curr_level;
6434 #endif
6435 
6436 /*
6437  * SD_flags allowed in topology descriptions.
6438  *
6439  * These flags are purely descriptive of the topology and do not prescribe
6440  * behaviour. Behaviour is artificial and mapped in the below sd_init()
6441  * function:
6442  *
6443  *   SD_SHARE_CPUCAPACITY   - describes SMT topologies
6444  *   SD_SHARE_PKG_RESOURCES - describes shared caches
6445  *   SD_NUMA                - describes NUMA topologies
6446  *   SD_SHARE_POWERDOMAIN   - describes shared power domain
6447  *   SD_ASYM_CPUCAPACITY    - describes mixed capacity topologies
6448  *
6449  * Odd one out, which beside describing the topology has a quirk also
6450  * prescribes the desired behaviour that goes along with it:
6451  *
6452  *   SD_ASYM_PACKING        - describes SMT quirks
6453  */
6454 #define TOPOLOGY_SD_FLAGS               \
6455         (SD_SHARE_CPUCAPACITY |         \
6456          SD_SHARE_PKG_RESOURCES |       \
6457          SD_NUMA |                      \
6458          SD_ASYM_PACKING |              \
6459          SD_ASYM_CPUCAPACITY |          \
6460          SD_SHARE_POWERDOMAIN)
6461 
6462 static struct sched_domain *
6463 sd_init(struct sched_domain_topology_level *tl,
6464         const struct cpumask *cpu_map,
6465         struct sched_domain *child, int cpu)
6466 {
6467         struct sd_data *sdd = &tl->data;
6468         struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
6469         int sd_id, sd_weight, sd_flags = 0;
6470 
6471 #ifdef CONFIG_NUMA
6472         /*
6473          * Ugly hack to pass state to sd_numa_mask()...
6474          */
6475         sched_domains_curr_level = tl->numa_level;
6476 #endif
6477 
6478         sd_weight = cpumask_weight(tl->mask(cpu));
6479 
6480         if (tl->sd_flags)
6481                 sd_flags = (*tl->sd_flags)();
6482         if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
6483                         "wrong sd_flags in topology description\n"))
6484                 sd_flags &= ~TOPOLOGY_SD_FLAGS;
6485 
6486         *sd = (struct sched_domain){
6487                 .min_interval           = sd_weight,
6488                 .max_interval           = 2*sd_weight,
6489                 .busy_factor            = 32,
6490                 .imbalance_pct          = 125,
6491 
6492                 .cache_nice_tries       = 0,
6493                 .busy_idx               = 0,
6494                 .idle_idx               = 0,
6495                 .newidle_idx            = 0,
6496                 .wake_idx               = 0,
6497                 .forkexec_idx           = 0,
6498 
6499                 .flags                  = 1*SD_LOAD_BALANCE
6500                                         | 1*SD_BALANCE_NEWIDLE
6501                                         | 1*SD_BALANCE_EXEC
6502                                         | 1*SD_BALANCE_FORK
6503                                         | 0*SD_BALANCE_WAKE
6504                                         | 1*SD_WAKE_AFFINE
6505                                         | 0*SD_SHARE_CPUCAPACITY
6506                                         | 0*SD_SHARE_PKG_RESOURCES
6507                                         | 0*SD_SERIALIZE
6508                                         | 0*SD_PREFER_SIBLING
6509                                         | 0*SD_NUMA
6510                                         | sd_flags
6511                                         ,
6512 
6513                 .last_balance           = jiffies,
6514                 .balance_interval       = sd_weight,
6515                 .smt_gain               = 0,
6516                 .max_newidle_lb_cost    = 0,
6517                 .next_decay_max_lb_cost = jiffies,
6518                 .child                  = child,
6519 #ifdef CONFIG_SCHED_DEBUG
6520                 .name                   = tl->name,
6521 #endif
6522         };
6523 
6524         cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
6525         sd_id = cpumask_first(sched_domain_span(sd));
6526 
6527         /*
6528          * Convert topological properties into behaviour.
6529          */
6530 
6531         if (sd->flags & SD_ASYM_CPUCAPACITY) {
6532                 struct sched_domain *t = sd;
6533 
6534                 for_each_lower_domain(t)
6535                         t->flags |= SD_BALANCE_WAKE;
6536         }
6537 
6538         if (sd->flags & SD_SHARE_CPUCAPACITY) {
6539                 sd->flags |= SD_PREFER_SIBLING;
6540                 sd->imbalance_pct = 110;
6541                 sd->smt_gain = 1178; /* ~15% */
6542 
6543         } else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
6544                 sd->imbalance_pct = 117;
6545                 sd->cache_nice_tries = 1;
6546                 sd->busy_idx = 2;
6547 
6548 #ifdef CONFIG_NUMA
6549         } else if (sd->flags & SD_NUMA) {
6550                 sd->cache_nice_tries = 2;
6551                 sd->busy_idx = 3;
6552                 sd->idle_idx = 2;
6553 
6554                 sd->flags |= SD_SERIALIZE;
6555                 if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
6556                         sd->flags &= ~(SD_BALANCE_EXEC |
6557                                        SD_BALANCE_FORK |
6558                                        SD_WAKE_AFFINE);
6559                 }
6560 
6561 #endif
6562         } else {
6563                 sd->flags |= SD_PREFER_SIBLING;
6564                 sd->cache_nice_tries = 1;
6565                 sd->busy_idx = 2;
6566                 sd->idle_idx = 1;
6567         }
6568 
6569         /*
6570          * For all levels sharing cache; connect a sched_domain_shared
6571          * instance.
6572          */
6573         if (sd->flags & SD_SHARE_PKG_RESOURCES) {
6574                 sd->shared = *per_cpu_ptr(sdd->sds, sd_id);
6575                 atomic_inc(&sd->shared->ref);
6576                 atomic_set(&sd->shared->nr_busy_cpus, sd_weight);
6577         }
6578 
6579         sd->private = sdd;
6580 
6581         return sd;
6582 }
6583 
6584 /*
6585  * Topology list, bottom-up.
6586  */
6587 static struct sched_domain_topology_level default_topology[] = {
6588 #ifdef CONFIG_SCHED_SMT
6589         { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
6590 #endif
6591 #ifdef CONFIG_SCHED_MC
6592         { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
6593 #endif
6594         { cpu_cpu_mask, SD_INIT_NAME(DIE) },
6595         { NULL, },
6596 };
6597 
6598 static struct sched_domain_topology_level *sched_domain_topology =
6599         default_topology;
6600 
6601 #define for_each_sd_topology(tl)                        \
6602         for (tl = sched_domain_topology; tl->mask; tl++)
6603 
6604 void set_sched_topology(struct sched_domain_topology_level *tl)
6605 {
6606         if (WARN_ON_ONCE(sched_smp_initialized))
6607                 return;
6608 
6609         sched_domain_topology = tl;
6610 }
6611 
6612 #ifdef CONFIG_NUMA
6613 
6614 static const struct cpumask *sd_numa_mask(int cpu)
6615 {
6616         return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
6617 }
6618 
6619 static void sched_numa_warn(const char *str)
6620 {
6621         static int done = false;
6622         int i,j;
6623 
6624         if (done)
6625                 return;
6626 
6627         done = true;
6628 
6629         printk(KERN_WARNING "ERROR: %s\n\n", str);
6630 
6631         for (i = 0; i < nr_node_ids; i++) {
6632                 printk(KERN_WARNING "  ");
6633                 for (j = 0; j < nr_node_ids; j++)
6634                         printk(KERN_CONT "%02d ", node_distance(i,j));
6635                 printk(KERN_CONT "\n");
6636         }
6637         printk(KERN_WARNING "\n");
6638 }
6639 
6640 bool find_numa_distance(int distance)
6641 {
6642         int i;
6643 
6644         if (distance == node_distance(0, 0))
6645                 return true;
6646 
6647         for (i = 0; i < sched_domains_numa_levels; i++) {
6648                 if (sched_domains_numa_distance[i] == distance)
6649                         return true;
6650         }
6651 
6652         return false;
6653 }
6654 
6655 /*
6656  * A system can have three types of NUMA topology:
6657  * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system
6658  * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes
6659  * NUMA_BACKPLANE: nodes can reach other nodes through a backplane
6660  *
6661  * The difference between a glueless mesh topology and a backplane
6662  * topology lies in whether communication between not directly
6663  * connected nodes goes through intermediary nodes (where programs
6664  * could run), or through backplane controllers. This affects
6665  * placement of programs.
6666  *
6667  * The type of topology can be discerned with the following tests:
6668  * - If the maximum distance between any nodes is 1 hop, the system
6669  *   is directly connected.
6670  * - If for two nodes A and B, located N > 1 hops away from each other,
6671  *   there is an intermediary node C, which is < N hops away from both
6672  *   nodes A and B, the system is a glueless mesh.
6673  */
6674 static void init_numa_topology_type(void)
6675 {
6676         int a, b, c, n;
6677 
6678         n = sched_max_numa_distance;
6679 
6680         if (sched_domains_numa_levels <= 1) {
6681                 sched_numa_topology_type = NUMA_DIRECT;
6682                 return;
6683         }
6684 
6685         for_each_online_node(a) {
6686                 for_each_online_node(b) {
6687                         /* Find two nodes furthest removed from each other. */
6688                         if (node_distance(a, b) < n)
6689                                 continue;
6690 
6691                         /* Is there an intermediary node between a and b? */
6692                         for_each_online_node(c) {
6693                                 if (node_distance(a, c) < n &&
6694                                     node_distance(b, c) < n) {
6695                                         sched_numa_topology_type =
6696                                                         NUMA_GLUELESS_MESH;
6697                                         return;
6698                                 }
6699                         }
6700 
6701                         sched_numa_topology_type = NUMA_BACKPLANE;
6702                         return;
6703                 }
6704         }
6705 }
6706 
6707 static void sched_init_numa(void)
6708 {
6709         int next_distance, curr_distance = node_distance(0, 0);
6710         struct sched_domain_topology_level *tl;
6711         int level = 0;
6712         int i, j, k;
6713 
6714         sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
6715         if (!sched_domains_numa_distance)
6716                 return;
6717 
6718         /*
6719          * O(nr_nodes^2) deduplicating selection sort -- in order to find the
6720          * unique distances in the node_distance() table.
6721          *
6722          * Assumes node_distance(0,j) includes all distances in
6723          * node_distance(i,j) in order to avoid cubic time.
6724          */
6725         next_distance = curr_distance;
6726         for (i = 0; i < nr_node_ids; i++) {
6727                 for (j = 0; j < nr_node_ids; j++) {
6728                         for (k = 0; k < nr_node_ids; k++) {
6729                                 int distance = node_distance(i, k);
6730 
6731                                 if (distance > curr_distance &&
6732                                     (distance < next_distance ||
6733                                      next_distance == curr_distance))
6734                                         next_distance = distance;
6735 
6736                                 /*
6737                                  * While not a strong assumption it would be nice to know
6738                                  * about cases where if node A is connected to B, B is not
6739                                  * equally connected to A.
6740                                  */
6741                                 if (sched_debug() && node_distance(k, i) != distance)
6742                                         sched_numa_warn("Node-distance not symmetric");
6743 
6744                                 if (sched_debug() && i && !find_numa_distance(distance))
6745                                         sched_numa_warn("Node-0 not representative");
6746                         }
6747                         if (next_distance != curr_distance) {
6748                                 sched_domains_numa_distance[level++] = next_distance;
6749                                 sched_domains_numa_levels = level;
6750                                 curr_distance = next_distance;
6751                         } else break;
6752                 }
6753 
6754                 /*
6755                  * In case of sched_debug() we verify the above assumption.
6756                  */
6757                 if (!sched_debug())
6758                         break;
6759         }
6760 
6761         if (!level)
6762                 return;
6763 
6764         /*
6765          * 'level' contains the number of unique distances, excluding the
6766          * identity distance node_distance(i,i).
6767          *
6768          * The sched_domains_numa_distance[] array includes the actual distance
6769          * numbers.
6770          */
6771 
6772         /*
6773          * Here, we should temporarily reset sched_domains_numa_levels to 0.
6774          * If it fails to allocate memory for array sched_domains_numa_masks[][],
6775          * the array will contain less then 'level' members. This could be
6776          * dangerous when we use it to iterate array sched_domains_numa_masks[][]
6777          * in other functions.
6778          *
6779          * We reset it to 'level' at the end of this function.
6780          */
6781         sched_domains_numa_levels = 0;
6782 
6783         sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
6784         if (!sched_domains_numa_masks)
6785                 return;
6786 
6787         /*
6788          * Now for each level, construct a mask per node which contains all
6789          * cpus of nodes that are that many hops away from us.
6790          */
6791         for (i = 0; i < level; i++) {
6792                 sched_domains_numa_masks[i] =
6793                         kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
6794                 if (!sched_domains_numa_masks[i])
6795                         return;
6796 
6797                 for (j = 0; j < nr_node_ids; j++) {
6798                         struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
6799                         if (!mask)
6800                                 return;
6801 
6802                         sched_domains_numa_masks[i][j] = mask;
6803 
6804                         for_each_node(k) {
6805                                 if (node_distance(j, k) > sched_domains_numa_distance[i])
6806                                         continue;
6807 
6808                                 cpumask_or(mask, mask, cpumask_of_node(k));
6809                         }
6810                 }
6811         }
6812 
6813         /* Compute default topology size */
6814         for (i = 0; sched_domain_topology[i].mask; i++);
6815 
6816         tl = kzalloc((i + level + 1) *
6817                         sizeof(struct sched_domain_topology_level), GFP_KERNEL);
6818         if (!tl)
6819                 return;
6820 
6821         /*
6822          * Copy the default topology bits..
6823          */
6824         for (i = 0; sched_domain_topology[i].mask; i++)
6825                 tl[i] = sched_domain_topology[i];
6826 
6827         /*
6828          * .. and append 'j' levels of NUMA goodness.
6829          */
6830         for (j = 0; j < level; i++, j++) {
6831                 tl[i] = (struct sched_domain_topology_level){
6832                         .mask = sd_numa_mask,
6833                         .sd_flags = cpu_numa_flags,
6834                         .flags = SDTL_OVERLAP,
6835                         .numa_level = j,
6836                         SD_INIT_NAME(NUMA)
6837                 };
6838         }
6839 
6840         sched_domain_topology = tl;
6841 
6842         sched_domains_numa_levels = level;
6843         sched_max_numa_distance = sched_domains_numa_distance[level - 1];
6844 
6845         init_numa_topology_type();
6846 }
6847 
6848 static void sched_domains_numa_masks_set(unsigned int cpu)
6849 {
6850         int node = cpu_to_node(cpu);
6851         int i, j;
6852 
6853         for (i = 0; i < sched_domains_numa_levels; i++) {
6854                 for (j = 0; j < nr_node_ids; j++) {
6855                         if (node_distance(j, node) <= sched_domains_numa_distance[i])
6856                                 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
6857                 }
6858         }
6859 }
6860 
6861 static void sched_domains_numa_masks_clear(unsigned int cpu)
6862 {
6863         int i, j;
6864 
6865         for (i = 0; i < sched_domains_numa_levels; i++) {
6866                 for (j = 0; j < nr_node_ids; j++)
6867                         cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
6868         }
6869 }
6870 
6871 #else
6872 static inline void sched_init_numa(void) { }
6873 static void sched_domains_numa_masks_set(unsigned int cpu) { }
6874 static void sched_domains_numa_masks_clear(unsigned int cpu) { }
6875 #endif /* CONFIG_NUMA */
6876 
6877 static int __sdt_alloc(const struct cpumask *cpu_map)
6878 {
6879         struct sched_domain_topology_level *tl;
6880         int j;
6881 
6882         for_each_sd_topology(tl) {
6883                 struct sd_data *sdd = &tl->data;
6884 
6885                 sdd->sd = alloc_percpu(struct sched_domain *);
6886                 if (!sdd->sd)
6887                         return -ENOMEM;
6888 
6889                 sdd->sds = alloc_percpu(struct sched_domain_shared *);
6890                 if (!sdd->sds)
6891                         return -ENOMEM;
6892 
6893                 sdd->sg = alloc_percpu(struct sched_group *);
6894                 if (!sdd->sg)
6895                         return -ENOMEM;
6896 
6897                 sdd->sgc = alloc_percpu(struct sched_group_capacity *);
6898                 if (!sdd->sgc)
6899                         return -ENOMEM;
6900 
6901                 for_each_cpu(j, cpu_map) {
6902                         struct sched_domain *sd;
6903                         struct sched_domain_shared *sds;
6904                         struct sched_group *sg;
6905                         struct sched_group_capacity *sgc;
6906 
6907                         sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
6908                                         GFP_KERNEL, cpu_to_node(j));
6909                         if (!sd)
6910                                 return -ENOMEM;
6911 
6912                         *per_cpu_ptr(sdd->sd, j) = sd;
6913 
6914                         sds = kzalloc_node(sizeof(struct sched_domain_shared),
6915                                         GFP_KERNEL, cpu_to_node(j));
6916                         if (!sds)
6917                                 return -ENOMEM;
6918 
6919                         *per_cpu_ptr(sdd->sds, j) = sds;
6920 
6921                         sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
6922                                         GFP_KERNEL, cpu_to_node(j));
6923                         if (!sg)
6924                                 return -ENOMEM;
6925 
6926                         sg->next = sg;
6927 
6928                         *per_cpu_ptr(sdd->sg, j) = sg;
6929 
6930                         sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
6931                                         GFP_KERNEL, cpu_to_node(j));
6932                         if (!sgc)
6933                                 return -ENOMEM;
6934 
6935                         *per_cpu_ptr(sdd->sgc, j) = sgc;
6936                 }
6937         }
6938 
6939         return 0;
6940 }
6941 
6942 static void __sdt_free(const struct cpumask *cpu_map)
6943 {
6944         struct sched_domain_topology_level *tl;
6945         int j;
6946 
6947         for_each_sd_topology(tl) {
6948                 struct sd_data *sdd = &tl->data;
6949 
6950                 for_each_cpu(j, cpu_map) {
6951                         struct sched_domain *sd;
6952 
6953                         if (sdd->sd) {
6954                                 sd = *per_cpu_ptr(sdd->sd, j);
6955                                 if (sd && (sd->flags & SD_OVERLAP))
6956                                         free_sched_groups(sd->groups, 0);
6957                                 kfree(*per_cpu_ptr(sdd->sd, j));
6958                         }
6959 
6960                         if (sdd->sds)
6961                                 kfree(*per_cpu_ptr(sdd->sds, j));
6962                         if (sdd->sg)
6963                                 kfree(*per_cpu_ptr(sdd->sg, j));
6964                         if (sdd->sgc)
6965                                 kfree(*per_cpu_ptr(sdd->sgc, j));
6966                 }
6967                 free_percpu(sdd->sd);
6968                 sdd->sd = NULL;
6969                 free_percpu(sdd->sds);
6970                 sdd->sds = NULL;
6971                 free_percpu(sdd->sg);
6972                 sdd->sg = NULL;
6973                 free_percpu(sdd->sgc);
6974                 sdd->sgc = NULL;
6975         }
6976 }
6977 
6978 struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
6979                 const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6980                 struct sched_domain *child, int cpu)
6981 {
6982         struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu);
6983 
6984         if (child) {
6985                 sd->level = child->level + 1;
6986                 sched_domain_level_max = max(sched_domain_level_max, sd->level);
6987                 child->parent = sd;
6988 
6989                 if (!cpumask_subset(sched_domain_span(child),
6990                                     sched_domain_span(sd))) {
6991                         pr_err("BUG: arch topology borken\n");
6992 #ifdef CONFIG_SCHED_DEBUG
6993                         pr_err("     the %s domain not a subset of the %s domain\n",
6994                                         child->name, sd->name);
6995 #endif
6996                         /* Fixup, ensure @sd has at least @child cpus. */
6997                         cpumask_or(sched_domain_span(sd),
6998                                    sched_domain_span(sd),
6999                                    sched_domain_span(child));
7000                 }
7001 
7002         }
7003         set_domain_attribute(sd, attr);
7004 
7005         return sd;
7006 }
7007 
7008 /*
7009  * Build sched domains for a given set of cpus and attach the sched domains
7010  * to the individual cpus
7011  */
7012 static int build_sched_domains(const struct cpumask *cpu_map,
7013                                struct sched_domain_attr *attr)
7014 {
7015         enum s_alloc alloc_state;
7016         struct sched_domain *sd;
7017         struct s_data d;
7018         struct rq *rq = NULL;
7019         int i, ret = -ENOMEM;
7020 
7021         alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
7022         if (alloc_state != sa_rootdomain)
7023                 goto error;
7024 
7025         /* Set up domains for cpus specified by the cpu_map. */
7026         for_each_cpu(i, cpu_map) {
7027                 struct sched_domain_topology_level *tl;
7028 
7029                 sd = NULL;
7030                 for_each_sd_topology(tl) {
7031                         sd = build_sched_domain(tl, cpu_map, attr, sd, i);
7032                         if (tl == sched_domain_topology)
7033                                 *per_cpu_ptr(d.sd, i) = sd;
7034                         if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
7035                                 sd->flags |= SD_OVERLAP;
7036                         if (cpumask_equal(cpu_map, sched_domain_span(sd)))
7037                                 break;
7038                 }
7039         }
7040 
7041         /* Build the groups for the domains */
7042         for_each_cpu(i, cpu_map) {
7043                 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
7044                         sd->span_weight = cpumask_weight(sched_domain_span(sd));
7045                         if (sd->flags & SD_OVERLAP) {
7046                                 if (build_overlap_sched_groups(sd, i))
7047                                         goto error;
7048                         } else {
7049                                 if (build_sched_groups(sd, i))
7050                                         goto error;
7051                         }
7052                 }
7053         }
7054 
7055         /* Calculate CPU capacity for physical packages and nodes */
7056         for (i = nr_cpumask_bits-1; i >= 0; i--) {
7057                 if (!cpumask_test_cpu(i, cpu_map))
7058                         continue;
7059 
7060                 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
7061                         claim_allocations(i, sd);
7062                         init_sched_groups_capacity(i, sd);
7063                 }
7064         }
7065 
7066         /* Attach the domains */
7067         rcu_read_lock();
7068         for_each_cpu(i, cpu_map) {
7069                 rq = cpu_rq(i);
7070                 sd = *per_cpu_ptr(d.sd, i);
7071 
7072                 /* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */
7073                 if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity))
7074                         WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig);
7075 
7076                 cpu_attach_domain(sd, d.rd, i);
7077         }
7078         rcu_read_unlock();
7079 
7080         if (rq && sched_debug_enabled) {
7081                 pr_info("span: %*pbl (max cpu_capacity = %lu)\n",
7082                         cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);
7083         }
7084 
7085         ret = 0;
7086 error:
7087         __free_domain_allocs(&d, alloc_state, cpu_map);
7088         return ret;
7089 }
7090 
7091 static cpumask_var_t *doms_cur; /* current sched domains */
7092 static int ndoms_cur;           /* number of sched domains in 'doms_cur' */
7093 static struct sched_domain_attr *dattr_cur;
7094                                 /* attribues of custom domains in 'doms_cur' */
7095 
7096 /*
7097  * Special case: If a kmalloc of a doms_cur partition (array of
7098  * cpumask) fails, then fallback to a single sched domain,
7099  * as determined by the single cpumask fallback_doms.
7100  */
7101 static cpumask_var_t fallback_doms;
7102 
7103 /*
7104  * arch_update_cpu_topology lets virtualized architectures update the
7105  * cpu core maps. It is supposed to return 1 if the topology changed
7106  * or 0 if it stayed the same.
7107  */
7108 int __weak arch_update_cpu_topology(void)
7109 {
7110         return 0;
7111 }
7112 
7113 cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
7114 {
7115         int i;
7116         cpumask_var_t *doms;
7117 
7118         doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
7119         if (!doms)
7120                 return NULL;
7121         for (i = 0; i < ndoms; i++) {
7122                 if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
7123                         free_sched_domains(doms, i);
7124                         return NULL;
7125                 }
7126         }
7127         return doms;
7128 }
7129 
7130 void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
7131 {
7132         unsigned int i;
7133         for (i = 0; i < ndoms; i++)
7134                 free_cpumask_var(doms[i]);
7135         kfree(doms);
7136 }
7137 
7138 /*
7139  * Set up scheduler domains and groups. Callers must hold the hotplug lock.
7140  * For now this just excludes isolated cpus, but could be used to
7141  * exclude other special cases in the future.
7142  */
7143 static int init_sched_domains(const struct cpumask *cpu_map)
7144 {
7145         int err;
7146 
7147         arch_update_cpu_topology();
7148         ndoms_cur = 1;
7149         doms_cur = alloc_sched_domains(ndoms_cur);
7150         if (!doms_cur)
7151                 doms_cur = &fallback_doms;
7152         cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
7153         err = build_sched_domains(doms_cur[0], NULL);
7154         register_sched_domain_sysctl();
7155 
7156         return err;
7157 }
7158 
7159 /*
7160  * Detach sched domains from a group of cpus specified in cpu_map
7161  * These cpus will now be attached to the NULL domain
7162  */
7163 static void detach_destroy_domains(const struct cpumask *cpu_map)
7164 {
7165         int i;
7166 
7167         rcu_read_lock();
7168         for_each_cpu(i, cpu_map)
7169                 cpu_attach_domain(NULL, &def_root_domain, i);
7170         rcu_read_unlock();
7171 }
7172 
7173 /* handle null as "default" */
7174 static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7175                         struct sched_domain_attr *new, int idx_new)
7176 {
7177         struct sched_domain_attr tmp;
7178 
7179         /* fast path */
7180         if (!new && !cur)
7181                 return 1;
7182 
7183         tmp = SD_ATTR_INIT;
7184         return !memcmp(cur ? (cur + idx_cur) : &tmp,
7185                         new ? (new + idx_new) : &tmp,
7186                         sizeof(struct sched_domain_attr));
7187 }
7188 
7189 /*
7190  * Partition sched domains as specified by the 'ndoms_new'
7191  * cpumasks in the array doms_new[] of cpumasks. This compares
7192  * doms_new[] to the current sched domain partitioning, doms_cur[].
7193  * It destroys each deleted domain and builds each new domain.
7194  *
7195  * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
7196  * The masks don't intersect (don't overlap.) We should setup one
7197  * sched domain for each mask. CPUs not in any of the cpumasks will
7198  * not be load balanced. If the same cpumask appears both in the
7199  * current 'doms_cur' domains and in the new 'doms_new', we can leave
7200  * it as it is.
7201  *
7202  * The passed in 'doms_new' should be allocated using
7203  * alloc_sched_domains.  This routine takes ownership of it and will
7204  * free_sched_domains it when done with it. If the caller failed the
7205  * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
7206  * and partition_sched_domains() will fallback to the single partition
7207  * 'fallback_doms', it also forces the domains to be rebuilt.
7208  *
7209  * If doms_new == NULL it will be replaced with cpu_online_mask.
7210  * ndoms_new == 0 is a special case for destroying existing domains,
7211  * and it will not create the default domain.
7212  *
7213  * Call with hotplug lock held
7214  */
7215 void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
7216                              struct sched_domain_attr *dattr_new)
7217 {
7218         int i, j, n;
7219         int new_topology;
7220 
7221         mutex_lock(&sched_domains_mutex);
7222 
7223         /* always unregister in case we don't destroy any domains */
7224         unregister_sched_domain_sysctl();
7225 
7226         /* Let architecture update cpu core mappings. */
7227         new_topology = arch_update_cpu_topology();
7228 
7229         n = doms_new ? ndoms_new : 0;
7230 
7231         /* Destroy deleted domains */
7232         for (i = 0; i < ndoms_cur; i++) {
7233                 for (j = 0; j < n && !new_topology; j++) {
7234                         if (cpumask_equal(doms_cur[i], doms_new[j])
7235                             && dattrs_equal(dattr_cur, i, dattr_new, j))
7236                                 goto match1;
7237                 }
7238                 /* no match - a current sched domain not in new doms_new[] */
7239                 detach_destroy_domains(doms_cur[i]);
7240 match1:
7241                 ;
7242         }
7243 
7244         n = ndoms_cur;
7245         if (doms_new == NULL) {
7246                 n = 0;
7247                 doms_new = &fallback_doms;
7248                 cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
7249                 WARN_ON_ONCE(dattr_new);
7250         }
7251 
7252         /* Build new domains */
7253         for (i = 0; i < ndoms_new; i++) {
7254                 for (j = 0; j < n && !new_topology; j++) {
7255                         if (cpumask_equal(doms_new[i], doms_cur[j])
7256                             && dattrs_equal(dattr_new, i, dattr_cur, j))
7257                                 goto match2;
7258                 }
7259                 /* no match - add a new doms_new */
7260                 build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
7261 match2:
7262                 ;
7263         }
7264 
7265         /* Remember the new sched domains */
7266         if (doms_cur != &fallback_doms)
7267                 free_sched_domains(doms_cur, ndoms_cur);
7268         kfree(dattr_cur);       /* kfree(NULL) is safe */
7269         doms_cur = doms_new;
7270         dattr_cur = dattr_new;
7271         ndoms_cur = ndoms_new;
7272 
7273         register_sched_domain_sysctl();
7274 
7275         mutex_unlock(&sched_domains_mutex);
7276 }
7277 
7278 static int num_cpus_frozen;     /* used to mark begin/end of suspend/resume */
7279 
7280 /*
7281  * Update cpusets according to cpu_active mask.  If cpusets are
7282  * disabled, cpuset_update_active_cpus() becomes a simple wrapper
7283  * around partition_sched_domains().
7284  *
7285  * If we come here as part of a suspend/resume, don't touch cpusets because we
7286  * want to restore it back to its original state upon resume anyway.
7287  */
7288 static void cpuset_cpu_active(void)
7289 {
7290         if (cpuhp_tasks_frozen) {
7291                 /*
7292                  * num_cpus_frozen tracks how many CPUs are involved in suspend
7293                  * resume sequence. As long as this is not the last online
7294                  * operation in the resume sequence, just build a single sched
7295                  * domain, ignoring cpusets.
7296                  */
7297                 num_cpus_frozen--;
7298                 if (likely(num_cpus_frozen)) {
7299                         partition_sched_domains(1, NULL, NULL);
7300                         return;
7301                 }
7302                 /*
7303                  * This is the last CPU online operation. So fall through and
7304                  * restore the original sched domains by considering the
7305                  * cpuset configurations.
7306                  */
7307         }
7308         cpuset_update_active_cpus(true);
7309 }
7310 
7311 static int cpuset_cpu_inactive(unsigned int cpu)
7312 {
7313         unsigned long flags;
7314         struct dl_bw *dl_b;
7315         bool overflow;
7316         int cpus;
7317 
7318         if (!cpuhp_tasks_frozen) {
7319                 rcu_read_lock_sched();
7320                 dl_b = dl_bw_of(cpu);
7321 
7322                 raw_spin_lock_irqsave(&dl_b->lock, flags);
7323                 cpus = dl_bw_cpus(cpu);
7324                 overflow = __dl_overflow(dl_b, cpus, 0, 0);
7325                 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
7326 
7327                 rcu_read_unlock_sched();
7328 
7329                 if (overflow)
7330                         return -EBUSY;
7331                 cpuset_update_active_cpus(false);
7332         } else {
7333                 num_cpus_frozen++;
7334                 partition_sched_domains(1, NULL, NULL);
7335         }
7336         return 0;
7337 }
7338 
7339 int sched_cpu_activate(unsigned int cpu)
7340 {
7341         struct rq *rq = cpu_rq(cpu);
7342         unsigned long flags;
7343 
7344         set_cpu_active(cpu, true);
7345 
7346         if (sched_smp_initialized) {
7347                 sched_domains_numa_masks_set(cpu);
7348                 cpuset_cpu_active();
7349         }
7350 
7351         /*
7352          * Put the rq online, if not already. This happens:
7353          *
7354          * 1) In the early boot process, because we build the real domains
7355          *    after all cpus have been brought up.
7356          *
7357          * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
7358          *    domains.
7359          */
7360         raw_spin_lock_irqsave(&rq->lock, flags);
7361         if (rq->rd) {
7362                 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7363                 set_rq_online(rq);
7364         }
7365         raw_spin_unlock_irqrestore(&rq->lock, flags);
7366 
7367         update_max_interval();
7368 
7369         return 0;
7370 }
7371 
7372 int sched_cpu_deactivate(unsigned int cpu)
7373 {
7374         int ret;
7375 
7376         set_cpu_active(cpu, false);
7377         /*
7378          * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU
7379          * users of this state to go away such that all new such users will
7380          * observe it.
7381          *
7382          * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
7383          * not imply sync_sched(), so wait for both.
7384          *
7385          * Do sync before park smpboot threads to take care the rcu boost case.
7386          */
7387         if (IS_ENABLED(CONFIG_PREEMPT))
7388                 synchronize_rcu_mult(call_rcu, call_rcu_sched);
7389         else
7390                 synchronize_rcu();
7391 
7392         if (!sched_smp_initialized)
7393                 return 0;
7394 
7395         ret = cpuset_cpu_inactive(cpu);
7396         if (ret) {
7397                 set_cpu_active(cpu, true);
7398                 return ret;
7399         }
7400         sched_domains_numa_masks_clear(cpu);
7401         return 0;
7402 }
7403 
7404 static void sched_rq_cpu_starting(unsigned int cpu)
7405 {
7406         struct rq *rq = cpu_rq(cpu);
7407 
7408         rq->calc_load_update = calc_load_update;
7409         update_max_interval();
7410 }
7411 
7412 int sched_cpu_starting(unsigned int cpu)
7413 {
7414         set_cpu_rq_start_time(cpu);
7415         sched_rq_cpu_starting(cpu);
7416         return 0;
7417 }
7418 
7419 #ifdef CONFIG_HOTPLUG_CPU
7420 int sched_cpu_dying(unsigned int cpu)
7421 {
7422         struct rq *rq = cpu_rq(cpu);
7423         unsigned long flags;
7424 
7425         /* Handle pending wakeups and then migrate everything off */
7426         sched_ttwu_pending();
7427         raw_spin_lock_irqsave(&rq->lock, flags);
7428         if (rq->rd) {
7429                 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7430                 set_rq_offline(rq);
7431         }
7432         migrate_tasks(rq);
7433         BUG_ON(rq->nr_running != 1);
7434         raw_spin_unlock_irqrestore(&rq->lock, flags);
7435         calc_load_migrate(rq);
7436         update_max_interval();
7437         nohz_balance_exit_idle(cpu);
7438         hrtick_clear(rq);
7439         return 0;
7440 }
7441 #endif
7442 
7443 #ifdef CONFIG_SCHED_SMT
7444 DEFINE_STATIC_KEY_FALSE(sched_smt_present);
7445 
7446 static void sched_init_smt(void)
7447 {
7448         /*
7449          * We've enumerated all CPUs and will assume that if any CPU
7450          * has SMT siblings, CPU0 will too.
7451          */
7452         if (cpumask_weight(cpu_smt_mask(0)) > 1)
7453                 static_branch_enable(&sched_smt_present);
7454 }
7455 #else
7456 static inline void sched_init_smt(void) { }
7457 #endif
7458 
7459 void __init sched_init_smp(void)
7460 {
7461         cpumask_var_t non_isolated_cpus;
7462 
7463         alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7464         alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
7465 
7466         sched_init_numa();
7467 
7468         /*
7469          * There's no userspace yet to cause hotplug operations; hence all the
7470          * cpu masks are stable and all blatant races in the below code cannot
7471          * happen.
7472          */
7473         mutex_lock(&sched_domains_mutex);
7474         init_sched_domains(cpu_active_mask);
7475         cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
7476         if (cpumask_empty(non_isolated_cpus))
7477                 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
7478         mutex_unlock(&sched_domains_mutex);
7479 
7480         /* Move init over to a non-isolated CPU */
7481         if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
7482                 BUG();
7483         sched_init_granularity();
7484         free_cpumask_var(non_isolated_cpus);
7485 
7486         init_sched_rt_class();
7487         init_sched_dl_class();
7488 
7489         sched_init_smt();
7490 
7491         sched_smp_initialized = true;
7492 }
7493 
7494 static int __init migration_init(void)
7495 {
7496         sched_rq_cpu_starting(smp_processor_id());
7497         return 0;
7498 }
7499 early_initcall(migration_init);
7500 
7501 #else
7502 void __init sched_init_smp(void)
7503 {
7504         sched_init_granularity();
7505 }
7506 #endif /* CONFIG_SMP */
7507 
7508 int in_sched_functions(unsigned long addr)
7509 {
7510         return in_lock_functions(addr) ||
7511                 (addr >= (unsigned long)__sched_text_start
7512                 && addr < (unsigned long)__sched_text_end);
7513 }
7514 
7515 #ifdef CONFIG_CGROUP_SCHED
7516 /*
7517  * Default task group.
7518  * Every task in system belongs to this group at bootup.
7519  */
7520 struct task_group root_task_group;
7521 LIST_HEAD(task_groups);
7522 
7523 /* Cacheline aligned slab cache for task_group */
7524 static struct kmem_cache *task_group_cache __read_mostly;
7525 #endif
7526 
7527 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
7528 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
7529 
7530 #define WAIT_TABLE_BITS 8
7531 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
7532 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
7533 
7534 wait_queue_head_t *bit_waitqueue(void *word, int bit)
7535 {
7536         const int shift = BITS_PER_LONG == 32 ? 5 : 6;
7537         unsigned long val = (unsigned long)word << shift | bit;
7538 
7539         return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
7540 }
7541 EXPORT_SYMBOL(bit_waitqueue);
7542 
7543 void __init sched_init(void)
7544 {
7545         int i, j;
7546         unsigned long alloc_size = 0, ptr;
7547 
7548         for (i = 0; i < WAIT_TABLE_SIZE; i++)
7549                 init_waitqueue_head(bit_wait_table + i);
7550 
7551 #ifdef CONFIG_FAIR_GROUP_SCHED
7552         alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7553 #endif
7554 #ifdef CONFIG_RT_GROUP_SCHED
7555         alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7556 #endif
7557         if (alloc_size) {
7558                 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
7559 
7560 #ifdef CONFIG_FAIR_GROUP_SCHED
7561                 root_task_group.se = (struct sched_entity **)ptr;
7562                 ptr += nr_cpu_ids * sizeof(void **);
7563 
7564                 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
7565                 ptr += nr_cpu_ids * sizeof(void **);
7566 
7567 #endif /* CONFIG_FAIR_GROUP_SCHED */
7568 #ifdef CONFIG_RT_GROUP_SCHED
7569                 root_task_group.rt_se = (struct sched_rt_entity **)ptr;
7570                 ptr += nr_cpu_ids * sizeof(void **);
7571 
7572                 root_task_group.rt_rq = (struct rt_rq **)ptr;
7573                 ptr += nr_cpu_ids * sizeof(void **);
7574 
7575 #endif /* CONFIG_RT_GROUP_SCHED */
7576         }
7577 #ifdef CONFIG_CPUMASK_OFFSTACK
7578         for_each_possible_cpu(i) {
7579                 per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
7580                         cpumask_size(), GFP_KERNEL, cpu_to_node(i));
7581                 per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
7582                         cpumask_size(), GFP_KERNEL, cpu_to_node(i));
7583         }
7584 #endif /* CONFIG_CPUMASK_OFFSTACK */
7585 
7586         init_rt_bandwidth(&def_rt_bandwidth,
7587                         global_rt_period(), global_rt_runtime());
7588         init_dl_bandwidth(&def_dl_bandwidth,
7589                         global_rt_period(), global_rt_runtime());
7590 
7591 #ifdef CONFIG_SMP
7592         init_defrootdomain();
7593 #endif
7594 
7595 #ifdef CONFIG_RT_GROUP_SCHED
7596         init_rt_bandwidth(&root_task_group.rt_bandwidth,
7597                         global_rt_period(), global_rt_runtime());
7598 #endif /* CONFIG_RT_GROUP_SCHED */
7599 
7600 #ifdef CONFIG_CGROUP_SCHED
7601         task_group_cache = KMEM_CACHE(task_group, 0);
7602 
7603         list_add(&root_task_group.list, &task_groups);
7604         INIT_LIST_HEAD(&root_task_group.children);
7605         INIT_LIST_HEAD(&root_task_group.siblings);
7606         autogroup_init(&init_task);
7607 #endif /* CONFIG_CGROUP_SCHED */
7608 
7609         for_each_possible_cpu(i) {
7610                 struct rq *rq;
7611 
7612                 rq = cpu_rq(i);
7613                 raw_spin_lock_init(&rq->lock);
7614                 rq->nr_running = 0;
7615                 rq->calc_load_active = 0;
7616                 rq->calc_load_update = jiffies + LOAD_FREQ;
7617                 init_cfs_rq(&rq->cfs);
7618                 init_rt_rq(&rq->rt);
7619                 init_dl_rq(&rq->dl);
7620 #ifdef CONFIG_FAIR_GROUP_SCHED
7621                 root_task_group.shares = ROOT_TASK_GROUP_LOAD;
7622                 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
7623                 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
7624                 /*
7625                  * How much cpu bandwidth does root_task_group get?
7626                  *
7627                  * In case of task-groups formed thr' the cgroup filesystem, it
7628                  * gets 100% of the cpu resources in the system. This overall
7629                  * system cpu resource is divided among the tasks of
7630                  * root_task_group and its child task-groups in a fair manner,
7631                  * based on each entity's (task or task-group's) weight
7632                  * (se->load.weight).
7633                  *
7634                  * In other words, if root_task_group has 10 tasks of weight
7635                  * 1024) and two child groups A0 and A1 (of weight 1024 each),
7636                  * then A0's share of the cpu resource is:
7637                  *
7638                  *      A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
7639                  *
7640                  * We achieve this by letting root_task_group's tasks sit
7641                  * directly in rq->cfs (i.e root_task_group->se[] = NULL).
7642                  */
7643                 init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
7644                 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
7645 #endif /* CONFIG_FAIR_GROUP_SCHED */
7646 
7647                 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
7648 #ifdef CONFIG_RT_GROUP_SCHED
7649                 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
7650 #endif
7651 
7652                 for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
7653                         rq->cpu_load[j] = 0;
7654 
7655 #ifdef CONFIG_SMP
7656                 rq->sd = NULL;
7657                 rq->rd = NULL;
7658                 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE;
7659                 rq->balance_callback = NULL;
7660                 rq->active_balance = 0;
7661                 rq->next_balance = jiffies;
7662                 rq->push_cpu = 0;
7663                 rq->cpu = i;
7664                 rq->online = 0;
7665                 rq->idle_stamp = 0;
7666                 rq->avg_idle = 2*sysctl_sched_migration_cost;
7667                 rq->max_idle_balance_cost = sysctl_sched_migration_cost;
7668 
7669                 INIT_LIST_HEAD(&rq->cfs_tasks);
7670 
7671                 rq_attach_root(rq, &def_root_domain);
7672 #ifdef CONFIG_NO_HZ_COMMON
7673                 rq->last_load_update_tick = jiffies;
7674                 rq->nohz_flags = 0;
7675 #endif
7676 #ifdef CONFIG_NO_HZ_FULL
7677                 rq->last_sched_tick = 0;
7678 #endif
7679 #endif /* CONFIG_SMP */
7680                 init_rq_hrtick(rq);
7681                 atomic_set(&rq->nr_iowait, 0);
7682         }
7683 
7684         set_load_weight(&init_task);
7685 
7686         /*
7687          * The boot idle thread does lazy MMU switching as well:
7688          */
7689         atomic_inc(&init_mm.mm_count);
7690         enter_lazy_tlb(&init_mm, current);
7691 
7692         /*
7693          * Make us the idle thread. Technically, schedule() should not be
7694          * called from this thread, however somewhere below it might be,
7695          * but because we are the idle thread, we just pick up running again
7696          * when this runqueue becomes "idle".
7697          */
7698         init_idle(current, smp_processor_id());
7699 
7700         calc_load_update = jiffies + LOAD_FREQ;
7701 
7702 #ifdef CONFIG_SMP
7703         zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
7704         /* May be allocated at isolcpus cmdline parse time */
7705         if (cpu_isolated_map == NULL)
7706                 zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
7707         idle_thread_set_boot_cpu();
7708         set_cpu_rq_start_time(smp_processor_id());
7709 #endif
7710         init_sched_fair_class();
7711 
7712         init_schedstats();
7713 
7714         scheduler_running = 1;
7715 }
7716 
7717 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
7718 static inline int preempt_count_equals(int preempt_offset)
7719 {
7720         int nested = preempt_count() + rcu_preempt_depth();
7721 
7722         return (nested == preempt_offset);
7723 }
7724 
7725 void __might_sleep(const char *file, int line, int preempt_offset)
7726 {
7727         /*
7728          * Blocking primitives will set (and therefore destroy) current->state,
7729          * since we will exit with TASK_RUNNING make sure we enter with it,
7730          * otherwise we will destroy state.
7731          */
7732         WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
7733                         "do not call blocking ops when !TASK_RUNNING; "
7734                         "state=%lx set at [<%p>] %pS\n",
7735                         current->state,
7736                         (void *)current->task_state_change,
7737                         (void *)current->task_state_change);
7738 
7739         ___might_sleep(file, line, preempt_offset);
7740 }
7741 EXPORT_SYMBOL(__might_sleep);
7742 
7743 void ___might_sleep(const char *file, int line, int preempt_offset)
7744 {
7745         static unsigned long prev_jiffy;        /* ratelimiting */
7746         unsigned long preempt_disable_ip;
7747 
7748         rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
7749         if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
7750              !is_idle_task(current)) ||
7751             system_state != SYSTEM_RUNNING || oops_in_progress)
7752                 return;
7753         if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
7754                 return;
7755         prev_jiffy = jiffies;
7756 
7757         /* Save this before calling printk(), since that will clobber it */
7758         preempt_disable_ip = get_preempt_disable_ip(current);
7759 
7760         printk(KERN_ERR
7761                 "BUG: sleeping function called from invalid context at %s:%d\n",
7762                         file, line);
7763         printk(KERN_ERR
7764                 "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
7765                         in_atomic(), irqs_disabled(),
7766                         current->pid, current->comm);
7767 
7768         if (task_stack_end_corrupted(current))
7769                 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
7770 
7771         debug_show_held_locks(current);
7772         if (irqs_disabled())
7773                 print_irqtrace_events(current);
7774         if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
7775             && !preempt_count_equals(preempt_offset)) {
7776                 pr_err("Preemption disabled at:");
7777                 print_ip_sym(preempt_disable_ip);
7778                 pr_cont("\n");
7779         }
7780         dump_stack();
7781         add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
7782 }
7783 EXPORT_SYMBOL(___might_sleep);
7784 #endif
7785 
7786 #ifdef CONFIG_MAGIC_SYSRQ
7787 void normalize_rt_tasks(void)
7788 {
7789         struct task_struct *g, *p;
7790         struct sched_attr attr = {
7791                 .sched_policy = SCHED_NORMAL,
7792         };
7793 
7794         read_lock(&tasklist_lock);
7795         for_each_process_thread(g, p) {
7796                 /*
7797                  * Only normalize user tasks:
7798                  */
7799                 if (p->flags & PF_KTHREAD)
7800                         continue;
7801 
7802                 p->se.exec_start = 0;
7803                 schedstat_set(p->se.statistics.wait_start,  0);
7804                 schedstat_set(p->se.statistics.sleep_start, 0);
7805                 schedstat_set(p->se.statistics.block_start, 0);
7806 
7807                 if (!dl_task(p) && !rt_task(p)) {
7808                         /*
7809                          * Renice negative nice level userspace
7810                          * tasks back to 0:
7811                          */
7812                         if (task_nice(p) < 0)
7813                                 set_user_nice(p, 0);
7814                         continue;
7815                 }
7816 
7817                 __sched_setscheduler(p, &attr, false, false);
7818         }
7819         read_unlock(&tasklist_lock);
7820 }
7821 
7822 #endif /* CONFIG_MAGIC_SYSRQ */
7823 
7824 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
7825 /*
7826  * These functions are only useful for the IA64 MCA handling, or kdb.
7827  *
7828  * They can only be called when the whole system has been
7829  * stopped - every CPU needs to be quiescent, and no scheduling
7830  * activity can take place. Using them for anything else would
7831  * be a serious bug, and as a result, they aren't even visible
7832  * under any other configuration.
7833  */
7834 
7835 /**
7836  * curr_task - return the current task for a given cpu.
7837  * @cpu: the processor in question.
7838  *
7839  * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7840  *
7841  * Return: The current task for @cpu.
7842  */
7843 struct task_struct *curr_task(int cpu)
7844 {
7845         return cpu_curr(cpu);
7846 }
7847 
7848 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
7849 
7850 #ifdef CONFIG_IA64
7851 /**
7852  * set_curr_task - set the current task for a given cpu.
7853  * @cpu: the processor in question.
7854  * @p: the task pointer to set.
7855  *
7856  * Description: This function must only be used when non-maskable interrupts
7857  * are serviced on a separate stack. It allows the architecture to switch the
7858  * notion of the current task on a cpu in a non-blocking manner. This function
7859  * must be called with all CPU's synchronized, and interrupts disabled, the
7860  * and caller must save the original value of the current task (see
7861  * curr_task() above) and restore that value before reenabling interrupts and
7862  * re-starting the system.
7863  *
7864  * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7865  */
7866 void ia64_set_curr_task(int cpu, struct task_struct *p)
7867 {
7868         cpu_curr(cpu) = p;
7869 }
7870 
7871 #endif
7872 
7873 #ifdef CONFIG_CGROUP_SCHED
7874 /* task_group_lock serializes the addition/removal of task groups */
7875 static DEFINE_SPINLOCK(task_group_lock);
7876 
7877 static void sched_free_group(struct task_group *tg)
7878 {
7879         free_fair_sched_group(tg);
7880         free_rt_sched_group(tg);
7881         autogroup_free(tg);
7882         kmem_cache_free(task_group_cache, tg);
7883 }
7884 
7885 /* allocate runqueue etc for a new task group */
7886 struct task_group *sched_create_group(struct task_group *parent)
7887 {
7888         struct task_group *tg;
7889 
7890         tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
7891         if (!tg)
7892                 return ERR_PTR(-ENOMEM);
7893 
7894         if (!alloc_fair_sched_group(tg, parent))
7895                 goto err;
7896 
7897         if (!alloc_rt_sched_group(tg, parent))
7898                 goto err;
7899 
7900         return tg;
7901 
7902 err:
7903         sched_free_group(tg);
7904         return ERR_PTR(-ENOMEM);
7905 }
7906 
7907 void sched_online_group(struct task_group *tg, struct task_group *parent)
7908 {
7909         unsigned long flags;
7910 
7911         spin_lock_irqsave(&task_group_lock, flags);
7912         list_add_rcu(&tg->list, &task_groups);
7913 
7914         WARN_ON(!parent); /* root should already exist */
7915 
7916         tg->parent = parent;
7917         INIT_LIST_HEAD(&tg->children);
7918         list_add_rcu(&tg->siblings, &parent->children);
7919         spin_unlock_irqrestore(&task_group_lock, flags);
7920 
7921         online_fair_sched_group(tg);
7922 }
7923 
7924 /* rcu callback to free various structures associated with a task group */
7925 static void sched_free_group_rcu(struct rcu_head *rhp)
7926 {
7927         /* now it should be safe to free those cfs_rqs */
7928         sched_free_group(container_of(rhp, struct task_group, rcu));
7929 }
7930 
7931 void sched_destroy_group(struct task_group *tg)
7932 {
7933         /* wait for possible concurrent references to cfs_rqs complete */
7934         call_rcu(&tg->rcu, sched_free_group_rcu);
7935 }
7936 
7937 void sched_offline_group(struct task_group *tg)
7938 {
7939         unsigned long flags;
7940 
7941         /* end participation in shares distribution */
7942         unregister_fair_sched_group(tg);
7943 
7944         spin_lock_irqsave(&task_group_lock, flags);
7945         list_del_rcu(&tg->list);
7946         list_del_rcu(&tg->siblings);
7947         spin_unlock_irqrestore(&task_group_lock, flags);
7948 }
7949 
7950 static void sched_change_group(struct task_struct *tsk, int type)
7951 {
7952         struct task_group *tg;
7953 
7954         /*
7955          * All callers are synchronized by task_rq_lock(); we do not use RCU
7956          * which is pointless here. Thus, we pass "true" to task_css_check()
7957          * to prevent lockdep warnings.
7958          */
7959         tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
7960                           struct task_group, css);
7961         tg = autogroup_task_group(tsk, tg);
7962         tsk->sched_task_group = tg;
7963 
7964 #ifdef CONFIG_FAIR_GROUP_SCHED
7965         if (tsk->sched_class->task_change_group)
7966                 tsk->sched_class->task_change_group(tsk, type);
7967         else
7968 #endif
7969                 set_task_rq(tsk, task_cpu(tsk));
7970 }
7971 
7972 /*
7973  * Change task's runqueue when it moves between groups.
7974  *
7975  * The caller of this function should have put the task in its new group by
7976  * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect
7977  * its new group.
7978  */
7979 void sched_move_task(struct task_struct *tsk)
7980 {
7981         int queued, running;
7982         struct rq_flags rf;
7983         struct rq *rq;
7984 
7985         rq = task_rq_lock(tsk, &rf);
7986 
7987         running = task_current(rq, tsk);
7988         queued = task_on_rq_queued(tsk);
7989 
7990         if (queued)
7991                 dequeue_task(rq, tsk, DEQUEUE_SAVE | DEQUEUE_MOVE);
7992         if (unlikely(running))
7993                 put_prev_task(rq, tsk);
7994 
7995         sched_change_group(tsk, TASK_MOVE_GROUP);
7996 
7997         if (queued)
7998                 enqueue_task(rq, tsk, ENQUEUE_RESTORE | ENQUEUE_MOVE);
7999         if (unlikely(running))
8000                 set_curr_task(rq, tsk);
8001 
8002         task_rq_unlock(rq, tsk, &rf);
8003 }
8004 #endif /* CONFIG_CGROUP_SCHED */
8005 
8006 #ifdef CONFIG_RT_GROUP_SCHED
8007 /*
8008  * Ensure that the real time constraints are schedulable.
8009  */
8010 static DEFINE_MUTEX(rt_constraints_mutex);
8011 
8012 /* Must be called with tasklist_lock held */
8013 static inline int tg_has_rt_tasks(struct task_group *tg)
8014 {
8015         struct task_struct *g, *p;
8016 
8017         /*
8018          * Autogroups do not have RT tasks; see autogroup_create().
8019          */
8020         if (task_group_is_autogroup(tg))
8021                 return 0;
8022 
8023         for_each_process_thread(g, p) {
8024                 if (rt_task(p) && task_group(p) == tg)
8025                         return 1;
8026         }
8027 
8028         return 0;
8029 }
8030 
8031 struct rt_schedulable_data {
8032         struct task_group *tg;
8033         u64 rt_period;
8034         u64 rt_runtime;
8035 };
8036 
8037 static int tg_rt_schedulable(struct task_group *tg, void *data)
8038 {
8039         struct rt_schedulable_data *d = data;
8040         struct task_group *child;
8041         unsigned long total, sum = 0;
8042         u64 period, runtime;
8043 
8044         period = ktime_to_ns(tg->rt_bandwidth.rt_period);
8045         runtime = tg->rt_bandwidth.rt_runtime;
8046 
8047         if (tg == d->tg) {
8048                 period = d->rt_period;
8049                 runtime = d->rt_runtime;
8050         }
8051 
8052         /*
8053          * Cannot have more runtime than the period.
8054          */
8055         if (runtime > period && runtime != RUNTIME_INF)
8056                 return -EINVAL;
8057 
8058         /*
8059          * Ensure we don't starve existing RT tasks.
8060          */
8061         if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
8062                 return -EBUSY;
8063 
8064         total = to_ratio(period, runtime);
8065 
8066         /*
8067          * Nobody can have more than the global setting allows.
8068          */
8069         if (total > to_ratio(global_rt_period(), global_rt_runtime()))
8070                 return -EINVAL;
8071 
8072         /*
8073          * The sum of our children's runtime should not exceed our own.
8074          */
8075         list_for_each_entry_rcu(child, &tg->children, siblings) {
8076                 period = ktime_to_ns(child->rt_bandwidth.rt_period);
8077                 runtime = child->rt_bandwidth.rt_runtime;
8078 
8079                 if (child == d->tg) {
8080                         period = d->rt_period;
8081                         runtime = d->rt_runtime;
8082                 }
8083 
8084                 sum += to_ratio(period, runtime);
8085         }
8086 
8087         if (sum > total)
8088                 return -EINVAL;
8089 
8090         return 0;
8091 }
8092 
8093 static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
8094 {
8095         int ret;
8096 
8097         struct rt_schedulable_data data = {
8098                 .tg = tg,
8099                 .rt_period = period,
8100                 .rt_runtime = runtime,
8101         };
8102 
8103         rcu_read_lock();
8104         ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
8105         rcu_read_unlock();
8106 
8107         return ret;
8108 }
8109 
8110 static int tg_set_rt_bandwidth(struct task_group *tg,
8111                 u64 rt_period, u64 rt_runtime)
8112 {
8113         int i, err = 0;
8114 
8115         /*
8116          * Disallowing the root group RT runtime is BAD, it would disallow the
8117          * kernel creating (and or operating) RT threads.
8118          */
8119         if (tg == &root_task_group && rt_runtime == 0)
8120                 return -EINVAL;
8121 
8122         /* No period doesn't make any sense. */
8123         if (rt_period == 0)
8124                 return -EINVAL;
8125 
8126         mutex_lock(&rt_constraints_mutex);
8127         read_lock(&tasklist_lock);
8128         err = __rt_schedulable(tg, rt_period, rt_runtime);
8129         if (err)
8130                 goto unlock;
8131 
8132         raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
8133         tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
8134         tg->rt_bandwidth.rt_runtime = rt_runtime;
8135 
8136         for_each_possible_cpu(i) {
8137                 struct rt_rq *rt_rq = tg->rt_rq[i];
8138 
8139                 raw_spin_lock(&rt_rq->rt_runtime_lock);
8140                 rt_rq->rt_runtime = rt_runtime;
8141                 raw_spin_unlock(&rt_rq->rt_runtime_lock);
8142         }
8143         raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
8144 unlock:
8145         read_unlock(&tasklist_lock);
8146         mutex_unlock(&rt_constraints_mutex);
8147 
8148         return err;