Version:  2.0.40 2.2.26 2.4.37 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5

Linux/kernel/time/tick-sched.c

  1 /*
  2  *  linux/kernel/time/tick-sched.c
  3  *
  4  *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  5  *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  6  *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
  7  *
  8  *  No idle tick implementation for low and high resolution timers
  9  *
 10  *  Started by: Thomas Gleixner and Ingo Molnar
 11  *
 12  *  Distribute under GPLv2.
 13  */
 14 #include <linux/cpu.h>
 15 #include <linux/err.h>
 16 #include <linux/hrtimer.h>
 17 #include <linux/interrupt.h>
 18 #include <linux/kernel_stat.h>
 19 #include <linux/percpu.h>
 20 #include <linux/profile.h>
 21 #include <linux/sched.h>
 22 #include <linux/module.h>
 23 #include <linux/irq_work.h>
 24 #include <linux/posix-timers.h>
 25 #include <linux/perf_event.h>
 26 #include <linux/context_tracking.h>
 27 
 28 #include <asm/irq_regs.h>
 29 
 30 #include "tick-internal.h"
 31 
 32 #include <trace/events/timer.h>
 33 
 34 /*
 35  * Per cpu nohz control structure
 36  */
 37 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
 38 
 39 struct tick_sched *tick_get_tick_sched(int cpu)
 40 {
 41         return &per_cpu(tick_cpu_sched, cpu);
 42 }
 43 
 44 #if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
 45 /*
 46  * The time, when the last jiffy update happened. Protected by jiffies_lock.
 47  */
 48 static ktime_t last_jiffies_update;
 49 
 50 /*
 51  * Must be called with interrupts disabled !
 52  */
 53 static void tick_do_update_jiffies64(ktime_t now)
 54 {
 55         unsigned long ticks = 0;
 56         ktime_t delta;
 57 
 58         /*
 59          * Do a quick check without holding jiffies_lock:
 60          */
 61         delta = ktime_sub(now, last_jiffies_update);
 62         if (delta.tv64 < tick_period.tv64)
 63                 return;
 64 
 65         /* Reevalute with jiffies_lock held */
 66         write_seqlock(&jiffies_lock);
 67 
 68         delta = ktime_sub(now, last_jiffies_update);
 69         if (delta.tv64 >= tick_period.tv64) {
 70 
 71                 delta = ktime_sub(delta, tick_period);
 72                 last_jiffies_update = ktime_add(last_jiffies_update,
 73                                                 tick_period);
 74 
 75                 /* Slow path for long timeouts */
 76                 if (unlikely(delta.tv64 >= tick_period.tv64)) {
 77                         s64 incr = ktime_to_ns(tick_period);
 78 
 79                         ticks = ktime_divns(delta, incr);
 80 
 81                         last_jiffies_update = ktime_add_ns(last_jiffies_update,
 82                                                            incr * ticks);
 83                 }
 84                 do_timer(++ticks);
 85 
 86                 /* Keep the tick_next_period variable up to date */
 87                 tick_next_period = ktime_add(last_jiffies_update, tick_period);
 88         } else {
 89                 write_sequnlock(&jiffies_lock);
 90                 return;
 91         }
 92         write_sequnlock(&jiffies_lock);
 93         update_wall_time();
 94 }
 95 
 96 /*
 97  * Initialize and return retrieve the jiffies update.
 98  */
 99 static ktime_t tick_init_jiffy_update(void)
100 {
101         ktime_t period;
102 
103         write_seqlock(&jiffies_lock);
104         /* Did we start the jiffies update yet ? */
105         if (last_jiffies_update.tv64 == 0)
106                 last_jiffies_update = tick_next_period;
107         period = last_jiffies_update;
108         write_sequnlock(&jiffies_lock);
109         return period;
110 }
111 
112 
113 static void tick_sched_do_timer(ktime_t now)
114 {
115         int cpu = smp_processor_id();
116 
117 #ifdef CONFIG_NO_HZ_COMMON
118         /*
119          * Check if the do_timer duty was dropped. We don't care about
120          * concurrency: This happens only when the cpu in charge went
121          * into a long sleep. If two cpus happen to assign themself to
122          * this duty, then the jiffies update is still serialized by
123          * jiffies_lock.
124          */
125         if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)
126             && !tick_nohz_full_cpu(cpu))
127                 tick_do_timer_cpu = cpu;
128 #endif
129 
130         /* Check, if the jiffies need an update */
131         if (tick_do_timer_cpu == cpu)
132                 tick_do_update_jiffies64(now);
133 }
134 
135 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
136 {
137 #ifdef CONFIG_NO_HZ_COMMON
138         /*
139          * When we are idle and the tick is stopped, we have to touch
140          * the watchdog as we might not schedule for a really long
141          * time. This happens on complete idle SMP systems while
142          * waiting on the login prompt. We also increment the "start of
143          * idle" jiffy stamp so the idle accounting adjustment we do
144          * when we go busy again does not account too much ticks.
145          */
146         if (ts->tick_stopped) {
147                 touch_softlockup_watchdog_sched();
148                 if (is_idle_task(current))
149                         ts->idle_jiffies++;
150         }
151 #endif
152         update_process_times(user_mode(regs));
153         profile_tick(CPU_PROFILING);
154 }
155 #endif
156 
157 #ifdef CONFIG_NO_HZ_FULL
158 cpumask_var_t tick_nohz_full_mask;
159 cpumask_var_t housekeeping_mask;
160 bool tick_nohz_full_running;
161 
162 static bool can_stop_full_tick(void)
163 {
164         WARN_ON_ONCE(!irqs_disabled());
165 
166         if (!sched_can_stop_tick()) {
167                 trace_tick_stop(0, "more than 1 task in runqueue\n");
168                 return false;
169         }
170 
171         if (!posix_cpu_timers_can_stop_tick(current)) {
172                 trace_tick_stop(0, "posix timers running\n");
173                 return false;
174         }
175 
176         if (!perf_event_can_stop_tick()) {
177                 trace_tick_stop(0, "perf events running\n");
178                 return false;
179         }
180 
181         /* sched_clock_tick() needs us? */
182 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
183         /*
184          * TODO: kick full dynticks CPUs when
185          * sched_clock_stable is set.
186          */
187         if (!sched_clock_stable()) {
188                 trace_tick_stop(0, "unstable sched clock\n");
189                 /*
190                  * Don't allow the user to think they can get
191                  * full NO_HZ with this machine.
192                  */
193                 WARN_ONCE(tick_nohz_full_running,
194                           "NO_HZ FULL will not work with unstable sched clock");
195                 return false;
196         }
197 #endif
198 
199         return true;
200 }
201 
202 static void nohz_full_kick_work_func(struct irq_work *work)
203 {
204         /* Empty, the tick restart happens on tick_nohz_irq_exit() */
205 }
206 
207 static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
208         .func = nohz_full_kick_work_func,
209 };
210 
211 /*
212  * Kick this CPU if it's full dynticks in order to force it to
213  * re-evaluate its dependency on the tick and restart it if necessary.
214  * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
215  * is NMI safe.
216  */
217 void tick_nohz_full_kick(void)
218 {
219         if (!tick_nohz_full_cpu(smp_processor_id()))
220                 return;
221 
222         irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
223 }
224 
225 /*
226  * Kick the CPU if it's full dynticks in order to force it to
227  * re-evaluate its dependency on the tick and restart it if necessary.
228  */
229 void tick_nohz_full_kick_cpu(int cpu)
230 {
231         if (!tick_nohz_full_cpu(cpu))
232                 return;
233 
234         irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
235 }
236 
237 static void nohz_full_kick_ipi(void *info)
238 {
239         /* Empty, the tick restart happens on tick_nohz_irq_exit() */
240 }
241 
242 /*
243  * Kick all full dynticks CPUs in order to force these to re-evaluate
244  * their dependency on the tick and restart it if necessary.
245  */
246 void tick_nohz_full_kick_all(void)
247 {
248         if (!tick_nohz_full_running)
249                 return;
250 
251         preempt_disable();
252         smp_call_function_many(tick_nohz_full_mask,
253                                nohz_full_kick_ipi, NULL, false);
254         tick_nohz_full_kick();
255         preempt_enable();
256 }
257 
258 /*
259  * Re-evaluate the need for the tick as we switch the current task.
260  * It might need the tick due to per task/process properties:
261  * perf events, posix cpu timers, ...
262  */
263 void __tick_nohz_task_switch(void)
264 {
265         unsigned long flags;
266 
267         local_irq_save(flags);
268 
269         if (!tick_nohz_full_cpu(smp_processor_id()))
270                 goto out;
271 
272         if (tick_nohz_tick_stopped() && !can_stop_full_tick())
273                 tick_nohz_full_kick();
274 
275 out:
276         local_irq_restore(flags);
277 }
278 
279 /* Parse the boot-time nohz CPU list from the kernel parameters. */
280 static int __init tick_nohz_full_setup(char *str)
281 {
282         alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
283         if (cpulist_parse(str, tick_nohz_full_mask) < 0) {
284                 pr_warning("NOHZ: Incorrect nohz_full cpumask\n");
285                 free_bootmem_cpumask_var(tick_nohz_full_mask);
286                 return 1;
287         }
288         tick_nohz_full_running = true;
289 
290         return 1;
291 }
292 __setup("nohz_full=", tick_nohz_full_setup);
293 
294 static int tick_nohz_cpu_down_callback(struct notifier_block *nfb,
295                                        unsigned long action,
296                                        void *hcpu)
297 {
298         unsigned int cpu = (unsigned long)hcpu;
299 
300         switch (action & ~CPU_TASKS_FROZEN) {
301         case CPU_DOWN_PREPARE:
302                 /*
303                  * The boot CPU handles housekeeping duty (unbound timers,
304                  * workqueues, timekeeping, ...) on behalf of full dynticks
305                  * CPUs. It must remain online when nohz full is enabled.
306                  */
307                 if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
308                         return NOTIFY_BAD;
309                 break;
310         }
311         return NOTIFY_OK;
312 }
313 
314 static int tick_nohz_init_all(void)
315 {
316         int err = -1;
317 
318 #ifdef CONFIG_NO_HZ_FULL_ALL
319         if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) {
320                 WARN(1, "NO_HZ: Can't allocate full dynticks cpumask\n");
321                 return err;
322         }
323         err = 0;
324         cpumask_setall(tick_nohz_full_mask);
325         tick_nohz_full_running = true;
326 #endif
327         return err;
328 }
329 
330 void __init tick_nohz_init(void)
331 {
332         int cpu;
333 
334         if (!tick_nohz_full_running) {
335                 if (tick_nohz_init_all() < 0)
336                         return;
337         }
338 
339         if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) {
340                 WARN(1, "NO_HZ: Can't allocate not-full dynticks cpumask\n");
341                 cpumask_clear(tick_nohz_full_mask);
342                 tick_nohz_full_running = false;
343                 return;
344         }
345 
346         /*
347          * Full dynticks uses irq work to drive the tick rescheduling on safe
348          * locking contexts. But then we need irq work to raise its own
349          * interrupts to avoid circular dependency on the tick
350          */
351         if (!arch_irq_work_has_interrupt()) {
352                 pr_warning("NO_HZ: Can't run full dynticks because arch doesn't "
353                            "support irq work self-IPIs\n");
354                 cpumask_clear(tick_nohz_full_mask);
355                 cpumask_copy(housekeeping_mask, cpu_possible_mask);
356                 tick_nohz_full_running = false;
357                 return;
358         }
359 
360         cpu = smp_processor_id();
361 
362         if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
363                 pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu);
364                 cpumask_clear_cpu(cpu, tick_nohz_full_mask);
365         }
366 
367         cpumask_andnot(housekeeping_mask,
368                        cpu_possible_mask, tick_nohz_full_mask);
369 
370         for_each_cpu(cpu, tick_nohz_full_mask)
371                 context_tracking_cpu_set(cpu);
372 
373         cpu_notifier(tick_nohz_cpu_down_callback, 0);
374         pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
375                 cpumask_pr_args(tick_nohz_full_mask));
376 
377         /*
378          * We need at least one CPU to handle housekeeping work such
379          * as timekeeping, unbound timers, workqueues, ...
380          */
381         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
382 }
383 #endif
384 
385 /*
386  * NOHZ - aka dynamic tick functionality
387  */
388 #ifdef CONFIG_NO_HZ_COMMON
389 /*
390  * NO HZ enabled ?
391  */
392 int tick_nohz_enabled __read_mostly = 1;
393 unsigned long tick_nohz_active  __read_mostly;
394 /*
395  * Enable / Disable tickless mode
396  */
397 static int __init setup_tick_nohz(char *str)
398 {
399         if (!strcmp(str, "off"))
400                 tick_nohz_enabled = 0;
401         else if (!strcmp(str, "on"))
402                 tick_nohz_enabled = 1;
403         else
404                 return 0;
405         return 1;
406 }
407 
408 __setup("nohz=", setup_tick_nohz);
409 
410 int tick_nohz_tick_stopped(void)
411 {
412         return __this_cpu_read(tick_cpu_sched.tick_stopped);
413 }
414 
415 /**
416  * tick_nohz_update_jiffies - update jiffies when idle was interrupted
417  *
418  * Called from interrupt entry when the CPU was idle
419  *
420  * In case the sched_tick was stopped on this CPU, we have to check if jiffies
421  * must be updated. Otherwise an interrupt handler could use a stale jiffy
422  * value. We do this unconditionally on any cpu, as we don't know whether the
423  * cpu, which has the update task assigned is in a long sleep.
424  */
425 static void tick_nohz_update_jiffies(ktime_t now)
426 {
427         unsigned long flags;
428 
429         __this_cpu_write(tick_cpu_sched.idle_waketime, now);
430 
431         local_irq_save(flags);
432         tick_do_update_jiffies64(now);
433         local_irq_restore(flags);
434 
435         touch_softlockup_watchdog_sched();
436 }
437 
438 /*
439  * Updates the per cpu time idle statistics counters
440  */
441 static void
442 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
443 {
444         ktime_t delta;
445 
446         if (ts->idle_active) {
447                 delta = ktime_sub(now, ts->idle_entrytime);
448                 if (nr_iowait_cpu(cpu) > 0)
449                         ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
450                 else
451                         ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
452                 ts->idle_entrytime = now;
453         }
454 
455         if (last_update_time)
456                 *last_update_time = ktime_to_us(now);
457 
458 }
459 
460 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
461 {
462         update_ts_time_stats(smp_processor_id(), ts, now, NULL);
463         ts->idle_active = 0;
464 
465         sched_clock_idle_wakeup_event(0);
466 }
467 
468 static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
469 {
470         ktime_t now = ktime_get();
471 
472         ts->idle_entrytime = now;
473         ts->idle_active = 1;
474         sched_clock_idle_sleep_event();
475         return now;
476 }
477 
478 /**
479  * get_cpu_idle_time_us - get the total idle time of a cpu
480  * @cpu: CPU number to query
481  * @last_update_time: variable to store update time in. Do not update
482  * counters if NULL.
483  *
484  * Return the cummulative idle time (since boot) for a given
485  * CPU, in microseconds.
486  *
487  * This time is measured via accounting rather than sampling,
488  * and is as accurate as ktime_get() is.
489  *
490  * This function returns -1 if NOHZ is not enabled.
491  */
492 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
493 {
494         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
495         ktime_t now, idle;
496 
497         if (!tick_nohz_active)
498                 return -1;
499 
500         now = ktime_get();
501         if (last_update_time) {
502                 update_ts_time_stats(cpu, ts, now, last_update_time);
503                 idle = ts->idle_sleeptime;
504         } else {
505                 if (ts->idle_active && !nr_iowait_cpu(cpu)) {
506                         ktime_t delta = ktime_sub(now, ts->idle_entrytime);
507 
508                         idle = ktime_add(ts->idle_sleeptime, delta);
509                 } else {
510                         idle = ts->idle_sleeptime;
511                 }
512         }
513 
514         return ktime_to_us(idle);
515 
516 }
517 EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
518 
519 /**
520  * get_cpu_iowait_time_us - get the total iowait time of a cpu
521  * @cpu: CPU number to query
522  * @last_update_time: variable to store update time in. Do not update
523  * counters if NULL.
524  *
525  * Return the cummulative iowait time (since boot) for a given
526  * CPU, in microseconds.
527  *
528  * This time is measured via accounting rather than sampling,
529  * and is as accurate as ktime_get() is.
530  *
531  * This function returns -1 if NOHZ is not enabled.
532  */
533 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
534 {
535         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
536         ktime_t now, iowait;
537 
538         if (!tick_nohz_active)
539                 return -1;
540 
541         now = ktime_get();
542         if (last_update_time) {
543                 update_ts_time_stats(cpu, ts, now, last_update_time);
544                 iowait = ts->iowait_sleeptime;
545         } else {
546                 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
547                         ktime_t delta = ktime_sub(now, ts->idle_entrytime);
548 
549                         iowait = ktime_add(ts->iowait_sleeptime, delta);
550                 } else {
551                         iowait = ts->iowait_sleeptime;
552                 }
553         }
554 
555         return ktime_to_us(iowait);
556 }
557 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
558 
559 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
560 {
561         hrtimer_cancel(&ts->sched_timer);
562         hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
563 
564         /* Forward the time to expire in the future */
565         hrtimer_forward(&ts->sched_timer, now, tick_period);
566 
567         if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
568                 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
569         else
570                 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
571 }
572 
573 static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
574                                          ktime_t now, int cpu)
575 {
576         struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
577         u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
578         unsigned long seq, basejiff;
579         ktime_t tick;
580 
581         /* Read jiffies and the time when jiffies were updated last */
582         do {
583                 seq = read_seqbegin(&jiffies_lock);
584                 basemono = last_jiffies_update.tv64;
585                 basejiff = jiffies;
586         } while (read_seqretry(&jiffies_lock, seq));
587         ts->last_jiffies = basejiff;
588 
589         if (rcu_needs_cpu(basemono, &next_rcu) ||
590             arch_needs_cpu() || irq_work_needs_cpu()) {
591                 next_tick = basemono + TICK_NSEC;
592         } else {
593                 /*
594                  * Get the next pending timer. If high resolution
595                  * timers are enabled this only takes the timer wheel
596                  * timers into account. If high resolution timers are
597                  * disabled this also looks at the next expiring
598                  * hrtimer.
599                  */
600                 next_tmr = get_next_timer_interrupt(basejiff, basemono);
601                 ts->next_timer = next_tmr;
602                 /* Take the next rcu event into account */
603                 next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
604         }
605 
606         /*
607          * If the tick is due in the next period, keep it ticking or
608          * force prod the timer.
609          */
610         delta = next_tick - basemono;
611         if (delta <= (u64)TICK_NSEC) {
612                 tick.tv64 = 0;
613                 /*
614                  * We've not stopped the tick yet, and there's a timer in the
615                  * next period, so no point in stopping it either, bail.
616                  */
617                 if (!ts->tick_stopped)
618                         goto out;
619 
620                 /*
621                  * If, OTOH, we did stop it, but there's a pending (expired)
622                  * timer reprogram the timer hardware to fire now.
623                  *
624                  * We will not restart the tick proper, just prod the timer
625                  * hardware into firing an interrupt to process the pending
626                  * timers. Just like tick_irq_exit() will not restart the tick
627                  * for 'normal' interrupts.
628                  *
629                  * Only once we exit the idle loop will we re-enable the tick,
630                  * see tick_nohz_idle_exit().
631                  */
632                 if (delta == 0) {
633                         tick_nohz_restart(ts, now);
634                         goto out;
635                 }
636         }
637 
638         /*
639          * If this cpu is the one which updates jiffies, then give up
640          * the assignment and let it be taken by the cpu which runs
641          * the tick timer next, which might be this cpu as well. If we
642          * don't drop this here the jiffies might be stale and
643          * do_timer() never invoked. Keep track of the fact that it
644          * was the one which had the do_timer() duty last. If this cpu
645          * is the one which had the do_timer() duty last, we limit the
646          * sleep time to the timekeeping max_deferement value.
647          * Otherwise we can sleep as long as we want.
648          */
649         delta = timekeeping_max_deferment();
650         if (cpu == tick_do_timer_cpu) {
651                 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
652                 ts->do_timer_last = 1;
653         } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
654                 delta = KTIME_MAX;
655                 ts->do_timer_last = 0;
656         } else if (!ts->do_timer_last) {
657                 delta = KTIME_MAX;
658         }
659 
660 #ifdef CONFIG_NO_HZ_FULL
661         /* Limit the tick delta to the maximum scheduler deferment */
662         if (!ts->inidle)
663                 delta = min(delta, scheduler_tick_max_deferment());
664 #endif
665 
666         /* Calculate the next expiry time */
667         if (delta < (KTIME_MAX - basemono))
668                 expires = basemono + delta;
669         else
670                 expires = KTIME_MAX;
671 
672         expires = min_t(u64, expires, next_tick);
673         tick.tv64 = expires;
674 
675         /* Skip reprogram of event if its not changed */
676         if (ts->tick_stopped && (expires == dev->next_event.tv64))
677                 goto out;
678 
679         /*
680          * nohz_stop_sched_tick can be called several times before
681          * the nohz_restart_sched_tick is called. This happens when
682          * interrupts arrive which do not cause a reschedule. In the
683          * first call we save the current tick time, so we can restart
684          * the scheduler tick in nohz_restart_sched_tick.
685          */
686         if (!ts->tick_stopped) {
687                 nohz_balance_enter_idle(cpu);
688                 calc_load_enter_idle();
689 
690                 ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
691                 ts->tick_stopped = 1;
692                 trace_tick_stop(1, " ");
693         }
694 
695         /*
696          * If the expiration time == KTIME_MAX, then we simply stop
697          * the tick timer.
698          */
699         if (unlikely(expires == KTIME_MAX)) {
700                 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
701                         hrtimer_cancel(&ts->sched_timer);
702                 goto out;
703         }
704 
705         if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
706                 hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);
707         else
708                 tick_program_event(tick, 1);
709 out:
710         /* Update the estimated sleep length */
711         ts->sleep_length = ktime_sub(dev->next_event, now);
712         return tick;
713 }
714 
715 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now, int active)
716 {
717         /* Update jiffies first */
718         tick_do_update_jiffies64(now);
719         update_cpu_load_nohz(active);
720 
721         calc_load_exit_idle();
722         touch_softlockup_watchdog_sched();
723         /*
724          * Cancel the scheduled timer and restore the tick
725          */
726         ts->tick_stopped  = 0;
727         ts->idle_exittime = now;
728 
729         tick_nohz_restart(ts, now);
730 }
731 
732 static void tick_nohz_full_update_tick(struct tick_sched *ts)
733 {
734 #ifdef CONFIG_NO_HZ_FULL
735         int cpu = smp_processor_id();
736 
737         if (!tick_nohz_full_cpu(cpu))
738                 return;
739 
740         if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
741                 return;
742 
743         if (can_stop_full_tick())
744                 tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);
745         else if (ts->tick_stopped)
746                 tick_nohz_restart_sched_tick(ts, ktime_get(), 1);
747 #endif
748 }
749 
750 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
751 {
752         /*
753          * If this cpu is offline and it is the one which updates
754          * jiffies, then give up the assignment and let it be taken by
755          * the cpu which runs the tick timer next. If we don't drop
756          * this here the jiffies might be stale and do_timer() never
757          * invoked.
758          */
759         if (unlikely(!cpu_online(cpu))) {
760                 if (cpu == tick_do_timer_cpu)
761                         tick_do_timer_cpu = TICK_DO_TIMER_NONE;
762                 return false;
763         }
764 
765         if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
766                 ts->sleep_length = (ktime_t) { .tv64 = NSEC_PER_SEC/HZ };
767                 return false;
768         }
769 
770         if (need_resched())
771                 return false;
772 
773         if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
774                 static int ratelimit;
775 
776                 if (ratelimit < 10 &&
777                     (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
778                         pr_warn("NOHZ: local_softirq_pending %02x\n",
779                                 (unsigned int) local_softirq_pending());
780                         ratelimit++;
781                 }
782                 return false;
783         }
784 
785         if (tick_nohz_full_enabled()) {
786                 /*
787                  * Keep the tick alive to guarantee timekeeping progression
788                  * if there are full dynticks CPUs around
789                  */
790                 if (tick_do_timer_cpu == cpu)
791                         return false;
792                 /*
793                  * Boot safety: make sure the timekeeping duty has been
794                  * assigned before entering dyntick-idle mode,
795                  */
796                 if (tick_do_timer_cpu == TICK_DO_TIMER_NONE)
797                         return false;
798         }
799 
800         return true;
801 }
802 
803 static void __tick_nohz_idle_enter(struct tick_sched *ts)
804 {
805         ktime_t now, expires;
806         int cpu = smp_processor_id();
807 
808         now = tick_nohz_start_idle(ts);
809 
810         if (can_stop_idle_tick(cpu, ts)) {
811                 int was_stopped = ts->tick_stopped;
812 
813                 ts->idle_calls++;
814 
815                 expires = tick_nohz_stop_sched_tick(ts, now, cpu);
816                 if (expires.tv64 > 0LL) {
817                         ts->idle_sleeps++;
818                         ts->idle_expires = expires;
819                 }
820 
821                 if (!was_stopped && ts->tick_stopped)
822                         ts->idle_jiffies = ts->last_jiffies;
823         }
824 }
825 
826 /**
827  * tick_nohz_idle_enter - stop the idle tick from the idle task
828  *
829  * When the next event is more than a tick into the future, stop the idle tick
830  * Called when we start the idle loop.
831  *
832  * The arch is responsible of calling:
833  *
834  * - rcu_idle_enter() after its last use of RCU before the CPU is put
835  *  to sleep.
836  * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
837  */
838 void tick_nohz_idle_enter(void)
839 {
840         struct tick_sched *ts;
841 
842         WARN_ON_ONCE(irqs_disabled());
843 
844         /*
845          * Update the idle state in the scheduler domain hierarchy
846          * when tick_nohz_stop_sched_tick() is called from the idle loop.
847          * State will be updated to busy during the first busy tick after
848          * exiting idle.
849          */
850         set_cpu_sd_state_idle();
851 
852         local_irq_disable();
853 
854         ts = this_cpu_ptr(&tick_cpu_sched);
855         ts->inidle = 1;
856         __tick_nohz_idle_enter(ts);
857 
858         local_irq_enable();
859 }
860 
861 /**
862  * tick_nohz_irq_exit - update next tick event from interrupt exit
863  *
864  * When an interrupt fires while we are idle and it doesn't cause
865  * a reschedule, it may still add, modify or delete a timer, enqueue
866  * an RCU callback, etc...
867  * So we need to re-calculate and reprogram the next tick event.
868  */
869 void tick_nohz_irq_exit(void)
870 {
871         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
872 
873         if (ts->inidle)
874                 __tick_nohz_idle_enter(ts);
875         else
876                 tick_nohz_full_update_tick(ts);
877 }
878 
879 /**
880  * tick_nohz_get_sleep_length - return the length of the current sleep
881  *
882  * Called from power state control code with interrupts disabled
883  */
884 ktime_t tick_nohz_get_sleep_length(void)
885 {
886         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
887 
888         return ts->sleep_length;
889 }
890 
891 static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
892 {
893 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
894         unsigned long ticks;
895 
896         if (vtime_accounting_cpu_enabled())
897                 return;
898         /*
899          * We stopped the tick in idle. Update process times would miss the
900          * time we slept as update_process_times does only a 1 tick
901          * accounting. Enforce that this is accounted to idle !
902          */
903         ticks = jiffies - ts->idle_jiffies;
904         /*
905          * We might be one off. Do not randomly account a huge number of ticks!
906          */
907         if (ticks && ticks < LONG_MAX)
908                 account_idle_ticks(ticks);
909 #endif
910 }
911 
912 /**
913  * tick_nohz_idle_exit - restart the idle tick from the idle task
914  *
915  * Restart the idle tick when the CPU is woken up from idle
916  * This also exit the RCU extended quiescent state. The CPU
917  * can use RCU again after this function is called.
918  */
919 void tick_nohz_idle_exit(void)
920 {
921         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
922         ktime_t now;
923 
924         local_irq_disable();
925 
926         WARN_ON_ONCE(!ts->inidle);
927 
928         ts->inidle = 0;
929 
930         if (ts->idle_active || ts->tick_stopped)
931                 now = ktime_get();
932 
933         if (ts->idle_active)
934                 tick_nohz_stop_idle(ts, now);
935 
936         if (ts->tick_stopped) {
937                 tick_nohz_restart_sched_tick(ts, now, 0);
938                 tick_nohz_account_idle_ticks(ts);
939         }
940 
941         local_irq_enable();
942 }
943 
944 /*
945  * The nohz low res interrupt handler
946  */
947 static void tick_nohz_handler(struct clock_event_device *dev)
948 {
949         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
950         struct pt_regs *regs = get_irq_regs();
951         ktime_t now = ktime_get();
952 
953         dev->next_event.tv64 = KTIME_MAX;
954 
955         tick_sched_do_timer(now);
956         tick_sched_handle(ts, regs);
957 
958         /* No need to reprogram if we are running tickless  */
959         if (unlikely(ts->tick_stopped))
960                 return;
961 
962         hrtimer_forward(&ts->sched_timer, now, tick_period);
963         tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
964 }
965 
966 static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
967 {
968         if (!tick_nohz_enabled)
969                 return;
970         ts->nohz_mode = mode;
971         /* One update is enough */
972         if (!test_and_set_bit(0, &tick_nohz_active))
973                 timers_update_migration(true);
974 }
975 
976 /**
977  * tick_nohz_switch_to_nohz - switch to nohz mode
978  */
979 static void tick_nohz_switch_to_nohz(void)
980 {
981         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
982         ktime_t next;
983 
984         if (!tick_nohz_enabled)
985                 return;
986 
987         if (tick_switch_to_oneshot(tick_nohz_handler))
988                 return;
989 
990         /*
991          * Recycle the hrtimer in ts, so we can share the
992          * hrtimer_forward with the highres code.
993          */
994         hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
995         /* Get the next period */
996         next = tick_init_jiffy_update();
997 
998         hrtimer_set_expires(&ts->sched_timer, next);
999         hrtimer_forward_now(&ts->sched_timer, tick_period);
1000         tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1001         tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
1002 }
1003 
1004 /*
1005  * When NOHZ is enabled and the tick is stopped, we need to kick the
1006  * tick timer from irq_enter() so that the jiffies update is kept
1007  * alive during long running softirqs. That's ugly as hell, but
1008  * correctness is key even if we need to fix the offending softirq in
1009  * the first place.
1010  *
1011  * Note, this is different to tick_nohz_restart. We just kick the
1012  * timer and do not touch the other magic bits which need to be done
1013  * when idle is left.
1014  */
1015 static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now)
1016 {
1017 #if 0
1018         /* Switch back to 2.6.27 behaviour */
1019         ktime_t delta;
1020 
1021         /*
1022          * Do not touch the tick device, when the next expiry is either
1023          * already reached or less/equal than the tick period.
1024          */
1025         delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
1026         if (delta.tv64 <= tick_period.tv64)
1027                 return;
1028 
1029         tick_nohz_restart(ts, now);
1030 #endif
1031 }
1032 
1033 static inline void tick_nohz_irq_enter(void)
1034 {
1035         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1036         ktime_t now;
1037 
1038         if (!ts->idle_active && !ts->tick_stopped)
1039                 return;
1040         now = ktime_get();
1041         if (ts->idle_active)
1042                 tick_nohz_stop_idle(ts, now);
1043         if (ts->tick_stopped) {
1044                 tick_nohz_update_jiffies(now);
1045                 tick_nohz_kick_tick(ts, now);
1046         }
1047 }
1048 
1049 #else
1050 
1051 static inline void tick_nohz_switch_to_nohz(void) { }
1052 static inline void tick_nohz_irq_enter(void) { }
1053 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
1054 
1055 #endif /* CONFIG_NO_HZ_COMMON */
1056 
1057 /*
1058  * Called from irq_enter to notify about the possible interruption of idle()
1059  */
1060 void tick_irq_enter(void)
1061 {
1062         tick_check_oneshot_broadcast_this_cpu();
1063         tick_nohz_irq_enter();
1064 }
1065 
1066 /*
1067  * High resolution timer specific code
1068  */
1069 #ifdef CONFIG_HIGH_RES_TIMERS
1070 /*
1071  * We rearm the timer until we get disabled by the idle code.
1072  * Called with interrupts disabled.
1073  */
1074 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
1075 {
1076         struct tick_sched *ts =
1077                 container_of(timer, struct tick_sched, sched_timer);
1078         struct pt_regs *regs = get_irq_regs();
1079         ktime_t now = ktime_get();
1080 
1081         tick_sched_do_timer(now);
1082 
1083         /*
1084          * Do not call, when we are not in irq context and have
1085          * no valid regs pointer
1086          */
1087         if (regs)
1088                 tick_sched_handle(ts, regs);
1089 
1090         /* No need to reprogram if we are in idle or full dynticks mode */
1091         if (unlikely(ts->tick_stopped))
1092                 return HRTIMER_NORESTART;
1093 
1094         hrtimer_forward(timer, now, tick_period);
1095 
1096         return HRTIMER_RESTART;
1097 }
1098 
1099 static int sched_skew_tick;
1100 
1101 static int __init skew_tick(char *str)
1102 {
1103         get_option(&str, &sched_skew_tick);
1104 
1105         return 0;
1106 }
1107 early_param("skew_tick", skew_tick);
1108 
1109 /**
1110  * tick_setup_sched_timer - setup the tick emulation timer
1111  */
1112 void tick_setup_sched_timer(void)
1113 {
1114         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1115         ktime_t now = ktime_get();
1116 
1117         /*
1118          * Emulate tick processing via per-CPU hrtimers:
1119          */
1120         hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1121         ts->sched_timer.function = tick_sched_timer;
1122 
1123         /* Get the next period (per cpu) */
1124         hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1125 
1126         /* Offset the tick to avert jiffies_lock contention. */
1127         if (sched_skew_tick) {
1128                 u64 offset = ktime_to_ns(tick_period) >> 1;
1129                 do_div(offset, num_possible_cpus());
1130                 offset *= smp_processor_id();
1131                 hrtimer_add_expires_ns(&ts->sched_timer, offset);
1132         }
1133 
1134         hrtimer_forward(&ts->sched_timer, now, tick_period);
1135         hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
1136         tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
1137 }
1138 #endif /* HIGH_RES_TIMERS */
1139 
1140 #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
1141 void tick_cancel_sched_timer(int cpu)
1142 {
1143         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1144 
1145 # ifdef CONFIG_HIGH_RES_TIMERS
1146         if (ts->sched_timer.base)
1147                 hrtimer_cancel(&ts->sched_timer);
1148 # endif
1149 
1150         memset(ts, 0, sizeof(*ts));
1151 }
1152 #endif
1153 
1154 /**
1155  * Async notification about clocksource changes
1156  */
1157 void tick_clock_notify(void)
1158 {
1159         int cpu;
1160 
1161         for_each_possible_cpu(cpu)
1162                 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
1163 }
1164 
1165 /*
1166  * Async notification about clock event changes
1167  */
1168 void tick_oneshot_notify(void)
1169 {
1170         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1171 
1172         set_bit(0, &ts->check_clocks);
1173 }
1174 
1175 /**
1176  * Check, if a change happened, which makes oneshot possible.
1177  *
1178  * Called cyclic from the hrtimer softirq (driven by the timer
1179  * softirq) allow_nohz signals, that we can switch into low-res nohz
1180  * mode, because high resolution timers are disabled (either compile
1181  * or runtime). Called with interrupts disabled.
1182  */
1183 int tick_check_oneshot_change(int allow_nohz)
1184 {
1185         struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1186 
1187         if (!test_and_clear_bit(0, &ts->check_clocks))
1188                 return 0;
1189 
1190         if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
1191                 return 0;
1192 
1193         if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
1194                 return 0;
1195 
1196         if (!allow_nohz)
1197                 return 1;
1198 
1199         tick_nohz_switch_to_nohz();
1200         return 0;
1201 }
1202 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us