Version:  2.0.40 2.2.26 2.4.37 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8

Linux/drivers/clocksource/arm_arch_timer.c

  1 /*
  2  *  linux/drivers/clocksource/arm_arch_timer.c
  3  *
  4  *  Copyright (C) 2011 ARM Ltd.
  5  *  All Rights Reserved
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  */
 11 
 12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
 13 
 14 #include <linux/init.h>
 15 #include <linux/kernel.h>
 16 #include <linux/device.h>
 17 #include <linux/smp.h>
 18 #include <linux/cpu.h>
 19 #include <linux/cpu_pm.h>
 20 #include <linux/clockchips.h>
 21 #include <linux/clocksource.h>
 22 #include <linux/interrupt.h>
 23 #include <linux/of_irq.h>
 24 #include <linux/of_address.h>
 25 #include <linux/io.h>
 26 #include <linux/slab.h>
 27 #include <linux/sched_clock.h>
 28 #include <linux/acpi.h>
 29 
 30 #include <asm/arch_timer.h>
 31 #include <asm/virt.h>
 32 
 33 #include <clocksource/arm_arch_timer.h>
 34 
 35 #define CNTTIDR         0x08
 36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
 37 
 38 #define CNTACR(n)       (0x40 + ((n) * 4))
 39 #define CNTACR_RPCT     BIT(0)
 40 #define CNTACR_RVCT     BIT(1)
 41 #define CNTACR_RFRQ     BIT(2)
 42 #define CNTACR_RVOFF    BIT(3)
 43 #define CNTACR_RWVT     BIT(4)
 44 #define CNTACR_RWPT     BIT(5)
 45 
 46 #define CNTVCT_LO       0x08
 47 #define CNTVCT_HI       0x0c
 48 #define CNTFRQ          0x10
 49 #define CNTP_TVAL       0x28
 50 #define CNTP_CTL        0x2c
 51 #define CNTV_TVAL       0x38
 52 #define CNTV_CTL        0x3c
 53 
 54 #define ARCH_CP15_TIMER BIT(0)
 55 #define ARCH_MEM_TIMER  BIT(1)
 56 static unsigned arch_timers_present __initdata;
 57 
 58 static void __iomem *arch_counter_base;
 59 
 60 struct arch_timer {
 61         void __iomem *base;
 62         struct clock_event_device evt;
 63 };
 64 
 65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
 66 
 67 static u32 arch_timer_rate;
 68 
 69 enum ppi_nr {
 70         PHYS_SECURE_PPI,
 71         PHYS_NONSECURE_PPI,
 72         VIRT_PPI,
 73         HYP_PPI,
 74         MAX_TIMER_PPI
 75 };
 76 
 77 static int arch_timer_ppi[MAX_TIMER_PPI];
 78 
 79 static struct clock_event_device __percpu *arch_timer_evt;
 80 
 81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
 82 static bool arch_timer_c3stop;
 83 static bool arch_timer_mem_use_virtual;
 84 
 85 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
 86 
 87 static int __init early_evtstrm_cfg(char *buf)
 88 {
 89         return strtobool(buf, &evtstrm_enable);
 90 }
 91 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
 92 
 93 /*
 94  * Architected system timer support.
 95  */
 96 
 97 static __always_inline
 98 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
 99                           struct clock_event_device *clk)
100 {
101         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
102                 struct arch_timer *timer = to_arch_timer(clk);
103                 switch (reg) {
104                 case ARCH_TIMER_REG_CTRL:
105                         writel_relaxed(val, timer->base + CNTP_CTL);
106                         break;
107                 case ARCH_TIMER_REG_TVAL:
108                         writel_relaxed(val, timer->base + CNTP_TVAL);
109                         break;
110                 }
111         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
112                 struct arch_timer *timer = to_arch_timer(clk);
113                 switch (reg) {
114                 case ARCH_TIMER_REG_CTRL:
115                         writel_relaxed(val, timer->base + CNTV_CTL);
116                         break;
117                 case ARCH_TIMER_REG_TVAL:
118                         writel_relaxed(val, timer->base + CNTV_TVAL);
119                         break;
120                 }
121         } else {
122                 arch_timer_reg_write_cp15(access, reg, val);
123         }
124 }
125 
126 static __always_inline
127 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
128                         struct clock_event_device *clk)
129 {
130         u32 val;
131 
132         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
133                 struct arch_timer *timer = to_arch_timer(clk);
134                 switch (reg) {
135                 case ARCH_TIMER_REG_CTRL:
136                         val = readl_relaxed(timer->base + CNTP_CTL);
137                         break;
138                 case ARCH_TIMER_REG_TVAL:
139                         val = readl_relaxed(timer->base + CNTP_TVAL);
140                         break;
141                 }
142         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
143                 struct arch_timer *timer = to_arch_timer(clk);
144                 switch (reg) {
145                 case ARCH_TIMER_REG_CTRL:
146                         val = readl_relaxed(timer->base + CNTV_CTL);
147                         break;
148                 case ARCH_TIMER_REG_TVAL:
149                         val = readl_relaxed(timer->base + CNTV_TVAL);
150                         break;
151                 }
152         } else {
153                 val = arch_timer_reg_read_cp15(access, reg);
154         }
155 
156         return val;
157 }
158 
159 static __always_inline irqreturn_t timer_handler(const int access,
160                                         struct clock_event_device *evt)
161 {
162         unsigned long ctrl;
163 
164         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
165         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
166                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
167                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
168                 evt->event_handler(evt);
169                 return IRQ_HANDLED;
170         }
171 
172         return IRQ_NONE;
173 }
174 
175 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
176 {
177         struct clock_event_device *evt = dev_id;
178 
179         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
180 }
181 
182 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
183 {
184         struct clock_event_device *evt = dev_id;
185 
186         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
187 }
188 
189 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
190 {
191         struct clock_event_device *evt = dev_id;
192 
193         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
194 }
195 
196 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
197 {
198         struct clock_event_device *evt = dev_id;
199 
200         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
201 }
202 
203 static __always_inline int timer_shutdown(const int access,
204                                           struct clock_event_device *clk)
205 {
206         unsigned long ctrl;
207 
208         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
209         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
210         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
211 
212         return 0;
213 }
214 
215 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
216 {
217         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
218 }
219 
220 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
221 {
222         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
223 }
224 
225 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
226 {
227         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
228 }
229 
230 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
231 {
232         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
233 }
234 
235 static __always_inline void set_next_event(const int access, unsigned long evt,
236                                            struct clock_event_device *clk)
237 {
238         unsigned long ctrl;
239         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
240         ctrl |= ARCH_TIMER_CTRL_ENABLE;
241         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
242         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
243         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
244 }
245 
246 static int arch_timer_set_next_event_virt(unsigned long evt,
247                                           struct clock_event_device *clk)
248 {
249         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
250         return 0;
251 }
252 
253 static int arch_timer_set_next_event_phys(unsigned long evt,
254                                           struct clock_event_device *clk)
255 {
256         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
257         return 0;
258 }
259 
260 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
261                                               struct clock_event_device *clk)
262 {
263         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
264         return 0;
265 }
266 
267 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
268                                               struct clock_event_device *clk)
269 {
270         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
271         return 0;
272 }
273 
274 static void __arch_timer_setup(unsigned type,
275                                struct clock_event_device *clk)
276 {
277         clk->features = CLOCK_EVT_FEAT_ONESHOT;
278 
279         if (type == ARCH_CP15_TIMER) {
280                 if (arch_timer_c3stop)
281                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
282                 clk->name = "arch_sys_timer";
283                 clk->rating = 450;
284                 clk->cpumask = cpumask_of(smp_processor_id());
285                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
286                 switch (arch_timer_uses_ppi) {
287                 case VIRT_PPI:
288                         clk->set_state_shutdown = arch_timer_shutdown_virt;
289                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
290                         clk->set_next_event = arch_timer_set_next_event_virt;
291                         break;
292                 case PHYS_SECURE_PPI:
293                 case PHYS_NONSECURE_PPI:
294                 case HYP_PPI:
295                         clk->set_state_shutdown = arch_timer_shutdown_phys;
296                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
297                         clk->set_next_event = arch_timer_set_next_event_phys;
298                         break;
299                 default:
300                         BUG();
301                 }
302         } else {
303                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
304                 clk->name = "arch_mem_timer";
305                 clk->rating = 400;
306                 clk->cpumask = cpu_all_mask;
307                 if (arch_timer_mem_use_virtual) {
308                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
309                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
310                         clk->set_next_event =
311                                 arch_timer_set_next_event_virt_mem;
312                 } else {
313                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
314                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
315                         clk->set_next_event =
316                                 arch_timer_set_next_event_phys_mem;
317                 }
318         }
319 
320         clk->set_state_shutdown(clk);
321 
322         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
323 }
324 
325 static void arch_timer_evtstrm_enable(int divider)
326 {
327         u32 cntkctl = arch_timer_get_cntkctl();
328 
329         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
330         /* Set the divider and enable virtual event stream */
331         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
332                         | ARCH_TIMER_VIRT_EVT_EN;
333         arch_timer_set_cntkctl(cntkctl);
334         elf_hwcap |= HWCAP_EVTSTRM;
335 #ifdef CONFIG_COMPAT
336         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
337 #endif
338 }
339 
340 static void arch_timer_configure_evtstream(void)
341 {
342         int evt_stream_div, pos;
343 
344         /* Find the closest power of two to the divisor */
345         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
346         pos = fls(evt_stream_div);
347         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
348                 pos--;
349         /* enable event stream */
350         arch_timer_evtstrm_enable(min(pos, 15));
351 }
352 
353 static void arch_counter_set_user_access(void)
354 {
355         u32 cntkctl = arch_timer_get_cntkctl();
356 
357         /* Disable user access to the timers and the physical counter */
358         /* Also disable virtual event stream */
359         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
360                         | ARCH_TIMER_USR_VT_ACCESS_EN
361                         | ARCH_TIMER_VIRT_EVT_EN
362                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
363 
364         /* Enable user access to the virtual counter */
365         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
366 
367         arch_timer_set_cntkctl(cntkctl);
368 }
369 
370 static bool arch_timer_has_nonsecure_ppi(void)
371 {
372         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
373                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
374 }
375 
376 static u32 check_ppi_trigger(int irq)
377 {
378         u32 flags = irq_get_trigger_type(irq);
379 
380         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
381                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
382                 pr_warn("WARNING: Please fix your firmware\n");
383                 flags = IRQF_TRIGGER_LOW;
384         }
385 
386         return flags;
387 }
388 
389 static int arch_timer_starting_cpu(unsigned int cpu)
390 {
391         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
392         u32 flags;
393 
394         __arch_timer_setup(ARCH_CP15_TIMER, clk);
395 
396         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
397         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
398 
399         if (arch_timer_has_nonsecure_ppi()) {
400                 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
401                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
402         }
403 
404         arch_counter_set_user_access();
405         if (evtstrm_enable)
406                 arch_timer_configure_evtstream();
407 
408         return 0;
409 }
410 
411 static void
412 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
413 {
414         /* Who has more than one independent system counter? */
415         if (arch_timer_rate)
416                 return;
417 
418         /*
419          * Try to determine the frequency from the device tree or CNTFRQ,
420          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
421          */
422         if (!acpi_disabled ||
423             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
424                 if (cntbase)
425                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
426                 else
427                         arch_timer_rate = arch_timer_get_cntfrq();
428         }
429 
430         /* Check the timer frequency. */
431         if (arch_timer_rate == 0)
432                 pr_warn("Architected timer frequency not available\n");
433 }
434 
435 static void arch_timer_banner(unsigned type)
436 {
437         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
438                      type & ARCH_CP15_TIMER ? "cp15" : "",
439                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
440                      type & ARCH_MEM_TIMER ? "mmio" : "",
441                      (unsigned long)arch_timer_rate / 1000000,
442                      (unsigned long)(arch_timer_rate / 10000) % 100,
443                      type & ARCH_CP15_TIMER ?
444                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
445                         "",
446                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
447                      type & ARCH_MEM_TIMER ?
448                         arch_timer_mem_use_virtual ? "virt" : "phys" :
449                         "");
450 }
451 
452 u32 arch_timer_get_rate(void)
453 {
454         return arch_timer_rate;
455 }
456 
457 static u64 arch_counter_get_cntvct_mem(void)
458 {
459         u32 vct_lo, vct_hi, tmp_hi;
460 
461         do {
462                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
463                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
464                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
465         } while (vct_hi != tmp_hi);
466 
467         return ((u64) vct_hi << 32) | vct_lo;
468 }
469 
470 /*
471  * Default to cp15 based access because arm64 uses this function for
472  * sched_clock() before DT is probed and the cp15 method is guaranteed
473  * to exist on arm64. arm doesn't use this before DT is probed so even
474  * if we don't have the cp15 accessors we won't have a problem.
475  */
476 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
477 
478 static cycle_t arch_counter_read(struct clocksource *cs)
479 {
480         return arch_timer_read_counter();
481 }
482 
483 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
484 {
485         return arch_timer_read_counter();
486 }
487 
488 static struct clocksource clocksource_counter = {
489         .name   = "arch_sys_counter",
490         .rating = 400,
491         .read   = arch_counter_read,
492         .mask   = CLOCKSOURCE_MASK(56),
493         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
494 };
495 
496 static struct cyclecounter cyclecounter = {
497         .read   = arch_counter_read_cc,
498         .mask   = CLOCKSOURCE_MASK(56),
499 };
500 
501 static struct arch_timer_kvm_info arch_timer_kvm_info;
502 
503 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
504 {
505         return &arch_timer_kvm_info;
506 }
507 
508 static void __init arch_counter_register(unsigned type)
509 {
510         u64 start_count;
511 
512         /* Register the CP15 based counter if we have one */
513         if (type & ARCH_CP15_TIMER) {
514                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
515                         arch_timer_read_counter = arch_counter_get_cntvct;
516                 else
517                         arch_timer_read_counter = arch_counter_get_cntpct;
518         } else {
519                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
520 
521                 /* If the clocksource name is "arch_sys_counter" the
522                  * VDSO will attempt to read the CP15-based counter.
523                  * Ensure this does not happen when CP15-based
524                  * counter is not available.
525                  */
526                 clocksource_counter.name = "arch_mem_counter";
527         }
528 
529         start_count = arch_timer_read_counter();
530         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
531         cyclecounter.mult = clocksource_counter.mult;
532         cyclecounter.shift = clocksource_counter.shift;
533         timecounter_init(&arch_timer_kvm_info.timecounter,
534                          &cyclecounter, start_count);
535 
536         /* 56 bits minimum, so we assume worst case rollover */
537         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
538 }
539 
540 static void arch_timer_stop(struct clock_event_device *clk)
541 {
542         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
543                  clk->irq, smp_processor_id());
544 
545         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
546         if (arch_timer_has_nonsecure_ppi())
547                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
548 
549         clk->set_state_shutdown(clk);
550 }
551 
552 static int arch_timer_dying_cpu(unsigned int cpu)
553 {
554         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
555 
556         arch_timer_stop(clk);
557         return 0;
558 }
559 
560 #ifdef CONFIG_CPU_PM
561 static unsigned int saved_cntkctl;
562 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
563                                     unsigned long action, void *hcpu)
564 {
565         if (action == CPU_PM_ENTER)
566                 saved_cntkctl = arch_timer_get_cntkctl();
567         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
568                 arch_timer_set_cntkctl(saved_cntkctl);
569         return NOTIFY_OK;
570 }
571 
572 static struct notifier_block arch_timer_cpu_pm_notifier = {
573         .notifier_call = arch_timer_cpu_pm_notify,
574 };
575 
576 static int __init arch_timer_cpu_pm_init(void)
577 {
578         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
579 }
580 
581 static void __init arch_timer_cpu_pm_deinit(void)
582 {
583         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
584 }
585 
586 #else
587 static int __init arch_timer_cpu_pm_init(void)
588 {
589         return 0;
590 }
591 
592 static void __init arch_timer_cpu_pm_deinit(void)
593 {
594 }
595 #endif
596 
597 static int __init arch_timer_register(void)
598 {
599         int err;
600         int ppi;
601 
602         arch_timer_evt = alloc_percpu(struct clock_event_device);
603         if (!arch_timer_evt) {
604                 err = -ENOMEM;
605                 goto out;
606         }
607 
608         ppi = arch_timer_ppi[arch_timer_uses_ppi];
609         switch (arch_timer_uses_ppi) {
610         case VIRT_PPI:
611                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
612                                          "arch_timer", arch_timer_evt);
613                 break;
614         case PHYS_SECURE_PPI:
615         case PHYS_NONSECURE_PPI:
616                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
617                                          "arch_timer", arch_timer_evt);
618                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
619                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
620                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
621                                                  "arch_timer", arch_timer_evt);
622                         if (err)
623                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
624                                                 arch_timer_evt);
625                 }
626                 break;
627         case HYP_PPI:
628                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
629                                          "arch_timer", arch_timer_evt);
630                 break;
631         default:
632                 BUG();
633         }
634 
635         if (err) {
636                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
637                        ppi, err);
638                 goto out_free;
639         }
640 
641         err = arch_timer_cpu_pm_init();
642         if (err)
643                 goto out_unreg_notify;
644 
645 
646         /* Register and immediately configure the timer on the boot CPU */
647         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
648                                 "AP_ARM_ARCH_TIMER_STARTING",
649                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
650         if (err)
651                 goto out_unreg_cpupm;
652         return 0;
653 
654 out_unreg_cpupm:
655         arch_timer_cpu_pm_deinit();
656 
657 out_unreg_notify:
658         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
659         if (arch_timer_has_nonsecure_ppi())
660                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
661                                 arch_timer_evt);
662 
663 out_free:
664         free_percpu(arch_timer_evt);
665 out:
666         return err;
667 }
668 
669 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
670 {
671         int ret;
672         irq_handler_t func;
673         struct arch_timer *t;
674 
675         t = kzalloc(sizeof(*t), GFP_KERNEL);
676         if (!t)
677                 return -ENOMEM;
678 
679         t->base = base;
680         t->evt.irq = irq;
681         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
682 
683         if (arch_timer_mem_use_virtual)
684                 func = arch_timer_handler_virt_mem;
685         else
686                 func = arch_timer_handler_phys_mem;
687 
688         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
689         if (ret) {
690                 pr_err("arch_timer: Failed to request mem timer irq\n");
691                 kfree(t);
692         }
693 
694         return ret;
695 }
696 
697 static const struct of_device_id arch_timer_of_match[] __initconst = {
698         { .compatible   = "arm,armv7-timer",    },
699         { .compatible   = "arm,armv8-timer",    },
700         {},
701 };
702 
703 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
704         { .compatible   = "arm,armv7-timer-mem", },
705         {},
706 };
707 
708 static bool __init
709 arch_timer_needs_probing(int type, const struct of_device_id *matches)
710 {
711         struct device_node *dn;
712         bool needs_probing = false;
713 
714         dn = of_find_matching_node(NULL, matches);
715         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
716                 needs_probing = true;
717         of_node_put(dn);
718 
719         return needs_probing;
720 }
721 
722 static int __init arch_timer_common_init(void)
723 {
724         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
725 
726         /* Wait until both nodes are probed if we have two timers */
727         if ((arch_timers_present & mask) != mask) {
728                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
729                         return 0;
730                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
731                         return 0;
732         }
733 
734         arch_timer_banner(arch_timers_present);
735         arch_counter_register(arch_timers_present);
736         return arch_timer_arch_init();
737 }
738 
739 static int __init arch_timer_init(void)
740 {
741         int ret;
742         /*
743          * If HYP mode is available, we know that the physical timer
744          * has been configured to be accessible from PL1. Use it, so
745          * that a guest can use the virtual timer instead.
746          *
747          * If no interrupt provided for virtual timer, we'll have to
748          * stick to the physical timer. It'd better be accessible...
749          *
750          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
751          * accesses to CNTP_*_EL1 registers are silently redirected to
752          * their CNTHP_*_EL2 counterparts, and use a different PPI
753          * number.
754          */
755         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
756                 bool has_ppi;
757 
758                 if (is_kernel_in_hyp_mode()) {
759                         arch_timer_uses_ppi = HYP_PPI;
760                         has_ppi = !!arch_timer_ppi[HYP_PPI];
761                 } else {
762                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
763                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
764                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
765                 }
766 
767                 if (!has_ppi) {
768                         pr_warn("arch_timer: No interrupt available, giving up\n");
769                         return -EINVAL;
770                 }
771         }
772 
773         ret = arch_timer_register();
774         if (ret)
775                 return ret;
776 
777         ret = arch_timer_common_init();
778         if (ret)
779                 return ret;
780 
781         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
782         
783         return 0;
784 }
785 
786 static int __init arch_timer_of_init(struct device_node *np)
787 {
788         int i;
789 
790         if (arch_timers_present & ARCH_CP15_TIMER) {
791                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
792                 return 0;
793         }
794 
795         arch_timers_present |= ARCH_CP15_TIMER;
796         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
797                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
798 
799         arch_timer_detect_rate(NULL, np);
800 
801         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
802 
803         /*
804          * If we cannot rely on firmware initializing the timer registers then
805          * we should use the physical timers instead.
806          */
807         if (IS_ENABLED(CONFIG_ARM) &&
808             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
809                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
810 
811         return arch_timer_init();
812 }
813 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
814 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
815 
816 static int __init arch_timer_mem_init(struct device_node *np)
817 {
818         struct device_node *frame, *best_frame = NULL;
819         void __iomem *cntctlbase, *base;
820         unsigned int irq, ret = -EINVAL;
821         u32 cnttidr;
822 
823         arch_timers_present |= ARCH_MEM_TIMER;
824         cntctlbase = of_iomap(np, 0);
825         if (!cntctlbase) {
826                 pr_err("arch_timer: Can't find CNTCTLBase\n");
827                 return -ENXIO;
828         }
829 
830         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
831 
832         /*
833          * Try to find a virtual capable frame. Otherwise fall back to a
834          * physical capable frame.
835          */
836         for_each_available_child_of_node(np, frame) {
837                 int n;
838                 u32 cntacr;
839 
840                 if (of_property_read_u32(frame, "frame-number", &n)) {
841                         pr_err("arch_timer: Missing frame-number\n");
842                         of_node_put(frame);
843                         goto out;
844                 }
845 
846                 /* Try enabling everything, and see what sticks */
847                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
848                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
849                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
850                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
851 
852                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
853                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
854                         of_node_put(best_frame);
855                         best_frame = frame;
856                         arch_timer_mem_use_virtual = true;
857                         break;
858                 }
859 
860                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
861                         continue;
862 
863                 of_node_put(best_frame);
864                 best_frame = of_node_get(frame);
865         }
866 
867         ret= -ENXIO;
868         base = arch_counter_base = of_iomap(best_frame, 0);
869         if (!base) {
870                 pr_err("arch_timer: Can't map frame's registers\n");
871                 goto out;
872         }
873 
874         if (arch_timer_mem_use_virtual)
875                 irq = irq_of_parse_and_map(best_frame, 1);
876         else
877                 irq = irq_of_parse_and_map(best_frame, 0);
878 
879         ret = -EINVAL;
880         if (!irq) {
881                 pr_err("arch_timer: Frame missing %s irq",
882                        arch_timer_mem_use_virtual ? "virt" : "phys");
883                 goto out;
884         }
885 
886         arch_timer_detect_rate(base, np);
887         ret = arch_timer_mem_register(base, irq);
888         if (ret)
889                 goto out;
890 
891         return arch_timer_common_init();
892 out:
893         iounmap(cntctlbase);
894         of_node_put(best_frame);
895         return ret;
896 }
897 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
898                        arch_timer_mem_init);
899 
900 #ifdef CONFIG_ACPI
901 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
902 {
903         int trigger, polarity;
904 
905         if (!interrupt)
906                 return 0;
907 
908         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
909                         : ACPI_LEVEL_SENSITIVE;
910 
911         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
912                         : ACPI_ACTIVE_HIGH;
913 
914         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
915 }
916 
917 /* Initialize per-processor generic timer */
918 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
919 {
920         struct acpi_table_gtdt *gtdt;
921 
922         if (arch_timers_present & ARCH_CP15_TIMER) {
923                 pr_warn("arch_timer: already initialized, skipping\n");
924                 return -EINVAL;
925         }
926 
927         gtdt = container_of(table, struct acpi_table_gtdt, header);
928 
929         arch_timers_present |= ARCH_CP15_TIMER;
930 
931         arch_timer_ppi[PHYS_SECURE_PPI] =
932                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
933                 gtdt->secure_el1_flags);
934 
935         arch_timer_ppi[PHYS_NONSECURE_PPI] =
936                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
937                 gtdt->non_secure_el1_flags);
938 
939         arch_timer_ppi[VIRT_PPI] =
940                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
941                 gtdt->virtual_timer_flags);
942 
943         arch_timer_ppi[HYP_PPI] =
944                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
945                 gtdt->non_secure_el2_flags);
946 
947         /* Get the frequency from CNTFRQ */
948         arch_timer_detect_rate(NULL, NULL);
949 
950         /* Always-on capability */
951         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
952 
953         arch_timer_init();
954         return 0;
955 }
956 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
957 #endif
958 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us