Version:  2.0.40 2.2.26 2.4.37 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7

Linux/drivers/clocksource/arm_arch_timer.c

  1 /*
  2  *  linux/drivers/clocksource/arm_arch_timer.c
  3  *
  4  *  Copyright (C) 2011 ARM Ltd.
  5  *  All Rights Reserved
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  */
 11 #include <linux/init.h>
 12 #include <linux/kernel.h>
 13 #include <linux/device.h>
 14 #include <linux/smp.h>
 15 #include <linux/cpu.h>
 16 #include <linux/cpu_pm.h>
 17 #include <linux/clockchips.h>
 18 #include <linux/clocksource.h>
 19 #include <linux/interrupt.h>
 20 #include <linux/of_irq.h>
 21 #include <linux/of_address.h>
 22 #include <linux/io.h>
 23 #include <linux/slab.h>
 24 #include <linux/sched_clock.h>
 25 #include <linux/acpi.h>
 26 
 27 #include <asm/arch_timer.h>
 28 #include <asm/virt.h>
 29 
 30 #include <clocksource/arm_arch_timer.h>
 31 
 32 #define CNTTIDR         0x08
 33 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
 34 
 35 #define CNTACR(n)       (0x40 + ((n) * 4))
 36 #define CNTACR_RPCT     BIT(0)
 37 #define CNTACR_RVCT     BIT(1)
 38 #define CNTACR_RFRQ     BIT(2)
 39 #define CNTACR_RVOFF    BIT(3)
 40 #define CNTACR_RWVT     BIT(4)
 41 #define CNTACR_RWPT     BIT(5)
 42 
 43 #define CNTVCT_LO       0x08
 44 #define CNTVCT_HI       0x0c
 45 #define CNTFRQ          0x10
 46 #define CNTP_TVAL       0x28
 47 #define CNTP_CTL        0x2c
 48 #define CNTV_TVAL       0x38
 49 #define CNTV_CTL        0x3c
 50 
 51 #define ARCH_CP15_TIMER BIT(0)
 52 #define ARCH_MEM_TIMER  BIT(1)
 53 static unsigned arch_timers_present __initdata;
 54 
 55 static void __iomem *arch_counter_base;
 56 
 57 struct arch_timer {
 58         void __iomem *base;
 59         struct clock_event_device evt;
 60 };
 61 
 62 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
 63 
 64 static u32 arch_timer_rate;
 65 
 66 enum ppi_nr {
 67         PHYS_SECURE_PPI,
 68         PHYS_NONSECURE_PPI,
 69         VIRT_PPI,
 70         HYP_PPI,
 71         MAX_TIMER_PPI
 72 };
 73 
 74 static int arch_timer_ppi[MAX_TIMER_PPI];
 75 
 76 static struct clock_event_device __percpu *arch_timer_evt;
 77 
 78 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
 79 static bool arch_timer_c3stop;
 80 static bool arch_timer_mem_use_virtual;
 81 
 82 /*
 83  * Architected system timer support.
 84  */
 85 
 86 static __always_inline
 87 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
 88                           struct clock_event_device *clk)
 89 {
 90         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 91                 struct arch_timer *timer = to_arch_timer(clk);
 92                 switch (reg) {
 93                 case ARCH_TIMER_REG_CTRL:
 94                         writel_relaxed(val, timer->base + CNTP_CTL);
 95                         break;
 96                 case ARCH_TIMER_REG_TVAL:
 97                         writel_relaxed(val, timer->base + CNTP_TVAL);
 98                         break;
 99                 }
100         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
101                 struct arch_timer *timer = to_arch_timer(clk);
102                 switch (reg) {
103                 case ARCH_TIMER_REG_CTRL:
104                         writel_relaxed(val, timer->base + CNTV_CTL);
105                         break;
106                 case ARCH_TIMER_REG_TVAL:
107                         writel_relaxed(val, timer->base + CNTV_TVAL);
108                         break;
109                 }
110         } else {
111                 arch_timer_reg_write_cp15(access, reg, val);
112         }
113 }
114 
115 static __always_inline
116 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
117                         struct clock_event_device *clk)
118 {
119         u32 val;
120 
121         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
122                 struct arch_timer *timer = to_arch_timer(clk);
123                 switch (reg) {
124                 case ARCH_TIMER_REG_CTRL:
125                         val = readl_relaxed(timer->base + CNTP_CTL);
126                         break;
127                 case ARCH_TIMER_REG_TVAL:
128                         val = readl_relaxed(timer->base + CNTP_TVAL);
129                         break;
130                 }
131         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
132                 struct arch_timer *timer = to_arch_timer(clk);
133                 switch (reg) {
134                 case ARCH_TIMER_REG_CTRL:
135                         val = readl_relaxed(timer->base + CNTV_CTL);
136                         break;
137                 case ARCH_TIMER_REG_TVAL:
138                         val = readl_relaxed(timer->base + CNTV_TVAL);
139                         break;
140                 }
141         } else {
142                 val = arch_timer_reg_read_cp15(access, reg);
143         }
144 
145         return val;
146 }
147 
148 static __always_inline irqreturn_t timer_handler(const int access,
149                                         struct clock_event_device *evt)
150 {
151         unsigned long ctrl;
152 
153         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
154         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
155                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
156                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
157                 evt->event_handler(evt);
158                 return IRQ_HANDLED;
159         }
160 
161         return IRQ_NONE;
162 }
163 
164 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
165 {
166         struct clock_event_device *evt = dev_id;
167 
168         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
169 }
170 
171 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
172 {
173         struct clock_event_device *evt = dev_id;
174 
175         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
176 }
177 
178 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
179 {
180         struct clock_event_device *evt = dev_id;
181 
182         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
183 }
184 
185 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
186 {
187         struct clock_event_device *evt = dev_id;
188 
189         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
190 }
191 
192 static __always_inline int timer_shutdown(const int access,
193                                           struct clock_event_device *clk)
194 {
195         unsigned long ctrl;
196 
197         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
198         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
199         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
200 
201         return 0;
202 }
203 
204 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
205 {
206         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
207 }
208 
209 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
210 {
211         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
212 }
213 
214 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
215 {
216         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
217 }
218 
219 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
220 {
221         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
222 }
223 
224 static __always_inline void set_next_event(const int access, unsigned long evt,
225                                            struct clock_event_device *clk)
226 {
227         unsigned long ctrl;
228         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
229         ctrl |= ARCH_TIMER_CTRL_ENABLE;
230         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
231         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
232         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
233 }
234 
235 static int arch_timer_set_next_event_virt(unsigned long evt,
236                                           struct clock_event_device *clk)
237 {
238         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
239         return 0;
240 }
241 
242 static int arch_timer_set_next_event_phys(unsigned long evt,
243                                           struct clock_event_device *clk)
244 {
245         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
246         return 0;
247 }
248 
249 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
250                                               struct clock_event_device *clk)
251 {
252         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
253         return 0;
254 }
255 
256 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
257                                               struct clock_event_device *clk)
258 {
259         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
260         return 0;
261 }
262 
263 static void __arch_timer_setup(unsigned type,
264                                struct clock_event_device *clk)
265 {
266         clk->features = CLOCK_EVT_FEAT_ONESHOT;
267 
268         if (type == ARCH_CP15_TIMER) {
269                 if (arch_timer_c3stop)
270                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
271                 clk->name = "arch_sys_timer";
272                 clk->rating = 450;
273                 clk->cpumask = cpumask_of(smp_processor_id());
274                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
275                 switch (arch_timer_uses_ppi) {
276                 case VIRT_PPI:
277                         clk->set_state_shutdown = arch_timer_shutdown_virt;
278                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
279                         clk->set_next_event = arch_timer_set_next_event_virt;
280                         break;
281                 case PHYS_SECURE_PPI:
282                 case PHYS_NONSECURE_PPI:
283                 case HYP_PPI:
284                         clk->set_state_shutdown = arch_timer_shutdown_phys;
285                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
286                         clk->set_next_event = arch_timer_set_next_event_phys;
287                         break;
288                 default:
289                         BUG();
290                 }
291         } else {
292                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
293                 clk->name = "arch_mem_timer";
294                 clk->rating = 400;
295                 clk->cpumask = cpu_all_mask;
296                 if (arch_timer_mem_use_virtual) {
297                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
298                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
299                         clk->set_next_event =
300                                 arch_timer_set_next_event_virt_mem;
301                 } else {
302                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
303                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
304                         clk->set_next_event =
305                                 arch_timer_set_next_event_phys_mem;
306                 }
307         }
308 
309         clk->set_state_shutdown(clk);
310 
311         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
312 }
313 
314 static void arch_timer_evtstrm_enable(int divider)
315 {
316         u32 cntkctl = arch_timer_get_cntkctl();
317 
318         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
319         /* Set the divider and enable virtual event stream */
320         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
321                         | ARCH_TIMER_VIRT_EVT_EN;
322         arch_timer_set_cntkctl(cntkctl);
323         elf_hwcap |= HWCAP_EVTSTRM;
324 #ifdef CONFIG_COMPAT
325         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
326 #endif
327 }
328 
329 static void arch_timer_configure_evtstream(void)
330 {
331         int evt_stream_div, pos;
332 
333         /* Find the closest power of two to the divisor */
334         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
335         pos = fls(evt_stream_div);
336         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
337                 pos--;
338         /* enable event stream */
339         arch_timer_evtstrm_enable(min(pos, 15));
340 }
341 
342 static void arch_counter_set_user_access(void)
343 {
344         u32 cntkctl = arch_timer_get_cntkctl();
345 
346         /* Disable user access to the timers and the physical counter */
347         /* Also disable virtual event stream */
348         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
349                         | ARCH_TIMER_USR_VT_ACCESS_EN
350                         | ARCH_TIMER_VIRT_EVT_EN
351                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
352 
353         /* Enable user access to the virtual counter */
354         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
355 
356         arch_timer_set_cntkctl(cntkctl);
357 }
358 
359 static bool arch_timer_has_nonsecure_ppi(void)
360 {
361         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
362                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
363 }
364 
365 static int arch_timer_setup(struct clock_event_device *clk)
366 {
367         __arch_timer_setup(ARCH_CP15_TIMER, clk);
368 
369         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], 0);
370 
371         if (arch_timer_has_nonsecure_ppi())
372                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
373 
374         arch_counter_set_user_access();
375         if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
376                 arch_timer_configure_evtstream();
377 
378         return 0;
379 }
380 
381 static void
382 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
383 {
384         /* Who has more than one independent system counter? */
385         if (arch_timer_rate)
386                 return;
387 
388         /*
389          * Try to determine the frequency from the device tree or CNTFRQ,
390          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
391          */
392         if (!acpi_disabled ||
393             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
394                 if (cntbase)
395                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
396                 else
397                         arch_timer_rate = arch_timer_get_cntfrq();
398         }
399 
400         /* Check the timer frequency. */
401         if (arch_timer_rate == 0)
402                 pr_warn("Architected timer frequency not available\n");
403 }
404 
405 static void arch_timer_banner(unsigned type)
406 {
407         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
408                      type & ARCH_CP15_TIMER ? "cp15" : "",
409                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
410                      type & ARCH_MEM_TIMER ? "mmio" : "",
411                      (unsigned long)arch_timer_rate / 1000000,
412                      (unsigned long)(arch_timer_rate / 10000) % 100,
413                      type & ARCH_CP15_TIMER ?
414                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
415                         "",
416                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
417                      type & ARCH_MEM_TIMER ?
418                         arch_timer_mem_use_virtual ? "virt" : "phys" :
419                         "");
420 }
421 
422 u32 arch_timer_get_rate(void)
423 {
424         return arch_timer_rate;
425 }
426 
427 static u64 arch_counter_get_cntvct_mem(void)
428 {
429         u32 vct_lo, vct_hi, tmp_hi;
430 
431         do {
432                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
433                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
434                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
435         } while (vct_hi != tmp_hi);
436 
437         return ((u64) vct_hi << 32) | vct_lo;
438 }
439 
440 /*
441  * Default to cp15 based access because arm64 uses this function for
442  * sched_clock() before DT is probed and the cp15 method is guaranteed
443  * to exist on arm64. arm doesn't use this before DT is probed so even
444  * if we don't have the cp15 accessors we won't have a problem.
445  */
446 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
447 
448 static cycle_t arch_counter_read(struct clocksource *cs)
449 {
450         return arch_timer_read_counter();
451 }
452 
453 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
454 {
455         return arch_timer_read_counter();
456 }
457 
458 static struct clocksource clocksource_counter = {
459         .name   = "arch_sys_counter",
460         .rating = 400,
461         .read   = arch_counter_read,
462         .mask   = CLOCKSOURCE_MASK(56),
463         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
464 };
465 
466 static struct cyclecounter cyclecounter = {
467         .read   = arch_counter_read_cc,
468         .mask   = CLOCKSOURCE_MASK(56),
469 };
470 
471 static struct arch_timer_kvm_info arch_timer_kvm_info;
472 
473 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
474 {
475         return &arch_timer_kvm_info;
476 }
477 
478 static void __init arch_counter_register(unsigned type)
479 {
480         u64 start_count;
481 
482         /* Register the CP15 based counter if we have one */
483         if (type & ARCH_CP15_TIMER) {
484                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
485                         arch_timer_read_counter = arch_counter_get_cntvct;
486                 else
487                         arch_timer_read_counter = arch_counter_get_cntpct;
488         } else {
489                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
490 
491                 /* If the clocksource name is "arch_sys_counter" the
492                  * VDSO will attempt to read the CP15-based counter.
493                  * Ensure this does not happen when CP15-based
494                  * counter is not available.
495                  */
496                 clocksource_counter.name = "arch_mem_counter";
497         }
498 
499         start_count = arch_timer_read_counter();
500         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
501         cyclecounter.mult = clocksource_counter.mult;
502         cyclecounter.shift = clocksource_counter.shift;
503         timecounter_init(&arch_timer_kvm_info.timecounter,
504                          &cyclecounter, start_count);
505 
506         /* 56 bits minimum, so we assume worst case rollover */
507         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
508 }
509 
510 static void arch_timer_stop(struct clock_event_device *clk)
511 {
512         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
513                  clk->irq, smp_processor_id());
514 
515         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
516         if (arch_timer_has_nonsecure_ppi())
517                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
518 
519         clk->set_state_shutdown(clk);
520 }
521 
522 static int arch_timer_cpu_notify(struct notifier_block *self,
523                                            unsigned long action, void *hcpu)
524 {
525         /*
526          * Grab cpu pointer in each case to avoid spurious
527          * preemptible warnings
528          */
529         switch (action & ~CPU_TASKS_FROZEN) {
530         case CPU_STARTING:
531                 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
532                 break;
533         case CPU_DYING:
534                 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
535                 break;
536         }
537 
538         return NOTIFY_OK;
539 }
540 
541 static struct notifier_block arch_timer_cpu_nb = {
542         .notifier_call = arch_timer_cpu_notify,
543 };
544 
545 #ifdef CONFIG_CPU_PM
546 static unsigned int saved_cntkctl;
547 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
548                                     unsigned long action, void *hcpu)
549 {
550         if (action == CPU_PM_ENTER)
551                 saved_cntkctl = arch_timer_get_cntkctl();
552         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
553                 arch_timer_set_cntkctl(saved_cntkctl);
554         return NOTIFY_OK;
555 }
556 
557 static struct notifier_block arch_timer_cpu_pm_notifier = {
558         .notifier_call = arch_timer_cpu_pm_notify,
559 };
560 
561 static int __init arch_timer_cpu_pm_init(void)
562 {
563         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
564 }
565 #else
566 static int __init arch_timer_cpu_pm_init(void)
567 {
568         return 0;
569 }
570 #endif
571 
572 static int __init arch_timer_register(void)
573 {
574         int err;
575         int ppi;
576 
577         arch_timer_evt = alloc_percpu(struct clock_event_device);
578         if (!arch_timer_evt) {
579                 err = -ENOMEM;
580                 goto out;
581         }
582 
583         ppi = arch_timer_ppi[arch_timer_uses_ppi];
584         switch (arch_timer_uses_ppi) {
585         case VIRT_PPI:
586                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
587                                          "arch_timer", arch_timer_evt);
588                 break;
589         case PHYS_SECURE_PPI:
590         case PHYS_NONSECURE_PPI:
591                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
592                                          "arch_timer", arch_timer_evt);
593                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
594                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
595                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
596                                                  "arch_timer", arch_timer_evt);
597                         if (err)
598                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
599                                                 arch_timer_evt);
600                 }
601                 break;
602         case HYP_PPI:
603                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
604                                          "arch_timer", arch_timer_evt);
605                 break;
606         default:
607                 BUG();
608         }
609 
610         if (err) {
611                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
612                        ppi, err);
613                 goto out_free;
614         }
615 
616         err = register_cpu_notifier(&arch_timer_cpu_nb);
617         if (err)
618                 goto out_free_irq;
619 
620         err = arch_timer_cpu_pm_init();
621         if (err)
622                 goto out_unreg_notify;
623 
624         /* Immediately configure the timer on the boot CPU */
625         arch_timer_setup(this_cpu_ptr(arch_timer_evt));
626 
627         return 0;
628 
629 out_unreg_notify:
630         unregister_cpu_notifier(&arch_timer_cpu_nb);
631 out_free_irq:
632         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
633         if (arch_timer_has_nonsecure_ppi())
634                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
635                                 arch_timer_evt);
636 
637 out_free:
638         free_percpu(arch_timer_evt);
639 out:
640         return err;
641 }
642 
643 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
644 {
645         int ret;
646         irq_handler_t func;
647         struct arch_timer *t;
648 
649         t = kzalloc(sizeof(*t), GFP_KERNEL);
650         if (!t)
651                 return -ENOMEM;
652 
653         t->base = base;
654         t->evt.irq = irq;
655         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
656 
657         if (arch_timer_mem_use_virtual)
658                 func = arch_timer_handler_virt_mem;
659         else
660                 func = arch_timer_handler_phys_mem;
661 
662         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
663         if (ret) {
664                 pr_err("arch_timer: Failed to request mem timer irq\n");
665                 kfree(t);
666         }
667 
668         return ret;
669 }
670 
671 static const struct of_device_id arch_timer_of_match[] __initconst = {
672         { .compatible   = "arm,armv7-timer",    },
673         { .compatible   = "arm,armv8-timer",    },
674         {},
675 };
676 
677 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
678         { .compatible   = "arm,armv7-timer-mem", },
679         {},
680 };
681 
682 static bool __init
683 arch_timer_needs_probing(int type, const struct of_device_id *matches)
684 {
685         struct device_node *dn;
686         bool needs_probing = false;
687 
688         dn = of_find_matching_node(NULL, matches);
689         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
690                 needs_probing = true;
691         of_node_put(dn);
692 
693         return needs_probing;
694 }
695 
696 static void __init arch_timer_common_init(void)
697 {
698         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
699 
700         /* Wait until both nodes are probed if we have two timers */
701         if ((arch_timers_present & mask) != mask) {
702                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
703                         return;
704                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
705                         return;
706         }
707 
708         arch_timer_banner(arch_timers_present);
709         arch_counter_register(arch_timers_present);
710         arch_timer_arch_init();
711 }
712 
713 static void __init arch_timer_init(void)
714 {
715         /*
716          * If HYP mode is available, we know that the physical timer
717          * has been configured to be accessible from PL1. Use it, so
718          * that a guest can use the virtual timer instead.
719          *
720          * If no interrupt provided for virtual timer, we'll have to
721          * stick to the physical timer. It'd better be accessible...
722          *
723          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
724          * accesses to CNTP_*_EL1 registers are silently redirected to
725          * their CNTHP_*_EL2 counterparts, and use a different PPI
726          * number.
727          */
728         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
729                 bool has_ppi;
730 
731                 if (is_kernel_in_hyp_mode()) {
732                         arch_timer_uses_ppi = HYP_PPI;
733                         has_ppi = !!arch_timer_ppi[HYP_PPI];
734                 } else {
735                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
736                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
737                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
738                 }
739 
740                 if (!has_ppi) {
741                         pr_warn("arch_timer: No interrupt available, giving up\n");
742                         return;
743                 }
744         }
745 
746         arch_timer_register();
747         arch_timer_common_init();
748 
749         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
750 }
751 
752 static void __init arch_timer_of_init(struct device_node *np)
753 {
754         int i;
755 
756         if (arch_timers_present & ARCH_CP15_TIMER) {
757                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
758                 return;
759         }
760 
761         arch_timers_present |= ARCH_CP15_TIMER;
762         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
763                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
764 
765         arch_timer_detect_rate(NULL, np);
766 
767         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
768 
769         /*
770          * If we cannot rely on firmware initializing the timer registers then
771          * we should use the physical timers instead.
772          */
773         if (IS_ENABLED(CONFIG_ARM) &&
774             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
775                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
776 
777         arch_timer_init();
778 }
779 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
780 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
781 
782 static void __init arch_timer_mem_init(struct device_node *np)
783 {
784         struct device_node *frame, *best_frame = NULL;
785         void __iomem *cntctlbase, *base;
786         unsigned int irq;
787         u32 cnttidr;
788 
789         arch_timers_present |= ARCH_MEM_TIMER;
790         cntctlbase = of_iomap(np, 0);
791         if (!cntctlbase) {
792                 pr_err("arch_timer: Can't find CNTCTLBase\n");
793                 return;
794         }
795 
796         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
797 
798         /*
799          * Try to find a virtual capable frame. Otherwise fall back to a
800          * physical capable frame.
801          */
802         for_each_available_child_of_node(np, frame) {
803                 int n;
804                 u32 cntacr;
805 
806                 if (of_property_read_u32(frame, "frame-number", &n)) {
807                         pr_err("arch_timer: Missing frame-number\n");
808                         of_node_put(frame);
809                         goto out;
810                 }
811 
812                 /* Try enabling everything, and see what sticks */
813                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
814                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
815                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
816                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
817 
818                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
819                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
820                         of_node_put(best_frame);
821                         best_frame = frame;
822                         arch_timer_mem_use_virtual = true;
823                         break;
824                 }
825 
826                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
827                         continue;
828 
829                 of_node_put(best_frame);
830                 best_frame = of_node_get(frame);
831         }
832 
833         base = arch_counter_base = of_iomap(best_frame, 0);
834         if (!base) {
835                 pr_err("arch_timer: Can't map frame's registers\n");
836                 goto out;
837         }
838 
839         if (arch_timer_mem_use_virtual)
840                 irq = irq_of_parse_and_map(best_frame, 1);
841         else
842                 irq = irq_of_parse_and_map(best_frame, 0);
843 
844         if (!irq) {
845                 pr_err("arch_timer: Frame missing %s irq",
846                        arch_timer_mem_use_virtual ? "virt" : "phys");
847                 goto out;
848         }
849 
850         arch_timer_detect_rate(base, np);
851         arch_timer_mem_register(base, irq);
852         arch_timer_common_init();
853 out:
854         iounmap(cntctlbase);
855         of_node_put(best_frame);
856 }
857 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
858                        arch_timer_mem_init);
859 
860 #ifdef CONFIG_ACPI
861 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
862 {
863         int trigger, polarity;
864 
865         if (!interrupt)
866                 return 0;
867 
868         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
869                         : ACPI_LEVEL_SENSITIVE;
870 
871         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
872                         : ACPI_ACTIVE_HIGH;
873 
874         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
875 }
876 
877 /* Initialize per-processor generic timer */
878 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
879 {
880         struct acpi_table_gtdt *gtdt;
881 
882         if (arch_timers_present & ARCH_CP15_TIMER) {
883                 pr_warn("arch_timer: already initialized, skipping\n");
884                 return -EINVAL;
885         }
886 
887         gtdt = container_of(table, struct acpi_table_gtdt, header);
888 
889         arch_timers_present |= ARCH_CP15_TIMER;
890 
891         arch_timer_ppi[PHYS_SECURE_PPI] =
892                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
893                 gtdt->secure_el1_flags);
894 
895         arch_timer_ppi[PHYS_NONSECURE_PPI] =
896                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
897                 gtdt->non_secure_el1_flags);
898 
899         arch_timer_ppi[VIRT_PPI] =
900                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
901                 gtdt->virtual_timer_flags);
902 
903         arch_timer_ppi[HYP_PPI] =
904                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
905                 gtdt->non_secure_el2_flags);
906 
907         /* Get the frequency from CNTFRQ */
908         arch_timer_detect_rate(NULL, NULL);
909 
910         /* Always-on capability */
911         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
912 
913         arch_timer_init();
914         return 0;
915 }
916 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
917 #endif
918 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us