Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/drivers/clocksource/arm_arch_timer.c

  1 /*
  2  *  linux/drivers/clocksource/arm_arch_timer.c
  3  *
  4  *  Copyright (C) 2011 ARM Ltd.
  5  *  All Rights Reserved
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  */
 11 
 12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
 13 
 14 #include <linux/init.h>
 15 #include <linux/kernel.h>
 16 #include <linux/device.h>
 17 #include <linux/smp.h>
 18 #include <linux/cpu.h>
 19 #include <linux/cpu_pm.h>
 20 #include <linux/clockchips.h>
 21 #include <linux/clocksource.h>
 22 #include <linux/interrupt.h>
 23 #include <linux/of_irq.h>
 24 #include <linux/of_address.h>
 25 #include <linux/io.h>
 26 #include <linux/slab.h>
 27 #include <linux/sched_clock.h>
 28 #include <linux/acpi.h>
 29 
 30 #include <asm/arch_timer.h>
 31 #include <asm/virt.h>
 32 
 33 #include <clocksource/arm_arch_timer.h>
 34 
 35 #define CNTTIDR         0x08
 36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
 37 
 38 #define CNTACR(n)       (0x40 + ((n) * 4))
 39 #define CNTACR_RPCT     BIT(0)
 40 #define CNTACR_RVCT     BIT(1)
 41 #define CNTACR_RFRQ     BIT(2)
 42 #define CNTACR_RVOFF    BIT(3)
 43 #define CNTACR_RWVT     BIT(4)
 44 #define CNTACR_RWPT     BIT(5)
 45 
 46 #define CNTVCT_LO       0x08
 47 #define CNTVCT_HI       0x0c
 48 #define CNTFRQ          0x10
 49 #define CNTP_TVAL       0x28
 50 #define CNTP_CTL        0x2c
 51 #define CNTV_TVAL       0x38
 52 #define CNTV_CTL        0x3c
 53 
 54 #define ARCH_CP15_TIMER BIT(0)
 55 #define ARCH_MEM_TIMER  BIT(1)
 56 static unsigned arch_timers_present __initdata;
 57 
 58 static void __iomem *arch_counter_base;
 59 
 60 struct arch_timer {
 61         void __iomem *base;
 62         struct clock_event_device evt;
 63 };
 64 
 65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
 66 
 67 static u32 arch_timer_rate;
 68 
 69 enum ppi_nr {
 70         PHYS_SECURE_PPI,
 71         PHYS_NONSECURE_PPI,
 72         VIRT_PPI,
 73         HYP_PPI,
 74         MAX_TIMER_PPI
 75 };
 76 
 77 static int arch_timer_ppi[MAX_TIMER_PPI];
 78 
 79 static struct clock_event_device __percpu *arch_timer_evt;
 80 
 81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
 82 static bool arch_timer_c3stop;
 83 static bool arch_timer_mem_use_virtual;
 84 static bool arch_counter_suspend_stop;
 85 
 86 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
 87 
 88 static int __init early_evtstrm_cfg(char *buf)
 89 {
 90         return strtobool(buf, &evtstrm_enable);
 91 }
 92 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
 93 
 94 /*
 95  * Architected system timer support.
 96  */
 97 
 98 #ifdef CONFIG_FSL_ERRATUM_A008585
 99 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
100 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
101 
102 static int fsl_a008585_enable = -1;
103 
104 static int __init early_fsl_a008585_cfg(char *buf)
105 {
106         int ret;
107         bool val;
108 
109         ret = strtobool(buf, &val);
110         if (ret)
111                 return ret;
112 
113         fsl_a008585_enable = val;
114         return 0;
115 }
116 early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg);
117 
118 u32 __fsl_a008585_read_cntp_tval_el0(void)
119 {
120         return __fsl_a008585_read_reg(cntp_tval_el0);
121 }
122 
123 u32 __fsl_a008585_read_cntv_tval_el0(void)
124 {
125         return __fsl_a008585_read_reg(cntv_tval_el0);
126 }
127 
128 u64 __fsl_a008585_read_cntvct_el0(void)
129 {
130         return __fsl_a008585_read_reg(cntvct_el0);
131 }
132 EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
133 #endif /* CONFIG_FSL_ERRATUM_A008585 */
134 
135 static __always_inline
136 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
137                           struct clock_event_device *clk)
138 {
139         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
140                 struct arch_timer *timer = to_arch_timer(clk);
141                 switch (reg) {
142                 case ARCH_TIMER_REG_CTRL:
143                         writel_relaxed(val, timer->base + CNTP_CTL);
144                         break;
145                 case ARCH_TIMER_REG_TVAL:
146                         writel_relaxed(val, timer->base + CNTP_TVAL);
147                         break;
148                 }
149         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
150                 struct arch_timer *timer = to_arch_timer(clk);
151                 switch (reg) {
152                 case ARCH_TIMER_REG_CTRL:
153                         writel_relaxed(val, timer->base + CNTV_CTL);
154                         break;
155                 case ARCH_TIMER_REG_TVAL:
156                         writel_relaxed(val, timer->base + CNTV_TVAL);
157                         break;
158                 }
159         } else {
160                 arch_timer_reg_write_cp15(access, reg, val);
161         }
162 }
163 
164 static __always_inline
165 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
166                         struct clock_event_device *clk)
167 {
168         u32 val;
169 
170         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
171                 struct arch_timer *timer = to_arch_timer(clk);
172                 switch (reg) {
173                 case ARCH_TIMER_REG_CTRL:
174                         val = readl_relaxed(timer->base + CNTP_CTL);
175                         break;
176                 case ARCH_TIMER_REG_TVAL:
177                         val = readl_relaxed(timer->base + CNTP_TVAL);
178                         break;
179                 }
180         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
181                 struct arch_timer *timer = to_arch_timer(clk);
182                 switch (reg) {
183                 case ARCH_TIMER_REG_CTRL:
184                         val = readl_relaxed(timer->base + CNTV_CTL);
185                         break;
186                 case ARCH_TIMER_REG_TVAL:
187                         val = readl_relaxed(timer->base + CNTV_TVAL);
188                         break;
189                 }
190         } else {
191                 val = arch_timer_reg_read_cp15(access, reg);
192         }
193 
194         return val;
195 }
196 
197 static __always_inline irqreturn_t timer_handler(const int access,
198                                         struct clock_event_device *evt)
199 {
200         unsigned long ctrl;
201 
202         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
203         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
204                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
205                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
206                 evt->event_handler(evt);
207                 return IRQ_HANDLED;
208         }
209 
210         return IRQ_NONE;
211 }
212 
213 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
214 {
215         struct clock_event_device *evt = dev_id;
216 
217         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
218 }
219 
220 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
221 {
222         struct clock_event_device *evt = dev_id;
223 
224         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
225 }
226 
227 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
228 {
229         struct clock_event_device *evt = dev_id;
230 
231         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
232 }
233 
234 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
235 {
236         struct clock_event_device *evt = dev_id;
237 
238         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
239 }
240 
241 static __always_inline int timer_shutdown(const int access,
242                                           struct clock_event_device *clk)
243 {
244         unsigned long ctrl;
245 
246         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
247         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
248         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
249 
250         return 0;
251 }
252 
253 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
254 {
255         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
256 }
257 
258 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
259 {
260         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
261 }
262 
263 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
264 {
265         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
266 }
267 
268 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
269 {
270         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
271 }
272 
273 static __always_inline void set_next_event(const int access, unsigned long evt,
274                                            struct clock_event_device *clk)
275 {
276         unsigned long ctrl;
277         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
278         ctrl |= ARCH_TIMER_CTRL_ENABLE;
279         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
280         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
281         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
282 }
283 
284 #ifdef CONFIG_FSL_ERRATUM_A008585
285 static __always_inline void fsl_a008585_set_next_event(const int access,
286                 unsigned long evt, struct clock_event_device *clk)
287 {
288         unsigned long ctrl;
289         u64 cval = evt + arch_counter_get_cntvct();
290 
291         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
292         ctrl |= ARCH_TIMER_CTRL_ENABLE;
293         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
294 
295         if (access == ARCH_TIMER_PHYS_ACCESS)
296                 write_sysreg(cval, cntp_cval_el0);
297         else if (access == ARCH_TIMER_VIRT_ACCESS)
298                 write_sysreg(cval, cntv_cval_el0);
299 
300         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
301 }
302 
303 static int fsl_a008585_set_next_event_virt(unsigned long evt,
304                                            struct clock_event_device *clk)
305 {
306         fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
307         return 0;
308 }
309 
310 static int fsl_a008585_set_next_event_phys(unsigned long evt,
311                                            struct clock_event_device *clk)
312 {
313         fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
314         return 0;
315 }
316 #endif /* CONFIG_FSL_ERRATUM_A008585 */
317 
318 static int arch_timer_set_next_event_virt(unsigned long evt,
319                                           struct clock_event_device *clk)
320 {
321         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
322         return 0;
323 }
324 
325 static int arch_timer_set_next_event_phys(unsigned long evt,
326                                           struct clock_event_device *clk)
327 {
328         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
329         return 0;
330 }
331 
332 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
333                                               struct clock_event_device *clk)
334 {
335         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
336         return 0;
337 }
338 
339 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
340                                               struct clock_event_device *clk)
341 {
342         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
343         return 0;
344 }
345 
346 static void fsl_a008585_set_sne(struct clock_event_device *clk)
347 {
348 #ifdef CONFIG_FSL_ERRATUM_A008585
349         if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
350                 return;
351 
352         if (arch_timer_uses_ppi == VIRT_PPI)
353                 clk->set_next_event = fsl_a008585_set_next_event_virt;
354         else
355                 clk->set_next_event = fsl_a008585_set_next_event_phys;
356 #endif
357 }
358 
359 static void __arch_timer_setup(unsigned type,
360                                struct clock_event_device *clk)
361 {
362         clk->features = CLOCK_EVT_FEAT_ONESHOT;
363 
364         if (type == ARCH_CP15_TIMER) {
365                 if (arch_timer_c3stop)
366                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
367                 clk->name = "arch_sys_timer";
368                 clk->rating = 450;
369                 clk->cpumask = cpumask_of(smp_processor_id());
370                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
371                 switch (arch_timer_uses_ppi) {
372                 case VIRT_PPI:
373                         clk->set_state_shutdown = arch_timer_shutdown_virt;
374                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
375                         clk->set_next_event = arch_timer_set_next_event_virt;
376                         break;
377                 case PHYS_SECURE_PPI:
378                 case PHYS_NONSECURE_PPI:
379                 case HYP_PPI:
380                         clk->set_state_shutdown = arch_timer_shutdown_phys;
381                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
382                         clk->set_next_event = arch_timer_set_next_event_phys;
383                         break;
384                 default:
385                         BUG();
386                 }
387 
388                 fsl_a008585_set_sne(clk);
389         } else {
390                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
391                 clk->name = "arch_mem_timer";
392                 clk->rating = 400;
393                 clk->cpumask = cpu_all_mask;
394                 if (arch_timer_mem_use_virtual) {
395                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
396                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
397                         clk->set_next_event =
398                                 arch_timer_set_next_event_virt_mem;
399                 } else {
400                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
401                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
402                         clk->set_next_event =
403                                 arch_timer_set_next_event_phys_mem;
404                 }
405         }
406 
407         clk->set_state_shutdown(clk);
408 
409         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
410 }
411 
412 static void arch_timer_evtstrm_enable(int divider)
413 {
414         u32 cntkctl = arch_timer_get_cntkctl();
415 
416         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
417         /* Set the divider and enable virtual event stream */
418         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
419                         | ARCH_TIMER_VIRT_EVT_EN;
420         arch_timer_set_cntkctl(cntkctl);
421         elf_hwcap |= HWCAP_EVTSTRM;
422 #ifdef CONFIG_COMPAT
423         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
424 #endif
425 }
426 
427 static void arch_timer_configure_evtstream(void)
428 {
429         int evt_stream_div, pos;
430 
431         /* Find the closest power of two to the divisor */
432         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
433         pos = fls(evt_stream_div);
434         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
435                 pos--;
436         /* enable event stream */
437         arch_timer_evtstrm_enable(min(pos, 15));
438 }
439 
440 static void arch_counter_set_user_access(void)
441 {
442         u32 cntkctl = arch_timer_get_cntkctl();
443 
444         /* Disable user access to the timers and the physical counter */
445         /* Also disable virtual event stream */
446         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
447                         | ARCH_TIMER_USR_VT_ACCESS_EN
448                         | ARCH_TIMER_VIRT_EVT_EN
449                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
450 
451         /* Enable user access to the virtual counter */
452         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
453 
454         arch_timer_set_cntkctl(cntkctl);
455 }
456 
457 static bool arch_timer_has_nonsecure_ppi(void)
458 {
459         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
460                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
461 }
462 
463 static u32 check_ppi_trigger(int irq)
464 {
465         u32 flags = irq_get_trigger_type(irq);
466 
467         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
468                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
469                 pr_warn("WARNING: Please fix your firmware\n");
470                 flags = IRQF_TRIGGER_LOW;
471         }
472 
473         return flags;
474 }
475 
476 static int arch_timer_starting_cpu(unsigned int cpu)
477 {
478         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
479         u32 flags;
480 
481         __arch_timer_setup(ARCH_CP15_TIMER, clk);
482 
483         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
484         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
485 
486         if (arch_timer_has_nonsecure_ppi()) {
487                 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
488                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
489         }
490 
491         arch_counter_set_user_access();
492         if (evtstrm_enable)
493                 arch_timer_configure_evtstream();
494 
495         return 0;
496 }
497 
498 static void
499 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
500 {
501         /* Who has more than one independent system counter? */
502         if (arch_timer_rate)
503                 return;
504 
505         /*
506          * Try to determine the frequency from the device tree or CNTFRQ,
507          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
508          */
509         if (!acpi_disabled ||
510             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
511                 if (cntbase)
512                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
513                 else
514                         arch_timer_rate = arch_timer_get_cntfrq();
515         }
516 
517         /* Check the timer frequency. */
518         if (arch_timer_rate == 0)
519                 pr_warn("Architected timer frequency not available\n");
520 }
521 
522 static void arch_timer_banner(unsigned type)
523 {
524         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
525                      type & ARCH_CP15_TIMER ? "cp15" : "",
526                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
527                      type & ARCH_MEM_TIMER ? "mmio" : "",
528                      (unsigned long)arch_timer_rate / 1000000,
529                      (unsigned long)(arch_timer_rate / 10000) % 100,
530                      type & ARCH_CP15_TIMER ?
531                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
532                         "",
533                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
534                      type & ARCH_MEM_TIMER ?
535                         arch_timer_mem_use_virtual ? "virt" : "phys" :
536                         "");
537 }
538 
539 u32 arch_timer_get_rate(void)
540 {
541         return arch_timer_rate;
542 }
543 
544 static u64 arch_counter_get_cntvct_mem(void)
545 {
546         u32 vct_lo, vct_hi, tmp_hi;
547 
548         do {
549                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
550                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
551                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
552         } while (vct_hi != tmp_hi);
553 
554         return ((u64) vct_hi << 32) | vct_lo;
555 }
556 
557 /*
558  * Default to cp15 based access because arm64 uses this function for
559  * sched_clock() before DT is probed and the cp15 method is guaranteed
560  * to exist on arm64. arm doesn't use this before DT is probed so even
561  * if we don't have the cp15 accessors we won't have a problem.
562  */
563 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
564 
565 static u64 arch_counter_read(struct clocksource *cs)
566 {
567         return arch_timer_read_counter();
568 }
569 
570 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
571 {
572         return arch_timer_read_counter();
573 }
574 
575 static struct clocksource clocksource_counter = {
576         .name   = "arch_sys_counter",
577         .rating = 400,
578         .read   = arch_counter_read,
579         .mask   = CLOCKSOURCE_MASK(56),
580         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
581 };
582 
583 static struct cyclecounter cyclecounter = {
584         .read   = arch_counter_read_cc,
585         .mask   = CLOCKSOURCE_MASK(56),
586 };
587 
588 static struct arch_timer_kvm_info arch_timer_kvm_info;
589 
590 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
591 {
592         return &arch_timer_kvm_info;
593 }
594 
595 static void __init arch_counter_register(unsigned type)
596 {
597         u64 start_count;
598 
599         /* Register the CP15 based counter if we have one */
600         if (type & ARCH_CP15_TIMER) {
601                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
602                         arch_timer_read_counter = arch_counter_get_cntvct;
603                 else
604                         arch_timer_read_counter = arch_counter_get_cntpct;
605 
606                 clocksource_counter.archdata.vdso_direct = true;
607 
608 #ifdef CONFIG_FSL_ERRATUM_A008585
609                 /*
610                  * Don't use the vdso fastpath if errata require using
611                  * the out-of-line counter accessor.
612                  */
613                 if (static_branch_unlikely(&arch_timer_read_ool_enabled))
614                         clocksource_counter.archdata.vdso_direct = false;
615 #endif
616         } else {
617                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
618         }
619 
620         if (!arch_counter_suspend_stop)
621                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
622         start_count = arch_timer_read_counter();
623         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
624         cyclecounter.mult = clocksource_counter.mult;
625         cyclecounter.shift = clocksource_counter.shift;
626         timecounter_init(&arch_timer_kvm_info.timecounter,
627                          &cyclecounter, start_count);
628 
629         /* 56 bits minimum, so we assume worst case rollover */
630         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
631 }
632 
633 static void arch_timer_stop(struct clock_event_device *clk)
634 {
635         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
636                  clk->irq, smp_processor_id());
637 
638         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
639         if (arch_timer_has_nonsecure_ppi())
640                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
641 
642         clk->set_state_shutdown(clk);
643 }
644 
645 static int arch_timer_dying_cpu(unsigned int cpu)
646 {
647         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
648 
649         arch_timer_stop(clk);
650         return 0;
651 }
652 
653 #ifdef CONFIG_CPU_PM
654 static unsigned int saved_cntkctl;
655 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
656                                     unsigned long action, void *hcpu)
657 {
658         if (action == CPU_PM_ENTER)
659                 saved_cntkctl = arch_timer_get_cntkctl();
660         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
661                 arch_timer_set_cntkctl(saved_cntkctl);
662         return NOTIFY_OK;
663 }
664 
665 static struct notifier_block arch_timer_cpu_pm_notifier = {
666         .notifier_call = arch_timer_cpu_pm_notify,
667 };
668 
669 static int __init arch_timer_cpu_pm_init(void)
670 {
671         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
672 }
673 
674 static void __init arch_timer_cpu_pm_deinit(void)
675 {
676         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
677 }
678 
679 #else
680 static int __init arch_timer_cpu_pm_init(void)
681 {
682         return 0;
683 }
684 
685 static void __init arch_timer_cpu_pm_deinit(void)
686 {
687 }
688 #endif
689 
690 static int __init arch_timer_register(void)
691 {
692         int err;
693         int ppi;
694 
695         arch_timer_evt = alloc_percpu(struct clock_event_device);
696         if (!arch_timer_evt) {
697                 err = -ENOMEM;
698                 goto out;
699         }
700 
701         ppi = arch_timer_ppi[arch_timer_uses_ppi];
702         switch (arch_timer_uses_ppi) {
703         case VIRT_PPI:
704                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
705                                          "arch_timer", arch_timer_evt);
706                 break;
707         case PHYS_SECURE_PPI:
708         case PHYS_NONSECURE_PPI:
709                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
710                                          "arch_timer", arch_timer_evt);
711                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
712                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
713                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
714                                                  "arch_timer", arch_timer_evt);
715                         if (err)
716                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
717                                                 arch_timer_evt);
718                 }
719                 break;
720         case HYP_PPI:
721                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
722                                          "arch_timer", arch_timer_evt);
723                 break;
724         default:
725                 BUG();
726         }
727 
728         if (err) {
729                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
730                        ppi, err);
731                 goto out_free;
732         }
733 
734         err = arch_timer_cpu_pm_init();
735         if (err)
736                 goto out_unreg_notify;
737 
738 
739         /* Register and immediately configure the timer on the boot CPU */
740         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
741                                 "clockevents/arm/arch_timer:starting",
742                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
743         if (err)
744                 goto out_unreg_cpupm;
745         return 0;
746 
747 out_unreg_cpupm:
748         arch_timer_cpu_pm_deinit();
749 
750 out_unreg_notify:
751         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
752         if (arch_timer_has_nonsecure_ppi())
753                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
754                                 arch_timer_evt);
755 
756 out_free:
757         free_percpu(arch_timer_evt);
758 out:
759         return err;
760 }
761 
762 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
763 {
764         int ret;
765         irq_handler_t func;
766         struct arch_timer *t;
767 
768         t = kzalloc(sizeof(*t), GFP_KERNEL);
769         if (!t)
770                 return -ENOMEM;
771 
772         t->base = base;
773         t->evt.irq = irq;
774         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
775 
776         if (arch_timer_mem_use_virtual)
777                 func = arch_timer_handler_virt_mem;
778         else
779                 func = arch_timer_handler_phys_mem;
780 
781         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
782         if (ret) {
783                 pr_err("arch_timer: Failed to request mem timer irq\n");
784                 kfree(t);
785         }
786 
787         return ret;
788 }
789 
790 static const struct of_device_id arch_timer_of_match[] __initconst = {
791         { .compatible   = "arm,armv7-timer",    },
792         { .compatible   = "arm,armv8-timer",    },
793         {},
794 };
795 
796 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
797         { .compatible   = "arm,armv7-timer-mem", },
798         {},
799 };
800 
801 static bool __init
802 arch_timer_needs_probing(int type, const struct of_device_id *matches)
803 {
804         struct device_node *dn;
805         bool needs_probing = false;
806 
807         dn = of_find_matching_node(NULL, matches);
808         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
809                 needs_probing = true;
810         of_node_put(dn);
811 
812         return needs_probing;
813 }
814 
815 static int __init arch_timer_common_init(void)
816 {
817         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
818 
819         /* Wait until both nodes are probed if we have two timers */
820         if ((arch_timers_present & mask) != mask) {
821                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
822                         return 0;
823                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
824                         return 0;
825         }
826 
827         arch_timer_banner(arch_timers_present);
828         arch_counter_register(arch_timers_present);
829         return arch_timer_arch_init();
830 }
831 
832 static int __init arch_timer_init(void)
833 {
834         int ret;
835         /*
836          * If HYP mode is available, we know that the physical timer
837          * has been configured to be accessible from PL1. Use it, so
838          * that a guest can use the virtual timer instead.
839          *
840          * If no interrupt provided for virtual timer, we'll have to
841          * stick to the physical timer. It'd better be accessible...
842          *
843          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
844          * accesses to CNTP_*_EL1 registers are silently redirected to
845          * their CNTHP_*_EL2 counterparts, and use a different PPI
846          * number.
847          */
848         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
849                 bool has_ppi;
850 
851                 if (is_kernel_in_hyp_mode()) {
852                         arch_timer_uses_ppi = HYP_PPI;
853                         has_ppi = !!arch_timer_ppi[HYP_PPI];
854                 } else {
855                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
856                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
857                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
858                 }
859 
860                 if (!has_ppi) {
861                         pr_warn("arch_timer: No interrupt available, giving up\n");
862                         return -EINVAL;
863                 }
864         }
865 
866         ret = arch_timer_register();
867         if (ret)
868                 return ret;
869 
870         ret = arch_timer_common_init();
871         if (ret)
872                 return ret;
873 
874         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
875         
876         return 0;
877 }
878 
879 static int __init arch_timer_of_init(struct device_node *np)
880 {
881         int i;
882 
883         if (arch_timers_present & ARCH_CP15_TIMER) {
884                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
885                 return 0;
886         }
887 
888         arch_timers_present |= ARCH_CP15_TIMER;
889         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
890                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
891 
892         arch_timer_detect_rate(NULL, np);
893 
894         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
895 
896 #ifdef CONFIG_FSL_ERRATUM_A008585
897         if (fsl_a008585_enable < 0)
898                 fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
899         if (fsl_a008585_enable) {
900                 static_branch_enable(&arch_timer_read_ool_enabled);
901                 pr_info("Enabling workaround for FSL erratum A-008585\n");
902         }
903 #endif
904 
905         /*
906          * If we cannot rely on firmware initializing the timer registers then
907          * we should use the physical timers instead.
908          */
909         if (IS_ENABLED(CONFIG_ARM) &&
910             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
911                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
912 
913         /* On some systems, the counter stops ticking when in suspend. */
914         arch_counter_suspend_stop = of_property_read_bool(np,
915                                                          "arm,no-tick-in-suspend");
916 
917         return arch_timer_init();
918 }
919 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
920 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
921 
922 static int __init arch_timer_mem_init(struct device_node *np)
923 {
924         struct device_node *frame, *best_frame = NULL;
925         void __iomem *cntctlbase, *base;
926         unsigned int irq, ret = -EINVAL;
927         u32 cnttidr;
928 
929         arch_timers_present |= ARCH_MEM_TIMER;
930         cntctlbase = of_iomap(np, 0);
931         if (!cntctlbase) {
932                 pr_err("arch_timer: Can't find CNTCTLBase\n");
933                 return -ENXIO;
934         }
935 
936         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
937 
938         /*
939          * Try to find a virtual capable frame. Otherwise fall back to a
940          * physical capable frame.
941          */
942         for_each_available_child_of_node(np, frame) {
943                 int n;
944                 u32 cntacr;
945 
946                 if (of_property_read_u32(frame, "frame-number", &n)) {
947                         pr_err("arch_timer: Missing frame-number\n");
948                         of_node_put(frame);
949                         goto out;
950                 }
951 
952                 /* Try enabling everything, and see what sticks */
953                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
954                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
955                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
956                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
957 
958                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
959                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
960                         of_node_put(best_frame);
961                         best_frame = frame;
962                         arch_timer_mem_use_virtual = true;
963                         break;
964                 }
965 
966                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
967                         continue;
968 
969                 of_node_put(best_frame);
970                 best_frame = of_node_get(frame);
971         }
972 
973         ret= -ENXIO;
974         base = arch_counter_base = of_io_request_and_map(best_frame, 0,
975                                                          "arch_mem_timer");
976         if (IS_ERR(base)) {
977                 pr_err("arch_timer: Can't map frame's registers\n");
978                 goto out;
979         }
980 
981         if (arch_timer_mem_use_virtual)
982                 irq = irq_of_parse_and_map(best_frame, 1);
983         else
984                 irq = irq_of_parse_and_map(best_frame, 0);
985 
986         ret = -EINVAL;
987         if (!irq) {
988                 pr_err("arch_timer: Frame missing %s irq",
989                        arch_timer_mem_use_virtual ? "virt" : "phys");
990                 goto out;
991         }
992 
993         arch_timer_detect_rate(base, np);
994         ret = arch_timer_mem_register(base, irq);
995         if (ret)
996                 goto out;
997 
998         return arch_timer_common_init();
999 out:
1000         iounmap(cntctlbase);
1001         of_node_put(best_frame);
1002         return ret;
1003 }
1004 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1005                        arch_timer_mem_init);
1006 
1007 #ifdef CONFIG_ACPI
1008 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
1009 {
1010         int trigger, polarity;
1011 
1012         if (!interrupt)
1013                 return 0;
1014 
1015         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
1016                         : ACPI_LEVEL_SENSITIVE;
1017 
1018         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
1019                         : ACPI_ACTIVE_HIGH;
1020 
1021         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
1022 }
1023 
1024 /* Initialize per-processor generic timer */
1025 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1026 {
1027         struct acpi_table_gtdt *gtdt;
1028 
1029         if (arch_timers_present & ARCH_CP15_TIMER) {
1030                 pr_warn("arch_timer: already initialized, skipping\n");
1031                 return -EINVAL;
1032         }
1033 
1034         gtdt = container_of(table, struct acpi_table_gtdt, header);
1035 
1036         arch_timers_present |= ARCH_CP15_TIMER;
1037 
1038         arch_timer_ppi[PHYS_SECURE_PPI] =
1039                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
1040                 gtdt->secure_el1_flags);
1041 
1042         arch_timer_ppi[PHYS_NONSECURE_PPI] =
1043                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
1044                 gtdt->non_secure_el1_flags);
1045 
1046         arch_timer_ppi[VIRT_PPI] =
1047                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
1048                 gtdt->virtual_timer_flags);
1049 
1050         arch_timer_ppi[HYP_PPI] =
1051                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
1052                 gtdt->non_secure_el2_flags);
1053 
1054         /* Get the frequency from CNTFRQ */
1055         arch_timer_detect_rate(NULL, NULL);
1056 
1057         /* Always-on capability */
1058         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
1059 
1060         arch_timer_init();
1061         return 0;
1062 }
1063 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1064 #endif
1065 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us