Version:  2.0.40 2.2.26 2.4.37 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9

Linux/drivers/clocksource/arm_arch_timer.c

  1 /*
  2  *  linux/drivers/clocksource/arm_arch_timer.c
  3  *
  4  *  Copyright (C) 2011 ARM Ltd.
  5  *  All Rights Reserved
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  */
 11 
 12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
 13 
 14 #include <linux/init.h>
 15 #include <linux/kernel.h>
 16 #include <linux/device.h>
 17 #include <linux/smp.h>
 18 #include <linux/cpu.h>
 19 #include <linux/cpu_pm.h>
 20 #include <linux/clockchips.h>
 21 #include <linux/clocksource.h>
 22 #include <linux/interrupt.h>
 23 #include <linux/of_irq.h>
 24 #include <linux/of_address.h>
 25 #include <linux/io.h>
 26 #include <linux/slab.h>
 27 #include <linux/sched_clock.h>
 28 #include <linux/acpi.h>
 29 
 30 #include <asm/arch_timer.h>
 31 #include <asm/virt.h>
 32 
 33 #include <clocksource/arm_arch_timer.h>
 34 
 35 #define CNTTIDR         0x08
 36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
 37 
 38 #define CNTACR(n)       (0x40 + ((n) * 4))
 39 #define CNTACR_RPCT     BIT(0)
 40 #define CNTACR_RVCT     BIT(1)
 41 #define CNTACR_RFRQ     BIT(2)
 42 #define CNTACR_RVOFF    BIT(3)
 43 #define CNTACR_RWVT     BIT(4)
 44 #define CNTACR_RWPT     BIT(5)
 45 
 46 #define CNTVCT_LO       0x08
 47 #define CNTVCT_HI       0x0c
 48 #define CNTFRQ          0x10
 49 #define CNTP_TVAL       0x28
 50 #define CNTP_CTL        0x2c
 51 #define CNTV_TVAL       0x38
 52 #define CNTV_CTL        0x3c
 53 
 54 #define ARCH_CP15_TIMER BIT(0)
 55 #define ARCH_MEM_TIMER  BIT(1)
 56 static unsigned arch_timers_present __initdata;
 57 
 58 static void __iomem *arch_counter_base;
 59 
 60 struct arch_timer {
 61         void __iomem *base;
 62         struct clock_event_device evt;
 63 };
 64 
 65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
 66 
 67 static u32 arch_timer_rate;
 68 
 69 enum ppi_nr {
 70         PHYS_SECURE_PPI,
 71         PHYS_NONSECURE_PPI,
 72         VIRT_PPI,
 73         HYP_PPI,
 74         MAX_TIMER_PPI
 75 };
 76 
 77 static int arch_timer_ppi[MAX_TIMER_PPI];
 78 
 79 static struct clock_event_device __percpu *arch_timer_evt;
 80 
 81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
 82 static bool arch_timer_c3stop;
 83 static bool arch_timer_mem_use_virtual;
 84 
 85 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
 86 
 87 static int __init early_evtstrm_cfg(char *buf)
 88 {
 89         return strtobool(buf, &evtstrm_enable);
 90 }
 91 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
 92 
 93 /*
 94  * Architected system timer support.
 95  */
 96 
 97 #ifdef CONFIG_FSL_ERRATUM_A008585
 98 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
 99 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
100 
101 static int fsl_a008585_enable = -1;
102 
103 static int __init early_fsl_a008585_cfg(char *buf)
104 {
105         int ret;
106         bool val;
107 
108         ret = strtobool(buf, &val);
109         if (ret)
110                 return ret;
111 
112         fsl_a008585_enable = val;
113         return 0;
114 }
115 early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg);
116 
117 u32 __fsl_a008585_read_cntp_tval_el0(void)
118 {
119         return __fsl_a008585_read_reg(cntp_tval_el0);
120 }
121 
122 u32 __fsl_a008585_read_cntv_tval_el0(void)
123 {
124         return __fsl_a008585_read_reg(cntv_tval_el0);
125 }
126 
127 u64 __fsl_a008585_read_cntvct_el0(void)
128 {
129         return __fsl_a008585_read_reg(cntvct_el0);
130 }
131 EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
132 #endif /* CONFIG_FSL_ERRATUM_A008585 */
133 
134 static __always_inline
135 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
136                           struct clock_event_device *clk)
137 {
138         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
139                 struct arch_timer *timer = to_arch_timer(clk);
140                 switch (reg) {
141                 case ARCH_TIMER_REG_CTRL:
142                         writel_relaxed(val, timer->base + CNTP_CTL);
143                         break;
144                 case ARCH_TIMER_REG_TVAL:
145                         writel_relaxed(val, timer->base + CNTP_TVAL);
146                         break;
147                 }
148         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
149                 struct arch_timer *timer = to_arch_timer(clk);
150                 switch (reg) {
151                 case ARCH_TIMER_REG_CTRL:
152                         writel_relaxed(val, timer->base + CNTV_CTL);
153                         break;
154                 case ARCH_TIMER_REG_TVAL:
155                         writel_relaxed(val, timer->base + CNTV_TVAL);
156                         break;
157                 }
158         } else {
159                 arch_timer_reg_write_cp15(access, reg, val);
160         }
161 }
162 
163 static __always_inline
164 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
165                         struct clock_event_device *clk)
166 {
167         u32 val;
168 
169         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
170                 struct arch_timer *timer = to_arch_timer(clk);
171                 switch (reg) {
172                 case ARCH_TIMER_REG_CTRL:
173                         val = readl_relaxed(timer->base + CNTP_CTL);
174                         break;
175                 case ARCH_TIMER_REG_TVAL:
176                         val = readl_relaxed(timer->base + CNTP_TVAL);
177                         break;
178                 }
179         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
180                 struct arch_timer *timer = to_arch_timer(clk);
181                 switch (reg) {
182                 case ARCH_TIMER_REG_CTRL:
183                         val = readl_relaxed(timer->base + CNTV_CTL);
184                         break;
185                 case ARCH_TIMER_REG_TVAL:
186                         val = readl_relaxed(timer->base + CNTV_TVAL);
187                         break;
188                 }
189         } else {
190                 val = arch_timer_reg_read_cp15(access, reg);
191         }
192 
193         return val;
194 }
195 
196 static __always_inline irqreturn_t timer_handler(const int access,
197                                         struct clock_event_device *evt)
198 {
199         unsigned long ctrl;
200 
201         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
202         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
203                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
204                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
205                 evt->event_handler(evt);
206                 return IRQ_HANDLED;
207         }
208 
209         return IRQ_NONE;
210 }
211 
212 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
213 {
214         struct clock_event_device *evt = dev_id;
215 
216         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
217 }
218 
219 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
220 {
221         struct clock_event_device *evt = dev_id;
222 
223         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
224 }
225 
226 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
227 {
228         struct clock_event_device *evt = dev_id;
229 
230         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
231 }
232 
233 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
234 {
235         struct clock_event_device *evt = dev_id;
236 
237         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
238 }
239 
240 static __always_inline int timer_shutdown(const int access,
241                                           struct clock_event_device *clk)
242 {
243         unsigned long ctrl;
244 
245         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
246         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
247         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
248 
249         return 0;
250 }
251 
252 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
253 {
254         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
255 }
256 
257 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
258 {
259         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
260 }
261 
262 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
263 {
264         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
265 }
266 
267 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
268 {
269         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
270 }
271 
272 static __always_inline void set_next_event(const int access, unsigned long evt,
273                                            struct clock_event_device *clk)
274 {
275         unsigned long ctrl;
276         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
277         ctrl |= ARCH_TIMER_CTRL_ENABLE;
278         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
279         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
280         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
281 }
282 
283 #ifdef CONFIG_FSL_ERRATUM_A008585
284 static __always_inline void fsl_a008585_set_next_event(const int access,
285                 unsigned long evt, struct clock_event_device *clk)
286 {
287         unsigned long ctrl;
288         u64 cval = evt + arch_counter_get_cntvct();
289 
290         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
291         ctrl |= ARCH_TIMER_CTRL_ENABLE;
292         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
293 
294         if (access == ARCH_TIMER_PHYS_ACCESS)
295                 write_sysreg(cval, cntp_cval_el0);
296         else if (access == ARCH_TIMER_VIRT_ACCESS)
297                 write_sysreg(cval, cntv_cval_el0);
298 
299         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
300 }
301 
302 static int fsl_a008585_set_next_event_virt(unsigned long evt,
303                                            struct clock_event_device *clk)
304 {
305         fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
306         return 0;
307 }
308 
309 static int fsl_a008585_set_next_event_phys(unsigned long evt,
310                                            struct clock_event_device *clk)
311 {
312         fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
313         return 0;
314 }
315 #endif /* CONFIG_FSL_ERRATUM_A008585 */
316 
317 static int arch_timer_set_next_event_virt(unsigned long evt,
318                                           struct clock_event_device *clk)
319 {
320         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
321         return 0;
322 }
323 
324 static int arch_timer_set_next_event_phys(unsigned long evt,
325                                           struct clock_event_device *clk)
326 {
327         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
328         return 0;
329 }
330 
331 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
332                                               struct clock_event_device *clk)
333 {
334         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
335         return 0;
336 }
337 
338 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
339                                               struct clock_event_device *clk)
340 {
341         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
342         return 0;
343 }
344 
345 static void fsl_a008585_set_sne(struct clock_event_device *clk)
346 {
347 #ifdef CONFIG_FSL_ERRATUM_A008585
348         if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
349                 return;
350 
351         if (arch_timer_uses_ppi == VIRT_PPI)
352                 clk->set_next_event = fsl_a008585_set_next_event_virt;
353         else
354                 clk->set_next_event = fsl_a008585_set_next_event_phys;
355 #endif
356 }
357 
358 static void __arch_timer_setup(unsigned type,
359                                struct clock_event_device *clk)
360 {
361         clk->features = CLOCK_EVT_FEAT_ONESHOT;
362 
363         if (type == ARCH_CP15_TIMER) {
364                 if (arch_timer_c3stop)
365                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
366                 clk->name = "arch_sys_timer";
367                 clk->rating = 450;
368                 clk->cpumask = cpumask_of(smp_processor_id());
369                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
370                 switch (arch_timer_uses_ppi) {
371                 case VIRT_PPI:
372                         clk->set_state_shutdown = arch_timer_shutdown_virt;
373                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
374                         clk->set_next_event = arch_timer_set_next_event_virt;
375                         break;
376                 case PHYS_SECURE_PPI:
377                 case PHYS_NONSECURE_PPI:
378                 case HYP_PPI:
379                         clk->set_state_shutdown = arch_timer_shutdown_phys;
380                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
381                         clk->set_next_event = arch_timer_set_next_event_phys;
382                         break;
383                 default:
384                         BUG();
385                 }
386 
387                 fsl_a008585_set_sne(clk);
388         } else {
389                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
390                 clk->name = "arch_mem_timer";
391                 clk->rating = 400;
392                 clk->cpumask = cpu_all_mask;
393                 if (arch_timer_mem_use_virtual) {
394                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
395                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
396                         clk->set_next_event =
397                                 arch_timer_set_next_event_virt_mem;
398                 } else {
399                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
400                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
401                         clk->set_next_event =
402                                 arch_timer_set_next_event_phys_mem;
403                 }
404         }
405 
406         clk->set_state_shutdown(clk);
407 
408         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
409 }
410 
411 static void arch_timer_evtstrm_enable(int divider)
412 {
413         u32 cntkctl = arch_timer_get_cntkctl();
414 
415         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
416         /* Set the divider and enable virtual event stream */
417         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
418                         | ARCH_TIMER_VIRT_EVT_EN;
419         arch_timer_set_cntkctl(cntkctl);
420         elf_hwcap |= HWCAP_EVTSTRM;
421 #ifdef CONFIG_COMPAT
422         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
423 #endif
424 }
425 
426 static void arch_timer_configure_evtstream(void)
427 {
428         int evt_stream_div, pos;
429 
430         /* Find the closest power of two to the divisor */
431         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
432         pos = fls(evt_stream_div);
433         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
434                 pos--;
435         /* enable event stream */
436         arch_timer_evtstrm_enable(min(pos, 15));
437 }
438 
439 static void arch_counter_set_user_access(void)
440 {
441         u32 cntkctl = arch_timer_get_cntkctl();
442 
443         /* Disable user access to the timers and the physical counter */
444         /* Also disable virtual event stream */
445         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
446                         | ARCH_TIMER_USR_VT_ACCESS_EN
447                         | ARCH_TIMER_VIRT_EVT_EN
448                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
449 
450         /* Enable user access to the virtual counter */
451         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
452 
453         arch_timer_set_cntkctl(cntkctl);
454 }
455 
456 static bool arch_timer_has_nonsecure_ppi(void)
457 {
458         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
459                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
460 }
461 
462 static u32 check_ppi_trigger(int irq)
463 {
464         u32 flags = irq_get_trigger_type(irq);
465 
466         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
467                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
468                 pr_warn("WARNING: Please fix your firmware\n");
469                 flags = IRQF_TRIGGER_LOW;
470         }
471 
472         return flags;
473 }
474 
475 static int arch_timer_starting_cpu(unsigned int cpu)
476 {
477         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
478         u32 flags;
479 
480         __arch_timer_setup(ARCH_CP15_TIMER, clk);
481 
482         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
483         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
484 
485         if (arch_timer_has_nonsecure_ppi()) {
486                 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
487                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
488         }
489 
490         arch_counter_set_user_access();
491         if (evtstrm_enable)
492                 arch_timer_configure_evtstream();
493 
494         return 0;
495 }
496 
497 static void
498 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
499 {
500         /* Who has more than one independent system counter? */
501         if (arch_timer_rate)
502                 return;
503 
504         /*
505          * Try to determine the frequency from the device tree or CNTFRQ,
506          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
507          */
508         if (!acpi_disabled ||
509             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
510                 if (cntbase)
511                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
512                 else
513                         arch_timer_rate = arch_timer_get_cntfrq();
514         }
515 
516         /* Check the timer frequency. */
517         if (arch_timer_rate == 0)
518                 pr_warn("Architected timer frequency not available\n");
519 }
520 
521 static void arch_timer_banner(unsigned type)
522 {
523         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
524                      type & ARCH_CP15_TIMER ? "cp15" : "",
525                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
526                      type & ARCH_MEM_TIMER ? "mmio" : "",
527                      (unsigned long)arch_timer_rate / 1000000,
528                      (unsigned long)(arch_timer_rate / 10000) % 100,
529                      type & ARCH_CP15_TIMER ?
530                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
531                         "",
532                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
533                      type & ARCH_MEM_TIMER ?
534                         arch_timer_mem_use_virtual ? "virt" : "phys" :
535                         "");
536 }
537 
538 u32 arch_timer_get_rate(void)
539 {
540         return arch_timer_rate;
541 }
542 
543 static u64 arch_counter_get_cntvct_mem(void)
544 {
545         u32 vct_lo, vct_hi, tmp_hi;
546 
547         do {
548                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
549                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
550                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
551         } while (vct_hi != tmp_hi);
552 
553         return ((u64) vct_hi << 32) | vct_lo;
554 }
555 
556 /*
557  * Default to cp15 based access because arm64 uses this function for
558  * sched_clock() before DT is probed and the cp15 method is guaranteed
559  * to exist on arm64. arm doesn't use this before DT is probed so even
560  * if we don't have the cp15 accessors we won't have a problem.
561  */
562 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
563 
564 static cycle_t arch_counter_read(struct clocksource *cs)
565 {
566         return arch_timer_read_counter();
567 }
568 
569 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
570 {
571         return arch_timer_read_counter();
572 }
573 
574 static struct clocksource clocksource_counter = {
575         .name   = "arch_sys_counter",
576         .rating = 400,
577         .read   = arch_counter_read,
578         .mask   = CLOCKSOURCE_MASK(56),
579         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
580 };
581 
582 static struct cyclecounter cyclecounter = {
583         .read   = arch_counter_read_cc,
584         .mask   = CLOCKSOURCE_MASK(56),
585 };
586 
587 static struct arch_timer_kvm_info arch_timer_kvm_info;
588 
589 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
590 {
591         return &arch_timer_kvm_info;
592 }
593 
594 static void __init arch_counter_register(unsigned type)
595 {
596         u64 start_count;
597 
598         /* Register the CP15 based counter if we have one */
599         if (type & ARCH_CP15_TIMER) {
600                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
601                         arch_timer_read_counter = arch_counter_get_cntvct;
602                 else
603                         arch_timer_read_counter = arch_counter_get_cntpct;
604 
605                 clocksource_counter.archdata.vdso_direct = true;
606 
607 #ifdef CONFIG_FSL_ERRATUM_A008585
608                 /*
609                  * Don't use the vdso fastpath if errata require using
610                  * the out-of-line counter accessor.
611                  */
612                 if (static_branch_unlikely(&arch_timer_read_ool_enabled))
613                         clocksource_counter.archdata.vdso_direct = false;
614 #endif
615         } else {
616                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
617         }
618 
619         start_count = arch_timer_read_counter();
620         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
621         cyclecounter.mult = clocksource_counter.mult;
622         cyclecounter.shift = clocksource_counter.shift;
623         timecounter_init(&arch_timer_kvm_info.timecounter,
624                          &cyclecounter, start_count);
625 
626         /* 56 bits minimum, so we assume worst case rollover */
627         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
628 }
629 
630 static void arch_timer_stop(struct clock_event_device *clk)
631 {
632         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
633                  clk->irq, smp_processor_id());
634 
635         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
636         if (arch_timer_has_nonsecure_ppi())
637                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
638 
639         clk->set_state_shutdown(clk);
640 }
641 
642 static int arch_timer_dying_cpu(unsigned int cpu)
643 {
644         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
645 
646         arch_timer_stop(clk);
647         return 0;
648 }
649 
650 #ifdef CONFIG_CPU_PM
651 static unsigned int saved_cntkctl;
652 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
653                                     unsigned long action, void *hcpu)
654 {
655         if (action == CPU_PM_ENTER)
656                 saved_cntkctl = arch_timer_get_cntkctl();
657         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
658                 arch_timer_set_cntkctl(saved_cntkctl);
659         return NOTIFY_OK;
660 }
661 
662 static struct notifier_block arch_timer_cpu_pm_notifier = {
663         .notifier_call = arch_timer_cpu_pm_notify,
664 };
665 
666 static int __init arch_timer_cpu_pm_init(void)
667 {
668         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
669 }
670 
671 static void __init arch_timer_cpu_pm_deinit(void)
672 {
673         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
674 }
675 
676 #else
677 static int __init arch_timer_cpu_pm_init(void)
678 {
679         return 0;
680 }
681 
682 static void __init arch_timer_cpu_pm_deinit(void)
683 {
684 }
685 #endif
686 
687 static int __init arch_timer_register(void)
688 {
689         int err;
690         int ppi;
691 
692         arch_timer_evt = alloc_percpu(struct clock_event_device);
693         if (!arch_timer_evt) {
694                 err = -ENOMEM;
695                 goto out;
696         }
697 
698         ppi = arch_timer_ppi[arch_timer_uses_ppi];
699         switch (arch_timer_uses_ppi) {
700         case VIRT_PPI:
701                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
702                                          "arch_timer", arch_timer_evt);
703                 break;
704         case PHYS_SECURE_PPI:
705         case PHYS_NONSECURE_PPI:
706                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
707                                          "arch_timer", arch_timer_evt);
708                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
709                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
710                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
711                                                  "arch_timer", arch_timer_evt);
712                         if (err)
713                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
714                                                 arch_timer_evt);
715                 }
716                 break;
717         case HYP_PPI:
718                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
719                                          "arch_timer", arch_timer_evt);
720                 break;
721         default:
722                 BUG();
723         }
724 
725         if (err) {
726                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
727                        ppi, err);
728                 goto out_free;
729         }
730 
731         err = arch_timer_cpu_pm_init();
732         if (err)
733                 goto out_unreg_notify;
734 
735 
736         /* Register and immediately configure the timer on the boot CPU */
737         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
738                                 "AP_ARM_ARCH_TIMER_STARTING",
739                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
740         if (err)
741                 goto out_unreg_cpupm;
742         return 0;
743 
744 out_unreg_cpupm:
745         arch_timer_cpu_pm_deinit();
746 
747 out_unreg_notify:
748         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
749         if (arch_timer_has_nonsecure_ppi())
750                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
751                                 arch_timer_evt);
752 
753 out_free:
754         free_percpu(arch_timer_evt);
755 out:
756         return err;
757 }
758 
759 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
760 {
761         int ret;
762         irq_handler_t func;
763         struct arch_timer *t;
764 
765         t = kzalloc(sizeof(*t), GFP_KERNEL);
766         if (!t)
767                 return -ENOMEM;
768 
769         t->base = base;
770         t->evt.irq = irq;
771         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
772 
773         if (arch_timer_mem_use_virtual)
774                 func = arch_timer_handler_virt_mem;
775         else
776                 func = arch_timer_handler_phys_mem;
777 
778         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
779         if (ret) {
780                 pr_err("arch_timer: Failed to request mem timer irq\n");
781                 kfree(t);
782         }
783 
784         return ret;
785 }
786 
787 static const struct of_device_id arch_timer_of_match[] __initconst = {
788         { .compatible   = "arm,armv7-timer",    },
789         { .compatible   = "arm,armv8-timer",    },
790         {},
791 };
792 
793 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
794         { .compatible   = "arm,armv7-timer-mem", },
795         {},
796 };
797 
798 static bool __init
799 arch_timer_needs_probing(int type, const struct of_device_id *matches)
800 {
801         struct device_node *dn;
802         bool needs_probing = false;
803 
804         dn = of_find_matching_node(NULL, matches);
805         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
806                 needs_probing = true;
807         of_node_put(dn);
808 
809         return needs_probing;
810 }
811 
812 static int __init arch_timer_common_init(void)
813 {
814         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
815 
816         /* Wait until both nodes are probed if we have two timers */
817         if ((arch_timers_present & mask) != mask) {
818                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
819                         return 0;
820                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
821                         return 0;
822         }
823 
824         arch_timer_banner(arch_timers_present);
825         arch_counter_register(arch_timers_present);
826         return arch_timer_arch_init();
827 }
828 
829 static int __init arch_timer_init(void)
830 {
831         int ret;
832         /*
833          * If HYP mode is available, we know that the physical timer
834          * has been configured to be accessible from PL1. Use it, so
835          * that a guest can use the virtual timer instead.
836          *
837          * If no interrupt provided for virtual timer, we'll have to
838          * stick to the physical timer. It'd better be accessible...
839          *
840          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
841          * accesses to CNTP_*_EL1 registers are silently redirected to
842          * their CNTHP_*_EL2 counterparts, and use a different PPI
843          * number.
844          */
845         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
846                 bool has_ppi;
847 
848                 if (is_kernel_in_hyp_mode()) {
849                         arch_timer_uses_ppi = HYP_PPI;
850                         has_ppi = !!arch_timer_ppi[HYP_PPI];
851                 } else {
852                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
853                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
854                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
855                 }
856 
857                 if (!has_ppi) {
858                         pr_warn("arch_timer: No interrupt available, giving up\n");
859                         return -EINVAL;
860                 }
861         }
862 
863         ret = arch_timer_register();
864         if (ret)
865                 return ret;
866 
867         ret = arch_timer_common_init();
868         if (ret)
869                 return ret;
870 
871         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
872         
873         return 0;
874 }
875 
876 static int __init arch_timer_of_init(struct device_node *np)
877 {
878         int i;
879 
880         if (arch_timers_present & ARCH_CP15_TIMER) {
881                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
882                 return 0;
883         }
884 
885         arch_timers_present |= ARCH_CP15_TIMER;
886         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
887                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
888 
889         arch_timer_detect_rate(NULL, np);
890 
891         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
892 
893 #ifdef CONFIG_FSL_ERRATUM_A008585
894         if (fsl_a008585_enable < 0)
895                 fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
896         if (fsl_a008585_enable) {
897                 static_branch_enable(&arch_timer_read_ool_enabled);
898                 pr_info("Enabling workaround for FSL erratum A-008585\n");
899         }
900 #endif
901 
902         /*
903          * If we cannot rely on firmware initializing the timer registers then
904          * we should use the physical timers instead.
905          */
906         if (IS_ENABLED(CONFIG_ARM) &&
907             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
908                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
909 
910         return arch_timer_init();
911 }
912 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
913 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
914 
915 static int __init arch_timer_mem_init(struct device_node *np)
916 {
917         struct device_node *frame, *best_frame = NULL;
918         void __iomem *cntctlbase, *base;
919         unsigned int irq, ret = -EINVAL;
920         u32 cnttidr;
921 
922         arch_timers_present |= ARCH_MEM_TIMER;
923         cntctlbase = of_iomap(np, 0);
924         if (!cntctlbase) {
925                 pr_err("arch_timer: Can't find CNTCTLBase\n");
926                 return -ENXIO;
927         }
928 
929         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
930 
931         /*
932          * Try to find a virtual capable frame. Otherwise fall back to a
933          * physical capable frame.
934          */
935         for_each_available_child_of_node(np, frame) {
936                 int n;
937                 u32 cntacr;
938 
939                 if (of_property_read_u32(frame, "frame-number", &n)) {
940                         pr_err("arch_timer: Missing frame-number\n");
941                         of_node_put(frame);
942                         goto out;
943                 }
944 
945                 /* Try enabling everything, and see what sticks */
946                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
947                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
948                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
949                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
950 
951                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
952                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
953                         of_node_put(best_frame);
954                         best_frame = frame;
955                         arch_timer_mem_use_virtual = true;
956                         break;
957                 }
958 
959                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
960                         continue;
961 
962                 of_node_put(best_frame);
963                 best_frame = of_node_get(frame);
964         }
965 
966         ret= -ENXIO;
967         base = arch_counter_base = of_iomap(best_frame, 0);
968         if (!base) {
969                 pr_err("arch_timer: Can't map frame's registers\n");
970                 goto out;
971         }
972 
973         if (arch_timer_mem_use_virtual)
974                 irq = irq_of_parse_and_map(best_frame, 1);
975         else
976                 irq = irq_of_parse_and_map(best_frame, 0);
977 
978         ret = -EINVAL;
979         if (!irq) {
980                 pr_err("arch_timer: Frame missing %s irq",
981                        arch_timer_mem_use_virtual ? "virt" : "phys");
982                 goto out;
983         }
984 
985         arch_timer_detect_rate(base, np);
986         ret = arch_timer_mem_register(base, irq);
987         if (ret)
988                 goto out;
989 
990         return arch_timer_common_init();
991 out:
992         iounmap(cntctlbase);
993         of_node_put(best_frame);
994         return ret;
995 }
996 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
997                        arch_timer_mem_init);
998 
999 #ifdef CONFIG_ACPI
1000 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
1001 {
1002         int trigger, polarity;
1003 
1004         if (!interrupt)
1005                 return 0;
1006 
1007         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
1008                         : ACPI_LEVEL_SENSITIVE;
1009 
1010         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
1011                         : ACPI_ACTIVE_HIGH;
1012 
1013         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
1014 }
1015 
1016 /* Initialize per-processor generic timer */
1017 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1018 {
1019         struct acpi_table_gtdt *gtdt;
1020 
1021         if (arch_timers_present & ARCH_CP15_TIMER) {
1022                 pr_warn("arch_timer: already initialized, skipping\n");
1023                 return -EINVAL;
1024         }
1025 
1026         gtdt = container_of(table, struct acpi_table_gtdt, header);
1027 
1028         arch_timers_present |= ARCH_CP15_TIMER;
1029 
1030         arch_timer_ppi[PHYS_SECURE_PPI] =
1031                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
1032                 gtdt->secure_el1_flags);
1033 
1034         arch_timer_ppi[PHYS_NONSECURE_PPI] =
1035                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
1036                 gtdt->non_secure_el1_flags);
1037 
1038         arch_timer_ppi[VIRT_PPI] =
1039                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
1040                 gtdt->virtual_timer_flags);
1041 
1042         arch_timer_ppi[HYP_PPI] =
1043                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
1044                 gtdt->non_secure_el2_flags);
1045 
1046         /* Get the frequency from CNTFRQ */
1047         arch_timer_detect_rate(NULL, NULL);
1048 
1049         /* Always-on capability */
1050         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
1051 
1052         arch_timer_init();
1053         return 0;
1054 }
1055 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1056 #endif
1057 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us