1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/drivers/clocksource/arm_arch_timer.c
4  *
5  *  Copyright (C) 2011 ARM Ltd.
6  *  All Rights Reserved
7  */
8 
9 #define pr_fmt(fmt) 	"arch_timer: " fmt
10 
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/device.h>
14 #include <linux/smp.h>
15 #include <linux/cpu.h>
16 #include <linux/cpu_pm.h>
17 #include <linux/clockchips.h>
18 #include <linux/clocksource.h>
19 #include <linux/clocksource_ids.h>
20 #include <linux/interrupt.h>
21 #include <linux/of_irq.h>
22 #include <linux/of_address.h>
23 #include <linux/io.h>
24 #include <linux/slab.h>
25 #include <linux/sched/clock.h>
26 #include <linux/sched_clock.h>
27 #include <linux/acpi.h>
28 #include <linux/arm-smccc.h>
29 #include <linux/ptp_kvm.h>
30 
31 #include <asm/arch_timer.h>
32 #include <asm/virt.h>
33 
34 #include <clocksource/arm_arch_timer.h>
35 
36 #define CNTTIDR		0x08
37 #define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))
38 
39 #define CNTACR(n)	(0x40 + ((n) * 4))
40 #define CNTACR_RPCT	BIT(0)
41 #define CNTACR_RVCT	BIT(1)
42 #define CNTACR_RFRQ	BIT(2)
43 #define CNTACR_RVOFF	BIT(3)
44 #define CNTACR_RWVT	BIT(4)
45 #define CNTACR_RWPT	BIT(5)
46 
47 #define CNTVCT_LO	0x00
48 #define CNTPCT_LO	0x08
49 #define CNTFRQ		0x10
50 #define CNTP_CVAL_LO	0x20
51 #define CNTP_CTL	0x2c
52 #define CNTV_CVAL_LO	0x30
53 #define CNTV_CTL	0x3c
54 
55 /*
56  * The minimum amount of time a generic counter is guaranteed to not roll over
57  * (40 years)
58  */
59 #define MIN_ROLLOVER_SECS	(40ULL * 365 * 24 * 3600)
60 
61 static unsigned arch_timers_present __initdata;
62 
63 struct arch_timer {
64 	void __iomem *base;
65 	struct clock_event_device evt;
66 };
67 
68 static struct arch_timer *arch_timer_mem __ro_after_init;
69 
70 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
71 
72 static u32 arch_timer_rate __ro_after_init;
73 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI] __ro_after_init;
74 
75 static const char *arch_timer_ppi_names[ARCH_TIMER_MAX_TIMER_PPI] = {
76 	[ARCH_TIMER_PHYS_SECURE_PPI]	= "sec-phys",
77 	[ARCH_TIMER_PHYS_NONSECURE_PPI]	= "phys",
78 	[ARCH_TIMER_VIRT_PPI]		= "virt",
79 	[ARCH_TIMER_HYP_PPI]		= "hyp-phys",
80 	[ARCH_TIMER_HYP_VIRT_PPI]	= "hyp-virt",
81 };
82 
83 static struct clock_event_device __percpu *arch_timer_evt;
84 
85 static enum arch_timer_ppi_nr arch_timer_uses_ppi __ro_after_init = ARCH_TIMER_VIRT_PPI;
86 static bool arch_timer_c3stop __ro_after_init;
87 static bool arch_timer_mem_use_virtual __ro_after_init;
88 static bool arch_counter_suspend_stop __ro_after_init;
89 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
90 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
91 #else
92 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
93 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
94 
95 static cpumask_t evtstrm_available = CPU_MASK_NONE;
96 static bool evtstrm_enable __ro_after_init = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
97 
early_evtstrm_cfg(char * buf)98 static int __init early_evtstrm_cfg(char *buf)
99 {
100 	return strtobool(buf, &evtstrm_enable);
101 }
102 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
103 
104 /*
105  * Makes an educated guess at a valid counter width based on the Generic Timer
106  * specification. Of note:
107  *   1) the system counter is at least 56 bits wide
108  *   2) a roll-over time of not less than 40 years
109  *
110  * See 'ARM DDI 0487G.a D11.1.2 ("The system counter")' for more details.
111  */
arch_counter_get_width(void)112 static int arch_counter_get_width(void)
113 {
114 	u64 min_cycles = MIN_ROLLOVER_SECS * arch_timer_rate;
115 
116 	/* guarantee the returned width is within the valid range */
117 	return clamp_val(ilog2(min_cycles - 1) + 1, 56, 64);
118 }
119 
120 /*
121  * Architected system timer support.
122  */
123 
124 static __always_inline
arch_timer_reg_write(int access,enum arch_timer_reg reg,u64 val,struct clock_event_device * clk)125 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u64 val,
126 			  struct clock_event_device *clk)
127 {
128 	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
129 		struct arch_timer *timer = to_arch_timer(clk);
130 		switch (reg) {
131 		case ARCH_TIMER_REG_CTRL:
132 			writel_relaxed((u32)val, timer->base + CNTP_CTL);
133 			break;
134 		case ARCH_TIMER_REG_CVAL:
135 			/*
136 			 * Not guaranteed to be atomic, so the timer
137 			 * must be disabled at this point.
138 			 */
139 			writeq_relaxed(val, timer->base + CNTP_CVAL_LO);
140 			break;
141 		default:
142 			BUILD_BUG();
143 		}
144 	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
145 		struct arch_timer *timer = to_arch_timer(clk);
146 		switch (reg) {
147 		case ARCH_TIMER_REG_CTRL:
148 			writel_relaxed((u32)val, timer->base + CNTV_CTL);
149 			break;
150 		case ARCH_TIMER_REG_CVAL:
151 			/* Same restriction as above */
152 			writeq_relaxed(val, timer->base + CNTV_CVAL_LO);
153 			break;
154 		default:
155 			BUILD_BUG();
156 		}
157 	} else {
158 		arch_timer_reg_write_cp15(access, reg, val);
159 	}
160 }
161 
162 static __always_inline
arch_timer_reg_read(int access,enum arch_timer_reg reg,struct clock_event_device * clk)163 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
164 			struct clock_event_device *clk)
165 {
166 	u32 val;
167 
168 	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
169 		struct arch_timer *timer = to_arch_timer(clk);
170 		switch (reg) {
171 		case ARCH_TIMER_REG_CTRL:
172 			val = readl_relaxed(timer->base + CNTP_CTL);
173 			break;
174 		default:
175 			BUILD_BUG();
176 		}
177 	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
178 		struct arch_timer *timer = to_arch_timer(clk);
179 		switch (reg) {
180 		case ARCH_TIMER_REG_CTRL:
181 			val = readl_relaxed(timer->base + CNTV_CTL);
182 			break;
183 		default:
184 			BUILD_BUG();
185 		}
186 	} else {
187 		val = arch_timer_reg_read_cp15(access, reg);
188 	}
189 
190 	return val;
191 }
192 
arch_counter_get_cntpct_stable(void)193 static notrace u64 arch_counter_get_cntpct_stable(void)
194 {
195 	return __arch_counter_get_cntpct_stable();
196 }
197 
arch_counter_get_cntpct(void)198 static notrace u64 arch_counter_get_cntpct(void)
199 {
200 	return __arch_counter_get_cntpct();
201 }
202 
arch_counter_get_cntvct_stable(void)203 static notrace u64 arch_counter_get_cntvct_stable(void)
204 {
205 	return __arch_counter_get_cntvct_stable();
206 }
207 
arch_counter_get_cntvct(void)208 static notrace u64 arch_counter_get_cntvct(void)
209 {
210 	return __arch_counter_get_cntvct();
211 }
212 
213 /*
214  * Default to cp15 based access because arm64 uses this function for
215  * sched_clock() before DT is probed and the cp15 method is guaranteed
216  * to exist on arm64. arm doesn't use this before DT is probed so even
217  * if we don't have the cp15 accessors we won't have a problem.
218  */
219 u64 (*arch_timer_read_counter)(void) __ro_after_init = arch_counter_get_cntvct;
220 EXPORT_SYMBOL_GPL(arch_timer_read_counter);
221 
arch_counter_read(struct clocksource * cs)222 static u64 arch_counter_read(struct clocksource *cs)
223 {
224 	return arch_timer_read_counter();
225 }
226 
arch_counter_read_cc(const struct cyclecounter * cc)227 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
228 {
229 	return arch_timer_read_counter();
230 }
231 
232 static struct clocksource clocksource_counter = {
233 	.name	= "arch_sys_counter",
234 	.id	= CSID_ARM_ARCH_COUNTER,
235 	.rating	= 400,
236 	.read	= arch_counter_read,
237 	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
238 };
239 
240 static struct cyclecounter cyclecounter __ro_after_init = {
241 	.read	= arch_counter_read_cc,
242 };
243 
244 struct ate_acpi_oem_info {
245 	char oem_id[ACPI_OEM_ID_SIZE + 1];
246 	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
247 	u32 oem_revision;
248 };
249 
250 #ifdef CONFIG_FSL_ERRATUM_A008585
251 /*
252  * The number of retries is an arbitrary value well beyond the highest number
253  * of iterations the loop has been observed to take.
254  */
255 #define __fsl_a008585_read_reg(reg) ({			\
256 	u64 _old, _new;					\
257 	int _retries = 200;				\
258 							\
259 	do {						\
260 		_old = read_sysreg(reg);		\
261 		_new = read_sysreg(reg);		\
262 		_retries--;				\
263 	} while (unlikely(_old != _new) && _retries);	\
264 							\
265 	WARN_ON_ONCE(!_retries);			\
266 	_new;						\
267 })
268 
fsl_a008585_read_cntpct_el0(void)269 static u64 notrace fsl_a008585_read_cntpct_el0(void)
270 {
271 	return __fsl_a008585_read_reg(cntpct_el0);
272 }
273 
fsl_a008585_read_cntvct_el0(void)274 static u64 notrace fsl_a008585_read_cntvct_el0(void)
275 {
276 	return __fsl_a008585_read_reg(cntvct_el0);
277 }
278 #endif
279 
280 #ifdef CONFIG_HISILICON_ERRATUM_161010101
281 /*
282  * Verify whether the value of the second read is larger than the first by
283  * less than 32 is the only way to confirm the value is correct, so clear the
284  * lower 5 bits to check whether the difference is greater than 32 or not.
285  * Theoretically the erratum should not occur more than twice in succession
286  * when reading the system counter, but it is possible that some interrupts
287  * may lead to more than twice read errors, triggering the warning, so setting
288  * the number of retries far beyond the number of iterations the loop has been
289  * observed to take.
290  */
291 #define __hisi_161010101_read_reg(reg) ({				\
292 	u64 _old, _new;						\
293 	int _retries = 50;					\
294 								\
295 	do {							\
296 		_old = read_sysreg(reg);			\
297 		_new = read_sysreg(reg);			\
298 		_retries--;					\
299 	} while (unlikely((_new - _old) >> 5) && _retries);	\
300 								\
301 	WARN_ON_ONCE(!_retries);				\
302 	_new;							\
303 })
304 
hisi_161010101_read_cntpct_el0(void)305 static u64 notrace hisi_161010101_read_cntpct_el0(void)
306 {
307 	return __hisi_161010101_read_reg(cntpct_el0);
308 }
309 
hisi_161010101_read_cntvct_el0(void)310 static u64 notrace hisi_161010101_read_cntvct_el0(void)
311 {
312 	return __hisi_161010101_read_reg(cntvct_el0);
313 }
314 
315 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
316 	/*
317 	 * Note that trailing spaces are required to properly match
318 	 * the OEM table information.
319 	 */
320 	{
321 		.oem_id		= "HISI  ",
322 		.oem_table_id	= "HIP05   ",
323 		.oem_revision	= 0,
324 	},
325 	{
326 		.oem_id		= "HISI  ",
327 		.oem_table_id	= "HIP06   ",
328 		.oem_revision	= 0,
329 	},
330 	{
331 		.oem_id		= "HISI  ",
332 		.oem_table_id	= "HIP07   ",
333 		.oem_revision	= 0,
334 	},
335 	{ /* Sentinel indicating the end of the OEM array */ },
336 };
337 #endif
338 
339 #ifdef CONFIG_ARM64_ERRATUM_858921
arm64_858921_read_cntpct_el0(void)340 static u64 notrace arm64_858921_read_cntpct_el0(void)
341 {
342 	u64 old, new;
343 
344 	old = read_sysreg(cntpct_el0);
345 	new = read_sysreg(cntpct_el0);
346 	return (((old ^ new) >> 32) & 1) ? old : new;
347 }
348 
arm64_858921_read_cntvct_el0(void)349 static u64 notrace arm64_858921_read_cntvct_el0(void)
350 {
351 	u64 old, new;
352 
353 	old = read_sysreg(cntvct_el0);
354 	new = read_sysreg(cntvct_el0);
355 	return (((old ^ new) >> 32) & 1) ? old : new;
356 }
357 #endif
358 
359 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
360 /*
361  * The low bits of the counter registers are indeterminate while bit 10 or
362  * greater is rolling over. Since the counter value can jump both backward
363  * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
364  * with all ones or all zeros in the low bits. Bound the loop by the maximum
365  * number of CPU cycles in 3 consecutive 24 MHz counter periods.
366  */
367 #define __sun50i_a64_read_reg(reg) ({					\
368 	u64 _val;							\
369 	int _retries = 150;						\
370 									\
371 	do {								\
372 		_val = read_sysreg(reg);				\
373 		_retries--;						\
374 	} while (((_val + 1) & GENMASK(8, 0)) <= 1 && _retries);	\
375 									\
376 	WARN_ON_ONCE(!_retries);					\
377 	_val;								\
378 })
379 
sun50i_a64_read_cntpct_el0(void)380 static u64 notrace sun50i_a64_read_cntpct_el0(void)
381 {
382 	return __sun50i_a64_read_reg(cntpct_el0);
383 }
384 
sun50i_a64_read_cntvct_el0(void)385 static u64 notrace sun50i_a64_read_cntvct_el0(void)
386 {
387 	return __sun50i_a64_read_reg(cntvct_el0);
388 }
389 #endif
390 
391 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
392 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
393 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
394 
395 static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
396 
397 /*
398  * Force the inlining of this function so that the register accesses
399  * can be themselves correctly inlined.
400  */
401 static __always_inline
erratum_set_next_event_generic(const int access,unsigned long evt,struct clock_event_device * clk)402 void erratum_set_next_event_generic(const int access, unsigned long evt,
403 				    struct clock_event_device *clk)
404 {
405 	unsigned long ctrl;
406 	u64 cval;
407 
408 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
409 	ctrl |= ARCH_TIMER_CTRL_ENABLE;
410 	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
411 
412 	if (access == ARCH_TIMER_PHYS_ACCESS) {
413 		cval = evt + arch_counter_get_cntpct_stable();
414 		write_sysreg(cval, cntp_cval_el0);
415 	} else {
416 		cval = evt + arch_counter_get_cntvct_stable();
417 		write_sysreg(cval, cntv_cval_el0);
418 	}
419 
420 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
421 }
422 
erratum_set_next_event_virt(unsigned long evt,struct clock_event_device * clk)423 static __maybe_unused int erratum_set_next_event_virt(unsigned long evt,
424 					    struct clock_event_device *clk)
425 {
426 	erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
427 	return 0;
428 }
429 
erratum_set_next_event_phys(unsigned long evt,struct clock_event_device * clk)430 static __maybe_unused int erratum_set_next_event_phys(unsigned long evt,
431 					    struct clock_event_device *clk)
432 {
433 	erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
434 	return 0;
435 }
436 
437 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
438 #ifdef CONFIG_FSL_ERRATUM_A008585
439 	{
440 		.match_type = ate_match_dt,
441 		.id = "fsl,erratum-a008585",
442 		.desc = "Freescale erratum a005858",
443 		.read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
444 		.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
445 		.set_next_event_phys = erratum_set_next_event_phys,
446 		.set_next_event_virt = erratum_set_next_event_virt,
447 	},
448 #endif
449 #ifdef CONFIG_HISILICON_ERRATUM_161010101
450 	{
451 		.match_type = ate_match_dt,
452 		.id = "hisilicon,erratum-161010101",
453 		.desc = "HiSilicon erratum 161010101",
454 		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
455 		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
456 		.set_next_event_phys = erratum_set_next_event_phys,
457 		.set_next_event_virt = erratum_set_next_event_virt,
458 	},
459 	{
460 		.match_type = ate_match_acpi_oem_info,
461 		.id = hisi_161010101_oem_info,
462 		.desc = "HiSilicon erratum 161010101",
463 		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
464 		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
465 		.set_next_event_phys = erratum_set_next_event_phys,
466 		.set_next_event_virt = erratum_set_next_event_virt,
467 	},
468 #endif
469 #ifdef CONFIG_ARM64_ERRATUM_858921
470 	{
471 		.match_type = ate_match_local_cap_id,
472 		.id = (void *)ARM64_WORKAROUND_858921,
473 		.desc = "ARM erratum 858921",
474 		.read_cntpct_el0 = arm64_858921_read_cntpct_el0,
475 		.read_cntvct_el0 = arm64_858921_read_cntvct_el0,
476 	},
477 #endif
478 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
479 	{
480 		.match_type = ate_match_dt,
481 		.id = "allwinner,erratum-unknown1",
482 		.desc = "Allwinner erratum UNKNOWN1",
483 		.read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
484 		.read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
485 		.set_next_event_phys = erratum_set_next_event_phys,
486 		.set_next_event_virt = erratum_set_next_event_virt,
487 	},
488 #endif
489 #ifdef CONFIG_ARM64_ERRATUM_1418040
490 	{
491 		.match_type = ate_match_local_cap_id,
492 		.id = (void *)ARM64_WORKAROUND_1418040,
493 		.desc = "ARM erratum 1418040",
494 		.disable_compat_vdso = true,
495 	},
496 #endif
497 };
498 
499 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
500 			       const void *);
501 
502 static
arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround * wa,const void * arg)503 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
504 				 const void *arg)
505 {
506 	const struct device_node *np = arg;
507 
508 	return of_property_read_bool(np, wa->id);
509 }
510 
511 static
arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround * wa,const void * arg)512 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
513 					const void *arg)
514 {
515 	return this_cpu_has_cap((uintptr_t)wa->id);
516 }
517 
518 
519 static
arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround * wa,const void * arg)520 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
521 				       const void *arg)
522 {
523 	static const struct ate_acpi_oem_info empty_oem_info = {};
524 	const struct ate_acpi_oem_info *info = wa->id;
525 	const struct acpi_table_header *table = arg;
526 
527 	/* Iterate over the ACPI OEM info array, looking for a match */
528 	while (memcmp(info, &empty_oem_info, sizeof(*info))) {
529 		if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
530 		    !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
531 		    info->oem_revision == table->oem_revision)
532 			return true;
533 
534 		info++;
535 	}
536 
537 	return false;
538 }
539 
540 static const struct arch_timer_erratum_workaround *
arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,ate_match_fn_t match_fn,void * arg)541 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
542 			  ate_match_fn_t match_fn,
543 			  void *arg)
544 {
545 	int i;
546 
547 	for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
548 		if (ool_workarounds[i].match_type != type)
549 			continue;
550 
551 		if (match_fn(&ool_workarounds[i], arg))
552 			return &ool_workarounds[i];
553 	}
554 
555 	return NULL;
556 }
557 
558 static
arch_timer_enable_workaround(const struct arch_timer_erratum_workaround * wa,bool local)559 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
560 				  bool local)
561 {
562 	int i;
563 
564 	if (local) {
565 		__this_cpu_write(timer_unstable_counter_workaround, wa);
566 	} else {
567 		for_each_possible_cpu(i)
568 			per_cpu(timer_unstable_counter_workaround, i) = wa;
569 	}
570 
571 	if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
572 		atomic_set(&timer_unstable_counter_workaround_in_use, 1);
573 
574 	/*
575 	 * Don't use the vdso fastpath if errata require using the
576 	 * out-of-line counter accessor. We may change our mind pretty
577 	 * late in the game (with a per-CPU erratum, for example), so
578 	 * change both the default value and the vdso itself.
579 	 */
580 	if (wa->read_cntvct_el0) {
581 		clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
582 		vdso_default = VDSO_CLOCKMODE_NONE;
583 	} else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
584 		vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
585 		clocksource_counter.vdso_clock_mode = vdso_default;
586 	}
587 }
588 
arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,void * arg)589 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
590 					    void *arg)
591 {
592 	const struct arch_timer_erratum_workaround *wa, *__wa;
593 	ate_match_fn_t match_fn = NULL;
594 	bool local = false;
595 
596 	switch (type) {
597 	case ate_match_dt:
598 		match_fn = arch_timer_check_dt_erratum;
599 		break;
600 	case ate_match_local_cap_id:
601 		match_fn = arch_timer_check_local_cap_erratum;
602 		local = true;
603 		break;
604 	case ate_match_acpi_oem_info:
605 		match_fn = arch_timer_check_acpi_oem_erratum;
606 		break;
607 	default:
608 		WARN_ON(1);
609 		return;
610 	}
611 
612 	wa = arch_timer_iterate_errata(type, match_fn, arg);
613 	if (!wa)
614 		return;
615 
616 	__wa = __this_cpu_read(timer_unstable_counter_workaround);
617 	if (__wa && wa != __wa)
618 		pr_warn("Can't enable workaround for %s (clashes with %s\n)",
619 			wa->desc, __wa->desc);
620 
621 	if (__wa)
622 		return;
623 
624 	arch_timer_enable_workaround(wa, local);
625 	pr_info("Enabling %s workaround for %s\n",
626 		local ? "local" : "global", wa->desc);
627 }
628 
arch_timer_this_cpu_has_cntvct_wa(void)629 static bool arch_timer_this_cpu_has_cntvct_wa(void)
630 {
631 	return has_erratum_handler(read_cntvct_el0);
632 }
633 
arch_timer_counter_has_wa(void)634 static bool arch_timer_counter_has_wa(void)
635 {
636 	return atomic_read(&timer_unstable_counter_workaround_in_use);
637 }
638 #else
639 #define arch_timer_check_ool_workaround(t,a)		do { } while(0)
640 #define arch_timer_this_cpu_has_cntvct_wa()		({false;})
641 #define arch_timer_counter_has_wa()			({false;})
642 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
643 
timer_handler(const int access,struct clock_event_device * evt)644 static __always_inline irqreturn_t timer_handler(const int access,
645 					struct clock_event_device *evt)
646 {
647 	unsigned long ctrl;
648 
649 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
650 	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
651 		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
652 		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
653 		evt->event_handler(evt);
654 		return IRQ_HANDLED;
655 	}
656 
657 	return IRQ_NONE;
658 }
659 
arch_timer_handler_virt(int irq,void * dev_id)660 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
661 {
662 	struct clock_event_device *evt = dev_id;
663 
664 	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
665 }
666 
arch_timer_handler_phys(int irq,void * dev_id)667 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
668 {
669 	struct clock_event_device *evt = dev_id;
670 
671 	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
672 }
673 
arch_timer_handler_phys_mem(int irq,void * dev_id)674 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
675 {
676 	struct clock_event_device *evt = dev_id;
677 
678 	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
679 }
680 
arch_timer_handler_virt_mem(int irq,void * dev_id)681 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
682 {
683 	struct clock_event_device *evt = dev_id;
684 
685 	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
686 }
687 
timer_shutdown(const int access,struct clock_event_device * clk)688 static __always_inline int timer_shutdown(const int access,
689 					  struct clock_event_device *clk)
690 {
691 	unsigned long ctrl;
692 
693 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
694 	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
695 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
696 
697 	return 0;
698 }
699 
arch_timer_shutdown_virt(struct clock_event_device * clk)700 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
701 {
702 	return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
703 }
704 
arch_timer_shutdown_phys(struct clock_event_device * clk)705 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
706 {
707 	return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
708 }
709 
arch_timer_shutdown_virt_mem(struct clock_event_device * clk)710 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
711 {
712 	return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
713 }
714 
arch_timer_shutdown_phys_mem(struct clock_event_device * clk)715 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
716 {
717 	return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
718 }
719 
set_next_event(const int access,unsigned long evt,struct clock_event_device * clk)720 static __always_inline void set_next_event(const int access, unsigned long evt,
721 					   struct clock_event_device *clk)
722 {
723 	unsigned long ctrl;
724 	u64 cnt;
725 
726 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
727 	ctrl |= ARCH_TIMER_CTRL_ENABLE;
728 	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
729 
730 	if (access == ARCH_TIMER_PHYS_ACCESS)
731 		cnt = __arch_counter_get_cntpct();
732 	else
733 		cnt = __arch_counter_get_cntvct();
734 
735 	arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
736 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
737 }
738 
arch_timer_set_next_event_virt(unsigned long evt,struct clock_event_device * clk)739 static int arch_timer_set_next_event_virt(unsigned long evt,
740 					  struct clock_event_device *clk)
741 {
742 	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
743 	return 0;
744 }
745 
arch_timer_set_next_event_phys(unsigned long evt,struct clock_event_device * clk)746 static int arch_timer_set_next_event_phys(unsigned long evt,
747 					  struct clock_event_device *clk)
748 {
749 	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
750 	return 0;
751 }
752 
arch_counter_get_cnt_mem(struct arch_timer * t,int offset_lo)753 static u64 arch_counter_get_cnt_mem(struct arch_timer *t, int offset_lo)
754 {
755 	u32 cnt_lo, cnt_hi, tmp_hi;
756 
757 	do {
758 		cnt_hi = readl_relaxed(t->base + offset_lo + 4);
759 		cnt_lo = readl_relaxed(t->base + offset_lo);
760 		tmp_hi = readl_relaxed(t->base + offset_lo + 4);
761 	} while (cnt_hi != tmp_hi);
762 
763 	return ((u64) cnt_hi << 32) | cnt_lo;
764 }
765 
set_next_event_mem(const int access,unsigned long evt,struct clock_event_device * clk)766 static __always_inline void set_next_event_mem(const int access, unsigned long evt,
767 					   struct clock_event_device *clk)
768 {
769 	struct arch_timer *timer = to_arch_timer(clk);
770 	unsigned long ctrl;
771 	u64 cnt;
772 
773 	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
774 	ctrl |= ARCH_TIMER_CTRL_ENABLE;
775 	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
776 
777 	if (access ==  ARCH_TIMER_MEM_VIRT_ACCESS)
778 		cnt = arch_counter_get_cnt_mem(timer, CNTVCT_LO);
779 	else
780 		cnt = arch_counter_get_cnt_mem(timer, CNTPCT_LO);
781 
782 	arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
783 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
784 }
785 
arch_timer_set_next_event_virt_mem(unsigned long evt,struct clock_event_device * clk)786 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
787 					      struct clock_event_device *clk)
788 {
789 	set_next_event_mem(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
790 	return 0;
791 }
792 
arch_timer_set_next_event_phys_mem(unsigned long evt,struct clock_event_device * clk)793 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
794 					      struct clock_event_device *clk)
795 {
796 	set_next_event_mem(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
797 	return 0;
798 }
799 
__arch_timer_check_delta(void)800 static u64 __arch_timer_check_delta(void)
801 {
802 #ifdef CONFIG_ARM64
803 	const struct midr_range broken_cval_midrs[] = {
804 		/*
805 		 * XGene-1 implements CVAL in terms of TVAL, meaning
806 		 * that the maximum timer range is 32bit. Shame on them.
807 		 */
808 		MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APM,
809 						 APM_CPU_PART_POTENZA)),
810 		{},
811 	};
812 
813 	if (is_midr_in_range_list(read_cpuid_id(), broken_cval_midrs)) {
814 		pr_warn_once("Broken CNTx_CVAL_EL1, limiting width to 32bits");
815 		return CLOCKSOURCE_MASK(32);
816 	}
817 #endif
818 	return CLOCKSOURCE_MASK(arch_counter_get_width());
819 }
820 
__arch_timer_setup(unsigned type,struct clock_event_device * clk)821 static void __arch_timer_setup(unsigned type,
822 			       struct clock_event_device *clk)
823 {
824 	u64 max_delta;
825 
826 	clk->features = CLOCK_EVT_FEAT_ONESHOT;
827 
828 	if (type == ARCH_TIMER_TYPE_CP15) {
829 		typeof(clk->set_next_event) sne;
830 
831 		arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
832 
833 		if (arch_timer_c3stop)
834 			clk->features |= CLOCK_EVT_FEAT_C3STOP;
835 		clk->name = "arch_sys_timer";
836 		clk->rating = 450;
837 		clk->cpumask = cpumask_of(smp_processor_id());
838 		clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
839 		switch (arch_timer_uses_ppi) {
840 		case ARCH_TIMER_VIRT_PPI:
841 			clk->set_state_shutdown = arch_timer_shutdown_virt;
842 			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
843 			sne = erratum_handler(set_next_event_virt);
844 			break;
845 		case ARCH_TIMER_PHYS_SECURE_PPI:
846 		case ARCH_TIMER_PHYS_NONSECURE_PPI:
847 		case ARCH_TIMER_HYP_PPI:
848 			clk->set_state_shutdown = arch_timer_shutdown_phys;
849 			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
850 			sne = erratum_handler(set_next_event_phys);
851 			break;
852 		default:
853 			BUG();
854 		}
855 
856 		clk->set_next_event = sne;
857 		max_delta = __arch_timer_check_delta();
858 	} else {
859 		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
860 		clk->name = "arch_mem_timer";
861 		clk->rating = 400;
862 		clk->cpumask = cpu_possible_mask;
863 		if (arch_timer_mem_use_virtual) {
864 			clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
865 			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
866 			clk->set_next_event =
867 				arch_timer_set_next_event_virt_mem;
868 		} else {
869 			clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
870 			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
871 			clk->set_next_event =
872 				arch_timer_set_next_event_phys_mem;
873 		}
874 
875 		max_delta = CLOCKSOURCE_MASK(56);
876 	}
877 
878 	clk->set_state_shutdown(clk);
879 
880 	clockevents_config_and_register(clk, arch_timer_rate, 0xf, max_delta);
881 }
882 
arch_timer_evtstrm_enable(unsigned int divider)883 static void arch_timer_evtstrm_enable(unsigned int divider)
884 {
885 	u32 cntkctl = arch_timer_get_cntkctl();
886 
887 #ifdef CONFIG_ARM64
888 	/* ECV is likely to require a large divider. Use the EVNTIS flag. */
889 	if (cpus_have_const_cap(ARM64_HAS_ECV) && divider > 15) {
890 		cntkctl |= ARCH_TIMER_EVT_INTERVAL_SCALE;
891 		divider -= 8;
892 	}
893 #endif
894 
895 	divider = min(divider, 15U);
896 	cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
897 	/* Set the divider and enable virtual event stream */
898 	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
899 			| ARCH_TIMER_VIRT_EVT_EN;
900 	arch_timer_set_cntkctl(cntkctl);
901 	arch_timer_set_evtstrm_feature();
902 	cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
903 }
904 
arch_timer_configure_evtstream(void)905 static void arch_timer_configure_evtstream(void)
906 {
907 	int evt_stream_div, lsb;
908 
909 	/*
910 	 * As the event stream can at most be generated at half the frequency
911 	 * of the counter, use half the frequency when computing the divider.
912 	 */
913 	evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
914 
915 	/*
916 	 * Find the closest power of two to the divisor. If the adjacent bit
917 	 * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
918 	 */
919 	lsb = fls(evt_stream_div) - 1;
920 	if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
921 		lsb++;
922 
923 	/* enable event stream */
924 	arch_timer_evtstrm_enable(max(0, lsb));
925 }
926 
arch_counter_set_user_access(void)927 static void arch_counter_set_user_access(void)
928 {
929 	u32 cntkctl = arch_timer_get_cntkctl();
930 
931 	/* Disable user access to the timers and both counters */
932 	/* Also disable virtual event stream */
933 	cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
934 			| ARCH_TIMER_USR_VT_ACCESS_EN
935 		        | ARCH_TIMER_USR_VCT_ACCESS_EN
936 			| ARCH_TIMER_VIRT_EVT_EN
937 			| ARCH_TIMER_USR_PCT_ACCESS_EN);
938 
939 	/*
940 	 * Enable user access to the virtual counter if it doesn't
941 	 * need to be workaround. The vdso may have been already
942 	 * disabled though.
943 	 */
944 	if (arch_timer_this_cpu_has_cntvct_wa())
945 		pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
946 	else
947 		cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
948 
949 	arch_timer_set_cntkctl(cntkctl);
950 }
951 
arch_timer_has_nonsecure_ppi(void)952 static bool arch_timer_has_nonsecure_ppi(void)
953 {
954 	return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
955 		arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
956 }
957 
check_ppi_trigger(int irq)958 static u32 check_ppi_trigger(int irq)
959 {
960 	u32 flags = irq_get_trigger_type(irq);
961 
962 	if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
963 		pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
964 		pr_warn("WARNING: Please fix your firmware\n");
965 		flags = IRQF_TRIGGER_LOW;
966 	}
967 
968 	return flags;
969 }
970 
arch_timer_starting_cpu(unsigned int cpu)971 static int arch_timer_starting_cpu(unsigned int cpu)
972 {
973 	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
974 	u32 flags;
975 
976 	__arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
977 
978 	flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
979 	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
980 
981 	if (arch_timer_has_nonsecure_ppi()) {
982 		flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
983 		enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
984 				  flags);
985 	}
986 
987 	arch_counter_set_user_access();
988 	if (evtstrm_enable)
989 		arch_timer_configure_evtstream();
990 
991 	return 0;
992 }
993 
validate_timer_rate(void)994 static int validate_timer_rate(void)
995 {
996 	if (!arch_timer_rate)
997 		return -EINVAL;
998 
999 	/* Arch timer frequency < 1MHz can cause trouble */
1000 	WARN_ON(arch_timer_rate < 1000000);
1001 
1002 	return 0;
1003 }
1004 
1005 /*
1006  * For historical reasons, when probing with DT we use whichever (non-zero)
1007  * rate was probed first, and don't verify that others match. If the first node
1008  * probed has a clock-frequency property, this overrides the HW register.
1009  */
arch_timer_of_configure_rate(u32 rate,struct device_node * np)1010 static void __init arch_timer_of_configure_rate(u32 rate, struct device_node *np)
1011 {
1012 	/* Who has more than one independent system counter? */
1013 	if (arch_timer_rate)
1014 		return;
1015 
1016 	if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
1017 		arch_timer_rate = rate;
1018 
1019 	/* Check the timer frequency. */
1020 	if (validate_timer_rate())
1021 		pr_warn("frequency not available\n");
1022 }
1023 
arch_timer_banner(unsigned type)1024 static void __init arch_timer_banner(unsigned type)
1025 {
1026 	pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
1027 		type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
1028 		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
1029 			" and " : "",
1030 		type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
1031 		(unsigned long)arch_timer_rate / 1000000,
1032 		(unsigned long)(arch_timer_rate / 10000) % 100,
1033 		type & ARCH_TIMER_TYPE_CP15 ?
1034 			(arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
1035 			"",
1036 		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
1037 		type & ARCH_TIMER_TYPE_MEM ?
1038 			arch_timer_mem_use_virtual ? "virt" : "phys" :
1039 			"");
1040 }
1041 
arch_timer_get_rate(void)1042 u32 arch_timer_get_rate(void)
1043 {
1044 	return arch_timer_rate;
1045 }
1046 
arch_timer_evtstrm_available(void)1047 bool arch_timer_evtstrm_available(void)
1048 {
1049 	/*
1050 	 * We might get called from a preemptible context. This is fine
1051 	 * because availability of the event stream should be always the same
1052 	 * for a preemptible context and context where we might resume a task.
1053 	 */
1054 	return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
1055 }
1056 
arch_counter_get_cntvct_mem(void)1057 static u64 arch_counter_get_cntvct_mem(void)
1058 {
1059 	return arch_counter_get_cnt_mem(arch_timer_mem, CNTVCT_LO);
1060 }
1061 
1062 static struct arch_timer_kvm_info arch_timer_kvm_info;
1063 
arch_timer_get_kvm_info(void)1064 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
1065 {
1066 	return &arch_timer_kvm_info;
1067 }
1068 
arch_counter_register(unsigned type)1069 static void __init arch_counter_register(unsigned type)
1070 {
1071 	u64 start_count;
1072 	int width;
1073 
1074 	/* Register the CP15 based counter if we have one */
1075 	if (type & ARCH_TIMER_TYPE_CP15) {
1076 		u64 (*rd)(void);
1077 
1078 		if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1079 		    arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
1080 			if (arch_timer_counter_has_wa())
1081 				rd = arch_counter_get_cntvct_stable;
1082 			else
1083 				rd = arch_counter_get_cntvct;
1084 		} else {
1085 			if (arch_timer_counter_has_wa())
1086 				rd = arch_counter_get_cntpct_stable;
1087 			else
1088 				rd = arch_counter_get_cntpct;
1089 		}
1090 
1091 		arch_timer_read_counter = rd;
1092 		clocksource_counter.vdso_clock_mode = vdso_default;
1093 	} else {
1094 		arch_timer_read_counter = arch_counter_get_cntvct_mem;
1095 	}
1096 
1097 	width = arch_counter_get_width();
1098 	clocksource_counter.mask = CLOCKSOURCE_MASK(width);
1099 	cyclecounter.mask = CLOCKSOURCE_MASK(width);
1100 
1101 	if (!arch_counter_suspend_stop)
1102 		clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1103 	start_count = arch_timer_read_counter();
1104 	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
1105 	cyclecounter.mult = clocksource_counter.mult;
1106 	cyclecounter.shift = clocksource_counter.shift;
1107 	timecounter_init(&arch_timer_kvm_info.timecounter,
1108 			 &cyclecounter, start_count);
1109 
1110 	sched_clock_register(arch_timer_read_counter, width, arch_timer_rate);
1111 }
1112 
arch_timer_stop(struct clock_event_device * clk)1113 static void arch_timer_stop(struct clock_event_device *clk)
1114 {
1115 	pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1116 
1117 	disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1118 	if (arch_timer_has_nonsecure_ppi())
1119 		disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1120 
1121 	clk->set_state_shutdown(clk);
1122 }
1123 
arch_timer_dying_cpu(unsigned int cpu)1124 static int arch_timer_dying_cpu(unsigned int cpu)
1125 {
1126 	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1127 
1128 	cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1129 
1130 	arch_timer_stop(clk);
1131 	return 0;
1132 }
1133 
1134 #ifdef CONFIG_CPU_PM
1135 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
arch_timer_cpu_pm_notify(struct notifier_block * self,unsigned long action,void * hcpu)1136 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1137 				    unsigned long action, void *hcpu)
1138 {
1139 	if (action == CPU_PM_ENTER) {
1140 		__this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1141 
1142 		cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1143 	} else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1144 		arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1145 
1146 		if (arch_timer_have_evtstrm_feature())
1147 			cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1148 	}
1149 	return NOTIFY_OK;
1150 }
1151 
1152 static struct notifier_block arch_timer_cpu_pm_notifier = {
1153 	.notifier_call = arch_timer_cpu_pm_notify,
1154 };
1155 
arch_timer_cpu_pm_init(void)1156 static int __init arch_timer_cpu_pm_init(void)
1157 {
1158 	return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1159 }
1160 
arch_timer_cpu_pm_deinit(void)1161 static void __init arch_timer_cpu_pm_deinit(void)
1162 {
1163 	WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1164 }
1165 
1166 #else
arch_timer_cpu_pm_init(void)1167 static int __init arch_timer_cpu_pm_init(void)
1168 {
1169 	return 0;
1170 }
1171 
arch_timer_cpu_pm_deinit(void)1172 static void __init arch_timer_cpu_pm_deinit(void)
1173 {
1174 }
1175 #endif
1176 
arch_timer_register(void)1177 static int __init arch_timer_register(void)
1178 {
1179 	int err;
1180 	int ppi;
1181 
1182 	arch_timer_evt = alloc_percpu(struct clock_event_device);
1183 	if (!arch_timer_evt) {
1184 		err = -ENOMEM;
1185 		goto out;
1186 	}
1187 
1188 	ppi = arch_timer_ppi[arch_timer_uses_ppi];
1189 	switch (arch_timer_uses_ppi) {
1190 	case ARCH_TIMER_VIRT_PPI:
1191 		err = request_percpu_irq(ppi, arch_timer_handler_virt,
1192 					 "arch_timer", arch_timer_evt);
1193 		break;
1194 	case ARCH_TIMER_PHYS_SECURE_PPI:
1195 	case ARCH_TIMER_PHYS_NONSECURE_PPI:
1196 		err = request_percpu_irq(ppi, arch_timer_handler_phys,
1197 					 "arch_timer", arch_timer_evt);
1198 		if (!err && arch_timer_has_nonsecure_ppi()) {
1199 			ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1200 			err = request_percpu_irq(ppi, arch_timer_handler_phys,
1201 						 "arch_timer", arch_timer_evt);
1202 			if (err)
1203 				free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1204 						arch_timer_evt);
1205 		}
1206 		break;
1207 	case ARCH_TIMER_HYP_PPI:
1208 		err = request_percpu_irq(ppi, arch_timer_handler_phys,
1209 					 "arch_timer", arch_timer_evt);
1210 		break;
1211 	default:
1212 		BUG();
1213 	}
1214 
1215 	if (err) {
1216 		pr_err("can't register interrupt %d (%d)\n", ppi, err);
1217 		goto out_free;
1218 	}
1219 
1220 	err = arch_timer_cpu_pm_init();
1221 	if (err)
1222 		goto out_unreg_notify;
1223 
1224 	/* Register and immediately configure the timer on the boot CPU */
1225 	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1226 				"clockevents/arm/arch_timer:starting",
1227 				arch_timer_starting_cpu, arch_timer_dying_cpu);
1228 	if (err)
1229 		goto out_unreg_cpupm;
1230 	return 0;
1231 
1232 out_unreg_cpupm:
1233 	arch_timer_cpu_pm_deinit();
1234 
1235 out_unreg_notify:
1236 	free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1237 	if (arch_timer_has_nonsecure_ppi())
1238 		free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1239 				arch_timer_evt);
1240 
1241 out_free:
1242 	free_percpu(arch_timer_evt);
1243 out:
1244 	return err;
1245 }
1246 
arch_timer_mem_register(void __iomem * base,unsigned int irq)1247 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1248 {
1249 	int ret;
1250 	irq_handler_t func;
1251 
1252 	arch_timer_mem = kzalloc(sizeof(*arch_timer_mem), GFP_KERNEL);
1253 	if (!arch_timer_mem)
1254 		return -ENOMEM;
1255 
1256 	arch_timer_mem->base = base;
1257 	arch_timer_mem->evt.irq = irq;
1258 	__arch_timer_setup(ARCH_TIMER_TYPE_MEM, &arch_timer_mem->evt);
1259 
1260 	if (arch_timer_mem_use_virtual)
1261 		func = arch_timer_handler_virt_mem;
1262 	else
1263 		func = arch_timer_handler_phys_mem;
1264 
1265 	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &arch_timer_mem->evt);
1266 	if (ret) {
1267 		pr_err("Failed to request mem timer irq\n");
1268 		kfree(arch_timer_mem);
1269 		arch_timer_mem = NULL;
1270 	}
1271 
1272 	return ret;
1273 }
1274 
1275 static const struct of_device_id arch_timer_of_match[] __initconst = {
1276 	{ .compatible   = "arm,armv7-timer",    },
1277 	{ .compatible   = "arm,armv8-timer",    },
1278 	{},
1279 };
1280 
1281 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1282 	{ .compatible   = "arm,armv7-timer-mem", },
1283 	{},
1284 };
1285 
arch_timer_needs_of_probing(void)1286 static bool __init arch_timer_needs_of_probing(void)
1287 {
1288 	struct device_node *dn;
1289 	bool needs_probing = false;
1290 	unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1291 
1292 	/* We have two timers, and both device-tree nodes are probed. */
1293 	if ((arch_timers_present & mask) == mask)
1294 		return false;
1295 
1296 	/*
1297 	 * Only one type of timer is probed,
1298 	 * check if we have another type of timer node in device-tree.
1299 	 */
1300 	if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1301 		dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1302 	else
1303 		dn = of_find_matching_node(NULL, arch_timer_of_match);
1304 
1305 	if (dn && of_device_is_available(dn))
1306 		needs_probing = true;
1307 
1308 	of_node_put(dn);
1309 
1310 	return needs_probing;
1311 }
1312 
arch_timer_common_init(void)1313 static int __init arch_timer_common_init(void)
1314 {
1315 	arch_timer_banner(arch_timers_present);
1316 	arch_counter_register(arch_timers_present);
1317 	return arch_timer_arch_init();
1318 }
1319 
1320 /**
1321  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1322  *
1323  * If HYP mode is available, we know that the physical timer
1324  * has been configured to be accessible from PL1. Use it, so
1325  * that a guest can use the virtual timer instead.
1326  *
1327  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1328  * accesses to CNTP_*_EL1 registers are silently redirected to
1329  * their CNTHP_*_EL2 counterparts, and use a different PPI
1330  * number.
1331  *
1332  * If no interrupt provided for virtual timer, we'll have to
1333  * stick to the physical timer. It'd better be accessible...
1334  * For arm64 we never use the secure interrupt.
1335  *
1336  * Return: a suitable PPI type for the current system.
1337  */
arch_timer_select_ppi(void)1338 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1339 {
1340 	if (is_kernel_in_hyp_mode())
1341 		return ARCH_TIMER_HYP_PPI;
1342 
1343 	if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1344 		return ARCH_TIMER_VIRT_PPI;
1345 
1346 	if (IS_ENABLED(CONFIG_ARM64))
1347 		return ARCH_TIMER_PHYS_NONSECURE_PPI;
1348 
1349 	return ARCH_TIMER_PHYS_SECURE_PPI;
1350 }
1351 
arch_timer_populate_kvm_info(void)1352 static void __init arch_timer_populate_kvm_info(void)
1353 {
1354 	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1355 	if (is_kernel_in_hyp_mode())
1356 		arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1357 }
1358 
arch_timer_of_init(struct device_node * np)1359 static int __init arch_timer_of_init(struct device_node *np)
1360 {
1361 	int i, irq, ret;
1362 	u32 rate;
1363 	bool has_names;
1364 
1365 	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1366 		pr_warn("multiple nodes in dt, skipping\n");
1367 		return 0;
1368 	}
1369 
1370 	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1371 
1372 	has_names = of_property_read_bool(np, "interrupt-names");
1373 
1374 	for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) {
1375 		if (has_names)
1376 			irq = of_irq_get_byname(np, arch_timer_ppi_names[i]);
1377 		else
1378 			irq = of_irq_get(np, i);
1379 		if (irq > 0)
1380 			arch_timer_ppi[i] = irq;
1381 	}
1382 
1383 	arch_timer_populate_kvm_info();
1384 
1385 	rate = arch_timer_get_cntfrq();
1386 	arch_timer_of_configure_rate(rate, np);
1387 
1388 	arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1389 
1390 	/* Check for globally applicable workarounds */
1391 	arch_timer_check_ool_workaround(ate_match_dt, np);
1392 
1393 	/*
1394 	 * If we cannot rely on firmware initializing the timer registers then
1395 	 * we should use the physical timers instead.
1396 	 */
1397 	if (IS_ENABLED(CONFIG_ARM) &&
1398 	    of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1399 		arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1400 	else
1401 		arch_timer_uses_ppi = arch_timer_select_ppi();
1402 
1403 	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1404 		pr_err("No interrupt available, giving up\n");
1405 		return -EINVAL;
1406 	}
1407 
1408 	/* On some systems, the counter stops ticking when in suspend. */
1409 	arch_counter_suspend_stop = of_property_read_bool(np,
1410 							 "arm,no-tick-in-suspend");
1411 
1412 	ret = arch_timer_register();
1413 	if (ret)
1414 		return ret;
1415 
1416 	if (arch_timer_needs_of_probing())
1417 		return 0;
1418 
1419 	return arch_timer_common_init();
1420 }
1421 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1422 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1423 
1424 static u32 __init
arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame * frame)1425 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1426 {
1427 	void __iomem *base;
1428 	u32 rate;
1429 
1430 	base = ioremap(frame->cntbase, frame->size);
1431 	if (!base) {
1432 		pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1433 		return 0;
1434 	}
1435 
1436 	rate = readl_relaxed(base + CNTFRQ);
1437 
1438 	iounmap(base);
1439 
1440 	return rate;
1441 }
1442 
1443 static struct arch_timer_mem_frame * __init
arch_timer_mem_find_best_frame(struct arch_timer_mem * timer_mem)1444 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1445 {
1446 	struct arch_timer_mem_frame *frame, *best_frame = NULL;
1447 	void __iomem *cntctlbase;
1448 	u32 cnttidr;
1449 	int i;
1450 
1451 	cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1452 	if (!cntctlbase) {
1453 		pr_err("Can't map CNTCTLBase @ %pa\n",
1454 			&timer_mem->cntctlbase);
1455 		return NULL;
1456 	}
1457 
1458 	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1459 
1460 	/*
1461 	 * Try to find a virtual capable frame. Otherwise fall back to a
1462 	 * physical capable frame.
1463 	 */
1464 	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1465 		u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1466 			     CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1467 
1468 		frame = &timer_mem->frame[i];
1469 		if (!frame->valid)
1470 			continue;
1471 
1472 		/* Try enabling everything, and see what sticks */
1473 		writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1474 		cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1475 
1476 		if ((cnttidr & CNTTIDR_VIRT(i)) &&
1477 		    !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1478 			best_frame = frame;
1479 			arch_timer_mem_use_virtual = true;
1480 			break;
1481 		}
1482 
1483 		if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1484 			continue;
1485 
1486 		best_frame = frame;
1487 	}
1488 
1489 	iounmap(cntctlbase);
1490 
1491 	return best_frame;
1492 }
1493 
1494 static int __init
arch_timer_mem_frame_register(struct arch_timer_mem_frame * frame)1495 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1496 {
1497 	void __iomem *base;
1498 	int ret, irq = 0;
1499 
1500 	if (arch_timer_mem_use_virtual)
1501 		irq = frame->virt_irq;
1502 	else
1503 		irq = frame->phys_irq;
1504 
1505 	if (!irq) {
1506 		pr_err("Frame missing %s irq.\n",
1507 		       arch_timer_mem_use_virtual ? "virt" : "phys");
1508 		return -EINVAL;
1509 	}
1510 
1511 	if (!request_mem_region(frame->cntbase, frame->size,
1512 				"arch_mem_timer"))
1513 		return -EBUSY;
1514 
1515 	base = ioremap(frame->cntbase, frame->size);
1516 	if (!base) {
1517 		pr_err("Can't map frame's registers\n");
1518 		return -ENXIO;
1519 	}
1520 
1521 	ret = arch_timer_mem_register(base, irq);
1522 	if (ret) {
1523 		iounmap(base);
1524 		return ret;
1525 	}
1526 
1527 	arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1528 
1529 	return 0;
1530 }
1531 
arch_timer_mem_of_init(struct device_node * np)1532 static int __init arch_timer_mem_of_init(struct device_node *np)
1533 {
1534 	struct arch_timer_mem *timer_mem;
1535 	struct arch_timer_mem_frame *frame;
1536 	struct device_node *frame_node;
1537 	struct resource res;
1538 	int ret = -EINVAL;
1539 	u32 rate;
1540 
1541 	timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1542 	if (!timer_mem)
1543 		return -ENOMEM;
1544 
1545 	if (of_address_to_resource(np, 0, &res))
1546 		goto out;
1547 	timer_mem->cntctlbase = res.start;
1548 	timer_mem->size = resource_size(&res);
1549 
1550 	for_each_available_child_of_node(np, frame_node) {
1551 		u32 n;
1552 		struct arch_timer_mem_frame *frame;
1553 
1554 		if (of_property_read_u32(frame_node, "frame-number", &n)) {
1555 			pr_err(FW_BUG "Missing frame-number.\n");
1556 			of_node_put(frame_node);
1557 			goto out;
1558 		}
1559 		if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1560 			pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1561 			       ARCH_TIMER_MEM_MAX_FRAMES - 1);
1562 			of_node_put(frame_node);
1563 			goto out;
1564 		}
1565 		frame = &timer_mem->frame[n];
1566 
1567 		if (frame->valid) {
1568 			pr_err(FW_BUG "Duplicated frame-number.\n");
1569 			of_node_put(frame_node);
1570 			goto out;
1571 		}
1572 
1573 		if (of_address_to_resource(frame_node, 0, &res)) {
1574 			of_node_put(frame_node);
1575 			goto out;
1576 		}
1577 		frame->cntbase = res.start;
1578 		frame->size = resource_size(&res);
1579 
1580 		frame->virt_irq = irq_of_parse_and_map(frame_node,
1581 						       ARCH_TIMER_VIRT_SPI);
1582 		frame->phys_irq = irq_of_parse_and_map(frame_node,
1583 						       ARCH_TIMER_PHYS_SPI);
1584 
1585 		frame->valid = true;
1586 	}
1587 
1588 	frame = arch_timer_mem_find_best_frame(timer_mem);
1589 	if (!frame) {
1590 		pr_err("Unable to find a suitable frame in timer @ %pa\n",
1591 			&timer_mem->cntctlbase);
1592 		ret = -EINVAL;
1593 		goto out;
1594 	}
1595 
1596 	rate = arch_timer_mem_frame_get_cntfrq(frame);
1597 	arch_timer_of_configure_rate(rate, np);
1598 
1599 	ret = arch_timer_mem_frame_register(frame);
1600 	if (!ret && !arch_timer_needs_of_probing())
1601 		ret = arch_timer_common_init();
1602 out:
1603 	kfree(timer_mem);
1604 	return ret;
1605 }
1606 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1607 		       arch_timer_mem_of_init);
1608 
1609 #ifdef CONFIG_ACPI_GTDT
1610 static int __init
arch_timer_mem_verify_cntfrq(struct arch_timer_mem * timer_mem)1611 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1612 {
1613 	struct arch_timer_mem_frame *frame;
1614 	u32 rate;
1615 	int i;
1616 
1617 	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1618 		frame = &timer_mem->frame[i];
1619 
1620 		if (!frame->valid)
1621 			continue;
1622 
1623 		rate = arch_timer_mem_frame_get_cntfrq(frame);
1624 		if (rate == arch_timer_rate)
1625 			continue;
1626 
1627 		pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1628 			&frame->cntbase,
1629 			(unsigned long)rate, (unsigned long)arch_timer_rate);
1630 
1631 		return -EINVAL;
1632 	}
1633 
1634 	return 0;
1635 }
1636 
arch_timer_mem_acpi_init(int platform_timer_count)1637 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1638 {
1639 	struct arch_timer_mem *timers, *timer;
1640 	struct arch_timer_mem_frame *frame, *best_frame = NULL;
1641 	int timer_count, i, ret = 0;
1642 
1643 	timers = kcalloc(platform_timer_count, sizeof(*timers),
1644 			    GFP_KERNEL);
1645 	if (!timers)
1646 		return -ENOMEM;
1647 
1648 	ret = acpi_arch_timer_mem_init(timers, &timer_count);
1649 	if (ret || !timer_count)
1650 		goto out;
1651 
1652 	/*
1653 	 * While unlikely, it's theoretically possible that none of the frames
1654 	 * in a timer expose the combination of feature we want.
1655 	 */
1656 	for (i = 0; i < timer_count; i++) {
1657 		timer = &timers[i];
1658 
1659 		frame = arch_timer_mem_find_best_frame(timer);
1660 		if (!best_frame)
1661 			best_frame = frame;
1662 
1663 		ret = arch_timer_mem_verify_cntfrq(timer);
1664 		if (ret) {
1665 			pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1666 			goto out;
1667 		}
1668 
1669 		if (!best_frame) /* implies !frame */
1670 			/*
1671 			 * Only complain about missing suitable frames if we
1672 			 * haven't already found one in a previous iteration.
1673 			 */
1674 			pr_err("Unable to find a suitable frame in timer @ %pa\n",
1675 				&timer->cntctlbase);
1676 	}
1677 
1678 	if (best_frame)
1679 		ret = arch_timer_mem_frame_register(best_frame);
1680 out:
1681 	kfree(timers);
1682 	return ret;
1683 }
1684 
1685 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
arch_timer_acpi_init(struct acpi_table_header * table)1686 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1687 {
1688 	int ret, platform_timer_count;
1689 
1690 	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1691 		pr_warn("already initialized, skipping\n");
1692 		return -EINVAL;
1693 	}
1694 
1695 	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1696 
1697 	ret = acpi_gtdt_init(table, &platform_timer_count);
1698 	if (ret)
1699 		return ret;
1700 
1701 	arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1702 		acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1703 
1704 	arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1705 		acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1706 
1707 	arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1708 		acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1709 
1710 	arch_timer_populate_kvm_info();
1711 
1712 	/*
1713 	 * When probing via ACPI, we have no mechanism to override the sysreg
1714 	 * CNTFRQ value. This *must* be correct.
1715 	 */
1716 	arch_timer_rate = arch_timer_get_cntfrq();
1717 	ret = validate_timer_rate();
1718 	if (ret) {
1719 		pr_err(FW_BUG "frequency not available.\n");
1720 		return ret;
1721 	}
1722 
1723 	arch_timer_uses_ppi = arch_timer_select_ppi();
1724 	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1725 		pr_err("No interrupt available, giving up\n");
1726 		return -EINVAL;
1727 	}
1728 
1729 	/* Always-on capability */
1730 	arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1731 
1732 	/* Check for globally applicable workarounds */
1733 	arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1734 
1735 	ret = arch_timer_register();
1736 	if (ret)
1737 		return ret;
1738 
1739 	if (platform_timer_count &&
1740 	    arch_timer_mem_acpi_init(platform_timer_count))
1741 		pr_err("Failed to initialize memory-mapped timer.\n");
1742 
1743 	return arch_timer_common_init();
1744 }
1745 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1746 #endif
1747 
kvm_arch_ptp_get_crosststamp(u64 * cycle,struct timespec64 * ts,struct clocksource ** cs)1748 int kvm_arch_ptp_get_crosststamp(u64 *cycle, struct timespec64 *ts,
1749 				 struct clocksource **cs)
1750 {
1751 	struct arm_smccc_res hvc_res;
1752 	u32 ptp_counter;
1753 	ktime_t ktime;
1754 
1755 	if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC_DISCOVERY))
1756 		return -EOPNOTSUPP;
1757 
1758 	if (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
1759 		ptp_counter = KVM_PTP_VIRT_COUNTER;
1760 	else
1761 		ptp_counter = KVM_PTP_PHYS_COUNTER;
1762 
1763 	arm_smccc_1_1_invoke(ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID,
1764 			     ptp_counter, &hvc_res);
1765 
1766 	if ((int)(hvc_res.a0) < 0)
1767 		return -EOPNOTSUPP;
1768 
1769 	ktime = (u64)hvc_res.a0 << 32 | hvc_res.a1;
1770 	*ts = ktime_to_timespec64(ktime);
1771 	if (cycle)
1772 		*cycle = (u64)hvc_res.a2 << 32 | hvc_res.a3;
1773 	if (cs)
1774 		*cs = &clocksource_counter;
1775 
1776 	return 0;
1777 }
1778 EXPORT_SYMBOL_GPL(kvm_arch_ptp_get_crosststamp);
1779