1 /*
2 * check TSC synchronization.
3 *
4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
5 *
6 * We check whether all boot CPUs have their TSC's synchronized,
7 * print a warning if not and turn off the TSC clock-source.
8 *
9 * The warp-check is point-to-point between two CPUs, the CPU
10 * initiating the bootup is the 'source CPU', the freshly booting
11 * CPU is the 'target CPU'.
12 *
13 * Only two CPUs may participate - they can enter in any order.
14 * ( The serial nature of the boot logic and the CPU hotplug lock
15 * protects against more than 2 CPUs entering this code. )
16 */
17 #include <linux/spinlock.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
20 #include <linux/smp.h>
21 #include <linux/nmi.h>
22 #include <asm/tsc.h>
23
24 /*
25 * Entry/exit counters that make sure that both CPUs
26 * run the measurement code at once:
27 */
28 static __cpuinitdata atomic_t start_count;
29 static __cpuinitdata atomic_t stop_count;
30
31 /*
32 * We use a raw spinlock in this exceptional case, because
33 * we want to have the fastest, inlined, non-debug version
34 * of a critical section, to be able to prove TSC time-warps:
35 */
36 static __cpuinitdata arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
37
38 static __cpuinitdata cycles_t last_tsc;
39 static __cpuinitdata cycles_t max_warp;
40 static __cpuinitdata int nr_warps;
41
42 /*
43 * TSC-warp measurement loop running on both CPUs:
44 */
check_tsc_warp(void)45 static __cpuinit void check_tsc_warp(void)
46 {
47 cycles_t start, now, prev, end;
48 int i;
49
50 rdtsc_barrier();
51 start = get_cycles();
52 rdtsc_barrier();
53 /*
54 * The measurement runs for 20 msecs:
55 */
56 end = start + tsc_khz * 20ULL;
57 now = start;
58
59 for (i = 0; ; i++) {
60 /*
61 * We take the global lock, measure TSC, save the
62 * previous TSC that was measured (possibly on
63 * another CPU) and update the previous TSC timestamp.
64 */
65 arch_spin_lock(&sync_lock);
66 prev = last_tsc;
67 rdtsc_barrier();
68 now = get_cycles();
69 rdtsc_barrier();
70 last_tsc = now;
71 arch_spin_unlock(&sync_lock);
72
73 /*
74 * Be nice every now and then (and also check whether
75 * measurement is done [we also insert a 10 million
76 * loops safety exit, so we dont lock up in case the
77 * TSC readout is totally broken]):
78 */
79 if (unlikely(!(i & 7))) {
80 if (now > end || i > 10000000)
81 break;
82 cpu_relax();
83 touch_nmi_watchdog();
84 }
85 /*
86 * Outside the critical section we can now see whether
87 * we saw a time-warp of the TSC going backwards:
88 */
89 if (unlikely(prev > now)) {
90 arch_spin_lock(&sync_lock);
91 max_warp = max(max_warp, prev - now);
92 nr_warps++;
93 arch_spin_unlock(&sync_lock);
94 }
95 }
96 WARN(!(now-start),
97 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
98 now-start, end-start);
99 }
100
101 /*
102 * Source CPU calls into this - it waits for the freshly booted
103 * target CPU to arrive and then starts the measurement:
104 */
check_tsc_sync_source(int cpu)105 void __cpuinit check_tsc_sync_source(int cpu)
106 {
107 int cpus = 2;
108
109 /*
110 * No need to check if we already know that the TSC is not
111 * synchronized:
112 */
113 if (unsynchronized_tsc())
114 return;
115
116 if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE)) {
117 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING)
118 pr_info(
119 "Skipped synchronization checks as TSC is reliable.\n");
120 return;
121 }
122
123 /*
124 * Reset it - in case this is a second bootup:
125 */
126 atomic_set(&stop_count, 0);
127
128 /*
129 * Wait for the target to arrive:
130 */
131 while (atomic_read(&start_count) != cpus-1)
132 cpu_relax();
133 /*
134 * Trigger the target to continue into the measurement too:
135 */
136 atomic_inc(&start_count);
137
138 check_tsc_warp();
139
140 while (atomic_read(&stop_count) != cpus-1)
141 cpu_relax();
142
143 if (nr_warps) {
144 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n",
145 smp_processor_id(), cpu);
146 pr_warning("Measured %Ld cycles TSC warp between CPUs, "
147 "turning off TSC clock.\n", max_warp);
148 mark_tsc_unstable("check_tsc_sync_source failed");
149 } else {
150 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
151 smp_processor_id(), cpu);
152 }
153
154 /*
155 * Reset it - just in case we boot another CPU later:
156 */
157 atomic_set(&start_count, 0);
158 nr_warps = 0;
159 max_warp = 0;
160 last_tsc = 0;
161
162 /*
163 * Let the target continue with the bootup:
164 */
165 atomic_inc(&stop_count);
166 }
167
168 /*
169 * Freshly booted CPUs call into this:
170 */
check_tsc_sync_target(void)171 void __cpuinit check_tsc_sync_target(void)
172 {
173 int cpus = 2;
174
175 if (unsynchronized_tsc() || boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
176 return;
177
178 /*
179 * Register this CPU's participation and wait for the
180 * source CPU to start the measurement:
181 */
182 atomic_inc(&start_count);
183 while (atomic_read(&start_count) != cpus)
184 cpu_relax();
185
186 check_tsc_warp();
187
188 /*
189 * Ok, we are done:
190 */
191 atomic_inc(&stop_count);
192
193 /*
194 * Wait for the source CPU to print stuff:
195 */
196 while (atomic_read(&stop_count) != cpus)
197 cpu_relax();
198 }
199