1 /*
2  * check TSC synchronization.
3  *
4  * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
5  *
6  * We check whether all boot CPUs have their TSC's synchronized,
7  * print a warning if not and turn off the TSC clock-source.
8  *
9  * The warp-check is point-to-point between two CPUs, the CPU
10  * initiating the bootup is the 'source CPU', the freshly booting
11  * CPU is the 'target CPU'.
12  *
13  * Only two CPUs may participate - they can enter in any order.
14  * ( The serial nature of the boot logic and the CPU hotplug lock
15  *   protects against more than 2 CPUs entering this code. )
16  */
17 #include <linux/spinlock.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
20 #include <linux/smp.h>
21 #include <linux/nmi.h>
22 #include <asm/tsc.h>
23 
24 /*
25  * Entry/exit counters that make sure that both CPUs
26  * run the measurement code at once:
27  */
28 static __cpuinitdata atomic_t start_count;
29 static __cpuinitdata atomic_t stop_count;
30 
31 /*
32  * We use a raw spinlock in this exceptional case, because
33  * we want to have the fastest, inlined, non-debug version
34  * of a critical section, to be able to prove TSC time-warps:
35  */
36 static __cpuinitdata arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
37 
38 static __cpuinitdata cycles_t last_tsc;
39 static __cpuinitdata cycles_t max_warp;
40 static __cpuinitdata int nr_warps;
41 
42 /*
43  * TSC-warp measurement loop running on both CPUs:
44  */
check_tsc_warp(unsigned int timeout)45 static __cpuinit void check_tsc_warp(unsigned int timeout)
46 {
47 	cycles_t start, now, prev, end;
48 	int i;
49 
50 	rdtsc_barrier();
51 	start = get_cycles();
52 	rdtsc_barrier();
53 	/*
54 	 * The measurement runs for 'timeout' msecs:
55 	 */
56 	end = start + (cycles_t) tsc_khz * timeout;
57 	now = start;
58 
59 	for (i = 0; ; i++) {
60 		/*
61 		 * We take the global lock, measure TSC, save the
62 		 * previous TSC that was measured (possibly on
63 		 * another CPU) and update the previous TSC timestamp.
64 		 */
65 		arch_spin_lock(&sync_lock);
66 		prev = last_tsc;
67 		rdtsc_barrier();
68 		now = get_cycles();
69 		rdtsc_barrier();
70 		last_tsc = now;
71 		arch_spin_unlock(&sync_lock);
72 
73 		/*
74 		 * Be nice every now and then (and also check whether
75 		 * measurement is done [we also insert a 10 million
76 		 * loops safety exit, so we dont lock up in case the
77 		 * TSC readout is totally broken]):
78 		 */
79 		if (unlikely(!(i & 7))) {
80 			if (now > end || i > 10000000)
81 				break;
82 			cpu_relax();
83 			touch_nmi_watchdog();
84 		}
85 		/*
86 		 * Outside the critical section we can now see whether
87 		 * we saw a time-warp of the TSC going backwards:
88 		 */
89 		if (unlikely(prev > now)) {
90 			arch_spin_lock(&sync_lock);
91 			max_warp = max(max_warp, prev - now);
92 			nr_warps++;
93 			arch_spin_unlock(&sync_lock);
94 		}
95 	}
96 	WARN(!(now-start),
97 		"Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
98 			now-start, end-start);
99 }
100 
101 /*
102  * If the target CPU coming online doesn't have any of its core-siblings
103  * online, a timeout of 20msec will be used for the TSC-warp measurement
104  * loop. Otherwise a smaller timeout of 2msec will be used, as we have some
105  * information about this socket already (and this information grows as we
106  * have more and more logical-siblings in that socket).
107  *
108  * Ideally we should be able to skip the TSC sync check on the other
109  * core-siblings, if the first logical CPU in a socket passed the sync test.
110  * But as the TSC is per-logical CPU and can potentially be modified wrongly
111  * by the bios, TSC sync test for smaller duration should be able
112  * to catch such errors. Also this will catch the condition where all the
113  * cores in the socket doesn't get reset at the same time.
114  */
loop_timeout(int cpu)115 static inline unsigned int loop_timeout(int cpu)
116 {
117 	return (cpumask_weight(cpu_core_mask(cpu)) > 1) ? 2 : 20;
118 }
119 
120 /*
121  * Source CPU calls into this - it waits for the freshly booted
122  * target CPU to arrive and then starts the measurement:
123  */
check_tsc_sync_source(int cpu)124 void __cpuinit check_tsc_sync_source(int cpu)
125 {
126 	int cpus = 2;
127 
128 	/*
129 	 * No need to check if we already know that the TSC is not
130 	 * synchronized:
131 	 */
132 	if (unsynchronized_tsc())
133 		return;
134 
135 	if (tsc_clocksource_reliable) {
136 		if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING)
137 			pr_info(
138 			"Skipped synchronization checks as TSC is reliable.\n");
139 		return;
140 	}
141 
142 	/*
143 	 * Reset it - in case this is a second bootup:
144 	 */
145 	atomic_set(&stop_count, 0);
146 
147 	/*
148 	 * Wait for the target to arrive:
149 	 */
150 	while (atomic_read(&start_count) != cpus-1)
151 		cpu_relax();
152 	/*
153 	 * Trigger the target to continue into the measurement too:
154 	 */
155 	atomic_inc(&start_count);
156 
157 	check_tsc_warp(loop_timeout(cpu));
158 
159 	while (atomic_read(&stop_count) != cpus-1)
160 		cpu_relax();
161 
162 	if (nr_warps) {
163 		pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n",
164 			smp_processor_id(), cpu);
165 		pr_warning("Measured %Ld cycles TSC warp between CPUs, "
166 			   "turning off TSC clock.\n", max_warp);
167 		mark_tsc_unstable("check_tsc_sync_source failed");
168 	} else {
169 		pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
170 			smp_processor_id(), cpu);
171 	}
172 
173 	/*
174 	 * Reset it - just in case we boot another CPU later:
175 	 */
176 	atomic_set(&start_count, 0);
177 	nr_warps = 0;
178 	max_warp = 0;
179 	last_tsc = 0;
180 
181 	/*
182 	 * Let the target continue with the bootup:
183 	 */
184 	atomic_inc(&stop_count);
185 }
186 
187 /*
188  * Freshly booted CPUs call into this:
189  */
check_tsc_sync_target(void)190 void __cpuinit check_tsc_sync_target(void)
191 {
192 	int cpus = 2;
193 
194 	if (unsynchronized_tsc() || tsc_clocksource_reliable)
195 		return;
196 
197 	/*
198 	 * Register this CPU's participation and wait for the
199 	 * source CPU to start the measurement:
200 	 */
201 	atomic_inc(&start_count);
202 	while (atomic_read(&start_count) != cpus)
203 		cpu_relax();
204 
205 	check_tsc_warp(loop_timeout(smp_processor_id()));
206 
207 	/*
208 	 * Ok, we are done:
209 	 */
210 	atomic_inc(&stop_count);
211 
212 	/*
213 	 * Wait for the source CPU to print stuff:
214 	 */
215 	while (atomic_read(&stop_count) != cpus)
216 		cpu_relax();
217 }
218