1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 #include <linux/suspend.h>
8 
9 #include "i915_drv.h"
10 #include "i915_params.h"
11 #include "intel_context.h"
12 #include "intel_engine_pm.h"
13 #include "intel_gt.h"
14 #include "intel_gt_clock_utils.h"
15 #include "intel_gt_pm.h"
16 #include "intel_gt_requests.h"
17 #include "intel_llc.h"
18 #include "intel_pm.h"
19 #include "intel_rc6.h"
20 #include "intel_rps.h"
21 #include "intel_wakeref.h"
22 #include "pxp/intel_pxp_pm.h"
23 
24 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
25 
user_forcewake(struct intel_gt * gt,bool suspend)26 static void user_forcewake(struct intel_gt *gt, bool suspend)
27 {
28 	int count = atomic_read(&gt->user_wakeref);
29 
30 	/* Inside suspend/resume so single threaded, no races to worry about. */
31 	if (likely(!count))
32 		return;
33 
34 	intel_gt_pm_get(gt);
35 	if (suspend) {
36 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
37 		atomic_sub(count, &gt->wakeref.count);
38 	} else {
39 		atomic_add(count, &gt->wakeref.count);
40 	}
41 	intel_gt_pm_put(gt);
42 }
43 
runtime_begin(struct intel_gt * gt)44 static void runtime_begin(struct intel_gt *gt)
45 {
46 	local_irq_disable();
47 	write_seqcount_begin(&gt->stats.lock);
48 	gt->stats.start = ktime_get();
49 	gt->stats.active = true;
50 	write_seqcount_end(&gt->stats.lock);
51 	local_irq_enable();
52 }
53 
runtime_end(struct intel_gt * gt)54 static void runtime_end(struct intel_gt *gt)
55 {
56 	local_irq_disable();
57 	write_seqcount_begin(&gt->stats.lock);
58 	gt->stats.active = false;
59 	gt->stats.total =
60 		ktime_add(gt->stats.total,
61 			  ktime_sub(ktime_get(), gt->stats.start));
62 	write_seqcount_end(&gt->stats.lock);
63 	local_irq_enable();
64 }
65 
__gt_unpark(struct intel_wakeref * wf)66 static int __gt_unpark(struct intel_wakeref *wf)
67 {
68 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
69 	struct drm_i915_private *i915 = gt->i915;
70 
71 	GT_TRACE(gt, "\n");
72 
73 	/*
74 	 * It seems that the DMC likes to transition between the DC states a lot
75 	 * when there are no connected displays (no active power domains) during
76 	 * command submission.
77 	 *
78 	 * This activity has negative impact on the performance of the chip with
79 	 * huge latencies observed in the interrupt handler and elsewhere.
80 	 *
81 	 * Work around it by grabbing a GT IRQ power domain whilst there is any
82 	 * GT activity, preventing any DC state transitions.
83 	 */
84 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
85 	GEM_BUG_ON(!gt->awake);
86 
87 	intel_rc6_unpark(&gt->rc6);
88 	intel_rps_unpark(&gt->rps);
89 	i915_pmu_gt_unparked(i915);
90 	intel_guc_busyness_unpark(gt);
91 
92 	intel_gt_unpark_requests(gt);
93 	runtime_begin(gt);
94 
95 	return 0;
96 }
97 
__gt_park(struct intel_wakeref * wf)98 static int __gt_park(struct intel_wakeref *wf)
99 {
100 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
101 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
102 	struct drm_i915_private *i915 = gt->i915;
103 
104 	GT_TRACE(gt, "\n");
105 
106 	runtime_end(gt);
107 	intel_gt_park_requests(gt);
108 
109 	intel_guc_busyness_park(gt);
110 	i915_vma_parked(gt);
111 	i915_pmu_gt_parked(i915);
112 	intel_rps_park(&gt->rps);
113 	intel_rc6_park(&gt->rc6);
114 
115 	/* Everything switched off, flush any residual interrupt just in case */
116 	intel_synchronize_irq(i915);
117 
118 	/* Defer dropping the display power well for 100ms, it's slow! */
119 	GEM_BUG_ON(!wakeref);
120 	intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
121 
122 	return 0;
123 }
124 
125 static const struct intel_wakeref_ops wf_ops = {
126 	.get = __gt_unpark,
127 	.put = __gt_park,
128 };
129 
intel_gt_pm_init_early(struct intel_gt * gt)130 void intel_gt_pm_init_early(struct intel_gt *gt)
131 {
132 	/*
133 	 * We access the runtime_pm structure via gt->i915 here rather than
134 	 * gt->uncore as we do elsewhere in the file because gt->uncore is not
135 	 * yet initialized for all tiles at this point in the driver startup.
136 	 * runtime_pm is per-device rather than per-tile, so this is still the
137 	 * correct structure.
138 	 */
139 	intel_wakeref_init(&gt->wakeref, &gt->i915->runtime_pm, &wf_ops);
140 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
141 }
142 
intel_gt_pm_init(struct intel_gt * gt)143 void intel_gt_pm_init(struct intel_gt *gt)
144 {
145 	/*
146 	 * Enabling power-management should be "self-healing". If we cannot
147 	 * enable a feature, simply leave it disabled with a notice to the
148 	 * user.
149 	 */
150 	intel_rc6_init(&gt->rc6);
151 	intel_rps_init(&gt->rps);
152 }
153 
reset_engines(struct intel_gt * gt)154 static bool reset_engines(struct intel_gt *gt)
155 {
156 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
157 		return false;
158 
159 	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
160 }
161 
gt_sanitize(struct intel_gt * gt,bool force)162 static void gt_sanitize(struct intel_gt *gt, bool force)
163 {
164 	struct intel_engine_cs *engine;
165 	enum intel_engine_id id;
166 	intel_wakeref_t wakeref;
167 
168 	GT_TRACE(gt, "force:%s", str_yes_no(force));
169 
170 	/* Use a raw wakeref to avoid calling intel_display_power_get early */
171 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
172 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
173 
174 	intel_gt_check_clock_frequency(gt);
175 
176 	/*
177 	 * As we have just resumed the machine and woken the device up from
178 	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
179 	 * back to defaults, recovering from whatever wedged state we left it
180 	 * in and so worth trying to use the device once more.
181 	 */
182 	if (intel_gt_is_wedged(gt))
183 		intel_gt_unset_wedged(gt);
184 
185 	/* For GuC mode, ensure submission is disabled before stopping ring */
186 	intel_uc_reset_prepare(&gt->uc);
187 
188 	for_each_engine(engine, gt, id) {
189 		if (engine->reset.prepare)
190 			engine->reset.prepare(engine);
191 
192 		if (engine->sanitize)
193 			engine->sanitize(engine);
194 	}
195 
196 	if (reset_engines(gt) || force) {
197 		for_each_engine(engine, gt, id)
198 			__intel_engine_reset(engine, false);
199 	}
200 
201 	intel_uc_reset(&gt->uc, false);
202 
203 	for_each_engine(engine, gt, id)
204 		if (engine->reset.finish)
205 			engine->reset.finish(engine);
206 
207 	intel_rps_sanitize(&gt->rps);
208 
209 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
210 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
211 }
212 
intel_gt_pm_fini(struct intel_gt * gt)213 void intel_gt_pm_fini(struct intel_gt *gt)
214 {
215 	intel_rc6_fini(&gt->rc6);
216 }
217 
intel_gt_resume(struct intel_gt * gt)218 int intel_gt_resume(struct intel_gt *gt)
219 {
220 	struct intel_engine_cs *engine;
221 	enum intel_engine_id id;
222 	int err;
223 
224 	err = intel_gt_has_unrecoverable_error(gt);
225 	if (err)
226 		return err;
227 
228 	GT_TRACE(gt, "\n");
229 
230 	/*
231 	 * After resume, we may need to poke into the pinned kernel
232 	 * contexts to paper over any damage caused by the sudden suspend.
233 	 * Only the kernel contexts should remain pinned over suspend,
234 	 * allowing us to fixup the user contexts on their first pin.
235 	 */
236 	gt_sanitize(gt, true);
237 
238 	intel_gt_pm_get(gt);
239 
240 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
241 	intel_rc6_sanitize(&gt->rc6);
242 	if (intel_gt_is_wedged(gt)) {
243 		err = -EIO;
244 		goto out_fw;
245 	}
246 
247 	/* Only when the HW is re-initialised, can we replay the requests */
248 	err = intel_gt_init_hw(gt);
249 	if (err) {
250 		i915_probe_error(gt->i915,
251 				 "Failed to initialize GPU, declaring it wedged!\n");
252 		goto err_wedged;
253 	}
254 
255 	intel_uc_reset_finish(&gt->uc);
256 
257 	intel_rps_enable(&gt->rps);
258 	intel_llc_enable(&gt->llc);
259 
260 	for_each_engine(engine, gt, id) {
261 		intel_engine_pm_get(engine);
262 
263 		engine->serial++; /* kernel context lost */
264 		err = intel_engine_resume(engine);
265 
266 		intel_engine_pm_put(engine);
267 		if (err) {
268 			drm_err(&gt->i915->drm,
269 				"Failed to restart %s (%d)\n",
270 				engine->name, err);
271 			goto err_wedged;
272 		}
273 	}
274 
275 	intel_rc6_enable(&gt->rc6);
276 
277 	intel_uc_resume(&gt->uc);
278 
279 	intel_pxp_resume(&gt->pxp);
280 
281 	user_forcewake(gt, false);
282 
283 out_fw:
284 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
285 	intel_gt_pm_put(gt);
286 	return err;
287 
288 err_wedged:
289 	intel_gt_set_wedged(gt);
290 	goto out_fw;
291 }
292 
wait_for_suspend(struct intel_gt * gt)293 static void wait_for_suspend(struct intel_gt *gt)
294 {
295 	if (!intel_gt_pm_is_awake(gt))
296 		return;
297 
298 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
299 		/*
300 		 * Forcibly cancel outstanding work and leave
301 		 * the gpu quiet.
302 		 */
303 		intel_gt_set_wedged(gt);
304 		intel_gt_retire_requests(gt);
305 	}
306 
307 	intel_gt_pm_wait_for_idle(gt);
308 }
309 
intel_gt_suspend_prepare(struct intel_gt * gt)310 void intel_gt_suspend_prepare(struct intel_gt *gt)
311 {
312 	user_forcewake(gt, true);
313 	wait_for_suspend(gt);
314 
315 	intel_pxp_suspend_prepare(&gt->pxp);
316 }
317 
pm_suspend_target(void)318 static suspend_state_t pm_suspend_target(void)
319 {
320 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
321 	return pm_suspend_target_state;
322 #else
323 	return PM_SUSPEND_TO_IDLE;
324 #endif
325 }
326 
intel_gt_suspend_late(struct intel_gt * gt)327 void intel_gt_suspend_late(struct intel_gt *gt)
328 {
329 	intel_wakeref_t wakeref;
330 
331 	/* We expect to be idle already; but also want to be independent */
332 	wait_for_suspend(gt);
333 
334 	if (is_mock_gt(gt))
335 		return;
336 
337 	GEM_BUG_ON(gt->awake);
338 
339 	intel_uc_suspend(&gt->uc);
340 	intel_pxp_suspend(&gt->pxp);
341 
342 	/*
343 	 * On disabling the device, we want to turn off HW access to memory
344 	 * that we no longer own.
345 	 *
346 	 * However, not all suspend-states disable the device. S0 (s2idle)
347 	 * is effectively runtime-suspend, the device is left powered on
348 	 * but needs to be put into a low power state. We need to keep
349 	 * powermanagement enabled, but we also retain system state and so
350 	 * it remains safe to keep on using our allocated memory.
351 	 */
352 	if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
353 		return;
354 
355 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
356 		intel_rps_disable(&gt->rps);
357 		intel_rc6_disable(&gt->rc6);
358 		intel_llc_disable(&gt->llc);
359 	}
360 
361 	gt_sanitize(gt, false);
362 
363 	GT_TRACE(gt, "\n");
364 }
365 
intel_gt_runtime_suspend(struct intel_gt * gt)366 void intel_gt_runtime_suspend(struct intel_gt *gt)
367 {
368 	intel_pxp_runtime_suspend(&gt->pxp);
369 	intel_uc_runtime_suspend(&gt->uc);
370 
371 	GT_TRACE(gt, "\n");
372 }
373 
intel_gt_runtime_resume(struct intel_gt * gt)374 int intel_gt_runtime_resume(struct intel_gt *gt)
375 {
376 	int ret;
377 
378 	GT_TRACE(gt, "\n");
379 	intel_gt_init_swizzling(gt);
380 	intel_ggtt_restore_fences(gt->ggtt);
381 
382 	ret = intel_uc_runtime_resume(&gt->uc);
383 	if (ret)
384 		return ret;
385 
386 	intel_pxp_runtime_resume(&gt->pxp);
387 
388 	return 0;
389 }
390 
__intel_gt_get_awake_time(const struct intel_gt * gt)391 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
392 {
393 	ktime_t total = gt->stats.total;
394 
395 	if (gt->stats.active)
396 		total = ktime_add(total,
397 				  ktime_sub(ktime_get(), gt->stats.start));
398 
399 	return total;
400 }
401 
intel_gt_get_awake_time(const struct intel_gt * gt)402 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
403 {
404 	unsigned int seq;
405 	ktime_t total;
406 
407 	do {
408 		seq = read_seqcount_begin(&gt->stats.lock);
409 		total = __intel_gt_get_awake_time(gt);
410 	} while (read_seqcount_retry(&gt->stats.lock, seq));
411 
412 	return total;
413 }
414 
415 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
416 #include "selftest_gt_pm.c"
417 #endif
418