1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 #include <linux/suspend.h>
8 
9 #include "i915_drv.h"
10 #include "i915_irq.h"
11 #include "i915_params.h"
12 #include "intel_context.h"
13 #include "intel_engine_pm.h"
14 #include "intel_gt.h"
15 #include "intel_gt_clock_utils.h"
16 #include "intel_gt_pm.h"
17 #include "intel_gt_print.h"
18 #include "intel_gt_requests.h"
19 #include "intel_llc.h"
20 #include "intel_rc6.h"
21 #include "intel_rps.h"
22 #include "intel_wakeref.h"
23 #include "pxp/intel_pxp_pm.h"
24 
25 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
26 
user_forcewake(struct intel_gt * gt,bool suspend)27 static void user_forcewake(struct intel_gt *gt, bool suspend)
28 {
29 	int count = atomic_read(&gt->user_wakeref);
30 
31 	/* Inside suspend/resume so single threaded, no races to worry about. */
32 	if (likely(!count))
33 		return;
34 
35 	intel_gt_pm_get(gt);
36 	if (suspend) {
37 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
38 		atomic_sub(count, &gt->wakeref.count);
39 	} else {
40 		atomic_add(count, &gt->wakeref.count);
41 	}
42 	intel_gt_pm_put(gt);
43 }
44 
runtime_begin(struct intel_gt * gt)45 static void runtime_begin(struct intel_gt *gt)
46 {
47 	local_irq_disable();
48 	write_seqcount_begin(&gt->stats.lock);
49 	gt->stats.start = ktime_get();
50 	gt->stats.active = true;
51 	write_seqcount_end(&gt->stats.lock);
52 	local_irq_enable();
53 }
54 
runtime_end(struct intel_gt * gt)55 static void runtime_end(struct intel_gt *gt)
56 {
57 	local_irq_disable();
58 	write_seqcount_begin(&gt->stats.lock);
59 	gt->stats.active = false;
60 	gt->stats.total =
61 		ktime_add(gt->stats.total,
62 			  ktime_sub(ktime_get(), gt->stats.start));
63 	write_seqcount_end(&gt->stats.lock);
64 	local_irq_enable();
65 }
66 
__gt_unpark(struct intel_wakeref * wf)67 static int __gt_unpark(struct intel_wakeref *wf)
68 {
69 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
70 	struct drm_i915_private *i915 = gt->i915;
71 
72 	GT_TRACE(gt, "\n");
73 
74 	/*
75 	 * It seems that the DMC likes to transition between the DC states a lot
76 	 * when there are no connected displays (no active power domains) during
77 	 * command submission.
78 	 *
79 	 * This activity has negative impact on the performance of the chip with
80 	 * huge latencies observed in the interrupt handler and elsewhere.
81 	 *
82 	 * Work around it by grabbing a GT IRQ power domain whilst there is any
83 	 * GT activity, preventing any DC state transitions.
84 	 */
85 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
86 	GEM_BUG_ON(!gt->awake);
87 
88 	intel_rc6_unpark(&gt->rc6);
89 	intel_rps_unpark(&gt->rps);
90 	i915_pmu_gt_unparked(gt);
91 	intel_guc_busyness_unpark(gt);
92 
93 	intel_gt_unpark_requests(gt);
94 	runtime_begin(gt);
95 
96 	return 0;
97 }
98 
__gt_park(struct intel_wakeref * wf)99 static int __gt_park(struct intel_wakeref *wf)
100 {
101 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
102 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
103 	struct drm_i915_private *i915 = gt->i915;
104 
105 	GT_TRACE(gt, "\n");
106 
107 	runtime_end(gt);
108 	intel_gt_park_requests(gt);
109 
110 	intel_guc_busyness_park(gt);
111 	i915_vma_parked(gt);
112 	i915_pmu_gt_parked(gt);
113 	intel_rps_park(&gt->rps);
114 	intel_rc6_park(&gt->rc6);
115 
116 	/* Everything switched off, flush any residual interrupt just in case */
117 	intel_synchronize_irq(i915);
118 
119 	/* Defer dropping the display power well for 100ms, it's slow! */
120 	GEM_BUG_ON(!wakeref);
121 	intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
122 
123 	return 0;
124 }
125 
126 static const struct intel_wakeref_ops wf_ops = {
127 	.get = __gt_unpark,
128 	.put = __gt_park,
129 };
130 
intel_gt_pm_init_early(struct intel_gt * gt)131 void intel_gt_pm_init_early(struct intel_gt *gt)
132 {
133 	/*
134 	 * We access the runtime_pm structure via gt->i915 here rather than
135 	 * gt->uncore as we do elsewhere in the file because gt->uncore is not
136 	 * yet initialized for all tiles at this point in the driver startup.
137 	 * runtime_pm is per-device rather than per-tile, so this is still the
138 	 * correct structure.
139 	 */
140 	intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops);
141 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
142 }
143 
intel_gt_pm_init(struct intel_gt * gt)144 void intel_gt_pm_init(struct intel_gt *gt)
145 {
146 	/*
147 	 * Enabling power-management should be "self-healing". If we cannot
148 	 * enable a feature, simply leave it disabled with a notice to the
149 	 * user.
150 	 */
151 	intel_rc6_init(&gt->rc6);
152 	intel_rps_init(&gt->rps);
153 }
154 
reset_engines(struct intel_gt * gt)155 static bool reset_engines(struct intel_gt *gt)
156 {
157 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
158 		return false;
159 
160 	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
161 }
162 
gt_sanitize(struct intel_gt * gt,bool force)163 static void gt_sanitize(struct intel_gt *gt, bool force)
164 {
165 	struct intel_engine_cs *engine;
166 	enum intel_engine_id id;
167 	intel_wakeref_t wakeref;
168 
169 	GT_TRACE(gt, "force:%s", str_yes_no(force));
170 
171 	/* Use a raw wakeref to avoid calling intel_display_power_get early */
172 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
173 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
174 
175 	intel_gt_check_clock_frequency(gt);
176 
177 	/*
178 	 * As we have just resumed the machine and woken the device up from
179 	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
180 	 * back to defaults, recovering from whatever wedged state we left it
181 	 * in and so worth trying to use the device once more.
182 	 */
183 	if (intel_gt_is_wedged(gt))
184 		intel_gt_unset_wedged(gt);
185 
186 	/* For GuC mode, ensure submission is disabled before stopping ring */
187 	intel_uc_reset_prepare(&gt->uc);
188 
189 	for_each_engine(engine, gt, id) {
190 		if (engine->reset.prepare)
191 			engine->reset.prepare(engine);
192 
193 		if (engine->sanitize)
194 			engine->sanitize(engine);
195 	}
196 
197 	if (reset_engines(gt) || force) {
198 		for_each_engine(engine, gt, id)
199 			__intel_engine_reset(engine, false);
200 	}
201 
202 	intel_uc_reset(&gt->uc, false);
203 
204 	for_each_engine(engine, gt, id)
205 		if (engine->reset.finish)
206 			engine->reset.finish(engine);
207 
208 	intel_rps_sanitize(&gt->rps);
209 
210 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
211 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
212 }
213 
intel_gt_pm_fini(struct intel_gt * gt)214 void intel_gt_pm_fini(struct intel_gt *gt)
215 {
216 	intel_rc6_fini(&gt->rc6);
217 }
218 
intel_gt_resume(struct intel_gt * gt)219 int intel_gt_resume(struct intel_gt *gt)
220 {
221 	struct intel_engine_cs *engine;
222 	enum intel_engine_id id;
223 	int err;
224 
225 	err = intel_gt_has_unrecoverable_error(gt);
226 	if (err)
227 		return err;
228 
229 	GT_TRACE(gt, "\n");
230 
231 	/*
232 	 * After resume, we may need to poke into the pinned kernel
233 	 * contexts to paper over any damage caused by the sudden suspend.
234 	 * Only the kernel contexts should remain pinned over suspend,
235 	 * allowing us to fixup the user contexts on their first pin.
236 	 */
237 	gt_sanitize(gt, true);
238 
239 	intel_gt_pm_get(gt);
240 
241 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
242 	intel_rc6_sanitize(&gt->rc6);
243 	if (intel_gt_is_wedged(gt)) {
244 		err = -EIO;
245 		goto out_fw;
246 	}
247 
248 	/* Only when the HW is re-initialised, can we replay the requests */
249 	err = intel_gt_init_hw(gt);
250 	if (err) {
251 		gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");
252 		goto err_wedged;
253 	}
254 
255 	intel_uc_reset_finish(&gt->uc);
256 
257 	intel_rps_enable(&gt->rps);
258 	intel_llc_enable(&gt->llc);
259 
260 	for_each_engine(engine, gt, id) {
261 		intel_engine_pm_get(engine);
262 
263 		engine->serial++; /* kernel context lost */
264 		err = intel_engine_resume(engine);
265 
266 		intel_engine_pm_put(engine);
267 		if (err) {
268 			gt_err(gt, "Failed to restart %s (%d)\n",
269 			       engine->name, err);
270 			goto err_wedged;
271 		}
272 	}
273 
274 	intel_rc6_enable(&gt->rc6);
275 
276 	intel_uc_resume(&gt->uc);
277 
278 	user_forcewake(gt, false);
279 
280 out_fw:
281 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
282 	intel_gt_pm_put(gt);
283 	return err;
284 
285 err_wedged:
286 	intel_gt_set_wedged(gt);
287 	goto out_fw;
288 }
289 
wait_for_suspend(struct intel_gt * gt)290 static void wait_for_suspend(struct intel_gt *gt)
291 {
292 	if (!intel_gt_pm_is_awake(gt))
293 		return;
294 
295 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
296 		/*
297 		 * Forcibly cancel outstanding work and leave
298 		 * the gpu quiet.
299 		 */
300 		intel_gt_set_wedged(gt);
301 		intel_gt_retire_requests(gt);
302 	}
303 
304 	intel_gt_pm_wait_for_idle(gt);
305 }
306 
intel_gt_suspend_prepare(struct intel_gt * gt)307 void intel_gt_suspend_prepare(struct intel_gt *gt)
308 {
309 	user_forcewake(gt, true);
310 	wait_for_suspend(gt);
311 }
312 
pm_suspend_target(void)313 static suspend_state_t pm_suspend_target(void)
314 {
315 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
316 	return pm_suspend_target_state;
317 #else
318 	return PM_SUSPEND_TO_IDLE;
319 #endif
320 }
321 
intel_gt_suspend_late(struct intel_gt * gt)322 void intel_gt_suspend_late(struct intel_gt *gt)
323 {
324 	intel_wakeref_t wakeref;
325 
326 	/* We expect to be idle already; but also want to be independent */
327 	wait_for_suspend(gt);
328 
329 	if (is_mock_gt(gt))
330 		return;
331 
332 	GEM_BUG_ON(gt->awake);
333 
334 	intel_uc_suspend(&gt->uc);
335 
336 	/*
337 	 * On disabling the device, we want to turn off HW access to memory
338 	 * that we no longer own.
339 	 *
340 	 * However, not all suspend-states disable the device. S0 (s2idle)
341 	 * is effectively runtime-suspend, the device is left powered on
342 	 * but needs to be put into a low power state. We need to keep
343 	 * powermanagement enabled, but we also retain system state and so
344 	 * it remains safe to keep on using our allocated memory.
345 	 */
346 	if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
347 		return;
348 
349 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
350 		intel_rps_disable(&gt->rps);
351 		intel_rc6_disable(&gt->rc6);
352 		intel_llc_disable(&gt->llc);
353 	}
354 
355 	gt_sanitize(gt, false);
356 
357 	GT_TRACE(gt, "\n");
358 }
359 
intel_gt_runtime_suspend(struct intel_gt * gt)360 void intel_gt_runtime_suspend(struct intel_gt *gt)
361 {
362 	intel_uc_runtime_suspend(&gt->uc);
363 
364 	GT_TRACE(gt, "\n");
365 }
366 
intel_gt_runtime_resume(struct intel_gt * gt)367 int intel_gt_runtime_resume(struct intel_gt *gt)
368 {
369 	int ret;
370 
371 	GT_TRACE(gt, "\n");
372 	intel_gt_init_swizzling(gt);
373 	intel_ggtt_restore_fences(gt->ggtt);
374 
375 	ret = intel_uc_runtime_resume(&gt->uc);
376 	if (ret)
377 		return ret;
378 
379 	return 0;
380 }
381 
__intel_gt_get_awake_time(const struct intel_gt * gt)382 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
383 {
384 	ktime_t total = gt->stats.total;
385 
386 	if (gt->stats.active)
387 		total = ktime_add(total,
388 				  ktime_sub(ktime_get(), gt->stats.start));
389 
390 	return total;
391 }
392 
intel_gt_get_awake_time(const struct intel_gt * gt)393 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
394 {
395 	unsigned int seq;
396 	ktime_t total;
397 
398 	do {
399 		seq = read_seqcount_begin(&gt->stats.lock);
400 		total = __intel_gt_get_awake_time(gt);
401 	} while (read_seqcount_retry(&gt->stats.lock, seq));
402 
403 	return total;
404 }
405 
406 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
407 #include "selftest_gt_pm.c"
408 #endif
409