1 /*
2  * drm_irq.c IRQ and vblank support
3  *
4  * \author Rickard E. (Rik) Faith <faith@valinux.com>
5  * \author Gareth Hughes <gareth@valinux.com>
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the "Software"),
9  * to deal in the Software without restriction, including without limitation
10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11  * and/or sell copies of the Software, and to permit persons to whom the
12  * Software is furnished to do so, subject to the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the next
15  * paragraph) shall be included in all copies or substantial portions of the
16  * Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
21  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24  * OTHER DEALINGS IN THE SOFTWARE.
25  */
26 
27 #include <linux/export.h>
28 #include <linux/kthread.h>
29 #include <linux/moduleparam.h>
30 
31 #include <drm/drm_crtc.h>
32 #include <drm/drm_drv.h>
33 #include <drm/drm_framebuffer.h>
34 #include <drm/drm_managed.h>
35 #include <drm/drm_modeset_helper_vtables.h>
36 #include <drm/drm_print.h>
37 #include <drm/drm_vblank.h>
38 
39 #include "drm_internal.h"
40 #include "drm_trace.h"
41 
42 /**
43  * DOC: vblank handling
44  *
45  * From the computer's perspective, every time the monitor displays
46  * a new frame the scanout engine has "scanned out" the display image
47  * from top to bottom, one row of pixels at a time. The current row
48  * of pixels is referred to as the current scanline.
49  *
50  * In addition to the display's visible area, there's usually a couple of
51  * extra scanlines which aren't actually displayed on the screen.
52  * These extra scanlines don't contain image data and are occasionally used
53  * for features like audio and infoframes. The region made up of these
54  * scanlines is referred to as the vertical blanking region, or vblank for
55  * short.
56  *
57  * For historical reference, the vertical blanking period was designed to
58  * give the electron gun (on CRTs) enough time to move back to the top of
59  * the screen to start scanning out the next frame. Similar for horizontal
60  * blanking periods. They were designed to give the electron gun enough
61  * time to move back to the other side of the screen to start scanning the
62  * next scanline.
63  *
64  * ::
65  *
66  *
67  *    physical →   ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
68  *    top of      |                                        |
69  *    display     |                                        |
70  *                |               New frame                |
71  *                |                                        |
72  *                |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
73  *                |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
74  *                |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|   updates the
75  *                |                                        |   frame as it
76  *                |                                        |   travels down
77  *                |                                        |   ("scan out")
78  *                |               Old frame                |
79  *                |                                        |
80  *                |                                        |
81  *                |                                        |
82  *                |                                        |   physical
83  *                |                                        |   bottom of
84  *    vertical    |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
85  *    blanking    ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
86  *    region   →  ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
87  *                ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
88  *    start of →   ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
89  *    new frame
90  *
91  * "Physical top of display" is the reference point for the high-precision/
92  * corrected timestamp.
93  *
94  * On a lot of display hardware, programming needs to take effect during the
95  * vertical blanking period so that settings like gamma, the image buffer
96  * buffer to be scanned out, etc. can safely be changed without showing
97  * any visual artifacts on the screen. In some unforgiving hardware, some of
98  * this programming has to both start and end in the same vblank. To help
99  * with the timing of the hardware programming, an interrupt is usually
100  * available to notify the driver when it can start the updating of registers.
101  * The interrupt is in this context named the vblank interrupt.
102  *
103  * The vblank interrupt may be fired at different points depending on the
104  * hardware. Some hardware implementations will fire the interrupt when the
105  * new frame start, other implementations will fire the interrupt at different
106  * points in time.
107  *
108  * Vertical blanking plays a major role in graphics rendering. To achieve
109  * tear-free display, users must synchronize page flips and/or rendering to
110  * vertical blanking. The DRM API offers ioctls to perform page flips
111  * synchronized to vertical blanking and wait for vertical blanking.
112  *
113  * The DRM core handles most of the vertical blanking management logic, which
114  * involves filtering out spurious interrupts, keeping race-free blanking
115  * counters, coping with counter wrap-around and resets and keeping use counts.
116  * It relies on the driver to generate vertical blanking interrupts and
117  * optionally provide a hardware vertical blanking counter.
118  *
119  * Drivers must initialize the vertical blanking handling core with a call to
120  * drm_vblank_init(). Minimally, a driver needs to implement
121  * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
122  * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
123  * support.
124  *
125  * Vertical blanking interrupts can be enabled by the DRM core or by drivers
126  * themselves (for instance to handle page flipping operations).  The DRM core
127  * maintains a vertical blanking use count to ensure that the interrupts are not
128  * disabled while a user still needs them. To increment the use count, drivers
129  * call drm_crtc_vblank_get() and release the vblank reference again with
130  * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
131  * guaranteed to be enabled.
132  *
133  * On many hardware disabling the vblank interrupt cannot be done in a race-free
134  * manner, see &drm_driver.vblank_disable_immediate and
135  * &drm_driver.max_vblank_count. In that case the vblank core only disables the
136  * vblanks after a timer has expired, which can be configured through the
137  * ``vblankoffdelay`` module parameter.
138  *
139  * Drivers for hardware without support for vertical-blanking interrupts
140  * must not call drm_vblank_init(). For such drivers, atomic helpers will
141  * automatically generate fake vblank events as part of the display update.
142  * This functionality also can be controlled by the driver by enabling and
143  * disabling struct drm_crtc_state.no_vblank.
144  */
145 
146 /* Retry timestamp calculation up to 3 times to satisfy
147  * drm_timestamp_precision before giving up.
148  */
149 #define DRM_TIMESTAMP_MAXRETRIES 3
150 
151 /* Threshold in nanoseconds for detection of redundant
152  * vblank irq in drm_handle_vblank(). 1 msec should be ok.
153  */
154 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
155 
156 static bool
157 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
158 			  ktime_t *tvblank, bool in_vblank_irq);
159 
160 static unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
161 
162 static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
163 
164 module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
165 module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
166 MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
167 MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
168 
store_vblank(struct drm_device * dev,unsigned int pipe,u32 vblank_count_inc,ktime_t t_vblank,u32 last)169 static void store_vblank(struct drm_device *dev, unsigned int pipe,
170 			 u32 vblank_count_inc,
171 			 ktime_t t_vblank, u32 last)
172 {
173 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
174 
175 	assert_spin_locked(&dev->vblank_time_lock);
176 
177 	vblank->last = last;
178 
179 	write_seqlock(&vblank->seqlock);
180 	vblank->time = t_vblank;
181 	atomic64_add(vblank_count_inc, &vblank->count);
182 	write_sequnlock(&vblank->seqlock);
183 }
184 
drm_max_vblank_count(struct drm_device * dev,unsigned int pipe)185 static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
186 {
187 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
188 
189 	return vblank->max_vblank_count ?: dev->max_vblank_count;
190 }
191 
192 /*
193  * "No hw counter" fallback implementation of .get_vblank_counter() hook,
194  * if there is no usable hardware frame counter available.
195  */
drm_vblank_no_hw_counter(struct drm_device * dev,unsigned int pipe)196 static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
197 {
198 	drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
199 	return 0;
200 }
201 
__get_vblank_counter(struct drm_device * dev,unsigned int pipe)202 static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
203 {
204 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
205 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
206 
207 		if (drm_WARN_ON(dev, !crtc))
208 			return 0;
209 
210 		if (crtc->funcs->get_vblank_counter)
211 			return crtc->funcs->get_vblank_counter(crtc);
212 	}
213 #ifdef CONFIG_DRM_LEGACY
214 	else if (dev->driver->get_vblank_counter) {
215 		return dev->driver->get_vblank_counter(dev, pipe);
216 	}
217 #endif
218 
219 	return drm_vblank_no_hw_counter(dev, pipe);
220 }
221 
222 /*
223  * Reset the stored timestamp for the current vblank count to correspond
224  * to the last vblank occurred.
225  *
226  * Only to be called from drm_crtc_vblank_on().
227  *
228  * Note: caller must hold &drm_device.vbl_lock since this reads & writes
229  * device vblank fields.
230  */
drm_reset_vblank_timestamp(struct drm_device * dev,unsigned int pipe)231 static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
232 {
233 	u32 cur_vblank;
234 	bool rc;
235 	ktime_t t_vblank;
236 	int count = DRM_TIMESTAMP_MAXRETRIES;
237 
238 	spin_lock(&dev->vblank_time_lock);
239 
240 	/*
241 	 * sample the current counter to avoid random jumps
242 	 * when drm_vblank_enable() applies the diff
243 	 */
244 	do {
245 		cur_vblank = __get_vblank_counter(dev, pipe);
246 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
247 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
248 
249 	/*
250 	 * Only reinitialize corresponding vblank timestamp if high-precision query
251 	 * available and didn't fail. Otherwise reinitialize delayed at next vblank
252 	 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
253 	 */
254 	if (!rc)
255 		t_vblank = 0;
256 
257 	/*
258 	 * +1 to make sure user will never see the same
259 	 * vblank counter value before and after a modeset
260 	 */
261 	store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
262 
263 	spin_unlock(&dev->vblank_time_lock);
264 }
265 
266 /*
267  * Call back into the driver to update the appropriate vblank counter
268  * (specified by @pipe).  Deal with wraparound, if it occurred, and
269  * update the last read value so we can deal with wraparound on the next
270  * call if necessary.
271  *
272  * Only necessary when going from off->on, to account for frames we
273  * didn't get an interrupt for.
274  *
275  * Note: caller must hold &drm_device.vbl_lock since this reads & writes
276  * device vblank fields.
277  */
drm_update_vblank_count(struct drm_device * dev,unsigned int pipe,bool in_vblank_irq)278 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
279 				    bool in_vblank_irq)
280 {
281 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
282 	u32 cur_vblank, diff;
283 	bool rc;
284 	ktime_t t_vblank;
285 	int count = DRM_TIMESTAMP_MAXRETRIES;
286 	int framedur_ns = vblank->framedur_ns;
287 	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
288 
289 	/*
290 	 * Interrupts were disabled prior to this call, so deal with counter
291 	 * wrap if needed.
292 	 * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
293 	 * here if the register is small or we had vblank interrupts off for
294 	 * a long time.
295 	 *
296 	 * We repeat the hardware vblank counter & timestamp query until
297 	 * we get consistent results. This to prevent races between gpu
298 	 * updating its hardware counter while we are retrieving the
299 	 * corresponding vblank timestamp.
300 	 */
301 	do {
302 		cur_vblank = __get_vblank_counter(dev, pipe);
303 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
304 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
305 
306 	if (max_vblank_count) {
307 		/* trust the hw counter when it's around */
308 		diff = (cur_vblank - vblank->last) & max_vblank_count;
309 	} else if (rc && framedur_ns) {
310 		u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
311 
312 		/*
313 		 * Figure out how many vblanks we've missed based
314 		 * on the difference in the timestamps and the
315 		 * frame/field duration.
316 		 */
317 
318 		drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
319 			    " diff_ns = %lld, framedur_ns = %d)\n",
320 			    pipe, (long long)diff_ns, framedur_ns);
321 
322 		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
323 
324 		if (diff == 0 && in_vblank_irq)
325 			drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
326 				    pipe);
327 	} else {
328 		/* some kind of default for drivers w/o accurate vbl timestamping */
329 		diff = in_vblank_irq ? 1 : 0;
330 	}
331 
332 	/*
333 	 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
334 	 * interval? If so then vblank irqs keep running and it will likely
335 	 * happen that the hardware vblank counter is not trustworthy as it
336 	 * might reset at some point in that interval and vblank timestamps
337 	 * are not trustworthy either in that interval. Iow. this can result
338 	 * in a bogus diff >> 1 which must be avoided as it would cause
339 	 * random large forward jumps of the software vblank counter.
340 	 */
341 	if (diff > 1 && (vblank->inmodeset & 0x2)) {
342 		drm_dbg_vbl(dev,
343 			    "clamping vblank bump to 1 on crtc %u: diffr=%u"
344 			    " due to pre-modeset.\n", pipe, diff);
345 		diff = 1;
346 	}
347 
348 	drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
349 		    " current=%llu, diff=%u, hw=%u hw_last=%u\n",
350 		    pipe, (unsigned long long)atomic64_read(&vblank->count),
351 		    diff, cur_vblank, vblank->last);
352 
353 	if (diff == 0) {
354 		drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
355 		return;
356 	}
357 
358 	/*
359 	 * Only reinitialize corresponding vblank timestamp if high-precision query
360 	 * available and didn't fail, or we were called from the vblank interrupt.
361 	 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
362 	 * for now, to mark the vblanktimestamp as invalid.
363 	 */
364 	if (!rc && !in_vblank_irq)
365 		t_vblank = 0;
366 
367 	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
368 }
369 
drm_vblank_count(struct drm_device * dev,unsigned int pipe)370 u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
371 {
372 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
373 	u64 count;
374 
375 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
376 		return 0;
377 
378 	count = atomic64_read(&vblank->count);
379 
380 	/*
381 	 * This read barrier corresponds to the implicit write barrier of the
382 	 * write seqlock in store_vblank(). Note that this is the only place
383 	 * where we need an explicit barrier, since all other access goes
384 	 * through drm_vblank_count_and_time(), which already has the required
385 	 * read barrier curtesy of the read seqlock.
386 	 */
387 	smp_rmb();
388 
389 	return count;
390 }
391 
392 /**
393  * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
394  * @crtc: which counter to retrieve
395  *
396  * This function is similar to drm_crtc_vblank_count() but this function
397  * interpolates to handle a race with vblank interrupts using the high precision
398  * timestamping support.
399  *
400  * This is mostly useful for hardware that can obtain the scanout position, but
401  * doesn't have a hardware frame counter.
402  */
drm_crtc_accurate_vblank_count(struct drm_crtc * crtc)403 u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
404 {
405 	struct drm_device *dev = crtc->dev;
406 	unsigned int pipe = drm_crtc_index(crtc);
407 	u64 vblank;
408 	unsigned long flags;
409 
410 	drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
411 		      !crtc->funcs->get_vblank_timestamp,
412 		      "This function requires support for accurate vblank timestamps.");
413 
414 	spin_lock_irqsave(&dev->vblank_time_lock, flags);
415 
416 	drm_update_vblank_count(dev, pipe, false);
417 	vblank = drm_vblank_count(dev, pipe);
418 
419 	spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
420 
421 	return vblank;
422 }
423 EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
424 
__disable_vblank(struct drm_device * dev,unsigned int pipe)425 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
426 {
427 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
428 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
429 
430 		if (drm_WARN_ON(dev, !crtc))
431 			return;
432 
433 		if (crtc->funcs->disable_vblank)
434 			crtc->funcs->disable_vblank(crtc);
435 	}
436 #ifdef CONFIG_DRM_LEGACY
437 	else {
438 		dev->driver->disable_vblank(dev, pipe);
439 	}
440 #endif
441 }
442 
443 /*
444  * Disable vblank irq's on crtc, make sure that last vblank count
445  * of hardware and corresponding consistent software vblank counter
446  * are preserved, even if there are any spurious vblank irq's after
447  * disable.
448  */
drm_vblank_disable_and_save(struct drm_device * dev,unsigned int pipe)449 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
450 {
451 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
452 	unsigned long irqflags;
453 
454 	assert_spin_locked(&dev->vbl_lock);
455 
456 	/* Prevent vblank irq processing while disabling vblank irqs,
457 	 * so no updates of timestamps or count can happen after we've
458 	 * disabled. Needed to prevent races in case of delayed irq's.
459 	 */
460 	spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
461 
462 	/*
463 	 * Update vblank count and disable vblank interrupts only if the
464 	 * interrupts were enabled. This avoids calling the ->disable_vblank()
465 	 * operation in atomic context with the hardware potentially runtime
466 	 * suspended.
467 	 */
468 	if (!vblank->enabled)
469 		goto out;
470 
471 	/*
472 	 * Update the count and timestamp to maintain the
473 	 * appearance that the counter has been ticking all along until
474 	 * this time. This makes the count account for the entire time
475 	 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
476 	 */
477 	drm_update_vblank_count(dev, pipe, false);
478 	__disable_vblank(dev, pipe);
479 	vblank->enabled = false;
480 
481 out:
482 	spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
483 }
484 
vblank_disable_fn(struct timer_list * t)485 static void vblank_disable_fn(struct timer_list *t)
486 {
487 	struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
488 	struct drm_device *dev = vblank->dev;
489 	unsigned int pipe = vblank->pipe;
490 	unsigned long irqflags;
491 
492 	spin_lock_irqsave(&dev->vbl_lock, irqflags);
493 	if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
494 		drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
495 		drm_vblank_disable_and_save(dev, pipe);
496 	}
497 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
498 }
499 
drm_vblank_init_release(struct drm_device * dev,void * ptr)500 static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
501 {
502 	struct drm_vblank_crtc *vblank = ptr;
503 
504 	drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
505 		    drm_core_check_feature(dev, DRIVER_MODESET));
506 
507 	drm_vblank_destroy_worker(vblank);
508 	del_timer_sync(&vblank->disable_timer);
509 }
510 
511 /**
512  * drm_vblank_init - initialize vblank support
513  * @dev: DRM device
514  * @num_crtcs: number of CRTCs supported by @dev
515  *
516  * This function initializes vblank support for @num_crtcs display pipelines.
517  * Cleanup is handled automatically through a cleanup function added with
518  * drmm_add_action_or_reset().
519  *
520  * Returns:
521  * Zero on success or a negative error code on failure.
522  */
drm_vblank_init(struct drm_device * dev,unsigned int num_crtcs)523 int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
524 {
525 	int ret;
526 	unsigned int i;
527 
528 	spin_lock_init(&dev->vbl_lock);
529 	spin_lock_init(&dev->vblank_time_lock);
530 
531 	dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
532 	if (!dev->vblank)
533 		return -ENOMEM;
534 
535 	dev->num_crtcs = num_crtcs;
536 
537 	for (i = 0; i < num_crtcs; i++) {
538 		struct drm_vblank_crtc *vblank = &dev->vblank[i];
539 
540 		vblank->dev = dev;
541 		vblank->pipe = i;
542 		init_waitqueue_head(&vblank->queue);
543 		timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
544 		seqlock_init(&vblank->seqlock);
545 
546 		ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
547 					       vblank);
548 		if (ret)
549 			return ret;
550 
551 		ret = drm_vblank_worker_init(vblank);
552 		if (ret)
553 			return ret;
554 	}
555 
556 	return 0;
557 }
558 EXPORT_SYMBOL(drm_vblank_init);
559 
560 /**
561  * drm_dev_has_vblank - test if vblanking has been initialized for
562  *                      a device
563  * @dev: the device
564  *
565  * Drivers may call this function to test if vblank support is
566  * initialized for a device. For most hardware this means that vblanking
567  * can also be enabled.
568  *
569  * Atomic helpers use this function to initialize
570  * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
571  *
572  * Returns:
573  * True if vblanking has been initialized for the given device, false
574  * otherwise.
575  */
drm_dev_has_vblank(const struct drm_device * dev)576 bool drm_dev_has_vblank(const struct drm_device *dev)
577 {
578 	return dev->num_crtcs != 0;
579 }
580 EXPORT_SYMBOL(drm_dev_has_vblank);
581 
582 /**
583  * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
584  * @crtc: which CRTC's vblank waitqueue to retrieve
585  *
586  * This function returns a pointer to the vblank waitqueue for the CRTC.
587  * Drivers can use this to implement vblank waits using wait_event() and related
588  * functions.
589  */
drm_crtc_vblank_waitqueue(struct drm_crtc * crtc)590 wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
591 {
592 	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
593 }
594 EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
595 
596 
597 /**
598  * drm_calc_timestamping_constants - calculate vblank timestamp constants
599  * @crtc: drm_crtc whose timestamp constants should be updated.
600  * @mode: display mode containing the scanout timings
601  *
602  * Calculate and store various constants which are later needed by vblank and
603  * swap-completion timestamping, e.g, by
604  * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
605  * CRTC's true scanout timing, so they take things like panel scaling or
606  * other adjustments into account.
607  */
drm_calc_timestamping_constants(struct drm_crtc * crtc,const struct drm_display_mode * mode)608 void drm_calc_timestamping_constants(struct drm_crtc *crtc,
609 				     const struct drm_display_mode *mode)
610 {
611 	struct drm_device *dev = crtc->dev;
612 	unsigned int pipe = drm_crtc_index(crtc);
613 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
614 	int linedur_ns = 0, framedur_ns = 0;
615 	int dotclock = mode->crtc_clock;
616 
617 	if (!drm_dev_has_vblank(dev))
618 		return;
619 
620 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
621 		return;
622 
623 	/* Valid dotclock? */
624 	if (dotclock > 0) {
625 		int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
626 
627 		/*
628 		 * Convert scanline length in pixels and video
629 		 * dot clock to line duration and frame duration
630 		 * in nanoseconds:
631 		 */
632 		linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
633 		framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
634 
635 		/*
636 		 * Fields of interlaced scanout modes are only half a frame duration.
637 		 */
638 		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
639 			framedur_ns /= 2;
640 	} else {
641 		drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
642 			crtc->base.id);
643 	}
644 
645 	vblank->linedur_ns  = linedur_ns;
646 	vblank->framedur_ns = framedur_ns;
647 	drm_mode_copy(&vblank->hwmode, mode);
648 
649 	drm_dbg_core(dev,
650 		     "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
651 		     crtc->base.id, mode->crtc_htotal,
652 		     mode->crtc_vtotal, mode->crtc_vdisplay);
653 	drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
654 		     crtc->base.id, dotclock, framedur_ns, linedur_ns);
655 }
656 EXPORT_SYMBOL(drm_calc_timestamping_constants);
657 
658 /**
659  * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
660  *                                                        timestamp helper
661  * @crtc: CRTC whose vblank timestamp to retrieve
662  * @max_error: Desired maximum allowable error in timestamps (nanosecs)
663  *             On return contains true maximum error of timestamp
664  * @vblank_time: Pointer to time which should receive the timestamp
665  * @in_vblank_irq:
666  *     True when called from drm_crtc_handle_vblank().  Some drivers
667  *     need to apply some workarounds for gpu-specific vblank irq quirks
668  *     if flag is set.
669  * @get_scanout_position:
670  *     Callback function to retrieve the scanout position. See
671  *     @struct drm_crtc_helper_funcs.get_scanout_position.
672  *
673  * Implements calculation of exact vblank timestamps from given drm_display_mode
674  * timings and current video scanout position of a CRTC.
675  *
676  * The current implementation only handles standard video modes. For double scan
677  * and interlaced modes the driver is supposed to adjust the hardware mode
678  * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
679  * match the scanout position reported.
680  *
681  * Note that atomic drivers must call drm_calc_timestamping_constants() before
682  * enabling a CRTC. The atomic helpers already take care of that in
683  * drm_atomic_helper_calc_timestamping_constants().
684  *
685  * Returns:
686  *
687  * Returns true on success, and false on failure, i.e. when no accurate
688  * timestamp could be acquired.
689  */
690 bool
drm_crtc_vblank_helper_get_vblank_timestamp_internal(struct drm_crtc * crtc,int * max_error,ktime_t * vblank_time,bool in_vblank_irq,drm_vblank_get_scanout_position_func get_scanout_position)691 drm_crtc_vblank_helper_get_vblank_timestamp_internal(
692 	struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
693 	bool in_vblank_irq,
694 	drm_vblank_get_scanout_position_func get_scanout_position)
695 {
696 	struct drm_device *dev = crtc->dev;
697 	unsigned int pipe = crtc->index;
698 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
699 	struct timespec64 ts_etime, ts_vblank_time;
700 	ktime_t stime, etime;
701 	bool vbl_status;
702 	const struct drm_display_mode *mode;
703 	int vpos, hpos, i;
704 	int delta_ns, duration_ns;
705 
706 	if (pipe >= dev->num_crtcs) {
707 		drm_err(dev, "Invalid crtc %u\n", pipe);
708 		return false;
709 	}
710 
711 	/* Scanout position query not supported? Should not happen. */
712 	if (!get_scanout_position) {
713 		drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
714 		return false;
715 	}
716 
717 	if (drm_drv_uses_atomic_modeset(dev))
718 		mode = &vblank->hwmode;
719 	else
720 		mode = &crtc->hwmode;
721 
722 	/* If mode timing undefined, just return as no-op:
723 	 * Happens during initial modesetting of a crtc.
724 	 */
725 	if (mode->crtc_clock == 0) {
726 		drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
727 			     pipe);
728 		drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
729 		return false;
730 	}
731 
732 	/* Get current scanout position with system timestamp.
733 	 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
734 	 * if single query takes longer than max_error nanoseconds.
735 	 *
736 	 * This guarantees a tight bound on maximum error if
737 	 * code gets preempted or delayed for some reason.
738 	 */
739 	for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
740 		/*
741 		 * Get vertical and horizontal scanout position vpos, hpos,
742 		 * and bounding timestamps stime, etime, pre/post query.
743 		 */
744 		vbl_status = get_scanout_position(crtc, in_vblank_irq,
745 						  &vpos, &hpos,
746 						  &stime, &etime,
747 						  mode);
748 
749 		/* Return as no-op if scanout query unsupported or failed. */
750 		if (!vbl_status) {
751 			drm_dbg_core(dev,
752 				     "crtc %u : scanoutpos query failed.\n",
753 				     pipe);
754 			return false;
755 		}
756 
757 		/* Compute uncertainty in timestamp of scanout position query. */
758 		duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
759 
760 		/* Accept result with <  max_error nsecs timing uncertainty. */
761 		if (duration_ns <= *max_error)
762 			break;
763 	}
764 
765 	/* Noisy system timing? */
766 	if (i == DRM_TIMESTAMP_MAXRETRIES) {
767 		drm_dbg_core(dev,
768 			     "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
769 			     pipe, duration_ns / 1000, *max_error / 1000, i);
770 	}
771 
772 	/* Return upper bound of timestamp precision error. */
773 	*max_error = duration_ns;
774 
775 	/* Convert scanout position into elapsed time at raw_time query
776 	 * since start of scanout at first display scanline. delta_ns
777 	 * can be negative if start of scanout hasn't happened yet.
778 	 */
779 	delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
780 			   mode->crtc_clock);
781 
782 	/* Subtract time delta from raw timestamp to get final
783 	 * vblank_time timestamp for end of vblank.
784 	 */
785 	*vblank_time = ktime_sub_ns(etime, delta_ns);
786 
787 	if (!drm_debug_enabled(DRM_UT_VBL))
788 		return true;
789 
790 	ts_etime = ktime_to_timespec64(etime);
791 	ts_vblank_time = ktime_to_timespec64(*vblank_time);
792 
793 	drm_dbg_vbl(dev,
794 		    "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
795 		    pipe, hpos, vpos,
796 		    (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
797 		    (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
798 		    duration_ns / 1000, i);
799 
800 	return true;
801 }
802 EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
803 
804 /**
805  * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
806  *                                               helper
807  * @crtc: CRTC whose vblank timestamp to retrieve
808  * @max_error: Desired maximum allowable error in timestamps (nanosecs)
809  *             On return contains true maximum error of timestamp
810  * @vblank_time: Pointer to time which should receive the timestamp
811  * @in_vblank_irq:
812  *     True when called from drm_crtc_handle_vblank().  Some drivers
813  *     need to apply some workarounds for gpu-specific vblank irq quirks
814  *     if flag is set.
815  *
816  * Implements calculation of exact vblank timestamps from given drm_display_mode
817  * timings and current video scanout position of a CRTC. This can be directly
818  * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
819  * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
820  *
821  * The current implementation only handles standard video modes. For double scan
822  * and interlaced modes the driver is supposed to adjust the hardware mode
823  * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
824  * match the scanout position reported.
825  *
826  * Note that atomic drivers must call drm_calc_timestamping_constants() before
827  * enabling a CRTC. The atomic helpers already take care of that in
828  * drm_atomic_helper_calc_timestamping_constants().
829  *
830  * Returns:
831  *
832  * Returns true on success, and false on failure, i.e. when no accurate
833  * timestamp could be acquired.
834  */
drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc * crtc,int * max_error,ktime_t * vblank_time,bool in_vblank_irq)835 bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
836 						 int *max_error,
837 						 ktime_t *vblank_time,
838 						 bool in_vblank_irq)
839 {
840 	return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
841 		crtc, max_error, vblank_time, in_vblank_irq,
842 		crtc->helper_private->get_scanout_position);
843 }
844 EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
845 
846 /**
847  * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most
848  *                                  recent vblank interval
849  * @crtc: CRTC whose vblank timestamp to retrieve
850  * @tvblank: Pointer to target time which should receive the timestamp
851  * @in_vblank_irq:
852  *     True when called from drm_crtc_handle_vblank().  Some drivers
853  *     need to apply some workarounds for gpu-specific vblank irq quirks
854  *     if flag is set.
855  *
856  * Fetches the system timestamp corresponding to the time of the most recent
857  * vblank interval on specified CRTC. May call into kms-driver to
858  * compute the timestamp with a high-precision GPU specific method.
859  *
860  * Returns zero if timestamp originates from uncorrected do_gettimeofday()
861  * call, i.e., it isn't very precisely locked to the true vblank.
862  *
863  * Returns:
864  * True if timestamp is considered to be very precise, false otherwise.
865  */
866 static bool
drm_crtc_get_last_vbltimestamp(struct drm_crtc * crtc,ktime_t * tvblank,bool in_vblank_irq)867 drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,
868 			       bool in_vblank_irq)
869 {
870 	bool ret = false;
871 
872 	/* Define requested maximum error on timestamps (nanoseconds). */
873 	int max_error = (int) drm_timestamp_precision * 1000;
874 
875 	/* Query driver if possible and precision timestamping enabled. */
876 	if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
877 		ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
878 							tvblank, in_vblank_irq);
879 	}
880 
881 	/* GPU high precision timestamp query unsupported or failed.
882 	 * Return current monotonic/gettimeofday timestamp as best estimate.
883 	 */
884 	if (!ret)
885 		*tvblank = ktime_get();
886 
887 	return ret;
888 }
889 
890 static bool
drm_get_last_vbltimestamp(struct drm_device * dev,unsigned int pipe,ktime_t * tvblank,bool in_vblank_irq)891 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
892 			  ktime_t *tvblank, bool in_vblank_irq)
893 {
894 	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
895 
896 	return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);
897 }
898 
899 /**
900  * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
901  * @crtc: which counter to retrieve
902  *
903  * Fetches the "cooked" vblank count value that represents the number of
904  * vblank events since the system was booted, including lost events due to
905  * modesetting activity. Note that this timer isn't correct against a racing
906  * vblank interrupt (since it only reports the software vblank counter), see
907  * drm_crtc_accurate_vblank_count() for such use-cases.
908  *
909  * Note that for a given vblank counter value drm_crtc_handle_vblank()
910  * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
911  * provide a barrier: Any writes done before calling
912  * drm_crtc_handle_vblank() will be visible to callers of the later
913  * functions, if the vblank count is the same or a later one.
914  *
915  * See also &drm_vblank_crtc.count.
916  *
917  * Returns:
918  * The software vblank counter.
919  */
drm_crtc_vblank_count(struct drm_crtc * crtc)920 u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
921 {
922 	return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
923 }
924 EXPORT_SYMBOL(drm_crtc_vblank_count);
925 
926 /**
927  * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
928  *     system timestamp corresponding to that vblank counter value.
929  * @dev: DRM device
930  * @pipe: index of CRTC whose counter to retrieve
931  * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
932  *
933  * Fetches the "cooked" vblank count value that represents the number of
934  * vblank events since the system was booted, including lost events due to
935  * modesetting activity. Returns corresponding system timestamp of the time
936  * of the vblank interval that corresponds to the current vblank counter value.
937  *
938  * This is the legacy version of drm_crtc_vblank_count_and_time().
939  */
drm_vblank_count_and_time(struct drm_device * dev,unsigned int pipe,ktime_t * vblanktime)940 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
941 				     ktime_t *vblanktime)
942 {
943 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
944 	u64 vblank_count;
945 	unsigned int seq;
946 
947 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
948 		*vblanktime = 0;
949 		return 0;
950 	}
951 
952 	do {
953 		seq = read_seqbegin(&vblank->seqlock);
954 		vblank_count = atomic64_read(&vblank->count);
955 		*vblanktime = vblank->time;
956 	} while (read_seqretry(&vblank->seqlock, seq));
957 
958 	return vblank_count;
959 }
960 
961 /**
962  * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
963  *     and the system timestamp corresponding to that vblank counter value
964  * @crtc: which counter to retrieve
965  * @vblanktime: Pointer to time to receive the vblank timestamp.
966  *
967  * Fetches the "cooked" vblank count value that represents the number of
968  * vblank events since the system was booted, including lost events due to
969  * modesetting activity. Returns corresponding system timestamp of the time
970  * of the vblank interval that corresponds to the current vblank counter value.
971  *
972  * Note that for a given vblank counter value drm_crtc_handle_vblank()
973  * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
974  * provide a barrier: Any writes done before calling
975  * drm_crtc_handle_vblank() will be visible to callers of the later
976  * functions, if the vblank count is the same or a later one.
977  *
978  * See also &drm_vblank_crtc.count.
979  */
drm_crtc_vblank_count_and_time(struct drm_crtc * crtc,ktime_t * vblanktime)980 u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
981 				   ktime_t *vblanktime)
982 {
983 	return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
984 					 vblanktime);
985 }
986 EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
987 
988 /**
989  * drm_crtc_next_vblank_start - calculate the time of the next vblank
990  * @crtc: the crtc for which to calculate next vblank time
991  * @vblanktime: pointer to time to receive the next vblank timestamp.
992  *
993  * Calculate the expected time of the start of the next vblank period,
994  * based on time of previous vblank and frame duration
995  */
drm_crtc_next_vblank_start(struct drm_crtc * crtc,ktime_t * vblanktime)996 int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
997 {
998 	unsigned int pipe = drm_crtc_index(crtc);
999 	struct drm_vblank_crtc *vblank;
1000 	struct drm_display_mode *mode;
1001 	u64 vblank_start;
1002 
1003 	if (!drm_dev_has_vblank(crtc->dev))
1004 		return -EINVAL;
1005 
1006 	vblank = &crtc->dev->vblank[pipe];
1007 	mode = &vblank->hwmode;
1008 
1009 	if (!vblank->framedur_ns || !vblank->linedur_ns)
1010 		return -EINVAL;
1011 
1012 	if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false))
1013 		return -EINVAL;
1014 
1015 	vblank_start = DIV_ROUND_DOWN_ULL(
1016 			(u64)vblank->framedur_ns * mode->crtc_vblank_start,
1017 			mode->crtc_vtotal);
1018 	*vblanktime  = ktime_add(*vblanktime, ns_to_ktime(vblank_start));
1019 
1020 	return 0;
1021 }
1022 EXPORT_SYMBOL(drm_crtc_next_vblank_start);
1023 
send_vblank_event(struct drm_device * dev,struct drm_pending_vblank_event * e,u64 seq,ktime_t now)1024 static void send_vblank_event(struct drm_device *dev,
1025 		struct drm_pending_vblank_event *e,
1026 		u64 seq, ktime_t now)
1027 {
1028 	struct timespec64 tv;
1029 
1030 	switch (e->event.base.type) {
1031 	case DRM_EVENT_VBLANK:
1032 	case DRM_EVENT_FLIP_COMPLETE:
1033 		tv = ktime_to_timespec64(now);
1034 		e->event.vbl.sequence = seq;
1035 		/*
1036 		 * e->event is a user space structure, with hardcoded unsigned
1037 		 * 32-bit seconds/microseconds. This is safe as we always use
1038 		 * monotonic timestamps since linux-4.15
1039 		 */
1040 		e->event.vbl.tv_sec = tv.tv_sec;
1041 		e->event.vbl.tv_usec = tv.tv_nsec / 1000;
1042 		break;
1043 	case DRM_EVENT_CRTC_SEQUENCE:
1044 		if (seq)
1045 			e->event.seq.sequence = seq;
1046 		e->event.seq.time_ns = ktime_to_ns(now);
1047 		break;
1048 	}
1049 	trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
1050 	/*
1051 	 * Use the same timestamp for any associated fence signal to avoid
1052 	 * mismatch in timestamps for vsync & fence events triggered by the
1053 	 * same HW event. Frameworks like SurfaceFlinger in Android expects the
1054 	 * retire-fence timestamp to match exactly with HW vsync as it uses it
1055 	 * for its software vsync modeling.
1056 	 */
1057 	drm_send_event_timestamp_locked(dev, &e->base, now);
1058 }
1059 
1060 /**
1061  * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1062  * @crtc: the source CRTC of the vblank event
1063  * @e: the event to send
1064  *
1065  * A lot of drivers need to generate vblank events for the very next vblank
1066  * interrupt. For example when the page flip interrupt happens when the page
1067  * flip gets armed, but not when it actually executes within the next vblank
1068  * period. This helper function implements exactly the required vblank arming
1069  * behaviour.
1070  *
1071  * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1072  * atomic commit must ensure that the next vblank happens at exactly the same
1073  * time as the atomic commit is committed to the hardware. This function itself
1074  * does **not** protect against the next vblank interrupt racing with either this
1075  * function call or the atomic commit operation. A possible sequence could be:
1076  *
1077  * 1. Driver commits new hardware state into vblank-synchronized registers.
1078  * 2. A vblank happens, committing the hardware state. Also the corresponding
1079  *    vblank interrupt is fired off and fully processed by the interrupt
1080  *    handler.
1081  * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1082  * 4. The event is only send out for the next vblank, which is wrong.
1083  *
1084  * An equivalent race can happen when the driver calls
1085  * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1086  *
1087  * The only way to make this work safely is to prevent the vblank from firing
1088  * (and the hardware from committing anything else) until the entire atomic
1089  * commit sequence has run to completion. If the hardware does not have such a
1090  * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1091  * Instead drivers need to manually send out the event from their interrupt
1092  * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1093  * possible race with the hardware committing the atomic update.
1094  *
1095  * Caller must hold a vblank reference for the event @e acquired by a
1096  * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1097  */
drm_crtc_arm_vblank_event(struct drm_crtc * crtc,struct drm_pending_vblank_event * e)1098 void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1099 			       struct drm_pending_vblank_event *e)
1100 {
1101 	struct drm_device *dev = crtc->dev;
1102 	unsigned int pipe = drm_crtc_index(crtc);
1103 
1104 	assert_spin_locked(&dev->event_lock);
1105 
1106 	e->pipe = pipe;
1107 	e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1108 	list_add_tail(&e->base.link, &dev->vblank_event_list);
1109 }
1110 EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1111 
1112 /**
1113  * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1114  * @crtc: the source CRTC of the vblank event
1115  * @e: the event to send
1116  *
1117  * Updates sequence # and timestamp on event for the most recently processed
1118  * vblank, and sends it to userspace.  Caller must hold event lock.
1119  *
1120  * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1121  * situation, especially to send out events for atomic commit operations.
1122  */
drm_crtc_send_vblank_event(struct drm_crtc * crtc,struct drm_pending_vblank_event * e)1123 void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1124 				struct drm_pending_vblank_event *e)
1125 {
1126 	struct drm_device *dev = crtc->dev;
1127 	u64 seq;
1128 	unsigned int pipe = drm_crtc_index(crtc);
1129 	ktime_t now;
1130 
1131 	if (drm_dev_has_vblank(dev)) {
1132 		seq = drm_vblank_count_and_time(dev, pipe, &now);
1133 	} else {
1134 		seq = 0;
1135 
1136 		now = ktime_get();
1137 	}
1138 	e->pipe = pipe;
1139 	send_vblank_event(dev, e, seq, now);
1140 }
1141 EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1142 
__enable_vblank(struct drm_device * dev,unsigned int pipe)1143 static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1144 {
1145 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1146 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1147 
1148 		if (drm_WARN_ON(dev, !crtc))
1149 			return 0;
1150 
1151 		if (crtc->funcs->enable_vblank)
1152 			return crtc->funcs->enable_vblank(crtc);
1153 	}
1154 #ifdef CONFIG_DRM_LEGACY
1155 	else if (dev->driver->enable_vblank) {
1156 		return dev->driver->enable_vblank(dev, pipe);
1157 	}
1158 #endif
1159 
1160 	return -EINVAL;
1161 }
1162 
drm_vblank_enable(struct drm_device * dev,unsigned int pipe)1163 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1164 {
1165 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1166 	int ret = 0;
1167 
1168 	assert_spin_locked(&dev->vbl_lock);
1169 
1170 	spin_lock(&dev->vblank_time_lock);
1171 
1172 	if (!vblank->enabled) {
1173 		/*
1174 		 * Enable vblank irqs under vblank_time_lock protection.
1175 		 * All vblank count & timestamp updates are held off
1176 		 * until we are done reinitializing master counter and
1177 		 * timestamps. Filtercode in drm_handle_vblank() will
1178 		 * prevent double-accounting of same vblank interval.
1179 		 */
1180 		ret = __enable_vblank(dev, pipe);
1181 		drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1182 			     pipe, ret);
1183 		if (ret) {
1184 			atomic_dec(&vblank->refcount);
1185 		} else {
1186 			drm_update_vblank_count(dev, pipe, 0);
1187 			/* drm_update_vblank_count() includes a wmb so we just
1188 			 * need to ensure that the compiler emits the write
1189 			 * to mark the vblank as enabled after the call
1190 			 * to drm_update_vblank_count().
1191 			 */
1192 			WRITE_ONCE(vblank->enabled, true);
1193 		}
1194 	}
1195 
1196 	spin_unlock(&dev->vblank_time_lock);
1197 
1198 	return ret;
1199 }
1200 
drm_vblank_get(struct drm_device * dev,unsigned int pipe)1201 int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1202 {
1203 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1204 	unsigned long irqflags;
1205 	int ret = 0;
1206 
1207 	if (!drm_dev_has_vblank(dev))
1208 		return -EINVAL;
1209 
1210 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1211 		return -EINVAL;
1212 
1213 	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1214 	/* Going from 0->1 means we have to enable interrupts again */
1215 	if (atomic_add_return(1, &vblank->refcount) == 1) {
1216 		ret = drm_vblank_enable(dev, pipe);
1217 	} else {
1218 		if (!vblank->enabled) {
1219 			atomic_dec(&vblank->refcount);
1220 			ret = -EINVAL;
1221 		}
1222 	}
1223 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1224 
1225 	return ret;
1226 }
1227 
1228 /**
1229  * drm_crtc_vblank_get - get a reference count on vblank events
1230  * @crtc: which CRTC to own
1231  *
1232  * Acquire a reference count on vblank events to avoid having them disabled
1233  * while in use.
1234  *
1235  * Returns:
1236  * Zero on success or a negative error code on failure.
1237  */
drm_crtc_vblank_get(struct drm_crtc * crtc)1238 int drm_crtc_vblank_get(struct drm_crtc *crtc)
1239 {
1240 	return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1241 }
1242 EXPORT_SYMBOL(drm_crtc_vblank_get);
1243 
drm_vblank_put(struct drm_device * dev,unsigned int pipe)1244 void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1245 {
1246 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1247 
1248 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1249 		return;
1250 
1251 	if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1252 		return;
1253 
1254 	/* Last user schedules interrupt disable */
1255 	if (atomic_dec_and_test(&vblank->refcount)) {
1256 		if (drm_vblank_offdelay == 0)
1257 			return;
1258 		else if (drm_vblank_offdelay < 0)
1259 			vblank_disable_fn(&vblank->disable_timer);
1260 		else if (!dev->vblank_disable_immediate)
1261 			mod_timer(&vblank->disable_timer,
1262 				  jiffies + ((drm_vblank_offdelay * HZ)/1000));
1263 	}
1264 }
1265 
1266 /**
1267  * drm_crtc_vblank_put - give up ownership of vblank events
1268  * @crtc: which counter to give up
1269  *
1270  * Release ownership of a given vblank counter, turning off interrupts
1271  * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1272  */
drm_crtc_vblank_put(struct drm_crtc * crtc)1273 void drm_crtc_vblank_put(struct drm_crtc *crtc)
1274 {
1275 	drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1276 }
1277 EXPORT_SYMBOL(drm_crtc_vblank_put);
1278 
1279 /**
1280  * drm_wait_one_vblank - wait for one vblank
1281  * @dev: DRM device
1282  * @pipe: CRTC index
1283  *
1284  * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1285  * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1286  * due to lack of driver support or because the crtc is off.
1287  *
1288  * This is the legacy version of drm_crtc_wait_one_vblank().
1289  */
drm_wait_one_vblank(struct drm_device * dev,unsigned int pipe)1290 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1291 {
1292 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1293 	int ret;
1294 	u64 last;
1295 
1296 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1297 		return;
1298 
1299 	ret = drm_vblank_get(dev, pipe);
1300 	if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1301 		     pipe, ret))
1302 		return;
1303 
1304 	last = drm_vblank_count(dev, pipe);
1305 
1306 	ret = wait_event_timeout(vblank->queue,
1307 				 last != drm_vblank_count(dev, pipe),
1308 				 msecs_to_jiffies(100));
1309 
1310 	drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1311 
1312 	drm_vblank_put(dev, pipe);
1313 }
1314 EXPORT_SYMBOL(drm_wait_one_vblank);
1315 
1316 /**
1317  * drm_crtc_wait_one_vblank - wait for one vblank
1318  * @crtc: DRM crtc
1319  *
1320  * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1321  * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1322  * due to lack of driver support or because the crtc is off.
1323  */
drm_crtc_wait_one_vblank(struct drm_crtc * crtc)1324 void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1325 {
1326 	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1327 }
1328 EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1329 
1330 /**
1331  * drm_crtc_vblank_off - disable vblank events on a CRTC
1332  * @crtc: CRTC in question
1333  *
1334  * Drivers can use this function to shut down the vblank interrupt handling when
1335  * disabling a crtc. This function ensures that the latest vblank frame count is
1336  * stored so that drm_vblank_on can restore it again.
1337  *
1338  * Drivers must use this function when the hardware vblank counter can get
1339  * reset, e.g. when suspending or disabling the @crtc in general.
1340  */
drm_crtc_vblank_off(struct drm_crtc * crtc)1341 void drm_crtc_vblank_off(struct drm_crtc *crtc)
1342 {
1343 	struct drm_device *dev = crtc->dev;
1344 	unsigned int pipe = drm_crtc_index(crtc);
1345 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1346 	struct drm_pending_vblank_event *e, *t;
1347 	ktime_t now;
1348 	u64 seq;
1349 
1350 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1351 		return;
1352 
1353 	/*
1354 	 * Grab event_lock early to prevent vblank work from being scheduled
1355 	 * while we're in the middle of shutting down vblank interrupts
1356 	 */
1357 	spin_lock_irq(&dev->event_lock);
1358 
1359 	spin_lock(&dev->vbl_lock);
1360 	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1361 		    pipe, vblank->enabled, vblank->inmodeset);
1362 
1363 	/* Avoid redundant vblank disables without previous
1364 	 * drm_crtc_vblank_on(). */
1365 	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1366 		drm_vblank_disable_and_save(dev, pipe);
1367 
1368 	wake_up(&vblank->queue);
1369 
1370 	/*
1371 	 * Prevent subsequent drm_vblank_get() from re-enabling
1372 	 * the vblank interrupt by bumping the refcount.
1373 	 */
1374 	if (!vblank->inmodeset) {
1375 		atomic_inc(&vblank->refcount);
1376 		vblank->inmodeset = 1;
1377 	}
1378 	spin_unlock(&dev->vbl_lock);
1379 
1380 	/* Send any queued vblank events, lest the natives grow disquiet */
1381 	seq = drm_vblank_count_and_time(dev, pipe, &now);
1382 
1383 	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1384 		if (e->pipe != pipe)
1385 			continue;
1386 		drm_dbg_core(dev, "Sending premature vblank event on disable: "
1387 			     "wanted %llu, current %llu\n",
1388 			     e->sequence, seq);
1389 		list_del(&e->base.link);
1390 		drm_vblank_put(dev, pipe);
1391 		send_vblank_event(dev, e, seq, now);
1392 	}
1393 
1394 	/* Cancel any leftover pending vblank work */
1395 	drm_vblank_cancel_pending_works(vblank);
1396 
1397 	spin_unlock_irq(&dev->event_lock);
1398 
1399 	/* Will be reset by the modeset helpers when re-enabling the crtc by
1400 	 * calling drm_calc_timestamping_constants(). */
1401 	vblank->hwmode.crtc_clock = 0;
1402 
1403 	/* Wait for any vblank work that's still executing to finish */
1404 	drm_vblank_flush_worker(vblank);
1405 }
1406 EXPORT_SYMBOL(drm_crtc_vblank_off);
1407 
1408 /**
1409  * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1410  * @crtc: CRTC in question
1411  *
1412  * Drivers can use this function to reset the vblank state to off at load time.
1413  * Drivers should use this together with the drm_crtc_vblank_off() and
1414  * drm_crtc_vblank_on() functions. The difference compared to
1415  * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1416  * and hence doesn't need to call any driver hooks.
1417  *
1418  * This is useful for recovering driver state e.g. on driver load, or on resume.
1419  */
drm_crtc_vblank_reset(struct drm_crtc * crtc)1420 void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1421 {
1422 	struct drm_device *dev = crtc->dev;
1423 	unsigned int pipe = drm_crtc_index(crtc);
1424 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1425 
1426 	spin_lock_irq(&dev->vbl_lock);
1427 	/*
1428 	 * Prevent subsequent drm_vblank_get() from enabling the vblank
1429 	 * interrupt by bumping the refcount.
1430 	 */
1431 	if (!vblank->inmodeset) {
1432 		atomic_inc(&vblank->refcount);
1433 		vblank->inmodeset = 1;
1434 	}
1435 	spin_unlock_irq(&dev->vbl_lock);
1436 
1437 	drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1438 	drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
1439 }
1440 EXPORT_SYMBOL(drm_crtc_vblank_reset);
1441 
1442 /**
1443  * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1444  * @crtc: CRTC in question
1445  * @max_vblank_count: max hardware vblank counter value
1446  *
1447  * Update the maximum hardware vblank counter value for @crtc
1448  * at runtime. Useful for hardware where the operation of the
1449  * hardware vblank counter depends on the currently active
1450  * display configuration.
1451  *
1452  * For example, if the hardware vblank counter does not work
1453  * when a specific connector is active the maximum can be set
1454  * to zero. And when that specific connector isn't active the
1455  * maximum can again be set to the appropriate non-zero value.
1456  *
1457  * If used, must be called before drm_vblank_on().
1458  */
drm_crtc_set_max_vblank_count(struct drm_crtc * crtc,u32 max_vblank_count)1459 void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1460 				   u32 max_vblank_count)
1461 {
1462 	struct drm_device *dev = crtc->dev;
1463 	unsigned int pipe = drm_crtc_index(crtc);
1464 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1465 
1466 	drm_WARN_ON(dev, dev->max_vblank_count);
1467 	drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1468 
1469 	vblank->max_vblank_count = max_vblank_count;
1470 }
1471 EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1472 
1473 /**
1474  * drm_crtc_vblank_on - enable vblank events on a CRTC
1475  * @crtc: CRTC in question
1476  *
1477  * This functions restores the vblank interrupt state captured with
1478  * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1479  * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1480  * unbalanced and so can also be unconditionally called in driver load code to
1481  * reflect the current hardware state of the crtc.
1482  */
drm_crtc_vblank_on(struct drm_crtc * crtc)1483 void drm_crtc_vblank_on(struct drm_crtc *crtc)
1484 {
1485 	struct drm_device *dev = crtc->dev;
1486 	unsigned int pipe = drm_crtc_index(crtc);
1487 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1488 
1489 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1490 		return;
1491 
1492 	spin_lock_irq(&dev->vbl_lock);
1493 	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1494 		    pipe, vblank->enabled, vblank->inmodeset);
1495 
1496 	/* Drop our private "prevent drm_vblank_get" refcount */
1497 	if (vblank->inmodeset) {
1498 		atomic_dec(&vblank->refcount);
1499 		vblank->inmodeset = 0;
1500 	}
1501 
1502 	drm_reset_vblank_timestamp(dev, pipe);
1503 
1504 	/*
1505 	 * re-enable interrupts if there are users left, or the
1506 	 * user wishes vblank interrupts to be enabled all the time.
1507 	 */
1508 	if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1509 		drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1510 	spin_unlock_irq(&dev->vbl_lock);
1511 }
1512 EXPORT_SYMBOL(drm_crtc_vblank_on);
1513 
drm_vblank_restore(struct drm_device * dev,unsigned int pipe)1514 static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1515 {
1516 	ktime_t t_vblank;
1517 	struct drm_vblank_crtc *vblank;
1518 	int framedur_ns;
1519 	u64 diff_ns;
1520 	u32 cur_vblank, diff = 1;
1521 	int count = DRM_TIMESTAMP_MAXRETRIES;
1522 	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
1523 
1524 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1525 		return;
1526 
1527 	assert_spin_locked(&dev->vbl_lock);
1528 	assert_spin_locked(&dev->vblank_time_lock);
1529 
1530 	vblank = &dev->vblank[pipe];
1531 	drm_WARN_ONCE(dev,
1532 		      drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1533 		      "Cannot compute missed vblanks without frame duration\n");
1534 	framedur_ns = vblank->framedur_ns;
1535 
1536 	do {
1537 		cur_vblank = __get_vblank_counter(dev, pipe);
1538 		drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1539 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1540 
1541 	diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1542 	if (framedur_ns)
1543 		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1544 
1545 
1546 	drm_dbg_vbl(dev,
1547 		    "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1548 		    diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1549 	vblank->last = (cur_vblank - diff) & max_vblank_count;
1550 }
1551 
1552 /**
1553  * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1554  * @crtc: CRTC in question
1555  *
1556  * Power manamement features can cause frame counter resets between vblank
1557  * disable and enable. Drivers can use this function in their
1558  * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1559  * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1560  * vblank counter.
1561  *
1562  * Note that drivers must have race-free high-precision timestamping support,
1563  * i.e.  &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
1564  * &drm_driver.vblank_disable_immediate must be set to indicate the
1565  * time-stamping functions are race-free against vblank hardware counter
1566  * increments.
1567  */
drm_crtc_vblank_restore(struct drm_crtc * crtc)1568 void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1569 {
1570 	WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp);
1571 	WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate);
1572 
1573 	drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1574 }
1575 EXPORT_SYMBOL(drm_crtc_vblank_restore);
1576 
drm_legacy_vblank_pre_modeset(struct drm_device * dev,unsigned int pipe)1577 static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1578 					  unsigned int pipe)
1579 {
1580 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1581 
1582 	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1583 	if (!drm_dev_has_vblank(dev))
1584 		return;
1585 
1586 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1587 		return;
1588 
1589 	/*
1590 	 * To avoid all the problems that might happen if interrupts
1591 	 * were enabled/disabled around or between these calls, we just
1592 	 * have the kernel take a reference on the CRTC (just once though
1593 	 * to avoid corrupting the count if multiple, mismatch calls occur),
1594 	 * so that interrupts remain enabled in the interim.
1595 	 */
1596 	if (!vblank->inmodeset) {
1597 		vblank->inmodeset = 0x1;
1598 		if (drm_vblank_get(dev, pipe) == 0)
1599 			vblank->inmodeset |= 0x2;
1600 	}
1601 }
1602 
drm_legacy_vblank_post_modeset(struct drm_device * dev,unsigned int pipe)1603 static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1604 					   unsigned int pipe)
1605 {
1606 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1607 
1608 	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1609 	if (!drm_dev_has_vblank(dev))
1610 		return;
1611 
1612 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1613 		return;
1614 
1615 	if (vblank->inmodeset) {
1616 		spin_lock_irq(&dev->vbl_lock);
1617 		drm_reset_vblank_timestamp(dev, pipe);
1618 		spin_unlock_irq(&dev->vbl_lock);
1619 
1620 		if (vblank->inmodeset & 0x2)
1621 			drm_vblank_put(dev, pipe);
1622 
1623 		vblank->inmodeset = 0;
1624 	}
1625 }
1626 
drm_legacy_modeset_ctl_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1627 int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1628 				 struct drm_file *file_priv)
1629 {
1630 	struct drm_modeset_ctl *modeset = data;
1631 	unsigned int pipe;
1632 
1633 	/* If drm_vblank_init() hasn't been called yet, just no-op */
1634 	if (!drm_dev_has_vblank(dev))
1635 		return 0;
1636 
1637 	/* KMS drivers handle this internally */
1638 	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1639 		return 0;
1640 
1641 	pipe = modeset->crtc;
1642 	if (pipe >= dev->num_crtcs)
1643 		return -EINVAL;
1644 
1645 	switch (modeset->cmd) {
1646 	case _DRM_PRE_MODESET:
1647 		drm_legacy_vblank_pre_modeset(dev, pipe);
1648 		break;
1649 	case _DRM_POST_MODESET:
1650 		drm_legacy_vblank_post_modeset(dev, pipe);
1651 		break;
1652 	default:
1653 		return -EINVAL;
1654 	}
1655 
1656 	return 0;
1657 }
1658 
drm_queue_vblank_event(struct drm_device * dev,unsigned int pipe,u64 req_seq,union drm_wait_vblank * vblwait,struct drm_file * file_priv)1659 static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1660 				  u64 req_seq,
1661 				  union drm_wait_vblank *vblwait,
1662 				  struct drm_file *file_priv)
1663 {
1664 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1665 	struct drm_pending_vblank_event *e;
1666 	ktime_t now;
1667 	u64 seq;
1668 	int ret;
1669 
1670 	e = kzalloc(sizeof(*e), GFP_KERNEL);
1671 	if (e == NULL) {
1672 		ret = -ENOMEM;
1673 		goto err_put;
1674 	}
1675 
1676 	e->pipe = pipe;
1677 	e->event.base.type = DRM_EVENT_VBLANK;
1678 	e->event.base.length = sizeof(e->event.vbl);
1679 	e->event.vbl.user_data = vblwait->request.signal;
1680 	e->event.vbl.crtc_id = 0;
1681 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1682 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1683 
1684 		if (crtc)
1685 			e->event.vbl.crtc_id = crtc->base.id;
1686 	}
1687 
1688 	spin_lock_irq(&dev->event_lock);
1689 
1690 	/*
1691 	 * drm_crtc_vblank_off() might have been called after we called
1692 	 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1693 	 * vblank disable, so no need for further locking.  The reference from
1694 	 * drm_vblank_get() protects against vblank disable from another source.
1695 	 */
1696 	if (!READ_ONCE(vblank->enabled)) {
1697 		ret = -EINVAL;
1698 		goto err_unlock;
1699 	}
1700 
1701 	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1702 					    &e->event.base);
1703 
1704 	if (ret)
1705 		goto err_unlock;
1706 
1707 	seq = drm_vblank_count_and_time(dev, pipe, &now);
1708 
1709 	drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1710 		     req_seq, seq, pipe);
1711 
1712 	trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1713 
1714 	e->sequence = req_seq;
1715 	if (drm_vblank_passed(seq, req_seq)) {
1716 		drm_vblank_put(dev, pipe);
1717 		send_vblank_event(dev, e, seq, now);
1718 		vblwait->reply.sequence = seq;
1719 	} else {
1720 		/* drm_handle_vblank_events will call drm_vblank_put */
1721 		list_add_tail(&e->base.link, &dev->vblank_event_list);
1722 		vblwait->reply.sequence = req_seq;
1723 	}
1724 
1725 	spin_unlock_irq(&dev->event_lock);
1726 
1727 	return 0;
1728 
1729 err_unlock:
1730 	spin_unlock_irq(&dev->event_lock);
1731 	kfree(e);
1732 err_put:
1733 	drm_vblank_put(dev, pipe);
1734 	return ret;
1735 }
1736 
drm_wait_vblank_is_query(union drm_wait_vblank * vblwait)1737 static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1738 {
1739 	if (vblwait->request.sequence)
1740 		return false;
1741 
1742 	return _DRM_VBLANK_RELATIVE ==
1743 		(vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1744 					  _DRM_VBLANK_EVENT |
1745 					  _DRM_VBLANK_NEXTONMISS));
1746 }
1747 
1748 /*
1749  * Widen a 32-bit param to 64-bits.
1750  *
1751  * \param narrow 32-bit value (missing upper 32 bits)
1752  * \param near 64-bit value that should be 'close' to near
1753  *
1754  * This function returns a 64-bit value using the lower 32-bits from
1755  * 'narrow' and constructing the upper 32-bits so that the result is
1756  * as close as possible to 'near'.
1757  */
1758 
widen_32_to_64(u32 narrow,u64 near)1759 static u64 widen_32_to_64(u32 narrow, u64 near)
1760 {
1761 	return near + (s32) (narrow - near);
1762 }
1763 
drm_wait_vblank_reply(struct drm_device * dev,unsigned int pipe,struct drm_wait_vblank_reply * reply)1764 static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1765 				  struct drm_wait_vblank_reply *reply)
1766 {
1767 	ktime_t now;
1768 	struct timespec64 ts;
1769 
1770 	/*
1771 	 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1772 	 * to store the seconds. This is safe as we always use monotonic
1773 	 * timestamps since linux-4.15.
1774 	 */
1775 	reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1776 	ts = ktime_to_timespec64(now);
1777 	reply->tval_sec = (u32)ts.tv_sec;
1778 	reply->tval_usec = ts.tv_nsec / 1000;
1779 }
1780 
drm_wait_vblank_supported(struct drm_device * dev)1781 static bool drm_wait_vblank_supported(struct drm_device *dev)
1782 {
1783 #if IS_ENABLED(CONFIG_DRM_LEGACY)
1784 	if (unlikely(drm_core_check_feature(dev, DRIVER_LEGACY)))
1785 		return dev->irq_enabled;
1786 #endif
1787 	return drm_dev_has_vblank(dev);
1788 }
1789 
drm_wait_vblank_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1790 int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1791 			  struct drm_file *file_priv)
1792 {
1793 	struct drm_crtc *crtc;
1794 	struct drm_vblank_crtc *vblank;
1795 	union drm_wait_vblank *vblwait = data;
1796 	int ret;
1797 	u64 req_seq, seq;
1798 	unsigned int pipe_index;
1799 	unsigned int flags, pipe, high_pipe;
1800 
1801 	if (!drm_wait_vblank_supported(dev))
1802 		return -EOPNOTSUPP;
1803 
1804 	if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1805 		return -EINVAL;
1806 
1807 	if (vblwait->request.type &
1808 	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1809 	      _DRM_VBLANK_HIGH_CRTC_MASK)) {
1810 		drm_dbg_core(dev,
1811 			     "Unsupported type value 0x%x, supported mask 0x%x\n",
1812 			     vblwait->request.type,
1813 			     (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1814 			      _DRM_VBLANK_HIGH_CRTC_MASK));
1815 		return -EINVAL;
1816 	}
1817 
1818 	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1819 	high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1820 	if (high_pipe)
1821 		pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1822 	else
1823 		pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1824 
1825 	/* Convert lease-relative crtc index into global crtc index */
1826 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1827 		pipe = 0;
1828 		drm_for_each_crtc(crtc, dev) {
1829 			if (drm_lease_held(file_priv, crtc->base.id)) {
1830 				if (pipe_index == 0)
1831 					break;
1832 				pipe_index--;
1833 			}
1834 			pipe++;
1835 		}
1836 	} else {
1837 		pipe = pipe_index;
1838 	}
1839 
1840 	if (pipe >= dev->num_crtcs)
1841 		return -EINVAL;
1842 
1843 	vblank = &dev->vblank[pipe];
1844 
1845 	/* If the counter is currently enabled and accurate, short-circuit
1846 	 * queries to return the cached timestamp of the last vblank.
1847 	 */
1848 	if (dev->vblank_disable_immediate &&
1849 	    drm_wait_vblank_is_query(vblwait) &&
1850 	    READ_ONCE(vblank->enabled)) {
1851 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1852 		return 0;
1853 	}
1854 
1855 	ret = drm_vblank_get(dev, pipe);
1856 	if (ret) {
1857 		drm_dbg_core(dev,
1858 			     "crtc %d failed to acquire vblank counter, %d\n",
1859 			     pipe, ret);
1860 		return ret;
1861 	}
1862 	seq = drm_vblank_count(dev, pipe);
1863 
1864 	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1865 	case _DRM_VBLANK_RELATIVE:
1866 		req_seq = seq + vblwait->request.sequence;
1867 		vblwait->request.sequence = req_seq;
1868 		vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1869 		break;
1870 	case _DRM_VBLANK_ABSOLUTE:
1871 		req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1872 		break;
1873 	default:
1874 		ret = -EINVAL;
1875 		goto done;
1876 	}
1877 
1878 	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1879 	    drm_vblank_passed(seq, req_seq)) {
1880 		req_seq = seq + 1;
1881 		vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1882 		vblwait->request.sequence = req_seq;
1883 	}
1884 
1885 	if (flags & _DRM_VBLANK_EVENT) {
1886 		/* must hold on to the vblank ref until the event fires
1887 		 * drm_vblank_put will be called asynchronously
1888 		 */
1889 		return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1890 	}
1891 
1892 	if (req_seq != seq) {
1893 		int wait;
1894 
1895 		drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1896 			     req_seq, pipe);
1897 		wait = wait_event_interruptible_timeout(vblank->queue,
1898 			drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1899 				      !READ_ONCE(vblank->enabled),
1900 			msecs_to_jiffies(3000));
1901 
1902 		switch (wait) {
1903 		case 0:
1904 			/* timeout */
1905 			ret = -EBUSY;
1906 			break;
1907 		case -ERESTARTSYS:
1908 			/* interrupted by signal */
1909 			ret = -EINTR;
1910 			break;
1911 		default:
1912 			ret = 0;
1913 			break;
1914 		}
1915 	}
1916 
1917 	if (ret != -EINTR) {
1918 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1919 
1920 		drm_dbg_core(dev, "crtc %d returning %u to client\n",
1921 			     pipe, vblwait->reply.sequence);
1922 	} else {
1923 		drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1924 			     pipe);
1925 	}
1926 
1927 done:
1928 	drm_vblank_put(dev, pipe);
1929 	return ret;
1930 }
1931 
drm_handle_vblank_events(struct drm_device * dev,unsigned int pipe)1932 static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1933 {
1934 	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1935 	bool high_prec = false;
1936 	struct drm_pending_vblank_event *e, *t;
1937 	ktime_t now;
1938 	u64 seq;
1939 
1940 	assert_spin_locked(&dev->event_lock);
1941 
1942 	seq = drm_vblank_count_and_time(dev, pipe, &now);
1943 
1944 	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1945 		if (e->pipe != pipe)
1946 			continue;
1947 		if (!drm_vblank_passed(seq, e->sequence))
1948 			continue;
1949 
1950 		drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1951 			     e->sequence, seq);
1952 
1953 		list_del(&e->base.link);
1954 		drm_vblank_put(dev, pipe);
1955 		send_vblank_event(dev, e, seq, now);
1956 	}
1957 
1958 	if (crtc && crtc->funcs->get_vblank_timestamp)
1959 		high_prec = true;
1960 
1961 	trace_drm_vblank_event(pipe, seq, now, high_prec);
1962 }
1963 
1964 /**
1965  * drm_handle_vblank - handle a vblank event
1966  * @dev: DRM device
1967  * @pipe: index of CRTC where this event occurred
1968  *
1969  * Drivers should call this routine in their vblank interrupt handlers to
1970  * update the vblank counter and send any signals that may be pending.
1971  *
1972  * This is the legacy version of drm_crtc_handle_vblank().
1973  */
drm_handle_vblank(struct drm_device * dev,unsigned int pipe)1974 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1975 {
1976 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1977 	unsigned long irqflags;
1978 	bool disable_irq;
1979 
1980 	if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1981 		return false;
1982 
1983 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1984 		return false;
1985 
1986 	spin_lock_irqsave(&dev->event_lock, irqflags);
1987 
1988 	/* Need timestamp lock to prevent concurrent execution with
1989 	 * vblank enable/disable, as this would cause inconsistent
1990 	 * or corrupted timestamps and vblank counts.
1991 	 */
1992 	spin_lock(&dev->vblank_time_lock);
1993 
1994 	/* Vblank irq handling disabled. Nothing to do. */
1995 	if (!vblank->enabled) {
1996 		spin_unlock(&dev->vblank_time_lock);
1997 		spin_unlock_irqrestore(&dev->event_lock, irqflags);
1998 		return false;
1999 	}
2000 
2001 	drm_update_vblank_count(dev, pipe, true);
2002 
2003 	spin_unlock(&dev->vblank_time_lock);
2004 
2005 	wake_up(&vblank->queue);
2006 
2007 	/* With instant-off, we defer disabling the interrupt until after
2008 	 * we finish processing the following vblank after all events have
2009 	 * been signaled. The disable has to be last (after
2010 	 * drm_handle_vblank_events) so that the timestamp is always accurate.
2011 	 */
2012 	disable_irq = (dev->vblank_disable_immediate &&
2013 		       drm_vblank_offdelay > 0 &&
2014 		       !atomic_read(&vblank->refcount));
2015 
2016 	drm_handle_vblank_events(dev, pipe);
2017 	drm_handle_vblank_works(vblank);
2018 
2019 	spin_unlock_irqrestore(&dev->event_lock, irqflags);
2020 
2021 	if (disable_irq)
2022 		vblank_disable_fn(&vblank->disable_timer);
2023 
2024 	return true;
2025 }
2026 EXPORT_SYMBOL(drm_handle_vblank);
2027 
2028 /**
2029  * drm_crtc_handle_vblank - handle a vblank event
2030  * @crtc: where this event occurred
2031  *
2032  * Drivers should call this routine in their vblank interrupt handlers to
2033  * update the vblank counter and send any signals that may be pending.
2034  *
2035  * This is the native KMS version of drm_handle_vblank().
2036  *
2037  * Note that for a given vblank counter value drm_crtc_handle_vblank()
2038  * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
2039  * provide a barrier: Any writes done before calling
2040  * drm_crtc_handle_vblank() will be visible to callers of the later
2041  * functions, if the vblank count is the same or a later one.
2042  *
2043  * See also &drm_vblank_crtc.count.
2044  *
2045  * Returns:
2046  * True if the event was successfully handled, false on failure.
2047  */
drm_crtc_handle_vblank(struct drm_crtc * crtc)2048 bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
2049 {
2050 	return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
2051 }
2052 EXPORT_SYMBOL(drm_crtc_handle_vblank);
2053 
2054 /*
2055  * Get crtc VBLANK count.
2056  *
2057  * \param dev DRM device
2058  * \param data user argument, pointing to a drm_crtc_get_sequence structure.
2059  * \param file_priv drm file private for the user's open file descriptor
2060  */
2061 
drm_crtc_get_sequence_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)2062 int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
2063 				struct drm_file *file_priv)
2064 {
2065 	struct drm_crtc *crtc;
2066 	struct drm_vblank_crtc *vblank;
2067 	int pipe;
2068 	struct drm_crtc_get_sequence *get_seq = data;
2069 	ktime_t now;
2070 	bool vblank_enabled;
2071 	int ret;
2072 
2073 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
2074 		return -EOPNOTSUPP;
2075 
2076 	if (!drm_dev_has_vblank(dev))
2077 		return -EOPNOTSUPP;
2078 
2079 	crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
2080 	if (!crtc)
2081 		return -ENOENT;
2082 
2083 	pipe = drm_crtc_index(crtc);
2084 
2085 	vblank = &dev->vblank[pipe];
2086 	vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
2087 
2088 	if (!vblank_enabled) {
2089 		ret = drm_crtc_vblank_get(crtc);
2090 		if (ret) {
2091 			drm_dbg_core(dev,
2092 				     "crtc %d failed to acquire vblank counter, %d\n",
2093 				     pipe, ret);
2094 			return ret;
2095 		}
2096 	}
2097 	drm_modeset_lock(&crtc->mutex, NULL);
2098 	if (crtc->state)
2099 		get_seq->active = crtc->state->enable;
2100 	else
2101 		get_seq->active = crtc->enabled;
2102 	drm_modeset_unlock(&crtc->mutex);
2103 	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
2104 	get_seq->sequence_ns = ktime_to_ns(now);
2105 	if (!vblank_enabled)
2106 		drm_crtc_vblank_put(crtc);
2107 	return 0;
2108 }
2109 
2110 /*
2111  * Queue a event for VBLANK sequence
2112  *
2113  * \param dev DRM device
2114  * \param data user argument, pointing to a drm_crtc_queue_sequence structure.
2115  * \param file_priv drm file private for the user's open file descriptor
2116  */
2117 
drm_crtc_queue_sequence_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)2118 int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2119 				  struct drm_file *file_priv)
2120 {
2121 	struct drm_crtc *crtc;
2122 	struct drm_vblank_crtc *vblank;
2123 	int pipe;
2124 	struct drm_crtc_queue_sequence *queue_seq = data;
2125 	ktime_t now;
2126 	struct drm_pending_vblank_event *e;
2127 	u32 flags;
2128 	u64 seq;
2129 	u64 req_seq;
2130 	int ret;
2131 
2132 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
2133 		return -EOPNOTSUPP;
2134 
2135 	if (!drm_dev_has_vblank(dev))
2136 		return -EOPNOTSUPP;
2137 
2138 	crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
2139 	if (!crtc)
2140 		return -ENOENT;
2141 
2142 	flags = queue_seq->flags;
2143 	/* Check valid flag bits */
2144 	if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2145 		      DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2146 		return -EINVAL;
2147 
2148 	pipe = drm_crtc_index(crtc);
2149 
2150 	vblank = &dev->vblank[pipe];
2151 
2152 	e = kzalloc(sizeof(*e), GFP_KERNEL);
2153 	if (e == NULL)
2154 		return -ENOMEM;
2155 
2156 	ret = drm_crtc_vblank_get(crtc);
2157 	if (ret) {
2158 		drm_dbg_core(dev,
2159 			     "crtc %d failed to acquire vblank counter, %d\n",
2160 			     pipe, ret);
2161 		goto err_free;
2162 	}
2163 
2164 	seq = drm_vblank_count_and_time(dev, pipe, &now);
2165 	req_seq = queue_seq->sequence;
2166 
2167 	if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2168 		req_seq += seq;
2169 
2170 	if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq))
2171 		req_seq = seq + 1;
2172 
2173 	e->pipe = pipe;
2174 	e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2175 	e->event.base.length = sizeof(e->event.seq);
2176 	e->event.seq.user_data = queue_seq->user_data;
2177 
2178 	spin_lock_irq(&dev->event_lock);
2179 
2180 	/*
2181 	 * drm_crtc_vblank_off() might have been called after we called
2182 	 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2183 	 * vblank disable, so no need for further locking.  The reference from
2184 	 * drm_crtc_vblank_get() protects against vblank disable from another source.
2185 	 */
2186 	if (!READ_ONCE(vblank->enabled)) {
2187 		ret = -EINVAL;
2188 		goto err_unlock;
2189 	}
2190 
2191 	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
2192 					    &e->event.base);
2193 
2194 	if (ret)
2195 		goto err_unlock;
2196 
2197 	e->sequence = req_seq;
2198 
2199 	if (drm_vblank_passed(seq, req_seq)) {
2200 		drm_crtc_vblank_put(crtc);
2201 		send_vblank_event(dev, e, seq, now);
2202 		queue_seq->sequence = seq;
2203 	} else {
2204 		/* drm_handle_vblank_events will call drm_vblank_put */
2205 		list_add_tail(&e->base.link, &dev->vblank_event_list);
2206 		queue_seq->sequence = req_seq;
2207 	}
2208 
2209 	spin_unlock_irq(&dev->event_lock);
2210 	return 0;
2211 
2212 err_unlock:
2213 	spin_unlock_irq(&dev->event_lock);
2214 	drm_crtc_vblank_put(crtc);
2215 err_free:
2216 	kfree(e);
2217 	return ret;
2218 }
2219 
2220