1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include "display/intel_frontbuffer.h"
8 #include "gt/intel_gt.h"
9 
10 #include "i915_drv.h"
11 #include "i915_gem_clflush.h"
12 #include "i915_gem_domain.h"
13 #include "i915_gem_gtt.h"
14 #include "i915_gem_ioctls.h"
15 #include "i915_gem_lmem.h"
16 #include "i915_gem_mman.h"
17 #include "i915_gem_object.h"
18 #include "i915_vma.h"
19 
gpu_write_needs_clflush(struct drm_i915_gem_object * obj)20 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
21 {
22 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
23 
24 	if (IS_DGFX(i915))
25 		return false;
26 
27 	return !(obj->cache_level == I915_CACHE_NONE ||
28 		 obj->cache_level == I915_CACHE_WT);
29 }
30 
i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object * obj)31 bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
32 {
33 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
34 
35 	if (obj->cache_dirty)
36 		return false;
37 
38 	if (IS_DGFX(i915))
39 		return false;
40 
41 	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
42 		return true;
43 
44 	/* Currently in use by HW (display engine)? Keep flushed. */
45 	return i915_gem_object_is_framebuffer(obj);
46 }
47 
48 static void
flush_write_domain(struct drm_i915_gem_object * obj,unsigned int flush_domains)49 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
50 {
51 	struct i915_vma *vma;
52 
53 	assert_object_held(obj);
54 
55 	if (!(obj->write_domain & flush_domains))
56 		return;
57 
58 	switch (obj->write_domain) {
59 	case I915_GEM_DOMAIN_GTT:
60 		spin_lock(&obj->vma.lock);
61 		for_each_ggtt_vma(vma, obj) {
62 			if (i915_vma_unset_ggtt_write(vma))
63 				intel_gt_flush_ggtt_writes(vma->vm->gt);
64 		}
65 		spin_unlock(&obj->vma.lock);
66 
67 		i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
68 		break;
69 
70 	case I915_GEM_DOMAIN_WC:
71 		wmb();
72 		break;
73 
74 	case I915_GEM_DOMAIN_CPU:
75 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
76 		break;
77 
78 	case I915_GEM_DOMAIN_RENDER:
79 		if (gpu_write_needs_clflush(obj))
80 			obj->cache_dirty = true;
81 		break;
82 	}
83 
84 	obj->write_domain = 0;
85 }
86 
__i915_gem_object_flush_for_display(struct drm_i915_gem_object * obj)87 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
88 {
89 	/*
90 	 * We manually flush the CPU domain so that we can override and
91 	 * force the flush for the display, and perform it asyncrhonously.
92 	 */
93 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
94 	if (obj->cache_dirty)
95 		i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
96 	obj->write_domain = 0;
97 }
98 
i915_gem_object_flush_if_display(struct drm_i915_gem_object * obj)99 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
100 {
101 	if (!i915_gem_object_is_framebuffer(obj))
102 		return;
103 
104 	i915_gem_object_lock(obj, NULL);
105 	__i915_gem_object_flush_for_display(obj);
106 	i915_gem_object_unlock(obj);
107 }
108 
i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object * obj)109 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
110 {
111 	if (i915_gem_object_is_framebuffer(obj))
112 		__i915_gem_object_flush_for_display(obj);
113 }
114 
115 /**
116  * Moves a single object to the WC read, and possibly write domain.
117  * @obj: object to act on
118  * @write: ask for write access or read only
119  *
120  * This function returns when the move is complete, including waiting on
121  * flushes to occur.
122  */
123 int
i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object * obj,bool write)124 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
125 {
126 	int ret;
127 
128 	assert_object_held(obj);
129 
130 	ret = i915_gem_object_wait(obj,
131 				   I915_WAIT_INTERRUPTIBLE |
132 				   (write ? I915_WAIT_ALL : 0),
133 				   MAX_SCHEDULE_TIMEOUT);
134 	if (ret)
135 		return ret;
136 
137 	if (obj->write_domain == I915_GEM_DOMAIN_WC)
138 		return 0;
139 
140 	/* Flush and acquire obj->pages so that we are coherent through
141 	 * direct access in memory with previous cached writes through
142 	 * shmemfs and that our cache domain tracking remains valid.
143 	 * For example, if the obj->filp was moved to swap without us
144 	 * being notified and releasing the pages, we would mistakenly
145 	 * continue to assume that the obj remained out of the CPU cached
146 	 * domain.
147 	 */
148 	ret = i915_gem_object_pin_pages(obj);
149 	if (ret)
150 		return ret;
151 
152 	flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
153 
154 	/* Serialise direct access to this object with the barriers for
155 	 * coherent writes from the GPU, by effectively invalidating the
156 	 * WC domain upon first access.
157 	 */
158 	if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
159 		mb();
160 
161 	/* It should now be out of any other write domains, and we can update
162 	 * the domain values for our changes.
163 	 */
164 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
165 	obj->read_domains |= I915_GEM_DOMAIN_WC;
166 	if (write) {
167 		obj->read_domains = I915_GEM_DOMAIN_WC;
168 		obj->write_domain = I915_GEM_DOMAIN_WC;
169 		obj->mm.dirty = true;
170 	}
171 
172 	i915_gem_object_unpin_pages(obj);
173 	return 0;
174 }
175 
176 /**
177  * Moves a single object to the GTT read, and possibly write domain.
178  * @obj: object to act on
179  * @write: ask for write access or read only
180  *
181  * This function returns when the move is complete, including waiting on
182  * flushes to occur.
183  */
184 int
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object * obj,bool write)185 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
186 {
187 	int ret;
188 
189 	assert_object_held(obj);
190 
191 	ret = i915_gem_object_wait(obj,
192 				   I915_WAIT_INTERRUPTIBLE |
193 				   (write ? I915_WAIT_ALL : 0),
194 				   MAX_SCHEDULE_TIMEOUT);
195 	if (ret)
196 		return ret;
197 
198 	if (obj->write_domain == I915_GEM_DOMAIN_GTT)
199 		return 0;
200 
201 	/* Flush and acquire obj->pages so that we are coherent through
202 	 * direct access in memory with previous cached writes through
203 	 * shmemfs and that our cache domain tracking remains valid.
204 	 * For example, if the obj->filp was moved to swap without us
205 	 * being notified and releasing the pages, we would mistakenly
206 	 * continue to assume that the obj remained out of the CPU cached
207 	 * domain.
208 	 */
209 	ret = i915_gem_object_pin_pages(obj);
210 	if (ret)
211 		return ret;
212 
213 	flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
214 
215 	/* Serialise direct access to this object with the barriers for
216 	 * coherent writes from the GPU, by effectively invalidating the
217 	 * GTT domain upon first access.
218 	 */
219 	if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
220 		mb();
221 
222 	/* It should now be out of any other write domains, and we can update
223 	 * the domain values for our changes.
224 	 */
225 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
226 	obj->read_domains |= I915_GEM_DOMAIN_GTT;
227 	if (write) {
228 		struct i915_vma *vma;
229 
230 		obj->read_domains = I915_GEM_DOMAIN_GTT;
231 		obj->write_domain = I915_GEM_DOMAIN_GTT;
232 		obj->mm.dirty = true;
233 
234 		spin_lock(&obj->vma.lock);
235 		for_each_ggtt_vma(vma, obj)
236 			if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
237 				i915_vma_set_ggtt_write(vma);
238 		spin_unlock(&obj->vma.lock);
239 	}
240 
241 	i915_gem_object_unpin_pages(obj);
242 	return 0;
243 }
244 
245 /**
246  * Changes the cache-level of an object across all VMA.
247  * @obj: object to act on
248  * @cache_level: new cache level to set for the object
249  *
250  * After this function returns, the object will be in the new cache-level
251  * across all GTT and the contents of the backing storage will be coherent,
252  * with respect to the new cache-level. In order to keep the backing storage
253  * coherent for all users, we only allow a single cache level to be set
254  * globally on the object and prevent it from being changed whilst the
255  * hardware is reading from the object. That is if the object is currently
256  * on the scanout it will be set to uncached (or equivalent display
257  * cache coherency) and all non-MOCS GPU access will also be uncached so
258  * that all direct access to the scanout remains coherent.
259  */
i915_gem_object_set_cache_level(struct drm_i915_gem_object * obj,enum i915_cache_level cache_level)260 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
261 				    enum i915_cache_level cache_level)
262 {
263 	int ret;
264 
265 	if (obj->cache_level == cache_level)
266 		return 0;
267 
268 	ret = i915_gem_object_wait(obj,
269 				   I915_WAIT_INTERRUPTIBLE |
270 				   I915_WAIT_ALL,
271 				   MAX_SCHEDULE_TIMEOUT);
272 	if (ret)
273 		return ret;
274 
275 	/* Always invalidate stale cachelines */
276 	if (obj->cache_level != cache_level) {
277 		i915_gem_object_set_cache_coherency(obj, cache_level);
278 		obj->cache_dirty = true;
279 	}
280 
281 	/* The cache-level will be applied when each vma is rebound. */
282 	return i915_gem_object_unbind(obj,
283 				      I915_GEM_OBJECT_UNBIND_ACTIVE |
284 				      I915_GEM_OBJECT_UNBIND_BARRIER);
285 }
286 
i915_gem_get_caching_ioctl(struct drm_device * dev,void * data,struct drm_file * file)287 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
288 			       struct drm_file *file)
289 {
290 	struct drm_i915_gem_caching *args = data;
291 	struct drm_i915_gem_object *obj;
292 	int err = 0;
293 
294 	if (IS_DGFX(to_i915(dev)))
295 		return -ENODEV;
296 
297 	rcu_read_lock();
298 	obj = i915_gem_object_lookup_rcu(file, args->handle);
299 	if (!obj) {
300 		err = -ENOENT;
301 		goto out;
302 	}
303 
304 	switch (obj->cache_level) {
305 	case I915_CACHE_LLC:
306 	case I915_CACHE_L3_LLC:
307 		args->caching = I915_CACHING_CACHED;
308 		break;
309 
310 	case I915_CACHE_WT:
311 		args->caching = I915_CACHING_DISPLAY;
312 		break;
313 
314 	default:
315 		args->caching = I915_CACHING_NONE;
316 		break;
317 	}
318 out:
319 	rcu_read_unlock();
320 	return err;
321 }
322 
i915_gem_set_caching_ioctl(struct drm_device * dev,void * data,struct drm_file * file)323 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
324 			       struct drm_file *file)
325 {
326 	struct drm_i915_private *i915 = to_i915(dev);
327 	struct drm_i915_gem_caching *args = data;
328 	struct drm_i915_gem_object *obj;
329 	enum i915_cache_level level;
330 	int ret = 0;
331 
332 	if (IS_DGFX(i915))
333 		return -ENODEV;
334 
335 	switch (args->caching) {
336 	case I915_CACHING_NONE:
337 		level = I915_CACHE_NONE;
338 		break;
339 	case I915_CACHING_CACHED:
340 		/*
341 		 * Due to a HW issue on BXT A stepping, GPU stores via a
342 		 * snooped mapping may leave stale data in a corresponding CPU
343 		 * cacheline, whereas normally such cachelines would get
344 		 * invalidated.
345 		 */
346 		if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
347 			return -ENODEV;
348 
349 		level = I915_CACHE_LLC;
350 		break;
351 	case I915_CACHING_DISPLAY:
352 		level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
353 		break;
354 	default:
355 		return -EINVAL;
356 	}
357 
358 	obj = i915_gem_object_lookup(file, args->handle);
359 	if (!obj)
360 		return -ENOENT;
361 
362 	/*
363 	 * The caching mode of proxy object is handled by its generator, and
364 	 * not allowed to be changed by userspace.
365 	 */
366 	if (i915_gem_object_is_proxy(obj)) {
367 		/*
368 		 * Silently allow cached for userptr; the vulkan driver
369 		 * sets all objects to cached
370 		 */
371 		if (!i915_gem_object_is_userptr(obj) ||
372 		    args->caching != I915_CACHING_CACHED)
373 			ret = -ENXIO;
374 
375 		goto out;
376 	}
377 
378 	ret = i915_gem_object_lock_interruptible(obj, NULL);
379 	if (ret)
380 		goto out;
381 
382 	ret = i915_gem_object_set_cache_level(obj, level);
383 	i915_gem_object_unlock(obj);
384 
385 out:
386 	i915_gem_object_put(obj);
387 	return ret;
388 }
389 
390 /*
391  * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
392  * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
393  * (for pageflips). We only flush the caches while preparing the buffer for
394  * display, the callers are responsible for frontbuffer flush.
395  */
396 struct i915_vma *
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object * obj,struct i915_gem_ww_ctx * ww,u32 alignment,const struct i915_gtt_view * view,unsigned int flags)397 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
398 				     struct i915_gem_ww_ctx *ww,
399 				     u32 alignment,
400 				     const struct i915_gtt_view *view,
401 				     unsigned int flags)
402 {
403 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
404 	struct i915_vma *vma;
405 	int ret;
406 
407 	/* Frame buffer must be in LMEM */
408 	if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
409 		return ERR_PTR(-EINVAL);
410 
411 	/*
412 	 * The display engine is not coherent with the LLC cache on gen6.  As
413 	 * a result, we make sure that the pinning that is about to occur is
414 	 * done with uncached PTEs. This is lowest common denominator for all
415 	 * chipsets.
416 	 *
417 	 * However for gen6+, we could do better by using the GFDT bit instead
418 	 * of uncaching, which would allow us to flush all the LLC-cached data
419 	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
420 	 */
421 	ret = i915_gem_object_set_cache_level(obj,
422 					      HAS_WT(i915) ?
423 					      I915_CACHE_WT : I915_CACHE_NONE);
424 	if (ret)
425 		return ERR_PTR(ret);
426 
427 	/*
428 	 * As the user may map the buffer once pinned in the display plane
429 	 * (e.g. libkms for the bootup splash), we have to ensure that we
430 	 * always use map_and_fenceable for all scanout buffers. However,
431 	 * it may simply be too big to fit into mappable, in which case
432 	 * put it anyway and hope that userspace can cope (but always first
433 	 * try to preserve the existing ABI).
434 	 */
435 	vma = ERR_PTR(-ENOSPC);
436 	if ((flags & PIN_MAPPABLE) == 0 &&
437 	    (!view || view->type == I915_GTT_VIEW_NORMAL))
438 		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,
439 						  flags | PIN_MAPPABLE |
440 						  PIN_NONBLOCK);
441 	if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
442 		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,
443 						  alignment, flags);
444 	if (IS_ERR(vma))
445 		return vma;
446 
447 	vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
448 	i915_vma_mark_scanout(vma);
449 
450 	i915_gem_object_flush_if_display_locked(obj);
451 
452 	return vma;
453 }
454 
455 /**
456  * Moves a single object to the CPU read, and possibly write domain.
457  * @obj: object to act on
458  * @write: requesting write or read-only access
459  *
460  * This function returns when the move is complete, including waiting on
461  * flushes to occur.
462  */
463 int
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object * obj,bool write)464 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
465 {
466 	int ret;
467 
468 	assert_object_held(obj);
469 
470 	ret = i915_gem_object_wait(obj,
471 				   I915_WAIT_INTERRUPTIBLE |
472 				   (write ? I915_WAIT_ALL : 0),
473 				   MAX_SCHEDULE_TIMEOUT);
474 	if (ret)
475 		return ret;
476 
477 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
478 
479 	/* Flush the CPU cache if it's still invalid. */
480 	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
481 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
482 		obj->read_domains |= I915_GEM_DOMAIN_CPU;
483 	}
484 
485 	/* It should now be out of any other write domains, and we can update
486 	 * the domain values for our changes.
487 	 */
488 	GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
489 
490 	/* If we're writing through the CPU, then the GPU read domains will
491 	 * need to be invalidated at next use.
492 	 */
493 	if (write)
494 		__start_cpu_write(obj);
495 
496 	return 0;
497 }
498 
499 /**
500  * Called when user space prepares to use an object with the CPU, either
501  * through the mmap ioctl's mapping or a GTT mapping.
502  * @dev: drm device
503  * @data: ioctl data blob
504  * @file: drm file
505  */
506 int
i915_gem_set_domain_ioctl(struct drm_device * dev,void * data,struct drm_file * file)507 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
508 			  struct drm_file *file)
509 {
510 	struct drm_i915_gem_set_domain *args = data;
511 	struct drm_i915_gem_object *obj;
512 	u32 read_domains = args->read_domains;
513 	u32 write_domain = args->write_domain;
514 	int err;
515 
516 	if (IS_DGFX(to_i915(dev)))
517 		return -ENODEV;
518 
519 	/* Only handle setting domains to types used by the CPU. */
520 	if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
521 		return -EINVAL;
522 
523 	/*
524 	 * Having something in the write domain implies it's in the read
525 	 * domain, and only that read domain.  Enforce that in the request.
526 	 */
527 	if (write_domain && read_domains != write_domain)
528 		return -EINVAL;
529 
530 	if (!read_domains)
531 		return 0;
532 
533 	obj = i915_gem_object_lookup(file, args->handle);
534 	if (!obj)
535 		return -ENOENT;
536 
537 	/*
538 	 * Try to flush the object off the GPU without holding the lock.
539 	 * We will repeat the flush holding the lock in the normal manner
540 	 * to catch cases where we are gazumped.
541 	 */
542 	err = i915_gem_object_wait(obj,
543 				   I915_WAIT_INTERRUPTIBLE |
544 				   I915_WAIT_PRIORITY |
545 				   (write_domain ? I915_WAIT_ALL : 0),
546 				   MAX_SCHEDULE_TIMEOUT);
547 	if (err)
548 		goto out;
549 
550 	if (i915_gem_object_is_userptr(obj)) {
551 		/*
552 		 * Try to grab userptr pages, iris uses set_domain to check
553 		 * userptr validity
554 		 */
555 		err = i915_gem_object_userptr_validate(obj);
556 		if (!err)
557 			err = i915_gem_object_wait(obj,
558 						   I915_WAIT_INTERRUPTIBLE |
559 						   I915_WAIT_PRIORITY |
560 						   (write_domain ? I915_WAIT_ALL : 0),
561 						   MAX_SCHEDULE_TIMEOUT);
562 		goto out;
563 	}
564 
565 	/*
566 	 * Proxy objects do not control access to the backing storage, ergo
567 	 * they cannot be used as a means to manipulate the cache domain
568 	 * tracking for that backing storage. The proxy object is always
569 	 * considered to be outside of any cache domain.
570 	 */
571 	if (i915_gem_object_is_proxy(obj)) {
572 		err = -ENXIO;
573 		goto out;
574 	}
575 
576 	err = i915_gem_object_lock_interruptible(obj, NULL);
577 	if (err)
578 		goto out;
579 
580 	/*
581 	 * Flush and acquire obj->pages so that we are coherent through
582 	 * direct access in memory with previous cached writes through
583 	 * shmemfs and that our cache domain tracking remains valid.
584 	 * For example, if the obj->filp was moved to swap without us
585 	 * being notified and releasing the pages, we would mistakenly
586 	 * continue to assume that the obj remained out of the CPU cached
587 	 * domain.
588 	 */
589 	err = i915_gem_object_pin_pages(obj);
590 	if (err)
591 		goto out_unlock;
592 
593 	/*
594 	 * Already in the desired write domain? Nothing for us to do!
595 	 *
596 	 * We apply a little bit of cunning here to catch a broader set of
597 	 * no-ops. If obj->write_domain is set, we must be in the same
598 	 * obj->read_domains, and only that domain. Therefore, if that
599 	 * obj->write_domain matches the request read_domains, we are
600 	 * already in the same read/write domain and can skip the operation,
601 	 * without having to further check the requested write_domain.
602 	 */
603 	if (READ_ONCE(obj->write_domain) == read_domains)
604 		goto out_unpin;
605 
606 	if (read_domains & I915_GEM_DOMAIN_WC)
607 		err = i915_gem_object_set_to_wc_domain(obj, write_domain);
608 	else if (read_domains & I915_GEM_DOMAIN_GTT)
609 		err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
610 	else
611 		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
612 
613 out_unpin:
614 	i915_gem_object_unpin_pages(obj);
615 
616 out_unlock:
617 	i915_gem_object_unlock(obj);
618 
619 	if (!err && write_domain)
620 		i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
621 
622 out:
623 	i915_gem_object_put(obj);
624 	return err;
625 }
626 
627 /*
628  * Pins the specified object's pages and synchronizes the object with
629  * GPU accesses. Sets needs_clflush to non-zero if the caller should
630  * flush the object from the CPU cache.
631  */
i915_gem_object_prepare_read(struct drm_i915_gem_object * obj,unsigned int * needs_clflush)632 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
633 				 unsigned int *needs_clflush)
634 {
635 	int ret;
636 
637 	*needs_clflush = 0;
638 	if (!i915_gem_object_has_struct_page(obj))
639 		return -ENODEV;
640 
641 	assert_object_held(obj);
642 
643 	ret = i915_gem_object_wait(obj,
644 				   I915_WAIT_INTERRUPTIBLE,
645 				   MAX_SCHEDULE_TIMEOUT);
646 	if (ret)
647 		return ret;
648 
649 	ret = i915_gem_object_pin_pages(obj);
650 	if (ret)
651 		return ret;
652 
653 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
654 	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
655 		ret = i915_gem_object_set_to_cpu_domain(obj, false);
656 		if (ret)
657 			goto err_unpin;
658 		else
659 			goto out;
660 	}
661 
662 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
663 
664 	/* If we're not in the cpu read domain, set ourself into the gtt
665 	 * read domain and manually flush cachelines (if required). This
666 	 * optimizes for the case when the gpu will dirty the data
667 	 * anyway again before the next pread happens.
668 	 */
669 	if (!obj->cache_dirty &&
670 	    !(obj->read_domains & I915_GEM_DOMAIN_CPU))
671 		*needs_clflush = CLFLUSH_BEFORE;
672 
673 out:
674 	/* return with the pages pinned */
675 	return 0;
676 
677 err_unpin:
678 	i915_gem_object_unpin_pages(obj);
679 	return ret;
680 }
681 
i915_gem_object_prepare_write(struct drm_i915_gem_object * obj,unsigned int * needs_clflush)682 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
683 				  unsigned int *needs_clflush)
684 {
685 	int ret;
686 
687 	*needs_clflush = 0;
688 	if (!i915_gem_object_has_struct_page(obj))
689 		return -ENODEV;
690 
691 	assert_object_held(obj);
692 
693 	ret = i915_gem_object_wait(obj,
694 				   I915_WAIT_INTERRUPTIBLE |
695 				   I915_WAIT_ALL,
696 				   MAX_SCHEDULE_TIMEOUT);
697 	if (ret)
698 		return ret;
699 
700 	ret = i915_gem_object_pin_pages(obj);
701 	if (ret)
702 		return ret;
703 
704 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
705 	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
706 		ret = i915_gem_object_set_to_cpu_domain(obj, true);
707 		if (ret)
708 			goto err_unpin;
709 		else
710 			goto out;
711 	}
712 
713 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
714 
715 	/* If we're not in the cpu write domain, set ourself into the
716 	 * gtt write domain and manually flush cachelines (as required).
717 	 * This optimizes for the case when the gpu will use the data
718 	 * right away and we therefore have to clflush anyway.
719 	 */
720 	if (!obj->cache_dirty) {
721 		*needs_clflush |= CLFLUSH_AFTER;
722 
723 		/*
724 		 * Same trick applies to invalidate partially written
725 		 * cachelines read before writing.
726 		 */
727 		if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
728 			*needs_clflush |= CLFLUSH_BEFORE;
729 	}
730 
731 out:
732 	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
733 	obj->mm.dirty = true;
734 	/* return with the pages pinned */
735 	return 0;
736 
737 err_unpin:
738 	i915_gem_object_unpin_pages(obj);
739 	return ret;
740 }
741