54 	struct i915_sw_fence *fence;  member
61 static const char *i915_fence_get_driver_name(struct dma_fence *fence)  in i915_fence_get_driver_name()  argument
63 	return dev_name(to_request(fence)->i915->drm.dev);  in i915_fence_get_driver_name()
66 static const char *i915_fence_get_timeline_name(struct dma_fence *fence)  in i915_fence_get_timeline_name()  argument
79 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in i915_fence_get_timeline_name()
82 	ctx = i915_request_gem_context(to_request(fence));  in i915_fence_get_timeline_name()
89 static bool i915_fence_signaled(struct dma_fence *fence)  in i915_fence_signaled()  argument
91 	return i915_request_completed(to_request(fence));  in i915_fence_signaled()
94 static bool i915_fence_enable_signaling(struct dma_fence *fence)  in i915_fence_enable_signaling()  argument
96 	return i915_request_enable_breadcrumb(to_request(fence));  in i915_fence_enable_signaling()
99 static signed long i915_fence_wait(struct dma_fence *fence,  in i915_fence_wait()  argument
103 	return i915_request_wait_timeout(to_request(fence),  in i915_fence_wait()
113 static void i915_fence_release(struct dma_fence *fence)  in i915_fence_release()  argument
115 	struct i915_request *rq = to_request(fence);  in i915_fence_release()
187 	i915_sw_fence_complete(cb->fence);  in irq_execute_cb()
388 		dma_fence_signal_locked(&rq->fence);  in i915_request_retire()
392 	if (test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags))  in i915_request_retire()
491 			inflight = i915_seqno_passed(rq->fence.seqno,  in __request_in_flight()
492 						     signal->fence.seqno);  in __request_in_flight()
515 	cb->fence = &rq->submit;  in __await_execution()
516 	i915_sw_fence_await(cb->fence);  in __await_execution()
555 	GEM_BUG_ON(!fatal_error(rq->fence.error));  in __i915_request_skip()
560 	RQ_TRACE(rq, "error: %d\n", rq->fence.error);  in __i915_request_skip()
580 	old = READ_ONCE(rq->fence.error);  in i915_request_set_error_once()
584 	} while (!try_cmpxchg(&rq->fence.error, &old, error));  in i915_request_set_error_once()
639 	if (unlikely(fatal_error(request->fence.error)))  in __i915_request_submit()
673 	GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));  in __i915_request_submit()
676 	clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags);  in __i915_request_submit()
677 	set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);  in __i915_request_submit()
692 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))  in __i915_request_submit()
731 	GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));  in __i915_request_unsubmit()
732 	clear_bit_unlock(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);  in __i915_request_unsubmit()
733 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))  in __i915_request_unsubmit()
767 	set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);  in i915_request_cancel()
773 submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)  in submit_notify()  argument
776 		container_of(fence, typeof(*request), submit);  in submit_notify()
782 		if (unlikely(fence->error))  in submit_notify()
783 			i915_request_set_error_once(request, fence->error);  in submit_notify()
809 semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)  in semaphore_notify()  argument
811 	struct i915_request *rq = container_of(fence, typeof(*rq), semaphore);  in semaphore_notify()
956 	dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock,  in __i915_request_create()
1062 	struct dma_fence *fence;  in i915_request_await_start()  local
1079 	fence = NULL;  in i915_request_await_start()
1109 		fence = &prev->fence;  in i915_request_await_start()
1112 	if (!fence)  in i915_request_await_start()
1116 	if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence))  in i915_request_await_start()
1118 						    fence, 0,  in i915_request_await_start()
1120 	dma_fence_put(fence);  in i915_request_await_start()
1237 	if (__emit_semaphore_wait(to, from, from->fence.seqno))  in emit_semaphore_wait()
1245 					     &from->fence, 0,  in emit_semaphore_wait()
1250 					  struct dma_fence *fence)  in intel_timeline_sync_has_start()  argument
1253 					      fence->context,  in intel_timeline_sync_has_start()
1254 					      fence->seqno - 1);  in intel_timeline_sync_has_start()
1258 					 const struct dma_fence *fence)  in intel_timeline_sync_set_start()  argument
1260 	return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);  in intel_timeline_sync_set_start()
1278 					  &from->fence))  in __i915_request_await_execution()
1317 		err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);  in __i915_request_await_execution()
1332 					     &from->fence);  in __i915_request_await_execution()
1349 __i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)  in __i915_request_await_external()  argument
1352 	return i915_sw_fence_await_dma_fence(&rq->submit, fence,  in __i915_request_await_external()
1354 									fence->context),  in __i915_request_await_external()
1359 i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)  in i915_request_await_external()  argument
1364 	if (!to_dma_fence_chain(fence))  in i915_request_await_external()
1365 		return __i915_request_await_external(rq, fence);  in i915_request_await_external()
1367 	dma_fence_chain_for_each(iter, fence) {  in i915_request_await_external()
1370 		if (!dma_fence_is_i915(chain->fence)) {  in i915_request_await_external()
1375 		err = i915_request_await_dma_fence(rq, chain->fence);  in i915_request_await_external()
1405 			     struct dma_fence *fence)  in i915_request_await_execution()  argument
1407 	struct dma_fence **child = &fence;  in i915_request_await_execution()
1411 	if (dma_fence_is_array(fence)) {  in i915_request_await_execution()
1412 		struct dma_fence_array *array = to_dma_fence_array(fence);  in i915_request_await_execution()
1422 		fence = *child++;  in i915_request_await_execution()
1423 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in i915_request_await_execution()
1426 		if (fence->context == rq->fence.context)  in i915_request_await_execution()
1434 		if (dma_fence_is_i915(fence)) {  in i915_request_await_execution()
1435 			if (is_same_parallel_context(rq, to_request(fence)))  in i915_request_await_execution()
1438 							     to_request(fence));  in i915_request_await_execution()
1440 			ret = i915_request_await_external(rq, fence);  in i915_request_await_execution()
1477 		i915_sw_fence_set_error_once(&to->submit, from->fence.error);  in i915_request_await_request()
1501 i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)  in i915_request_await_dma_fence()  argument
1503 	struct dma_fence **child = &fence;  in i915_request_await_dma_fence()
1515 	if (dma_fence_is_array(fence)) {  in i915_request_await_dma_fence()
1516 		struct dma_fence_array *array = to_dma_fence_array(fence);  in i915_request_await_dma_fence()
1524 		fence = *child++;  in i915_request_await_dma_fence()
1525 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in i915_request_await_dma_fence()
1533 		if (fence->context == rq->fence.context)  in i915_request_await_dma_fence()
1537 		if (fence->context &&  in i915_request_await_dma_fence()
1539 						 fence))  in i915_request_await_dma_fence()
1542 		if (dma_fence_is_i915(fence)) {  in i915_request_await_dma_fence()
1543 			if (is_same_parallel_context(rq, to_request(fence)))  in i915_request_await_dma_fence()
1545 			ret = i915_request_await_request(rq, to_request(fence));  in i915_request_await_dma_fence()
1547 			ret = i915_request_await_external(rq, fence);  in i915_request_await_dma_fence()
1553 		if (fence->context)  in i915_request_await_dma_fence()
1555 						fence);  in i915_request_await_dma_fence()
1608 	struct dma_fence *fence;  in i915_request_await_object()  local
1612 				dma_resv_usage_rw(write), fence) {  in i915_request_await_object()
1613 		ret = i915_request_await_dma_fence(to, fence);  in i915_request_await_object()
1631 					     &huc->delayed_load.fence,  in i915_request_await_huc()
1667 						  &rq->fence));  in __i915_request_ensure_parallel_ordering()
1679 						  &rq->fence));  in __i915_request_ensure_ordering()
1694 			   i915_seqno_passed(prev->fence.seqno,  in __i915_request_ensure_ordering()
1695 					     rq->fence.seqno));  in __i915_request_ensure_ordering()
1703 							&prev->fence,  in __i915_request_ensure_ordering()
1777 	GEM_BUG_ON(timeline->seqno != rq->fence.seqno);  in __i915_request_add_to_timeline()
1935 		if (dma_fence_is_signaled(&rq->fence))  in __i915_spin_request()
1955 static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb)  in request_wait_wake()  argument
1992 	if (dma_fence_is_signaled(&rq->fence))  in i915_request_wait_timeout()
2051 	if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake))  in i915_request_wait_timeout()
2075 		if (dma_fence_is_signaled(&rq->fence))  in i915_request_wait_timeout()
2093 		dma_fence_remove_callback(&rq->fence, &wait.cb);  in i915_request_wait_timeout()
2174 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))  in fence_status()
2177 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))  in fence_status()
2188 	const char *name = rq->fence.ops->get_timeline_name((struct dma_fence *)&rq->fence);  in i915_request_show()
2227 		   rq->fence.context, rq->fence.seqno,  in i915_request_show()