1 /*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Dave Airlie
30 */
31 #include <linux/seq_file.h>
32 #include <linux/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/list.h>
35 #include <linux/kref.h>
36 #include <linux/slab.h>
37 #include "drmP.h"
38 #include "drm.h"
39 #include "radeon_reg.h"
40 #include "radeon.h"
41 #include "radeon_trace.h"
42
radeon_fence_write(struct radeon_device * rdev,u32 seq,int ring)43 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
44 {
45 if (rdev->wb.enabled) {
46 *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
47 } else {
48 WREG32(rdev->fence_drv[ring].scratch_reg, seq);
49 }
50 }
51
radeon_fence_read(struct radeon_device * rdev,int ring)52 static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
53 {
54 u32 seq = 0;
55
56 if (rdev->wb.enabled) {
57 seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
58 } else {
59 seq = RREG32(rdev->fence_drv[ring].scratch_reg);
60 }
61 return seq;
62 }
63
radeon_fence_emit(struct radeon_device * rdev,struct radeon_fence * fence)64 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
65 {
66 unsigned long irq_flags;
67
68 write_lock_irqsave(&rdev->fence_lock, irq_flags);
69 if (fence->emitted) {
70 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
71 return 0;
72 }
73 fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
74 if (!rdev->ring[fence->ring].ready)
75 /* FIXME: cp is not running assume everythings is done right
76 * away
77 */
78 radeon_fence_write(rdev, fence->seq, fence->ring);
79 else
80 radeon_fence_ring_emit(rdev, fence->ring, fence);
81
82 trace_radeon_fence_emit(rdev->ddev, fence->seq);
83 fence->emitted = true;
84 list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
85 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
86 return 0;
87 }
88
radeon_fence_poll_locked(struct radeon_device * rdev,int ring)89 static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
90 {
91 struct radeon_fence *fence;
92 struct list_head *i, *n;
93 uint32_t seq;
94 bool wake = false;
95 unsigned long cjiffies;
96
97 seq = radeon_fence_read(rdev, ring);
98 if (seq != rdev->fence_drv[ring].last_seq) {
99 rdev->fence_drv[ring].last_seq = seq;
100 rdev->fence_drv[ring].last_jiffies = jiffies;
101 rdev->fence_drv[ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
102 } else {
103 cjiffies = jiffies;
104 if (time_after(cjiffies, rdev->fence_drv[ring].last_jiffies)) {
105 cjiffies -= rdev->fence_drv[ring].last_jiffies;
106 if (time_after(rdev->fence_drv[ring].last_timeout, cjiffies)) {
107 /* update the timeout */
108 rdev->fence_drv[ring].last_timeout -= cjiffies;
109 } else {
110 /* the 500ms timeout is elapsed we should test
111 * for GPU lockup
112 */
113 rdev->fence_drv[ring].last_timeout = 1;
114 }
115 } else {
116 /* wrap around update last jiffies, we will just wait
117 * a little longer
118 */
119 rdev->fence_drv[ring].last_jiffies = cjiffies;
120 }
121 return false;
122 }
123 n = NULL;
124 list_for_each(i, &rdev->fence_drv[ring].emitted) {
125 fence = list_entry(i, struct radeon_fence, list);
126 if (fence->seq == seq) {
127 n = i;
128 break;
129 }
130 }
131 /* all fence previous to this one are considered as signaled */
132 if (n) {
133 i = n;
134 do {
135 n = i->prev;
136 list_move_tail(i, &rdev->fence_drv[ring].signaled);
137 fence = list_entry(i, struct radeon_fence, list);
138 fence->signaled = true;
139 i = n;
140 } while (i != &rdev->fence_drv[ring].emitted);
141 wake = true;
142 }
143 return wake;
144 }
145
radeon_fence_destroy(struct kref * kref)146 static void radeon_fence_destroy(struct kref *kref)
147 {
148 unsigned long irq_flags;
149 struct radeon_fence *fence;
150
151 fence = container_of(kref, struct radeon_fence, kref);
152 write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
153 list_del(&fence->list);
154 fence->emitted = false;
155 write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
156 if (fence->semaphore)
157 radeon_semaphore_free(fence->rdev, fence->semaphore);
158 kfree(fence);
159 }
160
radeon_fence_create(struct radeon_device * rdev,struct radeon_fence ** fence,int ring)161 int radeon_fence_create(struct radeon_device *rdev,
162 struct radeon_fence **fence,
163 int ring)
164 {
165 unsigned long irq_flags;
166
167 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
168 if ((*fence) == NULL) {
169 return -ENOMEM;
170 }
171 kref_init(&((*fence)->kref));
172 (*fence)->rdev = rdev;
173 (*fence)->emitted = false;
174 (*fence)->signaled = false;
175 (*fence)->seq = 0;
176 (*fence)->ring = ring;
177 (*fence)->semaphore = NULL;
178 INIT_LIST_HEAD(&(*fence)->list);
179
180 write_lock_irqsave(&rdev->fence_lock, irq_flags);
181 list_add_tail(&(*fence)->list, &rdev->fence_drv[ring].created);
182 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
183 return 0;
184 }
185
radeon_fence_signaled(struct radeon_fence * fence)186 bool radeon_fence_signaled(struct radeon_fence *fence)
187 {
188 unsigned long irq_flags;
189 bool signaled = false;
190
191 if (!fence)
192 return true;
193
194 if (fence->rdev->gpu_lockup)
195 return true;
196
197 write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
198 signaled = fence->signaled;
199 /* if we are shuting down report all fence as signaled */
200 if (fence->rdev->shutdown) {
201 signaled = true;
202 }
203 if (!fence->emitted) {
204 WARN(1, "Querying an unemitted fence : %p !\n", fence);
205 signaled = true;
206 }
207 if (!signaled) {
208 radeon_fence_poll_locked(fence->rdev, fence->ring);
209 signaled = fence->signaled;
210 }
211 write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
212 return signaled;
213 }
214
radeon_fence_wait(struct radeon_fence * fence,bool intr)215 int radeon_fence_wait(struct radeon_fence *fence, bool intr)
216 {
217 struct radeon_device *rdev;
218 unsigned long irq_flags, timeout;
219 u32 seq;
220 int r;
221
222 if (fence == NULL) {
223 WARN(1, "Querying an invalid fence : %p !\n", fence);
224 return 0;
225 }
226 rdev = fence->rdev;
227 if (radeon_fence_signaled(fence)) {
228 return 0;
229 }
230 timeout = rdev->fence_drv[fence->ring].last_timeout;
231 retry:
232 /* save current sequence used to check for GPU lockup */
233 seq = rdev->fence_drv[fence->ring].last_seq;
234 trace_radeon_fence_wait_begin(rdev->ddev, seq);
235 if (intr) {
236 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
237 r = wait_event_interruptible_timeout(rdev->fence_drv[fence->ring].queue,
238 radeon_fence_signaled(fence), timeout);
239 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
240 if (unlikely(r < 0)) {
241 return r;
242 }
243 } else {
244 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
245 r = wait_event_timeout(rdev->fence_drv[fence->ring].queue,
246 radeon_fence_signaled(fence), timeout);
247 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
248 }
249 trace_radeon_fence_wait_end(rdev->ddev, seq);
250 if (unlikely(!radeon_fence_signaled(fence))) {
251 /* we were interrupted for some reason and fence isn't
252 * isn't signaled yet, resume wait
253 */
254 if (r) {
255 timeout = r;
256 goto retry;
257 }
258 /* don't protect read access to rdev->fence_drv[t].last_seq
259 * if we experiencing a lockup the value doesn't change
260 */
261 if (seq == rdev->fence_drv[fence->ring].last_seq &&
262 radeon_gpu_is_lockup(rdev, &rdev->ring[fence->ring])) {
263 /* good news we believe it's a lockup */
264 printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
265 fence->seq, seq);
266 /* FIXME: what should we do ? marking everyone
267 * as signaled for now
268 */
269 rdev->gpu_lockup = true;
270 r = radeon_gpu_reset(rdev);
271 if (r)
272 return r;
273 radeon_fence_write(rdev, fence->seq, fence->ring);
274 rdev->gpu_lockup = false;
275 }
276 timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
277 write_lock_irqsave(&rdev->fence_lock, irq_flags);
278 rdev->fence_drv[fence->ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
279 rdev->fence_drv[fence->ring].last_jiffies = jiffies;
280 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
281 goto retry;
282 }
283 return 0;
284 }
285
radeon_fence_wait_next(struct radeon_device * rdev,int ring)286 int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
287 {
288 unsigned long irq_flags;
289 struct radeon_fence *fence;
290 int r;
291
292 if (rdev->gpu_lockup) {
293 return 0;
294 }
295 write_lock_irqsave(&rdev->fence_lock, irq_flags);
296 if (list_empty(&rdev->fence_drv[ring].emitted)) {
297 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
298 return 0;
299 }
300 fence = list_entry(rdev->fence_drv[ring].emitted.next,
301 struct radeon_fence, list);
302 radeon_fence_ref(fence);
303 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
304 r = radeon_fence_wait(fence, false);
305 radeon_fence_unref(&fence);
306 return r;
307 }
308
radeon_fence_wait_last(struct radeon_device * rdev,int ring)309 int radeon_fence_wait_last(struct radeon_device *rdev, int ring)
310 {
311 unsigned long irq_flags;
312 struct radeon_fence *fence;
313 int r;
314
315 if (rdev->gpu_lockup) {
316 return 0;
317 }
318 write_lock_irqsave(&rdev->fence_lock, irq_flags);
319 if (list_empty(&rdev->fence_drv[ring].emitted)) {
320 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
321 return 0;
322 }
323 fence = list_entry(rdev->fence_drv[ring].emitted.prev,
324 struct radeon_fence, list);
325 radeon_fence_ref(fence);
326 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
327 r = radeon_fence_wait(fence, false);
328 radeon_fence_unref(&fence);
329 return r;
330 }
331
radeon_fence_ref(struct radeon_fence * fence)332 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
333 {
334 kref_get(&fence->kref);
335 return fence;
336 }
337
radeon_fence_unref(struct radeon_fence ** fence)338 void radeon_fence_unref(struct radeon_fence **fence)
339 {
340 struct radeon_fence *tmp = *fence;
341
342 *fence = NULL;
343 if (tmp) {
344 kref_put(&tmp->kref, radeon_fence_destroy);
345 }
346 }
347
radeon_fence_process(struct radeon_device * rdev,int ring)348 void radeon_fence_process(struct radeon_device *rdev, int ring)
349 {
350 unsigned long irq_flags;
351 bool wake;
352
353 write_lock_irqsave(&rdev->fence_lock, irq_flags);
354 wake = radeon_fence_poll_locked(rdev, ring);
355 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
356 if (wake) {
357 wake_up_all(&rdev->fence_drv[ring].queue);
358 }
359 }
360
radeon_fence_count_emitted(struct radeon_device * rdev,int ring)361 int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
362 {
363 unsigned long irq_flags;
364 int not_processed = 0;
365
366 read_lock_irqsave(&rdev->fence_lock, irq_flags);
367 if (!rdev->fence_drv[ring].initialized) {
368 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
369 return 0;
370 }
371
372 if (!list_empty(&rdev->fence_drv[ring].emitted)) {
373 struct list_head *ptr;
374 list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
375 /* count up to 3, that's enought info */
376 if (++not_processed >= 3)
377 break;
378 }
379 }
380 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
381 return not_processed;
382 }
383
radeon_fence_driver_start_ring(struct radeon_device * rdev,int ring)384 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
385 {
386 unsigned long irq_flags;
387 uint64_t index;
388 int r;
389
390 write_lock_irqsave(&rdev->fence_lock, irq_flags);
391 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
392 if (rdev->wb.use_event) {
393 rdev->fence_drv[ring].scratch_reg = 0;
394 index = R600_WB_EVENT_OFFSET + ring * 4;
395 } else {
396 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
397 if (r) {
398 dev_err(rdev->dev, "fence failed to get scratch register\n");
399 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
400 return r;
401 }
402 index = RADEON_WB_SCRATCH_OFFSET +
403 rdev->fence_drv[ring].scratch_reg -
404 rdev->scratch.reg_base;
405 }
406 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
407 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
408 radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
409 rdev->fence_drv[ring].initialized = true;
410 DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
411 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
412 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
413 return 0;
414 }
415
radeon_fence_driver_init_ring(struct radeon_device * rdev,int ring)416 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
417 {
418 rdev->fence_drv[ring].scratch_reg = -1;
419 rdev->fence_drv[ring].cpu_addr = NULL;
420 rdev->fence_drv[ring].gpu_addr = 0;
421 atomic_set(&rdev->fence_drv[ring].seq, 0);
422 INIT_LIST_HEAD(&rdev->fence_drv[ring].created);
423 INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
424 INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
425 init_waitqueue_head(&rdev->fence_drv[ring].queue);
426 rdev->fence_drv[ring].initialized = false;
427 }
428
radeon_fence_driver_init(struct radeon_device * rdev)429 int radeon_fence_driver_init(struct radeon_device *rdev)
430 {
431 unsigned long irq_flags;
432 int ring;
433
434 write_lock_irqsave(&rdev->fence_lock, irq_flags);
435 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
436 radeon_fence_driver_init_ring(rdev, ring);
437 }
438 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
439 if (radeon_debugfs_fence_init(rdev)) {
440 dev_err(rdev->dev, "fence debugfs file creation failed\n");
441 }
442 return 0;
443 }
444
radeon_fence_driver_fini(struct radeon_device * rdev)445 void radeon_fence_driver_fini(struct radeon_device *rdev)
446 {
447 unsigned long irq_flags;
448 int ring;
449
450 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
451 if (!rdev->fence_drv[ring].initialized)
452 continue;
453 radeon_fence_wait_last(rdev, ring);
454 wake_up_all(&rdev->fence_drv[ring].queue);
455 write_lock_irqsave(&rdev->fence_lock, irq_flags);
456 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
457 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
458 rdev->fence_drv[ring].initialized = false;
459 }
460 }
461
462
463 /*
464 * Fence debugfs
465 */
466 #if defined(CONFIG_DEBUG_FS)
radeon_debugfs_fence_info(struct seq_file * m,void * data)467 static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
468 {
469 struct drm_info_node *node = (struct drm_info_node *)m->private;
470 struct drm_device *dev = node->minor->dev;
471 struct radeon_device *rdev = dev->dev_private;
472 struct radeon_fence *fence;
473 int i;
474
475 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
476 if (!rdev->fence_drv[i].initialized)
477 continue;
478
479 seq_printf(m, "--- ring %d ---\n", i);
480 seq_printf(m, "Last signaled fence 0x%08X\n",
481 radeon_fence_read(rdev, i));
482 if (!list_empty(&rdev->fence_drv[i].emitted)) {
483 fence = list_entry(rdev->fence_drv[i].emitted.prev,
484 struct radeon_fence, list);
485 seq_printf(m, "Last emitted fence %p with 0x%08X\n",
486 fence, fence->seq);
487 }
488 }
489 return 0;
490 }
491
492 static struct drm_info_list radeon_debugfs_fence_list[] = {
493 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
494 };
495 #endif
496
radeon_debugfs_fence_init(struct radeon_device * rdev)497 int radeon_debugfs_fence_init(struct radeon_device *rdev)
498 {
499 #if defined(CONFIG_DEBUG_FS)
500 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
501 #else
502 return 0;
503 #endif
504 }
505