1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Tegra host1x Command DMA
4 *
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 */
7
8 #include <linux/slab.h>
9 #include <linux/scatterlist.h>
10 #include <linux/dma-mapping.h>
11
12 #include "../cdma.h"
13 #include "../channel.h"
14 #include "../dev.h"
15 #include "../debug.h"
16
17 /*
18 * Put the restart at the end of pushbuffer memory
19 */
push_buffer_init(struct push_buffer * pb)20 static void push_buffer_init(struct push_buffer *pb)
21 {
22 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
23 }
24
25 /*
26 * Increment timedout buffer's syncpt via CPU.
27 */
cdma_timeout_cpu_incr(struct host1x_cdma * cdma,u32 getptr,u32 syncpt_incrs,u32 syncval,u32 nr_slots)28 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
29 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
30 {
31 unsigned int i;
32
33 for (i = 0; i < syncpt_incrs; i++)
34 host1x_syncpt_incr(cdma->timeout.syncpt);
35
36 /* after CPU incr, ensure shadow is up to date */
37 host1x_syncpt_load(cdma->timeout.syncpt);
38 }
39
40 /*
41 * Start channel DMA
42 */
cdma_start(struct host1x_cdma * cdma)43 static void cdma_start(struct host1x_cdma *cdma)
44 {
45 struct host1x_channel *ch = cdma_to_channel(cdma);
46 u64 start, end;
47
48 if (cdma->running)
49 return;
50
51 cdma->last_pos = cdma->push_buffer.pos;
52 start = cdma->push_buffer.dma;
53 end = cdma->push_buffer.size + 4;
54
55 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
56 HOST1X_CHANNEL_DMACTRL);
57
58 /* set base, put and end pointer */
59 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
60 #if HOST1X_HW >= 6
61 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
62 #endif
63 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
64 #if HOST1X_HW >= 6
65 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);
66 #endif
67 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
68 #if HOST1X_HW >= 6
69 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
70 #endif
71
72 /* reset GET */
73 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
74 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
75 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
76 HOST1X_CHANNEL_DMACTRL);
77
78 /* start the command DMA */
79 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
80
81 cdma->running = true;
82 }
83
84 /*
85 * Similar to cdma_start(), but rather than starting from an idle
86 * state (where DMA GET is set to DMA PUT), on a timeout we restore
87 * DMA GET from an explicit value (so DMA may again be pending).
88 */
cdma_timeout_restart(struct host1x_cdma * cdma,u32 getptr)89 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
90 {
91 struct host1x *host1x = cdma_to_host1x(cdma);
92 struct host1x_channel *ch = cdma_to_channel(cdma);
93 u64 start, end;
94
95 if (cdma->running)
96 return;
97
98 cdma->last_pos = cdma->push_buffer.pos;
99
100 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
101 HOST1X_CHANNEL_DMACTRL);
102
103 start = cdma->push_buffer.dma;
104 end = cdma->push_buffer.size + 4;
105
106 /* set base, end pointer (all of memory) */
107 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
108 #if HOST1X_HW >= 6
109 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
110 #endif
111 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
112 #if HOST1X_HW >= 6
113 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
114 #endif
115
116 /* set GET, by loading the value in PUT (then reset GET) */
117 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
118 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
119 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
120 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
121 HOST1X_CHANNEL_DMACTRL);
122
123 dev_dbg(host1x->dev,
124 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
125 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
126 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
127 cdma->last_pos);
128
129 /* deassert GET reset and set PUT */
130 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
131 HOST1X_CHANNEL_DMACTRL);
132 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
133
134 /* start the command DMA */
135 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
136
137 cdma->running = true;
138 }
139
140 /*
141 * Kick channel DMA into action by writing its PUT offset (if it has changed)
142 */
cdma_flush(struct host1x_cdma * cdma)143 static void cdma_flush(struct host1x_cdma *cdma)
144 {
145 struct host1x_channel *ch = cdma_to_channel(cdma);
146
147 if (cdma->push_buffer.pos != cdma->last_pos) {
148 host1x_ch_writel(ch, cdma->push_buffer.pos,
149 HOST1X_CHANNEL_DMAPUT);
150 cdma->last_pos = cdma->push_buffer.pos;
151 }
152 }
153
cdma_stop(struct host1x_cdma * cdma)154 static void cdma_stop(struct host1x_cdma *cdma)
155 {
156 struct host1x_channel *ch = cdma_to_channel(cdma);
157
158 mutex_lock(&cdma->lock);
159
160 if (cdma->running) {
161 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
162 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
163 HOST1X_CHANNEL_DMACTRL);
164 cdma->running = false;
165 }
166
167 mutex_unlock(&cdma->lock);
168 }
169
cdma_hw_cmdproc_stop(struct host1x * host,struct host1x_channel * ch,bool stop)170 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
171 bool stop)
172 {
173 #if HOST1X_HW >= 6
174 host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
175 #else
176 u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
177 if (stop)
178 cmdproc_stop |= BIT(ch->id);
179 else
180 cmdproc_stop &= ~BIT(ch->id);
181 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
182 #endif
183 }
184
cdma_hw_teardown(struct host1x * host,struct host1x_channel * ch)185 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
186 {
187 #if HOST1X_HW >= 6
188 host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
189 #else
190 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
191 #endif
192 }
193
194 /*
195 * Stops both channel's command processor and CDMA immediately.
196 * Also, tears down the channel and resets corresponding module.
197 */
cdma_freeze(struct host1x_cdma * cdma)198 static void cdma_freeze(struct host1x_cdma *cdma)
199 {
200 struct host1x *host = cdma_to_host1x(cdma);
201 struct host1x_channel *ch = cdma_to_channel(cdma);
202
203 if (cdma->torndown && !cdma->running) {
204 dev_warn(host->dev, "Already torn down\n");
205 return;
206 }
207
208 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
209
210 cdma_hw_cmdproc_stop(host, ch, true);
211
212 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
213 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
214 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
215 cdma->last_pos);
216
217 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
218 HOST1X_CHANNEL_DMACTRL);
219
220 cdma_hw_teardown(host, ch);
221
222 cdma->running = false;
223 cdma->torndown = true;
224 }
225
cdma_resume(struct host1x_cdma * cdma,u32 getptr)226 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
227 {
228 struct host1x *host1x = cdma_to_host1x(cdma);
229 struct host1x_channel *ch = cdma_to_channel(cdma);
230
231 dev_dbg(host1x->dev,
232 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
233 ch->id, getptr);
234
235 cdma_hw_cmdproc_stop(host1x, ch, false);
236
237 cdma->torndown = false;
238 cdma_timeout_restart(cdma, getptr);
239 }
240
timeout_release_mlock(struct host1x_cdma * cdma)241 static void timeout_release_mlock(struct host1x_cdma *cdma)
242 {
243 #if HOST1X_HW >= 8
244 /* Tegra186 and Tegra194 require a more complicated MLOCK release
245 * sequence. Furthermore, those chips by default don't enforce MLOCKs,
246 * so it turns out that if we don't /actually/ need MLOCKs, we can just
247 * ignore them.
248 *
249 * As such, for now just implement this on Tegra234 where things are
250 * stricter but also easy to implement.
251 */
252 struct host1x_channel *ch = cdma_to_channel(cdma);
253 struct host1x *host1x = cdma_to_host1x(cdma);
254 u32 offset;
255
256 switch (ch->client->class) {
257 case HOST1X_CLASS_VIC:
258 offset = HOST1X_COMMON_VIC_MLOCK;
259 break;
260 case HOST1X_CLASS_NVDEC:
261 offset = HOST1X_COMMON_NVDEC_MLOCK;
262 break;
263 default:
264 WARN(1, "%s was not updated for class %u", __func__, ch->client->class);
265 return;
266 }
267
268 host1x_common_writel(host1x, 0x0, offset);
269 #endif
270 }
271
272 /*
273 * If this timeout fires, it indicates the current sync_queue entry has
274 * exceeded its TTL and the userctx should be timed out and remaining
275 * submits already issued cleaned up (future submits return an error).
276 */
cdma_timeout_handler(struct work_struct * work)277 static void cdma_timeout_handler(struct work_struct *work)
278 {
279 u32 syncpt_val;
280 struct host1x_cdma *cdma;
281 struct host1x *host1x;
282 struct host1x_channel *ch;
283
284 cdma = container_of(to_delayed_work(work), struct host1x_cdma,
285 timeout.wq);
286 host1x = cdma_to_host1x(cdma);
287 ch = cdma_to_channel(cdma);
288
289 host1x_debug_dump(cdma_to_host1x(cdma));
290
291 mutex_lock(&cdma->lock);
292
293 if (!cdma->timeout.client) {
294 dev_dbg(host1x->dev,
295 "cdma_timeout: expired, but has no clientid\n");
296 mutex_unlock(&cdma->lock);
297 return;
298 }
299
300 /* stop processing to get a clean snapshot */
301 cdma_hw_cmdproc_stop(host1x, ch, true);
302
303 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
304
305 /* has buffer actually completed? */
306 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
307 dev_dbg(host1x->dev,
308 "cdma_timeout: expired, but buffer had completed\n");
309 /* restore */
310 cdma_hw_cmdproc_stop(host1x, ch, false);
311 mutex_unlock(&cdma->lock);
312 return;
313 }
314
315 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
316 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
317 syncpt_val, cdma->timeout.syncpt_val);
318
319 /* stop HW, resetting channel/module */
320 host1x_hw_cdma_freeze(host1x, cdma);
321
322 /* release any held MLOCK */
323 timeout_release_mlock(cdma);
324
325 host1x_cdma_update_sync_queue(cdma, ch->dev);
326 mutex_unlock(&cdma->lock);
327 }
328
329 /*
330 * Init timeout resources
331 */
cdma_timeout_init(struct host1x_cdma * cdma)332 static int cdma_timeout_init(struct host1x_cdma *cdma)
333 {
334 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
335 cdma->timeout.initialized = true;
336
337 return 0;
338 }
339
340 /*
341 * Clean up timeout resources
342 */
cdma_timeout_destroy(struct host1x_cdma * cdma)343 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
344 {
345 if (cdma->timeout.initialized)
346 cancel_delayed_work(&cdma->timeout.wq);
347
348 cdma->timeout.initialized = false;
349 }
350
351 static const struct host1x_cdma_ops host1x_cdma_ops = {
352 .start = cdma_start,
353 .stop = cdma_stop,
354 .flush = cdma_flush,
355
356 .timeout_init = cdma_timeout_init,
357 .timeout_destroy = cdma_timeout_destroy,
358 .freeze = cdma_freeze,
359 .resume = cdma_resume,
360 .timeout_cpu_incr = cdma_timeout_cpu_incr,
361 };
362
363 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
364 .init = push_buffer_init,
365 };
366