1 /*
2  * Copyright 2012 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24 #include "priv.h"
25 #include "cgrp.h"
26 #include "chan.h"
27 #include "chid.h"
28 #include "runl.h"
29 
30 #include <core/ramht.h>
31 #include <subdev/timer.h>
32 
33 #include <nvif/class.h>
34 
35 void
nv50_eobj_ramht_del(struct nvkm_chan * chan,int hash)36 nv50_eobj_ramht_del(struct nvkm_chan *chan, int hash)
37 {
38 	nvkm_ramht_remove(chan->ramht, hash);
39 }
40 
41 int
nv50_eobj_ramht_add(struct nvkm_engn * engn,struct nvkm_object * eobj,struct nvkm_chan * chan)42 nv50_eobj_ramht_add(struct nvkm_engn *engn, struct nvkm_object *eobj, struct nvkm_chan *chan)
43 {
44 	return nvkm_ramht_insert(chan->ramht, eobj, 0, 4, eobj->handle, engn->id << 20);
45 }
46 
47 void
nv50_chan_stop(struct nvkm_chan * chan)48 nv50_chan_stop(struct nvkm_chan *chan)
49 {
50 	struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;
51 
52 	nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x00000000);
53 }
54 
55 void
nv50_chan_start(struct nvkm_chan * chan)56 nv50_chan_start(struct nvkm_chan *chan)
57 {
58 	struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;
59 
60 	nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x80000000);
61 }
62 
63 void
nv50_chan_unbind(struct nvkm_chan * chan)64 nv50_chan_unbind(struct nvkm_chan *chan)
65 {
66 	struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;
67 
68 	nvkm_wr32(device, 0x002600 + (chan->id * 4), 0x00000000);
69 }
70 
71 static void
nv50_chan_bind(struct nvkm_chan * chan)72 nv50_chan_bind(struct nvkm_chan *chan)
73 {
74 	struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;
75 
76 	nvkm_wr32(device, 0x002600 + (chan->id * 4), chan->ramfc->addr >> 12);
77 }
78 
79 static int
nv50_chan_ramfc_write(struct nvkm_chan * chan,u64 offset,u64 length,u32 devm,bool priv)80 nv50_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv)
81 {
82 	struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;
83 	const u32 limit2 = ilog2(length / 8);
84 	int ret;
85 
86 	ret = nvkm_gpuobj_new(device, 0x0200, 0x1000, true, chan->inst, &chan->ramfc);
87 	if (ret)
88 		return ret;
89 
90 	ret = nvkm_gpuobj_new(device, 0x1200, 0, true, chan->inst, &chan->eng);
91 	if (ret)
92 		return ret;
93 
94 	ret = nvkm_gpuobj_new(device, 0x4000, 0, false, chan->inst, &chan->pgd);
95 	if (ret)
96 		return ret;
97 
98 	ret = nvkm_ramht_new(device, 0x8000, 16, chan->inst, &chan->ramht);
99 	if (ret)
100 		return ret;
101 
102 	nvkm_kmap(chan->ramfc);
103 	nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078);
104 	nvkm_wo32(chan->ramfc, 0x44, 0x01003fff);
105 	nvkm_wo32(chan->ramfc, 0x48, chan->push->node->offset >> 4);
106 	nvkm_wo32(chan->ramfc, 0x50, lower_32_bits(offset));
107 	nvkm_wo32(chan->ramfc, 0x54, upper_32_bits(offset) | (limit2 << 16));
108 	nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff);
109 	nvkm_wo32(chan->ramfc, 0x78, 0x00000000);
110 	nvkm_wo32(chan->ramfc, 0x7c, 0x30000000 | devm);
111 	nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
112 				     (4 << 24) /* SEARCH_FULL */ |
113 				     (chan->ramht->gpuobj->node->offset >> 4));
114 	nvkm_done(chan->ramfc);
115 	return 0;
116 }
117 
118 static const struct nvkm_chan_func_ramfc
119 nv50_chan_ramfc = {
120 	.write = nv50_chan_ramfc_write,
121 	.ctxdma = true,
122 	.devm = 0xfff,
123 };
124 
125 const struct nvkm_chan_func_userd
126 nv50_chan_userd = {
127 	.bar = 0,
128 	.base = 0xc00000,
129 	.size = 0x002000,
130 };
131 
132 const struct nvkm_chan_func_inst
133 nv50_chan_inst = {
134 	.size = 0x10000,
135 	.vmm = true,
136 };
137 
138 static const struct nvkm_chan_func
139 nv50_chan = {
140 	.inst = &nv50_chan_inst,
141 	.userd = &nv50_chan_userd,
142 	.ramfc = &nv50_chan_ramfc,
143 	.bind = nv50_chan_bind,
144 	.unbind = nv50_chan_unbind,
145 	.start = nv50_chan_start,
146 	.stop = nv50_chan_stop,
147 };
148 
149 static void
nv50_ectx_bind(struct nvkm_engn * engn,struct nvkm_cctx * cctx,struct nvkm_chan * chan)150 nv50_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan)
151 {
152 	struct nvkm_subdev *subdev = &chan->cgrp->runl->fifo->engine.subdev;
153 	struct nvkm_device *device = subdev->device;
154 	u64 start = 0, limit = 0;
155 	u32 flags = 0, ptr0, save;
156 
157 	switch (engn->engine->subdev.type) {
158 	case NVKM_ENGINE_GR    : ptr0 = 0x0000; break;
159 	case NVKM_ENGINE_MPEG  : ptr0 = 0x0060; break;
160 	default:
161 		WARN_ON(1);
162 		return;
163 	}
164 
165 	if (!cctx) {
166 		/* HW bug workaround:
167 		 *
168 		 * PFIFO will hang forever if the connected engines don't report
169 		 * that they've processed the context switch request.
170 		 *
171 		 * In order for the kickoff to work, we need to ensure all the
172 		 * connected engines are in a state where they can answer.
173 		 *
174 		 * Newer chipsets don't seem to suffer from this issue, and well,
175 		 * there's also a "ignore these engines" bitmask reg we can use
176 		 * if we hit the issue there..
177 		 */
178 		save = nvkm_mask(device, 0x00b860, 0x00000001, 0x00000001);
179 
180 		/* Tell engines to save out contexts. */
181 		nvkm_wr32(device, 0x0032fc, chan->inst->addr >> 12);
182 		nvkm_msec(device, 2000,
183 			if (nvkm_rd32(device, 0x0032fc) != 0xffffffff)
184 				break;
185 		);
186 		nvkm_wr32(device, 0x00b860, save);
187 	} else {
188 		flags = 0x00190000;
189 		start = cctx->vctx->inst->addr;
190 		limit = start + cctx->vctx->inst->size - 1;
191 	}
192 
193 	nvkm_kmap(chan->eng);
194 	nvkm_wo32(chan->eng, ptr0 + 0x00, flags);
195 	nvkm_wo32(chan->eng, ptr0 + 0x04, lower_32_bits(limit));
196 	nvkm_wo32(chan->eng, ptr0 + 0x08, lower_32_bits(start));
197 	nvkm_wo32(chan->eng, ptr0 + 0x0c, upper_32_bits(limit) << 24 |
198 					  lower_32_bits(start));
199 	nvkm_wo32(chan->eng, ptr0 + 0x10, 0x00000000);
200 	nvkm_wo32(chan->eng, ptr0 + 0x14, 0x00000000);
201 	nvkm_done(chan->eng);
202 }
203 
204 static const struct nvkm_engn_func
205 nv50_engn = {
206 	.bind = nv50_ectx_bind,
207 	.ramht_add = nv50_eobj_ramht_add,
208 	.ramht_del = nv50_eobj_ramht_del,
209 };
210 
211 const struct nvkm_engn_func
212 nv50_engn_sw = {
213 	.ramht_add = nv50_eobj_ramht_add,
214 	.ramht_del = nv50_eobj_ramht_del,
215 };
216 
217 static bool
nv50_runl_pending(struct nvkm_runl * runl)218 nv50_runl_pending(struct nvkm_runl *runl)
219 {
220 	return nvkm_rd32(runl->fifo->engine.subdev.device, 0x0032ec) & 0x00000100;
221 }
222 
223 int
nv50_runl_wait(struct nvkm_runl * runl)224 nv50_runl_wait(struct nvkm_runl *runl)
225 {
226 	struct nvkm_fifo *fifo = runl->fifo;
227 
228 	nvkm_msec(fifo->engine.subdev.device, fifo->timeout.chan_msec,
229 		if (!nvkm_runl_update_pending(runl))
230 			return 0;
231 		usleep_range(1, 2);
232 	);
233 
234 	return -ETIMEDOUT;
235 }
236 
237 static void
nv50_runl_commit(struct nvkm_runl * runl,struct nvkm_memory * memory,u32 start,int count)238 nv50_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count)
239 {
240 	struct nvkm_device *device = runl->fifo->engine.subdev.device;
241 	u64 addr = nvkm_memory_addr(memory) + start;
242 
243 	nvkm_wr32(device, 0x0032f4, addr >> 12);
244 	nvkm_wr32(device, 0x0032ec, count);
245 }
246 
247 static void
nv50_runl_insert_chan(struct nvkm_chan * chan,struct nvkm_memory * memory,u64 offset)248 nv50_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset)
249 {
250 	nvkm_wo32(memory, offset, chan->id);
251 }
252 
253 static struct nvkm_memory *
nv50_runl_alloc(struct nvkm_runl * runl,u32 * offset)254 nv50_runl_alloc(struct nvkm_runl *runl, u32 *offset)
255 {
256 	const u32 segment = ALIGN((runl->cgrp_nr + runl->chan_nr) * runl->func->size, 0x1000);
257 	const u32 maxsize = (runl->cgid ? runl->cgid->nr : 0) + runl->chid->nr;
258 	int ret;
259 
260 	if (unlikely(!runl->mem)) {
261 		ret = nvkm_memory_new(runl->fifo->engine.subdev.device, NVKM_MEM_TARGET_INST,
262 				      maxsize * 2 * runl->func->size, 0, false, &runl->mem);
263 		if (ret) {
264 			RUNL_ERROR(runl, "alloc %d\n", ret);
265 			return ERR_PTR(ret);
266 		}
267 	} else {
268 		if (runl->offset + segment >= nvkm_memory_size(runl->mem)) {
269 			ret = runl->func->wait(runl);
270 			if (ret) {
271 				RUNL_DEBUG(runl, "rewind timeout");
272 				return ERR_PTR(ret);
273 			}
274 
275 			runl->offset = 0;
276 		}
277 	}
278 
279 	*offset = runl->offset;
280 	runl->offset += segment;
281 	return runl->mem;
282 }
283 
284 int
nv50_runl_update(struct nvkm_runl * runl)285 nv50_runl_update(struct nvkm_runl *runl)
286 {
287 	struct nvkm_memory *memory;
288 	struct nvkm_cgrp *cgrp;
289 	struct nvkm_chan *chan;
290 	u32 start, offset, count;
291 
292 	/*TODO: prio, interleaving. */
293 
294 	RUNL_TRACE(runl, "RAMRL: update cgrps:%d chans:%d", runl->cgrp_nr, runl->chan_nr);
295 	memory = nv50_runl_alloc(runl, &start);
296 	if (IS_ERR(memory))
297 		return PTR_ERR(memory);
298 
299 	RUNL_TRACE(runl, "RAMRL: update start:%08x", start);
300 	offset = start;
301 
302 	nvkm_kmap(memory);
303 	nvkm_runl_foreach_cgrp(cgrp, runl) {
304 		if (cgrp->hw) {
305 			CGRP_TRACE(cgrp, "     RAMRL+%08x: chans:%d", offset, cgrp->chan_nr);
306 			runl->func->insert_cgrp(cgrp, memory, offset);
307 			offset += runl->func->size;
308 		}
309 
310 		nvkm_cgrp_foreach_chan(chan, cgrp) {
311 			CHAN_TRACE(chan, "RAMRL+%08x: [%s]", offset, chan->name);
312 			runl->func->insert_chan(chan, memory, offset);
313 			offset += runl->func->size;
314 		}
315 	}
316 	nvkm_done(memory);
317 
318 	/*TODO: look into using features on newer HW to guarantee forward progress. */
319 	list_rotate_left(&runl->cgrps);
320 
321 	count = (offset - start) / runl->func->size;
322 	RUNL_TRACE(runl, "RAMRL: commit start:%08x count:%d", start, count);
323 
324 	runl->func->commit(runl, memory, start, count);
325 	return 0;
326 }
327 
328 const struct nvkm_runl_func
329 nv50_runl = {
330 	.size = 4,
331 	.update = nv50_runl_update,
332 	.insert_chan = nv50_runl_insert_chan,
333 	.commit = nv50_runl_commit,
334 	.wait = nv50_runl_wait,
335 	.pending = nv50_runl_pending,
336 };
337 
338 void
nv50_fifo_init(struct nvkm_fifo * fifo)339 nv50_fifo_init(struct nvkm_fifo *fifo)
340 {
341 	struct nvkm_runl *runl = nvkm_runl_first(fifo);
342 	struct nvkm_device *device = fifo->engine.subdev.device;
343 	int i;
344 
345 	nvkm_mask(device, 0x000200, 0x00000100, 0x00000000);
346 	nvkm_mask(device, 0x000200, 0x00000100, 0x00000100);
347 	nvkm_wr32(device, 0x00250c, 0x6f3cfc34);
348 	nvkm_wr32(device, 0x002044, 0x01003fff);
349 
350 	nvkm_wr32(device, 0x002100, 0xffffffff);
351 	nvkm_wr32(device, 0x002140, 0xbfffffff);
352 
353 	for (i = 0; i < 128; i++)
354 		nvkm_wr32(device, 0x002600 + (i * 4), 0x00000000);
355 
356 	atomic_set(&runl->changed, 1);
357 	runl->func->update(runl);
358 
359 	nvkm_wr32(device, 0x003200, 0x00000001);
360 	nvkm_wr32(device, 0x003250, 0x00000001);
361 	nvkm_wr32(device, 0x002500, 0x00000001);
362 }
363 
364 int
nv50_fifo_chid_ctor(struct nvkm_fifo * fifo,int nr)365 nv50_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr)
366 {
367 	/* CHID 0 is unusable (some kind of PIO channel?), 127 is "channel invalid". */
368 	return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 1, nr - 2, &fifo->chid);
369 }
370 
371 int
nv50_fifo_chid_nr(struct nvkm_fifo * fifo)372 nv50_fifo_chid_nr(struct nvkm_fifo *fifo)
373 {
374 	return 128;
375 }
376 
377 static const struct nvkm_fifo_func
378 nv50_fifo = {
379 	.chid_nr = nv50_fifo_chid_nr,
380 	.chid_ctor = nv50_fifo_chid_ctor,
381 	.runl_ctor = nv04_fifo_runl_ctor,
382 	.init = nv50_fifo_init,
383 	.intr = nv04_fifo_intr,
384 	.pause = nv04_fifo_pause,
385 	.start = nv04_fifo_start,
386 	.runl = &nv50_runl,
387 	.engn = &nv50_engn,
388 	.engn_sw = &nv50_engn_sw,
389 	.cgrp = {{                           }, &nv04_cgrp },
390 	.chan = {{ 0, 0, NV50_CHANNEL_GPFIFO }, &nv50_chan },
391 };
392 
393 int
nv50_fifo_new(struct nvkm_device * device,enum nvkm_subdev_type type,int inst,struct nvkm_fifo ** pfifo)394 nv50_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
395 	      struct nvkm_fifo **pfifo)
396 {
397 	return nvkm_fifo_new_(&nv50_fifo, device, type, inst, pfifo);
398 }
399