1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_vcn.h"
27 #include "amdgpu_pm.h"
28 #include "amdgpu_cs.h"
29 #include "soc15.h"
30 #include "soc15d.h"
31 #include "vcn_v2_0.h"
32 #include "mmsch_v3_0.h"
33 #include "vcn_sw_ring.h"
34 
35 #include "vcn/vcn_3_0_0_offset.h"
36 #include "vcn/vcn_3_0_0_sh_mask.h"
37 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
38 
39 #include <drm/drm_drv.h>
40 
41 #define VCN_VID_SOC_ADDRESS_2_0					0x1fa00
42 #define VCN1_VID_SOC_ADDRESS_3_0				0x48200
43 
44 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET			0x27
45 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET			0x0f
46 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET			0x10
47 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET			0x11
48 #define mmUVD_NO_OP_INTERNAL_OFFSET				0x29
49 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET			0x66
50 #define mmUVD_SCRATCH9_INTERNAL_OFFSET				0xc01d
51 
52 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET			0x431
53 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET		0x3b4
54 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET		0x3b5
55 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET			0x25c
56 
57 #define VCN_INSTANCES_SIENNA_CICHLID				2
58 #define DEC_SW_RING_ENABLED					FALSE
59 
60 #define RDECODE_MSG_CREATE					0x00000000
61 #define RDECODE_MESSAGE_CREATE					0x00000001
62 
63 static int amdgpu_ih_clientid_vcns[] = {
64 	SOC15_IH_CLIENTID_VCN,
65 	SOC15_IH_CLIENTID_VCN1
66 };
67 
68 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
69 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
70 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
71 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
72 static int vcn_v3_0_set_powergating_state(void *handle,
73 			enum amd_powergating_state state);
74 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
75 			int inst_idx, struct dpg_pause_state *new_state);
76 
77 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
78 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
79 
80 /**
81  * vcn_v3_0_early_init - set function pointers and load microcode
82  *
83  * @handle: amdgpu_device pointer
84  *
85  * Set ring and irq function pointers
86  * Load microcode from filesystem
87  */
vcn_v3_0_early_init(void * handle)88 static int vcn_v3_0_early_init(void *handle)
89 {
90 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
91 
92 	if (amdgpu_sriov_vf(adev)) {
93 		adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
94 		adev->vcn.harvest_config = 0;
95 		adev->vcn.num_enc_rings = 1;
96 
97 	} else {
98 		if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
99 						 AMDGPU_VCN_HARVEST_VCN1))
100 			/* both instances are harvested, disable the block */
101 			return -ENOENT;
102 
103 		if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 0, 33))
104 			adev->vcn.num_enc_rings = 0;
105 		else
106 			adev->vcn.num_enc_rings = 2;
107 	}
108 
109 	vcn_v3_0_set_dec_ring_funcs(adev);
110 	vcn_v3_0_set_enc_ring_funcs(adev);
111 	vcn_v3_0_set_irq_funcs(adev);
112 
113 	return amdgpu_vcn_early_init(adev);
114 }
115 
116 /**
117  * vcn_v3_0_sw_init - sw init for VCN block
118  *
119  * @handle: amdgpu_device pointer
120  *
121  * Load firmware and sw initialization
122  */
vcn_v3_0_sw_init(void * handle)123 static int vcn_v3_0_sw_init(void *handle)
124 {
125 	struct amdgpu_ring *ring;
126 	int i, j, r;
127 	int vcn_doorbell_index = 0;
128 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
129 
130 	r = amdgpu_vcn_sw_init(adev);
131 	if (r)
132 		return r;
133 
134 	amdgpu_vcn_setup_ucode(adev);
135 
136 	r = amdgpu_vcn_resume(adev);
137 	if (r)
138 		return r;
139 
140 	/*
141 	 * Note: doorbell assignment is fixed for SRIOV multiple VCN engines
142 	 * Formula:
143 	 *   vcn_db_base  = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
144 	 *   dec_ring_i   = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)
145 	 *   enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j
146 	 */
147 	if (amdgpu_sriov_vf(adev)) {
148 		vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
149 		/* get DWORD offset */
150 		vcn_doorbell_index = vcn_doorbell_index << 1;
151 	}
152 
153 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
154 		volatile struct amdgpu_fw_shared *fw_shared;
155 
156 		if (adev->vcn.harvest_config & (1 << i))
157 			continue;
158 
159 		adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
160 		adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
161 		adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
162 		adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
163 		adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
164 		adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
165 
166 		adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
167 		adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
168 		adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
169 		adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
170 		adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
171 		adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
172 		adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
173 		adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
174 		adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
175 		adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
176 
177 		/* VCN DEC TRAP */
178 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
179 				VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);
180 		if (r)
181 			return r;
182 
183 		atomic_set(&adev->vcn.inst[i].sched_score, 0);
184 
185 		ring = &adev->vcn.inst[i].ring_dec;
186 		ring->use_doorbell = true;
187 		if (amdgpu_sriov_vf(adev)) {
188 			ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);
189 		} else {
190 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
191 		}
192 		ring->vm_hub = AMDGPU_MMHUB0(0);
193 		sprintf(ring->name, "vcn_dec_%d", i);
194 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
195 				     AMDGPU_RING_PRIO_DEFAULT,
196 				     &adev->vcn.inst[i].sched_score);
197 		if (r)
198 			return r;
199 
200 		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
201 			enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(j);
202 
203 			/* VCN ENC TRAP */
204 			r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
205 				j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
206 			if (r)
207 				return r;
208 
209 			ring = &adev->vcn.inst[i].ring_enc[j];
210 			ring->use_doorbell = true;
211 			if (amdgpu_sriov_vf(adev)) {
212 				ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;
213 			} else {
214 				ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
215 			}
216 			ring->vm_hub = AMDGPU_MMHUB0(0);
217 			sprintf(ring->name, "vcn_enc_%d.%d", i, j);
218 			r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
219 					     hw_prio, &adev->vcn.inst[i].sched_score);
220 			if (r)
221 				return r;
222 		}
223 
224 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
225 		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
226 					     cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG) |
227 					     cpu_to_le32(AMDGPU_VCN_FW_SHARED_FLAG_0_RB);
228 		fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
229 		fw_shared->present_flag_0 |= AMDGPU_VCN_SMU_VERSION_INFO_FLAG;
230 		if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 1, 2))
231 			fw_shared->smu_interface_info.smu_interface_type = 2;
232 		else if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 1, 1))
233 			fw_shared->smu_interface_info.smu_interface_type = 1;
234 
235 		if (amdgpu_vcnfw_log)
236 			amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
237 	}
238 
239 	if (amdgpu_sriov_vf(adev)) {
240 		r = amdgpu_virt_alloc_mm_table(adev);
241 		if (r)
242 			return r;
243 	}
244 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
245 		adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
246 
247 	return 0;
248 }
249 
250 /**
251  * vcn_v3_0_sw_fini - sw fini for VCN block
252  *
253  * @handle: amdgpu_device pointer
254  *
255  * VCN suspend and free up sw allocation
256  */
vcn_v3_0_sw_fini(void * handle)257 static int vcn_v3_0_sw_fini(void *handle)
258 {
259 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
260 	int i, r, idx;
261 
262 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
263 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
264 			volatile struct amdgpu_fw_shared *fw_shared;
265 
266 			if (adev->vcn.harvest_config & (1 << i))
267 				continue;
268 			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
269 			fw_shared->present_flag_0 = 0;
270 			fw_shared->sw_ring.is_enabled = false;
271 		}
272 
273 		drm_dev_exit(idx);
274 	}
275 
276 	if (amdgpu_sriov_vf(adev))
277 		amdgpu_virt_free_mm_table(adev);
278 
279 	r = amdgpu_vcn_suspend(adev);
280 	if (r)
281 		return r;
282 
283 	r = amdgpu_vcn_sw_fini(adev);
284 
285 	return r;
286 }
287 
288 /**
289  * vcn_v3_0_hw_init - start and test VCN block
290  *
291  * @handle: amdgpu_device pointer
292  *
293  * Initialize the hardware, boot up the VCPU and do some testing
294  */
vcn_v3_0_hw_init(void * handle)295 static int vcn_v3_0_hw_init(void *handle)
296 {
297 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
298 	struct amdgpu_ring *ring;
299 	int i, j, r;
300 
301 	if (amdgpu_sriov_vf(adev)) {
302 		r = vcn_v3_0_start_sriov(adev);
303 		if (r)
304 			goto done;
305 
306 		/* initialize VCN dec and enc ring buffers */
307 		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
308 			if (adev->vcn.harvest_config & (1 << i))
309 				continue;
310 
311 			ring = &adev->vcn.inst[i].ring_dec;
312 			if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) {
313 				ring->sched.ready = false;
314 				ring->no_scheduler = true;
315 				dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
316 			} else {
317 				ring->wptr = 0;
318 				ring->wptr_old = 0;
319 				vcn_v3_0_dec_ring_set_wptr(ring);
320 				ring->sched.ready = true;
321 			}
322 
323 			for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
324 				ring = &adev->vcn.inst[i].ring_enc[j];
325 				if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
326 					ring->sched.ready = false;
327 					ring->no_scheduler = true;
328 					dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
329 				} else {
330 					ring->wptr = 0;
331 					ring->wptr_old = 0;
332 					vcn_v3_0_enc_ring_set_wptr(ring);
333 					ring->sched.ready = true;
334 				}
335 			}
336 		}
337 	} else {
338 		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
339 			if (adev->vcn.harvest_config & (1 << i))
340 				continue;
341 
342 			ring = &adev->vcn.inst[i].ring_dec;
343 
344 			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
345 						     ring->doorbell_index, i);
346 
347 			r = amdgpu_ring_test_helper(ring);
348 			if (r)
349 				goto done;
350 
351 			for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
352 				ring = &adev->vcn.inst[i].ring_enc[j];
353 				r = amdgpu_ring_test_helper(ring);
354 				if (r)
355 					goto done;
356 			}
357 		}
358 	}
359 
360 done:
361 	if (!r)
362 		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
363 			(adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
364 
365 	return r;
366 }
367 
368 /**
369  * vcn_v3_0_hw_fini - stop the hardware block
370  *
371  * @handle: amdgpu_device pointer
372  *
373  * Stop the VCN block, mark ring as not ready any more
374  */
vcn_v3_0_hw_fini(void * handle)375 static int vcn_v3_0_hw_fini(void *handle)
376 {
377 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
378 	int i;
379 
380 	cancel_delayed_work_sync(&adev->vcn.idle_work);
381 
382 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
383 		if (adev->vcn.harvest_config & (1 << i))
384 			continue;
385 
386 		if (!amdgpu_sriov_vf(adev)) {
387 			if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
388 					(adev->vcn.cur_state != AMD_PG_STATE_GATE &&
389 					 RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
390 				vcn_v3_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
391 			}
392 		}
393 	}
394 
395 	return 0;
396 }
397 
398 /**
399  * vcn_v3_0_suspend - suspend VCN block
400  *
401  * @handle: amdgpu_device pointer
402  *
403  * HW fini and suspend VCN block
404  */
vcn_v3_0_suspend(void * handle)405 static int vcn_v3_0_suspend(void *handle)
406 {
407 	int r;
408 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
409 
410 	r = vcn_v3_0_hw_fini(adev);
411 	if (r)
412 		return r;
413 
414 	r = amdgpu_vcn_suspend(adev);
415 
416 	return r;
417 }
418 
419 /**
420  * vcn_v3_0_resume - resume VCN block
421  *
422  * @handle: amdgpu_device pointer
423  *
424  * Resume firmware and hw init VCN block
425  */
vcn_v3_0_resume(void * handle)426 static int vcn_v3_0_resume(void *handle)
427 {
428 	int r;
429 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
430 
431 	r = amdgpu_vcn_resume(adev);
432 	if (r)
433 		return r;
434 
435 	r = vcn_v3_0_hw_init(adev);
436 
437 	return r;
438 }
439 
440 /**
441  * vcn_v3_0_mc_resume - memory controller programming
442  *
443  * @adev: amdgpu_device pointer
444  * @inst: instance number
445  *
446  * Let the VCN memory controller know it's offsets
447  */
vcn_v3_0_mc_resume(struct amdgpu_device * adev,int inst)448 static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)
449 {
450 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
451 	uint32_t offset;
452 
453 	/* cache window 0: fw */
454 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
455 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
456 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
457 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
458 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
459 		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
460 		offset = 0;
461 	} else {
462 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
463 			lower_32_bits(adev->vcn.inst[inst].gpu_addr));
464 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
465 			upper_32_bits(adev->vcn.inst[inst].gpu_addr));
466 		offset = size;
467 		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
468 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
469 	}
470 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
471 
472 	/* cache window 1: stack */
473 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
474 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
475 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
476 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
477 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
478 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
479 
480 	/* cache window 2: context */
481 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
482 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
483 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
484 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
485 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
486 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
487 
488 	/* non-cache window */
489 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
490 		lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
491 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
492 		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
493 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
494 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
495 		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
496 }
497 
vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device * adev,int inst_idx,bool indirect)498 static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
499 {
500 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
501 	uint32_t offset;
502 
503 	/* cache window 0: fw */
504 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
505 		if (!indirect) {
506 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
507 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
508 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
509 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
510 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
511 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
512 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
513 				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
514 		} else {
515 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
516 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
517 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
518 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
519 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
520 				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
521 		}
522 		offset = 0;
523 	} else {
524 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
525 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
526 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
527 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
528 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
529 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
530 		offset = size;
531 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
532 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
533 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
534 	}
535 
536 	if (!indirect)
537 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
538 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
539 	else
540 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
541 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
542 
543 	/* cache window 1: stack */
544 	if (!indirect) {
545 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
546 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
547 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
548 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
549 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
550 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
551 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
552 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
553 	} else {
554 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
555 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
556 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
557 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
558 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
559 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
560 	}
561 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
562 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
563 
564 	/* cache window 2: context */
565 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
566 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
567 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
568 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
569 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
570 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
571 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
572 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
573 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
574 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
575 
576 	/* non-cache window */
577 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
578 			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
579 			lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
580 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
581 			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
582 			upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
583 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
584 			VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
585 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
586 			VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
587 			AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
588 
589 	/* VCN global tiling registers */
590 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
591 		UVD, inst_idx, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
592 }
593 
vcn_v3_0_disable_static_power_gating(struct amdgpu_device * adev,int inst)594 static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
595 {
596 	uint32_t data = 0;
597 
598 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
599 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
600 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
601 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
602 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
603 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
604 			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
605 			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
606 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
607 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
608 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
609 			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
610 			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
611 			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
612 			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
613 
614 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
615 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
616 			UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
617 	} else {
618 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
619 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
620 			| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
621 			| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
622 			| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
623 			| 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
624 			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
625 			| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
626 			| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
627 			| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
628 			| 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
629 			| 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
630 			| 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
631 			| 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
632 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
633 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0,  0x3F3FFFFF);
634 	}
635 
636 	data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
637 	data &= ~0x103;
638 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
639 		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
640 			UVD_POWER_STATUS__UVD_PG_EN_MASK;
641 
642 	WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
643 }
644 
vcn_v3_0_enable_static_power_gating(struct amdgpu_device * adev,int inst)645 static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
646 {
647 	uint32_t data;
648 
649 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
650 		/* Before power off, this indicator has to be turned on */
651 		data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
652 		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
653 		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
654 		WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
655 
656 		data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
657 			| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
658 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
659 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
660 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
661 			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
662 			| 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
663 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
664 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
665 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
666 			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
667 			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
668 			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
669 			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
670 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
671 
672 		data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
673 			| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
674 			| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
675 			| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
676 			| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
677 			| 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
678 			| 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
679 			| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
680 			| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
681 			| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
682 			| 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
683 			| 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
684 			| 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
685 			| 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
686 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
687 	}
688 }
689 
690 /**
691  * vcn_v3_0_disable_clock_gating - disable VCN clock gating
692  *
693  * @adev: amdgpu_device pointer
694  * @inst: instance number
695  *
696  * Disable clock gating for VCN block
697  */
vcn_v3_0_disable_clock_gating(struct amdgpu_device * adev,int inst)698 static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
699 {
700 	uint32_t data;
701 
702 	/* VCN disable CGC */
703 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
704 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
705 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
706 	else
707 		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
708 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
709 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
710 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
711 
712 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
713 	data &= ~(UVD_CGC_GATE__SYS_MASK
714 		| UVD_CGC_GATE__UDEC_MASK
715 		| UVD_CGC_GATE__MPEG2_MASK
716 		| UVD_CGC_GATE__REGS_MASK
717 		| UVD_CGC_GATE__RBC_MASK
718 		| UVD_CGC_GATE__LMI_MC_MASK
719 		| UVD_CGC_GATE__LMI_UMC_MASK
720 		| UVD_CGC_GATE__IDCT_MASK
721 		| UVD_CGC_GATE__MPRD_MASK
722 		| UVD_CGC_GATE__MPC_MASK
723 		| UVD_CGC_GATE__LBSI_MASK
724 		| UVD_CGC_GATE__LRBBM_MASK
725 		| UVD_CGC_GATE__UDEC_RE_MASK
726 		| UVD_CGC_GATE__UDEC_CM_MASK
727 		| UVD_CGC_GATE__UDEC_IT_MASK
728 		| UVD_CGC_GATE__UDEC_DB_MASK
729 		| UVD_CGC_GATE__UDEC_MP_MASK
730 		| UVD_CGC_GATE__WCB_MASK
731 		| UVD_CGC_GATE__VCPU_MASK
732 		| UVD_CGC_GATE__MMSCH_MASK);
733 
734 	WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
735 
736 	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0,  0xFFFFFFFF);
737 
738 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
739 	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
740 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
741 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
742 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
743 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
744 		| UVD_CGC_CTRL__SYS_MODE_MASK
745 		| UVD_CGC_CTRL__UDEC_MODE_MASK
746 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
747 		| UVD_CGC_CTRL__REGS_MODE_MASK
748 		| UVD_CGC_CTRL__RBC_MODE_MASK
749 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
750 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
751 		| UVD_CGC_CTRL__IDCT_MODE_MASK
752 		| UVD_CGC_CTRL__MPRD_MODE_MASK
753 		| UVD_CGC_CTRL__MPC_MODE_MASK
754 		| UVD_CGC_CTRL__LBSI_MODE_MASK
755 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
756 		| UVD_CGC_CTRL__WCB_MODE_MASK
757 		| UVD_CGC_CTRL__VCPU_MODE_MASK
758 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
759 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
760 
761 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
762 	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
763 		| UVD_SUVD_CGC_GATE__SIT_MASK
764 		| UVD_SUVD_CGC_GATE__SMP_MASK
765 		| UVD_SUVD_CGC_GATE__SCM_MASK
766 		| UVD_SUVD_CGC_GATE__SDB_MASK
767 		| UVD_SUVD_CGC_GATE__SRE_H264_MASK
768 		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
769 		| UVD_SUVD_CGC_GATE__SIT_H264_MASK
770 		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
771 		| UVD_SUVD_CGC_GATE__SCM_H264_MASK
772 		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
773 		| UVD_SUVD_CGC_GATE__SDB_H264_MASK
774 		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
775 		| UVD_SUVD_CGC_GATE__SCLR_MASK
776 		| UVD_SUVD_CGC_GATE__ENT_MASK
777 		| UVD_SUVD_CGC_GATE__IME_MASK
778 		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
779 		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
780 		| UVD_SUVD_CGC_GATE__SITE_MASK
781 		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
782 		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
783 		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
784 		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
785 		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK
786 		| UVD_SUVD_CGC_GATE__EFC_MASK
787 		| UVD_SUVD_CGC_GATE__SAOE_MASK
788 		| UVD_SUVD_CGC_GATE__SRE_AV1_MASK
789 		| UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
790 		| UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
791 		| UVD_SUVD_CGC_GATE__SCM_AV1_MASK
792 		| UVD_SUVD_CGC_GATE__SMPA_MASK);
793 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
794 
795 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
796 	data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
797 		| UVD_SUVD_CGC_GATE2__MPBE1_MASK
798 		| UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
799 		| UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
800 		| UVD_SUVD_CGC_GATE2__MPC1_MASK);
801 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
802 
803 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
804 	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
805 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
806 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
807 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
808 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
809 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
810 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
811 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
812 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
813 		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
814 		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
815 		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
816 		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
817 		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
818 		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
819 		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
820 		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
821 		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
822 		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
823 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
824 }
825 
vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device * adev,uint8_t sram_sel,int inst_idx,uint8_t indirect)826 static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
827 		uint8_t sram_sel, int inst_idx, uint8_t indirect)
828 {
829 	uint32_t reg_data = 0;
830 
831 	/* enable sw clock gating control */
832 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
833 		reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
834 	else
835 		reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
836 	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
837 	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
838 	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
839 		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
840 		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
841 		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
842 		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
843 		 UVD_CGC_CTRL__SYS_MODE_MASK |
844 		 UVD_CGC_CTRL__UDEC_MODE_MASK |
845 		 UVD_CGC_CTRL__MPEG2_MODE_MASK |
846 		 UVD_CGC_CTRL__REGS_MODE_MASK |
847 		 UVD_CGC_CTRL__RBC_MODE_MASK |
848 		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
849 		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
850 		 UVD_CGC_CTRL__IDCT_MODE_MASK |
851 		 UVD_CGC_CTRL__MPRD_MODE_MASK |
852 		 UVD_CGC_CTRL__MPC_MODE_MASK |
853 		 UVD_CGC_CTRL__LBSI_MODE_MASK |
854 		 UVD_CGC_CTRL__LRBBM_MODE_MASK |
855 		 UVD_CGC_CTRL__WCB_MODE_MASK |
856 		 UVD_CGC_CTRL__VCPU_MODE_MASK |
857 		 UVD_CGC_CTRL__MMSCH_MODE_MASK);
858 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
859 		VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
860 
861 	/* turn off clock gating */
862 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
863 		VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
864 
865 	/* turn on SUVD clock gating */
866 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
867 		VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
868 
869 	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
870 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
871 		VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
872 }
873 
874 /**
875  * vcn_v3_0_enable_clock_gating - enable VCN clock gating
876  *
877  * @adev: amdgpu_device pointer
878  * @inst: instance number
879  *
880  * Enable clock gating for VCN block
881  */
vcn_v3_0_enable_clock_gating(struct amdgpu_device * adev,int inst)882 static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
883 {
884 	uint32_t data;
885 
886 	/* enable VCN CGC */
887 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
888 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
889 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
890 	else
891 		data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
892 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
893 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
894 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
895 
896 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
897 	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
898 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
899 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
900 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
901 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
902 		| UVD_CGC_CTRL__SYS_MODE_MASK
903 		| UVD_CGC_CTRL__UDEC_MODE_MASK
904 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
905 		| UVD_CGC_CTRL__REGS_MODE_MASK
906 		| UVD_CGC_CTRL__RBC_MODE_MASK
907 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
908 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
909 		| UVD_CGC_CTRL__IDCT_MODE_MASK
910 		| UVD_CGC_CTRL__MPRD_MODE_MASK
911 		| UVD_CGC_CTRL__MPC_MODE_MASK
912 		| UVD_CGC_CTRL__LBSI_MODE_MASK
913 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
914 		| UVD_CGC_CTRL__WCB_MODE_MASK
915 		| UVD_CGC_CTRL__VCPU_MODE_MASK
916 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
917 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
918 
919 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
920 	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
921 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
922 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
923 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
924 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
925 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
926 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
927 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
928 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
929 		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
930 		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
931 		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
932 		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
933 		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
934 		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
935 		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
936 		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
937 		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
938 		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
939 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
940 }
941 
vcn_v3_0_start_dpg_mode(struct amdgpu_device * adev,int inst_idx,bool indirect)942 static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
943 {
944 	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
945 	struct amdgpu_ring *ring;
946 	uint32_t rb_bufsz, tmp;
947 
948 	/* disable register anti-hang mechanism */
949 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
950 		~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
951 	/* enable dynamic power gating mode */
952 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
953 	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
954 	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
955 	WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
956 
957 	if (indirect)
958 		adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
959 
960 	/* enable clock gating */
961 	vcn_v3_0_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
962 
963 	/* enable VCPU clock */
964 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
965 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
966 	tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
967 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
968 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
969 
970 	/* disable master interupt */
971 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
972 		VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
973 
974 	/* setup mmUVD_LMI_CTRL */
975 	tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
976 		UVD_LMI_CTRL__REQ_MODE_MASK |
977 		UVD_LMI_CTRL__CRC_RESET_MASK |
978 		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
979 		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
980 		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
981 		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
982 		0x00100000L);
983 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
984 		VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
985 
986 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
987 		VCN, inst_idx, mmUVD_MPC_CNTL),
988 		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
989 
990 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
991 		VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
992 		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
993 		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
994 		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
995 		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
996 
997 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
998 		VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
999 		 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1000 		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1001 		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1002 		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1003 
1004 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1005 		VCN, inst_idx, mmUVD_MPC_SET_MUX),
1006 		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1007 		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1008 		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1009 
1010 	vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
1011 
1012 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1013 		VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1014 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1015 		VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1016 
1017 	/* enable LMI MC and UMC channels */
1018 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1019 		VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1020 
1021 	/* unblock VCPU register access */
1022 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1023 		VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1024 
1025 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1026 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1027 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1028 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1029 
1030 	/* enable master interrupt */
1031 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1032 		VCN, inst_idx, mmUVD_MASTINT_EN),
1033 		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1034 
1035 	/* add nop to workaround PSP size check */
1036 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1037 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1038 
1039 	if (indirect)
1040 		amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
1041 
1042 	ring = &adev->vcn.inst[inst_idx].ring_dec;
1043 	/* force RBC into idle state */
1044 	rb_bufsz = order_base_2(ring->ring_size);
1045 	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1046 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1047 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1048 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1049 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1050 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1051 
1052 	/* Stall DPG before WPTR/RPTR reset */
1053 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1054 		UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1055 		~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1056 	fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1057 
1058 	/* set the write pointer delay */
1059 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1060 
1061 	/* set the wb address */
1062 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1063 		(upper_32_bits(ring->gpu_addr) >> 2));
1064 
1065 	/* programm the RB_BASE for ring buffer */
1066 	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1067 		lower_32_bits(ring->gpu_addr));
1068 	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1069 		upper_32_bits(ring->gpu_addr));
1070 
1071 	/* Initialize the ring buffer's read and write pointers */
1072 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1073 
1074 	WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1075 
1076 	ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1077 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1078 		lower_32_bits(ring->wptr));
1079 
1080 	/* Reset FW shared memory RBC WPTR/RPTR */
1081 	fw_shared->rb.rptr = 0;
1082 	fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1083 
1084 	/*resetting done, fw can check RB ring */
1085 	fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1086 
1087 	/* Unstall DPG */
1088 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1089 		0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1090 
1091 	return 0;
1092 }
1093 
vcn_v3_0_start(struct amdgpu_device * adev)1094 static int vcn_v3_0_start(struct amdgpu_device *adev)
1095 {
1096 	volatile struct amdgpu_fw_shared *fw_shared;
1097 	struct amdgpu_ring *ring;
1098 	uint32_t rb_bufsz, tmp;
1099 	int i, j, k, r;
1100 
1101 	if (adev->pm.dpm_enabled)
1102 		amdgpu_dpm_enable_uvd(adev, true);
1103 
1104 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1105 		if (adev->vcn.harvest_config & (1 << i))
1106 			continue;
1107 
1108 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1109 			r = vcn_v3_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1110 			continue;
1111 		}
1112 
1113 		/* disable VCN power gating */
1114 		vcn_v3_0_disable_static_power_gating(adev, i);
1115 
1116 		/* set VCN status busy */
1117 		tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1118 		WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1119 
1120 		/*SW clock gating */
1121 		vcn_v3_0_disable_clock_gating(adev, i);
1122 
1123 		/* enable VCPU clock */
1124 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1125 			UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1126 
1127 		/* disable master interrupt */
1128 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1129 			~UVD_MASTINT_EN__VCPU_EN_MASK);
1130 
1131 		/* enable LMI MC and UMC channels */
1132 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1133 			~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1134 
1135 		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1136 		tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1137 		tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1138 		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1139 
1140 		/* setup mmUVD_LMI_CTRL */
1141 		tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1142 		WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1143 			UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK	|
1144 			UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1145 			UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1146 			UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1147 
1148 		/* setup mmUVD_MPC_CNTL */
1149 		tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1150 		tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1151 		tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1152 		WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1153 
1154 		/* setup UVD_MPC_SET_MUXA0 */
1155 		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1156 			((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1157 			(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1158 			(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1159 			(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1160 
1161 		/* setup UVD_MPC_SET_MUXB0 */
1162 		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1163 			((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1164 			(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1165 			(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1166 			(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1167 
1168 		/* setup mmUVD_MPC_SET_MUX */
1169 		WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1170 			((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1171 			(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1172 			(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1173 
1174 		vcn_v3_0_mc_resume(adev, i);
1175 
1176 		/* VCN global tiling registers */
1177 		WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1178 			adev->gfx.config.gb_addr_config);
1179 
1180 		/* unblock VCPU register access */
1181 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1182 			~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1183 
1184 		/* release VCPU reset to boot */
1185 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1186 			~UVD_VCPU_CNTL__BLK_RST_MASK);
1187 
1188 		for (j = 0; j < 10; ++j) {
1189 			uint32_t status;
1190 
1191 			for (k = 0; k < 100; ++k) {
1192 				status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1193 				if (status & 2)
1194 					break;
1195 				mdelay(10);
1196 			}
1197 			r = 0;
1198 			if (status & 2)
1199 				break;
1200 
1201 			DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1202 			WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1203 				UVD_VCPU_CNTL__BLK_RST_MASK,
1204 				~UVD_VCPU_CNTL__BLK_RST_MASK);
1205 			mdelay(10);
1206 			WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1207 				~UVD_VCPU_CNTL__BLK_RST_MASK);
1208 
1209 			mdelay(10);
1210 			r = -1;
1211 		}
1212 
1213 		if (r) {
1214 			DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1215 			return r;
1216 		}
1217 
1218 		/* enable master interrupt */
1219 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1220 			UVD_MASTINT_EN__VCPU_EN_MASK,
1221 			~UVD_MASTINT_EN__VCPU_EN_MASK);
1222 
1223 		/* clear the busy bit of VCN_STATUS */
1224 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1225 			~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1226 
1227 		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1228 
1229 		ring = &adev->vcn.inst[i].ring_dec;
1230 		/* force RBC into idle state */
1231 		rb_bufsz = order_base_2(ring->ring_size);
1232 		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1233 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1234 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1235 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1236 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1237 		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1238 
1239 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1240 		fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1241 
1242 		/* programm the RB_BASE for ring buffer */
1243 		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1244 			lower_32_bits(ring->gpu_addr));
1245 		WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1246 			upper_32_bits(ring->gpu_addr));
1247 
1248 		/* Initialize the ring buffer's read and write pointers */
1249 		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1250 
1251 		WREG32_SOC15(VCN, i, mmUVD_SCRATCH2, 0);
1252 		ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1253 		WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1254 			lower_32_bits(ring->wptr));
1255 		fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1256 		fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1257 
1258 		if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1259 			fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1260 			ring = &adev->vcn.inst[i].ring_enc[0];
1261 			WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1262 			WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1263 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1264 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1265 			WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1266 			fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1267 
1268 			fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1269 			ring = &adev->vcn.inst[i].ring_enc[1];
1270 			WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1271 			WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1272 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1273 			WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1274 			WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1275 			fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1276 		}
1277 	}
1278 
1279 	return 0;
1280 }
1281 
vcn_v3_0_start_sriov(struct amdgpu_device * adev)1282 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1283 {
1284 	int i, j;
1285 	struct amdgpu_ring *ring;
1286 	uint64_t cache_addr;
1287 	uint64_t rb_addr;
1288 	uint64_t ctx_addr;
1289 	uint32_t param, resp, expected;
1290 	uint32_t offset, cache_size;
1291 	uint32_t tmp, timeout;
1292 
1293 	struct amdgpu_mm_table *table = &adev->virt.mm_table;
1294 	uint32_t *table_loc;
1295 	uint32_t table_size;
1296 	uint32_t size, size_dw;
1297 
1298 	struct mmsch_v3_0_cmd_direct_write
1299 		direct_wt = { {0} };
1300 	struct mmsch_v3_0_cmd_direct_read_modify_write
1301 		direct_rd_mod_wt = { {0} };
1302 	struct mmsch_v3_0_cmd_end end = { {0} };
1303 	struct mmsch_v3_0_init_header header;
1304 
1305 	direct_wt.cmd_header.command_type =
1306 		MMSCH_COMMAND__DIRECT_REG_WRITE;
1307 	direct_rd_mod_wt.cmd_header.command_type =
1308 		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1309 	end.cmd_header.command_type =
1310 		MMSCH_COMMAND__END;
1311 
1312 	header.version = MMSCH_VERSION;
1313 	header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1314 	for (i = 0; i < MMSCH_V3_0_VCN_INSTANCES; i++) {
1315 		header.inst[i].init_status = 0;
1316 		header.inst[i].table_offset = 0;
1317 		header.inst[i].table_size = 0;
1318 	}
1319 
1320 	table_loc = (uint32_t *)table->cpu_addr;
1321 	table_loc += header.total_size;
1322 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1323 		if (adev->vcn.harvest_config & (1 << i))
1324 			continue;
1325 
1326 		table_size = 0;
1327 
1328 		MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1329 			mmUVD_STATUS),
1330 			~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1331 
1332 		cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1333 
1334 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1335 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1336 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1337 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1338 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1339 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1340 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1341 			offset = 0;
1342 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1343 				mmUVD_VCPU_CACHE_OFFSET0),
1344 				0);
1345 		} else {
1346 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1347 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1348 				lower_32_bits(adev->vcn.inst[i].gpu_addr));
1349 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1350 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1351 				upper_32_bits(adev->vcn.inst[i].gpu_addr));
1352 			offset = cache_size;
1353 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1354 				mmUVD_VCPU_CACHE_OFFSET0),
1355 				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1356 		}
1357 
1358 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1359 			mmUVD_VCPU_CACHE_SIZE0),
1360 			cache_size);
1361 
1362 		cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1363 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1364 			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1365 			lower_32_bits(cache_addr));
1366 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1367 			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1368 			upper_32_bits(cache_addr));
1369 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1370 			mmUVD_VCPU_CACHE_OFFSET1),
1371 			0);
1372 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1373 			mmUVD_VCPU_CACHE_SIZE1),
1374 			AMDGPU_VCN_STACK_SIZE);
1375 
1376 		cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1377 			AMDGPU_VCN_STACK_SIZE;
1378 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1379 			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1380 			lower_32_bits(cache_addr));
1381 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1382 			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1383 			upper_32_bits(cache_addr));
1384 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1385 			mmUVD_VCPU_CACHE_OFFSET2),
1386 			0);
1387 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1388 			mmUVD_VCPU_CACHE_SIZE2),
1389 			AMDGPU_VCN_CONTEXT_SIZE);
1390 
1391 		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1392 			ring = &adev->vcn.inst[i].ring_enc[j];
1393 			ring->wptr = 0;
1394 			rb_addr = ring->gpu_addr;
1395 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1396 				mmUVD_RB_BASE_LO),
1397 				lower_32_bits(rb_addr));
1398 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1399 				mmUVD_RB_BASE_HI),
1400 				upper_32_bits(rb_addr));
1401 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1402 				mmUVD_RB_SIZE),
1403 				ring->ring_size / 4);
1404 		}
1405 
1406 		ring = &adev->vcn.inst[i].ring_dec;
1407 		ring->wptr = 0;
1408 		rb_addr = ring->gpu_addr;
1409 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1410 			mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1411 			lower_32_bits(rb_addr));
1412 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1413 			mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1414 			upper_32_bits(rb_addr));
1415 		/* force RBC into idle state */
1416 		tmp = order_base_2(ring->ring_size);
1417 		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1418 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1419 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1420 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1421 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1422 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1423 			mmUVD_RBC_RB_CNTL),
1424 			tmp);
1425 
1426 		/* add end packet */
1427 		MMSCH_V3_0_INSERT_END();
1428 
1429 		/* refine header */
1430 		header.inst[i].init_status = 0;
1431 		header.inst[i].table_offset = header.total_size;
1432 		header.inst[i].table_size = table_size;
1433 		header.total_size += table_size;
1434 	}
1435 
1436 	/* Update init table header in memory */
1437 	size = sizeof(struct mmsch_v3_0_init_header);
1438 	table_loc = (uint32_t *)table->cpu_addr;
1439 	memcpy((void *)table_loc, &header, size);
1440 
1441 	/* message MMSCH (in VCN[0]) to initialize this client
1442 	 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1443 	 * of memory descriptor location
1444 	 */
1445 	ctx_addr = table->gpu_addr;
1446 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1447 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1448 
1449 	/* 2, update vmid of descriptor */
1450 	tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1451 	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1452 	/* use domain0 for MM scheduler */
1453 	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1454 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1455 
1456 	/* 3, notify mmsch about the size of this descriptor */
1457 	size = header.total_size;
1458 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1459 
1460 	/* 4, set resp to zero */
1461 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1462 
1463 	/* 5, kick off the initialization and wait until
1464 	 * MMSCH_VF_MAILBOX_RESP becomes non-zero
1465 	 */
1466 	param = 0x10000001;
1467 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1468 	tmp = 0;
1469 	timeout = 1000;
1470 	resp = 0;
1471 	expected = param + 1;
1472 	while (resp != expected) {
1473 		resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1474 		if (resp == expected)
1475 			break;
1476 
1477 		udelay(10);
1478 		tmp = tmp + 10;
1479 		if (tmp >= timeout) {
1480 			DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1481 				" waiting for mmMMSCH_VF_MAILBOX_RESP "\
1482 				"(expected=0x%08x, readback=0x%08x)\n",
1483 				tmp, expected, resp);
1484 			return -EBUSY;
1485 		}
1486 	}
1487 
1488 	return 0;
1489 }
1490 
vcn_v3_0_stop_dpg_mode(struct amdgpu_device * adev,int inst_idx)1491 static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1492 {
1493 	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1494 	uint32_t tmp;
1495 
1496 	vcn_v3_0_pause_dpg_mode(adev, inst_idx, &state);
1497 
1498 	/* Wait for power status to be 1 */
1499 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1500 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1501 
1502 	/* wait for read ptr to be equal to write ptr */
1503 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1504 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1505 
1506 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1507 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1508 
1509 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1510 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1511 
1512 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1513 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1514 
1515 	/* disable dynamic power gating mode */
1516 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1517 		~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1518 
1519 	return 0;
1520 }
1521 
vcn_v3_0_stop(struct amdgpu_device * adev)1522 static int vcn_v3_0_stop(struct amdgpu_device *adev)
1523 {
1524 	uint32_t tmp;
1525 	int i, r = 0;
1526 
1527 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1528 		if (adev->vcn.harvest_config & (1 << i))
1529 			continue;
1530 
1531 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1532 			r = vcn_v3_0_stop_dpg_mode(adev, i);
1533 			continue;
1534 		}
1535 
1536 		/* wait for vcn idle */
1537 		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1538 		if (r)
1539 			return r;
1540 
1541 		tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1542 			UVD_LMI_STATUS__READ_CLEAN_MASK |
1543 			UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1544 			UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1545 		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1546 		if (r)
1547 			return r;
1548 
1549 		/* disable LMI UMC channel */
1550 		tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1551 		tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1552 		WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1553 		tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1554 			UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1555 		r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1556 		if (r)
1557 			return r;
1558 
1559 		/* block VCPU register access */
1560 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1561 			UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1562 			~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1563 
1564 		/* reset VCPU */
1565 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1566 			UVD_VCPU_CNTL__BLK_RST_MASK,
1567 			~UVD_VCPU_CNTL__BLK_RST_MASK);
1568 
1569 		/* disable VCPU clock */
1570 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1571 			~(UVD_VCPU_CNTL__CLK_EN_MASK));
1572 
1573 		/* apply soft reset */
1574 		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1575 		tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1576 		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1577 		tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1578 		tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1579 		WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1580 
1581 		/* clear status */
1582 		WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1583 
1584 		/* apply HW clock gating */
1585 		vcn_v3_0_enable_clock_gating(adev, i);
1586 
1587 		/* enable VCN power gating */
1588 		vcn_v3_0_enable_static_power_gating(adev, i);
1589 	}
1590 
1591 	if (adev->pm.dpm_enabled)
1592 		amdgpu_dpm_enable_uvd(adev, false);
1593 
1594 	return 0;
1595 }
1596 
vcn_v3_0_pause_dpg_mode(struct amdgpu_device * adev,int inst_idx,struct dpg_pause_state * new_state)1597 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
1598 		   int inst_idx, struct dpg_pause_state *new_state)
1599 {
1600 	volatile struct amdgpu_fw_shared *fw_shared;
1601 	struct amdgpu_ring *ring;
1602 	uint32_t reg_data = 0;
1603 	int ret_code;
1604 
1605 	/* pause/unpause if state is changed */
1606 	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1607 		DRM_DEBUG("dpg pause state changed %d -> %d",
1608 			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
1609 		reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1610 			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1611 
1612 		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1613 			ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1614 				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1615 
1616 			if (!ret_code) {
1617 				/* pause DPG */
1618 				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1619 				WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1620 
1621 				/* wait for ACK */
1622 				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1623 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1624 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1625 
1626 				/* Stall DPG before WPTR/RPTR reset */
1627 				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1628 					UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1629 					~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1630 
1631 				if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1632 					/* Restore */
1633 					fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1634 					fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1635 					ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1636 					ring->wptr = 0;
1637 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1638 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1639 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1640 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1641 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1642 					fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1643 
1644 					fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1645 					ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1646 					ring->wptr = 0;
1647 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1648 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1649 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1650 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1651 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1652 					fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1653 
1654 					/* restore wptr/rptr with pointers saved in FW shared memory*/
1655 					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, fw_shared->rb.rptr);
1656 					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, fw_shared->rb.wptr);
1657 				}
1658 
1659 				/* Unstall DPG */
1660 				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1661 					0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1662 
1663 				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1664 					UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1665 			}
1666 		} else {
1667 			/* unpause dpg, no need to wait */
1668 			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1669 			WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1670 		}
1671 		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1672 	}
1673 
1674 	return 0;
1675 }
1676 
1677 /**
1678  * vcn_v3_0_dec_ring_get_rptr - get read pointer
1679  *
1680  * @ring: amdgpu_ring pointer
1681  *
1682  * Returns the current hardware read pointer
1683  */
vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring * ring)1684 static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1685 {
1686 	struct amdgpu_device *adev = ring->adev;
1687 
1688 	return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1689 }
1690 
1691 /**
1692  * vcn_v3_0_dec_ring_get_wptr - get write pointer
1693  *
1694  * @ring: amdgpu_ring pointer
1695  *
1696  * Returns the current hardware write pointer
1697  */
vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring * ring)1698 static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1699 {
1700 	struct amdgpu_device *adev = ring->adev;
1701 
1702 	if (ring->use_doorbell)
1703 		return *ring->wptr_cpu_addr;
1704 	else
1705 		return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1706 }
1707 
1708 /**
1709  * vcn_v3_0_dec_ring_set_wptr - set write pointer
1710  *
1711  * @ring: amdgpu_ring pointer
1712  *
1713  * Commits the write pointer to the hardware
1714  */
vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring * ring)1715 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1716 {
1717 	struct amdgpu_device *adev = ring->adev;
1718 	volatile struct amdgpu_fw_shared *fw_shared;
1719 
1720 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1721 		/*whenever update RBC_RB_WPTR, we save the wptr in shared rb.wptr and scratch2 */
1722 		fw_shared = adev->vcn.inst[ring->me].fw_shared.cpu_addr;
1723 		fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1724 		WREG32_SOC15(VCN, ring->me, mmUVD_SCRATCH2,
1725 			lower_32_bits(ring->wptr));
1726 	}
1727 
1728 	if (ring->use_doorbell) {
1729 		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1730 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1731 	} else {
1732 		WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1733 	}
1734 }
1735 
1736 static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1737 	.type = AMDGPU_RING_TYPE_VCN_DEC,
1738 	.align_mask = 0x3f,
1739 	.nop = VCN_DEC_SW_CMD_NO_OP,
1740 	.secure_submission_supported = true,
1741 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
1742 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
1743 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
1744 	.emit_frame_size =
1745 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1746 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1747 		VCN_SW_RING_EMIT_FRAME_SIZE,
1748 	.emit_ib_size = 5, /* vcn_dec_sw_ring_emit_ib */
1749 	.emit_ib = vcn_dec_sw_ring_emit_ib,
1750 	.emit_fence = vcn_dec_sw_ring_emit_fence,
1751 	.emit_vm_flush = vcn_dec_sw_ring_emit_vm_flush,
1752 	.test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1753 	.test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,
1754 	.insert_nop = amdgpu_ring_insert_nop,
1755 	.insert_end = vcn_dec_sw_ring_insert_end,
1756 	.pad_ib = amdgpu_ring_generic_pad_ib,
1757 	.begin_use = amdgpu_vcn_ring_begin_use,
1758 	.end_use = amdgpu_vcn_ring_end_use,
1759 	.emit_wreg = vcn_dec_sw_ring_emit_wreg,
1760 	.emit_reg_wait = vcn_dec_sw_ring_emit_reg_wait,
1761 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1762 };
1763 
vcn_v3_0_limit_sched(struct amdgpu_cs_parser * p,struct amdgpu_job * job)1764 static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p,
1765 				struct amdgpu_job *job)
1766 {
1767 	struct drm_gpu_scheduler **scheds;
1768 
1769 	/* The create msg must be in the first IB submitted */
1770 	if (atomic_read(&job->base.entity->fence_seq))
1771 		return -EINVAL;
1772 
1773 	/* if VCN0 is harvested, we can't support AV1 */
1774 	if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)
1775 		return -EINVAL;
1776 
1777 	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]
1778 		[AMDGPU_RING_PRIO_DEFAULT].sched;
1779 	drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
1780 	return 0;
1781 }
1782 
vcn_v3_0_dec_msg(struct amdgpu_cs_parser * p,struct amdgpu_job * job,uint64_t addr)1783 static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1784 			    uint64_t addr)
1785 {
1786 	struct ttm_operation_ctx ctx = { false, false };
1787 	struct amdgpu_bo_va_mapping *map;
1788 	uint32_t *msg, num_buffers;
1789 	struct amdgpu_bo *bo;
1790 	uint64_t start, end;
1791 	unsigned int i;
1792 	void *ptr;
1793 	int r;
1794 
1795 	addr &= AMDGPU_GMC_HOLE_MASK;
1796 	r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1797 	if (r) {
1798 		DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
1799 		return r;
1800 	}
1801 
1802 	start = map->start * AMDGPU_GPU_PAGE_SIZE;
1803 	end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1804 	if (addr & 0x7) {
1805 		DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1806 		return -EINVAL;
1807 	}
1808 
1809 	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1810 	amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1811 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1812 	if (r) {
1813 		DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1814 		return r;
1815 	}
1816 
1817 	r = amdgpu_bo_kmap(bo, &ptr);
1818 	if (r) {
1819 		DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1820 		return r;
1821 	}
1822 
1823 	msg = ptr + addr - start;
1824 
1825 	/* Check length */
1826 	if (msg[1] > end - addr) {
1827 		r = -EINVAL;
1828 		goto out;
1829 	}
1830 
1831 	if (msg[3] != RDECODE_MSG_CREATE)
1832 		goto out;
1833 
1834 	num_buffers = msg[2];
1835 	for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1836 		uint32_t offset, size, *create;
1837 
1838 		if (msg[0] != RDECODE_MESSAGE_CREATE)
1839 			continue;
1840 
1841 		offset = msg[1];
1842 		size = msg[2];
1843 
1844 		if (offset + size > end) {
1845 			r = -EINVAL;
1846 			goto out;
1847 		}
1848 
1849 		create = ptr + addr + offset - start;
1850 
1851 		/* H246, HEVC and VP9 can run on any instance */
1852 		if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1853 			continue;
1854 
1855 		r = vcn_v3_0_limit_sched(p, job);
1856 		if (r)
1857 			goto out;
1858 	}
1859 
1860 out:
1861 	amdgpu_bo_kunmap(bo);
1862 	return r;
1863 }
1864 
vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser * p,struct amdgpu_job * job,struct amdgpu_ib * ib)1865 static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1866 					   struct amdgpu_job *job,
1867 					   struct amdgpu_ib *ib)
1868 {
1869 	struct amdgpu_ring *ring = amdgpu_job_ring(job);
1870 	uint32_t msg_lo = 0, msg_hi = 0;
1871 	unsigned i;
1872 	int r;
1873 
1874 	/* The first instance can decode anything */
1875 	if (!ring->me)
1876 		return 0;
1877 
1878 	for (i = 0; i < ib->length_dw; i += 2) {
1879 		uint32_t reg = amdgpu_ib_get_value(ib, i);
1880 		uint32_t val = amdgpu_ib_get_value(ib, i + 1);
1881 
1882 		if (reg == PACKET0(p->adev->vcn.internal.data0, 0)) {
1883 			msg_lo = val;
1884 		} else if (reg == PACKET0(p->adev->vcn.internal.data1, 0)) {
1885 			msg_hi = val;
1886 		} else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) &&
1887 			   val == 0) {
1888 			r = vcn_v3_0_dec_msg(p, job,
1889 					     ((u64)msg_hi) << 32 | msg_lo);
1890 			if (r)
1891 				return r;
1892 		}
1893 	}
1894 	return 0;
1895 }
1896 
1897 static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
1898 	.type = AMDGPU_RING_TYPE_VCN_DEC,
1899 	.align_mask = 0xf,
1900 	.secure_submission_supported = true,
1901 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
1902 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
1903 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
1904 	.patch_cs_in_place = vcn_v3_0_ring_patch_cs_in_place,
1905 	.emit_frame_size =
1906 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1907 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1908 		8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1909 		14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1910 		6,
1911 	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1912 	.emit_ib = vcn_v2_0_dec_ring_emit_ib,
1913 	.emit_fence = vcn_v2_0_dec_ring_emit_fence,
1914 	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1915 	.test_ring = vcn_v2_0_dec_ring_test_ring,
1916 	.test_ib = amdgpu_vcn_dec_ring_test_ib,
1917 	.insert_nop = vcn_v2_0_dec_ring_insert_nop,
1918 	.insert_start = vcn_v2_0_dec_ring_insert_start,
1919 	.insert_end = vcn_v2_0_dec_ring_insert_end,
1920 	.pad_ib = amdgpu_ring_generic_pad_ib,
1921 	.begin_use = amdgpu_vcn_ring_begin_use,
1922 	.end_use = amdgpu_vcn_ring_end_use,
1923 	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1924 	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1925 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1926 };
1927 
1928 /**
1929  * vcn_v3_0_enc_ring_get_rptr - get enc read pointer
1930  *
1931  * @ring: amdgpu_ring pointer
1932  *
1933  * Returns the current hardware enc read pointer
1934  */
vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring * ring)1935 static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1936 {
1937 	struct amdgpu_device *adev = ring->adev;
1938 
1939 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1940 		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1941 	else
1942 		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1943 }
1944 
1945 /**
1946  * vcn_v3_0_enc_ring_get_wptr - get enc write pointer
1947  *
1948  * @ring: amdgpu_ring pointer
1949  *
1950  * Returns the current hardware enc write pointer
1951  */
vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring * ring)1952 static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1953 {
1954 	struct amdgpu_device *adev = ring->adev;
1955 
1956 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1957 		if (ring->use_doorbell)
1958 			return *ring->wptr_cpu_addr;
1959 		else
1960 			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1961 	} else {
1962 		if (ring->use_doorbell)
1963 			return *ring->wptr_cpu_addr;
1964 		else
1965 			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1966 	}
1967 }
1968 
1969 /**
1970  * vcn_v3_0_enc_ring_set_wptr - set enc write pointer
1971  *
1972  * @ring: amdgpu_ring pointer
1973  *
1974  * Commits the enc write pointer to the hardware
1975  */
vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring * ring)1976 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1977 {
1978 	struct amdgpu_device *adev = ring->adev;
1979 
1980 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1981 		if (ring->use_doorbell) {
1982 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1983 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1984 		} else {
1985 			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1986 		}
1987 	} else {
1988 		if (ring->use_doorbell) {
1989 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1990 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1991 		} else {
1992 			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1993 		}
1994 	}
1995 }
1996 
1997 static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
1998 	.type = AMDGPU_RING_TYPE_VCN_ENC,
1999 	.align_mask = 0x3f,
2000 	.nop = VCN_ENC_CMD_NO_OP,
2001 	.get_rptr = vcn_v3_0_enc_ring_get_rptr,
2002 	.get_wptr = vcn_v3_0_enc_ring_get_wptr,
2003 	.set_wptr = vcn_v3_0_enc_ring_set_wptr,
2004 	.emit_frame_size =
2005 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2006 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2007 		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2008 		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2009 		1, /* vcn_v2_0_enc_ring_insert_end */
2010 	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2011 	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
2012 	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
2013 	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2014 	.test_ring = amdgpu_vcn_enc_ring_test_ring,
2015 	.test_ib = amdgpu_vcn_enc_ring_test_ib,
2016 	.insert_nop = amdgpu_ring_insert_nop,
2017 	.insert_end = vcn_v2_0_enc_ring_insert_end,
2018 	.pad_ib = amdgpu_ring_generic_pad_ib,
2019 	.begin_use = amdgpu_vcn_ring_begin_use,
2020 	.end_use = amdgpu_vcn_ring_end_use,
2021 	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2022 	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2023 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2024 };
2025 
vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device * adev)2026 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2027 {
2028 	int i;
2029 
2030 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2031 		if (adev->vcn.harvest_config & (1 << i))
2032 			continue;
2033 
2034 		if (!DEC_SW_RING_ENABLED)
2035 			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
2036 		else
2037 			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
2038 		adev->vcn.inst[i].ring_dec.me = i;
2039 		DRM_INFO("VCN(%d) decode%s is enabled in VM mode\n", i,
2040 			  DEC_SW_RING_ENABLED?"(Software Ring)":"");
2041 	}
2042 }
2043 
vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device * adev)2044 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2045 {
2046 	int i, j;
2047 
2048 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2049 		if (adev->vcn.harvest_config & (1 << i))
2050 			continue;
2051 
2052 		for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
2053 			adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
2054 			adev->vcn.inst[i].ring_enc[j].me = i;
2055 		}
2056 		if (adev->vcn.num_enc_rings > 0)
2057 			DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
2058 	}
2059 }
2060 
vcn_v3_0_is_idle(void * handle)2061 static bool vcn_v3_0_is_idle(void *handle)
2062 {
2063 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2064 	int i, ret = 1;
2065 
2066 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2067 		if (adev->vcn.harvest_config & (1 << i))
2068 			continue;
2069 
2070 		ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
2071 	}
2072 
2073 	return ret;
2074 }
2075 
vcn_v3_0_wait_for_idle(void * handle)2076 static int vcn_v3_0_wait_for_idle(void *handle)
2077 {
2078 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2079 	int i, ret = 0;
2080 
2081 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2082 		if (adev->vcn.harvest_config & (1 << i))
2083 			continue;
2084 
2085 		ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2086 			UVD_STATUS__IDLE);
2087 		if (ret)
2088 			return ret;
2089 	}
2090 
2091 	return ret;
2092 }
2093 
vcn_v3_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)2094 static int vcn_v3_0_set_clockgating_state(void *handle,
2095 					  enum amd_clockgating_state state)
2096 {
2097 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2098 	bool enable = state == AMD_CG_STATE_GATE;
2099 	int i;
2100 
2101 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2102 		if (adev->vcn.harvest_config & (1 << i))
2103 			continue;
2104 
2105 		if (enable) {
2106 			if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2107 				return -EBUSY;
2108 			vcn_v3_0_enable_clock_gating(adev, i);
2109 		} else {
2110 			vcn_v3_0_disable_clock_gating(adev, i);
2111 		}
2112 	}
2113 
2114 	return 0;
2115 }
2116 
vcn_v3_0_set_powergating_state(void * handle,enum amd_powergating_state state)2117 static int vcn_v3_0_set_powergating_state(void *handle,
2118 					  enum amd_powergating_state state)
2119 {
2120 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2121 	int ret;
2122 
2123 	/* for SRIOV, guest should not control VCN Power-gating
2124 	 * MMSCH FW should control Power-gating and clock-gating
2125 	 * guest should avoid touching CGC and PG
2126 	 */
2127 	if (amdgpu_sriov_vf(adev)) {
2128 		adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
2129 		return 0;
2130 	}
2131 
2132 	if (state == adev->vcn.cur_state)
2133 		return 0;
2134 
2135 	if (state == AMD_PG_STATE_GATE)
2136 		ret = vcn_v3_0_stop(adev);
2137 	else
2138 		ret = vcn_v3_0_start(adev);
2139 
2140 	if (!ret)
2141 		adev->vcn.cur_state = state;
2142 
2143 	return ret;
2144 }
2145 
vcn_v3_0_set_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)2146 static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2147 					struct amdgpu_irq_src *source,
2148 					unsigned type,
2149 					enum amdgpu_interrupt_state state)
2150 {
2151 	return 0;
2152 }
2153 
vcn_v3_0_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)2154 static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2155 				      struct amdgpu_irq_src *source,
2156 				      struct amdgpu_iv_entry *entry)
2157 {
2158 	uint32_t ip_instance;
2159 
2160 	switch (entry->client_id) {
2161 	case SOC15_IH_CLIENTID_VCN:
2162 		ip_instance = 0;
2163 		break;
2164 	case SOC15_IH_CLIENTID_VCN1:
2165 		ip_instance = 1;
2166 		break;
2167 	default:
2168 		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2169 		return 0;
2170 	}
2171 
2172 	DRM_DEBUG("IH: VCN TRAP\n");
2173 
2174 	switch (entry->src_id) {
2175 	case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2176 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2177 		break;
2178 	case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2179 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2180 		break;
2181 	case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2182 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2183 		break;
2184 	default:
2185 		DRM_ERROR("Unhandled interrupt: %d %d\n",
2186 			  entry->src_id, entry->src_data[0]);
2187 		break;
2188 	}
2189 
2190 	return 0;
2191 }
2192 
2193 static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2194 	.set = vcn_v3_0_set_interrupt_state,
2195 	.process = vcn_v3_0_process_interrupt,
2196 };
2197 
vcn_v3_0_set_irq_funcs(struct amdgpu_device * adev)2198 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2199 {
2200 	int i;
2201 
2202 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2203 		if (adev->vcn.harvest_config & (1 << i))
2204 			continue;
2205 
2206 		adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2207 		adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2208 	}
2209 }
2210 
2211 static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2212 	.name = "vcn_v3_0",
2213 	.early_init = vcn_v3_0_early_init,
2214 	.late_init = NULL,
2215 	.sw_init = vcn_v3_0_sw_init,
2216 	.sw_fini = vcn_v3_0_sw_fini,
2217 	.hw_init = vcn_v3_0_hw_init,
2218 	.hw_fini = vcn_v3_0_hw_fini,
2219 	.suspend = vcn_v3_0_suspend,
2220 	.resume = vcn_v3_0_resume,
2221 	.is_idle = vcn_v3_0_is_idle,
2222 	.wait_for_idle = vcn_v3_0_wait_for_idle,
2223 	.check_soft_reset = NULL,
2224 	.pre_soft_reset = NULL,
2225 	.soft_reset = NULL,
2226 	.post_soft_reset = NULL,
2227 	.set_clockgating_state = vcn_v3_0_set_clockgating_state,
2228 	.set_powergating_state = vcn_v3_0_set_powergating_state,
2229 };
2230 
2231 const struct amdgpu_ip_block_version vcn_v3_0_ip_block = {
2232 	.type = AMD_IP_BLOCK_TYPE_VCN,
2233 	.major = 3,
2234 	.minor = 0,
2235 	.rev = 0,
2236 	.funcs = &vcn_v3_0_ip_funcs,
2237 };
2238