90 static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,  in mes_v11_0_submit_pkt_and_poll_completion()  argument
98 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_submit_pkt_and_poll_completion()
99 	struct amdgpu_ring *ring = &mes->ring;  in mes_v11_0_submit_pkt_and_poll_completion()
111 	spin_lock_irqsave(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
113 		spin_unlock_irqrestore(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
118 	api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;  in mes_v11_0_submit_pkt_and_poll_completion()
119 	api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;  in mes_v11_0_submit_pkt_and_poll_completion()
123 	spin_unlock_irqrestore(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
151 static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,  in mes_v11_0_add_hw_queue()  argument
154 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_add_hw_queue()
180 	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>  in mes_v11_0_add_hw_queue()
200 	if (!(((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 4) &&  in mes_v11_0_add_hw_queue()
209 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_add_hw_queue()
214 static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,  in mes_v11_0_remove_hw_queue()  argument
228 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_remove_hw_queue()
233 static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,  in mes_v11_0_unmap_legacy_queue()  argument
261 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_unmap_legacy_queue()
266 static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,  in mes_v11_0_suspend_gang()  argument
272 static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,  in mes_v11_0_resume_gang()  argument
278 static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)  in mes_v11_0_query_sched_status()  argument
288 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_query_sched_status()
293 static int mes_v11_0_misc_op(struct amdgpu_mes *mes,  in mes_v11_0_misc_op()  argument
336 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_misc_op()
341 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)  in mes_v11_0_set_hw_resources()  argument
344 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_set_hw_resources()
353 	mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;  in mes_v11_0_set_hw_resources()
354 	mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;  in mes_v11_0_set_hw_resources()
357 	mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr;  in mes_v11_0_set_hw_resources()
359 		mes->query_status_fence_gpu_addr;  in mes_v11_0_set_hw_resources()
363 			mes->compute_hqd_mask[i];  in mes_v11_0_set_hw_resources()
366 		mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];  in mes_v11_0_set_hw_resources()
369 		mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];  in mes_v11_0_set_hw_resources()
373 			mes->aggregated_doorbells[i];  in mes_v11_0_set_hw_resources()
388 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_set_hw_resources()
393 static void mes_v11_0_init_aggregated_doorbell(struct amdgpu_mes *mes)  in mes_v11_0_init_aggregated_doorbell()  argument
395 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_init_aggregated_doorbell()
402 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<  in mes_v11_0_init_aggregated_doorbell()
411 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<  in mes_v11_0_init_aggregated_doorbell()
420 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<  in mes_v11_0_init_aggregated_doorbell()
429 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<  in mes_v11_0_init_aggregated_doorbell()
438 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<  in mes_v11_0_init_aggregated_doorbell()
474 	err = request_firmware(&adev->mes.fw[pipe], fw_name, adev->dev);  in mes_v11_0_init_microcode()
478 	err = amdgpu_ucode_validate(adev->mes.fw[pipe]);  in mes_v11_0_init_microcode()
480 		release_firmware(adev->mes.fw[pipe]);  in mes_v11_0_init_microcode()
481 		adev->mes.fw[pipe] = NULL;  in mes_v11_0_init_microcode()
486 		adev->mes.fw[pipe]->data;  in mes_v11_0_init_microcode()
487 	adev->mes.ucode_fw_version[pipe] =  in mes_v11_0_init_microcode()
489 	adev->mes.ucode_fw_version[pipe] =  in mes_v11_0_init_microcode()
491 	adev->mes.uc_start_addr[pipe] =  in mes_v11_0_init_microcode()
494 	adev->mes.data_start_addr[pipe] =  in mes_v11_0_init_microcode()
511 		info->fw = adev->mes.fw[pipe];  in mes_v11_0_init_microcode()
518 		info->fw = adev->mes.fw[pipe];  in mes_v11_0_init_microcode()
530 	release_firmware(adev->mes.fw[pipe]);  in mes_v11_0_free_microcode()
531 	adev->mes.fw[pipe] = NULL;  in mes_v11_0_free_microcode()
543 		adev->mes.fw[pipe]->data;  in mes_v11_0_allocate_ucode_buffer()
545 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v11_0_allocate_ucode_buffer()
551 				      &adev->mes.ucode_fw_obj[pipe],  in mes_v11_0_allocate_ucode_buffer()
552 				      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v11_0_allocate_ucode_buffer()
553 				      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v11_0_allocate_ucode_buffer()
559 	memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);  in mes_v11_0_allocate_ucode_buffer()
561 	amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_buffer()
562 	amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_buffer()
576 		adev->mes.fw[pipe]->data;  in mes_v11_0_allocate_ucode_data_buffer()
578 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v11_0_allocate_ucode_data_buffer()
584 				      &adev->mes.data_fw_obj[pipe],  in mes_v11_0_allocate_ucode_data_buffer()
585 				      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v11_0_allocate_ucode_data_buffer()
586 				      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
592 	memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);  in mes_v11_0_allocate_ucode_data_buffer()
594 	amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
595 	amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
603 	amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],  in mes_v11_0_free_ucode_buffers()
604 			      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v11_0_free_ucode_buffers()
605 			      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v11_0_free_ucode_buffers()
607 	amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],  in mes_v11_0_free_ucode_buffers()
608 			      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v11_0_free_ucode_buffers()
609 			      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v11_0_free_ucode_buffers()
632 			ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v11_0_enable()
675 	if (!adev->mes.fw[pipe])  in mes_v11_0_load_microcode()
695 	ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v11_0_load_microcode()
703 		     lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
705 		     upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
712 		     lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
714 		     upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
746 			      &adev->mes.eop_gpu_obj[pipe],  in mes_v11_0_allocate_eop_buf()
747 			      &adev->mes.eop_gpu_addr[pipe],  in mes_v11_0_allocate_eop_buf()
755 	       adev->mes.eop_gpu_obj[pipe]->tbo.base.size);  in mes_v11_0_allocate_eop_buf()
757 	amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);  in mes_v11_0_allocate_eop_buf()
758 	amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);  in mes_v11_0_allocate_eop_buf()
942 	kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring);  in mes_v11_0_kiq_enable_queue()
961 		ring = &adev->mes.ring;  in mes_v11_0_queue_init()
989 		adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v11_0_queue_init()
991 		adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v11_0_queue_init()
1003 	ring = &adev->mes.ring;  in mes_v11_0_ring_init()
1014 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];  in mes_v11_0_ring_init()
1038 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];  in mes_v11_0_kiq_ring_init()
1056 		ring = &adev->mes.ring;  in mes_v11_0_mqd_sw_init()
1074 	adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);  in mes_v11_0_mqd_sw_init()
1075 	if (!adev->mes.mqd_backup[pipe])  in mes_v11_0_mqd_sw_init()
1088 	adev->mes.adev = adev;  in mes_v11_0_sw_init()
1089 	adev->mes.funcs = &mes_v11_0_funcs;  in mes_v11_0_sw_init()
1090 	adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;  in mes_v11_0_sw_init()
1091 	adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;  in mes_v11_0_sw_init()
1132 	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);  in mes_v11_0_sw_fini()
1133 	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);  in mes_v11_0_sw_fini()
1136 		kfree(adev->mes.mqd_backup[pipe]);  in mes_v11_0_sw_fini()
1138 		amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],  in mes_v11_0_sw_fini()
1139 				      &adev->mes.eop_gpu_addr[pipe],  in mes_v11_0_sw_fini()
1149 	amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj,  in mes_v11_0_sw_fini()
1150 			      &adev->mes.ring.mqd_gpu_addr,  in mes_v11_0_sw_fini()
1151 			      &adev->mes.ring.mqd_ptr);  in mes_v11_0_sw_fini()
1154 	amdgpu_ring_fini(&adev->mes.ring);  in mes_v11_0_sw_fini()
1198 	adev->mes.ring.sched.ready = false;  in mes_v11_0_kiq_dequeue_sched()
1252 	if (adev->mes.ring.sched.ready)  in mes_v11_0_kiq_hw_fini()
1281 	r = mes_v11_0_set_hw_resources(&adev->mes);  in mes_v11_0_hw_init()
1285 	mes_v11_0_init_aggregated_doorbell(&adev->mes);  in mes_v11_0_hw_init()
1287 	r = mes_v11_0_query_sched_status(&adev->mes);  in mes_v11_0_hw_init()
1299 	adev->mes.ring.sched.ready = true;  in mes_v11_0_hw_init()