310 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_init_ring()  local
313 	spin_lock_init(&kiq->ring_lock);  in amdgpu_gfx_kiq_init_ring()
321 		(adev->doorbell_index.kiq +  in amdgpu_gfx_kiq_init_ring()
329 	ring->eop_gpu_addr = kiq->eop_gpu_addr;  in amdgpu_gfx_kiq_init_ring()
347 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_fini()  local
349 	amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);  in amdgpu_gfx_kiq_fini()
357 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_init()  local
360 				    AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,  in amdgpu_gfx_kiq_init()
361 				    &kiq->eop_gpu_addr, (void **)&hpd);  in amdgpu_gfx_kiq_init()
369 	r = amdgpu_bo_reserve(kiq->eop_obj, true);  in amdgpu_gfx_kiq_init()
372 	amdgpu_bo_kunmap(kiq->eop_obj);  in amdgpu_gfx_kiq_init()
373 	amdgpu_bo_unreserve(kiq->eop_obj);  in amdgpu_gfx_kiq_init()
383 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_mqd_sw_init()  local
384 	struct amdgpu_ring *ring = &kiq->ring;  in amdgpu_gfx_mqd_sw_init()
412 		kiq->mqd_backup = kmalloc(mqd_size, GFP_KERNEL);  in amdgpu_gfx_mqd_sw_init()
413 		if (!kiq->mqd_backup) {  in amdgpu_gfx_mqd_sw_init()
474 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_mqd_sw_fini()  local
495 	ring = &kiq->ring;  in amdgpu_gfx_mqd_sw_fini()
496 	kfree(kiq->mqd_backup);  in amdgpu_gfx_mqd_sw_fini()
504 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_disable_kcq()  local
505 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_disable_kcq()
509 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)  in amdgpu_gfx_disable_kcq()
512 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_disable_kcq()
513 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *  in amdgpu_gfx_disable_kcq()
515 		spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kcq()
521 		kiq->pmf->kiq_unmap_queues(kiq_ring,  in amdgpu_gfx_disable_kcq()
528 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kcq()
535 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_disable_kgq()  local
536 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_disable_kgq()
540 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)  in amdgpu_gfx_disable_kgq()
543 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_disable_kgq()
545 		if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *  in amdgpu_gfx_disable_kgq()
547 			spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kgq()
553 			kiq->pmf->kiq_unmap_queues(kiq_ring,  in amdgpu_gfx_disable_kgq()
559 	if (adev->gfx.kiq[0].ring.sched.ready && !adev->job_hang)  in amdgpu_gfx_disable_kgq()
561 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kgq()
581 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_enable_kcq()  local
582 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_enable_kcq()
586 	if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)  in amdgpu_gfx_enable_kcq()
608 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_enable_kcq()
609 	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *  in amdgpu_gfx_enable_kcq()
611 					kiq->pmf->set_resources_size);  in amdgpu_gfx_enable_kcq()
614 		spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kcq()
621 	kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);  in amdgpu_gfx_enable_kcq()
624 			kiq->pmf->kiq_map_queues(kiq_ring,  in amdgpu_gfx_enable_kcq()
629 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kcq()
638 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_enable_kgq()  local
639 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_enable_kgq()
642 	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)  in amdgpu_gfx_enable_kgq()
647 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_enable_kgq()
650 		r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *  in amdgpu_gfx_enable_kgq()
654 			spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kgq()
660 			kiq->pmf->kiq_map_queues(kiq_ring,  in amdgpu_gfx_enable_kgq()
666 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kgq()
925 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in amdgpu_kiq_rreg()  local
926 	struct amdgpu_ring *ring = &kiq->ring;  in amdgpu_kiq_rreg()
936 	spin_lock_irqsave(&kiq->ring_lock, flags);  in amdgpu_kiq_rreg()
948 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_rreg()
980 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_rreg()
993 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in amdgpu_kiq_wreg()  local
994 	struct amdgpu_ring *ring = &kiq->ring;  in amdgpu_kiq_wreg()
1006 	spin_lock_irqsave(&kiq->ring_lock, flags);  in amdgpu_kiq_wreg()
1014 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_wreg()
1043 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_wreg()