71 		struct kfd_process_device *pdd = process->pdds[i];  in kfd_dbg_ev_query_debug_event()  local
73 						& pdd->exception_status;  in kfd_dbg_ev_query_debug_event()
78 		*event_status = pdd->exception_status;  in kfd_dbg_ev_query_debug_event()
79 		*gpu_id = pdd->dev->id;  in kfd_dbg_ev_query_debug_event()
80 		pdd->exception_status &= ~exception_clear_mask;  in kfd_dbg_ev_query_debug_event()
131 			struct kfd_process_device *pdd = process->pdds[i];  in kfd_dbg_ev_raise()  local
133 			if (pdd->dev != dev)  in kfd_dbg_ev_raise()
136 			pdd->exception_status |= event_mask & KFD_EC_MASK_DEVICE;  in kfd_dbg_ev_raise()
139 				if (!pdd->vm_fault_exc_data) {  in kfd_dbg_ev_raise()
140 					pdd->vm_fault_exc_data = kmemdup(  in kfd_dbg_ev_raise()
144 					if (!pdd->vm_fault_exc_data)  in kfd_dbg_ev_raise()
260 		struct kfd_process_device *pdd = NULL;  in kfd_dbg_send_exception_to_runtime()  local
266 				pdd = p->pdds[i];  in kfd_dbg_send_exception_to_runtime()
271 		if (!pdd)  in kfd_dbg_send_exception_to_runtime()
275 						pdd->vm_fault_exc_data;  in kfd_dbg_send_exception_to_runtime()
277 		kfd_dqm_evict_pasid(pdd->dev->dqm, p->pasid);  in kfd_dbg_send_exception_to_runtime()
278 		kfd_signal_vm_fault_event(pdd->dev, p->pasid, NULL, data);  in kfd_dbg_send_exception_to_runtime()
347 int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en)  in kfd_dbg_set_mes_debug_mode()  argument
349 	uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode;  in kfd_dbg_set_mes_debug_mode()
350 	uint32_t flags = pdd->process->dbg_flags;  in kfd_dbg_set_mes_debug_mode()
352 	if (!kfd_dbg_is_per_vmid_supported(pdd->dev))  in kfd_dbg_set_mes_debug_mode()
355 	return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl,  in kfd_dbg_set_mes_debug_mode()
356 						pdd->watch_points, flags, sq_trap_en);  in kfd_dbg_set_mes_debug_mode()
360 static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_id)  in kfd_dbg_get_dev_watch_id()  argument
366 	spin_lock(&pdd->dev->kfd->watch_points_lock);  in kfd_dbg_get_dev_watch_id()
370 		if ((pdd->dev->kfd->alloc_watch_ids >> i) & 0x1)  in kfd_dbg_get_dev_watch_id()
373 		pdd->alloc_watch_ids |= 0x1 << i;  in kfd_dbg_get_dev_watch_id()
374 		pdd->dev->kfd->alloc_watch_ids |= 0x1 << i;  in kfd_dbg_get_dev_watch_id()
376 		spin_unlock(&pdd->dev->kfd->watch_points_lock);  in kfd_dbg_get_dev_watch_id()
380 	spin_unlock(&pdd->dev->kfd->watch_points_lock);  in kfd_dbg_get_dev_watch_id()
385 static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, int watch_id)  in kfd_dbg_clear_dev_watch_id()  argument
387 	spin_lock(&pdd->dev->kfd->watch_points_lock);  in kfd_dbg_clear_dev_watch_id()
390 	if ((pdd->alloc_watch_ids >> watch_id) & 0x1) {  in kfd_dbg_clear_dev_watch_id()
391 		pdd->alloc_watch_ids &= ~(0x1 << watch_id);  in kfd_dbg_clear_dev_watch_id()
392 		pdd->dev->kfd->alloc_watch_ids &= ~(0x1 << watch_id);  in kfd_dbg_clear_dev_watch_id()
395 	spin_unlock(&pdd->dev->kfd->watch_points_lock);  in kfd_dbg_clear_dev_watch_id()
398 static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, int watch_id)  in kfd_dbg_owns_dev_watch_id()  argument
402 	spin_lock(&pdd->dev->kfd->watch_points_lock);  in kfd_dbg_owns_dev_watch_id()
404 			((pdd->alloc_watch_ids >> watch_id) & 0x1);  in kfd_dbg_owns_dev_watch_id()
406 	spin_unlock(&pdd->dev->kfd->watch_points_lock);  in kfd_dbg_owns_dev_watch_id()
411 int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd,  in kfd_dbg_trap_clear_dev_address_watch()  argument
416 	if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id))  in kfd_dbg_trap_clear_dev_address_watch()
419 	if (!pdd->dev->kfd->shared_resources.enable_mes) {  in kfd_dbg_trap_clear_dev_address_watch()
420 		r = debug_lock_and_unmap(pdd->dev->dqm);  in kfd_dbg_trap_clear_dev_address_watch()
425 	amdgpu_gfx_off_ctrl(pdd->dev->adev, false);  in kfd_dbg_trap_clear_dev_address_watch()
426 	pdd->watch_points[watch_id] = pdd->dev->kfd2kgd->clear_address_watch(  in kfd_dbg_trap_clear_dev_address_watch()
427 							pdd->dev->adev,  in kfd_dbg_trap_clear_dev_address_watch()
429 	amdgpu_gfx_off_ctrl(pdd->dev->adev, true);  in kfd_dbg_trap_clear_dev_address_watch()
431 	if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_clear_dev_address_watch()
432 		r = debug_map_and_unlock(pdd->dev->dqm);  in kfd_dbg_trap_clear_dev_address_watch()
434 		r = kfd_dbg_set_mes_debug_mode(pdd, true);  in kfd_dbg_trap_clear_dev_address_watch()
436 	kfd_dbg_clear_dev_watch_id(pdd, watch_id);  in kfd_dbg_trap_clear_dev_address_watch()
441 int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd,  in kfd_dbg_trap_set_dev_address_watch()  argument
447 	int xcc_id, r = kfd_dbg_get_dev_watch_id(pdd, watch_id);  in kfd_dbg_trap_set_dev_address_watch()
448 	uint32_t xcc_mask = pdd->dev->xcc_mask;  in kfd_dbg_trap_set_dev_address_watch()
453 	if (!pdd->dev->kfd->shared_resources.enable_mes) {  in kfd_dbg_trap_set_dev_address_watch()
454 		r = debug_lock_and_unmap(pdd->dev->dqm);  in kfd_dbg_trap_set_dev_address_watch()
456 			kfd_dbg_clear_dev_watch_id(pdd, *watch_id);  in kfd_dbg_trap_set_dev_address_watch()
461 	amdgpu_gfx_off_ctrl(pdd->dev->adev, false);  in kfd_dbg_trap_set_dev_address_watch()
463 		pdd->watch_points[*watch_id] = pdd->dev->kfd2kgd->set_address_watch(  in kfd_dbg_trap_set_dev_address_watch()
464 				pdd->dev->adev,  in kfd_dbg_trap_set_dev_address_watch()
469 				pdd->dev->vm_info.last_vmid_kfd,  in kfd_dbg_trap_set_dev_address_watch()
471 	amdgpu_gfx_off_ctrl(pdd->dev->adev, true);  in kfd_dbg_trap_set_dev_address_watch()
473 	if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_set_dev_address_watch()
474 		r = debug_map_and_unlock(pdd->dev->dqm);  in kfd_dbg_trap_set_dev_address_watch()
476 		r = kfd_dbg_set_mes_debug_mode(pdd, true);  in kfd_dbg_trap_set_dev_address_watch()
480 		kfd_dbg_clear_dev_watch_id(pdd, *watch_id);  in kfd_dbg_trap_set_dev_address_watch()
510 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_set_flags()  local
512 		if (!kfd_dbg_is_per_vmid_supported(pdd->dev))  in kfd_dbg_trap_set_flags()
515 		if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_set_flags()
516 			r = debug_refresh_runlist(pdd->dev->dqm);  in kfd_dbg_trap_set_flags()
518 			r = kfd_dbg_set_mes_debug_mode(pdd, true);  in kfd_dbg_trap_set_flags()
533 			struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_set_flags()  local
535 			if (!kfd_dbg_is_per_vmid_supported(pdd->dev))  in kfd_dbg_trap_set_flags()
538 			if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_set_flags()
539 				debug_refresh_runlist(pdd->dev->dqm);  in kfd_dbg_trap_set_flags()
541 				kfd_dbg_set_mes_debug_mode(pdd, true);  in kfd_dbg_trap_set_flags()
575 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_deactivate()  local
584 		kfd_process_set_trap_debug_flag(&pdd->qpd, false);  in kfd_dbg_trap_deactivate()
587 		if (kfd_dbg_is_rlc_restore_supported(pdd->dev))  in kfd_dbg_trap_deactivate()
588 			amdgpu_gfx_off_ctrl(pdd->dev->adev, false);  in kfd_dbg_trap_deactivate()
589 		pdd->spi_dbg_override =  in kfd_dbg_trap_deactivate()
590 				pdd->dev->kfd2kgd->disable_debug_trap(  in kfd_dbg_trap_deactivate()
591 				pdd->dev->adev,  in kfd_dbg_trap_deactivate()
593 				pdd->dev->vm_info.last_vmid_kfd);  in kfd_dbg_trap_deactivate()
594 		amdgpu_gfx_off_ctrl(pdd->dev->adev, true);  in kfd_dbg_trap_deactivate()
596 		if (!kfd_dbg_is_per_vmid_supported(pdd->dev) &&  in kfd_dbg_trap_deactivate()
597 				release_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd))  in kfd_dbg_trap_deactivate()
598 			pr_err("Failed to release debug vmid on [%i]\n", pdd->dev->id);  in kfd_dbg_trap_deactivate()
600 		if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_deactivate()
601 			debug_refresh_runlist(pdd->dev->dqm);  in kfd_dbg_trap_deactivate()
603 			kfd_dbg_set_mes_debug_mode(pdd, !kfd_dbg_has_cwsr_workaround(pdd->dev));  in kfd_dbg_trap_deactivate()
616 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_clean_exception_status()  local
618 		kfd_process_drain_interrupts(pdd);  in kfd_dbg_clean_exception_status()
620 		pdd->exception_status = 0;  in kfd_dbg_clean_exception_status()
672 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_activate()  local
674 		if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) {  in kfd_dbg_trap_activate()
675 			r = reserve_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd);  in kfd_dbg_trap_activate()
693 		amdgpu_gfx_off_ctrl(pdd->dev->adev, false);  in kfd_dbg_trap_activate()
694 		if (!(kfd_dbg_is_rlc_restore_supported(pdd->dev) ||  in kfd_dbg_trap_activate()
696 			pdd->dev->kfd2kgd->enable_debug_trap(pdd->dev->adev, true,  in kfd_dbg_trap_activate()
697 								pdd->dev->vm_info.last_vmid_kfd);  in kfd_dbg_trap_activate()
699 		pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(  in kfd_dbg_trap_activate()
700 					pdd->dev->adev,  in kfd_dbg_trap_activate()
702 					pdd->dev->vm_info.last_vmid_kfd);  in kfd_dbg_trap_activate()
704 		if (kfd_dbg_is_rlc_restore_supported(pdd->dev))  in kfd_dbg_trap_activate()
705 			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);  in kfd_dbg_trap_activate()
714 		kfd_process_set_trap_debug_flag(&pdd->qpd, true);  in kfd_dbg_trap_activate()
716 		if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_activate()
717 			r = debug_refresh_runlist(pdd->dev->dqm);  in kfd_dbg_trap_activate()
719 			r = kfd_dbg_set_mes_debug_mode(pdd, true);  in kfd_dbg_trap_activate()
750 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_enable()  local
752 		if (!KFD_IS_SOC15(pdd->dev))  in kfd_dbg_trap_enable()
755 		if (pdd->qpd.num_gws && (!kfd_dbg_has_gws_support(pdd->dev) ||  in kfd_dbg_trap_enable()
756 					 kfd_dbg_has_cwsr_workaround(pdd->dev)))  in kfd_dbg_trap_enable()
803 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_dbg_validate_trap_override_request()  local
804 		int err = pdd->dev->kfd2kgd->validate_trap_override_request(  in kfd_dbg_validate_trap_override_request()
805 								pdd->dev->adev,  in kfd_dbg_validate_trap_override_request()
837 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_set_wave_launch_override()  local
839 		amdgpu_gfx_off_ctrl(pdd->dev->adev, false);  in kfd_dbg_trap_set_wave_launch_override()
840 		pdd->spi_dbg_override = pdd->dev->kfd2kgd->set_wave_launch_trap_override(  in kfd_dbg_trap_set_wave_launch_override()
841 				pdd->dev->adev,  in kfd_dbg_trap_set_wave_launch_override()
842 				pdd->dev->vm_info.last_vmid_kfd,  in kfd_dbg_trap_set_wave_launch_override()
847 				pdd->spi_dbg_override);  in kfd_dbg_trap_set_wave_launch_override()
848 		amdgpu_gfx_off_ctrl(pdd->dev->adev, true);  in kfd_dbg_trap_set_wave_launch_override()
850 		if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_set_wave_launch_override()
851 			r = debug_refresh_runlist(pdd->dev->dqm);  in kfd_dbg_trap_set_wave_launch_override()
853 			r = kfd_dbg_set_mes_debug_mode(pdd, true);  in kfd_dbg_trap_set_wave_launch_override()
873 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_set_wave_launch_mode()  local
875 		amdgpu_gfx_off_ctrl(pdd->dev->adev, false);  in kfd_dbg_trap_set_wave_launch_mode()
876 		pdd->spi_dbg_launch_mode = pdd->dev->kfd2kgd->set_wave_launch_mode(  in kfd_dbg_trap_set_wave_launch_mode()
877 				pdd->dev->adev,  in kfd_dbg_trap_set_wave_launch_mode()
879 				pdd->dev->vm_info.last_vmid_kfd);  in kfd_dbg_trap_set_wave_launch_mode()
880 		amdgpu_gfx_off_ctrl(pdd->dev->adev, true);  in kfd_dbg_trap_set_wave_launch_mode()
882 		if (!pdd->dev->kfd->shared_resources.enable_mes)  in kfd_dbg_trap_set_wave_launch_mode()
883 			r = debug_refresh_runlist(pdd->dev->dqm);  in kfd_dbg_trap_set_wave_launch_mode()
885 			r = kfd_dbg_set_mes_debug_mode(pdd, true);  in kfd_dbg_trap_set_wave_launch_mode()
920 			struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_query_exception_info()  local
921 			struct qcm_process_device *qpd = &pdd->qpd;  in kfd_dbg_trap_query_exception_info()
945 		struct kfd_process_device *pdd = NULL;  in kfd_dbg_trap_query_exception_info()  local
949 			pdd = target->pdds[i];  in kfd_dbg_trap_query_exception_info()
950 			if (pdd->dev->id == source_id) {  in kfd_dbg_trap_query_exception_info()
961 		if (!(pdd->exception_status & KFD_EC_MASK(exception_code))) {  in kfd_dbg_trap_query_exception_info()
967 			copy_size = min((size_t)(*info_size), pdd->vm_fault_exc_data_size);  in kfd_dbg_trap_query_exception_info()
969 			if (copy_to_user(info, pdd->vm_fault_exc_data, copy_size)) {  in kfd_dbg_trap_query_exception_info()
973 			actual_info_size = pdd->vm_fault_exc_data_size;  in kfd_dbg_trap_query_exception_info()
975 				kfree(pdd->vm_fault_exc_data);  in kfd_dbg_trap_query_exception_info()
976 				pdd->vm_fault_exc_data = NULL;  in kfd_dbg_trap_query_exception_info()
977 				pdd->vm_fault_exc_data_size = 0;  in kfd_dbg_trap_query_exception_info()
980 		exception_status_ptr = &pdd->exception_status;  in kfd_dbg_trap_query_exception_info()
1040 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_trap_device_snapshot()  local
1041 		struct kfd_topology_device *topo_dev = kfd_topology_device_by_id(pdd->dev->id);  in kfd_dbg_trap_device_snapshot()
1043 		device_info.gpu_id = pdd->dev->id;  in kfd_dbg_trap_device_snapshot()
1044 		device_info.exception_status = pdd->exception_status;  in kfd_dbg_trap_device_snapshot()
1045 		device_info.lds_base = pdd->lds_base;  in kfd_dbg_trap_device_snapshot()
1046 		device_info.lds_limit = pdd->lds_limit;  in kfd_dbg_trap_device_snapshot()
1047 		device_info.scratch_base = pdd->scratch_base;  in kfd_dbg_trap_device_snapshot()
1048 		device_info.scratch_limit = pdd->scratch_limit;  in kfd_dbg_trap_device_snapshot()
1049 		device_info.gpuvm_base = pdd->gpuvm_base;  in kfd_dbg_trap_device_snapshot()
1050 		device_info.gpuvm_limit = pdd->gpuvm_limit;  in kfd_dbg_trap_device_snapshot()
1054 		device_info.revision_id = pdd->dev->adev->pdev->revision;  in kfd_dbg_trap_device_snapshot()
1055 		device_info.subsystem_vendor_id = pdd->dev->adev->pdev->subsystem_vendor;  in kfd_dbg_trap_device_snapshot()
1056 		device_info.subsystem_device_id = pdd->dev->adev->pdev->subsystem_device;  in kfd_dbg_trap_device_snapshot()
1057 		device_info.fw_version = pdd->dev->kfd->mec_fw_version;  in kfd_dbg_trap_device_snapshot()
1066 		device_info.num_xcc = NUM_XCC(pdd->dev->xcc_mask);  in kfd_dbg_trap_device_snapshot()
1071 			pdd->exception_status &= ~exception_clear_mask;  in kfd_dbg_trap_device_snapshot()
1109 		struct kfd_process_device *pdd = target->pdds[i];  in kfd_dbg_set_enabled_debug_exception_mask()  local
1111 		found_mask |= pdd->exception_status;  in kfd_dbg_set_enabled_debug_exception_mask()