82 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)  in kfd_process_dequeue_from_device()  argument
84 	struct kfd_node *dev = pdd->dev;  in kfd_process_dequeue_from_device()
86 	if (pdd->already_dequeued)  in kfd_process_dequeue_from_device()
89 	dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);  in kfd_process_dequeue_from_device()
91 		amdgpu_mes_flush_shader_debugger(dev->adev, pdd->proc_ctx_gpu_addr);  in kfd_process_dequeue_from_device()
92 	pdd->already_dequeued = true;  in kfd_process_dequeue_from_device()
100 	struct kfd_process_device *pdd;  in pqm_set_gws()  local
115 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_set_gws()
116 	if (!pdd) {  in pqm_set_gws()
122 	if (gws && pdd->qpd.num_gws)  in pqm_set_gws()
125 	if (!gws && pdd->qpd.num_gws == 0)  in pqm_set_gws()
130 			ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,  in pqm_set_gws()
133 			ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,  in pqm_set_gws()
148 	pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;  in pqm_set_gws()
178 	struct kfd_process_device *pdd;  in pqm_clean_queue_resource()  local
182 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_clean_queue_resource()
183 	if (!pdd) {  in pqm_clean_queue_resource()
193 		pdd->qpd.num_gws = 0;  in pqm_clean_queue_resource()
280 	struct kfd_process_device *pdd;  in pqm_create_queue()  local
297 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_create_queue()
298 	if (!pdd) {  in pqm_create_queue()
309 	if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))  in pqm_create_queue()
312 	if (pdd->qpd.queue_count >= max_queues)  in pqm_create_queue()
324 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_create_queue()
325 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_create_queue()
326 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);  in pqm_create_queue()
348 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,  in pqm_create_queue()
369 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,  in pqm_create_queue()
382 		retval = kfd_process_drain_interrupts(pdd);  in pqm_create_queue()
387 							kq, &pdd->qpd);  in pqm_create_queue()
407 		uint32_t first_db_index = amdgpu_doorbell_index_on_bar(pdd->dev->adev,  in pqm_create_queue()
408 								       pdd->qpd.proc_doorbells,  in pqm_create_queue()
410 								       pdd->dev->kfd->device_info.doorbell_size);  in pqm_create_queue()
436 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_create_queue()
437 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_create_queue()
438 		dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);  in pqm_create_queue()
445 	struct kfd_process_device *pdd;  in pqm_destroy_queue()  local
468 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_destroy_queue()
469 	if (!pdd) {  in pqm_destroy_queue()
477 		dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);  in pqm_destroy_queue()
484 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);  in pqm_destroy_queue()
501 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_destroy_queue()
502 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_destroy_queue()
503 		dqm->ops.unregister_process(dqm, &pdd->qpd);  in pqm_destroy_queue()
661 static int get_queue_data_sizes(struct kfd_process_device *pdd,  in get_queue_data_sizes()  argument
668 	ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,  in get_queue_data_sizes()
691 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_get_queue_info()  local
693 		list_for_each_entry(q, &pdd->qpd.queues_list, list) {  in kfd_process_get_queue_info()
701 				ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);  in kfd_process_get_queue_info()
740 static int criu_checkpoint_queue(struct kfd_process_device *pdd,  in criu_checkpoint_queue()  argument
750 	q_data->gpu_id = pdd->user_gpu_id;  in criu_checkpoint_queue()
777 	ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);  in criu_checkpoint_queue()
787 static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,  in criu_checkpoint_queues_device()  argument
797 	list_for_each_entry(q, &pdd->qpd.queues_list, list) {  in criu_checkpoint_queues_device()
812 		ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);  in criu_checkpoint_queues_device()
836 		ret = criu_checkpoint_queue(pdd, q, q_data);  in criu_checkpoint_queues_device()
864 		struct kfd_process_device *pdd = p->pdds[pdd_index];  in kfd_criu_checkpoint_queues()  local
870 		ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,  in kfd_criu_checkpoint_queues()
906 	struct kfd_process_device *pdd;  in kfd_criu_restore_queue()  local
947 	pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);  in kfd_criu_restore_queue()
948 	if (!pdd) {  in kfd_criu_restore_queue()
963 	ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, NULL, q_data, mqd, ctl_stack,  in kfd_criu_restore_queue()
971 		ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws);  in kfd_criu_restore_queue()