43 	struct hl_device *hdev = cs->ctx->hdev;  in hl_hw_queue_update_ci()  local
47 	if (hdev->disabled)  in hl_hw_queue_update_ci()
50 	q = &hdev->kernel_queues[0];  in hl_hw_queue_update_ci()
53 	if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)  in hl_hw_queue_update_ci()
61 	for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {  in hl_hw_queue_update_ci()
83 void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,  in hl_hw_queue_submit_bd()  argument
95 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in hl_hw_queue_submit_bd()
116 static int ext_queue_sanity_checks(struct hl_device *hdev,  in ext_queue_sanity_checks()  argument
121 			&hdev->completion_queue[q->cq_id].free_slots_cnt;  in ext_queue_sanity_checks()
128 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in ext_queue_sanity_checks()
142 			dev_dbg(hdev->dev, "No space for %d on CQ %d\n",  in ext_queue_sanity_checks()
165 static int int_queue_sanity_checks(struct hl_device *hdev,  in int_queue_sanity_checks()  argument
172 		dev_err(hdev->dev,  in int_queue_sanity_checks()
182 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in int_queue_sanity_checks()
200 static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,  in hw_queue_sanity_checks()  argument
209 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in hw_queue_sanity_checks()
228 int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,  in hl_hw_queue_send_cb_no_cmpl()  argument
231 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_send_cb_no_cmpl()
234 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_hw_queue_send_cb_no_cmpl()
236 	if (hdev->disabled) {  in hl_hw_queue_send_cb_no_cmpl()
247 		rc = ext_queue_sanity_checks(hdev, q, 1, false);  in hl_hw_queue_send_cb_no_cmpl()
252 	hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);  in hl_hw_queue_send_cb_no_cmpl()
255 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_hw_queue_send_cb_no_cmpl()
270 	struct hl_device *hdev = job->cs->ctx->hdev;  in ext_queue_schedule_job()  local
271 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in ext_queue_schedule_job()
307 	cq = &hdev->completion_queue[q->cq_id];  in ext_queue_schedule_job()
310 	hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,  in ext_queue_schedule_job()
322 	hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in ext_queue_schedule_job()
335 	struct hl_device *hdev = job->cs->ctx->hdev;  in int_queue_schedule_job()  local
336 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in int_queue_schedule_job()
356 	hdev->asic_funcs->pqe_write(hdev, pi, &bd);  in int_queue_schedule_job()
358 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in int_queue_schedule_job()
371 	struct hl_device *hdev = job->cs->ctx->hdev;  in hw_queue_schedule_job()  local
372 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in hw_queue_schedule_job()
382 	offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1);  in hw_queue_schedule_job()
401 	hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in hw_queue_schedule_job()
404 static int init_signal_cs(struct hl_device *hdev,  in init_signal_cs()  argument
413 	prop = &hdev->kernel_queues[q_idx].sync_stream_prop;  in init_signal_cs()
419 	dev_dbg(hdev->dev,  in init_signal_cs()
427 	hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,  in init_signal_cs()
430 	rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1,  in init_signal_cs()
439 void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,  in hl_hw_queue_encaps_sig_set_sob_info()  argument
463 static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,  in init_wait_cs()  argument
472 	prop = &hdev->kernel_queues[q_idx].sync_stream_prop;  in init_wait_cs()
483 		hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl);  in init_wait_cs()
485 …dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, of…  in init_wait_cs()
519 	dev_dbg(hdev->dev,  in init_wait_cs()
532 	hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);  in init_wait_cs()
550 	struct hl_device *hdev = ctx->hdev;  in init_signal_wait_cs()  local
561 		rc = init_signal_cs(hdev, job, cs_cmpl);  in init_signal_wait_cs()
563 		rc = init_wait_cs(hdev, cs, job, cs_cmpl);  in init_signal_wait_cs()
569 			(struct hl_device *hdev, struct hl_cs *cs)  in encaps_sig_first_staged_cs_handler()  argument
603 …dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob…  in encaps_sig_first_staged_cs_handler()
611 		dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n",  in encaps_sig_first_staged_cs_handler()
630 	struct hl_device *hdev = ctx->hdev;  in hl_hw_queue_schedule_cs()  local
637 	cntr = &hdev->aggregated_cs_counters;  in hl_hw_queue_schedule_cs()
639 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_hw_queue_schedule_cs()
641 	if (!hl_device_operational(hdev, &status)) {  in hl_hw_queue_schedule_cs()
644 		dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
645 			"device is %s, CS rejected!\n", hdev->status[status]);  in hl_hw_queue_schedule_cs()
650 	max_queues = hdev->asic_prop.max_queues;  in hl_hw_queue_schedule_cs()
652 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
657 				rc = ext_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
663 				rc = int_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
667 				rc = hw_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
671 				dev_err(hdev->dev, "Queue type %d is invalid\n",  in hl_hw_queue_schedule_cs()
694 		rc = hdev->asic_funcs->collective_wait_init_cs(cs);  in hl_hw_queue_schedule_cs()
699 	rc = hdev->asic_funcs->pre_schedule_cs(cs);  in hl_hw_queue_schedule_cs()
701 		dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
707 	hdev->shadow_cs_queue[cs->sequence &  in hl_hw_queue_schedule_cs()
708 				(hdev->asic_prop.max_pending_cs - 1)] = cs;  in hl_hw_queue_schedule_cs()
711 		rc = encaps_sig_first_staged_cs_handler(hdev, cs);  in hl_hw_queue_schedule_cs()
716 	spin_lock(&hdev->cs_mirror_lock);  in hl_hw_queue_schedule_cs()
722 		staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);  in hl_hw_queue_schedule_cs()
724 			dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
731 		if (is_staged_cs_last_exists(hdev, staged_cs)) {  in hl_hw_queue_schedule_cs()
732 			dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
742 		if (hdev->supports_wait_for_multi_cs)  in hl_hw_queue_schedule_cs()
747 	list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);  in hl_hw_queue_schedule_cs()
750 	first_entry = list_first_entry(&hdev->cs_mirror_list,  in hl_hw_queue_schedule_cs()
752 	if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&  in hl_hw_queue_schedule_cs()
759 	spin_unlock(&hdev->cs_mirror_lock);  in hl_hw_queue_schedule_cs()
781 	spin_unlock(&hdev->cs_mirror_lock);  in hl_hw_queue_schedule_cs()
783 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
788 				&hdev->completion_queue[i].free_slots_cnt;  in hl_hw_queue_schedule_cs()
795 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_hw_queue_schedule_cs()
806 void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)  in hl_hw_queue_inc_ci_kernel()  argument
808 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_inc_ci_kernel()
813 static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in ext_and_cpu_queue_init()  argument
820 		p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address);  in ext_and_cpu_queue_init()
822 		p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,  in ext_and_cpu_queue_init()
831 		dev_err(hdev->dev,  in ext_and_cpu_queue_init()
846 		hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);  in ext_and_cpu_queue_init()
848 		hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,  in ext_and_cpu_queue_init()
854 static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in int_queue_init()  argument
858 	p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,  in int_queue_init()
861 		dev_err(hdev->dev,  in int_queue_init()
874 static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in cpu_queue_init()  argument
876 	return ext_and_cpu_queue_init(hdev, q, true);  in cpu_queue_init()
879 static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in ext_queue_init()  argument
881 	return ext_and_cpu_queue_init(hdev, q, false);  in ext_queue_init()
884 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in hw_queue_init()  argument
888 	p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,  in hw_queue_init()
902 static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx)  in sync_stream_queue_init()  argument
905 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in sync_stream_queue_init()
909 	sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop;  in sync_stream_queue_init()
916 	if (hdev->kernel_queues[q_idx].collective_mode ==  in sync_stream_queue_init()
918 		reserved_mon_idx = hdev->collective_mon_idx;  in sync_stream_queue_init()
928 		hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS;  in sync_stream_queue_init()
929 	} else if (hdev->kernel_queues[q_idx].collective_mode ==  in sync_stream_queue_init()
931 		reserved_mon_idx = hdev->collective_mon_idx++;  in sync_stream_queue_init()
938 	if (!hdev->kernel_queues[q_idx].supports_sync_stream)  in sync_stream_queue_init()
941 	queue_idx = hdev->sync_stream_queue_idx++;  in sync_stream_queue_init()
952 		hw_sob->hdev = hdev;  in sync_stream_queue_init()
955 			hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);  in sync_stream_queue_init()
961 static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx)  in sync_stream_queue_reset()  argument
964 			&hdev->kernel_queues[q_idx].sync_stream_prop;  in sync_stream_queue_reset()
985 static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in queue_init()  argument
994 		rc = ext_queue_init(hdev, q);  in queue_init()
997 		rc = int_queue_init(hdev, q);  in queue_init()
1000 		rc = cpu_queue_init(hdev, q);  in queue_init()
1003 		rc = hw_queue_init(hdev, q);  in queue_init()
1009 		dev_crit(hdev->dev, "wrong queue type %d during init\n",  in queue_init()
1015 	sync_stream_queue_init(hdev, q->hw_queue_id);  in queue_init()
1033 static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)  in queue_fini()  argument
1062 		hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);  in queue_fini()
1064 		hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,  in queue_fini()
1068 int hl_hw_queues_create(struct hl_device *hdev)  in hl_hw_queues_create()  argument
1070 	struct asic_fixed_properties *asic = &hdev->asic_prop;  in hl_hw_queues_create()
1074 	hdev->kernel_queues = kcalloc(asic->max_queues,  in hl_hw_queues_create()
1075 				sizeof(*hdev->kernel_queues), GFP_KERNEL);  in hl_hw_queues_create()
1077 	if (!hdev->kernel_queues) {  in hl_hw_queues_create()
1078 		dev_err(hdev->dev, "Not enough memory for H/W queues\n");  in hl_hw_queues_create()
1083 	for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;  in hl_hw_queues_create()
1090 		rc = queue_init(hdev, q, i);  in hl_hw_queues_create()
1092 			dev_err(hdev->dev,  in hl_hw_queues_create()
1101 	for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)  in hl_hw_queues_create()
1102 		queue_fini(hdev, q);  in hl_hw_queues_create()
1104 	kfree(hdev->kernel_queues);  in hl_hw_queues_create()
1109 void hl_hw_queues_destroy(struct hl_device *hdev)  in hl_hw_queues_destroy()  argument
1112 	u32 max_queues = hdev->asic_prop.max_queues;  in hl_hw_queues_destroy()
1115 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)  in hl_hw_queues_destroy()
1116 		queue_fini(hdev, q);  in hl_hw_queues_destroy()
1118 	kfree(hdev->kernel_queues);  in hl_hw_queues_destroy()
1121 void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)  in hl_hw_queue_reset()  argument
1124 	u32 max_queues = hdev->asic_prop.max_queues;  in hl_hw_queue_reset()
1127 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {  in hl_hw_queue_reset()
1135 			sync_stream_queue_reset(hdev, q->hw_queue_id);  in hl_hw_queue_reset()