1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2016-2022 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
10 
11 #include <linux/firmware.h>
12 #include <linux/crc32.h>
13 #include <linux/slab.h>
14 #include <linux/ctype.h>
15 #include <linux/vmalloc.h>
16 
17 #include <trace/events/habanalabs.h>
18 
19 #define FW_FILE_MAX_SIZE		0x1400000 /* maximum size of 20MB */
20 
21 static char *comms_cmd_str_arr[COMMS_INVLD_LAST] = {
22 	[COMMS_NOOP] = __stringify(COMMS_NOOP),
23 	[COMMS_CLR_STS] = __stringify(COMMS_CLR_STS),
24 	[COMMS_RST_STATE] = __stringify(COMMS_RST_STATE),
25 	[COMMS_PREP_DESC] = __stringify(COMMS_PREP_DESC),
26 	[COMMS_DATA_RDY] = __stringify(COMMS_DATA_RDY),
27 	[COMMS_EXEC] = __stringify(COMMS_EXEC),
28 	[COMMS_RST_DEV] = __stringify(COMMS_RST_DEV),
29 	[COMMS_GOTO_WFE] = __stringify(COMMS_GOTO_WFE),
30 	[COMMS_SKIP_BMC] = __stringify(COMMS_SKIP_BMC),
31 	[COMMS_PREP_DESC_ELBI] = __stringify(COMMS_PREP_DESC_ELBI),
32 };
33 
34 static char *comms_sts_str_arr[COMMS_STS_INVLD_LAST] = {
35 	[COMMS_STS_NOOP] = __stringify(COMMS_STS_NOOP),
36 	[COMMS_STS_ACK] = __stringify(COMMS_STS_ACK),
37 	[COMMS_STS_OK] = __stringify(COMMS_STS_OK),
38 	[COMMS_STS_ERR] = __stringify(COMMS_STS_ERR),
39 	[COMMS_STS_VALID_ERR] = __stringify(COMMS_STS_VALID_ERR),
40 	[COMMS_STS_TIMEOUT_ERR] = __stringify(COMMS_STS_TIMEOUT_ERR),
41 };
42 
extract_fw_ver_from_str(const char * fw_str)43 static char *extract_fw_ver_from_str(const char *fw_str)
44 {
45 	char *str, *fw_ver, *whitespace;
46 	u32 ver_offset;
47 
48 	fw_ver = kmalloc(VERSION_MAX_LEN, GFP_KERNEL);
49 	if (!fw_ver)
50 		return NULL;
51 
52 	str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
53 	if (!str)
54 		goto free_fw_ver;
55 
56 	/* Skip the fw- part */
57 	str += 3;
58 	ver_offset = str - fw_str;
59 
60 	/* Copy until the next whitespace */
61 	whitespace = strnstr(str, " ", VERSION_MAX_LEN - ver_offset);
62 	if (!whitespace)
63 		goto free_fw_ver;
64 
65 	strscpy(fw_ver, str, whitespace - str + 1);
66 
67 	return fw_ver;
68 
69 free_fw_ver:
70 	kfree(fw_ver);
71 	return NULL;
72 }
73 
74 /**
75  * extract_u32_until_given_char() - given a string of the format "<u32><char>*", extract the u32.
76  * @str: the given string
77  * @ver_num: the pointer to the extracted u32 to be returned to the caller.
78  * @given_char: the given char at the end of the u32 in the string
79  *
80  * Return: Upon success, return a pointer to the given_char in the string. Upon failure, return NULL
81  */
extract_u32_until_given_char(char * str,u32 * ver_num,char given_char)82 static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_char)
83 {
84 	char num_str[8] = {}, *ch;
85 
86 	ch = strchrnul(str, given_char);
87 	if (*ch == '\0' || ch == str || ch - str >= sizeof(num_str))
88 		return NULL;
89 
90 	memcpy(num_str, str, ch - str);
91 	if (kstrtou32(num_str, 10, ver_num))
92 		return NULL;
93 	return ch;
94 }
95 
96 /**
97  * hl_get_sw_major_minor_subminor() - extract the FW's SW version major, minor, sub-minor
98  *				      from the version string
99  * @hdev: pointer to the hl_device
100  * @fw_str: the FW's version string
101  *
102  * The extracted version is set in the hdev fields: fw_sw_{major/minor/sub_minor}_ver.
103  *
104  * fw_str is expected to have one of two possible formats, examples:
105  * 1) 'Preboot version hl-gaudi2-1.9.0-fw-42.0.1-sec-3'
106  * 2) 'Preboot version hl-gaudi2-1.9.0-rc-fw-42.0.1-sec-3'
107  * In those examples, the SW major,minor,subminor are correspondingly: 1,9,0.
108  *
109  * Return: 0 for success or a negative error code for failure.
110  */
hl_get_sw_major_minor_subminor(struct hl_device * hdev,const char * fw_str)111 static int hl_get_sw_major_minor_subminor(struct hl_device *hdev, const char *fw_str)
112 {
113 	char *end, *start;
114 
115 	end = strnstr(fw_str, "-rc-", VERSION_MAX_LEN);
116 	if (end == fw_str)
117 		return -EINVAL;
118 
119 	if (!end)
120 		end = strnstr(fw_str, "-fw-", VERSION_MAX_LEN);
121 
122 	if (end == fw_str)
123 		return -EINVAL;
124 
125 	if (!end)
126 		return -EINVAL;
127 
128 	for (start = end - 1; start != fw_str; start--) {
129 		if (*start == '-')
130 			break;
131 	}
132 
133 	if (start == fw_str)
134 		return -EINVAL;
135 
136 	/* start/end point each to the starting and ending hyphen of the sw version e.g. -1.9.0- */
137 	start++;
138 	start = extract_u32_until_given_char(start, &hdev->fw_sw_major_ver, '.');
139 	if (!start)
140 		goto err_zero_ver;
141 
142 	start++;
143 	start = extract_u32_until_given_char(start, &hdev->fw_sw_minor_ver, '.');
144 	if (!start)
145 		goto err_zero_ver;
146 
147 	start++;
148 	start = extract_u32_until_given_char(start, &hdev->fw_sw_sub_minor_ver, '-');
149 	if (!start)
150 		goto err_zero_ver;
151 
152 	return 0;
153 
154 err_zero_ver:
155 	hdev->fw_sw_major_ver = 0;
156 	hdev->fw_sw_minor_ver = 0;
157 	hdev->fw_sw_sub_minor_ver = 0;
158 	return -EINVAL;
159 }
160 
161 /**
162  * hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string.
163  * @hdev: pointer to the hl_device
164  * @preboot_ver: the FW's version string
165  *
166  * preboot_ver is expected to be the format of <major>.<minor>.<sub minor>*, e.g: 42.0.1-sec-3
167  * The extracted version is set in the hdev fields: fw_inner_{major/minor}_ver.
168  *
169  * Return: 0 on success, negative error code for failure.
170  */
hl_get_preboot_major_minor(struct hl_device * hdev,char * preboot_ver)171 static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver)
172 {
173 	preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_major_ver, '.');
174 	if (!preboot_ver) {
175 		dev_err(hdev->dev, "Error parsing preboot major version\n");
176 		goto err_zero_ver;
177 	}
178 
179 	preboot_ver++;
180 
181 	preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_minor_ver, '.');
182 	if (!preboot_ver) {
183 		dev_err(hdev->dev, "Error parsing preboot minor version\n");
184 		goto err_zero_ver;
185 	}
186 	return 0;
187 
188 err_zero_ver:
189 	hdev->fw_inner_major_ver = 0;
190 	hdev->fw_inner_minor_ver = 0;
191 	return -EINVAL;
192 }
193 
hl_request_fw(struct hl_device * hdev,const struct firmware ** firmware_p,const char * fw_name)194 static int hl_request_fw(struct hl_device *hdev,
195 				const struct firmware **firmware_p,
196 				const char *fw_name)
197 {
198 	size_t fw_size;
199 	int rc;
200 
201 	rc = request_firmware(firmware_p, fw_name, hdev->dev);
202 	if (rc) {
203 		dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
204 				fw_name, rc);
205 		goto out;
206 	}
207 
208 	fw_size = (*firmware_p)->size;
209 	if ((fw_size % 4) != 0) {
210 		dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
211 				fw_name, fw_size);
212 		rc = -EINVAL;
213 		goto release_fw;
214 	}
215 
216 	dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
217 
218 	if (fw_size > FW_FILE_MAX_SIZE) {
219 		dev_err(hdev->dev,
220 			"FW file size %zu exceeds maximum of %u bytes\n",
221 			fw_size, FW_FILE_MAX_SIZE);
222 		rc = -EINVAL;
223 		goto release_fw;
224 	}
225 
226 	return 0;
227 
228 release_fw:
229 	release_firmware(*firmware_p);
230 out:
231 	return rc;
232 }
233 
234 /**
235  * hl_release_firmware() - release FW
236  *
237  * @fw: fw descriptor
238  *
239  * note: this inline function added to serve as a comprehensive mirror for the
240  *       hl_request_fw function.
241  */
hl_release_firmware(const struct firmware * fw)242 static inline void hl_release_firmware(const struct firmware *fw)
243 {
244 	release_firmware(fw);
245 }
246 
247 /**
248  * hl_fw_copy_fw_to_device() - copy FW to device
249  *
250  * @hdev: pointer to hl_device structure.
251  * @fw: fw descriptor
252  * @dst: IO memory mapped address space to copy firmware to
253  * @src_offset: offset in src FW to copy from
254  * @size: amount of bytes to copy (0 to copy the whole binary)
255  *
256  * actual copy of FW binary data to device, shared by static and dynamic loaders
257  */
hl_fw_copy_fw_to_device(struct hl_device * hdev,const struct firmware * fw,void __iomem * dst,u32 src_offset,u32 size)258 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
259 				const struct firmware *fw, void __iomem *dst,
260 				u32 src_offset, u32 size)
261 {
262 	const void *fw_data;
263 
264 	/* size 0 indicates to copy the whole file */
265 	if (!size)
266 		size = fw->size;
267 
268 	if (src_offset + size > fw->size) {
269 		dev_err(hdev->dev,
270 			"size to copy(%u) and offset(%u) are invalid\n",
271 			size, src_offset);
272 		return -EINVAL;
273 	}
274 
275 	fw_data = (const void *) fw->data;
276 
277 	memcpy_toio(dst, fw_data + src_offset, size);
278 	return 0;
279 }
280 
281 /**
282  * hl_fw_copy_msg_to_device() - copy message to device
283  *
284  * @hdev: pointer to hl_device structure.
285  * @msg: message
286  * @dst: IO memory mapped address space to copy firmware to
287  * @src_offset: offset in src message to copy from
288  * @size: amount of bytes to copy (0 to copy the whole binary)
289  *
290  * actual copy of message data to device.
291  */
hl_fw_copy_msg_to_device(struct hl_device * hdev,struct lkd_msg_comms * msg,void __iomem * dst,u32 src_offset,u32 size)292 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
293 		struct lkd_msg_comms *msg, void __iomem *dst,
294 		u32 src_offset, u32 size)
295 {
296 	void *msg_data;
297 
298 	/* size 0 indicates to copy the whole file */
299 	if (!size)
300 		size = sizeof(struct lkd_msg_comms);
301 
302 	if (src_offset + size > sizeof(struct lkd_msg_comms)) {
303 		dev_err(hdev->dev,
304 			"size to copy(%u) and offset(%u) are invalid\n",
305 			size, src_offset);
306 		return -EINVAL;
307 	}
308 
309 	msg_data = (void *) msg;
310 
311 	memcpy_toio(dst, msg_data + src_offset, size);
312 
313 	return 0;
314 }
315 
316 /**
317  * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
318  *
319  * @hdev: pointer to hl_device structure.
320  * @fw_name: the firmware image name
321  * @dst: IO memory mapped address space to copy firmware to
322  * @src_offset: offset in src FW to copy from
323  * @size: amount of bytes to copy (0 to copy the whole binary)
324  *
325  * Copy fw code from firmware file to device memory.
326  *
327  * Return: 0 on success, non-zero for failure.
328  */
hl_fw_load_fw_to_device(struct hl_device * hdev,const char * fw_name,void __iomem * dst,u32 src_offset,u32 size)329 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
330 				void __iomem *dst, u32 src_offset, u32 size)
331 {
332 	const struct firmware *fw;
333 	int rc;
334 
335 	rc = hl_request_fw(hdev, &fw, fw_name);
336 	if (rc)
337 		return rc;
338 
339 	rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
340 
341 	hl_release_firmware(fw);
342 	return rc;
343 }
344 
hl_fw_send_pci_access_msg(struct hl_device * hdev,u32 opcode,u64 value)345 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value)
346 {
347 	struct cpucp_packet pkt = {};
348 
349 	pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
350 	pkt.value = cpu_to_le64(value);
351 
352 	return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
353 }
354 
hl_fw_send_cpu_message(struct hl_device * hdev,u32 hw_queue_id,u32 * msg,u16 len,u32 timeout,u64 * result)355 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
356 				u16 len, u32 timeout, u64 *result)
357 {
358 	struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
359 	struct asic_fixed_properties *prop = &hdev->asic_prop;
360 	struct cpucp_packet *pkt;
361 	dma_addr_t pkt_dma_addr;
362 	struct hl_bd *sent_bd;
363 	u32 tmp, expected_ack_val, pi, opcode;
364 	int rc;
365 
366 	pkt = hl_cpu_accessible_dma_pool_alloc(hdev, len, &pkt_dma_addr);
367 	if (!pkt) {
368 		dev_err(hdev->dev,
369 			"Failed to allocate DMA memory for packet to CPU\n");
370 		return -ENOMEM;
371 	}
372 
373 	memcpy(pkt, msg, len);
374 
375 	mutex_lock(&hdev->send_cpu_message_lock);
376 
377 	/* CPU-CP messages can be sent during soft-reset */
378 	if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
379 		rc = 0;
380 		goto out;
381 	}
382 
383 	if (hdev->device_cpu_disabled) {
384 		rc = -EIO;
385 		goto out;
386 	}
387 
388 	/* set fence to a non valid value */
389 	pkt->fence = cpu_to_le32(UINT_MAX);
390 	pi = queue->pi;
391 
392 	/*
393 	 * The CPU queue is a synchronous queue with an effective depth of
394 	 * a single entry (although it is allocated with room for multiple
395 	 * entries). We lock on it using 'send_cpu_message_lock' which
396 	 * serializes accesses to the CPU queue.
397 	 * Which means that we don't need to lock the access to the entire H/W
398 	 * queues module when submitting a JOB to the CPU queue.
399 	 */
400 	hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), len, pkt_dma_addr);
401 
402 	if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
403 		expected_ack_val = queue->pi;
404 	else
405 		expected_ack_val = CPUCP_PACKET_FENCE_VAL;
406 
407 	rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
408 				(tmp == expected_ack_val), 1000,
409 				timeout, true);
410 
411 	hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
412 
413 	if (rc == -ETIMEDOUT) {
414 		/* If FW performed reset just before sending it a packet, we will get a timeout.
415 		 * This is expected behavior, hence no need for error message.
416 		 */
417 		if (!hl_device_operational(hdev, NULL) && !hdev->reset_info.in_compute_reset)
418 			dev_dbg(hdev->dev, "Device CPU packet timeout (0x%x) due to FW reset\n",
419 					tmp);
420 		else
421 			dev_err(hdev->dev, "Device CPU packet timeout (status = 0x%x)\n", tmp);
422 		hdev->device_cpu_disabled = true;
423 		goto out;
424 	}
425 
426 	tmp = le32_to_cpu(pkt->ctl);
427 
428 	rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
429 	if (rc) {
430 		opcode = (tmp & CPUCP_PKT_CTL_OPCODE_MASK) >> CPUCP_PKT_CTL_OPCODE_SHIFT;
431 
432 		if (!prop->supports_advanced_cpucp_rc) {
433 			dev_dbg(hdev->dev, "F/W ERROR %d for CPU packet %d\n", rc, opcode);
434 			rc = -EIO;
435 			goto scrub_descriptor;
436 		}
437 
438 		switch (rc) {
439 		case cpucp_packet_invalid:
440 			dev_err(hdev->dev,
441 				"CPU packet %d is not supported by F/W\n", opcode);
442 			break;
443 		case cpucp_packet_fault:
444 			dev_err(hdev->dev,
445 				"F/W failed processing CPU packet %d\n", opcode);
446 			break;
447 		case cpucp_packet_invalid_pkt:
448 			dev_dbg(hdev->dev,
449 				"CPU packet %d is not supported by F/W\n", opcode);
450 			break;
451 		case cpucp_packet_invalid_params:
452 			dev_err(hdev->dev,
453 				"F/W reports invalid parameters for CPU packet %d\n", opcode);
454 			break;
455 
456 		default:
457 			dev_err(hdev->dev,
458 				"Unknown F/W ERROR %d for CPU packet %d\n", rc, opcode);
459 		}
460 
461 		/* propagate the return code from the f/w to the callers who want to check it */
462 		if (result)
463 			*result = rc;
464 
465 		rc = -EIO;
466 
467 	} else if (result) {
468 		*result = le64_to_cpu(pkt->result);
469 	}
470 
471 scrub_descriptor:
472 	/* Scrub previous buffer descriptor 'ctl' field which contains the
473 	 * previous PI value written during packet submission.
474 	 * We must do this or else F/W can read an old value upon queue wraparound.
475 	 */
476 	sent_bd = queue->kernel_address;
477 	sent_bd += hl_pi_2_offset(pi);
478 	sent_bd->ctl = cpu_to_le32(UINT_MAX);
479 
480 out:
481 	mutex_unlock(&hdev->send_cpu_message_lock);
482 
483 	hl_cpu_accessible_dma_pool_free(hdev, len, pkt);
484 
485 	return rc;
486 }
487 
hl_fw_unmask_irq(struct hl_device * hdev,u16 event_type)488 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
489 {
490 	struct cpucp_packet pkt;
491 	u64 result;
492 	int rc;
493 
494 	memset(&pkt, 0, sizeof(pkt));
495 
496 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
497 				CPUCP_PKT_CTL_OPCODE_SHIFT);
498 	pkt.value = cpu_to_le64(event_type);
499 
500 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
501 						0, &result);
502 
503 	if (rc)
504 		dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
505 
506 	return rc;
507 }
508 
hl_fw_unmask_irq_arr(struct hl_device * hdev,const u32 * irq_arr,size_t irq_arr_size)509 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
510 		size_t irq_arr_size)
511 {
512 	struct cpucp_unmask_irq_arr_packet *pkt;
513 	size_t total_pkt_size;
514 	u64 result;
515 	int rc;
516 
517 	total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
518 			irq_arr_size;
519 
520 	/* data should be aligned to 8 bytes in order to CPU-CP to copy it */
521 	total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
522 
523 	/* total_pkt_size is casted to u16 later on */
524 	if (total_pkt_size > USHRT_MAX) {
525 		dev_err(hdev->dev, "too many elements in IRQ array\n");
526 		return -EINVAL;
527 	}
528 
529 	pkt = kzalloc(total_pkt_size, GFP_KERNEL);
530 	if (!pkt)
531 		return -ENOMEM;
532 
533 	pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
534 	memcpy(&pkt->irqs, irq_arr, irq_arr_size);
535 
536 	pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
537 						CPUCP_PKT_CTL_OPCODE_SHIFT);
538 
539 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
540 						total_pkt_size, 0, &result);
541 
542 	if (rc)
543 		dev_err(hdev->dev, "failed to unmask IRQ array\n");
544 
545 	kfree(pkt);
546 
547 	return rc;
548 }
549 
hl_fw_test_cpu_queue(struct hl_device * hdev)550 int hl_fw_test_cpu_queue(struct hl_device *hdev)
551 {
552 	struct cpucp_packet test_pkt = {};
553 	u64 result;
554 	int rc;
555 
556 	test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
557 					CPUCP_PKT_CTL_OPCODE_SHIFT);
558 	test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
559 
560 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
561 						sizeof(test_pkt), 0, &result);
562 
563 	if (!rc) {
564 		if (result != CPUCP_PACKET_FENCE_VAL)
565 			dev_err(hdev->dev,
566 				"CPU queue test failed (%#08llx)\n", result);
567 	} else {
568 		dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
569 	}
570 
571 	return rc;
572 }
573 
hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device * hdev,size_t size,dma_addr_t * dma_handle)574 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
575 						dma_addr_t *dma_handle)
576 {
577 	u64 kernel_addr;
578 
579 	kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
580 
581 	*dma_handle = hdev->cpu_accessible_dma_address +
582 		(kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
583 
584 	return (void *) (uintptr_t) kernel_addr;
585 }
586 
hl_fw_cpu_accessible_dma_pool_free(struct hl_device * hdev,size_t size,void * vaddr)587 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
588 					void *vaddr)
589 {
590 	gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
591 			size);
592 }
593 
hl_fw_send_soft_reset(struct hl_device * hdev)594 int hl_fw_send_soft_reset(struct hl_device *hdev)
595 {
596 	struct cpucp_packet pkt;
597 	int rc;
598 
599 	memset(&pkt, 0, sizeof(pkt));
600 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_SOFT_RESET << CPUCP_PKT_CTL_OPCODE_SHIFT);
601 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
602 	if (rc)
603 		dev_err(hdev->dev, "failed to send soft-reset msg (err = %d)\n", rc);
604 
605 	return rc;
606 }
607 
hl_fw_send_device_activity(struct hl_device * hdev,bool open)608 int hl_fw_send_device_activity(struct hl_device *hdev, bool open)
609 {
610 	struct cpucp_packet pkt;
611 	int rc;
612 
613 	memset(&pkt, 0, sizeof(pkt));
614 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_ACTIVE_STATUS_SET <<	CPUCP_PKT_CTL_OPCODE_SHIFT);
615 	pkt.value = cpu_to_le64(open);
616 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
617 	if (rc)
618 		dev_err(hdev->dev, "failed to send device activity msg(%u)\n", open);
619 
620 	return rc;
621 }
622 
hl_fw_send_heartbeat(struct hl_device * hdev)623 int hl_fw_send_heartbeat(struct hl_device *hdev)
624 {
625 	struct cpucp_packet hb_pkt;
626 	u64 result;
627 	int rc;
628 
629 	memset(&hb_pkt, 0, sizeof(hb_pkt));
630 	hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
631 					CPUCP_PKT_CTL_OPCODE_SHIFT);
632 	hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
633 
634 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
635 						sizeof(hb_pkt), 0, &result);
636 
637 	if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
638 		return -EIO;
639 
640 	if (le32_to_cpu(hb_pkt.status_mask) &
641 					CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
642 		dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
643 		rc = -EIO;
644 	}
645 
646 	return rc;
647 }
648 
fw_report_boot_dev0(struct hl_device * hdev,u32 err_val,u32 sts_val)649 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
650 								u32 sts_val)
651 {
652 	bool err_exists = false;
653 
654 	if (!(err_val & CPU_BOOT_ERR0_ENABLED))
655 		return false;
656 
657 	if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
658 		dev_err(hdev->dev,
659 			"Device boot error - DRAM initialization failed\n");
660 		err_exists = true;
661 	}
662 
663 	if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
664 		dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
665 		err_exists = true;
666 	}
667 
668 	if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
669 		dev_err(hdev->dev,
670 			"Device boot error - Thermal Sensor initialization failed\n");
671 		err_exists = true;
672 	}
673 
674 	if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
675 		if (hdev->bmc_enable) {
676 			dev_err(hdev->dev,
677 				"Device boot error - Skipped waiting for BMC\n");
678 			err_exists = true;
679 		} else {
680 			dev_info(hdev->dev,
681 				"Device boot message - Skipped waiting for BMC\n");
682 			/* This is an info so we don't want it to disable the
683 			 * device
684 			 */
685 			err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
686 		}
687 	}
688 
689 	if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
690 		dev_err(hdev->dev,
691 			"Device boot error - Serdes data from BMC not available\n");
692 		err_exists = true;
693 	}
694 
695 	if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
696 		dev_err(hdev->dev,
697 			"Device boot error - NIC F/W initialization failed\n");
698 		err_exists = true;
699 	}
700 
701 	if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
702 		dev_err(hdev->dev,
703 			"Device boot warning - security not ready\n");
704 		err_exists = true;
705 	}
706 
707 	if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
708 		dev_err(hdev->dev, "Device boot error - security failure\n");
709 		err_exists = true;
710 	}
711 
712 	if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
713 		dev_err(hdev->dev, "Device boot error - eFuse failure\n");
714 		err_exists = true;
715 	}
716 
717 	if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
718 		dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
719 		err_exists = true;
720 	}
721 
722 	if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
723 		dev_err(hdev->dev, "Device boot error - PLL failure\n");
724 		err_exists = true;
725 	}
726 
727 	if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
728 		/* Ignore this bit, don't prevent driver loading */
729 		dev_dbg(hdev->dev, "device unusable status is set\n");
730 		err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
731 	}
732 
733 	if (err_val & CPU_BOOT_ERR0_BINNING_FAIL) {
734 		dev_err(hdev->dev, "Device boot error - binning failure\n");
735 		err_exists = true;
736 	}
737 
738 	if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
739 		dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
740 
741 	if (err_val & CPU_BOOT_ERR0_EEPROM_FAIL) {
742 		dev_err(hdev->dev, "Device boot error - EEPROM failure detected\n");
743 		err_exists = true;
744 	}
745 
746 	/* All warnings should go here in order not to reach the unknown error validation */
747 	if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
748 		dev_warn(hdev->dev,
749 			"Device boot warning - Skipped DRAM initialization\n");
750 		/* This is a warning so we don't want it to disable the
751 		 * device
752 		 */
753 		err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
754 	}
755 
756 	if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
757 		dev_warn(hdev->dev,
758 			"Device boot warning - Failed to load preboot primary image\n");
759 		/* This is a warning so we don't want it to disable the
760 		 * device as we have a secondary preboot image
761 		 */
762 		err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
763 	}
764 
765 	if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
766 		dev_warn(hdev->dev,
767 			"Device boot warning - TPM failure\n");
768 		/* This is a warning so we don't want it to disable the
769 		 * device
770 		 */
771 		err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
772 	}
773 
774 	if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
775 		dev_err(hdev->dev,
776 			"Device boot error - unknown ERR0 error 0x%08x\n", err_val);
777 		err_exists = true;
778 	}
779 
780 	/* return error only if it's in the predefined mask */
781 	if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
782 				lower_32_bits(hdev->boot_error_status_mask)))
783 		return true;
784 
785 	return false;
786 }
787 
788 /* placeholder for ERR1 as no errors defined there yet */
fw_report_boot_dev1(struct hl_device * hdev,u32 err_val,u32 sts_val)789 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
790 								u32 sts_val)
791 {
792 	/*
793 	 * keep this variable to preserve the logic of the function.
794 	 * this way it would require less modifications when error will be
795 	 * added to DEV_ERR1
796 	 */
797 	bool err_exists = false;
798 
799 	if (!(err_val & CPU_BOOT_ERR1_ENABLED))
800 		return false;
801 
802 	if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
803 		dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
804 
805 	if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
806 		dev_err(hdev->dev,
807 			"Device boot error - unknown ERR1 error 0x%08x\n",
808 								err_val);
809 		err_exists = true;
810 	}
811 
812 	/* return error only if it's in the predefined mask */
813 	if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
814 				upper_32_bits(hdev->boot_error_status_mask)))
815 		return true;
816 
817 	return false;
818 }
819 
fw_read_errors(struct hl_device * hdev,u32 boot_err0_reg,u32 boot_err1_reg,u32 cpu_boot_dev_status0_reg,u32 cpu_boot_dev_status1_reg)820 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
821 				u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
822 				u32 cpu_boot_dev_status1_reg)
823 {
824 	u32 err_val, status_val;
825 	bool err_exists = false;
826 
827 	/* Some of the firmware status codes are deprecated in newer f/w
828 	 * versions. In those versions, the errors are reported
829 	 * in different registers. Therefore, we need to check those
830 	 * registers and print the exact errors. Moreover, there
831 	 * may be multiple errors, so we need to report on each error
832 	 * separately. Some of the error codes might indicate a state
833 	 * that is not an error per-se, but it is an error in production
834 	 * environment
835 	 */
836 	err_val = RREG32(boot_err0_reg);
837 	status_val = RREG32(cpu_boot_dev_status0_reg);
838 	err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
839 
840 	err_val = RREG32(boot_err1_reg);
841 	status_val = RREG32(cpu_boot_dev_status1_reg);
842 	err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
843 
844 	if (err_exists)
845 		return -EIO;
846 
847 	return 0;
848 }
849 
hl_fw_cpucp_info_get(struct hl_device * hdev,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg)850 int hl_fw_cpucp_info_get(struct hl_device *hdev,
851 				u32 sts_boot_dev_sts0_reg,
852 				u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
853 				u32 boot_err1_reg)
854 {
855 	struct asic_fixed_properties *prop = &hdev->asic_prop;
856 	struct cpucp_packet pkt = {};
857 	dma_addr_t cpucp_info_dma_addr;
858 	void *cpucp_info_cpu_addr;
859 	char *kernel_ver;
860 	u64 result;
861 	int rc;
862 
863 	cpucp_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, sizeof(struct cpucp_info),
864 								&cpucp_info_dma_addr);
865 	if (!cpucp_info_cpu_addr) {
866 		dev_err(hdev->dev,
867 			"Failed to allocate DMA memory for CPU-CP info packet\n");
868 		return -ENOMEM;
869 	}
870 
871 	memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
872 
873 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
874 				CPUCP_PKT_CTL_OPCODE_SHIFT);
875 	pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
876 	pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
877 
878 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
879 					HL_CPUCP_INFO_TIMEOUT_USEC, &result);
880 	if (rc) {
881 		dev_err(hdev->dev,
882 			"Failed to handle CPU-CP info pkt, error %d\n", rc);
883 		goto out;
884 	}
885 
886 	rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
887 				sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
888 	if (rc) {
889 		dev_err(hdev->dev, "Errors in device boot\n");
890 		goto out;
891 	}
892 
893 	memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
894 			sizeof(prop->cpucp_info));
895 
896 	rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
897 	if (rc) {
898 		dev_err(hdev->dev,
899 			"Failed to build hwmon channel info, error %d\n", rc);
900 		rc = -EFAULT;
901 		goto out;
902 	}
903 
904 	kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
905 	if (kernel_ver) {
906 		dev_info(hdev->dev, "Linux version %s", kernel_ver);
907 		kfree(kernel_ver);
908 	}
909 
910 	/* assume EQ code doesn't need to check eqe index */
911 	hdev->event_queue.check_eqe_index = false;
912 
913 	/* Read FW application security bits again */
914 	if (prop->fw_cpu_boot_dev_sts0_valid) {
915 		prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
916 		if (prop->fw_app_cpu_boot_dev_sts0 &
917 				CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
918 			hdev->event_queue.check_eqe_index = true;
919 	}
920 
921 	if (prop->fw_cpu_boot_dev_sts1_valid)
922 		prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
923 
924 out:
925 	hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_info), cpucp_info_cpu_addr);
926 
927 	return rc;
928 }
929 
hl_fw_send_msi_info_msg(struct hl_device * hdev)930 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
931 {
932 	struct cpucp_array_data_packet *pkt;
933 	size_t total_pkt_size, data_size;
934 	u64 result;
935 	int rc;
936 
937 	/* skip sending this info for unsupported ASICs */
938 	if (!hdev->asic_funcs->get_msi_info)
939 		return 0;
940 
941 	data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
942 	total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
943 
944 	/* data should be aligned to 8 bytes in order to CPU-CP to copy it */
945 	total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
946 
947 	/* total_pkt_size is casted to u16 later on */
948 	if (total_pkt_size > USHRT_MAX) {
949 		dev_err(hdev->dev, "CPUCP array data is too big\n");
950 		return -EINVAL;
951 	}
952 
953 	pkt = kzalloc(total_pkt_size, GFP_KERNEL);
954 	if (!pkt)
955 		return -ENOMEM;
956 
957 	pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
958 
959 	memset((void *) &pkt->data, 0xFF, data_size);
960 	hdev->asic_funcs->get_msi_info(pkt->data);
961 
962 	pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
963 						CPUCP_PKT_CTL_OPCODE_SHIFT);
964 
965 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
966 						total_pkt_size, 0, &result);
967 
968 	/*
969 	 * in case packet result is invalid it means that FW does not support
970 	 * this feature and will use default/hard coded MSI values. no reason
971 	 * to stop the boot
972 	 */
973 	if (rc && result == cpucp_packet_invalid)
974 		rc = 0;
975 
976 	if (rc)
977 		dev_err(hdev->dev, "failed to send CPUCP array data\n");
978 
979 	kfree(pkt);
980 
981 	return rc;
982 }
983 
hl_fw_cpucp_handshake(struct hl_device * hdev,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg)984 int hl_fw_cpucp_handshake(struct hl_device *hdev,
985 				u32 sts_boot_dev_sts0_reg,
986 				u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
987 				u32 boot_err1_reg)
988 {
989 	int rc;
990 
991 	rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
992 					sts_boot_dev_sts1_reg, boot_err0_reg,
993 					boot_err1_reg);
994 	if (rc)
995 		return rc;
996 
997 	return hl_fw_send_msi_info_msg(hdev);
998 }
999 
hl_fw_get_eeprom_data(struct hl_device * hdev,void * data,size_t max_size)1000 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
1001 {
1002 	struct cpucp_packet pkt = {};
1003 	void *eeprom_info_cpu_addr;
1004 	dma_addr_t eeprom_info_dma_addr;
1005 	u64 result;
1006 	int rc;
1007 
1008 	eeprom_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, max_size,
1009 									&eeprom_info_dma_addr);
1010 	if (!eeprom_info_cpu_addr) {
1011 		dev_err(hdev->dev,
1012 			"Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
1013 		return -ENOMEM;
1014 	}
1015 
1016 	memset(eeprom_info_cpu_addr, 0, max_size);
1017 
1018 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
1019 				CPUCP_PKT_CTL_OPCODE_SHIFT);
1020 	pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
1021 	pkt.data_max_size = cpu_to_le32(max_size);
1022 
1023 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1024 			HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
1025 
1026 	if (rc) {
1027 		dev_err(hdev->dev,
1028 			"Failed to handle CPU-CP EEPROM packet, error %d\n",
1029 			rc);
1030 		goto out;
1031 	}
1032 
1033 	/* result contains the actual size */
1034 	memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
1035 
1036 out:
1037 	hl_cpu_accessible_dma_pool_free(hdev, max_size, eeprom_info_cpu_addr);
1038 
1039 	return rc;
1040 }
1041 
hl_fw_get_monitor_dump(struct hl_device * hdev,void * data)1042 int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data)
1043 {
1044 	struct cpucp_monitor_dump *mon_dump_cpu_addr;
1045 	dma_addr_t mon_dump_dma_addr;
1046 	struct cpucp_packet pkt = {};
1047 	size_t data_size;
1048 	__le32 *src_ptr;
1049 	u32 *dst_ptr;
1050 	u64 result;
1051 	int i, rc;
1052 
1053 	data_size = sizeof(struct cpucp_monitor_dump);
1054 	mon_dump_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, data_size, &mon_dump_dma_addr);
1055 	if (!mon_dump_cpu_addr) {
1056 		dev_err(hdev->dev,
1057 			"Failed to allocate DMA memory for CPU-CP monitor-dump packet\n");
1058 		return -ENOMEM;
1059 	}
1060 
1061 	memset(mon_dump_cpu_addr, 0, data_size);
1062 
1063 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_MONITOR_DUMP_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1064 	pkt.addr = cpu_to_le64(mon_dump_dma_addr);
1065 	pkt.data_max_size = cpu_to_le32(data_size);
1066 
1067 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1068 							HL_CPUCP_MON_DUMP_TIMEOUT_USEC, &result);
1069 	if (rc) {
1070 		dev_err(hdev->dev, "Failed to handle CPU-CP monitor-dump packet, error %d\n", rc);
1071 		goto out;
1072 	}
1073 
1074 	/* result contains the actual size */
1075 	src_ptr = (__le32 *) mon_dump_cpu_addr;
1076 	dst_ptr = data;
1077 	for (i = 0; i < (data_size / sizeof(u32)); i++) {
1078 		*dst_ptr = le32_to_cpu(*src_ptr);
1079 		src_ptr++;
1080 		dst_ptr++;
1081 	}
1082 
1083 out:
1084 	hl_cpu_accessible_dma_pool_free(hdev, data_size, mon_dump_cpu_addr);
1085 
1086 	return rc;
1087 }
1088 
hl_fw_cpucp_pci_counters_get(struct hl_device * hdev,struct hl_info_pci_counters * counters)1089 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
1090 		struct hl_info_pci_counters *counters)
1091 {
1092 	struct cpucp_packet pkt = {};
1093 	u64 result;
1094 	int rc;
1095 
1096 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1097 			CPUCP_PKT_CTL_OPCODE_SHIFT);
1098 
1099 	/* Fetch PCI rx counter */
1100 	pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
1101 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1102 					HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1103 	if (rc) {
1104 		dev_err(hdev->dev,
1105 			"Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1106 		return rc;
1107 	}
1108 	counters->rx_throughput = result;
1109 
1110 	memset(&pkt, 0, sizeof(pkt));
1111 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1112 			CPUCP_PKT_CTL_OPCODE_SHIFT);
1113 
1114 	/* Fetch PCI tx counter */
1115 	pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
1116 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1117 					HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1118 	if (rc) {
1119 		dev_err(hdev->dev,
1120 			"Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1121 		return rc;
1122 	}
1123 	counters->tx_throughput = result;
1124 
1125 	/* Fetch PCI replay counter */
1126 	memset(&pkt, 0, sizeof(pkt));
1127 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
1128 			CPUCP_PKT_CTL_OPCODE_SHIFT);
1129 
1130 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1131 			HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1132 	if (rc) {
1133 		dev_err(hdev->dev,
1134 			"Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1135 		return rc;
1136 	}
1137 	counters->replay_cnt = (u32) result;
1138 
1139 	return rc;
1140 }
1141 
hl_fw_cpucp_total_energy_get(struct hl_device * hdev,u64 * total_energy)1142 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
1143 {
1144 	struct cpucp_packet pkt = {};
1145 	u64 result;
1146 	int rc;
1147 
1148 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
1149 				CPUCP_PKT_CTL_OPCODE_SHIFT);
1150 
1151 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1152 					HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1153 	if (rc) {
1154 		dev_err(hdev->dev,
1155 			"Failed to handle CpuCP total energy pkt, error %d\n",
1156 				rc);
1157 		return rc;
1158 	}
1159 
1160 	*total_energy = result;
1161 
1162 	return rc;
1163 }
1164 
get_used_pll_index(struct hl_device * hdev,u32 input_pll_index,enum pll_index * pll_index)1165 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
1166 						enum pll_index *pll_index)
1167 {
1168 	struct asic_fixed_properties *prop = &hdev->asic_prop;
1169 	u8 pll_byte, pll_bit_off;
1170 	bool dynamic_pll;
1171 	int fw_pll_idx;
1172 
1173 	dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
1174 						CPU_BOOT_DEV_STS0_DYN_PLL_EN);
1175 
1176 	if (!dynamic_pll) {
1177 		/*
1178 		 * in case we are working with legacy FW (each asic has unique
1179 		 * PLL numbering) use the driver based index as they are
1180 		 * aligned with fw legacy numbering
1181 		 */
1182 		*pll_index = input_pll_index;
1183 		return 0;
1184 	}
1185 
1186 	/* retrieve a FW compatible PLL index based on
1187 	 * ASIC specific user request
1188 	 */
1189 	fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
1190 	if (fw_pll_idx < 0) {
1191 		dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
1192 			input_pll_index, fw_pll_idx);
1193 		return -EINVAL;
1194 	}
1195 
1196 	/* PLL map is a u8 array */
1197 	pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
1198 	pll_bit_off = fw_pll_idx & 0x7;
1199 
1200 	if (!(pll_byte & BIT(pll_bit_off))) {
1201 		dev_err(hdev->dev, "PLL index %d is not supported\n",
1202 			fw_pll_idx);
1203 		return -EINVAL;
1204 	}
1205 
1206 	*pll_index = fw_pll_idx;
1207 
1208 	return 0;
1209 }
1210 
hl_fw_cpucp_pll_info_get(struct hl_device * hdev,u32 pll_index,u16 * pll_freq_arr)1211 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
1212 		u16 *pll_freq_arr)
1213 {
1214 	struct cpucp_packet pkt;
1215 	enum pll_index used_pll_idx;
1216 	u64 result;
1217 	int rc;
1218 
1219 	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
1220 	if (rc)
1221 		return rc;
1222 
1223 	memset(&pkt, 0, sizeof(pkt));
1224 
1225 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
1226 				CPUCP_PKT_CTL_OPCODE_SHIFT);
1227 	pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
1228 
1229 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1230 			HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1231 	if (rc) {
1232 		dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
1233 		return rc;
1234 	}
1235 
1236 	pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
1237 	pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
1238 	pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
1239 	pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
1240 
1241 	return 0;
1242 }
1243 
hl_fw_cpucp_power_get(struct hl_device * hdev,u64 * power)1244 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
1245 {
1246 	struct cpucp_packet pkt;
1247 	u64 result;
1248 	int rc;
1249 
1250 	memset(&pkt, 0, sizeof(pkt));
1251 
1252 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
1253 				CPUCP_PKT_CTL_OPCODE_SHIFT);
1254 	pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);
1255 
1256 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1257 			HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1258 	if (rc) {
1259 		dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
1260 		return rc;
1261 	}
1262 
1263 	*power = result;
1264 
1265 	return rc;
1266 }
1267 
hl_fw_dram_replaced_row_get(struct hl_device * hdev,struct cpucp_hbm_row_info * info)1268 int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
1269 				struct cpucp_hbm_row_info *info)
1270 {
1271 	struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;
1272 	dma_addr_t cpucp_repl_rows_info_dma_addr;
1273 	struct cpucp_packet pkt = {};
1274 	u64 result;
1275 	int rc;
1276 
1277 	cpucp_repl_rows_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev,
1278 							sizeof(struct cpucp_hbm_row_info),
1279 							&cpucp_repl_rows_info_dma_addr);
1280 	if (!cpucp_repl_rows_info_cpu_addr) {
1281 		dev_err(hdev->dev,
1282 			"Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");
1283 		return -ENOMEM;
1284 	}
1285 
1286 	memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));
1287 
1288 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<
1289 					CPUCP_PKT_CTL_OPCODE_SHIFT);
1290 	pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);
1291 	pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));
1292 
1293 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1294 					HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1295 	if (rc) {
1296 		dev_err(hdev->dev,
1297 			"Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);
1298 		goto out;
1299 	}
1300 
1301 	memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));
1302 
1303 out:
1304 	hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_hbm_row_info),
1305 						cpucp_repl_rows_info_cpu_addr);
1306 
1307 	return rc;
1308 }
1309 
hl_fw_dram_pending_row_get(struct hl_device * hdev,u32 * pend_rows_num)1310 int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)
1311 {
1312 	struct cpucp_packet pkt;
1313 	u64 result;
1314 	int rc;
1315 
1316 	memset(&pkt, 0, sizeof(pkt));
1317 
1318 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);
1319 
1320 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
1321 	if (rc) {
1322 		dev_err(hdev->dev,
1323 				"Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);
1324 		goto out;
1325 	}
1326 
1327 	*pend_rows_num = (u32) result;
1328 out:
1329 	return rc;
1330 }
1331 
hl_fw_cpucp_engine_core_asid_set(struct hl_device * hdev,u32 asid)1332 int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)
1333 {
1334 	struct cpucp_packet pkt;
1335 	int rc;
1336 
1337 	memset(&pkt, 0, sizeof(pkt));
1338 
1339 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1340 	pkt.value = cpu_to_le64(asid);
1341 
1342 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1343 						HL_CPUCP_INFO_TIMEOUT_USEC, NULL);
1344 	if (rc)
1345 		dev_err(hdev->dev,
1346 			"Failed on ASID configuration request for engine core, error %d\n",
1347 			rc);
1348 
1349 	return rc;
1350 }
1351 
hl_fw_ask_hard_reset_without_linux(struct hl_device * hdev)1352 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
1353 {
1354 	struct static_fw_load_mgr *static_loader =
1355 			&hdev->fw_loader.static_loader;
1356 	int rc;
1357 
1358 	if (hdev->asic_prop.dynamic_fw_load) {
1359 		rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1360 				COMMS_RST_DEV, 0, false,
1361 				hdev->fw_loader.cpu_timeout);
1362 		if (rc)
1363 			dev_err(hdev->dev, "Failed sending COMMS_RST_DEV\n");
1364 	} else {
1365 		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
1366 	}
1367 }
1368 
hl_fw_ask_halt_machine_without_linux(struct hl_device * hdev)1369 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
1370 {
1371 	struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1372 	u32 status, cpu_boot_status_reg, cpu_timeout;
1373 	struct static_fw_load_mgr *static_loader;
1374 	struct pre_fw_load_props *pre_fw_load;
1375 	int rc;
1376 
1377 	if (hdev->device_cpu_is_halted)
1378 		return;
1379 
1380 	/* Stop device CPU to make sure nothing bad happens */
1381 	if (hdev->asic_prop.dynamic_fw_load) {
1382 		pre_fw_load = &fw_loader->pre_fw_load;
1383 		cpu_timeout = fw_loader->cpu_timeout;
1384 		cpu_boot_status_reg = pre_fw_load->cpu_boot_status_reg;
1385 
1386 		rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1387 				COMMS_GOTO_WFE, 0, false, cpu_timeout);
1388 		if (rc) {
1389 			dev_err(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1390 		} else {
1391 			rc = hl_poll_timeout(
1392 				hdev,
1393 				cpu_boot_status_reg,
1394 				status,
1395 				status == CPU_BOOT_STATUS_IN_WFE,
1396 				hdev->fw_poll_interval_usec,
1397 				cpu_timeout);
1398 			if (rc)
1399 				dev_err(hdev->dev, "Current status=%u. Timed-out updating to WFE\n",
1400 						status);
1401 		}
1402 	} else {
1403 		static_loader = &hdev->fw_loader.static_loader;
1404 		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1405 		msleep(static_loader->cpu_reset_wait_msec);
1406 
1407 		/* Must clear this register in order to prevent preboot
1408 		 * from reading WFE after reboot
1409 		 */
1410 		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1411 	}
1412 
1413 	hdev->device_cpu_is_halted = true;
1414 }
1415 
detect_cpu_boot_status(struct hl_device * hdev,u32 status)1416 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1417 {
1418 	/* Some of the status codes below are deprecated in newer f/w
1419 	 * versions but we keep them here for backward compatibility
1420 	 */
1421 	switch (status) {
1422 	case CPU_BOOT_STATUS_NA:
1423 		dev_err(hdev->dev,
1424 			"Device boot progress - BTL/ROM did NOT run\n");
1425 		break;
1426 	case CPU_BOOT_STATUS_IN_WFE:
1427 		dev_err(hdev->dev,
1428 			"Device boot progress - Stuck inside WFE loop\n");
1429 		break;
1430 	case CPU_BOOT_STATUS_IN_BTL:
1431 		dev_err(hdev->dev,
1432 			"Device boot progress - Stuck in BTL\n");
1433 		break;
1434 	case CPU_BOOT_STATUS_IN_PREBOOT:
1435 		dev_err(hdev->dev,
1436 			"Device boot progress - Stuck in Preboot\n");
1437 		break;
1438 	case CPU_BOOT_STATUS_IN_SPL:
1439 		dev_err(hdev->dev,
1440 			"Device boot progress - Stuck in SPL\n");
1441 		break;
1442 	case CPU_BOOT_STATUS_IN_UBOOT:
1443 		dev_err(hdev->dev,
1444 			"Device boot progress - Stuck in u-boot\n");
1445 		break;
1446 	case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1447 		dev_err(hdev->dev,
1448 			"Device boot progress - DRAM initialization failed\n");
1449 		break;
1450 	case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1451 		dev_err(hdev->dev,
1452 			"Device boot progress - Cannot boot\n");
1453 		break;
1454 	case CPU_BOOT_STATUS_TS_INIT_FAIL:
1455 		dev_err(hdev->dev,
1456 			"Device boot progress - Thermal Sensor initialization failed\n");
1457 		break;
1458 	case CPU_BOOT_STATUS_SECURITY_READY:
1459 		dev_err(hdev->dev,
1460 			"Device boot progress - Stuck in preboot after security initialization\n");
1461 		break;
1462 	default:
1463 		dev_err(hdev->dev,
1464 			"Device boot progress - Invalid or unexpected status code %d\n", status);
1465 		break;
1466 	}
1467 }
1468 
hl_fw_wait_preboot_ready(struct hl_device * hdev)1469 int hl_fw_wait_preboot_ready(struct hl_device *hdev)
1470 {
1471 	struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
1472 	u32 status;
1473 	int rc;
1474 
1475 	/* Need to check two possible scenarios:
1476 	 *
1477 	 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1478 	 * the preboot is waiting for the boot fit
1479 	 *
1480 	 * All other status values - for older firmwares where the uboot was
1481 	 * loaded from the FLASH
1482 	 */
1483 	rc = hl_poll_timeout(
1484 		hdev,
1485 		pre_fw_load->cpu_boot_status_reg,
1486 		status,
1487 		(status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1488 		(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1489 		(status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1490 		hdev->fw_poll_interval_usec,
1491 		pre_fw_load->wait_for_preboot_timeout);
1492 
1493 	if (rc) {
1494 		detect_cpu_boot_status(hdev, status);
1495 		dev_err(hdev->dev, "CPU boot ready timeout (status = %d)\n", status);
1496 
1497 		/* If we read all FF, then something is totally wrong, no point
1498 		 * of reading specific errors
1499 		 */
1500 		if (status != -1)
1501 			fw_read_errors(hdev, pre_fw_load->boot_err0_reg,
1502 						pre_fw_load->boot_err1_reg,
1503 						pre_fw_load->sts_boot_dev_sts0_reg,
1504 						pre_fw_load->sts_boot_dev_sts1_reg);
1505 		return -EIO;
1506 	}
1507 
1508 	hdev->fw_loader.fw_comp_loaded |= FW_TYPE_PREBOOT_CPU;
1509 
1510 	return 0;
1511 }
1512 
hl_fw_read_preboot_caps(struct hl_device * hdev)1513 static int hl_fw_read_preboot_caps(struct hl_device *hdev)
1514 {
1515 	struct pre_fw_load_props *pre_fw_load;
1516 	struct asic_fixed_properties *prop;
1517 	u32 reg_val;
1518 	int rc;
1519 
1520 	prop = &hdev->asic_prop;
1521 	pre_fw_load = &hdev->fw_loader.pre_fw_load;
1522 
1523 	rc = hl_fw_wait_preboot_ready(hdev);
1524 	if (rc)
1525 		return rc;
1526 
1527 	/*
1528 	 * the registers DEV_STS* contain FW capabilities/features.
1529 	 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1530 	 * is set.
1531 	 * In the first read of this register we store the value of this
1532 	 * register ONLY if the register is enabled (which will be propagated
1533 	 * to next stages) and also mark the register as valid.
1534 	 * In case it is not enabled the stored value will be left 0- all
1535 	 * caps/features are off
1536 	 */
1537 	reg_val = RREG32(pre_fw_load->sts_boot_dev_sts0_reg);
1538 	if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1539 		prop->fw_cpu_boot_dev_sts0_valid = true;
1540 		prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1541 	}
1542 
1543 	reg_val = RREG32(pre_fw_load->sts_boot_dev_sts1_reg);
1544 	if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1545 		prop->fw_cpu_boot_dev_sts1_valid = true;
1546 		prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1547 	}
1548 
1549 	prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1550 						CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1551 
1552 	/* initialize FW loader once we know what load protocol is used */
1553 	hdev->asic_funcs->init_firmware_loader(hdev);
1554 
1555 	dev_dbg(hdev->dev, "Attempting %s FW load\n",
1556 			prop->dynamic_fw_load ? "dynamic" : "legacy");
1557 	return 0;
1558 }
1559 
hl_fw_static_read_device_fw_version(struct hl_device * hdev,enum hl_fw_component fwc)1560 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1561 					enum hl_fw_component fwc)
1562 {
1563 	struct asic_fixed_properties *prop = &hdev->asic_prop;
1564 	struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1565 	struct static_fw_load_mgr *static_loader;
1566 	char *dest, *boot_ver, *preboot_ver;
1567 	u32 ver_off, limit;
1568 	const char *name;
1569 	char btl_ver[32];
1570 
1571 	static_loader = &hdev->fw_loader.static_loader;
1572 
1573 	switch (fwc) {
1574 	case FW_COMP_BOOT_FIT:
1575 		ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1576 		dest = prop->uboot_ver;
1577 		name = "Boot-fit";
1578 		limit = static_loader->boot_fit_version_max_off;
1579 		break;
1580 	case FW_COMP_PREBOOT:
1581 		ver_off = RREG32(static_loader->preboot_version_offset_reg);
1582 		dest = prop->preboot_ver;
1583 		name = "Preboot";
1584 		limit = static_loader->preboot_version_max_off;
1585 		break;
1586 	default:
1587 		dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1588 		return -EIO;
1589 	}
1590 
1591 	ver_off &= static_loader->sram_offset_mask;
1592 
1593 	if (ver_off < limit) {
1594 		memcpy_fromio(dest,
1595 			hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1596 			VERSION_MAX_LEN);
1597 	} else {
1598 		dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1599 								name, ver_off);
1600 		strscpy(dest, "unavailable", VERSION_MAX_LEN);
1601 		return -EIO;
1602 	}
1603 
1604 	if (fwc == FW_COMP_BOOT_FIT) {
1605 		boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1606 		if (boot_ver) {
1607 			dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1608 			kfree(boot_ver);
1609 		}
1610 	} else if (fwc == FW_COMP_PREBOOT) {
1611 		preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1612 						VERSION_MAX_LEN);
1613 		if (preboot_ver && preboot_ver != prop->preboot_ver) {
1614 			strscpy(btl_ver, prop->preboot_ver,
1615 				min((int) (preboot_ver - prop->preboot_ver),
1616 									31));
1617 			dev_info(hdev->dev, "%s\n", btl_ver);
1618 		}
1619 
1620 		preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1621 		if (preboot_ver) {
1622 			dev_info(hdev->dev, "preboot version %s\n",
1623 								preboot_ver);
1624 			kfree(preboot_ver);
1625 		}
1626 	}
1627 
1628 	return 0;
1629 }
1630 
1631 /**
1632  * hl_fw_preboot_update_state - update internal data structures during
1633  *                              handshake with preboot
1634  *
1635  *
1636  * @hdev: pointer to the habanalabs device structure
1637  *
1638  * @return 0 on success, otherwise non-zero error code
1639  */
hl_fw_preboot_update_state(struct hl_device * hdev)1640 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1641 {
1642 	struct asic_fixed_properties *prop = &hdev->asic_prop;
1643 	u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1644 
1645 	cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1646 	cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1647 
1648 	/* We read boot_dev_sts registers multiple times during boot:
1649 	 * 1. preboot - a. Check whether the security status bits are valid
1650 	 *              b. Check whether fw security is enabled
1651 	 *              c. Check whether hard reset is done by preboot
1652 	 * 2. boot cpu - a. Fetch boot cpu security status
1653 	 *               b. Check whether hard reset is done by boot cpu
1654 	 * 3. FW application - a. Fetch fw application security status
1655 	 *                     b. Check whether hard reset is done by fw app
1656 	 */
1657 	prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1658 
1659 	prop->fw_security_enabled = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_SECURITY_EN);
1660 
1661 	dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1662 							cpu_boot_dev_sts0);
1663 
1664 	dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1665 							cpu_boot_dev_sts1);
1666 
1667 	dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1668 			prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1669 
1670 	dev_dbg(hdev->dev, "firmware-level security is %s\n",
1671 			prop->fw_security_enabled ? "enabled" : "disabled");
1672 
1673 	dev_dbg(hdev->dev, "GIC controller is %s\n",
1674 			prop->gic_interrupts_enable ? "enabled" : "disabled");
1675 }
1676 
hl_fw_static_read_preboot_status(struct hl_device * hdev)1677 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1678 {
1679 	int rc;
1680 
1681 	rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1682 	if (rc)
1683 		return rc;
1684 
1685 	return 0;
1686 }
1687 
hl_fw_read_preboot_status(struct hl_device * hdev)1688 int hl_fw_read_preboot_status(struct hl_device *hdev)
1689 {
1690 	int rc;
1691 
1692 	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
1693 		return 0;
1694 
1695 	/* get FW pre-load parameters  */
1696 	hdev->asic_funcs->init_firmware_preload_params(hdev);
1697 
1698 	/*
1699 	 * In order to determine boot method (static VS dynamic) we need to
1700 	 * read the boot caps register
1701 	 */
1702 	rc = hl_fw_read_preboot_caps(hdev);
1703 	if (rc)
1704 		return rc;
1705 
1706 	hl_fw_preboot_update_state(hdev);
1707 
1708 	/* no need to read preboot status in dynamic load */
1709 	if (hdev->asic_prop.dynamic_fw_load)
1710 		return 0;
1711 
1712 	return hl_fw_static_read_preboot_status(hdev);
1713 }
1714 
1715 /* associate string with COMM status */
1716 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1717 	[COMMS_STS_NOOP] = "NOOP",
1718 	[COMMS_STS_ACK] = "ACK",
1719 	[COMMS_STS_OK] = "OK",
1720 	[COMMS_STS_ERR] = "ERR",
1721 	[COMMS_STS_VALID_ERR] = "VALID_ERR",
1722 	[COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1723 };
1724 
1725 /**
1726  * hl_fw_dynamic_report_error_status - report error status
1727  *
1728  * @hdev: pointer to the habanalabs device structure
1729  * @status: value of FW status register
1730  * @expected_status: the expected status
1731  */
hl_fw_dynamic_report_error_status(struct hl_device * hdev,u32 status,enum comms_sts expected_status)1732 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1733 						u32 status,
1734 						enum comms_sts expected_status)
1735 {
1736 	enum comms_sts comm_status =
1737 				FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1738 
1739 	if (comm_status < COMMS_STS_INVLD_LAST)
1740 		dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1741 				hl_dynamic_fw_status_str[comm_status],
1742 				hl_dynamic_fw_status_str[expected_status]);
1743 	else
1744 		dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1745 				comm_status,
1746 				hl_dynamic_fw_status_str[expected_status]);
1747 }
1748 
1749 /**
1750  * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1751  *
1752  * @hdev: pointer to the habanalabs device structure
1753  * @fw_loader: managing structure for loading device's FW
1754  * @cmd: LKD to FW cmd code
1755  * @size: size of next FW component to be loaded (0 if not necessary)
1756  *
1757  * LDK to FW exact command layout is defined at struct comms_command.
1758  * note: the size argument is used only when the next FW component should be
1759  *       loaded, otherwise it shall be 0. the size is used by the FW in later
1760  *       protocol stages and when sending only indicating the amount of memory
1761  *       to be allocated by the FW to receive the next boot component.
1762  */
hl_fw_dynamic_send_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_cmd cmd,unsigned int size)1763 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1764 				struct fw_load_mgr *fw_loader,
1765 				enum comms_cmd cmd, unsigned int size)
1766 {
1767 	struct cpu_dyn_regs *dyn_regs;
1768 	u32 val;
1769 
1770 	dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1771 
1772 	val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1773 	val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1774 
1775 	trace_habanalabs_comms_send_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1776 	WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1777 }
1778 
1779 /**
1780  * hl_fw_dynamic_extract_fw_response - update the FW response
1781  *
1782  * @hdev: pointer to the habanalabs device structure
1783  * @fw_loader: managing structure for loading device's FW
1784  * @response: FW response
1785  * @status: the status read from CPU status register
1786  *
1787  * @return 0 on success, otherwise non-zero error code
1788  */
hl_fw_dynamic_extract_fw_response(struct hl_device * hdev,struct fw_load_mgr * fw_loader,struct fw_response * response,u32 status)1789 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1790 						struct fw_load_mgr *fw_loader,
1791 						struct fw_response *response,
1792 						u32 status)
1793 {
1794 	response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1795 	response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1796 						COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1797 	response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1798 
1799 	if ((response->ram_type != COMMS_SRAM) &&
1800 					(response->ram_type != COMMS_DRAM)) {
1801 		dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1802 							response->ram_type);
1803 		return -EIO;
1804 	}
1805 
1806 	return 0;
1807 }
1808 
1809 /**
1810  * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1811  *
1812  * @hdev: pointer to the habanalabs device structure
1813  * @fw_loader: managing structure for loading device's FW
1814  * @expected_status: expected status to wait for
1815  * @timeout: timeout for status wait
1816  *
1817  * @return 0 on success, otherwise non-zero error code
1818  *
1819  * waiting for status from FW include polling the FW status register until
1820  * expected status is received or timeout occurs (whatever occurs first).
1821  */
hl_fw_dynamic_wait_for_status(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_sts expected_status,u32 timeout)1822 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1823 						struct fw_load_mgr *fw_loader,
1824 						enum comms_sts expected_status,
1825 						u32 timeout)
1826 {
1827 	struct cpu_dyn_regs *dyn_regs;
1828 	u32 status;
1829 	int rc;
1830 
1831 	dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1832 
1833 	trace_habanalabs_comms_wait_status(hdev->dev, comms_sts_str_arr[expected_status]);
1834 
1835 	/* Wait for expected status */
1836 	rc = hl_poll_timeout(
1837 		hdev,
1838 		le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1839 		status,
1840 		FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1841 		hdev->fw_comms_poll_interval_usec,
1842 		timeout);
1843 
1844 	if (rc) {
1845 		hl_fw_dynamic_report_error_status(hdev, status,
1846 							expected_status);
1847 		return -EIO;
1848 	}
1849 
1850 	trace_habanalabs_comms_wait_status_done(hdev->dev, comms_sts_str_arr[expected_status]);
1851 
1852 	/*
1853 	 * skip storing FW response for NOOP to preserve the actual desired
1854 	 * FW status
1855 	 */
1856 	if (expected_status == COMMS_STS_NOOP)
1857 		return 0;
1858 
1859 	rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1860 					&fw_loader->dynamic_loader.response,
1861 					status);
1862 	return rc;
1863 }
1864 
1865 /**
1866  * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1867  *
1868  * @hdev: pointer to the habanalabs device structure
1869  * @fw_loader: managing structure for loading device's FW
1870  *
1871  * @return 0 on success, otherwise non-zero error code
1872  *
1873  * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1874  * from FW) we need to clear the CPU status register in order to avoid garbage
1875  * between command cycles.
1876  * This is done by sending clear command and polling the CPU to LKD status
1877  * register to hold the status NOOP
1878  */
hl_fw_dynamic_send_clear_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader)1879 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1880 						struct fw_load_mgr *fw_loader)
1881 {
1882 	hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1883 
1884 	return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1885 							fw_loader->cpu_timeout);
1886 }
1887 
1888 /**
1889  * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1890  *
1891  * @hdev: pointer to the habanalabs device structure
1892  * @fw_loader: managing structure for loading device's FW
1893  * @cmd: LKD to FW cmd code
1894  * @size: size of next FW component to be loaded (0 if not necessary)
1895  * @wait_ok: if true also wait for OK response from FW
1896  * @timeout: timeout for status wait
1897  *
1898  * @return 0 on success, otherwise non-zero error code
1899  *
1900  * brief:
1901  * when sending protocol command we have the following steps:
1902  * - send clear (clear command and verify clear status register)
1903  * - send the actual protocol command
1904  * - wait for ACK on the protocol command
1905  * - send clear
1906  * - send NOOP
1907  * if, in addition, the specific protocol command should wait for OK then:
1908  * - wait for OK
1909  * - send clear
1910  * - send NOOP
1911  *
1912  * NOTES:
1913  * send clear: this is necessary in order to clear the status register to avoid
1914  *             leftovers between command
1915  * NOOP command: necessary to avoid loop on the clear command by the FW
1916  */
hl_fw_dynamic_send_protocol_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_cmd cmd,unsigned int size,bool wait_ok,u32 timeout)1917 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1918 				struct fw_load_mgr *fw_loader,
1919 				enum comms_cmd cmd, unsigned int size,
1920 				bool wait_ok, u32 timeout)
1921 {
1922 	int rc;
1923 
1924 	trace_habanalabs_comms_protocol_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1925 
1926 	/* first send clear command to clean former commands */
1927 	rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1928 	if (rc)
1929 		return rc;
1930 
1931 	/* send the actual command */
1932 	hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1933 
1934 	/* wait for ACK for the command */
1935 	rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1936 								timeout);
1937 	if (rc)
1938 		return rc;
1939 
1940 	/* clear command to prepare for NOOP command */
1941 	rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1942 	if (rc)
1943 		return rc;
1944 
1945 	/* send the actual NOOP command */
1946 	hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1947 
1948 	if (!wait_ok)
1949 		return 0;
1950 
1951 	rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1952 								timeout);
1953 	if (rc)
1954 		return rc;
1955 
1956 	/* clear command to prepare for NOOP command */
1957 	rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1958 	if (rc)
1959 		return rc;
1960 
1961 	/* send the actual NOOP command */
1962 	hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1963 
1964 	return 0;
1965 }
1966 
1967 /**
1968  * hl_fw_compat_crc32 - CRC compatible with FW
1969  *
1970  * @data: pointer to the data
1971  * @size: size of the data
1972  *
1973  * @return the CRC32 result
1974  *
1975  * NOTE: kernel's CRC32 differs from standard CRC32 calculation.
1976  *       in order to be aligned we need to flip the bits of both the input
1977  *       initial CRC and kernel's CRC32 result.
1978  *       in addition both sides use initial CRC of 0,
1979  */
hl_fw_compat_crc32(u8 * data,size_t size)1980 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1981 {
1982 	return ~crc32_le(~((u32)0), data, size);
1983 }
1984 
1985 /**
1986  * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1987  *                                        transfer (image or descriptor) between
1988  *                                        host and FW
1989  *
1990  * @hdev: pointer to the habanalabs device structure
1991  * @addr: device address of memory transfer
1992  * @size: memory transfer size
1993  * @region: PCI memory region
1994  *
1995  * @return 0 on success, otherwise non-zero error code
1996  */
hl_fw_dynamic_validate_memory_bound(struct hl_device * hdev,u64 addr,size_t size,struct pci_mem_region * region)1997 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1998 						u64 addr, size_t size,
1999 						struct pci_mem_region *region)
2000 {
2001 	u64 end_addr;
2002 
2003 	/* now make sure that the memory transfer is within region's bounds */
2004 	end_addr = addr + size;
2005 	if (end_addr >= region->region_base + region->region_size) {
2006 		dev_err(hdev->dev,
2007 			"dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
2008 							end_addr);
2009 		return -EIO;
2010 	}
2011 
2012 	/*
2013 	 * now make sure memory transfer is within predefined BAR bounds.
2014 	 * this is to make sure we do not need to set the bar (e.g. for DRAM
2015 	 * memory transfers)
2016 	 */
2017 	if (end_addr >= region->region_base - region->offset_in_bar +
2018 							region->bar_size) {
2019 		dev_err(hdev->dev,
2020 			"FW image beyond PCI BAR bounds\n");
2021 		return -EIO;
2022 	}
2023 
2024 	return 0;
2025 }
2026 
2027 /**
2028  * hl_fw_dynamic_validate_descriptor - validate FW descriptor
2029  *
2030  * @hdev: pointer to the habanalabs device structure
2031  * @fw_loader: managing structure for loading device's FW
2032  * @fw_desc: the descriptor from FW
2033  *
2034  * @return 0 on success, otherwise non-zero error code
2035  */
hl_fw_dynamic_validate_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader,struct lkd_fw_comms_desc * fw_desc)2036 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
2037 					struct fw_load_mgr *fw_loader,
2038 					struct lkd_fw_comms_desc *fw_desc)
2039 {
2040 	struct pci_mem_region *region;
2041 	enum pci_region region_id;
2042 	size_t data_size;
2043 	u32 data_crc32;
2044 	u8 *data_ptr;
2045 	u64 addr;
2046 	int rc;
2047 
2048 	if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC)
2049 		dev_dbg(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
2050 				fw_desc->header.magic);
2051 
2052 	if (fw_desc->header.version != HL_COMMS_DESC_VER)
2053 		dev_dbg(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
2054 				fw_desc->header.version);
2055 
2056 	/*
2057 	 * Calc CRC32 of data without header. use the size of the descriptor
2058 	 * reported by firmware, without calculating it ourself, to allow adding
2059 	 * more fields to the lkd_fw_comms_desc structure.
2060 	 * note that no alignment/stride address issues here as all structures
2061 	 * are 64 bit padded.
2062 	 */
2063 	data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
2064 	data_size = le16_to_cpu(fw_desc->header.size);
2065 
2066 	data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
2067 	if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
2068 		dev_err(hdev->dev, "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
2069 			data_crc32, fw_desc->header.crc32);
2070 		return -EIO;
2071 	}
2072 
2073 	/* find memory region to which to copy the image */
2074 	addr = le64_to_cpu(fw_desc->img_addr);
2075 	region_id = hl_get_pci_memory_region(hdev, addr);
2076 	if ((region_id != PCI_REGION_SRAM) && ((region_id != PCI_REGION_DRAM))) {
2077 		dev_err(hdev->dev, "Invalid region to copy FW image address=%llx\n", addr);
2078 		return -EIO;
2079 	}
2080 
2081 	region = &hdev->pci_mem_region[region_id];
2082 
2083 	/* store the region for the copy stage */
2084 	fw_loader->dynamic_loader.image_region = region;
2085 
2086 	/*
2087 	 * here we know that the start address is valid, now make sure that the
2088 	 * image is within region's bounds
2089 	 */
2090 	rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
2091 					fw_loader->dynamic_loader.fw_image_size,
2092 					region);
2093 	if (rc) {
2094 		dev_err(hdev->dev, "invalid mem transfer request for FW image\n");
2095 		return rc;
2096 	}
2097 
2098 	/* here we can mark the descriptor as valid as the content has been validated */
2099 	fw_loader->dynamic_loader.fw_desc_valid = true;
2100 
2101 	return 0;
2102 }
2103 
hl_fw_dynamic_validate_response(struct hl_device * hdev,struct fw_response * response,struct pci_mem_region * region)2104 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
2105 						struct fw_response *response,
2106 						struct pci_mem_region *region)
2107 {
2108 	u64 device_addr;
2109 	int rc;
2110 
2111 	device_addr = region->region_base + response->ram_offset;
2112 
2113 	/*
2114 	 * validate that the descriptor is within region's bounds
2115 	 * Note that as the start address was supplied according to the RAM
2116 	 * type- testing only the end address is enough
2117 	 */
2118 	rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
2119 					sizeof(struct lkd_fw_comms_desc),
2120 					region);
2121 	return rc;
2122 }
2123 
2124 /*
2125  * hl_fw_dynamic_read_descriptor_msg - read and show the ascii msg that sent by fw
2126  *
2127  * @hdev: pointer to the habanalabs device structure
2128  * @fw_desc: the descriptor from FW
2129  */
hl_fw_dynamic_read_descriptor_msg(struct hl_device * hdev,struct lkd_fw_comms_desc * fw_desc)2130 static void hl_fw_dynamic_read_descriptor_msg(struct hl_device *hdev,
2131 					struct lkd_fw_comms_desc *fw_desc)
2132 {
2133 	int i;
2134 	char *msg;
2135 
2136 	for (i = 0 ; i < LKD_FW_ASCII_MSG_MAX ; i++) {
2137 		if (!fw_desc->ascii_msg[i].valid)
2138 			return;
2139 
2140 		/* force NULL termination */
2141 		msg = fw_desc->ascii_msg[i].msg;
2142 		msg[LKD_FW_ASCII_MSG_MAX_LEN - 1] = '\0';
2143 
2144 		switch (fw_desc->ascii_msg[i].msg_lvl) {
2145 		case LKD_FW_ASCII_MSG_ERR:
2146 			dev_err(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2147 			break;
2148 		case LKD_FW_ASCII_MSG_WRN:
2149 			dev_warn(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2150 			break;
2151 		case LKD_FW_ASCII_MSG_INF:
2152 			dev_info(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2153 			break;
2154 		default:
2155 			dev_dbg(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2156 			break;
2157 		}
2158 	}
2159 }
2160 
2161 /**
2162  * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
2163  *
2164  * @hdev: pointer to the habanalabs device structure
2165  * @fw_loader: managing structure for loading device's FW
2166  *
2167  * @return 0 on success, otherwise non-zero error code
2168  */
hl_fw_dynamic_read_and_validate_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2169 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
2170 						struct fw_load_mgr *fw_loader)
2171 {
2172 	struct lkd_fw_comms_desc *fw_desc;
2173 	struct pci_mem_region *region;
2174 	struct fw_response *response;
2175 	void *temp_fw_desc;
2176 	void __iomem *src;
2177 	u16 fw_data_size;
2178 	enum pci_region region_id;
2179 	int rc;
2180 
2181 	fw_desc = &fw_loader->dynamic_loader.comm_desc;
2182 	response = &fw_loader->dynamic_loader.response;
2183 
2184 	region_id = (response->ram_type == COMMS_SRAM) ?
2185 					PCI_REGION_SRAM : PCI_REGION_DRAM;
2186 
2187 	region = &hdev->pci_mem_region[region_id];
2188 
2189 	rc = hl_fw_dynamic_validate_response(hdev, response, region);
2190 	if (rc) {
2191 		dev_err(hdev->dev,
2192 			"invalid mem transfer request for FW descriptor\n");
2193 		return rc;
2194 	}
2195 
2196 	/*
2197 	 * extract address to copy the descriptor from
2198 	 * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
2199 	 * as invalid.
2200 	 * it will be marked again as valid once validated
2201 	 */
2202 	fw_loader->dynamic_loader.fw_desc_valid = false;
2203 	src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2204 							response->ram_offset;
2205 
2206 	/*
2207 	 * We do the copy of the fw descriptor in 2 phases:
2208 	 * 1. copy the header + data info according to our lkd_fw_comms_desc definition.
2209 	 *    then we're able to read the actual data size provided by fw.
2210 	 *    this is needed for cases where data in descriptor was changed(add/remove)
2211 	 *    in embedded specs header file before updating lkd copy of the header file
2212 	 * 2. copy descriptor to temporary buffer with aligned size and send it to validation
2213 	 */
2214 	memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
2215 	fw_data_size = le16_to_cpu(fw_desc->header.size);
2216 
2217 	temp_fw_desc = vzalloc(sizeof(struct comms_desc_header) + fw_data_size);
2218 	if (!temp_fw_desc)
2219 		return -ENOMEM;
2220 
2221 	memcpy_fromio(temp_fw_desc, src, sizeof(struct comms_desc_header) + fw_data_size);
2222 
2223 	rc = hl_fw_dynamic_validate_descriptor(hdev, fw_loader,
2224 					(struct lkd_fw_comms_desc *) temp_fw_desc);
2225 
2226 	if (!rc)
2227 		hl_fw_dynamic_read_descriptor_msg(hdev, temp_fw_desc);
2228 
2229 	vfree(temp_fw_desc);
2230 
2231 	return rc;
2232 }
2233 
2234 /**
2235  * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
2236  *
2237  * @hdev: pointer to the habanalabs device structure
2238  * @fw_loader: managing structure for loading device's FW
2239  * @next_image_size: size to allocate for next FW component
2240  *
2241  * @return 0 on success, otherwise non-zero error code
2242  */
hl_fw_dynamic_request_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader,size_t next_image_size)2243 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
2244 						struct fw_load_mgr *fw_loader,
2245 						size_t next_image_size)
2246 {
2247 	int rc;
2248 
2249 	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
2250 						next_image_size, true,
2251 						fw_loader->cpu_timeout);
2252 	if (rc)
2253 		return rc;
2254 
2255 	return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
2256 }
2257 
2258 /**
2259  * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
2260  *
2261  * @hdev: pointer to the habanalabs device structure
2262  * @fwc: the firmware component
2263  * @fw_version: fw component's version string
2264  */
hl_fw_dynamic_read_device_fw_version(struct hl_device * hdev,enum hl_fw_component fwc,const char * fw_version)2265 static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
2266 					enum hl_fw_component fwc,
2267 					const char *fw_version)
2268 {
2269 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2270 	char *preboot_ver, *boot_ver;
2271 	char btl_ver[32];
2272 	int rc;
2273 
2274 	switch (fwc) {
2275 	case FW_COMP_BOOT_FIT:
2276 		strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
2277 		boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
2278 		if (boot_ver) {
2279 			dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
2280 			kfree(boot_ver);
2281 		}
2282 
2283 		break;
2284 	case FW_COMP_PREBOOT:
2285 		strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
2286 		preboot_ver = strnstr(prop->preboot_ver, "Preboot", VERSION_MAX_LEN);
2287 		dev_info(hdev->dev, "preboot full version: '%s'\n", preboot_ver);
2288 
2289 		if (preboot_ver && preboot_ver != prop->preboot_ver) {
2290 			strscpy(btl_ver, prop->preboot_ver,
2291 				min((int) (preboot_ver - prop->preboot_ver), 31));
2292 			dev_info(hdev->dev, "%s\n", btl_ver);
2293 		}
2294 
2295 		rc = hl_get_sw_major_minor_subminor(hdev, preboot_ver);
2296 		if (rc)
2297 			return rc;
2298 		preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
2299 		if (preboot_ver) {
2300 			rc = hl_get_preboot_major_minor(hdev, preboot_ver);
2301 			kfree(preboot_ver);
2302 			if (rc)
2303 				return rc;
2304 		}
2305 
2306 		break;
2307 	default:
2308 		dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2309 		return -EINVAL;
2310 	}
2311 
2312 	return 0;
2313 }
2314 
2315 /**
2316  * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
2317  *
2318  * @hdev: pointer to the habanalabs device structure
2319  * @fw: fw descriptor
2320  * @fw_loader: managing structure for loading device's FW
2321  */
hl_fw_dynamic_copy_image(struct hl_device * hdev,const struct firmware * fw,struct fw_load_mgr * fw_loader)2322 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
2323 						const struct firmware *fw,
2324 						struct fw_load_mgr *fw_loader)
2325 {
2326 	struct lkd_fw_comms_desc *fw_desc;
2327 	struct pci_mem_region *region;
2328 	void __iomem *dest;
2329 	u64 addr;
2330 	int rc;
2331 
2332 	fw_desc = &fw_loader->dynamic_loader.comm_desc;
2333 	addr = le64_to_cpu(fw_desc->img_addr);
2334 
2335 	/* find memory region to which to copy the image */
2336 	region = fw_loader->dynamic_loader.image_region;
2337 
2338 	dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2339 					(addr - region->region_base);
2340 
2341 	rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
2342 					fw_loader->boot_fit_img.src_off,
2343 					fw_loader->boot_fit_img.copy_size);
2344 
2345 	return rc;
2346 }
2347 
2348 /**
2349  * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
2350  *
2351  * @hdev: pointer to the habanalabs device structure
2352  * @msg: message
2353  * @fw_loader: managing structure for loading device's FW
2354  */
hl_fw_dynamic_copy_msg(struct hl_device * hdev,struct lkd_msg_comms * msg,struct fw_load_mgr * fw_loader)2355 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
2356 		struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
2357 {
2358 	struct lkd_fw_comms_desc *fw_desc;
2359 	struct pci_mem_region *region;
2360 	void __iomem *dest;
2361 	u64 addr;
2362 	int rc;
2363 
2364 	fw_desc = &fw_loader->dynamic_loader.comm_desc;
2365 	addr = le64_to_cpu(fw_desc->img_addr);
2366 
2367 	/* find memory region to which to copy the image */
2368 	region = fw_loader->dynamic_loader.image_region;
2369 
2370 	dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2371 					(addr - region->region_base);
2372 
2373 	rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
2374 
2375 	return rc;
2376 }
2377 
2378 /**
2379  * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
2380  *                               is loaded
2381  *
2382  * @hdev: pointer to the habanalabs device structure
2383  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2384  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2385  *
2386  * @return 0 on success, otherwise non-zero error code
2387  */
hl_fw_boot_fit_update_state(struct hl_device * hdev,u32 cpu_boot_dev_sts0_reg,u32 cpu_boot_dev_sts1_reg)2388 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
2389 						u32 cpu_boot_dev_sts0_reg,
2390 						u32 cpu_boot_dev_sts1_reg)
2391 {
2392 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2393 
2394 	hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;
2395 
2396 	/* Read boot_cpu status bits */
2397 	if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
2398 		prop->fw_bootfit_cpu_boot_dev_sts0 =
2399 				RREG32(cpu_boot_dev_sts0_reg);
2400 
2401 		prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &
2402 							CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2403 
2404 		dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
2405 					prop->fw_bootfit_cpu_boot_dev_sts0);
2406 	}
2407 
2408 	if (prop->fw_cpu_boot_dev_sts1_valid) {
2409 		prop->fw_bootfit_cpu_boot_dev_sts1 =
2410 				RREG32(cpu_boot_dev_sts1_reg);
2411 
2412 		dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
2413 					prop->fw_bootfit_cpu_boot_dev_sts1);
2414 	}
2415 
2416 	dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
2417 			prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2418 }
2419 
hl_fw_dynamic_update_linux_interrupt_if(struct hl_device * hdev)2420 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
2421 {
2422 	struct cpu_dyn_regs *dyn_regs =
2423 			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2424 
2425 	/* Check whether all 3 interrupt interfaces are set, if not use a
2426 	 * single interface
2427 	 */
2428 	if (!hdev->asic_prop.gic_interrupts_enable &&
2429 			!(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2430 				CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
2431 		dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
2432 		dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
2433 
2434 		dev_warn(hdev->dev,
2435 			"Using a single interrupt interface towards cpucp");
2436 	}
2437 }
2438 /**
2439  * hl_fw_dynamic_load_image - load FW image using dynamic protocol
2440  *
2441  * @hdev: pointer to the habanalabs device structure
2442  * @fw_loader: managing structure for loading device's FW
2443  * @load_fwc: the FW component to be loaded
2444  * @img_ld_timeout: image load timeout
2445  *
2446  * @return 0 on success, otherwise non-zero error code
2447  */
hl_fw_dynamic_load_image(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum hl_fw_component load_fwc,u32 img_ld_timeout)2448 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
2449 						struct fw_load_mgr *fw_loader,
2450 						enum hl_fw_component load_fwc,
2451 						u32 img_ld_timeout)
2452 {
2453 	enum hl_fw_component cur_fwc;
2454 	const struct firmware *fw;
2455 	char *fw_name;
2456 	int rc = 0;
2457 
2458 	/*
2459 	 * when loading image we have one of 2 scenarios:
2460 	 * 1. current FW component is preboot and we want to load boot-fit
2461 	 * 2. current FW component is boot-fit and we want to load linux
2462 	 */
2463 	if (load_fwc == FW_COMP_BOOT_FIT) {
2464 		cur_fwc = FW_COMP_PREBOOT;
2465 		fw_name = fw_loader->boot_fit_img.image_name;
2466 	} else {
2467 		cur_fwc = FW_COMP_BOOT_FIT;
2468 		fw_name = fw_loader->linux_img.image_name;
2469 	}
2470 
2471 	/* request FW in order to communicate to FW the size to be allocated */
2472 	rc = hl_request_fw(hdev, &fw, fw_name);
2473 	if (rc)
2474 		return rc;
2475 
2476 	/* store the image size for future validation */
2477 	fw_loader->dynamic_loader.fw_image_size = fw->size;
2478 
2479 	rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2480 	if (rc)
2481 		goto release_fw;
2482 
2483 	/* read preboot version */
2484 	rc = hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2485 				fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2486 	if (rc)
2487 		goto release_fw;
2488 
2489 	/* copy boot fit to space allocated by FW */
2490 	rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2491 	if (rc)
2492 		goto release_fw;
2493 
2494 	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2495 						0, true,
2496 						fw_loader->cpu_timeout);
2497 	if (rc)
2498 		goto release_fw;
2499 
2500 	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2501 						0, false,
2502 						img_ld_timeout);
2503 
2504 release_fw:
2505 	hl_release_firmware(fw);
2506 	return rc;
2507 }
2508 
hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2509 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2510 					struct fw_load_mgr *fw_loader)
2511 {
2512 	struct dynamic_fw_load_mgr *dyn_loader;
2513 	u32 status;
2514 	int rc;
2515 
2516 	dyn_loader = &fw_loader->dynamic_loader;
2517 
2518 	/*
2519 	 * Make sure CPU boot-loader is running
2520 	 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2521 	 * yet there is a debug scenario in which we loading uboot (without Linux)
2522 	 * which at later stage is relocated to DRAM. In this case we expect
2523 	 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2524 	 * poll flags
2525 	 */
2526 	rc = hl_poll_timeout(
2527 		hdev,
2528 		le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2529 		status,
2530 		(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2531 		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
2532 		hdev->fw_poll_interval_usec,
2533 		dyn_loader->wait_for_bl_timeout);
2534 	if (rc) {
2535 		dev_err(hdev->dev, "failed to wait for boot (status = %d)\n", status);
2536 		return rc;
2537 	}
2538 
2539 	dev_dbg(hdev->dev, "uboot status = %d\n", status);
2540 	return 0;
2541 }
2542 
hl_fw_dynamic_wait_for_linux_active(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2543 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2544 						struct fw_load_mgr *fw_loader)
2545 {
2546 	struct dynamic_fw_load_mgr *dyn_loader;
2547 	u32 status;
2548 	int rc;
2549 
2550 	dyn_loader = &fw_loader->dynamic_loader;
2551 
2552 	/* Make sure CPU linux is running */
2553 
2554 	rc = hl_poll_timeout(
2555 		hdev,
2556 		le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2557 		status,
2558 		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
2559 		hdev->fw_poll_interval_usec,
2560 		fw_loader->cpu_timeout);
2561 	if (rc) {
2562 		dev_err(hdev->dev, "failed to wait for Linux (status = %d)\n", status);
2563 		return rc;
2564 	}
2565 
2566 	dev_dbg(hdev->dev, "Boot status = %d\n", status);
2567 	return 0;
2568 }
2569 
2570 /**
2571  * hl_fw_linux_update_state -	update internal data structures after Linux
2572  *				is loaded.
2573  *				Note: Linux initialization is comprised mainly
2574  *				of two stages - loading kernel (SRAM_AVAIL)
2575  *				& loading ARMCP.
2576  *				Therefore reading boot device status in any of
2577  *				these stages might result in different values.
2578  *
2579  * @hdev: pointer to the habanalabs device structure
2580  * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2581  * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2582  *
2583  * @return 0 on success, otherwise non-zero error code
2584  */
hl_fw_linux_update_state(struct hl_device * hdev,u32 cpu_boot_dev_sts0_reg,u32 cpu_boot_dev_sts1_reg)2585 static void hl_fw_linux_update_state(struct hl_device *hdev,
2586 						u32 cpu_boot_dev_sts0_reg,
2587 						u32 cpu_boot_dev_sts1_reg)
2588 {
2589 	struct asic_fixed_properties *prop = &hdev->asic_prop;
2590 
2591 	hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;
2592 
2593 	/* Read FW application security bits */
2594 	if (prop->fw_cpu_boot_dev_sts0_valid) {
2595 		prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2596 
2597 		prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &
2598 							CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2599 
2600 		if (prop->fw_app_cpu_boot_dev_sts0 &
2601 				CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2602 			prop->gic_interrupts_enable = false;
2603 
2604 		dev_dbg(hdev->dev,
2605 			"Firmware application CPU status0 %#x\n",
2606 			prop->fw_app_cpu_boot_dev_sts0);
2607 
2608 		dev_dbg(hdev->dev, "GIC controller is %s\n",
2609 				prop->gic_interrupts_enable ?
2610 						"enabled" : "disabled");
2611 	}
2612 
2613 	if (prop->fw_cpu_boot_dev_sts1_valid) {
2614 		prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2615 
2616 		dev_dbg(hdev->dev,
2617 			"Firmware application CPU status1 %#x\n",
2618 			prop->fw_app_cpu_boot_dev_sts1);
2619 	}
2620 
2621 	dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2622 			prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2623 
2624 	dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2625 }
2626 
2627 /**
2628  * hl_fw_dynamic_send_msg - send a COMMS message with attached data
2629  *
2630  * @hdev: pointer to the habanalabs device structure
2631  * @fw_loader: managing structure for loading device's FW
2632  * @msg_type: message type
2633  * @data: data to be sent
2634  *
2635  * @return 0 on success, otherwise non-zero error code
2636  */
hl_fw_dynamic_send_msg(struct hl_device * hdev,struct fw_load_mgr * fw_loader,u8 msg_type,void * data)2637 static int hl_fw_dynamic_send_msg(struct hl_device *hdev,
2638 		struct fw_load_mgr *fw_loader, u8 msg_type, void *data)
2639 {
2640 	struct lkd_msg_comms *msg;
2641 	int rc;
2642 
2643 	msg = kzalloc(sizeof(*msg), GFP_KERNEL);
2644 	if (!msg)
2645 		return -ENOMEM;
2646 
2647 	/* create message to be sent */
2648 	msg->header.type = msg_type;
2649 	msg->header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2650 	msg->header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2651 
2652 	switch (msg_type) {
2653 	case HL_COMMS_RESET_CAUSE_TYPE:
2654 		msg->reset_cause = *(__u8 *) data;
2655 		break;
2656 
2657 	default:
2658 		dev_err(hdev->dev,
2659 			"Send COMMS message - invalid message type %u\n",
2660 			msg_type);
2661 		rc = -EINVAL;
2662 		goto out;
2663 	}
2664 
2665 	rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2666 			sizeof(struct lkd_msg_comms));
2667 	if (rc)
2668 		goto out;
2669 
2670 	/* copy message to space allocated by FW */
2671 	rc = hl_fw_dynamic_copy_msg(hdev, msg, fw_loader);
2672 	if (rc)
2673 		goto out;
2674 
2675 	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2676 						0, true,
2677 						fw_loader->cpu_timeout);
2678 	if (rc)
2679 		goto out;
2680 
2681 	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2682 						0, true,
2683 						fw_loader->cpu_timeout);
2684 
2685 out:
2686 	kfree(msg);
2687 	return rc;
2688 }
2689 
2690 /**
2691  * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2692  *
2693  * @hdev: pointer to the habanalabs device structure
2694  * @fw_loader: managing structure for loading device's FW
2695  *
2696  * @return 0 on success, otherwise non-zero error code
2697  *
2698  * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2699  * the communication is done using registers:
2700  * - LKD command register
2701  * - FW status register
2702  * the protocol is race free. this goal is achieved by splitting the requests
2703  * and response to known synchronization points between the LKD and the FW.
2704  * each response to LKD request is known and bound to a predefined timeout.
2705  * in case of timeout expiration without the desired status from FW- the
2706  * protocol (and hence the boot) will fail.
2707  */
hl_fw_dynamic_init_cpu(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2708 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2709 					struct fw_load_mgr *fw_loader)
2710 {
2711 	struct cpu_dyn_regs *dyn_regs;
2712 	int rc, fw_error_rc;
2713 
2714 	dev_info(hdev->dev,
2715 		"Loading %sfirmware to device, may take some time...\n",
2716 		hdev->asic_prop.fw_security_enabled ? "secured " : "");
2717 
2718 	/* initialize FW descriptor as invalid */
2719 	fw_loader->dynamic_loader.fw_desc_valid = false;
2720 
2721 	/*
2722 	 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2723 	 * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2724 	 */
2725 	dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2726 
2727 	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2728 						0, true,
2729 						fw_loader->cpu_timeout);
2730 	if (rc)
2731 		goto protocol_err;
2732 
2733 	if (hdev->reset_info.curr_reset_cause) {
2734 		rc = hl_fw_dynamic_send_msg(hdev, fw_loader,
2735 				HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);
2736 		if (rc)
2737 			goto protocol_err;
2738 
2739 		/* Clear current reset cause */
2740 		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2741 	}
2742 
2743 	if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2744 		struct lkd_fw_binning_info *binning_info;
2745 
2746 		rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2747 		if (rc)
2748 			goto protocol_err;
2749 
2750 		/* read preboot version */
2751 		rc = hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2752 				fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2753 
2754 		if (rc)
2755 			return rc;
2756 
2757 		/* read binning info from preboot */
2758 		if (hdev->support_preboot_binning) {
2759 			binning_info = &fw_loader->dynamic_loader.comm_desc.binning_info;
2760 			hdev->tpc_binning = le64_to_cpu(binning_info->tpc_mask_l);
2761 			hdev->dram_binning = le32_to_cpu(binning_info->dram_mask);
2762 			hdev->edma_binning = le32_to_cpu(binning_info->edma_mask);
2763 			hdev->decoder_binning = le32_to_cpu(binning_info->dec_mask);
2764 			hdev->rotator_binning = le32_to_cpu(binning_info->rot_mask);
2765 
2766 			rc = hdev->asic_funcs->set_dram_properties(hdev);
2767 			if (rc)
2768 				return rc;
2769 
2770 			rc = hdev->asic_funcs->set_binning_masks(hdev);
2771 			if (rc)
2772 				return rc;
2773 
2774 			dev_dbg(hdev->dev,
2775 				"Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x, rot:0x%x\n",
2776 				hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
2777 				hdev->decoder_binning, hdev->rotator_binning);
2778 		}
2779 
2780 		return 0;
2781 	}
2782 
2783 	/* load boot fit to FW */
2784 	rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2785 						fw_loader->boot_fit_timeout);
2786 	if (rc) {
2787 		dev_err(hdev->dev, "failed to load boot fit\n");
2788 		goto protocol_err;
2789 	}
2790 
2791 	rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2792 	if (rc)
2793 		goto protocol_err;
2794 
2795 	hl_fw_boot_fit_update_state(hdev,
2796 			le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2797 			le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2798 
2799 	/*
2800 	 * when testing FW load (without Linux) on PLDM we don't want to
2801 	 * wait until boot fit is active as it may take several hours.
2802 	 * instead, we load the bootfit and let it do all initialization in
2803 	 * the background.
2804 	 */
2805 	if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))
2806 		return 0;
2807 
2808 	/* Enable DRAM scrambling before Linux boot and after successful
2809 	 *  UBoot
2810 	 */
2811 	hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2812 
2813 	if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2814 		dev_info(hdev->dev, "Skip loading Linux F/W\n");
2815 		return 0;
2816 	}
2817 
2818 	if (fw_loader->skip_bmc) {
2819 		rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2820 							COMMS_SKIP_BMC, 0,
2821 							true,
2822 							fw_loader->cpu_timeout);
2823 		if (rc) {
2824 			dev_err(hdev->dev, "failed to load boot fit\n");
2825 			goto protocol_err;
2826 		}
2827 	}
2828 
2829 	/* load Linux image to FW */
2830 	rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2831 							fw_loader->cpu_timeout);
2832 	if (rc) {
2833 		dev_err(hdev->dev, "failed to load Linux\n");
2834 		goto protocol_err;
2835 	}
2836 
2837 	rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2838 	if (rc)
2839 		goto protocol_err;
2840 
2841 	hl_fw_linux_update_state(hdev,
2842 				le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2843 				le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2844 
2845 	hl_fw_dynamic_update_linux_interrupt_if(hdev);
2846 
2847 protocol_err:
2848 	if (fw_loader->dynamic_loader.fw_desc_valid) {
2849 		fw_error_rc = fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2850 				le32_to_cpu(dyn_regs->cpu_boot_err1),
2851 				le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2852 				le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2853 
2854 		if (fw_error_rc)
2855 			return fw_error_rc;
2856 	}
2857 
2858 	return rc;
2859 }
2860 
2861 /**
2862  * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2863  *
2864  * @hdev: pointer to the habanalabs device structure
2865  * @fw_loader: managing structure for loading device's FW
2866  *
2867  * @return 0 on success, otherwise non-zero error code
2868  */
hl_fw_static_init_cpu(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2869 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2870 					struct fw_load_mgr *fw_loader)
2871 {
2872 	u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2873 	u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2874 	struct static_fw_load_mgr *static_loader;
2875 	u32 cpu_boot_status_reg;
2876 	int rc;
2877 
2878 	if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2879 		return 0;
2880 
2881 	/* init common loader parameters */
2882 	cpu_timeout = fw_loader->cpu_timeout;
2883 
2884 	/* init static loader parameters */
2885 	static_loader = &fw_loader->static_loader;
2886 	cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2887 	msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2888 	cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2889 	cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2890 	cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2891 
2892 	dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2893 		cpu_timeout / USEC_PER_SEC);
2894 
2895 	/* Wait for boot FIT request */
2896 	rc = hl_poll_timeout(
2897 		hdev,
2898 		cpu_boot_status_reg,
2899 		status,
2900 		status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2901 		hdev->fw_poll_interval_usec,
2902 		fw_loader->boot_fit_timeout);
2903 
2904 	if (rc) {
2905 		dev_dbg(hdev->dev,
2906 			"No boot fit request received (status = %d), resuming boot\n", status);
2907 	} else {
2908 		rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2909 		if (rc)
2910 			goto out;
2911 
2912 		/* Clear device CPU message status */
2913 		WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2914 
2915 		/* Signal device CPU that boot loader is ready */
2916 		WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2917 
2918 		/* Poll for CPU device ack */
2919 		rc = hl_poll_timeout(
2920 			hdev,
2921 			cpu_msg_status_reg,
2922 			status,
2923 			status == CPU_MSG_OK,
2924 			hdev->fw_poll_interval_usec,
2925 			fw_loader->boot_fit_timeout);
2926 
2927 		if (rc) {
2928 			dev_err(hdev->dev,
2929 				"Timeout waiting for boot fit load ack (status = %d)\n", status);
2930 			goto out;
2931 		}
2932 
2933 		/* Clear message */
2934 		WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2935 	}
2936 
2937 	/*
2938 	 * Make sure CPU boot-loader is running
2939 	 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2940 	 * yet there is a debug scenario in which we loading uboot (without Linux)
2941 	 * which at later stage is relocated to DRAM. In this case we expect
2942 	 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2943 	 * poll flags
2944 	 */
2945 	rc = hl_poll_timeout(
2946 		hdev,
2947 		cpu_boot_status_reg,
2948 		status,
2949 		(status == CPU_BOOT_STATUS_DRAM_RDY) ||
2950 		(status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2951 		(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2952 		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
2953 		hdev->fw_poll_interval_usec,
2954 		cpu_timeout);
2955 
2956 	dev_dbg(hdev->dev, "uboot status = %d\n", status);
2957 
2958 	/* Read U-Boot version now in case we will later fail */
2959 	hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2960 
2961 	/* update state according to boot stage */
2962 	hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2963 						cpu_boot_dev_status1_reg);
2964 
2965 	if (rc) {
2966 		detect_cpu_boot_status(hdev, status);
2967 		rc = -EIO;
2968 		goto out;
2969 	}
2970 
2971 	/* Enable DRAM scrambling before Linux boot and after successful
2972 	 *  UBoot
2973 	 */
2974 	hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2975 
2976 	if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2977 		dev_info(hdev->dev, "Skip loading Linux F/W\n");
2978 		rc = 0;
2979 		goto out;
2980 	}
2981 
2982 	if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2983 		rc = 0;
2984 		goto out;
2985 	}
2986 
2987 	dev_info(hdev->dev,
2988 		"Loading firmware to device, may take some time...\n");
2989 
2990 	rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2991 	if (rc)
2992 		goto out;
2993 
2994 	if (fw_loader->skip_bmc) {
2995 		WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2996 
2997 		rc = hl_poll_timeout(
2998 			hdev,
2999 			cpu_boot_status_reg,
3000 			status,
3001 			(status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
3002 			hdev->fw_poll_interval_usec,
3003 			cpu_timeout);
3004 
3005 		if (rc) {
3006 			dev_err(hdev->dev,
3007 				"Failed to get ACK on skipping BMC (status = %d)\n",
3008 				status);
3009 			WREG32(msg_to_cpu_reg, KMD_MSG_NA);
3010 			rc = -EIO;
3011 			goto out;
3012 		}
3013 	}
3014 
3015 	WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
3016 
3017 	rc = hl_poll_timeout(
3018 		hdev,
3019 		cpu_boot_status_reg,
3020 		status,
3021 		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
3022 		hdev->fw_poll_interval_usec,
3023 		cpu_timeout);
3024 
3025 	/* Clear message */
3026 	WREG32(msg_to_cpu_reg, KMD_MSG_NA);
3027 
3028 	if (rc) {
3029 		if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
3030 			dev_err(hdev->dev,
3031 				"Device reports FIT image is corrupted\n");
3032 		else
3033 			dev_err(hdev->dev,
3034 				"Failed to load firmware to device (status = %d)\n",
3035 				status);
3036 
3037 		rc = -EIO;
3038 		goto out;
3039 	}
3040 
3041 	rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3042 					fw_loader->static_loader.boot_err1_reg,
3043 					cpu_boot_dev_status0_reg,
3044 					cpu_boot_dev_status1_reg);
3045 	if (rc)
3046 		return rc;
3047 
3048 	hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
3049 						cpu_boot_dev_status1_reg);
3050 
3051 	return 0;
3052 
3053 out:
3054 	fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3055 					fw_loader->static_loader.boot_err1_reg,
3056 					cpu_boot_dev_status0_reg,
3057 					cpu_boot_dev_status1_reg);
3058 
3059 	return rc;
3060 }
3061 
3062 /**
3063  * hl_fw_init_cpu - initialize the device CPU
3064  *
3065  * @hdev: pointer to the habanalabs device structure
3066  *
3067  * @return 0 on success, otherwise non-zero error code
3068  *
3069  * perform necessary initializations for device's CPU. takes into account if
3070  * init protocol is static or dynamic.
3071  */
hl_fw_init_cpu(struct hl_device * hdev)3072 int hl_fw_init_cpu(struct hl_device *hdev)
3073 {
3074 	struct asic_fixed_properties *prop = &hdev->asic_prop;
3075 	struct fw_load_mgr *fw_loader = &hdev->fw_loader;
3076 
3077 	return  prop->dynamic_fw_load ?
3078 			hl_fw_dynamic_init_cpu(hdev, fw_loader) :
3079 			hl_fw_static_init_cpu(hdev, fw_loader);
3080 }
3081 
hl_fw_set_pll_profile(struct hl_device * hdev)3082 void hl_fw_set_pll_profile(struct hl_device *hdev)
3083 {
3084 	hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,
3085 				hdev->asic_prop.max_freq_value);
3086 }
3087 
hl_fw_get_clk_rate(struct hl_device * hdev,u32 * cur_clk,u32 * max_clk)3088 int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
3089 {
3090 	long value;
3091 
3092 	if (!hl_device_operational(hdev, NULL))
3093 		return -ENODEV;
3094 
3095 	if (!hdev->pdev) {
3096 		*cur_clk = 0;
3097 		*max_clk = 0;
3098 		return 0;
3099 	}
3100 
3101 	value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);
3102 
3103 	if (value < 0) {
3104 		dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);
3105 		return value;
3106 	}
3107 
3108 	*max_clk = (value / 1000 / 1000);
3109 
3110 	value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);
3111 
3112 	if (value < 0) {
3113 		dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);
3114 		return value;
3115 	}
3116 
3117 	*cur_clk = (value / 1000 / 1000);
3118 
3119 	return 0;
3120 }
3121 
hl_fw_get_frequency(struct hl_device * hdev,u32 pll_index,bool curr)3122 long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
3123 {
3124 	struct cpucp_packet pkt;
3125 	u32 used_pll_idx;
3126 	u64 result;
3127 	int rc;
3128 
3129 	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3130 	if (rc)
3131 		return rc;
3132 
3133 	memset(&pkt, 0, sizeof(pkt));
3134 
3135 	if (curr)
3136 		pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
3137 						CPUCP_PKT_CTL_OPCODE_SHIFT);
3138 	else
3139 		pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3140 
3141 	pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3142 
3143 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3144 
3145 	if (rc) {
3146 		dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",
3147 			used_pll_idx, rc);
3148 		return rc;
3149 	}
3150 
3151 	return (long) result;
3152 }
3153 
hl_fw_set_frequency(struct hl_device * hdev,u32 pll_index,u64 freq)3154 void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
3155 {
3156 	struct cpucp_packet pkt;
3157 	u32 used_pll_idx;
3158 	int rc;
3159 
3160 	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3161 	if (rc)
3162 		return;
3163 
3164 	memset(&pkt, 0, sizeof(pkt));
3165 
3166 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3167 	pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3168 	pkt.value = cpu_to_le64(freq);
3169 
3170 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3171 
3172 	if (rc)
3173 		dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",
3174 			used_pll_idx, rc);
3175 }
3176 
hl_fw_get_max_power(struct hl_device * hdev)3177 long hl_fw_get_max_power(struct hl_device *hdev)
3178 {
3179 	struct cpucp_packet pkt;
3180 	u64 result;
3181 	int rc;
3182 
3183 	memset(&pkt, 0, sizeof(pkt));
3184 
3185 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3186 
3187 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3188 
3189 	if (rc) {
3190 		dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);
3191 		return rc;
3192 	}
3193 
3194 	return result;
3195 }
3196 
hl_fw_set_max_power(struct hl_device * hdev)3197 void hl_fw_set_max_power(struct hl_device *hdev)
3198 {
3199 	struct cpucp_packet pkt;
3200 	int rc;
3201 
3202 	/* TODO: remove this after simulator supports this packet */
3203 	if (!hdev->pdev)
3204 		return;
3205 
3206 	memset(&pkt, 0, sizeof(pkt));
3207 
3208 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3209 	pkt.value = cpu_to_le64(hdev->max_power);
3210 
3211 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3212 
3213 	if (rc)
3214 		dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);
3215 }
3216 
hl_fw_get_sec_attest_data(struct hl_device * hdev,u32 packet_id,void * data,u32 size,u32 nonce,u32 timeout)3217 static int hl_fw_get_sec_attest_data(struct hl_device *hdev, u32 packet_id, void *data, u32 size,
3218 					u32 nonce, u32 timeout)
3219 {
3220 	struct cpucp_packet pkt = {};
3221 	dma_addr_t req_dma_addr;
3222 	void *req_cpu_addr;
3223 	int rc;
3224 
3225 	req_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, size, &req_dma_addr);
3226 	if (!req_cpu_addr) {
3227 		dev_err(hdev->dev,
3228 			"Failed to allocate DMA memory for CPU-CP packet %u\n", packet_id);
3229 		return -ENOMEM;
3230 	}
3231 
3232 	memset(data, 0, size);
3233 
3234 	pkt.ctl = cpu_to_le32(packet_id << CPUCP_PKT_CTL_OPCODE_SHIFT);
3235 	pkt.addr = cpu_to_le64(req_dma_addr);
3236 	pkt.data_max_size = cpu_to_le32(size);
3237 	pkt.nonce = cpu_to_le32(nonce);
3238 
3239 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
3240 					timeout, NULL);
3241 	if (rc) {
3242 		dev_err(hdev->dev,
3243 			"Failed to handle CPU-CP pkt %u, error %d\n", packet_id, rc);
3244 		goto out;
3245 	}
3246 
3247 	memcpy(data, req_cpu_addr, size);
3248 
3249 out:
3250 	hl_cpu_accessible_dma_pool_free(hdev, size, req_cpu_addr);
3251 
3252 	return rc;
3253 }
3254 
hl_fw_get_sec_attest_info(struct hl_device * hdev,struct cpucp_sec_attest_info * sec_attest_info,u32 nonce)3255 int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info,
3256 				u32 nonce)
3257 {
3258 	return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_SEC_ATTEST_GET, sec_attest_info,
3259 					sizeof(struct cpucp_sec_attest_info), nonce,
3260 					HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC);
3261 }
3262 
hl_fw_send_generic_request(struct hl_device * hdev,enum hl_passthrough_type sub_opcode,dma_addr_t buff,u32 * size)3263 int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode,
3264 						dma_addr_t buff, u32 *size)
3265 {
3266 	struct cpucp_packet pkt = {};
3267 	u64 result;
3268 	int rc = 0;
3269 
3270 	pkt.ctl = cpu_to_le32(CPUCP_PACKET_GENERIC_PASSTHROUGH << CPUCP_PKT_CTL_OPCODE_SHIFT);
3271 	pkt.addr = cpu_to_le64(buff);
3272 	pkt.data_max_size = cpu_to_le32(*size);
3273 	pkt.pkt_subidx = cpu_to_le32(sub_opcode);
3274 
3275 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)&pkt, sizeof(pkt),
3276 						HL_CPUCP_INFO_TIMEOUT_USEC, &result);
3277 	if (rc)
3278 		dev_err(hdev->dev, "failed to send CPUCP data of generic fw pkt\n");
3279 	else
3280 		dev_dbg(hdev->dev, "generic pkt was successful, result: 0x%llx\n", result);
3281 
3282 	*size = (u32)result;
3283 
3284 	return rc;
3285 }
3286