1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4 * Copyright 2016-2022 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
10
11 #include <linux/firmware.h>
12 #include <linux/crc32.h>
13 #include <linux/slab.h>
14 #include <linux/ctype.h>
15 #include <linux/vmalloc.h>
16
17 #include <trace/events/habanalabs.h>
18
19 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
20
21 static char *comms_cmd_str_arr[COMMS_INVLD_LAST] = {
22 [COMMS_NOOP] = __stringify(COMMS_NOOP),
23 [COMMS_CLR_STS] = __stringify(COMMS_CLR_STS),
24 [COMMS_RST_STATE] = __stringify(COMMS_RST_STATE),
25 [COMMS_PREP_DESC] = __stringify(COMMS_PREP_DESC),
26 [COMMS_DATA_RDY] = __stringify(COMMS_DATA_RDY),
27 [COMMS_EXEC] = __stringify(COMMS_EXEC),
28 [COMMS_RST_DEV] = __stringify(COMMS_RST_DEV),
29 [COMMS_GOTO_WFE] = __stringify(COMMS_GOTO_WFE),
30 [COMMS_SKIP_BMC] = __stringify(COMMS_SKIP_BMC),
31 [COMMS_PREP_DESC_ELBI] = __stringify(COMMS_PREP_DESC_ELBI),
32 };
33
34 static char *comms_sts_str_arr[COMMS_STS_INVLD_LAST] = {
35 [COMMS_STS_NOOP] = __stringify(COMMS_STS_NOOP),
36 [COMMS_STS_ACK] = __stringify(COMMS_STS_ACK),
37 [COMMS_STS_OK] = __stringify(COMMS_STS_OK),
38 [COMMS_STS_ERR] = __stringify(COMMS_STS_ERR),
39 [COMMS_STS_VALID_ERR] = __stringify(COMMS_STS_VALID_ERR),
40 [COMMS_STS_TIMEOUT_ERR] = __stringify(COMMS_STS_TIMEOUT_ERR),
41 };
42
extract_fw_ver_from_str(const char * fw_str)43 static char *extract_fw_ver_from_str(const char *fw_str)
44 {
45 char *str, *fw_ver, *whitespace;
46 u32 ver_offset;
47
48 fw_ver = kmalloc(VERSION_MAX_LEN, GFP_KERNEL);
49 if (!fw_ver)
50 return NULL;
51
52 str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
53 if (!str)
54 goto free_fw_ver;
55
56 /* Skip the fw- part */
57 str += 3;
58 ver_offset = str - fw_str;
59
60 /* Copy until the next whitespace */
61 whitespace = strnstr(str, " ", VERSION_MAX_LEN - ver_offset);
62 if (!whitespace)
63 goto free_fw_ver;
64
65 strscpy(fw_ver, str, whitespace - str + 1);
66
67 return fw_ver;
68
69 free_fw_ver:
70 kfree(fw_ver);
71 return NULL;
72 }
73
74 /**
75 * extract_u32_until_given_char() - given a string of the format "<u32><char>*", extract the u32.
76 * @str: the given string
77 * @ver_num: the pointer to the extracted u32 to be returned to the caller.
78 * @given_char: the given char at the end of the u32 in the string
79 *
80 * Return: Upon success, return a pointer to the given_char in the string. Upon failure, return NULL
81 */
extract_u32_until_given_char(char * str,u32 * ver_num,char given_char)82 static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_char)
83 {
84 char num_str[8] = {}, *ch;
85
86 ch = strchrnul(str, given_char);
87 if (*ch == '\0' || ch == str || ch - str >= sizeof(num_str))
88 return NULL;
89
90 memcpy(num_str, str, ch - str);
91 if (kstrtou32(num_str, 10, ver_num))
92 return NULL;
93 return ch;
94 }
95
96 /**
97 * hl_get_sw_major_minor_subminor() - extract the FW's SW version major, minor, sub-minor
98 * from the version string
99 * @hdev: pointer to the hl_device
100 * @fw_str: the FW's version string
101 *
102 * The extracted version is set in the hdev fields: fw_sw_{major/minor/sub_minor}_ver.
103 *
104 * fw_str is expected to have one of two possible formats, examples:
105 * 1) 'Preboot version hl-gaudi2-1.9.0-fw-42.0.1-sec-3'
106 * 2) 'Preboot version hl-gaudi2-1.9.0-rc-fw-42.0.1-sec-3'
107 * In those examples, the SW major,minor,subminor are correspondingly: 1,9,0.
108 *
109 * Return: 0 for success or a negative error code for failure.
110 */
hl_get_sw_major_minor_subminor(struct hl_device * hdev,const char * fw_str)111 static int hl_get_sw_major_minor_subminor(struct hl_device *hdev, const char *fw_str)
112 {
113 char *end, *start;
114
115 end = strnstr(fw_str, "-rc-", VERSION_MAX_LEN);
116 if (end == fw_str)
117 return -EINVAL;
118
119 if (!end)
120 end = strnstr(fw_str, "-fw-", VERSION_MAX_LEN);
121
122 if (end == fw_str)
123 return -EINVAL;
124
125 if (!end)
126 return -EINVAL;
127
128 for (start = end - 1; start != fw_str; start--) {
129 if (*start == '-')
130 break;
131 }
132
133 if (start == fw_str)
134 return -EINVAL;
135
136 /* start/end point each to the starting and ending hyphen of the sw version e.g. -1.9.0- */
137 start++;
138 start = extract_u32_until_given_char(start, &hdev->fw_sw_major_ver, '.');
139 if (!start)
140 goto err_zero_ver;
141
142 start++;
143 start = extract_u32_until_given_char(start, &hdev->fw_sw_minor_ver, '.');
144 if (!start)
145 goto err_zero_ver;
146
147 start++;
148 start = extract_u32_until_given_char(start, &hdev->fw_sw_sub_minor_ver, '-');
149 if (!start)
150 goto err_zero_ver;
151
152 return 0;
153
154 err_zero_ver:
155 hdev->fw_sw_major_ver = 0;
156 hdev->fw_sw_minor_ver = 0;
157 hdev->fw_sw_sub_minor_ver = 0;
158 return -EINVAL;
159 }
160
161 /**
162 * hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string.
163 * @hdev: pointer to the hl_device
164 * @preboot_ver: the FW's version string
165 *
166 * preboot_ver is expected to be the format of <major>.<minor>.<sub minor>*, e.g: 42.0.1-sec-3
167 * The extracted version is set in the hdev fields: fw_inner_{major/minor}_ver.
168 *
169 * Return: 0 on success, negative error code for failure.
170 */
hl_get_preboot_major_minor(struct hl_device * hdev,char * preboot_ver)171 static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver)
172 {
173 preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_major_ver, '.');
174 if (!preboot_ver) {
175 dev_err(hdev->dev, "Error parsing preboot major version\n");
176 goto err_zero_ver;
177 }
178
179 preboot_ver++;
180
181 preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_minor_ver, '.');
182 if (!preboot_ver) {
183 dev_err(hdev->dev, "Error parsing preboot minor version\n");
184 goto err_zero_ver;
185 }
186 return 0;
187
188 err_zero_ver:
189 hdev->fw_inner_major_ver = 0;
190 hdev->fw_inner_minor_ver = 0;
191 return -EINVAL;
192 }
193
hl_request_fw(struct hl_device * hdev,const struct firmware ** firmware_p,const char * fw_name)194 static int hl_request_fw(struct hl_device *hdev,
195 const struct firmware **firmware_p,
196 const char *fw_name)
197 {
198 size_t fw_size;
199 int rc;
200
201 rc = request_firmware(firmware_p, fw_name, hdev->dev);
202 if (rc) {
203 dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
204 fw_name, rc);
205 goto out;
206 }
207
208 fw_size = (*firmware_p)->size;
209 if ((fw_size % 4) != 0) {
210 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
211 fw_name, fw_size);
212 rc = -EINVAL;
213 goto release_fw;
214 }
215
216 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
217
218 if (fw_size > FW_FILE_MAX_SIZE) {
219 dev_err(hdev->dev,
220 "FW file size %zu exceeds maximum of %u bytes\n",
221 fw_size, FW_FILE_MAX_SIZE);
222 rc = -EINVAL;
223 goto release_fw;
224 }
225
226 return 0;
227
228 release_fw:
229 release_firmware(*firmware_p);
230 out:
231 return rc;
232 }
233
234 /**
235 * hl_release_firmware() - release FW
236 *
237 * @fw: fw descriptor
238 *
239 * note: this inline function added to serve as a comprehensive mirror for the
240 * hl_request_fw function.
241 */
hl_release_firmware(const struct firmware * fw)242 static inline void hl_release_firmware(const struct firmware *fw)
243 {
244 release_firmware(fw);
245 }
246
247 /**
248 * hl_fw_copy_fw_to_device() - copy FW to device
249 *
250 * @hdev: pointer to hl_device structure.
251 * @fw: fw descriptor
252 * @dst: IO memory mapped address space to copy firmware to
253 * @src_offset: offset in src FW to copy from
254 * @size: amount of bytes to copy (0 to copy the whole binary)
255 *
256 * actual copy of FW binary data to device, shared by static and dynamic loaders
257 */
hl_fw_copy_fw_to_device(struct hl_device * hdev,const struct firmware * fw,void __iomem * dst,u32 src_offset,u32 size)258 static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
259 const struct firmware *fw, void __iomem *dst,
260 u32 src_offset, u32 size)
261 {
262 const void *fw_data;
263
264 /* size 0 indicates to copy the whole file */
265 if (!size)
266 size = fw->size;
267
268 if (src_offset + size > fw->size) {
269 dev_err(hdev->dev,
270 "size to copy(%u) and offset(%u) are invalid\n",
271 size, src_offset);
272 return -EINVAL;
273 }
274
275 fw_data = (const void *) fw->data;
276
277 memcpy_toio(dst, fw_data + src_offset, size);
278 return 0;
279 }
280
281 /**
282 * hl_fw_copy_msg_to_device() - copy message to device
283 *
284 * @hdev: pointer to hl_device structure.
285 * @msg: message
286 * @dst: IO memory mapped address space to copy firmware to
287 * @src_offset: offset in src message to copy from
288 * @size: amount of bytes to copy (0 to copy the whole binary)
289 *
290 * actual copy of message data to device.
291 */
hl_fw_copy_msg_to_device(struct hl_device * hdev,struct lkd_msg_comms * msg,void __iomem * dst,u32 src_offset,u32 size)292 static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
293 struct lkd_msg_comms *msg, void __iomem *dst,
294 u32 src_offset, u32 size)
295 {
296 void *msg_data;
297
298 /* size 0 indicates to copy the whole file */
299 if (!size)
300 size = sizeof(struct lkd_msg_comms);
301
302 if (src_offset + size > sizeof(struct lkd_msg_comms)) {
303 dev_err(hdev->dev,
304 "size to copy(%u) and offset(%u) are invalid\n",
305 size, src_offset);
306 return -EINVAL;
307 }
308
309 msg_data = (void *) msg;
310
311 memcpy_toio(dst, msg_data + src_offset, size);
312
313 return 0;
314 }
315
316 /**
317 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
318 *
319 * @hdev: pointer to hl_device structure.
320 * @fw_name: the firmware image name
321 * @dst: IO memory mapped address space to copy firmware to
322 * @src_offset: offset in src FW to copy from
323 * @size: amount of bytes to copy (0 to copy the whole binary)
324 *
325 * Copy fw code from firmware file to device memory.
326 *
327 * Return: 0 on success, non-zero for failure.
328 */
hl_fw_load_fw_to_device(struct hl_device * hdev,const char * fw_name,void __iomem * dst,u32 src_offset,u32 size)329 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
330 void __iomem *dst, u32 src_offset, u32 size)
331 {
332 const struct firmware *fw;
333 int rc;
334
335 rc = hl_request_fw(hdev, &fw, fw_name);
336 if (rc)
337 return rc;
338
339 rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
340
341 hl_release_firmware(fw);
342 return rc;
343 }
344
hl_fw_send_pci_access_msg(struct hl_device * hdev,u32 opcode,u64 value)345 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value)
346 {
347 struct cpucp_packet pkt = {};
348
349 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
350 pkt.value = cpu_to_le64(value);
351
352 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
353 }
354
hl_fw_send_cpu_message(struct hl_device * hdev,u32 hw_queue_id,u32 * msg,u16 len,u32 timeout,u64 * result)355 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
356 u16 len, u32 timeout, u64 *result)
357 {
358 struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
359 struct asic_fixed_properties *prop = &hdev->asic_prop;
360 struct cpucp_packet *pkt;
361 dma_addr_t pkt_dma_addr;
362 struct hl_bd *sent_bd;
363 u32 tmp, expected_ack_val, pi, opcode;
364 int rc;
365
366 pkt = hl_cpu_accessible_dma_pool_alloc(hdev, len, &pkt_dma_addr);
367 if (!pkt) {
368 dev_err(hdev->dev,
369 "Failed to allocate DMA memory for packet to CPU\n");
370 return -ENOMEM;
371 }
372
373 memcpy(pkt, msg, len);
374
375 mutex_lock(&hdev->send_cpu_message_lock);
376
377 /* CPU-CP messages can be sent during soft-reset */
378 if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
379 rc = 0;
380 goto out;
381 }
382
383 if (hdev->device_cpu_disabled) {
384 rc = -EIO;
385 goto out;
386 }
387
388 /* set fence to a non valid value */
389 pkt->fence = cpu_to_le32(UINT_MAX);
390 pi = queue->pi;
391
392 /*
393 * The CPU queue is a synchronous queue with an effective depth of
394 * a single entry (although it is allocated with room for multiple
395 * entries). We lock on it using 'send_cpu_message_lock' which
396 * serializes accesses to the CPU queue.
397 * Which means that we don't need to lock the access to the entire H/W
398 * queues module when submitting a JOB to the CPU queue.
399 */
400 hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), len, pkt_dma_addr);
401
402 if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
403 expected_ack_val = queue->pi;
404 else
405 expected_ack_val = CPUCP_PACKET_FENCE_VAL;
406
407 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
408 (tmp == expected_ack_val), 1000,
409 timeout, true);
410
411 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
412
413 if (rc == -ETIMEDOUT) {
414 /* If FW performed reset just before sending it a packet, we will get a timeout.
415 * This is expected behavior, hence no need for error message.
416 */
417 if (!hl_device_operational(hdev, NULL) && !hdev->reset_info.in_compute_reset)
418 dev_dbg(hdev->dev, "Device CPU packet timeout (0x%x) due to FW reset\n",
419 tmp);
420 else
421 dev_err(hdev->dev, "Device CPU packet timeout (status = 0x%x)\n", tmp);
422 hdev->device_cpu_disabled = true;
423 goto out;
424 }
425
426 tmp = le32_to_cpu(pkt->ctl);
427
428 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
429 if (rc) {
430 opcode = (tmp & CPUCP_PKT_CTL_OPCODE_MASK) >> CPUCP_PKT_CTL_OPCODE_SHIFT;
431
432 if (!prop->supports_advanced_cpucp_rc) {
433 dev_dbg(hdev->dev, "F/W ERROR %d for CPU packet %d\n", rc, opcode);
434 rc = -EIO;
435 goto scrub_descriptor;
436 }
437
438 switch (rc) {
439 case cpucp_packet_invalid:
440 dev_err(hdev->dev,
441 "CPU packet %d is not supported by F/W\n", opcode);
442 break;
443 case cpucp_packet_fault:
444 dev_err(hdev->dev,
445 "F/W failed processing CPU packet %d\n", opcode);
446 break;
447 case cpucp_packet_invalid_pkt:
448 dev_dbg(hdev->dev,
449 "CPU packet %d is not supported by F/W\n", opcode);
450 break;
451 case cpucp_packet_invalid_params:
452 dev_err(hdev->dev,
453 "F/W reports invalid parameters for CPU packet %d\n", opcode);
454 break;
455
456 default:
457 dev_err(hdev->dev,
458 "Unknown F/W ERROR %d for CPU packet %d\n", rc, opcode);
459 }
460
461 /* propagate the return code from the f/w to the callers who want to check it */
462 if (result)
463 *result = rc;
464
465 rc = -EIO;
466
467 } else if (result) {
468 *result = le64_to_cpu(pkt->result);
469 }
470
471 scrub_descriptor:
472 /* Scrub previous buffer descriptor 'ctl' field which contains the
473 * previous PI value written during packet submission.
474 * We must do this or else F/W can read an old value upon queue wraparound.
475 */
476 sent_bd = queue->kernel_address;
477 sent_bd += hl_pi_2_offset(pi);
478 sent_bd->ctl = cpu_to_le32(UINT_MAX);
479
480 out:
481 mutex_unlock(&hdev->send_cpu_message_lock);
482
483 hl_cpu_accessible_dma_pool_free(hdev, len, pkt);
484
485 return rc;
486 }
487
hl_fw_unmask_irq(struct hl_device * hdev,u16 event_type)488 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
489 {
490 struct cpucp_packet pkt;
491 u64 result;
492 int rc;
493
494 memset(&pkt, 0, sizeof(pkt));
495
496 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
497 CPUCP_PKT_CTL_OPCODE_SHIFT);
498 pkt.value = cpu_to_le64(event_type);
499
500 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
501 0, &result);
502
503 if (rc)
504 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
505
506 return rc;
507 }
508
hl_fw_unmask_irq_arr(struct hl_device * hdev,const u32 * irq_arr,size_t irq_arr_size)509 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
510 size_t irq_arr_size)
511 {
512 struct cpucp_unmask_irq_arr_packet *pkt;
513 size_t total_pkt_size;
514 u64 result;
515 int rc;
516
517 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
518 irq_arr_size;
519
520 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
521 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
522
523 /* total_pkt_size is casted to u16 later on */
524 if (total_pkt_size > USHRT_MAX) {
525 dev_err(hdev->dev, "too many elements in IRQ array\n");
526 return -EINVAL;
527 }
528
529 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
530 if (!pkt)
531 return -ENOMEM;
532
533 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
534 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
535
536 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
537 CPUCP_PKT_CTL_OPCODE_SHIFT);
538
539 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
540 total_pkt_size, 0, &result);
541
542 if (rc)
543 dev_err(hdev->dev, "failed to unmask IRQ array\n");
544
545 kfree(pkt);
546
547 return rc;
548 }
549
hl_fw_test_cpu_queue(struct hl_device * hdev)550 int hl_fw_test_cpu_queue(struct hl_device *hdev)
551 {
552 struct cpucp_packet test_pkt = {};
553 u64 result;
554 int rc;
555
556 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
557 CPUCP_PKT_CTL_OPCODE_SHIFT);
558 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
559
560 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
561 sizeof(test_pkt), 0, &result);
562
563 if (!rc) {
564 if (result != CPUCP_PACKET_FENCE_VAL)
565 dev_err(hdev->dev,
566 "CPU queue test failed (%#08llx)\n", result);
567 } else {
568 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
569 }
570
571 return rc;
572 }
573
hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device * hdev,size_t size,dma_addr_t * dma_handle)574 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
575 dma_addr_t *dma_handle)
576 {
577 u64 kernel_addr;
578
579 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
580
581 *dma_handle = hdev->cpu_accessible_dma_address +
582 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
583
584 return (void *) (uintptr_t) kernel_addr;
585 }
586
hl_fw_cpu_accessible_dma_pool_free(struct hl_device * hdev,size_t size,void * vaddr)587 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
588 void *vaddr)
589 {
590 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
591 size);
592 }
593
hl_fw_send_soft_reset(struct hl_device * hdev)594 int hl_fw_send_soft_reset(struct hl_device *hdev)
595 {
596 struct cpucp_packet pkt;
597 int rc;
598
599 memset(&pkt, 0, sizeof(pkt));
600 pkt.ctl = cpu_to_le32(CPUCP_PACKET_SOFT_RESET << CPUCP_PKT_CTL_OPCODE_SHIFT);
601 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
602 if (rc)
603 dev_err(hdev->dev, "failed to send soft-reset msg (err = %d)\n", rc);
604
605 return rc;
606 }
607
hl_fw_send_device_activity(struct hl_device * hdev,bool open)608 int hl_fw_send_device_activity(struct hl_device *hdev, bool open)
609 {
610 struct cpucp_packet pkt;
611 int rc;
612
613 memset(&pkt, 0, sizeof(pkt));
614 pkt.ctl = cpu_to_le32(CPUCP_PACKET_ACTIVE_STATUS_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
615 pkt.value = cpu_to_le64(open);
616 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
617 if (rc)
618 dev_err(hdev->dev, "failed to send device activity msg(%u)\n", open);
619
620 return rc;
621 }
622
hl_fw_send_heartbeat(struct hl_device * hdev)623 int hl_fw_send_heartbeat(struct hl_device *hdev)
624 {
625 struct cpucp_packet hb_pkt;
626 u64 result;
627 int rc;
628
629 memset(&hb_pkt, 0, sizeof(hb_pkt));
630 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
631 CPUCP_PKT_CTL_OPCODE_SHIFT);
632 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
633
634 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
635 sizeof(hb_pkt), 0, &result);
636
637 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
638 return -EIO;
639
640 if (le32_to_cpu(hb_pkt.status_mask) &
641 CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
642 dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
643 rc = -EIO;
644 }
645
646 return rc;
647 }
648
fw_report_boot_dev0(struct hl_device * hdev,u32 err_val,u32 sts_val)649 static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
650 u32 sts_val)
651 {
652 bool err_exists = false;
653
654 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
655 return false;
656
657 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
658 dev_err(hdev->dev,
659 "Device boot error - DRAM initialization failed\n");
660 err_exists = true;
661 }
662
663 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
664 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
665 err_exists = true;
666 }
667
668 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
669 dev_err(hdev->dev,
670 "Device boot error - Thermal Sensor initialization failed\n");
671 err_exists = true;
672 }
673
674 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
675 if (hdev->bmc_enable) {
676 dev_err(hdev->dev,
677 "Device boot error - Skipped waiting for BMC\n");
678 err_exists = true;
679 } else {
680 dev_info(hdev->dev,
681 "Device boot message - Skipped waiting for BMC\n");
682 /* This is an info so we don't want it to disable the
683 * device
684 */
685 err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;
686 }
687 }
688
689 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
690 dev_err(hdev->dev,
691 "Device boot error - Serdes data from BMC not available\n");
692 err_exists = true;
693 }
694
695 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
696 dev_err(hdev->dev,
697 "Device boot error - NIC F/W initialization failed\n");
698 err_exists = true;
699 }
700
701 if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
702 dev_err(hdev->dev,
703 "Device boot warning - security not ready\n");
704 err_exists = true;
705 }
706
707 if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
708 dev_err(hdev->dev, "Device boot error - security failure\n");
709 err_exists = true;
710 }
711
712 if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
713 dev_err(hdev->dev, "Device boot error - eFuse failure\n");
714 err_exists = true;
715 }
716
717 if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
718 dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
719 err_exists = true;
720 }
721
722 if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
723 dev_err(hdev->dev, "Device boot error - PLL failure\n");
724 err_exists = true;
725 }
726
727 if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
728 /* Ignore this bit, don't prevent driver loading */
729 dev_dbg(hdev->dev, "device unusable status is set\n");
730 err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
731 }
732
733 if (err_val & CPU_BOOT_ERR0_BINNING_FAIL) {
734 dev_err(hdev->dev, "Device boot error - binning failure\n");
735 err_exists = true;
736 }
737
738 if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
739 dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
740
741 if (err_val & CPU_BOOT_ERR0_EEPROM_FAIL) {
742 dev_err(hdev->dev, "Device boot error - EEPROM failure detected\n");
743 err_exists = true;
744 }
745
746 /* All warnings should go here in order not to reach the unknown error validation */
747 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
748 dev_warn(hdev->dev,
749 "Device boot warning - Skipped DRAM initialization\n");
750 /* This is a warning so we don't want it to disable the
751 * device
752 */
753 err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
754 }
755
756 if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
757 dev_warn(hdev->dev,
758 "Device boot warning - Failed to load preboot primary image\n");
759 /* This is a warning so we don't want it to disable the
760 * device as we have a secondary preboot image
761 */
762 err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
763 }
764
765 if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
766 dev_warn(hdev->dev,
767 "Device boot warning - TPM failure\n");
768 /* This is a warning so we don't want it to disable the
769 * device
770 */
771 err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
772 }
773
774 if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
775 dev_err(hdev->dev,
776 "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
777 err_exists = true;
778 }
779
780 /* return error only if it's in the predefined mask */
781 if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
782 lower_32_bits(hdev->boot_error_status_mask)))
783 return true;
784
785 return false;
786 }
787
788 /* placeholder for ERR1 as no errors defined there yet */
fw_report_boot_dev1(struct hl_device * hdev,u32 err_val,u32 sts_val)789 static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
790 u32 sts_val)
791 {
792 /*
793 * keep this variable to preserve the logic of the function.
794 * this way it would require less modifications when error will be
795 * added to DEV_ERR1
796 */
797 bool err_exists = false;
798
799 if (!(err_val & CPU_BOOT_ERR1_ENABLED))
800 return false;
801
802 if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
803 dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
804
805 if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
806 dev_err(hdev->dev,
807 "Device boot error - unknown ERR1 error 0x%08x\n",
808 err_val);
809 err_exists = true;
810 }
811
812 /* return error only if it's in the predefined mask */
813 if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
814 upper_32_bits(hdev->boot_error_status_mask)))
815 return true;
816
817 return false;
818 }
819
fw_read_errors(struct hl_device * hdev,u32 boot_err0_reg,u32 boot_err1_reg,u32 cpu_boot_dev_status0_reg,u32 cpu_boot_dev_status1_reg)820 static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
821 u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
822 u32 cpu_boot_dev_status1_reg)
823 {
824 u32 err_val, status_val;
825 bool err_exists = false;
826
827 /* Some of the firmware status codes are deprecated in newer f/w
828 * versions. In those versions, the errors are reported
829 * in different registers. Therefore, we need to check those
830 * registers and print the exact errors. Moreover, there
831 * may be multiple errors, so we need to report on each error
832 * separately. Some of the error codes might indicate a state
833 * that is not an error per-se, but it is an error in production
834 * environment
835 */
836 err_val = RREG32(boot_err0_reg);
837 status_val = RREG32(cpu_boot_dev_status0_reg);
838 err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
839
840 err_val = RREG32(boot_err1_reg);
841 status_val = RREG32(cpu_boot_dev_status1_reg);
842 err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
843
844 if (err_exists)
845 return -EIO;
846
847 return 0;
848 }
849
hl_fw_cpucp_info_get(struct hl_device * hdev,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg)850 int hl_fw_cpucp_info_get(struct hl_device *hdev,
851 u32 sts_boot_dev_sts0_reg,
852 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
853 u32 boot_err1_reg)
854 {
855 struct asic_fixed_properties *prop = &hdev->asic_prop;
856 struct cpucp_packet pkt = {};
857 dma_addr_t cpucp_info_dma_addr;
858 void *cpucp_info_cpu_addr;
859 char *kernel_ver;
860 u64 result;
861 int rc;
862
863 cpucp_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, sizeof(struct cpucp_info),
864 &cpucp_info_dma_addr);
865 if (!cpucp_info_cpu_addr) {
866 dev_err(hdev->dev,
867 "Failed to allocate DMA memory for CPU-CP info packet\n");
868 return -ENOMEM;
869 }
870
871 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
872
873 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
874 CPUCP_PKT_CTL_OPCODE_SHIFT);
875 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
876 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
877
878 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
879 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
880 if (rc) {
881 dev_err(hdev->dev,
882 "Failed to handle CPU-CP info pkt, error %d\n", rc);
883 goto out;
884 }
885
886 rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
887 sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
888 if (rc) {
889 dev_err(hdev->dev, "Errors in device boot\n");
890 goto out;
891 }
892
893 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
894 sizeof(prop->cpucp_info));
895
896 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
897 if (rc) {
898 dev_err(hdev->dev,
899 "Failed to build hwmon channel info, error %d\n", rc);
900 rc = -EFAULT;
901 goto out;
902 }
903
904 kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
905 if (kernel_ver) {
906 dev_info(hdev->dev, "Linux version %s", kernel_ver);
907 kfree(kernel_ver);
908 }
909
910 /* assume EQ code doesn't need to check eqe index */
911 hdev->event_queue.check_eqe_index = false;
912
913 /* Read FW application security bits again */
914 if (prop->fw_cpu_boot_dev_sts0_valid) {
915 prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);
916 if (prop->fw_app_cpu_boot_dev_sts0 &
917 CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
918 hdev->event_queue.check_eqe_index = true;
919 }
920
921 if (prop->fw_cpu_boot_dev_sts1_valid)
922 prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);
923
924 out:
925 hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_info), cpucp_info_cpu_addr);
926
927 return rc;
928 }
929
hl_fw_send_msi_info_msg(struct hl_device * hdev)930 static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
931 {
932 struct cpucp_array_data_packet *pkt;
933 size_t total_pkt_size, data_size;
934 u64 result;
935 int rc;
936
937 /* skip sending this info for unsupported ASICs */
938 if (!hdev->asic_funcs->get_msi_info)
939 return 0;
940
941 data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);
942 total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;
943
944 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
945 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
946
947 /* total_pkt_size is casted to u16 later on */
948 if (total_pkt_size > USHRT_MAX) {
949 dev_err(hdev->dev, "CPUCP array data is too big\n");
950 return -EINVAL;
951 }
952
953 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
954 if (!pkt)
955 return -ENOMEM;
956
957 pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
958
959 memset((void *) &pkt->data, 0xFF, data_size);
960 hdev->asic_funcs->get_msi_info(pkt->data);
961
962 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
963 CPUCP_PKT_CTL_OPCODE_SHIFT);
964
965 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,
966 total_pkt_size, 0, &result);
967
968 /*
969 * in case packet result is invalid it means that FW does not support
970 * this feature and will use default/hard coded MSI values. no reason
971 * to stop the boot
972 */
973 if (rc && result == cpucp_packet_invalid)
974 rc = 0;
975
976 if (rc)
977 dev_err(hdev->dev, "failed to send CPUCP array data\n");
978
979 kfree(pkt);
980
981 return rc;
982 }
983
hl_fw_cpucp_handshake(struct hl_device * hdev,u32 sts_boot_dev_sts0_reg,u32 sts_boot_dev_sts1_reg,u32 boot_err0_reg,u32 boot_err1_reg)984 int hl_fw_cpucp_handshake(struct hl_device *hdev,
985 u32 sts_boot_dev_sts0_reg,
986 u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
987 u32 boot_err1_reg)
988 {
989 int rc;
990
991 rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
992 sts_boot_dev_sts1_reg, boot_err0_reg,
993 boot_err1_reg);
994 if (rc)
995 return rc;
996
997 return hl_fw_send_msi_info_msg(hdev);
998 }
999
hl_fw_get_eeprom_data(struct hl_device * hdev,void * data,size_t max_size)1000 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
1001 {
1002 struct cpucp_packet pkt = {};
1003 void *eeprom_info_cpu_addr;
1004 dma_addr_t eeprom_info_dma_addr;
1005 u64 result;
1006 int rc;
1007
1008 eeprom_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, max_size,
1009 &eeprom_info_dma_addr);
1010 if (!eeprom_info_cpu_addr) {
1011 dev_err(hdev->dev,
1012 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
1013 return -ENOMEM;
1014 }
1015
1016 memset(eeprom_info_cpu_addr, 0, max_size);
1017
1018 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
1019 CPUCP_PKT_CTL_OPCODE_SHIFT);
1020 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
1021 pkt.data_max_size = cpu_to_le32(max_size);
1022
1023 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1024 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
1025
1026 if (rc) {
1027 dev_err(hdev->dev,
1028 "Failed to handle CPU-CP EEPROM packet, error %d\n",
1029 rc);
1030 goto out;
1031 }
1032
1033 /* result contains the actual size */
1034 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
1035
1036 out:
1037 hl_cpu_accessible_dma_pool_free(hdev, max_size, eeprom_info_cpu_addr);
1038
1039 return rc;
1040 }
1041
hl_fw_get_monitor_dump(struct hl_device * hdev,void * data)1042 int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data)
1043 {
1044 struct cpucp_monitor_dump *mon_dump_cpu_addr;
1045 dma_addr_t mon_dump_dma_addr;
1046 struct cpucp_packet pkt = {};
1047 size_t data_size;
1048 __le32 *src_ptr;
1049 u32 *dst_ptr;
1050 u64 result;
1051 int i, rc;
1052
1053 data_size = sizeof(struct cpucp_monitor_dump);
1054 mon_dump_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, data_size, &mon_dump_dma_addr);
1055 if (!mon_dump_cpu_addr) {
1056 dev_err(hdev->dev,
1057 "Failed to allocate DMA memory for CPU-CP monitor-dump packet\n");
1058 return -ENOMEM;
1059 }
1060
1061 memset(mon_dump_cpu_addr, 0, data_size);
1062
1063 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MONITOR_DUMP_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1064 pkt.addr = cpu_to_le64(mon_dump_dma_addr);
1065 pkt.data_max_size = cpu_to_le32(data_size);
1066
1067 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1068 HL_CPUCP_MON_DUMP_TIMEOUT_USEC, &result);
1069 if (rc) {
1070 dev_err(hdev->dev, "Failed to handle CPU-CP monitor-dump packet, error %d\n", rc);
1071 goto out;
1072 }
1073
1074 /* result contains the actual size */
1075 src_ptr = (__le32 *) mon_dump_cpu_addr;
1076 dst_ptr = data;
1077 for (i = 0; i < (data_size / sizeof(u32)); i++) {
1078 *dst_ptr = le32_to_cpu(*src_ptr);
1079 src_ptr++;
1080 dst_ptr++;
1081 }
1082
1083 out:
1084 hl_cpu_accessible_dma_pool_free(hdev, data_size, mon_dump_cpu_addr);
1085
1086 return rc;
1087 }
1088
hl_fw_cpucp_pci_counters_get(struct hl_device * hdev,struct hl_info_pci_counters * counters)1089 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
1090 struct hl_info_pci_counters *counters)
1091 {
1092 struct cpucp_packet pkt = {};
1093 u64 result;
1094 int rc;
1095
1096 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1097 CPUCP_PKT_CTL_OPCODE_SHIFT);
1098
1099 /* Fetch PCI rx counter */
1100 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
1101 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1102 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1103 if (rc) {
1104 dev_err(hdev->dev,
1105 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1106 return rc;
1107 }
1108 counters->rx_throughput = result;
1109
1110 memset(&pkt, 0, sizeof(pkt));
1111 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
1112 CPUCP_PKT_CTL_OPCODE_SHIFT);
1113
1114 /* Fetch PCI tx counter */
1115 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
1116 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1117 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1118 if (rc) {
1119 dev_err(hdev->dev,
1120 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1121 return rc;
1122 }
1123 counters->tx_throughput = result;
1124
1125 /* Fetch PCI replay counter */
1126 memset(&pkt, 0, sizeof(pkt));
1127 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
1128 CPUCP_PKT_CTL_OPCODE_SHIFT);
1129
1130 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1131 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1132 if (rc) {
1133 dev_err(hdev->dev,
1134 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
1135 return rc;
1136 }
1137 counters->replay_cnt = (u32) result;
1138
1139 return rc;
1140 }
1141
hl_fw_cpucp_total_energy_get(struct hl_device * hdev,u64 * total_energy)1142 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
1143 {
1144 struct cpucp_packet pkt = {};
1145 u64 result;
1146 int rc;
1147
1148 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
1149 CPUCP_PKT_CTL_OPCODE_SHIFT);
1150
1151 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1152 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1153 if (rc) {
1154 dev_err(hdev->dev,
1155 "Failed to handle CpuCP total energy pkt, error %d\n",
1156 rc);
1157 return rc;
1158 }
1159
1160 *total_energy = result;
1161
1162 return rc;
1163 }
1164
get_used_pll_index(struct hl_device * hdev,u32 input_pll_index,enum pll_index * pll_index)1165 int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
1166 enum pll_index *pll_index)
1167 {
1168 struct asic_fixed_properties *prop = &hdev->asic_prop;
1169 u8 pll_byte, pll_bit_off;
1170 bool dynamic_pll;
1171 int fw_pll_idx;
1172
1173 dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
1174 CPU_BOOT_DEV_STS0_DYN_PLL_EN);
1175
1176 if (!dynamic_pll) {
1177 /*
1178 * in case we are working with legacy FW (each asic has unique
1179 * PLL numbering) use the driver based index as they are
1180 * aligned with fw legacy numbering
1181 */
1182 *pll_index = input_pll_index;
1183 return 0;
1184 }
1185
1186 /* retrieve a FW compatible PLL index based on
1187 * ASIC specific user request
1188 */
1189 fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);
1190 if (fw_pll_idx < 0) {
1191 dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",
1192 input_pll_index, fw_pll_idx);
1193 return -EINVAL;
1194 }
1195
1196 /* PLL map is a u8 array */
1197 pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];
1198 pll_bit_off = fw_pll_idx & 0x7;
1199
1200 if (!(pll_byte & BIT(pll_bit_off))) {
1201 dev_err(hdev->dev, "PLL index %d is not supported\n",
1202 fw_pll_idx);
1203 return -EINVAL;
1204 }
1205
1206 *pll_index = fw_pll_idx;
1207
1208 return 0;
1209 }
1210
hl_fw_cpucp_pll_info_get(struct hl_device * hdev,u32 pll_index,u16 * pll_freq_arr)1211 int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
1212 u16 *pll_freq_arr)
1213 {
1214 struct cpucp_packet pkt;
1215 enum pll_index used_pll_idx;
1216 u64 result;
1217 int rc;
1218
1219 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
1220 if (rc)
1221 return rc;
1222
1223 memset(&pkt, 0, sizeof(pkt));
1224
1225 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<
1226 CPUCP_PKT_CTL_OPCODE_SHIFT);
1227 pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);
1228
1229 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1230 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1231 if (rc) {
1232 dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);
1233 return rc;
1234 }
1235
1236 pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);
1237 pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);
1238 pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);
1239 pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);
1240
1241 return 0;
1242 }
1243
hl_fw_cpucp_power_get(struct hl_device * hdev,u64 * power)1244 int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
1245 {
1246 struct cpucp_packet pkt;
1247 u64 result;
1248 int rc;
1249
1250 memset(&pkt, 0, sizeof(pkt));
1251
1252 pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<
1253 CPUCP_PKT_CTL_OPCODE_SHIFT);
1254 pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);
1255
1256 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1257 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1258 if (rc) {
1259 dev_err(hdev->dev, "Failed to read power, error %d\n", rc);
1260 return rc;
1261 }
1262
1263 *power = result;
1264
1265 return rc;
1266 }
1267
hl_fw_dram_replaced_row_get(struct hl_device * hdev,struct cpucp_hbm_row_info * info)1268 int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
1269 struct cpucp_hbm_row_info *info)
1270 {
1271 struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;
1272 dma_addr_t cpucp_repl_rows_info_dma_addr;
1273 struct cpucp_packet pkt = {};
1274 u64 result;
1275 int rc;
1276
1277 cpucp_repl_rows_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev,
1278 sizeof(struct cpucp_hbm_row_info),
1279 &cpucp_repl_rows_info_dma_addr);
1280 if (!cpucp_repl_rows_info_cpu_addr) {
1281 dev_err(hdev->dev,
1282 "Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");
1283 return -ENOMEM;
1284 }
1285
1286 memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));
1287
1288 pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<
1289 CPUCP_PKT_CTL_OPCODE_SHIFT);
1290 pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);
1291 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));
1292
1293 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1294 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
1295 if (rc) {
1296 dev_err(hdev->dev,
1297 "Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);
1298 goto out;
1299 }
1300
1301 memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));
1302
1303 out:
1304 hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_hbm_row_info),
1305 cpucp_repl_rows_info_cpu_addr);
1306
1307 return rc;
1308 }
1309
hl_fw_dram_pending_row_get(struct hl_device * hdev,u32 * pend_rows_num)1310 int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)
1311 {
1312 struct cpucp_packet pkt;
1313 u64 result;
1314 int rc;
1315
1316 memset(&pkt, 0, sizeof(pkt));
1317
1318 pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);
1319
1320 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
1321 if (rc) {
1322 dev_err(hdev->dev,
1323 "Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);
1324 goto out;
1325 }
1326
1327 *pend_rows_num = (u32) result;
1328 out:
1329 return rc;
1330 }
1331
hl_fw_cpucp_engine_core_asid_set(struct hl_device * hdev,u32 asid)1332 int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)
1333 {
1334 struct cpucp_packet pkt;
1335 int rc;
1336
1337 memset(&pkt, 0, sizeof(pkt));
1338
1339 pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
1340 pkt.value = cpu_to_le64(asid);
1341
1342 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
1343 HL_CPUCP_INFO_TIMEOUT_USEC, NULL);
1344 if (rc)
1345 dev_err(hdev->dev,
1346 "Failed on ASID configuration request for engine core, error %d\n",
1347 rc);
1348
1349 return rc;
1350 }
1351
hl_fw_ask_hard_reset_without_linux(struct hl_device * hdev)1352 void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
1353 {
1354 struct static_fw_load_mgr *static_loader =
1355 &hdev->fw_loader.static_loader;
1356 int rc;
1357
1358 if (hdev->asic_prop.dynamic_fw_load) {
1359 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1360 COMMS_RST_DEV, 0, false,
1361 hdev->fw_loader.cpu_timeout);
1362 if (rc)
1363 dev_err(hdev->dev, "Failed sending COMMS_RST_DEV\n");
1364 } else {
1365 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
1366 }
1367 }
1368
hl_fw_ask_halt_machine_without_linux(struct hl_device * hdev)1369 void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
1370 {
1371 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1372 u32 status, cpu_boot_status_reg, cpu_timeout;
1373 struct static_fw_load_mgr *static_loader;
1374 struct pre_fw_load_props *pre_fw_load;
1375 int rc;
1376
1377 if (hdev->device_cpu_is_halted)
1378 return;
1379
1380 /* Stop device CPU to make sure nothing bad happens */
1381 if (hdev->asic_prop.dynamic_fw_load) {
1382 pre_fw_load = &fw_loader->pre_fw_load;
1383 cpu_timeout = fw_loader->cpu_timeout;
1384 cpu_boot_status_reg = pre_fw_load->cpu_boot_status_reg;
1385
1386 rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
1387 COMMS_GOTO_WFE, 0, false, cpu_timeout);
1388 if (rc) {
1389 dev_err(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
1390 } else {
1391 rc = hl_poll_timeout(
1392 hdev,
1393 cpu_boot_status_reg,
1394 status,
1395 status == CPU_BOOT_STATUS_IN_WFE,
1396 hdev->fw_poll_interval_usec,
1397 cpu_timeout);
1398 if (rc)
1399 dev_err(hdev->dev, "Current status=%u. Timed-out updating to WFE\n",
1400 status);
1401 }
1402 } else {
1403 static_loader = &hdev->fw_loader.static_loader;
1404 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
1405 msleep(static_loader->cpu_reset_wait_msec);
1406
1407 /* Must clear this register in order to prevent preboot
1408 * from reading WFE after reboot
1409 */
1410 WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);
1411 }
1412
1413 hdev->device_cpu_is_halted = true;
1414 }
1415
detect_cpu_boot_status(struct hl_device * hdev,u32 status)1416 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
1417 {
1418 /* Some of the status codes below are deprecated in newer f/w
1419 * versions but we keep them here for backward compatibility
1420 */
1421 switch (status) {
1422 case CPU_BOOT_STATUS_NA:
1423 dev_err(hdev->dev,
1424 "Device boot progress - BTL/ROM did NOT run\n");
1425 break;
1426 case CPU_BOOT_STATUS_IN_WFE:
1427 dev_err(hdev->dev,
1428 "Device boot progress - Stuck inside WFE loop\n");
1429 break;
1430 case CPU_BOOT_STATUS_IN_BTL:
1431 dev_err(hdev->dev,
1432 "Device boot progress - Stuck in BTL\n");
1433 break;
1434 case CPU_BOOT_STATUS_IN_PREBOOT:
1435 dev_err(hdev->dev,
1436 "Device boot progress - Stuck in Preboot\n");
1437 break;
1438 case CPU_BOOT_STATUS_IN_SPL:
1439 dev_err(hdev->dev,
1440 "Device boot progress - Stuck in SPL\n");
1441 break;
1442 case CPU_BOOT_STATUS_IN_UBOOT:
1443 dev_err(hdev->dev,
1444 "Device boot progress - Stuck in u-boot\n");
1445 break;
1446 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
1447 dev_err(hdev->dev,
1448 "Device boot progress - DRAM initialization failed\n");
1449 break;
1450 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
1451 dev_err(hdev->dev,
1452 "Device boot progress - Cannot boot\n");
1453 break;
1454 case CPU_BOOT_STATUS_TS_INIT_FAIL:
1455 dev_err(hdev->dev,
1456 "Device boot progress - Thermal Sensor initialization failed\n");
1457 break;
1458 case CPU_BOOT_STATUS_SECURITY_READY:
1459 dev_err(hdev->dev,
1460 "Device boot progress - Stuck in preboot after security initialization\n");
1461 break;
1462 default:
1463 dev_err(hdev->dev,
1464 "Device boot progress - Invalid or unexpected status code %d\n", status);
1465 break;
1466 }
1467 }
1468
hl_fw_wait_preboot_ready(struct hl_device * hdev)1469 int hl_fw_wait_preboot_ready(struct hl_device *hdev)
1470 {
1471 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
1472 u32 status;
1473 int rc;
1474
1475 /* Need to check two possible scenarios:
1476 *
1477 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
1478 * the preboot is waiting for the boot fit
1479 *
1480 * All other status values - for older firmwares where the uboot was
1481 * loaded from the FLASH
1482 */
1483 rc = hl_poll_timeout(
1484 hdev,
1485 pre_fw_load->cpu_boot_status_reg,
1486 status,
1487 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
1488 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
1489 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
1490 hdev->fw_poll_interval_usec,
1491 pre_fw_load->wait_for_preboot_timeout);
1492
1493 if (rc) {
1494 detect_cpu_boot_status(hdev, status);
1495 dev_err(hdev->dev, "CPU boot ready timeout (status = %d)\n", status);
1496
1497 /* If we read all FF, then something is totally wrong, no point
1498 * of reading specific errors
1499 */
1500 if (status != -1)
1501 fw_read_errors(hdev, pre_fw_load->boot_err0_reg,
1502 pre_fw_load->boot_err1_reg,
1503 pre_fw_load->sts_boot_dev_sts0_reg,
1504 pre_fw_load->sts_boot_dev_sts1_reg);
1505 return -EIO;
1506 }
1507
1508 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_PREBOOT_CPU;
1509
1510 return 0;
1511 }
1512
hl_fw_read_preboot_caps(struct hl_device * hdev)1513 static int hl_fw_read_preboot_caps(struct hl_device *hdev)
1514 {
1515 struct pre_fw_load_props *pre_fw_load;
1516 struct asic_fixed_properties *prop;
1517 u32 reg_val;
1518 int rc;
1519
1520 prop = &hdev->asic_prop;
1521 pre_fw_load = &hdev->fw_loader.pre_fw_load;
1522
1523 rc = hl_fw_wait_preboot_ready(hdev);
1524 if (rc)
1525 return rc;
1526
1527 /*
1528 * the registers DEV_STS* contain FW capabilities/features.
1529 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
1530 * is set.
1531 * In the first read of this register we store the value of this
1532 * register ONLY if the register is enabled (which will be propagated
1533 * to next stages) and also mark the register as valid.
1534 * In case it is not enabled the stored value will be left 0- all
1535 * caps/features are off
1536 */
1537 reg_val = RREG32(pre_fw_load->sts_boot_dev_sts0_reg);
1538 if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
1539 prop->fw_cpu_boot_dev_sts0_valid = true;
1540 prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
1541 }
1542
1543 reg_val = RREG32(pre_fw_load->sts_boot_dev_sts1_reg);
1544 if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
1545 prop->fw_cpu_boot_dev_sts1_valid = true;
1546 prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
1547 }
1548
1549 prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
1550 CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
1551
1552 /* initialize FW loader once we know what load protocol is used */
1553 hdev->asic_funcs->init_firmware_loader(hdev);
1554
1555 dev_dbg(hdev->dev, "Attempting %s FW load\n",
1556 prop->dynamic_fw_load ? "dynamic" : "legacy");
1557 return 0;
1558 }
1559
hl_fw_static_read_device_fw_version(struct hl_device * hdev,enum hl_fw_component fwc)1560 static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
1561 enum hl_fw_component fwc)
1562 {
1563 struct asic_fixed_properties *prop = &hdev->asic_prop;
1564 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
1565 struct static_fw_load_mgr *static_loader;
1566 char *dest, *boot_ver, *preboot_ver;
1567 u32 ver_off, limit;
1568 const char *name;
1569 char btl_ver[32];
1570
1571 static_loader = &hdev->fw_loader.static_loader;
1572
1573 switch (fwc) {
1574 case FW_COMP_BOOT_FIT:
1575 ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
1576 dest = prop->uboot_ver;
1577 name = "Boot-fit";
1578 limit = static_loader->boot_fit_version_max_off;
1579 break;
1580 case FW_COMP_PREBOOT:
1581 ver_off = RREG32(static_loader->preboot_version_offset_reg);
1582 dest = prop->preboot_ver;
1583 name = "Preboot";
1584 limit = static_loader->preboot_version_max_off;
1585 break;
1586 default:
1587 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
1588 return -EIO;
1589 }
1590
1591 ver_off &= static_loader->sram_offset_mask;
1592
1593 if (ver_off < limit) {
1594 memcpy_fromio(dest,
1595 hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
1596 VERSION_MAX_LEN);
1597 } else {
1598 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
1599 name, ver_off);
1600 strscpy(dest, "unavailable", VERSION_MAX_LEN);
1601 return -EIO;
1602 }
1603
1604 if (fwc == FW_COMP_BOOT_FIT) {
1605 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
1606 if (boot_ver) {
1607 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
1608 kfree(boot_ver);
1609 }
1610 } else if (fwc == FW_COMP_PREBOOT) {
1611 preboot_ver = strnstr(prop->preboot_ver, "Preboot",
1612 VERSION_MAX_LEN);
1613 if (preboot_ver && preboot_ver != prop->preboot_ver) {
1614 strscpy(btl_ver, prop->preboot_ver,
1615 min((int) (preboot_ver - prop->preboot_ver),
1616 31));
1617 dev_info(hdev->dev, "%s\n", btl_ver);
1618 }
1619
1620 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
1621 if (preboot_ver) {
1622 dev_info(hdev->dev, "preboot version %s\n",
1623 preboot_ver);
1624 kfree(preboot_ver);
1625 }
1626 }
1627
1628 return 0;
1629 }
1630
1631 /**
1632 * hl_fw_preboot_update_state - update internal data structures during
1633 * handshake with preboot
1634 *
1635 *
1636 * @hdev: pointer to the habanalabs device structure
1637 *
1638 * @return 0 on success, otherwise non-zero error code
1639 */
hl_fw_preboot_update_state(struct hl_device * hdev)1640 static void hl_fw_preboot_update_state(struct hl_device *hdev)
1641 {
1642 struct asic_fixed_properties *prop = &hdev->asic_prop;
1643 u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
1644
1645 cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
1646 cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
1647
1648 /* We read boot_dev_sts registers multiple times during boot:
1649 * 1. preboot - a. Check whether the security status bits are valid
1650 * b. Check whether fw security is enabled
1651 * c. Check whether hard reset is done by preboot
1652 * 2. boot cpu - a. Fetch boot cpu security status
1653 * b. Check whether hard reset is done by boot cpu
1654 * 3. FW application - a. Fetch fw application security status
1655 * b. Check whether hard reset is done by fw app
1656 */
1657 prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
1658
1659 prop->fw_security_enabled = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_SECURITY_EN);
1660
1661 dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
1662 cpu_boot_dev_sts0);
1663
1664 dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
1665 cpu_boot_dev_sts1);
1666
1667 dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
1668 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
1669
1670 dev_dbg(hdev->dev, "firmware-level security is %s\n",
1671 prop->fw_security_enabled ? "enabled" : "disabled");
1672
1673 dev_dbg(hdev->dev, "GIC controller is %s\n",
1674 prop->gic_interrupts_enable ? "enabled" : "disabled");
1675 }
1676
hl_fw_static_read_preboot_status(struct hl_device * hdev)1677 static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
1678 {
1679 int rc;
1680
1681 rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
1682 if (rc)
1683 return rc;
1684
1685 return 0;
1686 }
1687
hl_fw_read_preboot_status(struct hl_device * hdev)1688 int hl_fw_read_preboot_status(struct hl_device *hdev)
1689 {
1690 int rc;
1691
1692 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
1693 return 0;
1694
1695 /* get FW pre-load parameters */
1696 hdev->asic_funcs->init_firmware_preload_params(hdev);
1697
1698 /*
1699 * In order to determine boot method (static VS dynamic) we need to
1700 * read the boot caps register
1701 */
1702 rc = hl_fw_read_preboot_caps(hdev);
1703 if (rc)
1704 return rc;
1705
1706 hl_fw_preboot_update_state(hdev);
1707
1708 /* no need to read preboot status in dynamic load */
1709 if (hdev->asic_prop.dynamic_fw_load)
1710 return 0;
1711
1712 return hl_fw_static_read_preboot_status(hdev);
1713 }
1714
1715 /* associate string with COMM status */
1716 static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
1717 [COMMS_STS_NOOP] = "NOOP",
1718 [COMMS_STS_ACK] = "ACK",
1719 [COMMS_STS_OK] = "OK",
1720 [COMMS_STS_ERR] = "ERR",
1721 [COMMS_STS_VALID_ERR] = "VALID_ERR",
1722 [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
1723 };
1724
1725 /**
1726 * hl_fw_dynamic_report_error_status - report error status
1727 *
1728 * @hdev: pointer to the habanalabs device structure
1729 * @status: value of FW status register
1730 * @expected_status: the expected status
1731 */
hl_fw_dynamic_report_error_status(struct hl_device * hdev,u32 status,enum comms_sts expected_status)1732 static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
1733 u32 status,
1734 enum comms_sts expected_status)
1735 {
1736 enum comms_sts comm_status =
1737 FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1738
1739 if (comm_status < COMMS_STS_INVLD_LAST)
1740 dev_err(hdev->dev, "Device status %s, expected status: %s\n",
1741 hl_dynamic_fw_status_str[comm_status],
1742 hl_dynamic_fw_status_str[expected_status]);
1743 else
1744 dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
1745 comm_status,
1746 hl_dynamic_fw_status_str[expected_status]);
1747 }
1748
1749 /**
1750 * hl_fw_dynamic_send_cmd - send LKD to FW cmd
1751 *
1752 * @hdev: pointer to the habanalabs device structure
1753 * @fw_loader: managing structure for loading device's FW
1754 * @cmd: LKD to FW cmd code
1755 * @size: size of next FW component to be loaded (0 if not necessary)
1756 *
1757 * LDK to FW exact command layout is defined at struct comms_command.
1758 * note: the size argument is used only when the next FW component should be
1759 * loaded, otherwise it shall be 0. the size is used by the FW in later
1760 * protocol stages and when sending only indicating the amount of memory
1761 * to be allocated by the FW to receive the next boot component.
1762 */
hl_fw_dynamic_send_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_cmd cmd,unsigned int size)1763 static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
1764 struct fw_load_mgr *fw_loader,
1765 enum comms_cmd cmd, unsigned int size)
1766 {
1767 struct cpu_dyn_regs *dyn_regs;
1768 u32 val;
1769
1770 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1771
1772 val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
1773 val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
1774
1775 trace_habanalabs_comms_send_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1776 WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
1777 }
1778
1779 /**
1780 * hl_fw_dynamic_extract_fw_response - update the FW response
1781 *
1782 * @hdev: pointer to the habanalabs device structure
1783 * @fw_loader: managing structure for loading device's FW
1784 * @response: FW response
1785 * @status: the status read from CPU status register
1786 *
1787 * @return 0 on success, otherwise non-zero error code
1788 */
hl_fw_dynamic_extract_fw_response(struct hl_device * hdev,struct fw_load_mgr * fw_loader,struct fw_response * response,u32 status)1789 static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
1790 struct fw_load_mgr *fw_loader,
1791 struct fw_response *response,
1792 u32 status)
1793 {
1794 response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
1795 response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
1796 COMMS_STATUS_OFFSET_ALIGN_SHIFT;
1797 response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
1798
1799 if ((response->ram_type != COMMS_SRAM) &&
1800 (response->ram_type != COMMS_DRAM)) {
1801 dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
1802 response->ram_type);
1803 return -EIO;
1804 }
1805
1806 return 0;
1807 }
1808
1809 /**
1810 * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
1811 *
1812 * @hdev: pointer to the habanalabs device structure
1813 * @fw_loader: managing structure for loading device's FW
1814 * @expected_status: expected status to wait for
1815 * @timeout: timeout for status wait
1816 *
1817 * @return 0 on success, otherwise non-zero error code
1818 *
1819 * waiting for status from FW include polling the FW status register until
1820 * expected status is received or timeout occurs (whatever occurs first).
1821 */
hl_fw_dynamic_wait_for_status(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_sts expected_status,u32 timeout)1822 static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
1823 struct fw_load_mgr *fw_loader,
1824 enum comms_sts expected_status,
1825 u32 timeout)
1826 {
1827 struct cpu_dyn_regs *dyn_regs;
1828 u32 status;
1829 int rc;
1830
1831 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
1832
1833 trace_habanalabs_comms_wait_status(hdev->dev, comms_sts_str_arr[expected_status]);
1834
1835 /* Wait for expected status */
1836 rc = hl_poll_timeout(
1837 hdev,
1838 le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
1839 status,
1840 FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
1841 hdev->fw_comms_poll_interval_usec,
1842 timeout);
1843
1844 if (rc) {
1845 hl_fw_dynamic_report_error_status(hdev, status,
1846 expected_status);
1847 return -EIO;
1848 }
1849
1850 trace_habanalabs_comms_wait_status_done(hdev->dev, comms_sts_str_arr[expected_status]);
1851
1852 /*
1853 * skip storing FW response for NOOP to preserve the actual desired
1854 * FW status
1855 */
1856 if (expected_status == COMMS_STS_NOOP)
1857 return 0;
1858
1859 rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
1860 &fw_loader->dynamic_loader.response,
1861 status);
1862 return rc;
1863 }
1864
1865 /**
1866 * hl_fw_dynamic_send_clear_cmd - send clear command to FW
1867 *
1868 * @hdev: pointer to the habanalabs device structure
1869 * @fw_loader: managing structure for loading device's FW
1870 *
1871 * @return 0 on success, otherwise non-zero error code
1872 *
1873 * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
1874 * from FW) we need to clear the CPU status register in order to avoid garbage
1875 * between command cycles.
1876 * This is done by sending clear command and polling the CPU to LKD status
1877 * register to hold the status NOOP
1878 */
hl_fw_dynamic_send_clear_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader)1879 static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
1880 struct fw_load_mgr *fw_loader)
1881 {
1882 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
1883
1884 return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
1885 fw_loader->cpu_timeout);
1886 }
1887
1888 /**
1889 * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
1890 *
1891 * @hdev: pointer to the habanalabs device structure
1892 * @fw_loader: managing structure for loading device's FW
1893 * @cmd: LKD to FW cmd code
1894 * @size: size of next FW component to be loaded (0 if not necessary)
1895 * @wait_ok: if true also wait for OK response from FW
1896 * @timeout: timeout for status wait
1897 *
1898 * @return 0 on success, otherwise non-zero error code
1899 *
1900 * brief:
1901 * when sending protocol command we have the following steps:
1902 * - send clear (clear command and verify clear status register)
1903 * - send the actual protocol command
1904 * - wait for ACK on the protocol command
1905 * - send clear
1906 * - send NOOP
1907 * if, in addition, the specific protocol command should wait for OK then:
1908 * - wait for OK
1909 * - send clear
1910 * - send NOOP
1911 *
1912 * NOTES:
1913 * send clear: this is necessary in order to clear the status register to avoid
1914 * leftovers between command
1915 * NOOP command: necessary to avoid loop on the clear command by the FW
1916 */
hl_fw_dynamic_send_protocol_cmd(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum comms_cmd cmd,unsigned int size,bool wait_ok,u32 timeout)1917 int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
1918 struct fw_load_mgr *fw_loader,
1919 enum comms_cmd cmd, unsigned int size,
1920 bool wait_ok, u32 timeout)
1921 {
1922 int rc;
1923
1924 trace_habanalabs_comms_protocol_cmd(hdev->dev, comms_cmd_str_arr[cmd]);
1925
1926 /* first send clear command to clean former commands */
1927 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1928 if (rc)
1929 return rc;
1930
1931 /* send the actual command */
1932 hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
1933
1934 /* wait for ACK for the command */
1935 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
1936 timeout);
1937 if (rc)
1938 return rc;
1939
1940 /* clear command to prepare for NOOP command */
1941 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1942 if (rc)
1943 return rc;
1944
1945 /* send the actual NOOP command */
1946 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1947
1948 if (!wait_ok)
1949 return 0;
1950
1951 rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
1952 timeout);
1953 if (rc)
1954 return rc;
1955
1956 /* clear command to prepare for NOOP command */
1957 rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
1958 if (rc)
1959 return rc;
1960
1961 /* send the actual NOOP command */
1962 hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
1963
1964 return 0;
1965 }
1966
1967 /**
1968 * hl_fw_compat_crc32 - CRC compatible with FW
1969 *
1970 * @data: pointer to the data
1971 * @size: size of the data
1972 *
1973 * @return the CRC32 result
1974 *
1975 * NOTE: kernel's CRC32 differs from standard CRC32 calculation.
1976 * in order to be aligned we need to flip the bits of both the input
1977 * initial CRC and kernel's CRC32 result.
1978 * in addition both sides use initial CRC of 0,
1979 */
hl_fw_compat_crc32(u8 * data,size_t size)1980 static u32 hl_fw_compat_crc32(u8 *data, size_t size)
1981 {
1982 return ~crc32_le(~((u32)0), data, size);
1983 }
1984
1985 /**
1986 * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
1987 * transfer (image or descriptor) between
1988 * host and FW
1989 *
1990 * @hdev: pointer to the habanalabs device structure
1991 * @addr: device address of memory transfer
1992 * @size: memory transfer size
1993 * @region: PCI memory region
1994 *
1995 * @return 0 on success, otherwise non-zero error code
1996 */
hl_fw_dynamic_validate_memory_bound(struct hl_device * hdev,u64 addr,size_t size,struct pci_mem_region * region)1997 static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
1998 u64 addr, size_t size,
1999 struct pci_mem_region *region)
2000 {
2001 u64 end_addr;
2002
2003 /* now make sure that the memory transfer is within region's bounds */
2004 end_addr = addr + size;
2005 if (end_addr >= region->region_base + region->region_size) {
2006 dev_err(hdev->dev,
2007 "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
2008 end_addr);
2009 return -EIO;
2010 }
2011
2012 /*
2013 * now make sure memory transfer is within predefined BAR bounds.
2014 * this is to make sure we do not need to set the bar (e.g. for DRAM
2015 * memory transfers)
2016 */
2017 if (end_addr >= region->region_base - region->offset_in_bar +
2018 region->bar_size) {
2019 dev_err(hdev->dev,
2020 "FW image beyond PCI BAR bounds\n");
2021 return -EIO;
2022 }
2023
2024 return 0;
2025 }
2026
2027 /**
2028 * hl_fw_dynamic_validate_descriptor - validate FW descriptor
2029 *
2030 * @hdev: pointer to the habanalabs device structure
2031 * @fw_loader: managing structure for loading device's FW
2032 * @fw_desc: the descriptor from FW
2033 *
2034 * @return 0 on success, otherwise non-zero error code
2035 */
hl_fw_dynamic_validate_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader,struct lkd_fw_comms_desc * fw_desc)2036 static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
2037 struct fw_load_mgr *fw_loader,
2038 struct lkd_fw_comms_desc *fw_desc)
2039 {
2040 struct pci_mem_region *region;
2041 enum pci_region region_id;
2042 size_t data_size;
2043 u32 data_crc32;
2044 u8 *data_ptr;
2045 u64 addr;
2046 int rc;
2047
2048 if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC)
2049 dev_dbg(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
2050 fw_desc->header.magic);
2051
2052 if (fw_desc->header.version != HL_COMMS_DESC_VER)
2053 dev_dbg(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
2054 fw_desc->header.version);
2055
2056 /*
2057 * Calc CRC32 of data without header. use the size of the descriptor
2058 * reported by firmware, without calculating it ourself, to allow adding
2059 * more fields to the lkd_fw_comms_desc structure.
2060 * note that no alignment/stride address issues here as all structures
2061 * are 64 bit padded.
2062 */
2063 data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
2064 data_size = le16_to_cpu(fw_desc->header.size);
2065
2066 data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
2067 if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
2068 dev_err(hdev->dev, "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
2069 data_crc32, fw_desc->header.crc32);
2070 return -EIO;
2071 }
2072
2073 /* find memory region to which to copy the image */
2074 addr = le64_to_cpu(fw_desc->img_addr);
2075 region_id = hl_get_pci_memory_region(hdev, addr);
2076 if ((region_id != PCI_REGION_SRAM) && ((region_id != PCI_REGION_DRAM))) {
2077 dev_err(hdev->dev, "Invalid region to copy FW image address=%llx\n", addr);
2078 return -EIO;
2079 }
2080
2081 region = &hdev->pci_mem_region[region_id];
2082
2083 /* store the region for the copy stage */
2084 fw_loader->dynamic_loader.image_region = region;
2085
2086 /*
2087 * here we know that the start address is valid, now make sure that the
2088 * image is within region's bounds
2089 */
2090 rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
2091 fw_loader->dynamic_loader.fw_image_size,
2092 region);
2093 if (rc) {
2094 dev_err(hdev->dev, "invalid mem transfer request for FW image\n");
2095 return rc;
2096 }
2097
2098 /* here we can mark the descriptor as valid as the content has been validated */
2099 fw_loader->dynamic_loader.fw_desc_valid = true;
2100
2101 return 0;
2102 }
2103
hl_fw_dynamic_validate_response(struct hl_device * hdev,struct fw_response * response,struct pci_mem_region * region)2104 static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
2105 struct fw_response *response,
2106 struct pci_mem_region *region)
2107 {
2108 u64 device_addr;
2109 int rc;
2110
2111 device_addr = region->region_base + response->ram_offset;
2112
2113 /*
2114 * validate that the descriptor is within region's bounds
2115 * Note that as the start address was supplied according to the RAM
2116 * type- testing only the end address is enough
2117 */
2118 rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
2119 sizeof(struct lkd_fw_comms_desc),
2120 region);
2121 return rc;
2122 }
2123
2124 /*
2125 * hl_fw_dynamic_read_descriptor_msg - read and show the ascii msg that sent by fw
2126 *
2127 * @hdev: pointer to the habanalabs device structure
2128 * @fw_desc: the descriptor from FW
2129 */
hl_fw_dynamic_read_descriptor_msg(struct hl_device * hdev,struct lkd_fw_comms_desc * fw_desc)2130 static void hl_fw_dynamic_read_descriptor_msg(struct hl_device *hdev,
2131 struct lkd_fw_comms_desc *fw_desc)
2132 {
2133 int i;
2134 char *msg;
2135
2136 for (i = 0 ; i < LKD_FW_ASCII_MSG_MAX ; i++) {
2137 if (!fw_desc->ascii_msg[i].valid)
2138 return;
2139
2140 /* force NULL termination */
2141 msg = fw_desc->ascii_msg[i].msg;
2142 msg[LKD_FW_ASCII_MSG_MAX_LEN - 1] = '\0';
2143
2144 switch (fw_desc->ascii_msg[i].msg_lvl) {
2145 case LKD_FW_ASCII_MSG_ERR:
2146 dev_err(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2147 break;
2148 case LKD_FW_ASCII_MSG_WRN:
2149 dev_warn(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2150 break;
2151 case LKD_FW_ASCII_MSG_INF:
2152 dev_info(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2153 break;
2154 default:
2155 dev_dbg(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);
2156 break;
2157 }
2158 }
2159 }
2160
2161 /**
2162 * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
2163 *
2164 * @hdev: pointer to the habanalabs device structure
2165 * @fw_loader: managing structure for loading device's FW
2166 *
2167 * @return 0 on success, otherwise non-zero error code
2168 */
hl_fw_dynamic_read_and_validate_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2169 static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
2170 struct fw_load_mgr *fw_loader)
2171 {
2172 struct lkd_fw_comms_desc *fw_desc;
2173 struct pci_mem_region *region;
2174 struct fw_response *response;
2175 void *temp_fw_desc;
2176 void __iomem *src;
2177 u16 fw_data_size;
2178 enum pci_region region_id;
2179 int rc;
2180
2181 fw_desc = &fw_loader->dynamic_loader.comm_desc;
2182 response = &fw_loader->dynamic_loader.response;
2183
2184 region_id = (response->ram_type == COMMS_SRAM) ?
2185 PCI_REGION_SRAM : PCI_REGION_DRAM;
2186
2187 region = &hdev->pci_mem_region[region_id];
2188
2189 rc = hl_fw_dynamic_validate_response(hdev, response, region);
2190 if (rc) {
2191 dev_err(hdev->dev,
2192 "invalid mem transfer request for FW descriptor\n");
2193 return rc;
2194 }
2195
2196 /*
2197 * extract address to copy the descriptor from
2198 * in addition, as the descriptor value is going to be over-ridden by new data- we mark it
2199 * as invalid.
2200 * it will be marked again as valid once validated
2201 */
2202 fw_loader->dynamic_loader.fw_desc_valid = false;
2203 src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2204 response->ram_offset;
2205
2206 /*
2207 * We do the copy of the fw descriptor in 2 phases:
2208 * 1. copy the header + data info according to our lkd_fw_comms_desc definition.
2209 * then we're able to read the actual data size provided by fw.
2210 * this is needed for cases where data in descriptor was changed(add/remove)
2211 * in embedded specs header file before updating lkd copy of the header file
2212 * 2. copy descriptor to temporary buffer with aligned size and send it to validation
2213 */
2214 memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
2215 fw_data_size = le16_to_cpu(fw_desc->header.size);
2216
2217 temp_fw_desc = vzalloc(sizeof(struct comms_desc_header) + fw_data_size);
2218 if (!temp_fw_desc)
2219 return -ENOMEM;
2220
2221 memcpy_fromio(temp_fw_desc, src, sizeof(struct comms_desc_header) + fw_data_size);
2222
2223 rc = hl_fw_dynamic_validate_descriptor(hdev, fw_loader,
2224 (struct lkd_fw_comms_desc *) temp_fw_desc);
2225
2226 if (!rc)
2227 hl_fw_dynamic_read_descriptor_msg(hdev, temp_fw_desc);
2228
2229 vfree(temp_fw_desc);
2230
2231 return rc;
2232 }
2233
2234 /**
2235 * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
2236 *
2237 * @hdev: pointer to the habanalabs device structure
2238 * @fw_loader: managing structure for loading device's FW
2239 * @next_image_size: size to allocate for next FW component
2240 *
2241 * @return 0 on success, otherwise non-zero error code
2242 */
hl_fw_dynamic_request_descriptor(struct hl_device * hdev,struct fw_load_mgr * fw_loader,size_t next_image_size)2243 static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
2244 struct fw_load_mgr *fw_loader,
2245 size_t next_image_size)
2246 {
2247 int rc;
2248
2249 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
2250 next_image_size, true,
2251 fw_loader->cpu_timeout);
2252 if (rc)
2253 return rc;
2254
2255 return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
2256 }
2257
2258 /**
2259 * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
2260 *
2261 * @hdev: pointer to the habanalabs device structure
2262 * @fwc: the firmware component
2263 * @fw_version: fw component's version string
2264 */
hl_fw_dynamic_read_device_fw_version(struct hl_device * hdev,enum hl_fw_component fwc,const char * fw_version)2265 static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
2266 enum hl_fw_component fwc,
2267 const char *fw_version)
2268 {
2269 struct asic_fixed_properties *prop = &hdev->asic_prop;
2270 char *preboot_ver, *boot_ver;
2271 char btl_ver[32];
2272 int rc;
2273
2274 switch (fwc) {
2275 case FW_COMP_BOOT_FIT:
2276 strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
2277 boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
2278 if (boot_ver) {
2279 dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
2280 kfree(boot_ver);
2281 }
2282
2283 break;
2284 case FW_COMP_PREBOOT:
2285 strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
2286 preboot_ver = strnstr(prop->preboot_ver, "Preboot", VERSION_MAX_LEN);
2287 dev_info(hdev->dev, "preboot full version: '%s'\n", preboot_ver);
2288
2289 if (preboot_ver && preboot_ver != prop->preboot_ver) {
2290 strscpy(btl_ver, prop->preboot_ver,
2291 min((int) (preboot_ver - prop->preboot_ver), 31));
2292 dev_info(hdev->dev, "%s\n", btl_ver);
2293 }
2294
2295 rc = hl_get_sw_major_minor_subminor(hdev, preboot_ver);
2296 if (rc)
2297 return rc;
2298 preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
2299 if (preboot_ver) {
2300 rc = hl_get_preboot_major_minor(hdev, preboot_ver);
2301 kfree(preboot_ver);
2302 if (rc)
2303 return rc;
2304 }
2305
2306 break;
2307 default:
2308 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2309 return -EINVAL;
2310 }
2311
2312 return 0;
2313 }
2314
2315 /**
2316 * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
2317 *
2318 * @hdev: pointer to the habanalabs device structure
2319 * @fw: fw descriptor
2320 * @fw_loader: managing structure for loading device's FW
2321 */
hl_fw_dynamic_copy_image(struct hl_device * hdev,const struct firmware * fw,struct fw_load_mgr * fw_loader)2322 static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
2323 const struct firmware *fw,
2324 struct fw_load_mgr *fw_loader)
2325 {
2326 struct lkd_fw_comms_desc *fw_desc;
2327 struct pci_mem_region *region;
2328 void __iomem *dest;
2329 u64 addr;
2330 int rc;
2331
2332 fw_desc = &fw_loader->dynamic_loader.comm_desc;
2333 addr = le64_to_cpu(fw_desc->img_addr);
2334
2335 /* find memory region to which to copy the image */
2336 region = fw_loader->dynamic_loader.image_region;
2337
2338 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2339 (addr - region->region_base);
2340
2341 rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
2342 fw_loader->boot_fit_img.src_off,
2343 fw_loader->boot_fit_img.copy_size);
2344
2345 return rc;
2346 }
2347
2348 /**
2349 * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
2350 *
2351 * @hdev: pointer to the habanalabs device structure
2352 * @msg: message
2353 * @fw_loader: managing structure for loading device's FW
2354 */
hl_fw_dynamic_copy_msg(struct hl_device * hdev,struct lkd_msg_comms * msg,struct fw_load_mgr * fw_loader)2355 static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
2356 struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
2357 {
2358 struct lkd_fw_comms_desc *fw_desc;
2359 struct pci_mem_region *region;
2360 void __iomem *dest;
2361 u64 addr;
2362 int rc;
2363
2364 fw_desc = &fw_loader->dynamic_loader.comm_desc;
2365 addr = le64_to_cpu(fw_desc->img_addr);
2366
2367 /* find memory region to which to copy the image */
2368 region = fw_loader->dynamic_loader.image_region;
2369
2370 dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
2371 (addr - region->region_base);
2372
2373 rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
2374
2375 return rc;
2376 }
2377
2378 /**
2379 * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
2380 * is loaded
2381 *
2382 * @hdev: pointer to the habanalabs device structure
2383 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2384 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2385 *
2386 * @return 0 on success, otherwise non-zero error code
2387 */
hl_fw_boot_fit_update_state(struct hl_device * hdev,u32 cpu_boot_dev_sts0_reg,u32 cpu_boot_dev_sts1_reg)2388 static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
2389 u32 cpu_boot_dev_sts0_reg,
2390 u32 cpu_boot_dev_sts1_reg)
2391 {
2392 struct asic_fixed_properties *prop = &hdev->asic_prop;
2393
2394 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;
2395
2396 /* Read boot_cpu status bits */
2397 if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
2398 prop->fw_bootfit_cpu_boot_dev_sts0 =
2399 RREG32(cpu_boot_dev_sts0_reg);
2400
2401 prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &
2402 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2403
2404 dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
2405 prop->fw_bootfit_cpu_boot_dev_sts0);
2406 }
2407
2408 if (prop->fw_cpu_boot_dev_sts1_valid) {
2409 prop->fw_bootfit_cpu_boot_dev_sts1 =
2410 RREG32(cpu_boot_dev_sts1_reg);
2411
2412 dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
2413 prop->fw_bootfit_cpu_boot_dev_sts1);
2414 }
2415
2416 dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
2417 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2418 }
2419
hl_fw_dynamic_update_linux_interrupt_if(struct hl_device * hdev)2420 static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
2421 {
2422 struct cpu_dyn_regs *dyn_regs =
2423 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
2424
2425 /* Check whether all 3 interrupt interfaces are set, if not use a
2426 * single interface
2427 */
2428 if (!hdev->asic_prop.gic_interrupts_enable &&
2429 !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
2430 CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
2431 dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;
2432 dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;
2433
2434 dev_warn(hdev->dev,
2435 "Using a single interrupt interface towards cpucp");
2436 }
2437 }
2438 /**
2439 * hl_fw_dynamic_load_image - load FW image using dynamic protocol
2440 *
2441 * @hdev: pointer to the habanalabs device structure
2442 * @fw_loader: managing structure for loading device's FW
2443 * @load_fwc: the FW component to be loaded
2444 * @img_ld_timeout: image load timeout
2445 *
2446 * @return 0 on success, otherwise non-zero error code
2447 */
hl_fw_dynamic_load_image(struct hl_device * hdev,struct fw_load_mgr * fw_loader,enum hl_fw_component load_fwc,u32 img_ld_timeout)2448 static int hl_fw_dynamic_load_image(struct hl_device *hdev,
2449 struct fw_load_mgr *fw_loader,
2450 enum hl_fw_component load_fwc,
2451 u32 img_ld_timeout)
2452 {
2453 enum hl_fw_component cur_fwc;
2454 const struct firmware *fw;
2455 char *fw_name;
2456 int rc = 0;
2457
2458 /*
2459 * when loading image we have one of 2 scenarios:
2460 * 1. current FW component is preboot and we want to load boot-fit
2461 * 2. current FW component is boot-fit and we want to load linux
2462 */
2463 if (load_fwc == FW_COMP_BOOT_FIT) {
2464 cur_fwc = FW_COMP_PREBOOT;
2465 fw_name = fw_loader->boot_fit_img.image_name;
2466 } else {
2467 cur_fwc = FW_COMP_BOOT_FIT;
2468 fw_name = fw_loader->linux_img.image_name;
2469 }
2470
2471 /* request FW in order to communicate to FW the size to be allocated */
2472 rc = hl_request_fw(hdev, &fw, fw_name);
2473 if (rc)
2474 return rc;
2475
2476 /* store the image size for future validation */
2477 fw_loader->dynamic_loader.fw_image_size = fw->size;
2478
2479 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
2480 if (rc)
2481 goto release_fw;
2482
2483 /* read preboot version */
2484 rc = hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
2485 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2486 if (rc)
2487 goto release_fw;
2488
2489 /* copy boot fit to space allocated by FW */
2490 rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
2491 if (rc)
2492 goto release_fw;
2493
2494 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2495 0, true,
2496 fw_loader->cpu_timeout);
2497 if (rc)
2498 goto release_fw;
2499
2500 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2501 0, false,
2502 img_ld_timeout);
2503
2504 release_fw:
2505 hl_release_firmware(fw);
2506 return rc;
2507 }
2508
hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2509 static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
2510 struct fw_load_mgr *fw_loader)
2511 {
2512 struct dynamic_fw_load_mgr *dyn_loader;
2513 u32 status;
2514 int rc;
2515
2516 dyn_loader = &fw_loader->dynamic_loader;
2517
2518 /*
2519 * Make sure CPU boot-loader is running
2520 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2521 * yet there is a debug scenario in which we loading uboot (without Linux)
2522 * which at later stage is relocated to DRAM. In this case we expect
2523 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2524 * poll flags
2525 */
2526 rc = hl_poll_timeout(
2527 hdev,
2528 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2529 status,
2530 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2531 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2532 hdev->fw_poll_interval_usec,
2533 dyn_loader->wait_for_bl_timeout);
2534 if (rc) {
2535 dev_err(hdev->dev, "failed to wait for boot (status = %d)\n", status);
2536 return rc;
2537 }
2538
2539 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2540 return 0;
2541 }
2542
hl_fw_dynamic_wait_for_linux_active(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2543 static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
2544 struct fw_load_mgr *fw_loader)
2545 {
2546 struct dynamic_fw_load_mgr *dyn_loader;
2547 u32 status;
2548 int rc;
2549
2550 dyn_loader = &fw_loader->dynamic_loader;
2551
2552 /* Make sure CPU linux is running */
2553
2554 rc = hl_poll_timeout(
2555 hdev,
2556 le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
2557 status,
2558 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2559 hdev->fw_poll_interval_usec,
2560 fw_loader->cpu_timeout);
2561 if (rc) {
2562 dev_err(hdev->dev, "failed to wait for Linux (status = %d)\n", status);
2563 return rc;
2564 }
2565
2566 dev_dbg(hdev->dev, "Boot status = %d\n", status);
2567 return 0;
2568 }
2569
2570 /**
2571 * hl_fw_linux_update_state - update internal data structures after Linux
2572 * is loaded.
2573 * Note: Linux initialization is comprised mainly
2574 * of two stages - loading kernel (SRAM_AVAIL)
2575 * & loading ARMCP.
2576 * Therefore reading boot device status in any of
2577 * these stages might result in different values.
2578 *
2579 * @hdev: pointer to the habanalabs device structure
2580 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
2581 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
2582 *
2583 * @return 0 on success, otherwise non-zero error code
2584 */
hl_fw_linux_update_state(struct hl_device * hdev,u32 cpu_boot_dev_sts0_reg,u32 cpu_boot_dev_sts1_reg)2585 static void hl_fw_linux_update_state(struct hl_device *hdev,
2586 u32 cpu_boot_dev_sts0_reg,
2587 u32 cpu_boot_dev_sts1_reg)
2588 {
2589 struct asic_fixed_properties *prop = &hdev->asic_prop;
2590
2591 hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;
2592
2593 /* Read FW application security bits */
2594 if (prop->fw_cpu_boot_dev_sts0_valid) {
2595 prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);
2596
2597 prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &
2598 CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
2599
2600 if (prop->fw_app_cpu_boot_dev_sts0 &
2601 CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
2602 prop->gic_interrupts_enable = false;
2603
2604 dev_dbg(hdev->dev,
2605 "Firmware application CPU status0 %#x\n",
2606 prop->fw_app_cpu_boot_dev_sts0);
2607
2608 dev_dbg(hdev->dev, "GIC controller is %s\n",
2609 prop->gic_interrupts_enable ?
2610 "enabled" : "disabled");
2611 }
2612
2613 if (prop->fw_cpu_boot_dev_sts1_valid) {
2614 prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);
2615
2616 dev_dbg(hdev->dev,
2617 "Firmware application CPU status1 %#x\n",
2618 prop->fw_app_cpu_boot_dev_sts1);
2619 }
2620
2621 dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
2622 prop->hard_reset_done_by_fw ? "enabled" : "disabled");
2623
2624 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2625 }
2626
2627 /**
2628 * hl_fw_dynamic_send_msg - send a COMMS message with attached data
2629 *
2630 * @hdev: pointer to the habanalabs device structure
2631 * @fw_loader: managing structure for loading device's FW
2632 * @msg_type: message type
2633 * @data: data to be sent
2634 *
2635 * @return 0 on success, otherwise non-zero error code
2636 */
hl_fw_dynamic_send_msg(struct hl_device * hdev,struct fw_load_mgr * fw_loader,u8 msg_type,void * data)2637 static int hl_fw_dynamic_send_msg(struct hl_device *hdev,
2638 struct fw_load_mgr *fw_loader, u8 msg_type, void *data)
2639 {
2640 struct lkd_msg_comms *msg;
2641 int rc;
2642
2643 msg = kzalloc(sizeof(*msg), GFP_KERNEL);
2644 if (!msg)
2645 return -ENOMEM;
2646
2647 /* create message to be sent */
2648 msg->header.type = msg_type;
2649 msg->header.size = cpu_to_le16(sizeof(struct comms_msg_header));
2650 msg->header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
2651
2652 switch (msg_type) {
2653 case HL_COMMS_RESET_CAUSE_TYPE:
2654 msg->reset_cause = *(__u8 *) data;
2655 break;
2656
2657 default:
2658 dev_err(hdev->dev,
2659 "Send COMMS message - invalid message type %u\n",
2660 msg_type);
2661 rc = -EINVAL;
2662 goto out;
2663 }
2664
2665 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
2666 sizeof(struct lkd_msg_comms));
2667 if (rc)
2668 goto out;
2669
2670 /* copy message to space allocated by FW */
2671 rc = hl_fw_dynamic_copy_msg(hdev, msg, fw_loader);
2672 if (rc)
2673 goto out;
2674
2675 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
2676 0, true,
2677 fw_loader->cpu_timeout);
2678 if (rc)
2679 goto out;
2680
2681 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
2682 0, true,
2683 fw_loader->cpu_timeout);
2684
2685 out:
2686 kfree(msg);
2687 return rc;
2688 }
2689
2690 /**
2691 * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
2692 *
2693 * @hdev: pointer to the habanalabs device structure
2694 * @fw_loader: managing structure for loading device's FW
2695 *
2696 * @return 0 on success, otherwise non-zero error code
2697 *
2698 * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
2699 * the communication is done using registers:
2700 * - LKD command register
2701 * - FW status register
2702 * the protocol is race free. this goal is achieved by splitting the requests
2703 * and response to known synchronization points between the LKD and the FW.
2704 * each response to LKD request is known and bound to a predefined timeout.
2705 * in case of timeout expiration without the desired status from FW- the
2706 * protocol (and hence the boot) will fail.
2707 */
hl_fw_dynamic_init_cpu(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2708 static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
2709 struct fw_load_mgr *fw_loader)
2710 {
2711 struct cpu_dyn_regs *dyn_regs;
2712 int rc, fw_error_rc;
2713
2714 dev_info(hdev->dev,
2715 "Loading %sfirmware to device, may take some time...\n",
2716 hdev->asic_prop.fw_security_enabled ? "secured " : "");
2717
2718 /* initialize FW descriptor as invalid */
2719 fw_loader->dynamic_loader.fw_desc_valid = false;
2720
2721 /*
2722 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!
2723 * It will be updated from FW after hl_fw_dynamic_request_descriptor().
2724 */
2725 dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
2726
2727 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
2728 0, true,
2729 fw_loader->cpu_timeout);
2730 if (rc)
2731 goto protocol_err;
2732
2733 if (hdev->reset_info.curr_reset_cause) {
2734 rc = hl_fw_dynamic_send_msg(hdev, fw_loader,
2735 HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);
2736 if (rc)
2737 goto protocol_err;
2738
2739 /* Clear current reset cause */
2740 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
2741 }
2742
2743 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
2744 struct lkd_fw_binning_info *binning_info;
2745
2746 rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
2747 if (rc)
2748 goto protocol_err;
2749
2750 /* read preboot version */
2751 rc = hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
2752 fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
2753
2754 if (rc)
2755 return rc;
2756
2757 /* read binning info from preboot */
2758 if (hdev->support_preboot_binning) {
2759 binning_info = &fw_loader->dynamic_loader.comm_desc.binning_info;
2760 hdev->tpc_binning = le64_to_cpu(binning_info->tpc_mask_l);
2761 hdev->dram_binning = le32_to_cpu(binning_info->dram_mask);
2762 hdev->edma_binning = le32_to_cpu(binning_info->edma_mask);
2763 hdev->decoder_binning = le32_to_cpu(binning_info->dec_mask);
2764 hdev->rotator_binning = le32_to_cpu(binning_info->rot_mask);
2765
2766 rc = hdev->asic_funcs->set_dram_properties(hdev);
2767 if (rc)
2768 return rc;
2769
2770 rc = hdev->asic_funcs->set_binning_masks(hdev);
2771 if (rc)
2772 return rc;
2773
2774 dev_dbg(hdev->dev,
2775 "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x, rot:0x%x\n",
2776 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,
2777 hdev->decoder_binning, hdev->rotator_binning);
2778 }
2779
2780 return 0;
2781 }
2782
2783 /* load boot fit to FW */
2784 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
2785 fw_loader->boot_fit_timeout);
2786 if (rc) {
2787 dev_err(hdev->dev, "failed to load boot fit\n");
2788 goto protocol_err;
2789 }
2790
2791 rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
2792 if (rc)
2793 goto protocol_err;
2794
2795 hl_fw_boot_fit_update_state(hdev,
2796 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2797 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2798
2799 /*
2800 * when testing FW load (without Linux) on PLDM we don't want to
2801 * wait until boot fit is active as it may take several hours.
2802 * instead, we load the bootfit and let it do all initialization in
2803 * the background.
2804 */
2805 if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))
2806 return 0;
2807
2808 /* Enable DRAM scrambling before Linux boot and after successful
2809 * UBoot
2810 */
2811 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2812
2813 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2814 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2815 return 0;
2816 }
2817
2818 if (fw_loader->skip_bmc) {
2819 rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
2820 COMMS_SKIP_BMC, 0,
2821 true,
2822 fw_loader->cpu_timeout);
2823 if (rc) {
2824 dev_err(hdev->dev, "failed to load boot fit\n");
2825 goto protocol_err;
2826 }
2827 }
2828
2829 /* load Linux image to FW */
2830 rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
2831 fw_loader->cpu_timeout);
2832 if (rc) {
2833 dev_err(hdev->dev, "failed to load Linux\n");
2834 goto protocol_err;
2835 }
2836
2837 rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
2838 if (rc)
2839 goto protocol_err;
2840
2841 hl_fw_linux_update_state(hdev,
2842 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2843 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2844
2845 hl_fw_dynamic_update_linux_interrupt_if(hdev);
2846
2847 protocol_err:
2848 if (fw_loader->dynamic_loader.fw_desc_valid) {
2849 fw_error_rc = fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
2850 le32_to_cpu(dyn_regs->cpu_boot_err1),
2851 le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
2852 le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
2853
2854 if (fw_error_rc)
2855 return fw_error_rc;
2856 }
2857
2858 return rc;
2859 }
2860
2861 /**
2862 * hl_fw_static_init_cpu - initialize the device CPU using static protocol
2863 *
2864 * @hdev: pointer to the habanalabs device structure
2865 * @fw_loader: managing structure for loading device's FW
2866 *
2867 * @return 0 on success, otherwise non-zero error code
2868 */
hl_fw_static_init_cpu(struct hl_device * hdev,struct fw_load_mgr * fw_loader)2869 static int hl_fw_static_init_cpu(struct hl_device *hdev,
2870 struct fw_load_mgr *fw_loader)
2871 {
2872 u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
2873 u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
2874 struct static_fw_load_mgr *static_loader;
2875 u32 cpu_boot_status_reg;
2876 int rc;
2877
2878 if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
2879 return 0;
2880
2881 /* init common loader parameters */
2882 cpu_timeout = fw_loader->cpu_timeout;
2883
2884 /* init static loader parameters */
2885 static_loader = &fw_loader->static_loader;
2886 cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
2887 msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
2888 cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
2889 cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
2890 cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
2891
2892 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
2893 cpu_timeout / USEC_PER_SEC);
2894
2895 /* Wait for boot FIT request */
2896 rc = hl_poll_timeout(
2897 hdev,
2898 cpu_boot_status_reg,
2899 status,
2900 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
2901 hdev->fw_poll_interval_usec,
2902 fw_loader->boot_fit_timeout);
2903
2904 if (rc) {
2905 dev_dbg(hdev->dev,
2906 "No boot fit request received (status = %d), resuming boot\n", status);
2907 } else {
2908 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
2909 if (rc)
2910 goto out;
2911
2912 /* Clear device CPU message status */
2913 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
2914
2915 /* Signal device CPU that boot loader is ready */
2916 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
2917
2918 /* Poll for CPU device ack */
2919 rc = hl_poll_timeout(
2920 hdev,
2921 cpu_msg_status_reg,
2922 status,
2923 status == CPU_MSG_OK,
2924 hdev->fw_poll_interval_usec,
2925 fw_loader->boot_fit_timeout);
2926
2927 if (rc) {
2928 dev_err(hdev->dev,
2929 "Timeout waiting for boot fit load ack (status = %d)\n", status);
2930 goto out;
2931 }
2932
2933 /* Clear message */
2934 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
2935 }
2936
2937 /*
2938 * Make sure CPU boot-loader is running
2939 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux
2940 * yet there is a debug scenario in which we loading uboot (without Linux)
2941 * which at later stage is relocated to DRAM. In this case we expect
2942 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the
2943 * poll flags
2944 */
2945 rc = hl_poll_timeout(
2946 hdev,
2947 cpu_boot_status_reg,
2948 status,
2949 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2950 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
2951 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
2952 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2953 hdev->fw_poll_interval_usec,
2954 cpu_timeout);
2955
2956 dev_dbg(hdev->dev, "uboot status = %d\n", status);
2957
2958 /* Read U-Boot version now in case we will later fail */
2959 hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
2960
2961 /* update state according to boot stage */
2962 hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
2963 cpu_boot_dev_status1_reg);
2964
2965 if (rc) {
2966 detect_cpu_boot_status(hdev, status);
2967 rc = -EIO;
2968 goto out;
2969 }
2970
2971 /* Enable DRAM scrambling before Linux boot and after successful
2972 * UBoot
2973 */
2974 hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
2975
2976 if (!(hdev->fw_components & FW_TYPE_LINUX)) {
2977 dev_info(hdev->dev, "Skip loading Linux F/W\n");
2978 rc = 0;
2979 goto out;
2980 }
2981
2982 if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
2983 rc = 0;
2984 goto out;
2985 }
2986
2987 dev_info(hdev->dev,
2988 "Loading firmware to device, may take some time...\n");
2989
2990 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
2991 if (rc)
2992 goto out;
2993
2994 if (fw_loader->skip_bmc) {
2995 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
2996
2997 rc = hl_poll_timeout(
2998 hdev,
2999 cpu_boot_status_reg,
3000 status,
3001 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
3002 hdev->fw_poll_interval_usec,
3003 cpu_timeout);
3004
3005 if (rc) {
3006 dev_err(hdev->dev,
3007 "Failed to get ACK on skipping BMC (status = %d)\n",
3008 status);
3009 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
3010 rc = -EIO;
3011 goto out;
3012 }
3013 }
3014
3015 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
3016
3017 rc = hl_poll_timeout(
3018 hdev,
3019 cpu_boot_status_reg,
3020 status,
3021 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
3022 hdev->fw_poll_interval_usec,
3023 cpu_timeout);
3024
3025 /* Clear message */
3026 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
3027
3028 if (rc) {
3029 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
3030 dev_err(hdev->dev,
3031 "Device reports FIT image is corrupted\n");
3032 else
3033 dev_err(hdev->dev,
3034 "Failed to load firmware to device (status = %d)\n",
3035 status);
3036
3037 rc = -EIO;
3038 goto out;
3039 }
3040
3041 rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3042 fw_loader->static_loader.boot_err1_reg,
3043 cpu_boot_dev_status0_reg,
3044 cpu_boot_dev_status1_reg);
3045 if (rc)
3046 return rc;
3047
3048 hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
3049 cpu_boot_dev_status1_reg);
3050
3051 return 0;
3052
3053 out:
3054 fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
3055 fw_loader->static_loader.boot_err1_reg,
3056 cpu_boot_dev_status0_reg,
3057 cpu_boot_dev_status1_reg);
3058
3059 return rc;
3060 }
3061
3062 /**
3063 * hl_fw_init_cpu - initialize the device CPU
3064 *
3065 * @hdev: pointer to the habanalabs device structure
3066 *
3067 * @return 0 on success, otherwise non-zero error code
3068 *
3069 * perform necessary initializations for device's CPU. takes into account if
3070 * init protocol is static or dynamic.
3071 */
hl_fw_init_cpu(struct hl_device * hdev)3072 int hl_fw_init_cpu(struct hl_device *hdev)
3073 {
3074 struct asic_fixed_properties *prop = &hdev->asic_prop;
3075 struct fw_load_mgr *fw_loader = &hdev->fw_loader;
3076
3077 return prop->dynamic_fw_load ?
3078 hl_fw_dynamic_init_cpu(hdev, fw_loader) :
3079 hl_fw_static_init_cpu(hdev, fw_loader);
3080 }
3081
hl_fw_set_pll_profile(struct hl_device * hdev)3082 void hl_fw_set_pll_profile(struct hl_device *hdev)
3083 {
3084 hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,
3085 hdev->asic_prop.max_freq_value);
3086 }
3087
hl_fw_get_clk_rate(struct hl_device * hdev,u32 * cur_clk,u32 * max_clk)3088 int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
3089 {
3090 long value;
3091
3092 if (!hl_device_operational(hdev, NULL))
3093 return -ENODEV;
3094
3095 if (!hdev->pdev) {
3096 *cur_clk = 0;
3097 *max_clk = 0;
3098 return 0;
3099 }
3100
3101 value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);
3102
3103 if (value < 0) {
3104 dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);
3105 return value;
3106 }
3107
3108 *max_clk = (value / 1000 / 1000);
3109
3110 value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);
3111
3112 if (value < 0) {
3113 dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);
3114 return value;
3115 }
3116
3117 *cur_clk = (value / 1000 / 1000);
3118
3119 return 0;
3120 }
3121
hl_fw_get_frequency(struct hl_device * hdev,u32 pll_index,bool curr)3122 long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)
3123 {
3124 struct cpucp_packet pkt;
3125 u32 used_pll_idx;
3126 u64 result;
3127 int rc;
3128
3129 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3130 if (rc)
3131 return rc;
3132
3133 memset(&pkt, 0, sizeof(pkt));
3134
3135 if (curr)
3136 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<
3137 CPUCP_PKT_CTL_OPCODE_SHIFT);
3138 else
3139 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3140
3141 pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3142
3143 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3144
3145 if (rc) {
3146 dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",
3147 used_pll_idx, rc);
3148 return rc;
3149 }
3150
3151 return (long) result;
3152 }
3153
hl_fw_set_frequency(struct hl_device * hdev,u32 pll_index,u64 freq)3154 void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)
3155 {
3156 struct cpucp_packet pkt;
3157 u32 used_pll_idx;
3158 int rc;
3159
3160 rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);
3161 if (rc)
3162 return;
3163
3164 memset(&pkt, 0, sizeof(pkt));
3165
3166 pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3167 pkt.pll_index = cpu_to_le32((u32)used_pll_idx);
3168 pkt.value = cpu_to_le64(freq);
3169
3170 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3171
3172 if (rc)
3173 dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",
3174 used_pll_idx, rc);
3175 }
3176
hl_fw_get_max_power(struct hl_device * hdev)3177 long hl_fw_get_max_power(struct hl_device *hdev)
3178 {
3179 struct cpucp_packet pkt;
3180 u64 result;
3181 int rc;
3182
3183 memset(&pkt, 0, sizeof(pkt));
3184
3185 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3186
3187 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);
3188
3189 if (rc) {
3190 dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);
3191 return rc;
3192 }
3193
3194 return result;
3195 }
3196
hl_fw_set_max_power(struct hl_device * hdev)3197 void hl_fw_set_max_power(struct hl_device *hdev)
3198 {
3199 struct cpucp_packet pkt;
3200 int rc;
3201
3202 /* TODO: remove this after simulator supports this packet */
3203 if (!hdev->pdev)
3204 return;
3205
3206 memset(&pkt, 0, sizeof(pkt));
3207
3208 pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);
3209 pkt.value = cpu_to_le64(hdev->max_power);
3210
3211 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);
3212
3213 if (rc)
3214 dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);
3215 }
3216
hl_fw_get_sec_attest_data(struct hl_device * hdev,u32 packet_id,void * data,u32 size,u32 nonce,u32 timeout)3217 static int hl_fw_get_sec_attest_data(struct hl_device *hdev, u32 packet_id, void *data, u32 size,
3218 u32 nonce, u32 timeout)
3219 {
3220 struct cpucp_packet pkt = {};
3221 dma_addr_t req_dma_addr;
3222 void *req_cpu_addr;
3223 int rc;
3224
3225 req_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, size, &req_dma_addr);
3226 if (!req_cpu_addr) {
3227 dev_err(hdev->dev,
3228 "Failed to allocate DMA memory for CPU-CP packet %u\n", packet_id);
3229 return -ENOMEM;
3230 }
3231
3232 memset(data, 0, size);
3233
3234 pkt.ctl = cpu_to_le32(packet_id << CPUCP_PKT_CTL_OPCODE_SHIFT);
3235 pkt.addr = cpu_to_le64(req_dma_addr);
3236 pkt.data_max_size = cpu_to_le32(size);
3237 pkt.nonce = cpu_to_le32(nonce);
3238
3239 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
3240 timeout, NULL);
3241 if (rc) {
3242 dev_err(hdev->dev,
3243 "Failed to handle CPU-CP pkt %u, error %d\n", packet_id, rc);
3244 goto out;
3245 }
3246
3247 memcpy(data, req_cpu_addr, size);
3248
3249 out:
3250 hl_cpu_accessible_dma_pool_free(hdev, size, req_cpu_addr);
3251
3252 return rc;
3253 }
3254
hl_fw_get_sec_attest_info(struct hl_device * hdev,struct cpucp_sec_attest_info * sec_attest_info,u32 nonce)3255 int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info,
3256 u32 nonce)
3257 {
3258 return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_SEC_ATTEST_GET, sec_attest_info,
3259 sizeof(struct cpucp_sec_attest_info), nonce,
3260 HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC);
3261 }
3262
hl_fw_send_generic_request(struct hl_device * hdev,enum hl_passthrough_type sub_opcode,dma_addr_t buff,u32 * size)3263 int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode,
3264 dma_addr_t buff, u32 *size)
3265 {
3266 struct cpucp_packet pkt = {};
3267 u64 result;
3268 int rc = 0;
3269
3270 pkt.ctl = cpu_to_le32(CPUCP_PACKET_GENERIC_PASSTHROUGH << CPUCP_PKT_CTL_OPCODE_SHIFT);
3271 pkt.addr = cpu_to_le64(buff);
3272 pkt.data_max_size = cpu_to_le32(*size);
3273 pkt.pkt_subidx = cpu_to_le32(sub_opcode);
3274
3275 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)&pkt, sizeof(pkt),
3276 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
3277 if (rc)
3278 dev_err(hdev->dev, "failed to send CPUCP data of generic fw pkt\n");
3279 else
3280 dev_dbg(hdev->dev, "generic pkt was successful, result: 0x%llx\n", result);
3281
3282 *size = (u32)result;
3283
3284 return rc;
3285 }
3286