Lines Matching refs:dev
188 static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili, in wait_fw_init() argument
197 fw_initializing = ioread32be(&dev->iseg->initializing); in wait_fw_init()
201 test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) { in wait_fw_init()
206 mlx5_core_warn(dev, "Waiting for FW initialization, timeout abort in %ds (0x%x)\n", in wait_fw_init()
210 msleep(mlx5_tout_ms(dev, FW_PRE_INIT_WAIT)); in wait_fw_init()
216 static void mlx5_set_driver_version(struct mlx5_core_dev *dev) in mlx5_set_driver_version() argument
224 if (!MLX5_CAP_GEN(dev, driver_version)) in mlx5_set_driver_version()
250 mlx5_cmd_exec_in(dev, set_driver_version, in); in mlx5_set_driver_version()
257 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); in set_dma_caps()
259 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); in set_dma_caps()
260 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); in set_dma_caps()
262 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); in set_dma_caps()
267 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); in set_dma_caps()
271 static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) in mlx5_pci_enable_device() argument
273 struct pci_dev *pdev = dev->pdev; in mlx5_pci_enable_device()
276 mutex_lock(&dev->pci_status_mutex); in mlx5_pci_enable_device()
277 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { in mlx5_pci_enable_device()
280 dev->pci_status = MLX5_PCI_STATUS_ENABLED; in mlx5_pci_enable_device()
282 mutex_unlock(&dev->pci_status_mutex); in mlx5_pci_enable_device()
287 static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) in mlx5_pci_disable_device() argument
289 struct pci_dev *pdev = dev->pdev; in mlx5_pci_disable_device()
291 mutex_lock(&dev->pci_status_mutex); in mlx5_pci_disable_device()
292 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { in mlx5_pci_disable_device()
294 dev->pci_status = MLX5_PCI_STATUS_DISABLED; in mlx5_pci_disable_device()
296 mutex_unlock(&dev->pci_status_mutex); in mlx5_pci_disable_device()
304 dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); in request_bar()
310 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); in request_bar()
325 static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) in to_fw_pkey_sz() argument
341 mlx5_core_warn(dev, "invalid pkey table size %d\n", size); in to_fw_pkey_sz()
346 void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *dev, struct net_device *netdev) in mlx5_core_uplink_netdev_set() argument
348 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); in mlx5_core_uplink_netdev_set()
349 dev->mlx5e_res.uplink_netdev = netdev; in mlx5_core_uplink_netdev_set()
350 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV, in mlx5_core_uplink_netdev_set()
352 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock); in mlx5_core_uplink_netdev_set()
355 void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *dev) in mlx5_core_uplink_netdev_event_replay() argument
357 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); in mlx5_core_uplink_netdev_event_replay()
358 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV, in mlx5_core_uplink_netdev_event_replay()
359 dev->mlx5e_res.uplink_netdev); in mlx5_core_uplink_netdev_event_replay()
360 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock); in mlx5_core_uplink_netdev_event_replay()
364 void mlx5_core_mp_event_replay(struct mlx5_core_dev *dev, u32 event, void *data) in mlx5_core_mp_event_replay() argument
366 mlx5_blocking_notifier_call_chain(dev, event, data); in mlx5_core_mp_event_replay()
370 int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, in mlx5_core_get_caps_mode() argument
386 err = mlx5_cmd_exec_inout(dev, query_hca_cap, in, out); in mlx5_core_get_caps_mode()
388 mlx5_core_warn(dev, in mlx5_core_get_caps_mode()
398 memcpy(dev->caps.hca[cap_type]->max, hca_caps, in mlx5_core_get_caps_mode()
402 memcpy(dev->caps.hca[cap_type]->cur, hca_caps, in mlx5_core_get_caps_mode()
406 mlx5_core_warn(dev, in mlx5_core_get_caps_mode()
417 int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) in mlx5_core_get_caps() argument
421 ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR); in mlx5_core_get_caps()
424 return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX); in mlx5_core_get_caps()
427 static int set_caps(struct mlx5_core_dev *dev, void *in, int opmod) in set_caps() argument
431 return mlx5_cmd_exec_in(dev, set_hca_cap, in); in set_caps()
434 static int handle_hca_cap_atomic(struct mlx5_core_dev *dev, void *set_ctx) in handle_hca_cap_atomic() argument
440 if (!MLX5_CAP_GEN(dev, atomic)) in handle_hca_cap_atomic()
443 err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); in handle_hca_cap_atomic()
448 MLX5_CAP_ATOMIC(dev, in handle_hca_cap_atomic()
460 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC); in handle_hca_cap_atomic()
463 static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx) in handle_hca_cap_odp() argument
470 !MLX5_CAP_GEN(dev, pg)) in handle_hca_cap_odp()
473 err = mlx5_core_get_caps(dev, MLX5_CAP_ODP); in handle_hca_cap_odp()
478 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ODP]->cur, in handle_hca_cap_odp()
481 #define ODP_CAP_SET_MAX(dev, field) \ in handle_hca_cap_odp() argument
483 u32 _res = MLX5_CAP_ODP_MAX(dev, field); \ in handle_hca_cap_odp()
490 ODP_CAP_SET_MAX(dev, ud_odp_caps.srq_receive); in handle_hca_cap_odp()
491 ODP_CAP_SET_MAX(dev, rc_odp_caps.srq_receive); in handle_hca_cap_odp()
492 ODP_CAP_SET_MAX(dev, xrc_odp_caps.srq_receive); in handle_hca_cap_odp()
493 ODP_CAP_SET_MAX(dev, xrc_odp_caps.send); in handle_hca_cap_odp()
494 ODP_CAP_SET_MAX(dev, xrc_odp_caps.receive); in handle_hca_cap_odp()
495 ODP_CAP_SET_MAX(dev, xrc_odp_caps.write); in handle_hca_cap_odp()
496 ODP_CAP_SET_MAX(dev, xrc_odp_caps.read); in handle_hca_cap_odp()
497 ODP_CAP_SET_MAX(dev, xrc_odp_caps.atomic); in handle_hca_cap_odp()
498 ODP_CAP_SET_MAX(dev, dc_odp_caps.srq_receive); in handle_hca_cap_odp()
499 ODP_CAP_SET_MAX(dev, dc_odp_caps.send); in handle_hca_cap_odp()
500 ODP_CAP_SET_MAX(dev, dc_odp_caps.receive); in handle_hca_cap_odp()
501 ODP_CAP_SET_MAX(dev, dc_odp_caps.write); in handle_hca_cap_odp()
502 ODP_CAP_SET_MAX(dev, dc_odp_caps.read); in handle_hca_cap_odp()
503 ODP_CAP_SET_MAX(dev, dc_odp_caps.atomic); in handle_hca_cap_odp()
508 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ODP); in handle_hca_cap_odp()
511 static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev) in max_uc_list_get_devlink_param() argument
513 struct devlink *devlink = priv_to_devlink(dev); in max_uc_list_get_devlink_param()
522 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err); in max_uc_list_get_devlink_param()
526 bool mlx5_is_roce_on(struct mlx5_core_dev *dev) in mlx5_is_roce_on() argument
528 struct devlink *devlink = priv_to_devlink(dev); in mlx5_is_roce_on()
539 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err); in mlx5_is_roce_on()
540 return MLX5_CAP_GEN(dev, roce); in mlx5_is_roce_on()
544 static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx) in handle_hca_cap_2() argument
549 if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2)) in handle_hca_cap_2()
552 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2); in handle_hca_cap_2()
556 if (!MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) || in handle_hca_cap_2()
557 !(dev->priv.sw_vhca_id > 0)) in handle_hca_cap_2()
562 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur, in handle_hca_cap_2()
566 return set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2); in handle_hca_cap_2()
569 static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) in handle_hca_cap() argument
571 struct mlx5_profile *prof = &dev->profile; in handle_hca_cap()
576 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); in handle_hca_cap()
582 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL]->cur, in handle_hca_cap()
585 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", in handle_hca_cap()
586 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), in handle_hca_cap()
590 to_fw_pkey_sz(dev, 128)); in handle_hca_cap()
594 prof->log_max_qp = min_t(u8, 18, MLX5_CAP_GEN_MAX(dev, log_max_qp)); in handle_hca_cap()
595 } else if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) { in handle_hca_cap()
596 …mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limi… in handle_hca_cap()
598 MLX5_CAP_GEN_MAX(dev, log_max_qp)); in handle_hca_cap()
599 prof->log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); in handle_hca_cap()
611 if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096) in handle_hca_cap()
616 if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte)) in handle_hca_cap()
622 if (MLX5_CAP_GEN_MAX(dev, dct)) in handle_hca_cap()
625 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_event)) in handle_hca_cap()
627 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_with_driver_unload)) in handle_hca_cap()
631 if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports)) in handle_hca_cap()
635 MLX5_CAP_GEN_MAX(dev, num_vhca_ports)); in handle_hca_cap()
637 if (MLX5_CAP_GEN_MAX(dev, release_all_pages)) in handle_hca_cap()
640 if (MLX5_CAP_GEN_MAX(dev, mkey_by_name)) in handle_hca_cap()
643 mlx5_vhca_state_cap_handle(dev, set_hca_cap); in handle_hca_cap()
645 if (MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix)) in handle_hca_cap()
647 MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix)); in handle_hca_cap()
649 if (MLX5_CAP_GEN(dev, roce_rw_supported) && MLX5_CAP_GEN_MAX(dev, roce)) in handle_hca_cap()
651 mlx5_is_roce_on(dev)); in handle_hca_cap()
653 max_uc_list = max_uc_list_get_devlink_param(dev); in handle_hca_cap()
658 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); in handle_hca_cap()
675 static bool is_roce_fw_disabled(struct mlx5_core_dev *dev) in is_roce_fw_disabled() argument
677 return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_on(dev)) || in is_roce_fw_disabled()
678 (!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce)); in is_roce_fw_disabled()
681 static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx) in handle_hca_cap_roce() argument
686 if (is_roce_fw_disabled(dev)) in handle_hca_cap_roce()
689 err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE); in handle_hca_cap_roce()
693 if (MLX5_CAP_ROCE(dev, sw_r_roce_src_udp_port) || in handle_hca_cap_roce()
694 !MLX5_CAP_ROCE_MAX(dev, sw_r_roce_src_udp_port)) in handle_hca_cap_roce()
698 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ROCE]->cur, in handle_hca_cap_roce()
702 if (MLX5_CAP_ROCE_MAX(dev, qp_ooo_transmit_default)) in handle_hca_cap_roce()
705 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ROCE); in handle_hca_cap_roce()
709 static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev, in handle_hca_cap_port_selection() argument
715 if (!MLX5_CAP_GEN(dev, port_selection_cap)) in handle_hca_cap_port_selection()
718 err = mlx5_core_get_caps(dev, MLX5_CAP_PORT_SELECTION); in handle_hca_cap_port_selection()
722 if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) || in handle_hca_cap_port_selection()
723 !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass)) in handle_hca_cap_port_selection()
727 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur, in handle_hca_cap_port_selection()
731 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_PORT_SELECTION); in handle_hca_cap_port_selection()
736 static int set_hca_cap(struct mlx5_core_dev *dev) in set_hca_cap() argument
746 err = handle_hca_cap(dev, set_ctx); in set_hca_cap()
748 mlx5_core_err(dev, "handle_hca_cap failed\n"); in set_hca_cap()
753 err = handle_hca_cap_atomic(dev, set_ctx); in set_hca_cap()
755 mlx5_core_err(dev, "handle_hca_cap_atomic failed\n"); in set_hca_cap()
760 err = handle_hca_cap_odp(dev, set_ctx); in set_hca_cap()
762 mlx5_core_err(dev, "handle_hca_cap_odp failed\n"); in set_hca_cap()
767 err = handle_hca_cap_roce(dev, set_ctx); in set_hca_cap()
769 mlx5_core_err(dev, "handle_hca_cap_roce failed\n"); in set_hca_cap()
774 err = handle_hca_cap_2(dev, set_ctx); in set_hca_cap()
776 mlx5_core_err(dev, "handle_hca_cap_2 failed\n"); in set_hca_cap()
781 err = handle_hca_cap_port_selection(dev, set_ctx); in set_hca_cap()
783 mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n"); in set_hca_cap()
792 static int set_hca_ctrl(struct mlx5_core_dev *dev) in set_hca_ctrl() argument
798 if (!mlx5_core_is_pf(dev)) in set_hca_ctrl()
803 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), in set_hca_ctrl()
809 static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev) in mlx5_core_set_hca_defaults() argument
814 if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) in mlx5_core_set_hca_defaults()
815 ret = mlx5_nic_vport_update_local_lb(dev, false); in mlx5_core_set_hca_defaults()
820 int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id) in mlx5_core_enable_hca() argument
827 dev->caps.embedded_cpu); in mlx5_core_enable_hca()
828 return mlx5_cmd_exec_in(dev, enable_hca, in); in mlx5_core_enable_hca()
831 int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) in mlx5_core_disable_hca() argument
838 dev->caps.embedded_cpu); in mlx5_core_disable_hca()
839 return mlx5_cmd_exec_in(dev, disable_hca, in); in mlx5_core_disable_hca()
842 static int mlx5_core_set_issi(struct mlx5_core_dev *dev) in mlx5_core_set_issi() argument
850 err = mlx5_cmd_exec_inout(dev, query_issi, query_in, query_out); in mlx5_core_set_issi()
856 mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n", in mlx5_core_set_issi()
861 mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n"); in mlx5_core_set_issi()
862 dev->issi = 0; in mlx5_core_set_issi()
873 err = mlx5_cmd_exec_in(dev, set_issi, set_in); in mlx5_core_set_issi()
875 mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n", in mlx5_core_set_issi()
880 dev->issi = 1; in mlx5_core_set_issi()
890 static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev, in mlx5_pci_init() argument
895 mutex_init(&dev->pci_status_mutex); in mlx5_pci_init()
896 pci_set_drvdata(dev->pdev, dev); in mlx5_pci_init()
898 dev->bar_addr = pci_resource_start(pdev, 0); in mlx5_pci_init()
900 err = mlx5_pci_enable_device(dev); in mlx5_pci_init()
902 mlx5_core_err(dev, "Cannot enable PCI device, aborting\n"); in mlx5_pci_init()
908 mlx5_core_err(dev, "error requesting BARs, aborting\n"); in mlx5_pci_init()
916 mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n"); in mlx5_pci_init()
923 mlx5_core_dbg(dev, "Enabling pci atomics failed\n"); in mlx5_pci_init()
925 dev->iseg_base = dev->bar_addr; in mlx5_pci_init()
926 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); in mlx5_pci_init()
927 if (!dev->iseg) { in mlx5_pci_init()
929 mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n"); in mlx5_pci_init()
933 mlx5_pci_vsc_init(dev); in mlx5_pci_init()
937 release_bar(dev->pdev); in mlx5_pci_init()
939 mlx5_pci_disable_device(dev); in mlx5_pci_init()
943 static void mlx5_pci_close(struct mlx5_core_dev *dev) in mlx5_pci_close() argument
949 mlx5_drain_health_wq(dev); in mlx5_pci_close()
950 iounmap(dev->iseg); in mlx5_pci_close()
951 release_bar(dev->pdev); in mlx5_pci_close()
952 mlx5_pci_disable_device(dev); in mlx5_pci_close()
955 static int mlx5_init_once(struct mlx5_core_dev *dev) in mlx5_init_once() argument
959 dev->priv.devc = mlx5_devcom_register_device(dev); in mlx5_init_once()
960 if (IS_ERR(dev->priv.devc)) in mlx5_init_once()
961 mlx5_core_warn(dev, "failed to register devcom device %ld\n", in mlx5_init_once()
962 PTR_ERR(dev->priv.devc)); in mlx5_init_once()
964 err = mlx5_query_board_id(dev); in mlx5_init_once()
966 mlx5_core_err(dev, "query board id failed\n"); in mlx5_init_once()
970 err = mlx5_irq_table_init(dev); in mlx5_init_once()
972 mlx5_core_err(dev, "failed to initialize irq table\n"); in mlx5_init_once()
976 err = mlx5_eq_table_init(dev); in mlx5_init_once()
978 mlx5_core_err(dev, "failed to initialize eq\n"); in mlx5_init_once()
982 err = mlx5_events_init(dev); in mlx5_init_once()
984 mlx5_core_err(dev, "failed to initialize events\n"); in mlx5_init_once()
988 err = mlx5_fw_reset_init(dev); in mlx5_init_once()
990 mlx5_core_err(dev, "failed to initialize fw reset events\n"); in mlx5_init_once()
994 mlx5_cq_debugfs_init(dev); in mlx5_init_once()
996 mlx5_init_reserved_gids(dev); in mlx5_init_once()
998 mlx5_init_clock(dev); in mlx5_init_once()
1000 dev->vxlan = mlx5_vxlan_create(dev); in mlx5_init_once()
1001 dev->geneve = mlx5_geneve_create(dev); in mlx5_init_once()
1003 err = mlx5_init_rl_table(dev); in mlx5_init_once()
1005 mlx5_core_err(dev, "Failed to init rate limiting\n"); in mlx5_init_once()
1009 err = mlx5_mpfs_init(dev); in mlx5_init_once()
1011 mlx5_core_err(dev, "Failed to init l2 table %d\n", err); in mlx5_init_once()
1015 err = mlx5_sriov_init(dev); in mlx5_init_once()
1017 mlx5_core_err(dev, "Failed to init sriov %d\n", err); in mlx5_init_once()
1021 err = mlx5_eswitch_init(dev); in mlx5_init_once()
1023 mlx5_core_err(dev, "Failed to init eswitch %d\n", err); in mlx5_init_once()
1027 err = mlx5_fpga_init(dev); in mlx5_init_once()
1029 mlx5_core_err(dev, "Failed to init fpga device %d\n", err); in mlx5_init_once()
1033 err = mlx5_vhca_event_init(dev); in mlx5_init_once()
1035 mlx5_core_err(dev, "Failed to init vhca event notifier %d\n", err); in mlx5_init_once()
1039 err = mlx5_sf_hw_table_init(dev); in mlx5_init_once()
1041 mlx5_core_err(dev, "Failed to init SF HW table %d\n", err); in mlx5_init_once()
1045 err = mlx5_sf_table_init(dev); in mlx5_init_once()
1047 mlx5_core_err(dev, "Failed to init SF table %d\n", err); in mlx5_init_once()
1051 err = mlx5_fs_core_alloc(dev); in mlx5_init_once()
1053 mlx5_core_err(dev, "Failed to alloc flow steering\n"); in mlx5_init_once()
1057 dev->dm = mlx5_dm_create(dev); in mlx5_init_once()
1058 if (IS_ERR(dev->dm)) in mlx5_init_once()
1059 mlx5_core_warn(dev, "Failed to init device memory %ld\n", PTR_ERR(dev->dm)); in mlx5_init_once()
1061 dev->tracer = mlx5_fw_tracer_create(dev); in mlx5_init_once()
1062 dev->hv_vhca = mlx5_hv_vhca_create(dev); in mlx5_init_once()
1063 dev->rsc_dump = mlx5_rsc_dump_create(dev); in mlx5_init_once()
1068 mlx5_sf_table_cleanup(dev); in mlx5_init_once()
1070 mlx5_sf_hw_table_cleanup(dev); in mlx5_init_once()
1072 mlx5_vhca_event_cleanup(dev); in mlx5_init_once()
1074 mlx5_fpga_cleanup(dev); in mlx5_init_once()
1076 mlx5_eswitch_cleanup(dev->priv.eswitch); in mlx5_init_once()
1078 mlx5_sriov_cleanup(dev); in mlx5_init_once()
1080 mlx5_mpfs_cleanup(dev); in mlx5_init_once()
1082 mlx5_cleanup_rl_table(dev); in mlx5_init_once()
1084 mlx5_geneve_destroy(dev->geneve); in mlx5_init_once()
1085 mlx5_vxlan_destroy(dev->vxlan); in mlx5_init_once()
1086 mlx5_cleanup_clock(dev); in mlx5_init_once()
1087 mlx5_cleanup_reserved_gids(dev); in mlx5_init_once()
1088 mlx5_cq_debugfs_cleanup(dev); in mlx5_init_once()
1089 mlx5_fw_reset_cleanup(dev); in mlx5_init_once()
1091 mlx5_events_cleanup(dev); in mlx5_init_once()
1093 mlx5_eq_table_cleanup(dev); in mlx5_init_once()
1095 mlx5_irq_table_cleanup(dev); in mlx5_init_once()
1097 mlx5_devcom_unregister_device(dev->priv.devc); in mlx5_init_once()
1102 static void mlx5_cleanup_once(struct mlx5_core_dev *dev) in mlx5_cleanup_once() argument
1104 mlx5_rsc_dump_destroy(dev); in mlx5_cleanup_once()
1105 mlx5_hv_vhca_destroy(dev->hv_vhca); in mlx5_cleanup_once()
1106 mlx5_fw_tracer_destroy(dev->tracer); in mlx5_cleanup_once()
1107 mlx5_dm_cleanup(dev); in mlx5_cleanup_once()
1108 mlx5_fs_core_free(dev); in mlx5_cleanup_once()
1109 mlx5_sf_table_cleanup(dev); in mlx5_cleanup_once()
1110 mlx5_sf_hw_table_cleanup(dev); in mlx5_cleanup_once()
1111 mlx5_vhca_event_cleanup(dev); in mlx5_cleanup_once()
1112 mlx5_fpga_cleanup(dev); in mlx5_cleanup_once()
1113 mlx5_eswitch_cleanup(dev->priv.eswitch); in mlx5_cleanup_once()
1114 mlx5_sriov_cleanup(dev); in mlx5_cleanup_once()
1115 mlx5_mpfs_cleanup(dev); in mlx5_cleanup_once()
1116 mlx5_cleanup_rl_table(dev); in mlx5_cleanup_once()
1117 mlx5_geneve_destroy(dev->geneve); in mlx5_cleanup_once()
1118 mlx5_vxlan_destroy(dev->vxlan); in mlx5_cleanup_once()
1119 mlx5_cleanup_clock(dev); in mlx5_cleanup_once()
1120 mlx5_cleanup_reserved_gids(dev); in mlx5_cleanup_once()
1121 mlx5_cq_debugfs_cleanup(dev); in mlx5_cleanup_once()
1122 mlx5_fw_reset_cleanup(dev); in mlx5_cleanup_once()
1123 mlx5_events_cleanup(dev); in mlx5_cleanup_once()
1124 mlx5_eq_table_cleanup(dev); in mlx5_cleanup_once()
1125 mlx5_irq_table_cleanup(dev); in mlx5_cleanup_once()
1126 mlx5_devcom_unregister_device(dev->priv.devc); in mlx5_cleanup_once()
1129 static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeout) in mlx5_function_enable() argument
1133 mlx5_core_info(dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), in mlx5_function_enable()
1134 fw_rev_min(dev), fw_rev_sub(dev)); in mlx5_function_enable()
1137 if (mlx5_core_is_pf(dev)) in mlx5_function_enable()
1138 pcie_print_link_status(dev->pdev); in mlx5_function_enable()
1142 err = wait_fw_init(dev, timeout, in mlx5_function_enable()
1143 mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL)); in mlx5_function_enable()
1145 mlx5_core_err(dev, "Firmware over %llu MS in pre-initializing state, aborting\n", in mlx5_function_enable()
1150 err = mlx5_cmd_enable(dev); in mlx5_function_enable()
1152 mlx5_core_err(dev, "Failed initializing command interface, aborting\n"); in mlx5_function_enable()
1156 mlx5_tout_query_iseg(dev); in mlx5_function_enable()
1158 err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), 0); in mlx5_function_enable()
1160 mlx5_core_err(dev, "Firmware over %llu MS in initializing state, aborting\n", in mlx5_function_enable()
1161 mlx5_tout_ms(dev, FW_INIT)); in mlx5_function_enable()
1165 dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev); in mlx5_function_enable()
1166 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP); in mlx5_function_enable()
1168 mlx5_start_health_poll(dev); in mlx5_function_enable()
1170 err = mlx5_core_enable_hca(dev, 0); in mlx5_function_enable()
1172 mlx5_core_err(dev, "enable hca failed\n"); in mlx5_function_enable()
1176 err = mlx5_core_set_issi(dev); in mlx5_function_enable()
1178 mlx5_core_err(dev, "failed to set issi\n"); in mlx5_function_enable()
1182 err = mlx5_satisfy_startup_pages(dev, 1); in mlx5_function_enable()
1184 mlx5_core_err(dev, "failed to allocate boot pages\n"); in mlx5_function_enable()
1188 err = mlx5_tout_query_dtor(dev); in mlx5_function_enable()
1190 mlx5_core_err(dev, "failed to read dtor\n"); in mlx5_function_enable()
1197 mlx5_reclaim_startup_pages(dev); in mlx5_function_enable()
1199 mlx5_core_disable_hca(dev, 0); in mlx5_function_enable()
1201 mlx5_stop_health_poll(dev, boot); in mlx5_function_enable()
1203 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); in mlx5_function_enable()
1204 mlx5_cmd_disable(dev); in mlx5_function_enable()
1209 static void mlx5_function_disable(struct mlx5_core_dev *dev, bool boot) in mlx5_function_disable() argument
1211 mlx5_reclaim_startup_pages(dev); in mlx5_function_disable()
1212 mlx5_core_disable_hca(dev, 0); in mlx5_function_disable()
1213 mlx5_stop_health_poll(dev, boot); in mlx5_function_disable()
1214 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); in mlx5_function_disable()
1215 mlx5_cmd_disable(dev); in mlx5_function_disable()
1218 static int mlx5_function_open(struct mlx5_core_dev *dev) in mlx5_function_open() argument
1222 err = set_hca_ctrl(dev); in mlx5_function_open()
1224 mlx5_core_err(dev, "set_hca_ctrl failed\n"); in mlx5_function_open()
1228 err = set_hca_cap(dev); in mlx5_function_open()
1230 mlx5_core_err(dev, "set_hca_cap failed\n"); in mlx5_function_open()
1234 err = mlx5_satisfy_startup_pages(dev, 0); in mlx5_function_open()
1236 mlx5_core_err(dev, "failed to allocate init pages\n"); in mlx5_function_open()
1240 err = mlx5_cmd_init_hca(dev, sw_owner_id); in mlx5_function_open()
1242 mlx5_core_err(dev, "init hca failed\n"); in mlx5_function_open()
1246 mlx5_set_driver_version(dev); in mlx5_function_open()
1248 err = mlx5_query_hca_caps(dev); in mlx5_function_open()
1250 mlx5_core_err(dev, "query hca failed\n"); in mlx5_function_open()
1253 mlx5_start_health_fw_log_up(dev); in mlx5_function_open()
1257 static int mlx5_function_close(struct mlx5_core_dev *dev) in mlx5_function_close() argument
1261 err = mlx5_cmd_teardown_hca(dev); in mlx5_function_close()
1263 mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n"); in mlx5_function_close()
1270 static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout) in mlx5_function_setup() argument
1274 err = mlx5_function_enable(dev, boot, timeout); in mlx5_function_setup()
1278 err = mlx5_function_open(dev); in mlx5_function_setup()
1280 mlx5_function_disable(dev, boot); in mlx5_function_setup()
1284 static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot) in mlx5_function_teardown() argument
1286 int err = mlx5_function_close(dev); in mlx5_function_teardown()
1289 mlx5_function_disable(dev, boot); in mlx5_function_teardown()
1293 static int mlx5_load(struct mlx5_core_dev *dev) in mlx5_load() argument
1297 dev->priv.uar = mlx5_get_uars_page(dev); in mlx5_load()
1298 if (IS_ERR(dev->priv.uar)) { in mlx5_load()
1299 mlx5_core_err(dev, "Failed allocating uar, aborting\n"); in mlx5_load()
1300 err = PTR_ERR(dev->priv.uar); in mlx5_load()
1304 mlx5_events_start(dev); in mlx5_load()
1305 mlx5_pagealloc_start(dev); in mlx5_load()
1307 err = mlx5_irq_table_create(dev); in mlx5_load()
1309 mlx5_core_err(dev, "Failed to alloc IRQs\n"); in mlx5_load()
1313 err = mlx5_eq_table_create(dev); in mlx5_load()
1315 mlx5_core_err(dev, "Failed to create EQs\n"); in mlx5_load()
1319 err = mlx5_fw_tracer_init(dev->tracer); in mlx5_load()
1321 mlx5_core_err(dev, "Failed to init FW tracer %d\n", err); in mlx5_load()
1322 mlx5_fw_tracer_destroy(dev->tracer); in mlx5_load()
1323 dev->tracer = NULL; in mlx5_load()
1326 mlx5_fw_reset_events_start(dev); in mlx5_load()
1327 mlx5_hv_vhca_init(dev->hv_vhca); in mlx5_load()
1329 err = mlx5_rsc_dump_init(dev); in mlx5_load()
1331 mlx5_core_err(dev, "Failed to init Resource dump %d\n", err); in mlx5_load()
1332 mlx5_rsc_dump_destroy(dev); in mlx5_load()
1333 dev->rsc_dump = NULL; in mlx5_load()
1336 err = mlx5_fpga_device_start(dev); in mlx5_load()
1338 mlx5_core_err(dev, "fpga device start failed %d\n", err); in mlx5_load()
1342 err = mlx5_fs_core_init(dev); in mlx5_load()
1344 mlx5_core_err(dev, "Failed to init flow steering\n"); in mlx5_load()
1348 err = mlx5_core_set_hca_defaults(dev); in mlx5_load()
1350 mlx5_core_err(dev, "Failed to set hca defaults\n"); in mlx5_load()
1354 mlx5_vhca_event_start(dev); in mlx5_load()
1356 err = mlx5_sf_hw_table_create(dev); in mlx5_load()
1358 mlx5_core_err(dev, "sf table create failed %d\n", err); in mlx5_load()
1362 err = mlx5_ec_init(dev); in mlx5_load()
1364 mlx5_core_err(dev, "Failed to init embedded CPU\n"); in mlx5_load()
1368 mlx5_lag_add_mdev(dev); in mlx5_load()
1369 err = mlx5_sriov_attach(dev); in mlx5_load()
1371 mlx5_core_err(dev, "sriov init failed %d\n", err); in mlx5_load()
1375 mlx5_sf_dev_table_create(dev); in mlx5_load()
1377 err = mlx5_devlink_traps_register(priv_to_devlink(dev)); in mlx5_load()
1384 mlx5_sf_dev_table_destroy(dev); in mlx5_load()
1385 mlx5_sriov_detach(dev); in mlx5_load()
1387 mlx5_lag_remove_mdev(dev); in mlx5_load()
1388 mlx5_ec_cleanup(dev); in mlx5_load()
1390 mlx5_sf_hw_table_destroy(dev); in mlx5_load()
1392 mlx5_vhca_event_stop(dev); in mlx5_load()
1394 mlx5_fs_core_cleanup(dev); in mlx5_load()
1396 mlx5_fpga_device_stop(dev); in mlx5_load()
1398 mlx5_rsc_dump_cleanup(dev); in mlx5_load()
1399 mlx5_hv_vhca_cleanup(dev->hv_vhca); in mlx5_load()
1400 mlx5_fw_reset_events_stop(dev); in mlx5_load()
1401 mlx5_fw_tracer_cleanup(dev->tracer); in mlx5_load()
1402 mlx5_eq_table_destroy(dev); in mlx5_load()
1404 mlx5_irq_table_destroy(dev); in mlx5_load()
1406 mlx5_pagealloc_stop(dev); in mlx5_load()
1407 mlx5_events_stop(dev); in mlx5_load()
1408 mlx5_put_uars_page(dev, dev->priv.uar); in mlx5_load()
1412 static void mlx5_unload(struct mlx5_core_dev *dev) in mlx5_unload() argument
1414 mlx5_devlink_traps_unregister(priv_to_devlink(dev)); in mlx5_unload()
1415 mlx5_sf_dev_table_destroy(dev); in mlx5_unload()
1416 mlx5_eswitch_disable(dev->priv.eswitch); in mlx5_unload()
1417 mlx5_sriov_detach(dev); in mlx5_unload()
1418 mlx5_lag_remove_mdev(dev); in mlx5_unload()
1419 mlx5_ec_cleanup(dev); in mlx5_unload()
1420 mlx5_sf_hw_table_destroy(dev); in mlx5_unload()
1421 mlx5_vhca_event_stop(dev); in mlx5_unload()
1422 mlx5_fs_core_cleanup(dev); in mlx5_unload()
1423 mlx5_fpga_device_stop(dev); in mlx5_unload()
1424 mlx5_rsc_dump_cleanup(dev); in mlx5_unload()
1425 mlx5_hv_vhca_cleanup(dev->hv_vhca); in mlx5_unload()
1426 mlx5_fw_reset_events_stop(dev); in mlx5_unload()
1427 mlx5_fw_tracer_cleanup(dev->tracer); in mlx5_unload()
1428 mlx5_eq_table_destroy(dev); in mlx5_unload()
1429 mlx5_irq_table_destroy(dev); in mlx5_unload()
1430 mlx5_pagealloc_stop(dev); in mlx5_unload()
1431 mlx5_events_stop(dev); in mlx5_unload()
1432 mlx5_put_uars_page(dev, dev->priv.uar); in mlx5_unload()
1435 int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev) in mlx5_init_one_devl_locked() argument
1437 bool light_probe = mlx5_dev_is_lightweight(dev); in mlx5_init_one_devl_locked()
1440 mutex_lock(&dev->intf_state_mutex); in mlx5_init_one_devl_locked()
1441 dev->state = MLX5_DEVICE_STATE_UP; in mlx5_init_one_devl_locked()
1443 err = mlx5_function_setup(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); in mlx5_init_one_devl_locked()
1447 err = mlx5_init_once(dev); in mlx5_init_one_devl_locked()
1449 mlx5_core_err(dev, "sw objs init failed\n"); in mlx5_init_one_devl_locked()
1457 err = mlx5_devlink_params_register(priv_to_devlink(dev)); in mlx5_init_one_devl_locked()
1462 err = mlx5_load(dev); in mlx5_init_one_devl_locked()
1466 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); in mlx5_init_one_devl_locked()
1468 err = mlx5_register_device(dev); in mlx5_init_one_devl_locked()
1472 mutex_unlock(&dev->intf_state_mutex); in mlx5_init_one_devl_locked()
1476 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); in mlx5_init_one_devl_locked()
1477 mlx5_unload(dev); in mlx5_init_one_devl_locked()
1480 mlx5_devlink_params_unregister(priv_to_devlink(dev)); in mlx5_init_one_devl_locked()
1482 mlx5_cleanup_once(dev); in mlx5_init_one_devl_locked()
1484 mlx5_function_teardown(dev, true); in mlx5_init_one_devl_locked()
1486 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; in mlx5_init_one_devl_locked()
1487 mutex_unlock(&dev->intf_state_mutex); in mlx5_init_one_devl_locked()
1491 int mlx5_init_one(struct mlx5_core_dev *dev) in mlx5_init_one() argument
1493 struct devlink *devlink = priv_to_devlink(dev); in mlx5_init_one()
1497 err = mlx5_init_one_devl_locked(dev); in mlx5_init_one()
1502 void mlx5_uninit_one(struct mlx5_core_dev *dev) in mlx5_uninit_one() argument
1504 struct devlink *devlink = priv_to_devlink(dev); in mlx5_uninit_one()
1507 mutex_lock(&dev->intf_state_mutex); in mlx5_uninit_one()
1509 mlx5_unregister_device(dev); in mlx5_uninit_one()
1511 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { in mlx5_uninit_one()
1512 mlx5_core_warn(dev, "%s: interface is down, NOP\n", in mlx5_uninit_one()
1514 mlx5_devlink_params_unregister(priv_to_devlink(dev)); in mlx5_uninit_one()
1515 mlx5_cleanup_once(dev); in mlx5_uninit_one()
1519 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); in mlx5_uninit_one()
1520 mlx5_unload(dev); in mlx5_uninit_one()
1521 mlx5_devlink_params_unregister(priv_to_devlink(dev)); in mlx5_uninit_one()
1522 mlx5_cleanup_once(dev); in mlx5_uninit_one()
1523 mlx5_function_teardown(dev, true); in mlx5_uninit_one()
1525 mutex_unlock(&dev->intf_state_mutex); in mlx5_uninit_one()
1529 int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery) in mlx5_load_one_devl_locked() argument
1534 devl_assert_locked(priv_to_devlink(dev)); in mlx5_load_one_devl_locked()
1535 mutex_lock(&dev->intf_state_mutex); in mlx5_load_one_devl_locked()
1536 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { in mlx5_load_one_devl_locked()
1537 mlx5_core_warn(dev, "interface is up, NOP\n"); in mlx5_load_one_devl_locked()
1541 dev->state = MLX5_DEVICE_STATE_UP; in mlx5_load_one_devl_locked()
1544 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT); in mlx5_load_one_devl_locked()
1546 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT); in mlx5_load_one_devl_locked()
1547 err = mlx5_function_setup(dev, false, timeout); in mlx5_load_one_devl_locked()
1551 err = mlx5_load(dev); in mlx5_load_one_devl_locked()
1555 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); in mlx5_load_one_devl_locked()
1557 err = mlx5_attach_device(dev); in mlx5_load_one_devl_locked()
1561 mutex_unlock(&dev->intf_state_mutex); in mlx5_load_one_devl_locked()
1565 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); in mlx5_load_one_devl_locked()
1566 mlx5_unload(dev); in mlx5_load_one_devl_locked()
1568 mlx5_function_teardown(dev, false); in mlx5_load_one_devl_locked()
1570 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; in mlx5_load_one_devl_locked()
1572 mutex_unlock(&dev->intf_state_mutex); in mlx5_load_one_devl_locked()
1576 int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery) in mlx5_load_one() argument
1578 struct devlink *devlink = priv_to_devlink(dev); in mlx5_load_one()
1582 ret = mlx5_load_one_devl_locked(dev, recovery); in mlx5_load_one()
1587 void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend) in mlx5_unload_one_devl_locked() argument
1589 devl_assert_locked(priv_to_devlink(dev)); in mlx5_unload_one_devl_locked()
1590 mutex_lock(&dev->intf_state_mutex); in mlx5_unload_one_devl_locked()
1592 mlx5_detach_device(dev, suspend); in mlx5_unload_one_devl_locked()
1594 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { in mlx5_unload_one_devl_locked()
1595 mlx5_core_warn(dev, "%s: interface is down, NOP\n", in mlx5_unload_one_devl_locked()
1600 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); in mlx5_unload_one_devl_locked()
1601 mlx5_unload(dev); in mlx5_unload_one_devl_locked()
1602 mlx5_function_teardown(dev, false); in mlx5_unload_one_devl_locked()
1604 mutex_unlock(&dev->intf_state_mutex); in mlx5_unload_one_devl_locked()
1607 void mlx5_unload_one(struct mlx5_core_dev *dev, bool suspend) in mlx5_unload_one() argument
1609 struct devlink *devlink = priv_to_devlink(dev); in mlx5_unload_one()
1612 mlx5_unload_one_devl_locked(dev, suspend); in mlx5_unload_one()
1619 static int mlx5_query_hca_caps_light(struct mlx5_core_dev *dev) in mlx5_query_hca_caps_light() argument
1623 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); in mlx5_query_hca_caps_light()
1627 if (MLX5_CAP_GEN(dev, eth_net_offloads)) { in mlx5_query_hca_caps_light()
1628 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_ETHERNET_OFFLOADS, in mlx5_query_hca_caps_light()
1634 if (MLX5_CAP_GEN(dev, nic_flow_table) || in mlx5_query_hca_caps_light()
1635 MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) { in mlx5_query_hca_caps_light()
1636 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_FLOW_TABLE, in mlx5_query_hca_caps_light()
1642 if (MLX5_CAP_GEN_64(dev, general_obj_types) & in mlx5_query_hca_caps_light()
1644 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_VDPA_EMULATION, in mlx5_query_hca_caps_light()
1653 int mlx5_init_one_light(struct mlx5_core_dev *dev) in mlx5_init_one_light() argument
1655 struct devlink *devlink = priv_to_devlink(dev); in mlx5_init_one_light()
1658 dev->state = MLX5_DEVICE_STATE_UP; in mlx5_init_one_light()
1659 err = mlx5_function_enable(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); in mlx5_init_one_light()
1661 mlx5_core_warn(dev, "mlx5_function_enable err=%d\n", err); in mlx5_init_one_light()
1665 err = mlx5_query_hca_caps_light(dev); in mlx5_init_one_light()
1667 mlx5_core_warn(dev, "mlx5_query_hca_caps_light err=%d\n", err); in mlx5_init_one_light()
1672 err = mlx5_devlink_params_register(priv_to_devlink(dev)); in mlx5_init_one_light()
1675 mlx5_core_warn(dev, "mlx5_devlink_param_reg err = %d\n", err); in mlx5_init_one_light()
1682 mlx5_function_disable(dev, true); in mlx5_init_one_light()
1684 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; in mlx5_init_one_light()
1688 void mlx5_uninit_one_light(struct mlx5_core_dev *dev) in mlx5_uninit_one_light() argument
1690 struct devlink *devlink = priv_to_devlink(dev); in mlx5_uninit_one_light()
1693 mlx5_devlink_params_unregister(priv_to_devlink(dev)); in mlx5_uninit_one_light()
1695 if (dev->state != MLX5_DEVICE_STATE_UP) in mlx5_uninit_one_light()
1697 mlx5_function_disable(dev, true); in mlx5_uninit_one_light()
1705 void mlx5_unload_one_light(struct mlx5_core_dev *dev) in mlx5_unload_one_light() argument
1707 if (dev->state != MLX5_DEVICE_STATE_UP) in mlx5_unload_one_light()
1709 mlx5_function_disable(dev, false); in mlx5_unload_one_light()
1710 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; in mlx5_unload_one_light()
1738 static void mlx5_hca_caps_free(struct mlx5_core_dev *dev) in mlx5_hca_caps_free() argument
1745 kfree(dev->caps.hca[type]); in mlx5_hca_caps_free()
1749 static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev) in mlx5_hca_caps_alloc() argument
1760 dev->caps.hca[type] = cap; in mlx5_hca_caps_alloc()
1766 mlx5_hca_caps_free(dev); in mlx5_hca_caps_alloc()
1772 struct mlx5_core_dev *dev = file->private; in vhca_id_show() local
1774 seq_printf(file, "0x%x\n", MLX5_CAP_GEN(dev, vhca_id)); in vhca_id_show()
1780 int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) in mlx5_mdev_init() argument
1782 struct mlx5_priv *priv = &dev->priv; in mlx5_mdev_init()
1785 memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile)); in mlx5_mdev_init()
1786 lockdep_register_key(&dev->lock_key); in mlx5_mdev_init()
1787 mutex_init(&dev->intf_state_mutex); in mlx5_mdev_init()
1788 lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key); in mlx5_mdev_init()
1789 mutex_init(&dev->mlx5e_res.uplink_netdev_lock); in mlx5_mdev_init()
1800 priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev)); in mlx5_mdev_init()
1801 priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device), in mlx5_mdev_init()
1803 debugfs_create_file("vhca_id", 0400, priv->dbg.dbg_root, dev, &vhca_id_fops); in mlx5_mdev_init()
1806 err = mlx5_cmd_init(dev); in mlx5_mdev_init()
1808 mlx5_core_err(dev, "Failed initializing cmdif SW structs, aborting\n"); in mlx5_mdev_init()
1812 err = mlx5_tout_init(dev); in mlx5_mdev_init()
1814 mlx5_core_err(dev, "Failed initializing timeouts, aborting\n"); in mlx5_mdev_init()
1818 err = mlx5_health_init(dev); in mlx5_mdev_init()
1822 err = mlx5_pagealloc_init(dev); in mlx5_mdev_init()
1826 err = mlx5_adev_init(dev); in mlx5_mdev_init()
1830 err = mlx5_hca_caps_alloc(dev); in mlx5_mdev_init()
1839 dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1, in mlx5_mdev_init()
1842 if (dev->priv.sw_vhca_id < 0) in mlx5_mdev_init()
1843 mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n", in mlx5_mdev_init()
1844 dev->priv.sw_vhca_id); in mlx5_mdev_init()
1849 mlx5_adev_cleanup(dev); in mlx5_mdev_init()
1851 mlx5_pagealloc_cleanup(dev); in mlx5_mdev_init()
1853 mlx5_health_cleanup(dev); in mlx5_mdev_init()
1855 mlx5_tout_cleanup(dev); in mlx5_mdev_init()
1857 mlx5_cmd_cleanup(dev); in mlx5_mdev_init()
1859 debugfs_remove(dev->priv.dbg.dbg_root); in mlx5_mdev_init()
1864 mutex_destroy(&dev->intf_state_mutex); in mlx5_mdev_init()
1865 lockdep_unregister_key(&dev->lock_key); in mlx5_mdev_init()
1869 void mlx5_mdev_uninit(struct mlx5_core_dev *dev) in mlx5_mdev_uninit() argument
1871 struct mlx5_priv *priv = &dev->priv; in mlx5_mdev_uninit()
1874 ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id); in mlx5_mdev_uninit()
1876 mlx5_hca_caps_free(dev); in mlx5_mdev_uninit()
1877 mlx5_adev_cleanup(dev); in mlx5_mdev_uninit()
1878 mlx5_pagealloc_cleanup(dev); in mlx5_mdev_uninit()
1879 mlx5_health_cleanup(dev); in mlx5_mdev_uninit()
1880 mlx5_tout_cleanup(dev); in mlx5_mdev_uninit()
1881 mlx5_cmd_cleanup(dev); in mlx5_mdev_uninit()
1882 debugfs_remove_recursive(dev->priv.dbg.dbg_root); in mlx5_mdev_uninit()
1887 mutex_destroy(&dev->mlx5e_res.uplink_netdev_lock); in mlx5_mdev_uninit()
1888 mutex_destroy(&dev->intf_state_mutex); in mlx5_mdev_uninit()
1889 lockdep_unregister_key(&dev->lock_key); in mlx5_mdev_uninit()
1894 struct mlx5_core_dev *dev; in probe_one() local
1898 devlink = mlx5_devlink_alloc(&pdev->dev); in probe_one()
1900 dev_err(&pdev->dev, "devlink alloc failed\n"); in probe_one()
1904 dev = devlink_priv(devlink); in probe_one()
1905 dev->device = &pdev->dev; in probe_one()
1906 dev->pdev = pdev; in probe_one()
1908 dev->coredev_type = id->driver_data & MLX5_PCI_DEV_IS_VF ? in probe_one()
1911 dev->priv.adev_idx = mlx5_adev_idx_alloc(); in probe_one()
1912 if (dev->priv.adev_idx < 0) { in probe_one()
1913 err = dev->priv.adev_idx; in probe_one()
1917 err = mlx5_mdev_init(dev, prof_sel); in probe_one()
1921 err = mlx5_pci_init(dev, pdev, id); in probe_one()
1923 mlx5_core_err(dev, "mlx5_pci_init failed with error code %d\n", in probe_one()
1928 err = mlx5_init_one(dev); in probe_one()
1930 mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n", in probe_one()
1935 err = mlx5_crdump_enable(dev); in probe_one()
1937 dev_err(&pdev->dev, "mlx5_crdump_enable failed with error code %d\n", err); in probe_one()
1939 err = mlx5_hwmon_dev_register(dev); in probe_one()
1941 mlx5_core_err(dev, "mlx5_hwmon_dev_register failed with error code %d\n", err); in probe_one()
1948 mlx5_pci_close(dev); in probe_one()
1950 mlx5_mdev_uninit(dev); in probe_one()
1952 mlx5_adev_idx_free(dev->priv.adev_idx); in probe_one()
1961 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in remove_one() local
1962 struct devlink *devlink = priv_to_devlink(dev); in remove_one()
1964 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state); in remove_one()
1969 mlx5_drain_fw_reset(dev); in remove_one()
1970 mlx5_drain_health_wq(dev); in remove_one()
1973 mlx5_hwmon_dev_unregister(dev); in remove_one()
1974 mlx5_crdump_disable(dev); in remove_one()
1975 mlx5_uninit_one(dev); in remove_one()
1976 mlx5_pci_close(dev); in remove_one()
1977 mlx5_mdev_uninit(dev); in remove_one()
1978 mlx5_adev_idx_free(dev->priv.adev_idx); in remove_one()
1982 #define mlx5_pci_trace(dev, fmt, ...) ({ \ argument
1983 struct mlx5_core_dev *__dev = (dev); \
2000 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in mlx5_pci_err_detected() local
2003 mlx5_pci_trace(dev, "Enter, pci channel state = %d\n", state); in mlx5_pci_err_detected()
2005 mlx5_enter_error_state(dev, false); in mlx5_pci_err_detected()
2006 mlx5_error_sw_reset(dev); in mlx5_pci_err_detected()
2007 mlx5_unload_one(dev, false); in mlx5_pci_err_detected()
2008 mlx5_drain_health_wq(dev); in mlx5_pci_err_detected()
2009 mlx5_pci_disable_device(dev); in mlx5_pci_err_detected()
2014 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, result = %d, %s\n", in mlx5_pci_err_detected()
2015 __func__, dev->state, dev->pci_status, res, result2str(res)); in mlx5_pci_err_detected()
2024 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in wait_vital() local
2025 struct mlx5_core_health *health = &dev->priv.health; in wait_vital()
2035 mlx5_core_info(dev, in wait_vital()
2051 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in mlx5_pci_slot_reset() local
2054 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Enter\n", in mlx5_pci_slot_reset()
2055 __func__, dev->state, dev->pci_status); in mlx5_pci_slot_reset()
2057 err = mlx5_pci_enable_device(dev); in mlx5_pci_slot_reset()
2059 mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n", in mlx5_pci_slot_reset()
2070 mlx5_core_err(dev, "%s: wait vital failed with error code: %d\n", in mlx5_pci_slot_reset()
2077 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, err = %d, result = %d, %s\n", in mlx5_pci_slot_reset()
2078 __func__, dev->state, dev->pci_status, err, res, result2str(res)); in mlx5_pci_slot_reset()
2084 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in mlx5_pci_resume() local
2087 mlx5_pci_trace(dev, "Enter, loading driver..\n"); in mlx5_pci_resume()
2089 err = mlx5_load_one(dev, false); in mlx5_pci_resume()
2092 devlink_health_reporter_state_update(dev->priv.health.fw_fatal_reporter, in mlx5_pci_resume()
2095 mlx5_pci_trace(dev, "Done, err = %d, device %s\n", err, in mlx5_pci_resume()
2105 static int mlx5_try_fast_unload(struct mlx5_core_dev *dev) in mlx5_try_fast_unload() argument
2110 fast_teardown = MLX5_CAP_GEN(dev, fast_teardown); in mlx5_try_fast_unload()
2111 force_teardown = MLX5_CAP_GEN(dev, force_teardown); in mlx5_try_fast_unload()
2113 mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown); in mlx5_try_fast_unload()
2114 mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown); in mlx5_try_fast_unload()
2119 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { in mlx5_try_fast_unload()
2120 mlx5_core_dbg(dev, "Device in internal error state, giving up\n"); in mlx5_try_fast_unload()
2127 mlx5_drain_health_wq(dev); in mlx5_try_fast_unload()
2128 mlx5_stop_health_poll(dev, false); in mlx5_try_fast_unload()
2130 ret = mlx5_cmd_fast_teardown_hca(dev); in mlx5_try_fast_unload()
2134 ret = mlx5_cmd_force_teardown_hca(dev); in mlx5_try_fast_unload()
2138 mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", ret); in mlx5_try_fast_unload()
2139 mlx5_start_health_poll(dev); in mlx5_try_fast_unload()
2143 mlx5_enter_error_state(dev, true); in mlx5_try_fast_unload()
2150 mlx5_core_eq_free_irqs(dev); in mlx5_try_fast_unload()
2157 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in shutdown() local
2160 mlx5_core_info(dev, "Shutdown was called\n"); in shutdown()
2161 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state); in shutdown()
2162 err = mlx5_try_fast_unload(dev); in shutdown()
2164 mlx5_unload_one(dev, false); in shutdown()
2165 mlx5_pci_disable_device(dev); in shutdown()
2170 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in mlx5_suspend() local
2172 mlx5_unload_one(dev, true); in mlx5_suspend()
2179 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); in mlx5_resume() local
2181 return mlx5_load_one(dev, false); in mlx5_resume()
2212 void mlx5_disable_device(struct mlx5_core_dev *dev) in mlx5_disable_device() argument
2214 mlx5_error_sw_reset(dev); in mlx5_disable_device()
2215 mlx5_unload_one_devl_locked(dev, false); in mlx5_disable_device()
2218 int mlx5_recover_device(struct mlx5_core_dev *dev) in mlx5_recover_device() argument
2220 if (!mlx5_core_is_sf(dev)) { in mlx5_recover_device()
2221 mlx5_pci_disable_device(dev); in mlx5_recover_device()
2222 if (mlx5_pci_slot_reset(dev->pdev) != PCI_ERS_RESULT_RECOVERED) in mlx5_recover_device()
2226 return mlx5_load_one_devl_locked(dev, true); in mlx5_recover_device()