16 ice_sched_add_root_node(struct ice_port_info *pi,  in ice_sched_add_root_node()  argument
22 	if (!pi)  in ice_sched_add_root_node()
25 	hw = pi->hw;  in ice_sched_add_root_node()
40 	pi->root = root;  in ice_sched_add_root_node()
151 ice_sched_add_node(struct ice_port_info *pi, u8 layer,  in ice_sched_add_node()  argument
161 	if (!pi)  in ice_sched_add_node()
164 	hw = pi->hw;  in ice_sched_add_node()
167 	parent = ice_sched_find_node_by_teid(pi->root,  in ice_sched_add_node()
275 ice_sched_get_first_node(struct ice_port_info *pi,  in ice_sched_get_first_node()  argument
278 	return pi->sib_head[parent->tc_num][layer];  in ice_sched_get_first_node()
288 struct ice_sched_node *ice_sched_get_tc_node(struct ice_port_info *pi, u8 tc)  in ice_sched_get_tc_node()  argument
292 	if (!pi || !pi->root)  in ice_sched_get_tc_node()
294 	for (i = 0; i < pi->root->num_children; i++)  in ice_sched_get_tc_node()
295 		if (pi->root->children[i]->tc_num == tc)  in ice_sched_get_tc_node()
296 			return pi->root->children[i];  in ice_sched_get_tc_node()
309 void ice_free_sched_node(struct ice_port_info *pi, struct ice_sched_node *node)  in ice_free_sched_node()  argument
312 	struct ice_hw *hw = pi->hw;  in ice_free_sched_node()
320 		ice_free_sched_node(pi, node->children[0]);  in ice_free_sched_node()
346 		p = ice_sched_get_first_node(pi, node, node->tx_sched_layer);  in ice_free_sched_node()
356 		if (pi->sib_head[node->tc_num][node->tx_sched_layer] == node)  in ice_free_sched_node()
357 			pi->sib_head[node->tc_num][node->tx_sched_layer] =  in ice_free_sched_node()
363 	xa_erase(&pi->sched_node_ids, node->id);  in ice_free_sched_node()
774 static void ice_sched_clear_rl_prof(struct ice_port_info *pi)  in ice_sched_clear_rl_prof()  argument
778 	for (ln = 0; ln < pi->hw->num_tx_sched_layers; ln++) {  in ice_sched_clear_rl_prof()
783 					 &pi->rl_prof_list[ln], list_entry) {  in ice_sched_clear_rl_prof()
784 			struct ice_hw *hw = pi->hw;  in ice_sched_clear_rl_prof()
831 static void ice_sched_clear_tx_topo(struct ice_port_info *pi)  in ice_sched_clear_tx_topo()  argument
833 	if (!pi)  in ice_sched_clear_tx_topo()
836 	ice_sched_clear_rl_prof(pi);  in ice_sched_clear_tx_topo()
837 	if (pi->root) {  in ice_sched_clear_tx_topo()
838 		ice_free_sched_node(pi, pi->root);  in ice_sched_clear_tx_topo()
839 		pi->root = NULL;  in ice_sched_clear_tx_topo()
849 void ice_sched_clear_port(struct ice_port_info *pi)  in ice_sched_clear_port()  argument
851 	if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY)  in ice_sched_clear_port()
854 	pi->port_state = ICE_SCHED_PORT_STATE_INIT;  in ice_sched_clear_port()
855 	mutex_lock(&pi->sched_lock);  in ice_sched_clear_port()
856 	ice_sched_clear_tx_topo(pi);  in ice_sched_clear_port()
857 	mutex_unlock(&pi->sched_lock);  in ice_sched_clear_port()
858 	mutex_destroy(&pi->sched_lock);  in ice_sched_clear_port()
897 ice_sched_add_elems(struct ice_port_info *pi, struct ice_sched_node *tc_node,  in ice_sched_add_elems()  argument
905 	struct ice_hw *hw = pi->hw;  in ice_sched_add_elems()
947 			status = ice_sched_add_node(pi, layer, &buf->generic[i], prealloc_nodes[i]);  in ice_sched_add_elems()
949 			status = ice_sched_add_node(pi, layer, &buf->generic[i], NULL);  in ice_sched_add_elems()
973 		status = xa_alloc(&pi->sched_node_ids, &new_node->id, NULL, XA_LIMIT(0, UINT_MAX),  in ice_sched_add_elems()
985 		prev = ice_sched_get_first_node(pi, tc_node, layer);  in ice_sched_add_elems()
993 		if (!pi->sib_head[tc_node->tc_num][layer])  in ice_sched_add_elems()
994 			pi->sib_head[tc_node->tc_num][layer] = new_node;  in ice_sched_add_elems()
1017 ice_sched_add_nodes_to_hw_layer(struct ice_port_info *pi,  in ice_sched_add_nodes_to_hw_layer()  argument
1030 	if (!parent || layer < pi->hw->sw_entry_point_layer)  in ice_sched_add_nodes_to_hw_layer()
1034 	max_child_nodes = pi->hw->max_children[parent->tx_sched_layer];  in ice_sched_add_nodes_to_hw_layer()
1044 	return ice_sched_add_elems(pi, tc_node, parent, layer, num_nodes,  in ice_sched_add_nodes_to_hw_layer()
1061 ice_sched_add_nodes_to_layer(struct ice_port_info *pi,  in ice_sched_add_nodes_to_layer()  argument
1076 		status = ice_sched_add_nodes_to_hw_layer(pi, tc_node, parent,  in ice_sched_add_nodes_to_layer()
1084 			ice_debug(pi->hw, ICE_DBG_SCHED, "added extra nodes %d %d\n", num_nodes,  in ice_sched_add_nodes_to_layer()
1096 		max_child_nodes = pi->hw->max_children[parent->tx_sched_layer];  in ice_sched_add_nodes_to_layer()
1181 static void ice_rm_dflt_leaf_node(struct ice_port_info *pi)  in ice_rm_dflt_leaf_node()  argument
1185 	node = pi->root;  in ice_rm_dflt_leaf_node()
1196 		status = ice_sched_remove_elems(pi->hw, node->parent, 1, &teid);  in ice_rm_dflt_leaf_node()
1198 			ice_free_sched_node(pi, node);  in ice_rm_dflt_leaf_node()
1209 static void ice_sched_rm_dflt_nodes(struct ice_port_info *pi)  in ice_sched_rm_dflt_nodes()  argument
1213 	ice_rm_dflt_leaf_node(pi);  in ice_sched_rm_dflt_nodes()
1216 	node = pi->root;  in ice_sched_rm_dflt_nodes()
1218 		if (node->tx_sched_layer >= pi->hw->sw_entry_point_layer &&  in ice_sched_rm_dflt_nodes()
1221 			ice_free_sched_node(pi, node);  in ice_sched_rm_dflt_nodes()
1239 int ice_sched_init_port(struct ice_port_info *pi)  in ice_sched_init_port()  argument
1248 	if (!pi)  in ice_sched_init_port()
1250 	hw = pi->hw;  in ice_sched_init_port()
1258 	status = ice_aq_get_dflt_topo(hw, pi->lport, buf, ICE_AQ_MAX_BUF_LEN,  in ice_sched_init_port()
1287 		pi->last_node_teid =  in ice_sched_init_port()
1290 		pi->last_node_teid =  in ice_sched_init_port()
1294 	status = ice_sched_add_root_node(pi, &buf[0].generic[0]);  in ice_sched_init_port()
1309 			status = ice_sched_add_node(pi, j, &buf[i].generic[j], NULL);  in ice_sched_init_port()
1316 	if (pi->root)  in ice_sched_init_port()
1317 		ice_sched_rm_dflt_nodes(pi);  in ice_sched_init_port()
1320 	pi->port_state = ICE_SCHED_PORT_STATE_READY;  in ice_sched_init_port()
1321 	mutex_init(&pi->sched_lock);  in ice_sched_init_port()
1323 		INIT_LIST_HEAD(&pi->rl_prof_list[i]);  in ice_sched_init_port()
1326 	if (status && pi->root) {  in ice_sched_init_port()
1327 		ice_free_sched_node(pi, pi->root);  in ice_sched_init_port()
1328 		pi->root = NULL;  in ice_sched_init_port()
1476 ice_sched_get_free_qgrp(struct ice_port_info *pi,  in ice_sched_get_free_qgrp()  argument
1496 		if (ice_sched_find_node_in_subtree(pi->hw, vsi_node, qgrp_node))  in ice_sched_get_free_qgrp()
1521 ice_sched_get_free_qparent(struct ice_port_info *pi, u16 vsi_handle, u8 tc,  in ice_sched_get_free_qparent()  argument
1529 	qgrp_layer = ice_sched_get_qgrp_layer(pi->hw);  in ice_sched_get_free_qparent()
1530 	max_children = pi->hw->max_children[qgrp_layer];  in ice_sched_get_free_qparent()
1532 	vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle);  in ice_sched_get_free_qparent()
1541 	qgrp_node = ice_sched_get_first_node(pi, vsi_node, qgrp_layer);  in ice_sched_get_free_qparent()
1544 		if (ice_sched_find_node_in_subtree(pi->hw, vsi_node, qgrp_node))  in ice_sched_get_free_qparent()
1552 	return ice_sched_get_free_qgrp(pi, vsi_node, qgrp_node, owner);  in ice_sched_get_free_qparent()
1565 ice_sched_get_vsi_node(struct ice_port_info *pi, struct ice_sched_node *tc_node,  in ice_sched_get_vsi_node()  argument
1571 	vsi_layer = ice_sched_get_vsi_layer(pi->hw);  in ice_sched_get_vsi_node()
1572 	node = ice_sched_get_first_node(pi, tc_node, vsi_layer);  in ice_sched_get_vsi_node()
1594 ice_sched_get_agg_node(struct ice_port_info *pi, struct ice_sched_node *tc_node,  in ice_sched_get_agg_node()  argument
1598 	struct ice_hw *hw = pi->hw;  in ice_sched_get_agg_node()
1604 	node = ice_sched_get_first_node(pi, tc_node, agg_layer);  in ice_sched_get_agg_node()
1656 ice_sched_add_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle,  in ice_sched_add_vsi_child_nodes()  argument
1661 	struct ice_hw *hw = pi->hw;  in ice_sched_add_vsi_child_nodes()
1668 	parent = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);  in ice_sched_add_vsi_child_nodes()
1675 		status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, i,  in ice_sched_add_vsi_child_nodes()
1712 ice_sched_calc_vsi_support_nodes(struct ice_port_info *pi,  in ice_sched_calc_vsi_support_nodes()  argument
1719 	vsil = ice_sched_get_vsi_layer(pi->hw);  in ice_sched_calc_vsi_support_nodes()
1720 	for (i = vsil; i >= pi->hw->sw_entry_point_layer; i--)  in ice_sched_calc_vsi_support_nodes()
1730 			node = ice_sched_get_first_node(pi, tc_node, (u8)i);  in ice_sched_calc_vsi_support_nodes()
1733 				if (node->num_children < pi->hw->max_children[i])  in ice_sched_calc_vsi_support_nodes()
1760 ice_sched_add_vsi_support_nodes(struct ice_port_info *pi, u16 vsi_handle,  in ice_sched_add_vsi_support_nodes()  argument
1768 	if (!pi)  in ice_sched_add_vsi_support_nodes()
1771 	vsil = ice_sched_get_vsi_layer(pi->hw);  in ice_sched_add_vsi_support_nodes()
1772 	for (i = pi->hw->sw_entry_point_layer; i <= vsil; i++) {  in ice_sched_add_vsi_support_nodes()
1775 		status = ice_sched_add_nodes_to_layer(pi, tc_node, parent,  in ice_sched_add_vsi_support_nodes()
1810 ice_sched_add_vsi_to_topo(struct ice_port_info *pi, u16 vsi_handle, u8 tc)  in ice_sched_add_vsi_to_topo()  argument
1815 	tc_node = ice_sched_get_tc_node(pi, tc);  in ice_sched_add_vsi_to_topo()
1820 	ice_sched_calc_vsi_support_nodes(pi, tc_node, num_nodes);  in ice_sched_add_vsi_to_topo()
1823 	return ice_sched_add_vsi_support_nodes(pi, vsi_handle, tc_node,  in ice_sched_add_vsi_to_topo()
1838 ice_sched_update_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle,  in ice_sched_update_vsi_child_nodes()  argument
1845 	struct ice_hw *hw = pi->hw;  in ice_sched_update_vsi_child_nodes()
1849 	tc_node = ice_sched_get_tc_node(pi, tc);  in ice_sched_update_vsi_child_nodes()
1853 	vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);  in ice_sched_update_vsi_child_nodes()
1887 	status = ice_sched_add_vsi_child_nodes(pi, vsi_handle, tc_node,  in ice_sched_update_vsi_child_nodes()
1913 ice_sched_cfg_vsi(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u16 maxqs,  in ice_sched_cfg_vsi()  argument
1918 	struct ice_hw *hw = pi->hw;  in ice_sched_cfg_vsi()
1921 	ice_debug(pi->hw, ICE_DBG_SCHED, "add/config VSI %d\n", vsi_handle);  in ice_sched_cfg_vsi()
1922 	tc_node = ice_sched_get_tc_node(pi, tc);  in ice_sched_cfg_vsi()
1928 	vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);  in ice_sched_cfg_vsi()
1945 		status = ice_sched_add_vsi_to_topo(pi, vsi_handle, tc);  in ice_sched_cfg_vsi()
1949 		vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);  in ice_sched_cfg_vsi()
1964 	status = ice_sched_update_vsi_child_nodes(pi, vsi_handle, tc, maxqs,  in ice_sched_cfg_vsi()
1989 static void ice_sched_rm_agg_vsi_info(struct ice_port_info *pi, u16 vsi_handle)  in ice_sched_rm_agg_vsi_info()  argument
1994 	list_for_each_entry_safe(agg_info, atmp, &pi->hw->agg_list,  in ice_sched_rm_agg_vsi_info()
2003 				devm_kfree(ice_hw_to_dev(pi->hw),  in ice_sched_rm_agg_vsi_info()
2037 ice_sched_rm_vsi_cfg(struct ice_port_info *pi, u16 vsi_handle, u8 owner)  in ice_sched_rm_vsi_cfg()  argument
2043 	ice_debug(pi->hw, ICE_DBG_SCHED, "removing VSI %d\n", vsi_handle);  in ice_sched_rm_vsi_cfg()
2044 	if (!ice_is_vsi_valid(pi->hw, vsi_handle))  in ice_sched_rm_vsi_cfg()
2046 	mutex_lock(&pi->sched_lock);  in ice_sched_rm_vsi_cfg()
2047 	vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle);  in ice_sched_rm_vsi_cfg()
2055 		tc_node = ice_sched_get_tc_node(pi, i);  in ice_sched_rm_vsi_cfg()
2059 		vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);  in ice_sched_rm_vsi_cfg()
2064 			ice_debug(pi->hw, ICE_DBG_SCHED, "VSI has leaf nodes in TC %d\n", i);  in ice_sched_rm_vsi_cfg()
2070 				ice_free_sched_node(pi, vsi_node->children[j]);  in ice_sched_rm_vsi_cfg()
2082 			ice_free_sched_node(pi, vsi_node);  in ice_sched_rm_vsi_cfg()
2086 			ice_sched_rm_agg_vsi_info(pi, vsi_handle);  in ice_sched_rm_vsi_cfg()
2096 	mutex_unlock(&pi->sched_lock);  in ice_sched_rm_vsi_cfg()
2108 int ice_rm_vsi_lan_cfg(struct ice_port_info *pi, u16 vsi_handle)  in ice_rm_vsi_lan_cfg()  argument
2110 	return ice_sched_rm_vsi_cfg(pi, vsi_handle, ICE_SCHED_NODE_OWNER_LAN);  in ice_rm_vsi_lan_cfg()
2121 int ice_rm_vsi_rdma_cfg(struct ice_port_info *pi, u16 vsi_handle)  in ice_rm_vsi_rdma_cfg()  argument
2123 	return ice_sched_rm_vsi_cfg(pi, vsi_handle, ICE_SCHED_NODE_OWNER_RDMA);  in ice_rm_vsi_rdma_cfg()
2232 ice_sched_move_nodes(struct ice_port_info *pi, struct ice_sched_node *parent,  in ice_sched_move_nodes()  argument
2242 	hw = pi->hw;  in ice_sched_move_nodes()
2258 		node = ice_sched_find_node_by_teid(pi->root, list[i]);  in ice_sched_move_nodes()
2295 ice_sched_move_vsi_to_agg(struct ice_port_info *pi, u16 vsi_handle, u32 agg_id,  in ice_sched_move_vsi_to_agg()  argument
2305 	tc_node = ice_sched_get_tc_node(pi, tc);  in ice_sched_move_vsi_to_agg()
2309 	agg_node = ice_sched_get_agg_node(pi, tc_node, agg_id);  in ice_sched_move_vsi_to_agg()
2313 	vsi_node = ice_sched_get_vsi_node(pi, tc_node, vsi_handle);  in ice_sched_move_vsi_to_agg()
2318 	if (ice_sched_find_node_in_subtree(pi->hw, agg_node, vsi_node))  in ice_sched_move_vsi_to_agg()
2321 	aggl = ice_sched_get_agg_layer(pi->hw);  in ice_sched_move_vsi_to_agg()
2322 	vsil = ice_sched_get_vsi_layer(pi->hw);  in ice_sched_move_vsi_to_agg()
2330 		parent = ice_sched_get_free_vsi_parent(pi->hw,  in ice_sched_move_vsi_to_agg()
2340 		status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, i,  in ice_sched_move_vsi_to_agg()
2362 	return ice_sched_move_nodes(pi, parent, 1, &vsi_teid);  in ice_sched_move_vsi_to_agg()
2377 ice_move_all_vsi_to_dflt_agg(struct ice_port_info *pi,  in ice_move_all_vsi_to_dflt_agg()  argument
2393 		status = ice_sched_move_vsi_to_agg(pi, vsi_handle,  in ice_move_all_vsi_to_dflt_agg()
2401 			devm_kfree(ice_hw_to_dev(pi->hw), agg_vsi_info);  in ice_move_all_vsi_to_dflt_agg()
2416 ice_sched_is_agg_inuse(struct ice_port_info *pi, struct ice_sched_node *node)  in ice_sched_is_agg_inuse()  argument
2420 	vsil = ice_sched_get_vsi_layer(pi->hw);  in ice_sched_is_agg_inuse()
2423 			if (ice_sched_is_agg_inuse(pi, node->children[i]))  in ice_sched_is_agg_inuse()
2441 ice_sched_rm_agg_cfg(struct ice_port_info *pi, u32 agg_id, u8 tc)  in ice_sched_rm_agg_cfg()  argument
2444 	struct ice_hw *hw = pi->hw;  in ice_sched_rm_agg_cfg()
2446 	tc_node = ice_sched_get_tc_node(pi, tc);  in ice_sched_rm_agg_cfg()
2450 	agg_node = ice_sched_get_agg_node(pi, tc_node, agg_id);  in ice_sched_rm_agg_cfg()
2455 	if (ice_sched_is_agg_inuse(pi, agg_node))  in ice_sched_rm_agg_cfg()
2473 	ice_free_sched_node(pi, agg_node);  in ice_sched_rm_agg_cfg()
2489 ice_rm_agg_cfg_tc(struct ice_port_info *pi, struct ice_sched_agg_info *agg_info,  in ice_rm_agg_cfg_tc()  argument
2498 	status = ice_move_all_vsi_to_dflt_agg(pi, agg_info, tc, rm_vsi_info);  in ice_rm_agg_cfg_tc()
2503 	status = ice_sched_rm_agg_cfg(pi, agg_info->agg_id, tc);  in ice_rm_agg_cfg_tc()
2522 ice_save_agg_tc_bitmap(struct ice_port_info *pi, u32 agg_id,  in ice_save_agg_tc_bitmap()  argument
2527 	agg_info = ice_get_agg_info(pi->hw, agg_id);  in ice_save_agg_tc_bitmap()
2545 ice_sched_add_agg_cfg(struct ice_port_info *pi, u32 agg_id, u8 tc)  in ice_sched_add_agg_cfg()  argument
2549 	struct ice_hw *hw = pi->hw;  in ice_sched_add_agg_cfg()
2555 	tc_node = ice_sched_get_tc_node(pi, tc);  in ice_sched_add_agg_cfg()
2559 	agg_node = ice_sched_get_agg_node(pi, tc_node, agg_id);  in ice_sched_add_agg_cfg()
2574 		parent = ice_sched_get_first_node(pi, tc_node, i);  in ice_sched_add_agg_cfg()
2594 		status = ice_sched_add_nodes_to_layer(pi, tc_node, parent, i,  in ice_sched_add_agg_cfg()
2635 ice_sched_cfg_agg(struct ice_port_info *pi, u32 agg_id,  in ice_sched_cfg_agg()  argument
2639 	struct ice_hw *hw = pi->hw;  in ice_sched_cfg_agg()
2665 			status = ice_rm_agg_cfg_tc(pi, agg_info, tc, false);  in ice_sched_cfg_agg()
2676 		status = ice_sched_add_agg_cfg(pi, agg_id, tc);  in ice_sched_cfg_agg()
2697 ice_cfg_agg(struct ice_port_info *pi, u32 agg_id, enum ice_agg_type agg_type,  in ice_cfg_agg()  argument
2703 	mutex_lock(&pi->sched_lock);  in ice_cfg_agg()
2704 	status = ice_sched_cfg_agg(pi, agg_id, agg_type, &bitmap);  in ice_cfg_agg()
2706 		status = ice_save_agg_tc_bitmap(pi, agg_id, &bitmap);  in ice_cfg_agg()
2707 	mutex_unlock(&pi->sched_lock);  in ice_cfg_agg()
2766 ice_save_agg_vsi_tc_bitmap(struct ice_port_info *pi, u32 agg_id, u16 vsi_handle,  in ice_save_agg_vsi_tc_bitmap()  argument
2772 	agg_info = ice_get_agg_info(pi->hw, agg_id);  in ice_save_agg_vsi_tc_bitmap()
2796 ice_sched_assoc_vsi_to_agg(struct ice_port_info *pi, u32 agg_id,  in ice_sched_assoc_vsi_to_agg()  argument
2801 	struct ice_hw *hw = pi->hw;  in ice_sched_assoc_vsi_to_agg()
2805 	if (!ice_is_vsi_valid(pi->hw, vsi_handle))  in ice_sched_assoc_vsi_to_agg()
2845 		status = ice_sched_move_vsi_to_agg(pi, vsi_handle, agg_id, tc);  in ice_sched_assoc_vsi_to_agg()
2855 		devm_kfree(ice_hw_to_dev(pi->hw), old_agg_vsi_info);  in ice_sched_assoc_vsi_to_agg()
2867 static void ice_sched_rm_unused_rl_prof(struct ice_port_info *pi)  in ice_sched_rm_unused_rl_prof()  argument
2871 	for (ln = 0; ln < pi->hw->num_tx_sched_layers; ln++) {  in ice_sched_rm_unused_rl_prof()
2876 					 &pi->rl_prof_list[ln], list_entry) {  in ice_sched_rm_unused_rl_prof()
2877 			if (!ice_sched_del_rl_profile(pi->hw, rl_prof_elem))  in ice_sched_rm_unused_rl_prof()
2878 				ice_debug(pi->hw, ICE_DBG_SCHED, "Removed rl profile\n");  in ice_sched_rm_unused_rl_prof()
2969 ice_move_vsi_to_agg(struct ice_port_info *pi, u32 agg_id, u16 vsi_handle,  in ice_move_vsi_to_agg()  argument
2975 	mutex_lock(&pi->sched_lock);  in ice_move_vsi_to_agg()
2976 	status = ice_sched_assoc_vsi_to_agg(pi, agg_id, vsi_handle,  in ice_move_vsi_to_agg()
2979 		status = ice_save_agg_vsi_tc_bitmap(pi, agg_id, vsi_handle,  in ice_move_vsi_to_agg()
2981 	mutex_unlock(&pi->sched_lock);  in ice_move_vsi_to_agg()
3065 ice_sched_save_vsi_bw(struct ice_port_info *pi, u16 vsi_handle, u8 tc,  in ice_sched_save_vsi_bw()  argument
3070 	if (!ice_is_vsi_valid(pi->hw, vsi_handle))  in ice_sched_save_vsi_bw()
3072 	vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle);  in ice_sched_save_vsi_bw()
3213 ice_sched_add_rl_profile(struct ice_port_info *pi,  in ice_sched_add_rl_profile()  argument
3239 	if (!pi)  in ice_sched_add_rl_profile()
3241 	hw = pi->hw;  in ice_sched_add_rl_profile()
3242 	list_for_each_entry(rl_prof_elem, &pi->rl_prof_list[layer_num],  in ice_sched_add_rl_profile()
3275 	list_add(&rl_prof_elem->list_entry, &pi->rl_prof_list[layer_num]);  in ice_sched_add_rl_profile()
3395 ice_sched_get_rl_prof_layer(struct ice_port_info *pi, enum ice_rl_type rl_type,  in ice_sched_get_rl_prof_layer()  argument
3398 	struct ice_hw *hw = pi->hw;  in ice_sched_get_rl_prof_layer()
3464 ice_sched_rm_rl_profile(struct ice_port_info *pi, u8 layer_num, u8 profile_type,  in ice_sched_rm_rl_profile()  argument
3473 	list_for_each_entry(rl_prof_elem, &pi->rl_prof_list[layer_num],  in ice_sched_rm_rl_profile()
3483 			status = ice_sched_del_rl_profile(pi->hw, rl_prof_elem);  in ice_sched_rm_rl_profile()
3485 				ice_debug(pi->hw, ICE_DBG_SCHED, "Remove rl profile failed\n");  in ice_sched_rm_rl_profile()
3505 ice_sched_set_node_bw_dflt(struct ice_port_info *pi,  in ice_sched_set_node_bw_dflt()  argument
3515 	hw = pi->hw;  in ice_sched_set_node_bw_dflt()
3545 	return ice_sched_rm_rl_profile(pi, layer_num, profile_type, old_id);  in ice_sched_set_node_bw_dflt()
3562 ice_sched_set_eir_srl_excl(struct ice_port_info *pi,  in ice_sched_set_eir_srl_excl()  argument
3579 		return ice_sched_set_node_bw_dflt(pi, node, ICE_MAX_BW,  in ice_sched_set_eir_srl_excl()
3586 		return ice_sched_set_node_bw_dflt(pi, node,  in ice_sched_set_eir_srl_excl()
3606 ice_sched_set_node_bw(struct ice_port_info *pi, struct ice_sched_node *node,  in ice_sched_set_node_bw()  argument
3610 	struct ice_hw *hw = pi->hw;  in ice_sched_set_node_bw()
3614 	rl_prof_info = ice_sched_add_rl_profile(pi, rl_type, bw, layer_num);  in ice_sched_set_node_bw()
3636 	return ice_sched_rm_rl_profile(pi, layer_num,  in ice_sched_set_node_bw()
3650 ice_sched_set_node_priority(struct ice_port_info *pi, struct ice_sched_node *node,  in ice_sched_set_node_priority()  argument
3662 	return ice_sched_update_elem(pi->hw, node, &buf);  in ice_sched_set_node_priority()
3674 ice_sched_set_node_weight(struct ice_port_info *pi, struct ice_sched_node *node, u16 weight)  in ice_sched_set_node_weight()  argument
3689 	return ice_sched_update_elem(pi->hw, node, &buf);  in ice_sched_set_node_weight()
3703 ice_sched_set_node_bw_lmt(struct ice_port_info *pi, struct ice_sched_node *node,  in ice_sched_set_node_bw_lmt()  argument
3712 	if (!pi)  in ice_sched_set_node_bw_lmt()
3714 	hw = pi->hw;  in ice_sched_set_node_bw_lmt()
3716 	ice_sched_rm_unused_rl_prof(pi);  in ice_sched_set_node_bw_lmt()
3717 	layer_num = ice_sched_get_rl_prof_layer(pi, rl_type,  in ice_sched_set_node_bw_lmt()
3731 	status = ice_sched_set_eir_srl_excl(pi, cfg_node, layer_num, rl_type,  in ice_sched_set_node_bw_lmt()
3736 		return ice_sched_set_node_bw_dflt(pi, cfg_node, rl_type,  in ice_sched_set_node_bw_lmt()
3738 	return ice_sched_set_node_bw(pi, cfg_node, rl_type, bw, layer_num);  in ice_sched_set_node_bw_lmt()
3752 ice_sched_set_node_bw_dflt_lmt(struct ice_port_info *pi,  in ice_sched_set_node_bw_dflt_lmt()  argument
3756 	return ice_sched_set_node_bw_lmt(pi, node, rl_type,  in ice_sched_set_node_bw_dflt_lmt()
3826 ice_sched_set_q_bw_lmt(struct ice_port_info *pi, u16 vsi_handle, u8 tc,  in ice_sched_set_q_bw_lmt()  argument
3833 	if (!ice_is_vsi_valid(pi->hw, vsi_handle))  in ice_sched_set_q_bw_lmt()
3835 	mutex_lock(&pi->sched_lock);  in ice_sched_set_q_bw_lmt()
3836 	q_ctx = ice_get_lan_q_ctx(pi->hw, vsi_handle, tc, q_handle);  in ice_sched_set_q_bw_lmt()
3839 	node = ice_sched_find_node_by_teid(pi->root, q_ctx->q_teid);  in ice_sched_set_q_bw_lmt()
3841 		ice_debug(pi->hw, ICE_DBG_SCHED, "Wrong q_teid\n");  in ice_sched_set_q_bw_lmt()
3853 		sel_layer = ice_sched_get_rl_prof_layer(pi, rl_type,  in ice_sched_set_q_bw_lmt()
3855 		if (sel_layer >= pi->hw->num_tx_sched_layers) {  in ice_sched_set_q_bw_lmt()
3865 		status = ice_sched_set_node_bw_dflt_lmt(pi, node, rl_type);  in ice_sched_set_q_bw_lmt()
3867 		status = ice_sched_set_node_bw_lmt(pi, node, rl_type, bw);  in ice_sched_set_q_bw_lmt()
3873 	mutex_unlock(&pi->sched_lock);  in ice_sched_set_q_bw_lmt()
3889 ice_cfg_q_bw_lmt(struct ice_port_info *pi, u16 vsi_handle, u8 tc,  in ice_cfg_q_bw_lmt()  argument
3892 	return ice_sched_set_q_bw_lmt(pi, vsi_handle, tc, q_handle, rl_type,  in ice_cfg_q_bw_lmt()
3907 ice_cfg_q_bw_dflt_lmt(struct ice_port_info *pi, u16 vsi_handle, u8 tc,  in ice_cfg_q_bw_dflt_lmt()  argument
3910 	return ice_sched_set_q_bw_lmt(pi, vsi_handle, tc, q_handle, rl_type,  in ice_cfg_q_bw_dflt_lmt()
3926 ice_sched_get_node_by_id_type(struct ice_port_info *pi, u32 id,  in ice_sched_get_node_by_id_type()  argument
3936 		if (!ice_is_vsi_valid(pi->hw, vsi_handle))  in ice_sched_get_node_by_id_type()
3939 		vsi_ctx = ice_get_vsi_ctx(pi->hw, vsi_handle);  in ice_sched_get_node_by_id_type()
3949 		tc_node = ice_sched_get_tc_node(pi, tc);  in ice_sched_get_node_by_id_type()
3951 			node = ice_sched_get_agg_node(pi, tc_node, id);  in ice_sched_get_node_by_id_type()
3975 ice_sched_set_node_bw_lmt_per_tc(struct ice_port_info *pi, u32 id,  in ice_sched_set_node_bw_lmt_per_tc()  argument
3982 	if (!pi)  in ice_sched_set_node_bw_lmt_per_tc()
3988 	mutex_lock(&pi->sched_lock);  in ice_sched_set_node_bw_lmt_per_tc()
3989 	node = ice_sched_get_node_by_id_type(pi, id, agg_type, tc);  in ice_sched_set_node_bw_lmt_per_tc()
3991 		ice_debug(pi->hw, ICE_DBG_SCHED, "Wrong id, agg type, or tc\n");  in ice_sched_set_node_bw_lmt_per_tc()
3995 		status = ice_sched_set_node_bw_dflt_lmt(pi, node, rl_type);  in ice_sched_set_node_bw_lmt_per_tc()
3997 		status = ice_sched_set_node_bw_lmt(pi, node, rl_type, bw);  in ice_sched_set_node_bw_lmt_per_tc()
4000 	mutex_unlock(&pi->sched_lock);  in ice_sched_set_node_bw_lmt_per_tc()
4016 ice_cfg_vsi_bw_lmt_per_tc(struct ice_port_info *pi, u16 vsi_handle, u8 tc,  in ice_cfg_vsi_bw_lmt_per_tc()  argument
4021 	status = ice_sched_set_node_bw_lmt_per_tc(pi, vsi_handle,  in ice_cfg_vsi_bw_lmt_per_tc()
4025 		mutex_lock(&pi->sched_lock);  in ice_cfg_vsi_bw_lmt_per_tc()
4026 		status = ice_sched_save_vsi_bw(pi, vsi_handle, tc, rl_type, bw);  in ice_cfg_vsi_bw_lmt_per_tc()
4027 		mutex_unlock(&pi->sched_lock);  in ice_cfg_vsi_bw_lmt_per_tc()
4043 ice_cfg_vsi_bw_dflt_lmt_per_tc(struct ice_port_info *pi, u16 vsi_handle, u8 tc,  in ice_cfg_vsi_bw_dflt_lmt_per_tc()  argument
4048 	status = ice_sched_set_node_bw_lmt_per_tc(pi, vsi_handle,  in ice_cfg_vsi_bw_dflt_lmt_per_tc()
4053 		mutex_lock(&pi->sched_lock);  in ice_cfg_vsi_bw_dflt_lmt_per_tc()
4054 		status = ice_sched_save_vsi_bw(pi, vsi_handle, tc, rl_type,  in ice_cfg_vsi_bw_dflt_lmt_per_tc()
4056 		mutex_unlock(&pi->sched_lock);  in ice_cfg_vsi_bw_dflt_lmt_per_tc()
4142 	struct ice_port_info *pi = hw->port_info;  in ice_sched_replay_node_bw()  local
4157 		status = ice_sched_set_node_bw_lmt(pi, node, ICE_MIN_BW,  in ice_sched_replay_node_bw()
4170 		status = ice_sched_set_node_bw_lmt(pi, node, ICE_MAX_BW,  in ice_sched_replay_node_bw()
4183 		status = ice_sched_set_node_bw_lmt(pi, node, ICE_SHARED_BW,  in ice_sched_replay_node_bw()
4199 ice_sched_get_ena_tc_bitmap(struct ice_port_info *pi,  in ice_sched_get_ena_tc_bitmap()  argument
4208 		    (ice_sched_get_tc_node(pi, tc)))  in ice_sched_get_ena_tc_bitmap()
4222 	struct ice_port_info *pi = hw->port_info;  in ice_sched_replay_agg()  local
4225 	mutex_lock(&pi->sched_lock);  in ice_sched_replay_agg()
4234 			ice_sched_get_ena_tc_bitmap(pi,  in ice_sched_replay_agg()
4249 	mutex_unlock(&pi->sched_lock);  in ice_sched_replay_agg()
4261 	struct ice_port_info *pi = hw->port_info;  in ice_sched_replay_agg_vsi_preinit()  local
4264 	mutex_lock(&pi->sched_lock);  in ice_sched_replay_agg_vsi_preinit()
4273 	mutex_unlock(&pi->sched_lock);  in ice_sched_replay_agg_vsi_preinit()
4289 	struct ice_port_info *pi = hw->port_info;  in ice_sched_replay_vsi_agg()  local
4302 	ice_sched_get_ena_tc_bitmap(pi, agg_info->replay_tc_bitmap,  in ice_sched_replay_vsi_agg()
4311 	ice_sched_get_ena_tc_bitmap(pi, agg_vsi_info->replay_tc_bitmap,  in ice_sched_replay_vsi_agg()
4314 	return ice_sched_assoc_vsi_to_agg(pi, agg_info->agg_id, vsi_handle,  in ice_sched_replay_vsi_agg()
4328 	struct ice_port_info *pi = hw->port_info;  in ice_replay_vsi_agg()  local
4331 	mutex_lock(&pi->sched_lock);  in ice_replay_vsi_agg()
4333 	mutex_unlock(&pi->sched_lock);  in ice_replay_vsi_agg()
4345 int ice_sched_replay_q_bw(struct ice_port_info *pi, struct ice_q_ctx *q_ctx)  in ice_sched_replay_q_bw()  argument
4350 	q_node = ice_sched_find_node_by_teid(pi->root, q_ctx->q_teid);  in ice_sched_replay_q_bw()
4353 	return ice_sched_replay_node_bw(pi->hw, q_node, &q_ctx->bw_t_info);  in ice_sched_replay_q_bw()