25 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)  in ppe_w32()  argument
27 	writel(val, ppe->base + reg);  in ppe_w32()
30 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg)  in ppe_r32()  argument
32 	return readl(ppe->base + reg);  in ppe_r32()
35 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set)  in ppe_m32()  argument
39 	val = ppe_r32(ppe, reg);  in ppe_m32()
42 	ppe_w32(ppe, reg, val);  in ppe_m32()
47 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val)  in ppe_set()  argument
49 	return ppe_m32(ppe, reg, 0, val);  in ppe_set()
52 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)  in ppe_clear()  argument
54 	return ppe_m32(ppe, reg, val, 0);  in ppe_clear()
62 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)  in mtk_ppe_wait_busy()  argument
67 	ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val,  in mtk_ppe_wait_busy()
72 		dev_err(ppe->dev, "PPE table busy");  in mtk_ppe_wait_busy()
77 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe)  in mtk_ppe_cache_clear()  argument
79 	ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);  in mtk_ppe_cache_clear()
80 	ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);  in mtk_ppe_cache_clear()
83 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable)  in mtk_ppe_cache_enable()  argument
85 	mtk_ppe_cache_clear(ppe);  in mtk_ppe_cache_enable()
87 	ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN,  in mtk_ppe_cache_enable()
419 __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)  in __mtk_foe_entry_clear()  argument
425 		rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node,  in __mtk_foe_entry_clear()
430 			__mtk_foe_entry_clear(ppe, entry);  in __mtk_foe_entry_clear()
436 		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash);  in __mtk_foe_entry_clear()
451 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1)  in __mtk_foe_entry_idle_time()  argument
453 	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);  in __mtk_foe_entry_idle_time()
454 	u16 now = mtk_eth_timestamp(ppe->eth);  in __mtk_foe_entry_idle_time()
464 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)  in mtk_flow_entry_update_l2()  argument
466 	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);  in mtk_flow_entry_update_l2()
472 	idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1);  in mtk_flow_entry_update_l2()
477 		hwe = mtk_foe_get_entry(ppe, cur->hash);  in mtk_flow_entry_update_l2()
482 			__mtk_foe_entry_clear(ppe, cur);  in mtk_flow_entry_update_l2()
486 		cur_idle = __mtk_foe_entry_idle_time(ppe, ib1);  in mtk_flow_entry_update_l2()
497 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)  in mtk_flow_entry_update()  argument
505 		mtk_flow_entry_update_l2(ppe, entry);  in mtk_flow_entry_update()
512 	hwe = mtk_foe_get_entry(ppe, entry->hash);  in mtk_flow_entry_update()
513 	memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size);  in mtk_flow_entry_update()
514 	if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) {  in mtk_flow_entry_update()
526 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,  in __mtk_foe_entry_commit()  argument
529 	struct mtk_eth *eth = ppe->eth;  in __mtk_foe_entry_commit()
543 	hwe = mtk_foe_get_entry(ppe, hash);  in __mtk_foe_entry_commit()
550 	mtk_ppe_cache_clear(ppe);  in __mtk_foe_entry_commit()
553 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)  in mtk_foe_entry_clear()  argument
556 	__mtk_foe_entry_clear(ppe, entry);  in mtk_foe_entry_clear()
561 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)  in mtk_foe_entry_commit_l2()  argument
565 	return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node,  in mtk_foe_entry_commit_l2()
569 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)  in mtk_foe_entry_commit()  argument
571 	const struct mtk_soc_data *soc = ppe->eth->soc;  in mtk_foe_entry_commit()
572 	int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1);  in mtk_foe_entry_commit()
576 		return mtk_foe_entry_commit_l2(ppe, entry);  in mtk_foe_entry_commit()
578 	hash = mtk_ppe_hash_entry(ppe->eth, &entry->data);  in mtk_foe_entry_commit()
581 	hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]);  in mtk_foe_entry_commit()
588 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry,  in mtk_foe_entry_commit_subflow()  argument
591 	const struct mtk_soc_data *soc = ppe->eth->soc;  in mtk_foe_entry_commit_subflow()
595 	u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP;  in mtk_foe_entry_commit_subflow()
607 		       &ppe->foe_flow[hash / soc->hash_offset]);  in mtk_foe_entry_commit_subflow()
610 	hwe = mtk_foe_get_entry(ppe, hash);  in mtk_foe_entry_commit_subflow()
615 	l2 = mtk_foe_entry_l2(ppe->eth, &foe);  in mtk_foe_entry_commit_subflow()
618 	type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1);  in mtk_foe_entry_commit_subflow()
624 	*mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2;  in mtk_foe_entry_commit_subflow()
626 	__mtk_foe_entry_commit(ppe, &foe, hash);  in mtk_foe_entry_commit_subflow()
629 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)  in __mtk_ppe_check_skb()  argument
631 	const struct mtk_soc_data *soc = ppe->eth->soc;  in __mtk_ppe_check_skb()
632 	struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset];  in __mtk_ppe_check_skb()
633 	struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash);  in __mtk_ppe_check_skb()
653 			__mtk_foe_entry_clear(ppe, entry);  in __mtk_ppe_check_skb()
657 		if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) {  in __mtk_ppe_check_skb()
664 		__mtk_foe_entry_commit(ppe, &entry->data, hash);  in __mtk_ppe_check_skb()
696 	entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params);  in __mtk_ppe_check_skb()
700 	mtk_foe_entry_commit_subflow(ppe, entry, hash);  in __mtk_ppe_check_skb()
706 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)  in mtk_foe_entry_idle_time()  argument
708 	mtk_flow_entry_update(ppe, entry);  in mtk_foe_entry_idle_time()
710 	return __mtk_foe_entry_idle_time(ppe, entry->data.ib1);  in mtk_foe_entry_idle_time()
718 	struct mtk_ppe *ppe;  in mtk_ppe_init()  local
722 	ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL);  in mtk_ppe_init()
723 	if (!ppe)  in mtk_ppe_init()
726 	rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params);  in mtk_ppe_init()
731 	ppe->base = base;  in mtk_ppe_init()
732 	ppe->eth = eth;  in mtk_ppe_init()
733 	ppe->dev = dev;  in mtk_ppe_init()
734 	ppe->version = version;  in mtk_ppe_init()
736 	foe = dmam_alloc_coherent(ppe->dev,  in mtk_ppe_init()
738 				  &ppe->foe_phys, GFP_KERNEL);  in mtk_ppe_init()
742 	ppe->foe_table = foe;  in mtk_ppe_init()
745 			sizeof(*ppe->foe_flow);  in mtk_ppe_init()
746 	ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL);  in mtk_ppe_init()
747 	if (!ppe->foe_flow)  in mtk_ppe_init()
750 	mtk_ppe_debugfs_init(ppe, index);  in mtk_ppe_init()
752 	return ppe;  in mtk_ppe_init()
755 	rhashtable_destroy(&ppe->l2_flows);  in mtk_ppe_init()
763 	for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) {  in mtk_ppe_deinit()
764 		if (!eth->ppe[i])  in mtk_ppe_deinit()
766 		rhashtable_destroy(&eth->ppe[i]->l2_flows);  in mtk_ppe_deinit()
770 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)  in mtk_ppe_init_foe_table()  argument
775 	memset(ppe->foe_table, 0,  in mtk_ppe_init_foe_table()
776 	       MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size);  in mtk_ppe_init_foe_table()
786 			hwe = mtk_foe_get_entry(ppe, i + skip[k]);  in mtk_ppe_init_foe_table()
792 void mtk_ppe_start(struct mtk_ppe *ppe)  in mtk_ppe_start()  argument
796 	if (!ppe)  in mtk_ppe_start()
799 	mtk_ppe_init_foe_table(ppe);  in mtk_ppe_start()
800 	ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys);  in mtk_ppe_start()
817 	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))  in mtk_ppe_start()
819 	ppe_w32(ppe, MTK_PPE_TB_CFG, val);  in mtk_ppe_start()
821 	ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK,  in mtk_ppe_start()
824 	mtk_ppe_cache_enable(ppe, true);  in mtk_ppe_start()
833 	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))  in mtk_ppe_start()
841 	ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);  in mtk_ppe_start()
845 	ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val);  in mtk_ppe_start()
849 	ppe_w32(ppe, MTK_PPE_BIND_AGE0, val);  in mtk_ppe_start()
853 	ppe_w32(ppe, MTK_PPE_BIND_AGE1, val);  in mtk_ppe_start()
856 	ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val);  in mtk_ppe_start()
860 	ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val);  in mtk_ppe_start()
864 	ppe_w32(ppe, MTK_PPE_BIND_RATE, val);  in mtk_ppe_start()
871 	ppe_w32(ppe, MTK_PPE_GLO_CFG, val);  in mtk_ppe_start()
873 	ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);  in mtk_ppe_start()
875 	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) {  in mtk_ppe_start()
876 		ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777);  in mtk_ppe_start()
877 		ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f);  in mtk_ppe_start()
881 int mtk_ppe_stop(struct mtk_ppe *ppe)  in mtk_ppe_stop()  argument
886 	if (!ppe)  in mtk_ppe_stop()
890 		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i);  in mtk_ppe_stop()
896 	mtk_ppe_cache_enable(ppe, false);  in mtk_ppe_stop()
899 	ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN);  in mtk_ppe_stop()
900 	ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0);  in mtk_ppe_stop()
908 	ppe_clear(ppe, MTK_PPE_TB_CFG, val);  in mtk_ppe_stop()
910 	return mtk_ppe_wait_busy(ppe);  in mtk_ppe_stop()