73 static unsigned int fq_pie_hash(const struct fq_pie_sched_data *q,  in fq_pie_hash()  argument
76 	return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);  in fq_pie_hash()
82 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_classify()  local
89 	    TC_H_MIN(skb->priority) <= q->flows_cnt)  in fq_pie_classify()
92 	filter = rcu_dereference_bh(q->filter_list);  in fq_pie_classify()
94 		return fq_pie_hash(q, skb) + 1;  in fq_pie_classify()
110 		if (TC_H_MIN(res.classid) <= q->flows_cnt)  in fq_pie_classify()
131 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_enqueue()  local
149 	sel_flow = &q->flows[idx];  in fq_pie_qdisc_enqueue()
152 	memory_limited = q->memory_usage > q->memory_limit + skb->truesize;  in fq_pie_qdisc_enqueue()
156 		q->stats.overlimit++;  in fq_pie_qdisc_enqueue()
159 		q->overmemory++;  in fq_pie_qdisc_enqueue()
162 	if (!pie_drop_early(sch, &q->p_params, &sel_flow->vars,  in fq_pie_qdisc_enqueue()
165 	} else if (q->p_params.ecn &&  in fq_pie_qdisc_enqueue()
166 		   sel_flow->vars.prob <= (MAX_PROB / 100) * q->ecn_prob &&  in fq_pie_qdisc_enqueue()
171 		q->stats.ecn_mark++;  in fq_pie_qdisc_enqueue()
176 		if (!q->p_params.dq_rate_estimator)  in fq_pie_qdisc_enqueue()
180 		q->stats.packets_in++;  in fq_pie_qdisc_enqueue()
181 		q->memory_usage += skb->truesize;  in fq_pie_qdisc_enqueue()
183 		sch->q.qlen++;  in fq_pie_qdisc_enqueue()
186 			list_add_tail(&sel_flow->flowchain, &q->new_flows);  in fq_pie_qdisc_enqueue()
187 			q->new_flow_count++;  in fq_pie_qdisc_enqueue()
188 			sel_flow->deficit = q->quantum;  in fq_pie_qdisc_enqueue()
197 	q->stats.dropped++;  in fq_pie_qdisc_enqueue()
230 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_dequeue()  local
237 	head = &q->new_flows;  in fq_pie_qdisc_dequeue()
239 		head = &q->old_flows;  in fq_pie_qdisc_dequeue()
247 		flow->deficit += q->quantum;  in fq_pie_qdisc_dequeue()
248 		list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
256 		sch->q.qlen--;  in fq_pie_qdisc_dequeue()
262 		if (head == &q->new_flows && !list_empty(&q->old_flows))  in fq_pie_qdisc_dequeue()
263 			list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
272 	q->memory_usage -= get_pie_cb(skb)->mem_usage;  in fq_pie_qdisc_dequeue()
273 	pie_process_dequeue(skb, &q->p_params, &flow->vars, flow->backlog);  in fq_pie_qdisc_dequeue()
280 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_change()  local
294 		q->p_params.limit = limit;  in fq_pie_change()
298 		if (q->flows) {  in fq_pie_change()
303 		q->flows_cnt = nla_get_u32(tb[TCA_FQ_PIE_FLOWS]);  in fq_pie_change()
304 		if (!q->flows_cnt || q->flows_cnt > 65536) {  in fq_pie_change()
317 		q->p_params.target =  in fq_pie_change()
323 		q->p_params.tupdate =  in fq_pie_change()
327 		q->p_params.alpha = nla_get_u32(tb[TCA_FQ_PIE_ALPHA]);  in fq_pie_change()
330 		q->p_params.beta = nla_get_u32(tb[TCA_FQ_PIE_BETA]);  in fq_pie_change()
333 		q->quantum = nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]);  in fq_pie_change()
336 		q->memory_limit = nla_get_u32(tb[TCA_FQ_PIE_MEMORY_LIMIT]);  in fq_pie_change()
339 		q->ecn_prob = nla_get_u32(tb[TCA_FQ_PIE_ECN_PROB]);  in fq_pie_change()
342 		q->p_params.ecn = nla_get_u32(tb[TCA_FQ_PIE_ECN]);  in fq_pie_change()
345 		q->p_params.bytemode = nla_get_u32(tb[TCA_FQ_PIE_BYTEMODE]);  in fq_pie_change()
348 		q->p_params.dq_rate_estimator =  in fq_pie_change()
352 	while (sch->q.qlen > sch->limit) {  in fq_pie_change()
371 	struct fq_pie_sched_data *q = from_timer(q, t, adapt_timer);  in fq_pie_timer()  local
372 	struct Qdisc *sch = q->sch;  in fq_pie_timer()
379 	for (idx = 0; idx < q->flows_cnt; idx++)  in fq_pie_timer()
380 		pie_calculate_probability(&q->p_params, &q->flows[idx].vars,  in fq_pie_timer()
381 					  q->flows[idx].backlog);  in fq_pie_timer()
384 	if (q->p_params.tupdate)  in fq_pie_timer()
385 		mod_timer(&q->adapt_timer, jiffies + q->p_params.tupdate);  in fq_pie_timer()
393 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_init()  local
397 	pie_params_init(&q->p_params);  in fq_pie_init()
399 	q->p_params.limit = sch->limit;  in fq_pie_init()
400 	q->quantum = psched_mtu(qdisc_dev(sch));  in fq_pie_init()
401 	q->sch = sch;  in fq_pie_init()
402 	q->ecn_prob = 10;  in fq_pie_init()
403 	q->flows_cnt = 1024;  in fq_pie_init()
404 	q->memory_limit = SZ_32M;  in fq_pie_init()
406 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_init()
407 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_init()
408 	timer_setup(&q->adapt_timer, fq_pie_timer, 0);  in fq_pie_init()
417 	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);  in fq_pie_init()
421 	q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_pie_flow),  in fq_pie_init()
423 	if (!q->flows) {  in fq_pie_init()
427 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_init()
428 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_init()
434 	mod_timer(&q->adapt_timer, jiffies + HZ / 2);  in fq_pie_init()
439 	q->flows_cnt = 0;  in fq_pie_init()
446 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump()  local
455 	    nla_put_u32(skb, TCA_FQ_PIE_FLOWS, q->flows_cnt) ||  in fq_pie_dump()
457 			((u32)PSCHED_TICKS2NS(q->p_params.target)) /  in fq_pie_dump()
460 			jiffies_to_usecs(q->p_params.tupdate)) ||  in fq_pie_dump()
461 	    nla_put_u32(skb, TCA_FQ_PIE_ALPHA, q->p_params.alpha) ||  in fq_pie_dump()
462 	    nla_put_u32(skb, TCA_FQ_PIE_BETA, q->p_params.beta) ||  in fq_pie_dump()
463 	    nla_put_u32(skb, TCA_FQ_PIE_QUANTUM, q->quantum) ||  in fq_pie_dump()
464 	    nla_put_u32(skb, TCA_FQ_PIE_MEMORY_LIMIT, q->memory_limit) ||  in fq_pie_dump()
465 	    nla_put_u32(skb, TCA_FQ_PIE_ECN_PROB, q->ecn_prob) ||  in fq_pie_dump()
466 	    nla_put_u32(skb, TCA_FQ_PIE_ECN, q->p_params.ecn) ||  in fq_pie_dump()
467 	    nla_put_u32(skb, TCA_FQ_PIE_BYTEMODE, q->p_params.bytemode) ||  in fq_pie_dump()
469 			q->p_params.dq_rate_estimator))  in fq_pie_dump()
481 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump_stats()  local
483 		.packets_in	= q->stats.packets_in,  in fq_pie_dump_stats()
484 		.overlimit	= q->stats.overlimit,  in fq_pie_dump_stats()
485 		.overmemory	= q->overmemory,  in fq_pie_dump_stats()
486 		.dropped	= q->stats.dropped,  in fq_pie_dump_stats()
487 		.ecn_mark	= q->stats.ecn_mark,  in fq_pie_dump_stats()
488 		.new_flow_count = q->new_flow_count,  in fq_pie_dump_stats()
489 		.memory_usage   = q->memory_usage,  in fq_pie_dump_stats()
494 	list_for_each(pos, &q->new_flows)  in fq_pie_dump_stats()
497 	list_for_each(pos, &q->old_flows)  in fq_pie_dump_stats()
506 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_reset()  local
509 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_reset()
510 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_reset()
511 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_reset()
512 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_reset()
525 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_destroy()  local
527 	tcf_block_put(q->block);  in fq_pie_destroy()
528 	q->p_params.tupdate = 0;  in fq_pie_destroy()
529 	del_timer_sync(&q->adapt_timer);  in fq_pie_destroy()
530 	kvfree(q->flows);  in fq_pie_destroy()