16 __fq_adjust_removal(struct fq *fq, struct fq_flow *flow, unsigned int packets,  in __fq_adjust_removal()  argument
19 	struct fq_tin *tin = flow->tin;  in __fq_adjust_removal()
24 	flow->backlog -= bytes;  in __fq_adjust_removal()
28 	if (flow->backlog)  in __fq_adjust_removal()
31 	if (flow == &tin->default_flow) {  in __fq_adjust_removal()
36 	idx = flow - fq->flows;  in __fq_adjust_removal()
41 			      struct fq_flow *flow,  in fq_adjust_removal()  argument
44 	__fq_adjust_removal(fq, flow, 1, skb->len, skb->truesize);  in fq_adjust_removal()
48 				       struct fq_flow *flow)  in fq_flow_dequeue()  argument
54 	skb = __skb_dequeue(&flow->queue);  in fq_flow_dequeue()
58 	fq_adjust_removal(fq, flow, skb);  in fq_flow_dequeue()
63 static int fq_flow_drop(struct fq *fq, struct fq_flow *flow,  in fq_flow_drop()  argument
67 	struct fq_tin *tin = flow->tin;  in fq_flow_drop()
73 	pending = min_t(int, 32, skb_queue_len(&flow->queue) / 2);  in fq_flow_drop()
75 		skb = __skb_dequeue(&flow->queue);  in fq_flow_drop()
82 		free_func(fq, tin, flow, skb);  in fq_flow_drop()
85 	__fq_adjust_removal(fq, flow, packets, bytes, truesize);  in fq_flow_drop()
94 	struct fq_flow *flow;  in fq_tin_dequeue()  local
108 	flow = list_first_entry(head, struct fq_flow, flowchain);  in fq_tin_dequeue()
110 	if (flow->deficit <= 0) {  in fq_tin_dequeue()
111 		flow->deficit += fq->quantum;  in fq_tin_dequeue()
112 		list_move_tail(&flow->flowchain,  in fq_tin_dequeue()
117 	skb = dequeue_func(fq, tin, flow);  in fq_tin_dequeue()
122 			list_move_tail(&flow->flowchain, &tin->old_flows);  in fq_tin_dequeue()
124 			list_del_init(&flow->flowchain);  in fq_tin_dequeue()
125 			flow->tin = NULL;  in fq_tin_dequeue()
130 	flow->deficit -= skb->len;  in fq_tin_dequeue()
148 	struct fq_flow *flow;  in fq_flow_classify()  local
152 	flow = &fq->flows[idx];  in fq_flow_classify()
153 	if (flow->tin && flow->tin != tin) {  in fq_flow_classify()
154 		flow = &tin->default_flow;  in fq_flow_classify()
159 	if (!flow->tin)  in fq_flow_classify()
162 	return flow;  in fq_flow_classify()
168 	struct fq_flow *flow = NULL;  in fq_find_fattest_flow()  local
180 		flow = cur;  in fq_find_fattest_flow()
190 		flow = &tin->default_flow;  in fq_find_fattest_flow()
194 	return flow;  in fq_find_fattest_flow()
202 	struct fq_flow *flow;  in fq_tin_enqueue()  local
207 	flow = fq_flow_classify(fq, tin, idx, skb);  in fq_tin_enqueue()
209 	if (!flow->backlog) {  in fq_tin_enqueue()
210 		if (flow != &tin->default_flow)  in fq_tin_enqueue()
216 	flow->tin = tin;  in fq_tin_enqueue()
217 	flow->backlog += skb->len;  in fq_tin_enqueue()
223 	if (list_empty(&flow->flowchain)) {  in fq_tin_enqueue()
224 		flow->deficit = fq->quantum;  in fq_tin_enqueue()
225 		list_add_tail(&flow->flowchain,  in fq_tin_enqueue()
229 	__skb_queue_tail(&flow->queue, skb);  in fq_tin_enqueue()
232 		flow = fq_find_fattest_flow(fq);  in fq_tin_enqueue()
233 		if (!flow)  in fq_tin_enqueue()
236 		if (!fq_flow_drop(fq, flow, free_func))  in fq_tin_enqueue()
239 		flow->tin->overlimit++;  in fq_tin_enqueue()
249 			   struct fq_flow *flow,  in fq_flow_filter()  argument
254 	struct fq_tin *tin = flow->tin;  in fq_flow_filter()
259 	skb_queue_walk_safe(&flow->queue, skb, tmp) {  in fq_flow_filter()
260 		if (!filter_func(fq, tin, flow, skb, filter_data))  in fq_flow_filter()
263 		__skb_unlink(skb, &flow->queue);  in fq_flow_filter()
264 		fq_adjust_removal(fq, flow, skb);  in fq_flow_filter()
265 		free_func(fq, tin, flow, skb);  in fq_flow_filter()
275 	struct fq_flow *flow;  in fq_tin_filter()  local
279 	list_for_each_entry(flow, &tin->new_flows, flowchain)  in fq_tin_filter()
280 		fq_flow_filter(fq, flow, filter_func, filter_data, free_func);  in fq_tin_filter()
281 	list_for_each_entry(flow, &tin->old_flows, flowchain)  in fq_tin_filter()
282 		fq_flow_filter(fq, flow, filter_func, filter_data, free_func);  in fq_tin_filter()
286 			  struct fq_flow *flow,  in fq_flow_reset()  argument
289 	struct fq_tin *tin = flow->tin;  in fq_flow_reset()
292 	while ((skb = fq_flow_dequeue(fq, flow)))  in fq_flow_reset()
293 		free_func(fq, tin, flow, skb);  in fq_flow_reset()
295 	if (!list_empty(&flow->flowchain)) {  in fq_flow_reset()
296 		list_del_init(&flow->flowchain);  in fq_flow_reset()
302 	flow->tin = NULL;  in fq_flow_reset()
304 	WARN_ON_ONCE(flow->backlog);  in fq_flow_reset()
312 	struct fq_flow *flow;  in fq_tin_reset()  local
322 		flow = list_first_entry(head, struct fq_flow, flowchain);  in fq_tin_reset()
323 		fq_flow_reset(fq, flow, free_func);  in fq_tin_reset()
331 static void fq_flow_init(struct fq_flow *flow)  in fq_flow_init()  argument
333 	INIT_LIST_HEAD(&flow->flowchain);  in fq_flow_init()
334 	__skb_queue_head_init(&flow->queue);  in fq_flow_init()