138 	struct qfq_aggregate *agg;	/* Parent aggregate. */  member
257 static void qfq_init_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_init_agg()  argument
260 	INIT_LIST_HEAD(&agg->active);  in qfq_init_agg()
261 	hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs);  in qfq_init_agg()
263 	agg->lmax = lmax;  in qfq_init_agg()
264 	agg->class_weight = weight;  in qfq_init_agg()
270 	struct qfq_aggregate *agg;  in qfq_find_agg()  local
272 	hlist_for_each_entry(agg, &q->nonfull_aggs, nonfull_next)  in qfq_find_agg()
273 		if (agg->lmax == lmax && agg->class_weight == weight)  in qfq_find_agg()
274 			return agg;  in qfq_find_agg()
281 static void qfq_update_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_update_agg()  argument
287 		hlist_del_init(&agg->nonfull_next);  in qfq_update_agg()
289 	if (agg->num_classes > new_num_classes &&  in qfq_update_agg()
291 		hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs);  in qfq_update_agg()
297 	agg->budgetmax = new_num_classes * agg->lmax;  in qfq_update_agg()
298 	new_agg_weight = agg->class_weight * new_num_classes;  in qfq_update_agg()
299 	agg->inv_w = ONE_FP/new_agg_weight;  in qfq_update_agg()
301 	if (agg->grp == NULL) {  in qfq_update_agg()
302 		int i = qfq_calc_index(agg->inv_w, agg->budgetmax,  in qfq_update_agg()
304 		agg->grp = &q->groups[i];  in qfq_update_agg()
308 		(int) agg->class_weight * (new_num_classes - agg->num_classes);  in qfq_update_agg()
311 	agg->num_classes = new_num_classes;  in qfq_update_agg()
316 			   struct qfq_aggregate *agg,  in qfq_add_to_agg()  argument
319 	cl->agg = agg;  in qfq_add_to_agg()
321 	qfq_update_agg(q, agg, agg->num_classes+1);  in qfq_add_to_agg()
323 		list_add_tail(&cl->alist, &agg->active);  in qfq_add_to_agg()
324 		if (list_first_entry(&agg->active, struct qfq_class, alist) ==  in qfq_add_to_agg()
325 		    cl && q->in_serv_agg != agg) /* agg was inactive */  in qfq_add_to_agg()
326 			qfq_activate_agg(q, agg, enqueue); /* schedule agg */  in qfq_add_to_agg()
332 static void qfq_destroy_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_destroy_agg()  argument
334 	hlist_del_init(&agg->nonfull_next);  in qfq_destroy_agg()
335 	q->wsum -= agg->class_weight;  in qfq_destroy_agg()
339 	if (q->in_serv_agg == agg)  in qfq_destroy_agg()
341 	kfree(agg);  in qfq_destroy_agg()
347 	struct qfq_aggregate *agg = cl->agg;  in qfq_deactivate_class()  local
351 	if (list_empty(&agg->active)) /* agg is now inactive */  in qfq_deactivate_class()
352 		qfq_deactivate_agg(q, agg);  in qfq_deactivate_class()
358 	struct qfq_aggregate *agg = cl->agg;  in qfq_rm_from_agg()  local
360 	cl->agg = NULL;  in qfq_rm_from_agg()
361 	if (agg->num_classes == 1) { /* agg being emptied, destroy it */  in qfq_rm_from_agg()
362 		qfq_destroy_agg(q, agg);  in qfq_rm_from_agg()
365 	qfq_update_agg(q, agg, agg->num_classes-1);  in qfq_rm_from_agg()
445 	    lmax == cl->agg->lmax &&  in qfq_change_class()
446 	    weight == cl->agg->class_weight)  in qfq_change_class()
449 	delta_w = weight - (cl ? cl->agg->class_weight : 0);  in qfq_change_class()
633 	if (nla_put_u32(skb, TCA_QFQ_WEIGHT, cl->agg->class_weight) ||  in qfq_dump_class()
634 	    nla_put_u32(skb, TCA_QFQ_LMAX, cl->agg->lmax))  in qfq_dump_class()
651 	xstats.weight = cl->agg->class_weight;  in qfq_dump_class_stats()
652 	xstats.lmax = cl->agg->lmax;  in qfq_dump_class_stats()
879 static void qfq_slot_insert(struct qfq_group *grp, struct qfq_aggregate *agg,  in qfq_slot_insert()  argument
888 		agg->S -= deltaS;  in qfq_slot_insert()
889 		agg->F -= deltaS;  in qfq_slot_insert()
895 	hlist_add_head(&agg->next, &grp->slots[i]);  in qfq_slot_insert()
911 	struct qfq_aggregate *agg = qfq_slot_head(grp);  in qfq_front_slot_remove()  local
913 	BUG_ON(!agg);  in qfq_front_slot_remove()
914 	hlist_del(&agg->next);  in qfq_front_slot_remove()
977 static struct sk_buff *agg_dequeue(struct qfq_aggregate *agg,  in agg_dequeue()  argument
990 		cl->deficit += agg->lmax;  in agg_dequeue()
991 		list_move_tail(&cl->alist, &agg->active);  in agg_dequeue()
997 static inline struct sk_buff *qfq_peek_skb(struct qfq_aggregate *agg,  in qfq_peek_skb()  argument
1003 	*cl = list_first_entry(&agg->active, struct qfq_class, alist);  in qfq_peek_skb()
1014 static inline void charge_actual_service(struct qfq_aggregate *agg)  in charge_actual_service()  argument
1021 	u32 service_received = min(agg->budgetmax,  in charge_actual_service()
1022 				   agg->initial_budget - agg->budget);  in charge_actual_service()
1024 	agg->F = agg->S + (u64)service_received * agg->inv_w;  in charge_actual_service()
1039 static void qfq_update_start(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_update_start()  argument
1043 	int slot_shift = agg->grp->slot_shift;  in qfq_update_start()
1045 	roundedF = qfq_round_down(agg->F, slot_shift);  in qfq_update_start()
1048 	if (!qfq_gt(agg->F, q->V) || qfq_gt(roundedF, limit)) {  in qfq_update_start()
1050 		mask = mask_from(q->bitmaps[ER], agg->grp->index);  in qfq_update_start()
1055 					agg->S = next->F;  in qfq_update_start()
1057 					agg->S = limit;  in qfq_update_start()
1061 		agg->S = q->V;  in qfq_update_start()
1063 		agg->S = agg->F;  in qfq_update_start()
1073 		    struct qfq_aggregate *agg, enum update_reason reason)  in qfq_update_agg_ts()  argument
1076 		qfq_update_start(q, agg);  in qfq_update_agg_ts()
1078 		agg->S = agg->F;  in qfq_update_agg_ts()
1080 	agg->F = agg->S + (u64)agg->budgetmax * agg->inv_w;  in qfq_update_agg_ts()
1083 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg);
1172 	struct qfq_aggregate *agg, *new_front_agg;  in qfq_choose_next_agg()  local
1184 	agg = qfq_slot_head(grp);  in qfq_choose_next_agg()
1199 			return agg;  in qfq_choose_next_agg()
1209 	return agg;  in qfq_choose_next_agg()
1218 	struct qfq_aggregate *agg;  in qfq_enqueue()  local
1231 	if (unlikely(cl->agg->lmax < len)) {  in qfq_enqueue()
1233 			 cl->agg->lmax, len, cl->common.classid);  in qfq_enqueue()
1234 		err = qfq_change_agg(sch, cl, cl->agg->class_weight, len);  in qfq_enqueue()
1257 	agg = cl->agg;  in qfq_enqueue()
1261 		    list_first_entry(&agg->active, struct qfq_class, alist)  in qfq_enqueue()
1263 			list_move_tail(&cl->alist, &agg->active);  in qfq_enqueue()
1269 	cl->deficit = agg->lmax;  in qfq_enqueue()
1270 	list_add_tail(&cl->alist, &agg->active);  in qfq_enqueue()
1272 	if (list_first_entry(&agg->active, struct qfq_class, alist) != cl ||  in qfq_enqueue()
1273 	    q->in_serv_agg == agg)  in qfq_enqueue()
1276 	qfq_activate_agg(q, agg, enqueue);  in qfq_enqueue()
1284 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_schedule_agg()  argument
1286 	struct qfq_group *grp = agg->grp;  in qfq_schedule_agg()
1290 	roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_schedule_agg()
1302 		if (!qfq_gt(grp->S, agg->S))  in qfq_schedule_agg()
1321 		 (unsigned long long) agg->S,  in qfq_schedule_agg()
1322 		 (unsigned long long) agg->F,  in qfq_schedule_agg()
1326 	qfq_slot_insert(grp, agg, roundedS);  in qfq_schedule_agg()
1331 static void qfq_activate_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_activate_agg()  argument
1334 	agg->initial_budget = agg->budget = agg->budgetmax; /* recharge budg. */  in qfq_activate_agg()
1336 	qfq_update_agg_ts(q, agg, reason);  in qfq_activate_agg()
1338 		q->in_serv_agg = agg; /* start serving this aggregate */  in qfq_activate_agg()
1340 		q->oldV = q->V = agg->S;  in qfq_activate_agg()
1341 	} else if (agg != q->in_serv_agg)  in qfq_activate_agg()
1342 		qfq_schedule_agg(q, agg);  in qfq_activate_agg()
1346 			    struct qfq_aggregate *agg)  in qfq_slot_remove()  argument
1351 	roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_slot_remove()
1356 	hlist_del(&agg->next);  in qfq_slot_remove()
1368 static void qfq_deactivate_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_deactivate_agg()  argument
1370 	struct qfq_group *grp = agg->grp;  in qfq_deactivate_agg()
1375 	if (agg == q->in_serv_agg) {  in qfq_deactivate_agg()
1376 		charge_actual_service(agg);  in qfq_deactivate_agg()
1381 	agg->F = agg->S;  in qfq_deactivate_agg()
1382 	qfq_slot_remove(q, grp, agg);  in qfq_deactivate_agg()
1401 		agg = qfq_slot_scan(grp);  in qfq_deactivate_agg()
1402 		roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_deactivate_agg()