166 static void qfq_purge_queue(struct qfq_class *cl)  in qfq_purge_queue()  argument
168 	unsigned int len = cl->qdisc->q.qlen;  in qfq_purge_queue()
170 	qdisc_reset(cl->qdisc);  in qfq_purge_queue()
171 	qdisc_tree_decrease_qlen(cl->qdisc, len);  in qfq_purge_queue()
210 	struct qfq_class *cl = (struct qfq_class *)*arg;  in qfq_change_class()  local
236 	delta_w = weight - (cl ? ONE_FP / cl->inv_w : 0);  in qfq_change_class()
252 	if (cl != NULL) {  in qfq_change_class()
254 			err = gen_replace_estimator(&cl->bstats, &cl->rate_est,  in qfq_change_class()
261 		if (inv_w != cl->inv_w) {  in qfq_change_class()
264 			cl->inv_w = inv_w;  in qfq_change_class()
270 	cl = kzalloc(sizeof(struct qfq_class), GFP_KERNEL);  in qfq_change_class()
271 	if (cl == NULL)  in qfq_change_class()
274 	cl->refcnt = 1;  in qfq_change_class()
275 	cl->common.classid = classid;  in qfq_change_class()
276 	cl->lmax = lmax;  in qfq_change_class()
277 	cl->inv_w = inv_w;  in qfq_change_class()
278 	i = qfq_calc_index(cl->inv_w, cl->lmax);  in qfq_change_class()
280 	cl->grp = &q->groups[i];  in qfq_change_class()
282 	cl->qdisc = qdisc_create_dflt(sch->dev_queue,  in qfq_change_class()
284 	if (cl->qdisc == NULL)  in qfq_change_class()
285 		cl->qdisc = &noop_qdisc;  in qfq_change_class()
288 		err = gen_new_estimator(&cl->bstats, &cl->rate_est,  in qfq_change_class()
292 			qdisc_destroy(cl->qdisc);  in qfq_change_class()
293 			kfree(cl);  in qfq_change_class()
300 	qdisc_class_hash_insert(&q->clhash, &cl->common);  in qfq_change_class()
305 	*arg = (unsigned long)cl;  in qfq_change_class()
309 static void qfq_destroy_class(struct Qdisc *sch, struct qfq_class *cl)  in qfq_destroy_class()  argument
313 	if (cl->inv_w) {  in qfq_destroy_class()
314 		q->wsum -= ONE_FP / cl->inv_w;  in qfq_destroy_class()
315 		cl->inv_w = 0;  in qfq_destroy_class()
318 	gen_kill_estimator(&cl->bstats, &cl->rate_est);  in qfq_destroy_class()
319 	qdisc_destroy(cl->qdisc);  in qfq_destroy_class()
320 	kfree(cl);  in qfq_destroy_class()
326 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_delete_class()  local
328 	if (cl->filter_cnt > 0)  in qfq_delete_class()
333 	qfq_purge_queue(cl);  in qfq_delete_class()
334 	qdisc_class_hash_remove(&q->clhash, &cl->common);  in qfq_delete_class()
336 	BUG_ON(--cl->refcnt == 0);  in qfq_delete_class()
348 	struct qfq_class *cl = qfq_find_class(sch, classid);  in qfq_get_class()  local
350 	if (cl != NULL)  in qfq_get_class()
351 		cl->refcnt++;  in qfq_get_class()
353 	return (unsigned long)cl;  in qfq_get_class()
358 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_put_class()  local
360 	if (--cl->refcnt == 0)  in qfq_put_class()
361 		qfq_destroy_class(sch, cl);  in qfq_put_class()
364 static struct tcf_proto **qfq_tcf_chain(struct Qdisc *sch, unsigned long cl)  in qfq_tcf_chain()  argument
368 	if (cl)  in qfq_tcf_chain()
377 	struct qfq_class *cl = qfq_find_class(sch, classid);  in qfq_bind_tcf()  local
379 	if (cl != NULL)  in qfq_bind_tcf()
380 		cl->filter_cnt++;  in qfq_bind_tcf()
382 	return (unsigned long)cl;  in qfq_bind_tcf()
387 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_unbind_tcf()  local
389 	cl->filter_cnt--;  in qfq_unbind_tcf()
395 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_graft_class()  local
399 					&pfifo_qdisc_ops, cl->common.classid);  in qfq_graft_class()
405 	qfq_purge_queue(cl);  in qfq_graft_class()
406 	*old = cl->qdisc;  in qfq_graft_class()
407 	cl->qdisc = new;  in qfq_graft_class()
414 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_class_leaf()  local
416 	return cl->qdisc;  in qfq_class_leaf()
422 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_dump_class()  local
426 	tcm->tcm_handle	= cl->common.classid;  in qfq_dump_class()
427 	tcm->tcm_info	= cl->qdisc->handle;  in qfq_dump_class()
432 	NLA_PUT_U32(skb, TCA_QFQ_WEIGHT, ONE_FP/cl->inv_w);  in qfq_dump_class()
433 	NLA_PUT_U32(skb, TCA_QFQ_LMAX, cl->lmax);  in qfq_dump_class()
444 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_dump_class_stats()  local
448 	cl->qdisc->qstats.qlen = cl->qdisc->q.qlen;  in qfq_dump_class_stats()
450 	xstats.weight = ONE_FP/cl->inv_w;  in qfq_dump_class_stats()
451 	xstats.lmax = cl->lmax;  in qfq_dump_class_stats()
453 	if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||  in qfq_dump_class_stats()
454 	    gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||  in qfq_dump_class_stats()
455 	    gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)  in qfq_dump_class_stats()
464 	struct qfq_class *cl;  in qfq_walk()  local
472 		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode) {  in qfq_walk()
477 			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {  in qfq_walk()
490 	struct qfq_class *cl;  in qfq_classify()  local
496 		cl = qfq_find_class(sch, skb->priority);  in qfq_classify()
497 		if (cl != NULL)  in qfq_classify()
498 			return cl;  in qfq_classify()
513 		cl = (struct qfq_class *)res.class;  in qfq_classify()
514 		if (cl == NULL)  in qfq_classify()
515 			cl = qfq_find_class(sch, res.classid);  in qfq_classify()
516 		return cl;  in qfq_classify()
626 static void qfq_slot_insert(struct qfq_group *grp, struct qfq_class *cl,  in qfq_slot_insert()  argument
632 	hlist_add_head(&cl->next, &grp->slots[i]);  in qfq_slot_insert()
648 	struct qfq_class *cl = qfq_slot_head(grp);  in qfq_front_slot_remove()  local
650 	BUG_ON(!cl);  in qfq_front_slot_remove()
651 	hlist_del(&cl->next);  in qfq_front_slot_remove()
725 static bool qfq_update_class(struct qfq_group *grp, struct qfq_class *cl)  in qfq_update_class()  argument
727 	unsigned int len = qdisc_peek_len(cl->qdisc);  in qfq_update_class()
729 	cl->S = cl->F;  in qfq_update_class()
735 		cl->F = cl->S + (u64)len * cl->inv_w;  in qfq_update_class()
736 		roundedS = qfq_round_down(cl->S, grp->slot_shift);  in qfq_update_class()
741 		qfq_slot_insert(grp, cl, roundedS);  in qfq_update_class()
751 	struct qfq_class *cl;  in qfq_dequeue()  local
761 	cl = qfq_slot_head(grp);  in qfq_dequeue()
762 	skb = qdisc_dequeue_peeked(cl->qdisc);  in qfq_dequeue()
775 		 len, (unsigned long long) cl->F, (unsigned long long) q->V);  in qfq_dequeue()
777 	if (qfq_update_class(grp, cl)) {  in qfq_dequeue()
780 		cl = qfq_slot_scan(grp);  in qfq_dequeue()
781 		if (!cl)  in qfq_dequeue()
784 			u64 roundedS = qfq_round_down(cl->S, grp->slot_shift);  in qfq_dequeue()
818 static void qfq_update_start(struct qfq_sched *q, struct qfq_class *cl)  in qfq_update_start()  argument
822 	int slot_shift = cl->grp->slot_shift;  in qfq_update_start()
824 	roundedF = qfq_round_down(cl->F, slot_shift);  in qfq_update_start()
827 	if (!qfq_gt(cl->F, q->V) || qfq_gt(roundedF, limit)) {  in qfq_update_start()
829 		mask = mask_from(q->bitmaps[ER], cl->grp->index);  in qfq_update_start()
834 					cl->S = next->F;  in qfq_update_start()
836 					cl->S = limit;  in qfq_update_start()
840 		cl->S = q->V;  in qfq_update_start()
842 		cl->S = cl->F;  in qfq_update_start()
849 	struct qfq_class *cl;  in qfq_enqueue()  local
854 	cl = qfq_classify(skb, sch, &err);  in qfq_enqueue()
855 	if (cl == NULL) {  in qfq_enqueue()
861 	pr_debug("qfq_enqueue: cl = %x\n", cl->common.classid);  in qfq_enqueue()
863 	err = qdisc_enqueue(skb, cl->qdisc);  in qfq_enqueue()
867 			cl->qstats.drops++;  in qfq_enqueue()
873 	bstats_update(&cl->bstats, skb);  in qfq_enqueue()
877 	if (cl->qdisc->q.qlen != 1)  in qfq_enqueue()
881 	grp = cl->grp;  in qfq_enqueue()
882 	qfq_update_start(q, cl);  in qfq_enqueue()
885 	cl->F = cl->S + (u64)qdisc_pkt_len(skb) * cl->inv_w;  in qfq_enqueue()
886 	roundedS = qfq_round_down(cl->S, grp->slot_shift);  in qfq_enqueue()
898 		if (!qfq_gt(grp->S, cl->S))  in qfq_enqueue()
916 		 (unsigned long long) cl->S,  in qfq_enqueue()
917 		 (unsigned long long) cl->F,  in qfq_enqueue()
921 	qfq_slot_insert(grp, cl, roundedS);  in qfq_enqueue()
928 			    struct qfq_class *cl)  in qfq_slot_remove()  argument
933 	roundedS = qfq_round_down(cl->S, grp->slot_shift);  in qfq_slot_remove()
937 	hlist_del(&cl->next);  in qfq_slot_remove()
949 static void qfq_deactivate_class(struct qfq_sched *q, struct qfq_class *cl)  in qfq_deactivate_class()  argument
951 	struct qfq_group *grp = cl->grp;  in qfq_deactivate_class()
956 	cl->F = cl->S;  in qfq_deactivate_class()
957 	qfq_slot_remove(q, grp, cl);  in qfq_deactivate_class()
976 		cl = qfq_slot_scan(grp);  in qfq_deactivate_class()
977 		roundedS = qfq_round_down(cl->S, grp->slot_shift);  in qfq_deactivate_class()
996 	struct qfq_class *cl = (struct qfq_class *)arg;  in qfq_qlen_notify()  local
998 	if (cl->qdisc->q.qlen == 0)  in qfq_qlen_notify()
999 		qfq_deactivate_class(q, cl);  in qfq_qlen_notify()
1011 			struct qfq_class *cl;  in qfq_drop()  local
1014 			hlist_for_each_entry(cl, n, &grp->slots[j], next) {  in qfq_drop()
1016 				if (!cl->qdisc->ops->drop)  in qfq_drop()
1019 				len = cl->qdisc->ops->drop(cl->qdisc);  in qfq_drop()
1022 					if (!cl->qdisc->q.qlen)  in qfq_drop()
1023 						qfq_deactivate_class(q, cl);  in qfq_drop()
1060 	struct qfq_class *cl;  in qfq_reset_qdisc()  local
1067 			hlist_for_each_entry_safe(cl, n, tmp,  in qfq_reset_qdisc()
1069 				qfq_deactivate_class(q, cl);  in qfq_reset_qdisc()
1075 		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode)  in qfq_reset_qdisc()
1076 			qdisc_reset(cl->qdisc);  in qfq_reset_qdisc()
1084 	struct qfq_class *cl;  in qfq_destroy_qdisc()  local
1091 		hlist_for_each_entry_safe(cl, n, next, &q->clhash.hash[i],  in qfq_destroy_qdisc()
1093 			qfq_destroy_class(sch, cl);  in qfq_destroy_qdisc()