628 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in noop_enqueue()  argument
635 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)  in noop_dequeue()  argument
650 	RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
677 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,  in noqueue_init()  argument
683 	qdisc->enqueue = NULL;  in noqueue_init()
721 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in pfifo_fast_enqueue()  argument
725 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_enqueue()
733 		if (qdisc_is_percpu_stats(qdisc))  in pfifo_fast_enqueue()
734 			return qdisc_drop_cpu(skb, qdisc, to_free);  in pfifo_fast_enqueue()
736 			return qdisc_drop(skb, qdisc, to_free);  in pfifo_fast_enqueue()
739 	qdisc_update_stats_at_enqueue(qdisc, pkt_len);  in pfifo_fast_enqueue()
743 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)  in pfifo_fast_dequeue()  argument
745 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_dequeue()
760 		qdisc_update_stats_at_dequeue(qdisc, skb);  in pfifo_fast_dequeue()
762 		   READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) {  in pfifo_fast_dequeue()
768 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);  in pfifo_fast_dequeue()
769 		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);  in pfifo_fast_dequeue()
784 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)  in pfifo_fast_peek()  argument
786 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_peek()
799 static void pfifo_fast_reset(struct Qdisc *qdisc)  in pfifo_fast_reset()  argument
802 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_reset()
818 	if (qdisc_is_percpu_stats(qdisc)) {  in pfifo_fast_reset()
822 			q = per_cpu_ptr(qdisc->cpu_qstats, i);  in pfifo_fast_reset()
829 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)  in pfifo_fast_dump()  argument
842 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,  in pfifo_fast_init()  argument
845 	unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;  in pfifo_fast_init()
846 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_init()
863 	qdisc->flags |= TCQ_F_CAN_BYPASS;  in pfifo_fast_init()
1015 void qdisc_reset(struct Qdisc *qdisc)  in qdisc_reset()  argument
1017 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_reset()
1019 	trace_qdisc_reset(qdisc);  in qdisc_reset()
1022 		ops->reset(qdisc);  in qdisc_reset()
1024 	__skb_queue_purge(&qdisc->gso_skb);  in qdisc_reset()
1025 	__skb_queue_purge(&qdisc->skb_bad_txq);  in qdisc_reset()
1027 	qdisc->q.qlen = 0;  in qdisc_reset()
1028 	qdisc->qstats.backlog = 0;  in qdisc_reset()
1032 void qdisc_free(struct Qdisc *qdisc)  in qdisc_free()  argument
1034 	if (qdisc_is_percpu_stats(qdisc)) {  in qdisc_free()
1035 		free_percpu(qdisc->cpu_bstats);  in qdisc_free()
1036 		free_percpu(qdisc->cpu_qstats);  in qdisc_free()
1039 	kfree(qdisc);  in qdisc_free()
1049 static void __qdisc_destroy(struct Qdisc *qdisc)  in __qdisc_destroy()  argument
1051 	const struct Qdisc_ops  *ops = qdisc->ops;  in __qdisc_destroy()
1054 	qdisc_hash_del(qdisc);  in __qdisc_destroy()
1056 	qdisc_put_stab(rtnl_dereference(qdisc->stab));  in __qdisc_destroy()
1058 	gen_kill_estimator(&qdisc->rate_est);  in __qdisc_destroy()
1060 	qdisc_reset(qdisc);  in __qdisc_destroy()
1063 		ops->destroy(qdisc);  in __qdisc_destroy()
1066 	netdev_put(qdisc_dev(qdisc), &qdisc->dev_tracker);  in __qdisc_destroy()
1068 	trace_qdisc_destroy(qdisc);  in __qdisc_destroy()
1070 	call_rcu(&qdisc->rcu, qdisc_free_cb);  in __qdisc_destroy()
1073 void qdisc_destroy(struct Qdisc *qdisc)  in qdisc_destroy()  argument
1075 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_destroy()
1078 	__qdisc_destroy(qdisc);  in qdisc_destroy()
1081 void qdisc_put(struct Qdisc *qdisc)  in qdisc_put()  argument
1083 	if (!qdisc)  in qdisc_put()
1086 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put()
1087 	    !refcount_dec_and_test(&qdisc->refcnt))  in qdisc_put()
1090 	__qdisc_destroy(qdisc);  in qdisc_put()
1099 void qdisc_put_unlocked(struct Qdisc *qdisc)  in qdisc_put_unlocked()  argument
1101 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put_unlocked()
1102 	    !refcount_dec_and_rtnl_lock(&qdisc->refcnt))  in qdisc_put_unlocked()
1105 	__qdisc_destroy(qdisc);  in qdisc_put_unlocked()
1112 			      struct Qdisc *qdisc)  in dev_graft_qdisc()  argument
1121 	if (qdisc == NULL)  in dev_graft_qdisc()
1122 		qdisc = &noop_qdisc;  in dev_graft_qdisc()
1123 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);  in dev_graft_qdisc()
1124 	rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);  in dev_graft_qdisc()
1136 	struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in shutdown_scheduler_queue()  local
1139 	if (qdisc) {  in shutdown_scheduler_queue()
1140 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in shutdown_scheduler_queue()
1143 		qdisc_put(qdisc);  in shutdown_scheduler_queue()
1151 	struct Qdisc *qdisc;  in attach_one_default_qdisc()  local
1159 	qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);  in attach_one_default_qdisc()
1160 	if (!qdisc)  in attach_one_default_qdisc()
1164 		qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;  in attach_one_default_qdisc()
1165 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);  in attach_one_default_qdisc()
1171 	struct Qdisc *qdisc;  in attach_default_qdiscs()  local
1178 		qdisc = rtnl_dereference(txq->qdisc_sleeping);  in attach_default_qdiscs()
1179 		rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1180 		qdisc_refcount_inc(qdisc);  in attach_default_qdiscs()
1182 		qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);  in attach_default_qdiscs()
1183 		if (qdisc) {  in attach_default_qdiscs()
1184 			rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1185 			qdisc->ops->attach(qdisc);  in attach_default_qdiscs()
1188 	qdisc = rtnl_dereference(dev->qdisc);  in attach_default_qdiscs()
1191 	if (qdisc == &noop_qdisc) {  in attach_default_qdiscs()
1197 		qdisc = rtnl_dereference(txq->qdisc_sleeping);  in attach_default_qdiscs()
1198 		rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1199 		qdisc_refcount_inc(qdisc);  in attach_default_qdiscs()
1204 	if (qdisc != &noop_qdisc)  in attach_default_qdiscs()
1205 		qdisc_hash_add(qdisc, false);  in attach_default_qdiscs()
1219 	rcu_assign_pointer(dev_queue->qdisc, new_qdisc);  in transition_one_qdisc()
1235 	if (rtnl_dereference(dev->qdisc) == &noop_qdisc)  in dev_activate()
1254 static void qdisc_deactivate(struct Qdisc *qdisc)  in qdisc_deactivate()  argument
1256 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_deactivate()
1259 	set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);  in qdisc_deactivate()
1267 	struct Qdisc *qdisc;  in dev_deactivate_queue()  local
1269 	qdisc = rtnl_dereference(dev_queue->qdisc);  in dev_deactivate_queue()
1270 	if (qdisc) {  in dev_deactivate_queue()
1271 		qdisc_deactivate(qdisc);  in dev_deactivate_queue()
1272 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in dev_deactivate_queue()
1280 	struct Qdisc *qdisc;  in dev_reset_queue()  local
1283 	qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in dev_reset_queue()
1284 	if (!qdisc)  in dev_reset_queue()
1287 	nolock = qdisc->flags & TCQ_F_NOLOCK;  in dev_reset_queue()
1290 		spin_lock_bh(&qdisc->seqlock);  in dev_reset_queue()
1291 	spin_lock_bh(qdisc_lock(qdisc));  in dev_reset_queue()
1293 	qdisc_reset(qdisc);  in dev_reset_queue()
1295 	spin_unlock_bh(qdisc_lock(qdisc));  in dev_reset_queue()
1297 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);  in dev_reset_queue()
1298 		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);  in dev_reset_queue()
1299 		spin_unlock_bh(&qdisc->seqlock);  in dev_reset_queue()
1390 	struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in qdisc_change_tx_queue_len()  local
1391 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_change_tx_queue_len()
1394 		return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);  in qdisc_change_tx_queue_len()
1401 	struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);  in dev_qdisc_change_real_num_tx()  local
1403 	if (qdisc->ops->change_real_num_tx)  in dev_qdisc_change_real_num_tx()
1404 		qdisc->ops->change_real_num_tx(qdisc, new_real_tx);  in dev_qdisc_change_real_num_tx()
1411 	struct Qdisc *qdisc;  in mq_change_real_num_tx()  local
1415 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);  in mq_change_real_num_tx()
1419 		if (qdisc != &noop_qdisc && !qdisc->handle)  in mq_change_real_num_tx()
1420 			qdisc_hash_del(qdisc);  in mq_change_real_num_tx()
1423 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);  in mq_change_real_num_tx()
1424 		if (qdisc != &noop_qdisc && !qdisc->handle)  in mq_change_real_num_tx()
1425 			qdisc_hash_add(qdisc, false);  in mq_change_real_num_tx()
1457 	struct Qdisc *qdisc = _qdisc;  in dev_init_scheduler_queue()  local
1459 	rcu_assign_pointer(dev_queue->qdisc, qdisc);  in dev_init_scheduler_queue()
1460 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);  in dev_init_scheduler_queue()
1465 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);  in dev_init_scheduler()
1478 	qdisc_put(rtnl_dereference(dev->qdisc));  in dev_shutdown()
1479 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);  in dev_shutdown()
1593 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,  in mini_qdisc_pair_init()  argument
1596 	miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1597 	miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()
1598 	miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1599 	miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()