527 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in noop_enqueue()  argument
534 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)  in noop_dequeue()  argument
549 	RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
577 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,  in noqueue_init()  argument
583 	qdisc->enqueue = NULL;  in noqueue_init()
621 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in pfifo_fast_enqueue()  argument
625 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_enqueue()
633 		if (qdisc_is_percpu_stats(qdisc))  in pfifo_fast_enqueue()
634 			return qdisc_drop_cpu(skb, qdisc, to_free);  in pfifo_fast_enqueue()
636 			return qdisc_drop(skb, qdisc, to_free);  in pfifo_fast_enqueue()
639 	qdisc_update_stats_at_enqueue(qdisc, pkt_len);  in pfifo_fast_enqueue()
643 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)  in pfifo_fast_dequeue()  argument
645 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_dequeue()
658 		qdisc_update_stats_at_dequeue(qdisc, skb);  in pfifo_fast_dequeue()
660 		qdisc->empty = true;  in pfifo_fast_dequeue()
666 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)  in pfifo_fast_peek()  argument
668 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_peek()
681 static void pfifo_fast_reset(struct Qdisc *qdisc)  in pfifo_fast_reset()  argument
684 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_reset()
700 	if (qdisc_is_percpu_stats(qdisc)) {  in pfifo_fast_reset()
704 			q = per_cpu_ptr(qdisc->cpu_qstats, i);  in pfifo_fast_reset()
711 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)  in pfifo_fast_dump()  argument
724 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,  in pfifo_fast_init()  argument
727 	unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;  in pfifo_fast_init()
728 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_init()
745 	qdisc->flags |= TCQ_F_CAN_BYPASS;  in pfifo_fast_init()
909 void qdisc_reset(struct Qdisc *qdisc)  in qdisc_reset()  argument
911 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_reset()
915 		ops->reset(qdisc);  in qdisc_reset()
917 	skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {  in qdisc_reset()
918 		__skb_unlink(skb, &qdisc->gso_skb);  in qdisc_reset()
922 	skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {  in qdisc_reset()
923 		__skb_unlink(skb, &qdisc->skb_bad_txq);  in qdisc_reset()
927 	qdisc->q.qlen = 0;  in qdisc_reset()
928 	qdisc->qstats.backlog = 0;  in qdisc_reset()
932 void qdisc_free(struct Qdisc *qdisc)  in qdisc_free()  argument
934 	if (qdisc_is_percpu_stats(qdisc)) {  in qdisc_free()
935 		free_percpu(qdisc->cpu_bstats);  in qdisc_free()
936 		free_percpu(qdisc->cpu_qstats);  in qdisc_free()
939 	kfree((char *) qdisc - qdisc->padded);  in qdisc_free()
949 static void qdisc_destroy(struct Qdisc *qdisc)  in qdisc_destroy()  argument
951 	const struct Qdisc_ops  *ops = qdisc->ops;  in qdisc_destroy()
955 	qdisc_hash_del(qdisc);  in qdisc_destroy()
957 	qdisc_put_stab(rtnl_dereference(qdisc->stab));  in qdisc_destroy()
959 	gen_kill_estimator(&qdisc->rate_est);  in qdisc_destroy()
961 		ops->reset(qdisc);  in qdisc_destroy()
963 		ops->destroy(qdisc);  in qdisc_destroy()
966 	dev_put(qdisc_dev(qdisc));  in qdisc_destroy()
968 	skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {  in qdisc_destroy()
969 		__skb_unlink(skb, &qdisc->gso_skb);  in qdisc_destroy()
973 	skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {  in qdisc_destroy()
974 		__skb_unlink(skb, &qdisc->skb_bad_txq);  in qdisc_destroy()
978 	call_rcu(&qdisc->rcu, qdisc_free_cb);  in qdisc_destroy()
981 void qdisc_put(struct Qdisc *qdisc)  in qdisc_put()  argument
983 	if (!qdisc)  in qdisc_put()
986 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put()
987 	    !refcount_dec_and_test(&qdisc->refcnt))  in qdisc_put()
990 	qdisc_destroy(qdisc);  in qdisc_put()
999 void qdisc_put_unlocked(struct Qdisc *qdisc)  in qdisc_put_unlocked()  argument
1001 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put_unlocked()
1002 	    !refcount_dec_and_rtnl_lock(&qdisc->refcnt))  in qdisc_put_unlocked()
1005 	qdisc_destroy(qdisc);  in qdisc_put_unlocked()
1012 			      struct Qdisc *qdisc)  in dev_graft_qdisc()  argument
1021 	if (qdisc == NULL)  in dev_graft_qdisc()
1022 		qdisc = &noop_qdisc;  in dev_graft_qdisc()
1023 	dev_queue->qdisc_sleeping = qdisc;  in dev_graft_qdisc()
1024 	rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);  in dev_graft_qdisc()
1036 	struct Qdisc *qdisc;  in attach_one_default_qdisc()  local
1044 	qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);  in attach_one_default_qdisc()
1045 	if (!qdisc) {  in attach_one_default_qdisc()
1050 		qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;  in attach_one_default_qdisc()
1051 	dev_queue->qdisc_sleeping = qdisc;  in attach_one_default_qdisc()
1057 	struct Qdisc *qdisc;  in attach_default_qdiscs()  local
1064 		dev->qdisc = txq->qdisc_sleeping;  in attach_default_qdiscs()
1065 		qdisc_refcount_inc(dev->qdisc);  in attach_default_qdiscs()
1067 		qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);  in attach_default_qdiscs()
1068 		if (qdisc) {  in attach_default_qdiscs()
1069 			dev->qdisc = qdisc;  in attach_default_qdiscs()
1070 			qdisc->ops->attach(qdisc);  in attach_default_qdiscs()
1074 	if (dev->qdisc != &noop_qdisc)  in attach_default_qdiscs()
1075 		qdisc_hash_add(dev->qdisc, false);  in attach_default_qdiscs()
1089 	rcu_assign_pointer(dev_queue->qdisc, new_qdisc);  in transition_one_qdisc()
1105 	if (dev->qdisc == &noop_qdisc)  in dev_activate()
1129 	struct Qdisc *qdisc;  in dev_deactivate_queue()  local
1131 	qdisc = rtnl_dereference(dev_queue->qdisc);  in dev_deactivate_queue()
1132 	if (qdisc) {  in dev_deactivate_queue()
1133 		bool nolock = qdisc->flags & TCQ_F_NOLOCK;  in dev_deactivate_queue()
1136 			spin_lock_bh(&qdisc->seqlock);  in dev_deactivate_queue()
1137 		spin_lock_bh(qdisc_lock(qdisc));  in dev_deactivate_queue()
1139 		if (!(qdisc->flags & TCQ_F_BUILTIN))  in dev_deactivate_queue()
1140 			set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);  in dev_deactivate_queue()
1142 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in dev_deactivate_queue()
1143 		qdisc_reset(qdisc);  in dev_deactivate_queue()
1145 		spin_unlock_bh(qdisc_lock(qdisc));  in dev_deactivate_queue()
1147 			spin_unlock_bh(&qdisc->seqlock);  in dev_deactivate_queue()
1182 	struct Qdisc *qdisc = dev_queue->qdisc_sleeping;  in dev_qdisc_reset()  local
1184 	if (qdisc)  in dev_qdisc_reset()
1185 		qdisc_reset(qdisc);  in dev_qdisc_reset()
1241 	struct Qdisc *qdisc = dev_queue->qdisc_sleeping;  in qdisc_change_tx_queue_len()  local
1242 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_change_tx_queue_len()
1245 		return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);  in qdisc_change_tx_queue_len()
1275 	struct Qdisc *qdisc = _qdisc;  in dev_init_scheduler_queue()  local
1277 	rcu_assign_pointer(dev_queue->qdisc, qdisc);  in dev_init_scheduler_queue()
1278 	dev_queue->qdisc_sleeping = qdisc;  in dev_init_scheduler_queue()
1283 	dev->qdisc = &noop_qdisc;  in dev_init_scheduler()
1295 	struct Qdisc *qdisc = dev_queue->qdisc_sleeping;  in shutdown_scheduler_queue()  local
1298 	if (qdisc) {  in shutdown_scheduler_queue()
1299 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in shutdown_scheduler_queue()
1302 		qdisc_put(qdisc);  in shutdown_scheduler_queue()
1311 	qdisc_put(dev->qdisc);  in dev_shutdown()
1312 	dev->qdisc = &noop_qdisc;  in dev_shutdown()
1394 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,  in mini_qdisc_pair_init()  argument
1397 	miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1398 	miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()
1399 	miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1400 	miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()