120 static inline void qdisc_refcount_inc(struct Qdisc *qdisc)  in qdisc_refcount_inc()  argument
122 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_refcount_inc()
124 	refcount_inc(&qdisc->refcnt);  in qdisc_refcount_inc()
131 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc)  in qdisc_refcount_inc_nz()  argument
133 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_refcount_inc_nz()
134 		return qdisc;  in qdisc_refcount_inc_nz()
135 	if (refcount_inc_not_zero(&qdisc->refcnt))  in qdisc_refcount_inc_nz()
136 		return qdisc;  in qdisc_refcount_inc_nz()
140 static inline bool qdisc_is_running(struct Qdisc *qdisc)  in qdisc_is_running()  argument
142 	if (qdisc->flags & TCQ_F_NOLOCK)  in qdisc_is_running()
143 		return spin_is_locked(&qdisc->seqlock);  in qdisc_is_running()
144 	return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;  in qdisc_is_running()
152 static inline bool qdisc_is_empty(const struct Qdisc *qdisc)  in qdisc_is_empty()  argument
154 	if (qdisc_is_percpu_stats(qdisc))  in qdisc_is_empty()
155 		return READ_ONCE(qdisc->empty);  in qdisc_is_empty()
156 	return !READ_ONCE(qdisc->q.qlen);  in qdisc_is_empty()
159 static inline bool qdisc_run_begin(struct Qdisc *qdisc)  in qdisc_run_begin()  argument
161 	if (qdisc->flags & TCQ_F_NOLOCK) {  in qdisc_run_begin()
162 		if (!spin_trylock(&qdisc->seqlock))  in qdisc_run_begin()
164 		WRITE_ONCE(qdisc->empty, false);  in qdisc_run_begin()
165 	} else if (qdisc_is_running(qdisc)) {  in qdisc_run_begin()
171 	raw_write_seqcount_begin(&qdisc->running);  in qdisc_run_begin()
172 	seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_);  in qdisc_run_begin()
176 static inline void qdisc_run_end(struct Qdisc *qdisc)  in qdisc_run_end()  argument
178 	write_seqcount_end(&qdisc->running);  in qdisc_run_end()
179 	if (qdisc->flags & TCQ_F_NOLOCK)  in qdisc_run_end()
180 		spin_unlock(&qdisc->seqlock);  in qdisc_run_end()
183 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)  in qdisc_may_bulk()  argument
185 	return qdisc->flags & TCQ_F_ONETXQUEUE;  in qdisc_may_bulk()
504 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc)  in qdisc_lock()  argument
506 	return &qdisc->q.lock;  in qdisc_lock()
509 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc)  in qdisc_root()  argument
511 	struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc);  in qdisc_root()
516 static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc)  in qdisc_root_bh()  argument
518 	return rcu_dereference_bh(qdisc->dev_queue->qdisc);  in qdisc_root_bh()
521 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc)  in qdisc_root_sleeping()  argument
523 	return qdisc->dev_queue->qdisc_sleeping;  in qdisc_root_sleeping()
537 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc)  in qdisc_root_lock()  argument
539 	struct Qdisc *root = qdisc_root(qdisc);  in qdisc_root_lock()
545 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc)  in qdisc_root_sleeping_lock()  argument
547 	struct Qdisc *root = qdisc_root_sleeping(qdisc);  in qdisc_root_sleeping_lock()
553 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)  in qdisc_root_sleeping_running()  argument
555 	struct Qdisc *root = qdisc_root_sleeping(qdisc);  in qdisc_root_sleeping_running()
561 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc)  in qdisc_dev()  argument
563 	return qdisc->dev_queue->dev;  in qdisc_dev()
647 			      struct Qdisc *qdisc);
648 void qdisc_reset(struct Qdisc *qdisc);
649 void qdisc_put(struct Qdisc *qdisc);
650 void qdisc_put_unlocked(struct Qdisc *qdisc);
651 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
679 void qdisc_free(struct Qdisc *qdisc);
710 	struct Qdisc *qdisc;  in qdisc_reset_all_tx_gt()  local
713 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc);  in qdisc_reset_all_tx_gt()
714 		if (qdisc) {  in qdisc_reset_all_tx_gt()
715 			spin_lock_bh(qdisc_lock(qdisc));  in qdisc_reset_all_tx_gt()
716 			qdisc_reset(qdisc);  in qdisc_reset_all_tx_gt()
717 			spin_unlock_bh(qdisc_lock(qdisc));  in qdisc_reset_all_tx_gt()
730 		const struct Qdisc *q = rcu_dereference(txq->qdisc);  in qdisc_all_tx_empty()
748 		if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping)  in qdisc_tx_changing()
761 		if (rcu_access_pointer(txq->qdisc) != &noop_qdisc)  in qdisc_tx_is_noop()
1279 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,