70 static unsigned int fq_codel_hash(const struct fq_codel_sched_data *q,  in fq_codel_hash()  argument
73 	return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);  in fq_codel_hash()
79 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_classify()  local
86 	    TC_H_MIN(skb->priority) <= q->flows_cnt)  in fq_codel_classify()
89 	filter = rcu_dereference_bh(q->filter_list);  in fq_codel_classify()
91 		return fq_codel_hash(q, skb) + 1;  in fq_codel_classify()
107 		if (TC_H_MIN(res.classid) <= q->flows_cnt)  in fq_codel_classify()
140 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_drop()  local
154 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_drop()
155 		if (q->backlogs[i] > maxbacklog) {  in fq_codel_drop()
156 			maxbacklog = q->backlogs[i];  in fq_codel_drop()
164 	flow = &q->flows[idx];  in fq_codel_drop()
176 	q->backlogs[idx] -= len;  in fq_codel_drop()
177 	q->memory_usage -= mem;  in fq_codel_drop()
180 	sch->q.qlen -= i;  in fq_codel_drop()
187 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_enqueue()  local
204 	flow = &q->flows[idx];  in fq_codel_enqueue()
206 	q->backlogs[idx] += qdisc_pkt_len(skb);  in fq_codel_enqueue()
210 		list_add_tail(&flow->flowchain, &q->new_flows);  in fq_codel_enqueue()
211 		q->new_flow_count++;  in fq_codel_enqueue()
212 		flow->deficit = q->quantum;  in fq_codel_enqueue()
215 	q->memory_usage += get_codel_cb(skb)->mem_usage;  in fq_codel_enqueue()
216 	memory_limited = q->memory_usage > q->memory_limit;  in fq_codel_enqueue()
217 	if (++sch->q.qlen <= sch->limit && !memory_limited)  in fq_codel_enqueue()
221 	prev_qlen = sch->q.qlen;  in fq_codel_enqueue()
226 	 * in q->backlogs[] to find a fat flow.  in fq_codel_enqueue()
230 	ret = fq_codel_drop(sch, q->drop_batch_size, to_free);  in fq_codel_enqueue()
232 	prev_qlen -= sch->q.qlen;  in fq_codel_enqueue()
234 	q->drop_overlimit += prev_qlen;  in fq_codel_enqueue()
236 		q->drop_overmemory += prev_qlen;  in fq_codel_enqueue()
258 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in dequeue_func()  local
265 		q->backlogs[flow - q->flows] -= qdisc_pkt_len(skb);  in dequeue_func()
266 		q->memory_usage -= get_codel_cb(skb)->mem_usage;  in dequeue_func()
267 		sch->q.qlen--;  in dequeue_func()
283 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dequeue()  local
289 	head = &q->new_flows;  in fq_codel_dequeue()
291 		head = &q->old_flows;  in fq_codel_dequeue()
298 		flow->deficit += q->quantum;  in fq_codel_dequeue()
299 		list_move_tail(&flow->flowchain, &q->old_flows);  in fq_codel_dequeue()
303 	skb = codel_dequeue(sch, &sch->qstats.backlog, &q->cparams,  in fq_codel_dequeue()
304 			    &flow->cvars, &q->cstats, qdisc_pkt_len,  in fq_codel_dequeue()
309 		if ((head == &q->new_flows) && !list_empty(&q->old_flows))  in fq_codel_dequeue()
310 			list_move_tail(&flow->flowchain, &q->old_flows);  in fq_codel_dequeue()
320 	if (q->cstats.drop_count && sch->q.qlen) {  in fq_codel_dequeue()
321 		qdisc_tree_reduce_backlog(sch, q->cstats.drop_count,  in fq_codel_dequeue()
322 					  q->cstats.drop_len);  in fq_codel_dequeue()
323 		q->cstats.drop_count = 0;  in fq_codel_dequeue()
324 		q->cstats.drop_len = 0;  in fq_codel_dequeue()
337 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_reset()  local
340 	INIT_LIST_HEAD(&q->new_flows);  in fq_codel_reset()
341 	INIT_LIST_HEAD(&q->old_flows);  in fq_codel_reset()
342 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_reset()
343 		struct fq_codel_flow *flow = q->flows + i;  in fq_codel_reset()
349 	memset(q->backlogs, 0, q->flows_cnt * sizeof(u32));  in fq_codel_reset()
350 	sch->q.qlen = 0;  in fq_codel_reset()
352 	q->memory_usage = 0;  in fq_codel_reset()
370 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_change()  local
382 		if (q->flows)  in fq_codel_change()
384 		q->flows_cnt = nla_get_u32(tb[TCA_FQ_CODEL_FLOWS]);  in fq_codel_change()
385 		if (!q->flows_cnt ||  in fq_codel_change()
386 		    q->flows_cnt > 65536)  in fq_codel_change()
394 		q->cparams.target = (target * NSEC_PER_USEC) >> CODEL_SHIFT;  in fq_codel_change()
400 		q->cparams.ce_threshold = (val * NSEC_PER_USEC) >> CODEL_SHIFT;  in fq_codel_change()
406 		q->cparams.interval = (interval * NSEC_PER_USEC) >> CODEL_SHIFT;  in fq_codel_change()
413 		q->cparams.ecn = !!nla_get_u32(tb[TCA_FQ_CODEL_ECN]);  in fq_codel_change()
416 		q->quantum = max(256U, nla_get_u32(tb[TCA_FQ_CODEL_QUANTUM]));  in fq_codel_change()
419 		q->drop_batch_size = max(1U, nla_get_u32(tb[TCA_FQ_CODEL_DROP_BATCH_SIZE]));  in fq_codel_change()
422 		q->memory_limit = min(1U << 31, nla_get_u32(tb[TCA_FQ_CODEL_MEMORY_LIMIT]));  in fq_codel_change()
424 	while (sch->q.qlen > sch->limit ||  in fq_codel_change()
425 	       q->memory_usage > q->memory_limit) {  in fq_codel_change()
428 		q->cstats.drop_len += qdisc_pkt_len(skb);  in fq_codel_change()
430 		q->cstats.drop_count++;  in fq_codel_change()
432 	qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len);  in fq_codel_change()
433 	q->cstats.drop_count = 0;  in fq_codel_change()
434 	q->cstats.drop_len = 0;  in fq_codel_change()
442 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_destroy()  local
444 	tcf_block_put(q->block);  in fq_codel_destroy()
445 	kvfree(q->backlogs);  in fq_codel_destroy()
446 	kvfree(q->flows);  in fq_codel_destroy()
452 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_init()  local
457 	q->flows_cnt = 1024;  in fq_codel_init()
458 	q->memory_limit = 32 << 20; /* 32 MBytes */  in fq_codel_init()
459 	q->drop_batch_size = 64;  in fq_codel_init()
460 	q->quantum = psched_mtu(qdisc_dev(sch));  in fq_codel_init()
461 	INIT_LIST_HEAD(&q->new_flows);  in fq_codel_init()
462 	INIT_LIST_HEAD(&q->old_flows);  in fq_codel_init()
463 	codel_params_init(&q->cparams);  in fq_codel_init()
464 	codel_stats_init(&q->cstats);  in fq_codel_init()
465 	q->cparams.ecn = true;  in fq_codel_init()
466 	q->cparams.mtu = psched_mtu(qdisc_dev(sch));  in fq_codel_init()
474 	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);  in fq_codel_init()
478 	if (!q->flows) {  in fq_codel_init()
479 		q->flows = kvcalloc(q->flows_cnt,  in fq_codel_init()
482 		if (!q->flows) {  in fq_codel_init()
486 		q->backlogs = kvcalloc(q->flows_cnt, sizeof(u32), GFP_KERNEL);  in fq_codel_init()
487 		if (!q->backlogs) {  in fq_codel_init()
491 		for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_init()
492 			struct fq_codel_flow *flow = q->flows + i;  in fq_codel_init()
505 	kvfree(q->flows);  in fq_codel_init()
506 	q->flows = NULL;  in fq_codel_init()
508 	q->flows_cnt = 0;  in fq_codel_init()
514 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump()  local
522 			codel_time_to_us(q->cparams.target)) ||  in fq_codel_dump()
526 			codel_time_to_us(q->cparams.interval)) ||  in fq_codel_dump()
528 			q->cparams.ecn) ||  in fq_codel_dump()
530 			q->quantum) ||  in fq_codel_dump()
532 			q->drop_batch_size) ||  in fq_codel_dump()
534 			q->memory_limit) ||  in fq_codel_dump()
536 			q->flows_cnt))  in fq_codel_dump()
539 	if (q->cparams.ce_threshold != CODEL_DISABLED_THRESHOLD &&  in fq_codel_dump()
541 			codel_time_to_us(q->cparams.ce_threshold)))  in fq_codel_dump()
552 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump_stats()  local
558 	st.qdisc_stats.maxpacket = q->cstats.maxpacket;  in fq_codel_dump_stats()
559 	st.qdisc_stats.drop_overlimit = q->drop_overlimit;  in fq_codel_dump_stats()
560 	st.qdisc_stats.ecn_mark = q->cstats.ecn_mark;  in fq_codel_dump_stats()
561 	st.qdisc_stats.new_flow_count = q->new_flow_count;  in fq_codel_dump_stats()
562 	st.qdisc_stats.ce_mark = q->cstats.ce_mark;  in fq_codel_dump_stats()
563 	st.qdisc_stats.memory_usage  = q->memory_usage;  in fq_codel_dump_stats()
564 	st.qdisc_stats.drop_overmemory = q->drop_overmemory;  in fq_codel_dump_stats()
567 	list_for_each(pos, &q->new_flows)  in fq_codel_dump_stats()
570 	list_for_each(pos, &q->old_flows)  in fq_codel_dump_stats()
593 static void fq_codel_unbind(struct Qdisc *q, unsigned long cl)  in fq_codel_unbind()  argument
600 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_tcf_block()  local
604 	return q->block;  in fq_codel_tcf_block()
617 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump_class_stats()  local
622 	if (idx < q->flows_cnt) {  in fq_codel_dump_class_stats()
623 		const struct fq_codel_flow *flow = &q->flows[idx];  in fq_codel_dump_class_stats()
651 		qs.backlog = q->backlogs[idx];  in fq_codel_dump_class_stats()
656 	if (idx < q->flows_cnt)  in fq_codel_dump_class_stats()
663 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_walk()  local
669 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_walk()
670 		if (list_empty(&q->flows[i].flowchain) ||  in fq_codel_walk()