94 	struct pie_sched_data *q = qdisc_priv(sch);  in drop_early()  local
96 	u64 local_prob = q->vars.prob;  in drop_early()
100 	if (q->vars.burst_time > 0)  in drop_early()
106 	if ((q->vars.qdelay < q->params.target / 2) &&  in drop_early()
107 	    (q->vars.prob < MAX_PROB / 5))  in drop_early()
119 	if (q->params.bytemode && packet_size <= mtu)  in drop_early()
122 		local_prob = q->vars.prob;  in drop_early()
125 		q->vars.accu_prob = 0;  in drop_early()
126 		q->vars.accu_prob_overflows = 0;  in drop_early()
129 	if (local_prob > MAX_PROB - q->vars.accu_prob)  in drop_early()
130 		q->vars.accu_prob_overflows++;  in drop_early()
132 	q->vars.accu_prob += local_prob;  in drop_early()
134 	if (q->vars.accu_prob_overflows == 0 &&  in drop_early()
135 	    q->vars.accu_prob < (MAX_PROB / 100) * 85)  in drop_early()
137 	if (q->vars.accu_prob_overflows == 8 &&  in drop_early()
138 	    q->vars.accu_prob >= MAX_PROB / 2)  in drop_early()
143 		q->vars.accu_prob = 0;  in drop_early()
144 		q->vars.accu_prob_overflows = 0;  in drop_early()
154 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_qdisc_enqueue()  local
158 		q->stats.overlimit++;  in pie_qdisc_enqueue()
164 	} else if (q->params.ecn && (q->vars.prob <= MAX_PROB / 10) &&  in pie_qdisc_enqueue()
169 		q->stats.ecn_mark++;  in pie_qdisc_enqueue()
175 		q->stats.packets_in++;  in pie_qdisc_enqueue()
176 		if (qdisc_qlen(sch) > q->stats.maxq)  in pie_qdisc_enqueue()
177 			q->stats.maxq = qdisc_qlen(sch);  in pie_qdisc_enqueue()
183 	q->stats.dropped++;  in pie_qdisc_enqueue()
184 	q->vars.accu_prob = 0;  in pie_qdisc_enqueue()
185 	q->vars.accu_prob_overflows = 0;  in pie_qdisc_enqueue()
202 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_change()  local
223 		q->params.target = PSCHED_NS2TICKS((u64)target * NSEC_PER_USEC);  in pie_change()
228 		q->params.tupdate =  in pie_change()
234 		q->params.limit = limit;  in pie_change()
239 		q->params.alpha = nla_get_u32(tb[TCA_PIE_ALPHA]);  in pie_change()
242 		q->params.beta = nla_get_u32(tb[TCA_PIE_BETA]);  in pie_change()
245 		q->params.ecn = nla_get_u32(tb[TCA_PIE_ECN]);  in pie_change()
248 		q->params.bytemode = nla_get_u32(tb[TCA_PIE_BYTEMODE]);  in pie_change()
251 	qlen = sch->q.qlen;  in pie_change()
252 	while (sch->q.qlen > sch->limit) {  in pie_change()
253 		struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);  in pie_change()
259 	qdisc_tree_reduce_backlog(sch, qlen - sch->q.qlen, dropped);  in pie_change()
267 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_process_dequeue()  local
274 	if (qlen >= QUEUE_THRESHOLD && q->vars.dq_count == DQCOUNT_INVALID) {  in pie_process_dequeue()
275 		q->vars.dq_tstamp = psched_get_time();  in pie_process_dequeue()
276 		q->vars.dq_count = 0;  in pie_process_dequeue()
288 	if (q->vars.dq_count != DQCOUNT_INVALID) {  in pie_process_dequeue()
289 		q->vars.dq_count += skb->len;  in pie_process_dequeue()
291 		if (q->vars.dq_count >= QUEUE_THRESHOLD) {  in pie_process_dequeue()
293 			u32 dtime = now - q->vars.dq_tstamp;  in pie_process_dequeue()
294 			u32 count = q->vars.dq_count << PIE_SCALE;  in pie_process_dequeue()
301 			if (q->vars.avg_dq_rate == 0)  in pie_process_dequeue()
302 				q->vars.avg_dq_rate = count;  in pie_process_dequeue()
304 				q->vars.avg_dq_rate =  in pie_process_dequeue()
305 				    (q->vars.avg_dq_rate -  in pie_process_dequeue()
306 				     (q->vars.avg_dq_rate >> 3)) + (count >> 3);  in pie_process_dequeue()
314 				q->vars.dq_count = DQCOUNT_INVALID;  in pie_process_dequeue()
316 				q->vars.dq_count = 0;  in pie_process_dequeue()
317 				q->vars.dq_tstamp = psched_get_time();  in pie_process_dequeue()
320 			if (q->vars.burst_time > 0) {  in pie_process_dequeue()
321 				if (q->vars.burst_time > dtime)  in pie_process_dequeue()
322 					q->vars.burst_time -= dtime;  in pie_process_dequeue()
324 					q->vars.burst_time = 0;  in pie_process_dequeue()
332 	struct pie_sched_data *q = qdisc_priv(sch);  in calculate_probability()  local
335 	psched_time_t qdelay_old = q->vars.qdelay;	/* in pschedtime */  in calculate_probability()
342 	q->vars.qdelay_old = q->vars.qdelay;  in calculate_probability()
344 	if (q->vars.avg_dq_rate > 0)  in calculate_probability()
345 		qdelay = (qlen << PIE_SCALE) / q->vars.avg_dq_rate;  in calculate_probability()
362 	alpha = ((u64)q->params.alpha * (MAX_PROB / PSCHED_TICKS_PER_SEC)) >> 4;  in calculate_probability()
363 	beta = ((u64)q->params.beta * (MAX_PROB / PSCHED_TICKS_PER_SEC)) >> 4;  in calculate_probability()
368 	if (q->vars.prob < MAX_PROB / 10) {  in calculate_probability()
373 		while (q->vars.prob < div_u64(MAX_PROB, power) &&  in calculate_probability()
382 	delta += alpha * (u64)(qdelay - q->params.target);  in calculate_probability()
385 	oldprob = q->vars.prob;  in calculate_probability()
389 	    q->vars.prob >= MAX_PROB / 10)  in calculate_probability()
400 	q->vars.prob += delta;  in calculate_probability()
404 		if (q->vars.prob < oldprob) {  in calculate_probability()
405 			q->vars.prob = MAX_PROB;  in calculate_probability()
415 		if (q->vars.prob > oldprob)  in calculate_probability()
416 			q->vars.prob = 0;  in calculate_probability()
425 		q->vars.prob -= q->vars.prob / 64u;  in calculate_probability()
427 	q->vars.qdelay = qdelay;  in calculate_probability()
428 	q->vars.qlen_old = qlen;  in calculate_probability()
436 	if ((q->vars.qdelay < q->params.target / 2) &&  in calculate_probability()
437 	    (q->vars.qdelay_old < q->params.target / 2) &&  in calculate_probability()
438 	    q->vars.prob == 0 &&  in calculate_probability()
439 	    q->vars.avg_dq_rate > 0)  in calculate_probability()
440 		pie_vars_init(&q->vars);  in calculate_probability()
445 	struct pie_sched_data *q = from_timer(q, t, adapt_timer);  in pie_timer()  local
446 	struct Qdisc *sch = q->sch;  in pie_timer()
453 	if (q->params.tupdate)  in pie_timer()
454 		mod_timer(&q->adapt_timer, jiffies + q->params.tupdate);  in pie_timer()
461 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_init()  local
463 	pie_params_init(&q->params);  in pie_init()
464 	pie_vars_init(&q->vars);  in pie_init()
465 	sch->limit = q->params.limit;  in pie_init()
467 	q->sch = sch;  in pie_init()
468 	timer_setup(&q->adapt_timer, pie_timer, 0);  in pie_init()
477 	mod_timer(&q->adapt_timer, jiffies + HZ / 2);  in pie_init()
483 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_dump()  local
492 			((u32)PSCHED_TICKS2NS(q->params.target)) /  in pie_dump()
496 			jiffies_to_usecs(q->params.tupdate)) ||  in pie_dump()
497 	    nla_put_u32(skb, TCA_PIE_ALPHA, q->params.alpha) ||  in pie_dump()
498 	    nla_put_u32(skb, TCA_PIE_BETA, q->params.beta) ||  in pie_dump()
499 	    nla_put_u32(skb, TCA_PIE_ECN, q->params.ecn) ||  in pie_dump()
500 	    nla_put_u32(skb, TCA_PIE_BYTEMODE, q->params.bytemode))  in pie_dump()
512 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_dump_stats()  local
514 		.prob		= q->vars.prob,  in pie_dump_stats()
515 		.delay		= ((u32)PSCHED_TICKS2NS(q->vars.qdelay)) /  in pie_dump_stats()
518 		.avg_dq_rate	= q->vars.avg_dq_rate *  in pie_dump_stats()
520 		.packets_in	= q->stats.packets_in,  in pie_dump_stats()
521 		.overlimit	= q->stats.overlimit,  in pie_dump_stats()
522 		.maxq		= q->stats.maxq,  in pie_dump_stats()
523 		.dropped	= q->stats.dropped,  in pie_dump_stats()
524 		.ecn_mark	= q->stats.ecn_mark,  in pie_dump_stats()
543 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_reset()  local
546 	pie_vars_init(&q->vars);  in pie_reset()
551 	struct pie_sched_data *q = qdisc_priv(sch);  in pie_destroy()  local
553 	q->params.tupdate = 0;  in pie_destroy()
554 	del_timer_sync(&q->adapt_timer);  in pie_destroy()