207 static bool loss_4state(struct netem_sched_data *q)  in loss_4state()  argument
209 	struct clgstate *clg = &q->clg;  in loss_4state()
210 	u32 rnd = prandom_u32_state(&q->prng.prng_state);  in loss_4state()
272 static bool loss_gilb_ell(struct netem_sched_data *q)  in loss_gilb_ell()  argument
274 	struct clgstate *clg = &q->clg;  in loss_gilb_ell()
275 	struct rnd_state *s = &q->prng.prng_state;  in loss_gilb_ell()
294 static bool loss_event(struct netem_sched_data *q)  in loss_event()  argument
296 	switch (q->loss_model) {  in loss_event()
299 		return q->loss && q->loss >= get_crandom(&q->loss_cor, &q->prng);  in loss_event()
307 		return loss_4state(q);  in loss_event()
315 		return loss_gilb_ell(q);  in loss_event()
354 static u64 packet_time_ns(u64 len, const struct netem_sched_data *q)  in packet_time_ns()  argument
356 	len += q->packet_overhead;  in packet_time_ns()
358 	if (q->cell_size) {  in packet_time_ns()
359 		u32 cells = reciprocal_divide(len, q->cell_size_reciprocal);  in packet_time_ns()
361 		if (len > cells * q->cell_size)	/* extra cell needed for remainder */  in packet_time_ns()
363 		len = cells * (q->cell_size + q->cell_overhead);  in packet_time_ns()
366 	return div64_u64(len * NSEC_PER_SEC, q->rate);  in packet_time_ns()
371 	struct netem_sched_data *q = qdisc_priv(sch);  in tfifo_reset()  local
372 	struct rb_node *p = rb_first(&q->t_root);  in tfifo_reset()
378 		rb_erase(&skb->rbnode, &q->t_root);  in tfifo_reset()
382 	rtnl_kfree_skbs(q->t_head, q->t_tail);  in tfifo_reset()
383 	q->t_head = NULL;  in tfifo_reset()
384 	q->t_tail = NULL;  in tfifo_reset()
389 	struct netem_sched_data *q = qdisc_priv(sch);  in tfifo_enqueue()  local
392 	if (!q->t_tail || tnext >= netem_skb_cb(q->t_tail)->time_to_send) {  in tfifo_enqueue()
393 		if (q->t_tail)  in tfifo_enqueue()
394 			q->t_tail->next = nskb;  in tfifo_enqueue()
396 			q->t_head = nskb;  in tfifo_enqueue()
397 		q->t_tail = nskb;  in tfifo_enqueue()
399 		struct rb_node **p = &q->t_root.rb_node, *parent = NULL;  in tfifo_enqueue()
412 		rb_insert_color(&nskb->rbnode, &q->t_root);  in tfifo_enqueue()
414 	sch->q.qlen++;  in tfifo_enqueue()
446 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_enqueue()  local
460 	if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor, &q->prng))  in netem_enqueue()
464 	if (loss_event(q)) {  in netem_enqueue()
465 		if (q->ecn && INET_ECN_set_ce(skb))  in netem_enqueue()
479 	if (q->latency || q->jitter || q->rate)  in netem_enqueue()
489 		u32 dupsave = q->duplicate; /* prevent duplicating a dup... */  in netem_enqueue()
491 		q->duplicate = 0;  in netem_enqueue()
493 		q->duplicate = dupsave;  in netem_enqueue()
503 	if (q->corrupt && q->corrupt >= get_crandom(&q->corrupt_cor, &q->prng)) {  in netem_enqueue()
529 	if (unlikely(sch->q.qlen >= sch->limit)) {  in netem_enqueue()
539 	if (q->gap == 0 ||		/* not doing reordering */  in netem_enqueue()
540 	    q->counter < q->gap - 1 ||	/* inside last reordering gap */  in netem_enqueue()
541 	    q->reorder < get_crandom(&q->reorder_cor, &q->prng)) {  in netem_enqueue()
545 		delay = tabledist(q->latency, q->jitter,  in netem_enqueue()
546 				  &q->delay_cor, &q->prng, q->delay_dist);  in netem_enqueue()
550 		if (q->rate) {  in netem_enqueue()
553 			if (sch->q.tail)  in netem_enqueue()
554 				last = netem_skb_cb(sch->q.tail);  in netem_enqueue()
555 			if (q->t_root.rb_node) {  in netem_enqueue()
559 				t_skb = skb_rb_last(&q->t_root);  in netem_enqueue()
565 			if (q->t_tail) {  in netem_enqueue()
567 					netem_skb_cb(q->t_tail);  in netem_enqueue()
585 			delay += packet_time_ns(qdisc_pkt_len(skb), q);  in netem_enqueue()
589 		++q->counter;  in netem_enqueue()
597 		q->counter = 0;  in netem_enqueue()
599 		__qdisc_enqueue_head(skb, &sch->q);  in netem_enqueue()
638 static void get_slot_next(struct netem_sched_data *q, u64 now)  in get_slot_next()  argument
642 	if (!q->slot_dist)  in get_slot_next()
643 		next_delay = q->slot_config.min_delay +  in get_slot_next()
645 				 (q->slot_config.max_delay -  in get_slot_next()
646 				  q->slot_config.min_delay) >> 32);  in get_slot_next()
648 		next_delay = tabledist(q->slot_config.dist_delay,  in get_slot_next()
649 				       (s32)(q->slot_config.dist_jitter),  in get_slot_next()
650 				       NULL, &q->prng, q->slot_dist);  in get_slot_next()
652 	q->slot.slot_next = now + next_delay;  in get_slot_next()
653 	q->slot.packets_left = q->slot_config.max_packets;  in get_slot_next()
654 	q->slot.bytes_left = q->slot_config.max_bytes;  in get_slot_next()
657 static struct sk_buff *netem_peek(struct netem_sched_data *q)  in netem_peek()  argument
659 	struct sk_buff *skb = skb_rb_first(&q->t_root);  in netem_peek()
663 		return q->t_head;  in netem_peek()
664 	if (!q->t_head)  in netem_peek()
668 	t2 = netem_skb_cb(q->t_head)->time_to_send;  in netem_peek()
671 	return q->t_head;  in netem_peek()
674 static void netem_erase_head(struct netem_sched_data *q, struct sk_buff *skb)  in netem_erase_head()  argument
676 	if (skb == q->t_head) {  in netem_erase_head()
677 		q->t_head = skb->next;  in netem_erase_head()
678 		if (!q->t_head)  in netem_erase_head()
679 			q->t_tail = NULL;  in netem_erase_head()
681 		rb_erase(&skb->rbnode, &q->t_root);  in netem_erase_head()
687 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_dequeue()  local
691 	skb = __qdisc_dequeue_head(&sch->q);  in netem_dequeue()
698 	skb = netem_peek(q);  in netem_dequeue()
705 		if (q->slot.slot_next && q->slot.slot_next < time_to_send)  in netem_dequeue()
706 			get_slot_next(q, now);  in netem_dequeue()
708 		if (time_to_send <= now && q->slot.slot_next <= now) {  in netem_dequeue()
709 			netem_erase_head(q, skb);  in netem_dequeue()
710 			sch->q.qlen--;  in netem_dequeue()
719 			if (q->slot.slot_next) {  in netem_dequeue()
720 				q->slot.packets_left--;  in netem_dequeue()
721 				q->slot.bytes_left -= qdisc_pkt_len(skb);  in netem_dequeue()
722 				if (q->slot.packets_left <= 0 ||  in netem_dequeue()
723 				    q->slot.bytes_left <= 0)  in netem_dequeue()
724 					get_slot_next(q, now);  in netem_dequeue()
727 			if (q->qdisc) {  in netem_dequeue()
732 				err = qdisc_enqueue(skb, q->qdisc, &to_free);  in netem_dequeue()
745 		if (q->qdisc) {  in netem_dequeue()
746 			skb = q->qdisc->ops->dequeue(q->qdisc);  in netem_dequeue()
751 		qdisc_watchdog_schedule_ns(&q->watchdog,  in netem_dequeue()
753 					       q->slot.slot_next));  in netem_dequeue()
756 	if (q->qdisc) {  in netem_dequeue()
757 		skb = q->qdisc->ops->dequeue(q->qdisc);  in netem_dequeue()
766 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_reset()  local
770 	if (q->qdisc)  in netem_reset()
771 		qdisc_reset(q->qdisc);  in netem_reset()
772 	qdisc_watchdog_cancel(&q->watchdog);  in netem_reset()
807 static void get_slot(struct netem_sched_data *q, const struct nlattr *attr)  in get_slot()  argument
811 	q->slot_config = *c;  in get_slot()
812 	if (q->slot_config.max_packets == 0)  in get_slot()
813 		q->slot_config.max_packets = INT_MAX;  in get_slot()
814 	if (q->slot_config.max_bytes == 0)  in get_slot()
815 		q->slot_config.max_bytes = INT_MAX;  in get_slot()
818 	q->slot_config.dist_jitter = min_t(__s64, INT_MAX, abs(q->slot_config.dist_jitter));  in get_slot()
820 	q->slot.packets_left = q->slot_config.max_packets;  in get_slot()
821 	q->slot.bytes_left = q->slot_config.max_bytes;  in get_slot()
822 	if (q->slot_config.min_delay | q->slot_config.max_delay |  in get_slot()
823 	    q->slot_config.dist_jitter)  in get_slot()
824 		q->slot.slot_next = ktime_get_ns();  in get_slot()
826 		q->slot.slot_next = 0;  in get_slot()
829 static void get_correlation(struct netem_sched_data *q, const struct nlattr *attr)  in get_correlation()  argument
833 	init_crandom(&q->delay_cor, c->delay_corr);  in get_correlation()
834 	init_crandom(&q->loss_cor, c->loss_corr);  in get_correlation()
835 	init_crandom(&q->dup_cor, c->dup_corr);  in get_correlation()
838 static void get_reorder(struct netem_sched_data *q, const struct nlattr *attr)  in get_reorder()  argument
842 	q->reorder = r->probability;  in get_reorder()
843 	init_crandom(&q->reorder_cor, r->correlation);  in get_reorder()
846 static void get_corrupt(struct netem_sched_data *q, const struct nlattr *attr)  in get_corrupt()  argument
850 	q->corrupt = r->probability;  in get_corrupt()
851 	init_crandom(&q->corrupt_cor, r->correlation);  in get_corrupt()
854 static void get_rate(struct netem_sched_data *q, const struct nlattr *attr)  in get_rate()  argument
858 	q->rate = r->rate;  in get_rate()
859 	q->packet_overhead = r->packet_overhead;  in get_rate()
860 	q->cell_size = r->cell_size;  in get_rate()
861 	q->cell_overhead = r->cell_overhead;  in get_rate()
862 	if (q->cell_size)  in get_rate()
863 		q->cell_size_reciprocal = reciprocal_value(q->cell_size);  in get_rate()
865 		q->cell_size_reciprocal = (struct reciprocal_value) { 0 };  in get_rate()
868 static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr)  in get_loss_clg()  argument
885 			q->loss_model = CLG_4_STATES;  in get_loss_clg()
887 			q->clg.state = TX_IN_GAP_PERIOD;  in get_loss_clg()
888 			q->clg.a1 = gi->p13;  in get_loss_clg()
889 			q->clg.a2 = gi->p31;  in get_loss_clg()
890 			q->clg.a3 = gi->p32;  in get_loss_clg()
891 			q->clg.a4 = gi->p14;  in get_loss_clg()
892 			q->clg.a5 = gi->p23;  in get_loss_clg()
904 			q->loss_model = CLG_GILB_ELL;  in get_loss_clg()
905 			q->clg.state = GOOD_STATE;  in get_loss_clg()
906 			q->clg.a1 = ge->p;  in get_loss_clg()
907 			q->clg.a2 = ge->r;  in get_loss_clg()
908 			q->clg.a3 = ge->h;  in get_loss_clg()
909 			q->clg.a4 = ge->k1;  in get_loss_clg()
959 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_change()  local
986 	/* backup q->clg and q->loss_model */  in netem_change()
987 	old_clg = q->clg;  in netem_change()
988 	old_loss_model = q->loss_model;  in netem_change()
991 		ret = get_loss_clg(q, tb[TCA_NETEM_LOSS]);  in netem_change()
993 			q->loss_model = old_loss_model;  in netem_change()
994 			q->clg = old_clg;  in netem_change()
998 		q->loss_model = CLG_RANDOM;  in netem_change()
1002 		swap(q->delay_dist, delay_dist);  in netem_change()
1004 		swap(q->slot_dist, slot_dist);  in netem_change()
1007 	q->latency = PSCHED_TICKS2NS(qopt->latency);  in netem_change()
1008 	q->jitter = PSCHED_TICKS2NS(qopt->jitter);  in netem_change()
1009 	q->limit = qopt->limit;  in netem_change()
1010 	q->gap = qopt->gap;  in netem_change()
1011 	q->counter = 0;  in netem_change()
1012 	q->loss = qopt->loss;  in netem_change()
1013 	q->duplicate = qopt->duplicate;  in netem_change()
1018 	if (q->gap)  in netem_change()
1019 		q->reorder = ~0;  in netem_change()
1022 		get_correlation(q, tb[TCA_NETEM_CORR]);  in netem_change()
1025 		get_reorder(q, tb[TCA_NETEM_REORDER]);  in netem_change()
1028 		get_corrupt(q, tb[TCA_NETEM_CORRUPT]);  in netem_change()
1031 		get_rate(q, tb[TCA_NETEM_RATE]);  in netem_change()
1034 		q->rate = max_t(u64, q->rate,  in netem_change()
1038 		q->latency = nla_get_s64(tb[TCA_NETEM_LATENCY64]);  in netem_change()
1041 		q->jitter = nla_get_s64(tb[TCA_NETEM_JITTER64]);  in netem_change()
1044 		q->ecn = nla_get_u32(tb[TCA_NETEM_ECN]);  in netem_change()
1047 		get_slot(q, tb[TCA_NETEM_SLOT]);  in netem_change()
1050 	q->jitter = min_t(s64, abs(q->jitter), INT_MAX);  in netem_change()
1053 		q->prng.seed = nla_get_u64(tb[TCA_NETEM_PRNG_SEED]);  in netem_change()
1055 		q->prng.seed = get_random_u64();  in netem_change()
1056 	prandom_seed_state(&q->prng.prng_state, q->prng.seed);  in netem_change()
1070 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_init()  local
1073 	qdisc_watchdog_init(&q->watchdog, sch);  in netem_init()
1078 	q->loss_model = CLG_RANDOM;  in netem_init()
1087 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_destroy()  local
1089 	qdisc_watchdog_cancel(&q->watchdog);  in netem_destroy()
1090 	if (q->qdisc)  in netem_destroy()
1091 		qdisc_put(q->qdisc);  in netem_destroy()
1092 	dist_free(q->delay_dist);  in netem_destroy()
1093 	dist_free(q->slot_dist);  in netem_destroy()
1096 static int dump_loss_model(const struct netem_sched_data *q,  in dump_loss_model()  argument
1105 	switch (q->loss_model) {  in dump_loss_model()
1113 			.p13 = q->clg.a1,  in dump_loss_model()
1114 			.p31 = q->clg.a2,  in dump_loss_model()
1115 			.p32 = q->clg.a3,  in dump_loss_model()
1116 			.p14 = q->clg.a4,  in dump_loss_model()
1117 			.p23 = q->clg.a5,  in dump_loss_model()
1126 			.p = q->clg.a1,  in dump_loss_model()
1127 			.r = q->clg.a2,  in dump_loss_model()
1128 			.h = q->clg.a3,  in dump_loss_model()
1129 			.k1 = q->clg.a4,  in dump_loss_model()
1148 	const struct netem_sched_data *q = qdisc_priv(sch);  in netem_dump()  local
1157 	qopt.latency = min_t(psched_time_t, PSCHED_NS2TICKS(q->latency),  in netem_dump()
1159 	qopt.jitter = min_t(psched_time_t, PSCHED_NS2TICKS(q->jitter),  in netem_dump()
1161 	qopt.limit = q->limit;  in netem_dump()
1162 	qopt.loss = q->loss;  in netem_dump()
1163 	qopt.gap = q->gap;  in netem_dump()
1164 	qopt.duplicate = q->duplicate;  in netem_dump()
1168 	if (nla_put(skb, TCA_NETEM_LATENCY64, sizeof(q->latency), &q->latency))  in netem_dump()
1171 	if (nla_put(skb, TCA_NETEM_JITTER64, sizeof(q->jitter), &q->jitter))  in netem_dump()
1174 	cor.delay_corr = q->delay_cor.rho;  in netem_dump()
1175 	cor.loss_corr = q->loss_cor.rho;  in netem_dump()
1176 	cor.dup_corr = q->dup_cor.rho;  in netem_dump()
1180 	reorder.probability = q->reorder;  in netem_dump()
1181 	reorder.correlation = q->reorder_cor.rho;  in netem_dump()
1185 	corrupt.probability = q->corrupt;  in netem_dump()
1186 	corrupt.correlation = q->corrupt_cor.rho;  in netem_dump()
1190 	if (q->rate >= (1ULL << 32)) {  in netem_dump()
1191 		if (nla_put_u64_64bit(skb, TCA_NETEM_RATE64, q->rate,  in netem_dump()
1196 		rate.rate = q->rate;  in netem_dump()
1198 	rate.packet_overhead = q->packet_overhead;  in netem_dump()
1199 	rate.cell_size = q->cell_size;  in netem_dump()
1200 	rate.cell_overhead = q->cell_overhead;  in netem_dump()
1204 	if (q->ecn && nla_put_u32(skb, TCA_NETEM_ECN, q->ecn))  in netem_dump()
1207 	if (dump_loss_model(q, skb) != 0)  in netem_dump()
1210 	if (q->slot_config.min_delay | q->slot_config.max_delay |  in netem_dump()
1211 	    q->slot_config.dist_jitter) {  in netem_dump()
1212 		slot = q->slot_config;  in netem_dump()
1221 	if (nla_put_u64_64bit(skb, TCA_NETEM_PRNG_SEED, q->prng.seed,  in netem_dump()
1235 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_dump_class()  local
1237 	if (cl != 1 || !q->qdisc) 	/* only one class */  in netem_dump_class()
1241 	tcm->tcm_info = q->qdisc->handle;  in netem_dump_class()
1249 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_graft()  local
1251 	*old = qdisc_replace(sch, new, &q->qdisc);  in netem_graft()
1257 	struct netem_sched_data *q = qdisc_priv(sch);  in netem_leaf()  local
1258 	return q->qdisc;  in netem_leaf()