15 			      struct fq_flow *flow,  in fq_adjust_removal()  argument
18 	struct fq_tin *tin = flow->tin;  in fq_adjust_removal()
22 	flow->backlog -= skb->len;  in fq_adjust_removal()
27 static void fq_rejigger_backlog(struct fq *fq, struct fq_flow *flow)  in fq_rejigger_backlog()  argument
31 	if (flow->backlog == 0) {  in fq_rejigger_backlog()
32 		list_del_init(&flow->backlogchain);  in fq_rejigger_backlog()
34 		i = flow;  in fq_rejigger_backlog()
37 			if (i->backlog < flow->backlog)  in fq_rejigger_backlog()
40 		list_move_tail(&flow->backlogchain,  in fq_rejigger_backlog()
46 				       struct fq_flow *flow)  in fq_flow_dequeue()  argument
52 	skb = __skb_dequeue(&flow->queue);  in fq_flow_dequeue()
56 	fq_adjust_removal(fq, flow, skb);  in fq_flow_dequeue()
57 	fq_rejigger_backlog(fq, flow);  in fq_flow_dequeue()
66 	struct fq_flow *flow;  in fq_tin_dequeue()  local
80 	flow = list_first_entry(head, struct fq_flow, flowchain);  in fq_tin_dequeue()
82 	if (flow->deficit <= 0) {  in fq_tin_dequeue()
83 		flow->deficit += fq->quantum;  in fq_tin_dequeue()
84 		list_move_tail(&flow->flowchain,  in fq_tin_dequeue()
89 	skb = dequeue_func(fq, tin, flow);  in fq_tin_dequeue()
94 			list_move_tail(&flow->flowchain, &tin->old_flows);  in fq_tin_dequeue()
96 			list_del_init(&flow->flowchain);  in fq_tin_dequeue()
97 			flow->tin = NULL;  in fq_tin_dequeue()
102 	flow->deficit -= skb->len;  in fq_tin_dequeue()
121 	struct fq_flow *flow;  in fq_flow_classify()  local
125 	flow = &fq->flows[idx];  in fq_flow_classify()
126 	if (flow->tin && flow->tin != tin) {  in fq_flow_classify()
127 		flow = get_default_func(fq, tin, idx, skb);  in fq_flow_classify()
132 	if (!flow->tin)  in fq_flow_classify()
135 	return flow;  in fq_flow_classify()
140 			      struct fq_flow *flow)  in fq_recalc_backlog()  argument
144 	if (list_empty(&flow->backlogchain))  in fq_recalc_backlog()
145 		list_add_tail(&flow->backlogchain, &fq->backlogs);  in fq_recalc_backlog()
147 	i = flow;  in fq_recalc_backlog()
150 		if (i->backlog > flow->backlog)  in fq_recalc_backlog()
153 	list_move(&flow->backlogchain, &i->backlogchain);  in fq_recalc_backlog()
162 	struct fq_flow *flow;  in fq_tin_enqueue()  local
167 	flow = fq_flow_classify(fq, tin, idx, skb, get_default_func);  in fq_tin_enqueue()
169 	flow->tin = tin;  in fq_tin_enqueue()
170 	flow->backlog += skb->len;  in fq_tin_enqueue()
176 	fq_recalc_backlog(fq, tin, flow);  in fq_tin_enqueue()
178 	if (list_empty(&flow->flowchain)) {  in fq_tin_enqueue()
179 		flow->deficit = fq->quantum;  in fq_tin_enqueue()
180 		list_add_tail(&flow->flowchain,  in fq_tin_enqueue()
184 	__skb_queue_tail(&flow->queue, skb);  in fq_tin_enqueue()
187 		flow = list_first_entry_or_null(&fq->backlogs,  in fq_tin_enqueue()
190 		if (!flow)  in fq_tin_enqueue()
193 		skb = fq_flow_dequeue(fq, flow);  in fq_tin_enqueue()
197 		free_func(fq, flow->tin, flow, skb);  in fq_tin_enqueue()
199 		flow->tin->overlimit++;  in fq_tin_enqueue()
209 			   struct fq_flow *flow,  in fq_flow_filter()  argument
214 	struct fq_tin *tin = flow->tin;  in fq_flow_filter()
219 	skb_queue_walk_safe(&flow->queue, skb, tmp) {  in fq_flow_filter()
220 		if (!filter_func(fq, tin, flow, skb, filter_data))  in fq_flow_filter()
223 		__skb_unlink(skb, &flow->queue);  in fq_flow_filter()
224 		fq_adjust_removal(fq, flow, skb);  in fq_flow_filter()
225 		free_func(fq, tin, flow, skb);  in fq_flow_filter()
228 	fq_rejigger_backlog(fq, flow);  in fq_flow_filter()
237 	struct fq_flow *flow;  in fq_tin_filter()  local
241 	list_for_each_entry(flow, &tin->new_flows, flowchain)  in fq_tin_filter()
242 		fq_flow_filter(fq, flow, filter_func, filter_data, free_func);  in fq_tin_filter()
243 	list_for_each_entry(flow, &tin->old_flows, flowchain)  in fq_tin_filter()
244 		fq_flow_filter(fq, flow, filter_func, filter_data, free_func);  in fq_tin_filter()
248 			  struct fq_flow *flow,  in fq_flow_reset()  argument
253 	while ((skb = fq_flow_dequeue(fq, flow)))  in fq_flow_reset()
254 		free_func(fq, flow->tin, flow, skb);  in fq_flow_reset()
256 	if (!list_empty(&flow->flowchain))  in fq_flow_reset()
257 		list_del_init(&flow->flowchain);  in fq_flow_reset()
259 	if (!list_empty(&flow->backlogchain))  in fq_flow_reset()
260 		list_del_init(&flow->backlogchain);  in fq_flow_reset()
262 	flow->tin = NULL;  in fq_flow_reset()
264 	WARN_ON_ONCE(flow->backlog);  in fq_flow_reset()
272 	struct fq_flow *flow;  in fq_tin_reset()  local
282 		flow = list_first_entry(head, struct fq_flow, flowchain);  in fq_tin_reset()
283 		fq_flow_reset(fq, flow, free_func);  in fq_tin_reset()
290 static void fq_flow_init(struct fq_flow *flow)  in fq_flow_init()  argument
292 	INIT_LIST_HEAD(&flow->flowchain);  in fq_flow_init()
293 	INIT_LIST_HEAD(&flow->backlogchain);  in fq_flow_init()
294 	__skb_queue_head_init(&flow->queue);  in fq_flow_init()