68 	struct sw_flow *flow;  in ovs_flow_alloc()  local
71 	flow = kmem_cache_zalloc(flow_cache, GFP_KERNEL);  in ovs_flow_alloc()
72 	if (!flow)  in ovs_flow_alloc()
75 	flow->stats_last_writer = -1;  in ovs_flow_alloc()
86 	RCU_INIT_POINTER(flow->stats[0], stats);  in ovs_flow_alloc()
88 	cpumask_set_cpu(0, &flow->cpu_used_mask);  in ovs_flow_alloc()
90 	return flow;  in ovs_flow_alloc()
92 	kmem_cache_free(flow_cache, flow);  in ovs_flow_alloc()
101 static void flow_free(struct sw_flow *flow)  in flow_free()  argument
105 	if (ovs_identifier_is_key(&flow->id))  in flow_free()
106 		kfree(flow->id.unmasked_key);  in flow_free()
107 	if (flow->sf_acts)  in flow_free()
108 		ovs_nla_free_flow_actions((struct sw_flow_actions __force *)flow->sf_acts);  in flow_free()
110 	for (cpu = 0; cpu < nr_cpu_ids; cpu = cpumask_next(cpu, &flow->cpu_used_mask))  in flow_free()
111 		if (flow->stats[cpu])  in flow_free()
113 					(struct sw_flow_stats __force *)flow->stats[cpu]);  in flow_free()
114 	kmem_cache_free(flow_cache, flow);  in flow_free()
119 	struct sw_flow *flow = container_of(rcu, struct sw_flow, rcu);  in rcu_free_flow_callback()  local
121 	flow_free(flow);  in rcu_free_flow_callback()
124 void ovs_flow_free(struct sw_flow *flow, bool deferred)  in ovs_flow_free()  argument
126 	if (!flow)  in ovs_flow_free()
130 		call_rcu(&flow->rcu, rcu_free_flow_callback);  in ovs_flow_free()
132 		flow_free(flow);  in ovs_flow_free()
214 		struct sw_flow *flow;  in table_instance_destroy()  local
220 		hlist_for_each_entry_safe(flow, n, head, flow_table.node[ver]) {  in table_instance_destroy()
221 			hlist_del_rcu(&flow->flow_table.node[ver]);  in table_instance_destroy()
222 			if (ovs_identifier_is_ufid(&flow->id))  in table_instance_destroy()
223 				hlist_del_rcu(&flow->ufid_table.node[ufid_ver]);  in table_instance_destroy()
224 			ovs_flow_free(flow, deferred);  in table_instance_destroy()
252 	struct sw_flow *flow;  in ovs_flow_tbl_dump_next()  local
261 		hlist_for_each_entry_rcu(flow, head, flow_table.node[ver]) {  in ovs_flow_tbl_dump_next()
267 			return flow;  in ovs_flow_tbl_dump_next()
283 				  struct sw_flow *flow)  in table_instance_insert()  argument
287 	head = find_bucket(ti, flow->flow_table.hash);  in table_instance_insert()
288 	hlist_add_head_rcu(&flow->flow_table.node[ti->node_ver], head);  in table_instance_insert()
292 				       struct sw_flow *flow)  in ufid_table_instance_insert()  argument
296 	head = find_bucket(ti, flow->ufid_table.hash);  in ufid_table_instance_insert()
297 	hlist_add_head_rcu(&flow->ufid_table.node[ti->node_ver], head);  in ufid_table_instance_insert()
311 		struct sw_flow *flow;  in flow_table_copy_flows()  local
315 			hlist_for_each_entry(flow, head,  in flow_table_copy_flows()
317 				ufid_table_instance_insert(new, flow);  in flow_table_copy_flows()
319 			hlist_for_each_entry(flow, head,  in flow_table_copy_flows()
321 				table_instance_insert(new, flow);  in flow_table_copy_flows()
408 static bool flow_cmp_masked_key(const struct sw_flow *flow,  in flow_cmp_masked_key()  argument
412 	return cmp_key(&flow->key, key, range->start, range->end);  in flow_cmp_masked_key()
415 static bool ovs_flow_cmp_unmasked_key(const struct sw_flow *flow,  in ovs_flow_cmp_unmasked_key()  argument
422 	BUG_ON(ovs_identifier_is_ufid(&flow->id));  in ovs_flow_cmp_unmasked_key()
423 	return cmp_key(flow->id.unmasked_key, key, key_start, key_end);  in ovs_flow_cmp_unmasked_key()
430 	struct sw_flow *flow;  in masked_flow_lookup()  local
438 	hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver]) {  in masked_flow_lookup()
439 		if (flow->mask == mask && flow->flow_table.hash == hash &&  in masked_flow_lookup()
440 		    flow_cmp_masked_key(flow, &masked_key, &mask->range))  in masked_flow_lookup()
441 			return flow;  in masked_flow_lookup()
452 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_stats()  local
457 		flow = masked_flow_lookup(ti, key, mask);  in ovs_flow_tbl_lookup_stats()
458 		if (flow)  /* Found */  in ovs_flow_tbl_lookup_stats()
459 			return flow;  in ovs_flow_tbl_lookup_stats()
477 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_exact()  local
481 		flow = masked_flow_lookup(ti, match->key, mask);  in ovs_flow_tbl_lookup_exact()
482 		if (flow && ovs_identifier_is_key(&flow->id) &&  in ovs_flow_tbl_lookup_exact()
483 		    ovs_flow_cmp_unmasked_key(flow, match))  in ovs_flow_tbl_lookup_exact()
484 			return flow;  in ovs_flow_tbl_lookup_exact()
494 static bool ovs_flow_cmp_ufid(const struct sw_flow *flow,  in ovs_flow_cmp_ufid()  argument
497 	if (flow->id.ufid_len != sfid->ufid_len)  in ovs_flow_cmp_ufid()
500 	return !memcmp(flow->id.ufid, sfid->ufid, sfid->ufid_len);  in ovs_flow_cmp_ufid()
503 bool ovs_flow_cmp(const struct sw_flow *flow, const struct sw_flow_match *match)  in ovs_flow_cmp()  argument
505 	if (ovs_identifier_is_ufid(&flow->id))  in ovs_flow_cmp()
506 		return flow_cmp_masked_key(flow, match->key, &match->range);  in ovs_flow_cmp()
508 	return ovs_flow_cmp_unmasked_key(flow, match);  in ovs_flow_cmp()
515 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_ufid()  local
521 	hlist_for_each_entry_rcu(flow, head, ufid_table.node[ti->node_ver]) {  in ovs_flow_tbl_lookup_ufid()
522 		if (flow->ufid_table.hash == hash &&  in ovs_flow_tbl_lookup_ufid()
523 		    ovs_flow_cmp_ufid(flow, ufid))  in ovs_flow_tbl_lookup_ufid()
524 			return flow;  in ovs_flow_tbl_lookup_ufid()
565 void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow)  in ovs_flow_tbl_remove()  argument
571 	hlist_del_rcu(&flow->flow_table.node[ti->node_ver]);  in ovs_flow_tbl_remove()
573 	if (ovs_identifier_is_ufid(&flow->id)) {  in ovs_flow_tbl_remove()
574 		hlist_del_rcu(&flow->ufid_table.node[ufid_ti->node_ver]);  in ovs_flow_tbl_remove()
581 	flow_mask_remove(table, flow->mask);  in ovs_flow_tbl_remove()
622 static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow,  in flow_mask_insert()  argument
640 	flow->mask = mask;  in flow_mask_insert()
645 static void flow_key_insert(struct flow_table *table, struct sw_flow *flow)  in flow_key_insert()  argument
650 	flow->flow_table.hash = flow_hash(&flow->key, &flow->mask->range);  in flow_key_insert()
652 	table_instance_insert(ti, flow);  in flow_key_insert()
669 static void flow_ufid_insert(struct flow_table *table, struct sw_flow *flow)  in flow_ufid_insert()  argument
673 	flow->ufid_table.hash = ufid_hash(&flow->id);  in flow_ufid_insert()
675 	ufid_table_instance_insert(ti, flow);  in flow_ufid_insert()
691 int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,  in ovs_flow_tbl_insert()  argument
696 	err = flow_mask_insert(table, flow, mask);  in ovs_flow_tbl_insert()
699 	flow_key_insert(table, flow);  in ovs_flow_tbl_insert()
700 	if (ovs_identifier_is_ufid(&flow->id))  in ovs_flow_tbl_insert()
701 		flow_ufid_insert(table, flow);  in ovs_flow_tbl_insert()