176 static void __mask_array_destroy(struct mask_array *ma)  in __mask_array_destroy()  argument
178 	free_percpu(ma->masks_usage_stats);  in __mask_array_destroy()
179 	kfree(ma);  in __mask_array_destroy()
184 	struct mask_array *ma = container_of(rcu, struct mask_array, rcu);  in mask_array_rcu_cb()  local
186 	__mask_array_destroy(ma);  in mask_array_rcu_cb()
189 static void tbl_mask_array_reset_counters(struct mask_array *ma)  in tbl_mask_array_reset_counters()  argument
198 	for (i = 0; i < ma->max; i++) {  in tbl_mask_array_reset_counters()
199 		ma->masks_usage_zero_cntr[i] = 0;  in tbl_mask_array_reset_counters()
206 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in tbl_mask_array_reset_counters()
212 			ma->masks_usage_zero_cntr[i] += counter;  in tbl_mask_array_reset_counters()
275 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()  local
276 	int err, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_add_mask()
278 	if (ma_count >= ma->max) {  in tbl_mask_array_add_mask()
279 		err = tbl_mask_array_realloc(tbl, ma->max +  in tbl_mask_array_add_mask()
284 		ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()
289 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_add_mask()
292 	BUG_ON(ovsl_dereference(ma->masks[ma_count]));  in tbl_mask_array_add_mask()
294 	rcu_assign_pointer(ma->masks[ma_count], new);  in tbl_mask_array_add_mask()
295 	WRITE_ONCE(ma->count, ma_count + 1);  in tbl_mask_array_add_mask()
303 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_del_mask()  local
304 	int i, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_del_mask()
308 		if (mask == ovsl_dereference(ma->masks[i]))  in tbl_mask_array_del_mask()
316 	WRITE_ONCE(ma->count, ma_count - 1);  in tbl_mask_array_del_mask()
318 	rcu_assign_pointer(ma->masks[i], ma->masks[ma_count - 1]);  in tbl_mask_array_del_mask()
319 	RCU_INIT_POINTER(ma->masks[ma_count - 1], NULL);  in tbl_mask_array_del_mask()
324 	if (ma->max >= (MASK_ARRAY_SIZE_MIN * 2) &&  in tbl_mask_array_del_mask()
325 	    ma_count <= (ma->max / 3))  in tbl_mask_array_del_mask()
326 		tbl_mask_array_realloc(tbl, ma->max / 2);  in tbl_mask_array_del_mask()
328 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_del_mask()
417 	struct mask_array *ma;  in ovs_flow_tbl_init()  local
423 	ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN);  in ovs_flow_tbl_init()
424 	if (!ma)  in ovs_flow_tbl_init()
437 	rcu_assign_pointer(table->mask_array, ma);  in ovs_flow_tbl_init()
447 	__mask_array_destroy(ma);  in ovs_flow_tbl_init()
520 	struct mask_array *ma = rcu_dereference_raw(table->mask_array);  in ovs_flow_tbl_destroy()  local
523 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_tbl_destroy()
731 				   struct mask_array *ma,  in flow_lookup()  argument
737 	struct mask_array_stats *stats = this_cpu_ptr(ma->masks_usage_stats);  in flow_lookup()
742 	if (likely(*index < ma->max)) {  in flow_lookup()
743 		mask = rcu_dereference_ovsl(ma->masks[*index]);  in flow_lookup()
756 	for (i = 0; i < ma->max; i++)  {  in flow_lookup()
761 		mask = rcu_dereference_ovsl(ma->masks[i]);  in flow_lookup()
792 	struct mask_array *ma = rcu_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_stats()  local
805 		return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache,  in ovs_flow_tbl_lookup_stats()
826 			flow = flow_lookup(tbl, ti, ma, key, n_mask_hit,  in ovs_flow_tbl_lookup_stats()
840 	flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit,  in ovs_flow_tbl_lookup_stats()
853 	struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array);  in ovs_flow_tbl_lookup()  local
864 	flow = flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index);  in ovs_flow_tbl_lookup()
872 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_exact()  local
876 	for (i = 0; i < ma->max; i++) {  in ovs_flow_tbl_lookup_exact()
882 		mask = ovsl_dereference(ma->masks[i]);  in ovs_flow_tbl_lookup_exact()
940 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_tbl_num_masks()  local
941 	return READ_ONCE(ma->count);  in ovs_flow_tbl_num_masks()
992 	struct mask_array *ma;  in flow_mask_find()  local
995 	ma = ovsl_dereference(tbl->mask_array);  in flow_mask_find()
996 	for (i = 0; i < ma->max; i++) {  in flow_mask_find()
998 		t = ovsl_dereference(ma->masks[i]);  in flow_mask_find()
1109 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_masks_rebalance()  local
1116 	masks_and_count = kmalloc_array(ma->max, sizeof(*masks_and_count),  in ovs_flow_masks_rebalance()
1121 	for (i = 0; i < ma->max; i++) {  in ovs_flow_masks_rebalance()
1125 		mask = rcu_dereference_ovsl(ma->masks[i]);  in ovs_flow_masks_rebalance()
1137 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in ovs_flow_masks_rebalance()
1148 		masks_and_count[i].counter -= ma->masks_usage_zero_cntr[i];  in ovs_flow_masks_rebalance()
1153 		ma->masks_usage_zero_cntr[i] += masks_and_count[i].counter;  in ovs_flow_masks_rebalance()
1173 	new = tbl_mask_array_alloc(ma->max);  in ovs_flow_masks_rebalance()
1180 		if (ovsl_dereference(ma->masks[index]))  in ovs_flow_masks_rebalance()
1181 			new->masks[new->count++] = ma->masks[index];  in ovs_flow_masks_rebalance()
1185 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_masks_rebalance()