177 static void __mask_array_destroy(struct mask_array *ma)  in __mask_array_destroy()  argument
179 	free_percpu(ma->masks_usage_stats);  in __mask_array_destroy()
180 	kfree(ma);  in __mask_array_destroy()
185 	struct mask_array *ma = container_of(rcu, struct mask_array, rcu);  in mask_array_rcu_cb()  local
187 	__mask_array_destroy(ma);  in mask_array_rcu_cb()
190 static void tbl_mask_array_reset_counters(struct mask_array *ma)  in tbl_mask_array_reset_counters()  argument
199 	for (i = 0; i < ma->max; i++) {  in tbl_mask_array_reset_counters()
200 		ma->masks_usage_zero_cntr[i] = 0;  in tbl_mask_array_reset_counters()
207 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in tbl_mask_array_reset_counters()
213 			ma->masks_usage_zero_cntr[i] += counter;  in tbl_mask_array_reset_counters()
276 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()  local
277 	int err, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_add_mask()
279 	if (ma_count >= ma->max) {  in tbl_mask_array_add_mask()
280 		err = tbl_mask_array_realloc(tbl, ma->max +  in tbl_mask_array_add_mask()
285 		ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()
290 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_add_mask()
293 	BUG_ON(ovsl_dereference(ma->masks[ma_count]));  in tbl_mask_array_add_mask()
295 	rcu_assign_pointer(ma->masks[ma_count], new);  in tbl_mask_array_add_mask()
296 	WRITE_ONCE(ma->count, ma_count + 1);  in tbl_mask_array_add_mask()
304 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_del_mask()  local
305 	int i, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_del_mask()
309 		if (mask == ovsl_dereference(ma->masks[i]))  in tbl_mask_array_del_mask()
317 	WRITE_ONCE(ma->count, ma_count - 1);  in tbl_mask_array_del_mask()
319 	rcu_assign_pointer(ma->masks[i], ma->masks[ma_count - 1]);  in tbl_mask_array_del_mask()
320 	RCU_INIT_POINTER(ma->masks[ma_count - 1], NULL);  in tbl_mask_array_del_mask()
325 	if (ma->max >= (MASK_ARRAY_SIZE_MIN * 2) &&  in tbl_mask_array_del_mask()
326 	    ma_count <= (ma->max / 3))  in tbl_mask_array_del_mask()
327 		tbl_mask_array_realloc(tbl, ma->max / 2);  in tbl_mask_array_del_mask()
329 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_del_mask()
418 	struct mask_array *ma;  in ovs_flow_tbl_init()  local
424 	ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN);  in ovs_flow_tbl_init()
425 	if (!ma)  in ovs_flow_tbl_init()
438 	rcu_assign_pointer(table->mask_array, ma);  in ovs_flow_tbl_init()
448 	__mask_array_destroy(ma);  in ovs_flow_tbl_init()
521 	struct mask_array *ma = rcu_dereference_raw(table->mask_array);  in ovs_flow_tbl_destroy()  local
524 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_tbl_destroy()
732 				   struct mask_array *ma,  in flow_lookup()  argument
738 	struct mask_array_stats *stats = this_cpu_ptr(ma->masks_usage_stats);  in flow_lookup()
743 	if (likely(*index < ma->max)) {  in flow_lookup()
744 		mask = rcu_dereference_ovsl(ma->masks[*index]);  in flow_lookup()
757 	for (i = 0; i < ma->max; i++)  {  in flow_lookup()
762 		mask = rcu_dereference_ovsl(ma->masks[i]);  in flow_lookup()
793 	struct mask_array *ma = rcu_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_stats()  local
806 		return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache,  in ovs_flow_tbl_lookup_stats()
827 			flow = flow_lookup(tbl, ti, ma, key, n_mask_hit,  in ovs_flow_tbl_lookup_stats()
841 	flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit,  in ovs_flow_tbl_lookup_stats()
854 	struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array);  in ovs_flow_tbl_lookup()  local
865 	flow = flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index);  in ovs_flow_tbl_lookup()
873 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_exact()  local
877 	for (i = 0; i < ma->max; i++) {  in ovs_flow_tbl_lookup_exact()
883 		mask = ovsl_dereference(ma->masks[i]);  in ovs_flow_tbl_lookup_exact()
941 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_tbl_num_masks()  local
942 	return READ_ONCE(ma->count);  in ovs_flow_tbl_num_masks()
993 	struct mask_array *ma;  in flow_mask_find()  local
996 	ma = ovsl_dereference(tbl->mask_array);  in flow_mask_find()
997 	for (i = 0; i < ma->max; i++) {  in flow_mask_find()
999 		t = ovsl_dereference(ma->masks[i]);  in flow_mask_find()
1110 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_masks_rebalance()  local
1117 	masks_and_count = kmalloc_array(ma->max, sizeof(*masks_and_count),  in ovs_flow_masks_rebalance()
1122 	for (i = 0; i < ma->max; i++) {  in ovs_flow_masks_rebalance()
1126 		mask = rcu_dereference_ovsl(ma->masks[i]);  in ovs_flow_masks_rebalance()
1138 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in ovs_flow_masks_rebalance()
1148 		masks_and_count[i].counter -= ma->masks_usage_zero_cntr[i];  in ovs_flow_masks_rebalance()
1153 		ma->masks_usage_zero_cntr[i] += masks_and_count[i].counter;  in ovs_flow_masks_rebalance()
1173 	new = tbl_mask_array_alloc(ma->max);  in ovs_flow_masks_rebalance()
1180 		if (ovsl_dereference(ma->masks[index]))  in ovs_flow_masks_rebalance()
1181 			new->masks[new->count++] = ma->masks[index];  in ovs_flow_masks_rebalance()
1185 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_masks_rebalance()