1 /*
2  * Copyright (C) 2017 Netronome Systems, Inc.
3  *
4  * This software is dual licensed under the GNU General License Version 2,
5  * June 1991 as shown in the file COPYING in the top-level directory of this
6  * source tree or the BSD 2-Clause License provided below.  You have the
7  * option to license this software under the complete terms of either license.
8  *
9  * The BSD 2-Clause License:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      1. Redistributions of source code must retain the above
16  *         copyright notice, this list of conditions and the following
17  *         disclaimer.
18  *
19  *      2. Redistributions in binary form must reproduce the above
20  *         copyright notice, this list of conditions and the following
21  *         disclaimer in the documentation and/or other materials
22  *         provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/hash.h>
35 #include <linux/hashtable.h>
36 #include <linux/jhash.h>
37 #include <linux/vmalloc.h>
38 #include <net/pkt_cls.h>
39 
40 #include "cmsg.h"
41 #include "main.h"
42 #include "../nfp_app.h"
43 
44 struct nfp_mask_id_table {
45 	struct hlist_node link;
46 	u32 hash_key;
47 	u32 ref_cnt;
48 	u8 mask_id;
49 };
50 
nfp_release_stats_entry(struct nfp_app * app,u32 stats_context_id)51 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id)
52 {
53 	struct nfp_flower_priv *priv = app->priv;
54 	struct circ_buf *ring;
55 
56 	ring = &priv->stats_ids.free_list;
57 	/* Check if buffer is full. */
58 	if (!CIRC_SPACE(ring->head, ring->tail, NFP_FL_STATS_ENTRY_RS *
59 			NFP_FL_STATS_ELEM_RS -
60 			NFP_FL_STATS_ELEM_RS + 1))
61 		return -ENOBUFS;
62 
63 	memcpy(&ring->buf[ring->head], &stats_context_id, NFP_FL_STATS_ELEM_RS);
64 	ring->head = (ring->head + NFP_FL_STATS_ELEM_RS) %
65 		     (NFP_FL_STATS_ENTRY_RS * NFP_FL_STATS_ELEM_RS);
66 
67 	return 0;
68 }
69 
nfp_get_stats_entry(struct nfp_app * app,u32 * stats_context_id)70 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id)
71 {
72 	struct nfp_flower_priv *priv = app->priv;
73 	u32 freed_stats_id, temp_stats_id;
74 	struct circ_buf *ring;
75 
76 	ring = &priv->stats_ids.free_list;
77 	freed_stats_id = NFP_FL_STATS_ENTRY_RS;
78 	/* Check for unallocated entries first. */
79 	if (priv->stats_ids.init_unalloc > 0) {
80 		*stats_context_id = priv->stats_ids.init_unalloc - 1;
81 		priv->stats_ids.init_unalloc--;
82 		return 0;
83 	}
84 
85 	/* Check if buffer is empty. */
86 	if (ring->head == ring->tail) {
87 		*stats_context_id = freed_stats_id;
88 		return -ENOENT;
89 	}
90 
91 	memcpy(&temp_stats_id, &ring->buf[ring->tail], NFP_FL_STATS_ELEM_RS);
92 	*stats_context_id = temp_stats_id;
93 	memcpy(&ring->buf[ring->tail], &freed_stats_id, NFP_FL_STATS_ELEM_RS);
94 	ring->tail = (ring->tail + NFP_FL_STATS_ELEM_RS) %
95 		     (NFP_FL_STATS_ENTRY_RS * NFP_FL_STATS_ELEM_RS);
96 
97 	return 0;
98 }
99 
100 /* Must be called with either RTNL or rcu_read_lock */
101 struct nfp_fl_payload *
nfp_flower_search_fl_table(struct nfp_app * app,unsigned long tc_flower_cookie,struct net_device * netdev,__be32 host_ctx)102 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
103 			   struct net_device *netdev, __be32 host_ctx)
104 {
105 	struct nfp_flower_priv *priv = app->priv;
106 	struct nfp_fl_payload *flower_entry;
107 
108 	hash_for_each_possible_rcu(priv->flow_table, flower_entry, link,
109 				   tc_flower_cookie)
110 		if (flower_entry->tc_flower_cookie == tc_flower_cookie &&
111 		    (!netdev || flower_entry->ingress_dev == netdev) &&
112 		    (host_ctx == NFP_FL_STATS_CTX_DONT_CARE ||
113 		     flower_entry->meta.host_ctx_id == host_ctx))
114 			return flower_entry;
115 
116 	return NULL;
117 }
118 
119 static void
nfp_flower_update_stats(struct nfp_app * app,struct nfp_fl_stats_frame * stats)120 nfp_flower_update_stats(struct nfp_app *app, struct nfp_fl_stats_frame *stats)
121 {
122 	struct nfp_fl_payload *nfp_flow;
123 	unsigned long flower_cookie;
124 
125 	flower_cookie = be64_to_cpu(stats->stats_cookie);
126 
127 	rcu_read_lock();
128 	nfp_flow = nfp_flower_search_fl_table(app, flower_cookie, NULL,
129 					      stats->stats_con_id);
130 	if (!nfp_flow)
131 		goto exit_rcu_unlock;
132 
133 	spin_lock(&nfp_flow->lock);
134 	nfp_flow->stats.pkts += be32_to_cpu(stats->pkt_count);
135 	nfp_flow->stats.bytes += be64_to_cpu(stats->byte_count);
136 	nfp_flow->stats.used = jiffies;
137 	spin_unlock(&nfp_flow->lock);
138 
139 exit_rcu_unlock:
140 	rcu_read_unlock();
141 }
142 
nfp_flower_rx_flow_stats(struct nfp_app * app,struct sk_buff * skb)143 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb)
144 {
145 	unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb);
146 	struct nfp_fl_stats_frame *stats_frame;
147 	unsigned char *msg;
148 	int i;
149 
150 	msg = nfp_flower_cmsg_get_data(skb);
151 
152 	stats_frame = (struct nfp_fl_stats_frame *)msg;
153 	for (i = 0; i < msg_len / sizeof(*stats_frame); i++)
154 		nfp_flower_update_stats(app, stats_frame + i);
155 }
156 
nfp_release_mask_id(struct nfp_app * app,u8 mask_id)157 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id)
158 {
159 	struct nfp_flower_priv *priv = app->priv;
160 	struct circ_buf *ring;
161 
162 	ring = &priv->mask_ids.mask_id_free_list;
163 	/* Checking if buffer is full. */
164 	if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0)
165 		return -ENOBUFS;
166 
167 	memcpy(&ring->buf[ring->head], &mask_id, NFP_FLOWER_MASK_ELEMENT_RS);
168 	ring->head = (ring->head + NFP_FLOWER_MASK_ELEMENT_RS) %
169 		     (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS);
170 
171 	priv->mask_ids.last_used[mask_id] = ktime_get();
172 
173 	return 0;
174 }
175 
nfp_mask_alloc(struct nfp_app * app,u8 * mask_id)176 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id)
177 {
178 	struct nfp_flower_priv *priv = app->priv;
179 	ktime_t reuse_timeout;
180 	struct circ_buf *ring;
181 	u8 temp_id, freed_id;
182 
183 	ring = &priv->mask_ids.mask_id_free_list;
184 	freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1;
185 	/* Checking for unallocated entries first. */
186 	if (priv->mask_ids.init_unallocated > 0) {
187 		*mask_id = priv->mask_ids.init_unallocated;
188 		priv->mask_ids.init_unallocated--;
189 		return 0;
190 	}
191 
192 	/* Checking if buffer is empty. */
193 	if (ring->head == ring->tail)
194 		goto err_not_found;
195 
196 	memcpy(&temp_id, &ring->buf[ring->tail], NFP_FLOWER_MASK_ELEMENT_RS);
197 	*mask_id = temp_id;
198 
199 	reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id],
200 				     NFP_FL_MASK_REUSE_TIME_NS);
201 
202 	if (ktime_before(ktime_get(), reuse_timeout))
203 		goto err_not_found;
204 
205 	memcpy(&ring->buf[ring->tail], &freed_id, NFP_FLOWER_MASK_ELEMENT_RS);
206 	ring->tail = (ring->tail + NFP_FLOWER_MASK_ELEMENT_RS) %
207 		     (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS);
208 
209 	return 0;
210 
211 err_not_found:
212 	*mask_id = freed_id;
213 	return -ENOENT;
214 }
215 
216 static int
nfp_add_mask_table(struct nfp_app * app,char * mask_data,u32 mask_len)217 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
218 {
219 	struct nfp_flower_priv *priv = app->priv;
220 	struct nfp_mask_id_table *mask_entry;
221 	unsigned long hash_key;
222 	u8 mask_id;
223 
224 	if (nfp_mask_alloc(app, &mask_id))
225 		return -ENOENT;
226 
227 	mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL);
228 	if (!mask_entry) {
229 		nfp_release_mask_id(app, mask_id);
230 		return -ENOMEM;
231 	}
232 
233 	INIT_HLIST_NODE(&mask_entry->link);
234 	mask_entry->mask_id = mask_id;
235 	hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
236 	mask_entry->hash_key = hash_key;
237 	mask_entry->ref_cnt = 1;
238 	hash_add(priv->mask_table, &mask_entry->link, hash_key);
239 
240 	return mask_id;
241 }
242 
243 static struct nfp_mask_id_table *
nfp_search_mask_table(struct nfp_app * app,char * mask_data,u32 mask_len)244 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
245 {
246 	struct nfp_flower_priv *priv = app->priv;
247 	struct nfp_mask_id_table *mask_entry;
248 	unsigned long hash_key;
249 
250 	hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
251 
252 	hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key)
253 		if (mask_entry->hash_key == hash_key)
254 			return mask_entry;
255 
256 	return NULL;
257 }
258 
259 static int
nfp_find_in_mask_table(struct nfp_app * app,char * mask_data,u32 mask_len)260 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
261 {
262 	struct nfp_mask_id_table *mask_entry;
263 
264 	mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
265 	if (!mask_entry)
266 		return -ENOENT;
267 
268 	mask_entry->ref_cnt++;
269 
270 	/* Casting u8 to int for later use. */
271 	return mask_entry->mask_id;
272 }
273 
274 static bool
nfp_check_mask_add(struct nfp_app * app,char * mask_data,u32 mask_len,u8 * meta_flags,u8 * mask_id)275 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len,
276 		   u8 *meta_flags, u8 *mask_id)
277 {
278 	int id;
279 
280 	id = nfp_find_in_mask_table(app, mask_data, mask_len);
281 	if (id < 0) {
282 		id = nfp_add_mask_table(app, mask_data, mask_len);
283 		if (id < 0)
284 			return false;
285 		*meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
286 	}
287 	*mask_id = id;
288 
289 	return true;
290 }
291 
292 static bool
nfp_check_mask_remove(struct nfp_app * app,char * mask_data,u32 mask_len,u8 * meta_flags,u8 * mask_id)293 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len,
294 		      u8 *meta_flags, u8 *mask_id)
295 {
296 	struct nfp_mask_id_table *mask_entry;
297 
298 	mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
299 	if (!mask_entry)
300 		return false;
301 
302 	if (meta_flags)
303 		*meta_flags &= ~NFP_FL_META_FLAG_MANAGE_MASK;
304 
305 	*mask_id = mask_entry->mask_id;
306 	mask_entry->ref_cnt--;
307 	if (!mask_entry->ref_cnt) {
308 		hash_del(&mask_entry->link);
309 		nfp_release_mask_id(app, *mask_id);
310 		kfree(mask_entry);
311 		if (meta_flags)
312 			*meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
313 	}
314 
315 	return true;
316 }
317 
nfp_compile_flow_metadata(struct nfp_app * app,struct tc_cls_flower_offload * flow,struct nfp_fl_payload * nfp_flow,struct net_device * netdev)318 int nfp_compile_flow_metadata(struct nfp_app *app,
319 			      struct tc_cls_flower_offload *flow,
320 			      struct nfp_fl_payload *nfp_flow,
321 			      struct net_device *netdev)
322 {
323 	struct nfp_flower_priv *priv = app->priv;
324 	struct nfp_fl_payload *check_entry;
325 	u8 new_mask_id;
326 	u32 stats_cxt;
327 
328 	if (nfp_get_stats_entry(app, &stats_cxt))
329 		return -ENOENT;
330 
331 	nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt);
332 	nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie);
333 
334 	new_mask_id = 0;
335 	if (!nfp_check_mask_add(app, nfp_flow->mask_data,
336 				nfp_flow->meta.mask_len,
337 				&nfp_flow->meta.flags, &new_mask_id)) {
338 		if (nfp_release_stats_entry(app, stats_cxt))
339 			return -EINVAL;
340 		return -ENOENT;
341 	}
342 
343 	nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
344 	priv->flower_version++;
345 
346 	/* Update flow payload with mask ids. */
347 	nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
348 	nfp_flow->stats.pkts = 0;
349 	nfp_flow->stats.bytes = 0;
350 	nfp_flow->stats.used = jiffies;
351 
352 	check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev,
353 						 NFP_FL_STATS_CTX_DONT_CARE);
354 	if (check_entry) {
355 		if (nfp_release_stats_entry(app, stats_cxt))
356 			return -EINVAL;
357 
358 		if (!nfp_check_mask_remove(app, nfp_flow->mask_data,
359 					   nfp_flow->meta.mask_len,
360 					   NULL, &new_mask_id))
361 			return -EINVAL;
362 
363 		return -EEXIST;
364 	}
365 
366 	return 0;
367 }
368 
nfp_modify_flow_metadata(struct nfp_app * app,struct nfp_fl_payload * nfp_flow)369 int nfp_modify_flow_metadata(struct nfp_app *app,
370 			     struct nfp_fl_payload *nfp_flow)
371 {
372 	struct nfp_flower_priv *priv = app->priv;
373 	u8 new_mask_id = 0;
374 	u32 temp_ctx_id;
375 
376 	nfp_check_mask_remove(app, nfp_flow->mask_data,
377 			      nfp_flow->meta.mask_len, &nfp_flow->meta.flags,
378 			      &new_mask_id);
379 
380 	nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
381 	priv->flower_version++;
382 
383 	/* Update flow payload with mask ids. */
384 	nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
385 
386 	/* Release the stats ctx id. */
387 	temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
388 
389 	return nfp_release_stats_entry(app, temp_ctx_id);
390 }
391 
nfp_flower_metadata_init(struct nfp_app * app)392 int nfp_flower_metadata_init(struct nfp_app *app)
393 {
394 	struct nfp_flower_priv *priv = app->priv;
395 
396 	hash_init(priv->mask_table);
397 	hash_init(priv->flow_table);
398 	get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed));
399 
400 	/* Init ring buffer and unallocated mask_ids. */
401 	priv->mask_ids.mask_id_free_list.buf =
402 		kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
403 			      NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL);
404 	if (!priv->mask_ids.mask_id_free_list.buf)
405 		return -ENOMEM;
406 
407 	priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1;
408 
409 	/* Init timestamps for mask id*/
410 	priv->mask_ids.last_used =
411 		kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
412 			      sizeof(*priv->mask_ids.last_used), GFP_KERNEL);
413 	if (!priv->mask_ids.last_used)
414 		goto err_free_mask_id;
415 
416 	/* Init ring buffer and unallocated stats_ids. */
417 	priv->stats_ids.free_list.buf =
418 		vmalloc(array_size(NFP_FL_STATS_ELEM_RS,
419 				   NFP_FL_STATS_ENTRY_RS));
420 	if (!priv->stats_ids.free_list.buf)
421 		goto err_free_last_used;
422 
423 	priv->stats_ids.init_unalloc = NFP_FL_REPEATED_HASH_MAX;
424 
425 	return 0;
426 
427 err_free_last_used:
428 	kfree(priv->mask_ids.last_used);
429 err_free_mask_id:
430 	kfree(priv->mask_ids.mask_id_free_list.buf);
431 	return -ENOMEM;
432 }
433 
nfp_flower_metadata_cleanup(struct nfp_app * app)434 void nfp_flower_metadata_cleanup(struct nfp_app *app)
435 {
436 	struct nfp_flower_priv *priv = app->priv;
437 
438 	if (!priv)
439 		return;
440 
441 	kfree(priv->mask_ids.mask_id_free_list.buf);
442 	kfree(priv->mask_ids.last_used);
443 	vfree(priv->stats_ids.free_list.buf);
444 }
445