1 /*
2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #define CREATE_TRACE_POINTS
33 #include "lib/eq.h"
34 #include "fw_tracer.h"
35 #include "fw_tracer_tracepoint.h"
36
mlx5_query_mtrc_caps(struct mlx5_fw_tracer * tracer)37 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer)
38 {
39 u32 *string_db_base_address_out = tracer->str_db.base_address_out;
40 u32 *string_db_size_out = tracer->str_db.size_out;
41 struct mlx5_core_dev *dev = tracer->dev;
42 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
43 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
44 void *mtrc_cap_sp;
45 int err, i;
46
47 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
48 MLX5_REG_MTRC_CAP, 0, 0);
49 if (err) {
50 mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n",
51 err);
52 return err;
53 }
54
55 if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) {
56 mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n");
57 return -ENOTSUPP;
58 }
59
60 tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver);
61 tracer->str_db.first_string_trace =
62 MLX5_GET(mtrc_cap, out, first_string_trace);
63 tracer->str_db.num_string_trace =
64 MLX5_GET(mtrc_cap, out, num_string_trace);
65 tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db);
66 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
67
68 for (i = 0; i < tracer->str_db.num_string_db; i++) {
69 mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]);
70 string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param,
71 mtrc_cap_sp,
72 string_db_base_address);
73 string_db_size_out[i] = MLX5_GET(mtrc_string_db_param,
74 mtrc_cap_sp, string_db_size);
75 }
76
77 return err;
78 }
79
mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer * tracer,u32 * out,u32 out_size,u8 trace_owner)80 static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer,
81 u32 *out, u32 out_size,
82 u8 trace_owner)
83 {
84 struct mlx5_core_dev *dev = tracer->dev;
85 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
86
87 MLX5_SET(mtrc_cap, in, trace_owner, trace_owner);
88
89 return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size,
90 MLX5_REG_MTRC_CAP, 0, 1);
91 }
92
mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer * tracer)93 static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer)
94 {
95 struct mlx5_core_dev *dev = tracer->dev;
96 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
97 int err;
98
99 err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
100 MLX5_FW_TRACER_ACQUIRE_OWNERSHIP);
101 if (err) {
102 mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n",
103 err);
104 return err;
105 }
106
107 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
108
109 if (!tracer->owner)
110 return -EBUSY;
111
112 return 0;
113 }
114
mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer * tracer)115 static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer)
116 {
117 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
118
119 mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
120 MLX5_FW_TRACER_RELEASE_OWNERSHIP);
121 tracer->owner = false;
122 }
123
mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer * tracer)124 static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer)
125 {
126 struct mlx5_core_dev *dev = tracer->dev;
127 struct device *ddev = &dev->pdev->dev;
128 dma_addr_t dma;
129 void *buff;
130 gfp_t gfp;
131 int err;
132
133 tracer->buff.size = TRACE_BUFFER_SIZE_BYTE;
134
135 gfp = GFP_KERNEL | __GFP_ZERO;
136 buff = (void *)__get_free_pages(gfp,
137 get_order(tracer->buff.size));
138 if (!buff) {
139 err = -ENOMEM;
140 mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err);
141 return err;
142 }
143 tracer->buff.log_buf = buff;
144
145 dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE);
146 if (dma_mapping_error(ddev, dma)) {
147 mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n",
148 dma_mapping_error(ddev, dma));
149 err = -ENOMEM;
150 goto free_pages;
151 }
152 tracer->buff.dma = dma;
153
154 return 0;
155
156 free_pages:
157 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
158
159 return err;
160 }
161
mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer * tracer)162 static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer)
163 {
164 struct mlx5_core_dev *dev = tracer->dev;
165 struct device *ddev = &dev->pdev->dev;
166
167 if (!tracer->buff.log_buf)
168 return;
169
170 dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE);
171 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
172 }
173
mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer * tracer)174 static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer)
175 {
176 struct mlx5_core_dev *dev = tracer->dev;
177 int err, inlen, i;
178 __be64 *mtt;
179 void *mkc;
180 u32 *in;
181
182 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
183 sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2);
184
185 in = kvzalloc(inlen, GFP_KERNEL);
186 if (!in)
187 return -ENOMEM;
188
189 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
190 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
191 mtt = (u64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
192 for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++)
193 mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE);
194
195 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
196 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
197 MLX5_SET(mkc, mkc, lr, 1);
198 MLX5_SET(mkc, mkc, lw, 1);
199 MLX5_SET(mkc, mkc, pd, tracer->buff.pdn);
200 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
201 MLX5_SET(mkc, mkc, qpn, 0xffffff);
202 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
203 MLX5_SET(mkc, mkc, translations_octword_size,
204 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
205 MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma);
206 MLX5_SET64(mkc, mkc, len, tracer->buff.size);
207 err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen);
208 if (err)
209 mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err);
210
211 kvfree(in);
212
213 return err;
214 }
215
mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer * tracer)216 static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer)
217 {
218 u32 num_string_db = tracer->str_db.num_string_db;
219 int i;
220
221 for (i = 0; i < num_string_db; i++) {
222 kfree(tracer->str_db.buffer[i]);
223 tracer->str_db.buffer[i] = NULL;
224 }
225 }
226
mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer * tracer)227 static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer)
228 {
229 u32 *string_db_size_out = tracer->str_db.size_out;
230 u32 num_string_db = tracer->str_db.num_string_db;
231 int i;
232
233 for (i = 0; i < num_string_db; i++) {
234 tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL);
235 if (!tracer->str_db.buffer[i])
236 goto free_strings_db;
237 }
238
239 return 0;
240
241 free_strings_db:
242 mlx5_fw_tracer_free_strings_db(tracer);
243 return -ENOMEM;
244 }
245
246 static void
mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer * tracer)247 mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer *tracer)
248 {
249 tracer->st_arr.saved_traces_index = 0;
250 mutex_init(&tracer->st_arr.lock);
251 }
252
253 static void
mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer * tracer)254 mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer *tracer)
255 {
256 mutex_destroy(&tracer->st_arr.lock);
257 }
258
mlx5_tracer_read_strings_db(struct work_struct * work)259 static void mlx5_tracer_read_strings_db(struct work_struct *work)
260 {
261 struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer,
262 read_fw_strings_work);
263 u32 num_of_reads, num_string_db = tracer->str_db.num_string_db;
264 struct mlx5_core_dev *dev = tracer->dev;
265 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
266 u32 leftovers, offset;
267 int err = 0, i, j;
268 u32 *out, outlen;
269 void *out_value;
270
271 outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES;
272 out = kzalloc(outlen, GFP_KERNEL);
273 if (!out) {
274 err = -ENOMEM;
275 goto out;
276 }
277
278 for (i = 0; i < num_string_db; i++) {
279 offset = 0;
280 MLX5_SET(mtrc_stdb, in, string_db_index, i);
281 num_of_reads = tracer->str_db.size_out[i] /
282 STRINGS_DB_READ_SIZE_BYTES;
283 leftovers = (tracer->str_db.size_out[i] %
284 STRINGS_DB_READ_SIZE_BYTES) /
285 STRINGS_DB_LEFTOVER_SIZE_BYTES;
286
287 MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES);
288 for (j = 0; j < num_of_reads; j++) {
289 MLX5_SET(mtrc_stdb, in, start_offset, offset);
290
291 err = mlx5_core_access_reg(dev, in, sizeof(in), out,
292 outlen, MLX5_REG_MTRC_STDB,
293 0, 1);
294 if (err) {
295 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
296 err);
297 goto out_free;
298 }
299
300 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
301 memcpy(tracer->str_db.buffer[i] + offset, out_value,
302 STRINGS_DB_READ_SIZE_BYTES);
303 offset += STRINGS_DB_READ_SIZE_BYTES;
304 }
305
306 /* Strings database is aligned to 64, need to read leftovers*/
307 MLX5_SET(mtrc_stdb, in, read_size,
308 STRINGS_DB_LEFTOVER_SIZE_BYTES);
309 for (j = 0; j < leftovers; j++) {
310 MLX5_SET(mtrc_stdb, in, start_offset, offset);
311
312 err = mlx5_core_access_reg(dev, in, sizeof(in), out,
313 outlen, MLX5_REG_MTRC_STDB,
314 0, 1);
315 if (err) {
316 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
317 err);
318 goto out_free;
319 }
320
321 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
322 memcpy(tracer->str_db.buffer[i] + offset, out_value,
323 STRINGS_DB_LEFTOVER_SIZE_BYTES);
324 offset += STRINGS_DB_LEFTOVER_SIZE_BYTES;
325 }
326 }
327
328 tracer->str_db.loaded = true;
329
330 out_free:
331 kfree(out);
332 out:
333 return;
334 }
335
mlx5_fw_tracer_arm(struct mlx5_core_dev * dev)336 static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev)
337 {
338 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
339 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
340 int err;
341
342 MLX5_SET(mtrc_ctrl, in, arm_event, 1);
343
344 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
345 MLX5_REG_MTRC_CTRL, 0, 1);
346 if (err)
347 mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err);
348 }
349
350 static const char *VAL_PARM = "%llx";
351 static const char *REPLACE_64_VAL_PARM = "%x%x";
352 static const char *PARAM_CHAR = "%";
353
mlx5_tracer_message_hash(u32 message_id)354 static int mlx5_tracer_message_hash(u32 message_id)
355 {
356 return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1);
357 }
358
mlx5_tracer_message_insert(struct mlx5_fw_tracer * tracer,struct tracer_event * tracer_event)359 static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer,
360 struct tracer_event *tracer_event)
361 {
362 struct hlist_head *head =
363 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
364 struct tracer_string_format *cur_string;
365
366 cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL);
367 if (!cur_string)
368 return NULL;
369
370 hlist_add_head(&cur_string->hlist, head);
371
372 return cur_string;
373 }
374
mlx5_tracer_get_string(struct mlx5_fw_tracer * tracer,struct tracer_event * tracer_event)375 static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer,
376 struct tracer_event *tracer_event)
377 {
378 struct tracer_string_format *cur_string;
379 u32 str_ptr, offset;
380 int i;
381
382 str_ptr = tracer_event->string_event.string_param;
383
384 for (i = 0; i < tracer->str_db.num_string_db; i++) {
385 if (str_ptr > tracer->str_db.base_address_out[i] &&
386 str_ptr < tracer->str_db.base_address_out[i] +
387 tracer->str_db.size_out[i]) {
388 offset = str_ptr - tracer->str_db.base_address_out[i];
389 /* add it to the hash */
390 cur_string = mlx5_tracer_message_insert(tracer, tracer_event);
391 if (!cur_string)
392 return NULL;
393 cur_string->string = (char *)(tracer->str_db.buffer[i] +
394 offset);
395 return cur_string;
396 }
397 }
398
399 return NULL;
400 }
401
mlx5_tracer_clean_message(struct tracer_string_format * str_frmt)402 static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt)
403 {
404 hlist_del(&str_frmt->hlist);
405 kfree(str_frmt);
406 }
407
mlx5_tracer_get_num_of_params(char * str)408 static int mlx5_tracer_get_num_of_params(char *str)
409 {
410 char *substr, *pstr = str;
411 int num_of_params = 0;
412
413 /* replace %llx with %x%x */
414 substr = strstr(pstr, VAL_PARM);
415 while (substr) {
416 memcpy(substr, REPLACE_64_VAL_PARM, 4);
417 pstr = substr;
418 substr = strstr(pstr, VAL_PARM);
419 }
420
421 /* count all the % characters */
422 substr = strstr(str, PARAM_CHAR);
423 while (substr) {
424 num_of_params += 1;
425 str = substr + 1;
426 substr = strstr(str, PARAM_CHAR);
427 }
428
429 return num_of_params;
430 }
431
mlx5_tracer_message_find(struct hlist_head * head,u8 event_id,u32 tmsn)432 static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head,
433 u8 event_id, u32 tmsn)
434 {
435 struct tracer_string_format *message;
436
437 hlist_for_each_entry(message, head, hlist)
438 if (message->event_id == event_id && message->tmsn == tmsn)
439 return message;
440
441 return NULL;
442 }
443
mlx5_tracer_message_get(struct mlx5_fw_tracer * tracer,struct tracer_event * tracer_event)444 static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer,
445 struct tracer_event *tracer_event)
446 {
447 struct hlist_head *head =
448 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
449
450 return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn);
451 }
452
poll_trace(struct mlx5_fw_tracer * tracer,struct tracer_event * tracer_event,u64 * trace)453 static void poll_trace(struct mlx5_fw_tracer *tracer,
454 struct tracer_event *tracer_event, u64 *trace)
455 {
456 u32 timestamp_low, timestamp_mid, timestamp_high, urts;
457
458 tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id);
459 tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost);
460
461 switch (tracer_event->event_id) {
462 case TRACER_EVENT_TYPE_TIMESTAMP:
463 tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP;
464 urts = MLX5_GET(tracer_timestamp_event, trace, urts);
465 if (tracer->trc_ver == 0)
466 tracer_event->timestamp_event.unreliable = !!(urts >> 2);
467 else
468 tracer_event->timestamp_event.unreliable = !!(urts & 1);
469
470 timestamp_low = MLX5_GET(tracer_timestamp_event,
471 trace, timestamp7_0);
472 timestamp_mid = MLX5_GET(tracer_timestamp_event,
473 trace, timestamp39_8);
474 timestamp_high = MLX5_GET(tracer_timestamp_event,
475 trace, timestamp52_40);
476
477 tracer_event->timestamp_event.timestamp =
478 ((u64)timestamp_high << 40) |
479 ((u64)timestamp_mid << 8) |
480 (u64)timestamp_low;
481 break;
482 default:
483 if (tracer_event->event_id >= tracer->str_db.first_string_trace ||
484 tracer_event->event_id <= tracer->str_db.first_string_trace +
485 tracer->str_db.num_string_trace) {
486 tracer_event->type = TRACER_EVENT_TYPE_STRING;
487 tracer_event->string_event.timestamp =
488 MLX5_GET(tracer_string_event, trace, timestamp);
489 tracer_event->string_event.string_param =
490 MLX5_GET(tracer_string_event, trace, string_param);
491 tracer_event->string_event.tmsn =
492 MLX5_GET(tracer_string_event, trace, tmsn);
493 tracer_event->string_event.tdsn =
494 MLX5_GET(tracer_string_event, trace, tdsn);
495 } else {
496 tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED;
497 }
498 break;
499 }
500 }
501
get_block_timestamp(struct mlx5_fw_tracer * tracer,u64 * ts_event)502 static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event)
503 {
504 struct tracer_event tracer_event;
505 u8 event_id;
506
507 event_id = MLX5_GET(tracer_event, ts_event, event_id);
508
509 if (event_id == TRACER_EVENT_TYPE_TIMESTAMP)
510 poll_trace(tracer, &tracer_event, ts_event);
511 else
512 tracer_event.timestamp_event.timestamp = 0;
513
514 return tracer_event.timestamp_event.timestamp;
515 }
516
mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer * tracer)517 static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer)
518 {
519 struct tracer_string_format *str_frmt;
520 struct hlist_node *n;
521 int i;
522
523 for (i = 0; i < MESSAGE_HASH_SIZE; i++) {
524 hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist)
525 mlx5_tracer_clean_message(str_frmt);
526 }
527 }
528
mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer * tracer)529 static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer)
530 {
531 struct tracer_string_format *str_frmt, *tmp_str;
532
533 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list,
534 list)
535 list_del(&str_frmt->list);
536 }
537
mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer * tracer,u64 timestamp,bool lost,u8 event_id,char * msg)538 static void mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer *tracer,
539 u64 timestamp, bool lost,
540 u8 event_id, char *msg)
541 {
542 struct mlx5_fw_trace_data *trace_data;
543
544 mutex_lock(&tracer->st_arr.lock);
545 trace_data = &tracer->st_arr.straces[tracer->st_arr.saved_traces_index];
546 trace_data->timestamp = timestamp;
547 trace_data->lost = lost;
548 trace_data->event_id = event_id;
549 strscpy_pad(trace_data->msg, msg, TRACE_STR_MSG);
550
551 tracer->st_arr.saved_traces_index =
552 (tracer->st_arr.saved_traces_index + 1) & (SAVED_TRACES_NUM - 1);
553 mutex_unlock(&tracer->st_arr.lock);
554 }
555
556 static noinline
mlx5_tracer_print_trace(struct tracer_string_format * str_frmt,struct mlx5_core_dev * dev,u64 trace_timestamp)557 void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt,
558 struct mlx5_core_dev *dev,
559 u64 trace_timestamp)
560 {
561 char tmp[512];
562
563 snprintf(tmp, sizeof(tmp), str_frmt->string,
564 str_frmt->params[0],
565 str_frmt->params[1],
566 str_frmt->params[2],
567 str_frmt->params[3],
568 str_frmt->params[4],
569 str_frmt->params[5],
570 str_frmt->params[6]);
571
572 trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost,
573 str_frmt->event_id, tmp);
574
575 mlx5_fw_tracer_save_trace(dev->tracer, trace_timestamp,
576 str_frmt->lost, str_frmt->event_id, tmp);
577
578 /* remove it from hash */
579 mlx5_tracer_clean_message(str_frmt);
580 }
581
mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer * tracer,struct tracer_event * tracer_event)582 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
583 struct tracer_event *tracer_event)
584 {
585 struct tracer_string_format *cur_string;
586
587 if (tracer_event->string_event.tdsn == 0) {
588 cur_string = mlx5_tracer_get_string(tracer, tracer_event);
589 if (!cur_string)
590 return -1;
591
592 cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
593 cur_string->last_param_num = 0;
594 cur_string->event_id = tracer_event->event_id;
595 cur_string->tmsn = tracer_event->string_event.tmsn;
596 cur_string->timestamp = tracer_event->string_event.timestamp;
597 cur_string->lost = tracer_event->lost_event;
598 if (cur_string->num_of_params == 0) /* trace with no params */
599 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
600 } else {
601 cur_string = mlx5_tracer_message_get(tracer, tracer_event);
602 if (!cur_string) {
603 pr_debug("%s Got string event for unknown string tdsm: %d\n",
604 __func__, tracer_event->string_event.tmsn);
605 return -1;
606 }
607 cur_string->last_param_num += 1;
608 if (cur_string->last_param_num > TRACER_MAX_PARAMS) {
609 pr_debug("%s Number of params exceeds the max (%d)\n",
610 __func__, TRACER_MAX_PARAMS);
611 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
612 return 0;
613 }
614 /* keep the new parameter */
615 cur_string->params[cur_string->last_param_num - 1] =
616 tracer_event->string_event.string_param;
617 if (cur_string->last_param_num == cur_string->num_of_params)
618 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
619 }
620
621 return 0;
622 }
623
mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer * tracer,struct tracer_event * tracer_event)624 static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer,
625 struct tracer_event *tracer_event)
626 {
627 struct tracer_timestamp_event timestamp_event =
628 tracer_event->timestamp_event;
629 struct tracer_string_format *str_frmt, *tmp_str;
630 struct mlx5_core_dev *dev = tracer->dev;
631 u64 trace_timestamp;
632
633 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) {
634 list_del(&str_frmt->list);
635 if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0))
636 trace_timestamp = (timestamp_event.timestamp & MASK_52_7) |
637 (str_frmt->timestamp & MASK_6_0);
638 else
639 trace_timestamp = ((timestamp_event.timestamp & MASK_52_7) - 1) |
640 (str_frmt->timestamp & MASK_6_0);
641
642 mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp);
643 }
644 }
645
mlx5_tracer_handle_trace(struct mlx5_fw_tracer * tracer,struct tracer_event * tracer_event)646 static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer,
647 struct tracer_event *tracer_event)
648 {
649 if (tracer_event->type == TRACER_EVENT_TYPE_STRING) {
650 mlx5_tracer_handle_string_trace(tracer, tracer_event);
651 } else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) {
652 if (!tracer_event->timestamp_event.unreliable)
653 mlx5_tracer_handle_timestamp_trace(tracer, tracer_event);
654 } else {
655 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n",
656 __func__, tracer_event->type);
657 }
658 return 0;
659 }
660
mlx5_fw_tracer_handle_traces(struct work_struct * work)661 static void mlx5_fw_tracer_handle_traces(struct work_struct *work)
662 {
663 struct mlx5_fw_tracer *tracer =
664 container_of(work, struct mlx5_fw_tracer, handle_traces_work);
665 u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK];
666 u32 block_count, start_offset, prev_start_offset, prev_consumer_index;
667 u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event);
668 struct mlx5_core_dev *dev = tracer->dev;
669 struct tracer_event tracer_event;
670 int i;
671
672 mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner);
673 if (!tracer->owner)
674 return;
675
676 block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE;
677 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
678
679 /* Copy the block to local buffer to avoid HW override while being processed*/
680 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
681 TRACER_BLOCK_SIZE_BYTE);
682
683 block_timestamp =
684 get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]);
685
686 while (block_timestamp > tracer->last_timestamp) {
687 /* Check block override if its not the first block */
688 if (!tracer->last_timestamp) {
689 u64 *ts_event;
690 /* To avoid block override be the HW in case of buffer
691 * wraparound, the time stamp of the previous block
692 * should be compared to the last timestamp handled
693 * by the driver.
694 */
695 prev_consumer_index =
696 (tracer->buff.consumer_index - 1) & (block_count - 1);
697 prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE;
698
699 ts_event = tracer->buff.log_buf + prev_start_offset +
700 (TRACES_PER_BLOCK - 1) * trace_event_size;
701 last_block_timestamp = get_block_timestamp(tracer, ts_event);
702 /* If previous timestamp different from last stored
703 * timestamp then there is a good chance that the
704 * current buffer is overwritten and therefore should
705 * not be parsed.
706 */
707 if (tracer->last_timestamp != last_block_timestamp) {
708 mlx5_core_warn(dev, "FWTracer: Events were lost\n");
709 tracer->last_timestamp = block_timestamp;
710 tracer->buff.consumer_index =
711 (tracer->buff.consumer_index + 1) & (block_count - 1);
712 break;
713 }
714 }
715
716 /* Parse events */
717 for (i = 0; i < TRACES_PER_BLOCK ; i++) {
718 poll_trace(tracer, &tracer_event, &tmp_trace_block[i]);
719 mlx5_tracer_handle_trace(tracer, &tracer_event);
720 }
721
722 tracer->buff.consumer_index =
723 (tracer->buff.consumer_index + 1) & (block_count - 1);
724
725 tracer->last_timestamp = block_timestamp;
726 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
727 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
728 TRACER_BLOCK_SIZE_BYTE);
729 block_timestamp = get_block_timestamp(tracer,
730 &tmp_trace_block[TRACES_PER_BLOCK - 1]);
731 }
732
733 mlx5_fw_tracer_arm(dev);
734 }
735
mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer * tracer)736 static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer)
737 {
738 struct mlx5_core_dev *dev = tracer->dev;
739 u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
740 u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
741 int err;
742
743 MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY);
744 MLX5_SET(mtrc_conf, in, log_trace_buffer_size,
745 ilog2(TRACER_BUFFER_PAGE_NUM));
746 MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey.key);
747
748 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
749 MLX5_REG_MTRC_CONF, 0, 1);
750 if (err)
751 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err);
752
753 return err;
754 }
755
mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer * tracer,u8 status,u8 arm)756 static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm)
757 {
758 struct mlx5_core_dev *dev = tracer->dev;
759 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
760 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
761 int err;
762
763 MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS);
764 MLX5_SET(mtrc_ctrl, in, trace_status, status);
765 MLX5_SET(mtrc_ctrl, in, arm_event, arm);
766
767 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
768 MLX5_REG_MTRC_CTRL, 0, 1);
769
770 if (!err && status)
771 tracer->last_timestamp = 0;
772
773 return err;
774 }
775
mlx5_fw_tracer_start(struct mlx5_fw_tracer * tracer)776 static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer)
777 {
778 struct mlx5_core_dev *dev = tracer->dev;
779 int err;
780
781 err = mlx5_fw_tracer_ownership_acquire(tracer);
782 if (err) {
783 mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err);
784 /* Don't fail since ownership can be acquired on a later FW event */
785 return 0;
786 }
787
788 err = mlx5_fw_tracer_set_mtrc_conf(tracer);
789 if (err) {
790 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err);
791 goto release_ownership;
792 }
793
794 /* enable tracer & trace events */
795 err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1);
796 if (err) {
797 mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err);
798 goto release_ownership;
799 }
800
801 mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n");
802 return 0;
803
804 release_ownership:
805 mlx5_fw_tracer_ownership_release(tracer);
806 return err;
807 }
808
mlx5_fw_tracer_ownership_change(struct work_struct * work)809 static void mlx5_fw_tracer_ownership_change(struct work_struct *work)
810 {
811 struct mlx5_fw_tracer *tracer =
812 container_of(work, struct mlx5_fw_tracer, ownership_change_work);
813
814 mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner);
815 if (tracer->owner) {
816 tracer->owner = false;
817 tracer->buff.consumer_index = 0;
818 return;
819 }
820
821 mlx5_fw_tracer_start(tracer);
822 }
823
mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev * dev,u32 * in,int size_in)824 static int mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev *dev,
825 u32 *in, int size_in)
826 {
827 u32 out[MLX5_ST_SZ_DW(core_dump_reg)] = {};
828
829 if (!MLX5_CAP_DEBUG(dev, core_dump_general) &&
830 !MLX5_CAP_DEBUG(dev, core_dump_qp))
831 return -EOPNOTSUPP;
832
833 return mlx5_core_access_reg(dev, in, size_in, out, sizeof(out),
834 MLX5_REG_CORE_DUMP, 0, 1);
835 }
836
mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev * dev)837 int mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev *dev)
838 {
839 struct mlx5_fw_tracer *tracer = dev->tracer;
840 u32 in[MLX5_ST_SZ_DW(core_dump_reg)] = {};
841 int err;
842
843 if (!MLX5_CAP_DEBUG(dev, core_dump_general) || !tracer)
844 return -EOPNOTSUPP;
845 if (!tracer->owner)
846 return -EPERM;
847
848 MLX5_SET(core_dump_reg, in, core_dump_type, 0x0);
849
850 err = mlx5_fw_tracer_set_core_dump_reg(dev, in, sizeof(in));
851 if (err)
852 return err;
853 queue_work(tracer->work_queue, &tracer->handle_traces_work);
854 flush_workqueue(tracer->work_queue);
855 return 0;
856 }
857
858 static int
mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg * fmsg,struct mlx5_fw_trace_data * trace_data)859 mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg *fmsg,
860 struct mlx5_fw_trace_data *trace_data)
861 {
862 int err;
863
864 err = devlink_fmsg_obj_nest_start(fmsg);
865 if (err)
866 return err;
867
868 err = devlink_fmsg_u64_pair_put(fmsg, "timestamp", trace_data->timestamp);
869 if (err)
870 return err;
871
872 err = devlink_fmsg_bool_pair_put(fmsg, "lost", trace_data->lost);
873 if (err)
874 return err;
875
876 err = devlink_fmsg_u8_pair_put(fmsg, "event_id", trace_data->event_id);
877 if (err)
878 return err;
879
880 err = devlink_fmsg_string_pair_put(fmsg, "msg", trace_data->msg);
881 if (err)
882 return err;
883
884 err = devlink_fmsg_obj_nest_end(fmsg);
885 if (err)
886 return err;
887 return 0;
888 }
889
mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer * tracer,struct devlink_fmsg * fmsg)890 int mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer *tracer,
891 struct devlink_fmsg *fmsg)
892 {
893 struct mlx5_fw_trace_data *straces = tracer->st_arr.straces;
894 u32 index, start_index, end_index;
895 u32 saved_traces_index;
896 int err;
897
898 if (!straces[0].timestamp)
899 return -ENOMSG;
900
901 mutex_lock(&tracer->st_arr.lock);
902 saved_traces_index = tracer->st_arr.saved_traces_index;
903 if (straces[saved_traces_index].timestamp)
904 start_index = saved_traces_index;
905 else
906 start_index = 0;
907 end_index = (saved_traces_index - 1) & (SAVED_TRACES_NUM - 1);
908
909 err = devlink_fmsg_arr_pair_nest_start(fmsg, "dump fw traces");
910 if (err)
911 goto unlock;
912 index = start_index;
913 while (index != end_index) {
914 err = mlx5_devlink_fmsg_fill_trace(fmsg, &straces[index]);
915 if (err)
916 goto unlock;
917
918 index = (index + 1) & (SAVED_TRACES_NUM - 1);
919 }
920
921 err = devlink_fmsg_arr_pair_nest_end(fmsg);
922 unlock:
923 mutex_unlock(&tracer->st_arr.lock);
924 return err;
925 }
926
927 /* Create software resources (Buffers, etc ..) */
mlx5_fw_tracer_create(struct mlx5_core_dev * dev)928 struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev)
929 {
930 struct mlx5_fw_tracer *tracer = NULL;
931 int err;
932
933 if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) {
934 mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n");
935 return NULL;
936 }
937
938 tracer = kzalloc(sizeof(*tracer), GFP_KERNEL);
939 if (!tracer)
940 return ERR_PTR(-ENOMEM);
941
942 tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer");
943 if (!tracer->work_queue) {
944 err = -ENOMEM;
945 goto free_tracer;
946 }
947
948 tracer->dev = dev;
949
950 INIT_LIST_HEAD(&tracer->ready_strings_list);
951 INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change);
952 INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db);
953 INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces);
954
955
956 err = mlx5_query_mtrc_caps(tracer);
957 if (err) {
958 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err);
959 goto destroy_workqueue;
960 }
961
962 err = mlx5_fw_tracer_create_log_buf(tracer);
963 if (err) {
964 mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err);
965 goto destroy_workqueue;
966 }
967
968 err = mlx5_fw_tracer_allocate_strings_db(tracer);
969 if (err) {
970 mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err);
971 goto free_log_buf;
972 }
973
974 mlx5_fw_tracer_init_saved_traces_array(tracer);
975 mlx5_core_dbg(dev, "FWTracer: Tracer created\n");
976
977 return tracer;
978
979 free_log_buf:
980 mlx5_fw_tracer_destroy_log_buf(tracer);
981 destroy_workqueue:
982 tracer->dev = NULL;
983 destroy_workqueue(tracer->work_queue);
984 free_tracer:
985 kfree(tracer);
986 return ERR_PTR(err);
987 }
988
989 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data);
990
991 /* Create HW resources + start tracer */
mlx5_fw_tracer_init(struct mlx5_fw_tracer * tracer)992 int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer)
993 {
994 struct mlx5_core_dev *dev;
995 int err;
996
997 if (IS_ERR_OR_NULL(tracer))
998 return 0;
999
1000 dev = tracer->dev;
1001
1002 if (!tracer->str_db.loaded)
1003 queue_work(tracer->work_queue, &tracer->read_fw_strings_work);
1004
1005 err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn);
1006 if (err) {
1007 mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err);
1008 return err;
1009 }
1010
1011 err = mlx5_fw_tracer_create_mkey(tracer);
1012 if (err) {
1013 mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err);
1014 goto err_dealloc_pd;
1015 }
1016
1017 MLX5_NB_INIT(&tracer->nb, fw_tracer_event, DEVICE_TRACER);
1018 mlx5_eq_notifier_register(dev, &tracer->nb);
1019
1020 mlx5_fw_tracer_start(tracer);
1021
1022 return 0;
1023
1024 err_dealloc_pd:
1025 mlx5_core_dealloc_pd(dev, tracer->buff.pdn);
1026 return err;
1027 }
1028
1029 /* Stop tracer + Cleanup HW resources */
mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer * tracer)1030 void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer)
1031 {
1032 if (IS_ERR_OR_NULL(tracer))
1033 return;
1034
1035 mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n",
1036 tracer->owner);
1037 mlx5_eq_notifier_unregister(tracer->dev, &tracer->nb);
1038 cancel_work_sync(&tracer->ownership_change_work);
1039 cancel_work_sync(&tracer->handle_traces_work);
1040
1041 if (tracer->owner)
1042 mlx5_fw_tracer_ownership_release(tracer);
1043
1044 mlx5_core_destroy_mkey(tracer->dev, &tracer->buff.mkey);
1045 mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn);
1046 }
1047
1048 /* Free software resources (Buffers, etc ..) */
mlx5_fw_tracer_destroy(struct mlx5_fw_tracer * tracer)1049 void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer)
1050 {
1051 if (IS_ERR_OR_NULL(tracer))
1052 return;
1053
1054 mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n");
1055
1056 cancel_work_sync(&tracer->read_fw_strings_work);
1057 mlx5_fw_tracer_clean_ready_list(tracer);
1058 mlx5_fw_tracer_clean_print_hash(tracer);
1059 mlx5_fw_tracer_clean_saved_traces_array(tracer);
1060 mlx5_fw_tracer_free_strings_db(tracer);
1061 mlx5_fw_tracer_destroy_log_buf(tracer);
1062 flush_workqueue(tracer->work_queue);
1063 destroy_workqueue(tracer->work_queue);
1064 kfree(tracer);
1065 }
1066
fw_tracer_event(struct notifier_block * nb,unsigned long action,void * data)1067 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data)
1068 {
1069 struct mlx5_fw_tracer *tracer = mlx5_nb_cof(nb, struct mlx5_fw_tracer, nb);
1070 struct mlx5_core_dev *dev = tracer->dev;
1071 struct mlx5_eqe *eqe = data;
1072
1073 switch (eqe->sub_type) {
1074 case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE:
1075 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state))
1076 queue_work(tracer->work_queue, &tracer->ownership_change_work);
1077 break;
1078 case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE:
1079 if (likely(tracer->str_db.loaded))
1080 queue_work(tracer->work_queue, &tracer->handle_traces_work);
1081 break;
1082 default:
1083 mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n",
1084 eqe->sub_type);
1085 }
1086
1087 return NOTIFY_OK;
1088 }
1089
1090 EXPORT_TRACEPOINT_SYMBOL(mlx5_fw);
1091
