Lines Matching +full:spare +full:- +full:regs
1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
12 * Copyright (C) 2004-2006 Ingo Molnar
64 * A selftest will lurk into the ring-buffer to count the
66 * insertions into the ring-buffer such as trace_printk could occurred
72 * If boot-time tracing including tracers/events via kernel cmdline
122 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
296 if (export->flags & flag) { in trace_process_export()
299 export->write(export, entry, size); in trace_process_export()
313 if (export->flags & TRACE_EXPORT_FUNCTION) in ftrace_exports_enable()
316 if (export->flags & TRACE_EXPORT_EVENT) in ftrace_exports_enable()
319 if (export->flags & TRACE_EXPORT_MARKER) in ftrace_exports_enable()
325 if (export->flags & TRACE_EXPORT_FUNCTION) in ftrace_exports_disable()
328 if (export->flags & TRACE_EXPORT_EVENT) in ftrace_exports_disable()
331 if (export->flags & TRACE_EXPORT_MARKER) in ftrace_exports_disable()
344 export = rcu_dereference_raw_check(export->next); in ftrace_exports()
353 rcu_assign_pointer(export->next, *list); in add_trace_export()
357 * the export->next pointer is valid before another CPU sees in add_trace_export()
368 for (p = list; *p != NULL; p = &(*p)->next) in rm_trace_export()
373 return -1; in rm_trace_export()
375 rcu_assign_pointer(*p, (*p)->next); in rm_trace_export()
401 if (WARN_ON_ONCE(!export->write)) in register_ftrace_export()
402 return -1; in register_ftrace_export()
446 * The global_trace is the descriptor that holds the top-level tracing
458 int ret = -ENODEV; in trace_array_get()
463 tr->ref++; in trace_array_get()
475 WARN_ON(!this_tr->ref); in __trace_array_put()
476 this_tr->ref--; in __trace_array_put()
480 * trace_array_put - Decrement the reference counter for this trace array.
508 return -ENODEV; in tracing_check_open_get_tr()
511 return -ENODEV; in tracing_check_open_get_tr()
520 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && in call_filter_check_discard()
521 !filter_match_preds(call->filter, rec)) { in call_filter_check_discard()
530 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
543 * trace_ignore_this_task - should a task be ignored for tracing
566 !trace_find_filtered_pid(filtered_pids, task->pid)) || in trace_ignore_this_task()
568 trace_find_filtered_pid(filtered_no_pids, task->pid)); in trace_ignore_this_task()
572 * trace_filter_add_remove_task - Add or remove a task from a pid_list
592 if (!trace_find_filtered_pid(pid_list, self->pid)) in trace_filter_add_remove_task()
598 trace_pid_list_set(pid_list, task->pid); in trace_filter_add_remove_task()
600 trace_pid_list_clear(pid_list, task->pid); in trace_filter_add_remove_task()
604 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
633 * trace_pid_start - Used for seq_file to start reading pid lists
662 * trace_pid_show - show the current pid in seq_file processing
671 unsigned long pid = (unsigned long)v - 1; in trace_pid_show()
694 return -ENOMEM; in trace_pid_write()
705 return -ENOMEM; in trace_pid_write()
729 cnt -= ret; in trace_pid_write()
734 ret = -EINVAL; in trace_pid_write()
741 ret = -1; in trace_pid_write()
772 if (!buf->buffer) in buffer_ftrace_now()
775 ts = ring_buffer_time_stamp(buf->buffer); in buffer_ftrace_now()
776 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); in buffer_ftrace_now()
787 * tracing_is_enabled - Show if global_trace has been enabled
846 * These primitives don't distinguish read-only and read-consume access.
847 * Multi read-only access are also serialized.
913 int skip, struct pt_regs *regs);
917 int skip, struct pt_regs *regs);
922 int skip, struct pt_regs *regs) in __ftrace_trace_stack() argument
928 int skip, struct pt_regs *regs) in ftrace_trace_stack() argument
960 if (tr->array_buffer.buffer) in tracer_tracing_on()
961 ring_buffer_record_on(tr->array_buffer.buffer); in tracer_tracing_on()
970 tr->buffer_disabled = 0; in tracer_tracing_on()
976 * tracing_on - enable tracing buffers
995 /* Length is in event->array[0] */ in __buffer_unlock_commit()
996 ring_buffer_write(buffer, event->array[0], &event->array[1]); in __buffer_unlock_commit()
1006 * __trace_puts - write a constant string into the trace buffer.
1038 entry->ip = ip; in __trace_puts()
1040 memcpy(&entry->buf, str, size); in __trace_puts()
1043 if (entry->buf[size - 1] != '\n') { in __trace_puts()
1044 entry->buf[size] = '\n'; in __trace_puts()
1045 entry->buf[size + 1] = '\0'; in __trace_puts()
1047 entry->buf[size] = '\0'; in __trace_puts()
1058 * __trace_bputs - write the pointer to a constant string into trace buffer
1087 entry->ip = ip; in __trace_bputs()
1088 entry->str = str; in __trace_bputs()
1104 struct tracer *tracer = tr->current_trace; in tracing_snapshot_instance_cond()
1113 if (!tr->allocated_snapshot) { in tracing_snapshot_instance_cond()
1121 if (tracer->use_max_tr) { in tracing_snapshot_instance_cond()
1138 * tracing_snapshot - take a snapshot of the current buffer.
1160 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1165 * conditional - the snapshot will only happen if the
1179 * tracing_cond_snapshot_data - get the user data associated with a snapshot
1183 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1187 * the tr->max_lock lock, which the code calling
1197 arch_spin_lock(&tr->max_lock); in tracing_cond_snapshot_data()
1199 if (tr->cond_snapshot) in tracing_cond_snapshot_data()
1200 cond_data = tr->cond_snapshot->cond_data; in tracing_cond_snapshot_data()
1202 arch_spin_unlock(&tr->max_lock); in tracing_cond_snapshot_data()
1217 if (!tr->allocated_snapshot) { in tracing_alloc_snapshot_instance()
1219 /* allocate spare buffer */ in tracing_alloc_snapshot_instance()
1220 ret = resize_buffer_duplicate_size(&tr->max_buffer, in tracing_alloc_snapshot_instance()
1221 &tr->array_buffer, RING_BUFFER_ALL_CPUS); in tracing_alloc_snapshot_instance()
1225 tr->allocated_snapshot = true; in tracing_alloc_snapshot_instance()
1235 * The max_tr ring buffer has some state (e.g. ring->clock) and in free_snapshot()
1238 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); in free_snapshot()
1239 set_buffer_entries(&tr->max_buffer, 1); in free_snapshot()
1240 tracing_reset_online_cpus(&tr->max_buffer); in free_snapshot()
1241 tr->allocated_snapshot = false; in free_snapshot()
1245 * tracing_alloc_snapshot - allocate snapshot buffer.
1248 * allocated - it doesn't also take a snapshot.
1267 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1290 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1297 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1310 return -ENOMEM; in tracing_snapshot_cond_enable()
1312 cond_snapshot->cond_data = cond_data; in tracing_snapshot_cond_enable()
1313 cond_snapshot->update = update; in tracing_snapshot_cond_enable()
1321 if (tr->current_trace->use_max_tr) { in tracing_snapshot_cond_enable()
1322 ret = -EBUSY; in tracing_snapshot_cond_enable()
1334 if (tr->cond_snapshot) { in tracing_snapshot_cond_enable()
1335 ret = -EBUSY; in tracing_snapshot_cond_enable()
1340 arch_spin_lock(&tr->max_lock); in tracing_snapshot_cond_enable()
1341 tr->cond_snapshot = cond_snapshot; in tracing_snapshot_cond_enable()
1342 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_cond_enable()
1357 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1362 * otherwise return -EINVAL.
1371 arch_spin_lock(&tr->max_lock); in tracing_snapshot_cond_disable()
1373 if (!tr->cond_snapshot) in tracing_snapshot_cond_disable()
1374 ret = -EINVAL; in tracing_snapshot_cond_disable()
1376 kfree(tr->cond_snapshot); in tracing_snapshot_cond_disable()
1377 tr->cond_snapshot = NULL; in tracing_snapshot_cond_disable()
1380 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_cond_disable()
1400 return -ENODEV; in tracing_alloc_snapshot()
1416 return -ENODEV; in tracing_snapshot_cond_enable()
1428 if (tr->array_buffer.buffer) in tracer_tracing_off()
1429 ring_buffer_record_off(tr->array_buffer.buffer); in tracer_tracing_off()
1438 tr->buffer_disabled = 1; in tracer_tracing_off()
1444 * tracing_off - turn off tracing buffers
1467 * tracer_tracing_is_on - show real state of ring buffer enabled
1474 if (tr->array_buffer.buffer) in tracer_tracing_is_on()
1475 return ring_buffer_record_is_on(tr->array_buffer.buffer); in tracer_tracing_is_on()
1476 return !tr->buffer_disabled; in tracer_tracing_is_on()
1480 * tracing_is_on - show state of ring buffers enabled
1559 if (trace_clocks[tr->clock_id].in_ns) in trace_clock_in_ns()
1566 * trace_parser_get_init - gets the buffer for trace parser
1572 parser->buffer = kmalloc(size, GFP_KERNEL); in trace_parser_get_init()
1573 if (!parser->buffer) in trace_parser_get_init()
1576 parser->size = size; in trace_parser_get_init()
1581 * trace_parser_put - frees the buffer for trace parser
1585 kfree(parser->buffer); in trace_parser_put()
1586 parser->buffer = NULL; in trace_parser_put()
1590 * trace_get_user - reads the user input string separated by space
1615 cnt--; in trace_get_user()
1621 if (!parser->cont) { in trace_get_user()
1628 cnt--; in trace_get_user()
1631 parser->idx = 0; in trace_get_user()
1641 /* read the non-space input */ in trace_get_user()
1643 if (parser->idx < parser->size - 1) in trace_get_user()
1644 parser->buffer[parser->idx++] = ch; in trace_get_user()
1646 ret = -EINVAL; in trace_get_user()
1653 cnt--; in trace_get_user()
1658 parser->buffer[parser->idx] = 0; in trace_get_user()
1659 parser->cont = false; in trace_get_user()
1660 } else if (parser->idx < parser->size - 1) { in trace_get_user()
1661 parser->cont = true; in trace_get_user()
1662 parser->buffer[parser->idx++] = ch; in trace_get_user()
1664 parser->buffer[parser->idx] = 0; in trace_get_user()
1666 ret = -EINVAL; in trace_get_user()
1682 if (trace_seq_used(s) <= s->seq.readpos) in trace_seq_to_buffer()
1683 return -EBUSY; in trace_seq_to_buffer()
1685 len = trace_seq_used(s) - s->seq.readpos; in trace_seq_to_buffer()
1688 memcpy(buf, s->buffer + s->seq.readpos, cnt); in trace_seq_to_buffer()
1690 s->seq.readpos += cnt; in trace_seq_to_buffer()
1705 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY); in latency_fsnotify_workfn()
1712 queue_work(fsnotify_wq, &tr->fsnotify_work); in latency_fsnotify_workfn_irq()
1718 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); in trace_create_maxlat_file()
1719 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); in trace_create_maxlat_file()
1720 tr->d_max_latency = trace_create_file("tracing_max_latency", in trace_create_maxlat_file()
1722 d_tracer, &tr->max_latency, in trace_create_maxlat_file()
1732 return -ENOMEM; in latency_fsnotify_init()
1744 * We cannot call queue_work(&tr->fsnotify_work) from here because it's in latency_fsnotify()
1748 irq_work_queue(&tr->fsnotify_irqwork); in latency_fsnotify()
1756 d_tracer, &tr->max_latency, &tracing_max_lat_fops)
1764 * Copy the new maximum trace into the separate maximum-trace
1771 struct array_buffer *trace_buf = &tr->array_buffer; in __update_max_tr()
1772 struct array_buffer *max_buf = &tr->max_buffer; in __update_max_tr()
1773 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); in __update_max_tr()
1774 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); in __update_max_tr()
1776 max_buf->cpu = cpu; in __update_max_tr()
1777 max_buf->time_start = data->preempt_timestamp; in __update_max_tr()
1779 max_data->saved_latency = tr->max_latency; in __update_max_tr()
1780 max_data->critical_start = data->critical_start; in __update_max_tr()
1781 max_data->critical_end = data->critical_end; in __update_max_tr()
1783 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN); in __update_max_tr()
1784 max_data->pid = tsk->pid; in __update_max_tr()
1790 max_data->uid = current_uid(); in __update_max_tr()
1792 max_data->uid = task_uid(tsk); in __update_max_tr()
1794 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; in __update_max_tr()
1795 max_data->policy = tsk->policy; in __update_max_tr()
1796 max_data->rt_priority = tsk->rt_priority; in __update_max_tr()
1804 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1817 if (tr->stop_count) in update_max_tr()
1822 if (!tr->allocated_snapshot) { in update_max_tr()
1824 WARN_ON_ONCE(tr->current_trace != &nop_trace); in update_max_tr()
1828 arch_spin_lock(&tr->max_lock); in update_max_tr()
1831 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) in update_max_tr()
1832 ring_buffer_record_on(tr->max_buffer.buffer); in update_max_tr()
1834 ring_buffer_record_off(tr->max_buffer.buffer); in update_max_tr()
1837 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) in update_max_tr()
1840 swap(tr->array_buffer.buffer, tr->max_buffer.buffer); in update_max_tr()
1845 arch_spin_unlock(&tr->max_lock); in update_max_tr()
1849 * update_max_tr_single - only copy one trace over, and reset the rest
1861 if (tr->stop_count) in update_max_tr_single()
1865 if (!tr->allocated_snapshot) { in update_max_tr_single()
1867 WARN_ON_ONCE(tr->current_trace != &nop_trace); in update_max_tr_single()
1871 arch_spin_lock(&tr->max_lock); in update_max_tr_single()
1873 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu); in update_max_tr_single()
1875 if (ret == -EBUSY) { in update_max_tr_single()
1882 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, in update_max_tr_single()
1886 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); in update_max_tr_single()
1889 arch_spin_unlock(&tr->max_lock); in update_max_tr_single()
1896 if (trace_buffer_iter(iter, iter->cpu_file)) in wait_on_pipe()
1899 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, in wait_on_pipe()
1919 return -ENOMEM; in save_selftest()
1921 selftest->type = type; in save_selftest()
1922 list_add(&selftest->list, &postponed_selftests); in save_selftest()
1929 struct tracer *saved_tracer = tr->current_trace; in run_tracer_selftest()
1932 if (!type->selftest || tracing_selftest_disabled) in run_tracer_selftest()
1945 type->name); in run_tracer_selftest()
1956 tracing_reset_online_cpus(&tr->array_buffer); in run_tracer_selftest()
1958 tr->current_trace = type; in run_tracer_selftest()
1961 if (type->use_max_tr) { in run_tracer_selftest()
1964 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, in run_tracer_selftest()
1966 tr->allocated_snapshot = true; in run_tracer_selftest()
1971 pr_info("Testing tracer %s: ", type->name); in run_tracer_selftest()
1972 ret = type->selftest(type, tr); in run_tracer_selftest()
1974 tr->current_trace = saved_tracer; in run_tracer_selftest()
1979 return -1; in run_tracer_selftest()
1982 tracing_reset_online_cpus(&tr->array_buffer); in run_tracer_selftest()
1985 if (type->use_max_tr) { in run_tracer_selftest()
1986 tr->allocated_snapshot = false; in run_tracer_selftest()
1990 ring_buffer_resize(tr->max_buffer.buffer, 1, in run_tracer_selftest()
2020 ret = run_tracer_selftest(p->type); in init_trace_selftests()
2024 p->type->name); in init_trace_selftests()
2026 for (t = trace_types; t; t = t->next) { in init_trace_selftests()
2027 if (t == p->type) { in init_trace_selftests()
2028 *last = t->next; in init_trace_selftests()
2031 last = &t->next; in init_trace_selftests()
2034 list_del(&p->list); in init_trace_selftests()
2057 * register_tracer - register a tracer with the ftrace system.
2067 if (!type->name) { in register_tracer()
2069 return -1; in register_tracer()
2072 if (strlen(type->name) >= MAX_TRACER_SIZE) { in register_tracer()
2074 return -1; in register_tracer()
2079 type->name); in register_tracer()
2080 return -EPERM; in register_tracer()
2087 for (t = trace_types; t; t = t->next) { in register_tracer()
2088 if (strcmp(type->name, t->name) == 0) { in register_tracer()
2091 type->name); in register_tracer()
2092 ret = -1; in register_tracer()
2097 if (!type->set_flag) in register_tracer()
2098 type->set_flag = &dummy_set_flag; in register_tracer()
2099 if (!type->flags) { in register_tracer()
2101 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL); in register_tracer()
2102 if (!type->flags) { in register_tracer()
2103 ret = -ENOMEM; in register_tracer()
2106 type->flags->val = 0; in register_tracer()
2107 type->flags->opts = dummy_tracer_opt; in register_tracer()
2109 if (!type->flags->opts) in register_tracer()
2110 type->flags->opts = dummy_tracer_opt; in register_tracer()
2113 type->flags->trace = type; in register_tracer()
2119 type->next = trace_types; in register_tracer()
2130 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) in register_tracer()
2133 printk(KERN_INFO "Starting tracer '%s'\n", type->name); in register_tracer()
2135 tracing_set_tracer(&global_trace, type->name); in register_tracer()
2149 struct trace_buffer *buffer = buf->buffer; in tracing_reset_cpu()
2165 struct trace_buffer *buffer = buf->buffer; in tracing_reset_online_cpus()
2175 buf->time_start = buffer_ftrace_now(buf, buf->cpu); in tracing_reset_online_cpus()
2190 if (!tr->clear_trace) in tracing_reset_all_online_cpus_unlocked()
2192 tr->clear_trace = false; in tracing_reset_all_online_cpus_unlocked()
2193 tracing_reset_online_cpus(&tr->array_buffer); in tracing_reset_all_online_cpus_unlocked()
2195 tracing_reset_online_cpus(&tr->max_buffer); in tracing_reset_all_online_cpus_unlocked()
2235 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; in get_saved_cmdlines()
2246 s->map_cmdline_to_pid = kmalloc_array(val, in allocate_cmdlines_buffer()
2247 sizeof(*s->map_cmdline_to_pid), in allocate_cmdlines_buffer()
2249 if (!s->map_cmdline_to_pid) in allocate_cmdlines_buffer()
2250 return -ENOMEM; in allocate_cmdlines_buffer()
2252 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL); in allocate_cmdlines_buffer()
2253 if (!s->saved_cmdlines) { in allocate_cmdlines_buffer()
2254 kfree(s->map_cmdline_to_pid); in allocate_cmdlines_buffer()
2255 return -ENOMEM; in allocate_cmdlines_buffer()
2258 s->cmdline_idx = 0; in allocate_cmdlines_buffer()
2259 s->cmdline_num = val; in allocate_cmdlines_buffer()
2260 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, in allocate_cmdlines_buffer()
2261 sizeof(s->map_pid_to_cmdline)); in allocate_cmdlines_buffer()
2262 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, in allocate_cmdlines_buffer()
2263 val * sizeof(*s->map_cmdline_to_pid)); in allocate_cmdlines_buffer()
2274 return -ENOMEM; in trace_create_savedcmd()
2280 return -ENOMEM; in trace_create_savedcmd()
2292 * tracing_start - quick start of the tracer
2306 if (--global_trace.stop_count) { in tracing_start()
2343 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) in tracing_start_tr()
2346 raw_spin_lock_irqsave(&tr->start_lock, flags); in tracing_start_tr()
2348 if (--tr->stop_count) { in tracing_start_tr()
2349 if (tr->stop_count < 0) { in tracing_start_tr()
2352 tr->stop_count = 0; in tracing_start_tr()
2357 buffer = tr->array_buffer.buffer; in tracing_start_tr()
2362 raw_spin_unlock_irqrestore(&tr->start_lock, flags); in tracing_start_tr()
2366 * tracing_stop - quick stop of the tracer
2405 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) in tracing_stop_tr()
2408 raw_spin_lock_irqsave(&tr->start_lock, flags); in tracing_stop_tr()
2409 if (tr->stop_count++) in tracing_stop_tr()
2412 buffer = tr->array_buffer.buffer; in tracing_stop_tr()
2417 raw_spin_unlock_irqrestore(&tr->start_lock, flags); in tracing_stop_tr()
2425 if (!tsk->pid) in trace_save_cmdline()
2428 tpid = tsk->pid & (PID_MAX_DEFAULT - 1); in trace_save_cmdline()
2443 idx = savedcmd->map_pid_to_cmdline[tpid]; in trace_save_cmdline()
2445 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; in trace_save_cmdline()
2447 savedcmd->map_pid_to_cmdline[tpid] = idx; in trace_save_cmdline()
2448 savedcmd->cmdline_idx = idx; in trace_save_cmdline()
2451 savedcmd->map_cmdline_to_pid[idx] = tsk->pid; in trace_save_cmdline()
2452 set_cmdline(idx, tsk->comm); in trace_save_cmdline()
2474 tpid = pid & (PID_MAX_DEFAULT - 1); in __trace_find_cmdline()
2475 map = savedcmd->map_pid_to_cmdline[tpid]; in __trace_find_cmdline()
2477 tpid = savedcmd->map_cmdline_to_pid[map]; in __trace_find_cmdline()
2501 * if we observe a non-NULL tgid_map then we also observe the correct in trace_find_tgid_ptr()
2524 if (!tsk->pid) in trace_save_tgid()
2527 ptr = trace_find_tgid_ptr(tsk->pid); in trace_save_tgid()
2531 *ptr = tsk->tgid; in trace_save_tgid()
2545 * tracing_record_taskinfo - record the task info of a task
2573 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2630 return current->migration_disabled; in migration_disable_value()
2674 * trace_buffered_event_enable - enable buffering events
2735 * trace_buffered_event_disable - disable buffering events
2751 if (--trace_buffered_event_ref) in trace_buffered_event_disable()
2789 struct trace_array *tr = trace_file->tr; in trace_event_buffer_lock_reserve()
2792 *current_rb = tr->array_buffer.buffer; in trace_event_buffer_lock_reserve()
2794 if (!tr->no_filter_buffering_ref && in trace_event_buffer_lock_reserve()
2795 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) { in trace_event_buffer_lock_reserve()
2802 * (see include/linux/ring-buffer.h for details on in trace_event_buffer_lock_reserve()
2815 int max_len = PAGE_SIZE - struct_size(entry, array, 1); in trace_event_buffer_lock_reserve()
2838 entry->array[0] = len; in trace_event_buffer_lock_reserve()
2856 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { in trace_event_buffer_lock_reserve()
2880 event_call = fbuffer->trace_file->event_call; in output_printk()
2881 if (!event_call || !event_call->event.funcs || in output_printk()
2882 !event_call->event.funcs->trace) in output_printk()
2885 file = fbuffer->trace_file; in output_printk()
2886 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) || in output_printk()
2887 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) && in output_printk()
2888 !filter_match_preds(file->filter, fbuffer->entry))) in output_printk()
2891 event = &fbuffer->trace_file->event_call->event; in output_printk()
2894 trace_seq_init(&iter->seq); in output_printk()
2895 iter->ent = fbuffer->entry; in output_printk()
2896 event_call->event.funcs->trace(iter, 0, event); in output_printk()
2897 trace_seq_putc(&iter->seq, 0); in output_printk()
2898 printk("%s", iter->seq.buffer); in output_printk()
2939 struct trace_event_file *file = fbuffer->trace_file; in trace_event_buffer_commit()
2941 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event, in trace_event_buffer_commit()
2942 fbuffer->entry, &tt)) in trace_event_buffer_commit()
2949 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT); in trace_event_buffer_commit()
2951 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer, in trace_event_buffer_commit()
2952 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs); in trace_event_buffer_commit()
2974 struct pt_regs *regs) in trace_buffer_unlock_commit_regs() argument
2979 * If regs is not set, then skip the necessary functions. in trace_buffer_unlock_commit_regs()
2984 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs); in trace_buffer_unlock_commit_regs()
3003 struct trace_buffer *buffer = tr->array_buffer.buffer; in trace_function()
3012 entry->ip = ip; in trace_function()
3013 entry->parent_ip = parent_ip; in trace_function()
3043 int skip, struct pt_regs *regs) in __ftrace_trace_stack() argument
3054 * If regs is set, then these functions will not be in the way. in __ftrace_trace_stack()
3057 if (!regs) in __ftrace_trace_stack()
3063 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1; in __ftrace_trace_stack()
3079 size = ARRAY_SIZE(fstack->calls); in __ftrace_trace_stack()
3081 if (regs) { in __ftrace_trace_stack()
3082 nr_entries = stack_trace_save_regs(regs, fstack->calls, in __ftrace_trace_stack()
3085 nr_entries = stack_trace_save(fstack->calls, size, skip); in __ftrace_trace_stack()
3090 (sizeof(*entry) - sizeof(entry->caller)) + size, in __ftrace_trace_stack()
3096 memcpy(&entry->caller, fstack->calls, size); in __ftrace_trace_stack()
3097 entry->size = nr_entries; in __ftrace_trace_stack()
3113 int skip, struct pt_regs *regs) in ftrace_trace_stack() argument
3115 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) in ftrace_trace_stack()
3118 __ftrace_trace_stack(buffer, trace_ctx, skip, regs); in ftrace_trace_stack()
3124 struct trace_buffer *buffer = tr->array_buffer.buffer; in __trace_stack()
3146 * trace_dump_stack - record a stack back trace in the trace buffer
3174 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE)) in ftrace_trace_userstack()
3200 entry->tgid = current->tgid; in ftrace_trace_userstack()
3201 memset(&entry->caller, 0, sizeof(entry->caller)); in ftrace_trace_userstack()
3203 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES); in ftrace_trace_userstack()
3226 entry->bottom_delta_ts = delta & U32_MAX; in func_repeats_set_delta_ts()
3227 entry->top_delta_ts = (delta >> 32); in func_repeats_set_delta_ts()
3234 struct trace_buffer *buffer = tr->array_buffer.buffer; in trace_last_func_repeats()
3244 delta = ring_buffer_event_time_stamp(buffer, event) - in trace_last_func_repeats()
3245 last_info->ts_last_call; in trace_last_func_repeats()
3248 entry->ip = last_info->ip; in trace_last_func_repeats()
3249 entry->parent_ip = last_info->parent_ip; in trace_last_func_repeats()
3250 entry->count = last_info->count; in trace_last_func_repeats()
3272 if (!trace_percpu_buffer || buffer->nesting >= 4) in get_trace_buf()
3275 buffer->nesting++; in get_trace_buf()
3279 return &buffer->buffer[buffer->nesting - 1][0]; in get_trace_buf()
3286 this_cpu_dec(trace_percpu_buffer->nesting); in put_trace_buf()
3298 return -ENOMEM; in alloc_percpu_trace_buffer()
3367 * trace_vbprintk - write binary msg to tracing buffer
3404 buffer = tr->array_buffer.buffer; in trace_vbprintk()
3411 entry->ip = ip; in trace_vbprintk()
3412 entry->fmt = fmt; in trace_vbprintk()
3414 memcpy(entry->buf, tbuffer, sizeof(u32) * len); in trace_vbprintk()
3470 entry->ip = ip; in __trace_array_vprintk()
3472 memcpy(&entry->buf, tbuffer, len + 1); in __trace_array_vprintk()
3493 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args); in trace_array_vprintk()
3497 * trace_array_printk - Print a message to a specific instance
3524 return -ENOENT; in trace_array_printk()
3530 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) in trace_array_printk()
3541 * trace_array_init_printk - Initialize buffers for trace_array_printk()
3551 return -ENOENT; in trace_array_init_printk()
3555 return -EINVAL; in trace_array_init_printk()
3586 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); in trace_iterator_increment()
3588 iter->idx++; in trace_iterator_increment()
3604 (unsigned long)-1 : 0; in peek_next_entry()
3606 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts, in peek_next_entry()
3611 iter->ent_size = ring_buffer_event_length(event); in peek_next_entry()
3614 iter->ent_size = 0; in peek_next_entry()
3622 struct trace_buffer *buffer = iter->array_buffer->buffer; in __find_next_entry()
3625 int cpu_file = iter->cpu_file; in __find_next_entry()
3627 int next_cpu = -1; in __find_next_entry()
3660 next_size = iter->ent_size; in __find_next_entry()
3664 iter->ent_size = next_size; in __find_next_entry()
3686 * iter->tr is NULL when used with tp_printk, which makes in trace_iter_expand_format()
3689 if (!iter->tr || iter->fmt == static_fmt_buf) in trace_iter_expand_format()
3692 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE, in trace_iter_expand_format()
3695 iter->fmt_size += STATIC_FMT_BUF_SIZE; in trace_iter_expand_format()
3696 iter->fmt = tmp; in trace_iter_expand_format()
3715 if ((addr >= (unsigned long)iter->ent) && in trace_safe_str()
3716 (addr < (unsigned long)iter->ent + iter->ent_size)) in trace_safe_str()
3720 if ((addr >= (unsigned long)iter->tmp_seq.buffer) && in trace_safe_str()
3721 (addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE)) in trace_safe_str()
3735 if (!iter->ent) in trace_safe_str()
3738 trace_event = ftrace_find_event(iter->ent->type); in trace_safe_str()
3743 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module) in trace_safe_str()
3747 if (within_module_core(addr, event->module)) in trace_safe_str()
3755 struct seq_buf *seq = &s->seq; in show_buffer()
3759 return seq->buffer; in show_buffer()
3796 * trace_check_vprintf - Check dereferenced strings while writing to the seq buffer
3801 * This writes the data into the @iter->seq buffer using the data from
3821 if (iter->fmt == static_fmt_buf) in trace_check_vprintf()
3832 if (i + 1 >= iter->fmt_size) { in trace_check_vprintf()
3868 strncpy(iter->fmt, p, i); in trace_check_vprintf()
3869 iter->fmt[i] = '\0'; in trace_check_vprintf()
3870 trace_seq_vprintf(&iter->seq, iter->fmt, ap); in trace_check_vprintf()
3873 * If iter->seq is full, the above call no longer guarantees in trace_check_vprintf()
3879 if (iter->seq.full) { in trace_check_vprintf()
3896 * instead. See samples/trace_events/trace-events-sample.h in trace_check_vprintf()
3901 fmt, show_buffer(&iter->seq))) { in trace_check_vprintf()
3906 if (len + 1 > iter->fmt_size) in trace_check_vprintf()
3907 len = iter->fmt_size - 1; in trace_check_vprintf()
3910 ret = copy_from_kernel_nofault(iter->fmt, str, len); in trace_check_vprintf()
3911 iter->fmt[len] = 0; in trace_check_vprintf()
3914 ret = strncpy_from_kernel_nofault(iter->fmt, str, in trace_check_vprintf()
3915 iter->fmt_size); in trace_check_vprintf()
3918 trace_seq_printf(&iter->seq, "(0x%px)", str); in trace_check_vprintf()
3920 trace_seq_printf(&iter->seq, "(0x%px:%s)", in trace_check_vprintf()
3921 str, iter->fmt); in trace_check_vprintf()
3922 str = "[UNSAFE-MEMORY]"; in trace_check_vprintf()
3923 strcpy(iter->fmt, "%s"); in trace_check_vprintf()
3925 strncpy(iter->fmt, p + i, j + 1); in trace_check_vprintf()
3926 iter->fmt[j+1] = '\0'; in trace_check_vprintf()
3929 trace_seq_printf(&iter->seq, iter->fmt, len, str); in trace_check_vprintf()
3931 trace_seq_printf(&iter->seq, iter->fmt, str); in trace_check_vprintf()
3937 trace_seq_vprintf(&iter->seq, p, ap); in trace_check_vprintf()
3948 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR) in trace_event_format()
3952 new_fmt = q = iter->fmt; in trace_event_format()
3954 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) { in trace_event_format()
3958 q += iter->fmt - new_fmt; in trace_event_format()
3959 new_fmt = iter->fmt; in trace_event_format()
3965 if (p[-1] == '%') { in trace_event_format()
3987 int ent_size = iter->ent_size; in trace_find_next_entry()
3991 * If called from ftrace_dump(), then the iter->temp buffer in trace_find_next_entry()
3998 if (iter->temp == static_temp_buf && in trace_find_next_entry()
4004 * call ring_buffer_peek() that may make the contents of iter->ent in trace_find_next_entry()
4005 * undefined. Need to copy iter->ent now. in trace_find_next_entry()
4007 if (iter->ent && iter->ent != iter->temp) { in trace_find_next_entry()
4008 if ((!iter->temp || iter->temp_size < iter->ent_size) && in trace_find_next_entry()
4009 !WARN_ON_ONCE(iter->temp == static_temp_buf)) { in trace_find_next_entry()
4011 temp = kmalloc(iter->ent_size, GFP_KERNEL); in trace_find_next_entry()
4014 kfree(iter->temp); in trace_find_next_entry()
4015 iter->temp = temp; in trace_find_next_entry()
4016 iter->temp_size = iter->ent_size; in trace_find_next_entry()
4018 memcpy(iter->temp, iter->ent, iter->ent_size); in trace_find_next_entry()
4019 iter->ent = iter->temp; in trace_find_next_entry()
4023 iter->ent_size = ent_size; in trace_find_next_entry()
4031 iter->ent = __find_next_entry(iter, &iter->cpu, in trace_find_next_entry_inc()
4032 &iter->lost_events, &iter->ts); in trace_find_next_entry_inc()
4034 if (iter->ent) in trace_find_next_entry_inc()
4037 return iter->ent ? iter : NULL; in trace_find_next_entry_inc()
4042 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts, in trace_consume()
4043 &iter->lost_events); in trace_consume()
4048 struct trace_iterator *iter = m->private; in s_next()
4052 WARN_ON_ONCE(iter->leftover); in s_next()
4057 if (iter->idx > i) in s_next()
4060 if (iter->idx < 0) in s_next()
4065 while (ent && iter->idx < i) in s_next()
4068 iter->pos = *pos; in s_next()
4079 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0; in tracing_iter_reset()
4093 if (ts >= iter->array_buffer->time_start) in tracing_iter_reset()
4099 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries; in tracing_iter_reset()
4108 struct trace_iterator *iter = m->private; in s_start()
4109 struct trace_array *tr = iter->tr; in s_start()
4110 int cpu_file = iter->cpu_file; in s_start()
4117 * iter->trace is a copy of current_trace, the pointer to the in s_start()
4118 * name may be used instead of a strcmp(), as iter->trace->name in s_start()
4119 * will point to the same string as current_trace->name. in s_start()
4122 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) in s_start()
4123 *iter->trace = *tr->current_trace; in s_start()
4127 if (iter->snapshot && iter->trace->use_max_tr) in s_start()
4128 return ERR_PTR(-EBUSY); in s_start()
4131 if (*pos != iter->pos) { in s_start()
4132 iter->ent = NULL; in s_start()
4133 iter->cpu = 0; in s_start()
4134 iter->idx = -1; in s_start()
4142 iter->leftover = 0; in s_start()
4151 if (iter->leftover) in s_start()
4154 l = *pos - 1; in s_start()
4166 struct trace_iterator *iter = m->private; in s_stop()
4169 if (iter->snapshot && iter->trace->use_max_tr) in s_stop()
4173 trace_access_unlock(iter->cpu_file); in s_stop()
4183 count = ring_buffer_entries_cpu(buf->buffer, cpu); in get_total_entries_cpu()
4189 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { in get_total_entries_cpu()
4190 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; in get_total_entries_cpu()
4195 ring_buffer_overrun_cpu(buf->buffer, cpu); in get_total_entries_cpu()
4223 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu); in trace_total_entries_cpu()
4235 get_total_entries(&tr->array_buffer, &total, &entries); in trace_total_entries()
4242 seq_puts(m, "# _------=> CPU# \n" in print_lat_help_header()
4243 "# / _-----=> irqs-off/BH-disabled\n" in print_lat_help_header()
4244 "# | / _----=> need-resched \n" in print_lat_help_header()
4245 "# || / _---=> hardirq/softirq \n" in print_lat_help_header()
4246 "# ||| / _--=> preempt-depth \n" in print_lat_help_header()
4247 "# |||| / _-=> migrate-disable \n" in print_lat_help_header()
4259 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", in print_event_info()
4271 …seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "… in print_func_help_header()
4284 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space); in print_func_help_header_irq()
4285 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space); in print_func_help_header_irq()
4286 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space); in print_func_help_header_irq()
4287 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space); in print_func_help_header_irq()
4288 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space); in print_func_help_header_irq()
4290 …seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID … in print_func_help_header_irq()
4298 struct array_buffer *buf = iter->array_buffer; in print_trace_header()
4299 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); in print_trace_header()
4300 struct tracer *type = iter->trace; in print_trace_header()
4303 const char *name = type->name; in print_trace_header()
4309 seq_puts(m, "# -----------------------------------" in print_trace_header()
4310 "---------------------------------\n"); in print_trace_header()
4313 nsecs_to_usecs(data->saved_latency), in print_trace_header()
4316 buf->cpu, in print_trace_header()
4329 seq_puts(m, "# -----------------\n"); in print_trace_header()
4330 seq_printf(m, "# | task: %.16s-%d " in print_trace_header()
4332 data->comm, data->pid, in print_trace_header()
4333 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, in print_trace_header()
4334 data->policy, data->rt_priority); in print_trace_header()
4335 seq_puts(m, "# -----------------\n"); in print_trace_header()
4337 if (data->critical_start) { in print_trace_header()
4339 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); in print_trace_header()
4340 trace_print_seq(m, &iter->seq); in print_trace_header()
4342 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); in print_trace_header()
4343 trace_print_seq(m, &iter->seq); in print_trace_header()
4352 struct trace_seq *s = &iter->seq; in test_cpu_buff_start()
4353 struct trace_array *tr = iter->tr; in test_cpu_buff_start()
4355 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) in test_cpu_buff_start()
4358 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) in test_cpu_buff_start()
4361 if (cpumask_available(iter->started) && in test_cpu_buff_start()
4362 cpumask_test_cpu(iter->cpu, iter->started)) in test_cpu_buff_start()
4365 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries) in test_cpu_buff_start()
4368 if (cpumask_available(iter->started)) in test_cpu_buff_start()
4369 cpumask_set_cpu(iter->cpu, iter->started); in test_cpu_buff_start()
4372 if (iter->idx > 1) in test_cpu_buff_start()
4374 iter->cpu); in test_cpu_buff_start()
4379 struct trace_array *tr = iter->tr; in print_trace_fmt()
4380 struct trace_seq *s = &iter->seq; in print_trace_fmt()
4381 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); in print_trace_fmt()
4385 entry = iter->ent; in print_trace_fmt()
4389 event = ftrace_find_event(entry->type); in print_trace_fmt()
4391 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_trace_fmt()
4392 if (iter->iter_flags & TRACE_FILE_LAT_FMT) in print_trace_fmt()
4402 return event->funcs->trace(iter, sym_flags, event); in print_trace_fmt()
4404 trace_seq_printf(s, "Unknown type %d\n", entry->type); in print_trace_fmt()
4411 struct trace_array *tr = iter->tr; in print_raw_fmt()
4412 struct trace_seq *s = &iter->seq; in print_raw_fmt()
4416 entry = iter->ent; in print_raw_fmt()
4418 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) in print_raw_fmt()
4420 entry->pid, iter->cpu, iter->ts); in print_raw_fmt()
4425 event = ftrace_find_event(entry->type); in print_raw_fmt()
4427 return event->funcs->raw(iter, 0, event); in print_raw_fmt()
4429 trace_seq_printf(s, "%d ?\n", entry->type); in print_raw_fmt()
4436 struct trace_array *tr = iter->tr; in print_hex_fmt()
4437 struct trace_seq *s = &iter->seq; in print_hex_fmt()
4442 entry = iter->ent; in print_hex_fmt()
4444 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_hex_fmt()
4445 SEQ_PUT_HEX_FIELD(s, entry->pid); in print_hex_fmt()
4446 SEQ_PUT_HEX_FIELD(s, iter->cpu); in print_hex_fmt()
4447 SEQ_PUT_HEX_FIELD(s, iter->ts); in print_hex_fmt()
4452 event = ftrace_find_event(entry->type); in print_hex_fmt()
4454 enum print_line_t ret = event->funcs->hex(iter, 0, event); in print_hex_fmt()
4466 struct trace_array *tr = iter->tr; in print_bin_fmt()
4467 struct trace_seq *s = &iter->seq; in print_bin_fmt()
4471 entry = iter->ent; in print_bin_fmt()
4473 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_bin_fmt()
4474 SEQ_PUT_FIELD(s, entry->pid); in print_bin_fmt()
4475 SEQ_PUT_FIELD(s, iter->cpu); in print_bin_fmt()
4476 SEQ_PUT_FIELD(s, iter->ts); in print_bin_fmt()
4481 event = ftrace_find_event(entry->type); in print_bin_fmt()
4482 return event ? event->funcs->binary(iter, 0, event) : in print_bin_fmt()
4492 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in trace_empty()
4493 cpu = iter->cpu_file; in trace_empty()
4499 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) in trace_empty()
4511 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) in trace_empty()
4522 struct trace_array *tr = iter->tr; in print_trace_line()
4523 unsigned long trace_flags = tr->trace_flags; in print_trace_line()
4526 if (iter->lost_events) { in print_trace_line()
4527 if (iter->lost_events == (unsigned long)-1) in print_trace_line()
4528 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n", in print_trace_line()
4529 iter->cpu); in print_trace_line()
4531 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", in print_trace_line()
4532 iter->cpu, iter->lost_events); in print_trace_line()
4533 if (trace_seq_has_overflowed(&iter->seq)) in print_trace_line()
4537 if (iter->trace && iter->trace->print_line) { in print_trace_line()
4538 ret = iter->trace->print_line(iter); in print_trace_line()
4543 if (iter->ent->type == TRACE_BPUTS && in print_trace_line()
4548 if (iter->ent->type == TRACE_BPRINT && in print_trace_line()
4553 if (iter->ent->type == TRACE_PRINT && in print_trace_line()
4572 struct trace_iterator *iter = m->private; in trace_latency_header()
4573 struct trace_array *tr = iter->tr; in trace_latency_header()
4579 if (iter->iter_flags & TRACE_FILE_LAT_FMT) in trace_latency_header()
4582 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) in trace_latency_header()
4588 struct trace_iterator *iter = m->private; in trace_default_header()
4589 struct trace_array *tr = iter->tr; in trace_default_header()
4590 unsigned long trace_flags = tr->trace_flags; in trace_default_header()
4595 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { in trace_default_header()
4605 print_func_help_header_irq(iter->array_buffer, in trace_default_header()
4608 print_func_help_header(iter->array_buffer, m, in trace_default_header()
4650 if (iter->tr->allocated_snapshot) in print_snapshot_help()
4656 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) in print_snapshot_help()
4671 if (iter->ent == NULL) { in s_show()
4672 if (iter->tr) { in s_show()
4673 seq_printf(m, "# tracer: %s\n", iter->trace->name); in s_show()
4677 if (iter->snapshot && trace_empty(iter)) in s_show()
4679 else if (iter->trace && iter->trace->print_header) in s_show()
4680 iter->trace->print_header(m); in s_show()
4684 } else if (iter->leftover) { in s_show()
4689 ret = trace_print_seq(m, &iter->seq); in s_show()
4692 iter->leftover = ret; in s_show()
4696 ret = trace_print_seq(m, &iter->seq); in s_show()
4702 * -1 otherwise. in s_show()
4704 iter->leftover = ret; in s_show()
4716 if (inode->i_cdev) /* See trace_create_cpu_file() */ in tracing_get_cpu()
4717 return (long)inode->i_cdev - 1; in tracing_get_cpu()
4731 struct trace_array *tr = inode->i_private; in __tracing_open()
4736 return ERR_PTR(-ENODEV); in __tracing_open()
4740 return ERR_PTR(-ENOMEM); in __tracing_open()
4742 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), in __tracing_open()
4744 if (!iter->buffer_iter) in __tracing_open()
4748 * trace_find_next_entry() may need to save off iter->ent. in __tracing_open()
4749 * It will place it into the iter->temp buffer. As most in __tracing_open()
4752 * allocate a new buffer to adjust for the bigger iter->ent. in __tracing_open()
4755 iter->temp = kmalloc(128, GFP_KERNEL); in __tracing_open()
4756 if (iter->temp) in __tracing_open()
4757 iter->temp_size = 128; in __tracing_open()
4766 iter->fmt = NULL; in __tracing_open()
4767 iter->fmt_size = 0; in __tracing_open()
4774 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); in __tracing_open()
4775 if (!iter->trace) in __tracing_open()
4778 *iter->trace = *tr->current_trace; in __tracing_open()
4780 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) in __tracing_open()
4783 iter->tr = tr; in __tracing_open()
4787 if (tr->current_trace->print_max || snapshot) in __tracing_open()
4788 iter->array_buffer = &tr->max_buffer; in __tracing_open()
4791 iter->array_buffer = &tr->array_buffer; in __tracing_open()
4792 iter->snapshot = snapshot; in __tracing_open()
4793 iter->pos = -1; in __tracing_open()
4794 iter->cpu_file = tracing_get_cpu(inode); in __tracing_open()
4795 mutex_init(&iter->mutex); in __tracing_open()
4798 if (iter->trace->open) in __tracing_open()
4799 iter->trace->open(iter); in __tracing_open()
4802 if (ring_buffer_overruns(iter->array_buffer->buffer)) in __tracing_open()
4803 iter->iter_flags |= TRACE_FILE_ANNOTATE; in __tracing_open()
4806 if (trace_clocks[tr->clock_id].in_ns) in __tracing_open()
4807 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in __tracing_open()
4810 * If pause-on-trace is enabled, then stop the trace while in __tracing_open()
4813 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE)) in __tracing_open()
4816 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { in __tracing_open()
4818 iter->buffer_iter[cpu] = in __tracing_open()
4819 ring_buffer_read_prepare(iter->array_buffer->buffer, in __tracing_open()
4824 ring_buffer_read_start(iter->buffer_iter[cpu]); in __tracing_open()
4828 cpu = iter->cpu_file; in __tracing_open()
4829 iter->buffer_iter[cpu] = in __tracing_open()
4830 ring_buffer_read_prepare(iter->array_buffer->buffer, in __tracing_open()
4833 ring_buffer_read_start(iter->buffer_iter[cpu]); in __tracing_open()
4843 kfree(iter->trace); in __tracing_open()
4844 kfree(iter->temp); in __tracing_open()
4845 kfree(iter->buffer_iter); in __tracing_open()
4848 return ERR_PTR(-ENOMEM); in __tracing_open()
4859 filp->private_data = inode->i_private; in tracing_open_generic()
4874 struct trace_array *tr = inode->i_private; in tracing_open_generic_tr()
4881 filp->private_data = inode->i_private; in tracing_open_generic_tr()
4894 struct trace_array *tr = inode->i_private; in tracing_release()
4895 struct seq_file *m = file->private_data; in tracing_release()
4899 if (!(file->f_mode & FMODE_READ)) { in tracing_release()
4905 iter = m->private; in tracing_release()
4909 if (iter->buffer_iter[cpu]) in tracing_release()
4910 ring_buffer_read_finish(iter->buffer_iter[cpu]); in tracing_release()
4913 if (iter->trace && iter->trace->close) in tracing_release()
4914 iter->trace->close(iter); in tracing_release()
4916 if (!iter->snapshot && tr->stop_count) in tracing_release()
4924 mutex_destroy(&iter->mutex); in tracing_release()
4925 free_cpumask_var(iter->started); in tracing_release()
4926 kfree(iter->fmt); in tracing_release()
4927 kfree(iter->temp); in tracing_release()
4928 kfree(iter->trace); in tracing_release()
4929 kfree(iter->buffer_iter); in tracing_release()
4937 struct trace_array *tr = inode->i_private; in tracing_release_generic_tr()
4945 struct trace_array *tr = inode->i_private; in tracing_single_release_tr()
4954 struct trace_array *tr = inode->i_private; in tracing_open()
4963 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { in tracing_open()
4965 struct array_buffer *trace_buf = &tr->array_buffer; in tracing_open()
4968 if (tr->current_trace->print_max) in tracing_open()
4969 trace_buf = &tr->max_buffer; in tracing_open()
4978 if (file->f_mode & FMODE_READ) { in tracing_open()
4982 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) in tracing_open()
4983 iter->iter_flags |= TRACE_FILE_LAT_FMT; in tracing_open()
5000 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; in trace_ok_for_array()
5008 t = t->next; in get_tracer_for_array()
5016 struct trace_array *tr = m->private; in t_next()
5022 t = get_tracer_for_array(tr, t->next); in t_next()
5029 struct trace_array *tr = m->private; in t_start()
5054 seq_puts(m, t->name); in t_show()
5055 if (t->next) in t_show()
5072 struct trace_array *tr = inode->i_private; in show_traces_open()
5086 m = file->private_data; in show_traces_open()
5087 m->private = tr; in show_traces_open()
5094 struct trace_array *tr = inode->i_private; in show_traces_release()
5111 if (file->f_mode & FMODE_READ) in tracing_lseek()
5114 file->f_pos = ret = 0; in tracing_lseek()
5138 struct trace_array *tr = file_inode(filp)->i_private; in tracing_cpumask_read()
5143 cpumask_pr_args(tr->tracing_cpumask)) + 1; in tracing_cpumask_read()
5146 return -ENOMEM; in tracing_cpumask_read()
5149 cpumask_pr_args(tr->tracing_cpumask)); in tracing_cpumask_read()
5151 count = -EINVAL; in tracing_cpumask_read()
5168 return -EINVAL; in tracing_set_cpumask()
5171 arch_spin_lock(&tr->max_lock); in tracing_set_cpumask()
5177 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && in tracing_set_cpumask()
5179 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); in tracing_set_cpumask()
5180 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu); in tracing_set_cpumask()
5182 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && in tracing_set_cpumask()
5184 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); in tracing_set_cpumask()
5185 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu); in tracing_set_cpumask()
5188 arch_spin_unlock(&tr->max_lock); in tracing_set_cpumask()
5191 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); in tracing_set_cpumask()
5200 struct trace_array *tr = file_inode(filp)->i_private; in tracing_cpumask_write()
5205 return -ENOMEM; in tracing_cpumask_write()
5236 struct trace_array *tr = m->private; in tracing_trace_options_show()
5241 tracer_flags = tr->current_trace->flags->val; in tracing_trace_options_show()
5242 trace_opts = tr->current_trace->flags->opts; in tracing_trace_options_show()
5245 if (tr->trace_flags & (1 << i)) in tracing_trace_options_show()
5266 struct tracer *trace = tracer_flags->trace; in __set_tracer_option()
5269 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); in __set_tracer_option()
5274 tracer_flags->val &= ~opts->bit; in __set_tracer_option()
5276 tracer_flags->val |= opts->bit; in __set_tracer_option()
5283 struct tracer *trace = tr->current_trace; in set_tracer_option()
5284 struct tracer_flags *tracer_flags = trace->flags; in set_tracer_option()
5288 for (i = 0; tracer_flags->opts[i].name; i++) { in set_tracer_option()
5289 opts = &tracer_flags->opts[i]; in set_tracer_option()
5291 if (strcmp(cmp, opts->name) == 0) in set_tracer_option()
5292 return __set_tracer_option(tr, trace->flags, opts, neg); in set_tracer_option()
5295 return -EINVAL; in set_tracer_option()
5301 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) in trace_keep_overwrite()
5302 return -1; in trace_keep_overwrite()
5316 if (!!(tr->trace_flags & mask) == !!enabled) in set_tracer_flag()
5320 if (tr->current_trace->flag_changed) in set_tracer_flag()
5321 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) in set_tracer_flag()
5322 return -EINVAL; in set_tracer_flag()
5325 tr->trace_flags |= mask; in set_tracer_flag()
5327 tr->trace_flags &= ~mask; in set_tracer_flag()
5347 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; in set_tracer_flag()
5348 return -ENOMEM; in set_tracer_flag()
5361 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled); in set_tracer_flag()
5363 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); in set_tracer_flag()
5394 ret = match_string(trace_options, -1, cmp); in trace_set_options()
5430 *(buf - 1) = ','; in apply_trace_boot_options()
5438 struct seq_file *m = filp->private_data; in tracing_trace_options_write()
5439 struct trace_array *tr = m->private; in tracing_trace_options_write()
5444 return -EINVAL; in tracing_trace_options_write()
5447 return -EFAULT; in tracing_trace_options_write()
5462 struct trace_array *tr = inode->i_private; in tracing_trace_options_open()
5469 ret = single_open(file, tracing_trace_options_show, inode->i_private); in tracing_trace_options_open()
5485 "tracing mini-HOWTO:\n\n"
5487 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5489 " trace\t\t\t- The static contents of the buffer\n"
5491 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5492 " current_tracer\t- function and latency tracers\n"
5493 " available_tracers\t- list of configured tracers for current_tracer\n"
5494 " error_log\t- error log for failed commands (that support it)\n"
5495 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
5496 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
5497 " trace_clock\t\t- change the clock used to order events\n"
5504 " x86-tsc: TSC cycle counter\n"
5506 "\n timestamp_mode\t- view the mode used to timestamp events\n"
5507 " delta: Delta difference against a buffer-wide timestamp\n"
5509 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5510 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5511 " tracing_cpumask\t- Limit which CPUs to trace\n"
5512 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5513 "\t\t\t Remove sub-buffer with rmdir\n"
5514 " trace_options\t\t- Set format or modify how tracing happens\n"
5517 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5519 "\n available_filter_functions - list of functions that can be filtered on\n"
5520 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5522 "\t accepts: func_full_name or glob-matching-pattern\n"
5524 "\t Format: :mod:<module-name>\n"
5551 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5557 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5559 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5563 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5564 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5565 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5568 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5573 " stack_trace\t\t- Shows the max stack trace when active\n"
5574 " stack_max_size\t- Shows current max stack size that was traced\n"
5578 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5583 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5587 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5591 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5595 "\t accepts: event-definitions (one definition per line)\n"
5601 "\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>]\n"
5602 "\t -:[<group>/][<event>]\n"
5611 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
5617 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5619 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5620 "\t <type>\\[<array-size>\\]\n"
5627 "\t of the <attached-group>/<attached-event>.\n"
5629 " events/\t\t- Directory containing all trace event subsystems:\n"
5630 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5631 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5632 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5634 " filter\t\t- If set, only events passing filter are traced\n"
5635 " events/<system>/<event>/\t- Directory containing control files for\n"
5637 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5638 " filter\t\t- If set, only events passing filter are traced\n"
5639 " trigger\t\t- If set, a command to perform when event is hit\n"
5673 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5684 "\t common_timestamp - to record current timestamp\n"
5685 "\t common_cpu - to record the CPU the event happened on\n"
5688 "\t - a reference to a field e.g. x=current_timestamp,\n"
5689 "\t - a reference to another variable e.g. y=$x,\n"
5690 "\t - a numeric literal: e.g. ms_per_sec=1000,\n"
5691 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5693 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5722 "\t .sym-offset display an address as a symbol and offset\n"
5737 "\t already-attached hist trigger. The syntax is analogous to\n"
5743 "\t onmatch(matching.event) - invoke on addition or update\n"
5744 "\t onmax(var) - invoke if var exceeds current max\n"
5745 "\t onchange(var) - invoke action if var changes\n\n"
5747 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5748 "\t save(field,...) - save current event fields\n"
5750 "\t snapshot() - snapshot the trace buffer\n\n"
5753 " events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5795 int pid = entry - tgid_map; in saved_tgids_show()
5835 if (*pos || m->count) in saved_cmdlines_next()
5840 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; in saved_cmdlines_next()
5842 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) in saved_cmdlines_next()
5859 v = &savedcmd->map_cmdline_to_pid[0]; in saved_cmdlines_start()
5919 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); in tracing_saved_cmdlines_size_read()
5928 kfree(s->saved_cmdlines); in free_saved_cmdlines_buffer()
5929 kfree(s->map_cmdline_to_pid); in free_saved_cmdlines_buffer()
5939 return -ENOMEM; in tracing_resize_saved_cmdlines()
5943 return -ENOMEM; in tracing_resize_saved_cmdlines()
5970 return -EINVAL; in tracing_saved_cmdlines_size_write()
5991 if (!ptr->map.eval_string) { in update_eval_map()
5992 if (ptr->tail.next) { in update_eval_map()
5993 ptr = ptr->tail.next; in update_eval_map()
6049 ptr->map.eval_string, ptr->map.eval_value, in eval_map_show()
6050 ptr->map.system); in eval_map_show()
6084 return ptr + ptr->head.length + 1; in trace_eval_jmp_to_tail()
6117 if (!ptr->tail.next) in trace_insert_eval_map_file()
6119 ptr = ptr->tail.next; in trace_insert_eval_map_file()
6122 ptr->tail.next = map_array; in trace_insert_eval_map_file()
6124 map_array->head.mod = mod; in trace_insert_eval_map_file()
6125 map_array->head.length = len; in trace_insert_eval_map_file()
6129 map_array->map = **map; in trace_insert_eval_map_file()
6168 struct trace_array *tr = filp->private_data; in tracing_set_trace_read()
6173 r = sprintf(buf, "%s\n", tr->current_trace->name); in tracing_set_trace_read()
6181 tracing_reset_online_cpus(&tr->array_buffer); in tracer_init()
6182 return t->init(tr); in tracer_init()
6190 per_cpu_ptr(buf->data, cpu)->entries = val; in set_buffer_entries()
6202 ret = ring_buffer_resize(trace_buf->buffer, in resize_buffer_duplicate_size()
6203 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); in resize_buffer_duplicate_size()
6206 per_cpu_ptr(trace_buf->data, cpu)->entries = in resize_buffer_duplicate_size()
6207 per_cpu_ptr(size_buf->data, cpu)->entries; in resize_buffer_duplicate_size()
6210 ret = ring_buffer_resize(trace_buf->buffer, in resize_buffer_duplicate_size()
6211 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); in resize_buffer_duplicate_size()
6213 per_cpu_ptr(trace_buf->data, cpu_id)->entries = in resize_buffer_duplicate_size()
6214 per_cpu_ptr(size_buf->data, cpu_id)->entries; in resize_buffer_duplicate_size()
6234 if (!tr->array_buffer.buffer) in __tracing_resize_ring_buffer()
6237 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu); in __tracing_resize_ring_buffer()
6242 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) || in __tracing_resize_ring_buffer()
6243 !tr->current_trace->use_max_tr) in __tracing_resize_ring_buffer()
6246 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); in __tracing_resize_ring_buffer()
6248 int r = resize_buffer_duplicate_size(&tr->array_buffer, in __tracing_resize_ring_buffer()
6249 &tr->array_buffer, cpu); in __tracing_resize_ring_buffer()
6272 set_buffer_entries(&tr->max_buffer, size); in __tracing_resize_ring_buffer()
6274 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size; in __tracing_resize_ring_buffer()
6280 set_buffer_entries(&tr->array_buffer, size); in __tracing_resize_ring_buffer()
6282 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size; in __tracing_resize_ring_buffer()
6297 ret = -EINVAL; in tracing_resize_ring_buffer()
6304 ret = -ENOMEM; in tracing_resize_ring_buffer()
6314 * tracing_update_buffers - used by tracing facility to expand ring buffers
6347 if (tr->current_trace == &nop_trace) in tracing_set_nop()
6350 tr->current_trace->enabled--; in tracing_set_nop()
6352 if (tr->current_trace->reset) in tracing_set_nop()
6353 tr->current_trace->reset(tr); in tracing_set_nop()
6355 tr->current_trace = &nop_trace; in tracing_set_nop()
6363 if (!tr->dir) in add_tracer_options()
6391 for (t = trace_types; t; t = t->next) { in tracing_set_tracer()
6392 if (strcmp(t->name, buf) == 0) in tracing_set_tracer()
6396 ret = -EINVAL; in tracing_set_tracer()
6399 if (t == tr->current_trace) in tracing_set_tracer()
6403 if (t->use_max_tr) { in tracing_set_tracer()
6405 arch_spin_lock(&tr->max_lock); in tracing_set_tracer()
6406 if (tr->cond_snapshot) in tracing_set_tracer()
6407 ret = -EBUSY; in tracing_set_tracer()
6408 arch_spin_unlock(&tr->max_lock); in tracing_set_tracer()
6415 if (system_state < SYSTEM_RUNNING && t->noboot) { in tracing_set_tracer()
6417 t->name); in tracing_set_tracer()
6423 ret = -EINVAL; in tracing_set_tracer()
6428 if (tr->trace_ref) { in tracing_set_tracer()
6429 ret = -EBUSY; in tracing_set_tracer()
6435 tr->current_trace->enabled--; in tracing_set_tracer()
6437 if (tr->current_trace->reset) in tracing_set_tracer()
6438 tr->current_trace->reset(tr); in tracing_set_tracer()
6441 had_max_tr = tr->current_trace->use_max_tr; in tracing_set_tracer()
6444 tr->current_trace = &nop_trace; in tracing_set_tracer()
6446 if (had_max_tr && !t->use_max_tr) { in tracing_set_tracer()
6458 if (t->use_max_tr && !tr->allocated_snapshot) { in tracing_set_tracer()
6464 tr->current_trace = &nop_trace; in tracing_set_tracer()
6467 if (t->init) { in tracing_set_tracer()
6473 tr->current_trace = t; in tracing_set_tracer()
6474 tr->current_trace->enabled++; in tracing_set_tracer()
6486 struct trace_array *tr = filp->private_data; in tracing_set_trace_write()
6498 return -EFAULT; in tracing_set_trace_write()
6521 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); in tracing_nsecs_read()
6554 struct trace_array *tr = filp->private_data; in tracing_thresh_write()
6562 if (tr->current_trace->update_thresh) { in tracing_thresh_write()
6563 ret = tr->current_trace->update_thresh(tr); in tracing_thresh_write()
6581 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos); in tracing_max_lat_read()
6588 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos); in tracing_max_lat_write()
6595 struct trace_array *tr = inode->i_private; in tracing_open_pipe()
6608 ret = -ENOMEM; in tracing_open_pipe()
6613 trace_seq_init(&iter->seq); in tracing_open_pipe()
6614 iter->trace = tr->current_trace; in tracing_open_pipe()
6616 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { in tracing_open_pipe()
6617 ret = -ENOMEM; in tracing_open_pipe()
6622 cpumask_setall(iter->started); in tracing_open_pipe()
6624 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) in tracing_open_pipe()
6625 iter->iter_flags |= TRACE_FILE_LAT_FMT; in tracing_open_pipe()
6628 if (trace_clocks[tr->clock_id].in_ns) in tracing_open_pipe()
6629 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in tracing_open_pipe()
6631 iter->tr = tr; in tracing_open_pipe()
6632 iter->array_buffer = &tr->array_buffer; in tracing_open_pipe()
6633 iter->cpu_file = tracing_get_cpu(inode); in tracing_open_pipe()
6634 mutex_init(&iter->mutex); in tracing_open_pipe()
6635 filp->private_data = iter; in tracing_open_pipe()
6637 if (iter->trace->pipe_open) in tracing_open_pipe()
6638 iter->trace->pipe_open(iter); in tracing_open_pipe()
6642 tr->trace_ref++; in tracing_open_pipe()
6656 struct trace_iterator *iter = file->private_data; in tracing_release_pipe()
6657 struct trace_array *tr = inode->i_private; in tracing_release_pipe()
6661 tr->trace_ref--; in tracing_release_pipe()
6663 if (iter->trace->pipe_close) in tracing_release_pipe()
6664 iter->trace->pipe_close(iter); in tracing_release_pipe()
6668 free_cpumask_var(iter->started); in tracing_release_pipe()
6669 kfree(iter->fmt); in tracing_release_pipe()
6670 mutex_destroy(&iter->mutex); in tracing_release_pipe()
6681 struct trace_array *tr = iter->tr; in trace_poll()
6684 if (trace_buffer_iter(iter, iter->cpu_file)) in trace_poll()
6687 if (tr->trace_flags & TRACE_ITER_BLOCK) in trace_poll()
6693 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file, in trace_poll()
6694 filp, poll_table, iter->tr->buffer_percent); in trace_poll()
6700 struct trace_iterator *iter = filp->private_data; in tracing_poll_pipe()
6705 /* Must be called with iter->mutex held. */
6708 struct trace_iterator *iter = filp->private_data; in tracing_wait_pipe()
6713 if ((filp->f_flags & O_NONBLOCK)) { in tracing_wait_pipe()
6714 return -EAGAIN; in tracing_wait_pipe()
6724 * iter->pos will be 0 if we haven't read anything. in tracing_wait_pipe()
6726 if (!tracer_tracing_is_on(iter->tr) && iter->pos) in tracing_wait_pipe()
6729 mutex_unlock(&iter->mutex); in tracing_wait_pipe()
6733 mutex_lock(&iter->mutex); in tracing_wait_pipe()
6749 struct trace_iterator *iter = filp->private_data; in tracing_read_pipe()
6757 mutex_lock(&iter->mutex); in tracing_read_pipe()
6760 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); in tracing_read_pipe()
6761 if (sret != -EBUSY) in tracing_read_pipe()
6764 trace_seq_init(&iter->seq); in tracing_read_pipe()
6766 if (iter->trace->read) { in tracing_read_pipe()
6767 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); in tracing_read_pipe()
6784 cnt = PAGE_SIZE - 1; in tracing_read_pipe()
6788 cpumask_clear(iter->started); in tracing_read_pipe()
6789 trace_seq_init(&iter->seq); in tracing_read_pipe()
6792 trace_access_lock(iter->cpu_file); in tracing_read_pipe()
6795 int save_len = iter->seq.seq.len; in tracing_read_pipe()
6800 iter->seq.seq.len = save_len; in tracing_read_pipe()
6806 if (trace_seq_used(&iter->seq) >= cnt) in tracing_read_pipe()
6814 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", in tracing_read_pipe()
6815 iter->ent->type); in tracing_read_pipe()
6817 trace_access_unlock(iter->cpu_file); in tracing_read_pipe()
6821 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); in tracing_read_pipe()
6822 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq)) in tracing_read_pipe()
6823 trace_seq_init(&iter->seq); in tracing_read_pipe()
6829 if (sret == -EBUSY) in tracing_read_pipe()
6833 mutex_unlock(&iter->mutex); in tracing_read_pipe()
6841 __free_page(spd->pages[idx]); in tracing_spd_release_pipe()
6851 /* Seq buffer is page-sized, exactly what we need. */ in tracing_fill_pipe_page()
6853 save_len = iter->seq.seq.len; in tracing_fill_pipe_page()
6856 if (trace_seq_has_overflowed(&iter->seq)) { in tracing_fill_pipe_page()
6857 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6863 * be set if the iter->seq overflowed. But check it in tracing_fill_pipe_page()
6867 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6871 count = trace_seq_used(&iter->seq) - save_len; in tracing_fill_pipe_page()
6874 iter->seq.seq.len = save_len; in tracing_fill_pipe_page()
6880 rem -= count; in tracing_fill_pipe_page()
6883 iter->ent = NULL; in tracing_fill_pipe_page()
6899 struct trace_iterator *iter = filp->private_data; in tracing_splice_read_pipe()
6913 return -ENOMEM; in tracing_splice_read_pipe()
6915 mutex_lock(&iter->mutex); in tracing_splice_read_pipe()
6917 if (iter->trace->splice_read) { in tracing_splice_read_pipe()
6918 ret = iter->trace->splice_read(iter, filp, in tracing_splice_read_pipe()
6928 if (!iter->ent && !trace_find_next_entry_inc(iter)) { in tracing_splice_read_pipe()
6929 ret = -EFAULT; in tracing_splice_read_pipe()
6934 trace_access_lock(iter->cpu_file); in tracing_splice_read_pipe()
6945 ret = trace_seq_to_buffer(&iter->seq, in tracing_splice_read_pipe()
6947 trace_seq_used(&iter->seq)); in tracing_splice_read_pipe()
6953 spd.partial[i].len = trace_seq_used(&iter->seq); in tracing_splice_read_pipe()
6955 trace_seq_init(&iter->seq); in tracing_splice_read_pipe()
6958 trace_access_unlock(iter->cpu_file); in tracing_splice_read_pipe()
6960 mutex_unlock(&iter->mutex); in tracing_splice_read_pipe()
6973 mutex_unlock(&iter->mutex); in tracing_splice_read_pipe()
6982 struct trace_array *tr = inode->i_private; in tracing_entries_read()
7000 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries; in tracing_entries_read()
7001 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) { in tracing_entries_read()
7017 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10); in tracing_entries_read()
7030 struct trace_array *tr = inode->i_private; in tracing_entries_write()
7040 return -EINVAL; in tracing_entries_write()
7057 struct trace_array *tr = filp->private_data; in tracing_total_entries_read()
7064 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10; in tracing_total_entries_read()
7094 struct trace_array *tr = inode->i_private; in tracing_free_buffer_release()
7097 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) in tracing_free_buffer_release()
7111 struct trace_array *tr = filp->private_data; in tracing_mark_write()
7122 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */ in tracing_mark_write()
7125 return -EINVAL; in tracing_mark_write()
7127 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) in tracing_mark_write()
7128 return -EINVAL; in tracing_mark_write()
7139 size += FAULTED_SIZE - cnt; in tracing_mark_write()
7141 buffer = tr->array_buffer.buffer; in tracing_mark_write()
7146 return -EBADF; in tracing_mark_write()
7149 entry->ip = _THIS_IP_; in tracing_mark_write()
7151 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt); in tracing_mark_write()
7153 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); in tracing_mark_write()
7155 written = -EFAULT; in tracing_mark_write()
7159 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) { in tracing_mark_write()
7161 entry->buf[cnt] = '\0'; in tracing_mark_write()
7162 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event); in tracing_mark_write()
7165 if (entry->buf[cnt - 1] != '\n') { in tracing_mark_write()
7166 entry->buf[cnt] = '\n'; in tracing_mark_write()
7167 entry->buf[cnt + 1] = '\0'; in tracing_mark_write()
7169 entry->buf[cnt] = '\0'; in tracing_mark_write()
7176 event_triggers_post_call(tr->trace_marker_file, tt); in tracing_mark_write()
7188 struct trace_array *tr = filp->private_data; in tracing_mark_raw_write()
7199 return -EINVAL; in tracing_mark_raw_write()
7201 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) in tracing_mark_raw_write()
7202 return -EINVAL; in tracing_mark_raw_write()
7206 return -EINVAL; in tracing_mark_raw_write()
7215 size += FAULT_SIZE_ID - cnt; in tracing_mark_raw_write()
7217 buffer = tr->array_buffer.buffer; in tracing_mark_raw_write()
7222 return -EBADF; in tracing_mark_raw_write()
7226 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt); in tracing_mark_raw_write()
7228 entry->id = -1; in tracing_mark_raw_write()
7229 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); in tracing_mark_raw_write()
7230 written = -EFAULT; in tracing_mark_raw_write()
7241 struct trace_array *tr = m->private; in tracing_clock_show()
7247 i == tr->clock_id ? "[" : "", trace_clocks[i].name, in tracing_clock_show()
7248 i == tr->clock_id ? "]" : ""); in tracing_clock_show()
7263 return -EINVAL; in tracing_set_clock()
7267 tr->clock_id = i; in tracing_set_clock()
7269 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func); in tracing_set_clock()
7275 tracing_reset_online_cpus(&tr->array_buffer); in tracing_set_clock()
7278 if (tr->max_buffer.buffer) in tracing_set_clock()
7279 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); in tracing_set_clock()
7280 tracing_reset_online_cpus(&tr->max_buffer); in tracing_set_clock()
7291 struct seq_file *m = filp->private_data; in tracing_clock_write()
7292 struct trace_array *tr = m->private; in tracing_clock_write()
7298 return -EINVAL; in tracing_clock_write()
7301 return -EFAULT; in tracing_clock_write()
7318 struct trace_array *tr = inode->i_private; in tracing_clock_open()
7325 ret = single_open(file, tracing_clock_show, inode->i_private); in tracing_clock_open()
7334 struct trace_array *tr = m->private; in tracing_time_stamp_mode_show()
7338 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer)) in tracing_time_stamp_mode_show()
7350 struct trace_array *tr = inode->i_private; in tracing_time_stamp_mode_open()
7357 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private); in tracing_time_stamp_mode_open()
7381 if (set && tr->no_filter_buffering_ref++) in tracing_set_filter_buffering()
7385 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) { in tracing_set_filter_buffering()
7386 ret = -EINVAL; in tracing_set_filter_buffering()
7390 --tr->no_filter_buffering_ref; in tracing_set_filter_buffering()
7400 void *spare; member
7408 struct trace_array *tr = inode->i_private; in tracing_snapshot_open()
7417 if (file->f_mode & FMODE_READ) { in tracing_snapshot_open()
7423 ret = -ENOMEM; in tracing_snapshot_open()
7434 iter->tr = tr; in tracing_snapshot_open()
7435 iter->array_buffer = &tr->max_buffer; in tracing_snapshot_open()
7436 iter->cpu_file = tracing_get_cpu(inode); in tracing_snapshot_open()
7437 m->private = iter; in tracing_snapshot_open()
7438 file->private_data = m; in tracing_snapshot_open()
7451 struct seq_file *m = filp->private_data; in tracing_snapshot_write()
7452 struct trace_iterator *iter = m->private; in tracing_snapshot_write()
7453 struct trace_array *tr = iter->tr; in tracing_snapshot_write()
7467 if (tr->current_trace->use_max_tr) { in tracing_snapshot_write()
7468 ret = -EBUSY; in tracing_snapshot_write()
7473 arch_spin_lock(&tr->max_lock); in tracing_snapshot_write()
7474 if (tr->cond_snapshot) in tracing_snapshot_write()
7475 ret = -EBUSY; in tracing_snapshot_write()
7476 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_write()
7483 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7484 ret = -EINVAL; in tracing_snapshot_write()
7487 if (tr->allocated_snapshot) in tracing_snapshot_write()
7491 /* Only allow per-cpu swap if the ring buffer supports it */ in tracing_snapshot_write()
7493 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { in tracing_snapshot_write()
7494 ret = -EINVAL; in tracing_snapshot_write()
7498 if (tr->allocated_snapshot) in tracing_snapshot_write()
7499 ret = resize_buffer_duplicate_size(&tr->max_buffer, in tracing_snapshot_write()
7500 &tr->array_buffer, iter->cpu_file); in tracing_snapshot_write()
7507 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) in tracing_snapshot_write()
7510 update_max_tr_single(tr, current, iter->cpu_file); in tracing_snapshot_write()
7514 if (tr->allocated_snapshot) { in tracing_snapshot_write()
7515 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) in tracing_snapshot_write()
7516 tracing_reset_online_cpus(&tr->max_buffer); in tracing_snapshot_write()
7518 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file); in tracing_snapshot_write()
7534 struct seq_file *m = file->private_data; in tracing_snapshot_release()
7539 if (file->f_mode & FMODE_READ) in tracing_snapshot_release()
7544 kfree(m->private); in tracing_snapshot_release()
7567 info = filp->private_data; in snapshot_raw_open()
7569 if (info->iter.trace->use_max_tr) { in snapshot_raw_open()
7571 return -EBUSY; in snapshot_raw_open()
7574 info->iter.snapshot = true; in snapshot_raw_open()
7575 info->iter.array_buffer = &info->iter.tr->max_buffer; in snapshot_raw_open()
7683 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7690 * The filp->private_data must point to a trace_min_max_param structure that
7697 struct trace_min_max_param *param = filp->private_data; in trace_min_max_write()
7702 return -EFAULT; in trace_min_max_write()
7708 if (param->lock) in trace_min_max_write()
7709 mutex_lock(param->lock); in trace_min_max_write()
7711 if (param->min && val < *param->min) in trace_min_max_write()
7712 err = -EINVAL; in trace_min_max_write()
7714 if (param->max && val > *param->max) in trace_min_max_write()
7715 err = -EINVAL; in trace_min_max_write()
7718 *param->val = val; in trace_min_max_write()
7720 if (param->lock) in trace_min_max_write()
7721 mutex_unlock(param->lock); in trace_min_max_write()
7730 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7737 * The filp->private_data must point to a trace_min_max_param struct with valid
7743 struct trace_min_max_param *param = filp->private_data; in trace_min_max_read()
7749 return -EFAULT; in trace_min_max_read()
7751 val = *param->val; in trace_min_max_read()
7773 const char **errs; /* ptr to loc-specific array of err strings */
7774 u8 type; /* index into errs -> specific err string */
7794 return ERR_PTR(-ENOMEM); in alloc_tracing_log_err()
7796 err->cmd = kzalloc(len, GFP_KERNEL); in alloc_tracing_log_err()
7797 if (!err->cmd) { in alloc_tracing_log_err()
7799 return ERR_PTR(-ENOMEM); in alloc_tracing_log_err()
7807 kfree(err->cmd); in free_tracing_log_err()
7817 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) { in get_tracing_log_err()
7819 if (PTR_ERR(err) != -ENOMEM) in get_tracing_log_err()
7820 tr->n_err_log_entries++; in get_tracing_log_err()
7826 return ERR_PTR(-ENOMEM); in get_tracing_log_err()
7827 err = list_first_entry(&tr->err_log, struct tracing_log_err, list); in get_tracing_log_err()
7828 kfree(err->cmd); in get_tracing_log_err()
7829 err->cmd = cmd; in get_tracing_log_err()
7830 list_del(&err->list); in get_tracing_log_err()
7836 * err_pos - find the position of a string within a command for error careting
7856 return found - cmd; in err_pos()
7862 * tracing_log_err - write an error to the tracing error log
7866 * @errs: The array of loc-specific static error strings
7883 * produce a static error string - this string is not copied and saved
7884 * when the error is logged - only a pointer to it is saved. See
7902 if (PTR_ERR(err) == -ENOMEM) { in tracing_log_err()
7907 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc); in tracing_log_err()
7908 snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd); in tracing_log_err()
7910 err->info.errs = errs; in tracing_log_err()
7911 err->info.type = type; in tracing_log_err()
7912 err->info.pos = pos; in tracing_log_err()
7913 err->info.ts = local_clock(); in tracing_log_err()
7915 list_add_tail(&err->list, &tr->err_log); in tracing_log_err()
7924 list_for_each_entry_safe(err, next, &tr->err_log, list) { in clear_tracing_err_log()
7925 list_del(&err->list); in clear_tracing_err_log()
7929 tr->n_err_log_entries = 0; in clear_tracing_err_log()
7935 struct trace_array *tr = m->private; in tracing_err_log_seq_start()
7939 return seq_list_start(&tr->err_log, *pos); in tracing_err_log_seq_start()
7944 struct trace_array *tr = m->private; in tracing_err_log_seq_next()
7946 return seq_list_next(v, &tr->err_log, pos); in tracing_err_log_seq_next()
7958 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++) in tracing_err_log_show_pos()
7970 const char *err_text = err->info.errs[err->info.type]; in tracing_err_log_seq_show()
7971 u64 sec = err->info.ts; in tracing_err_log_seq_show()
7976 err->loc, err_text); in tracing_err_log_seq_show()
7977 seq_printf(m, "%s", err->cmd); in tracing_err_log_seq_show()
7978 tracing_err_log_show_pos(m, err->info.pos); in tracing_err_log_seq_show()
7993 struct trace_array *tr = inode->i_private; in tracing_err_log_open()
8001 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) in tracing_err_log_open()
8004 if (file->f_mode & FMODE_READ) { in tracing_err_log_open()
8007 struct seq_file *m = file->private_data; in tracing_err_log_open()
8008 m->private = tr; in tracing_err_log_open()
8025 struct trace_array *tr = inode->i_private; in tracing_err_log_release()
8029 if (file->f_mode & FMODE_READ) in tracing_err_log_release()
8045 struct trace_array *tr = inode->i_private; in tracing_buffers_open()
8056 return -ENOMEM; in tracing_buffers_open()
8061 info->iter.tr = tr; in tracing_buffers_open()
8062 info->iter.cpu_file = tracing_get_cpu(inode); in tracing_buffers_open()
8063 info->iter.trace = tr->current_trace; in tracing_buffers_open()
8064 info->iter.array_buffer = &tr->array_buffer; in tracing_buffers_open()
8065 info->spare = NULL; in tracing_buffers_open()
8067 info->read = (unsigned int)-1; in tracing_buffers_open()
8069 filp->private_data = info; in tracing_buffers_open()
8071 tr->trace_ref++; in tracing_buffers_open()
8085 struct ftrace_buffer_info *info = filp->private_data; in tracing_buffers_poll()
8086 struct trace_iterator *iter = &info->iter; in tracing_buffers_poll()
8095 struct ftrace_buffer_info *info = filp->private_data; in tracing_buffers_read()
8096 struct trace_iterator *iter = &info->iter; in tracing_buffers_read()
8104 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_read()
8105 return -EBUSY; in tracing_buffers_read()
8108 if (!info->spare) { in tracing_buffers_read()
8109 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, in tracing_buffers_read()
8110 iter->cpu_file); in tracing_buffers_read()
8111 if (IS_ERR(info->spare)) { in tracing_buffers_read()
8112 ret = PTR_ERR(info->spare); in tracing_buffers_read()
8113 info->spare = NULL; in tracing_buffers_read()
8115 info->spare_cpu = iter->cpu_file; in tracing_buffers_read()
8118 if (!info->spare) in tracing_buffers_read()
8122 if (info->read < PAGE_SIZE) in tracing_buffers_read()
8126 trace_access_lock(iter->cpu_file); in tracing_buffers_read()
8127 ret = ring_buffer_read_page(iter->array_buffer->buffer, in tracing_buffers_read()
8128 &info->spare, in tracing_buffers_read()
8130 iter->cpu_file, 0); in tracing_buffers_read()
8131 trace_access_unlock(iter->cpu_file); in tracing_buffers_read()
8135 if ((filp->f_flags & O_NONBLOCK)) in tracing_buffers_read()
8136 return -EAGAIN; in tracing_buffers_read()
8147 info->read = 0; in tracing_buffers_read()
8149 size = PAGE_SIZE - info->read; in tracing_buffers_read()
8153 ret = copy_to_user(ubuf, info->spare + info->read, size); in tracing_buffers_read()
8155 return -EFAULT; in tracing_buffers_read()
8157 size -= ret; in tracing_buffers_read()
8160 info->read += size; in tracing_buffers_read()
8167 struct ftrace_buffer_info *info = file->private_data; in tracing_buffers_release()
8168 struct trace_iterator *iter = &info->iter; in tracing_buffers_release()
8172 iter->tr->trace_ref--; in tracing_buffers_release()
8174 __trace_array_put(iter->tr); in tracing_buffers_release()
8176 iter->wait_index++; in tracing_buffers_release()
8180 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_release()
8182 if (info->spare) in tracing_buffers_release()
8183 ring_buffer_free_read_page(iter->array_buffer->buffer, in tracing_buffers_release()
8184 info->spare_cpu, info->spare); in tracing_buffers_release()
8201 if (!refcount_dec_and_test(&ref->refcount)) in buffer_ref_release()
8203 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); in buffer_ref_release()
8210 struct buffer_ref *ref = (struct buffer_ref *)buf->private; in buffer_pipe_buf_release()
8213 buf->private = 0; in buffer_pipe_buf_release()
8219 struct buffer_ref *ref = (struct buffer_ref *)buf->private; in buffer_pipe_buf_get()
8221 if (refcount_read(&ref->refcount) > INT_MAX/2) in buffer_pipe_buf_get()
8224 refcount_inc(&ref->refcount); in buffer_pipe_buf_get()
8241 (struct buffer_ref *)spd->partial[i].private; in buffer_spd_release()
8244 spd->partial[i].private = 0; in buffer_spd_release()
8252 struct ftrace_buffer_info *info = file->private_data; in tracing_buffers_splice_read()
8253 struct trace_iterator *iter = &info->iter; in tracing_buffers_splice_read()
8268 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_splice_read()
8269 return -EBUSY; in tracing_buffers_splice_read()
8272 if (*ppos & (PAGE_SIZE - 1)) in tracing_buffers_splice_read()
8273 return -EINVAL; in tracing_buffers_splice_read()
8275 if (len & (PAGE_SIZE - 1)) { in tracing_buffers_splice_read()
8277 return -EINVAL; in tracing_buffers_splice_read()
8282 return -ENOMEM; in tracing_buffers_splice_read()
8285 trace_access_lock(iter->cpu_file); in tracing_buffers_splice_read()
8286 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8288 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) { in tracing_buffers_splice_read()
8294 ret = -ENOMEM; in tracing_buffers_splice_read()
8298 refcount_set(&ref->refcount, 1); in tracing_buffers_splice_read()
8299 ref->buffer = iter->array_buffer->buffer; in tracing_buffers_splice_read()
8300 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8301 if (IS_ERR(ref->page)) { in tracing_buffers_splice_read()
8302 ret = PTR_ERR(ref->page); in tracing_buffers_splice_read()
8303 ref->page = NULL; in tracing_buffers_splice_read()
8307 ref->cpu = iter->cpu_file; in tracing_buffers_splice_read()
8309 r = ring_buffer_read_page(ref->buffer, &ref->page, in tracing_buffers_splice_read()
8310 len, iter->cpu_file, 1); in tracing_buffers_splice_read()
8312 ring_buffer_free_read_page(ref->buffer, ref->cpu, in tracing_buffers_splice_read()
8313 ref->page); in tracing_buffers_splice_read()
8318 page = virt_to_page(ref->page); in tracing_buffers_splice_read()
8327 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_splice_read()
8330 trace_access_unlock(iter->cpu_file); in tracing_buffers_splice_read()
8340 ret = -EAGAIN; in tracing_buffers_splice_read()
8341 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) in tracing_buffers_splice_read()
8344 wait_index = READ_ONCE(iter->wait_index); in tracing_buffers_splice_read()
8346 ret = wait_on_pipe(iter, iter->tr->buffer_percent); in tracing_buffers_splice_read()
8351 if (!tracer_tracing_is_on(iter->tr)) in tracing_buffers_splice_read()
8356 if (wait_index != iter->wait_index) in tracing_buffers_splice_read()
8372 struct ftrace_buffer_info *info = file->private_data; in tracing_buffers_ioctl()
8373 struct trace_iterator *iter = &info->iter; in tracing_buffers_ioctl()
8376 return -ENOIOCTLCMD; in tracing_buffers_ioctl()
8380 iter->wait_index++; in tracing_buffers_ioctl()
8384 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); in tracing_buffers_ioctl()
8405 struct trace_array *tr = inode->i_private; in tracing_stats_read()
8406 struct array_buffer *trace_buf = &tr->array_buffer; in tracing_stats_read()
8415 return -ENOMEM; in tracing_stats_read()
8419 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8422 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8425 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8428 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8431 if (trace_clocks[tr->clock_id].in_ns) { in tracing_stats_read()
8433 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); in tracing_stats_read()
8438 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer)); in tracing_stats_read()
8444 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); in tracing_stats_read()
8447 ring_buffer_time_stamp(trace_buf->buffer)); in tracing_stats_read()
8450 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8453 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); in tracing_stats_read()
8457 s->buffer, trace_seq_used(s)); in tracing_stats_read()
8484 return -ENOMEM; in tracing_read_dyn_info()
8528 (*count)--; in ftrace_count_snapshot()
8565 return -ENOMEM; in ftrace_snapshot_init()
8605 void *count = (void *)-1; in ftrace_trace_snapshot_callback()
8610 return -ENODEV; in ftrace_trace_snapshot_callback()
8614 return -EINVAL; in ftrace_trace_snapshot_callback()
8663 if (WARN_ON(!tr->dir)) in tracing_get_dentry()
8664 return ERR_PTR(-ENODEV); in tracing_get_dentry()
8667 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) in tracing_get_dentry()
8671 return tr->dir; in tracing_get_dentry()
8678 if (tr->percpu_dir) in tracing_dentry_percpu()
8679 return tr->percpu_dir; in tracing_dentry_percpu()
8685 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); in tracing_dentry_percpu()
8687 MEM_FAIL(!tr->percpu_dir, in tracing_dentry_percpu()
8690 return tr->percpu_dir; in tracing_dentry_percpu()
8700 d_inode(ret)->i_cdev = (void *)(cpu + 1); in trace_create_cpu_file()
8756 struct trace_option_dentry *topt = filp->private_data; in trace_options_read()
8759 if (topt->flags->val & topt->opt->bit) in trace_options_read()
8771 struct trace_option_dentry *topt = filp->private_data; in trace_options_write()
8780 return -EINVAL; in trace_options_write()
8782 if (!!(topt->flags->val & topt->opt->bit) != val) { in trace_options_write()
8784 ret = __set_tracer_option(topt->tr, topt->flags, in trace_options_write()
8785 topt->opt, !val); in trace_options_write()
8823 * ptr - idx == &index[0]
8833 *ptr = container_of(data - *pindex, struct trace_array, in get_tr_index()
8841 void *tr_index = filp->private_data; in trace_options_core_read()
8848 if (tr->trace_flags & (1 << index)) in trace_options_core_read()
8860 void *tr_index = filp->private_data; in trace_options_core_write()
8873 return -EINVAL; in trace_options_core_write()
8916 if (tr->options) in trace_options_init_dentry()
8917 return tr->options; in trace_options_init_dentry()
8923 tr->options = tracefs_create_dir("options", d_tracer); in trace_options_init_dentry()
8924 if (!tr->options) { in trace_options_init_dentry()
8929 return tr->options; in trace_options_init_dentry()
8944 topt->flags = flags; in create_trace_option_file()
8945 topt->opt = opt; in create_trace_option_file()
8946 topt->tr = tr; in create_trace_option_file()
8948 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE, in create_trace_option_file()
8966 flags = tracer->flags; in create_trace_option_files()
8968 if (!flags || !flags->opts) in create_trace_option_files()
8978 for (i = 0; i < tr->nr_topts; i++) { in create_trace_option_files()
8980 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags)) in create_trace_option_files()
8984 opts = flags->opts; in create_trace_option_files()
8993 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), in create_trace_option_files()
9000 tr->topts = tr_topts; in create_trace_option_files()
9001 tr->topts[tr->nr_topts].tracer = tracer; in create_trace_option_files()
9002 tr->topts[tr->nr_topts].topts = topts; in create_trace_option_files()
9003 tr->nr_topts++; in create_trace_option_files()
9025 (void *)&tr->trace_flags_index[index], in create_trace_option_core_file()
9050 struct trace_array *tr = filp->private_data; in rb_simple_read()
9064 struct trace_array *tr = filp->private_data; in rb_simple_write()
9065 struct trace_buffer *buffer = tr->array_buffer.buffer; in rb_simple_write()
9079 if (tr->current_trace->start) in rb_simple_write()
9080 tr->current_trace->start(tr); in rb_simple_write()
9083 if (tr->current_trace->stop) in rb_simple_write()
9084 tr->current_trace->stop(tr); in rb_simple_write()
9108 struct trace_array *tr = filp->private_data; in buffer_percent_read()
9112 r = tr->buffer_percent; in buffer_percent_read()
9122 struct trace_array *tr = filp->private_data; in buffer_percent_write()
9131 return -EINVAL; in buffer_percent_write()
9136 tr->buffer_percent = val; in buffer_percent_write()
9161 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; in allocate_trace_buffer()
9163 buf->tr = tr; in allocate_trace_buffer()
9165 buf->buffer = ring_buffer_alloc(size, rb_flags); in allocate_trace_buffer()
9166 if (!buf->buffer) in allocate_trace_buffer()
9167 return -ENOMEM; in allocate_trace_buffer()
9169 buf->data = alloc_percpu(struct trace_array_cpu); in allocate_trace_buffer()
9170 if (!buf->data) { in allocate_trace_buffer()
9171 ring_buffer_free(buf->buffer); in allocate_trace_buffer()
9172 buf->buffer = NULL; in allocate_trace_buffer()
9173 return -ENOMEM; in allocate_trace_buffer()
9177 set_buffer_entries(&tr->array_buffer, in allocate_trace_buffer()
9178 ring_buffer_size(tr->array_buffer.buffer, 0)); in allocate_trace_buffer()
9185 if (buf->buffer) { in free_trace_buffer()
9186 ring_buffer_free(buf->buffer); in free_trace_buffer()
9187 buf->buffer = NULL; in free_trace_buffer()
9188 free_percpu(buf->data); in free_trace_buffer()
9189 buf->data = NULL; in free_trace_buffer()
9197 ret = allocate_trace_buffer(tr, &tr->array_buffer, size); in allocate_trace_buffers()
9202 ret = allocate_trace_buffer(tr, &tr->max_buffer, in allocate_trace_buffers()
9205 free_trace_buffer(&tr->array_buffer); in allocate_trace_buffers()
9206 return -ENOMEM; in allocate_trace_buffers()
9208 tr->allocated_snapshot = allocate_snapshot; in allocate_trace_buffers()
9225 free_trace_buffer(&tr->array_buffer); in free_trace_buffers()
9228 free_trace_buffer(&tr->max_buffer); in free_trace_buffers()
9238 tr->trace_flags_index[i] = i; in init_trace_flags_index()
9245 for (t = trace_types; t; t = t->next) in __update_tracer_options()
9263 if (tr->name && strcmp(tr->name, instance) == 0) { in trace_array_find()
9279 tr->ref++; in trace_array_find_get()
9289 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir); in trace_array_create_dir()
9290 if (!tr->dir) in trace_array_create_dir()
9291 return -EINVAL; in trace_array_create_dir()
9293 ret = event_trace_add_tracer(tr->dir, tr); in trace_array_create_dir()
9295 tracefs_remove(tr->dir); in trace_array_create_dir()
9299 init_tracer_tracefs(tr, tr->dir); in trace_array_create_dir()
9310 ret = -ENOMEM; in trace_array_create()
9315 tr->name = kstrdup(name, GFP_KERNEL); in trace_array_create()
9316 if (!tr->name) in trace_array_create()
9319 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) in trace_array_create()
9322 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; in trace_array_create()
9324 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); in trace_array_create()
9326 raw_spin_lock_init(&tr->start_lock); in trace_array_create()
9328 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; in trace_array_create()
9330 tr->current_trace = &nop_trace; in trace_array_create()
9332 INIT_LIST_HEAD(&tr->systems); in trace_array_create()
9333 INIT_LIST_HEAD(&tr->events); in trace_array_create()
9334 INIT_LIST_HEAD(&tr->hist_vars); in trace_array_create()
9335 INIT_LIST_HEAD(&tr->err_log); in trace_array_create()
9354 list_add(&tr->list, &ftrace_trace_arrays); in trace_array_create()
9356 tr->ref++; in trace_array_create()
9363 free_cpumask_var(tr->tracing_cpumask); in trace_array_create()
9364 kfree(tr->name); in trace_array_create()
9378 ret = -EEXIST; in instance_mkdir()
9393 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
9416 if (tr->name && strcmp(tr->name, name) == 0) in trace_array_get_by_name()
9426 tr->ref++; in trace_array_get_by_name()
9439 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref)) in __remove_instance()
9440 return -EBUSY; in __remove_instance()
9442 list_del(&tr->list); in __remove_instance()
9455 tracefs_remove(tr->dir); in __remove_instance()
9456 free_percpu(tr->last_func_repeats); in __remove_instance()
9459 for (i = 0; i < tr->nr_topts; i++) { in __remove_instance()
9460 kfree(tr->topts[i].topts); in __remove_instance()
9462 kfree(tr->topts); in __remove_instance()
9464 free_cpumask_var(tr->tracing_cpumask); in __remove_instance()
9465 kfree(tr->name); in __remove_instance()
9477 return -EINVAL; in trace_array_destroy()
9482 ret = -ENODEV; in trace_array_destroy()
9507 ret = -ENODEV; in instance_rmdir()
9532 if (!tr->name) in create_trace_instances()
9580 if (file && file->dir) in init_tracer_tracefs()
9581 trace_create_file("trigger", TRACE_MODE_WRITE, file->dir, in init_tracer_tracefs()
9583 tr->trace_marker_file = file; in init_tracer_tracefs()
9597 tr->buffer_percent = 50; in init_tracer_tracefs()
9646 * tracing_init_dentry - initialize top level trace array
9658 return -EPERM; in tracing_init_dentry()
9662 if (tr->dir) in tracing_init_dentry()
9666 return -ENODEV; in tracing_init_dentry()
9674 tr->dir = debugfs_create_automount("tracing", NULL, in tracing_init_dentry()
9691 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps; in eval_map_work_func()
9704 return -ENOMEM; in trace_eval_init()
9727 if (!mod->num_trace_evals) in trace_module_add_evals()
9737 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals); in trace_module_add_evals()
9746 if (!mod->num_trace_evals) in trace_module_remove_evals()
9754 if (map->head.mod == mod) in trace_module_remove_evals()
9757 last = &map->tail.next; in trace_module_remove_evals()
9758 map = map->tail.next; in trace_module_remove_evals()
9763 *last = trace_eval_jmp_to_tail(map)->tail.next; in trace_module_remove_evals()
9909 if (s->seq.len >= TRACE_MAX_PRINT) in trace_printk_seq()
9910 s->seq.len = TRACE_MAX_PRINT; in trace_printk_seq()
9917 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) in trace_printk_seq()
9918 s->seq.len = s->seq.size - 1; in trace_printk_seq()
9921 s->buffer[s->seq.len] = 0; in trace_printk_seq()
9923 printk(KERN_TRACE "%s", s->buffer); in trace_printk_seq()
9930 iter->tr = &global_trace; in trace_init_global_iter()
9931 iter->trace = iter->tr->current_trace; in trace_init_global_iter()
9932 iter->cpu_file = RING_BUFFER_ALL_CPUS; in trace_init_global_iter()
9933 iter->array_buffer = &global_trace.array_buffer; in trace_init_global_iter()
9935 if (iter->trace && iter->trace->open) in trace_init_global_iter()
9936 iter->trace->open(iter); in trace_init_global_iter()
9939 if (ring_buffer_overruns(iter->array_buffer->buffer)) in trace_init_global_iter()
9940 iter->iter_flags |= TRACE_FILE_ANNOTATE; in trace_init_global_iter()
9943 if (trace_clocks[iter->tr->clock_id].in_ns) in trace_init_global_iter()
9944 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; in trace_init_global_iter()
9947 iter->temp = static_temp_buf; in trace_init_global_iter()
9948 iter->temp_size = STATIC_TEMP_BUF_SIZE; in trace_init_global_iter()
9949 iter->fmt = static_fmt_buf; in trace_init_global_iter()
9950 iter->fmt_size = STATIC_FMT_BUF_SIZE; in trace_init_global_iter()
9974 * If the user does a sysrq-z, then they can re-enable in ftrace_dump()
9985 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); in ftrace_dump()
9988 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; in ftrace_dump()
9991 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; in ftrace_dump()
10025 printk(KERN_TRACE "---------------------------------\n"); in ftrace_dump()
10047 printk(KERN_TRACE "---------------------------------\n"); in ftrace_dump()
10050 tr->trace_flags |= old_userobj; in ftrace_dump()
10053 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); in ftrace_dump()
10073 return -ENOMEM; in trace_parse_run_command()
10076 size = count - done; in trace_parse_run_command()
10079 size = WRITE_BUFSIZE - 1; in trace_parse_run_command()
10082 ret = -EFAULT; in trace_parse_run_command()
10091 size = tmp - buf + 1; in trace_parse_run_command()
10097 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */ in trace_parse_run_command()
10099 WRITE_BUFSIZE - 2); in trace_parse_run_command()
10100 ret = -EINVAL; in trace_parse_run_command()
10130 int ret = -ENOMEM; in tracer_alloc_buffers()
10135 return -EPERM; in tracer_alloc_buffers()
10178 ret = -ENOMEM; in tracer_alloc_buffers()