Lines Matching +full:- +full:alert
1 // SPDX-License-Identifier: GPL-2.0
3 * Performance event support for the System z CPU-measurement Sampling Facility
26 /* Minimum number of sample-data-block-tables:
28 * A single table contains up to 511 pointers to sample-data-blocks.
32 /* Number of sample-data-blocks per sample-data-block-table (SDBT):
33 * A table contains SDB pointers (8 bytes) and one table-link entry
36 #define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
38 /* Maximum page offset for an SDBT table-link entry:
39 * If this page offset is reached, a table-link entry to the next SDBT
51 * the number of sample-data-block-tables into account. Note that these
52 * numbers apply to the basic-sampling function only.
54 * the diagnostic-sampling function is active.
57 * ---------------------------------------------------
59 * 1 page for SDB-tables
63 * 16 pages for SDB-tables
71 unsigned long *sdbt; /* Sample-data-block-table origin */
73 unsigned long num_sdb; /* Number of sample-data-blocks */
74 unsigned long num_sdbt; /* Number of sample-data-block-tables */
75 unsigned long *tail; /* last sample-data-block-table */
81 unsigned long alert_mark; /* index of SDB of alert request position */
88 /* CPU-measurement sampling information block */
90 /* CPU-measurement sampling control block */
103 * sf_disable() - Switch off sampling facility
114 * sf_buffer_available() - Check for an allocated sampling buffer
118 return !!cpuhw->sfb.sdbt; in sf_buffer_available()
128 if (!sfb->sdbt) in free_sampling_buffer()
131 sdbt = sfb->sdbt; in free_sampling_buffer()
139 /* Process table-link entries */ in free_sampling_buffer()
146 if (curr == sfb->sdbt) in free_sampling_buffer()
160 (unsigned long)sfb->sdbt); in free_sampling_buffer()
168 /* Allocate and initialize sample-data-block */ in alloc_sample_data_block()
171 return -ENOMEM; in alloc_sample_data_block()
175 /* Link SDB into the sample-data-block-table */ in alloc_sample_data_block()
182 * realloc_sampling_buffer() - extend sampler memory
184 * Allocates new sample-data-blocks and adds them to the specified sampling
190 * Returns zero on success, non-zero otherwise.
198 if (!sfb->sdbt || !sfb->tail) in realloc_sampling_buffer()
199 return -EINVAL; in realloc_sampling_buffer()
201 if (!is_link_entry(sfb->tail)) in realloc_sampling_buffer()
202 return -EINVAL; in realloc_sampling_buffer()
204 /* Append to the existing sampling buffer, overwriting the table-link in realloc_sampling_buffer()
206 * The tail variables always points to the "tail" (last and table-link) in realloc_sampling_buffer()
207 * entry in an SDB-table. in realloc_sampling_buffer()
209 tail = sfb->tail; in realloc_sampling_buffer()
211 /* Do a sanity check whether the table-link entry points to in realloc_sampling_buffer()
214 if (sfb->sdbt != get_next_sdbt(tail)) { in realloc_sampling_buffer()
218 (unsigned long)sfb->sdbt, in realloc_sampling_buffer()
220 return -EINVAL; in realloc_sampling_buffer()
226 /* Allocate a new SDB-table if it is full. */ in realloc_sampling_buffer()
230 rc = -ENOMEM; in realloc_sampling_buffer()
233 sfb->num_sdbt++; in realloc_sampling_buffer()
240 /* Allocate a new sample-data-block. in realloc_sampling_buffer()
253 sfb->num_sdbt--; in realloc_sampling_buffer()
259 sfb->num_sdb++; in realloc_sampling_buffer()
265 *tail = (unsigned long) sfb->sdbt + 1; in realloc_sampling_buffer()
266 sfb->tail = tail; in realloc_sampling_buffer()
270 sfb->num_sdbt, sfb->num_sdb); in realloc_sampling_buffer()
275 * allocate_sampling_buffer() - allocate sampler memory
278 * specified number of sample-data-blocks (SDB). For each allocation,
279 * a 4K page is used. The number of sample-data-block-tables (SDBT)
283 * Returns zero on success, non-zero otherwise.
289 if (sfb->sdbt) in alloc_sampling_buffer()
290 return -EINVAL; in alloc_sampling_buffer()
292 /* Allocate the sample-data-block-table origin */ in alloc_sampling_buffer()
293 sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL); in alloc_sampling_buffer()
294 if (!sfb->sdbt) in alloc_sampling_buffer()
295 return -ENOMEM; in alloc_sampling_buffer()
296 sfb->num_sdb = 0; in alloc_sampling_buffer()
297 sfb->num_sdbt = 1; in alloc_sampling_buffer()
302 sfb->tail = sfb->sdbt; in alloc_sampling_buffer()
303 *sfb->tail = (unsigned long)(void *) sfb->sdbt + 1; in alloc_sampling_buffer()
305 /* Allocate requested number of sample-data-blocks */ in alloc_sampling_buffer()
315 (unsigned long)sfb->sdbt, (unsigned long)*sfb->sdbt); in alloc_sampling_buffer()
340 if (!sfb->sdbt) in sfb_pending_allocs()
342 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb) in sfb_pending_allocs()
343 return SFB_ALLOC_REG(hwc) - sfb->num_sdb; in sfb_pending_allocs()
356 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc)); in sfb_account_allocs()
369 if (cpuhw->sfb.sdbt) in deallocate_buffers()
370 free_sampling_buffer(&cpuhw->sfb); in deallocate_buffers()
386 * 2. Function alloc_sampling_buffer() sets the Alert Request in allocate_buffers()
387 * Control indicator to trigger a measurement-alert to harvest in allocate_buffers()
388 * sample-data-blocks (SDB). This is done per SDB. This in allocate_buffers()
389 * measurement alert interrupt fires quick enough to handle in allocate_buffers()
392 * Currently there is no need for setup alert request on every in allocate_buffers()
393 * n-th page. This is counterproductive as one IRQ triggers in allocate_buffers()
403 * 4. Compute the number of sample-data-block-tables (SDBT) and in allocate_buffers()
408 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)); in allocate_buffers()
429 return alloc_sampling_buffer(&cpuhw->sfb, in allocate_buffers()
430 sfb_pending_allocs(&cpuhw->sfb, hwc)); in allocate_buffers()
441 /* Use a percentage-based approach to extend the sampling facility in compute_sfb_extent()
444 * sample-data-blocks. in compute_sfb_extent()
471 * that has been lost because sample-data-blocks were full. in sfb_account_overflows()
477 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb, in sfb_account_overflows()
478 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc))); in sfb_account_overflows()
480 /* Compute number of sample-data-blocks */ in sfb_account_overflows()
481 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb); in sfb_account_overflows()
490 /* extend_sampling_buffer() - Extend sampling buffer
510 num_old = sfb->num_sdb; in extend_sampling_buffer()
520 * new SDB-request cannot be satisfied immediately. in extend_sampling_buffer()
530 __func__, num, sfb->num_sdb - num_old, in extend_sampling_buffer()
551 err = qsi(&cpusf->qsi); in setup_pmc_cpu()
554 cpusf->flags |= PMU_F_RESERVED; in setup_pmc_cpu()
564 cpusf->flags &= ~PMU_F_RESERVED; in setup_pmc_cpu()
595 return -ENODEV; in reserve_pmc_hardware()
605 if (!atomic_add_unless(&num_events, -1, 1)) { in hw_perf_event_destroy()
615 hwc->sample_period = period; in hw_init_period()
616 hwc->last_period = hwc->sample_period; in hw_init_period()
617 local64_set(&hwc->period_left, hwc->sample_period); in hw_init_period()
624 si->min_sampl_rate, si->max_sampl_rate); in hw_limit_rate()
637 pid = -1; in cpumsf_pid_type()
643 if (event->parent) in cpumsf_pid_type()
644 event = event->parent; in cpumsf_pid_type()
645 pid = __task_pid_nr_ns(tsk, type, event->ns); in cpumsf_pid_type()
651 pid = -1; in cpumsf_pid_type()
666 * Obtain the PID from the basic-sampling data entry and in cpumsf_output_event_pid()
667 * correct the data->tid_entry.pid value. in cpumsf_output_event_pid()
669 pid = data->tid_entry.pid; in cpumsf_output_event_pid()
679 data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID); in cpumsf_output_event_pid()
680 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID); in cpumsf_output_event_pid()
740 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init_rate()
741 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init_rate()
744 if (attr->freq) { in __hw_perf_event_init_rate()
745 if (!attr->sample_freq) in __hw_perf_event_init_rate()
746 return -EINVAL; in __hw_perf_event_init_rate()
747 rate = getrate(attr->freq, attr->sample_freq, si); in __hw_perf_event_init_rate()
748 attr->freq = 0; /* Don't call perf_adjust_period() */ in __hw_perf_event_init_rate()
751 rate = getrate(attr->freq, attr->sample_period, si); in __hw_perf_event_init_rate()
753 return -EINVAL; in __hw_perf_event_init_rate()
755 attr->sample_period = rate; in __hw_perf_event_init_rate()
759 __func__, event->cpu, event->attr.sample_period, in __hw_perf_event_init_rate()
760 event->attr.freq, SAMPLE_FREQ_MODE(hwc)); in __hw_perf_event_init_rate()
768 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init()
769 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init()
772 /* Reserve CPU-measurement sampling facility */ in __hw_perf_event_init()
777 err = -EBUSY; in __hw_perf_event_init()
782 event->destroy = hw_perf_event_destroy; in __hw_perf_event_init()
787 /* Access per-CPU sampling information (query sampling info) */ in __hw_perf_event_init()
789 * The event->cpu value can be -1 to count on every CPU, for example, in __hw_perf_event_init()
791 * sampling info from the current CPU, otherwise use event->cpu to in __hw_perf_event_init()
792 * retrieve the per-CPU information. in __hw_perf_event_init()
798 if (event->cpu == -1) in __hw_perf_event_init()
801 /* Event is pinned to a particular CPU, retrieve the per-CPU in __hw_perf_event_init()
802 * sampling structure for accessing the CPU-specific QSI. in __hw_perf_event_init()
804 cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in __hw_perf_event_init()
805 si = cpuhw->qsi; in __hw_perf_event_init()
813 err = -ENOENT; in __hw_perf_event_init()
819 err = -EBUSY; in __hw_perf_event_init()
829 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) { in __hw_perf_event_init()
831 err = -EPERM; in __hw_perf_event_init()
838 if (attr->config1 & PERF_CPUM_SF_FULL_BLOCKS) in __hw_perf_event_init()
846 hwc->extra_reg.reg = REG_OVERFLOW; in __hw_perf_event_init()
850 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) in __hw_perf_event_init()
853 /* Allocate the per-CPU sampling buffer using the CPU information in __hw_perf_event_init()
855 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling in __hw_perf_event_init()
874 * handler to extract and resolve the PIDs from the basic-sampling in __hw_perf_event_init()
877 if (event->attr.sample_type & PERF_SAMPLE_TID) in __hw_perf_event_init()
879 event->overflow_handler = cpumsf_output_event_pid; in __hw_perf_event_init()
886 u64 sample_type = event->attr.sample_type; in is_callchain_event()
899 return -EOPNOTSUPP; in cpumsf_pmu_event_init()
901 switch (event->attr.type) { in cpumsf_pmu_event_init()
903 if ((event->attr.config != PERF_EVENT_CPUM_SF) && in cpumsf_pmu_event_init()
904 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG)) in cpumsf_pmu_event_init()
905 return -ENOENT; in cpumsf_pmu_event_init()
913 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES) in cpumsf_pmu_event_init()
914 return -ENOENT; in cpumsf_pmu_event_init()
916 return -ENOENT; in cpumsf_pmu_event_init()
919 return -ENOENT; in cpumsf_pmu_event_init()
923 if (event->cpu >= 0 && !cpu_online(event->cpu)) in cpumsf_pmu_event_init()
924 return -ENODEV; in cpumsf_pmu_event_init()
929 if (event->attr.exclude_hv) in cpumsf_pmu_event_init()
930 event->attr.exclude_hv = 0; in cpumsf_pmu_event_init()
931 if (event->attr.exclude_idle) in cpumsf_pmu_event_init()
932 event->attr.exclude_idle = 0; in cpumsf_pmu_event_init()
936 if (event->destroy) in cpumsf_pmu_event_init()
937 event->destroy(event); in cpumsf_pmu_event_init()
947 if (cpuhw->flags & PMU_F_ENABLED) in cpumsf_pmu_enable()
950 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_enable()
961 * facility, but it can be fully re-enabled using sampling controls that in cpumsf_pmu_enable()
964 if (cpuhw->event) { in cpumsf_pmu_enable()
965 hwc = &cpuhw->event->hw; in cpumsf_pmu_enable()
968 * Account number of overflow-designated in cpumsf_pmu_enable()
972 extend_sampling_buffer(&cpuhw->sfb, hwc); in cpumsf_pmu_enable()
975 cpuhw->lsctl.interval = SAMPL_RATE(&cpuhw->event->hw); in cpumsf_pmu_enable()
979 cpuhw->flags |= PMU_F_ENABLED; in cpumsf_pmu_enable()
982 err = lsctl(&cpuhw->lsctl); in cpumsf_pmu_enable()
984 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_enable()
995 cpuhw->lsctl.es, cpuhw->lsctl.cs, cpuhw->lsctl.ed, in cpumsf_pmu_enable()
996 cpuhw->lsctl.cd, cpuhw->lsctl.interval, in cpumsf_pmu_enable()
997 cpuhw->lsctl.tear, cpuhw->lsctl.dear); in cpumsf_pmu_enable()
1007 if (!(cpuhw->flags & PMU_F_ENABLED)) in cpumsf_pmu_disable()
1010 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_disable()
1014 inactive = cpuhw->lsctl; in cpumsf_pmu_disable()
1034 cpuhw->lsctl.tear = si.tear; in cpumsf_pmu_disable()
1035 cpuhw->lsctl.dear = si.dear; in cpumsf_pmu_disable()
1041 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_disable()
1044 /* perf_exclude_event() - Filter event
1047 * @sde_regs: Sample-data-entry (sde) regs structure
1051 * Return non-zero if the event shall be excluded.
1056 if (event->attr.exclude_user && user_mode(regs)) in perf_exclude_event()
1058 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_exclude_event()
1060 if (event->attr.exclude_guest && sde_regs->in_guest) in perf_exclude_event()
1062 if (event->attr.exclude_host && !sde_regs->in_guest) in perf_exclude_event()
1067 /* perf_push_sample() - Push samples to perf
1076 * Return non-zero if an event overflow occurred.
1087 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_push_sample()
1089 /* Setup pt_regs to look like an CPU-measurement external interrupt in perf_push_sample()
1090 * using the Program Request Alert code. The regs.int_parm_long in perf_push_sample()
1091 * field which is unused contains additional sample-data-entry related in perf_push_sample()
1099 psw_bits(regs.psw).ia = basic->ia; in perf_push_sample()
1100 psw_bits(regs.psw).dat = basic->T; in perf_push_sample()
1101 psw_bits(regs.psw).wait = basic->W; in perf_push_sample()
1102 psw_bits(regs.psw).pstate = basic->P; in perf_push_sample()
1103 psw_bits(regs.psw).as = basic->AS; in perf_push_sample()
1109 * A non-zero guest program parameter always indicates a guest in perf_push_sample()
1116 switch (basic->CL) { in perf_push_sample()
1118 sde_regs->in_guest = 0; in perf_push_sample()
1121 sde_regs->in_guest = 1; in perf_push_sample()
1124 if (basic->gpp || basic->prim_asn != 0xffff) in perf_push_sample()
1125 sde_regs->in_guest = 1; in perf_push_sample()
1130 * Store the PID value from the sample-data-entry to be in perf_push_sample()
1133 data.tid_entry.pid = basic->hpp & LPP_PID_MASK; in perf_push_sample()
1140 event->pmu->stop(event, 0); in perf_push_sample()
1149 local64_add(count, &event->count); in perf_event_count_update()
1152 /* hw_collect_samples() - Walk through a sample-data-block and collect samples
1154 * @sdbt: Sample-data-block table
1157 * Walks through a sample-data-block and collects sampling data entries that are
1159 * there can be either basic-sampling or combined-sampling data entries. A
1160 * combined-sampling data entry consists of a basic- and a diagnostic-sampling
1162 * event hardware structure. The function always works with a combined-sampling
1165 * Note that the implementation focuses on basic-sampling data entries and, if
1166 * such an entry is not valid, the entire combined-sampling data entry is
1182 if (!sample->def || sample->LS) in hw_collect_samples()
1186 perf_event_count_update(event, SAMPL_RATE(&event->hw)); in hw_collect_samples()
1189 if (sample->def == 0x0001) { in hw_collect_samples()
1196 if (sample->I == 0 && sample->W == 0) { in hw_collect_samples()
1207 " sampling data entry: te->f %i" in hw_collect_samples()
1209 te->f, sample->def, sample); in hw_collect_samples()
1213 * from a combined basic- and diagnostic-sampling. in hw_collect_samples()
1214 * If only basic-sampling is then active, entries are in hw_collect_samples()
1216 * This is typically the case for sample-data-blocks in hw_collect_samples()
1220 if (!te->f) in hw_collect_samples()
1225 sample->def = 0; in hw_collect_samples()
1230 /* hw_perf_event_update() - Process sampling buffer
1232 * @flush_all: Flag to also flush partially filled sample-data-blocks
1238 * Only full sample-data-blocks are processed. Specify the flash_all flag
1239 * to also walk through partially filled sample-data-blocks. It is ignored
1241 * enforces the processing of full sample-data-blocks only (trailer entries
1242 * with the block-full-indicator bit set).
1246 struct hw_perf_event *hwc = &event->hw; in hw_perf_event_update()
1256 if (SAMPL_DIAG_MODE(&event->hw)) in hw_perf_event_update()
1265 /* Get the trailer entry of the sample-data-block */ in hw_perf_event_update()
1269 if (!te->f) { in hw_perf_event_update()
1276 if (te->overflow) in hw_perf_event_update()
1281 sampl_overflow += te->overflow; in hw_perf_event_update()
1283 /* Timestamps are valid for full sample-data-blocks only */ in hw_perf_event_update()
1286 __func__, (unsigned long)sdbt, te->overflow, in hw_perf_event_update()
1287 (te->f) ? trailer_timestamp(te) : 0ULL); in hw_perf_event_update()
1289 /* Collect all samples from a single sample-data-block and in hw_perf_event_update()
1296 /* Reset trailer (using compare-double-and-swap) */ in hw_perf_event_update()
1298 te_flags = te->flags & ~SDB_TE_BUFFER_FULL_MASK; in hw_perf_event_update()
1300 } while (!cmpxchg_double(&te->flags, &te->overflow, in hw_perf_event_update()
1301 te->flags, te->overflow, in hw_perf_event_update()
1304 /* Advance to next sample-data-block */ in hw_perf_event_update()
1312 /* Stop processing sample-data if all samples of the current in hw_perf_event_update()
1313 * sample-data-block were flushed even if it was not full. in hw_perf_event_update()
1347 #define AUX_SDB_INDEX(aux, i) ((i) % aux->sfb.num_sdb)
1348 #define AUX_SDB_NUM(aux, start, end) (end >= start ? end - start + 1 : 0)
1349 #define AUX_SDB_NUM_ALERT(aux) AUX_SDB_NUM(aux, aux->head, aux->alert_mark)
1350 #define AUX_SDB_NUM_EMPTY(aux) AUX_SDB_NUM(aux, aux->head, aux->empty_mark)
1361 sdb = aux->sdb_index[index]; in aux_sdb_trailer()
1368 * the point of alert indicator. And ignore the SDBs which are not
1372 * 2. Remove alert indicator in the buffer.
1385 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) { in aux_output_end()
1387 if (!(te->flags & SDB_TE_BUFFER_FULL_MASK)) in aux_output_end()
1393 /* Remove alert indicators in the buffer */ in aux_output_end()
1394 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_end()
1395 te->flags &= ~SDB_TE_ALERT_REQ_MASK; in aux_output_end()
1398 __func__, i, range_scan, aux->head); in aux_output_end()
1419 if (WARN_ON_ONCE(handle->head & ~PAGE_MASK)) in aux_output_begin()
1420 return -EINVAL; in aux_output_begin()
1422 aux->head = handle->head >> PAGE_SHIFT; in aux_output_begin()
1423 range = (handle->size + 1) >> PAGE_SHIFT; in aux_output_begin()
1425 return -ENOMEM; in aux_output_begin()
1428 * SDBs between aux->head and aux->empty_mark are already ready in aux_output_begin()
1432 "%s: range %ld head %ld alert %ld empty %ld\n", in aux_output_begin()
1433 __func__, range, aux->head, aux->alert_mark, in aux_output_begin()
1434 aux->empty_mark); in aux_output_begin()
1436 range_scan = range - AUX_SDB_NUM_EMPTY(aux); in aux_output_begin()
1437 idx = aux->empty_mark + 1; in aux_output_begin()
1440 te->flags &= ~(SDB_TE_BUFFER_FULL_MASK | in aux_output_begin()
1442 te->overflow = 0; in aux_output_begin()
1445 aux->empty_mark = aux->head + range - 1; in aux_output_begin()
1448 /* Set alert indicator */ in aux_output_begin()
1449 aux->alert_mark = aux->head + range/2 - 1; in aux_output_begin()
1450 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_begin()
1451 te->flags = te->flags | SDB_TE_ALERT_REQ_MASK; in aux_output_begin()
1454 head = AUX_SDB_INDEX(aux, aux->head); in aux_output_begin()
1455 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE]; in aux_output_begin()
1457 cpuhw->lsctl.tear = base + offset * sizeof(unsigned long); in aux_output_begin()
1458 cpuhw->lsctl.dear = aux->sdb_index[head]; in aux_output_begin()
1460 debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld empty %ld " in aux_output_begin()
1462 aux->head, aux->alert_mark, aux->empty_mark, in aux_output_begin()
1464 cpuhw->lsctl.tear, cpuhw->lsctl.dear); in aux_output_begin()
1470 * Set alert indicator on SDB at index @alert_index while sampler is running.
1483 orig_flags = te->flags; in aux_set_alert()
1484 *overflow = orig_overflow = te->overflow; in aux_set_alert()
1494 } while (!cmpxchg_double(&te->flags, &te->overflow, in aux_set_alert()
1501 * aux_reset_buffer() - Scan and setup SDBs for new samples
1503 * @range: The range of SDBs to scan started from aux->head
1506 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1508 * If yes, that means new data is already there before we can set an alert
1509 * indicator. Caller should try to set alert indicator to some position behind.
1511 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1513 * (clear full indicator and alert indicator) for new data.
1514 * If aux->alert_mark fall in this area, just set it. Overflow count is
1517 * SDBs between aux->head and aux->empty_mark are already reset at last time.
1520 * Return true if alert indicator is set successfully and false if not.
1529 debug_sprintf_event(sfdbg, 6, "%s: range %ld head %ld alert %ld " in aux_reset_buffer()
1530 "empty %ld\n", __func__, range, aux->head, in aux_reset_buffer()
1531 aux->alert_mark, aux->empty_mark); in aux_reset_buffer()
1535 * Just set alert indicator. Should check race with hardware in aux_reset_buffer()
1538 return aux_set_alert(aux, aux->alert_mark, overflow); in aux_reset_buffer()
1540 if (aux->alert_mark <= aux->empty_mark) in aux_reset_buffer()
1542 * Set alert indicator on empty SDB. Should check race in aux_reset_buffer()
1545 if (!aux_set_alert(aux, aux->alert_mark, overflow)) in aux_reset_buffer()
1549 * Scan the SDBs to clear full and alert indicator used previously. in aux_reset_buffer()
1550 * Start scanning from one SDB behind empty_mark. If the new alert in aux_reset_buffer()
1553 range_scan = range - AUX_SDB_NUM_EMPTY(aux); in aux_reset_buffer()
1554 idx_old = idx = aux->empty_mark + 1; in aux_reset_buffer()
1558 orig_flags = te->flags; in aux_reset_buffer()
1559 orig_overflow = te->overflow; in aux_reset_buffer()
1561 if (idx == aux->alert_mark) in aux_reset_buffer()
1565 } while (!cmpxchg_double(&te->flags, &te->overflow, in aux_reset_buffer()
1572 aux->empty_mark = aux->head + range - 1; in aux_reset_buffer()
1576 idx - 1, aux->empty_mark); in aux_reset_buffer()
1581 * Measurement alert handler for diagnostic mode sampling.
1589 struct perf_output_handle *handle = &cpuhw->handle; in hw_collect_aux()
1598 debug_sprintf_event(sfdbg, 6, "%s: #alert %ld\n", __func__, in hw_collect_aux()
1602 num_sdb = aux->sfb.num_sdb; in hw_collect_aux()
1605 aux = perf_aux_output_begin(handle, cpuhw->event); in hw_collect_aux()
1606 if (handle->size == 0) { in hw_collect_aux()
1608 "diagnostic-sampling mode is full\n", in hw_collect_aux()
1619 aux->head = handle->head >> PAGE_SHIFT; in hw_collect_aux()
1620 range = (handle->size + 1) >> PAGE_SHIFT; in hw_collect_aux()
1622 aux->alert_mark = aux->head; in hw_collect_aux()
1624 aux->alert_mark = aux->head + range/2 - 1; in hw_collect_aux()
1632 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1634 "pages to overflow\n", aux->sfb.num_sdb); in hw_collect_aux()
1637 aux->head, range, overflow); in hw_collect_aux()
1640 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1641 debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld " in hw_collect_aux()
1644 aux->head, aux->alert_mark); in hw_collect_aux()
1649 debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld " in hw_collect_aux()
1650 "empty %ld\n", __func__, aux->head, in hw_collect_aux()
1651 aux->alert_mark, aux->empty_mark); in hw_collect_aux()
1666 num_sdbt = aux->sfb.num_sdbt; in aux_buffer_free()
1668 free_page(aux->sdbt_index[i]); in aux_buffer_free()
1670 kfree(aux->sdbt_index); in aux_buffer_free()
1671 kfree(aux->sdb_index); in aux_buffer_free()
1684 te->clock_base = 1; in aux_sdb_init()
1685 te->progusage2 = tod_clock_base.tod; in aux_sdb_init()
1689 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1690 * @event: Event the buffer is setup for, event->cpu == -1 means current
1698 * the task among online cpus when it is a per-thread event.
1729 sfb = &aux->sfb; in aux_buffer_setup()
1733 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1734 if (!aux->sdbt_index) in aux_buffer_setup()
1738 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1739 if (!aux->sdb_index) in aux_buffer_setup()
1743 sfb->num_sdbt = 0; in aux_buffer_setup()
1744 sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL); in aux_buffer_setup()
1745 if (!sfb->sdbt) in aux_buffer_setup()
1747 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt; in aux_buffer_setup()
1748 tail = sfb->tail = sfb->sdbt; in aux_buffer_setup()
1759 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new; in aux_buffer_setup()
1766 aux->sdb_index[i] = (unsigned long)pages[i]; in aux_buffer_setup()
1769 sfb->num_sdb = nr_pages; in aux_buffer_setup()
1772 *tail = (unsigned long) sfb->sdbt + 1; in aux_buffer_setup()
1773 sfb->tail = tail; in aux_buffer_setup()
1780 aux->empty_mark = sfb->num_sdb - 1; in aux_buffer_setup()
1783 sfb->num_sdbt, sfb->num_sdb); in aux_buffer_setup()
1789 for (i = 0; i < sfb->num_sdbt; i++) in aux_buffer_setup()
1790 free_page(aux->sdbt_index[i]); in aux_buffer_setup()
1791 kfree(aux->sdb_index); in aux_buffer_setup()
1793 kfree(aux->sdbt_index); in aux_buffer_setup()
1802 /* Nothing to do ... updates are interrupt-driven */ in cpumsf_pmu_read()
1807 * Return non-zero on error and zero on passed checks.
1816 if (event->cpu == -1) { in cpumsf_pmu_check_period()
1818 return -ENODEV; in cpumsf_pmu_check_period()
1820 /* Event is pinned to a particular CPU, retrieve the per-CPU in cpumsf_pmu_check_period()
1821 * sampling structure for accessing the CPU-specific QSI. in cpumsf_pmu_check_period()
1823 struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in cpumsf_pmu_check_period()
1825 si = cpuhw->qsi; in cpumsf_pmu_check_period()
1828 do_freq = !!SAMPLE_FREQ_MODE(&event->hw); in cpumsf_pmu_check_period()
1831 return -EINVAL; in cpumsf_pmu_check_period()
1833 event->attr.sample_period = rate; in cpumsf_pmu_check_period()
1834 SAMPL_RATE(&event->hw) = rate; in cpumsf_pmu_check_period()
1835 hw_init_period(&event->hw, SAMPL_RATE(&event->hw)); in cpumsf_pmu_check_period()
1838 __func__, event->cpu, value, in cpumsf_pmu_check_period()
1839 event->attr.sample_period, do_freq); in cpumsf_pmu_check_period()
1850 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) in cpumsf_pmu_start()
1854 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); in cpumsf_pmu_start()
1856 perf_pmu_disable(event->pmu); in cpumsf_pmu_start()
1857 event->hw.state = 0; in cpumsf_pmu_start()
1858 cpuhw->lsctl.cs = 1; in cpumsf_pmu_start()
1859 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_start()
1860 cpuhw->lsctl.cd = 1; in cpumsf_pmu_start()
1861 perf_pmu_enable(event->pmu); in cpumsf_pmu_start()
1871 if (event->hw.state & PERF_HES_STOPPED) in cpumsf_pmu_stop()
1874 perf_pmu_disable(event->pmu); in cpumsf_pmu_stop()
1875 cpuhw->lsctl.cs = 0; in cpumsf_pmu_stop()
1876 cpuhw->lsctl.cd = 0; in cpumsf_pmu_stop()
1877 event->hw.state |= PERF_HES_STOPPED; in cpumsf_pmu_stop()
1879 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) { in cpumsf_pmu_stop()
1881 event->hw.state |= PERF_HES_UPTODATE; in cpumsf_pmu_stop()
1883 perf_pmu_enable(event->pmu); in cpumsf_pmu_stop()
1892 if (cpuhw->flags & PMU_F_IN_USE) in cpumsf_pmu_add()
1893 return -EAGAIN; in cpumsf_pmu_add()
1895 if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt) in cpumsf_pmu_add()
1896 return -EINVAL; in cpumsf_pmu_add()
1899 perf_pmu_disable(event->pmu); in cpumsf_pmu_add()
1901 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in cpumsf_pmu_add()
1904 * using the SDB-table start. Reset TEAR_REG event hardware register in cpumsf_pmu_add()
1908 cpuhw->lsctl.s = 0; in cpumsf_pmu_add()
1909 cpuhw->lsctl.h = 1; in cpumsf_pmu_add()
1910 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw); in cpumsf_pmu_add()
1911 if (!SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1912 cpuhw->lsctl.tear = (unsigned long) cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1913 cpuhw->lsctl.dear = *(unsigned long *) cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1914 TEAR_REG(&event->hw) = (unsigned long) cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1919 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) { in cpumsf_pmu_add()
1920 err = -EAGAIN; in cpumsf_pmu_add()
1923 if (SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1924 aux = perf_aux_output_begin(&cpuhw->handle, event); in cpumsf_pmu_add()
1926 err = -EINVAL; in cpumsf_pmu_add()
1929 err = aux_output_begin(&cpuhw->handle, aux, cpuhw); in cpumsf_pmu_add()
1932 cpuhw->lsctl.ed = 1; in cpumsf_pmu_add()
1934 cpuhw->lsctl.es = 1; in cpumsf_pmu_add()
1937 cpuhw->event = event; in cpumsf_pmu_add()
1938 cpuhw->flags |= PMU_F_IN_USE; in cpumsf_pmu_add()
1944 perf_pmu_enable(event->pmu); in cpumsf_pmu_add()
1952 perf_pmu_disable(event->pmu); in cpumsf_pmu_del()
1955 cpuhw->lsctl.es = 0; in cpumsf_pmu_del()
1956 cpuhw->lsctl.ed = 0; in cpumsf_pmu_del()
1957 cpuhw->flags &= ~PMU_F_IN_USE; in cpumsf_pmu_del()
1958 cpuhw->event = NULL; in cpumsf_pmu_del()
1960 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_del()
1961 aux_output_end(&cpuhw->handle); in cpumsf_pmu_del()
1963 perf_pmu_enable(event->pmu); in cpumsf_pmu_del()
1995 PMU_FORMAT_ATTR(event, "config:0-63");
2039 unsigned int alert, unsigned long unused) in cpumf_measurement_alert() argument
2043 if (!(alert & CPU_MF_INT_SF_MASK)) in cpumf_measurement_alert()
2050 if (!(cpuhw->flags & PMU_F_RESERVED)) in cpumf_measurement_alert()
2053 /* The processing below must take care of multiple alert events that in cpumf_measurement_alert()
2056 /* Program alert request */ in cpumf_measurement_alert()
2057 if (alert & CPU_MF_INT_SF_PRA) { in cpumf_measurement_alert()
2058 if (cpuhw->flags & PMU_F_IN_USE) in cpumf_measurement_alert()
2059 if (SAMPL_DIAG_MODE(&cpuhw->event->hw)) in cpumf_measurement_alert()
2062 hw_perf_event_update(cpuhw->event, 0); in cpumf_measurement_alert()
2064 WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE)); in cpumf_measurement_alert()
2067 /* Report measurement alerts only for non-PRA codes */ in cpumf_measurement_alert()
2068 if (alert != CPU_MF_INT_SF_PRA) in cpumf_measurement_alert()
2069 debug_sprintf_event(sfdbg, 6, "%s: alert %#x\n", __func__, in cpumf_measurement_alert()
2070 alert); in cpumf_measurement_alert()
2073 if (alert & CPU_MF_INT_SF_SACA) in cpumf_measurement_alert()
2074 qsi(&cpuhw->qsi); in cpumf_measurement_alert()
2076 /* Loss of sample data due to high-priority machine activities */ in cpumf_measurement_alert()
2077 if (alert & CPU_MF_INT_SF_LSDA) { in cpumf_measurement_alert()
2079 cpuhw->flags |= PMU_F_ERR_LSDA; in cpumf_measurement_alert()
2084 if (alert & (CPU_MF_INT_SF_IAE|CPU_MF_INT_SF_ISE)) { in cpumf_measurement_alert()
2085 pr_err("A sampling buffer entry is incorrect (alert=0x%x)\n", in cpumf_measurement_alert()
2086 alert); in cpumf_measurement_alert()
2087 cpuhw->flags |= PMU_F_ERR_IBE; in cpumf_measurement_alert()
2119 return -ENODEV; in param_get_sfb_size()
2129 return -ENODEV; in param_set_sfb_size()
2131 return -EINVAL; in param_set_sfb_size()
2137 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL; in param_set_sfb_size()
2142 rc = -EINVAL; in param_set_sfb_size()
2175 return -ENODEV; in init_cpum_sampling_pmu()
2180 return -ENODEV; in init_cpum_sampling_pmu()
2184 return -ENODEV; in init_cpum_sampling_pmu()
2188 return -EINVAL; in init_cpum_sampling_pmu()
2203 return -ENOMEM; in init_cpum_sampling_pmu()