79 	struct sf_buffer sfb;  member
92 	struct sf_buffer sfb;	    /* Sampling buffer */  member
118 	return !!cpuhw->sfb.sdbt;  in sf_buffer_available()
124 static void free_sampling_buffer(struct sf_buffer *sfb)  in free_sampling_buffer()  argument
128 	if (!sfb->sdbt)  in free_sampling_buffer()
131 	sdbt = sfb->sdbt;  in free_sampling_buffer()
146 			if (curr == sfb->sdbt)  in free_sampling_buffer()
160 			    "free_sampling_buffer: freed sdbt=%p\n", sfb->sdbt);  in free_sampling_buffer()
161 	memset(sfb, 0, sizeof(*sfb));  in free_sampling_buffer()
192 static int realloc_sampling_buffer(struct sf_buffer *sfb,  in realloc_sampling_buffer()  argument
198 	if (!sfb->sdbt || !sfb->tail)  in realloc_sampling_buffer()
201 	if (!is_link_entry(sfb->tail))  in realloc_sampling_buffer()
209 	tail = sfb->tail;  in realloc_sampling_buffer()
214 	if (sfb->sdbt != get_next_sdbt(tail)) {  in realloc_sampling_buffer()
218 				    (void *) sfb->sdbt, (void *) tail);  in realloc_sampling_buffer()
232 			sfb->num_sdbt++;  in realloc_sampling_buffer()
246 		sfb->num_sdb++;  in realloc_sampling_buffer()
251 	*tail = (unsigned long) sfb->sdbt + 1;  in realloc_sampling_buffer()
252 	sfb->tail = tail;  in realloc_sampling_buffer()
256 			    sfb->num_sdbt, sfb->num_sdb);  in realloc_sampling_buffer()
271 static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb)  in alloc_sampling_buffer()  argument
275 	if (sfb->sdbt)  in alloc_sampling_buffer()
279 	sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL);  in alloc_sampling_buffer()
280 	if (!sfb->sdbt)  in alloc_sampling_buffer()
282 	sfb->num_sdb = 0;  in alloc_sampling_buffer()
283 	sfb->num_sdbt = 1;  in alloc_sampling_buffer()
288 	sfb->tail = sfb->sdbt;  in alloc_sampling_buffer()
289 	*sfb->tail = (unsigned long)(void *) sfb->sdbt + 1;  in alloc_sampling_buffer()
292 	rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL);  in alloc_sampling_buffer()
294 		free_sampling_buffer(sfb);  in alloc_sampling_buffer()
300 			sfb->sdbt, (void *) *sfb->sdbt);  in alloc_sampling_buffer()
322 static unsigned long sfb_pending_allocs(struct sf_buffer *sfb,  in sfb_pending_allocs()  argument
325 	if (!sfb->sdbt)  in sfb_pending_allocs()
327 	if (SFB_ALLOC_REG(hwc) > sfb->num_sdb)  in sfb_pending_allocs()
328 		return SFB_ALLOC_REG(hwc) - sfb->num_sdb;  in sfb_pending_allocs()
332 static int sfb_has_pending_allocs(struct sf_buffer *sfb,  in sfb_has_pending_allocs()  argument
335 	return sfb_pending_allocs(sfb, hwc) > 0;  in sfb_has_pending_allocs()
354 	if (cpuhw->sfb.sdbt)  in deallocate_buffers()
355 		free_sampling_buffer(&cpuhw->sfb);  in deallocate_buffers()
412 	return alloc_sampling_buffer(&cpuhw->sfb,  in allocate_buffers()
413 				     sfb_pending_allocs(&cpuhw->sfb, hwc));  in allocate_buffers()
460 	ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,  in sfb_account_overflows()
464 	num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb);  in sfb_account_overflows()
484 static void extend_sampling_buffer(struct sf_buffer *sfb,  in extend_sampling_buffer()  argument
490 	num = sfb_pending_allocs(sfb, hwc);  in extend_sampling_buffer()
493 	num_old = sfb->num_sdb;  in extend_sampling_buffer()
505 	rc = realloc_sampling_buffer(sfb, num, GFP_ATOMIC);  in extend_sampling_buffer()
510 	if (sfb_has_pending_allocs(sfb, hwc))  in extend_sampling_buffer()
513 				    num, sfb->num_sdb - num_old,  in extend_sampling_buffer()
514 				    sfb_pending_allocs(sfb, hwc));  in extend_sampling_buffer()
952 			if (sfb_has_pending_allocs(&cpuhw->sfb, hwc))  in cpumsf_pmu_enable()
953 				extend_sampling_buffer(&cpuhw->sfb, hwc);  in cpumsf_pmu_enable()
1321 #define AUX_SDB_INDEX(aux, i) ((i) % aux->sfb.num_sdb)
1567 	num_sdb = aux->sfb.num_sdb;  in hw_collect_aux()
1630 	num_sdbt = aux->sfb.num_sdbt;  in aux_buffer_free()
1670 	struct sf_buffer *sfb;  in aux_buffer_setup()  local
1694 	sfb = &aux->sfb;  in aux_buffer_setup()
1708 	sfb->num_sdbt = 0;  in aux_buffer_setup()
1709 	sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL);  in aux_buffer_setup()
1710 	if (!sfb->sdbt)  in aux_buffer_setup()
1712 	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;  in aux_buffer_setup()
1713 	tail = sfb->tail = sfb->sdbt;  in aux_buffer_setup()
1724 			aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;  in aux_buffer_setup()
1734 	sfb->num_sdb = nr_pages;  in aux_buffer_setup()
1737 	*tail = (unsigned long) sfb->sdbt + 1;  in aux_buffer_setup()
1738 	sfb->tail = tail;  in aux_buffer_setup()
1745 	aux->empty_mark = sfb->num_sdb - 1;  in aux_buffer_setup()
1749 			    sfb->num_sdbt, sfb->num_sdb);  in aux_buffer_setup()
1755 	for (i = 0; i < sfb->num_sdbt; i++)  in aux_buffer_setup()
1861 	if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt)  in cpumsf_pmu_add()
1878 		cpuhw->lsctl.tear = (unsigned long) cpuhw->sfb.sdbt;  in cpumsf_pmu_add()
1879 		cpuhw->lsctl.dear = *(unsigned long *) cpuhw->sfb.sdbt;  in cpumsf_pmu_add()
1880 		hw_reset_registers(&event->hw, cpuhw->sfb.sdbt);  in cpumsf_pmu_add()