346 				     u64 spe_events_id, u64 data_src)  in arm_spe__synth_mem_sample()  argument
359 	sample.data_src = data_src;  in arm_spe__synth_mem_sample()
384 					     u64 spe_events_id, u64 data_src)  in arm_spe__synth_instruction_sample()  argument
405 	sample.data_src = data_src;  in arm_spe__synth_instruction_sample()
420 						union perf_mem_data_src *data_src)  in arm_spe__synth_data_source_neoverse()  argument
435 		data_src->mem_lvl = PERF_MEM_LVL_NA;  in arm_spe__synth_data_source_neoverse()
436 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA;  in arm_spe__synth_data_source_neoverse()
437 		data_src->mem_snoop = PERF_MEM_SNOOP_NA;  in arm_spe__synth_data_source_neoverse()
443 		data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
444 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1;  in arm_spe__synth_data_source_neoverse()
445 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;  in arm_spe__synth_data_source_neoverse()
448 		data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
449 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;  in arm_spe__synth_data_source_neoverse()
450 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;  in arm_spe__synth_data_source_neoverse()
453 		data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
454 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;  in arm_spe__synth_data_source_neoverse()
455 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;  in arm_spe__synth_data_source_neoverse()
463 		data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
464 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;  in arm_spe__synth_data_source_neoverse()
465 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;  in arm_spe__synth_data_source_neoverse()
471 		data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
472 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;  in arm_spe__synth_data_source_neoverse()
473 		data_src->mem_snoop = PERF_MEM_SNOOP_HIT;  in arm_spe__synth_data_source_neoverse()
480 		data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1;  in arm_spe__synth_data_source_neoverse()
481 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE;  in arm_spe__synth_data_source_neoverse()
482 		data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;  in arm_spe__synth_data_source_neoverse()
483 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;  in arm_spe__synth_data_source_neoverse()
486 		data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
487 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM;  in arm_spe__synth_data_source_neoverse()
488 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;  in arm_spe__synth_data_source_neoverse()
496 					       union perf_mem_data_src *data_src)  in arm_spe__synth_data_source_generic()  argument
499 		data_src->mem_lvl = PERF_MEM_LVL_L3;  in arm_spe__synth_data_source_generic()
502 			data_src->mem_lvl |= PERF_MEM_LVL_MISS;  in arm_spe__synth_data_source_generic()
504 			data_src->mem_lvl |= PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_generic()
506 		data_src->mem_lvl = PERF_MEM_LVL_L1;  in arm_spe__synth_data_source_generic()
509 			data_src->mem_lvl |= PERF_MEM_LVL_MISS;  in arm_spe__synth_data_source_generic()
511 			data_src->mem_lvl |= PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_generic()
515 		data_src->mem_lvl |= PERF_MEM_LVL_REM_CCE1;  in arm_spe__synth_data_source_generic()
520 	union perf_mem_data_src	data_src = { .mem_op = PERF_MEM_OP_NA };  in arm_spe__synth_data_source()  local
524 		data_src.mem_op = PERF_MEM_OP_LOAD;  in arm_spe__synth_data_source()
526 		data_src.mem_op = PERF_MEM_OP_STORE;  in arm_spe__synth_data_source()
531 		arm_spe__synth_data_source_neoverse(record, &data_src);  in arm_spe__synth_data_source()
533 		arm_spe__synth_data_source_generic(record, &data_src);  in arm_spe__synth_data_source()
536 		data_src.mem_dtlb = PERF_MEM_TLB_WK;  in arm_spe__synth_data_source()
539 			data_src.mem_dtlb |= PERF_MEM_TLB_MISS;  in arm_spe__synth_data_source()
541 			data_src.mem_dtlb |= PERF_MEM_TLB_HIT;  in arm_spe__synth_data_source()
544 	return data_src.val;  in arm_spe__synth_data_source()
551 	u64 data_src;  in arm_spe_sample()  local
554 	data_src = arm_spe__synth_data_source(record, spe->midr);  in arm_spe_sample()
559 							data_src);  in arm_spe_sample()
566 							data_src);  in arm_spe_sample()
575 							data_src);  in arm_spe_sample()
582 							data_src);  in arm_spe_sample()
591 							data_src);  in arm_spe_sample()
598 							data_src);  in arm_spe_sample()
613 						data_src);  in arm_spe_sample()
622 	if (spe->sample_memory && data_src) {  in arm_spe_sample()
623 		err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src);  in arm_spe_sample()
629 		err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src);  in arm_spe_sample()