326 				     u64 spe_events_id, u64 data_src)  in arm_spe__synth_mem_sample()  argument
339 	sample.data_src = data_src;  in arm_spe__synth_mem_sample()
364 					     u64 spe_events_id, u64 data_src)  in arm_spe__synth_instruction_sample()  argument
385 	sample.data_src = data_src;  in arm_spe__synth_instruction_sample()
400 						union perf_mem_data_src *data_src)  in arm_spe__synth_data_source_neoverse()  argument
415 		data_src->mem_lvl = PERF_MEM_LVL_NA;  in arm_spe__synth_data_source_neoverse()
416 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA;  in arm_spe__synth_data_source_neoverse()
417 		data_src->mem_snoop = PERF_MEM_SNOOP_NA;  in arm_spe__synth_data_source_neoverse()
423 		data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
424 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1;  in arm_spe__synth_data_source_neoverse()
425 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;  in arm_spe__synth_data_source_neoverse()
428 		data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
429 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;  in arm_spe__synth_data_source_neoverse()
430 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;  in arm_spe__synth_data_source_neoverse()
433 		data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
434 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;  in arm_spe__synth_data_source_neoverse()
435 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;  in arm_spe__synth_data_source_neoverse()
443 		data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
444 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;  in arm_spe__synth_data_source_neoverse()
445 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;  in arm_spe__synth_data_source_neoverse()
451 		data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
452 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;  in arm_spe__synth_data_source_neoverse()
453 		data_src->mem_snoop = PERF_MEM_SNOOP_HIT;  in arm_spe__synth_data_source_neoverse()
460 		data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1;  in arm_spe__synth_data_source_neoverse()
461 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE;  in arm_spe__synth_data_source_neoverse()
462 		data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;  in arm_spe__synth_data_source_neoverse()
463 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;  in arm_spe__synth_data_source_neoverse()
466 		data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_neoverse()
467 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM;  in arm_spe__synth_data_source_neoverse()
468 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;  in arm_spe__synth_data_source_neoverse()
476 					       union perf_mem_data_src *data_src)  in arm_spe__synth_data_source_generic()  argument
479 		data_src->mem_lvl = PERF_MEM_LVL_L3;  in arm_spe__synth_data_source_generic()
482 			data_src->mem_lvl |= PERF_MEM_LVL_MISS;  in arm_spe__synth_data_source_generic()
484 			data_src->mem_lvl |= PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_generic()
486 		data_src->mem_lvl = PERF_MEM_LVL_L1;  in arm_spe__synth_data_source_generic()
489 			data_src->mem_lvl |= PERF_MEM_LVL_MISS;  in arm_spe__synth_data_source_generic()
491 			data_src->mem_lvl |= PERF_MEM_LVL_HIT;  in arm_spe__synth_data_source_generic()
495 		data_src->mem_lvl |= PERF_MEM_LVL_REM_CCE1;  in arm_spe__synth_data_source_generic()
500 	union perf_mem_data_src	data_src = { 0 };  in arm_spe__synth_data_source()  local
504 		data_src.mem_op = PERF_MEM_OP_LOAD;  in arm_spe__synth_data_source()
506 		data_src.mem_op = PERF_MEM_OP_STORE;  in arm_spe__synth_data_source()
511 		arm_spe__synth_data_source_neoverse(record, &data_src);  in arm_spe__synth_data_source()
513 		arm_spe__synth_data_source_generic(record, &data_src);  in arm_spe__synth_data_source()
516 		data_src.mem_dtlb = PERF_MEM_TLB_WK;  in arm_spe__synth_data_source()
519 			data_src.mem_dtlb |= PERF_MEM_TLB_MISS;  in arm_spe__synth_data_source()
521 			data_src.mem_dtlb |= PERF_MEM_TLB_HIT;  in arm_spe__synth_data_source()
524 	return data_src.val;  in arm_spe__synth_data_source()
531 	u64 data_src;  in arm_spe_sample()  local
534 	data_src = arm_spe__synth_data_source(record, spe->midr);  in arm_spe_sample()
539 							data_src);  in arm_spe_sample()
546 							data_src);  in arm_spe_sample()
555 							data_src);  in arm_spe_sample()
562 							data_src);  in arm_spe_sample()
571 							data_src);  in arm_spe_sample()
578 							data_src);  in arm_spe_sample()
593 						data_src);  in arm_spe_sample()
602 	if (spe->sample_memory && data_src) {  in arm_spe_sample()
603 		err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src);  in arm_spe_sample()
609 		err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src);  in arm_spe_sample()