706 static void perf_ibs_get_mem_op(union ibs_op_data3 *op_data3,  in perf_ibs_get_mem_op()  argument
713 	if (op_data3->ld_op)  in perf_ibs_get_mem_op()
715 	else if (op_data3->st_op)  in perf_ibs_get_mem_op()
762 				  union ibs_op_data3 *op_data3,  in perf_ibs_get_mem_lvl()  argument
775 	if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO)  in perf_ibs_get_mem_lvl()
779 	if (op_data3->dc_miss == 0)  in perf_ibs_get_mem_lvl()
783 	if (op_data3->l2_miss == 0) {  in perf_ibs_get_mem_lvl()
786 		    !(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc))  in perf_ibs_get_mem_lvl()
837 	if (op_data3->dc_miss_no_mab_alloc)  in perf_ibs_get_mem_lvl()
890 static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3,  in perf_ibs_get_tlb_lvl()  argument
897 	if (!op_data3->dc_lin_addr_valid)  in perf_ibs_get_tlb_lvl()
900 	if (!op_data3->dc_l1tlb_miss) {  in perf_ibs_get_tlb_lvl()
905 	if (!op_data3->dc_l2tlb_miss) {  in perf_ibs_get_tlb_lvl()
913 static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3,  in perf_ibs_get_mem_lock()  argument
920 	if (op_data3->dc_locked_op)  in perf_ibs_get_mem_lock()
929 				  union ibs_op_data3 *op_data3)  in perf_ibs_get_data_src()  argument
933 	data_src->val |= perf_ibs_get_mem_lvl(op_data2, op_data3, data);  in perf_ibs_get_data_src()
935 	perf_ibs_get_tlb_lvl(op_data3, data);  in perf_ibs_get_data_src()
936 	perf_ibs_get_mem_lock(op_data3, data);  in perf_ibs_get_data_src()
940 				   union ibs_op_data3 *op_data3)  in perf_ibs_get_op_data2()  argument
946 	    (op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) {  in perf_ibs_get_op_data2()
961 	union ibs_op_data3 op_data3;  in perf_ibs_parse_ld_st_data()  local
966 	op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)];  in perf_ibs_parse_ld_st_data()
968 	perf_ibs_get_mem_op(&op_data3, data);  in perf_ibs_parse_ld_st_data()
973 	op_data2.val = perf_ibs_get_op_data2(ibs_data, &op_data3);  in perf_ibs_parse_ld_st_data()
976 		perf_ibs_get_data_src(ibs_data, data, &op_data2, &op_data3);  in perf_ibs_parse_ld_st_data()
980 	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE && op_data3.dc_miss &&  in perf_ibs_parse_ld_st_data()
985 			data->weight.var1_dw = op_data3.dc_miss_lat;  in perf_ibs_parse_ld_st_data()
988 			data->weight.full = op_data3.dc_miss_lat;  in perf_ibs_parse_ld_st_data()
993 	if (sample_type & PERF_SAMPLE_ADDR && op_data3.dc_lin_addr_valid) {  in perf_ibs_parse_ld_st_data()
998 	if (sample_type & PERF_SAMPLE_PHYS_ADDR && op_data3.dc_phy_addr_valid) {  in perf_ibs_parse_ld_st_data()