1#!/bin/bash
2#
3# @echo off
4#
5# Compare results of two runs of tests
6#
7# (for comments see Do and asltsrun utilities).
8
9# Compare the summary information of two runs
10# arg1 - the first run test status lines file
11# arg2 - the second run test status lines file
12cmp_two_runs()
13{
14	local ret
15
16	echo "diff old: $1 -- new: $2"
17	diff -s "$1" "$2"
18	ret=$?
19	echo ""
20	return $ret
21}
22
23# Chose line of file by head
24# arg1 - file pathname
25# arg2 - delimiter
26# arg3 - string, the first element of line
27chose_line_of_file()
28{
29	cat "$1" |\
30	while [ 1 ]
31	do
32		read line
33		if [ $? -ne 0 ] ; then
34			break
35		fi
36
37		tcasename=`echo "$line" | awk -F"$2" '{print $2}'`
38		if [ "$tcasename" == "$3" ]; then
39			echo "$line"
40			return 1
41		fi
42	done
43}
44
45# Compare and report two run times
46#
47# arg1 - run time 1
48# arg2 - run time 2
49#
50# Return:
51#     0 - T1 == T2
52#     1 - T1 < T2
53#     2 - T1 > T2
54#   3 - bad layout of T1
55#   4 - bad layout of T2
56report_run_time_cmp()
57{
58	local rval=0 str ret t0 t1
59	local diff01 diff01str
60
61
62	str=`diff_of_str_times "$1" "$2"`
63	ret=$?
64
65	if [ $ret -gt 2 ] ; then
66		echo "report_run_time_cmp: bad layout, <$1>, <$2>"
67		return 3
68	fi
69
70	t0=`echo "$str" | awk -F"|" '{print $1}'`
71	t1=`echo "$str" | awk -F"|" '{print $2}'`
72	diff01=`echo "$str" | awk -F"|" '{print $3}'`
73	diff01str=`echo "$str" | awk -F"|" '{print $4}'`
74
75	if [ $ret -eq 2 ] ; then
76		# 2 means arg1 is greater than arg2
77		percent=$[ ($diff01 * 10000) / ($t1 * 100 + 1) ]
78		echo "WARNING, TOTAL, time of tests run grown from $2 to $1, (+$diff01str, $percent% of $2)"
79		rval=2
80	elif [ $ret -eq 1 ] ; then
81		# 1 means arg1 is less than arg2
82		percent=$[ ($diff01 * 10000) / ($t0 * 100 + 1) ]
83		echo "INFO,    TOTAL, time of tests run reduced from "$2" to "$1", (-$diff01str, $percent% of $2)"
84		rval=1
85	fi
86
87	return $rval
88}
89
90# Compare and report two memory statistics
91# arg1 - the test case name
92# arg2 - component
93# arg3 - the value on the first run
94# arg4 - the value on the second run
95report_mem_stat_cmp()
96{
97	local rval=0 x percent
98
99	if [[ $4 -eq 0 ]]; then
100		return 0
101	fi
102
103#
104# We are only interested in memory differences that are greater than 10%
105#
106	if [[ $3 -gt $4 ]]; then
107		x=$[ $3 - $4 ]
108		percent=$[ ($x * 100) / $4 ]
109
110		if [[ $percent -lt 10 ]]; then
111			return 1
112		fi
113		echo "WARNING, $1, memory use grown ($2): $3, $4, (+$x, $percent% of $4)"
114		rval=1
115	elif [[ $3 -lt $4 ]]; then
116		x=$[ $4 - $3 ]
117		percent=$[ ($x * 100) / $4 ]
118
119		if [[ $percent -lt 10 ]]; then
120			return 2
121		fi
122		echo "INFO,    $1, memory use reduced ($2): $3, $4, (-$x, $percent% of $4)"
123		rval=2
124	fi
125	return $rval
126}
127
128# Compare and report two exceptions statistics
129# arg1 - the test case name
130# arg2 - component
131# arg3 - the value on the first run
132# arg4 - the value on the second run
133report_exceptions_stat_cmp()
134{
135	local rval=0
136
137	if [[ $3 -gt $4 ]]; then
138		echo "WARNING, $1, exceptions occurrence number grown ($2): $3, $4"
139		rval=1
140	elif [[ $3 -lt $4 ]]; then
141		echo "INFO,    $1, exceptions occurrence number reduced ($2): $3, $4"
142		rval=2
143	fi
144	return $rval
145}
146
147# Compare status lines of the same test case of two different runs
148# arg1 - the first run test case status line
149# arg2 - the second run test case status line
150# arg3 - the test case name
151cmp_two_status_lines()
152{
153	local rval=0
154
155	local memtotal00
156	local outstand00
157	local outstand01
158	local max00 max01 max02 max03 max04 max05
159	local out00 out01 out02 out03 out04 out05
160
161	local memtotal10
162	local outstand10
163	local outstand11
164	local max10 max11 max12 max13 max14 max15
165	local out10 out11 out12 out13 out14 out15
166
167	local totalmem0
168	local totaloutstand00
169	local totaltime0
170	local totaloutstand01
171
172	local totalmem1
173	local totaloutstand10
174	local totaltime1
175	local totaloutstand11
176
177	local exceptionsnum0
178	local exceptionsnum1
179
180	if [ "$3" == TOTAL ]; then
181		totaloutstand00=`echo "$1" | awk -F"|" '{print $3}'`
182		totalmem0=`echo "$1" | awk -F"|" '{print $4}'`
183		totaltime0=`echo "$1" | awk -F"|" '{print $5}'`
184		totaloutstand01=`echo "$1" | awk -F"|" '{print $6}'`
185
186		totaloutstand10=`echo "$2" | awk -F"|" '{print $3}'`
187		totalmem1=`echo "$2" | awk -F"|" '{print $4}'`
188		totaltime1=`echo "$2" | awk -F"|" '{print $5}'`
189		totaloutstand11=`echo "$2" | awk -F"|" '{print $6}'`
190
191		if [ x$totaloutstand01 == x ]; then
192			# For obsolete layout
193			return 0
194		fi
195
196		if [ x$totaloutstand11 == x ]; then
197			# For obsolete layout
198			return 0
199		fi
200
201		report_run_time_cmp "$totaltime0" "$totaltime1"
202		if [ $? -eq 2 ]; then
203			rval=1
204		fi
205
206		report_mem_stat_cmp "$3" total_memory $totalmem0 $totalmem1
207		if [ $? -eq 1 ]; then
208			rval=1
209		fi
210
211		report_mem_stat_cmp "$3" total_outstand0 $totaloutstand00 $totaloutstand10
212		if [ $? -eq 1 ]; then
213			rval=1
214		fi
215
216		report_mem_stat_cmp "$3" total_outstand1 $totaloutstand01 $totaloutstand11
217		if [ $? -eq 1 ]; then
218			rval=1
219		fi
220
221		return $rval
222	fi
223
224	memtotal00=`echo "$1" | awk -F"|" '{print $12}'`
225	if [ x$memtotal00 == x ]; then
226		# For obsolete layout
227		return 0
228	fi
229
230	memtotal10=`echo "$2" | awk -F"|" '{print $12}'`
231	if [ x$memtotal10 == x ]; then
232		# For obsolete layout
233		return 0
234	fi
235
236	if [ "$DO_COMPARE_OF_EXCEPTIONS" == "yes" ]; then
237		exceptionsnum0=`echo "$1" | awk -F"|" '{print $28}'`
238		exceptionsnum1=`echo "$2" | awk -F"|" '{print $28}'`
239		report_exceptions_stat_cmp "$3" exceptionsnum $exceptionsnum0 $exceptionsnum1
240		if [ $? -eq 1 ]; then
241			rval=1
242		fi
243	fi
244
245	if [ "$DO_COMPARE_OF_TEST_CASES" == "yes" ]; then
246
247	outstand00=`echo "$1" | awk -F"|" '{print $10}'`
248	max00=`echo "$1" | awk -F"|" '{print $13}'`
249	max01=`echo "$1" | awk -F"|" '{print $14}'`
250	max02=`echo "$1" | awk -F"|" '{print $15}'`
251	max03=`echo "$1" | awk -F"|" '{print $16}'`
252	max04=`echo "$1" | awk -F"|" '{print $17}'`
253	max05=`echo "$1" | awk -F"|" '{print $18}'`
254	out00=`echo "$1" | awk -F"|" '{print $19}'`
255	out01=`echo "$1" | awk -F"|" '{print $20}'`
256	out02=`echo "$1" | awk -F"|" '{print $21}'`
257	out03=`echo "$1" | awk -F"|" '{print $22}'`
258	out04=`echo "$1" | awk -F"|" '{print $23}'`
259	out05=`echo "$1" | awk -F"|" '{print $24}'`
260	outstand01=`echo "$1" | awk -F"|" '{print $25}'`
261
262	outstand10=`echo "$2" | awk -F"|" '{print $10}'`
263	max10=`echo "$2" | awk -F"|" '{print $13}'`
264	max11=`echo "$2" | awk -F"|" '{print $14}'`
265	max12=`echo "$2" | awk -F"|" '{print $15}'`
266	max13=`echo "$2" | awk -F"|" '{print $16}'`
267	max14=`echo "$2" | awk -F"|" '{print $17}'`
268	max15=`echo "$2" | awk -F"|" '{print $18}'`
269	out10=`echo "$2" | awk -F"|" '{print $19}'`
270	out11=`echo "$2" | awk -F"|" '{print $20}'`
271	out12=`echo "$2" | awk -F"|" '{print $21}'`
272	out13=`echo "$2" | awk -F"|" '{print $22}'`
273	out14=`echo "$2" | awk -F"|" '{print $23}'`
274	out15=`echo "$2" | awk -F"|" '{print $24}'`
275	outstand11=`echo "$2" | awk -F"|" '{print $25}'`
276
277
278	report_mem_stat_cmp "$3" memtotal $memtotal00 $memtotal10
279	if [ $? -eq 1 ]; then
280		rval=1
281	fi
282	report_mem_stat_cmp "$3" outstand0 $outstand00 $outstand10
283	if [ $? -eq 1 ]; then
284		rval=1
285	fi
286	report_mem_stat_cmp "$3" outstand1 $outstand01 $outstand11
287	if [ $? -eq 1 ]; then
288		rval=1
289	fi
290	report_mem_stat_cmp "$3" max0 $max00 $max10
291	if [ $? -eq 1 ]; then
292		rval=1
293	fi
294	report_mem_stat_cmp "$3" max1 $max01 $max11
295	if [ $? -eq 1 ]; then
296		rval=1
297	fi
298	report_mem_stat_cmp "$3" max2 $max02 $max12
299	if [ $? -eq 1 ]; then
300		rval=1
301	fi
302	report_mem_stat_cmp "$3" max3 $max03 $max13
303	if [ $? -eq 1 ]; then
304		rval=1
305	fi
306	report_mem_stat_cmp "$3" max4 $max04 $max14
307	if [ $? -eq 1 ]; then
308		rval=1
309	fi
310	report_mem_stat_cmp "$3" max5 $max05 $max15
311	if [ $? -eq 1 ]; then
312		rval=1
313	fi
314
315	report_mem_stat_cmp "$3" out0 $out00 $out10
316	if [ $? -eq 1 ]; then
317		rval=1
318	fi
319	report_mem_stat_cmp "$3" out1 $out01 $out11
320	if [ $? -eq 1 ]; then
321		rval=1
322	fi
323	report_mem_stat_cmp "$3" out2 $out02 $out12
324	if [ $? -eq 1 ]; then
325		rval=1
326	fi
327	report_mem_stat_cmp "$3" out3 $out03 $out13
328	if [ $? -eq 1 ]; then
329		rval=1
330	fi
331	report_mem_stat_cmp "$3" out4 $out04 $out14
332	if [ $? -eq 1 ]; then
333		rval=1
334	fi
335	report_mem_stat_cmp "$3" out5 $out05 $out15
336	if [ $? -eq 1 ]; then
337		rval=1
338	fi
339
340	fi
341
342
343	return $rval
344}
345
346# Unpack file into array,
347# element of array is a line of file.
348# arg1 - path name of file
349unpack_file_into_array()
350{
351	x=`cat "$1" | sort |\
352	while [ 1 ]
353	do
354		read line
355		if [ $? -ne 0 ] ; then
356			break
357		fi
358		echo "$line"
359	done`
360	echo "$x"
361}
362
363# Compare the memory consumption statistics of two runs
364# arg1 - the first run test cases status lines file
365# arg2 - the second run test cases status lines file
366cmp_memory_of_two_runs()
367{
368	local rval=0 index=0 index_of_last_found=0 length=0
369
370	ARRAY=
371
372	file1=`unpack_file_into_array "$1"`
373	file2=`unpack_file_into_array "$2"`
374
375	# Put second input into array
376
377	for line in $file2
378	do
379		if [ x"$line" == x ]; then
380			continue
381		fi
382		ARRAY[$index]="$line"
383		index=$[ $index + 1 ]
384	done
385
386	length=$index
387
388	if [ "$length" -eq 0 ]; then
389		echo "Empty file <$2>"
390		return 0
391	fi
392
393	# Run through the first input and check equivalent line from the
394	# array of second input, do one pass only through both inputs (they
395	# are sorted identically).
396
397	for i in $file1
398	do
399		tcasename0=`echo "$i" | awk -F"|" '{print $2}'`
400
401		if [ x"$tcasename0" == x ]; then
402			continue
403		fi
404
405		index=$index_of_last_found
406
407		while [ 1 ]
408		do
409			if [ "$index" -ge "$length" ]; then
410				break
411			fi
412
413			line="${ARRAY[$index]}"
414			index=$[ $index + 1 ]
415			tcasename1=`echo "$line" | awk -F"|" '{print $2}'`
416			if [ "$tcasename1" == "$tcasename0" ]; then
417				index_of_last_found=$index
418				cmp_two_status_lines "$i" "$line" "$tcasename0"
419				if [ $? -ne 0 ]; then
420					rval=1
421				fi
422				if [ "$DO_COMPARE_TOTAL_ONLY" == yes -a "$tcasename0" == TOTAL ]; then
423					echo "!!!!!!!, REDUCED MODE OF STAT CMP: only TOTAL statistics was compared!"
424					return $rval
425				fi
426
427				break
428			fi
429		done
430	done
431
432	return $rval
433}
434
435# Determine the test status lines files
436# corresponding to the first and the second
437# runs and initiate comparing of them.
438# arg1 - second multi-result directory
439# arg2 - second bitmap of mode
440# arg3 - first multi-result directory
441# arg4 - first bitmap of mode
442do_compare_two_runs()
443{
444	local rval=100 ret mark path0 path1 exists0 exists1 path2 path3
445	local modepart0 modename0
446	local modepart1 modename1
447
448	modepart0=`get_mode_string $2 0`
449	modename0=`get_mode_string $2 1`
450	modepart1=`get_mode_string $4 0`
451	modename1=`get_mode_string $4 1`
452
453	path0="$1/$modepart0/__STATUS_OF_TESTS"
454	path1="$3/$modepart1/__STATUS_OF_TESTS"
455
456	echo ""
457	mark="================"
458	echo "$mark Comparing results of <$modename0> and <$modename1>:"
459
460	exists0=0
461	exists1=0
462	ret=1
463
464	if [ -f "$path0" ]; then
465		exists0=1
466	fi
467	if [ -f "$path1" ]; then
468		exists1=1
469	fi
470
471	if [ $exists0 == 1 -a $exists1 == 1 ]; then
472		cmp_two_runs "$path0" "$path1"
473		ret=$?
474
475		if [ "$DO_MEMSTAT" == "yes" ]; then
476			path2="$3/$modepart1/__STATUS_OF_TEST_CASES"
477			path3="$1/$modepart0/__STATUS_OF_TEST_CASES"
478			cmp_memory_of_two_runs "$path2" "$path3"
479		fi
480	fi
481
482	if [ $exists0 == 1 -a $exists1 == 1 ]; then
483		if [ $ret == 0 ]; then
484			rval=$CMP_CMP_OP
485			echo "$mark `cmp_result_opcode_to_str $rval`"
486		else
487			rval=$CMP_MISCMP_OP
488			echo "$mark `cmp_result_opcode_to_str $rval`"
489			UTILSTATUS=1
490		fi
491	elif [ $exists0 == 1 ]; then
492		rval=$CMP_NO_SECOND_OP
493		echo "$mark `cmp_result_opcode_to_str $rval`"
494	elif [ $exists1 == 1 ]; then
495		rval=$CMP_NO_FIRST_OP
496		echo "$mark `cmp_result_opcode_to_str $rval`"
497	else
498		rval=$CMP_NO_BOTH_OP
499		echo "$mark `cmp_result_opcode_to_str $rval`"
500	fi
501
502	echo ""
503	echo "Failures in new version: `grep -c FAIL $path1` -- $path1"
504	echo "Failures in old version: `grep -c FAIL $path0` -- $path0"
505	echo ""
506	return $rval
507}
508
509# Convert opcode of result of comparing to string
510# arg1 - opcode of result of comparing
511cmp_result_opcode_to_str()
512{
513	local msg
514
515	case $1 in
516		$CMP_CMP_OP) msg=compared;;
517		$CMP_MISCMP_OP) msg=miscompared;;
518		$CMP_NO_SECOND_OP) msg="- (the SUMMARY file of second run doesn't exist)";;
519		$CMP_NO_FIRST_OP) msg="- (the SUMMARY file of first run doesn't exist)";;
520		$CMP_NO_BOTH_OP) msg="- (the SUMMARY files of both runs don't exist)";;
521		*) msg="???"
522	esac
523
524	echo "$msg"
525}
526
527# ############################## MAIN ###############################
528
529# Initialize the common variables
530
531# Constants
532
533# Opcodes of result of comparing
534CMP_CMP_OP=0
535CMP_MISCMP_OP=1
536CMP_NO_SECOND_OP=2
537CMP_NO_FIRST_OP=3
538CMP_NO_BOTH_OP=4
539
540