Skip to content

Commit b986c7d

Browse files
author
Stefan Reinauer
committed
add testsuite scripts
git-svn-id: svn://coreboot.org/openbios/fcode-utils@92 f158a5a8-5612-0410-a976-696ce0be7e32
1 parent acf5213 commit b986c7d

2 files changed

Lines changed: 347 additions & 0 deletions

File tree

‎testsuite/AutoCompare‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
#! /bin/csh -f
2+
#
3+
# Part of automation of the testing process for the Tokenizer/De-Tokenizer
4+
# Automate Comparison of results of Auto-Execution of the various test-cases
5+
6+
# Updated Fri, 16 Jun 2006 at 12:12 PDT by David L. Paktor
7+
8+
# To Do:
9+
# Conditional comparison of .DeTok files:
10+
# Detect difference in .fc file, but do not display
11+
# Display differences in .Log file
12+
# If difference detected in .fc file, display mssg
13+
# Otherwise, detect difference in .DeTok file and display mssg
14+
# If either diff detected in .fc or in .DeTok file, show .DeTok diffs
15+
# Display differences in .RomHdr
16+
#
17+
# Compare the source (.fth) files (for regression testing...)
18+
19+
# Usage:
20+
# Run this from the parent-directory of the various Test categories.
21+
# This script uses the same TestArgs files that were used by
22+
# (and described in) the AutoExec script.
23+
#
24+
# This script needs a shell environment variable called CygTestLogs
25+
# whose value is the path to the parent-directory that contains
26+
# the results of Auto-Execution of the various test-cases under
27+
# Cygwin, which is the "Gold Standard" to which the other
28+
# platforms' results will be compared.
29+
#
30+
# This script accepts an optional environment variable called LogFilesDiff
31+
# which may be set to change the number of lines by which two .Log
32+
# files are expected to differ (due to different compilation-signatures)
33+
# If this is not supplied in the environment, a default will be provided.
34+
#
35+
# This script compares only one platform's results at a time;
36+
# the parent-directory that contains the Auto-Execution results
37+
# for the platform to be compared should be the current directory.
38+
#
39+
# This script displays both operational errors and detected discrepancies
40+
# in Standard Output and also collects the same in a log file whose
41+
# name is AutoComp.Log.<time-and-date-stamp>
42+
#
43+
# A secondary output file, called AutoComp.TKdiffs.<time-and-date-stamp>
44+
# is produced; it can be "source"d to cause a sequential tkdiff of
45+
# the detected discrepancies
46+
47+
48+
# Set the number of lines by which two .Log files are expected to differ.
49+
# (this is due to different compilation-signature lines)
50+
set LogFilesDiffLines = 6
51+
# Over-ride with optional shell env't variable called LogFilesDiff
52+
if ( $?LogFilesDiff ) then
53+
set notallnumbers = `echo $LogFilesDiff | tr '0-9' ' '`
54+
if ( "$notallnumbers" == "" ) then
55+
set LogFilesDiffLines = $LogFilesDiff
56+
endif
57+
endif
58+
59+
# Preliminary error checking:
60+
if ( ! $?CygTestLogs ) then
61+
echo 'Please define a shell environment variable called CygTestLogs'
62+
echo ' whose value is the path to the Cygwin test-results directory.'
63+
exit 1
64+
endif
65+
66+
set TArgFiles = `find . -name TestArgs -exec expr {} : '\./\(.*\)' \;`
67+
if ( $#TArgFiles == 0 ) then
68+
echo 'No TestArgs files found in subdirectories.'
69+
echo 'Starting in wrong directory?'
70+
exit 2
71+
endif
72+
73+
set TArgReslts = `find $CygTestLogs -name TestArgs -exec expr {} : $CygTestLogs/'\(.*\)' \;`
74+
if ( $#TArgReslts != $#TArgFiles ) then
75+
echo 'Number of TestArgs files in CygTestLogs ('$CygTestLogs') directory'
76+
echo ' does not match those found under current directory.'
77+
echo 'Please correct discrepancy and try again.'
78+
exit 4
79+
endif
80+
81+
if ( "$TArgReslts" != "$TArgFiles" ) then
82+
echo 'List of TestArgs files in CygTestLogs ('$CygTestLogs') directory'
83+
echo ' does not match those found under current directory.'
84+
echo 'Please correct discrepancy and try again.'
85+
exit 8
86+
endif
87+
88+
# Let's save mismatch identifiers:
89+
set datemark = `date '+%y%m%d.%H%M%S'`
90+
set ErrResltFil = AutoComp.Log.$datemark
91+
set ScriptResltFil = AutoComp.TKdiffs.$datemark
92+
93+
# Let's show a progress-downcount
94+
# Find the count of actual tests; leave the max number on display:
95+
set cnt = `egrep '^[^#].*' $TArgFiles | wc -l`
96+
echo ''
97+
echo $cnt
98+
99+
foreach TArgFil ( $TArgFiles )
100+
set dir = $TArgFil:h
101+
102+
set lindxmax = `cat $TArgFil | wc -l`
103+
set lindx = 0
104+
while ( $lindx < $lindxmax )
105+
@ lindx++
106+
set targline = `sed -n ${lindx}p $TArgFil`
107+
if ( $#targline == 0 ) continue
108+
if ( "$targline[1]" == "#" ) continue
109+
set ttarg = $dir/$targline[1]
110+
# One more error-check:
111+
if ( ! -f ${ttarg}.fth ) then
112+
echo File Not found: ${ttarg}.fth
113+
echo Please update ${TArgFil} file, line $lindx
114+
continue
115+
endif
116+
117+
set label = `echo $targline | awk -F , '{print $2}'`
118+
if ( "$label" != "" ) set label = .$label
119+
120+
# Now begins the real fun...
121+
# Show a running down-count
122+
echo -n X${cnt}' ' | tr X \\015
123+
@ cnt--
124+
125+
# Check the extensions that are text-type files
126+
foreach txext ( Log RomHdr fl fl.missing )
127+
# Is the file in both?
128+
if ( -f $CygTestLogs/${ttarg}${label}.${txext} && ! -f ${ttarg}${label}.${txext} ) then
129+
echo Missing ${ttarg}${label}.${txext} | tee -a $ErrResltFil
130+
else
131+
if ( -f ${ttarg}${label}.${txext} ) then
132+
set lim = 0
133+
set fxext = `echo $txext | tr '.' '_'`
134+
if ( `eval echo '$?'${fxext}FilesDiffLines` ) then
135+
set lim = `eval echo '$'${fxext}FilesDiffLines`
136+
endif
137+
set logdiff = `diff {,$CygTestLogs/}${ttarg}${label}.${txext} | wc -l`
138+
if ( $logdiff > $lim ) then
139+
echo ${ttarg}${label}.${txext} files differ. | tee -a $ErrResltFil
140+
diff {$CygTestLogs/,}${ttarg}${label}.${txext} | tee -a $ErrResltFil
141+
echo 'tkdiff {$CygTestLogs/,}'${ttarg}${label}.${txext} >> $ScriptResltFil
142+
endif
143+
endif
144+
endif
145+
end
146+
147+
# Now, check the FCode file.
148+
# If there's an FCode file in the reference directory ($CygTestLogs)
149+
# there should be one in the directory under test.
150+
if ( -f $CygTestLogs/${ttarg}${label}.fc && ! -f ${ttarg}${label}.fc ) then
151+
echo Missing ${ttarg}${label}.fc | tee -a $ErrResltFil
152+
else
153+
# Is there an FCode file in the directory under test?
154+
unset ShowDetokDiffs
155+
if ( -f ${ttarg}${label}.fc ) then
156+
set bindiff = `cmp {,$CygTestLogs/}${ttarg}${label}.fc`
157+
if ( "$bindiff" != "" ) then
158+
echo "$bindiff" | tee -a $ErrResltFil
159+
# If the binaries are different, show the difference in the DeTok form
160+
# If the DeTok form is missing, it's an even stranger error!
161+
if ( ! -f $CygTestLogs/${ttarg}${label}.DeTok || ! -f ${ttarg}${label}.DeTok ) then
162+
echo Missing DeTok file for ${ttarg}${label} | tee -a $ErrResltFil
163+
else
164+
set ShowDetokDiffs
165+
endif
166+
else
167+
# Even if the binaries are not different,
168+
# check for changes in in the DeTok form
169+
set detokdiff = `diff {,$CygTestLogs/}${ttarg}${label}.DeTok | wc -l`
170+
if ( $detokdiff > 0 ) set ShowDetokDiffs
171+
endif
172+
endif
173+
if ( $?ShowDetokDiffs ) then
174+
echo ${ttarg}${label}.DeTok files differ. | tee -a $ErrResltFil
175+
diff {$CygTestLogs/,}${ttarg}${label}.DeTok | tee -a $ErrResltFil
176+
echo 'tkdiff {$CygTestLogs/,}'${ttarg}${label}.DeTok >> $ScriptResltFil
177+
endif
178+
endif
179+
180+
end
181+
end
182+
183+

‎testsuite/AutoExec‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
#! /bin/csh -f
2+
#
3+
# Part of automation of the testing process for the Tokenizer/De-Tokenizer
4+
# Automate Execution of Tokenizing and De-Tokenizing the various test-cases
5+
6+
# Updated Thu, 09 Feb 2006 at 13:10 PST by David L. Paktor
7+
8+
# Run this from the parent-directory of the various Test categories.
9+
# Sym-links to the binaries should be here.
10+
11+
# Each Test category directory must have a TestArgs file,
12+
# formatted as follows:
13+
# Lines starting with # (Pound-sign Space) are comments,
14+
# and are ignored
15+
# Blank lines are also allowed, and are also ignored
16+
# Valid lines have four comma-separated fields:
17+
# Test-file base-name
18+
# Result-file label
19+
# Extra command-line switches
20+
# A script-command, to be run after, for specific
21+
# verification of this Test. Arguments can also
22+
# be included in this field, as long as they have
23+
# no commas.
24+
# An unspecified intermediate field must be represented as
25+
# an empty field separated by commas from the fields
26+
# preceding and following.
27+
# Lines that have no label, no switches, and no script
28+
# do not need any commas.
29+
# The -v (verbose) switch will always be used and does not need
30+
# to be explicitly specified. However, if the "script-command"
31+
# field starts with +V (case-sensitive), then the -v (verbose)
32+
# switch will not be used.
33+
#
34+
# The Test-file base-name field may have multiple base-names,
35+
# to run a multiple-file batch command.
36+
# If a multiple-file batch is run:
37+
# The Result-file label will apply only to the Log file.
38+
39+
# To Do:
40+
# Contrive a way to run the second through last input files as
41+
# separate individual jobs (with same command-line switches)
42+
# This needs to be co-ordinated with the Auto-Compare script:
43+
# It will need to compare the .FC files from the batch with
44+
# those from the individual runs.
45+
46+
47+
# Initial error-checking:
48+
if ( ( ! -x ./toke ) || ( ! -x ./detok ) ) then
49+
echo 'Starting in wrong directory. Executable toke and detok are not here.'
50+
exit 1
51+
endif
52+
53+
set TArgFiles = `find . -name TestArgs -exec expr {} : '\./\(.*\)' \;`
54+
if ( $#TArgFiles == 0 ) then
55+
echo 'No TestArgs files found in subdirectories.'
56+
echo 'Starting in wrong directory?'
57+
exit 1
58+
endif
59+
60+
set parent = `pwd`
61+
set tokex = `pwd`/toke
62+
set detokex = `pwd`/detok
63+
set romhdrex = `pwd`/romheaders
64+
65+
set chirren = `echo $TArgFiles | tr ' ' \\012 | sed -e 'sX/[^/]*$XXg'`
66+
67+
# The "verbose" flag is now a variable.
68+
set vflg = '-v'
69+
70+
# Let's show a progress-downcount and time, stamps and elapsed.
71+
# Find the count; leave the max number on display:
72+
set cnt = `egrep '^[^#].*' $TArgFiles | wc -l`
73+
echo ''
74+
set starttime = `date +'%T'`
75+
echo $starttime
76+
echo $cnt
77+
78+
foreach dir ( $chirren )
79+
cd $dir
80+
set lindxmax = `cat TestArgs | wc -l`
81+
set lindx = 0
82+
while ( $lindx < $lindxmax )
83+
@ lindx++
84+
set targline = `sed -n ${lindx}p TestArgs`
85+
if ( $#targline == 0 ) continue
86+
if ( "$targline[1]" == "#" ) continue
87+
set ttargs = `echo $targline | awk -F , '{print $1}'`
88+
# One more error-check:
89+
set tfths = ''
90+
set fcfils = ''
91+
unset FMissing
92+
foreach tfth ( $ttargs )
93+
if ( ! -f ${tfth}.fth ) then
94+
echo File Not found: ${dir}/${tfth}.fth
95+
echo Please update ${dir}/TestArgs file, line $lindx
96+
set FMissing
97+
endif
98+
set tfths = ( $tfths ${tfth}.fth )
99+
set fcfils = ( $fcfils ${tfth}.fc )
100+
end
101+
if ( $?FMissing ) continue
102+
103+
set ttarg = $ttargs[1]
104+
105+
set outfile = ''
106+
set label = `echo $targline | awk -F , '{print $2}'`
107+
if ( "$label" != "" ) then
108+
set label = .$label
109+
if ( $#tfths == 1 ) then
110+
set fcfils = ${ttarg}${label}.fc
111+
set outfile = ( -o $fcfils )
112+
endif
113+
endif
114+
# We don't set the switches as a shell-variable because we might
115+
# need to preserve quoted groupings that have embedded spaces.
116+
117+
# Collect the script-command.
118+
# Awkward place to expect a "+V" option, but it'll have to serve....
119+
set scriptcmd = `echo $targline | awk -F , '{print $4}'`
120+
if ( $#scriptcmd > 0 ) then
121+
if ( "$scriptcmd[1]" == "+V" ) then
122+
set vflg = ""
123+
if ( $#scriptcmd == 1 ) then
124+
set scriptcmd = ""
125+
else
126+
set scriptcmd = ( $scriptcmd[2-] )
127+
endif
128+
endif
129+
endif
130+
131+
echo -n X${cnt}' ' | tr X \\015
132+
@ cnt--
133+
134+
# Now do it.
135+
set doromhdr = 0
136+
echo toke $vflg `sed -n ${lindx}p TestArgs | awk -F , '{print $3}'` $outfile ${tfths} >& ${ttarg}${label}.Log
137+
echo '' >>& ${ttarg}${label}.Log
138+
eval $tokex $vflg `sed -n ${lindx}p TestArgs | awk -F , '{print $3}'` $outfile ${tfths} >>& ${ttarg}${label}.Log
139+
set vflg = '-v'
140+
141+
foreach fcfil ( $fcfils )
142+
if ( -f ${fcfil} ) then
143+
$detokex -v -o ${fcfil} > ${fcfil:r}.DeTok
144+
set doromhdr = `grep 'PCI Header identified' ${fcfil:r}.DeTok | wc -l`
145+
if ( $doromhdr ) then
146+
$romhdrex ${fcfil} > ${fcfil:r}.RomHdr
147+
endif
148+
endif
149+
end
150+
151+
152+
if ( "$scriptcmd" != "" ) then
153+
eval $scriptcmd
154+
endif
155+
156+
end
157+
cd $parent
158+
end
159+
# Clear the display-line
160+
echo -n X' 'X | tr X \\015
161+
# Show time, stamps and elapsed.
162+
date +'%T'
163+
echo $starttime ' (Started)'
164+

0 commit comments

Comments
 (0)