mirror of
https://github.com/wesnoth/wesnoth
synced 2025-04-15 18:27:08 +00:00
306 lines
7.3 KiB
Bash
Executable File
306 lines
7.3 KiB
Bash
Executable File
#!/bin/bash
|
|
#This script runs a sequence of wml unit test scenarios.
|
|
#Use -h to get help with usage.
|
|
|
|
usage()
|
|
{
|
|
echo "Usage:" $0 "[OPTIONS] [EXTRA-ARGS]"
|
|
echo "Executes a series of wml unit test scenarios found in a file."
|
|
echo
|
|
echo -e "Options:"
|
|
echo -e "\t-h\tShows this help."
|
|
echo -e "\t-v\tVerbose mode."
|
|
echo -e "\t-w\tVery verbose mode. (Debug script.)"
|
|
echo -e "\t-u\tUse unix timeout instead of wesnoth --timeout argument."
|
|
echo -e "\t \tUnix timeout will send a TERM signal, followed by KILL."
|
|
echo -e "\t-a arg\tAdditional arguments to go to wesnoth."
|
|
echo -e "\t-t arg\tNew timer value to use, instead of 10s as default."
|
|
echo -e "\t-s\tDisable strict mode. By default, we run wesnoth with option"
|
|
echo -e "\t \t'--log-strict=warning' to ensure errors result in a failed test."
|
|
echo -e "\t-d\tRun wesnoth-debug binary instead of wesnoth."
|
|
echo -e "\t-p arg\tPath to wesnoth binary. By default assume it is with this script."
|
|
echo -e "\t-l arg\tLoads list of tests from the given file."
|
|
echo -e "\t \tBy default, the file is wml_test_schedule."
|
|
echo
|
|
echo "Each line in the list of tests should be formatted:"
|
|
echo -e "\n\t<expected return code> <name of unit test scenario>\n"
|
|
echo "Lines beginning # are treated as comments."
|
|
echo "Expected return codes:"
|
|
for i in `seq 0 4`;
|
|
do
|
|
get_code_string $i
|
|
echo -e "\t" $i "-" $CodeString
|
|
done
|
|
echo
|
|
echo "Extra arguments besides these options are saved and passed on to wesnoth."
|
|
}
|
|
|
|
get_code_string()
|
|
{
|
|
case ${1} in
|
|
0)
|
|
CodeString="PASS"
|
|
;;
|
|
1)
|
|
CodeString="FAIL"
|
|
;;
|
|
2)
|
|
CodeString="FAIL (TIMEOUT)"
|
|
;;
|
|
3)
|
|
CodeString="FAIL (INVALID REPLAY)"
|
|
;;
|
|
4)
|
|
CodeString="FAIL (ERRORED REPLAY)"
|
|
;;
|
|
134)
|
|
CodeString="FAIL (ASSERTION FAILURE ? ? ?)"
|
|
;;
|
|
139)
|
|
CodeString="FAIL (SEGFAULT ? ? ?)"
|
|
;;
|
|
*)
|
|
CodeString="FAIL (? ? ?)"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
check_errs()
|
|
{
|
|
# Argument 1 is the name of the test.
|
|
# Argument 2 is the wesnoth error code for the test.
|
|
# Argument 3 is the expected error code.
|
|
if [ "${2}" -eq 134 -a "${3}" -eq 2 -a "$UnixTimeout" -eq 0 ]; then
|
|
if [ "$Verbose" -ge 2 ]; then
|
|
echo "Caught \'terminate called without an active exception\' return code 134"
|
|
echo "This means wesnoth tried to kill the thread but SDL threw an error..."
|
|
echo "(This happens occasionally when running the empty_test with a timeout.)"
|
|
echo "Since we expected timeout, the test passes."
|
|
echo ""
|
|
echo "*However*, review the logs, because it may also mean an assertion failure..."
|
|
fi
|
|
return 0
|
|
elif [ "${2}" -eq 124 -a "${3}" -eq 2 -a "$UnixTimeout" -eq 1 ]; then
|
|
if [ "$Verbose" -ge 2 ]; then
|
|
echo "Caught return code 124 from timeout"
|
|
echo "This signal means that the unix timeout utility killed wesnoth with TERM."
|
|
echo "Since we expected timeout, the test passes."
|
|
fi
|
|
return 0
|
|
elif [ "${2}" -eq 137 -a "${3}" -eq 2 -a "$UnixTimeout" -eq 1 ]; then
|
|
if [ "$Verbose" -ge 2 ]; then
|
|
echo "Caught return code 137 from timeout"
|
|
echo "This signal means that the unix timeout utility killed wesnoth with KILL."
|
|
echo "Since we expected timeout, the test passes."
|
|
fi
|
|
return 0
|
|
elif [ "${2}" -ne "${3}" ]; then
|
|
echo "${1}" ":"
|
|
get_code_string ${2}
|
|
printf '%-55s %3d - %s\n' " Observed result :" "${2}" "$CodeString"
|
|
get_code_string ${3}
|
|
printf '%-55s %3d - %s\n' " Expected result :" "${3}" "$CodeString"
|
|
if [ "$Verbose" -ge 2 -a -f "error.log" ]; then
|
|
echo ""
|
|
echo "Found error.log:"
|
|
cat error.log
|
|
fi
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
handle_error_log()
|
|
{
|
|
if [ -f "error.log" ]; then
|
|
if [ "${1}" -ne 0 ]; then
|
|
if [ -f "errors.log" ]; then
|
|
echo -e "\n--- next unit test ---\n" >> errors.log
|
|
cat error.log >> errors.log
|
|
else
|
|
cp error.log errors.log
|
|
fi
|
|
fi
|
|
|
|
rm error.log
|
|
fi
|
|
}
|
|
|
|
run_test()
|
|
{
|
|
# Argument 1 is the expected error code
|
|
# Argument 2 is the name of the test scenario
|
|
preopts=""
|
|
binary="$BinPath"
|
|
opts="-u $2 "
|
|
|
|
if [ "$DebugMode" -eq 1 ]; then
|
|
binary+="wesnoth-debug "
|
|
else
|
|
binary+="wesnoth "
|
|
fi
|
|
|
|
timer=$basetimer
|
|
# Use validcache on tests that aren't the first test.
|
|
if [ "$FirstTest" -eq 1 ]; then
|
|
((timer *= 2))
|
|
else
|
|
opts+="--validcache "
|
|
fi
|
|
# Add a timeout
|
|
if [ "$UnixTimeout" -eq 1 ]; then
|
|
timer1=$((timer+1))
|
|
preopts+="timeout --kill-after=$timer1 $timer "
|
|
else
|
|
timerms=$((timer*1000))
|
|
opts+="--timeout $timerms "
|
|
fi
|
|
# If running strict, then set strict level to "warning"
|
|
if [ "$StrictMode" -eq 1 ]; then
|
|
opts+="--log-strict=warning "
|
|
fi
|
|
# Assemble command
|
|
command="$preopts"
|
|
command+="$binary"
|
|
command+="$opts"
|
|
command+="$extra_opts"
|
|
if [ "$Verbose" -eq 1 ]; then
|
|
echo "$command"
|
|
elif [ "$Verbose" -eq 2 ]; then
|
|
echo "$command" "2> error.log"
|
|
fi
|
|
$command 2> error.log
|
|
if check_errs $2 $? $1; then
|
|
FirstTest=0 #Only start using validcache flag when at least one test has passed without error
|
|
handle_error_log 0
|
|
return 0
|
|
else
|
|
handle_error_log 1
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
|
|
### Main Script Starts Here ###
|
|
|
|
Verbose=0
|
|
UnixTimeout=0
|
|
LoadFile="wml_test_schedule"
|
|
BinPath="./"
|
|
StrictMode=1
|
|
DebugMode=0
|
|
extra_opts=""
|
|
basetimer=10
|
|
|
|
while getopts ":hvwusdp:l:a:t:" Option
|
|
do
|
|
case $Option in
|
|
h )
|
|
usage
|
|
exit 0;
|
|
;;
|
|
v )
|
|
if [ "$Verbose" -lt 1 ]; then
|
|
Verbose=1
|
|
fi
|
|
;;
|
|
w )
|
|
if [ "$Verbose" -lt 2 ]; then
|
|
Verbose=2
|
|
fi
|
|
;;
|
|
u )
|
|
UnixTimeout=1
|
|
;;
|
|
s )
|
|
StrictMode=0
|
|
;;
|
|
d )
|
|
DebugMode=1
|
|
;;
|
|
p )
|
|
BinPath="$OPTARG"
|
|
;;
|
|
l )
|
|
LoadFile="$OPTARG"
|
|
;;
|
|
a )
|
|
extra_opts+="$OPTARG"
|
|
;;
|
|
t )
|
|
echo "Replacing default timer of 10s with " "$OPTARG"
|
|
basetimer="$OPTARG"
|
|
;;
|
|
esac
|
|
done
|
|
shift $(($OPTIND - 1))
|
|
|
|
extra_opts+="$*"
|
|
|
|
if [ "$Verbose" -ge 2 ]; then
|
|
if [ "${#extra_opts}" -ge 0 ]; then
|
|
echo "Found additional arguments to wesnoth: " "$extra_opts"
|
|
fi
|
|
fi
|
|
|
|
echo "Getting tests from" "$LoadFile" "..."
|
|
|
|
old_IFS=$IFS
|
|
IFS=$'\n'
|
|
schedule=($(cat $LoadFile)) # array
|
|
IFS=$old_IFS
|
|
|
|
NumTests=0
|
|
NumComments=0
|
|
|
|
for line in "${schedule[@]}"
|
|
do
|
|
if [[ "$line" =~ \#.* ]]; then
|
|
NumComments=$((NumComments+1))
|
|
else
|
|
NumTests=$((NumTests+1))
|
|
fi
|
|
done
|
|
|
|
echo "Running" $NumTests "test scenarios."
|
|
|
|
if [ -f "errors.log" ]; then
|
|
rm errors.log
|
|
fi
|
|
|
|
AllPassed=1
|
|
FirstTest=1
|
|
TotalPassed=0
|
|
|
|
for line in "${schedule[@]}"
|
|
do
|
|
if [[ "$line" =~ \#.* ]]; then
|
|
if [ "$Verbose" -ge 2 ]; then
|
|
echo "comment:" $line
|
|
fi
|
|
else
|
|
if run_test $line; then #note: don't put run_test inside a pipe implicitly by using ! or something, this will cause the FirstTest variable not to work properly
|
|
if [ "$Verbose" -ge 2 ]; then
|
|
echo "good"
|
|
fi
|
|
TotalPassed=$((TotalPassed+1))
|
|
else
|
|
AllPassed=0
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$AllPassed" -eq 0 ]; then
|
|
if [ "$StrictMode" -eq 1 ]; then
|
|
echo "$TotalPassed" "out of" "$NumTests" "tests were correct."
|
|
else
|
|
echo "$TotalPassed" "out of" "$NumTests" "tests were correct. (However, some tests may expect to be running in strict mode, and not fail as expected otherwise.)"
|
|
fi
|
|
echo "Not all tests gave the correct result."
|
|
echo "Check errors.log for error reports."
|
|
exit 1
|
|
else
|
|
echo "All tests gave the correct result."
|
|
exit 0
|
|
fi
|