From d4ee67e90f5fb1f65273043176c33b9cfdd6681a Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Wed, 8 Apr 2020 01:09:57 +0800 Subject: [PATCH 1/8] keep track of first failure of each test --- src/test/framework/main.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/framework/main.js b/src/test/framework/main.js index c2cd6cc..54cac87 100644 --- a/src/test/framework/main.js +++ b/src/test/framework/main.js @@ -12,6 +12,7 @@ let start_time = 0; let elapsed_time = 0; let current_test_status = TEST_PASS; +let current_test_failure = null; /** * Checks if the given condition evaluates to true @@ -29,6 +30,10 @@ function assert(condition, message) { num_assert_fail = num_assert_fail + 1; current_test_status = TEST_FAIL; display(res); + if (current_test_failure === null) { + // only keep track of the first failure of any test + current_test_failure = res; + } else {} } else { num_assert_pass = num_assert_pass + 1; } @@ -51,7 +56,7 @@ function report_error(message) { * @param result */ function assert_equal(expected, result) { - const error_message = "assert_equal failed: " + stringify(result) + " (result) not equal to " + stringify(expected) + " (expected)"; + const error_message = stringify(result) + " (result) not equal to " + stringify(expected) + " (expected)"; assert(equal(expected, result), error_message); } @@ -137,7 +142,8 @@ function _test_result(test_name) { display(test_name + " PASSED"); num_pass = num_pass + 1; } else { - display(test_name + " FAILED"); + display(test_name + " FAILED: " + current_test_failure); + current_test_failure = null; } current_test_status = TEST_PASS; // reset test status From 765da5b6fae0582b182d9750d5ff445f6e9fc244 Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Wed, 8 Apr 2020 01:15:18 +0800 Subject: [PATCH 2/8] add test framework support --- scripts/test.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/scripts/test.sh b/scripts/test.sh index 8d0ec78..907d72f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,8 +1,10 @@ #! /usr/bin/env bash JS_SLANG="node node_modules/js-slang/dist/repl/repl.js" +TEST_FRAMEWORK="src/test/framework/main.js" SOURCEFILES=src/*/*.js +TEST_FRAMEWORK_FILE="source-test.js" red=`tput setaf 1` green=`tput setaf 2` @@ -28,6 +30,30 @@ $DIFF" } +# $1 is the Source test file which uses the test framework +test_source_framework() { + # run concatenation of test framework and test file + RESULTS=$($JS_SLANG -e --chapter=4 "$(cat $TEST_FRAMEWORK $1)") + + # retrieve names for tests that passed + while read test_name + do + passed=$(($passed+1)) + echo "${green}PASS $1 $test_name" + done < <(echo ${RESULTS} | grep -o '\w* PASSED' | awk -F 'PASSED' '{print $1}') + + # retrieve names and error messages for tests that failed + while read test_info + do + failed=$(($failed+1)) + echo $test_info | awk -F 'FAILED:' '{ print $1 ":" $2 }' | awk -F '"' '{ print $1 $2 }' | + while read test_name test_error + do echo "${red}FAIL $1 $test_name $test_error"; + done + done < <(echo ${RESULTS} | grep -o '\w* FAILED:[^"]*') + +} + main() { for s in ${SOURCEFILES} do @@ -40,6 +66,13 @@ main() { do test_source $s $i done + + # check if test framework is being used + TEST_PATH="$DIR/__tests__/$TEST_FRAMEWORK_FILE" + if [ -e $TEST_PATH ] + then + test_source_framework $TEST_PATH + fi fi done } From c70983b881ceec4df5dc9bd250a05c294931537b Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Wed, 8 Apr 2020 01:29:38 +0800 Subject: [PATCH 3/8] use source file --- scripts/test.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index dedf760..dac7813 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -30,16 +30,17 @@ $DIFF" } -# $1 is the Source test file which uses the test framework +# $1 is the source file to be tested +# $2 is the Source test file which uses the test framework test_source_framework() { # run concatenation of test framework and test file - RESULTS=$($JS_SLANG -e --chapter=4 "$(cat $TEST_FRAMEWORK $1)") + RESULTS=$($JS_SLANG -e --chapter=4 "$(cat $TEST_FRAMEWORK $1 $2)") # retrieve names for tests that passed while read test_name do passed=$(($passed+1)) - echo "${green}PASS $1 $test_name" + echo "${green}PASS $2 $test_name" done < <(echo ${RESULTS} | grep -o '\w* PASSED' | awk -F 'PASSED' '{print $1}') # retrieve names and error messages for tests that failed @@ -48,7 +49,7 @@ test_source_framework() { failed=$(($failed+1)) echo $test_info | awk -F 'FAILED:' '{ print $1 ":" $2 }' | awk -F '"' '{ print $1 $2 }' | while read test_name test_error - do echo "${red}FAIL $1 $test_name $test_error"; + do echo "${red}FAIL $2 $test_name $test_error"; done done < <(echo ${RESULTS} | grep -o '\w* FAILED:[^"]*') @@ -71,7 +72,7 @@ main() { TEST_PATH="$DIR/__tests__/$TEST_FRAMEWORK_FILE" if [ -e $TEST_PATH ] then - test_source_framework $TEST_PATH + test_source_framework $s $TEST_PATH fi fi From c7a4e650853ae6a575b5ed9b681c97cdc5cbe9db Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Wed, 8 Apr 2020 01:38:05 +0800 Subject: [PATCH 4/8] modify variable names --- scripts/test.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index dac7813..35bf455 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,10 +1,10 @@ #! /usr/bin/env bash JS_SLANG="node node_modules/js-slang/dist/repl/repl.js" -TEST_FRAMEWORK="src/test/framework/main.js" SOURCEFILES=src/*/*.js -TEST_FRAMEWORK_FILE="source-test.js" +SOURCE_TEST="src/test/framework/main.js" +SOURCE_TEST_FILENAME="source-test.js" red=`tput setaf 1` green=`tput setaf 2` @@ -31,10 +31,10 @@ $DIFF" } # $1 is the source file to be tested -# $2 is the Source test file which uses the test framework +# $2 is the Source test file which uses the source-test framework test_source_framework() { - # run concatenation of test framework and test file - RESULTS=$($JS_SLANG -e --chapter=4 "$(cat $TEST_FRAMEWORK $1 $2)") + # run concatenation of source-test framework and test file + RESULTS=$($JS_SLANG -e --chapter=4 "$(cat $SOURCE_TEST $1 $2)") # retrieve names for tests that passed while read test_name @@ -68,8 +68,8 @@ main() { test_source $s $i done - # check if test framework is being used - TEST_PATH="$DIR/__tests__/$TEST_FRAMEWORK_FILE" + # check if source-test framework is being used + TEST_PATH="$DIR/__tests__/$SOURCE_TEST_FILENAME" if [ -e $TEST_PATH ] then test_source_framework $s $TEST_PATH From a2e6f44ce85c76165910ea139ad21e1bf27e7267 Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Thu, 9 Apr 2020 23:27:37 +0800 Subject: [PATCH 5/8] add support for testing multiple source files --- scripts/test.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 35bf455..833bfb0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,7 +4,6 @@ JS_SLANG="node node_modules/js-slang/dist/repl/repl.js" SOURCEFILES=src/*/*.js SOURCE_TEST="src/test/framework/main.js" -SOURCE_TEST_FILENAME="source-test.js" red=`tput setaf 1` green=`tput setaf 2` @@ -33,7 +32,7 @@ $DIFF" # $1 is the source file to be tested # $2 is the Source test file which uses the source-test framework test_source_framework() { - # run concatenation of source-test framework and test file + # run concatenation of source-test framework, source and test files RESULTS=$($JS_SLANG -e --chapter=4 "$(cat $SOURCE_TEST $1 $2)") # retrieve names for tests that passed @@ -65,17 +64,14 @@ main() { # call test_source on each test case in __tests__ for i in "$DIR/__tests__/$(basename ${s} .js)".* do - test_source $s $i - done - - # check if source-test framework is being used - TEST_PATH="$DIR/__tests__/$SOURCE_TEST_FILENAME" - if [ -e $TEST_PATH ] - then - test_source_framework $s $TEST_PATH - fi + # check if first line of test file contains '// source-test' + use_source_test=$(awk 'FNR==1{if ($0~"//[[:space:]]*source-test") print "yes";}' $i) + if [[ $use_source_test == "yes" ]] + then test_source_framework $s $i + else test_source $s $i + fi + done fi - done } From 320ea5f2de3e028702f14530d39977fe1eeb52c5 Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Thu, 16 Apr 2020 22:46:43 +0800 Subject: [PATCH 6/8] add support for chapter and variant --- scripts/test.sh | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 833bfb0..e98c21d 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -5,6 +5,9 @@ JS_SLANG="node node_modules/js-slang/dist/repl/repl.js" SOURCEFILES=src/*/*.js SOURCE_TEST="src/test/framework/main.js" +DEFAULT_CHAP=4 +DEFAULT_VARIANT="default" + red=`tput setaf 1` green=`tput setaf 2` normal=`tput setaf 7` @@ -14,11 +17,14 @@ failed=0 # $1 is the source file to be tested # $2 is the test file +# $3 is the chapter +# $4 is the variant + test_source() { # run concatenation of source and test case # compare the result with the commented last line in test case - DIFF=$(diff <($JS_SLANG -e --chapter=4 "$(cat $1 $2)") \ + DIFF=$(diff <($JS_SLANG -e --chapter=$3 --variant=$4 "$(cat $1 $2)") \ <(cat $2 | tail -1 | cut -c4-)) if [ "$DIFF" = "" ] @@ -29,18 +35,17 @@ $DIFF" } -# $1 is the source file to be tested -# $2 is the Source test file which uses the source-test framework test_source_framework() { + # run concatenation of source-test framework, source and test files - RESULTS=$($JS_SLANG -e --chapter=4 "$(cat $SOURCE_TEST $1 $2)") + RESULTS=$($JS_SLANG -e --chapter=$3 --variant=$4 "$(cat $SOURCE_TEST $1 $2)") # retrieve names for tests that passed while read test_name do passed=$(($passed+1)) echo "${green}PASS $2 $test_name" - done < <(echo ${RESULTS} | grep -o '\w* PASSED' | awk -F 'PASSED' '{print $1}') + done < <(echo ${RESULTS} | grep -o '\w* PASSED' | awk -F 'PASSED' '{ print $1 }') # retrieve names and error messages for tests that failed while read test_info @@ -64,11 +69,17 @@ main() { # call test_source on each test case in __tests__ for i in "$DIR/__tests__/$(basename ${s} .js)".* do - # check if first line of test file contains '// source-test' - use_source_test=$(awk 'FNR==1{if ($0~"//[[:space:]]*source-test") print "yes";}' $i) + # check if first line of test file contains 'chap=' and retrieve its value. Set to the default chapter if it does not + chap=$(awk -F 'chap=' 'FNR==1{ if ($0~"chap=") { print $2 } else { print '$DEFAULT_CHAP' } }' $i | awk -F ' ' '{ print $1 }') + + # check if first line of test file contains 'variant=' and retrieve its value. Set to the default variant if it does not + variant=$(awk -F 'variant=' 'FNR==1{ if ($0~"variant=") { print $2 } else { print '$DEFAULT_VARIANT' } }' $i | awk -F ' ' '{ print $1 }') + + # check if first line of test file contains 'source-test' + use_source_test=$(awk 'FNR==1{ if ($0~"source-test") print "yes" }' $i) if [[ $use_source_test == "yes" ]] - then test_source_framework $s $i - else test_source $s $i + then test_source_framework $s $i $chap $variant + else test_source $s $i $chap $variant fi done fi From 127d0c88799aa7f44812251ade4e677cf57d5ecc Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Thu, 16 Apr 2020 23:04:06 +0800 Subject: [PATCH 7/8] set chapter to 4 for source-test --- scripts/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test.sh b/scripts/test.sh index e98c21d..f10aae0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -78,7 +78,7 @@ main() { # check if first line of test file contains 'source-test' use_source_test=$(awk 'FNR==1{ if ($0~"source-test") print "yes" }' $i) if [[ $use_source_test == "yes" ]] - then test_source_framework $s $i $chap $variant + then chap=4 ; test_source_framework $s $i $chap $variant else test_source $s $i $chap $variant fi done From 33805dd9f23d5ae857785deb1249e201f8671914 Mon Sep 17 00:00:00 2001 From: Arsalan Cheema Date: Thu, 16 Apr 2020 23:12:09 +0800 Subject: [PATCH 8/8] use chapter for consistency with js-slang --- scripts/test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index f10aae0..5e32bad 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -5,7 +5,7 @@ JS_SLANG="node node_modules/js-slang/dist/repl/repl.js" SOURCEFILES=src/*/*.js SOURCE_TEST="src/test/framework/main.js" -DEFAULT_CHAP=4 +DEFAULT_CHAPTER=4 DEFAULT_VARIANT="default" red=`tput setaf 1` @@ -69,8 +69,8 @@ main() { # call test_source on each test case in __tests__ for i in "$DIR/__tests__/$(basename ${s} .js)".* do - # check if first line of test file contains 'chap=' and retrieve its value. Set to the default chapter if it does not - chap=$(awk -F 'chap=' 'FNR==1{ if ($0~"chap=") { print $2 } else { print '$DEFAULT_CHAP' } }' $i | awk -F ' ' '{ print $1 }') + # check if first line of test file contains 'chapter=' and retrieve its value. Set to the default chapter if it does not + chapter=$(awk -F 'chapter=' 'FNR==1{ if ($0~"chapter=") { print $2 } else { print '$DEFAULT_CHAPTER' } }' $i | awk -F ' ' '{ print $1 }') # check if first line of test file contains 'variant=' and retrieve its value. Set to the default variant if it does not variant=$(awk -F 'variant=' 'FNR==1{ if ($0~"variant=") { print $2 } else { print '$DEFAULT_VARIANT' } }' $i | awk -F ' ' '{ print $1 }') @@ -78,8 +78,8 @@ main() { # check if first line of test file contains 'source-test' use_source_test=$(awk 'FNR==1{ if ($0~"source-test") print "yes" }' $i) if [[ $use_source_test == "yes" ]] - then chap=4 ; test_source_framework $s $i $chap $variant - else test_source $s $i $chap $variant + then chapter=4 ; test_source_framework $s $i $chapter $variant + else test_source $s $i $chapter $variant fi done fi