diff --git a/testing/run_tests.py b/testing/run_tests.py index f800a474544a6..0d7962f9dc98a 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -17,6 +17,7 @@ import multiprocessing import os import re +import shutil import subprocess import sys import tempfile @@ -752,21 +753,43 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None): ios_unit_test_dir = os.path.join( BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests' ) - # Avoid using xcpretty unless the following can be addressed: - # - Make sure all relevant failure output is printed on a failure. - # - Make sure that a failing exit code is set for CI. - # See https://github.com/flutter/flutter/issues/63742 - test_command = [ - 'xcodebuild ' - '-sdk iphonesimulator ' - '-scheme IosUnitTests ' - "-destination name='" + new_simulator_name + "' " - 'test ' - 'FLUTTER_ENGINE=' + ios_variant - ] - if test_filter is not None: - test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter - run_cmd(test_command, cwd=ios_unit_test_dir, shell=True) + + with tempfile.TemporaryDirectory(suffix='ios_embedding_xcresult' + ) as result_bundle_temp: + result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding') + + # Avoid using xcpretty unless the following can be addressed: + # - Make sure all relevant failure output is printed on a failure. + # - Make sure that a failing exit code is set for CI. + # See https://github.com/flutter/flutter/issues/63742 + test_command = [ + 'xcodebuild ' + '-sdk iphonesimulator ' + '-scheme IosUnitTests ' + '-resultBundlePath ' + result_bundle_path + ' ' + '-destination name=' + new_simulator_name + ' ' + 'test ' + 'FLUTTER_ENGINE=' + ios_variant + ] + if test_filter is not None: + test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter + run_cmd(test_command, cwd=ios_unit_test_dir, shell=True) + + # except: + # The LUCI environment may provide a variable containing a directory path + # for additional output files that will be uploaded to cloud storage. + # Upload the xcresult when the tests fail. + luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR') + xcresult_bundle = os.path.join( + result_bundle_temp, 'ios_embedding.xcresult' + ) + if luci_test_outputs_path and os.path.exists(xcresult_bundle): + dump_path = os.path.join( + luci_test_outputs_path, 'ios_embedding.xcresult' + ) + # xcresults contain many little files. Archive the bundle before upload. + shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle) + finally: delete_simulator(new_simulator_name) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index fd84de214150f..0034a42e4899a 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -41,25 +41,58 @@ fi # Can also be set via Simulator app Device > Rotate Device Automatically defaults write com.apple.iphonesimulator RotateWindowWhenSignaledByGuest -int 1 -cd $SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios +SCENARIO_PATH=$SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios +pushd . +cd $SCENARIO_PATH + +RESULT_BUNDLE_FOLDER="ios_scenario_xcresult" +RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}" + +# Zip and upload xcresult to luci. +# First parameter ($1) is the zip output name. +zip_and_upload_xcresult_to_luci () { + # We don't want the zip to contain the abusolute path, + # so use relative path (./$RESULT_BUNDLE_FOLDER) instead. + zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" + mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR + exit 1 +} echo "Running simulator tests with Skia" echo "" -set -o pipefail && xcodebuild -sdk iphonesimulator \ +mktemp -d $RESULT_BUNDLE_PATH + +if set -o pipefail && xcodebuild -sdk iphonesimulator \ -scheme Scenarios \ + -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \ clean test \ - FLUTTER_ENGINE="$FLUTTER_ENGINE" + FLUTTER_ENGINE="$FLUTTER_ENGINE"; then + echo "test success." +else + echo "test failed." + zip_and_upload_xcresult_to_luci "ios_scenario_xcresult.zip" +fi +rm -rf $RESULT_BUNDLE_PATH echo "Running simulator tests with Impeller" echo "" # Skip testFontRenderingWhenSuppliedWithBogusFont: https://github.com/flutter/flutter/issues/113250 -set -o pipefail && xcodebuild -sdk iphonesimulator \ +if set -o pipefail && xcodebuild -sdk iphonesimulator \ -scheme Scenarios \ + -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \ clean test \ FLUTTER_ENGINE="$FLUTTER_ENGINE" \ -skip-testing "ScenariosUITests/BogusFontTextTest/testFontRenderingWhenSuppliedWithBogusFont" \ - INFOPLIST_FILE="Scenarios/Info_Impeller.plist" # Plist with FLTEnableImpeller=YES + INFOPLIST_FILE="Scenarios/Info_Impeller.plist"; then # Plist with FLTEnableImpeller=YES + echo "test success." +else + echo "test failed." + zip_and_upload_xcresult_to_luci "ios_scenario_impeller_xcresult.zip" +fi +rm -rf $RESULT_BUNDLE_PATH + +popd