diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d4e2e060b..b214b9137 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -50,6 +50,7 @@ env: statusLabelFailed: "tests: failed" statusLabelSucceeded: "tests: succeeded" statusCommentIdentifier: "integration-test-status-comment" + mobileTestOn: "device" jobs: check_trigger: @@ -336,12 +337,16 @@ jobs: additional_flags+=(--cmake_flag=-DFIREBASE_USE_BORINGSSL=ON) fi fi - python scripts/gha/build_testapps.py --t ${{ needs.prepare_matrix.outputs.apis }} --p ${{ matrix.target_platform }} --output_directory "${{ github.workspace }}" --noadd_timestamp ${additional_flags[*]} --short_output_paths + python scripts/gha/build_testapps.py --t ${{ needs.prepare_matrix.outputs.apis }} --p ${{ matrix.target_platform }} --output_directory "${{ github.workspace }}" --ios_sdk "${{ env.mobileTestOn }}" --noadd_timestamp ${additional_flags[*]} --short_output_paths - name: Run desktop integration tests if: matrix.target_platform == 'Desktop' && !cancelled() run: | python scripts/gha/desktop_tester.py --testapp_dir ta + - name: Run iOS integration tests on Simulator locally + if: matrix.target_platform == 'iOS' && !cancelled() + run: | + python scripts/gha/test_simulator.py --testapp_dir ta # Workaround for https://github.com/GoogleCloudPlatform/github-actions/issues/100 # Must be run after the Python setup action - name: Set CLOUDSDK_PYTHON (Windows) @@ -355,7 +360,7 @@ jobs: if: matrix.target_platform == 'Desktop' && !cancelled() run: | python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json - - name: Run mobile integration tests + - name: Run mobile integration tests on Real Device via FTL if: matrix.target_platform != 'Desktop' && !cancelled() run: | python scripts/gha/test_lab.py --android_model ${{ needs.prepare_matrix.outputs.android_device }} --android_api ${{ needs.prepare_matrix.outputs.android_api }} --ios_model ${{ needs.prepare_matrix.outputs.ios_device }} --ios_version ${{ needs.prepare_matrix.outputs.ios_version }} --testapp_dir ta --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index a79d03d49..dc3ed2cc9 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -108,8 +108,7 @@ # Values for iOS SDK flag (where the iOS app will run) _IOS_SDK_DEVICE = "device" _IOS_SDK_SIMULATOR = "simulator" -_IOS_SDK_BOTH = "both" -_SUPPORTED_IOS_SDK = (_IOS_SDK_DEVICE, _IOS_SDK_SIMULATOR, _IOS_SDK_BOTH) +_SUPPORTED_IOS_SDK = (_IOS_SDK_DEVICE, _IOS_SDK_SIMULATOR) FLAGS = flags.FLAGS @@ -140,8 +139,8 @@ " Recommended when running locally, so each execution gets its own " " directory.") -flags.DEFINE_enum( - "ios_sdk", _IOS_SDK_DEVICE, _SUPPORTED_IOS_SDK, +flags.DEFINE_list( + "ios_sdk", _IOS_SDK_DEVICE, "(iOS only) Build for device (ipa), simulator (app), or both." " Building for both will produce both an .app and an .ipa.") @@ -168,6 +167,12 @@ message="Valid platforms: " + ",".join(_SUPPORTED_PLATFORMS), flag_values=FLAGS) +flags.register_validator( + "ios_sdk", + lambda s: all(ios_sdk in _SUPPORTED_IOS_SDK for ios_sdk in s), + message="Valid platforms: " + ",".join(_SUPPORTED_IOS_SDK), + flag_values=FLAGS) + flags.DEFINE_bool( "short_output_paths", False, "Use short directory names for output paths. Useful to avoid hitting file " @@ -501,7 +506,7 @@ def _build_ios( _run(xcode_patcher_args) xcode_path = os.path.join(project_dir, api_config.ios_target + ".xcworkspace") - if ios_sdk in [_IOS_SDK_SIMULATOR, _IOS_SDK_BOTH]: + if _IOS_SDK_SIMULATOR in ios_sdk: _run( xcodebuild.get_args_for_build( path=xcode_path, @@ -510,7 +515,7 @@ def _build_ios( ios_sdk=_IOS_SDK_SIMULATOR, configuration="Debug")) - if ios_sdk in [_IOS_SDK_DEVICE, _IOS_SDK_BOTH]: + if _IOS_SDK_DEVICE in ios_sdk: _run( xcodebuild.get_args_for_build( path=xcode_path, diff --git a/scripts/gha/integration_testing/gameloop/gameloop.xcodeproj/project.pbxproj b/scripts/gha/integration_testing/gameloop/gameloop.xcodeproj/project.pbxproj new file mode 100644 index 000000000..5e20e82c7 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop.xcodeproj/project.pbxproj @@ -0,0 +1,469 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 50; + objects = { + +/* Begin PBXBuildFile section */ + 6ABE2697260DD8AA00675C6B /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6ABE2696260DD8AA00675C6B /* AppDelegate.swift */; }; + 6ABE269B260DD8AA00675C6B /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6ABE269A260DD8AA00675C6B /* ViewController.swift */; }; + 6ABE269E260DD8AA00675C6B /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 6ABE269C260DD8AA00675C6B /* Main.storyboard */; }; + 6ABE26A0260DD8AB00675C6B /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 6ABE269F260DD8AB00675C6B /* Assets.xcassets */; }; + 6ABE26A3260DD8AB00675C6B /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 6ABE26A1260DD8AB00675C6B /* LaunchScreen.storyboard */; }; + 6ABE26AC260DD90600675C6B /* Constants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6ABE26AB260DD90600675C6B /* Constants.swift */; }; + 6ABE26B9260DDA4300675C6B /* gameloopUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6ABE26B8260DDA4300675C6B /* gameloopUITests.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 6ABE26BB260DDA4300675C6B /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 6ABE268B260DD8AA00675C6B /* Project object */; + proxyType = 1; + remoteGlobalIDString = 6ABE2692260DD8AA00675C6B; + remoteInfo = gameloop; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXFileReference section */ + 6ABE2693260DD8AA00675C6B /* gameloop.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = gameloop.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 6ABE2696260DD8AA00675C6B /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + 6ABE269A260DD8AA00675C6B /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; + 6ABE269D260DD8AA00675C6B /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; + 6ABE269F260DD8AB00675C6B /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 6ABE26A2260DD8AB00675C6B /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 6ABE26A4260DD8AB00675C6B /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 6ABE26AB260DD90600675C6B /* Constants.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Constants.swift; sourceTree = ""; }; + 6ABE26B6260DDA4300675C6B /* gameloopUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = gameloopUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 6ABE26B8260DDA4300675C6B /* gameloopUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = gameloopUITests.swift; sourceTree = ""; }; + 6ABE26BA260DDA4300675C6B /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 6ABE2690260DD8AA00675C6B /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 6ABE26B3260DDA4300675C6B /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 6ABE268A260DD8AA00675C6B = { + isa = PBXGroup; + children = ( + 6ABE2695260DD8AA00675C6B /* gameloop */, + 6ABE26B7260DDA4300675C6B /* gameloopUITests */, + 6ABE2694260DD8AA00675C6B /* Products */, + ); + sourceTree = ""; + }; + 6ABE2694260DD8AA00675C6B /* Products */ = { + isa = PBXGroup; + children = ( + 6ABE2693260DD8AA00675C6B /* gameloop.app */, + 6ABE26B6260DDA4300675C6B /* gameloopUITests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + 6ABE2695260DD8AA00675C6B /* gameloop */ = { + isa = PBXGroup; + children = ( + 6ABE2696260DD8AA00675C6B /* AppDelegate.swift */, + 6ABE269A260DD8AA00675C6B /* ViewController.swift */, + 6ABE269C260DD8AA00675C6B /* Main.storyboard */, + 6ABE269F260DD8AB00675C6B /* Assets.xcassets */, + 6ABE26A1260DD8AB00675C6B /* LaunchScreen.storyboard */, + 6ABE26A4260DD8AB00675C6B /* Info.plist */, + 6ABE26AB260DD90600675C6B /* Constants.swift */, + ); + path = gameloop; + sourceTree = ""; + }; + 6ABE26B7260DDA4300675C6B /* gameloopUITests */ = { + isa = PBXGroup; + children = ( + 6ABE26B8260DDA4300675C6B /* gameloopUITests.swift */, + 6ABE26BA260DDA4300675C6B /* Info.plist */, + ); + path = gameloopUITests; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 6ABE2692260DD8AA00675C6B /* gameloop */ = { + isa = PBXNativeTarget; + buildConfigurationList = 6ABE26A7260DD8AB00675C6B /* Build configuration list for PBXNativeTarget "gameloop" */; + buildPhases = ( + 6ABE268F260DD8AA00675C6B /* Sources */, + 6ABE2690260DD8AA00675C6B /* Frameworks */, + 6ABE2691260DD8AA00675C6B /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = gameloop; + productName = gameloop; + productReference = 6ABE2693260DD8AA00675C6B /* gameloop.app */; + productType = "com.apple.product-type.application"; + }; + 6ABE26B5260DDA4300675C6B /* gameloopUITests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 6ABE26BD260DDA4300675C6B /* Build configuration list for PBXNativeTarget "gameloopUITests" */; + buildPhases = ( + 6ABE26B2260DDA4300675C6B /* Sources */, + 6ABE26B3260DDA4300675C6B /* Frameworks */, + 6ABE26B4260DDA4300675C6B /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 6ABE26BC260DDA4300675C6B /* PBXTargetDependency */, + ); + name = gameloopUITests; + productName = gameloopUITests; + productReference = 6ABE26B6260DDA4300675C6B /* gameloopUITests.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 6ABE268B260DD8AA00675C6B /* Project object */ = { + isa = PBXProject; + attributes = { + LastSwiftUpdateCheck = 1230; + LastUpgradeCheck = 1230; + TargetAttributes = { + 6ABE2692260DD8AA00675C6B = { + CreatedOnToolsVersion = 12.3; + }; + 6ABE26B5260DDA4300675C6B = { + CreatedOnToolsVersion = 12.3; + TestTargetID = 6ABE2692260DD8AA00675C6B; + }; + }; + }; + buildConfigurationList = 6ABE268E260DD8AA00675C6B /* Build configuration list for PBXProject "gameloop" */; + compatibilityVersion = "Xcode 9.3"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 6ABE268A260DD8AA00675C6B; + productRefGroup = 6ABE2694260DD8AA00675C6B /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 6ABE2692260DD8AA00675C6B /* gameloop */, + 6ABE26B5260DDA4300675C6B /* gameloopUITests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 6ABE2691260DD8AA00675C6B /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 6ABE26A3260DD8AB00675C6B /* LaunchScreen.storyboard in Resources */, + 6ABE26A0260DD8AB00675C6B /* Assets.xcassets in Resources */, + 6ABE269E260DD8AA00675C6B /* Main.storyboard in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 6ABE26B4260DDA4300675C6B /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 6ABE268F260DD8AA00675C6B /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 6ABE269B260DD8AA00675C6B /* ViewController.swift in Sources */, + 6ABE2697260DD8AA00675C6B /* AppDelegate.swift in Sources */, + 6ABE26AC260DD90600675C6B /* Constants.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 6ABE26B2260DDA4300675C6B /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 6ABE26B9260DDA4300675C6B /* gameloopUITests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 6ABE26BC260DDA4300675C6B /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 6ABE2692260DD8AA00675C6B /* gameloop */; + targetProxy = 6ABE26BB260DDA4300675C6B /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin PBXVariantGroup section */ + 6ABE269C260DD8AA00675C6B /* Main.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 6ABE269D260DD8AA00675C6B /* Base */, + ); + name = Main.storyboard; + sourceTree = ""; + }; + 6ABE26A1260DD8AB00675C6B /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 6ABE26A2260DD8AB00675C6B /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 6ABE26A5260DD8AB00675C6B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + 6ABE26A6260DD8AB00675C6B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 6ABE26A8260DD8AB00675C6B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = gameloop/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.firebase.gameloop; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 6ABE26A9260DD8AB00675C6B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = gameloop/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.firebase.gameloop; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + 6ABE26BE260DDA4300675C6B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = gameloopUITests/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.firebase.gameloopUITests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = gameloop; + }; + name = Debug; + }; + 6ABE26BF260DDA4300675C6B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + INFOPLIST_FILE = gameloopUITests/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.firebase.gameloopUITests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = gameloop; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 6ABE268E260DD8AA00675C6B /* Build configuration list for PBXProject "gameloop" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 6ABE26A5260DD8AB00675C6B /* Debug */, + 6ABE26A6260DD8AB00675C6B /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 6ABE26A7260DD8AB00675C6B /* Build configuration list for PBXNativeTarget "gameloop" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 6ABE26A8260DD8AB00675C6B /* Debug */, + 6ABE26A9260DD8AB00675C6B /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 6ABE26BD260DDA4300675C6B /* Build configuration list for PBXNativeTarget "gameloopUITests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 6ABE26BE260DDA4300675C6B /* Debug */, + 6ABE26BF260DDA4300675C6B /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 6ABE268B260DD8AA00675C6B /* Project object */; +} diff --git a/scripts/gha/integration_testing/gameloop/gameloop/AppDelegate.swift b/scripts/gha/integration_testing/gameloop/gameloop/AppDelegate.swift new file mode 100644 index 000000000..a148a2c99 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/AppDelegate.swift @@ -0,0 +1,40 @@ +// +// Copyright (c) 2021 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// AppDelegate.swift +// gameloop +// + +import UIKit + +/// App delegate for handling calls to our custom URL scheme. +@UIApplicationMain +class AppDelegate: UIResponder, UIApplicationDelegate { + + var window: UIWindow? + + /// Handle calls to our custom URL scheme to signal the completion of the game loop + func application( + _ application: UIApplication, open url: URL, + options: [UIApplication.OpenURLOptionsKey: Any] + ) -> Bool { + let viewController = window!.rootViewController as! ViewController + viewController.markComplete() + return true + } + +} + + diff --git a/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/AccentColor.colorset/Contents.json b/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 000000000..eb8789700 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/AppIcon.appiconset/Contents.json b/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 000000000..9221b9bb1 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,98 @@ +{ + "images" : [ + { + "idiom" : "iphone", + "scale" : "2x", + "size" : "20x20" + }, + { + "idiom" : "iphone", + "scale" : "3x", + "size" : "20x20" + }, + { + "idiom" : "iphone", + "scale" : "2x", + "size" : "29x29" + }, + { + "idiom" : "iphone", + "scale" : "3x", + "size" : "29x29" + }, + { + "idiom" : "iphone", + "scale" : "2x", + "size" : "40x40" + }, + { + "idiom" : "iphone", + "scale" : "3x", + "size" : "40x40" + }, + { + "idiom" : "iphone", + "scale" : "2x", + "size" : "60x60" + }, + { + "idiom" : "iphone", + "scale" : "3x", + "size" : "60x60" + }, + { + "idiom" : "ipad", + "scale" : "1x", + "size" : "20x20" + }, + { + "idiom" : "ipad", + "scale" : "2x", + "size" : "20x20" + }, + { + "idiom" : "ipad", + "scale" : "1x", + "size" : "29x29" + }, + { + "idiom" : "ipad", + "scale" : "2x", + "size" : "29x29" + }, + { + "idiom" : "ipad", + "scale" : "1x", + "size" : "40x40" + }, + { + "idiom" : "ipad", + "scale" : "2x", + "size" : "40x40" + }, + { + "idiom" : "ipad", + "scale" : "1x", + "size" : "76x76" + }, + { + "idiom" : "ipad", + "scale" : "2x", + "size" : "76x76" + }, + { + "idiom" : "ipad", + "scale" : "2x", + "size" : "83.5x83.5" + }, + { + "idiom" : "ios-marketing", + "scale" : "1x", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/Contents.json b/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/Contents.json new file mode 100644 index 000000000..73c00596a --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/scripts/gha/integration_testing/gameloop/gameloop/Base.lproj/LaunchScreen.storyboard b/scripts/gha/integration_testing/gameloop/gameloop/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 000000000..bfa361294 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/gha/integration_testing/gameloop/gameloop/Base.lproj/Main.storyboard b/scripts/gha/integration_testing/gameloop/gameloop/Base.lproj/Main.storyboard new file mode 100644 index 000000000..f32740b5c --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/Base.lproj/Main.storyboard @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/gha/integration_testing/gameloop/gameloop/Constants.swift b/scripts/gha/integration_testing/gameloop/gameloop/Constants.swift new file mode 100644 index 000000000..940480dba --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/Constants.swift @@ -0,0 +1,14 @@ +/// Constants used across the loop launcher and UI test. +public struct Constants { + public static let completeText = "Game Loop Complete" + public static let gameLoopTimeout = "GAME_LOOP_TIMEOUT" + public static let gameLoopScenario = "GAME_LOOP_SCENARIO" + public static let gameLoopBundleId = "GAME_LOOP_BUNDLE_ID" + + // Custom URL schemes for starting and ending game loops + public static let gameLoopScheme = "firebase-game-loop://" + public static let gameLoopCompleteScheme = "firebase-game-loop-complete://" + + // Directory to look for the output results + public static let resultsDirectory = "GameLoopResults" +} diff --git a/scripts/gha/integration_testing/gameloop/gameloop/Info.plist b/scripts/gha/integration_testing/gameloop/gameloop/Info.plist new file mode 100644 index 000000000..0b5df7a54 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/Info.plist @@ -0,0 +1,60 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIApplicationSupportsIndirectInputEvents + + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + CFBundleURLTypes + + + CFBundleURLName + + CFBundleTypeRole + Editor + CFBundleURLSchemes + + firebase-game-loop-complete + + + + + diff --git a/scripts/gha/integration_testing/gameloop/gameloop/ViewController.swift b/scripts/gha/integration_testing/gameloop/gameloop/ViewController.swift new file mode 100644 index 000000000..1ffd503d3 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloop/ViewController.swift @@ -0,0 +1,84 @@ +// +// Copyright (c) 2021 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ViewController.swift +// gameloop +// + +import CoreFoundation +import UIKit + +/// Minimal view controller to launch the game loop. +class ViewController: UIViewController { + @IBOutlet var testingLabel: UILabel! + let scenarioFromEnv = Int(ProcessInfo.processInfo.environment[Constants.gameLoopScenario] ?? "") + let scenarioFromArg = Int(ProcessInfo.processInfo.arguments.last ?? "") + var scenario: Int { + scenarioFromArg ?? scenarioFromEnv ?? 1 + } + + /// Run the game loop as soon as we load. + override func viewDidLoad() { + super.viewDidLoad() + NSLog("Starting game loop with scenario \(scenario)") + launchGame() + } + + /// Launch the app under test by calling our custom URL scheme. + func launchGame() { + let url = getURL() + let pasteboard = UIPasteboard.general + pasteboard.string = String(scenario) + UIApplication.shared.open(url) { success in + if !success { + NSLog("Error launching game loop; skipping scenario \(self.scenario)") + self.markComplete() + } + } + } + + /// Derive the URL from our scheme and the scenario environment variable. + func getURL() -> URL { + let urlString = "\(Constants.gameLoopScheme)?scenario=\(scenario)" + return URL(string: urlString)! + } + + /// Signal to the UI test the game loop is done by changing the label. + func markComplete() { + NSLog("Completing game loop \(scenario)") + let center = CFNotificationCenterGetDarwinNotifyCenter() + let name = "com.google.game-loop" as CFString + CFNotificationCenterPostNotification(center, CFNotificationName(name), nil, nil, true) + testingLabel.text = Constants.completeText + writeResultSentinel() + } + + func writeResultSentinel() { + let text = "Scenario \(scenario) complete" + let fileName = "scenario.\(scenario)" + let fileManager = FileManager.default + do { + let docs = try fileManager.url( + for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + let fileURL = docs.appendingPathComponent(fileName) + NSLog("Writing to \(fileURL)") + try text.write(to: fileURL, atomically: false, encoding: .utf8) + NSLog("Sentinel file for scenario \(scenario) successfully written") + } catch { + NSLog("ERROR: There was an error writing to the file: \(error)") + } + } +} + diff --git a/scripts/gha/integration_testing/gameloop/gameloopUITests/Info.plist b/scripts/gha/integration_testing/gameloop/gameloopUITests/Info.plist new file mode 100644 index 000000000..64d65ca49 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloopUITests/Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/scripts/gha/integration_testing/gameloop/gameloopUITests/gameloopUITests.swift b/scripts/gha/integration_testing/gameloop/gameloopUITests/gameloopUITests.swift new file mode 100644 index 000000000..38d37d522 --- /dev/null +++ b/scripts/gha/integration_testing/gameloop/gameloopUITests/gameloopUITests.swift @@ -0,0 +1,104 @@ +// +// Copyright (c) 2021 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// gameloopUITests.swift +// gameloopUITests +// + +import XCTest + +/// Constants used across the loop launcher and UI test. +public struct Constants { + public static let completeText = "Game Loop Complete" + public static let gameLoopTimeout = "GAME_LOOP_TIMEOUT" + public static let gameLoopScenario = "GAME_LOOP_SCENARIO" + public static let gameLoopBundleId = "GAME_LOOP_BUNDLE_ID" + + // Custom URL schemes for starting and ending game loops + public static let gameLoopScheme = "firebase-game-loop://" + public static let gameLoopCompleteScheme = "firebase-game-loop-complete://" + + // Directory to look for the output results + public static let resultsDirectory = "GameLoopResults" +} +/// UI test suite as entry point to launching app under test. +class GameLoopLauncherUITests: XCTestCase { + + /// Launch the game loop through the view controller and wait for its completion. + func testLaunchGameLoop() { + // Add the scenario to our launcher app's environment + let app = XCUIApplication() + let environment = ProcessInfo.processInfo.environment + let scenario = environment[Constants.gameLoopScenario] ?? "0" + let bundleId = environment[Constants.gameLoopBundleId] ?? nil + app.launchEnvironment[Constants.gameLoopScenario] = scenario + + // Periodically check and dismiss dialogs with "Allow" or "OK" + Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { (_) in + let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") + for button in [springboard.buttons["Open"], springboard.buttons["Allow"], springboard.buttons["OK"]] { + if button.exists { + NSLog("Dismissing system dialog") + button.tap() + } + } + } + + app.launch() + + let result = waitForLoopCompletion(for: app) + attachResults() + // Terminate the app under test to clear its state + if bundleId != nil { + XCUIApplication(bundleIdentifier: bundleId!).terminate() + } + XCTAssert(result == .completed) + } + + /// Wait for the game loop to complete or time out + func waitForLoopCompletion(for app: XCUIApplication) -> XCTWaiter.Result { + let existsPredicate = NSPredicate(format: "exists == true") + let expectation = XCTNSPredicateExpectation( + predicate: existsPredicate, + object: app.staticTexts[Constants.completeText]) + let timeout = getTimeout(for: app) + return XCTWaiter().wait(for: [expectation], timeout: timeout) + } + + /// Obtain the timeout from the runtime environment. + func getTimeout(for app: XCUIApplication) -> TimeInterval { + if let timeoutString = ProcessInfo.processInfo.environment[Constants.gameLoopTimeout], + let timeoutSecs = TimeInterval(timeoutString) + { + return timeoutSecs + } else { + // Default 5 minutes + return TimeInterval(60 * 3) + } + } + + /// Collect all the strings in the general UIPasteboard and attach them as a test result. + func attachResults() { + let pasteboard = UIPasteboard.general + guard pasteboard.hasStrings else { + // No output data; do nothing + return + } + let allStrings = pasteboard.strings! + let joined = allStrings.joined(separator: "\n") + let attachment = XCTAttachment(string: joined) + add(attachment) + } +} diff --git a/scripts/gha/integration_testing/test_validation.py b/scripts/gha/integration_testing/test_validation.py index a46b480b3..84866cf54 100644 --- a/scripts/gha/integration_testing/test_validation.py +++ b/scripts/gha/integration_testing/test_validation.py @@ -150,7 +150,14 @@ def summarize_test_results(tests, platform, summary_dir): failures = [] errors = [] + test_on = "" for test in tests: + if not test_on: + if test.testapp_path.endswith(".ipa"): + test_on = " (ON REAL DEVICE VIA FTL)" + elif test.testapp_path.endswith(".app"): + test_on = " (ON SIMULATOR) " + results = validate_results(test.logs, platform) test_result_pair = (test, results) if not results.complete: @@ -176,7 +183,7 @@ def summarize_test_results(tests, platform, summary_dir): # The summary is much more terse, to minimize the time it takes to understand # what went wrong, without necessarily providing full debugging context. summary = [] - summary.append("TEST SUMMARY:") + summary.append("TEST SUMMARY%s:" % test_on) if successes: summary.append("%d TESTAPPS SUCCEEDED:" % len(successes)) summary.extend((test.testapp_path for (test, _) in successes)) diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index a970062d7..9860634f9 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -103,6 +103,7 @@ }, "config": { "apis": "admob,analytics,auth,database,dynamic_links,firestore,functions,installations,instance_id,messaging,remote_config,storage", + "mobile_test_on": "device,simulator", "android_device": "flame", "android_api": "29", "ios_device": "iphone8", diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py index 8b4da8728..b4b71663a 100644 --- a/scripts/gha/summarize_test_results.py +++ b/scripts/gha/summarize_test_results.py @@ -90,6 +90,9 @@ "desktop": "Desktop", } +SIMULATOR = "simulator" +HARDWARE = "hardware" + PLATFORM_HEADER = "Platform" BUILD_FAILURES_HEADER = "Build failures" TEST_FAILURES_HEADER = "Test failures" @@ -252,10 +255,19 @@ def main(argv): # For desktop, highlight the entire platform string. log_name[0] = "%s**" % log_name[0] log_name[1] = "**%s" % log_name[1] - # Rejoin matrix name with spaces. - log_name = ' '.join([log_name[1], log_name[0]]+log_name[2:]) with open(log_file, "r") as log_reader: - log_data[log_name] = log_reader.read() + log_reader_data = log_reader.read() + if "Android" in log_name or "iOS" in log_name: + # Rejoin matrix name with spaces. + log_name_str = ' '.join([log_name[1], SIMULATOR, log_name[0]]+log_name[2:]) + log_data[log_name_str] = log_reader_data + # iOS and Android repeat the list for simulator and device + log_name_str = ' '.join([log_name[1], HARDWARE, log_name[0]]+log_name[2:]) + log_data[log_name_str] = log_reader_data + else: + # Rejoin matrix name with spaces. + log_name_str = ' '.join([log_name[1], log_name[0]]+log_name[2:]) + log_data[log_name_str] = log_reader_data log_results = {} # Go through each log and extract out the build and test failures. @@ -279,20 +291,25 @@ def main(argv): any_failures = True # Extract test failures, which follow "TESTAPPS EXPERIENCED ERRORS:" - m = re.search(r'TESTAPPS (EXPERIENCED ERRORS|FAILED):\n(([^\n]*\n)+)', log_text, re.MULTILINE) + m = re.search(r'^TEST SUMMARY(.*)TESTAPPS (EXPERIENCED ERRORS|FAILED):\n(([^\n]*\n)+)', log_text, re.MULTILINE | re.DOTALL) + if m and ((SIMULATOR in platform and not "(ON SIMULATOR)" in m.group(1)) or + (HARDWARE in platform and not "(ON HARDWARE)" in m.group(1))): + m = None # don't process this if it's for the wrong hardware target if m: - for test_failure_line in m.group(2).strip("\n").split("\n"): + for test_failure_line in m.group(3).strip("\n").split("\n"): # Only get the lines showing paths. if "/firebase-cpp-sdk/" not in test_failure_line: continue test_filename = ""; if "log tail" in test_failure_line: test_filename = re.match(r'^(.*) log tail', test_failure_line).group(1) - if "lacks logs" in test_failure_line: + elif "lacks logs" in test_failure_line: test_filename = re.match(r'^(.*) lacks logs', test_failure_line).group(1) - if "it-debug.apk" in test_failure_line: + elif "it-debug.apk" in test_failure_line: test_filename = re.match(r'^(.*it-debug\.apk)', test_failure_line).group(1) - if "integration_test.ipa" in test_failure_line: + elif "integration_test.ipa" in test_failure_line: test_filename = re.match(r'^(.*integration_test\.ipa)', test_failure_line).group(1) + elif "integration_test.app" in test_failure_line: + test_filename = re.match(r'^(.*integration_test\.app)', test_failure_line).group(1) if test_filename: m2 = re.search(r'/ta/(firebase)?([^/]+)/iti?/', test_filename, re.IGNORECASE) @@ -309,6 +326,22 @@ def main(argv): log_results[platform]["test_failures"].union( log_results[platform]["build_failures"])) + # Also, if any simulator and hardware targets are identical, filter them. + to_del = set() + to_add = dict() + for platform in log_results.keys(): + simulator_str = (" %s " % SIMULATOR) + if simulator_str in platform: + other_platform = platform.replace(simulator_str, " %s " % HARDWARE) + if log_results[platform] == log_results[other_platform]: + targetless = platform.replace(simulator_str, " ") + to_add[targetless] = log_results[platform] + to_del.add(platform) + to_del.add(other_platform) + for platform_to_del in to_del: + del log_results[platform_to_del] + log_results.update(to_add) + if not any_failures and not FLAGS.include_successful: # No failures occurred, nothing to log. return(0) diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py new file mode 100644 index 000000000..c90b33d55 --- /dev/null +++ b/scripts/gha/test_simulator.py @@ -0,0 +1,259 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Tool for mobile testapps to Test on local simulators. + +Requires simulators installed. iOS simulator can be installed via tool: + https://github.com/xcpretty/xcode-install#simulators + + +Usage: + + python test_simulator.py --testapp_dir ~/testapps + +This will recursively search ~/testapps for apps, +test on local simulator, and validate their results. The validation is specific to +the structure of the Firebase Unity and C++ testapps. + + +If you wish to specify a particular device to test on, you will need the model +id and version (OS version for iOS). These change over time. You can listing all +available simulators (supported models and versions) with the following commands: + + xcrun simctl list + +Note: you need to combine Name and Version with "-". Examples: + +iPhone 11, OS 14.4: + --ios_device "iPhone 11-14.4" + +""" + +import json +import os +import pathlib +import subprocess + +from absl import app +from absl import flags +from absl import logging +import attr +from integration_testing import test_validation + + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + "testapp_dir", None, + "Testapps in this directory will be tested.") +flags.DEFINE_string( + "gameloop_project", "integration_testing/gameloop", + "A tool that enable game-loop test. This is a XCode project") +flags.DEFINE_string( + "ios_device", "iPhone 11-14.4", + "iOS device, which is a combination of device name and os version") + +@attr.s(frozen=False, eq=False) +class Test(object): + """Holds data related to the testing of one testapp.""" + testapp_path = attr.ib() + logs = attr.ib() + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + current_dir = pathlib.Path(__file__).parent.absolute() + testapp_dir = os.path.abspath(os.path.expanduser(FLAGS.testapp_dir)) + gameloop_project = os.path.join(current_dir, FLAGS.gameloop_project) + ios_device = FLAGS.ios_device + device_info = ios_device.split("-") + device_name = device_info[0] + device_os = device_info[1] + + config_path = os.path.join(current_dir, "integration_testing", "build_testapps.json") + with open(config_path, "r") as config: + config = json.load(config) + + if not config: + logging.info("No config found") + return 1 + + logging.info("Config found: %s", config) + + testapps = [] + for file_dir, directories, _ in os.walk(testapp_dir): + # .app is treated as a directory, not a file in MacOS + for directory in directories: + full_path = os.path.join(file_dir, directory) + if "simulator" in full_path and directory.endswith(".app"): + testapps.append((_get_bundle_id(full_path, config), full_path)) + + if not testapps: + logging.info("No testapps found") + return 1 + + logging.info("Testapps found: %s", "\n".join(path for _, path in testapps)) + + gameloop_app = _build_gameloop(gameloop_project, device_name, device_os) + if not gameloop_app: + logging.info("gameloop app not found") + return 2 + + device_id = _boot_simulator(device_name, device_os) + if not device_id: + logging.info("simulator created fail") + return 3 + + tests = [] + for bundle_id, app_path in testapps: + tests.append(Test( + testapp_path=app_path, + logs=_run_gameloop_test(bundle_id, app_path, gameloop_app, device_id))) + + return test_validation.summarize_test_results( + tests, test_validation.CPP, testapp_dir) + + +def _get_bundle_id(app_path, config): + """Get app bundle id from build_testapps.json file.""" + for api in config["apis"]: + if api["name"] != "app" and (api["name"] in app_path or api["full_name"] in app_path): + return api["bundle_id"] + + +def _build_gameloop(gameloop_project, device_name, device_os): + """Build gameloop UI Test app. + + This gameloop app can run integration_test app automatically. + """ + project_path = os.path.join(gameloop_project, "gameloop.xcodeproj") + output_path = os.path.join(gameloop_project, "Build") + + """Build the gameloop app for test.""" + args = ["xcodebuild", "-project", project_path, + "-scheme", "gameloop", + "-sdk", "iphonesimulator", + "build-for-testing", + "-destination", "platform=iOS Simulator,name=%s,OS=%s" % (device_name, device_os), + "SYMROOT=%s" % output_path] + logging.info("Running game-loop test: %s", " ".join(args)) + subprocess.run(args=args, check=True) + + for file_dir, _, file_names in os.walk(output_path): + for file_name in file_names: + if file_name.endswith(".xctestrun"): + return os.path.join(file_dir, file_name) + + +def _run_xctest(gameloop_app, device_id): + """Run the gameloop UI Test app. + This gameloop app can run integration_test app automatically. + """ + args = ["xcodebuild", "test-without-building", + "-xctestrun", gameloop_app, + "-destination", "id=%s" % device_id] + logging.info("Running game-loop test: %s", " ".join(args)) + result = subprocess.run(args=args, capture_output=True, text=True, check=False) + + if not result.stdout: + logging.info("No xctest result") + return + + result = result.stdout.splitlines() + log_path = next((line for line in result if ".xcresult" in line), None) + logging.info("game-loop xctest result: %s", log_path) + return log_path + + +def _boot_simulator(device_name, device_os): + """Create a simulator locally. Will wait until this simulator botted.""" + args = ["xcrun", "simctl", "shutdown", "all"] + logging.info("Shutdown all simulators: %s", " ".join(args)) + subprocess.run(args=args, check=True) + + command = "xcrun xctrace list devices 2>&1 | grep \"%s (%s)\" | awk -F'[()]' '{print $4}'" % (device_name, device_os) + logging.info("Get my simulator: %s", command) + result = subprocess.Popen(command, universal_newlines=True, shell=True, stdout=subprocess.PIPE) + device_id = result.stdout.read().strip() + + args = ["xcrun", "simctl", "boot", device_id] + logging.info("Boot my simulator: %s", " ".join(args)) + subprocess.run(args=args, check=True) + + args = ["xcrun", "simctl", "bootstatus", device_id] + subprocess.run(args=args, check=True) + return device_id + + +def _delete_simulator(device_id): + """Delete the created simulator.""" + args = ["xcrun", "simctl", "delete", device_id] + logging.info("Delete created simulator: %s", " ".join(args)) + subprocess.run(args=args, check=True) + + +def _run_gameloop_test(bundle_id, app_path, gameloop_app, device_id): + """Run gameloop test and collect test result.""" + logging.info("Running test: %s, %s, %s, %s", bundle_id, app_path, gameloop_app, device_id) + _install_app(app_path, device_id) + _run_xctest(gameloop_app, device_id) + logs = _get_test_log(bundle_id, app_path, device_id) + _uninstall_app(bundle_id, device_id) + return logs + + +def _install_app(app_path, device_id): + """Install integration_test app into the simulator.""" + args = ["xcrun", "simctl", "install", device_id, app_path] + logging.info("Install testapp: %s", " ".join(args)) + subprocess.run(args=args, check=True) + + +def _uninstall_app(bundle_id, device_id): + """Uninstall integration_test app from the simulator.""" + args = ["xcrun", "simctl", "uninstall", device_id, bundle_id] + logging.info("Uninstall testapp: %s", " ".join(args)) + subprocess.run(args=args, check=True) + + +def _get_test_log(bundle_id, app_path, device_id): + """Read integration_test app testing result.""" + args=["xcrun", "simctl", "get_app_container", device_id, bundle_id, "data"] + logging.info("Get test result: %s", " ".join(args)) + result = subprocess.run( + args=args, + capture_output=True, text=True, check=False) + + if not result.stdout: + logging.info("No test Result") + return None + + log_path = os.path.join(result.stdout.strip(), "Documents", "GameLoopResults", "Results1.json") + return _read_file(log_path) + + +def _read_file(path): + """Extracts the contents of a file.""" + with open(path, "r") as f: + test_result = f.read() + + logging.info("Reading file: %s", path) + logging.info("File contant: %s", test_result) + return test_result + + +if __name__ == '__main__': + flags.mark_flag_as_required("testapp_dir") + app.run(main)