From 9aa862a397fbc40dba967d0f938975e688a7dacf Mon Sep 17 00:00:00 2001 From: anonymous-akorn <66133366+anonymous-akorn@users.noreply.github.com> Date: Mon, 27 Jul 2020 09:57:11 -0700 Subject: [PATCH 1/3] Add tools for C++ integration tests Adds the tools needed to build and run C++ integration tests. These derive from the corresponding tools in google3, with some changes and exceptions: build_testapps.py and build_desktop_testapps.py have been merged into a single, simpler build_testapps.py. restore_secrets.py is a new script that decrypts the secret files in scripts/gha-encrypted and restores all the files/values that were stripped out when pushing the integration source code to Github. Except for the reverse ids in the Info.plists needed for iOS, which are not present yet. xcodebuild.py, provisioning.py and xcode_tool.rb are all for iOS, which is not yet supported until signing is figured out. This also includes a change to .gitignore to ignore temporary Python files, as well as unencrypted secrets. --- .gitignore | 11 + .../integration_testing/build_testapps.json | 232 +++++++++ testing/integration_testing/build_testapps.py | 482 ++++++++++++++++++ testing/integration_testing/config_reader.py | 183 +++++++ testing/integration_testing/export.plist | 12 + testing/integration_testing/provisioning.py | 141 +++++ testing/integration_testing/requirements.txt | 2 + .../integration_testing/restore_secrets.py | 137 +++++ testing/integration_testing/xcode_tool.rb | 188 +++++++ testing/integration_testing/xcodebuild.py | 144 ++++++ 10 files changed, 1532 insertions(+) create mode 100755 testing/integration_testing/build_testapps.json create mode 100644 testing/integration_testing/build_testapps.py create mode 100644 testing/integration_testing/config_reader.py create mode 100644 testing/integration_testing/export.plist create mode 100644 testing/integration_testing/provisioning.py create mode 100644 testing/integration_testing/requirements.txt create mode 100644 testing/integration_testing/restore_secrets.py create mode 100644 testing/integration_testing/xcode_tool.rb create mode 100644 testing/integration_testing/xcodebuild.py diff --git a/.gitignore b/.gitignore index e43b0f9889..fba3a99307 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,12 @@ .DS_Store + +# Python byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Unencrypted secret files +google-services.json +GoogleService-Info.plist +uri_prefix.txt +server_key.txt \ No newline at end of file diff --git a/testing/integration_testing/build_testapps.json b/testing/integration_testing/build_testapps.json new file mode 100755 index 0000000000..b0c6d2e52f --- /dev/null +++ b/testing/integration_testing/build_testapps.json @@ -0,0 +1,232 @@ +{ + "apis": [ + { + "name": "app", + "full_name": "FirebaseApp", + "bundle_id": "com.google.ios.analytics.testapp", + "ios_target": "integration_test", + "testapp_path": "analytics/integration_test", + "frameworks": [ + "firebase_analytics.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "admob", + "full_name": "FirebaseAdmob", + "bundle_id": "com.google.ios.admob.testapp", + "ios_target": "integration_test", + "testapp_path": "admob/integration_test", + "frameworks": [ + "firebase_admob.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "analytics", + "full_name": "FirebaseAnalytics", + "bundle_id": "com.google.ios.analytics.testapp", + "ios_target": "integration_test", + "testapp_path": "analytics/integration_test", + "frameworks": [ + "firebase_analytics.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "auth", + "full_name": "FirebaseAuth", + "bundle_id": "com.google.FirebaseCppAuthTestApp.dev", + "ios_target": "integration_test", + "testapp_path": "auth/integration_test", + "frameworks": [ + "firebase_auth.framework", + "firebase.framework" + ], + "provision": "Firebase_Cpp_Auth_Test_App_Dev.mobileprovision" + }, + { + "name": "database", + "full_name": "FirebaseDatabase", + "bundle_id": "com.google.firebase.cpp.database.testapp", + "ios_target": "integration_test", + "testapp_path": "database/integration_test", + "frameworks": [ + "firebase_auth.framework", + "firebase_database.framework", + "firebase.framework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision" + }, + { + "name": "dynamic_links", + "full_name": "FirebaseDynamicLinks", + "bundle_id": "com.google.FirebaseCppDynamicLinksTestApp.dev", + "ios_target": "integration_test", + "testapp_path": "dynamic_links/integration_test", + "frameworks": [ + "firebase_dynamic_links.framework", + "firebase.framework" + ], + "provision": "Firebase_Cpp_Dynamic_Links_Test_App_Dev.mobileprovision" + }, + { + "name": "functions", + "full_name": "FirebaseFunctions", + "bundle_id": "com.google.firebase.cpp.functions.testapp", + "ios_target": "integration_test", + "testapp_path": "functions/integration_test", + "frameworks": [ + "firebase_auth.framework", + "firebase_functions.framework", + "firebase.framework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision" + }, + { + "name": "instance_id", + "full_name": "FirebaseInstanceId", + "bundle_id": "com.google.firebase.instanceid.testapp", + "ios_target": "integration_test", + "testapp_path": "instance_id/integration_test", + "frameworks": [ + "firebase_instance_id.framework", + "firebase.framework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision" + }, + { + "name": "messaging", + "full_name": "FirebaseMessaging", + "bundle_id": "com.google.FirebaseCppMessagingTestApp.dev", + "ios_target": "integration_test", + "testapp_path": "messaging/integration_test", + "frameworks": [ + "firebase_messaging.framework", + "firebase.framework" + ], + "provision": "Firebase_Cpp_Messaging_Test_App_Dev.mobileprovision" + }, + { + "name": "remote_config", + "full_name": "FirebaseRemoteConfig", + "bundle_id": "com.google.ios.remoteconfig.testapp", + "ios_target": "integration_test", + "testapp_path": "remote_config/integration_test", + "frameworks": [ + "firebase_remote_config.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "storage", + "full_name": "FirebaseStorage", + "bundle_id": "com.google.firebase.cpp.storage.testapp", + "ios_target": "integration_test", + "testapp_path": "storage/integration_test", + "frameworks": [ + "firebase_storage.framework", + "firebase_auth.framework", + "firebase.framework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision" + }, + { + "name": "firestore", + "full_name": "FirebaseFirestore", + "bundle_id": "com.google.firebase.cpp.firestore.testapp", + "ios_target": "integration_test", + "testapp_path": "firestore/integration_test", + "frameworks": [ + "firebase_firestore.framework", + "firebase_auth.framework", + "firebase.framework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision", + "minify": "proguard" + }, + { + "name": "performance", + "full_name": "FirebasePerformance", + "bundle_id": "com.google.ios.performance.testapp", + "ios_target": "testapp", + "testapp_path": "firebase/performance/client/cpp/testapp", + "frameworks": [ + "firebase_performance.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "test_lab", + "full_name": "FirebaseTestLab", + "bundle_id": "com.google.ios.testlab.testapp", + "ios_target": "integration_test", + "testapp_path": "testlab/integration_test", + "frameworks": [ + "firebase_testlab.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + } + ], + "apple_team_id": "REPLACE_ME_TEMP_INVALID_ID", + "compiler_dict": { + "gcc-4.8": [ + "-DCMAKE_C_COMPILER=gcc-4.8", + "-DCMAKE_CXX_COMPILER=g++-4.8" + ], + "gcc-7": [ + "-DCMAKE_C_COMPILER=gcc-7", + "-DCMAKE_CXX_COMPILER=g++-7" + ], + "gcc-9": [ + "-DCMAKE_C_COMPILER=gcc-9", + "-DCMAKE_CXX_COMPILER=g++-9" + ], + "clang-5.0": [ + "-DCMAKE_C_COMPILER=clang-5.0", + "-DCMAKE_CXX_COMPILER=clang++-5.0" + ], + "VisualStudio2015": [ + "-G", + "Visual Studio 14 2015 Win64" + ], + "VisualStudio2017": [ + "-G", + "Visual Studio 15 2017 Win64" + ] + }, + "android_dep_key_mapping": { + "database": ["firebase-database"], + "dynamic_links": ["firebase-dynamic-links"], + "firestore": ["firebase-firestore"], + "functions": ["firebase-functions"], + "remote_config": ["firebase-config"], + "storage": ["firebase-storage"] + }, + "unreleased_dep_versions": [ + "def firebaseDependenciesMap = [", + "'app' : ['com.google.firebase:firebase-analytics:[0,)'],", + "'admob' : ['com.google.firebase:firebase-ads:[0,)',", + "'com.google.android.gms:play-services-measurement-sdk-api:[0,)'],", + "'analytics' : ['com.google.firebase:firebase-analytics:[0,)'],", + "'auth' : ['com.google.firebase:firebase-auth:[0,)'],", + "'database' : ['com.google.firebase:firebase-database:[0,)'],", + "'dynamic_links' : ['com.google.firebase:firebase-dynamic-links:[0,)'],", + "'firestore' : ['com.google.firebase:firebase-firestore:[0,)'],", + "'functions' : ['com.google.firebase:firebase-functions:[0,)'],", + "'instance_id' : ['com.google.firebase:firebase-iid:[0,)'],", + "'messaging' : ['com.google.firebase.messaging.cpp:firebase_messaging_cpp@aar',", + "'com.google.firebase:firebase-messaging:[0,)'],", + "'performance' : ['com.google.firebase:firebase-perf:[0,)'],", + "'remote_config' : ['com.google.firebase:firebase-config:[0,)'],", + "'storage' : ['com.google.firebase:firebase-storage:[0,)'],", + "'testlab' : []", + "]" + ] +} diff --git a/testing/integration_testing/build_testapps.py b/testing/integration_testing/build_testapps.py new file mode 100644 index 0000000000..af100c8968 --- /dev/null +++ b/testing/integration_testing/build_testapps.py @@ -0,0 +1,482 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Build automation tool for Firebase C++ testapps for desktop and mobile. + +USAGE: + +This tool has a number of dependencies (listed below). Once those are taken +care of, here is an example of an execution of the tool (on MacOS): + +python build_testapps.py --t auth,invites --p iOS + +Critical flags: +--t (full name: testapps, default: None) +--p (full name: platforms, default: None) + +Under most circumstances the other flags don't need to be set, but can be +seen by running --help. Note that all path flags will forcefully expand +the user ~. + +DEPENDENCIES: + +----Firebase SDK---- +The Firebase SDK (prebuilt) or repo must be locally present. +Path specified by the flag: + + --sdk_dir (default: current working directory), + +----Python Dependencies---- +The requirements.txt file has the required dependencies for this Python tool. + + pip install -r requirements.txt + +----CMake (Desktop only)---- +CMake must be installed and on the system path. + +----Environment Variables (Android only)---- +If building for Android, this tool requires several environment variables. +If any are missing, the tool will terminate and report the missing environment +variables. The following lists the required variables, and examples of what +a configured value may look like on MacOS: + + JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-8-latest/Contents/Home + ANDROID_HOME=/Users/user_name/Library/Android/sdk + ANDROID_SDK_HOME=/Users/user_name/Library/Android/sdk + ANDROID_NDK_HOME=/Users/user_name/Library/Android/sdk/ndk-bundle + +Or on Linux: + JAVA_HOME=/usr/local/buildtools/java/jdk/ + ANDROID_HOME=~/Android/Sdk + ANDROID_SDK_HOME=~/Android/Sdk + ANDROID_NDK_HOME=~/Android/Sdk/ndk + +If using this tool frequently, you will likely find it convenient to +modify your bashrc file to automatically set these variables. + +----Mobile Provisions (iOS only)---- +If building for iOS, the required mobile provisions must be installed and +locally present. +Path specified by the flag: + + --provision_dir (default: ~/Downloads/export) + +""" + +import datetime +from distutils import dir_util +import os +import pathlib +import platform +import shutil +import subprocess + +from absl import app +from absl import flags +from absl import logging + +import attr + +import config_reader +import provisioning +import xcodebuild + +# Platforms +_ANDROID = "Android" +_IOS = "iOS" +_DESKTOP = "Desktop" +_SUPPORTED_PLATFORMS = (_ANDROID, _IOS, _DESKTOP) + +# Values for iOS SDK flag (where the iOS app will run) +_IOS_SDK_DEVICE = "device" +_IOS_SDK_SIMULATOR = "simulator" +_IOS_SDK_BOTH = "both" +_SUPPORTED_IOS_SDK = (_IOS_SDK_DEVICE, _IOS_SDK_SIMULATOR, _IOS_SDK_BOTH) + +_REQUIRED_ANDROID_ENV_VARS = [ + "JAVA_HOME", "ANDROID_HOME", "ANDROID_SDK_HOME", "ANDROID_NDK_HOME" +] + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + "sdk_dir", os.getcwd(), "Unzipped Firebase C++ sdk OR Github repo.") + +flags.DEFINE_string( + "output_directory", "~", + "Build output will be placed in this directory.") + +flags.DEFINE_string( + "root_dir", os.getcwd(), + "Directory with which to join the relative paths in the config." + " Used to find e.g. the integration test projects. If using the SDK repo" + " this will be the same as the sdk dir, but not if using prebuilts." + " Defaults to the current directory.") + +flags.DEFINE_list( + "testapps", None, "Which testapps (Firebase APIs) to build, e.g." + " 'analytics,auth'.", + short_name="t") + +flags.DEFINE_list( + "platforms", None, "Which platforms to build. Can be Android, iOS and/or" + " Desktop", short_name="p") + +flags.DEFINE_bool( + "add_timestamp", True, + "Add a timestamp to the output directory for disambiguation." + " Recommended when running locally, so each execution gets its own " + " directory.") + +flags.DEFINE_enum( + "ios_sdk", _IOS_SDK_DEVICE, _SUPPORTED_IOS_SDK, + "(iOS only) Build for device (ipa), simulator (app), or both." + " Building for both will produce both an .app and an .ipa.") + +flags.DEFINE_bool( + "update_pod_repo", True, + "(iOS only) Will run 'pod repo update' before building for iOS to update" + " the local spec repos available on this machine. Must also include iOS" + " in platforms flag.") + +flags.DEFINE_string( + "provisions_dir", "~/Downloads/export", + "(iOS only) Directory containing the mobileprovision.") + +flags.DEFINE_bool( + "execute_desktop_testapp", True, + "(Desktop only) Run the testapp after building it. Will return non-zero" + " code if any tests fail inside the testapp.") + +flags.DEFINE_string( + "compiler", None, + "(Desktop only) Specify the compiler with CMake during the testapps build." + " Check the config file to see valid choices for this flag." + " If none, will invoke cmake without specifying a compiler.") + +flags.register_validator( + "platforms", + lambda p: all(platform in _SUPPORTED_PLATFORMS for platform in p), + message="Valid platforms: " + ",".join(_SUPPORTED_PLATFORMS), + flag_values=FLAGS) + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + platforms = FLAGS.platforms + testapps = FLAGS.testapps + + update_pod_repo = FLAGS.update_pod_repo + if FLAGS.add_timestamp: + timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") + else: + timestamp = "" + + if update_pod_repo and _IOS in platforms: + _run(["pod", "repo", "update"]) + + failures = [] + for testapp in testapps: + logging.info("BEGIN building for %s", testapp) + failures += _build( + testapp=testapp, + platforms=platforms, + config=config_reader.read_config(), + output_dir=os.path.expanduser(FLAGS.output_directory), + sdk_dir=os.path.expanduser(FLAGS.sdk_dir), + timestamp=timestamp, + builder_dir=pathlib.Path(__file__).parent.absolute(), + root_dir=os.path.expanduser(FLAGS.root_dir), + provisions_dir=os.path.expanduser(FLAGS.provisions_dir), + ios_sdk=FLAGS.ios_sdk, + desktop_compiler=FLAGS.compiler, + execute_desktop_testapp=FLAGS.execute_desktop_testapp) + logging.info("END building for %s", testapp) + + _summarize_results(testapps, platforms, failures) + return 1 if failures else 0 + + +def _build( + testapp, platforms, config, output_dir, sdk_dir, timestamp, builder_dir, + root_dir, provisions_dir, ios_sdk, desktop_compiler, + execute_desktop_testapp): + """Builds one testapp on each of the specified platforms.""" + api_config = APIConfig.from_config(config, testapp) + testapp_dir = os.path.join(root_dir, api_config.api_dir) + project_dir = os.path.join( + output_dir, "testapps" + timestamp, api_config.full_name, + os.path.basename(testapp_dir)) + + logging.info("Copying testapp project to %s", project_dir) + os.makedirs(project_dir) + dir_util.copy_tree(testapp_dir, project_dir) + + logging.info("Changing directory to %s", project_dir) + os.chdir(project_dir) + + _run_setup_script(sdk_dir, project_dir) + + failures = [] + + if _DESKTOP in platforms: + logging.info("BEGIN %s, %s", testapp, _DESKTOP) + try: + compiler_flags = _get_desktop_compiler_flags(desktop_compiler, config) + _build_desktop(sdk_dir, compiler_flags) + if execute_desktop_testapp: + _execute_desktop_testapp(project_dir) + except subprocess.CalledProcessError as e: + failures.append( + Failure(testapp=testapp, platform=_DESKTOP, error_message=str(e))) + logging.info("END %s, %s", testapp, _DESKTOP) + + if _ANDROID in platforms: + logging.info("BEGIN %s, %s", testapp, _ANDROID) + try: + _validate_android_environment_variables() + _build_android(project_dir, sdk_dir) + except subprocess.CalledProcessError as e: + failures.append( + Failure(testapp=testapp, platform=_ANDROID, error_message=str(e))) + logging.info("END %s, %s", testapp, _ANDROID) + + if _IOS in platforms: + logging.info("BEGIN %s, %s", testapp, _IOS) + try: + _build_ios( + sdk_dir=sdk_dir, + project_dir=project_dir, + testapp_builder_dir=builder_dir, + provisions_dir=provisions_dir, + api_config=api_config, + ios_sdk=ios_sdk, + dev_team=config_reader.read_general_config(config, "apple_team_id")) + except subprocess.CalledProcessError as e: + failures.append( + Failure(testapp=testapp, platform=_IOS, error_message=str(e))) + logging.info("END %s, %s", testapp, _IOS) + + logging.info("END building for API: %s", testapp) + return failures + + +def _summarize_results(testapps, platforms, failures): + """Logs a readable summary of the results of the build.""" + logging.info( + "FINISHED BUILDING TESTAPPS.\n\n\n" + "Tried to build these testapps: %s\n" + "On these platforms: %s", + ", ".join(testapps), ", ".join(platforms)) + if not failures: + logging.info("No failures occurred") + else: + # Collect lines, then log once, to reduce logging noise from timestamps etc. + lines = ["Some failures occurred:"] + for i, failure in enumerate(failures, start=1): + lines.append("%d: %s" % (i, failure.describe())) + logging.info("\n".join(lines)) + + +def _build_desktop(sdk_dir, compiler_flags): + _run(["cmake", ".", "-DFIREBASE_CPP_SDK_DIR=" + sdk_dir] + compiler_flags) + _run(["cmake", "--build", "."]) + + +def _execute_desktop_testapp(project_dir): + if platform.system() == "Windows": + testapp_path = os.path.join(project_dir, "Debug", "integration_test.exe") + else: + testapp_path = os.path.join(project_dir, "integration_test") + _run([testapp_path]) + + +def _get_desktop_compiler_flags(compiler, config): + """Returns the command line flags for this compiler.""" + if not compiler: # None is an acceptable default value + return [] + compilers = config_reader.read_general_config(config, "compiler_dict") + try: + return compilers[compiler] + except KeyError: + valid_keys = ", ".join(compilers.keys()) + raise ValueError( + "Given compiler: %s. Valid compilers: %s" % (compiler, valid_keys)) + + +def _build_android(project_dir, sdk_dir): + """Builds an Android binary (apk).""" + logging.info("Patching gradle properties with path to SDK") + gradle_properties = os.path.join(project_dir, "gradle.properties") + with open(gradle_properties, "a+") as f: + f.write("systemProp.firebase_cpp_sdk.dir=" + sdk_dir + "\n") + # This will log the versions of dependencies for debugging purposes. + _run( + ["./gradlew", "dependencies", "--configuration", "debugCompileClasspath"]) + _run(["./gradlew", "assembleDebug", "--stacktrace"]) + + +def _validate_android_environment_variables(): + """Raises an error if any environment variables for Android are missing.""" + # These are environment variables that must be set for gradle. + missing_env_vars = [ + var for var in _REQUIRED_ANDROID_ENV_VARS if not os.environ.get(var) + ] + if missing_env_vars: + raise ValueError( + "Missing required environment variable(s): " + + ", ".join(missing_env_vars)) + + +def _build_ios( + sdk_dir, project_dir, testapp_builder_dir, provisions_dir, api_config, + ios_sdk, dev_team): + """Builds an iOS application (.app, .ipa or both).""" + build_dir = os.path.join(project_dir, "ios_build") + os.makedirs(build_dir) + + logging.info("Copying XCode frameworks") + framework_src_dir = os.path.join(sdk_dir, "frameworks", "ios", "universal") + framework_paths = [] # Paths to the copied frameworks. + for framework in api_config.frameworks: + framework_src_path = os.path.join(framework_src_dir, framework) + framework_dest_path = os.path.join(project_dir, "Frameworks", framework) + dir_util.copy_tree(framework_src_path, framework_dest_path) + framework_paths.append(framework_dest_path) + + _run(["pod", "install"]) + + entitlements_path = os.path.join( + project_dir, api_config.ios_target + ".entitlements") + xcode_patcher_args = [ + "ruby", os.path.join(testapp_builder_dir, "xcode_tool.rb"), + "--XCodeCPP.xcodeProjectDir", project_dir, + "--XCodeCPP.target", api_config.ios_target, + "--XCodeCPP.frameworks", ",".join(framework_paths) + ] + if os.path.isfile(entitlements_path): # Not all testapps require entitlements + logging.info("Entitlements file detected.") + xcode_patcher_args.extend(("--XCodeCPP.entitlement", entitlements_path)) + else: + logging.info("No entitlements found at %s.", entitlements_path) + _run(xcode_patcher_args) + + xcode_path = os.path.join(project_dir, api_config.ios_target + ".xcworkspace") + if ios_sdk in [_IOS_SDK_SIMULATOR, _IOS_SDK_BOTH]: + _run( + xcodebuild.get_args_for_build( + path=xcode_path, + scheme=api_config.scheme, + output_dir=build_dir, + ios_sdk=_IOS_SDK_SIMULATOR, + dev_team=dev_team, + configuration="Debug")) + + if ios_sdk in [_IOS_SDK_DEVICE, _IOS_SDK_BOTH]: + logging.info("Creating 'export.plist' export options") + provision_id = provisioning.get_provision_id( + os.path.join(provisions_dir, api_config.provision)) + export_src = os.path.join(testapp_builder_dir, "export.plist") + export_dest = os.path.join(build_dir, "export.plist") + shutil.copy(export_src, export_dest) + provisioning.patch_provisioning_profile( + export_dest, api_config.bundle_id, provision_id) + + archive_path = os.path.join(build_dir, "app.xcarchive") + _run( + xcodebuild.get_args_for_archive( + path=xcode_path, + scheme=api_config.ios_target, + uuid=provision_id, + output_dir=build_dir, + archive_path=archive_path, + ios_sdk=_IOS_SDK_DEVICE, + dev_team=dev_team, + configuration="Debug")) + _run( + xcodebuild.get_args_for_export( + output_dir=build_dir, + archive_path=archive_path, + plist_path=export_dest)) + + +# This script is responsible for copying shared files into the integration +# test projects. Should be executed before performing any builds. +def _run_setup_script(sdk_dir, testapp_dir): + """Runs the setup_integration_tests.py script if needed.""" + script_path = os.path.join(sdk_dir, "setup_integration_tests.py") + if os.path.isfile(script_path): + _run(["python", script_path, testapp_dir]) + else: + logging.info("setup_integration_tests.py not found") + + +def _run(args, timeout=1200): + """Executes a command in a subprocess.""" + logging.info("Running in subprocess: %s", " ".join(args)) + return subprocess.run(args=args, timeout=timeout, check=True) + + +@attr.s(frozen=True, eq=False) +class Failure(object): + """Holds context for the failure of a testapp to build/run.""" + testapp = attr.ib() + platform = attr.ib() + error_message = attr.ib() + + def describe(self): + return "%s, %s: %s" % (self.testapp, self.platform, self.error_message) + + +@attr.s(frozen=True, eq=False) +class APIConfig(object): + """Holds all the configuration for a single testapp project.""" + name = attr.ib() + full_name = attr.ib() + bundle_id = attr.ib() + ios_target = attr.ib() + api_dir = attr.ib() # Integration test dir relative to sdk root + frameworks = attr.ib() # Required custom xcode frameworks + provision = attr.ib() # Path to the local mobile provision + + @classmethod + def from_config(cls, config, testapp): + """Builds the APIConfig for this testapp. + + Args: + config: The full configuration dictionary for the testapp builder. + testapp: Short name for the testapp, e.g. 'analytics'. + + Returns: + API-specific configuration for this testapp. + + """ + + if testapp not in config_reader.get_api_names(config): + raise ValueError("Invalid api name (not found in config): " + testapp) + + return cls( + name=testapp, + full_name=config_reader.read_api_config(config, testapp, "full_name"), + bundle_id=config_reader.read_api_config(config, testapp, "bundle_id"), + ios_target=config_reader.read_api_config(config, testapp, "ios_target"), + api_dir=config_reader.read_api_config(config, testapp, "testapp_path"), + frameworks=config_reader.read_api_config(config, testapp, "frameworks"), + provision=config_reader.read_api_config(config, testapp, "provision")) + + +if __name__ == "__main__": + app.run(main) diff --git a/testing/integration_testing/config_reader.py b/testing/integration_testing/config_reader.py new file mode 100644 index 0000000000..074e33a969 --- /dev/null +++ b/testing/integration_testing/config_reader.py @@ -0,0 +1,183 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A utility for working with testapp builder JSON files. + +This module handles loading the central configuration file for a testapp +builder, as well as offering simplified access to the loaded in-memory +object. + +The main motivation for this module is to provide descriptive error messages, +rather than an unhelpful KeyError that one would get from working with the +loaded JSON (a dictionary) directly. Instead, errors will dump a readable form +of the section of JSON being read, alongside the error. + +Example of such a configuration file: + +{ + "apis": [ + { + "name": "analytics", + "full_name": "FirebaseAnalytics", + "bundle_id": "com.google.ios.analytics.testapp", + "ios_target": "integration_test", + "testapp_path": "firebase/analytics/client/cpp/integration_test", + "frameworks": [ + "firebase_analytics.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "admob", + "full_name": "FirebaseAdmob", + "bundle_id": "com.google.ios.admob.testapp", + "ios_target": "integration_test", + "testapp_path": "firebase/admob/client/cpp/integration_test", + "frameworks": [ + "firebase_admob.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + } + ], + "dev_team": "ABCDEFGHIJK" +} + +Available methods: + + read_config reads the config into memory. + read_general_config provides access to the root level config + read_api_config provides access to an API's config, and requires its name. + This must occur in a root level "apis" list of objects. + get_api_names returns a sequence of the names of configured APIs, for use + with read_api_config. + +""" + +import json +import os +import pathlib + +# List of api-specific configurations. +_APIS_KEY = "apis" + +_NAME_KEY = "name" + +# These are used to contextualize where a value is being read within the config +# to provide better error messages. +_ROOT_CONFIG = "root config " +_API_CONFIG = "config in 'apis' list" + + +def read_config(path=None): + """Creates an in-memory config object out of a testapp config file. + + Args: + path: Path to a testapp builder config file. If not specified, will look + for 'build_testapps.json' in the same directory as this file. + + Returns: + An in-memory config object that can be used with the other methods in + this module to extract data from the config passed in. + + """ + if not path: + directory = pathlib.Path(__file__).parent.absolute() + path = os.path.join(directory, "build_testapps.json") + with open(path, "r") as config: + return json.load(config) + + +def read_general_config(config, key, optional=False): + """Reads a configuration value not tied to a particular API. + + Args: + config: Configuration created by read_config. + key: Key whose value is being read. + optional: Return a None if the key is not found. + + Returns: + Value associated to the given key. Could be any type supported by + Python's JSON module. + + """ + return _get_from_json(config, key, _ROOT_CONFIG, optional) + + +def read_api_config(config, api_name, key, optional=False): + """Reads a configuration value for a particular API. + + Args: + config: Configuration created by read_config. + api_name: Name of the API whose configuration is being read. + key: Key whose value is being read. + optional: Return a None if the key is not found. + + Returns: + Value associated to the given key. Could be any type supported by + Python's JSON module. + + """ + api_dicts = _get_from_json(config, _APIS_KEY, _ROOT_CONFIG) + api_dict = [api for api in api_dicts if api[_NAME_KEY] == api_name][0] + return _get_from_json(api_dict, key, _API_CONFIG, optional) + + +def get_api_names(config): + """Returns a list of all the APIs present in this config.""" + apis = _get_from_json(config, _APIS_KEY, _ROOT_CONFIG) + return [_get_from_json(api, _NAME_KEY, _API_CONFIG) for api in apis] + + +def _get_from_json(json_dict, key, json_description, optional=False): + """Attempts to retrieve key from the json dictionary. + + If the key is not found and is not marked as optional, an error + will be raised with detailed information including the key, a + pretty-printed dump of the given json, and a description of the json + for context. + + Args: + json_dict: A dictionary representation of a JSON object. + key: The key whose value is being extracted from the JSON dictionary. + json_description: A description of the json object to clarify where + the object came from. Will be included in an error message, if the key + is not found in the dict. Especially important for nested objects. + e.g. "API object from 'APIS' list in main config". + optional: Return a None if the key is not found. + + Raises: + ValueError: Key is None or empty, json_dict is not a dictionary. + RuntimeError: Non-optional Key not found in the dictionary. + + Returns: + The value of the key in the dictionary, equivalent to json_dict[key]. + + """ + if not key: + raise ValueError("Must supply a valid, non-empty key.") + try: + return json_dict[key] + except TypeError: # json_dict is not a dict + raise ValueError( + "Expected dictionary (JSON object) from %s, received %s: %s instead." % + (json_description, type(json_dict), json_dict)) + except KeyError: # key not found + if optional: + return None + formatted_json = json.dumps( + json_dict, sort_keys=True, indent=4, separators=(",", ":")) + raise RuntimeError( + "%s not found in %s:\n%s" % (key, json_description, formatted_json)) diff --git a/testing/integration_testing/export.plist b/testing/integration_testing/export.plist new file mode 100644 index 0000000000..2f7617ea62 --- /dev/null +++ b/testing/integration_testing/export.plist @@ -0,0 +1,12 @@ + + + + + compileBitcode + + stripSwiftSymbols + + provisioningProfiles + + + diff --git a/testing/integration_testing/provisioning.py b/testing/integration_testing/provisioning.py new file mode 100644 index 0000000000..0aecd1710f --- /dev/null +++ b/testing/integration_testing/provisioning.py @@ -0,0 +1,141 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper module for dealing with mobile provisioning profiles. + +This module provides functionality to extract a UUID from a mobile provisioning +file and patch it into an exports plist. The motivation for this arose when it +was discovered that the UUID changes each time the profile is updated. Since the +profile was pulled from a source where it was updated daily, manually specifying +the UUID in the exports plist was not feasible. + +The UUID is needed to tell xcodebuild which provisioning profile to use. + +This should only be run on MacOS, as it uses a mac-specific +application. + +""" + +import plistlib +import subprocess + + +def patch_provisioning_profile(plist_path, bundle_id, uuid): + """Patches the plist with a provisioning profile. + + Will add an entry to the provisioning profiles dictionary in the provided + plist. The entry will be bundle_id:uuid, associating a project with a + provisioning profile. This is needed for exporting an xcode archive using + xcodebuild. + + If an entry with the same bundle id already exists, its value will be + overwritten. + + Args: + plist_path: Path to a valid plist with a provisioningProfiles dictionary + in its root. + bundle_id: String representing a bundle id. + uuid: String representing the uuid of a provisioning profile. + + Raises: + ValueError: plist not in correct format. + + """ + # Need to explicitly specify binary format for load to work. + with open(plist_path, "rb") as f_read: + plist = plistlib.load(f_read) + _patch_provisioning_profile(plist, bundle_id, uuid) + with open(plist_path, "wb") as f_write: + plistlib.dump(plist, f_write) + + +def _patch_provisioning_profile(plist, bundle_id, uuid): + """Patches the plist dictionary with bundle_id:uuid.""" + try: + profiles = plist["provisioningProfiles"] + profiles[bundle_id] = uuid + except (TypeError, KeyError): + raise ValueError( + "Plist not in correct format. 'provisioningProfiles' misplaced?") + + +def get_provision_id(provision_path): + """Extracts the provisioning UUID in the provided provisioning profile. + + Can only be run on macOS. + + Args: + provision_path: Path to a .mobileprovision file. + + Returns: + A string corresponding to a UUID. + + Raises: + ValueError: If provision_path is empty or None, or isn't a mobileprovision. + RuntimeError: Called this script on anything but a Mac. + + """ + if not provision_path: + raise ValueError("Must provide valid directory for provisions") + + if not provision_path.endswith(".mobileprovision"): + raise ValueError("Path is not a .mobileprovision file: %s" % provision_path) + + plist = _extract_plist(provision_path) + uuid = _extract_uuid(plist) + if not uuid: + raise RuntimeError("No UUID found in provision %s." % provision_path) + return uuid + + +def _extract_plist(provision_path): + """Extract a plist as a string from the specified mobile provision. + + Mobile provisions are encoded files with an embedded plist. This method + will attempt to decode the file and return the plist as a string. + + Args: + provision_path: Path to a mobileprovisioning profile (.mobileprovision). + + Returns: + Bytes corresponding to a valid plist. + + Raises: + subprocess.CalledProcessError: Failed to decode the given file. + + """ + # -D specifies decode, -i specifies input. + return subprocess.check_output( + ["security", "cms", "-D", "-i", provision_path]) + + +def _extract_uuid(plist): + """Extract the UUID from the plist. + + Will extract the value of the UUID key from the mobile provisioning plist. + + Args: + plist: Plist bytes read from a mobileprovisioning file. + + Returns: + A string corresponding to a provisioning UUID. + + """ + # In Python 3, readPlistFromString is removed in favor of loads, which is not + # present in Python 2. + try: + root = plistlib.readPlistFromString(plist) # First try Python 2 method. + except AttributeError: + root = plistlib.loads(plist) # If nonexistent, try Python 3 method. + return root.get("UUID", None) diff --git a/testing/integration_testing/requirements.txt b/testing/integration_testing/requirements.txt new file mode 100644 index 0000000000..d265d0774c --- /dev/null +++ b/testing/integration_testing/requirements.txt @@ -0,0 +1,2 @@ +absl-py +attrs diff --git a/testing/integration_testing/restore_secrets.py b/testing/integration_testing/restore_secrets.py new file mode 100644 index 0000000000..e97112d13a --- /dev/null +++ b/testing/integration_testing/restore_secrets.py @@ -0,0 +1,137 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script for restoring secrets into the integration test projects. + +Usage: + +python restore_secrets.py --passphrase --repo_dir + +repo_dir refers to the C++ SDK github repository. Defaults to current directory. + +As an alternative to passing the passphrase as a flag, the password can be +stored in a file and passed via the --passphrase_file flag. + +This will perform the following: + +- Google Service files (plist and json) will be restored into the + integration_test directories. +- The server key will be patched into the Messaging project. +- The uri prefix will be patched into the Dynamic Links project. + +""" + +import argparse +import os +import subprocess + +parser = argparse.ArgumentParser( + description="Decrypt and restore the secrets. Must specify one of" + " passphrase or passphrase_file.") +parser.add_argument("--passphrase", help="The passphrase itself.") +parser.add_argument("--passphrase_file", help="Path to file with passphrase.") +parser.add_argument("--repo_dir", default=os.getcwd(), help="Path to SDK Repo") +args = parser.parse_args() + + +def main(): + repo_dir = args.repo_dir + # The passphrase is sensitive, do not log. + if args.passphrase: + passphrase = args.passphrase + elif args.passphrase_file: + with open(args.passphrase_file, "r") as f: + passphrase = f.read() + else: + raise ValueError("Must supply passphrase or passphrase_file arg.") + + secrets_dir = os.path.join(repo_dir, "scripts", "gha-encrypted") + encrypted_files = _find_encrypted_files(secrets_dir) + print("Found these encrypted files:\n%s" % "\n".join(encrypted_files)) + + for path in encrypted_files: + if "google-services" in path or "GoogleService" in path: + print("Encrypted Google Service file found: %s" % path) + # We infer the destination from the file's directory, example: + # /scripts/gha-encrypted/auth/google-services.json.gpg turns into + # //auth/integration_test/google-services.json + api = os.path.basename(os.path.dirname(path)) + file_name = os.path.basename(path).replace(".gpg", "") + dest_path = os.path.join(repo_dir, api, "integration_test", file_name) + decrypted_text = _decrypt(path, passphrase) + with open(dest_path, "w") as f: + f.write(decrypted_text) + print("Copied decrypted google service file to %s" % dest_path) + + print("Attempting to patch Dynamic Links uri prefix.") + uri_path = os.path.join(secrets_dir, "dynamic_links", "uri_prefix.txt.gpg") + uri_prefix = _decrypt(uri_path, passphrase) + dlinks_project = os.path.join(repo_dir, "dynamic_links", "integration_test") + _patch_main_src(dlinks_project, "REPLACE_WITH_YOUR_URI_PREFIX", uri_prefix) + + print("Attempting to patch Messaging server key.") + server_key_path = os.path.join(secrets_dir, "messaging", "server_key.txt.gpg") + server_key = _decrypt(server_key_path, passphrase) + messaging_project = os.path.join(repo_dir, "messaging", "integration_test") + _patch_main_src(messaging_project, "REPLACE_WITH_YOUR_SERVER_KEY", server_key) + + +def _find_encrypted_files(directory_to_search): + """Returns a list of full paths to all files encrypted with gpg.""" + encrypted_files = [] + for prefix, _, files in os.walk(directory_to_search): + for relative_path in files: + if relative_path.endswith(".gpg"): + encrypted_files.append(os.path.join(prefix, relative_path)) + return encrypted_files + + +def _decrypt(encrypted_file, passphrase): + """Generates a decrypted file with same path minus the '.gpg' extension.""" + print("Decrypting %s" % encrypted_file) + # Note: if setting check=True, be sure to catch the error and not rethrow it + # or print a traceback, as the message will include the passphrase. + result = subprocess.run( + args=[ + "gpg", + "--passphrase", passphrase, + "--quiet", + "--batch", + "--yes", + "--decrypt", + encrypted_file], + check=False, + text=True, + capture_output=True) + if result.returncode: + raise RuntimeError("ERROR: Failed to decrypt %s" % (encrypted_file)) + print("Decryption successful") + return result.stdout + + +def _patch_main_src(project_dir, placeholder, value): + """Patches the integration_test.cc file in the integration test project.""" + path = os.path.join(project_dir, "src", "integration_test.cc") + with open(path, "r") as f_read: + text = f_read.read() + # Count number of times placeholder appears for debugging purposes. + replacements = text.count(placeholder) + patched_text = text.replace(placeholder, value) + with open(path, "w") as f_write: + f_write.write(patched_text) + print("Patched %d instances of %s in %s " % (replacements, placeholder, path)) + + +if __name__ == "__main__": + main() diff --git a/testing/integration_testing/xcode_tool.rb b/testing/integration_testing/xcode_tool.rb new file mode 100644 index 0000000000..06aa1f9782 --- /dev/null +++ b/testing/integration_testing/xcode_tool.rb @@ -0,0 +1,188 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Configures C++ testapp XCode projects. +# +# This automates the configuration of XCode projects as outlined in the +# associated testapp readme files. This includes adding frameworks, enabling +# capabilities, and patching files with required data. +# See the C++ testapp readmes for details on these changes. +# +# Usage: xcode_tool.rb [options] +# -d [dir], XCode project directory (required) +# --XCodeCPP.xcodeProjectDir +# -t [target], Assume target to match the .xcodeproj name. (required) +# --XCodeCPP.target +# -f [frameworks], Paths to the custom frameworks. (required) +# --XCodeCPP.frameworks +# -e [entitlement_path], Path to entitlements (optional) +# --XCodeCPP.entitlement + +require 'optparse' +require 'xcodeproj' + +# Performs all the product-specific changes to the xcode project provided by +# the xcodeProjectDir flag. +def main + OptionParser.new do |opts| + opts.banner = 'Usage: xcode_tool.rb [options]' + opts.on('-d', '--XCodeCPP.xcodeProjectDir [dir]', + 'XCode project directory (required)') do |xcode_project_dir| + @xcode_project_dir = xcode_project_dir + end + opts.on('-t', '--XCodeCPP.target [target]', + 'Assume target to match the .xcodeproj name. (required)') do |target| + @target_name = target + end + opts.on('-f', '--XCodeCPP.frameworks [frameworks]', + 'Paths to the custom frameworks. (required)') do |frameworks| + @frameworks = frameworks.split(",") + end + opts.on('-e', '--XCodeCPP.entitlement [entitlement_path]', + 'Path to entitlements (optional)') do |entitlement_path| + @entitlement_path = entitlement_path + end + end.parse! + + raise OptionParser::MissingArgument,'-d' if @xcode_project_dir.nil? + raise OptionParser::MissingArgument,'-t' if @target_name.nil? + raise OptionParser::MissingArgument,'-f' if @frameworks.nil? + + project_path = "#@xcode_project_dir/#@target_name.xcodeproj" + @project = Xcodeproj::Project.open(project_path) + @target = @project.targets.first + + # Examine components rather than substrings to minimize false positives. + path_components = project_path.split('/') + if path_components.include?('FirebaseAuth') + make_changes_for_auth + elsif path_components.include?('FirebaseMessaging') + make_changes_for_messaging + elsif path_components.include?('FirebaseDynamicLinks') + make_changes_for_dynamiclinks + end + + framework_dir = "#@xcode_project_dir/Frameworks" + set_build_setting('FRAMEWORK_SEARCH_PATHS', ['${inherited}', framework_dir]) + + @frameworks.each do |framework| + add_custom_framework(framework) + end + + # Bitcode is unnecessary, as we are not submitting these to the Apple store. + # Disabling bitcode significantly speeds up builds. + set_build_setting('ENABLE_BITCODE', 'NO') + + @project.save +end + +def make_changes_for_auth + puts 'Auth testapp detected.' + add_entitlements + set_reverse_id + add_system_framework('UserNotifications') + puts 'Finished making auth-specific changes.' +end + +def make_changes_for_messaging + puts 'Messaging testapp detected.' + add_entitlements + set_reverse_id + add_system_framework('UserNotifications') + enable_romote_notification + puts 'Finished making messaging-specific changes.' +end + +def make_changes_for_dynamiclinks + puts 'Dynamic Links testapp detected.' + add_entitlements + puts 'Finished making Dynamic Links-specific changes.' +end + +def add_entitlements + raise OptionParser::MissingArgument,'-e' if @entitlement_path.nil? + + puts "Adding entitlement: #@entitlement_path" + entitlement_name = File.basename(@entitlement_path) + relative_entitlement_destination = "#@xcode_project_dir/#{entitlement_name}" + @project.new_file(relative_entitlement_destination) + set_build_setting('CODE_SIGN_ENTITLEMENTS', relative_entitlement_destination) + puts 'Added entitlement to xcode project.' +end + +# Configures the reverse client id in the xcode project. +# Needed for Google sign-in. +# Some testapps may have the REVERSED_CLIENT_ID already set, in which case +# this won't do anything. e.g. Auth +def set_reverse_id + puts 'Setting the Reverse Id...' + google_service_path = "#@xcode_project_dir/GoogleService-Info.plist" + google_plist = Xcodeproj::Plist.read_from_path(google_service_path) + reverse_id = google_plist['REVERSED_CLIENT_ID'] + puts "Found reverse id: #{reverse_id}" + info_plist_path = "#@xcode_project_dir/Info.plist" + info_plist_text = File.read(info_plist_path) + info_plist_text = info_plist_text.gsub('YOUR_REVERSED_CLIENT_ID', reverse_id) + File.open(info_plist_path, "w") {|file| file.puts info_plist_text} + puts "Finished setting the Reverse Id." +end + +def enable_romote_notification + puts 'Adding remote-notification to UIBackgroundModes...' + info_plist_path = "#@xcode_project_dir/Info.plist" + info_plist = Xcodeproj::Plist.read_from_path(info_plist_path) + url_types = {'UIBackgroundModes' => ['remote-notification']} + Xcodeproj::Plist.write_to_path(info_plist.merge(url_types), info_plist_path) + puts 'Finished adding remote-notification.' +end + +# A system framework is a framework included with MacOS. +# +# Args: +# - framework: string +# +def add_system_framework(framework) + puts "Adding framework to xcode project: #{framework}." + @target.add_system_framework(framework); + puts 'Finished adding framework.' +end + +# A custom framework is not included with MacOS, e.g. firebase_auth.framework. +# +# Args: +# - framework_path: string +# +def add_custom_framework(framework_path) + puts "Adding framework to xcode project: #{framework_path}." + framework_name = File.basename(framework_path); + local_framework_path = "Frameworks/#{framework_name}" + # Add the lib file as a reference + libRef = @project['Frameworks'].new_file(framework_path) + # Get the build phase + framework_buildphase = @project.objects.select{|x| x.class == Xcodeproj::Project::Object::PBXFrameworksBuildPhase}[0] + # Add it to the build phase + framework_buildphase.add_file_reference(libRef) + puts 'Finished adding framework.' +end + +def set_build_setting(key, value) + @target.build_configurations.each do |config| + config.build_settings[key] = value + end +end + +if __FILE__ == $0 + main() +end diff --git a/testing/integration_testing/xcodebuild.py b/testing/integration_testing/xcodebuild.py new file mode 100644 index 0000000000..135087eee6 --- /dev/null +++ b/testing/integration_testing/xcodebuild.py @@ -0,0 +1,144 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper module for working with xcode projects. + +The tool xcodebuild provides support to build xcode projects from the command +line. This module provides templates for building, archiving and exporting. The +motivation was to simplify usage of xcodebuild, since it was non-trivial to +figure out which flags were needed to get it working in a CI environment. +The options required by the methods in this module were found to work both +locally and on CI, with both the Unity and C++ projects. + +Note that instead of performing operations with xcodebuild directly, this module +returns arg sequences. These sequences can be passed to e.g. subprocess.run to +execute the operations. + +The methods support either device or simulator builds. For simulator +builds, it suffices to use gets_args_for_build to create a .app that can be +used with simulators. For device builds, it's necessary to use both a +get_args_for_archive step and a get_args_for_export step. The archive step +generates an .xcarchive, and the export step 'exports' this archive to create +an .ipa that can be installed onto devices. + +""" + + +def get_args_for_archive( + path, scheme, uuid, output_dir, archive_path, ios_sdk, dev_team, + configuration): + """Subprocess args for an iOS archive. Necessary step for creating an .ipa. + + Args: + path: Full path to the project or workspace to build. Must end in + either .xcodeproj or .xcworkspace. + scheme: Name of the scheme to build. + uuid: The mobileprovision's uuid. + output_dir: Directory for the resulting build artifacts. Will be created + if it doesn't already exist. + archive_path: Archive will be created at this path. + ios_sdk: Where this build will be run: device or simulator. + dev_team: Apple development team id. + configuration: Value for the -configuration flag. If building for Unity, + Unity will by default generate Debug, Release, ReleaseForRunning, and + ReleaseForProfiling configurations. + + Returns: + Sequence of strings, corresponding to valid args for a subprocess call. + """ + xcode_args = get_args_for_build( + path, scheme, output_dir, ios_sdk, dev_team, configuration) + additional_device_args = [ + "PROVISIONING_PROFILE=" + uuid, + "CODE_SIGN_STYLE=Manual", + "-archivePath", archive_path, + "archive" + ] + return xcode_args + additional_device_args + + +def get_args_for_build( + path, scheme, output_dir, ios_sdk, dev_team, configuration): + """Constructs subprocess args for an xcode build. + + Args: + path: Full path to the project or workspace to build. Must end in + either .xcodeproj or .xcworkspace. + scheme: Name of the scheme to build. + output_dir: Directory for the resulting build artifacts. Will be created + if it doesn't already exist. + ios_sdk: Where this build will be run: device or simulator. + dev_team: Apple development team id. + configuration: Value for the -configuration flag. + + Returns: + Sequence of strings, corresponding to valid args for a subprocess call. + + """ + args = [ + "xcodebuild", + "-sdk", _get_ios_env_from_target(ios_sdk), + "-scheme", scheme, + "-configuration", configuration, + "-quiet", + "DEVELOPMENT_TEAM=" + dev_team, + "BUILD_DIR=" + output_dir + ] + + if path is None: + raise ValueError("Must supply a path.") + if path.endswith(".xcworkspace"): + args.extend(("-workspace", path)) + elif path.endswith(".xcodeproj"): + args.extend(("-project", path)) + else: + raise ValueError("Path must end with .xcworkspace or .xcodeproj: %s" % path) + return args + + +def get_args_for_export(output_dir, archive_path, plist_path): + """Subprocess args to export an xcode archive, creating an ipa. + + Must have already performed an xcodebuild archive step before executing + the command given by the returned args. + + Args: + output_dir: Directory to contain the exported artifact. + archive_path: Path to the xcode archive (.xcarchive) to be exported. + plist_path: Path for the export plist. The export plist + contains configuration for the export process, and must be present + even if the file is vacuous (contains no configuration). + + Returns: + Sequence of strings, corresponding to valid args for a subprocess call. + + """ + return [ + "xcodebuild", + "-archivePath", archive_path, + "-exportOptionsPlist", plist_path, + "-exportPath", output_dir, + "-exportArchive", + "-quiet" + ] + + +def _get_ios_env_from_target(ios_sdk): + """Return a value for the -sdk flag based on the target (device/simulator).""" + if ios_sdk == "device": + return "iphoneos" + elif ios_sdk == "simulator": + return "iphonesimulator" + else: + raise ValueError("Unrecognized ios_sdk: %s" % ios_sdk) From 02d23c8d93f6381f417c78a4b6389d318584472b Mon Sep 17 00:00:00 2001 From: anonymous-akorn <66133366+anonymous-akorn@users.noreply.github.com> Date: Tue, 28 Jul 2020 13:16:43 -0700 Subject: [PATCH 2/3] Restore reverse id and simplify config reader. The following changes have been made: - config_reader.py has been simplified to just return a plain data object containing all the needed config upfront. Thus it's only needed to call one function from the module once. - Added types to args in docstrings. - restore_secrets now restores the reverse id in Info.plists, using the value from the decrypted GoogleService-Info.plist as the source of truth. --- testing/integration_testing/build_testapps.py | 63 ++----- testing/integration_testing/config_reader.py | 174 +++++++----------- testing/integration_testing/provisioning.py | 14 +- .../integration_testing/restore_secrets.py | 24 +++ testing/integration_testing/xcodebuild.py | 42 ++--- 5 files changed, 127 insertions(+), 190 deletions(-) diff --git a/testing/integration_testing/build_testapps.py b/testing/integration_testing/build_testapps.py index af100c8968..0ff0052840 100644 --- a/testing/integration_testing/build_testapps.py +++ b/testing/integration_testing/build_testapps.py @@ -188,13 +188,15 @@ def main(argv): if update_pod_repo and _IOS in platforms: _run(["pod", "repo", "update"]) + config = config_reader.read_config() + compiler_flags = _get_desktop_compiler_flags(FLAGS.compiler, config.compilers) failures = [] for testapp in testapps: logging.info("BEGIN building for %s", testapp) failures += _build( testapp=testapp, platforms=platforms, - config=config_reader.read_config(), + api_config=config.get_api(testapp), output_dir=os.path.expanduser(FLAGS.output_directory), sdk_dir=os.path.expanduser(FLAGS.sdk_dir), timestamp=timestamp, @@ -202,7 +204,8 @@ def main(argv): root_dir=os.path.expanduser(FLAGS.root_dir), provisions_dir=os.path.expanduser(FLAGS.provisions_dir), ios_sdk=FLAGS.ios_sdk, - desktop_compiler=FLAGS.compiler, + dev_team=config.apple_team_id, + compiler_flags=compiler_flags, execute_desktop_testapp=FLAGS.execute_desktop_testapp) logging.info("END building for %s", testapp) @@ -211,12 +214,11 @@ def main(argv): def _build( - testapp, platforms, config, output_dir, sdk_dir, timestamp, builder_dir, - root_dir, provisions_dir, ios_sdk, desktop_compiler, + testapp, platforms, api_config, output_dir, sdk_dir, timestamp, builder_dir, + root_dir, provisions_dir, ios_sdk, dev_team, compiler_flags, execute_desktop_testapp): """Builds one testapp on each of the specified platforms.""" - api_config = APIConfig.from_config(config, testapp) - testapp_dir = os.path.join(root_dir, api_config.api_dir) + testapp_dir = os.path.join(root_dir, api_config.testapp_path) project_dir = os.path.join( output_dir, "testapps" + timestamp, api_config.full_name, os.path.basename(testapp_dir)) @@ -235,7 +237,6 @@ def _build( if _DESKTOP in platforms: logging.info("BEGIN %s, %s", testapp, _DESKTOP) try: - compiler_flags = _get_desktop_compiler_flags(desktop_compiler, config) _build_desktop(sdk_dir, compiler_flags) if execute_desktop_testapp: _execute_desktop_testapp(project_dir) @@ -264,7 +265,7 @@ def _build( provisions_dir=provisions_dir, api_config=api_config, ios_sdk=ios_sdk, - dev_team=config_reader.read_general_config(config, "apple_team_id")) + dev_team=dev_team) except subprocess.CalledProcessError as e: failures.append( Failure(testapp=testapp, platform=_IOS, error_message=str(e))) @@ -304,15 +305,14 @@ def _execute_desktop_testapp(project_dir): _run([testapp_path]) -def _get_desktop_compiler_flags(compiler, config): +def _get_desktop_compiler_flags(compiler, compiler_table): """Returns the command line flags for this compiler.""" if not compiler: # None is an acceptable default value return [] - compilers = config_reader.read_general_config(config, "compiler_dict") try: - return compilers[compiler] + return compiler_table[compiler] except KeyError: - valid_keys = ", ".join(compilers.keys()) + valid_keys = ", ".join(compiler_table.keys()) raise ValueError( "Given compiler: %s. Valid compilers: %s" % (compiler, valid_keys)) @@ -399,7 +399,7 @@ def _build_ios( _run( xcodebuild.get_args_for_archive( path=xcode_path, - scheme=api_config.ios_target, + scheme=api_config.scheme, uuid=provision_id, output_dir=build_dir, archive_path=archive_path, @@ -441,42 +441,5 @@ def describe(self): return "%s, %s: %s" % (self.testapp, self.platform, self.error_message) -@attr.s(frozen=True, eq=False) -class APIConfig(object): - """Holds all the configuration for a single testapp project.""" - name = attr.ib() - full_name = attr.ib() - bundle_id = attr.ib() - ios_target = attr.ib() - api_dir = attr.ib() # Integration test dir relative to sdk root - frameworks = attr.ib() # Required custom xcode frameworks - provision = attr.ib() # Path to the local mobile provision - - @classmethod - def from_config(cls, config, testapp): - """Builds the APIConfig for this testapp. - - Args: - config: The full configuration dictionary for the testapp builder. - testapp: Short name for the testapp, e.g. 'analytics'. - - Returns: - API-specific configuration for this testapp. - - """ - - if testapp not in config_reader.get_api_names(config): - raise ValueError("Invalid api name (not found in config): " + testapp) - - return cls( - name=testapp, - full_name=config_reader.read_api_config(config, testapp, "full_name"), - bundle_id=config_reader.read_api_config(config, testapp, "bundle_id"), - ios_target=config_reader.read_api_config(config, testapp, "ios_target"), - api_dir=config_reader.read_api_config(config, testapp, "testapp_path"), - frameworks=config_reader.read_api_config(config, testapp, "frameworks"), - provision=config_reader.read_api_config(config, testapp, "provision")) - - if __name__ == "__main__": app.run(main) diff --git a/testing/integration_testing/config_reader.py b/testing/integration_testing/config_reader.py index 074e33a969..3f9017e19b 100644 --- a/testing/integration_testing/config_reader.py +++ b/testing/integration_testing/config_reader.py @@ -15,13 +15,11 @@ """A utility for working with testapp builder JSON files. This module handles loading the central configuration file for a testapp -builder, as well as offering simplified access to the loaded in-memory -object. +builder, returning a 'Config' object that exposes all the data. -The main motivation for this module is to provide descriptive error messages, -rather than an unhelpful KeyError that one would get from working with the -loaded JSON (a dictionary) directly. Instead, errors will dump a readable form -of the section of JSON being read, alongside the error. +The motivation for loading the config into a class as opposed to returning +the loaded JSON directly is to validate the data upfront, to fail fast if +anything is missing or formatted incorrectly. Example of such a configuration file: @@ -55,129 +53,81 @@ "dev_team": "ABCDEFGHIJK" } -Available methods: - - read_config reads the config into memory. - read_general_config provides access to the root level config - read_api_config provides access to an API's config, and requires its name. - This must occur in a root level "apis" list of objects. - get_api_names returns a sequence of the names of configured APIs, for use - with read_api_config. - """ import json import os import pathlib -# List of api-specific configurations. -_APIS_KEY = "apis" +import attr -_NAME_KEY = "name" - -# These are used to contextualize where a value is being read within the config -# to provide better error messages. -_ROOT_CONFIG = "root config " -_API_CONFIG = "config in 'apis' list" +_DEFAULT_CONFIG_NAME = "build_testapps.json" def read_config(path=None): - """Creates an in-memory config object out of a testapp config file. + """Creates an in-memory 'Config' object out of a testapp config file. Args: - path: Path to a testapp builder config file. If not specified, will look - for 'build_testapps.json' in the same directory as this file. + path (str): Path to a testapp builder config file. If not specified, will + look for 'build_testapps.json' in the same directory as this file. Returns: - An in-memory config object that can be used with the other methods in - this module to extract data from the config passed in. + A 'Config' containing all the testapp builder's configuration. """ if not path: directory = pathlib.Path(__file__).parent.absolute() - path = os.path.join(directory, "build_testapps.json") + path = os.path.join(directory, _DEFAULT_CONFIG_NAME) with open(path, "r") as config: - return json.load(config) - - -def read_general_config(config, key, optional=False): - """Reads a configuration value not tied to a particular API. - - Args: - config: Configuration created by read_config. - key: Key whose value is being read. - optional: Return a None if the key is not found. - - Returns: - Value associated to the given key. Could be any type supported by - Python's JSON module. - - """ - return _get_from_json(config, key, _ROOT_CONFIG, optional) - - -def read_api_config(config, api_name, key, optional=False): - """Reads a configuration value for a particular API. - - Args: - config: Configuration created by read_config. - api_name: Name of the API whose configuration is being read. - key: Key whose value is being read. - optional: Return a None if the key is not found. - - Returns: - Value associated to the given key. Could be any type supported by - Python's JSON module. - - """ - api_dicts = _get_from_json(config, _APIS_KEY, _ROOT_CONFIG) - api_dict = [api for api in api_dicts if api[_NAME_KEY] == api_name][0] - return _get_from_json(api_dict, key, _API_CONFIG, optional) - - -def get_api_names(config): - """Returns a list of all the APIs present in this config.""" - apis = _get_from_json(config, _APIS_KEY, _ROOT_CONFIG) - return [_get_from_json(api, _NAME_KEY, _API_CONFIG) for api in apis] - - -def _get_from_json(json_dict, key, json_description, optional=False): - """Attempts to retrieve key from the json dictionary. - - If the key is not found and is not marked as optional, an error - will be raised with detailed information including the key, a - pretty-printed dump of the given json, and a description of the json - for context. - - Args: - json_dict: A dictionary representation of a JSON object. - key: The key whose value is being extracted from the JSON dictionary. - json_description: A description of the json object to clarify where - the object came from. Will be included in an error message, if the key - is not found in the dict. Especially important for nested objects. - e.g. "API object from 'APIS' list in main config". - optional: Return a None if the key is not found. - - Raises: - ValueError: Key is None or empty, json_dict is not a dictionary. - RuntimeError: Non-optional Key not found in the dictionary. - - Returns: - The value of the key in the dictionary, equivalent to json_dict[key]. - - """ - if not key: - raise ValueError("Must supply a valid, non-empty key.") + config = json.load(config) + api_configs = dict() try: - return json_dict[key] - except TypeError: # json_dict is not a dict - raise ValueError( - "Expected dictionary (JSON object) from %s, received %s: %s instead." % - (json_description, type(json_dict), json_dict)) - except KeyError: # key not found - if optional: - return None - formatted_json = json.dumps( - json_dict, sort_keys=True, indent=4, separators=(",", ":")) - raise RuntimeError( - "%s not found in %s:\n%s" % (key, json_description, formatted_json)) + for api in config["apis"]: + api_name = api["name"] + api_configs[api_name] = APIConfig( + name=api_name, + full_name=api["full_name"], + bundle_id=api["bundle_id"], + ios_target=api["ios_target"], + scheme=api["ios_target"], # Scheme assumed to be same as target. + testapp_path=api["testapp_path"], + frameworks=api["frameworks"], + provision=api["provision"], + minify=api.get("minify", None)) + return Config( + apis=api_configs, + apple_team_id=config["apple_team_id"], + compilers=config["compiler_dict"]) + except (KeyError, TypeError, IndexError): + # The error will be cryptic on its own, so we dump the JSON to + # offer context, then reraise it the error. + print( + "Error occurred while parsing config. Full config dump:\n" + + json.dumps(config, sort_keys=True, indent=4, separators=(",", ":"))) + raise + + +@attr.s(frozen=True, eq=False) +class Config(object): + apis = attr.ib() # Mapping of str: APIConfig + apple_team_id = attr.ib() + compilers = attr.ib() + + def get_api(self, api): + """Returns the APIConfig object for the given api, e.g. 'analytics'.""" + return self.apis[api] + + +@attr.s(frozen=True, eq=False) +class APIConfig(object): + """Holds all the configuration for a single testapp project.""" + name = attr.ib() + full_name = attr.ib() + bundle_id = attr.ib() + ios_target = attr.ib() + scheme = attr.ib() + testapp_path = attr.ib() # Integration test dir relative to sdk root + frameworks = attr.ib() # Required custom xcode frameworks + provision = attr.ib() # Path to the local mobile provision + minify = attr.ib() # (Optional) Android minification. + diff --git a/testing/integration_testing/provisioning.py b/testing/integration_testing/provisioning.py index 0aecd1710f..ef7575bee7 100644 --- a/testing/integration_testing/provisioning.py +++ b/testing/integration_testing/provisioning.py @@ -43,10 +43,10 @@ def patch_provisioning_profile(plist_path, bundle_id, uuid): overwritten. Args: - plist_path: Path to a valid plist with a provisioningProfiles dictionary - in its root. - bundle_id: String representing a bundle id. - uuid: String representing the uuid of a provisioning profile. + plist_path (str): Path to a valid plist with a provisioningProfiles + dictionary in its root. + bundle_id (str): Bundle id identifying this project. + uuid (str): UUID from a mobile provisioning profile. Raises: ValueError: plist not in correct format. @@ -76,7 +76,7 @@ def get_provision_id(provision_path): Can only be run on macOS. Args: - provision_path: Path to a .mobileprovision file. + provision_path (str): Path to a .mobileprovision file. Returns: A string corresponding to a UUID. @@ -106,7 +106,7 @@ def _extract_plist(provision_path): will attempt to decode the file and return the plist as a string. Args: - provision_path: Path to a mobileprovisioning profile (.mobileprovision). + provision_path (str): Path to a mobileprovision profile. Returns: Bytes corresponding to a valid plist. @@ -126,7 +126,7 @@ def _extract_uuid(plist): Will extract the value of the UUID key from the mobile provisioning plist. Args: - plist: Plist bytes read from a mobileprovisioning file. + plist (bytes): Plist bytes read from a mobileprovisioning file. Returns: A string corresponding to a provisioning UUID. diff --git a/testing/integration_testing/restore_secrets.py b/testing/integration_testing/restore_secrets.py index e97112d13a..0373e59f35 100644 --- a/testing/integration_testing/restore_secrets.py +++ b/testing/integration_testing/restore_secrets.py @@ -29,11 +29,14 @@ integration_test directories. - The server key will be patched into the Messaging project. - The uri prefix will be patched into the Dynamic Links project. +- The reverse id will be patched into all Info.plist files, using the value from + the decrypted Google Service plist files as the source of truth. """ import argparse import os +import plistlib import subprocess parser = argparse.ArgumentParser( @@ -73,6 +76,10 @@ def main(): with open(dest_path, "w") as f: f.write(decrypted_text) print("Copied decrypted google service file to %s" % dest_path) + # We use a Google Service file as the source of truth for the reverse id + # that needs to be patched into the Info.plist files. + if dest_path.endswith(".plist"): + _patch_reverse_id(dest_path) print("Attempting to patch Dynamic Links uri prefix.") uri_path = os.path.join(secrets_dir, "dynamic_links", "uri_prefix.txt.gpg") @@ -120,9 +127,26 @@ def _decrypt(encrypted_file, passphrase): return result.stdout +def _patch_reverse_id(service_plist_path): + """Patches the Info.plist file with the reverse id from the Service plist.""" + print("Attempting to patch reverse id in Info.plist") + with open(service_plist_path, "rb") as f: + service_plist = plistlib.load(f) + _patch_file( + path=os.path.join(os.path.dirname(service_plist_path), "Info.plist"), + placeholder="REPLACE_WITH_REVERSED_CLIENT_ID", + value=service_plist["REVERSED_CLIENT_ID"]) + + def _patch_main_src(project_dir, placeholder, value): """Patches the integration_test.cc file in the integration test project.""" path = os.path.join(project_dir, "src", "integration_test.cc") + _patch_file(path, placeholder, value) + + +def _patch_file(path, placeholder, value): + """Patches instances of the placeholder with the given value.""" + # Note: value may be sensitive, so do not log. with open(path, "r") as f_read: text = f_read.read() # Count number of times placeholder appears for debugging purposes. diff --git a/testing/integration_testing/xcodebuild.py b/testing/integration_testing/xcodebuild.py index 135087eee6..9414740ef9 100644 --- a/testing/integration_testing/xcodebuild.py +++ b/testing/integration_testing/xcodebuild.py @@ -41,18 +41,18 @@ def get_args_for_archive( """Subprocess args for an iOS archive. Necessary step for creating an .ipa. Args: - path: Full path to the project or workspace to build. Must end in + path (str): Full path to the project or workspace to build. Must end in either .xcodeproj or .xcworkspace. - scheme: Name of the scheme to build. - uuid: The mobileprovision's uuid. - output_dir: Directory for the resulting build artifacts. Will be created - if it doesn't already exist. - archive_path: Archive will be created at this path. - ios_sdk: Where this build will be run: device or simulator. - dev_team: Apple development team id. - configuration: Value for the -configuration flag. If building for Unity, - Unity will by default generate Debug, Release, ReleaseForRunning, and - ReleaseForProfiling configurations. + scheme (str): Name of the scheme to build. + uuid (str): The mobileprovision's uuid. + output_dir (str): Directory for the resulting build artifacts. Will be + created if it doesn't already exist. + archive_path (str): Archive will be created at this path. + ios_sdk (str): Where this build will be run: device or simulator. + dev_team (str): Apple development team id. + configuration (str): Value for the -configuration flag. If building for + Unity, Unity will by default generate Debug, Release, ReleaseForRunning, + and ReleaseForProfiling configurations. Returns: Sequence of strings, corresponding to valid args for a subprocess call. @@ -73,14 +73,14 @@ def get_args_for_build( """Constructs subprocess args for an xcode build. Args: - path: Full path to the project or workspace to build. Must end in + path (str): Full path to the project or workspace to build. Must end in either .xcodeproj or .xcworkspace. - scheme: Name of the scheme to build. - output_dir: Directory for the resulting build artifacts. Will be created - if it doesn't already exist. - ios_sdk: Where this build will be run: device or simulator. - dev_team: Apple development team id. - configuration: Value for the -configuration flag. + scheme (str): Name of the scheme to build. + output_dir (str): Directory for the resulting build artifacts. Will be + created if it doesn't already exist. + ios_sdk (str): Where this build will be run: device or simulator. + dev_team (str): Apple development team id. + configuration (str): Value for the -configuration flag. Returns: Sequence of strings, corresponding to valid args for a subprocess call. @@ -114,9 +114,9 @@ def get_args_for_export(output_dir, archive_path, plist_path): the command given by the returned args. Args: - output_dir: Directory to contain the exported artifact. - archive_path: Path to the xcode archive (.xcarchive) to be exported. - plist_path: Path for the export plist. The export plist + output_dir (str): Directory to contain the exported artifact. + archive_path (str): Path to the xcode archive (.xcarchive) to be exported. + plist_path (str): Path for the export plist. The export plist contains configuration for the export process, and must be present even if the file is vacuous (contains no configuration). From d782567e416491b6667d1aefe96c9ff62f807dc3 Mon Sep 17 00:00:00 2001 From: anonymous-akorn <66133366+anonymous-akorn@users.noreply.github.com> Date: Thu, 30 Jul 2020 09:14:31 -0700 Subject: [PATCH 3/3] Replace argparse with absl.flags We've decided to use absl.flags as the flag-parsing library for Python in the repo. Also removed reverse id patching from the xcode tool, since that's being handled by the secret restoration script. --- testing/integration_testing/config_reader.py | 4 +- testing/integration_testing/provisioning.py | 6 +-- .../integration_testing/restore_secrets.py | 54 ++++++++++--------- testing/integration_testing/xcode_tool.rb | 30 +++-------- testing/integration_testing/xcodebuild.py | 2 +- 5 files changed, 44 insertions(+), 52 deletions(-) diff --git a/testing/integration_testing/config_reader.py b/testing/integration_testing/config_reader.py index 3f9017e19b..78826b4ec5 100644 --- a/testing/integration_testing/config_reader.py +++ b/testing/integration_testing/config_reader.py @@ -72,7 +72,7 @@ def read_config(path=None): look for 'build_testapps.json' in the same directory as this file. Returns: - A 'Config' containing all the testapp builder's configuration. + Config: All of the testapp builder's configuration. """ if not path: @@ -100,7 +100,7 @@ def read_config(path=None): compilers=config["compiler_dict"]) except (KeyError, TypeError, IndexError): # The error will be cryptic on its own, so we dump the JSON to - # offer context, then reraise it the error. + # offer context, then reraise the error. print( "Error occurred while parsing config. Full config dump:\n" + json.dumps(config, sort_keys=True, indent=4, separators=(",", ":"))) diff --git a/testing/integration_testing/provisioning.py b/testing/integration_testing/provisioning.py index ef7575bee7..b945551720 100644 --- a/testing/integration_testing/provisioning.py +++ b/testing/integration_testing/provisioning.py @@ -79,7 +79,7 @@ def get_provision_id(provision_path): provision_path (str): Path to a .mobileprovision file. Returns: - A string corresponding to a UUID. + str: UUID. Raises: ValueError: If provision_path is empty or None, or isn't a mobileprovision. @@ -109,7 +109,7 @@ def _extract_plist(provision_path): provision_path (str): Path to a mobileprovision profile. Returns: - Bytes corresponding to a valid plist. + bytes: Contents of the plist file. Raises: subprocess.CalledProcessError: Failed to decode the given file. @@ -129,7 +129,7 @@ def _extract_uuid(plist): plist (bytes): Plist bytes read from a mobileprovisioning file. Returns: - A string corresponding to a provisioning UUID. + str: UUID. """ # In Python 3, readPlistFromString is removed in favor of loads, which is not diff --git a/testing/integration_testing/restore_secrets.py b/testing/integration_testing/restore_secrets.py index 0373e59f35..6df0a7bb95 100644 --- a/testing/integration_testing/restore_secrets.py +++ b/testing/integration_testing/restore_secrets.py @@ -16,14 +16,16 @@ Usage: -python restore_secrets.py --passphrase --repo_dir +python restore_secrets.py --passphrase [--repo_dir ] +python restore_secrets.py --passphrase_file [--repo_dir ] -repo_dir refers to the C++ SDK github repository. Defaults to current directory. +--passphrase: Passphrase to decrypt the files. This option is insecure on a + multi-user machine; use the --passphrase_file option instead. +--passphrase_file: Specify a file to read the passphrase from (only reads the + first line). +--repo_dir: Path to C++ SDK Github repository. Defaults to current directory. -As an alternative to passing the passphrase as a flag, the password can be -stored in a file and passed via the --passphrase_file flag. - -This will perform the following: +This script will perform the following: - Google Service files (plist and json) will be restored into the integration_test directories. @@ -34,28 +36,32 @@ """ -import argparse import os import plistlib import subprocess -parser = argparse.ArgumentParser( - description="Decrypt and restore the secrets. Must specify one of" - " passphrase or passphrase_file.") -parser.add_argument("--passphrase", help="The passphrase itself.") -parser.add_argument("--passphrase_file", help="Path to file with passphrase.") -parser.add_argument("--repo_dir", default=os.getcwd(), help="Path to SDK Repo") -args = parser.parse_args() +from absl import app +from absl import flags + + +FLAGS = flags.FLAGS + +flags.DEFINE_string("repo_dir", os.getcwd(), "Path to C++ SDK Github repo.") +flags.DEFINE_string("passphrase", None, "The passphrase itself.") +flags.DEFINE_string("passphrase_file", None, "Path to file with passphrase.") + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") -def main(): - repo_dir = args.repo_dir + repo_dir = FLAGS.repo_dir # The passphrase is sensitive, do not log. - if args.passphrase: - passphrase = args.passphrase - elif args.passphrase_file: - with open(args.passphrase_file, "r") as f: - passphrase = f.read() + if FLAGS.passphrase: + passphrase = FLAGS.passphrase + elif FLAGS.passphrase_file: + with open(FLAGS.passphrase_file, "r") as f: + passphrase = f.readline().strip() else: raise ValueError("Must supply passphrase or passphrase_file arg.") @@ -105,7 +111,7 @@ def _find_encrypted_files(directory_to_search): def _decrypt(encrypted_file, passphrase): - """Generates a decrypted file with same path minus the '.gpg' extension.""" + """Returns the decrypted contents of the given .gpg file.""" print("Decrypting %s" % encrypted_file) # Note: if setting check=True, be sure to catch the error and not rethrow it # or print a traceback, as the message will include the passphrase. @@ -154,8 +160,8 @@ def _patch_file(path, placeholder, value): patched_text = text.replace(placeholder, value) with open(path, "w") as f_write: f_write.write(patched_text) - print("Patched %d instances of %s in %s " % (replacements, placeholder, path)) + print("Patched %d instances of %s in %s" % (replacements, placeholder, path)) if __name__ == "__main__": - main() + app.run(main) diff --git a/testing/integration_testing/xcode_tool.rb b/testing/integration_testing/xcode_tool.rb index 06aa1f9782..2c8fb89d81 100644 --- a/testing/integration_testing/xcode_tool.rb +++ b/testing/integration_testing/xcode_tool.rb @@ -13,7 +13,7 @@ # limitations under the License. # # -# Configures C++ testapp XCode projects. +# Configures C++ integration test XCode projects. # # This automates the configuration of XCode projects as outlined in the # associated testapp readme files. This includes adding frameworks, enabling @@ -65,6 +65,9 @@ def main @target = @project.targets.first # Examine components rather than substrings to minimize false positives. + # Note: this is not ideal. This tool should not be responsible for figuring + # out which project it's modifying. That responsibility should belong to + # the Python tool invoking this. path_components = project_path.split('/') if path_components.include?('FirebaseAuth') make_changes_for_auth @@ -91,7 +94,6 @@ def main def make_changes_for_auth puts 'Auth testapp detected.' add_entitlements - set_reverse_id add_system_framework('UserNotifications') puts 'Finished making auth-specific changes.' end @@ -99,9 +101,8 @@ def make_changes_for_auth def make_changes_for_messaging puts 'Messaging testapp detected.' add_entitlements - set_reverse_id add_system_framework('UserNotifications') - enable_romote_notification + enable_remote_notification puts 'Finished making messaging-specific changes.' end @@ -122,24 +123,9 @@ def add_entitlements puts 'Added entitlement to xcode project.' end -# Configures the reverse client id in the xcode project. -# Needed for Google sign-in. -# Some testapps may have the REVERSED_CLIENT_ID already set, in which case -# this won't do anything. e.g. Auth -def set_reverse_id - puts 'Setting the Reverse Id...' - google_service_path = "#@xcode_project_dir/GoogleService-Info.plist" - google_plist = Xcodeproj::Plist.read_from_path(google_service_path) - reverse_id = google_plist['REVERSED_CLIENT_ID'] - puts "Found reverse id: #{reverse_id}" - info_plist_path = "#@xcode_project_dir/Info.plist" - info_plist_text = File.read(info_plist_path) - info_plist_text = info_plist_text.gsub('YOUR_REVERSED_CLIENT_ID', reverse_id) - File.open(info_plist_path, "w") {|file| file.puts info_plist_text} - puts "Finished setting the Reverse Id." -end - -def enable_romote_notification +# This only involves patching a plist file, which should be moved to the +# Python tool. +def enable_remote_notification puts 'Adding remote-notification to UIBackgroundModes...' info_plist_path = "#@xcode_project_dir/Info.plist" info_plist = Xcodeproj::Plist.read_from_path(info_plist_path) diff --git a/testing/integration_testing/xcodebuild.py b/testing/integration_testing/xcodebuild.py index 9414740ef9..3e6cb0b5ba 100644 --- a/testing/integration_testing/xcodebuild.py +++ b/testing/integration_testing/xcodebuild.py @@ -96,7 +96,7 @@ def get_args_for_build( "BUILD_DIR=" + output_dir ] - if path is None: + if not path: raise ValueError("Must supply a path.") if path.endswith(".xcworkspace"): args.extend(("-workspace", path))