feat(liveness): integration tests (#118)

Co-authored-by: Thomas Leing <[email protected]>

Author: tjleing

.github/workflows/ci-integration-tests.yml

@@ -0,0 +1,31 @@
+name: Integration tests
+run-name: Integration tests for ${{ github.ref }}
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+  merge_group:
+    types: [checks_requested]
+
+env:
+  AWS_REGION: "us-east-1"
+
+jobs:
+  ci-integration-tests:
+    permissions:
+      id-token: write # This is required for requesting the JWT with configure-aws-credentials
+      contents: read # This is required for actions/checkout
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Source Code
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2
+        with:
+          role-to-assume: ${{ vars.AMPLIFY_UI_ANDROID_CI_TESTS_ROLE }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Run Integration Tests
+        uses: aws-actions/aws-codebuild-run-build@f202c327329cbbebd13f986f74af162a8539b5fd # v1
+        with:
+          project-name: Amplify-UI-Android-Integration-Test

authenticator/src/test/java/com/amplifyframework/ui/authenticator/AuthenticatorViewModelTest.kt

@@ -57,7 +57,7 @@ class AuthenticatorViewModelTest {
 
     @Test
     fun `start only executes once`() = runTest {
-        coEvery {authProvider.fetchAuthSession()}.returns(AmplifyResult.Error(mockAuthException()))
+        coEvery { authProvider.fetchAuthSession() }.returns(AmplifyResult.Error(mockAuthException()))
         viewModel.start(mockAuthConfiguration())
         viewModel.start(mockAuthConfiguration())
         advanceUntilIdle()

build-logic/plugins/src/main/kotlin/AndroidLibraryConventionPlugin.kt

@@ -113,8 +113,11 @@ class AndroidLibraryConventionPlugin : Plugin<Project> {
             // Needed when running integration tests. The oauth2 library uses relies on two
             // dependencies (Apache's httpcore and httpclient), both of which include
             // META-INF/DEPENDENCIES. Tried a couple other options to no avail.
+            // More collisions occurred using JUnit, so also adding LICENSE-*.md files.
             packaging {
                 resources.excludes.add("META-INF/DEPENDENCIES")
+                resources.excludes.add("META-INF/LICENSE.md")
+                resources.excludes.add("META-INF/LICENSE-notice.md")
             }
 
             buildFeatures {

build.gradle.kts

@@ -79,3 +79,5 @@ dependencies {
     kover(project(":authenticator"))
     kover(project(":liveness"))
 }
+
+apply(from = rootProject.file("configuration/instrumentation-tests.gradle"))

configuration/instrumentation-tests.gradle

@@ -0,0 +1,45 @@
+/*
+Need to create backends for s3, pinpoint, predictions, core
+ */
+def module_backends = [
+        'liveness' : 'LivenessIntegTests',
+]
+
+def canaryTests = [
+//        'liveness' : [''],
+]
+
+subprojects {
+    afterEvaluate { project ->
+        if (module_backends.containsKey(project.name)) {
+            task runTestsInDeviceFarm {
+                doLast {
+                    exec {
+                        commandLine("$rootDir.path/scripts/run_test_in_devicefarm.sh")
+                        args([project.name])
+                    }
+                }
+            }
+//            task runNightlyTestsInDeviceFarmPool {
+//                dependsOn(assembleAndroidTest)
+//                doLast {
+//                    exec {
+//                        commandLine("$rootDir.path/scripts/run_nightly_tests_in_devicefarm_pool.sh")
+//                        args([project.name])
+//                    }
+//                }
+//            }
+//            task runCanaryInDeviceFarm {
+//                dependsOn(assembleAndroidTest)
+//                doLast {
+//                    for (canary in canaryTests[project.name]) {
+//                        exec {
+//                            commandLine("$rootDir.path/scripts/run_canary_in_devicefarm.sh")
+//                            args(project.name, canary)
+//                        }
+//                    }
+//                }
+//            }
+        }
+    }
+}

gradle/libs.versions.toml

@@ -54,9 +54,8 @@ test-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", ve
 test-espresso = "androidx.test.espresso:espresso-core:3.5.1"
 test-junit = "junit:junit:4.13.2"
 test-kotest-assertions = { module = "io.kotest:kotest-assertions-core", version.ref = "kotest" }
-# downgrading mockk due to https://github.com/mockk/mockk/issues/1035
-test-mockk = "io.mockk:mockk:1.13.2"
-test-mockk-android = "io.mockk:mockk-android:1.13.2"
+test-mockk = "io.mockk:mockk:1.13.8"
+test-mockk-android = "io.mockk:mockk-android:1.13.8"
 test-robolectric = "org.robolectric:robolectric:4.9.2"
 test-turbine = { module = "app.cash.turbine:turbine", version.ref = "turbine" }
 debug-ui-test-manifest = "androidx.compose.ui:ui-test-manifest:1.5.0-beta01"

liveness/src/androidTest/java/com/amplifyframework/ui/liveness/LivenessFlowInstrumentationTest.kt

@@ -31,7 +31,6 @@ import com.amplifyframework.predictions.options.FaceLivenessSessionOptions
 import com.amplifyframework.ui.liveness.camera.FrameAnalyzer
 import com.amplifyframework.ui.liveness.ml.FaceDetector
 import com.amplifyframework.ui.liveness.model.LivenessCheckState
-import com.amplifyframework.ui.liveness.model.FaceLivenessDetectionException
 import com.amplifyframework.ui.liveness.state.LivenessState
 import com.amplifyframework.ui.liveness.ui.FaceLivenessDetector
 import io.mockk.CapturingSlot
@@ -46,6 +45,7 @@ import io.mockk.mockkObject
 import io.mockk.mockkStatic
 import io.mockk.slot
 import io.mockk.unmockkConstructor
+import java.util.Date
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertTrue
 import org.junit.Before
@@ -75,6 +75,7 @@ class LivenessFlowInstrumentationTest {
     private lateinit var connectingString: String
     private lateinit var moveCloserString: String
     private lateinit var holdStillString: String
+    private lateinit var verifyingString: String
     private lateinit var mockCredentialsProvider: MockCredentialsProvider
 
     private var framesSent = 0
@@ -123,6 +124,7 @@ class LivenessFlowInstrumentationTest {
         holdStillString = context.getString(
             R.string.amplify_ui_liveness_challenge_instruction_hold_face_during_freshness,
         )
+        verifyingString = context.getString(R.string.amplify_ui_liveness_challenge_verifying)
 
         mockCredentialsProvider = MockCredentialsProvider()
     }
@@ -255,7 +257,7 @@ class LivenessFlowInstrumentationTest {
         composeTestRule.setContent {
             FaceLivenessDetector(sessionId = sessionId, region = "us-east-1", onComplete = {
                 completesSuccessfully = true
-            }, onError = {assertTrue(false) })
+            }, onError = { assertTrue(false) })
         }
 
         composeTestRule.onNodeWithText(beginCheckString).assertExists()
@@ -306,10 +308,15 @@ class LivenessFlowInstrumentationTest {
         val sessionId = "sessionId"
         var completesSuccessfully = false
         composeTestRule.setContent {
-            FaceLivenessDetector(sessionId = sessionId, region = "us-east-1", credentialsProvider = mockCredentialsProvider,
+            FaceLivenessDetector(
+                sessionId = sessionId,
+                region = "us-east-1",
+                credentialsProvider = mockCredentialsProvider,
                 onComplete = {
-                completesSuccessfully = true
-            }, onError = { assertTrue(false) })
+                    completesSuccessfully = true
+                },
+                onError = { assertTrue(false) },
+            )
         }
 
         composeTestRule.onNodeWithText(beginCheckString).assertExists()
@@ -403,11 +410,23 @@ class LivenessFlowInstrumentationTest {
         )
         faceUpdates += 1
 
-        // now, the face is inside the oval.  wait for the colors to finish
         composeTestRule.waitForIdle()
 
-        assertEquals(livenessState?.readyToSendFinalEvents, true)
+        // countdown is now invsible, wait one second so that we can start freshness
+        composeTestRule.waitUntil(2000) {
+            livenessState?.faceMatchOvalStart?.let { (Date().time - it) > 1000 } ?: false
+        }
+        livenessState?.onFrameAvailable()
+        assert(livenessState?.runningFreshness!!)
+
+        // now, freshness is running.  wait for the colors to finish
+        composeTestRule.waitUntil(10000) {
+            composeTestRule.onAllNodesWithText(verifyingString)
+                .fetchSemanticsNodes().size == 1
+        }
+
         val state = livenessState?.livenessCheckState?.value
+        assertEquals(livenessState?.readyToSendFinalEvents, true)
         assertTrue(state is LivenessCheckState.Success)
         assertTrue((state as LivenessCheckState.Success).faceGuideRect == faceRect)
         // inconsistent number of frames sent

scripts/generate_df_testrun_report

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Python script that generates a user-readable report for a given DeviceFarm test run.
+"""
+
+import os
+import sys
+import subprocess
+import argparse
+import logging
+import boto3
+from botocore.config import Config
+from junit_xml import TestSuite, TestCase
+
+LOG_FORMATTER = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+CONSOLE_HANDLER = logging.StreamHandler()
+CONSOLE_HANDLER.setFormatter(LOG_FORMATTER)
+LOGGER = logging.getLogger("DeviceFarmTestRunReportGenerator")
+LOGGER.setLevel(os.getenv("LOG_LEVEL") if os.getenv("LOG_LEVEL") is not None else "INFO")
+LOGGER.addHandler(CONSOLE_HANDLER)
+
+client = boto3.client('devicefarm', config=Config(region_name='us-west-2'))
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Utility that generates a report for a DeviceFarm test run.")
+    parser.add_argument("--run_arn", help="The ARN of the DeviceFarm test run.", required=True)
+    parser.add_argument("--module_name", help="The module name for the test suite.", required=True)
+    parser.add_argument("--output_path", help="Destination path for the build reports.", required=True)
+    parser.add_argument("--pr", help="The github PR number.")
+    return parser.parse_args()
+
+def generate_junit_report(run_arn, output_path):
+    LOGGER.debug(f"Retrieving test jobs for run {run_arn}")
+    jobs = get_test_jobs(run_arn)
+    for job_no, job in enumerate(jobs):
+        LOGGER.debug(f"Retrieving test suites for job {job['arn']}")
+        suites = get_test_suites(job['arn'])
+        for suite in suites:
+            LOGGER.debug(f"Retrieving tests for suite {suite['arn']}")
+            tests = get_tests(suite['arn'])
+            test_cases = []
+            for test in tests:
+                tc = TestCase(test['name'],
+                                classname=suite['name'],
+                                elapsed_sec=test['deviceMinutes']['total']*60,
+                                stdout=test['message'],
+                                status=test['result'] )
+                if test['result'] == 'FAILED':
+                    tc.add_failure_info(message=test['message'])
+                if test['result'] == 'ERROR':
+                    tc.add_error_info(message=test['message'])
+                test_cases.append(tc)
+            ts = TestSuite(suite['name'] + "-" + str(job_no),test_cases=test_cases)
+            ts_output = TestSuite.to_xml_string([ts])
+            LOGGER.info(f"Saving test suite {suite['name']} report.")
+            if not os.path.exists(output_path):
+                os.makedirs(output_path)
+            f = open(output_path + suite['name'] + "-" + str(job_no) + ".xml", "w")
+            f.write(ts_output)
+            f.close()
+
+def get_test_jobs(run_arn):
+    result = client.list_jobs(arn=run_arn)
+    return result['jobs'] if result is not None else []
+
+def get_test_suites(job_arn):
+    result = client.list_suites(arn=job_arn)
+    return result['suites'] if result is not None else []
+
+def get_tests(suite_arn):
+    result = client.list_tests(arn=suite_arn)
+    return result['tests'] if result is not None else []
+
+def get_problems(run_arn):
+    return client.list_unique_problems(
+        arn=run_arn
+    )
+
+def main(arguments):
+    args = parse_arguments()
+    build_id = os.getenv("CODEBUILD_BUILD_ID")
+    source_version = os.getenv("CODEBUILD_SOURCE_VERSION")
+    arn_suffix = args.run_arn.split(':')[-1]
+    LOGGER.info(f"devicefarm_run: {arn_suffix} build_id: {build_id} source_version: {source_version}")
+    generate_junit_report(run_arn=args.run_arn,
+                            output_path=args.output_path)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))