diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0ae8fb08..32159a4b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -42,15 +42,20 @@ tensorflow-support = "org.tensorflow:tensorflow-lite-support:0.3.0" # Testing libraries test-androidx-junit = "androidx.test.ext:junit:1.1.4" +test-androidx-monitor = "androidx.test:monitor:1.5.0" +test-androidx-rules = "androidx.test:rules:1.5.0" test-compose-junit = { module = "androidx.compose.ui:ui-test-junit4", version.ref = "compose" } test-compose-manifest = { module = "androidx.compose.ui:ui-test-manifest", version.ref = "compose" } test-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" } test-espresso = "androidx.test.espresso:espresso-core:3.5.1" test-junit = "junit:junit:4.13.2" test-kotest-assertions = { module = "io.kotest:kotest-assertions-core", version.ref = "kotest" } -test-mockk = "io.mockk:mockk:1.13.4" +# downgrading mockk due to https://github.com/mockk/mockk/issues/1035 +test-mockk = "io.mockk:mockk:1.13.2" +test-mockk-android = "io.mockk:mockk-android:1.13.2" test-robolectric = "org.robolectric:robolectric:4.9.2" test-turbine = { module = "app.cash.turbine:turbine", version.ref = "turbine" } +debug-ui-test-manifest = "androidx.compose.ui:ui-test-manifest:1.5.0-beta01" # Dependencies for convention plugins plugin-android-gradle = { module = "com.android.tools.build:gradle", version.ref = "agp" } diff --git a/liveness/build.gradle.kts b/liveness/build.gradle.kts index 76dd1647..25417ecc 100644 --- a/liveness/build.gradle.kts +++ b/liveness/build.gradle.kts @@ -33,6 +33,11 @@ android { androidResources { noCompress += "tflite" } + + packagingOptions { + resources.excludes.add("META-INF/LICENSE.md") + resources.excludes.add("META-INF/LICENSE-notice.md") + } } dependencies { @@ -52,4 +57,11 @@ dependencies { implementation(libs.tensorflow.support) testImplementation(projects.testing) + androidTestImplementation(libs.amplify.auth) + androidTestImplementation(libs.test.compose.junit) + androidTestImplementation(libs.test.androidx.monitor) + androidTestImplementation(libs.test.androidx.rules) + androidTestImplementation(libs.test.junit) + androidTestImplementation(libs.test.mockk.android) + debugImplementation(libs.debug.ui.test.manifest) } diff --git a/liveness/src/androidTest/java/com/amplifyframework/ui/liveness/LivenessFlowInstrumentationTest.kt b/liveness/src/androidTest/java/com/amplifyframework/ui/liveness/LivenessFlowInstrumentationTest.kt new file mode 100644 index 00000000..a365c4d5 --- /dev/null +++ b/liveness/src/androidTest/java/com/amplifyframework/ui/liveness/LivenessFlowInstrumentationTest.kt @@ -0,0 +1,438 @@ +package com.amplifyframework.ui.liveness + +import android.Manifest +import android.content.Context +import android.graphics.RectF +import androidx.compose.ui.test.assertIsDisplayed +import androidx.compose.ui.test.junit4.createComposeRule +import androidx.compose.ui.test.onAllNodesWithText +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performClick +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.rule.GrantPermissionRule +import com.amplifyframework.auth.AWSCredentials +import com.amplifyframework.auth.AWSCredentialsProvider +import com.amplifyframework.auth.AuthException +import com.amplifyframework.auth.AuthSession +import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin +import com.amplifyframework.core.Action +import com.amplifyframework.core.Amplify +import com.amplifyframework.core.Consumer +import com.amplifyframework.predictions.aws.AWSPredictionsPlugin +import com.amplifyframework.predictions.aws.models.ColorChallenge +import com.amplifyframework.predictions.aws.models.ColorChallengeType +import com.amplifyframework.predictions.aws.models.ColorDisplayInformation +import com.amplifyframework.predictions.aws.models.FaceTargetChallenge +import com.amplifyframework.predictions.aws.models.FaceTargetMatchingParameters +import com.amplifyframework.predictions.aws.models.RgbColor +import com.amplifyframework.predictions.models.FaceLivenessSession +import com.amplifyframework.predictions.models.FaceLivenessSessionInformation +import com.amplifyframework.predictions.options.FaceLivenessSessionOptions +import com.amplifyframework.ui.liveness.camera.FrameAnalyzer +import com.amplifyframework.ui.liveness.ml.FaceDetector +import com.amplifyframework.ui.liveness.model.LivenessCheckState +import com.amplifyframework.ui.liveness.model.FaceLivenessDetectionException +import com.amplifyframework.ui.liveness.state.LivenessState +import com.amplifyframework.ui.liveness.ui.FaceLivenessDetector +import io.mockk.CapturingSlot +import io.mockk.InvokeMatcher +import io.mockk.OfTypeMatcher +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import io.mockk.mockkConstructor +import io.mockk.mockkObject +import io.mockk.mockkStatic +import io.mockk.slot +import io.mockk.unmockkConstructor +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Rule +import org.junit.Test + +class MockCredentialsProvider : AWSCredentialsProvider { + override fun fetchAWSCredentials( + onSuccess: Consumer, + onError: Consumer, + ) { + val creds: AWSCredentials = AWSCredentials.createAWSCredentials("asdf", "asdf", "asdf", 1000000L)!! + onSuccess.accept(creds) + } +} + +class LivenessFlowInstrumentationTest { + private lateinit var livenessSessionInformation: CapturingSlot + private lateinit var livenessSessionOptions: CapturingSlot + private lateinit var onSessionStarted: CapturingSlot> + private lateinit var onLivenessComplete: CapturingSlot + private lateinit var tooCloseString: String + private lateinit var beginCheckString: String + private lateinit var noFaceString: String + private lateinit var multipleFaceString: String + private lateinit var connectingString: String + private lateinit var moveCloserString: String + private lateinit var holdStillString: String + private lateinit var mockCredentialsProvider: MockCredentialsProvider + + private var framesSent = 0 + + @get:Rule + val composeTestRule = createComposeRule() + + @get:Rule + var mRuntimePermissionRule: GrantPermissionRule = GrantPermissionRule.grant(Manifest.permission.CAMERA) + + @Before + fun setup() { + val context = InstrumentationRegistry.getInstrumentation().targetContext.applicationContext + + livenessSessionInformation = slot() + livenessSessionOptions = slot() + onSessionStarted = slot() + onLivenessComplete = slot() + mockkStatic(AWSPredictionsPlugin::class) + every { + AWSPredictionsPlugin.startFaceLivenessSession( + any(), // sessionId + capture(livenessSessionInformation), // sessionInformation + capture(livenessSessionOptions), // options + any(), // version + capture(onSessionStarted), // onSessionStarted + capture(onLivenessComplete), // onComplete + any(), // onError + ) + } just Runs + + mockkConstructor(FaceLivenessSession::class) + every { anyConstructed().sendVideoEvent(any()) }.answers { + framesSent++ + } + + // string resources + beginCheckString = context.getString(R.string.amplify_ui_liveness_get_ready_begin_check) + tooCloseString = context.getString(R.string.amplify_ui_liveness_challenge_instruction_move_face_further) + noFaceString = context.getString(R.string.amplify_ui_liveness_challenge_instruction_move_face) + multipleFaceString = context.getString( + R.string.amplify_ui_liveness_challenge_instruction_multiple_faces_detected, + ) + connectingString = context.getString(R.string.amplify_ui_liveness_challenge_connecting) + moveCloserString = context.getString(R.string.amplify_ui_liveness_challenge_instruction_move_face_closer) + holdStillString = context.getString( + R.string.amplify_ui_liveness_challenge_instruction_hold_face_during_freshness, + ) + + mockCredentialsProvider = MockCredentialsProvider() + } + + @Test + fun testLivenessDefaultCameraGivesNoFaceError() { + val sessionId = "sessionId" + composeTestRule.setContent { + FaceLivenessDetector(sessionId = sessionId, region = "us-east-1", onComplete = { + }, onError = { assertTrue(false) }) + } + + composeTestRule.onNodeWithText(beginCheckString).assertExists() + composeTestRule.onNodeWithText(beginCheckString).performClick() + composeTestRule.waitUntil(5000) { + composeTestRule.onAllNodesWithText(noFaceString) + .fetchSemanticsNodes().size == 1 + } + // make sure compose flow reaches this point + composeTestRule.onNodeWithText(noFaceString).assertIsDisplayed() + } + + @Test + fun testLivenessFlowTooClose() { + mockkConstructor(FrameAnalyzer::class) + var livenessState: LivenessState? = null + every { + constructedWith( + OfTypeMatcher(Context::class), + InvokeMatcher { + livenessState = it + }, + ).analyze(any()) + } answers { + assert(livenessState != null) + + livenessState?.onFrameFaceCountUpdate(1) + + // Features too far apart, this face must be too close to the camera + livenessState?.onFrameFaceUpdate( + RectF(0f, 0f, 400f, 400f), + FaceDetector.Landmark(120f, 120f), + FaceDetector.Landmark(280f, 120f), + FaceDetector.Landmark(200f, 320f), + ) + } + + val sessionId = "sessionId" + composeTestRule.setContent { + FaceLivenessDetector(sessionId = sessionId, region = "us-east-1", onComplete = { + }, onError = { assertTrue(false) }) + } + + composeTestRule.onNodeWithText(beginCheckString).assertExists() + composeTestRule.onNodeWithText(beginCheckString).performClick() + composeTestRule.waitUntil(5000) { + composeTestRule.onAllNodesWithText(tooCloseString) + .fetchSemanticsNodes().size == 1 + } + + // make sure compose flow reaches this point + composeTestRule.onNodeWithText(tooCloseString).assertIsDisplayed() + + unmockkConstructor(FrameAnalyzer::class) + } + + @Test + fun testLivenessFlowTooManyFaces() { + mockkConstructor(FrameAnalyzer::class) + var livenessState: LivenessState? = null + every { + constructedWith( + OfTypeMatcher(Context::class), + InvokeMatcher { + livenessState = it + }, + ).analyze(any()) + } answers { + assert(livenessState != null) + + livenessState?.onFrameFaceCountUpdate(2) + } + + val sessionId = "sessionId" + composeTestRule.setContent { + FaceLivenessDetector(sessionId = sessionId, region = "us-east-1", onComplete = { + }, onError = { assertTrue(false) }) + } + + composeTestRule.onNodeWithText(beginCheckString).assertExists() + composeTestRule.onNodeWithText(beginCheckString).performClick() + composeTestRule.waitUntil(5000) { + composeTestRule.onAllNodesWithText(multipleFaceString) + .fetchSemanticsNodes().size == 1 + } + + // make sure compose flow reaches this point + composeTestRule.onNodeWithText(multipleFaceString).assertIsDisplayed() + + unmockkConstructor(FrameAnalyzer::class) + } + + @Test + fun testLivenessFlowNoChallenges() { + mockkConstructor(FrameAnalyzer::class) + var livenessState: LivenessState? = null + every { + constructedWith( + OfTypeMatcher(Context::class), + InvokeMatcher { + livenessState = it + }, + ).analyze(any()) + } answers { + assert(livenessState != null) + + livenessState?.onFrameFaceCountUpdate(1) + + // Features should be sized correctly here + livenessState?.onFrameFaceUpdate( + RectF(0f, 0f, 200f, 200f), + FaceDetector.Landmark(60f, 60f), + FaceDetector.Landmark(140f, 60f), + FaceDetector.Landmark(100f, 160f), + ) + } + + val sessionId = "sessionId" + var completesSuccessfully = false + composeTestRule.setContent { + FaceLivenessDetector(sessionId = sessionId, region = "us-east-1", onComplete = { + completesSuccessfully = true + }, onError = {assertTrue(false) }) + } + + composeTestRule.onNodeWithText(beginCheckString).assertExists() + composeTestRule.onNodeWithText(beginCheckString).performClick() + composeTestRule.waitUntil(5000) { + composeTestRule.onAllNodesWithText(connectingString) + .fetchSemanticsNodes().size == 1 + } + + composeTestRule.waitForIdle() + + val pause = 1 + onSessionStarted.captured.accept(FaceLivenessSession(emptyList(), {}, {}, {})) + + composeTestRule.waitForIdle() + + onLivenessComplete.captured.call() + assertTrue(completesSuccessfully) + + unmockkConstructor(FrameAnalyzer::class) + } + + @Test + fun testLivenessFlowWithChallenges() { + mockkConstructor(FrameAnalyzer::class) + var livenessState: LivenessState? = null + every { + constructedWith( + OfTypeMatcher(Context::class), + InvokeMatcher { + livenessState = it + }, + ).analyze(any()) + } answers { + assert(livenessState != null) + + livenessState?.onFrameFaceCountUpdate(1) + + // Features should be sized correctly here + livenessState?.onFrameFaceUpdate( + RectF(0f, 0f, 200f, 200f), + FaceDetector.Landmark(60f, 60f), + FaceDetector.Landmark(140f, 60f), + FaceDetector.Landmark(100f, 160f), + ) + } + + val sessionId = "sessionId" + var completesSuccessfully = false + composeTestRule.setContent { + FaceLivenessDetector(sessionId = sessionId, region = "us-east-1", credentialsProvider = mockCredentialsProvider, + onComplete = { + completesSuccessfully = true + }, onError = { assertTrue(false) }) + } + + composeTestRule.onNodeWithText(beginCheckString).assertExists() + composeTestRule.onNodeWithText(beginCheckString).performClick() + composeTestRule.waitUntil(5000) { + composeTestRule.onAllNodesWithText(connectingString) + .fetchSemanticsNodes().size == 1 + } + + val faceTargetMatchingParameters = mockk() + every { faceTargetMatchingParameters.targetIouThreshold }.returns(0.7f) + every { faceTargetMatchingParameters.targetIouWidthThreshold }.returns(0.25f) + every { faceTargetMatchingParameters.targetIouHeightThreshold }.returns(0.25f) + every { faceTargetMatchingParameters.faceIouWidthThreshold }.returns(0.15f) + every { faceTargetMatchingParameters.faceIouHeightThreshold }.returns(0.15f) + every { faceTargetMatchingParameters.ovalFitTimeout }.returns(10000) + + val faceTargetChallenge = mockk() + val faceRect = RectF(19f, -49f, 441f, 633f) + every { faceTargetChallenge.targetWidth }.returns(faceRect.right - faceRect.left) + every { faceTargetChallenge.targetHeight }.returns(faceRect.bottom - faceRect.top) + every { faceTargetChallenge.targetCenterX }.returns((faceRect.left + faceRect.right) / 2) + every { faceTargetChallenge.targetCenterY }.returns((faceRect.top + faceRect.bottom) / 2) + every { faceTargetChallenge.faceTargetMatching }.returns(faceTargetMatchingParameters) + + val colors = listOf( + RgbColor(0, 0, 0), + RgbColor(0, 255, 255), + RgbColor(255, 0, 0), + RgbColor(0, 255, 0), + RgbColor(0, 0, 255), + RgbColor(255, 255, 0), + RgbColor(0, 255, 0), + RgbColor(255, 0, 0), + ) + val durations = listOf( + 75f, + 475f, + 475f, + 475f, + 475f, + 475f, + 475f, + 475f, + ) + val challengeColors = List(colors.size) { + val colorDisplayInformation = mockk() + every { colorDisplayInformation.color }.returns(colors[it]) + every { colorDisplayInformation.duration }.returns(durations[it]) + every { colorDisplayInformation.shouldScroll }.returns(false) + colorDisplayInformation + } + val colorChallenge = mockk() + every { colorChallenge.challengeId }.returns("id") + every { colorChallenge.challengeType }.returns(ColorChallengeType.SEQUENTIAL) + every { colorChallenge.challengeColors }.returns(challengeColors) + + onSessionStarted.captured.accept( + FaceLivenessSession( + listOf(faceTargetChallenge, colorChallenge), + {}, // onVideoEvent + {}, // onChallengeResponseEvent + {}, // stopLivenessSession + ), + ) + var faceUpdates = 0 + + // update face location to show oval + livenessState?.onFrameFaceUpdate( + RectF(0f, 0f, 400f, 400f), + FaceDetector.Landmark(60f, 60f), + FaceDetector.Landmark(140f, 60f), + FaceDetector.Landmark(100f, 160f), + ) + faceUpdates += 1 + + // in the same spot as it was originally, the face is too far + composeTestRule.waitUntil(1000) { + composeTestRule.onAllNodesWithText(moveCloserString) + .fetchSemanticsNodes().size == 1 + } + + composeTestRule.waitForIdle() + + // update face to be inside the oval position + livenessState?.onFrameFaceUpdate( + faceRect, + FaceDetector.Landmark(60f, 60f), + FaceDetector.Landmark(140f, 60f), + FaceDetector.Landmark(100f, 160f), + ) + faceUpdates += 1 + + // now, the face is inside the oval. wait for the colors to finish + composeTestRule.waitForIdle() + + assertEquals(livenessState?.readyToSendFinalEvents, true) + val state = livenessState?.livenessCheckState?.value + assertTrue(state is LivenessCheckState.Success) + assertTrue((state as LivenessCheckState.Success).faceGuideRect == faceRect) + // inconsistent number of frames sent + assertTrue(framesSent >= faceUpdates) + + onLivenessComplete.captured.call() + assertTrue(completesSuccessfully) + + unmockkConstructor(FrameAnalyzer::class) + } + + companion object { + @BeforeClass + @JvmStatic + fun setupAmplify() { + val context = InstrumentationRegistry.getInstrumentation().targetContext.applicationContext + + // mock the Amplify Auth category + val authPlugin = AWSCognitoAuthPlugin() + mockkObject(authPlugin) + every { authPlugin.fetchAuthSession(any(), any()) } answers { + firstArg<(AuthSession) -> Unit>().invoke(AuthSession(true)) + } + Amplify.addPlugin(authPlugin) + Amplify.configure(context) + } + } +} diff --git a/liveness/src/test/java/com/amplifyframework/ui/liveness/state/LivenessStateTest.kt b/liveness/src/test/java/com/amplifyframework/ui/liveness/state/LivenessStateTest.kt index 0dbc5ffd..6144eb6c 100644 --- a/liveness/src/test/java/com/amplifyframework/ui/liveness/state/LivenessStateTest.kt +++ b/liveness/src/test/java/com/amplifyframework/ui/liveness/state/LivenessStateTest.kt @@ -17,6 +17,7 @@ package com.amplifyframework.ui.liveness.state import android.graphics.RectF import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 import com.amplifyframework.predictions.aws.models.ColorChallenge import com.amplifyframework.predictions.aws.models.FaceTargetChallenge import com.amplifyframework.predictions.aws.models.FaceTargetChallengeResponse @@ -36,9 +37,8 @@ import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -@RunWith(RobolectricTestRunner::class) +@RunWith(AndroidJUnit4::class) internal class LivenessStateTest { private lateinit var livenessState: LivenessState diff --git a/scripts/pull_backend_config_from_s3.sh b/scripts/pull_backend_config_from_s3.sh new file mode 100644 index 00000000..ff0e0d09 --- /dev/null +++ b/scripts/pull_backend_config_from_s3.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -x +set -e + +# This bucket contains a collection of config files that are used by the +# integration tests. The configuration files contain sensitive +# tokens/credentials/identifiers, so are not published publicly. +readonly config_bucket=$1 + +readonly config_files=( + # Liveness + "liveness/src/androidTest/res/raw/amplifyconfiguration.json" +) + +# Set up output path +declare -r dest_dir=$HOME/.aws-amplify/amplify-android +mkdir -p "$dest_dir" + +# Download remote files into a local directory outside of the project. +for config_file in ${config_files[@]}; do + aws s3 cp "s3://$config_bucket/$config_file" "$dest_dir/$config_file" & +done +wait + +# Create a symlink for each configuration file. +for config_file in ${config_files[@]}; do + mkdir -p "$(dirname "$config_file")" + ln -s "$dest_dir/$config_file" "$config_file" & +done +wait diff --git a/settings.gradle.kts b/settings.gradle.kts index dd5b24a8..09fe9c45 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -20,6 +20,7 @@ include(":liveness") include(":authenticator") include(":testing") include(":authenticator-screenshots") +include(":liveness-screenshots") // Enable typesafe accessor generation for cross-project references enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")