From d32627229191831bc57504791e65f1396fd88391 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Mon, 2 Jan 2023 12:20:00 -0500 Subject: [PATCH 1/6] skip rewarded ad loads on iOS except for manual testing. --- gma/integration_test/src/integration_test.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/gma/integration_test/src/integration_test.cc b/gma/integration_test/src/integration_test.cc index e02015d04e..13fc139a75 100644 --- a/gma/integration_test/src/integration_test.cc +++ b/gma/integration_test/src/integration_test.cc @@ -845,8 +845,7 @@ TEST_F(FirebaseGmaTest, TestRewardedAdLoad) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. + // TODO(@drsanta): remove when GMA whitelists CI devices. FLAKY_TEST_SECTION_BEGIN(); firebase::gma::RewardedAd* rewarded = new firebase::gma::RewardedAd(); @@ -1011,6 +1010,9 @@ TEST_F(FirebaseGmaUITest, TestRewardedAdLoadAndShow) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; + // TODO(@drsanta): remove when GMA whitelists CI devices. + TEST_REQUIRES_USER_INTERACTION_ON_IOS; + firebase::gma::RewardedAd* rewarded = new firebase::gma::RewardedAd(); WaitForCompletion(rewarded->Initialize(app_framework::GetWindowContext()), @@ -1851,9 +1853,8 @@ TEST_F(FirebaseGmaTest, TestRewardedAdLoadEmptyRequest) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); + // TODO(@drsanta): remove when GMA whitelists CI devices. + TEST_REQUIRES_USER_INTERACTION_ON_IOS; // Note: while showing an ad requires user interaction in another test, // this test is mean as a baseline loadAd functionality test. @@ -1878,8 +1879,6 @@ TEST_F(FirebaseGmaTest, TestRewardedAdLoadEmptyRequest) { EXPECT_FALSE(result_ptr->response_info().ToString().empty()); delete rewarded; - - FLAKY_TEST_SECTION_END(); } TEST_F(FirebaseGmaTest, TestRewardedAdErrorNotInitialized) { @@ -1938,6 +1937,9 @@ TEST_F(FirebaseGmaTest, TesRewardedAdErrorAlreadyInitialized) { TEST_F(FirebaseGmaTest, TestRewardedAdErrorLoadInProgress) { SKIP_TEST_ON_DESKTOP; + // TODO(@drsanta): remove when GMA whitelists CI devices. + TEST_REQUIRES_USER_INTERACTION_ON_IOS; + firebase::gma::RewardedAd* rewarded = new firebase::gma::RewardedAd(); WaitForCompletion(rewarded->Initialize(app_framework::GetWindowContext()), "Initialize"); @@ -2065,7 +2067,7 @@ TEST_F(FirebaseGmaTest, TestRewardedAdStress) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_EMULATOR; - // TODO(@drsanta): remove when GMA whitelists CI devices + // TODO(@drsanta): remove when GMA whitelists CI devices. TEST_REQUIRES_USER_INTERACTION_ON_IOS; for (int i = 0; i < 10; ++i) { From c62e442bc521d5df481699fa2ddab31620dc6f6b Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Mon, 2 Jan 2023 13:17:20 -0500 Subject: [PATCH 2/6] stress tests accept NoFill as a valid response --- gma/integration_test/src/integration_test.cc | 32 +++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/gma/integration_test/src/integration_test.cc b/gma/integration_test/src/integration_test.cc index 13fc139a75..7968e8dabf 100644 --- a/gma/integration_test/src/integration_test.cc +++ b/gma/integration_test/src/integration_test.cc @@ -2031,10 +2031,16 @@ TEST_F(FirebaseGmaTest, TestAdViewStress) { // Load the AdView ad. firebase::gma::AdRequest request = GetAdRequest(); - WaitForCompletion(ad_view->LoadAd(request), "TestAdViewStress LoadAd"); - EXPECT_EQ(ad_view->ad_size().width(), kBannerWidth); - EXPECT_EQ(ad_view->ad_size().height(), kBannerHeight); - + firebase::Future future = ad_view->LoadAd(request); + WaitForCompletionAnyResult(future, "TestAdViewStress LoadAd"); + // Stress tests may exhaust the ad pool. If so, loadAd will return + // kAdErrorCodeNoFill. + EXPECT_TRUE(future.error() == firebase::gma::kAdErrorCodeNone || + future.error() == firebase::gma::kAdErrorCodeNoFill); + if (future.error() == firebase::gma::kAdErrorCodeNone) { + EXPECT_EQ(ad_view->ad_size().width(), kBannerWidth); + EXPECT_EQ(ad_view->ad_size().height(), kBannerHeight); + } WaitForCompletion(ad_view->Destroy(), "Destroy the AdView"); delete ad_view; } @@ -2057,8 +2063,13 @@ TEST_F(FirebaseGmaTest, TestInterstitialAdStress) { // When the InterstitialAd is initialized, load an ad. firebase::gma::AdRequest request = GetAdRequest(); - WaitForCompletion(interstitial->LoadAd(kInterstitialAdUnit, request), - "TestInterstitialAdStress LoadAd"); + firebase::Future future = + interstitial->LoadAd(kInterstitialAdUnit, request); + WaitForCompletionAnyResult(future, "TestInterstitialAdStress LoadAd"); + // Stress tests may exhaust the ad pool. If so, loadAd will return + // kAdErrorCodeNoFill. + EXPECT_TRUE(future.error() == firebase::gma::kAdErrorCodeNone || + future.error() == firebase::gma::kAdErrorCodeNoFill); delete interstitial; } } @@ -2078,8 +2089,13 @@ TEST_F(FirebaseGmaTest, TestRewardedAdStress) { // When the RewardedAd is initialized, load an ad. firebase::gma::AdRequest request = GetAdRequest(); - WaitForCompletion(rewarded->LoadAd(kRewardedAdUnit, request), - "TestRewardedAdStress LoadAd"); + firebase::Future future = + rewarded->LoadAd(kRewardedAdUnit, request); + WaitForCompletionAnyResult(future, "TestRewardedAdStress LoadAd"); + // Stress tests may exhaust the ad pool. If so, loadAd will return + // kAdErrorCodeNoFill. + EXPECT_TRUE(future.error() == firebase::gma::kAdErrorCodeNone || + future.error() == firebase::gma::kAdErrorCodeNoFill); delete rewarded; } } From 2214b45f35a09c07a444af304f4f0b37ba29bcf3 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Tue, 3 Jan 2023 08:16:59 -0500 Subject: [PATCH 3/6] remove flaky test blocks --- gma/integration_test/src/integration_test.cc | 61 -------------------- 1 file changed, 61 deletions(-) diff --git a/gma/integration_test/src/integration_test.cc b/gma/integration_test/src/integration_test.cc index 7968e8dabf..e517dad718 100644 --- a/gma/integration_test/src/integration_test.cc +++ b/gma/integration_test/src/integration_test.cc @@ -778,10 +778,6 @@ TEST_F(FirebaseGmaTest, TestAdViewLoadAd) { firebase::Future load_ad_future; const firebase::gma::AdResult* result_ptr = nullptr; - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - load_ad_future = ad_view->LoadAd(GetAdRequest()); WaitForCompletion(load_ad_future, "LoadAd"); @@ -789,8 +785,6 @@ TEST_F(FirebaseGmaTest, TestAdViewLoadAd) { ASSERT_NE(result_ptr, nullptr); EXPECT_TRUE(result_ptr->is_successful()); - FLAKY_TEST_SECTION_END(); - ASSERT_NE(result_ptr, nullptr); EXPECT_FALSE(result_ptr->response_info().adapter_responses().empty()); EXPECT_FALSE( @@ -811,10 +805,6 @@ TEST_F(FirebaseGmaTest, TestInterstitialAdLoad) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - firebase::gma::InterstitialAd* interstitial = new firebase::gma::InterstitialAd(); @@ -837,17 +827,12 @@ TEST_F(FirebaseGmaTest, TestInterstitialAdLoad) { load_ad_future.Release(); delete interstitial; - - FLAKY_TEST_SECTION_END(); } TEST_F(FirebaseGmaTest, TestRewardedAdLoad) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; - // TODO(@drsanta): remove when GMA whitelists CI devices. - FLAKY_TEST_SECTION_BEGIN(); - firebase::gma::RewardedAd* rewarded = new firebase::gma::RewardedAd(); WaitForCompletion(rewarded->Initialize(app_framework::GetWindowContext()), @@ -869,8 +854,6 @@ TEST_F(FirebaseGmaTest, TestRewardedAdLoad) { load_ad_future.Release(); delete rewarded; - - FLAKY_TEST_SECTION_END(); } // Interactive test section. These have been placed up front so that the @@ -1095,18 +1078,12 @@ TEST_F(FirebaseGmaTest, TestAdViewLoadAdEmptyAdRequest) { firebase::Future load_ad_future; const firebase::gma::AdResult* result_ptr = nullptr; - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - load_ad_future = ad_view->LoadAd(request); WaitForCompletion(load_ad_future, "LoadAd"); result_ptr = load_ad_future.result(); ASSERT_NE(result_ptr, nullptr); EXPECT_TRUE(result_ptr->is_successful()); - FLAKY_TEST_SECTION_END(); - EXPECT_FALSE(result_ptr->response_info().adapter_responses().empty()); EXPECT_FALSE( result_ptr->response_info().mediation_adapter_class_name().empty()); @@ -1135,14 +1112,8 @@ TEST_F(FirebaseGmaTest, TestAdViewLoadAdAnchorAdaptiveAd) { kBannerAdUnit, banner_ad_size), "Initialize"); - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - WaitForCompletion(ad_view->LoadAd(GetAdRequest()), "LoadAd"); - FLAKY_TEST_SECTION_END(); - const AdSize ad_size = ad_view->ad_size(); EXPECT_EQ(ad_size.width(), kBannerWidth); EXPECT_NE(ad_size.height(), 0); @@ -1165,14 +1136,8 @@ TEST_F(FirebaseGmaTest, TestAdViewLoadAdInlineAdaptiveAd) { kBannerAdUnit, banner_ad_size), "Initialize"); - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - WaitForCompletion(ad_view->LoadAd(GetAdRequest()), "LoadAd"); - FLAKY_TEST_SECTION_END(); - const AdSize ad_size = ad_view->ad_size(); EXPECT_EQ(ad_size.width(), kBannerWidth); EXPECT_NE(ad_size.height(), 0); @@ -1194,14 +1159,8 @@ TEST_F(FirebaseGmaTest, TestAdViewLoadAdGetInlineAdaptiveBannerMaxHeight) { kBannerAdUnit, banner_ad_size), "Initialize"); - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - WaitForCompletion(ad_view->LoadAd(GetAdRequest()), "LoadAd"); - FLAKY_TEST_SECTION_END(); - const AdSize ad_size = ad_view->ad_size(); EXPECT_EQ(ad_size.width(), kBannerWidth); EXPECT_NE(ad_size.height(), 0); @@ -1221,15 +1180,7 @@ TEST_F(FirebaseGmaTest, TestAdViewLoadAdDestroyNotCalled) { WaitForCompletion(ad_view->Initialize(app_framework::GetWindowContext(), kBannerAdUnit, banner_ad_size), "Initialize"); - - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - WaitForCompletion(ad_view->LoadAd(GetAdRequest()), "LoadAd"); - - FLAKY_TEST_SECTION_END(); - delete ad_view; } @@ -1307,10 +1258,6 @@ TEST_F(FirebaseGmaTest, TestAdView) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - const firebase::gma::AdSize banner_ad_size(kBannerWidth, kBannerHeight); firebase::gma::AdView* ad_view = new firebase::gma::AdView(); WaitForCompletion(ad_view->Initialize(app_framework::GetWindowContext(), @@ -1506,8 +1453,6 @@ TEST_F(FirebaseGmaTest, TestAdView) { bounding_box_listener.bounding_box_changes_.back().height == -1); #endif // defined(ANDROID) || TARGET_OS_IPHONE } - - FLAKY_TEST_SECTION_END(); } TEST_F(FirebaseGmaTest, TestAdViewErrorNotInitialized) { @@ -1676,10 +1621,6 @@ TEST_F(FirebaseGmaTest, TestInterstitialAdLoadEmptyRequest) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; - // Loading Ads has been deemed flaky as the AdMob Service has a chance to - // return NoFill for valid ad requests if there aren't any ads to serve. - FLAKY_TEST_SECTION_BEGIN(); - firebase::gma::InterstitialAd* interstitial = new firebase::gma::InterstitialAd(); @@ -1703,8 +1644,6 @@ TEST_F(FirebaseGmaTest, TestInterstitialAdLoadEmptyRequest) { EXPECT_FALSE(result_ptr->response_info().ToString().empty()); delete interstitial; - - FLAKY_TEST_SECTION_END(); } TEST_F(FirebaseGmaTest, TestInterstitialAdErrorNotInitialized) { From cb9ac4ca2b743e5c5d04c950596b5a4928391dbb Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Tue, 3 Jan 2023 15:36:13 -0500 Subject: [PATCH 4/6] skip TestRewardedAdLoad --- gma/integration_test/src/integration_test.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gma/integration_test/src/integration_test.cc b/gma/integration_test/src/integration_test.cc index e517dad718..153170c4b0 100644 --- a/gma/integration_test/src/integration_test.cc +++ b/gma/integration_test/src/integration_test.cc @@ -833,6 +833,9 @@ TEST_F(FirebaseGmaTest, TestRewardedAdLoad) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_SIMULATOR; + // TODO(@drsanta): remove when GMA whitelists CI devices. + TEST_REQUIRES_USER_INTERACTION_ON_IOS; + firebase::gma::RewardedAd* rewarded = new firebase::gma::RewardedAd(); WaitForCompletion(rewarded->Initialize(app_framework::GetWindowContext()), From cefdbd3a39c1acb48d039e55d7f57bd49792bfc7 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Tue, 3 Jan 2023 15:43:04 -0500 Subject: [PATCH 5/6] added rewardedad stress tests, but succeed on NoFill responses --- gma/integration_test/src/integration_test.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/gma/integration_test/src/integration_test.cc b/gma/integration_test/src/integration_test.cc index 153170c4b0..1651b91f55 100644 --- a/gma/integration_test/src/integration_test.cc +++ b/gma/integration_test/src/integration_test.cc @@ -2020,9 +2020,6 @@ TEST_F(FirebaseGmaTest, TestRewardedAdStress) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_EMULATOR; - // TODO(@drsanta): remove when GMA whitelists CI devices. - TEST_REQUIRES_USER_INTERACTION_ON_IOS; - for (int i = 0; i < 10; ++i) { firebase::gma::RewardedAd* rewarded = new firebase::gma::RewardedAd(); From 8a9f953a213a408b17cd58b0a3bcbfea413e3413 Mon Sep 17 00:00:00 2001 From: "drsanta@google.com" Date: Tue, 3 Jan 2023 16:47:19 -0500 Subject: [PATCH 6/6] remove stress test --- gma/integration_test/src/integration_test.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gma/integration_test/src/integration_test.cc b/gma/integration_test/src/integration_test.cc index 1651b91f55..153170c4b0 100644 --- a/gma/integration_test/src/integration_test.cc +++ b/gma/integration_test/src/integration_test.cc @@ -2020,6 +2020,9 @@ TEST_F(FirebaseGmaTest, TestRewardedAdStress) { SKIP_TEST_ON_DESKTOP; SKIP_TEST_ON_EMULATOR; + // TODO(@drsanta): remove when GMA whitelists CI devices. + TEST_REQUIRES_USER_INTERACTION_ON_IOS; + for (int i = 0; i < 10; ++i) { firebase::gma::RewardedAd* rewarded = new firebase::gma::RewardedAd();