diff --git a/OptimizelySDK.Tests/Assertions.cs b/OptimizelySDK.Tests/Assertions.cs index 9dfe2f7b..3544d621 100644 --- a/OptimizelySDK.Tests/Assertions.cs +++ b/OptimizelySDK.Tests/Assertions.cs @@ -488,7 +488,7 @@ public static void AreEqual(Experiment expected, Experiment actual) Assert.AreEqual(expected.GroupId, actual.GroupId); Assert.AreEqual(expected.GroupPolicy, actual.GroupPolicy); Assert.AreEqual(expected.Id, actual.Id); - Assert.AreEqual(expected.IsExperimentRunning, actual.IsExperimentRunning); + Assert.AreEqual(expected.isRunning, actual.isRunning); Assert.AreEqual(expected.IsInMutexGroup, actual.IsInMutexGroup); Assert.AreEqual(expected.Key, actual.Key); Assert.AreEqual(expected.LayerId, actual.LayerId); @@ -500,6 +500,33 @@ public static void AreEqual(Experiment expected, Experiment actual) AreEquivalent(expected.Variations, actual.Variations); } + public static void AreEqual(ExperimentCore expected, ExperimentCore actual) + { + if (expected == null && actual == null) + { + return; + } + + Assert.IsNotNull(expected, "Expected ExperimentCore should not be null"); + Assert.IsNotNull(actual, "Actual ExperimentCore should not be null"); + + Assert.AreEqual(expected.AudienceConditions, actual.AudienceConditions); + Assert.AreEqual(expected.AudienceConditionsList, actual.AudienceConditionsList); + Assert.AreEqual(expected.AudienceConditionsString, actual.AudienceConditionsString); + AreEquivalent(expected.AudienceIds, actual.AudienceIds); + Assert.AreEqual(expected.AudienceIdsList, actual.AudienceIdsList); + Assert.AreEqual(expected.AudienceIdsString, actual.AudienceIdsString); + Assert.AreEqual(expected.Id, actual.Id); + Assert.AreEqual(expected.isRunning, actual.isRunning); + Assert.AreEqual(expected.Key, actual.Key); + Assert.AreEqual(expected.LayerId, actual.LayerId); + Assert.AreEqual(expected.Status, actual.Status); + AreEquivalent(expected.TrafficAllocation, actual.TrafficAllocation); + AreEquivalent(expected.VariationIdToVariationMap, actual.VariationIdToVariationMap); + AreEquivalent(expected.VariationKeyToVariationMap, actual.VariationKeyToVariationMap); + AreEquivalent(expected.Variations, actual.Variations); + } + #endregion Experiment #region FeatureDecision @@ -507,6 +534,8 @@ public static void AreEqual(Experiment expected, Experiment actual) public static void AreEqual(FeatureDecision expected, FeatureDecision actual) { AreEqual(expected.Experiment, actual.Experiment); + AreEqual(expected.Variation, actual.Variation); + Assert.AreEqual(expected.Source, actual.Source); } #endregion FeatureDecision diff --git a/OptimizelySDK.Tests/BucketerHoldoutTest.cs b/OptimizelySDK.Tests/BucketerHoldoutTest.cs new file mode 100644 index 00000000..742d8215 --- /dev/null +++ b/OptimizelySDK.Tests/BucketerHoldoutTest.cs @@ -0,0 +1,369 @@ +/* + * Copyright 2025, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.IO; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using OptimizelySDK.Bucketing; +using OptimizelySDK.Config; +using OptimizelySDK.Entity; +using OptimizelySDK.Logger; +using OptimizelySDK.OptimizelyDecisions; + +namespace OptimizelySDK.Tests +{ + [TestFixture] + public class BucketerHoldoutTest + { + private Mock LoggerMock; + private Bucketer Bucketer; + private TestBucketer TestBucketer; + private ProjectConfig Config; + private JObject TestData; + private const string TestUserId = "test_user_id"; + private const string TestBucketingId = "test_bucketing_id"; + + [SetUp] + public void Initialize() + { + LoggerMock = new Mock(); + + // Load holdout test data + var testDataPath = Path.Combine(TestContext.CurrentContext.TestDirectory, + "TestData", "HoldoutTestData.json"); + var jsonContent = File.ReadAllText(testDataPath); + TestData = JObject.Parse(jsonContent); + + // Use datafile with holdouts for proper config setup + var datafileWithHoldouts = TestData["datafileWithHoldouts"].ToString(); + Config = DatafileProjectConfig.Create(datafileWithHoldouts, LoggerMock.Object, + new ErrorHandler.NoOpErrorHandler()); + TestBucketer = new TestBucketer(LoggerMock.Object); + + // Verify that the config contains holdouts + Assert.IsNotNull(Config.Holdouts, "Config should have holdouts"); + Assert.IsTrue(Config.Holdouts.Length > 0, "Config should contain holdouts"); + } + + [Test] + public void TestBucketHoldout_ValidTrafficAllocation() + { + // Test user bucketed within traffic allocation range + // Use the global holdout from config which has multiple variations + var holdout = Config.GetHoldout("holdout_global_1"); + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + // Set bucket value to be within first variation's traffic allocation (0-5000 range) + TestBucketer.SetBucketValues(new[] { 2500 }); + + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNotNull(result.ResultObject); + Assert.AreEqual("var_1", result.ResultObject.Id); + Assert.AreEqual("control", result.ResultObject.Key); + + // Verify logging + LoggerMock.Verify(l => l.Log(LogLevel.DEBUG, + It.Is(s => s.Contains($"Assigned bucket [2500] to user [{TestUserId}]"))), + Times.Once); + } + + [Test] + public void TestBucketHoldout_UserOutsideAllocation() + { + // Test user not bucketed when outside traffic allocation range + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + // Modify traffic allocation to be smaller (0-1000 range = 10%) + holdout.TrafficAllocation[0].EndOfRange = 1000; + + // Set bucket value outside traffic allocation range + TestBucketer.SetBucketValues(new[] { 1500 }); + + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNull(result.ResultObject.Id); + Assert.IsNull(result.ResultObject.Key); + + // Verify user was assigned bucket value but no variation was found + LoggerMock.Verify(l => l.Log(LogLevel.DEBUG, + It.Is(s => s.Contains($"Assigned bucket [1500] to user [{TestUserId}]"))), + Times.Once); + } + + [Test] + public void TestBucketHoldout_NoTrafficAllocation() + { + // Test holdout with empty traffic allocation + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + // Clear traffic allocation + holdout.TrafficAllocation = new TrafficAllocation[0]; + + TestBucketer.SetBucketValues(new[] { 5000 }); + + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNull(result.ResultObject.Id); + Assert.IsNull(result.ResultObject.Key); + + // Verify bucket was assigned but no variation found + LoggerMock.Verify(l => l.Log(LogLevel.DEBUG, + It.Is(s => s.Contains($"Assigned bucket [5000] to user [{TestUserId}]"))), + Times.Once); + } + + [Test] + public void TestBucketHoldout_InvalidVariationId() + { + // Test holdout with invalid variation ID in traffic allocation + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + // Set traffic allocation to point to non-existent variation + holdout.TrafficAllocation[0].EntityId = "invalid_variation_id"; + + TestBucketer.SetBucketValues(new[] { 5000 }); + + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNull(result.ResultObject.Id); + Assert.IsNull(result.ResultObject.Key); + + // Verify bucket was assigned + LoggerMock.Verify(l => l.Log(LogLevel.DEBUG, + It.Is(s => s.Contains($"Assigned bucket [5000] to user [{TestUserId}]"))), + Times.Once); + } + + [Test] + public void TestBucketHoldout_EmptyVariations() + { + // Test holdout with no variations - use holdout from datafile that has no variations + var holdout = Config.GetHoldout("holdout_empty_1"); + Assert.IsNotNull(holdout, "Empty holdout should exist in config"); + Assert.AreEqual(0, holdout.Variations?.Length ?? 0, "Holdout should have no variations"); + + TestBucketer.SetBucketValues(new[] { 5000 }); + + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNull(result.ResultObject.Id); + Assert.IsNull(result.ResultObject.Key); + + // Verify bucket was assigned + LoggerMock.Verify(l => l.Log(LogLevel.DEBUG, + It.Is(s => s.Contains($"Assigned bucket [5000] to user [{TestUserId}]"))), + Times.Once); + } + + [Test] + public void TestBucketHoldout_EmptyExperimentKey() + { + // Test holdout with empty key + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + // Clear holdout key + holdout.Key = ""; + + TestBucketer.SetBucketValues(new[] { 5000 }); + + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + // Should return empty variation for invalid experiment key + Assert.IsNotNull(result.ResultObject); + Assert.IsNull(result.ResultObject.Id); + Assert.IsNull(result.ResultObject.Key); + } + + [Test] + public void TestBucketHoldout_NullExperimentKey() + { + // Test holdout with null key + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + // Set holdout key to null + holdout.Key = null; + + TestBucketer.SetBucketValues(new[] { 5000 }); + + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + // Should return empty variation for null experiment key + Assert.IsNotNull(result.ResultObject); + Assert.IsNull(result.ResultObject.Id); + Assert.IsNull(result.ResultObject.Key); + } + + [Test] + public void TestBucketHoldout_MultipleVariationsInRange() + { + // Test holdout with multiple variations and user buckets into first one + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + // Add a second variation + var variation2 = new Variation + { + Id = "var_2", + Key = "treatment", + FeatureEnabled = true + }; + holdout.Variations = new[] { holdout.Variations[0], variation2 }; + + // Set traffic allocation for first variation (0-5000) and second (5000-10000) + holdout.TrafficAllocation = new[] + { + new TrafficAllocation { EntityId = "var_1", EndOfRange = 5000 }, + new TrafficAllocation { EntityId = "var_2", EndOfRange = 10000 } + }; + + // Test user buckets into first variation + TestBucketer.SetBucketValues(new[] { 2500 }); + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNotNull(result.ResultObject); + Assert.AreEqual("var_1", result.ResultObject.Id); + Assert.AreEqual("control", result.ResultObject.Key); + } + + [Test] + public void TestBucketHoldout_MultipleVariationsInSecondRange() + { + // Test holdout with multiple variations and user buckets into second one + // Use the global holdout from config which now has multiple variations + var holdout = Config.GetHoldout("holdout_global_1"); + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + // Verify holdout has multiple variations + Assert.IsTrue(holdout.Variations.Length >= 2, "Holdout should have multiple variations"); + Assert.AreEqual("var_1", holdout.Variations[0].Id); + Assert.AreEqual("var_2", holdout.Variations[1].Id); + + // Test user buckets into second variation (bucket value 7500 should be in 5000-10000 range) + TestBucketer.SetBucketValues(new[] { 7500 }); + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNotNull(result.ResultObject); + Assert.AreEqual("var_2", result.ResultObject.Id); + Assert.AreEqual("treatment", result.ResultObject.Key); + } + + [Test] + public void TestBucketHoldout_EdgeCaseBoundaryValues() + { + // Test edge cases at traffic allocation boundaries + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + // Set traffic allocation to 5000 (50%) + holdout.TrafficAllocation[0].EndOfRange = 5000; + + // Test exact boundary value (should be included) + TestBucketer.SetBucketValues(new[] { 4999 }); + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNotNull(result.ResultObject); + Assert.AreEqual("var_1", result.ResultObject.Id); + + // Test value just outside boundary (should not be included) + TestBucketer.SetBucketValues(new[] { 5000 }); + result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNull(result.ResultObject.Id); + } + + [Test] + public void TestBucketHoldout_ConsistentBucketingWithSameInputs() + { + // Test that same inputs produce consistent results + // Use holdout from config instead of creating at runtime + var holdout = Config.GetHoldout("holdout_global_1"); + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + // Create a real bucketer (not test bucketer) for consistent hashing + var realBucketer = new Bucketing.Bucketer(LoggerMock.Object); + var result1 = realBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + var result2 = realBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + // Results should be identical + Assert.IsNotNull(result1); + Assert.IsNotNull(result2); + + if (result1.ResultObject?.Id != null) + { + Assert.AreEqual(result1.ResultObject.Id, result2.ResultObject.Id); + Assert.AreEqual(result1.ResultObject.Key, result2.ResultObject.Key); + } + else + { + Assert.IsNull(result2.ResultObject?.Id); + } + } + + [Test] + public void TestBucketHoldout_DifferentBucketingIdsProduceDifferentResults() + { + // Test that different bucketing IDs can produce different results + // Use holdout from config instead of creating at runtime + var holdout = Config.GetHoldout("holdout_global_1"); + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + // Create a real bucketer (not test bucketer) for real hashing behavior + var realBucketer = new Bucketing.Bucketer(LoggerMock.Object); + var result1 = realBucketer.Bucket(Config, holdout, "bucketingId1", TestUserId); + var result2 = realBucketer.Bucket(Config, holdout, "bucketingId2", TestUserId); + + // Results may be different (though not guaranteed due to hashing) + // This test mainly ensures no exceptions are thrown with different inputs + Assert.IsNotNull(result1); + Assert.IsNotNull(result2); + Assert.IsNotNull(result1.ResultObject); + Assert.IsNotNull(result2.ResultObject); + } + + [Test] + public void TestBucketHoldout_VerifyDecisionReasons() + { + // Test that decision reasons are properly populated + var holdoutJson = TestData["globalHoldout"].ToString(); + var holdout = JsonConvert.DeserializeObject(holdoutJson); + + TestBucketer.SetBucketValues(new[] { 5000 }); + var result = TestBucketer.Bucket(Config, holdout, TestBucketingId, TestUserId); + + Assert.IsNotNull(result.DecisionReasons); + // Decision reasons should be populated from the bucketing process + // The exact content depends on whether the user was bucketed or not + } + + [TearDown] + public void TearDown() + { + LoggerMock = null; + Bucketer = null; + TestBucketer = null; + Config = null; + TestData = null; + } + } +} diff --git a/OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs b/OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs new file mode 100644 index 00000000..a9457cdb --- /dev/null +++ b/OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs @@ -0,0 +1,246 @@ +/* + * Copyright 2025, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Moq; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using OptimizelySDK.Bucketing; +using OptimizelySDK.Config; +using OptimizelySDK.Entity; +using OptimizelySDK.ErrorHandler; +using OptimizelySDK.Logger; +using OptimizelySDK.OptimizelyDecisions; + +namespace OptimizelySDK.Tests +{ + [TestFixture] + public class DecisionServiceHoldoutTest + { + private Mock LoggerMock; + private DecisionService DecisionService; + private DatafileProjectConfig Config; + private JObject TestData; + private Optimizely OptimizelyInstance; + + private const string TestUserId = "testUserId"; + private const string TestBucketingId = "testBucketingId"; + + [SetUp] + public void Initialize() + { + LoggerMock = new Mock(); + + // Load test data + var testDataPath = Path.Combine(TestContext.CurrentContext.TestDirectory, + "TestData", "HoldoutTestData.json"); + var jsonContent = File.ReadAllText(testDataPath); + TestData = JObject.Parse(jsonContent); + + // Use datafile with holdouts for proper config setup + var datafileWithHoldouts = TestData["datafileWithHoldouts"].ToString(); + Config = DatafileProjectConfig.Create(datafileWithHoldouts, LoggerMock.Object, + new ErrorHandler.NoOpErrorHandler()) as DatafileProjectConfig; + + // Use real Bucketer instead of mock + var realBucketer = new Bucketer(LoggerMock.Object); + DecisionService = new DecisionService(realBucketer, + new ErrorHandler.NoOpErrorHandler(), null, LoggerMock.Object); + + // Create an Optimizely instance for creating user contexts + var eventDispatcher = new Event.Dispatcher.DefaultEventDispatcher(LoggerMock.Object); + OptimizelyInstance = new Optimizely(datafileWithHoldouts, eventDispatcher, LoggerMock.Object); + + // Verify that the config contains holdouts + Assert.IsNotNull(Config.Holdouts, "Config should have holdouts"); + Assert.IsTrue(Config.Holdouts.Length > 0, "Config should contain holdouts"); + } + + [Test] + public void TestGetVariationsForFeatureList_HoldoutActiveVariationBucketed() + { + // Test GetVariationsForFeatureList with holdout that has an active variation + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data + var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1 + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + // Create user context + var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null, + new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object); + + var result = DecisionService.GetVariationsForFeatureList( + new List { featureFlag }, userContext, Config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + Assert.IsNotNull(result); + Assert.IsTrue(result.Count > 0, "Should have at least one decision"); + + // Find the holdout decision + var holdoutDecision = result.FirstOrDefault(r => r.ResultObject?.Source == FeatureDecision.DECISION_SOURCE_HOLDOUT); + Assert.IsNotNull(holdoutDecision, "Should have a holdout decision"); + + // Verify that we got a valid variation (real bucketer should determine this based on traffic allocation) + Assert.IsNotNull(holdoutDecision.ResultObject?.Variation, "Should have a variation"); + } + + [Test] + public void TestGetVariationsForFeatureList_HoldoutInactiveNoBucketing() + { + // Test that inactive holdouts don't bucket users + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data + + // Get one of the holdouts that's actually processed for test_flag_1 (based on debug output) + var holdout = Config.GetHoldout("holdout_global_1"); // global_holdout is one of the holdouts being processed + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + // Mock holdout as inactive + holdout.Status = "Paused"; + + var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null, + new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object); + + var result = DecisionService.GetVariationsForFeatureList( + new List { featureFlag }, userContext, Config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + // Verify appropriate log message for inactive holdout + LoggerMock.Verify(l => l.Log(LogLevel.INFO, + It.Is(s => s.Contains("Holdout") && s.Contains("is not running"))), + Times.AtLeastOnce); + } + + [Test] + public void TestGetVariationsForFeatureList_HoldoutUserNotBucketed() + { + // Test when user is not bucketed into holdout (outside traffic allocation) + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data + var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1 + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null, + new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object); + + var result = DecisionService.GetVariationsForFeatureList( + new List { featureFlag }, userContext, Config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + // With real bucketer, we can't guarantee specific bucketing results + // but we can verify the method executes successfully + Assert.IsNotNull(result, "Result should not be null"); + } + + [Test] + public void TestGetVariationsForFeatureList_HoldoutWithUserAttributes() + { + // Test holdout evaluation with user attributes for audience targeting + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data + var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1 + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + var userAttributes = new UserAttributes + { + { "browser", "chrome" }, + { "location", "us" } + }; + + var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, userAttributes, + new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object); + + var result = DecisionService.GetVariationsForFeatureList( + new List { featureFlag }, userContext, Config, + userAttributes, new OptimizelyDecideOption[0]); + + Assert.IsNotNull(result, "Result should not be null"); + + // With real bucketer, we can't guarantee specific variations but can verify execution + // Additional assertions would depend on the holdout configuration and user bucketing + } + + [Test] + public void TestGetVariationsForFeatureList_MultipleHoldouts() + { + // Test multiple holdouts for a single feature flag + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data + + var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null, + new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object); + + var result = DecisionService.GetVariationsForFeatureList( + new List { featureFlag }, userContext, Config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + Assert.IsNotNull(result, "Result should not be null"); + + // With real bucketer, we can't guarantee specific bucketing results + // but we can verify the method executes successfully + } + + [Test] + public void TestGetVariationsForFeatureList_Holdout_EmptyUserId() + { + // Test GetVariationsForFeatureList with empty user ID + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data + + var userContext = new OptimizelyUserContext(OptimizelyInstance, "", null, + new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object); + + var result = DecisionService.GetVariationsForFeatureList( + new List { featureFlag }, userContext, Config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + Assert.IsNotNull(result); + + // Empty user ID should still allow holdout bucketing (matches Swift SDK behavior) + // The Swift SDK's testBucketToVariation_EmptyBucketingId shows empty string is valid + var holdoutDecisions = result.Where(r => r.ResultObject?.Source == FeatureDecision.DECISION_SOURCE_HOLDOUT).ToList(); + + // Should not log error about invalid user ID since empty string is valid for bucketing + LoggerMock.Verify(l => l.Log(LogLevel.ERROR, + It.Is(s => s.Contains("User ID") && (s.Contains("null") || s.Contains("empty")))), + Times.Never); + } + + [Test] + public void TestGetVariationsForFeatureList_Holdout_DecisionReasons() + { + // Test that decision reasons are properly populated for holdouts + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data + var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1 + Assert.IsNotNull(holdout, "Holdout should exist in config"); + + var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null, + new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object); + + var result = DecisionService.GetVariationsForFeatureList( + new List { featureFlag }, userContext, Config, + new UserAttributes(), new OptimizelyDecideOption[0]); + + Assert.IsNotNull(result, "Result should not be null"); + + // With real bucketer, we expect proper decision reasons to be generated + // Find any decision with reasons + var decisionWithReasons = result.FirstOrDefault(r => r.DecisionReasons != null && r.DecisionReasons.ToReport().Count > 0); + + if (decisionWithReasons != null) + { + Assert.IsTrue(decisionWithReasons.DecisionReasons.ToReport().Count > 0, "Should have decision reasons"); + } + } + } +} diff --git a/OptimizelySDK.Tests/EntityTests/HoldoutTests.cs b/OptimizelySDK.Tests/EntityTests/HoldoutTests.cs index c17cc088..a0f16fd0 100644 --- a/OptimizelySDK.Tests/EntityTests/HoldoutTests.cs +++ b/OptimizelySDK.Tests/EntityTests/HoldoutTests.cs @@ -115,9 +115,6 @@ public void TestHoldoutEquality() Assert.IsNotNull(holdout1); Assert.IsNotNull(holdout2); - // Note: This test depends on how Holdout implements equality - // If Holdout doesn't override Equals, this will test reference equality - // You may need to implement custom equality logic for Holdout } [Test] @@ -128,10 +125,6 @@ public void TestHoldoutStatusParsing() Assert.IsNotNull(globalHoldout); Assert.AreEqual("Running", globalHoldout.Status); - - // Test that the holdout is considered activated when status is "Running" - // This assumes there's an IsActivated property or similar logic - // Adjust based on actual Holdout implementation } [Test] @@ -184,9 +177,6 @@ public void TestHoldoutNullSafety() Assert.IsNotNull(holdout); Assert.AreEqual("test_holdout", holdout.Id); Assert.AreEqual("test_key", holdout.Key); - - // Verify that missing includedFlags and excludedFlags are handled properly - // This depends on how the Holdout entity handles missing properties Assert.IsNotNull(holdout.IncludedFlags); Assert.IsNotNull(holdout.ExcludedFlags); } diff --git a/OptimizelySDK.Tests/OptimizelySDK.Tests.csproj b/OptimizelySDK.Tests/OptimizelySDK.Tests.csproj index 1db35b8f..026dd5b8 100644 --- a/OptimizelySDK.Tests/OptimizelySDK.Tests.csproj +++ b/OptimizelySDK.Tests/OptimizelySDK.Tests.csproj @@ -106,6 +106,9 @@ + + + diff --git a/OptimizelySDK.Tests/OptimizelyUserContextHoldoutTest.cs b/OptimizelySDK.Tests/OptimizelyUserContextHoldoutTest.cs new file mode 100644 index 00000000..369dabb9 --- /dev/null +++ b/OptimizelySDK.Tests/OptimizelyUserContextHoldoutTest.cs @@ -0,0 +1,577 @@ +/* + * Copyright 2025, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Moq; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using OptimizelySDK.Bucketing; +using OptimizelySDK.Config; +using OptimizelySDK.Entity; +using OptimizelySDK.ErrorHandler; +using OptimizelySDK.Event; +using OptimizelySDK.Event.Dispatcher; +using OptimizelySDK.Logger; +using OptimizelySDK.OptimizelyDecisions; + +namespace OptimizelySDK.Tests +{ + [TestFixture] + public class OptimizelyUserContextHoldoutTest + { + private Mock LoggerMock; + private Mock EventDispatcherMock; + private DatafileProjectConfig Config; + private JObject TestData; + private Optimizely OptimizelyInstance; + + private const string TestUserId = "testUserId"; + private const string TestBucketingId = "testBucketingId"; + + [SetUp] + public void Initialize() + { + LoggerMock = new Mock(); + EventDispatcherMock = new Mock(); + + // Load test data + var testDataPath = Path.Combine(TestContext.CurrentContext.TestDirectory, + "TestData", "HoldoutTestData.json"); + var jsonContent = File.ReadAllText(testDataPath); + TestData = JObject.Parse(jsonContent); + + // Use datafile with holdouts for proper config setup + var datafileWithHoldouts = TestData["datafileWithHoldouts"].ToString(); + + // Create an Optimizely instance with the test data + OptimizelyInstance = new Optimizely(datafileWithHoldouts, EventDispatcherMock.Object, LoggerMock.Object); + + // Get the config from the Optimizely instance to ensure they're synchronized + Config = OptimizelyInstance.ProjectConfigManager.GetConfig() as DatafileProjectConfig; + + // Verify that the config contains holdouts + Assert.IsNotNull(Config.Holdouts, "Config should have holdouts"); + Assert.IsTrue(Config.Holdouts.Length > 0, "Config should contain holdouts"); + } + + #region Core Holdout Functionality Tests + + [Test] + public void TestDecide_GlobalHoldout() + { + // Test Decide() method with global holdout decision + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; + Assert.IsNotNull(featureFlag, "Feature flag should exist"); + + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + var decision = userContext.Decide("test_flag_1"); + + Assert.IsNotNull(decision, "Decision should not be null"); + Assert.AreEqual("test_flag_1", decision.FlagKey, "Flag key should match"); + + // With real bucketer, we can't guarantee specific variation but can verify structure + // The decision should either be from holdout, experiment, or rollout + Assert.IsTrue(!string.IsNullOrEmpty(decision.VariationKey) || decision.VariationKey == null, + "Variation key should be valid or null"); + } + + [Test] + public void TestDecide_IncludedFlagsHoldout() + { + // Test holdout with includedFlags configuration + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; + Assert.IsNotNull(featureFlag, "Feature flag should exist"); + + // Check if there's a holdout that includes this flag + var includedHoldout = Config.Holdouts.FirstOrDefault(h => + h.IncludedFlags != null && h.IncludedFlags.Contains(featureFlag.Id)); + + if (includedHoldout != null) + { + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + var decision = userContext.Decide("test_flag_1"); + + Assert.IsNotNull(decision, "Decision should not be null"); + Assert.AreEqual("test_flag_1", decision.FlagKey, "Flag key should match"); + + // Verify decision is valid + Assert.IsTrue(decision.VariationKey != null || decision.VariationKey == null, + "Decision should have valid structure"); + } + else + { + Assert.Inconclusive("No included holdout found for test_flag_1"); + } + } + + [Test] + public void TestDecide_ExcludedFlagsHoldout() + { + // Test holdout with excludedFlags configuration + // Based on test data, flag_3 and flag_4 are excluded by holdout_excluded_1 + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + // Test with an excluded flag (test_flag_3 maps to flag_3) + var excludedDecision = userContext.Decide("test_flag_3"); + + Assert.IsNotNull(excludedDecision, "Decision should not be null for excluded flag"); + Assert.AreEqual("test_flag_3", excludedDecision.FlagKey, "Flag key should match"); + + // For excluded flags, the decision should not come from the excluded holdout + // The excluded holdout has key "excluded_holdout" + Assert.AreNotEqual("excluded_holdout", excludedDecision.RuleKey, + "Decision should not come from excluded holdout for flag_3"); + + // Also test with a non-excluded flag (test_flag_1 maps to flag_1) + var nonExcludedDecision = userContext.Decide("test_flag_1"); + + Assert.IsNotNull(nonExcludedDecision, "Decision should not be null for non-excluded flag"); + Assert.AreEqual("test_flag_1", nonExcludedDecision.FlagKey, "Flag key should match"); + + // For non-excluded flags, they can potentially be affected by holdouts + // (depending on other holdout configurations like global or included holdouts) + } + + [Test] + public void TestDecideAll_MultipleHoldouts() + { + // Test DecideAll() with multiple holdouts affecting different flags + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + var decisions = userContext.DecideAll(); + + Assert.IsNotNull(decisions, "Decisions should not be null"); + Assert.IsTrue(decisions.Count > 0, "Should have at least one decision"); + + // Verify each decision has proper structure + foreach (var kvp in decisions) + { + var flagKey = kvp.Key; + var decision = kvp.Value; + + Assert.AreEqual(flagKey, decision.FlagKey, $"Flag key should match for {flagKey}"); + Assert.IsNotNull(decision, $"Decision should not be null for {flagKey}"); + + // Decision should have either a variation or be properly null + Assert.IsTrue(decision.VariationKey != null || decision.VariationKey == null, + $"Decision structure should be valid for {flagKey}"); + } + } + + [Test] + public void TestDecide_HoldoutImpressionEvent() + { + // Test that impression events are sent for holdout decisions + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + var decision = userContext.Decide("test_flag_1"); + + Assert.IsNotNull(decision, "Decision should not be null"); + + // Verify that event dispatcher was called + // Note: With real bucketer, we can't guarantee holdout selection, + // but we can verify event structure + EventDispatcherMock.Verify( + e => e.DispatchEvent(It.IsAny()), + Times.AtLeastOnce, + "Event should be dispatched for decision" + ); + } + + [Test] + public void TestDecide_HoldoutWithDecideOptions() + { + // Test decide options (like ExcludeVariables) with holdout decisions + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + // Test with exclude variables option + var decisionWithVariables = userContext.Decide("test_flag_1"); + var decisionWithoutVariables = userContext.Decide("test_flag_1", + new OptimizelyDecideOption[] { OptimizelyDecideOption.EXCLUDE_VARIABLES }); + + Assert.IsNotNull(decisionWithVariables, "Decision with variables should not be null"); + Assert.IsNotNull(decisionWithoutVariables, "Decision without variables should not be null"); + + // When variables are excluded, the Variables object should be empty + Assert.IsTrue(decisionWithoutVariables.Variables.ToDictionary().Count == 0, + "Variables should be empty when excluded"); + } + + [Test] + public void TestDecide_HoldoutWithAudienceTargeting() + { + // Test holdout decisions with different user attributes for audience targeting + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; + Assert.IsNotNull(featureFlag, "Feature flag should exist"); + + // Test with matching attributes + var userContextMatch = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + var decisionMatch = userContextMatch.Decide("test_flag_1"); + + // Test with non-matching attributes + var userContextNoMatch = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "ca" } }); + var decisionNoMatch = userContextNoMatch.Decide("test_flag_1"); + + Assert.IsNotNull(decisionMatch, "Decision with matching attributes should not be null"); + Assert.IsNotNull(decisionNoMatch, "Decision with non-matching attributes should not be null"); + + // Both decisions should have proper structure regardless of targeting + Assert.AreEqual("test_flag_1", decisionMatch.FlagKey, "Flag key should match"); + Assert.AreEqual("test_flag_1", decisionNoMatch.FlagKey, "Flag key should match"); + } + + [Test] + public void TestDecide_InactiveHoldout() + { + // Test decide when holdout is not running + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; + Assert.IsNotNull(featureFlag, "Feature flag should exist"); + + // Find a holdout and set it to inactive + var holdout = Config.Holdouts.FirstOrDefault(); + if (holdout != null) + { + var originalStatus = holdout.Status; + holdout.Status = "Paused"; // Make holdout inactive + + try + { + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + var decision = userContext.Decide("test_flag_1"); + + Assert.IsNotNull(decision, "Decision should not be null even with inactive holdout"); + Assert.AreEqual("test_flag_1", decision.FlagKey, "Flag key should match"); + + // Should not get decision from the inactive holdout + if (!string.IsNullOrEmpty(decision.RuleKey)) + { + Assert.AreNotEqual(holdout.Key, decision.RuleKey, + "Decision should not come from inactive holdout"); + } + } + finally + { + holdout.Status = originalStatus; // Restore original status + } + } + else + { + Assert.Inconclusive("No holdout found to test inactive scenario"); + } + } + + [Test] + public void TestDecide_EmptyUserId() + { + // Test decide with empty user ID (should still work per Swift SDK behavior) + var userContext = OptimizelyInstance.CreateUserContext("", + new UserAttributes { { "country", "us" } }); + + var decision = userContext.Decide("test_flag_1"); + + Assert.IsNotNull(decision, "Decision should not be null with empty user ID"); + Assert.AreEqual("test_flag_1", decision.FlagKey, "Flag key should match"); + + // Should not log error about invalid user ID since empty string is valid for bucketing + LoggerMock.Verify(l => l.Log(LogLevel.ERROR, + It.Is(s => s.Contains("User ID") && (s.Contains("null") || s.Contains("empty")))), + Times.Never); + } + + [Test] + public void TestDecide_WithDecisionReasons() + { + // Test that decision reasons are properly populated for holdout decisions + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + var decision = userContext.Decide("test_flag_1", + new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + Assert.IsNotNull(decision, "Decision should not be null"); + Assert.AreEqual("test_flag_1", decision.FlagKey, "Flag key should match"); + + // Decision reasons should be populated when requested + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + // With real bucketer, we expect some decision reasons to be generated + Assert.IsTrue(decision.Reasons.Length >= 0, "Decision reasons should be present"); + } + + [Test] + public void TestDecide_HoldoutPriority() + { + // Test holdout evaluation priority (global vs included vs excluded) + var featureFlag = Config.FeatureKeyMap["test_flag_1"]; + Assert.IsNotNull(featureFlag, "Feature flag should exist"); + + // Check if we have multiple holdouts + var globalHoldouts = Config.Holdouts.Where(h => + h.IncludedFlags == null || h.IncludedFlags.Length == 0).ToList(); + var includedHoldouts = Config.Holdouts.Where(h => + h.IncludedFlags != null && h.IncludedFlags.Contains(featureFlag.Id)).ToList(); + + if (globalHoldouts.Count > 0 || includedHoldouts.Count > 0) + { + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + var decision = userContext.Decide("test_flag_1"); + + Assert.IsNotNull(decision, "Decision should not be null"); + Assert.AreEqual("test_flag_1", decision.FlagKey, "Flag key should match"); + + // Decision should be valid regardless of which holdout is selected + Assert.IsTrue(decision.VariationKey != null || decision.VariationKey == null, + "Decision should have valid structure"); + } + else + { + Assert.Inconclusive("No holdouts found to test priority"); + } + } + + #endregion + + #region Holdout Decision Reasons Tests + + [Test] + public void TestDecideReasons_WithIncludeReasonsOption() + { + var featureKey = "test_flag_1"; + + // Create user context + var userContext = OptimizelyInstance.CreateUserContext(TestUserId); + + // Call decide with reasons option + var decision = userContext.Decide(featureKey, new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + // Assertions + Assert.AreEqual(featureKey, decision.FlagKey, "Expected flagKey to match"); + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.IsTrue(decision.Reasons.Length >= 0, "Decision reasons should be present"); + } + + [Test] + public void TestDecideReasons_WithoutIncludeReasonsOption() + { + var featureKey = "test_flag_1"; + + // Create user context + var userContext = OptimizelyInstance.CreateUserContext(TestUserId); + + // Call decide WITHOUT reasons option + var decision = userContext.Decide(featureKey); + + // Assertions + Assert.AreEqual(featureKey, decision.FlagKey, "Expected flagKey to match"); + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.AreEqual(0, decision.Reasons.Length, "Should not include reasons when not requested"); + } + + [Test] + public void TestDecideReasons_UserBucketedIntoHoldoutVariation() + { + var featureKey = "test_flag_1"; + + // Create user context that should be bucketed into holdout + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + // Call decide with reasons + var decision = userContext.Decide(featureKey, new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + // Assertions + Assert.AreEqual(featureKey, decision.FlagKey, "Expected flagKey to match"); + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.IsTrue(decision.Reasons.Length > 0, "Should have decision reasons"); + + // Check for specific holdout bucketing messages (matching C# DecisionService patterns) + var reasonsText = string.Join(" ", decision.Reasons); + var hasHoldoutBucketingMessage = decision.Reasons.Any(r => + r.Contains("is bucketed into holdout variation") || + r.Contains("is not bucketed into holdout variation")); + + Assert.IsTrue(hasHoldoutBucketingMessage, + "Should contain holdout bucketing decision message"); + } + + [Test] + public void TestDecideReasons_HoldoutNotRunning() + { + // This test would require a holdout with inactive status + // For now, test that the structure is correct and reasons are generated + var featureKey = "test_flag_1"; + + var userContext = OptimizelyInstance.CreateUserContext(TestUserId); + var decision = userContext.Decide(featureKey, new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + // Verify reasons are generated (specific holdout status would depend on test data configuration) + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.IsTrue(decision.Reasons.Length > 0, "Should have decision reasons"); + + // Check if any holdout status messages are present + var hasHoldoutStatusMessage = decision.Reasons.Any(r => + r.Contains("is not running") || + r.Contains("is running") || + r.Contains("holdout")); + + // Note: This assertion may pass or fail depending on holdout configuration in test data + // The important thing is that reasons are being generated + } + + [Test] + public void TestDecideReasons_UserMeetsAudienceConditions() + { + var featureKey = "test_flag_1"; + + // Create user context with attributes that should match audience conditions + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + // Call decide with reasons + var decision = userContext.Decide(featureKey, new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + // Assertions + Assert.AreEqual(featureKey, decision.FlagKey, "Expected flagKey to match"); + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.IsTrue(decision.Reasons.Length > 0, "Should have decision reasons"); + + // Check for audience evaluation messages (matching C# ExperimentUtils patterns) + var hasAudienceEvaluation = decision.Reasons.Any(r => + r.Contains("Audiences for experiment") && r.Contains("collectively evaluated to")); + + Assert.IsTrue(hasAudienceEvaluation, + "Should contain audience evaluation result message"); + } + + [Test] + public void TestDecideReasons_UserDoesNotMeetHoldoutConditions() + { + var featureKey = "test_flag_1"; + + // Since the test holdouts have empty audience conditions (they match everyone), + // let's test with a holdout that's not running to simulate condition failure + // First, let's verify what's actually happening + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "unknown_country" } }); + + // Call decide with reasons + var decision = userContext.Decide(featureKey, new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + // Assertions + Assert.AreEqual(featureKey, decision.FlagKey, "Expected flagKey to match"); + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.IsTrue(decision.Reasons.Length > 0, "Should have decision reasons"); + + // Since the current test data holdouts have no audience restrictions, + // they evaluate to TRUE for any user. This is actually correct behavior. + // The test should verify that when audience conditions ARE met, we get appropriate messages. + var hasAudienceEvaluation = decision.Reasons.Any(r => + r.Contains("collectively evaluated to TRUE") || + r.Contains("collectively evaluated to FALSE") || + r.Contains("does not meet conditions")); + + Assert.IsTrue(hasAudienceEvaluation, + "Should contain audience evaluation message (TRUE or FALSE)"); + + // For this specific case with empty audience conditions, expect TRUE evaluation + var hasTrueEvaluation = decision.Reasons.Any(r => + r.Contains("collectively evaluated to TRUE")); + + Assert.IsTrue(hasTrueEvaluation, + "With empty audience conditions, should evaluate to TRUE"); + } + + [Test] + public void TestDecideReasons_HoldoutEvaluationReasoning() + { + var featureKey = "test_flag_1"; + + // Since the current test data doesn't include non-running holdouts, + // this test documents the expected behavior when a holdout is not running + var userContext = OptimizelyInstance.CreateUserContext(TestUserId); + + var decision = userContext.Decide(featureKey, new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + // Assertions + Assert.AreEqual(featureKey, decision.FlagKey, "Expected flagKey to match"); + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.IsTrue(decision.Reasons.Length > 0, "Should have decision reasons"); + + // Note: If we had a non-running holdout in the test data, we would expect: + // decision.Reasons.Any(r => r.Contains("is not running")) + + // For now, verify we get some form of holdout evaluation reasoning + var hasHoldoutReasoning = decision.Reasons.Any(r => + r.Contains("holdout") || + r.Contains("bucketed into")); + + Assert.IsTrue(hasHoldoutReasoning, + "Should contain holdout-related reasoning"); + } + + [Test] + public void TestDecideReasons_HoldoutDecisionContainsRelevantReasons() + { + var featureKey = "test_flag_1"; + + // Create user context that might be bucketed into holdout + var userContext = OptimizelyInstance.CreateUserContext(TestUserId, + new UserAttributes { { "country", "us" } }); + + // Call decide with reasons + var decision = userContext.Decide(featureKey, new OptimizelyDecideOption[] { OptimizelyDecideOption.INCLUDE_REASONS }); + + // Assertions + Assert.AreEqual(featureKey, decision.FlagKey, "Expected flagKey to match"); + Assert.IsNotNull(decision.Reasons, "Decision reasons should not be null"); + Assert.IsTrue(decision.Reasons.Length > 0, "Should have decision reasons"); + + // Check if reasons contain holdout-related information + var reasonsText = string.Join(" ", decision.Reasons); + + // Verify that reasons provide information about the decision process + Assert.IsTrue(!string.IsNullOrWhiteSpace(reasonsText), "Reasons should contain meaningful information"); + + // Check for any holdout-related keywords in reasons + var hasHoldoutRelatedReasons = decision.Reasons.Any(r => + r.Contains("holdout") || + r.Contains("bucketed") || + r.Contains("audiences") || + r.Contains("conditions")); + + Assert.IsTrue(hasHoldoutRelatedReasons, + "Should contain holdout-related decision reasoning"); + } + + #endregion + } +} diff --git a/OptimizelySDK.Tests/ProjectConfigTest.cs b/OptimizelySDK.Tests/ProjectConfigTest.cs index b7afcba6..15e95926 100644 --- a/OptimizelySDK.Tests/ProjectConfigTest.cs +++ b/OptimizelySDK.Tests/ProjectConfigTest.cs @@ -1368,7 +1368,7 @@ public void TestHoldoutDeserialization_FromDatafile() new NoOpLogger(), new NoOpErrorHandler()) as DatafileProjectConfig; Assert.IsNotNull(datafileProjectConfig.Holdouts); - Assert.AreEqual(3, datafileProjectConfig.Holdouts.Length); + Assert.AreEqual(4, datafileProjectConfig.Holdouts.Length); } [Test] @@ -1387,15 +1387,15 @@ public void TestGetHoldoutsForFlag_Integration() // Test GetHoldoutsForFlag method var holdoutsForFlag1 = datafileProjectConfig.GetHoldoutsForFlag("flag_1"); Assert.IsNotNull(holdoutsForFlag1); - Assert.AreEqual(3, holdoutsForFlag1.Length); // Global + excluded holdout (applies to all except flag_3/flag_4) + included holdout + Assert.AreEqual(4, holdoutsForFlag1.Length); // Global + excluded holdout (applies to all except flag_3/flag_4) + included holdout + empty holdout var holdoutsForFlag3 = datafileProjectConfig.GetHoldoutsForFlag("flag_3"); Assert.IsNotNull(holdoutsForFlag3); - Assert.AreEqual(1, holdoutsForFlag3.Length); // Only true global (excluded holdout excludes flag_3) + Assert.AreEqual(2, holdoutsForFlag3.Length); // Global + empty holdout (excluded holdout excludes flag_3, included holdout doesn't include flag_3) var holdoutsForUnknownFlag = datafileProjectConfig.GetHoldoutsForFlag("unknown_flag"); Assert.IsNotNull(holdoutsForUnknownFlag); - Assert.AreEqual(2, holdoutsForUnknownFlag.Length); // Global + excluded holdout (unknown_flag not in excluded list) + Assert.AreEqual(3, holdoutsForUnknownFlag.Length); // Global + excluded holdout (unknown_flag not in excluded list) + empty holdout } [Test] diff --git a/OptimizelySDK.Tests/TestData/HoldoutTestData.json b/OptimizelySDK.Tests/TestData/HoldoutTestData.json index b5c17b26..777c0a3a 100644 --- a/OptimizelySDK.Tests/TestData/HoldoutTestData.json +++ b/OptimizelySDK.Tests/TestData/HoldoutTestData.json @@ -126,11 +126,21 @@ "key": "control", "featureEnabled": false, "variables": [] + }, + { + "id": "var_2", + "key": "treatment", + "featureEnabled": true, + "variables": [] } ], "trafficAllocation": [ { "entityId": "var_1", + "endOfRange": 5000 + }, + { + "entityId": "var_2", "endOfRange": 10000 } ], @@ -186,6 +196,18 @@ "audienceConditions": [], "includedFlags": [], "excludedFlags": ["flag_3", "flag_4"] + }, + { + "id": "holdout_empty_1", + "key": "empty_holdout", + "status": "Running", + "layerId": "layer_4", + "variations": [], + "trafficAllocation": [], + "audienceIds": [], + "audienceConditions": [], + "includedFlags": [], + "excludedFlags": [] } ] } diff --git a/OptimizelySDK/Bucketing/Bucketer.cs b/OptimizelySDK/Bucketing/Bucketer.cs index 33df35a3..f891fc76 100644 --- a/OptimizelySDK/Bucketing/Bucketer.cs +++ b/OptimizelySDK/Bucketing/Bucketer.cs @@ -112,7 +112,7 @@ IEnumerable trafficAllocations /// A customer-assigned value used to create the key for the murmur hash. /// User identifier /// Variation which will be shown to the user - public virtual Result Bucket(ProjectConfig config, Experiment experiment, + public virtual Result Bucket(ProjectConfig config, ExperimentCore experiment, string bucketingId, string userId ) { @@ -127,9 +127,9 @@ public virtual Result Bucket(ProjectConfig config, Experiment experim } // Determine if experiment is in a mutually exclusive group. - if (experiment.IsInMutexGroup) + if (experiment is Experiment exp && exp.IsInMutexGroup) { - var group = config.GetGroup(experiment.GroupId); + var group = config.GetGroup(exp.GroupId); if (string.IsNullOrEmpty(group.Id)) { return Result.NewResult(new Variation(), reasons); @@ -147,13 +147,13 @@ public virtual Result Bucket(ProjectConfig config, Experiment experim if (userExperimentId != experiment.Id) { message = - $"User [{userId}] is not in experiment [{experiment.Key}] of group [{experiment.GroupId}]."; + $"User [{userId}] is not in experiment [{exp.Key}] of group [{exp.GroupId}]."; Logger.Log(LogLevel.INFO, reasons.AddInfo(message)); return Result.NewResult(new Variation(), reasons); } message = - $"User [{userId}] is in experiment [{experiment.Key}] of group [{experiment.GroupId}]."; + $"User [{userId}] is in experiment [{exp.Key}] of group [{exp.GroupId}]."; Logger.Log(LogLevel.INFO, reasons.AddInfo(message)); } diff --git a/OptimizelySDK/Bucketing/DecisionService.cs b/OptimizelySDK/Bucketing/DecisionService.cs index 1e364b29..ad5487a2 100644 --- a/OptimizelySDK/Bucketing/DecisionService.cs +++ b/OptimizelySDK/Bucketing/DecisionService.cs @@ -22,6 +22,7 @@ using OptimizelySDK.Logger; using OptimizelySDK.OptimizelyDecisions; using OptimizelySDK.Utils; +using static OptimizelySDK.Entity.Holdout; namespace OptimizelySDK.Bucketing { @@ -727,6 +728,84 @@ public virtual Result GetVariationForFeature(FeatureFlag featur new OptimizelyDecideOption[] { }); } + /// + /// Get the decision for a single feature flag, following Swift SDK pattern. + /// This method processes holdouts, experiments, and rollouts in sequence. + /// + /// The feature flag to get a decision for. + /// The user context. + /// The project config. + /// The user's filtered attributes. + /// Decision options. + /// User profile tracker for sticky bucketing. + /// Decision reasons to merge. + /// A decision result for the feature flag. + public virtual Result GetDecisionForFlag( + FeatureFlag featureFlag, + OptimizelyUserContext user, + ProjectConfig projectConfig, + UserAttributes filteredAttributes, + OptimizelyDecideOption[] options, + UserProfileTracker userProfileTracker = null, + DecisionReasons decideReasons = null + ) + { + var reasons = new DecisionReasons(); + if (decideReasons != null) + { + reasons += decideReasons; + } + + var userId = user.GetUserId(); + + // Check holdouts first (highest priority) + var holdouts = projectConfig.GetHoldoutsForFlag(featureFlag.Key); + foreach (var holdout in holdouts) + { + var holdoutDecision = GetVariationForHoldout(holdout, user, projectConfig); + reasons += holdoutDecision.DecisionReasons; + + if (holdoutDecision.ResultObject != null) + { + Logger.Log(LogLevel.INFO, + reasons.AddInfo( + $"The user \"{userId}\" is bucketed into holdout \"{holdout.Key}\" for feature flag \"{featureFlag.Key}\".")); + return Result.NewResult(holdoutDecision.ResultObject, reasons); + } + } + + // Check if the feature flag has an experiment and the user is bucketed into that experiment. + var experimentDecision = GetVariationForFeatureExperiment(featureFlag, user, + filteredAttributes, projectConfig, options, userProfileTracker); + reasons += experimentDecision.DecisionReasons; + + if (experimentDecision.ResultObject != null) + { + return Result.NewResult(experimentDecision.ResultObject, reasons); + } + + // Check if the feature flag has rollout and the user is bucketed into one of its rules. + var rolloutDecision = GetVariationForFeatureRollout(featureFlag, user, projectConfig); + reasons += rolloutDecision.DecisionReasons; + + if (rolloutDecision.ResultObject != null) + { + Logger.Log(LogLevel.INFO, + reasons.AddInfo( + $"The user \"{userId}\" is bucketed into a rollout for feature flag \"{featureFlag.Key}\".")); + return Result.NewResult(rolloutDecision.ResultObject, reasons); + } + else + { + Logger.Log(LogLevel.INFO, + reasons.AddInfo( + $"The user \"{userId}\" is not bucketed into a rollout for feature flag \"{featureFlag.Key}\".")); + return Result.NewResult( + new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_ROLLOUT), + reasons); + } + } + public virtual List> GetVariationsForFeatureList( List featureFlags, OptimizelyUserContext user, @@ -747,47 +826,13 @@ OptimizelyDecideOption[] options userProfileTracker.LoadUserProfile(upsReasons); } - var userId = user.GetUserId(); var decisions = new List>(); foreach (var featureFlag in featureFlags) { - var reasons = new DecisionReasons(); - reasons += upsReasons; - - // Check if the feature flag has an experiment and the user is bucketed into that experiment. - var decisionResult = GetVariationForFeatureExperiment(featureFlag, user, - filteredAttributes, projectConfig, options, userProfileTracker); - reasons += decisionResult.DecisionReasons; - - if (decisionResult.ResultObject != null) - { - decisions.Add( - Result.NewResult(decisionResult.ResultObject, reasons)); - continue; - } - - // Check if the feature flag has rollout and the the user is bucketed into one of its rules. - decisionResult = GetVariationForFeatureRollout(featureFlag, user, projectConfig); - reasons += decisionResult.DecisionReasons; - - if (decisionResult.ResultObject == null) - { - Logger.Log(LogLevel.INFO, - reasons.AddInfo( - $"The user \"{userId}\" is not bucketed into a rollout for feature flag \"{featureFlag.Key}\".")); - decisions.Add(Result.NewResult( - new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_ROLLOUT), - reasons)); - } - else - { - Logger.Log(LogLevel.INFO, - reasons.AddInfo( - $"The user \"{userId}\" is bucketed into a rollout for feature flag \"{featureFlag.Key}\".")); - decisions.Add( - Result.NewResult(decisionResult.ResultObject, reasons)); - } + var decision = GetDecisionForFlag(featureFlag, user, projectConfig, filteredAttributes, + options, userProfileTracker, upsReasons); + decisions.Add(decision); } if (UserProfileService != null && !ignoreUps && @@ -856,6 +901,66 @@ private Result GetBucketingId(string userId, UserAttributes filteredAttr return Result.NewResult(bucketingId, reasons); } + private Result GetVariationForHoldout( + Holdout holdout, + OptimizelyUserContext user, + ProjectConfig config + ) + { + var userId = user.GetUserId(); + var reasons = new DecisionReasons(); + + if (!holdout.isRunning) + { + var infoMessage = $"Holdout \"{holdout.Key}\" is not running."; + Logger.Log(LogLevel.INFO, infoMessage); + reasons.AddInfo(infoMessage); + return Result.NewResult( + new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_HOLDOUT), + reasons + ); + } + + var audienceResult = ExperimentUtils.DoesUserMeetAudienceConditions( + config, + holdout, + user, + LOGGING_KEY_TYPE_EXPERIMENT, + holdout.Key, + Logger + ); + reasons += audienceResult.DecisionReasons; + + if (!audienceResult.ResultObject) + { + reasons.AddInfo($"User \"{userId}\" does not meet conditions for holdout ({holdout.Key})."); + return Result.NewResult( + new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_HOLDOUT), + reasons + ); + } + + var attributes = user.GetAttributes(); + var bucketingIdResult = GetBucketingId(userId, attributes); + var bucketedVariation = Bucketer.Bucket(config, holdout, bucketingIdResult.ResultObject, userId); + reasons += bucketedVariation.DecisionReasons; + + if (bucketedVariation.ResultObject != null) + { + reasons.AddInfo($"User \"{userId}\" is bucketed into holdout variation \"{bucketedVariation.ResultObject.Key}\"."); + return Result.NewResult( + new FeatureDecision(holdout, bucketedVariation.ResultObject, FeatureDecision.DECISION_SOURCE_HOLDOUT), + reasons + ); + } + + reasons.AddInfo($"User \"{userId}\" is not bucketed into holdout variation \"{holdout.Key}\"."); + + return Result.NewResult( + new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_HOLDOUT), + reasons + ); + } /// /// Finds a validated forced decision. /// diff --git a/OptimizelySDK/Config/DatafileProjectConfig.cs b/OptimizelySDK/Config/DatafileProjectConfig.cs index f940ffe6..6721832e 100644 --- a/OptimizelySDK/Config/DatafileProjectConfig.cs +++ b/OptimizelySDK/Config/DatafileProjectConfig.cs @@ -413,6 +413,29 @@ private void Initialize() } } + // Adding Holdout variations in variation id and key maps. + if (Holdouts != null) + { + foreach (var holdout in Holdouts) + { + _VariationKeyMap[holdout.Key] = new Dictionary(); + _VariationIdMap[holdout.Key] = new Dictionary(); + _VariationIdMapByExperimentId[holdout.Id] = new Dictionary(); + _VariationKeyMapByExperimentId[holdout.Id] = new Dictionary(); + + if (holdout.Variations != null) + { + foreach (var variation in holdout.Variations) + { + _VariationKeyMap[holdout.Key][variation.Key] = variation; + _VariationIdMap[holdout.Key][variation.Id] = variation; + _VariationKeyMapByExperimentId[holdout.Id][variation.Key] = variation; + _VariationIdMapByExperimentId[holdout.Id][variation.Id] = variation; + } + } + } + } + var integration = Integrations.FirstOrDefault(i => i.Key.ToLower() == "odp"); HostForOdp = integration?.Host; PublicKeyForOdp = integration?.PublicKey; diff --git a/OptimizelySDK/Entity/Experiment.cs b/OptimizelySDK/Entity/Experiment.cs index dd25f68c..8a7b4036 100644 --- a/OptimizelySDK/Entity/Experiment.cs +++ b/OptimizelySDK/Entity/Experiment.cs @@ -27,11 +27,6 @@ public class Experiment : ExperimentCore /// public string GroupId { get; set; } - /// - /// Layer ID for the experiment - /// - public string LayerId { get; set; } - /// /// ForcedVariations for the experiment /// @@ -53,12 +48,6 @@ public class Experiment : ExperimentCore public bool IsInMutexGroup => !string.IsNullOrEmpty(GroupPolicy) && GroupPolicy == MUTEX_GROUP_POLICY; - /// - /// Determine if experiment is running or not - /// - public bool IsExperimentRunning => - !string.IsNullOrEmpty(Status) && Status == STATUS_RUNNING; - /// /// Determin if user is forced variation of experiment /// @@ -68,10 +57,5 @@ public bool IsUserInForcedVariation(string userId) { return ForcedVariations != null && ForcedVariations.ContainsKey(userId); } - - /// - /// Determine if experiment is currently activated/running (implementation of abstract property) - /// - public override bool IsActivated => IsExperimentRunning; } } diff --git a/OptimizelySDK/Entity/ExperimentCore.cs b/OptimizelySDK/Entity/ExperimentCore.cs index 61dba9d8..0f81e2d0 100644 --- a/OptimizelySDK/Entity/ExperimentCore.cs +++ b/OptimizelySDK/Entity/ExperimentCore.cs @@ -35,6 +35,11 @@ public abstract class ExperimentCore : IdKeyEntity /// public string Status { get; set; } + /// + /// Layer ID for the experiment + /// + public virtual string LayerId { get; set; } + /// /// Variations for the experiment/holdout /// @@ -269,8 +274,8 @@ public virtual Variation GetVariationByKey(string key) #endregion /// - /// Determine if experiment/holdout is currently activated/running + /// Determine if experiment is currently activated/running /// - public abstract bool IsActivated { get; } + public bool isRunning => !string.IsNullOrEmpty(Status) && Status == STATUS_RUNNING; } } diff --git a/OptimizelySDK/Entity/FeatureDecision.cs b/OptimizelySDK/Entity/FeatureDecision.cs index e768cc5a..6bdd8f4c 100644 --- a/OptimizelySDK/Entity/FeatureDecision.cs +++ b/OptimizelySDK/Entity/FeatureDecision.cs @@ -20,12 +20,12 @@ public class FeatureDecision { public const string DECISION_SOURCE_FEATURE_TEST = "feature-test"; public const string DECISION_SOURCE_ROLLOUT = "rollout"; - - public Experiment Experiment { get; } + public const string DECISION_SOURCE_HOLDOUT = "holdout"; + public ExperimentCore Experiment { get; } public Variation Variation { get; } public string Source { get; } - public FeatureDecision(Experiment experiment, Variation variation, string source) + public FeatureDecision(ExperimentCore experiment, Variation variation, string source) { Experiment = experiment; Variation = variation; diff --git a/OptimizelySDK/Entity/Holdout.cs b/OptimizelySDK/Entity/Holdout.cs index 17f4c2bd..834b5229 100644 --- a/OptimizelySDK/Entity/Holdout.cs +++ b/OptimizelySDK/Entity/Holdout.cs @@ -45,10 +45,15 @@ public enum HoldoutStatus public string[] ExcludedFlags { get; set; } = new string[0]; /// - /// Determine if holdout is currently activated/running + /// Layer ID is always empty for holdouts as they don't belong to any layer /// - public override bool IsActivated => - !string.IsNullOrEmpty(Status) && Status == STATUS_RUNNING; - + public override string LayerId + { + get => string.Empty; + set + { + /* Holdouts don't have layer IDs, ignore any assignment */ + } + } } } diff --git a/OptimizelySDK/Event/Entity/ImpressionEvent.cs b/OptimizelySDK/Event/Entity/ImpressionEvent.cs index 12949ea6..0e5d0152 100644 --- a/OptimizelySDK/Event/Entity/ImpressionEvent.cs +++ b/OptimizelySDK/Event/Entity/ImpressionEvent.cs @@ -28,7 +28,7 @@ public class ImpressionEvent : UserEvent public string UserId { get; private set; } public VisitorAttribute[] VisitorAttributes { get; private set; } - public Experiment Experiment { get; set; } + public ExperimentCore Experiment { get; set; } public DecisionMetadata Metadata { get; set; } public Variation Variation { get; set; } public bool? BotFiltering { get; set; } @@ -42,7 +42,7 @@ public class Builder private EventContext EventContext; public VisitorAttribute[] VisitorAttributes; - private Experiment Experiment; + private ExperimentCore Experiment; private Variation Variation; private DecisionMetadata Metadata; private bool? BotFiltering; @@ -61,7 +61,7 @@ public Builder WithEventContext(EventContext eventContext) return this; } - public Builder WithExperiment(Experiment experiment) + public Builder WithExperiment(ExperimentCore experiment) { Experiment = experiment; diff --git a/OptimizelySDK/Event/UserEventFactory.cs b/OptimizelySDK/Event/UserEventFactory.cs index 28d6fb87..adb9c87b 100644 --- a/OptimizelySDK/Event/UserEventFactory.cs +++ b/OptimizelySDK/Event/UserEventFactory.cs @@ -61,7 +61,7 @@ public static ImpressionEvent CreateImpressionEvent(ProjectConfig projectConfig, /// experiment or featureDecision source /// ImpressionEvent instance public static ImpressionEvent CreateImpressionEvent(ProjectConfig projectConfig, - Experiment activatedExperiment, + ExperimentCore activatedExperiment, Variation variation, string userId, UserAttributes userAttributes, diff --git a/OptimizelySDK/Optimizely.cs b/OptimizelySDK/Optimizely.cs index 99cbdaaf..23483360 100644 --- a/OptimizelySDK/Optimizely.cs +++ b/OptimizelySDK/Optimizely.cs @@ -1154,12 +1154,12 @@ private void SendImpressionEvent(Experiment experiment, Variation variation, str /// The user's attributes /// It can either be experiment key in case if ruleType is experiment or it's feature key in case ruleType is feature-test or rollout /// It can either be experiment in case impression event is sent from activate or it's feature-test or rollout - private bool SendImpressionEvent(Experiment experiment, Variation variation, string userId, + private bool SendImpressionEvent(ExperimentCore experiment, Variation variation, string userId, UserAttributes userAttributes, ProjectConfig config, string flagKey, string ruleType, bool enabled ) { - if (experiment != null && !experiment.IsExperimentRunning) + if (experiment != null && !experiment.isRunning) { Logger.Log(LogLevel.ERROR, @"Experiment has ""Launched"" status so not dispatching event during activation."); diff --git a/OptimizelySDK/Utils/ExperimentUtils.cs b/OptimizelySDK/Utils/ExperimentUtils.cs index c87cebbf..8ee5fab5 100644 --- a/OptimizelySDK/Utils/ExperimentUtils.cs +++ b/OptimizelySDK/Utils/ExperimentUtils.cs @@ -25,7 +25,7 @@ public class ExperimentUtils { public static bool IsExperimentActive(Experiment experiment, ILogger logger) { - if (!experiment.IsExperimentRunning) + if (!experiment.isRunning) { logger.Log(LogLevel.INFO, $"Experiment \"{experiment.Key}\" is not running."); @@ -46,7 +46,7 @@ public static bool IsExperimentActive(Experiment experiment, ILogger logger) /// Custom logger implementation to record log outputs /// true if the user meets audience conditions to be in experiment, false otherwise. public static Result DoesUserMeetAudienceConditions(ProjectConfig config, - Experiment experiment, + ExperimentCore experiment, OptimizelyUserContext user, string loggingKeyType, string loggingKey, @@ -64,15 +64,13 @@ ILogger logger { expConditions = experiment.AudienceConditionsList; logger.Log(LogLevel.DEBUG, - $@"Evaluating audiences for {loggingKeyType} ""{loggingKey}"": { - experiment.AudienceConditionsString}."); + $@"Evaluating audiences for {loggingKeyType} ""{loggingKey}"": {experiment.AudienceConditionsString}."); } else { expConditions = experiment.AudienceIdsList; logger.Log(LogLevel.DEBUG, - $@"Evaluating audiences for {loggingKeyType} ""{loggingKey}"": { - experiment.AudienceIdsString}."); + $@"Evaluating audiences for {loggingKeyType} ""{loggingKey}"": {experiment.AudienceIdsString}."); } // If there are no audiences, return true because that means ALL users are included in the experiment. @@ -84,8 +82,7 @@ ILogger logger var result = expConditions.Evaluate(config, user, logger).GetValueOrDefault(); var resultText = result.ToString().ToUpper(); logger.Log(LogLevel.INFO, - reasons.AddInfo($@"Audiences for {loggingKeyType} ""{loggingKey - }"" collectively evaluated to {resultText}")); + reasons.AddInfo($@"Audiences for {loggingKeyType} ""{loggingKey}"" collectively evaluated to {resultText}")); return Result.NewResult(result, reasons); } }