Skip to content

Commit 2cfb505

Browse files
abhishekkumamsabhishekkumams
authored
Adding Merge config capabilities (#1446)
## Why make this change? - Closes #714 , #722 - To Allow dotnet like experience during local development where Development config gets merged with the original one. ## What is this change? - Added a new Class to enable merging of two different configs. - This merge feature is only applicable with `dab start`. - If the user provide a config file it will still be honored to ensure backward compatibility. - When the user doesn't provide a config file with `dab start`. It will check if `DAB_ENVIRONMENT` is set. if yes, then it will try to merge the `dab-config.json` with the `dab-config.{DAB_ENVIRONMENT}.json` and use the generated merged file `dab-config.{DAB_ENVIRONMENT}.merged.json` to startup the engine. - If only dab-config.{DAB_ENVIRONMENT}.json is available, it will be used. if not it will check for `dab-config.json`. if that is also not available, an error will be thrown. ## How the merge is performed? - if the type is same, then object from second config is given preference. merging of json object is done recurrsively. - if the second object is null, property from first config is given preference. - in case of JsonArray, the elements from second config completely overrides that of the original config, i.e. the merged config will have elements in the array from only second config. ## How was this tested? - [X] Unit Tests ## Sample Request(s) - set the `DAB_ENVIRONMENT` variable in your environment settings. for ex: Development. - you can keep your local changes in a file called `dab-config.Development.json`. It will have the connection string and other information that is for local development. - And you can have a file `dab-config.json` which can act as base file. - when we do `dab start` the merge will be performed, and the generated merged file `dab-config.Development.merged.json` will be used to fire up the engine. ## Example Scenarios: ### Case1: **Environment value:** $env:DAB_ENVIROMENT="" **command:** `dab start` **Files in Directory:** - dab-config.json - dab-config.DEVELOPMENT.json **Config Used for startup:** dab-config.json **Reasoning:** Environment value not setup ### Case2: **Environment value:** $env:DAB_ENVIROMENT="PRODUCTION" **command:** `dab start` **Files in Directory:** - dab-config.json - dab-config.DEVELOPMENT.json **Config Used for startup:** dab-config.json **Reasoning:** `dab-config.PRODUCTION.json` is not present in the directory ### Case3: **Environment value:** $env:DAB_ENVIROMENT="DEVELOPMENT" **command:** `dab start` **Files in Directory:** - dab-config.json - dab-config.DEVELOPMENT.json **Config Used for startup:** dab-config.json + dab-config.DEVELOPMENT.json (Merged) **Reasoning:** both `dab-config.json` and `dab-config.DEVELOPMENT.json` is present in the directory ### Case4: **Environment value:** $env:DAB_ENVIROMENT="DEVELOPMENT" **command:** `dab start -c dab-config.json` **Files in Directory:** - dab-config.json - dab-config.DEVELOPMENT.json **Config Used for startup:** dab-config.json **Reasoning:** If config is provided by the user it will always get precedence irrespective of the ENVIRONMENT value.
1 parent f6c11d8 commit 2cfb505

File tree

7 files changed

+437
-11
lines changed

7 files changed

+437
-11
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ log.txt
1111

1212
# only include appsettings for a few default environments
1313
appsettings.*.json
14-
dab-config.*.json
14+
dab-config*.json
1515
*.dab-config.json
1616
!dab-config.*reference.json
1717
!dab-config.*.example.json

‎src/Cli.Tests/TestHelper.cs‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -895,6 +895,207 @@ public static Process ExecuteDabCommand(string command, string flags)
895895
}
896896
}";
897897

898+
public const string BASE_CONFIG =
899+
@"{" +
900+
@"""$schema"": """ + DAB_DRAFT_SCHEMA_TEST_PATH + @"""" + "," +
901+
@"""data-source"": {
902+
""database-type"": ""mssql"",
903+
""connection-string"": """",
904+
""options"":{
905+
""set-session-context"": true
906+
}
907+
},
908+
""runtime"": {
909+
""rest"": {
910+
""path"": ""/api"",
911+
""enabled"": true
912+
},
913+
""graphql"": {
914+
""path"": ""/graphql"",
915+
""enabled"": true,
916+
""allow-introspection"": true
917+
},
918+
""host"": {
919+
""mode"": ""production"",
920+
""cors"": {
921+
""origins"": [],
922+
""allow-credentials"": false
923+
},
924+
""authentication"": {
925+
""provider"": ""StaticWebApps""
926+
}
927+
}
928+
},
929+
""entities"": {
930+
""book"": {
931+
""source"": ""s001.book"",
932+
""permissions"": [
933+
{
934+
""role"": ""anonymous"",
935+
""actions"": [
936+
""*""
937+
]
938+
}
939+
]
940+
},
941+
""author"": {
942+
""source"": ""s001.authors"",
943+
""permissions"": [
944+
{
945+
""role"": ""anonymous"",
946+
""actions"": [
947+
""*""
948+
]
949+
}
950+
]
951+
}
952+
}
953+
}";
954+
955+
public const string ENV_BASED_CONFIG =
956+
@"{" +
957+
@"""$schema"": """ + DAB_DRAFT_SCHEMA_TEST_PATH + @"""" + "," +
958+
@"""data-source"": {
959+
""database-type"": ""mssql"",
960+
""connection-string"": ""localhost:5000;User ID={USER_NAME};Password={USER_PASSWORD};MultipleActiveResultSets=False;""
961+
},
962+
""runtime"": {
963+
""graphql"": {
964+
""path"": ""/graphql"",
965+
""enabled"": true,
966+
""allow-introspection"": true
967+
},
968+
""host"": {
969+
""mode"": ""production"",
970+
""cors"": {
971+
""origins"": [ ""http://localhost:5000"" ],
972+
""allow-credentials"": false
973+
},
974+
""authentication"": {
975+
""provider"": ""StaticWebApps""
976+
}
977+
}
978+
},
979+
""entities"": {
980+
""source"":{
981+
""source"": ""src"",
982+
""rest"": ""true"",
983+
""permissions"": [
984+
{
985+
""role"": ""authenticated"",
986+
""actions"": [
987+
""*""
988+
]
989+
}
990+
]
991+
},
992+
""book"": {
993+
""source"": ""books"",
994+
""rest"": ""true"",
995+
""permissions"": [
996+
{
997+
""role"": ""authenticated"",
998+
""actions"": [
999+
""*""
1000+
]
1001+
}
1002+
]
1003+
},
1004+
""publisher"": {
1005+
""source"": ""publishers"",
1006+
""permissions"": [
1007+
{
1008+
""role"": ""anonymous"",
1009+
""actions"": [
1010+
""*""
1011+
]
1012+
}
1013+
]
1014+
}
1015+
}
1016+
}";
1017+
1018+
public const string MERGED_CONFIG =
1019+
@"{" +
1020+
@"""$schema"": """ + DAB_DRAFT_SCHEMA_TEST_PATH + @"""" + "," +
1021+
@"""data-source"": {
1022+
""database-type"": ""mssql"",
1023+
""connection-string"": ""localhost:5000;User ID={USER_NAME};Password={USER_PASSWORD};MultipleActiveResultSets=False;"",
1024+
""options"":{
1025+
""set-session-context"": true
1026+
}
1027+
},
1028+
""runtime"": {
1029+
""rest"": {
1030+
""path"": ""/api"",
1031+
""enabled"": true
1032+
},
1033+
""graphql"": {
1034+
""path"": ""/graphql"",
1035+
""enabled"": true,
1036+
""allow-introspection"": true
1037+
},
1038+
""host"": {
1039+
""mode"": ""production"",
1040+
""cors"": {
1041+
""origins"": [ ""http://localhost:5000"" ],
1042+
""allow-credentials"": false
1043+
},
1044+
""authentication"": {
1045+
""provider"": ""StaticWebApps""
1046+
}
1047+
}
1048+
},
1049+
""entities"": {
1050+
""source"":{
1051+
""source"": ""src"",
1052+
""rest"": ""true"",
1053+
""permissions"": [
1054+
{
1055+
""role"": ""authenticated"",
1056+
""actions"": [
1057+
""*""
1058+
]
1059+
}
1060+
]
1061+
},
1062+
""book"": {
1063+
""source"": ""books"",
1064+
""rest"": ""true"",
1065+
""permissions"": [
1066+
{
1067+
""role"": ""authenticated"",
1068+
""actions"": [
1069+
""*""
1070+
]
1071+
}
1072+
]
1073+
},
1074+
""author"": {
1075+
""source"": ""s001.authors"",
1076+
""permissions"": [
1077+
{
1078+
""role"": ""anonymous"",
1079+
""actions"": [
1080+
""*""
1081+
]
1082+
}
1083+
]
1084+
},
1085+
""publisher"": {
1086+
""source"": ""publishers"",
1087+
""permissions"": [
1088+
{
1089+
""role"": ""anonymous"",
1090+
""actions"": [
1091+
""*""
1092+
]
1093+
}
1094+
]
1095+
}
1096+
}
1097+
}";
1098+
8981099
/// <summary>
8991100
/// Helper method to create json string for runtime settings
9001101
/// for json comparison in tests.

‎src/Cli.Tests/UtilsTests.cs‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,9 @@ public class UtilsTests
1313
/// Setup the logger for CLI
1414
/// </summary>
1515
[ClassInitialize]
16-
public static void SetupLoggerForCLI(TestContext context)
16+
public static void Setup(TestContext context)
1717
{
18-
Mock<ILogger<Utils>> utilsLogger = new();
19-
Utils.SetCliUtilsLogger(utilsLogger.Object);
18+
TestHelper.SetupTestLoggerForCLI();
2019
}
2120

2221
/// <summary>
@@ -78,7 +77,7 @@ public void TestConfigSelectionBasedOnCliPrecedence(
7877
{
7978
if (!File.Exists(expectedRuntimeConfigFile))
8079
{
81-
File.Create(expectedRuntimeConfigFile);
80+
File.Create(expectedRuntimeConfigFile).Dispose();
8281
}
8382

8483
string? envValueBeforeTest = Environment.GetEnvironmentVariable(RUNTIME_ENVIRONMENT_VAR_NAME);
@@ -200,6 +199,41 @@ public void TestValidateAudienceAndIssuerForAuthenticationProvider(
200199
);
201200
}
202201

202+
/// <summary>
203+
/// Test to verify that when DAB_ENVIRONMENT variable is set, also base config and
204+
/// dab-config.{DAB_ENVIRONMENT}.json file is present, then when DAB engine is started, it will merge
205+
/// the two config and use the merged config to startup the engine.
206+
/// Here, baseConfig(dab-config.json) has no connection_string, while dab-config.Test.json has a defined connection string.
207+
/// once the `dab start` is executed the merge happens and the merged file contains the connection string from the
208+
/// Test config.
209+
/// Scenarios Covered:
210+
/// 1. Merging of Array: Complete override of Book permissions from the second config (environment based config).
211+
/// 2. Merging Property when present in both config: Connection string in the second config overrides that of the first.
212+
/// 3. Non-merging when a property in the environmentConfig file is null: {data-source.options} is null in the environment config,
213+
/// So it is added to the merged config as it is with no change.
214+
/// 4. Merging when a property is only present in the environmentConfig file: Publisher entity is present only in environment config,
215+
/// So it is directly added to the merged config.
216+
/// 5. Properties of same name but different level do not conflict: source is both a entityName and a property inside book entity, both are
217+
/// treated differently.
218+
/// </summary>
219+
[TestMethod]
220+
public void TestMergeConfig()
221+
{
222+
Environment.SetEnvironmentVariable(RUNTIME_ENVIRONMENT_VAR_NAME, "Test");
223+
File.WriteAllText("dab-config.json", BASE_CONFIG);
224+
File.WriteAllText("dab-config.Test.json", ENV_BASED_CONFIG);
225+
if (TryMergeConfigsIfAvailable(out string mergedConfig))
226+
{
227+
Assert.AreEqual(mergedConfig, "dab-config.Test.merged.json");
228+
Assert.IsTrue(File.Exists(mergedConfig));
229+
Assert.IsTrue(JToken.DeepEquals(JObject.Parse(MERGED_CONFIG), JObject.Parse(File.ReadAllText(mergedConfig))));
230+
}
231+
else
232+
{
233+
Assert.Fail("Failed to merge config files.");
234+
}
235+
}
236+
203237
[ClassCleanup]
204238
public static void Cleanup()
205239
{
@@ -217,6 +251,11 @@ public static void Cleanup()
217251
{
218252
File.Delete("my-config.json");
219253
}
254+
255+
if (File.Exists($"{CONFIGFILE_NAME}.Test.merged{CONFIG_EXTENSION}"))
256+
{
257+
File.Delete($"{CONFIGFILE_NAME}.Test.merged{CONFIG_EXTENSION}");
258+
}
220259
}
221260
}
222261
}

‎src/Cli/ConfigGenerator.cs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -943,12 +943,21 @@ public static bool VerifyCanUpdateRelationship(RuntimeConfig runtimeConfig, stri
943943

944944
/// <summary>
945945
/// This method will try starting the engine.
946-
/// It will use the config provided by the user, else will look for the default config.
947-
/// Does validation to check connection string is not null or empty.
946+
/// It will use the config provided by the user, else based on the environment value
947+
/// it will either merge the config if base config and environmentConfig is present
948+
/// else it will choose a single config based on precedence (left to right) of
949+
/// overrides < environmentConfig < defaultConfig
950+
/// Also preforms validation to check connection string is not null or empty.
948951
/// </summary>
949952
public static bool TryStartEngineWithOptions(StartOptions options)
950953
{
951-
if (!TryGetConfigFileBasedOnCliPrecedence(options.Config, out string runtimeConfigFile))
954+
string? configToBeUsed = options.Config;
955+
if (string.IsNullOrEmpty(configToBeUsed) && TryMergeConfigsIfAvailable(out configToBeUsed))
956+
{
957+
_logger.LogInformation($"Using merged config file based on environment:{configToBeUsed}.");
958+
}
959+
960+
if (!TryGetConfigFileBasedOnCliPrecedence(configToBeUsed, out string runtimeConfigFile))
952961
{
953962
_logger.LogError("Config not provided and default config file doesn't exist.");
954963
return false;

‎src/Cli/Utils.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
using Humanizer;
1313
using Microsoft.Extensions.Logging;
1414
using static Azure.DataApiBuilder.Config.AuthenticationConfig;
15+
using static Azure.DataApiBuilder.Config.MergeJsonProvider;
16+
using static Azure.DataApiBuilder.Config.RuntimeConfigPath;
1517
using static Azure.DataApiBuilder.Service.Configurations.RuntimeConfigValidator;
1618
using PermissionOperation = Azure.DataApiBuilder.Config.PermissionOperation;
1719

@@ -898,6 +900,44 @@ public static bool WriteJsonContentToFile(string file, string jsonContent)
898900
return true;
899901
}
900902

903+
/// <summary>
904+
/// This method will check if DAB_ENVIRONMENT value is set.
905+
/// If yes, it will try to merge dab-config.json with dab-config.{DAB_ENVIRONMENT}.json
906+
/// and create a merged file called dab-config.{DAB_ENVIRONMENT}.merged.json
907+
/// </summary>
908+
/// <returns>Returns the name of the merged Config if successful.</returns>
909+
public static bool TryMergeConfigsIfAvailable(out string mergedConfigFile)
910+
{
911+
string? environmentValue = Environment.GetEnvironmentVariable(RUNTIME_ENVIRONMENT_VAR_NAME);
912+
mergedConfigFile = string.Empty;
913+
if (!string.IsNullOrEmpty(environmentValue))
914+
{
915+
string baseConfigFile = RuntimeConfigPath.DefaultName;
916+
string environmentBasedConfigFile = RuntimeConfigPath.GetFileName(environmentValue, considerOverrides: false);
917+
mergedConfigFile = RuntimeConfigPath.GetMergedFileNameForEnvironment(CONFIGFILE_NAME, environmentValue);
918+
919+
if (DoesFileExistInCurrentDirectory(baseConfigFile) && !string.IsNullOrEmpty(environmentBasedConfigFile))
920+
{
921+
try
922+
{
923+
string baseConfigJson = File.ReadAllText(baseConfigFile);
924+
string overrideConfigJson = File.ReadAllText(environmentBasedConfigFile);
925+
string mergedConfigJson = Merge(baseConfigJson, overrideConfigJson);
926+
927+
File.WriteAllText(mergedConfigFile, mergedConfigJson);
928+
return true;
929+
}
930+
catch (Exception ex)
931+
{
932+
_logger.LogError(ex, $"Failed to merge the config files.");
933+
return false;
934+
}
935+
}
936+
}
937+
938+
return false;
939+
}
940+
901941
/// <summary>
902942
/// Utility method that converts REST HTTP verb string input to RestMethod Enum.
903943
/// The method returns true/false corresponding to successful/unsuccessful conversion.

0 commit comments

Comments
 (0)