Add support for .env file (#1497)

abhishekkumams · aaronpowell · web-flow · commit 0b3e3abf6085 · 2023-06-09T14:58:02.000+05:30
## Why make this change? - Closes #1374 - To allow users maintain a local `.env` to manage environment variables during local development. ## What is this change? - Using the package [DotNetEnv](https://www.nuget.org/packages/DotNetEnv/) - Now, users can maintain a local file `.env` to manage environment variables instead of setting environment variables in the system. - If `.env` file is used, it will take precedence over system environment variables. ## How was this tested? - [x] Unit Tests ## NOTE: - This feature is to enhance local development experience, and it works when used via DAB CLI. ## QnA 1. Where should the `.env` file be created? -> For `.env file` to be considered, it should be present in the same directory where the dab commands are executed. 2. Does it overwrite the existing environment variables? -> No, it doesn't update any existing variables. It just gives precedence to the variable defined in the file over the system defined variable in case it is present in both places. --------- Co-authored-by: Aaron Powell <me@aaron-powell.com>
diff --git a/src/Cli.Tests/EnvironmentTests.cs b/src/Cli.Tests/EnvironmentTests.cs
@@ -0,0 +1,172 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace Cli.Tests;
+
+/// <summary>
+/// Contains test involving environment variables.
+/// </summary>
+[TestClass]
+public class EnvironmentTests
+{
+    public const string TEST_ENV_VARIABLE = "DAB_TEST_ENVIRONMENT";
+    /// <summary>
+    /// Test to verify that environment variable setup in the system is picked up correctly
+    /// when no .env file is present.
+    /// </summary>
+    [TestMethod]
+    public void TestEnvironmentVariableIsConsumedCorrectly()
+    {
+        string jsonWithEnvVariable = @"{""envValue"": ""@env('DAB_TEST_ENVIRONMENT')""}";
+
+        // No environment File, No environment variable set in the system
+        Assert.AreEqual(null, Environment.GetEnvironmentVariable(TEST_ENV_VARIABLE));
+
+        // Configuring environment variable in the system
+        Environment.SetEnvironmentVariable(TEST_ENV_VARIABLE, "TEST");
+
+        // Test environment variable is correctly resolved in the config file
+        string? resolvedJson = RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(jsonWithEnvVariable);
+        Assert.IsNotNull(resolvedJson);
+        Assert.IsTrue(JToken.DeepEquals(
+            JObject.Parse(@"{""envValue"": ""TEST""}"),
+            JObject.Parse(resolvedJson)), "JSON resolved with environment variable correctly");
+
+        // removing Environment variable from the System
+        Environment.SetEnvironmentVariable(TEST_ENV_VARIABLE, null);
+        Assert.ThrowsException<DataApiBuilderException>(() =>
+            RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(jsonWithEnvVariable),
+            $"Environmental Variable, {TEST_ENV_VARIABLE}, not found.");
+    }
+
+    /// <summary>
+    /// This test creates a .env file and adds a variable and verifies that the variable is
+    /// correctly consumed.
+    /// For this test there were no existing environment variables. The values are picked up
+    /// directly from the `.env` file.
+    /// </summary>
+    [TestMethod]
+    public void TestEnvironmentFileIsConsumedCorrectly()
+    {
+        string jsonWithEnvVariable = @"{""envValue"": ""@env('DAB_TEST_ENVIRONMENT')""}";
+
+        // No environment File, no environment variable set in the system
+        Assert.IsNull(Environment.GetEnvironmentVariable(TEST_ENV_VARIABLE));
+
+        // Creating environment variable file
+        File.Create(".env").Close();
+        File.WriteAllText(".env", $"{TEST_ENV_VARIABLE}=DEVELOPMENT");
+        DotNetEnv.Env.Load();
+
+        // Test environment variable is picked up from the .env file and is correctly resolved in the config file.
+        Assert.AreEqual("DEVELOPMENT", Environment.GetEnvironmentVariable(TEST_ENV_VARIABLE));
+        string? resolvedJson = RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(jsonWithEnvVariable);
+        Assert.IsNotNull(resolvedJson);
+        Assert.IsTrue(JToken.DeepEquals(
+            JObject.Parse(@"{""envValue"": ""DEVELOPMENT""}"),
+            JObject.Parse(resolvedJson)), "JSON resolved with environment variable correctly");
+    }
+
+    /// <summary>
+    /// This test setups an environment variable in the system and also creates a .env file containing
+    /// the same variable with different value. In such a case, the value stored for variable in .env file is given
+    /// precedence over the value specified in the system.
+    /// </summary>
+    [TestMethod]
+    public void TestPrecedenceOfEnvironmentFileOverExistingVariables()
+    {
+        // The variable set in the .env file takes precedence over the environment value set in the system.
+        Environment.SetEnvironmentVariable(TEST_ENV_VARIABLE, "TEST");
+
+        // Creating environment variable file
+        File.Create(".env").Close();
+        File.WriteAllText(".env", $"{TEST_ENV_VARIABLE}=DEVELOPMENT");
+        DotNetEnv.Env.Load();     // It contains value DEVELOPMENT
+        Assert.AreEqual("DEVELOPMENT", Environment.GetEnvironmentVariable(TEST_ENV_VARIABLE));
+
+        // If a variable is not present in the .env file then the system defined variable would be used if defined.
+        Environment.SetEnvironmentVariable("HOSTING_TEST_ENVIRONMENT", "PHOENIX_TEST");
+        string? resolvedJson = RuntimeConfigPath.ParseConfigJsonAndReplaceEnvVariables(
+            @"{
+                ""envValue"": ""@env('DAB_TEST_ENVIRONMENT')"",
+                ""hostingEnvValue"": ""@env('HOSTING_TEST_ENVIRONMENT')""
+                }"
+        );
+        Assert.IsNotNull(resolvedJson);
+        Assert.IsTrue(JToken.DeepEquals(
+            JObject.Parse(
+                @"{
+                    ""envValue"": ""DEVELOPMENT"",
+                    ""hostingEnvValue"": ""PHOENIX_TEST""
+                }"),
+            JObject.Parse(resolvedJson)), "JSON resolved with environment variable correctly");
+    }
+
+    /// <summary>
+    /// Test to verify that no error is thrown if .env file is not present, and existing system variables are used.
+    /// </summary>
+    [TestMethod]
+    public void TestSystemEnvironmentVariableIsUsedInAbsenceOfEnvironmentFile()
+    {
+        Environment.SetEnvironmentVariable(TEST_ENV_VARIABLE, "TEST");
+        Assert.IsFalse(File.Exists(".env"));
+        DotNetEnv.Env.Load(); // No error is thrown
+        Assert.AreEqual("TEST", Environment.GetEnvironmentVariable(TEST_ENV_VARIABLE));
+    }
+
+    /// <summary>
+    /// Test to verify that if the environment variables are not resolved correctly, the runtime engine will not start.
+    /// Here, in the first scenario, engine fails to start because the variable defined in the environment file
+    /// is typed incorrectly and does not match the one present in the config.
+    /// </summary>
+    [DataRow("COMM_STRINX=test_connection_string", true, DisplayName = "Incorrect Variable name used in the environment file.")]
+    [DataRow("CONN_STRING=test_connection_string", false, DisplayName = "Correct Variable name used in the environment file.")]
+    [DataTestMethod]
+    public void TestFailureToStartWithUnresolvedJsonConfig(
+        string environmentFileContent,
+        bool isFailure
+    )
+    {
+        // Creating environment variable file
+        File.Create(".env").Close();
+        File.WriteAllText(".env", environmentFileContent);
+        if (File.Exists(TEST_RUNTIME_CONFIG_FILE))
+        {
+            File.Delete(TEST_RUNTIME_CONFIG_FILE);
+        }
+
+        string[] initArgs = { "init", "-c", TEST_RUNTIME_CONFIG_FILE, "--database-type", "mssql", "--connection-string", "@env('CONN_STRING')" };
+        Program.Main(initArgs);
+
+        // Trying to start the runtime engine
+        using Process process = ExecuteDabCommand(
+            "start",
+            $"-c {TEST_RUNTIME_CONFIG_FILE}"
+        );
+
+        string? output = process.StandardOutput.ReadToEnd();
+        Assert.IsNotNull(output);
+
+        if (isFailure)
+        {
+            // Failed to resolve the environment variables in the config.
+            Assert.IsFalse(output.Contains("Starting the runtime engine..."));
+            Assert.IsTrue(output.Contains("Error: Failed due to: Environmental Variable, CONN_STRING, not found."));
+        }
+        else
+        {
+            // config resolved correctly.
+            Assert.IsTrue(output.Contains("Starting the runtime engine..."));
+            Assert.IsFalse(output.Contains("Error: Failed due to: Environmental Variable, CONN_STRING, not found."));
+        }
+    }
+
+    [TestCleanup]
+    public void CleanUp()
+    {
+        if (File.Exists(".env"))
+        {
+            File.Delete(".env");
+        }
+    }
+}
diff --git a/src/Cli.Tests/Usings.cs b/src/Cli.Tests/Usings.cs
@@ -4,6 +4,7 @@
 global using System.Diagnostics;
 global using System.Text.Json;
 global using Azure.DataApiBuilder.Config;
+global using Azure.DataApiBuilder.Service.Exceptions;
 global using Microsoft.Extensions.Logging;
 global using Microsoft.VisualStudio.TestTools.UnitTesting;
 global using Moq;
diff --git a/src/Cli/Program.cs b/src/Cli/Program.cs
@@ -29,6 +29,9 @@ public static int Main(string[] args)
                 }
             );
 
+            // Load environment variables from .env file if present.
+            DotNetEnv.Env.Load();
+
             // Setting up Logger for CLI.
             ILoggerFactory loggerFactory = new LoggerFactory();
             loggerFactory.AddProvider(new CustomLoggerProvider());
diff --git a/src/Cli/Utils.cs b/src/Cli/Utils.cs
@@ -12,6 +12,7 @@
 using static Azure.DataApiBuilder.Config.AuthenticationConfig;
 using static Azure.DataApiBuilder.Config.MergeJsonProvider;
 using static Azure.DataApiBuilder.Config.RuntimeConfigPath;
+using static Azure.DataApiBuilder.Service.Configurations.RuntimeConfigProvider;
 using static Azure.DataApiBuilder.Service.Configurations.RuntimeConfigValidator;
 using PermissionOperation = Azure.DataApiBuilder.Config.PermissionOperation;
 
@@ -582,6 +583,7 @@ public static bool TryGetConfigFileBasedOnCliPrecedence(
             else
             {
                 _logger.LogInformation("Config not provided. Trying to get default config based on DAB_ENVIRONMENT...");
+                _logger.LogInformation("Environment variable DAB_ENVIRONMENT is {value}", Environment.GetEnvironmentVariable("DAB_ENVIRONMENT"));
                 /// Need to reset to true explicitly so any that any re-invocations of this function
                 /// get simulated as being called for the first time specifically useful for tests.
                 RuntimeConfigPath.CheckPrecedenceForConfigInEngine = true;
@@ -597,7 +599,7 @@ public static bool TryGetConfigFileBasedOnCliPrecedence(
         }
 
         /// <summary>
-        /// Checks if config can be correctly parsed by deserializing the
+        /// Checks if config can be correctly resolved and parsed by deserializing the
         /// json config into runtime config object.
         /// Also checks that connection-string is not null or empty whitespace.
         /// If parsing is successful and the config has valid connection-string, it
@@ -608,18 +610,25 @@ public static bool CanParseConfigCorrectly(
             [NotNullWhen(true)] out RuntimeConfig? deserializedRuntimeConfig)
         {
             deserializedRuntimeConfig = null;
-            if (!TryReadRuntimeConfig(configFile, out string runtimeConfigJson))
+            string? runtimeConfigJson;
+
+            try
+            {
+                // Tries to read the config and resolve environment variables.
+                runtimeConfigJson = GetRuntimeConfigJsonString(configFile);
+            }
+            catch (Exception e)
             {
-                _logger.LogError($"Failed to read the config file: {configFile}.");
+                _logger.LogError("Failed due to: {exceptionMessage}", e.Message);
                 return false;
             }
 
-            if (!RuntimeConfig.TryGetDeserializedRuntimeConfig(
+            if (string.IsNullOrEmpty(runtimeConfigJson) || !RuntimeConfig.TryGetDeserializedRuntimeConfig(
                     runtimeConfigJson,
                     out deserializedRuntimeConfig,
                     logger: null))
             {
-                _logger.LogError($"Failed to parse the config file: {configFile}.");
+                _logger.LogError("Failed to parse the config file: {configFile}.", configFile);
                 return false;
             }
 
@@ -906,6 +915,7 @@ public static bool TryMergeConfigsIfAvailable([NotNullWhen(true)] out string? me
                         string baseConfigJson = File.ReadAllText(baseConfigFile);
                         string overrideConfigJson = File.ReadAllText(environmentBasedConfigFile);
                         string currentDir = Directory.GetCurrentDirectory();
+                        _logger.LogInformation("Using DAB_ENVIRONMENT = {value}", environmentValue);
                         _logger.LogInformation($"Merging {Path.Combine(currentDir, baseConfigFile)}"
                             + $" and {Path.Combine(currentDir, environmentBasedConfigFile)}");
                         string mergedConfigJson = Merge(baseConfigJson, overrideConfigJson);
diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props
@@ -11,6 +11,7 @@
     <PackageVersion Include="HotChocolate.AspNetCore.Authorization" Version="12.18.0" />
     <PackageVersion Include="Humanizer" Version="2.14.1" />
     <PackageVersion Include="Humanizer.Core" Version="2.14.1" />
+    <PackageVersion Include="DotNetEnv" Version="2.5.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Authorization" Version="6.0.14" />
     <PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.14" />
     <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
diff --git a/src/Service/Azure.DataApiBuilder.Service.csproj b/src/Service/Azure.DataApiBuilder.Service.csproj
@@ -64,6 +64,7 @@
         <PackageReference Include="Newtonsoft.Json" />
         <PackageReference Include="Npgsql" />
         <PackageReference Include="Polly" />
+        <PackageReference Include="DotNetEnv" />
         <PackageReference Include="StyleCop.Analyzers">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,9 @@ public static int Main(string[] args)`
`29`	`29`	`}`
`30`	`30`	`);`
`31`	`31`
	`32`	`+ // Load environment variables from .env file if present.`
	`33`	`+ DotNetEnv.Env.Load();`
	`34`	`+`
`32`	`35`	`// Setting up Logger for CLI.`
`33`	`36`	`ILoggerFactory loggerFactory = new LoggerFactory();`
`34`	`37`	`loggerFactory.AddProvider(new CustomLoggerProvider());`