[Cherrypick] Support environment variable serialization for data source options #1980 (#2016)

seantleonard · abhishekkumams · web-flow · commit 573117a7ff19 · 2024-02-06T11:49:42.000-08:00
// only diff is excluded changes from mutationtests.cs for Cosmos that only exists in .11+. via #1893 ## Why make this change? - Closes #1956 - env variable replacement not happening in the `options` section of `data-source` property in our config. ## What is this change? - Added a DataSourceConverterFactory which will properly deserialize the data-source section considering the _replaceEnvVariable to decide weather to transform or not. ## How was this tested? - [X] Unit Tests --------- Co-authored-by: Abhishek Kumar <102276754+abhishekkumams@users.noreply.github.com>
diff --git a/src/Cli.Tests/Snapshots/EndToEndTests.TestAddingStoredProcedureWithRestMethodsAndGraphQLOperations.verified.txt b/src/Cli.Tests/Snapshots/EndToEndTests.TestAddingStoredProcedureWithRestMethodsAndGraphQLOperations.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: True
-      }
+      set-session-context: true
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/EndToEndTests.TestConfigGeneratedAfterAddingEntityWithSourceAsStoredProcedure.verified.txt b/src/Cli.Tests/Snapshots/EndToEndTests.TestConfigGeneratedAfterAddingEntityWithSourceAsStoredProcedure.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: True
-      }
+      set-session-context: true
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/EndToEndTests.TestConfigGeneratedAfterAddingEntityWithSourceWithDefaultType.verified.txt b/src/Cli.Tests/Snapshots/EndToEndTests.TestConfigGeneratedAfterAddingEntityWithSourceWithDefaultType.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: True
-      }
+      set-session-context: true
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/EndToEndTests.TestConfigGeneratedAfterAddingEntityWithoutIEnumerables.verified.txt b/src/Cli.Tests/Snapshots/EndToEndTests.TestConfigGeneratedAfterAddingEntityWithoutIEnumerables.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: True
-      }
+      set-session-context: true
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/EndToEndTests.TestInitForCosmosDBNoSql.verified.txt b/src/Cli.Tests/Snapshots/EndToEndTests.TestInitForCosmosDBNoSql.verified.txt
@@ -1,15 +1,9 @@
 ﻿{
   DataSource: {
     Options: {
-      container: {
-        ValueKind: String
-      },
-      database: {
-        ValueKind: String
-      },
-      schema: {
-        ValueKind: String
-      }
+      container: planet,
+      database: graphqldb,
+      schema: test-schema.gql
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/EndToEndTests.TestUpdatingStoredProcedureWithRestMethods.verified.txt b/src/Cli.Tests/Snapshots/EndToEndTests.TestUpdatingStoredProcedureWithRestMethods.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: True
-      }
+      set-session-context: true
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/EndToEndTests.TestUpdatingStoredProcedureWithRestMethodsAndGraphQLOperations.verified.txt b/src/Cli.Tests/Snapshots/EndToEndTests.TestUpdatingStoredProcedureWithRestMethodsAndGraphQLOperations.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: True
-      }
+      set-session-context: true
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.CosmosDbNoSqlDatabase.verified.txt b/src/Cli.Tests/Snapshots/InitTests.CosmosDbNoSqlDatabase.verified.txt
@@ -1,15 +1,9 @@
 ﻿{
   DataSource: {
     Options: {
-      container: {
-        ValueKind: String
-      },
-      database: {
-        ValueKind: String
-      },
-      schema: {
-        ValueKind: String
-      }
+      container: testcontainer,
+      database: testdb,
+      schema: test-schema.gql
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_171ea8114ff71814.verified.txt b/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_171ea8114ff71814.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_2df7a1794712f154.verified.txt b/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_2df7a1794712f154.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_59fe1a10aa78899d.verified.txt b/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_59fe1a10aa78899d.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_b95b637ea87f16a7.verified.txt b/src/Cli.Tests/Snapshots/InitTests.EnsureCorrectConfigGenerationWithDifferentAuthenticationProviders_b95b637ea87f16a7.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.GraphQLPathWithoutStartingSlashWillHaveItAdded.verified.txt b/src/Cli.Tests/Snapshots/InitTests.GraphQLPathWithoutStartingSlashWillHaveItAdded.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.MsSQLDatabase.verified.txt b/src/Cli.Tests/Snapshots/InitTests.MsSQLDatabase.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: True
-      }
+      set-session-context: true
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.RestPathWithoutStartingSlashWillHaveItAdded.verified.txt b/src/Cli.Tests/Snapshots/InitTests.RestPathWithoutStartingSlashWillHaveItAdded.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.TestInitializingConfigWithoutConnectionString.verified.txt b/src/Cli.Tests/Snapshots/InitTests.TestInitializingConfigWithoutConnectionString.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli.Tests/Snapshots/InitTests.TestSpecialCharactersInConnectionString.verified.txt b/src/Cli.Tests/Snapshots/InitTests.TestSpecialCharactersInConnectionString.verified.txt
@@ -2,9 +2,7 @@
   DataSource: {
     DatabaseType: MSSQL,
     Options: {
-      set-session-context: {
-        ValueKind: False
-      }
+      set-session-context: false
     }
   },
   Runtime: {
diff --git a/src/Cli/ConfigGenerator.cs b/src/Cli/ConfigGenerator.cs
@@ -4,7 +4,6 @@
 using System.Collections.ObjectModel;
 using System.Diagnostics.CodeAnalysis;
 using System.IO.Abstractions;
-using System.Text.Json;
 using Azure.DataApiBuilder.Config;
 using Azure.DataApiBuilder.Config.Converters;
 using Azure.DataApiBuilder.Config.NamingPolicies;
@@ -89,7 +88,7 @@ public static bool TryCreateRuntimeConfig(InitOptions options, FileSystemRuntime
             string? restPath = options.RestPath;
             string graphQLPath = options.GraphQLPath;
             string? runtimeBaseRoute = options.RuntimeBaseRoute;
-            Dictionary<string, JsonElement> dbOptions = new();
+            Dictionary<string, object?> dbOptions = new();
 
             HyphenatedNamingPolicy namingPolicy = new();
 
@@ -148,14 +147,14 @@ public static bool TryCreateRuntimeConfig(InitOptions options, FileSystemRuntime
                     }
 
                     restPath = null;
-                    dbOptions.Add(namingPolicy.ConvertName(nameof(CosmosDbNoSQLDataSourceOptions.Database)), JsonSerializer.SerializeToElement(cosmosDatabase));
-                    dbOptions.Add(namingPolicy.ConvertName(nameof(CosmosDbNoSQLDataSourceOptions.Container)), JsonSerializer.SerializeToElement(cosmosContainer));
-                    dbOptions.Add(namingPolicy.ConvertName(nameof(CosmosDbNoSQLDataSourceOptions.Schema)), JsonSerializer.SerializeToElement(graphQLSchemaPath));
+                    dbOptions.Add(namingPolicy.ConvertName(nameof(CosmosDbNoSQLDataSourceOptions.Database)), cosmosDatabase);
+                    dbOptions.Add(namingPolicy.ConvertName(nameof(CosmosDbNoSQLDataSourceOptions.Container)), cosmosContainer);
+                    dbOptions.Add(namingPolicy.ConvertName(nameof(CosmosDbNoSQLDataSourceOptions.Schema)), graphQLSchemaPath);
                     break;
 
                 case DatabaseType.DWSQL:
                 case DatabaseType.MSSQL:
-                    dbOptions.Add(namingPolicy.ConvertName(nameof(MsSqlOptions.SetSessionContext)), JsonSerializer.SerializeToElement(options.SetSessionContext));
+                    dbOptions.Add(namingPolicy.ConvertName(nameof(MsSqlOptions.SetSessionContext)), options.SetSessionContext);
 
                     break;
                 case DatabaseType.MySQL:
diff --git a/src/Config/Converters/DataSourceConverterFactory.cs b/src/Config/Converters/DataSourceConverterFactory.cs
@@ -0,0 +1,137 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Azure.DataApiBuilder.Config.ObjectModel;
+
+namespace Azure.DataApiBuilder.Config.Converters;
+
+internal class DataSourceConverterFactory : JsonConverterFactory
+{
+    // Determines whether to replace environment variable with its
+    // value or not while deserializing.
+    private bool _replaceEnvVar;
+
+    /// <inheritdoc/>
+    public override bool CanConvert(Type typeToConvert)
+    {
+        return typeToConvert.IsAssignableTo(typeof(DataSource));
+    }
+
+    /// <inheritdoc/>
+    public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
+    {
+        return new DataSourceConverter(_replaceEnvVar);
+    }
+
+    /// <param name="replaceEnvVar">Whether to replace environment variable with its
+    /// value or not while deserializing.</param>
+    internal DataSourceConverterFactory(bool replaceEnvVar)
+    {
+        _replaceEnvVar = replaceEnvVar;
+    }
+
+    private class DataSourceConverter : JsonConverter<DataSource>
+    {
+        // Determines whether to replace environment variable with its
+        // value or not while deserializing.
+        private bool _replaceEnvVar;
+
+        /// <param name="replaceEnvVar">Whether to replace environment variable with its
+        /// value or not while deserializing.</param>
+        public DataSourceConverter(bool replaceEnvVar)
+        {
+            _replaceEnvVar = replaceEnvVar;
+        }
+
+        public override DataSource? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            DataSource dataSource = new(DatabaseType.MSSQL, string.Empty, null);
+            if (reader.TokenType is JsonTokenType.StartObject)
+            {
+
+                while (reader.Read() && reader.TokenType is not JsonTokenType.EndObject)
+                {
+                    if (reader.TokenType is JsonTokenType.PropertyName)
+                    {
+                        string propertyName = reader.GetString() ?? string.Empty;
+                        reader.Read();
+                        switch (propertyName)
+                        {
+                            case "database-type":
+                                dataSource = dataSource with { DatabaseType = EnumExtensions.Deserialize<DatabaseType>(reader.DeserializeString(_replaceEnvVar)!) };
+                                break;
+                            case "connection-string":
+                                dataSource = dataSource with { ConnectionString = reader.DeserializeString(replaceEnvVar: _replaceEnvVar)! };
+                                break;
+                            case "options":
+                                if (reader.TokenType == JsonTokenType.Null)
+                                {
+                                    dataSource = dataSource with { Options = null };
+                                    break;
+                                }
+                                else
+                                {
+                                    Dictionary<string, object?> optionsDict = new();
+                                    while (reader.Read() && reader.TokenType is not JsonTokenType.EndObject)
+                                    {
+                                        string optionsSubproperty = reader.GetString()!;
+                                        reader.Read();
+                                        object? optionsSubpropertyValue;
+                                        if (reader.TokenType is JsonTokenType.String)
+                                        {
+                                            // Determine whether to resolve the environment variable or keep as-is.
+                                            string stringValue = reader.DeserializeString(replaceEnvVar: _replaceEnvVar)!;
+
+                                            if (bool.TryParse(stringValue, out bool boolValue))
+                                            {
+                                                // MsSqlOptions.SetSessionContext will contain a boolean value.
+                                                optionsSubpropertyValue = boolValue;
+                                            }
+                                            else
+                                            {
+                                                // CosmosDbNoSQLDataSourceOptions will contain string values.
+                                                optionsSubpropertyValue = stringValue;
+                                            }
+                                        }
+                                        else if (reader.TokenType is JsonTokenType.True or JsonTokenType.False)
+                                        {
+                                            optionsSubpropertyValue = reader.GetBoolean();
+                                        }
+                                        else if (reader.TokenType is JsonTokenType.Null)
+                                        {
+                                            optionsSubpropertyValue = null;
+                                        }
+                                        else
+                                        {
+                                            throw new JsonException($"Unexpected value for options {optionsSubproperty} while deserializing DataSource options.");
+                                        }
+
+                                        optionsDict.Add(optionsSubproperty, optionsSubpropertyValue);
+                                    }
+
+                                    dataSource = dataSource with { Options = optionsDict };
+                                    break;
+                                }
+                            default:
+                                throw new JsonException($"Unexpected property {propertyName} while deserializing DataSource.");
+                        }
+                    }
+                }
+
+            }
+
+            return dataSource;
+        }
+
+        public override void Write(Utf8JsonWriter writer, DataSource value, JsonSerializerOptions options)
+        {
+            // Remove the converter so we don't recurse.
+            JsonSerializerOptions innerOptions = new(options);
+            innerOptions.Converters.Remove(innerOptions.Converters.First(c => c is DataSourceConverterFactory));
+
+            JsonSerializer.Serialize(writer, value, innerOptions);
+        }
+    }
+}
diff --git a/src/Config/ObjectModel/DataSource.cs b/src/Config/ObjectModel/DataSource.cs
diff --git a/src/Config/RuntimeConfigLoader.cs b/src/Config/RuntimeConfigLoader.cs
diff --git a/src/Core/Configurations/RuntimeConfigProvider.cs b/src/Core/Configurations/RuntimeConfigProvider.cs
diff --git a/src/Service.Tests/CosmosTests/TestBase.cs b/src/Service.Tests/CosmosTests/TestBase.cs
diff --git a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForCosmos.verified.txt b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForCosmos.verified.txt
diff --git a/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt b/src/Service.Tests/Snapshots/ConfigurationTests.TestReadingRuntimeConfigForMsSql.verified.txt
diff --git a/src/Service.Tests/Unittests/RuntimeConfigLoaderJsonDeserializerTests.cs b/src/Service.Tests/Unittests/RuntimeConfigLoaderJsonDeserializerTests.cs
