Skip to content

Commit fb4c7d8

Browse files
aaronburtleCopilotAniruddh25
authored and
Anusha Kolan
committed
Update variable replacement during deserialization to use replacement settings class and add AKV replacement logic. (#2882)
## Why make this change? Adds AKV variable replacement and expands our design for doing variable replacements to be more extensible when new variable replacement logic is added. Closes #2708 Closes #2748 Related to #2863 ## What is this change? Change the way that variable replacement is handled to instead of simply using a `bool` to indicate that we want env variable replacement, we add a class which holds all of the replacement settings. This will hold whether or not we will do replacement for each kind of variable that we will handle replacement for during deserialization. We also include the replacement failure mode, and put the logic for handling the replacements into a strategy dictionary which pairs the replacement variable type with the strategy for doing that replacement. Because Azure Key Vault secret replacement requires having the retry and connection settings in order to do the AKV replacement, we must do a first pass where we only do non-AKV replacement and get the required settings so that if AKV replacement is used we have the required settings to do that replacement. We also have to keep in mind that the legacy of the `Configuration Controller` will ignore all variable replacement, so we construct the replacement settings for this code path to not use any variable replacement at all. ## How was this tested? We have updated the logic for the tests to use the new system, however manual testing using an actual AKV is still required. ## Sample Request(s) - Example REST and/or GraphQL request to demonstrate modifications - Example of CLI usage to demonstrate modifications --------- Co-authored-by: Copilot <[email protected]> Co-authored-by: Aniruddh Munde <[email protected]>
1 parent 8260986 commit fb4c7d8

37 files changed

+835
-357
lines changed

‎src/Cli.Tests/EndToEndTests.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,10 +116,11 @@ public void TestInitializingRestAndGraphQLGlobalSettings()
116116
string[] args = { "init", "-c", TEST_RUNTIME_CONFIG_FILE, "--connection-string", SAMPLE_TEST_CONN_STRING, "--database-type", "mssql", "--rest.path", "/rest-api", "--rest.enabled", "false", "--graphql.path", "/graphql-api" };
117117
Program.Execute(args, _cliLogger!, _fileSystem!, _runtimeConfigLoader!);
118118

119+
DeserializationVariableReplacementSettings replacementSettings = new(azureKeyVaultOptions: null, doReplaceEnvVar: true, doReplaceAkvVar: true);
119120
Assert.IsTrue(_runtimeConfigLoader!.TryLoadConfig(
120121
TEST_RUNTIME_CONFIG_FILE,
121122
out RuntimeConfig? runtimeConfig,
122-
replaceEnvVar: true));
123+
replacementSettings: replacementSettings));
123124

124125
SqlConnectionStringBuilder builder = new(runtimeConfig.DataSource.ConnectionString);
125126
Assert.AreEqual(ProductInfo.GetDataApiBuilderUserAgent(), builder.ApplicationName);
@@ -195,10 +196,11 @@ public void TestEnablingMultipleCreateOperation(CliBool isMultipleCreateEnabled,
195196

196197
Program.Execute(args.ToArray(), _cliLogger!, _fileSystem!, _runtimeConfigLoader!);
197198

199+
DeserializationVariableReplacementSettings replacementSettings = new(azureKeyVaultOptions: null, doReplaceEnvVar: true, doReplaceAkvVar: true);
198200
Assert.IsTrue(_runtimeConfigLoader!.TryLoadConfig(
199201
TEST_RUNTIME_CONFIG_FILE,
200202
out RuntimeConfig? runtimeConfig,
201-
replaceEnvVar: true));
203+
replacementSettings: replacementSettings));
202204

203205
Assert.IsNotNull(runtimeConfig);
204206
Assert.AreEqual(expectedDbType, runtimeConfig.DataSource.DatabaseType);

‎src/Cli.Tests/EnvironmentTests.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,13 @@ public class EnvironmentTests
1919
[TestInitialize]
2020
public void TestInitialize()
2121
{
22-
StringJsonConverterFactory converterFactory = new(EnvironmentVariableReplacementFailureMode.Throw);
22+
DeserializationVariableReplacementSettings replacementSettings = new(
23+
azureKeyVaultOptions: null,
24+
doReplaceEnvVar: true,
25+
doReplaceAkvVar: false,
26+
envFailureMode: EnvironmentVariableReplacementFailureMode.Throw);
27+
28+
StringJsonConverterFactory converterFactory = new(replacementSettings);
2329
_options = new()
2430
{
2531
PropertyNameCaseInsensitive = true

‎src/Cli/ConfigGenerator.cs‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2701,9 +2701,10 @@ private static bool TryUpdateConfiguredAzureKeyVaultOptions(
27012701
// Azure Key Vault Endpoint
27022702
if (options.AzureKeyVaultEndpoint is not null)
27032703
{
2704+
// Ensure endpoint flag is marked user provided so converter writes it.
27042705
updatedAzureKeyVaultOptions = updatedAzureKeyVaultOptions is not null
2705-
? updatedAzureKeyVaultOptions with { Endpoint = options.AzureKeyVaultEndpoint }
2706-
: new AzureKeyVaultOptions { Endpoint = options.AzureKeyVaultEndpoint };
2706+
? updatedAzureKeyVaultOptions with { Endpoint = options.AzureKeyVaultEndpoint, UserProvidedEndpoint = true }
2707+
: new AzureKeyVaultOptions(endpoint: options.AzureKeyVaultEndpoint);
27072708
_logger.LogInformation("Updated RuntimeConfig with azure-key-vault.endpoint as '{endpoint}'", options.AzureKeyVaultEndpoint);
27082709
}
27092710

@@ -2712,7 +2713,7 @@ private static bool TryUpdateConfiguredAzureKeyVaultOptions(
27122713
{
27132714
updatedRetryPolicyOptions = updatedRetryPolicyOptions is not null
27142715
? updatedRetryPolicyOptions with { Mode = options.AzureKeyVaultRetryPolicyMode.Value, UserProvidedMode = true }
2715-
: new AKVRetryPolicyOptions { Mode = options.AzureKeyVaultRetryPolicyMode.Value, UserProvidedMode = true };
2716+
: new AKVRetryPolicyOptions(mode: options.AzureKeyVaultRetryPolicyMode.Value);
27162717
_logger.LogInformation("Updated RuntimeConfig with azure-key-vault.retry-policy.mode as '{mode}'", options.AzureKeyVaultRetryPolicyMode.Value);
27172718
}
27182719

@@ -2727,7 +2728,7 @@ private static bool TryUpdateConfiguredAzureKeyVaultOptions(
27272728

27282729
updatedRetryPolicyOptions = updatedRetryPolicyOptions is not null
27292730
? updatedRetryPolicyOptions with { MaxCount = options.AzureKeyVaultRetryPolicyMaxCount.Value, UserProvidedMaxCount = true }
2730-
: new AKVRetryPolicyOptions { MaxCount = options.AzureKeyVaultRetryPolicyMaxCount.Value, UserProvidedMaxCount = true };
2731+
: new AKVRetryPolicyOptions(maxCount: options.AzureKeyVaultRetryPolicyMaxCount.Value);
27312732
_logger.LogInformation("Updated RuntimeConfig with azure-key-vault.retry-policy.max-count as '{maxCount}'", options.AzureKeyVaultRetryPolicyMaxCount.Value);
27322733
}
27332734

@@ -2742,7 +2743,7 @@ private static bool TryUpdateConfiguredAzureKeyVaultOptions(
27422743

27432744
updatedRetryPolicyOptions = updatedRetryPolicyOptions is not null
27442745
? updatedRetryPolicyOptions with { DelaySeconds = options.AzureKeyVaultRetryPolicyDelaySeconds.Value, UserProvidedDelaySeconds = true }
2745-
: new AKVRetryPolicyOptions { DelaySeconds = options.AzureKeyVaultRetryPolicyDelaySeconds.Value, UserProvidedDelaySeconds = true };
2746+
: new AKVRetryPolicyOptions(delaySeconds: options.AzureKeyVaultRetryPolicyDelaySeconds.Value);
27462747
_logger.LogInformation("Updated RuntimeConfig with azure-key-vault.retry-policy.delay-seconds as '{delaySeconds}'", options.AzureKeyVaultRetryPolicyDelaySeconds.Value);
27472748
}
27482749

@@ -2757,7 +2758,7 @@ private static bool TryUpdateConfiguredAzureKeyVaultOptions(
27572758

27582759
updatedRetryPolicyOptions = updatedRetryPolicyOptions is not null
27592760
? updatedRetryPolicyOptions with { MaxDelaySeconds = options.AzureKeyVaultRetryPolicyMaxDelaySeconds.Value, UserProvidedMaxDelaySeconds = true }
2760-
: new AKVRetryPolicyOptions { MaxDelaySeconds = options.AzureKeyVaultRetryPolicyMaxDelaySeconds.Value, UserProvidedMaxDelaySeconds = true };
2761+
: new AKVRetryPolicyOptions(maxDelaySeconds: options.AzureKeyVaultRetryPolicyMaxDelaySeconds.Value);
27612762
_logger.LogInformation("Updated RuntimeConfig with azure-key-vault.retry-policy.max-delay-seconds as '{maxDelaySeconds}'", options.AzureKeyVaultRetryPolicyMaxDelaySeconds.Value);
27622763
}
27632764

@@ -2772,16 +2773,17 @@ private static bool TryUpdateConfiguredAzureKeyVaultOptions(
27722773

27732774
updatedRetryPolicyOptions = updatedRetryPolicyOptions is not null
27742775
? updatedRetryPolicyOptions with { NetworkTimeoutSeconds = options.AzureKeyVaultRetryPolicyNetworkTimeoutSeconds.Value, UserProvidedNetworkTimeoutSeconds = true }
2775-
: new AKVRetryPolicyOptions { NetworkTimeoutSeconds = options.AzureKeyVaultRetryPolicyNetworkTimeoutSeconds.Value, UserProvidedNetworkTimeoutSeconds = true };
2776+
: new AKVRetryPolicyOptions(networkTimeoutSeconds: options.AzureKeyVaultRetryPolicyNetworkTimeoutSeconds.Value);
27762777
_logger.LogInformation("Updated RuntimeConfig with azure-key-vault.retry-policy.network-timeout-seconds as '{networkTimeoutSeconds}'", options.AzureKeyVaultRetryPolicyNetworkTimeoutSeconds.Value);
27772778
}
27782779

2779-
// Update Azure Key Vault options with retry policy if retry policy was modified
2780+
// Update Azure Key Vault options with retry policy if modified
27802781
if (updatedRetryPolicyOptions is not null)
27812782
{
2783+
// Ensure outer AKV object marks retry policy as user provided so it serializes.
27822784
updatedAzureKeyVaultOptions = updatedAzureKeyVaultOptions is not null
2783-
? updatedAzureKeyVaultOptions with { RetryPolicy = updatedRetryPolicyOptions }
2784-
: new AzureKeyVaultOptions { RetryPolicy = updatedRetryPolicyOptions };
2785+
? updatedAzureKeyVaultOptions with { RetryPolicy = updatedRetryPolicyOptions, UserProvidedRetryPolicy = true }
2786+
: new AzureKeyVaultOptions(retryPolicy: updatedRetryPolicyOptions);
27852787
}
27862788

27872789
// Update runtime config if Azure Key Vault options were modified

‎src/Cli/Exporter.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ public static bool Export(ExportOptions options, ILogger logger, FileSystemRunti
4444
}
4545

4646
// Load the runtime configuration from the file
47-
if (!loader.TryLoadConfig(runtimeConfigFile, out RuntimeConfig? runtimeConfig, replaceEnvVar: true))
47+
DeserializationVariableReplacementSettings replacementSettings = new(azureKeyVaultOptions: null, doReplaceEnvVar: true, doReplaceAkvVar: true);
48+
if (!loader.TryLoadConfig(runtimeConfigFile, out RuntimeConfig? runtimeConfig, replacementSettings: replacementSettings))
4849
{
4950
logger.LogError("Failed to read the config file: {0}.", runtimeConfigFile);
5051
return false;

‎src/Config/Azure.DataApiBuilder.Config.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
<ItemGroup>
1717
<PackageReference Include="Azure.Identity" />
18+
<PackageReference Include="Azure.Security.KeyVault.Secrets" />
1819
<PackageReference Include="Microsoft.AspNetCore.Authorization" />
1920
<PackageReference Include="Microsoft.IdentityModel.Protocols" />
2021
<PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" />

‎src/Config/Converters/AKVRetryPolicyOptionsConverterFactory.cs‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ namespace Azure.DataApiBuilder.Config.Converters;
1212
/// </summary>
1313
internal class AKVRetryPolicyOptionsConverterFactory : JsonConverterFactory
1414
{
15-
// Determines whether to replace environment variable with its
16-
// value or not while deserializing.
17-
private bool _replaceEnvVar;
15+
// Settings for variable replacement during deserialization.
16+
// Currently allows for Azure Key Vault (via @akv('secret-name')) and Environment Variable replacement.
17+
private readonly DeserializationVariableReplacementSettings? _replacementSettings;
1818

1919
/// <inheritdoc/>
2020
public override bool CanConvert(Type typeToConvert)
@@ -25,34 +25,34 @@ public override bool CanConvert(Type typeToConvert)
2525
/// <inheritdoc/>
2626
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
2727
{
28-
return new AKVRetryPolicyOptionsConverter(_replaceEnvVar);
28+
return new AKVRetryPolicyOptionsConverter(_replacementSettings);
2929
}
3030

31-
/// <param name="replaceEnvVar">Whether to replace environment variable with its
32-
/// value or not while deserializing.</param>
33-
internal AKVRetryPolicyOptionsConverterFactory(bool replaceEnvVar)
31+
/// <param name="replacementSettings">Settings for variable replacement during deserialization.
32+
/// If null, no variable replacement will be performed.</param>
33+
internal AKVRetryPolicyOptionsConverterFactory(DeserializationVariableReplacementSettings? replacementSettings = null)
3434
{
35-
_replaceEnvVar = replaceEnvVar;
35+
_replacementSettings = replacementSettings;
3636
}
3737

3838
private class AKVRetryPolicyOptionsConverter : JsonConverter<AKVRetryPolicyOptions>
3939
{
40-
// Determines whether to replace environment variable with its
41-
// value or not while deserializing.
42-
private bool _replaceEnvVar;
40+
// Settings for variable replacement during deserialization.
41+
// Currently allows for Azure Key Vault (via @akv('<secret>')) and Environment Variable replacement.
42+
private readonly DeserializationVariableReplacementSettings? _replacementSettings;
4343

44-
/// <param name="replaceEnvVar">Whether to replace environment variable with its
45-
/// value or not while deserializing.</param>
46-
public AKVRetryPolicyOptionsConverter(bool replaceEnvVar)
44+
/// <param name="replacementSettings">Settings for variable replacement during deserialization.
45+
/// If null, no variable replacement will be performed.</param>
46+
public AKVRetryPolicyOptionsConverter(DeserializationVariableReplacementSettings? replacementSettings)
4747
{
48-
_replaceEnvVar = replaceEnvVar;
48+
_replacementSettings = replacementSettings;
4949
}
5050

5151
/// <summary>
5252
/// Defines how DAB reads AKV Retry Policy options and defines which values are
5353
/// used to instantiate those options.
5454
/// </summary>
55-
/// <exception cref="JsonException">Thrown when improperly formatted cache options are provided.</exception>
55+
/// <exception cref="JsonException">Thrown when improperly formatted retry policy options are provided.</exception>
5656
public override AKVRetryPolicyOptions? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
5757
{
5858
if (reader.TokenType is JsonTokenType.StartObject)
@@ -82,7 +82,7 @@ public AKVRetryPolicyOptionsConverter(bool replaceEnvVar)
8282
}
8383
else
8484
{
85-
mode = EnumExtensions.Deserialize<AKVRetryPolicyMode>(reader.DeserializeString(_replaceEnvVar)!);
85+
mode = EnumExtensions.Deserialize<AKVRetryPolicyMode>(reader.DeserializeString(_replacementSettings)!);
8686
}
8787

8888
break;

‎src/Config/Converters/AzureKeyVaultOptionsConverterFactory.cs‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System.Text.Json;
5+
using System.Text.Json.Serialization;
6+
using Azure.DataApiBuilder.Config.ObjectModel;
7+
8+
namespace Azure.DataApiBuilder.Config.Converters;
9+
10+
/// <summary>
11+
/// Converter factory for AzureKeyVaultOptions that can optionally perform variable replacement.
12+
/// </summary>
13+
internal class AzureKeyVaultOptionsConverterFactory : JsonConverterFactory
14+
{
15+
// Determines whether to replace environment variable with its
16+
// value or not while deserializing.
17+
private readonly DeserializationVariableReplacementSettings? _replacementSettings;
18+
19+
/// <param name="replacementSettings">How to handle variable replacement during deserialization.</param>
20+
internal AzureKeyVaultOptionsConverterFactory(DeserializationVariableReplacementSettings? replacementSettings = null)
21+
{
22+
_replacementSettings = replacementSettings;
23+
}
24+
25+
/// <inheritdoc/>
26+
public override bool CanConvert(Type typeToConvert)
27+
{
28+
return typeToConvert.IsAssignableTo(typeof(AzureKeyVaultOptions));
29+
}
30+
31+
/// <inheritdoc/>
32+
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
33+
{
34+
return new AzureKeyVaultOptionsConverter(_replacementSettings);
35+
}
36+
37+
private class AzureKeyVaultOptionsConverter : JsonConverter<AzureKeyVaultOptions>
38+
{
39+
// Determines whether to replace environment variable with its
40+
// value or not while deserializing.
41+
private readonly DeserializationVariableReplacementSettings? _replacementSettings;
42+
43+
/// <param name="replaceEnvVar">Whether to replace environment variable with its
44+
/// value or not while deserializing.</param>
45+
public AzureKeyVaultOptionsConverter(DeserializationVariableReplacementSettings? replacementSettings)
46+
{
47+
_replacementSettings = replacementSettings;
48+
}
49+
50+
/// <summary>
51+
/// Reads AzureKeyVaultOptions with optional variable replacement.
52+
/// </summary>
53+
public override AzureKeyVaultOptions? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
54+
{
55+
if (reader.TokenType is JsonTokenType.Null)
56+
{
57+
return null;
58+
}
59+
60+
if (reader.TokenType is JsonTokenType.StartObject)
61+
{
62+
string? endpoint = null;
63+
AKVRetryPolicyOptions? retryPolicy = null;
64+
65+
while (reader.Read())
66+
{
67+
if (reader.TokenType is JsonTokenType.EndObject)
68+
{
69+
return new AzureKeyVaultOptions(endpoint, retryPolicy);
70+
}
71+
72+
string? property = reader.GetString();
73+
reader.Read();
74+
75+
switch (property)
76+
{
77+
case "endpoint":
78+
if (reader.TokenType is JsonTokenType.String)
79+
{
80+
endpoint = reader.DeserializeString(_replacementSettings);
81+
}
82+
83+
break;
84+
85+
case "retry-policy":
86+
if (reader.TokenType is JsonTokenType.StartObject)
87+
{
88+
// Uses the AKVRetryPolicyOptionsConverter to read the retry-policy object.
89+
retryPolicy = JsonSerializer.Deserialize<AKVRetryPolicyOptions>(ref reader, options);
90+
}
91+
92+
break;
93+
94+
default:
95+
throw new JsonException($"Unexpected property {property}");
96+
}
97+
}
98+
}
99+
100+
throw new JsonException("Invalid AzureKeyVaultOptions format");
101+
}
102+
103+
/// <summary>
104+
/// When writing the AzureKeyVaultOptions back to a JSON file, only write the properties
105+
/// if they are user provided. This avoids polluting the written JSON file with properties
106+
/// the user most likely omitted when writing the original DAB runtime config file.
107+
/// This Write operation is only used when a RuntimeConfig object is serialized to JSON.
108+
/// </summary>
109+
public override void Write(Utf8JsonWriter writer, AzureKeyVaultOptions value, JsonSerializerOptions options)
110+
{
111+
writer.WriteStartObject();
112+
113+
if (value?.UserProvidedEndpoint is true)
114+
{
115+
writer.WritePropertyName("endpoint");
116+
JsonSerializer.Serialize(writer, value.Endpoint, options);
117+
}
118+
119+
if (value?.UserProvidedRetryPolicy is true)
120+
{
121+
writer.WritePropertyName("retry-policy");
122+
JsonSerializer.Serialize(writer, value.RetryPolicy, options);
123+
}
124+
125+
writer.WriteEndObject();
126+
}
127+
}
128+
}

0 commit comments

Comments
 (0)