Add logic to connect to Azure Log Analytics (#2787)

## Why make this change?

Solves issue #2778

## What is this change?

This change adds logic to capture logs that DAB produces in order to
send them to the users' Azure Log Analytics tables. It does this by
creating a logger provider that is activated when the feature is enabled
and sends the logs that are produced to a custom log collector. This
custom log collector then flushes all of the saved logs periodically
every certain amount of time to Azure Log Analytics.
It is also important to note, that the name of one of the properties in
the feature was changed from `workspace-id` to `custom-table-name` as I
found that the feature did not need the `workspace-id` to connect to the
table, but it needed the name of custom log that is created as a
resource by the user.

Files Created:
- `AzureLogAnalyticsFlusherService` uploads the logs periodically to the
Azure Log Analytics Workspace Table.
- `AzureLogAnalyticsCustomLogCollector` it collects the logs from the
rest of the services and pushes them to
`AzureLogAnalyticsFlusherService`.
- `AzureLogAnalyticsLoggerProvider` creates the
`AzureLogAnalyticsLogger` to start the population of logs inside the
`AzureLogAnalyticsCustomLogCollector`.
- `AzureLogAnalyticsLog` is the base object that defines the structure
of the logs that will be sent to Azure Log Analytics.

## How was this tested?

- [ ] Integration Tests
- [X] Unit Tests
- [X] Manual Tests

For the manual testing, I had to create various resources inside of
Azure:
- Create Azure Log Analytics Workspace and creating a table inside of it
and decide on the `custom-table-name`.
- Create `DCE Endpoint` which is used the entry point for DAB to send
its logs
- Create DCR or Data Collection Rule and set rules on the structure of
the logs that it will receive, and set the workspace table and DCE
Endpoint to which it will connect.
- Create a VM that has a system assigned managed identity.
- Assign permission on DCR to allow VM to write telemetry data.

After creating all the necessary resources, you just need to run DAB
inside of the VM to have Azure Log Analytics logs sent.

You can follow the steps more in detail in the following link:

https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-api?tabs=dce#create-data-collection-rule

## Sample Request(s)
<img width="1105" height="400" alt="image"
src="https://github.com/user-attachments/assets/6cd3a597-69e8-4777-8c6d-bc167b6fd424"
/>

Author: RubenCerna2079

schemas/dab.draft.schema.json

@@ -423,9 +423,9 @@
                   "type": "object",
                   "additionalProperties": false,
                   "properties": {
-                    "workspace-id": {
+                    "custom-table-name": {
                       "type": [ "string", "null" ],
-                      "description": "Azure Log Analytics Workspace ID"
+                      "description": "Azure Log Analytics Custom Table Name for entra-id mode"
                     },
                     "dcr-immutable-id": {
                       "type": [ "string", "null" ],
@@ -459,9 +459,9 @@
                 "properties": {
                   "auth": {
                     "properties": {
-                      "workspace-id": {
+                      "custom-table-name": {
                         "type": "string",
-                        "description": "Azure Log Analytics Workspace ID"
+                        "description": "Azure Log Analytics Custom Table Name for entra-id mode"
                       },
                       "dcr-immutable-id": {
                         "type": "string",
@@ -472,7 +472,7 @@
                         "description": "DCE endpoint for entra-id mode"
                       }
                     },
-                    "required": [ "workspace-id", "dcr-immutable-id", "dce-endpoint" ]
+                    "required": [ "custom-table-name", "dcr-immutable-id", "dce-endpoint" ]
                   }
                 },
                 "required": [ "auth" ]

src/Cli.Tests/ConfigureOptionsTests.cs

@@ -164,7 +164,7 @@ public void TestAddAzureLogAnalyticsOptions()
                 azureLogAnalyticsEnabled: CliBool.True,
                 azureLogAnalyticsLogType: "log-type-test",
                 azureLogAnalyticsFlushIntervalSeconds: 1,
-                azureLogAnalyticsWorkspaceId: "workspace-id-test",
+                azureLogAnalyticsCustomTableName: "custom-table-name-test",
                 azureLogAnalyticsDcrImmutableId: "dcr-immutable-id-test",
                 azureLogAnalyticsDceEndpoint: "dce-endpoint-test",
                 config: TEST_RUNTIME_CONFIG_FILE
@@ -183,7 +183,7 @@ public void TestAddAzureLogAnalyticsOptions()
             Assert.AreEqual("log-type-test", config.Runtime.Telemetry.AzureLogAnalytics.LogType);
             Assert.AreEqual(1, config.Runtime.Telemetry.AzureLogAnalytics.FlushIntervalSeconds);
             Assert.IsNotNull(config.Runtime.Telemetry.AzureLogAnalytics.Auth);
-            Assert.AreEqual("workspace-id-test", config.Runtime.Telemetry.AzureLogAnalytics.Auth.WorkspaceId);
+            Assert.AreEqual("custom-table-name-test", config.Runtime.Telemetry.AzureLogAnalytics.Auth.CustomTableName);
             Assert.AreEqual("dcr-immutable-id-test", config.Runtime.Telemetry.AzureLogAnalytics.Auth.DcrImmutableId);
             Assert.AreEqual("dce-endpoint-test", config.Runtime.Telemetry.AzureLogAnalytics.Auth.DceEndpoint);
         }

src/Cli/Commands/ConfigureOptions.cs

@@ -51,7 +51,7 @@ public ConfigureOptions(
             CliBool? azureLogAnalyticsEnabled = null,
             string? azureLogAnalyticsLogType = null,
             int? azureLogAnalyticsFlushIntervalSeconds = null,
-            string? azureLogAnalyticsWorkspaceId = null,
+            string? azureLogAnalyticsCustomTableName = null,
             string? azureLogAnalyticsDcrImmutableId = null,
             string? azureLogAnalyticsDceEndpoint = null,
             string? config = null)
@@ -95,7 +95,7 @@ public ConfigureOptions(
             AzureLogAnalyticsEnabled = azureLogAnalyticsEnabled;
             AzureLogAnalyticsLogType = azureLogAnalyticsLogType;
             AzureLogAnalyticsFlushIntervalSeconds = azureLogAnalyticsFlushIntervalSeconds;
-            AzureLogAnalyticsWorkspaceId = azureLogAnalyticsWorkspaceId;
+            AzureLogAnalyticsCustomTableName = azureLogAnalyticsCustomTableName;
             AzureLogAnalyticsDcrImmutableId = azureLogAnalyticsDcrImmutableId;
             AzureLogAnalyticsDceEndpoint = azureLogAnalyticsDceEndpoint;
         }
@@ -184,17 +184,17 @@ public ConfigureOptions(
         [Option("azure-key-vault.retry-policy.network-timeout-seconds", Required = false, HelpText = "Configure the network timeout for requests in seconds. Default: 60.")]
         public int? AzureKeyVaultRetryPolicyNetworkTimeoutSeconds { get; }
 
-        [Option("runtime.telemetry.azure-log-analytics.enabled", Default = CliBool.False, Required = false, HelpText = "Enable/Disable Azure Log Analytics.")]
+        [Option("runtime.telemetry.azure-log-analytics.enabled", Required = false, HelpText = "Enable/Disable Azure Log Analytics. Default: False (boolean)")]
         public CliBool? AzureLogAnalyticsEnabled { get; }
 
-        [Option("runtime.telemetry.azure-log-analytics.log-type", Required = false, HelpText = "Configure Log Type for Azure Log Analytics to find table to send telemetry data")]
+        [Option("runtime.telemetry.azure-log-analytics.log-type", Required = false, HelpText = "Configure Log Type for Azure Log Analytics to find table to send telemetry data. Default: DABLogs")]
         public string? AzureLogAnalyticsLogType { get; }
 
-        [Option("runtime.telemetry.azure-log-analytics.flush-interval-seconds", Required = false, HelpText = "Configure Flush Interval in seconds for Azure Log Analytics to specify the time interval to send the telemetry data")]
+        [Option("runtime.telemetry.azure-log-analytics.flush-interval-seconds", Required = false, HelpText = "Configure Flush Interval in seconds for Azure Log Analytics to specify the time interval to send the telemetry data. Default: 5")]
         public int? AzureLogAnalyticsFlushIntervalSeconds { get; }
 
-        [Option("runtime.telemetry.azure-log-analytics.auth.workspace-id", Required = false, HelpText = "Configure Workspace ID for Azure Log Analytics used to find workspace to connect")]
-        public string? AzureLogAnalyticsWorkspaceId { get; }
+        [Option("runtime.telemetry.azure-log-analytics.auth.custom-table-name", Required = false, HelpText = "Configure Custom Table Name for Azure Log Analytics used to find table to connect")]
+        public string? AzureLogAnalyticsCustomTableName { get; }
 
         [Option("runtime.telemetry.azure-log-analytics.auth.dcr-immutable-id", Required = false, HelpText = "Configure DCR Immutable ID for Azure Log Analytics to find the data collection rule that defines how data is collected")]
         public string? AzureLogAnalyticsDcrImmutableId { get; }

src/Cli/ConfigGenerator.cs

@@ -782,7 +782,7 @@ private static bool TryUpdateConfiguredRuntimeOptions(
             if (options.AzureLogAnalyticsEnabled is not null ||
                 options.AzureLogAnalyticsLogType is not null ||
                 options.AzureLogAnalyticsFlushIntervalSeconds is not null ||
-                options.AzureLogAnalyticsWorkspaceId is not null ||
+                options.AzureLogAnalyticsCustomTableName is not null ||
                 options.AzureLogAnalyticsDcrImmutableId is not null ||
                 options.AzureLogAnalyticsDceEndpoint is not null)
             {
@@ -1157,13 +1157,13 @@ private static bool TryUpdateConfiguredAzureLogAnalyticsOptions(
                     _logger.LogInformation($"Updated configuration with runtime.telemetry.azure-log-analytics.flush-interval-seconds as '{options.AzureLogAnalyticsFlushIntervalSeconds}'");
                 }
 
-                // Runtime.Telemetry.AzureLogAnalytics.Auth.WorkspaceId
-                if (options.AzureLogAnalyticsWorkspaceId is not null)
+                // Runtime.Telemetry.AzureLogAnalytics.Auth.CustomTableName
+                if (options.AzureLogAnalyticsCustomTableName is not null)
                 {
                     updatedAuthOptions = updatedAuthOptions is not null
-                        ? updatedAuthOptions with { WorkspaceId = options.AzureLogAnalyticsWorkspaceId, UserProvidedWorkspaceId = true }
-                        : new AzureLogAnalyticsAuthOptions { WorkspaceId = options.AzureLogAnalyticsWorkspaceId, UserProvidedWorkspaceId = true };
-                    _logger.LogInformation($"Updated configuration with runtime.telemetry.azure-log-analytics.auth.workspace-id as '{options.AzureLogAnalyticsWorkspaceId}'");
+                        ? updatedAuthOptions with { CustomTableName = options.AzureLogAnalyticsCustomTableName, UserProvidedCustomTableName = true }
+                        : new AzureLogAnalyticsAuthOptions { CustomTableName = options.AzureLogAnalyticsCustomTableName, UserProvidedCustomTableName = true };
+                    _logger.LogInformation($"Updated configuration with runtime.telemetry.azure-log-analytics.auth.custom-table-name as '{options.AzureLogAnalyticsCustomTableName}'");
                 }
 
                 // Runtime.Telemetry.AzureLogAnalytics.Auth.DcrImmutableId

src/Config/Converters/AzureLogAnalyticsAuthOptionsConverter.cs

@@ -29,26 +29,26 @@ public AzureLogAnalyticsAuthOptionsConverter(bool replaceEnvVar)
     {
         if (reader.TokenType is JsonTokenType.StartObject)
         {
-            string? workspaceId = null;
+            string? customTableName = null;
             string? dcrImmutableId = null;
             string? dceEndpoint = null;
 
             while (reader.Read())
             {
                 if (reader.TokenType == JsonTokenType.EndObject)
                 {
-                    return new AzureLogAnalyticsAuthOptions(workspaceId, dcrImmutableId, dceEndpoint);
+                    return new AzureLogAnalyticsAuthOptions(customTableName, dcrImmutableId, dceEndpoint);
                 }
 
                 string? propertyName = reader.GetString();
 
                 reader.Read();
                 switch (propertyName)
                 {
-                    case "workspace-id":
+                    case "custom-table-name":
                         if (reader.TokenType is not JsonTokenType.Null)
                         {
-                            workspaceId = reader.DeserializeString(_replaceEnvVar);
+                            customTableName = reader.DeserializeString(_replaceEnvVar);
                         }
 
                         break;
@@ -88,10 +88,10 @@ public override void Write(Utf8JsonWriter writer, AzureLogAnalyticsAuthOptions v
     {
         writer.WriteStartObject();
 
-        if (value?.UserProvidedWorkspaceId is true)
+        if (value?.UserProvidedCustomTableName is true)
         {
-            writer.WritePropertyName("workspace-id");
-            JsonSerializer.Serialize(writer, value.WorkspaceId, options);
+            writer.WritePropertyName("custom-table-name");
+            JsonSerializer.Serialize(writer, value.CustomTableName, options);
         }
 
         if (value?.UserProvidedDcrImmutableId is true)

src/Config/Converters/AzureLogAnalyticsOptionsConverterFactory.cs

@@ -141,7 +141,7 @@ public override void Write(Utf8JsonWriter writer, AzureLogAnalyticsOptions value
                 JsonSerializer.Serialize(writer, value.Enabled, options);
             }
 
-            if (value?.Auth is not null && (value.Auth.UserProvidedWorkspaceId || value.Auth.UserProvidedDcrImmutableId || value.Auth.UserProvidedDceEndpoint))
+            if (value?.Auth is not null && (value.Auth.UserProvidedCustomTableName || value.Auth.UserProvidedDcrImmutableId || value.Auth.UserProvidedDceEndpoint))
             {
                 AzureLogAnalyticsAuthOptionsConverter authOptionsConverter = options.GetConverter(typeof(AzureLogAnalyticsAuthOptions)) as AzureLogAnalyticsAuthOptionsConverter ??
                                     throw new JsonException("Failed to get azure-log-analytics.auth options converter");

src/Config/ObjectModel/AzureLogAnalyticsAuthOptions.cs

@@ -14,7 +14,7 @@ public record AzureLogAnalyticsAuthOptions
     /// <summary>
     /// Whether Azure Log Analytics is enabled.
     /// </summary>
-    public string? WorkspaceId { get; init; }
+    public string? CustomTableName { get; init; }
 
     /// <summary>
     /// Authentication options for Azure Log Analytics.
@@ -27,12 +27,12 @@ public record AzureLogAnalyticsAuthOptions
     public string? DceEndpoint { get; init; }
 
     [JsonConstructor]
-    public AzureLogAnalyticsAuthOptions(string? workspaceId = null, string? dcrImmutableId = null, string? dceEndpoint = null)
+    public AzureLogAnalyticsAuthOptions(string? customTableName = null, string? dcrImmutableId = null, string? dceEndpoint = null)
     {
-        if (workspaceId is not null)
+        if (customTableName is not null)
         {
-            WorkspaceId = workspaceId;
-            UserProvidedWorkspaceId = true;
+            CustomTableName = customTableName;
+            UserProvidedCustomTableName = true;
         }
 
         if (dcrImmutableId is not null)
@@ -51,12 +51,12 @@ public AzureLogAnalyticsAuthOptions(string? workspaceId = null, string? dcrImmut
     /// <summary>
     /// Flag which informs CLI and JSON serializer whether to write workspace-id
     /// property and value to the runtime config file.
-    /// When user doesn't provide the workspace-id property/value, which signals DAB to not write anything,
+    /// When user doesn't provide the custom-table-name property/value, which signals DAB to not write anything,
     /// the DAB CLI should not write the current value to a serialized config.
     /// </summary>
     [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
-    [MemberNotNullWhen(true, nameof(WorkspaceId))]
-    public bool UserProvidedWorkspaceId { get; init; } = false;
+    [MemberNotNullWhen(true, nameof(CustomTableName))]
+    public bool UserProvidedCustomTableName { get; init; } = false;
 
     /// <summary>
     /// Flag which informs CLI and JSON serializer whether to write dcr-immutable-id

src/Config/ObjectModel/AzureLogAnalyticsLogs.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace Azure.DataApiBuilder.Config.ObjectModel;
+
+/// <summary>
+/// Class used to save the components for the logs that are sent to Azure Log Analytics
+/// </summary>
+public class AzureLogAnalyticsLogs
+{
+    public string Time { get; set; }
+    public string LogLevel { get; set; }
+    public string? Message { get; set; }
+    public string? Component { get; set; }
+    public string? LogType { get; set; }
+
+    public AzureLogAnalyticsLogs(string time, string logLevel, string? message, string? component, string? logType = null)
+    {
+        Time = time;
+        LogLevel = logLevel;
+        Message = message;
+        Component = component;
+        LogType = logType;
+    }
+}

src/Core/Configurations/RuntimeConfigValidator.cs

@@ -166,11 +166,11 @@ public void ValidateAzureLogAnalyticsAuth(RuntimeConfig runtimeConfig)
         {
             AzureLogAnalyticsOptions azureLogAnalyticsOptions = runtimeConfig.Runtime.Telemetry.AzureLogAnalytics;
             AzureLogAnalyticsAuthOptions? azureLogAnalyticsAuthOptions = azureLogAnalyticsOptions.Auth;
-            if (azureLogAnalyticsOptions.Enabled && (azureLogAnalyticsAuthOptions is null || string.IsNullOrWhiteSpace(azureLogAnalyticsAuthOptions.WorkspaceId) ||
+            if (azureLogAnalyticsOptions.Enabled && (azureLogAnalyticsAuthOptions is null || string.IsNullOrWhiteSpace(azureLogAnalyticsAuthOptions.CustomTableName) ||
                 string.IsNullOrWhiteSpace(azureLogAnalyticsAuthOptions.DcrImmutableId) || string.IsNullOrWhiteSpace(azureLogAnalyticsAuthOptions.DceEndpoint)))
             {
                 HandleOrRecordException(new DataApiBuilderException(
-                    message: "Azure Log Analytics Auth options 'workspace-id', 'dcr-immutable-id', and 'dce-endpoint' cannot be null or empty if enabled.",
+                    message: "Azure Log Analytics Auth options 'custom-table-name', 'dcr-immutable-id', and 'dce-endpoint' cannot be null or empty if enabled.",
                     statusCode: HttpStatusCode.ServiceUnavailable,
                     subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError));
             }