Add retry for Trigger tests (#836)

* add retry logic

* fix the csx compilation error

* use xRetry

* replace with RetryTheory

* add diagnostic messages to logs

Author: MaddyDev

Directory.Packages.props

@@ -15,6 +15,7 @@
     <PackageVersion Include="Moq" Version="4.18.2" />
     <PackageVersion Include="xunit" Version="2.4.2" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.4.5" />
+    <PackageVersion Include="xRetry" Version="1.9.0" />
     <PackageVersion Include="Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator" Version="4.0.1" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.2" />
     <PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.2.2" />

performance/packages.lock.json

@@ -1890,6 +1890,7 @@
           "Microsoft.NET.Test.Sdk": "[17.4.0, )",
           "Moq": "[4.18.2, )",
           "Newtonsoft.Json": "[13.0.2, )",
+          "xRetry": "[1.9.0, )",
           "xunit": "[2.4.2, )",
           "xunit.runner.visualstudio": "[2.4.5, )"
         }
@@ -2086,6 +2087,15 @@
           "System.Configuration.ConfigurationManager": "5.0.0"
         }
       },
+      "xRetry": {
+        "type": "CentralTransitive",
+        "requested": "[1.9.0, )",
+        "resolved": "1.9.0",
+        "contentHash": "NeIbJrwpc5EUPagx/mdd/7KzpR36BO8IWrsbgtvOVjxD2xtmNfUHieZ24PeZ4oCYiLBcTviCy+og/bE/OvPchw==",
+        "dependencies": {
+          "xunit.core": "[2.4.0, 3.0.0)"
+        }
+      },
       "xunit": {
         "type": "CentralTransitive",
         "requested": "[2.4.2, )",

samples/samples-outofproc/packages.lock.json

@@ -846,9 +846,9 @@
       "microsoft.azure.functions.worker.extensions.sql": {
         "type": "Project",
         "dependencies": {
-          "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": "1.1.0",
-          "Microsoft.Data.SqlClient": "5.0.1",
-          "System.Drawing.Common": "5.0.3"
+          "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": "[1.1.0, )",
+          "Microsoft.Data.SqlClient": "[5.0.1, )",
+          "System.Drawing.Common": "[5.0.3, )"
         }
       },
       "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": {

test/Integration/SqlTriggerBindingIntegrationTests.cs

@@ -15,6 +15,7 @@
 using Moq;
 using Xunit;
 using Xunit.Abstractions;
+using xRetry;
 
 namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration
 {
@@ -28,7 +29,7 @@ public SqlTriggerBindingIntegrationTests(ITestOutputHelper output = null) : base
         /// <summary>
         /// Ensures that the user function gets invoked for each of the insert, update and delete operation.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public async Task SingleOperationTriggerTest(SupportedLanguages lang)
         {
@@ -76,7 +77,7 @@ await this.WaitForProductChanges(
         /// Verifies that manually setting the batch size using the original config var correctly changes the
         /// number of changes processed at once.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public async Task BatchSizeOverrideTriggerTest(SupportedLanguages lang)
         {
@@ -119,7 +120,7 @@ await this.WaitForProductChanges(
         /// <summary>
         /// Verifies that manually setting the max batch size correctly changes the number of changes processed at once
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public async Task MaxBatchSizeOverrideTriggerTest(SupportedLanguages lang)
         {
@@ -162,7 +163,7 @@ await this.WaitForProductChanges(
         /// <summary>
         /// Verifies that manually setting the polling interval correctly changes the delay between processing each batch of changes
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public async Task PollingIntervalOverrideTriggerTest(SupportedLanguages lang)
         {
@@ -204,7 +205,7 @@ await this.WaitForProductChanges(
         /// Verifies that if several changes have happened to the table row since last invocation, then a single net
         /// change for that row is passed to the user function.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public async Task MultiOperationTriggerTest(SupportedLanguages lang)
         {
@@ -285,7 +286,7 @@ await this.WaitForProductChanges(
         /// <summary>
         /// Ensures correct functionality with multiple user functions tracking the same table.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public async Task MultiFunctionTriggerTest(SupportedLanguages lang)
         {
@@ -411,7 +412,7 @@ public async Task MultiFunctionTriggerTest(SupportedLanguages lang)
         /// <summary>
         /// Ensures correct functionality with user functions running across multiple functions host processes.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public async Task MultiHostTriggerTest(SupportedLanguages lang)
         {
@@ -462,7 +463,7 @@ await this.WaitForProductChanges(
         /// <summary>
         /// Tests the error message when the user table is not present in the database.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public void TableNotPresentTriggerTest(SupportedLanguages lang)
         {
@@ -476,7 +477,7 @@ public void TableNotPresentTriggerTest(SupportedLanguages lang)
         /// <summary>
         /// Tests the error message when the user table does not contain primary key.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public void PrimaryKeyNotCreatedTriggerTest(SupportedLanguages lang)
         {
@@ -491,7 +492,7 @@ public void PrimaryKeyNotCreatedTriggerTest(SupportedLanguages lang)
         /// Tests the error message when the user table contains one or more primary keys with names conflicting with
         /// column names in the leases table.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public void ReservedPrimaryKeyColumnNamesTriggerTest(SupportedLanguages lang)
         {
@@ -506,7 +507,7 @@ public void ReservedPrimaryKeyColumnNamesTriggerTest(SupportedLanguages lang)
         /// <summary>
         /// Tests the error message when the user table contains columns of unsupported SQL types.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public void UnsupportedColumnTypesTriggerTest(SupportedLanguages lang)
         {
@@ -521,7 +522,7 @@ public void UnsupportedColumnTypesTriggerTest(SupportedLanguages lang)
         /// <summary>
         /// Tests the error message when change tracking is not enabled on the user table.
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public void ChangeTrackingNotEnabledTriggerTest(SupportedLanguages lang)
         {
@@ -557,7 +558,7 @@ public async void GetMetricsTest()
         /// <summary>
         /// Tests that when using an unsupported database the expected error is thrown
         /// </summary>
-        [Theory]
+        [RetryTheory]
         [SqlInlineData()]
         public void UnsupportedDatabaseThrows(SupportedLanguages lang)
         {

test/Microsoft.Azure.WebJobs.Extensions.Sql.Tests.csproj

@@ -5,6 +5,7 @@
     <PackageReference Include="Moq" />
     <PackageReference Include="xunit" />
     <PackageReference Include="xunit.runner.visualstudio" />
+    <PackageReference Include="xRetry" />
     <PackageReference Include="Microsoft.NET.Sdk.Functions" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="Microsoft.AspNetCore.Http" />
@@ -18,8 +19,11 @@
 
   <ItemGroup>
     <None Update="Database\**\*.sql">
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-      </None>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="xunit.runner.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
   <Target Name="CopySamples" AfterTargets="Build">

test/packages.lock.json

@@ -74,6 +74,15 @@
         "resolved": "13.0.2",
         "contentHash": "R2pZ3B0UjeyHShm9vG+Tu0EBb2lC8b0dFzV9gVn50ofHXh9Smjk6kTn7A/FdAsC8B5cKib1OnGYOXxRBz5XQDg=="
       },
+      "xRetry": {
+        "type": "Direct",
+        "requested": "[1.9.0, )",
+        "resolved": "1.9.0",
+        "contentHash": "NeIbJrwpc5EUPagx/mdd/7KzpR36BO8IWrsbgtvOVjxD2xtmNfUHieZ24PeZ4oCYiLBcTviCy+og/bE/OvPchw==",
+        "dependencies": {
+          "xunit.core": "[2.4.0, 3.0.0)"
+        }
+      },
       "xunit": {
         "type": "Direct",
         "requested": "[2.4.2, )",

test/xunit.runner.json

@@ -0,0 +1,4 @@
+{
+    "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+    "diagnosticMessages": true
+}