Database policy support for PUT/PATCH operations - MsSql (#1428)

## Why make this change?
As per the behaviour expected from PUT/PATCH operations with database
policies discussed in
#1430, implement db
policy support for MsSql to fix
#1370.

## What is this change?
1. Prior to this change, there was only one database policy for each
operation. Since now database policies will be supported for both insert
(or create)/update actions via PUT/PATCH operations, these 2 operations
can have 2 database policies defined for them, one for each action.
Hence, the `DbPolicyPredicates` (`string`) property in
`BaseSqlQueryStructure` class is replaced by
`DbPolicyPredicatesForOperations` (dictionary: `Config.Action`,
`string`).

2. The query generated by
`MsSqlQueryBuilder.Build(SqlUpsertQueryStructure structure)` is modified
such that only one (or none) of the insert/update operations execute,
depending on:
     - Existence of record in the table/view
     - Nature of PK (auto gen, non-auto gen)
Previously, we always used to perform the update operation irrespective
of whether the row existed or not. From now onwards, we would first
determine if the row actually exists or not. We will perform update only
if it exists.

3. The method `IQueryExecutor.GetMultipleResultSetsIfAnyAsync` has been
provided another implementation specific to MsSql in
`MsSqlQueryExecutor`. The DbDataReader instance for the query being
executed for the PUT/PATCH return operation would always contain atleast
1 result set and atmost 2 result sets.
- The first result set will give us the number of records(0/1)
corresponding to given PK
- The second result set will give us the rows affected by the operation
executed. It maybe the case that we were not able to execute either of
the update/insert operations. This happens when we are dealing with auto
gen PK table/view, and there is no record for given PK. So we cannot
perform either of the operations in this case.
- If the first result sets says we have zero records present
corresponding to given PK, we will attempt an insert operation(provided
we are not dealing with auto gen PK table/view for which insert is not
possible).
- If the result set says we have one record present corresponding to
given PK, we go on to perform the update.
- The database policies are not taken into consideration when
determining number of records for given PK in the first result set. They
are only taken into consideration for the actual update/insert query.

4. The property `IS_FIRST_RESULT_SET` in `SqlMutationEngine` is renamed
to `IS_UPDATE_RESULT_SET` to better denote its purpose.

5. Different scenarios are added to the method
`MsSqlQueryExecutor.GetMultipleResultSetsIfAnyAsync` to throw
appropriate exceptions (Forbidden/Authorization failure - 403 and
NotFound - 404). Appropriate comments are added within the code to
demonstrate each case.

## How was this tested?
1. Integration Tests - Done

Author: ayush3797

config-generators/mssql-commands.txt

@@ -167,6 +167,8 @@ update Comic --config "dab-config.MsSql.json" --permissions "TestNestedFilterOne
 update Comic --config "dab-config.MsSql.json" --permissions "TestNestedFilterOneMany_EntityReadForbidden:create,update,delete"
 update Stock --config "dab-config.MsSql.json" --permissions "TestNestedFilterFieldIsNull_ColumnForbidden:read"
 update Stock --config "dab-config.MsSql.json" --permissions "TestNestedFilterFieldIsNull_EntityReadForbidden:read"
+update Stock --config "dab-config.MsSql.json" --permissions "database_policy_tester:update" --policy-database "@item.pieceid ne 1"
+update Stock --config "dab-config.MsSql.json" --permissions "database_policy_tester:create" --policy-database "@item.pieceid ne 6 and @item.piecesAvailable gt 0"
 update series --config "dab-config.MsSql.json" --permissions "TestNestedFilterManyOne_ColumnForbidden:read" --fields.exclude "name"
 update series --config "dab-config.MsSql.json" --permissions "TestNestedFilterManyOne_EntityReadForbidden:create,update,delete"
 update series --config "dab-config.MsSql.json" --permissions "TestNestedFilterOneMany_ColumnForbidden:read"

config-generators/postgresql-commands.txt

@@ -50,6 +50,8 @@ update Publisher --config "dab-config.PostgreSql.json" --permissions "database_p
 update Publisher --config "dab-config.PostgreSql.json" --permissions "database_policy_tester:create"
 update Publisher --config "dab-config.PostgreSql.json" --permissions "database_policy_tester:update" --policy-database "@item.id ne 1234"
 update Stock --config "dab-config.PostgreSql.json" --permissions "authenticated:create,read,update,delete" --rest commodities --graphql true --relationship stocks_price --target.entity stocks_price --cardinality one
+update Stock --config "dab-config.PostgreSql.json" --permissions "database_policy_tester:create"
+update Stock --config "dab-config.PostgreSql.json" --permissions "database_policy_tester:update" --policy-database "@item.pieceid ne 1"
 update Book --config "dab-config.PostgreSql.json" --permissions "authenticated:create,read,update,delete"
 update Book --config "dab-config.PostgreSql.json" --relationship publishers --target.entity Publisher --cardinality one
 update Book --config "dab-config.PostgreSql.json" --relationship websiteplacement --target.entity BookWebsitePlacement --cardinality one

src/Config/DataApiBuilderException.cs

@@ -16,6 +16,7 @@ public class DataApiBuilderException : Exception
         public const string CONNECTION_STRING_ERROR_MESSAGE = "A valid Connection String should be provided.";
         public const string GRAPHQL_FILTER_ENTITY_AUTHZ_FAILURE = "Access forbidden to the target entity described in the filter.";
         public const string GRAPHQL_FILTER_FIELD_AUTHZ_FAILURE = "Access forbidden to a field referenced in the filter.";
+        public const string AUTHORIZATION_FAILURE = "Authorization Failure: Access Not Allowed.";
 
         public enum SubStatusCodes
         {
@@ -37,6 +38,10 @@ public enum SubStatusCodes
             /// </summary>
             AuthorizationCheckFailed,
             /// <summary>
+            /// Request did not satisfy database policy for the operation.
+            /// </summary>
+            DatabasePolicyFailure,
+            /// <summary>
             /// The requested operation failed on the database.
             /// </summary>
             DatabaseOperationFailed,

src/Service.Tests/SqlTests/GraphQLMutationTests/GraphQLMutationTestBase.cs

@@ -64,7 +64,7 @@ public async Task InsertMutationFailingDatabasePolicy(string dbQuery, string err
             SqlTestHelper.TestForErrorInGraphQLResponse(
                 result.ToString(),
                 message: errorMessage,
-                statusCode: $"{DataApiBuilderException.SubStatusCodes.AuthorizationCheckFailed}"
+                statusCode: $"{DataApiBuilderException.SubStatusCodes.DatabasePolicyFailure}"
             );
 
             string dbResponse = await GetDatabaseResultAsync(dbQuery);
@@ -873,7 +873,7 @@ public virtual async Task TestDbPolicyForCreateOperationReferencingFieldAbsentIn
             SqlTestHelper.TestForErrorInGraphQLResponse(
                 actual.ToString(),
                 message: "One or more fields referenced by the database policy are not present in the request body.",
-                statusCode: $"{DataApiBuilderException.SubStatusCodes.BadRequest}");
+                statusCode: $"{DataApiBuilderException.SubStatusCodes.AuthorizationCheckFailed}");
         }
 
         #endregion

src/Service.Tests/SqlTests/RestApiTests/Insert/InsertApiTestBase.cs

@@ -664,7 +664,7 @@ await SetupAndRunRestApiTest(
                 requestBody: requestBody,
                 exceptionExpected: true,
                 expectedStatusCode: HttpStatusCode.Forbidden,
-                expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.AuthorizationCheckFailed.ToString(),
+                expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.DatabasePolicyFailure.ToString(),
                 expectedErrorMessage: "Could not insert row with given values.",
                 clientRoleHeader: "database_policy_tester"
             );
@@ -693,8 +693,8 @@ await SetupAndRunRestApiTest(
                 requestBody: requestBody,
                 clientRoleHeader: "database_policy_tester",
                 expectedErrorMessage: "One or more fields referenced by the database policy are not present in the request body.",
-                expectedStatusCode: HttpStatusCode.BadRequest,
-                expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest.ToString()
+                expectedStatusCode: HttpStatusCode.Forbidden,
+                expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.AuthorizationCheckFailed.ToString()
                 );
         }
 

src/Service.Tests/SqlTests/RestApiTests/Patch/MsSqlPatchApiTests.cs

@@ -90,6 +90,22 @@ public class MsSqlPatchApiTests : PatchApiTestBase
                 $"WHERE id = 1 AND title = 'The Hobbit Returns to The Shire' " +
                 $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER"
             },
+            {
+                "PatchOneUpdateWithDatabasePolicy",
+                $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," +
+                $"[piecesRequired] FROM { _Composite_NonAutoGenPK_TableName } " +
+                $"WHERE [categoryid] = 100 AND [pieceid] = 99 AND [categoryName] = 'Historical' " +
+                $"AND [piecesAvailable]= 4 AND [piecesRequired] = 0 AND [pieceid] != 1 " +
+                $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER"
+            },
+            {
+                "PatchOneInsertWithDatabasePolicy",
+                $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," +
+                $"[piecesRequired] FROM { _Composite_NonAutoGenPK_TableName } " +
+                $"WHERE [categoryid] = 0 AND [pieceid] = 7 AND [categoryName] = 'SciFi' " +
+                $"AND [piecesAvailable]= 4 AND [piecesRequired] = 0 AND ([pieceid] != 6 AND [piecesAvailable] > 0) " +
+                $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER"
+            },
             {
                 "PatchOne_Update_Default_Test",
                 $"SELECT [id], [book_id], [content] FROM { _tableWithCompositePrimaryKey } " +

src/Service.Tests/SqlTests/RestApiTests/Patch/MySqlPatchApiTests.cs

@@ -213,7 +213,28 @@ public void PatchOneViewBadRequestTest()
 
         [TestMethod]
         [Ignore]
-        public override Task PatchOneUpdateInAccessibleRowWithDatabasePolicy()
+        public override Task PatchOneUpdateWithUnsatisfiedDatabasePolicy()
+        {
+            throw new NotImplementedException();
+        }
+
+        [TestMethod]
+        [Ignore]
+        public override Task PatchOneInsertWithUnsatisfiedDatabasePolicy()
+        {
+            throw new NotImplementedException();
+        }
+
+        [TestMethod]
+        [Ignore]
+        public override Task PatchOneUpdateWithDatabasePolicy()
+        {
+            throw new NotImplementedException();
+        }
+
+        [TestMethod]
+        [Ignore]
+        public override Task PatchOneInsertWithDatabasePolicy()
         {
             throw new NotImplementedException();
         }

src/Service.Tests/SqlTests/RestApiTests/Patch/PatchApiTestBase.cs

@@ -334,7 +334,7 @@ await SetupAndRunRestApiTest(
                 );
         }
         /// <summary>
-        /// Tests the PatchOne functionality with a REST PUT request using
+        /// Tests the PatchOne functionality with a REST PATCH request using
         /// headers that include as a key "If-Match" with an item that does exist,
         /// resulting in an update occuring. Verify update with Find.
         /// </summary>
@@ -361,6 +361,58 @@ await SetupAndRunRestApiTest(
                 );
         }
 
+        /// <summary>
+        /// Test to validate successful execution of PATCH operation which satisfies the database policy for the update operation it resolves into.
+        /// </summary>
+        [TestMethod]
+        public virtual async Task PatchOneUpdateWithDatabasePolicy()
+        {
+            // PATCH operation resolves to update because we have a record present for given PK.
+            // Since the database policy for update operation ("@item.pieceid ne 1") is satisfied by the operation, it executes successfully.
+            string requestBody = @"
+            {
+                ""piecesAvailable"": 4
+            }";
+
+            await SetupAndRunRestApiTest(
+                    primaryKeyRoute: "categoryid/100/pieceid/99",
+                    queryString: null,
+                    entityNameOrPath: _Composite_NonAutoGenPK_EntityPath,
+                    sqlQuery: GetQuery("PatchOneUpdateWithDatabasePolicy"),
+                    operationType: Config.Operation.UpsertIncremental,
+                    requestBody: requestBody,
+                    expectedStatusCode: HttpStatusCode.OK,
+                    clientRoleHeader: "database_policy_tester"
+                );
+        }
+
+        /// <summary>
+        /// Test to validate successful execution of PATCH operation which satisfies the database policy for the insert operation it resolves into.
+        /// </summary>
+        [TestMethod]
+        public virtual async Task PatchOneInsertWithDatabasePolicy()
+        {
+            // PATCH operation resolves to insert because we don't have a record present for given PK.
+            // Since the database policy for insert operation ("@item.pieceid ne 6 and @item.piecesAvailable gt 0") is satisfied by the operation, it executes successfully.
+            string requestBody = @"
+            {
+                ""piecesAvailable"": 4,
+                ""categoryName"": ""SciFi""
+            }";
+
+            await SetupAndRunRestApiTest(
+                    primaryKeyRoute: "categoryid/0/pieceid/7",
+                    queryString: null,
+                    entityNameOrPath: _Composite_NonAutoGenPK_EntityPath,
+                    sqlQuery: GetQuery("PatchOneInsertWithDatabasePolicy"),
+                    operationType: Config.Operation.UpsertIncremental,
+                    requestBody: requestBody,
+                    expectedStatusCode: HttpStatusCode.Created,
+                    clientRoleHeader: "database_policy_tester",
+                    expectedLocationHeader: "categoryid/0/pieceid/7"
+                );
+        }
+
         #endregion
 
         #region Negative Tests
@@ -424,7 +476,7 @@ await SetupAndRunRestApiTest(
         }
 
         /// <summary>
-        /// Tests the PatchOne functionality with a REST PUT request using
+        /// Tests the PatchOne functionality with a REST PATCH request using
         /// headers that include as a key "If-Match" with an item that does not exist,
         /// resulting in a DataApiBuilderException with status code of Precondition Failed.
         /// </summary>
@@ -589,53 +641,66 @@ await SetupAndRunRestApiTest(
         }
 
         /// <summary>
-        /// Test to validate that PATCH operation fails because the database policy("@item.id ne 1234")
-        /// restricts modifying records where id is not 1234.
+        /// Test to validate failure of PATCH operation failing to satisfy the database policy for the update operation.
+        /// (because a record exists for given PK).
         /// </summary>
         [TestMethod]
-        public virtual async Task PatchOneUpdateInAccessibleRowWithDatabasePolicy()
+        public virtual async Task PatchOneUpdateWithUnsatisfiedDatabasePolicy()
         {
-            // Perform PATCH update with upsert incrmental semantics.
+            // PATCH operation resolves to update because we have a record present for given PK.
+            // However, the update fails to execute successfully because the database policy ("@item.pieceid ne 1") for update operation is not satisfied.
             string requestBody = @"
             {
-                ""name"": ""New Publisher""
+                ""categoryName"": ""SciFi"",
+                ""piecesRequired"": 5,
+                ""piecesAvailable"": 2
             }";
 
             await SetupAndRunRestApiTest(
-                    primaryKeyRoute: "id/1234",
+                    primaryKeyRoute: "categoryid/0/pieceid/1",
                     queryString: null,
-                    entityNameOrPath: _foreignKeyEntityName,
-                    sqlQuery: string.Empty,
-                    operationType: Config.Operation.UpsertIncremental,
+                    entityNameOrPath: _Composite_NonAutoGenPK_EntityPath,
+                    operationType: Config.Operation.Upsert,
                     requestBody: requestBody,
+                    sqlQuery: string.Empty,
                     exceptionExpected: true,
-                    expectedErrorMessage: $"Cannot perform INSERT and could not find {_foreignKeyEntityName} with primary key <id: 1234> to perform UPDATE on.",
-                    expectedStatusCode: HttpStatusCode.NotFound,
-                    expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.EntityNotFound.ToString(),
+                    expectedErrorMessage: DataApiBuilderException.AUTHORIZATION_FAILURE,
+                    expectedStatusCode: HttpStatusCode.Forbidden,
+                    expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.DatabasePolicyFailure.ToString(),
                     clientRoleHeader: "database_policy_tester"
                     );
+        }
 
-            // Perform PATCH update with update semantics.
-            Dictionary<string, StringValues> headerDictionary = new()
+        /// <summary>
+        /// Test to validate failure of PATCH operation failing to satisfy the database policy for the update operation.
+        /// (because no record exists for given PK).
+        /// </summary>
+        [TestMethod]
+        public virtual async Task PatchOneInsertWithUnsatisfiedDatabasePolicy()
+        {
+            // PATCH operation resolves to insert because we don't have a record present for given PK.
+            // However, the insert fails to execute successfully because the database policy ("@item.pieceid ne 6 and @item.piecesAvailable gt 6")
+            // for insert operation is not satisfied.
+            string requestBody = @"
             {
-                { "If-Match", "*" }
-            };
+                ""categoryName"": ""SciFi"",
+                ""piecesRequired"": 5,
+                ""piecesAvailable"": 2
+            }";
 
             await SetupAndRunRestApiTest(
-                    primaryKeyRoute: "id/1234",
+                    primaryKeyRoute: "categoryid/0/pieceid/6",
                     queryString: null,
-                    entityNameOrPath: _foreignKeyEntityName,
-                    sqlQuery: string.Empty,
-                    operationType: Config.Operation.UpsertIncremental,
+                    entityNameOrPath: _Composite_NonAutoGenPK_EntityPath,
+                    operationType: Config.Operation.Upsert,
                     requestBody: requestBody,
-                    headers: new HeaderDictionary(headerDictionary),
+                    sqlQuery: string.Empty,
                     exceptionExpected: true,
-                    expectedErrorMessage: $"No Update could be performed, record not found",
-                    expectedStatusCode: HttpStatusCode.PreconditionFailed,
-                    expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.DatabaseOperationFailed.ToString(),
+                    expectedErrorMessage: DataApiBuilderException.AUTHORIZATION_FAILURE,
+                    expectedStatusCode: HttpStatusCode.Forbidden,
+                    expectedSubStatusCode: DataApiBuilderException.SubStatusCodes.DatabasePolicyFailure.ToString(),
                     clientRoleHeader: "database_policy_tester"
                     );
-
         }
 
         /// <summary>

src/Service.Tests/SqlTests/RestApiTests/Patch/PostgreSqlPatchApiTests.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -117,6 +118,18 @@ SELECT to_jsonb(subq) AS data
                     ) AS subq
                 "
             },
+            {
+                "PatchOneUpdateWithDatabasePolicy",
+                @"
+                    SELECT to_jsonb(subq) AS data
+                    FROM (
+                        SELECT categoryid, pieceid, ""categoryName"", ""piecesAvailable"", ""piecesRequired""
+                        FROM " + _Composite_NonAutoGenPK_TableName + @"
+                        WHERE categoryid = 100 AND pieceid = 99 AND ""categoryName"" = 'Historical'
+                            AND ""piecesAvailable"" = 4 AND ""piecesRequired"" = 0 AND pieceid != 1
+                    ) AS subq
+                "
+            },
             {
                 "PatchOne_Update_Default_Test",
                 @"
@@ -207,6 +220,26 @@ await base.PatchOneViewBadRequestTest(
 
         #region overridden tests
 
+        [TestMethod]
+        [Ignore]
+        public override Task PatchOneUpdateWithUnsatisfiedDatabasePolicy()
+        {
+            throw new NotImplementedException();
+        }
+
+        [TestMethod]
+        [Ignore]
+        public override Task PatchOneInsertWithUnsatisfiedDatabasePolicy()
+        {
+            throw new NotImplementedException();
+        }
+
+        [TestMethod]
+        [Ignore]
+        public override Task PatchOneInsertWithDatabasePolicy()
+        {
+            throw new NotImplementedException();
+        }
         #endregion
 
         #region Test Fixture Setup

src/Service.Tests/SqlTests/RestApiTests/Put/MsSqlPutApiTests.cs

@@ -30,9 +30,19 @@ public class MsSqlPutApiTests : PutApiTestBase
                 $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER"
             },
             {
-                "PutOneUpdateAccessibleRowWithDatabasePolicy",
-                $"SELECT * FROM { _foreignKeyTableName } " +
-                $"WHERE id = 2345 AND name = 'New Publisher' AND (id != 1234) " +
+                "PutOneUpdateWithDatabasePolicy",
+                $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," +
+                $"[piecesRequired] FROM { _Composite_NonAutoGenPK_TableName } " +
+                $"WHERE [categoryid] = 100 AND [pieceid] = 99 AND [categoryName] = 'SciFi' " +
+                $"AND [piecesAvailable]= 4 AND [piecesRequired] = 5 AND [pieceid] != 1 " +
+                $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER"
+            },
+            {
+                "PutOneInsertWithDatabasePolicy",
+                $"SELECT [categoryid], [pieceid], [categoryName],[piecesAvailable]," +
+                $"[piecesRequired] FROM { _Composite_NonAutoGenPK_TableName } " +
+                $"WHERE [categoryid] = 0 AND [pieceid] = 7 AND [categoryName] = 'SciFi' " +
+                $"AND [piecesAvailable]= 4 AND [piecesRequired] = 0 AND ([pieceid] != 6 AND [piecesAvailable] > 0) " +
                 $"FOR JSON PATH, INCLUDE_NULL_VALUES, WITHOUT_ARRAY_WRAPPER"
             },
             {