Add MySQL RESTAPI POST/PUT support and Tests (#260)

* Add upsert Support 

* Use a different way to get the updated rowid

* Add tests to validate insert returns right JSON

* Enable most REST API tests

Author: junsu0ms

DataGateway.Service.Tests/SqlTests/MySqlRestApiTests.cs

@@ -338,14 +338,36 @@ SELECT JSON_OBJECT('id', id) AS data
             {
                 "PutOne_Insert_Test",
                 @"
-                    SELECT JSON_OBJECT('id', id) AS data
+                    SELECT JSON_OBJECT('id', id, 'title', title, 'issueNumber', issueNumber ) AS data
                     FROM (
-                        SELECT id, title, publisher_id
-                        FROM " + _integrationTableName + @"
-                        WHERE id > 5000 AND title = 'The Hobbit Returns to The Shire'
-                        AND publisher_id = 1234
+                        SELECT id, title, issueNumber
+                        FROM " + _integration_NonAutoGenPK_TableName + @"
+                        WHERE id > 5000 AND title = 'Batman Returns'
+                            AND issueNumber = 1234
                     ) AS subq
                 "
+            },
+            {
+                "PutOne_Insert_Nullable_Test",
+                @"SELECT JSON_OBJECT('id', id, 'title', title, 'issueNumber', issueNumber ) AS data
+                    FROM (
+                        SELECT id, title, issueNumber
+                        FROM " + _integration_NonAutoGenPK_TableName + @"
+                        WHERE id = " + $"{STARTING_ID_FOR_TEST_INSERTS + 1}" + @" AND title = 'Times'
+                        AND issueNumber is NULL
+                    ) as subq
+                "
+            },
+            {
+                "PutOne_Insert_AutoGenNonPK_Test",
+                @"SELECT JSON_OBJECT('id', id, 'title', title, 'volume', volume ) AS data
+                    FROM (
+                        SELECT id, title, volume
+                        FROM " + _integration_AutoGenNonPK_TableName + @"
+                        WHERE id = " + $"{STARTING_ID_FOR_TEST_INSERTS}" + @" AND title = 'Star Trek'
+                        AND volume IS NOT NULL
+                    ) as subq
+                "
             }
         };
 
@@ -376,60 +398,11 @@ public override string GetQuery(string key)
             return _queryMap[key];
         }
 
-        [TestMethod]
-        [Ignore]
-        public override Task InsertOneTest()
-        {
-            throw new NotImplementedException();
-        }
-
-        [TestMethod]
-        [Ignore]
-        public override Task InsertOneInCompositeKeyTableTest()
-        {
-            throw new NotImplementedException();
-        }
-
-        [TestMethod]
-        [Ignore]
-        public override Task PutOne_Update_Test()
-        {
-            throw new NotImplementedException();
-        }
-
-        [TestMethod]
-        [Ignore]
-        public override Task PutOne_Insert_Test()
-        {
-            throw new NotImplementedException();
-        }
-
-        [TestMethod]
-        [Ignore]
-        public override Task PutOne_Insert_BadReq_Test()
-        {
-            throw new NotImplementedException();
-        }
-
-        [TestMethod]
-        [Ignore]
-        public override Task PutOne_Insert_BadReq_NonNullable_Test()
-        {
-            throw new NotImplementedException();
-        }
-
         [TestMethod]
         [Ignore]
         public override Task PutOne_Insert_PKAutoGen_Test()
         {
-            throw new NotImplementedException();
-        }
-
-        [TestMethod]
-        [Ignore]
-        public override Task PutOne_Insert_BadReq_AutoGen_NonNullable_Test()
-        {
-            throw new NotImplementedException();
+            throw new NotImplementedException("Insert success");
         }
     }
 }

DataGateway.Service.Tests/SqlTests/SqlTestBase.cs

@@ -224,8 +224,8 @@ protected static async Task SetupAndRunRestApiTest(
                 actionResult,
                 expected,
                 expectedStatusCode,
-                expectedLocationHeader);
-
+                expectedLocationHeader,
+                !exception);
         }
 
         /// <summary>

DataGateway.Service.Tests/SqlTests/SqlTestHelper.cs

@@ -128,7 +128,8 @@ public static void VerifyResult(
             IActionResult actionResult,
             string expected,
             HttpStatusCode expectedStatusCode,
-            string expectedLocationHeader)
+            string expectedLocationHeader,
+            bool isJson = false)
         {
             string actual;
             switch (actionResult)
@@ -158,9 +159,15 @@ public static void VerifyResult(
                     break;
             }
 
-            // if whitespaces are not consistent JsonStringDeepEquals should be used
-            // this will require deserializing and then serializing the strings for JSON
-            Assert.AreEqual(expected, actual);
+            Console.WriteLine($"Expected: {expected}\nActual: {actual}");
+            if (isJson && !string.IsNullOrEmpty(expected))
+            {
+                Assert.IsTrue(JsonStringsDeepEqual(expected, actual));
+            }
+            else
+            {
+                Assert.AreEqual(expected, actual);
+            }
         }
     }
 }

DataGateway.Service/MySqlBooks.sql

@@ -4,6 +4,7 @@ DROP TABLE IF EXISTS authors;
 DROP TABLE IF EXISTS books;
 DROP TABLE IF EXISTS publishers;
 DROP TABLE IF EXISTS magazines;
+DROP TABLE IF EXISTS comics;
 
 CREATE TABLE publishers(
     id bigint AUTO_INCREMENT PRIMARY KEY,
@@ -43,6 +44,12 @@ CREATE TABLE magazines(
     issueNumber bigint NULL
 );
 
+CREATE TABLE comics(
+    id bigint PRIMARY KEY,
+    title text NOT NULL,
+    volume bigint AUTO_INCREMENT UNIQUE KEY
+);
+
 ALTER TABLE books
 ADD CONSTRAINT book_publisher_fk
 FOREIGN KEY (publisher_id)
@@ -80,5 +87,5 @@ ALTER TABLE books AUTO_INCREMENT = 5001;
 ALTER TABLE publishers AUTO_INCREMENT = 5001;
 ALTER TABLE authors AUTO_INCREMENT = 5001;
 ALTER TABLE reviews AUTO_INCREMENT = 5001;
-
+ALTER TABLE comics AUTO_INCREMENT = 5001
 

DataGateway.Service/Resolvers/MySqlQueryBuilder.cs

@@ -60,22 +60,25 @@ public string Build(SqlQueryStructure structure)
         /// <inheritdoc />
         public string Build(SqlInsertStructure structure)
         {
-            // TODO: these should be put in a transcation
+            // No need to put into transaction as LAST_INSERT_ID is session level variable
             return $"INSERT INTO {QuoteIdentifier(structure.TableName)} ({Build(structure.InsertColumns)}) " +
                     $"VALUES ({string.Join(", ", (structure.Values))}); " +
-                    $"SELECT {MakeInsertSelections(structure)}";
+                    $" SET @ROWCOUNT=ROW_COUNT(); " +
+                    $"SELECT {MakeInsertSelections(structure)} WHERE @ROWCOUNT > 0;";
         }
 
         /// <inheritdoc />
         public string Build(SqlUpdateStructure structure)
         {
-            // TODO: these should be put in a transaction
-            return $"UPDATE {QuoteIdentifier(structure.TableName)} " +
+            (string sets, string updates, string select) = MakeStoreUpdatePK(structure.PrimaryKey());
+
+            return sets + ";\n" +
+                    $"UPDATE {QuoteIdentifier(structure.TableName)} " +
                     $"SET {Build(structure.UpdateOperations, ", ")} " +
-                    $"WHERE {Build(structure.Predicates)}; " +
-                    $"SELECT {Build(structure.PrimaryKey())} " +
-                    $"FROM {QuoteIdentifier(structure.TableName)} " +
-                    $"WHERE {Build(structure.Predicates)}; ";
+                        ", " + updates +
+                    $" WHERE {Build(structure.Predicates)}; " +
+                    $" SET @ROWCOUNT=ROW_COUNT(); " +
+                    $"SELECT " + select + $" WHERE @ROWCOUNT > 0;";
         }
 
         /// <inheritdoc />
@@ -88,8 +91,36 @@ public string Build(SqlDeleteStructure structure)
         /// <inheritdoc />
         public string Build(SqlUpsertQueryStructure structure)
         {
-            // TODO: these should be put in a transcation
-            throw new NotImplementedException();
+            (string sets, string updates, string select) = MakeStoreUpdatePK(structure.PrimaryKey());
+
+            string insert = $"INSERT INTO {QuoteIdentifier(structure.TableName)} ({Build(structure.InsertColumns)}) " +
+                    $"VALUES ({string.Join(", ", (structure.Values))}) ";
+
+            return sets + ";\n" +
+                    insert + " ON DUPLICATE KEY " +
+                    $"UPDATE {Build(structure.UpdateOperations, ", ")}" +
+                    $", " + updates + ";" +
+                    $" SET @ROWCOUNT=ROW_COUNT(); " +
+                    $"SELECT " + select + $" WHERE @ROWCOUNT != 1;" +
+                    $"SELECT {MakeUpsertSelections(structure)} WHERE @ROWCOUNT = 1;";
+        }
+
+        /// <summary>
+        /// Makes the query segments to store PK during an update
+        /// </summary>
+        private (string, string, string) MakeStoreUpdatePK(List<string> primaryKey)
+        {
+            // Create local variables to store the pk columns
+            string sets = String.Join(";\n", primaryKey.Select((x, index) => $"SET {"@LU_" + index.ToString()} := 0"));
+
+            // Fetch the value to local variables
+            string updates = String.Join(", ", primaryKey.Select((x, index) =>
+                $"{QuoteIdentifier(x)} = (SELECT {"@LU_" + index.ToString()} := {QuoteIdentifier(x)})"));
+
+            // Select local variables and mapping to original column name
+            string select = String.Join(", ", primaryKey.Select((x, index) => $"{"@LU_" + index.ToString()} AS {QuoteIdentifier(x)}"));
+
+            return (sets, updates, select);
         }
 
         /// <summary>
@@ -126,8 +157,11 @@ private string MakeInsertSelections(SqlInsertStructure structure)
         {
             List<string> selections = new();
 
+            List<string> fields = structure.PrimaryKey()
+                .Union(structure.InsertColumns).ToList();
+
             int index = 0;
-            foreach (string colName in structure.PrimaryKey())
+            foreach (string colName in fields)
             {
                 string quotedColName = QuoteIdentifier(colName);
                 if (structure.InsertColumns.Contains(colName))
@@ -137,11 +171,41 @@ private string MakeInsertSelections(SqlInsertStructure structure)
                 }
                 else if (structure.GetColumnDefinition(colName).IsAutoGenerated)
                 {
+                    //TODO: This assumes one column PK
                     selections.Add($"LAST_INSERT_ID() AS {quotedColName}");
                 }
             }
 
             return string.Join(", ", selections);
         }
+
+        private string MakeUpsertSelections(SqlUpsertQueryStructure structure)
+        {
+            List<string> selections = new();
+
+            List<string> fields = structure.AllColumns();
+
+            int index = 0;
+            foreach (string colName in fields)
+            {
+                string quotedColName = QuoteIdentifier(colName);
+
+                if (structure.InsertColumns.Contains(colName))
+                {
+                    selections.Add($"{structure.Values[index]} AS {quotedColName}");
+                    index++;
+                }
+                else if (structure.GetColumnDefinition(colName).IsAutoGenerated)
+                {
+                    selections.Add($"LAST_INSERT_ID() AS {quotedColName}");
+                }
+                else
+                {
+                    selections.Add($"NULL AS {quotedColName}");
+                }
+            }
+
+            return string.Join(", ", selections);
+        }
     }
 }

DataGateway.Service/Resolvers/Sql Query Structures/BaseSqlQueryStructure.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using Azure.DataGateway.Service.Models;
 using Azure.DataGateway.Services;
 using HotChocolate.Language;
@@ -94,6 +95,14 @@ public List<string> PrimaryKey()
             return GetTableDefinition().PrimaryKey;
         }
 
+        /// <summary>
+        /// get all columns of the table
+        /// </summary>
+        public List<string> AllColumns()
+        {
+            return GetTableDefinition().Columns.Select(col => col.Key).ToList();
+        }
+
         /// <summary>
         ///  Add parameter to Parameters and return the name associated with it
         /// </summary>

DataGateway.Service/Resolvers/Sql Query Structures/SqlUpsertQueryStructure.cs

@@ -80,6 +80,14 @@ public SqlUpsertQueryStructure(string tableName, IMetadataStoreProvider metadata
             }
         }
 
+        /// <summary>
+        /// Get the definition of a column by name
+        /// </summary>
+        public ColumnDefinition GetColumnDefinition(string columnName)
+        {
+            return GetTableDefinition().Columns[columnName];
+        }
+
         private void PopulateColumns(
             IDictionary<string, object> mutationParams,
             TableDefinition tableDefinition)

DataGateway.Service/Resolvers/SqlMutationEngine.cs

@@ -124,7 +124,7 @@ await PerformMutationOperation(
                 /// In MsSQL upsert:
                 /// result set #1: result of the UPDATE operation.
                 /// result set #2: result of the INSERT operation.
-                if (await dbDataReader.NextResultAsync())
+                if (await dbDataReader.NextResultAsync() && resultRecord == null)
                 {
                     // Since no first result set exists, we overwrite Dictionary here.
                     resultRecord = await ExtractRowFromDbDataReader(dbDataReader);

DataGateway.Service/appsettings.MySql.json

@@ -3,7 +3,7 @@
     "DatabaseType": "MySql",
     "ResolverConfigFile": "sql-config.json",
     "DatabaseConnection": {
-      "ConnectionString": "server=localhost;database=graphql"
+      "ConnectionString": "server=localhost;database=graphql;Allow User Variables=true;"
     }
   }
 }

DataGateway.Service/appsettings.MySqlIntegrationTest.overrides.example.json

@@ -1,7 +1,7 @@
 {
   "DataGatewayConfig": {
     "DatabaseConnection": {
-       "ConnectionString": "server=localhost;database=datagatewaytest;uid=root;pwd=REPLACEME"
+       "ConnectionString": "server=localhost;database=datagatewaytest;Allow User Variables=true;uid=root;pwd=REPLACEME"
     }
   }
 }