Populate DatabaseSchema by reading it from database

DataGateway.Service.Tests/CosmosTests/MetadataStoreProviderForTest.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Threading.Tasks;
 using Azure.DataGateway.Service.Models;
 using Azure.DataGateway.Service.Services;
 
@@ -55,5 +56,10 @@ public FilterParser GetFilterParser()
         {
             return _filterParser;
         }
+
+        public Task<DatabaseSchema> RefreshDatabaseSchemaWithTablesAsync(string? schemaName)
+        {
+            throw new System.NotImplementedException();
+        }
     }
 }

DataGateway.Service.Tests/SqlTests/MsSqlMetadataProviderTests.cs

@@ -0,0 +1,15 @@
+using System.Threading.Tasks;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Azure.DataGateway.Service.Tests.SqlTests
+{
+    [TestClass, TestCategory(TestCategory.MSSQL)]
+    public class MsSqlMetadataProviderTests : SqlMetadataProviderTests
+    {
+        [ClassInitialize]
+        public static async Task InitializeTestFixture(TestContext context)
+        {
+            await InitializeTestFixture(context, TestCategory.MSSQL);
+        }
+    }
+}

DataGateway.Service.Tests/SqlTests/MySqlMetadataProviderTests.cs

@@ -0,0 +1,15 @@
+using System.Threading.Tasks;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Azure.DataGateway.Service.Tests.SqlTests
+{
+    [TestClass, TestCategory(TestCategory.MYSQL)]
+    public class MySqlMetadataProviderTests : SqlMetadataProviderTests
+    {
+        [ClassInitialize]
+        public static async Task InitializeTestFixture(TestContext context)
+        {
+            await InitializeTestFixture(context, TestCategory.MYSQL);
+        }
+    }
+}

DataGateway.Service.Tests/SqlTests/PostgreSqlMetadataProviderTests.cs

@@ -0,0 +1,15 @@
+using System.Threading.Tasks;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Azure.DataGateway.Service.Tests.SqlTests
+{
+    [TestClass, TestCategory(TestCategory.POSTGRESQL)]
+    public class PostgreSqlMetadataProviderTests : SqlMetadataProviderTests
+    {
+        [ClassInitialize]
+        public static async Task InitializeTestFixture(TestContext context)
+        {
+            await InitializeTestFixture(context, TestCategory.POSTGRESQL);
+        }
+    }
+}

DataGateway.Service.Tests/SqlTests/RestApiTestBase.cs

@@ -618,7 +618,7 @@ await SetupAndRunRestApiTest(
                 operationType: Operation.Insert,
                 requestBody: requestBody,
                 exception: true,
-                expectedErrorMessage: "Invalid request body. Missing field in body: publisher_id.",
+                expectedErrorMessage: "Invalid request body. Missing field in body: title.",
                 expectedStatusCode: HttpStatusCode.BadRequest
             );
         }

DataGateway.Service.Tests/SqlTests/SqlMetadataProviderTests.cs

@@ -0,0 +1,45 @@
+using System.Threading.Tasks;
+using Azure.DataGateway.Service.Models;
+using Azure.DataGateway.Service.Services;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Azure.DataGateway.Service.Tests.SqlTests
+{
+    [TestClass]
+    public abstract class SqlMetadataProviderTests : SqlTestBase
+    {
+        [TestMethod]
+        public async Task TestDerivedDatabaseSchemaIsValid()
+        {
+            ResolverConfig runtimeConfig = _metadataStoreProvider.GetResolvedConfig();
+            DatabaseSchema expectedSchema = runtimeConfig.DatabaseSchema;
+            DatabaseSchema derivedDatabaseSchema =
+                await _sqlMetadataProvider.RefreshDatabaseSchemaWithTablesAsync(_defaultSchemaName);
+
+            foreach ((string tableName, TableDefinition expectedTableDefinition) in expectedSchema.Tables)
+            {
+                TableDefinition actualTableDefinition;
+                Assert.IsTrue(derivedDatabaseSchema.Tables.TryGetValue(tableName, out actualTableDefinition),
+                    $"Could not find table definition for table '{tableName}'");
+
+                CollectionAssert.AreEqual(
+                    expectedTableDefinition.PrimaryKey,
+                    actualTableDefinition.PrimaryKey,
+                    $"Did not find the expected primary keys for table {tableName}");
+
+                foreach ((string columnName, ColumnDefinition expectedColumnDefinition) in expectedTableDefinition.Columns)
+                {
+                    ColumnDefinition actualColumnDefinition;
+                    Assert.IsTrue(actualTableDefinition.Columns.TryGetValue(columnName, out actualColumnDefinition),
+                        $"Could not find column definition for column '{columnName}' of table '{tableName}'");
+
+                    Assert.AreEqual(expectedColumnDefinition.IsAutoGenerated, actualColumnDefinition.IsAutoGenerated);
+                    Assert.AreEqual(expectedColumnDefinition.HasDefault, actualColumnDefinition.HasDefault,
+                        $"Expected HasDefault property of column '{columnName}' of table '{tableName}' does not match actual.");
+                    Assert.AreEqual(expectedColumnDefinition.IsNullable, actualColumnDefinition.IsNullable,
+                        $"Expected IsNullable property of column '{columnName}' of table '{tableName}' does not match actual.");
+                }
+            }
+        }
+    }
+}

DataGateway.Service.Tests/SqlTests/SqlTestBase.cs

@@ -40,6 +40,8 @@ public abstract class SqlTestBase
         protected static IMetadataStoreProvider _metadataStoreProvider;
         protected static Mock<IAuthorizationService> _authorizationService;
         protected static Mock<IHttpContextAccessor> _httpContextAccessor;
+        protected static IMetadataStoreProvider _sqlMetadataProvider;
+        protected static string _defaultSchemaName;
 
         /// <summary>
         /// Sets up test fixture for class, only to be run once per test run.
@@ -52,20 +54,30 @@ protected static async Task InitializeTestFixture(TestContext context, string te
             _testCategory = testCategory;
 
             IOptions<DataGatewayConfig> config = SqlTestHelper.LoadConfig($"{_testCategory}IntegrationTest");
+            string connectionString = config.Value.DatabaseConnection.ConnectionString;
 
             switch (_testCategory)
             {
                 case TestCategory.POSTGRESQL:
                     _queryExecutor = new QueryExecutor<NpgsqlConnection>(config);
                     _queryBuilder = new PostgresQueryBuilder();
+                    _sqlMetadataProvider =
+                        SqlMetadataProvider<NpgsqlConnection, NpgsqlDataAdapter, NpgsqlCommand>.GetSqlMetadataProvider(connectionString);
+                    _defaultSchemaName = "public";
                     break;
                 case TestCategory.MSSQL:
                     _queryExecutor = new QueryExecutor<SqlConnection>(config);
                     _queryBuilder = new MsSqlQueryBuilder();
+                    _sqlMetadataProvider =
+                      SqlMetadataProvider<SqlConnection, SqlDataAdapter, SqlCommand>.GetSqlMetadataProvider(connectionString);
+                    _defaultSchemaName = "dbo";
                     break;
                 case TestCategory.MYSQL:
                     _queryExecutor = new QueryExecutor<MySqlConnection>(config);
                     _queryBuilder = new MySqlQueryBuilder();
+                    _sqlMetadataProvider =
+                        SqlMetadataProvider<MySqlConnection, MySqlDataAdapter, MySqlCommand>.GetSqlMetadataProvider(connectionString);
+                    _defaultSchemaName = "mysql";
                     break;
             }
 
@@ -81,7 +93,10 @@ protected static async Task InitializeTestFixture(TestContext context, string te
             _httpContextAccessor = new Mock<IHttpContextAccessor>();
             _httpContextAccessor.Setup(x => x.HttpContext.User).Returns(new ClaimsPrincipal());
 
-            _metadataStoreProvider = new FileMetadataStoreProvider("sql-config.json");
+            _metadataStoreProvider = new FileMetadataStoreProvider(
+                "sql-config.json",
+                config.Value.DatabaseType,
+                config.Value.DatabaseConnection.ConnectionString);
             _queryEngine = new SqlQueryEngine(_metadataStoreProvider, _queryExecutor, _queryBuilder);
             _mutationEngine = new SqlMutationEngine(_queryEngine, _metadataStoreProvider, _queryExecutor, _queryBuilder);
 

DataGateway.Service/Configurations/SqlConfigValidatorExceptions.cs

@@ -130,7 +130,7 @@ private void ValidateNoMutationResolvers()
         /// </summary>
         private void ValidateDatabaseHasTables()
         {
-            if (_config.DatabaseSchema.Tables == null || _config.DatabaseSchema.Tables.Count == 0)
+            if (_config.DatabaseSchema!.Tables == null || _config.DatabaseSchema!.Tables.Count == 0)
             {
                 throw new ConfigValidationException(
                     "Database schema must have a non empty \"Tables\" element.",

DataGateway.Service/Configurations/SqlConfigValidatorUtil.cs

@@ -178,7 +178,7 @@ private static Dictionary<string, FieldDefinitionNode> GetObjTypeDefFields(Objec
         /// </summary>
         private Dictionary<string, TableDefinition> GetDatabaseTables()
         {
-            return _config.DatabaseSchema.Tables;
+            return _config.DatabaseSchema!.Tables;
         }
 
         /// <summary>
@@ -194,7 +194,7 @@ private Dictionary<string, GraphQLType> GetGraphQLTypes()
         /// </summary>
         private bool ExistsTableWithName(string tableName)
         {
-            return _config.DatabaseSchema.Tables.ContainsKey(tableName);
+            return _config.DatabaseSchema!.Tables.ContainsKey(tableName);
         }
 
         /// <summary>
@@ -211,7 +211,7 @@ private TableDefinition GetTableWithName(string tableName)
                 throw new ArgumentException("Invalid table name was provided.");
             }
 
-            return _config.DatabaseSchema.Tables[tableName];
+            return _config.DatabaseSchema!.Tables[tableName];
         }
 
         /// <summary>
@@ -540,7 +540,7 @@ private bool TableContainsForeignKey(string tableName, string foreignKeyName)
         /// <exception cref="KeyNotFoundException" />
         private ForeignKeyDefinition GetFkFromTable(string tableName, string fkName)
         {
-            return _config.DatabaseSchema.Tables[tableName].ForeignKeys[fkName];
+            return _config.DatabaseSchema!.Tables[tableName].ForeignKeys[fkName];
         }
 
         /// <summary>

DataGateway.Service/Models/DatabaseSchema.cs

@@ -9,7 +9,8 @@ namespace Azure.DataGateway.Service.Models
     /// </summary>
     public class DatabaseSchema
     {
-        public Dictionary<string, TableDefinition> Tables { get; set; } = new();
+        public Dictionary<string, TableDefinition> Tables { get; set; } =
+            new(StringComparer.InvariantCultureIgnoreCase);
     }
 
     public class TableDefinition
@@ -18,7 +19,8 @@ public class TableDefinition
         /// The list of columns that together form the primary key of the table.
         /// </summary>
         public List<string> PrimaryKey { get; set; } = new();
-        public Dictionary<string, ColumnDefinition> Columns { get; set; } = new();
+        public Dictionary<string, ColumnDefinition> Columns { get; set; } =
+            new(StringComparer.InvariantCultureIgnoreCase);
         public Dictionary<string, ForeignKeyDefinition> ForeignKeys { get; set; } = new();
         public Dictionary<string, AuthorizationRule> HttpVerbs { get; set; } = new();
     }

DataGateway.Service/MsSqlBooks.sql

@@ -30,7 +30,7 @@ CREATE TABLE authors(
 CREATE TABLE reviews(
     book_id bigint,
     id bigint IDENTITY(5001, 1),
-    content varchar(max),
+    content varchar(max) DEFAULT('Its a classic') NOT NULL,
     PRIMARY KEY(book_id, id)
 );
 
@@ -93,4 +93,4 @@ INSERT INTO book_author_link(book_id, author_id) VALUES (1, 123), (2, 124), (3,
 SET IDENTITY_INSERT reviews ON
 INSERT INTO reviews(id, book_id, content) VALUES (567, 1, 'Indeed a great book'), (568, 1, 'I loved it'), (569, 1, 'best book I read in years');
 SET IDENTITY_INSERT reviews OFF
-
+ 

DataGateway.Service/MySqlBooks.sql

@@ -27,7 +27,7 @@ CREATE TABLE authors(
 CREATE TABLE reviews(
     book_id bigint NOT NULL,
     id bigint AUTO_INCREMENT,
-    content text NOT NULL,
+    content text DEFAULT ('Its a classic') NOT NULL,
     PRIMARY KEY(book_id, id),
     INDEX (id)
 );

DataGateway.Service/PostgreSqlBooks.sql

@@ -4,6 +4,7 @@ DROP TABLE IF EXISTS authors;
 DROP TABLE IF EXISTS books;
 DROP TABLE IF EXISTS publishers;
 DROP TABLE IF EXISTS magazines;
+DROP TABLE IF EXISTS comics;
 
 CREATE TABLE publishers(
     id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
@@ -25,7 +26,7 @@ CREATE TABLE authors(
 CREATE TABLE reviews(
     book_id bigint,
     id bigint GENERATED BY DEFAULT AS IDENTITY,
-    content text,
+    content text DEFAULT 'Its a classic' NOT NULL,
     PRIMARY KEY(book_id, id)
 );
 
@@ -41,6 +42,12 @@ CREATE TABLE magazines(
     issueNumber bigint NULL
 );
 
+CREATE TABLE comics(
+    id bigint PRIMARY KEY,
+    title text NOT NULL,
+    volume bigint GENERATED BY DEFAULT AS IDENTITY
+);
+
 ALTER TABLE books
 ADD CONSTRAINT book_publisher_fk
 FOREIGN KEY (publisher_id)

DataGateway.Service/Resolvers/PostgresQueryBuilder.cs

@@ -86,7 +86,7 @@ public string Build(SqlUpsertQueryStructure structure)
         }
 
         /// <inheritdoc />
-        protected override string Build(KeysetPaginationPredicate? predicate)
+        protected override string Build(KeysetPaginationPredicate predicate)
         {
             if (predicate == null)
             {