sql-query-structure merged with main and resolve build issues

DataGateway.Service/Models/TypeMetadata.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+
+namespace Azure.DataGateway.Service.Models
+{
+    public class TypeMetadata
+    {
+        public string Table { get; set; }
+        public Dictionary<string, JoinMapping> JoinMappings { get; set; } = new();
+    }
+
+    public class JoinMapping
+    {
+        public string LeftColumn { get; set; }
+        public string RightColumn { get; set; }
+    }
+}

DataGateway.Service/Resolvers/CosmosQueryEngine.cs

@@ -1,5 +1,6 @@
 using Azure.DataGateway.Service.Models;
 using Azure.DataGateway.Service.Resolvers;
+using HotChocolate.Resolvers;
 using Microsoft.Azure.Cosmos;
 using Newtonsoft.Json.Linq;
 using System.Collections.Generic;
@@ -36,13 +37,14 @@ public void RegisterResolver(GraphQLQueryResolver resolver)
         // <summary>
         // ExecuteAsync the given named graphql query on the backend.
         // </summary>
-        public async Task<JsonDocument> ExecuteAsync(string graphQLQueryName, IDictionary<string, object> parameters)
+        public async Task<JsonDocument> ExecuteAsync(IMiddlewareContext context, IDictionary<string, object> parameters)
         {
             // TODO: fixme we have multiple rounds of serialization/deserialization JsomDocument/JObject
             // TODO: add support for nesting
             // TODO: add support for join query against another container
             // TODO: add support for TOP and Order-by push-down
 
+            string graphQLQueryName = context.Selection.Field.Name.Value;
             GraphQLQueryResolver resolver = this._metadataStoreProvider.GetQueryResolver(graphQLQueryName);
             Container container = this._clientProvider.Client.GetDatabase(resolver.DatabaseName).GetContainer(resolver.ContainerName);
             var querySpec = new QueryDefinition(resolver.ParametrizedQuery);
@@ -71,13 +73,14 @@ public async Task<JsonDocument> ExecuteAsync(string graphQLQueryName, IDictionar
             return jsonDocument;
         }
 
-        public async Task<IEnumerable<JsonDocument>> ExecuteListAsync(string graphQLQueryName, IDictionary<string, object> parameters)
+        public async Task<IEnumerable<JsonDocument>> ExecuteListAsync(IMiddlewareContext context, IDictionary<string, object> parameters)
         {
             // TODO: fixme we have multiple rounds of serialization/deserialization JsomDocument/JObject
             // TODO: add support for nesting
             // TODO: add support for join query against another container
             // TODO: add support for TOP and Order-by push-down
 
+            string graphQLQueryName = context.Selection.Field.Name.Value;
             GraphQLQueryResolver resolver = this._metadataStoreProvider.GetQueryResolver(graphQLQueryName);
             Container container = this._clientProvider.Client.GetDatabase(resolver.DatabaseName).GetContainer(resolver.ContainerName);
             var querySpec = new QueryDefinition(resolver.ParametrizedQuery);

DataGateway.Service/Resolvers/IQueryBuilder.cs

@@ -12,6 +12,8 @@ public interface IQueryBuilder
         // Modifies the inputQuery in such a way that it returns the results as
         // a JSON string.
         // </summary>
-        public string Build(string inputQuery, bool isList);
+        public string QuoteIdentifier(string ident);
+        public string WrapSubqueryColumn(string column, SqlQueryStructure subquery);
+        public string Build(SqlQueryStructure structure);
     }
 }

DataGateway.Service/Resolvers/IQueryEngine.cs

@@ -1,4 +1,5 @@
 using Azure.DataGateway.Service.Models;
+using HotChocolate.Resolvers;
 using System.Collections.Generic;
 using System.Text.Json;
 using System.Threading.Tasks;
@@ -18,11 +19,11 @@ public interface IQueryEngine
         // <summary>
         // Executes the given named graphql query on the backend and expecting a single Json back.
         // </summary>
-        public Task<JsonDocument> ExecuteAsync(string graphQLQueryName, IDictionary<string, object> parameters);
+        public Task<JsonDocument> ExecuteAsync(IMiddlewareContext context, IDictionary<string, object> parameters);
 
         // <summary>
         // Executes the given named graphql query on the backend and expecting a list of Jsons back.
         // </summary>
-        public Task<IEnumerable<JsonDocument>> ExecuteListAsync(string graphQLQueryName, IDictionary<string, object> parameters);
+        public Task<IEnumerable<JsonDocument>> ExecuteListAsync(IMiddlewareContext context, IDictionary<string, object> parameters);
     }
 }

DataGateway.Service/Resolvers/MsSqlQueryBuilder.cs

@@ -1,3 +1,7 @@
+using System;
+using System.Data.Common;
+using Microsoft.Data.SqlClient;
+using System.Linq;
 
 namespace Azure.DataGateway.Service.Resolvers
 {
@@ -6,19 +10,46 @@ namespace Azure.DataGateway.Service.Resolvers
     /// </summary>
     public class MsSqlQueryBuilder : IQueryBuilder
     {
-        private const string X_FOR_JSON_SUFFIX = " FOR JSON PATH, INCLUDE_NULL_VALUES";
-        private const string X_WITHOUT_ARRAY_WRAPPER_SUFFIX = "WITHOUT_ARRAY_WRAPPER";
+        private const string X_FORJSONSUFFIX = " FOR JSON PATH, INCLUDE_NULL_VALUES";
+        private const string X_WITHOUTARRAYWRAPPERSUFFIX = "WITHOUT_ARRAY_WRAPPER";
 
-        public string Build(string inputQuery, bool isList)
+        private static DbCommandBuilder _builder = new SqlCommandBuilder();
+        public string QuoteIdentifier(string ident)
         {
-            string queryText = inputQuery + X_FOR_JSON_SUFFIX;
-            if (!isList)
+            return _builder.QuoteIdentifier(ident);
+        }
+
+        public string WrapSubqueryColumn(string column, SqlQueryStructure subquery)
+        {
+            if (subquery.IsList())
             {
-                queryText += "," + X_WITHOUT_ARRAY_WRAPPER_SUFFIX;
+                return $"JSON_QUERY (COALESCE({column}, '[]'))";
             }
 
-            return queryText;
+            return $"JSON_QUERY ({column})";
         }
 
+        public string Build(SqlQueryStructure structure)
+        {
+            string selectedColumns = String.Join(", ", structure.Columns.Select(x => $"{x.Value} AS {QuoteIdentifier(x.Key)}"));
+            string fromPart = structure.Table(structure.TableName, structure.TableAlias);
+            fromPart += String.Join(
+                    "",
+                    structure.JoinQueries.Select(
+                        x => $" OUTER APPLY ({Build(x.Value)}) AS {QuoteIdentifier(x.Key)}({structure.DataIdent})"));
+            string query = $"SELECT {selectedColumns} FROM {fromPart}";
+            if (structure.Conditions.Count() > 0)
+            {
+                query += $" WHERE {String.Join(" AND ", structure.Conditions)}";
+            }
+
+            query += X_FORJSONSUFFIX;
+            if (!structure.IsList())
+            {
+                query += "," + X_WITHOUTARRAYWRAPPERSUFFIX;
+            }
+
+            return query;
+        }
     }
 }

DataGateway.Service/Resolvers/PostgresQueryBuilder.cs

@@ -1,3 +1,7 @@
+using System;
+using System.Linq;
+using System.Data.Common;
+using Npgsql;
 
 namespace Azure.DataGateway.Service.Resolvers
 {
@@ -6,15 +10,40 @@ namespace Azure.DataGateway.Service.Resolvers
     /// </summary>
     public class PostgresQueryBuilder : IQueryBuilder
     {
-        public string Build(string inputQuery, bool isList)
+        private static DbCommandBuilder Builder = new NpgsqlCommandBuilder();
+        public string QuoteIdentifier(string ident)
         {
-            if (!isList)
-            {
-                return $"SELECT row_to_json(q) FROM ({inputQuery}) q";
-            }
+            return Builder.QuoteIdentifier(ident);
+        }
 
-            return $"SELECT jsonb_agg(row_to_json(q)) FROM ({inputQuery}) q";
+        public string WrapSubqueryColumn(string column, SqlQueryStructure subquery)
+        {
+            return column;
         }
 
+        public string Build(SqlQueryStructure structure)
+        {
+            var selectedColumns = String.Join(", ", structure.Columns.Select(x => $"{x.Value} AS {QuoteIdentifier(x.Key)}"));
+            string fromPart = structure.Table(structure.TableName, structure.TableAlias);
+            fromPart += String.Join("", structure.JoinQueries.Select(x => $" LEFT OUTER JOIN LATERAL ({Build(x.Value)}) AS {QuoteIdentifier(x.Key)} ON TRUE"));
+            string query = $"SELECT {selectedColumns} FROM {fromPart}";
+            if (structure.Conditions.Count() > 0)
+            {
+                query += $" WHERE {String.Join(" AND ", structure.Conditions)}";
+            }
+            var subqName = QuoteIdentifier($"subq{structure.Counter.Next()}");
+            string start;
+            if (structure.IsList())
+            {
+                start = $"SELECT COALESCE(jsonb_agg(to_jsonb({subqName})), '[]') AS {structure.DataIdent} FROM (";
+            }
+            else
+            {
+                start = $"SELECT to_jsonb({subqName}) AS {structure.DataIdent} FROM (";
+            }
+            var end = $") AS {subqName}";
+            query = $"{start} {query} {end}";
+            return query;
+        }
     }
 }

DataGateway.Service/Resolvers/QueryExecutor.cs

@@ -46,7 +46,7 @@ public async Task<DbDataReader> ExecuteQueryAsync(string sqltext, IDictionary<st
                 }
             }
 
-            return await cmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess | CommandBehavior.CloseConnection);
+            return await cmd.ExecuteReaderAsync(CommandBehavior.CloseConnection);
         }
     }
 }

DataGateway.Service/Resolvers/SqlQueryEngine.cs

@@ -1,8 +1,10 @@
 using Azure.DataGateway.Service.Models;
 using Azure.DataGateway.Services;
+using HotChocolate.Resolvers;
 using System;
 using System.Collections.Generic;
 using System.Data.Common;
+using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
 
@@ -36,64 +38,69 @@ public void RegisterResolver(GraphQLQueryResolver resolver)
             // no-op
         }
 
+        private static async Task<string> GetJsonStringFromDbReader(DbDataReader dbDataReader)
+        {
+            var jsonString = new StringBuilder();
+            // Even though we only return a single cell, we need this loop for
+            // MS SQL. Sadly it splits FOR JSON PATH output across multiple
+            // cells if the JSON consists of more than 2033 bytes:
+            // Sources:
+            // 1. https://docs.microsoft.com/en-us/sql/relational-databases/json/format-query-results-as-json-with-for-json-sql-server?view=sql-server-2017#output-of-the-for-json-clause
+            // 2. https://stackoverflow.com/questions/54973536/for-json-path-results-in-ssms-truncated-to-2033-characters/54973676
+            // 3. https://docs.microsoft.com/en-us/sql/relational-databases/json/use-for-json-output-in-sql-server-and-in-client-apps-sql-server?view=sql-server-2017#use-for-json-output-in-a-c-client-app
+            if (await dbDataReader.ReadAsync())
+            {
+                jsonString.Append(dbDataReader.GetString(0));
+            }
+
+            return jsonString.ToString();
+        }
+
         // <summary>
         // ExecuteAsync the given named graphql query on the backend.
         // </summary>
-        public async Task<JsonDocument> ExecuteAsync(string graphQLQueryName, IDictionary<string, object> parameters)
+        public async Task<JsonDocument> ExecuteAsync(IMiddlewareContext context, IDictionary<string, object> parameters)
         {
             // TODO: add support for nesting
             // TODO: add support for join query against another table
             // TODO: add support for TOP and Order-by push-down
 
-            GraphQLQueryResolver resolver = _metadataStoreProvider.GetQueryResolver(graphQLQueryName);
-            var jsonDocument = JsonDocument.Parse("{ }");
-
-            string queryText = _queryBuilder.Build(resolver.ParametrizedQuery, false);
-
+            SqlQueryStructure structure = new(context, _metadataStoreProvider, _queryBuilder);
+            Console.WriteLine(structure.ToString());
             // Open connection and execute query using _queryExecutor
             //
-            DbDataReader dbDataReader = await _queryExecutor.ExecuteQueryAsync(queryText, parameters);
+            DbDataReader dbDataReader = await _queryExecutor.ExecuteQueryAsync(structure.ToString(), parameters);
 
             // Parse Results into Json and return
-            //
-            if (await dbDataReader.ReadAsync())
-            {
-                jsonDocument = JsonDocument.Parse(dbDataReader.GetString(0));
-            }
-            else
+            if (!dbDataReader.HasRows)
             {
-                Console.WriteLine("Did not return enough rows in the JSON result.");
+                return null;
             }
 
-            return jsonDocument;
+            return JsonDocument.Parse(await GetJsonStringFromDbReader(dbDataReader));
         }
 
         // <summary>
         // Executes the given named graphql query on the backend and expecting a list of Jsons back.
         // </summary>
-        public async Task<IEnumerable<JsonDocument>> ExecuteListAsync(string graphQLQueryName, IDictionary<string, object> parameters)
+        public async Task<IEnumerable<JsonDocument>> ExecuteListAsync(IMiddlewareContext context, IDictionary<string, object> parameters)
         {
             // TODO: add support for nesting
             // TODO: add support for join query against another container
             // TODO: add support for TOP and Order-by push-down
 
-            GraphQLQueryResolver resolver = _metadataStoreProvider.GetQueryResolver(graphQLQueryName);
-            var resultsAsList = new List<JsonDocument>();
-            string queryText = _queryBuilder.Build(resolver.ParametrizedQuery, true);
-            DbDataReader dbDataReader = await _queryExecutor.ExecuteQueryAsync(queryText, parameters);
+            SqlQueryStructure structure = new(context, _metadataStoreProvider, _queryBuilder);
+            Console.WriteLine(structure.ToString());
+            DbDataReader dbDataReader = await _queryExecutor.ExecuteQueryAsync(structure.ToString(), parameters);
 
-            // Deserialize results into list of JsonDocuments and return
+            // Parse Results into Json and return
             //
-            if (await dbDataReader.ReadAsync())
-            {
-                resultsAsList = JsonSerializer.Deserialize<List<JsonDocument>>(dbDataReader.GetString(0));
-            }
-            else
+            if (!dbDataReader.HasRows)
             {
-                Console.WriteLine("Did not return enough rows in the JSON result.");
+                return new List<JsonDocument>();
             }
 
-            return resultsAsList;
+            return JsonSerializer.Deserialize<List<JsonDocument>>(await GetJsonStringFromDbReader(dbDataReader));
         }
     }
 }