Allow Filtering by postman collection

src/Microsoft.OpenApi.Hidi/OpenApiService.cs

@@ -2,11 +2,13 @@
 // Licensed under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Text;
+using System.Text.Json;
 using Microsoft.OpenApi.Extensions;
 using Microsoft.OpenApi.Models;
 using Microsoft.OpenApi.Readers;
@@ -16,7 +18,7 @@
 
 namespace Microsoft.OpenApi.Hidi
 {
-    static class OpenApiService
+    public static class OpenApiService
     {
         public static void ProcessOpenApiDocument(
             string input,
@@ -25,6 +27,7 @@ public static void ProcessOpenApiDocument(
             OpenApiFormat format,
             string filterByOperationIds,
             string filterByTags,
+            string filterByCollection,
             bool inline,
             bool resolveExternal)
         {
@@ -49,23 +52,30 @@ public static void ProcessOpenApiDocument(
             }
             ).ReadAsync(stream).GetAwaiter().GetResult();
 
-            OpenApiDocument document;
-            document = result.OpenApiDocument;
+            var document = result.OpenApiDocument;
+            Func<string, OperationType?, OpenApiOperation, bool> predicate;
 
             // Check if filter options are provided, then execute
             if (!string.IsNullOrEmpty(filterByOperationIds) && !string.IsNullOrEmpty(filterByTags))
             {
                 throw new InvalidOperationException("Cannot filter by operationIds and tags at the same time.");
             }
-
             if (!string.IsNullOrEmpty(filterByOperationIds))
             {
-                var predicate = OpenApiFilterService.CreatePredicate(operationIds: filterByOperationIds);
+                predicate = OpenApiFilterService.CreatePredicate(operationIds: filterByOperationIds);
                 document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
             }
             if (!string.IsNullOrEmpty(filterByTags))
             {
-                var predicate = OpenApiFilterService.CreatePredicate(tags: filterByTags);
+                predicate = OpenApiFilterService.CreatePredicate(tags: filterByTags);
+                document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
+            }
+
+            if (!string.IsNullOrEmpty(filterByCollection))
+            {
+                var fileStream = GetStream(filterByCollection);
+                var requestUrls = ParseJsonCollectionFile(fileStream);
+                predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source:document);
                 document = OpenApiFilterService.CreateFilteredDocument(document, predicate);
             }
 
@@ -125,6 +135,38 @@ private static Stream GetStream(string input)
             return stream;
         }
 
+        /// <summary>
+        /// Takes in a file stream, parses the stream into a JsonDocument and gets a list of paths and Http methods
+        /// </summary>
+        /// <param name="stream"> A file stream.</param>
+        /// <returns> A dictionary of request urls and http methods from a collection.</returns>
+        public static Dictionary<string, List<string>> ParseJsonCollectionFile(Stream stream)
+        {
+            var requestUrls = new Dictionary<string, List<string>>();
+
+            // Convert file to JsonDocument
+            using var document = JsonDocument.Parse(stream);
+            var root = document.RootElement;
+            var itemElement = root.GetProperty("item");
+            foreach (var requestObject in itemElement.EnumerateArray().Select(item => item.GetProperty("request")))
+            {
+                // Fetch list of methods and urls from collection, store them in a dictionary
+                var path = requestObject.GetProperty("url").GetProperty("raw").ToString();
+                var method = requestObject.GetProperty("method").ToString();
+
+                if (!requestUrls.ContainsKey(path))
+                {
+                    requestUrls.Add(path, new List<string> { method });
+                }
+                else
+                {
+                    requestUrls[path].Add(method);
+                }
+            }
+
+            return requestUrls;
+        }
+
         internal static void ValidateOpenApiDocument(string input)
         {
             if (input == null)

src/Microsoft.OpenApi.Hidi/Program.cs

@@ -30,9 +30,10 @@ static async Task<int> Main(string[] args)
                 new Option("--inline", "Inline $ref instances", typeof(bool) ),
                 new Option("--resolveExternal","Resolve external $refs", typeof(bool)),
                 new Option("--filterByOperationIds", "Filters OpenApiDocument by OperationId(s) provided", typeof(string)),
-                new Option("--filterByTags", "Filters OpenApiDocument by Tag(s) provided", typeof(string))
+                new Option("--filterByTags", "Filters OpenApiDocument by Tag(s) provided", typeof(string)),
+                new Option("--filterByCollection", "Filters OpenApiDocument by Postman collection provided", typeof(string))
             };
-            transformCommand.Handler = CommandHandler.Create<string, FileInfo, OpenApiSpecVersion, OpenApiFormat, string, string, bool, bool>(
+            transformCommand.Handler = CommandHandler.Create<string, FileInfo, OpenApiSpecVersion, OpenApiFormat, string, string, string, bool, bool>(
                 OpenApiService.ProcessOpenApiDocument);
 
             rootCommand.Add(transformCommand);

src/Microsoft.OpenApi/Microsoft.OpenApi.csproj

@@ -38,6 +38,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
+        <PackageReference Include="System.Text.Json" Version="6.0.0" />
     </ItemGroup>
 
     <ItemGroup>

src/Microsoft.OpenApi/Services/OpenApiFilterService.cs

@@ -19,10 +19,18 @@ public static class OpenApiFilterService
         /// </summary>
         /// <param name="operationIds">Comma delimited list of operationIds or * for all operations.</param>
         /// <param name="tags">Comma delimited list of tags or a single regex.</param>
+        /// <param name="requestUrls">A dictionary of requests from a postman collection.</param>
+        /// <param name="source">The input OpenAPI document.</param>
         /// <returns>A predicate.</returns>
-        public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds = null, string tags = null)
+        public static Func<string, OperationType?, OpenApiOperation, bool> CreatePredicate(string operationIds = null,
+            string tags = null, Dictionary<string, List<string>> requestUrls = null, OpenApiDocument source = null)
         {
-            Func<OpenApiOperation, bool> predicate;
+            Func<string, OperationType?, OpenApiOperation, bool> predicate;
+
+            if (requestUrls != null && (operationIds != null || tags != null))
+            {
+                throw new InvalidOperationException("Cannot filter by Postman collection and either operationIds and tags at the same time.");
+            }
             if (!string.IsNullOrEmpty(operationIds) && !string.IsNullOrEmpty(tags))
             {
                 throw new InvalidOperationException("Cannot specify both operationIds and tags at the same time.");
@@ -31,12 +39,12 @@ public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds =
             {
                 if (operationIds == "*")
                 {
-                    predicate = (o) => true;  // All operations
+                    predicate = (url, operationType, operation) => true;  // All operations
                 }
                 else
                 {
                     var operationIdsArray = operationIds.Split(',');
-                    predicate = (o) => operationIdsArray.Contains(o.OperationId);
+                    predicate = (url, operationType, operation) => operationIdsArray.Contains(operation.OperationId);
                 }
             }
             else if (tags != null)
@@ -46,16 +54,59 @@ public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds =
                 {
                     var regex = new Regex(tagsArray[0]);
 
-                    predicate = (o) => o.Tags.Any(t => regex.IsMatch(t.Name));
+                    predicate = (url, operationType, operation) => operation.Tags.Any(tag => regex.IsMatch(tag.Name));
                 }
                 else
                 {
-                    predicate = (o) => o.Tags.Any(t => tagsArray.Contains(t.Name));
+                    predicate = (url, operationType, operation) => operation.Tags.Any(tag => tagsArray.Contains(tag.Name));
+                }
+            }
+            else if (requestUrls != null)
+            {
+                var operationTypes = new List<string>();
+
+                if (source != null)
+                {
+                    var apiVersion = source.Info.Version;
+
+                    var sources = new Dictionary<string, OpenApiDocument> {{ apiVersion, source}};
+                    var rootNode = CreateOpenApiUrlTreeNode(sources);
+
+                    // Iterate through urls dictionary and fetch operations for each url
+                    foreach (var path in requestUrls)
+                    {
+                        var serverList = source.Servers;
+                        var url = FormatUrlString(path.Key, serverList);
+
+                        var openApiOperations = GetOpenApiOperations(rootNode, url, apiVersion);
+                        if (openApiOperations == null)
+                        {
+                            continue;
+                        }
+
+                        // Add the available ops if they are in the postman collection. See path.Value
+                        foreach (var ops in openApiOperations)
+                        {
+                            if (path.Value.Contains(ops.Key.ToString().ToUpper()))
+                            {
+                                operationTypes.Add(ops.Key + url);
+                            }
+                        }
+                    }
+                }
+
+                if (!operationTypes.Any())
+                {
+                    throw new ArgumentException("The urls in the Postman collection supplied could not be found.");
                 }
+
+                // predicate for matching url and operationTypes
+                predicate = (path, operationType, operation) => operationTypes.Contains(operationType + path);
             }
+
             else
             {
-                throw new InvalidOperationException("Either operationId(s) or tag(s) need to be specified.");
+                throw new InvalidOperationException("Either operationId(s),tag(s) or Postman collection need to be specified.");
             }
 
             return predicate;
@@ -67,12 +118,12 @@ public static Func<OpenApiOperation, bool> CreatePredicate(string operationIds =
         /// <param name="source">The target <see cref="OpenApiDocument"/>.</param>
         /// <param name="predicate">A predicate function.</param>
         /// <returns>A partial OpenAPI document.</returns>
-        public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Func<OpenApiOperation, bool> predicate)
+        public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Func<string, OperationType?, OpenApiOperation, bool> predicate)
         {
             // Fetch and copy title, graphVersion and server info from OpenApiDoc
             var subset = new OpenApiDocument
             {
-                Info = new OpenApiInfo()
+                Info = new OpenApiInfo
                 {
                     Title = source.Info.Title + " - Subset",
                     Description = source.Info.Description,
@@ -83,13 +134,11 @@ public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Fun
                     Extensions = source.Info.Extensions
                 },
 
-                Components = new OpenApiComponents()
+                Components = new OpenApiComponents {SecuritySchemes = source.Components.SecuritySchemes},
+                SecurityRequirements = source.SecurityRequirements,
+                Servers = source.Servers
             };
 
-            subset.Components.SecuritySchemes = source.Components.SecuritySchemes;
-            subset.SecurityRequirements = source.SecurityRequirements;
-            subset.Servers = source.Servers;
-
             var results = FindOperations(source, predicate);
             foreach (var result in results)
             {
@@ -111,7 +160,10 @@ public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Fun
                     }
                 }
 
-                pathItem.Operations.Add((OperationType)result.CurrentKeys.Operation, result.Operation);
+                if (result.CurrentKeys.Operation != null)
+                {
+                    pathItem.Operations.Add((OperationType)result.CurrentKeys.Operation, result.Operation);
+                }
             }
 
             if (subset.Paths == null)
@@ -124,11 +176,103 @@ public static OpenApiDocument CreateFilteredDocument(OpenApiDocument source, Fun
             return subset;
         }
 
-        private static IList<SearchResult> FindOperations(OpenApiDocument graphOpenApi, Func<OpenApiOperation, bool> predicate)
+        /// <summary>
+        /// Creates an <see cref="OpenApiUrlTreeNode"/> from a collection of <see cref="OpenApiDocument"/>.
+        /// </summary>
+        /// <param name="sources">Dictionary of labels and their corresponding <see cref="OpenApiDocument"/> objects.</param>
+        /// <returns>The created <see cref="OpenApiUrlTreeNode"/>.</returns>
+        public static OpenApiUrlTreeNode CreateOpenApiUrlTreeNode(Dictionary<string, OpenApiDocument> sources)
+        {
+            var rootNode = OpenApiUrlTreeNode.Create();
+            foreach (var source in sources)
+            {
+                rootNode.Attach(source.Value, source.Key);
+            }
+            return rootNode;
+        }
+
+        private static IDictionary<OperationType, OpenApiOperation> GetOpenApiOperations(OpenApiUrlTreeNode rootNode, string relativeUrl, string label)
+        {
+            if (relativeUrl.Equals("/", StringComparison.Ordinal) && rootNode.HasOperations(label))
+            {
+                return rootNode.PathItems[label].Operations;
+            }
+
+            var urlSegments = relativeUrl.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
+
+            IDictionary<OperationType, OpenApiOperation> operations = null;
+
+            var targetChild = rootNode;
+
+            /* This will help keep track of whether we've skipped a segment
+             * in the target url due to a possible parameter naming mismatch
+             * with the corresponding OpenApiUrlTreeNode target child segment.
+             */
+            var parameterNameOffset = 0;
+
+            for (var i = 0; i < urlSegments?.Length; i++)
+            {
+                var tempTargetChild = targetChild?.Children?
+                                                  .FirstOrDefault(x => x.Key.Equals(urlSegments[i],
+                                                                    StringComparison.OrdinalIgnoreCase)).Value;
+
+                // Segment name mismatch
+                if (tempTargetChild == null)
+                {
+                    if (i == 0)
+                    {
+                        /* If no match and we are at the 1st segment of the relative url,
+                         * exit; no need to continue matching subsequent segments.
+                         */
+                        break;
+                    }
+
+                    /* Attempt to get the parameter segment from the children of the current node:
+                     * We are assuming a failed match because of different parameter namings
+                     * between the relative url segment and the corresponding OpenApiUrlTreeNode segment name
+                     * ex.: matching '/users/12345/messages' with '/users/{user-id}/messages'
+                     */
+                    tempTargetChild = targetChild?.Children?
+                                                 .FirstOrDefault(x => x.Value.IsParameter).Value;
+
+                    /* If no parameter segment exists in the children of the
+                     * current node or we've already skipped a parameter
+                     * segment in the relative url from the last pass,
+                     * then exit; there's no match.
+                     */
+                    if (tempTargetChild == null || parameterNameOffset > 0)
+                    {
+                        break;
+                    }
+
+                    /* To help us know we've skipped a
+                     * corresponding segment in the relative url.
+                     */
+                    parameterNameOffset++;
+                }
+                else
+                {
+                    parameterNameOffset = 0;
+                }
+
+                // Move to the next segment
+                targetChild = tempTargetChild;
+
+                // We want the operations of the last segment of the path.
+                if (i == urlSegments.Length - 1 && targetChild.HasOperations(label))
+                {
+                    operations = targetChild.PathItems[label].Operations;
+                }
+            }
+
+            return operations;
+        }
+
+        private static IList<SearchResult> FindOperations(OpenApiDocument sourceDocument, Func<string, OperationType?, OpenApiOperation, bool> predicate)
         {
             var search = new OperationSearch(predicate);
             var walker = new OpenApiWalker(search);
-            walker.Walk(graphOpenApi);
+            walker.Walk(sourceDocument);
             return search.SearchResults;
         }
 
@@ -177,5 +321,23 @@ private static bool AddReferences(OpenApiComponents newComponents, OpenApiCompon
             }
             return moreStuff;
         }
+
+        private static string FormatUrlString(string url, IList<OpenApiServer> serverList)
+        {
+            var queryPath = string.Empty;
+            foreach (var server in serverList)
+            {
+                var serverUrl = server.Url.TrimEnd('/');
+                if (!url.Contains(serverUrl))
+                {
+                    continue;
+                }
+
+                var querySegments = url.Split(new[]{ serverUrl }, StringSplitOptions.None);
+                queryPath = querySegments[1];
+            }
+
+            return queryPath;
+        }
     }
 }