feat: track $ref dependencies (#104)

* feat: track $ref dependencies

* chore: pr feed back

- revert changes to ICrawlerResult
- don't use srn
- add tests

* feat: ref graph

* chore: clean up

* fix: graph should not be an option

* fix: return a new graph instance when resolving a json object

* chore: fix tests

* chore: set node data in graph

* chore: update refgraph typing

* chore: add circular tests

* chore: remove ref graph functions

Author: pytlesk4

src/__tests__/__snapshots__/resolver.spec.ts.snap

@@ -0,0 +1,25 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`resolver print tree circular refs 1`] = `
+Array [
+  "#/ref3",
+  "#/ref1",
+  "#/ref2",
+]
+`;
+
+exports[`resolver print tree indirect circular refs 1`] = `
+Array [
+  "custom://obj3/",
+  "custom://obj2/",
+  "custom://obj1/",
+]
+`;
+
+exports[`resolver print tree should handle local refs 1`] = `
+Array [
+  "#/definitions/bear",
+]
+`;
+
+exports[`resolver print tree should resolve http relative paths + back pointing uri refs 1`] = `undefined`;

src/__tests__/resolver.spec.ts

@@ -819,7 +819,7 @@ describe('resolver', () => {
       };
 
       const runner = new ResolveRunner(source);
-      const resolved = await runner.resolve('#/inner2/marcsStreet');
+      const resolved = await runner.resolve({ jsonPointer: '#/inner2/marcsStreet' });
 
       // only marcStreet and related paths replaced
       const newObj = {
@@ -847,7 +847,7 @@ describe('resolver', () => {
 
       // now we use the same runner to resolve another portion of it
       // only the new portions are resolved (in addition to what has already been done)
-      await resolved.runner.resolve('#/inner3');
+      await resolved.runner.resolve({ jsonPointer: '#/inner3' });
 
       expect(runner.source).toEqual({
         ...newObj,
@@ -2070,6 +2070,130 @@ describe('resolver', () => {
     });
   });
 
+  describe('print tree', () => {
+    test('should handle local refs', async () => {
+      const data = {
+        title: 'Example',
+        type: 'object',
+        definitions: {
+          bear: {
+            type: 'object',
+            properties: {
+              type: {
+                type: 'string',
+              },
+              diet: {
+                type: 'string',
+              },
+              age: {
+                type: 'number',
+              },
+            },
+            required: ['type', 'diet', 'age'],
+          },
+        },
+        description: 'Bears are awesome',
+        properties: {
+          id: {
+            type: 'string',
+          },
+          bear: {
+            $ref: '#/definitions/bear',
+          },
+        },
+      };
+
+      const resolver = new Resolver();
+      const { graph } = await resolver.resolve(data);
+
+      expect(graph.dependenciesOf('root')).toMatchSnapshot();
+    });
+
+    // ./a#/foo -> ./b#bar -> ./a#/xxx -> ./c -> ./b#/zzz
+    test('should resolve http relative paths + back pointing uri refs', async () => {
+      const source = httpMocks['https://back-pointing.com/a'];
+
+      const resolver = new Resolver({
+        resolvers: {
+          https: new HttpReader(),
+        },
+      });
+
+      const baseUri = 'https://back-pointing.com/a';
+      const { graph } = await resolver.resolve(source, {
+        baseUri,
+      });
+
+      expect(graph.dependenciesOf[baseUri]).toMatchSnapshot();
+    });
+
+    test('circular refs', async () => {
+      const source = {
+        ref1: {
+          $ref: '#/ref3',
+        },
+        ref2: {
+          $ref: '#/ref1',
+        },
+        ref3: {
+          $ref: '#/ref2',
+        },
+      };
+
+      const resolver = new Resolver();
+      const { graph } = await resolver.resolve(source);
+
+      expect(graph.dependenciesOf('root')).toMatchSnapshot();
+    });
+
+    test('indirect circular refs', async () => {
+      const data = {
+        obj1: {
+          one: true,
+          foo: {
+            $ref: 'custom://obj2',
+          },
+        },
+        obj2: {
+          two: true,
+          foo: {
+            $ref: 'custom://obj3',
+          },
+        },
+        obj3: {
+          three: true,
+          foo: {
+            $ref: 'custom://obj1',
+          },
+        },
+      };
+
+      const source = {
+        inner: {
+          data: {
+            $ref: 'custom://obj1',
+          },
+        },
+      };
+
+      const reader: Types.IResolver = {
+        async resolve(ref: uri.URI): Promise<any> {
+          return data[ref.authority()];
+        },
+      };
+
+      const resolver = new Resolver({
+        resolvers: {
+          custom: reader,
+        },
+      });
+
+      const { graph } = await resolver.resolve(source);
+
+      expect(graph.dependenciesOf('root')).toMatchSnapshot();
+    });
+  });
+
   describe('use cases', () => {
     test('mixture of file and http', async () => {
       const data = {

src/crawler.ts

@@ -145,6 +145,10 @@ export class ResolveCrawler implements Types.ICrawler {
           this.pointerGraph.addNode(targetPointer);
         }
 
+        const targetRef = `${this._runner.baseUri.toString()}${targetPointer}`;
+        if (!this._runner.graph.hasNode(targetRef)) this._runner.graph.addNode(targetRef);
+        if (this._runner.root !== targetRef) this._runner.graph.addDependency(this._runner.root, targetRef);
+
         // register parent as a dependant of the target
         this.pointerGraph.addDependency(parentPointer, targetPointer);
 
@@ -161,6 +165,10 @@ export class ResolveCrawler implements Types.ICrawler {
       }
     } else {
       // remote pointer
+      const remoteRef = ref.toString();
+      if (!this._runner.graph.hasNode(remoteRef)) this._runner.graph.addNode(remoteRef);
+      if (this._runner.root !== remoteRef) this._runner.graph.addDependency(this._runner.root, remoteRef);
+
       if (this._runner.dereferenceRemote && !this._runner.atMaxUriDepth()) {
         this.resolvers.push(this._runner.lookupAndResolveUri(opts));
       }

src/resolver.ts

@@ -1,3 +1,5 @@
+import { DepGraph } from 'dependency-graph';
+
 import { Cache } from './cache';
 import { ResolveRunner } from './runner';
 import * as Types from './types';
@@ -35,7 +37,8 @@ export class Resolver {
   }
 
   public resolve(source: any, opts: Types.IResolveOpts = {}): Promise<Types.IResolveResult> {
-    const runner = new ResolveRunner(source, {
+    const graph = new DepGraph<any>({ circular: true });
+    const runner = new ResolveRunner(source, graph, {
       uriCache: this.uriCache,
       resolvers: this.resolvers,
       getRef: this.getRef,
@@ -48,6 +51,6 @@ export class Resolver {
       ctx: Object.assign({}, this.ctx || {}, opts.ctx || {}),
     });
 
-    return runner.resolve(opts.jsonPointer);
+    return runner.resolve(opts);
   }
 }

src/runner.ts

@@ -1,5 +1,6 @@
 import { pathToPointer, pointerToPath, startsWith, trimStart } from '@stoplight/json';
-import produce from 'immer';
+import { DepGraph } from 'dependency-graph';
+import produce, { original } from 'immer';
 import { get, set } from 'lodash';
 import { dirname, join } from 'path';
 import * as URI from 'urijs';
@@ -24,6 +25,8 @@ export class ResolveRunner implements Types.IResolveRunner {
   public readonly id: number;
   public readonly baseUri: uri.URI;
   public readonly uriCache: Types.ICache;
+  public readonly graph: DepGraph<any>;
+  public readonly root: string;
 
   public depth: number;
   public uriStack: string[];
@@ -44,7 +47,11 @@ export class ResolveRunner implements Types.IResolveRunner {
 
   private _source: any;
 
-  constructor(source: any, opts: Types.IResolveRunnerOpts = {}) {
+  constructor(
+    source: any,
+    graph: DepGraph<any> = new DepGraph<any>({ circular: true }),
+    opts: Types.IResolveRunnerOpts = {},
+  ) {
     this.id = resolveRunnerCount += 1;
     this.depth = opts.depth || 0;
     this._source = source;
@@ -60,6 +67,13 @@ export class ResolveRunner implements Types.IResolveRunner {
     this.uriStack = opts.uriStack || [];
     this.uriCache = opts.uriCache || new Cache();
 
+    this.root = (opts.root && opts.root.toString()) || this.baseUri.toString() || 'root';
+
+    this.graph = graph;
+    if (!this.graph.hasNode(this.root)) {
+      this.graph.addNode(this.root);
+    }
+
     if (this.baseUri && this.depth === 0) {
       // if this first runner has a baseUri, seed the cache so we don't create another one for this uri later
       this.uriCache.set(this.computeUriCacheKey(this.baseUri), this);
@@ -94,16 +108,17 @@ export class ResolveRunner implements Types.IResolveRunner {
     return this._source;
   }
 
-  public async resolve(jsonPointer?: string, opts?: Types.IResolveOpts): Promise<Types.IResolveResult> {
+  public async resolve(opts?: Types.IResolveOpts): Promise<Types.IResolveResult> {
     const resolved: Types.IResolveResult = {
       result: this.source,
+      graph: this.graph,
       refMap: {},
       errors: [],
       runner: this,
     };
 
     let targetPath: any;
-    jsonPointer = jsonPointer && jsonPointer.trim();
+    const jsonPointer = opts && opts.jsonPointer && opts.jsonPointer.trim();
     if (jsonPointer && jsonPointer !== '#' && jsonPointer !== '#/') {
       targetPath = pointerToPath(jsonPointer);
       resolved.result = get(resolved.result, targetPath);
@@ -165,6 +180,10 @@ export class ResolveRunner implements Types.IResolveRunner {
               return r.resolved.result;
             } else {
               set(draft, resolvedTargetPath, r.resolved.result);
+
+              if (this.graph.hasNode(String(r.uri))) {
+                this.graph.setNodeData(String(r.uri), r.resolved.result);
+              }
             }
           }
         });
@@ -209,6 +228,10 @@ export class ResolveRunner implements Types.IResolveRunner {
 
                 if (val !== void 0) {
                   set(draft, dependantPath, val);
+
+                  if (this.graph.hasNode(pathToPointer(pointerPath))) {
+                    this.graph.setNodeData(pathToPointer(pointerPath), original(val));
+                  }
                 } else {
                   resolved.errors.push({
                     code: 'POINTER_MISSING',
@@ -366,9 +389,10 @@ export class ResolveRunner implements Types.IResolveRunner {
       }
     }
 
-    return new ResolveRunner(result, {
+    return new ResolveRunner(result, this.graph, {
       depth: this.depth + 1,
       baseUri: ref.toString(),
+      root: ref,
       uriStack: this.uriStack,
       uriCache: this.uriCache,
       resolvers: this.resolvers,
@@ -397,6 +421,7 @@ export class ResolveRunner implements Types.IResolveRunner {
     if (this.uriStack.includes(uriCacheKey)) {
       lookupResult.resolved = {
         result: val,
+        graph: this.graph,
         refMap: {},
         errors: [],
         runner: this,
@@ -441,7 +466,10 @@ export class ResolveRunner implements Types.IResolveRunner {
       // only resolve the uri result if we were able to look it up and create the resolver
       // @ts-ignore
       if (uriResolver) {
-        lookupResult.resolved = await uriResolver.resolve(Utils.uriToJSONPointer(ref), { parentPath });
+        lookupResult.resolved = await uriResolver.resolve({
+          jsonPointer: Utils.uriToJSONPointer(ref),
+          parentPath,
+        });
 
         // if pointer resolution failed, revert to the original value (which will be a $ref most of the time)
         if (lookupResult.resolved.errors.length) {