From 30c95adf79911a87f82612839c27e7619ca7abda Mon Sep 17 00:00:00 2001 From: Manuel Spigolon Date: Sun, 10 May 2020 19:55:55 +0200 Subject: [PATCH 1/7] $id grabber --- .github/workflows/ci.yml | 31 +++++++++++++ .gitignore | 2 + README.md | 23 +++++++++- package.json | 38 ++++++++++++++++ ref-resolver.js | 94 +++++++++++++++++++++++++++++++++++++++ test/ref-resolver.test.js | 40 +++++++++++++++++ 6 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 package.json create mode 100644 ref-resolver.js create mode 100644 test/ref-resolver.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..051d95b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: ci + +on: [push, pull_request] + +env: + CI: true + +jobs: + test: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + node-version: [6.x, 8.x, 10.x, 12.x, 14.x] + os: [ubuntu-latest, windows-latest] + + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + + - name: Install + run: | + npm install --ignore-scripts + + - name: Run tests + run: | + npm test diff --git a/.gitignore b/.gitignore index 6704566..cab290e 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,5 @@ dist # TernJS port file .tern-port + +package-lock.json diff --git a/README.md b/README.md index 357130b..6cc1adc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,23 @@ # json-schema-resolver -Resolve all you $refs + +[![CI](https://github.com/Eomm/json-schema-resolver/workflows/ci/badge.svg)](https://github.com/Eomm/json-schema-resolver/actions?query=workflow%3Aci) +[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/) + +Resolve all `$refs` in your [JSON schema](https://json-schema.org/specification.html)! + + +## Install + +```sh +npm install json-schema-resolver +``` + +This plugin support Node.js >=6 + +## Usage + +TODO + +## License + +Licensed under [MIT](./LICENSE). diff --git a/package.json b/package.json new file mode 100644 index 0000000..4b7a81d --- /dev/null +++ b/package.json @@ -0,0 +1,38 @@ +{ + "name": "json-schema-resolver", + "version": "1.0.0", + "description": "Resolve all your $refs", + "main": "ref-resolver.js", + "scripts": { + "lint": "standard", + "lint:fix": "standard --fix", + "test": "tap test/**/*.test.js" + }, + "engines": { + "node": ">=6" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/Eomm/json-schema-resolver.git" + }, + "author": "Manuel Spigolon (https://github.com/Eomm)", + "license": "MIT", + "bugs": { + "url": "https://github.com/Eomm/json-schema-resolver/issues" + }, + "homepage": "https://github.com/Eomm/json-schema-resolver#readme", + "devDependencies": { + "standard": "^14.3.3", + "tap": "^12.7.0" + }, + "dependencies": { + "uri-js": "^4.2.2" + }, + "keywords": [ + "json", + "schema", + "json-schema", + "ref", + "$ref" + ] +} \ No newline at end of file diff --git a/ref-resolver.js b/ref-resolver.js new file mode 100644 index 0000000..f15fe88 --- /dev/null +++ b/ref-resolver.js @@ -0,0 +1,94 @@ +'use strict' + +const URI = require('uri-js') +const { EventEmitter } = require('events') + +// Target: DRAFT-07 +// https://tools.ietf.org/html/draft-handrews-json-schema-01 + +// Open to DRAFT 08 +// https://json-schema.org/draft/2019-09/json-schema-core.html + +// TODO logic: https://json-schema.org/draft/2019-09/json-schema-core.html#rfc.appendix.B.1 +function jschemaResolver (rootSchema, options) { + const opts = options || {} + const { externalSchemas } = opts + + // If present, the value for this keyword MUST be a string, and MUST + // represent a valid URI-reference [RFC3986]. This value SHOULD be + // normalized, and SHOULD NOT be an empty fragment <#> or an empty + // string <>. + const idExploded = URI.parse(rootSchema.$id) + idExploded.fragment = undefined // remove fragment + + const baseUri = URI.serialize(idExploded) // canonical absolute-URI + rootSchema.$id = baseUri // fix the schema $id value + + const allIds = new Map() + + if (externalSchemas) { + externalSchemas.forEach(_ => { + const ids = mapIds(idExploded, _) + ids.on('$id', collectIds) + }) + } + const ids = mapIds(idExploded, rootSchema) + ids.on('$id', collectIds) + + return { + + } + + function collectIds (json, baseUri, relative) { + const rel = (relative && URI.serialize(relative)) || '' + const id = URI.serialize(baseUri) + rel + console.log(id) + if (allIds.has(id)) { + console.log('WARN duplicated id .. IGNORED - ' + id) + } else { + allIds.set(id, json) + } + } +} + +function mapIds (baseUri, schema) { + const ee = new EventEmitter() + process.nextTick(() => { search(baseUri, schema) }) + return ee + + /** + * + * @param {URI} baseUri + * @param {*} json + */ + function search (baseUri, json) { + if (!(json instanceof Object)) return + + if (json.$id) { // TODO the $id should manage $anchor to support draft 08 + const $idUri = URI.parse(json.$id) + let fragment = null + + if ($idUri.reference === 'absolute') { + // "$id": "http://example.com/root.json" + baseUri = $idUri // a new baseURI for children + } else if ($idUri.reference === 'relative') { + // "$id": "other.json", + const newBaseUri = Object.assign({}, baseUri) + newBaseUri.path = $idUri.path + newBaseUri.fragment = $idUri.fragment + baseUri = newBaseUri + } else { + // { "$id": "#bar" } + fragment = $idUri + } + ee.emit('$id', json, baseUri, fragment) + } + + const fields = Object.keys(json) + for (const prop of fields) { + search(baseUri, json[prop]) + } + } +} + +module.exports = jschemaResolver diff --git a/test/ref-resolver.test.js b/test/ref-resolver.test.js new file mode 100644 index 0000000..4fa0c5a --- /dev/null +++ b/test/ref-resolver.test.js @@ -0,0 +1,40 @@ +'use strict' + +const { test } = require('tap') +const refResolver = require('../ref-resolver') + +// https://json-schema.org/draft/2019-09/json-schema-core.html#idExamples + +test('', t => { + const schema = { + $id: 'http://example.com/root.json', + definitions: { + A: { $id: '#foo' }, + B: { + $id: 'other.json', + definitions: { + X: { $id: '#bar' }, + Z: { $ref: 'commonSchema#' }, + Y: { $id: 't/inner.json' } + } + }, + C: { + $id: 'urn:uuid:ee564b8a-7a87-4125-8c96-e9f123d6766f' + } + } + } + + const opts = { + externalSchemas: [ + { + $id: 'commonSchema', + type: 'object', + properties: { + hello: { type: 'string' } + } + } + ] + } + refResolver(schema, opts) + t.end() +}) From 4a8f089cf132705ca24fcec4e3a655adf5965dc5 Mon Sep 17 00:00:00 2001 From: Manuel Spigolon Date: Sun, 10 May 2020 23:39:05 +0200 Subject: [PATCH 2/7] $ref tracing --- ref-resolver.js | 33 +++++++++++++++++++++++++++++++++ test/ref-resolver.test.js | 5 +++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/ref-resolver.js b/ref-resolver.js index f15fe88..c635e2d 100644 --- a/ref-resolver.js +++ b/ref-resolver.js @@ -25,15 +25,21 @@ function jschemaResolver (rootSchema, options) { rootSchema.$id = baseUri // fix the schema $id value const allIds = new Map() + const allRefs = [] if (externalSchemas) { externalSchemas.forEach(_ => { const ids = mapIds(idExploded, _) ids.on('$id', collectIds) + ids.on('$ref', collectRefs) }) } const ids = mapIds(idExploded, rootSchema) ids.on('$id', collectIds) + ids.on('$ref', collectRefs) + + // TODO attach every external schema to rootSchema/definition (or $defs for draft 08) + // TODO rewrite all the $ref pointing to /definitions return { @@ -49,6 +55,30 @@ function jschemaResolver (rootSchema, options) { allIds.set(id, json) } } + + function collectRefs (json, baseUri, refVal) { + const refUri = URI.parse(refVal) + console.log('Ref: ' + URI.serialize(refUri)) + + if (refUri.reference !== 'absolute') { + refUri.scheme = baseUri.scheme + refUri.userinfo = baseUri.userinfo + refUri.host = baseUri.host + refUri.port = baseUri.port + + if (refUri.reference === 'relative') { + const newBaseUri = Object.assign({}, baseUri) + newBaseUri.path = refUri.path + baseUri = newBaseUri + } + } + + allRefs.push({ + baseUri: URI.serialize(baseUri), + ref: URI.serialize(refUri), + json + }) + } } function mapIds (baseUri, schema) { @@ -86,6 +116,9 @@ function mapIds (baseUri, schema) { const fields = Object.keys(json) for (const prop of fields) { + if (prop === '$ref') { + ee.emit('$ref', json, baseUri, json[prop]) + } search(baseUri, json[prop]) } } diff --git a/test/ref-resolver.test.js b/test/ref-resolver.test.js index 4fa0c5a..b0fa8dd 100644 --- a/test/ref-resolver.test.js +++ b/test/ref-resolver.test.js @@ -14,7 +14,8 @@ test('', t => { $id: 'other.json', definitions: { X: { $id: '#bar' }, - Z: { $ref: 'commonSchema#' }, + Z: { $ref: 'commonSchema#/definitions/hello' }, + Zx: { $ref: 'http://wow/commonSchema#/definitions/hello' }, Y: { $id: 't/inner.json' } } }, @@ -29,7 +30,7 @@ test('', t => { { $id: 'commonSchema', type: 'object', - properties: { + definitions: { hello: { type: 'string' } } } From db5f94415ad2a77c22336b95093b74873671a0be Mon Sep 17 00:00:00 2001 From: Manuel Spigolon Date: Fri, 15 May 2020 20:37:54 +0200 Subject: [PATCH 3/7] $ref replacing --- package.json | 1 + ref-resolver.js | 163 +++++++++++++++++++++++--------------- test/ref-resolver.test.js | 117 +++++++++++++++++++++++---- 3 files changed, 201 insertions(+), 80 deletions(-) diff --git a/package.json b/package.json index 4b7a81d..a6f08c0 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ }, "homepage": "https://github.com/Eomm/json-schema-resolver#readme", "devDependencies": { + "rfdc": "^1.1.4", "standard": "^14.3.3", "tap": "^12.7.0" }, diff --git a/ref-resolver.js b/ref-resolver.js index c635e2d..2198735 100644 --- a/ref-resolver.js +++ b/ref-resolver.js @@ -1,66 +1,101 @@ 'use strict' +const debug = require('debug')('json-schema-resolver') const URI = require('uri-js') const { EventEmitter } = require('events') -// Target: DRAFT-07 +const kRefToDef = Symbol('json-schema-resolver.refToDef') +const kConsumed = Symbol('json-schema-resolver.consumed') +const kIgnore = Symbol('json-schema-resolver.ignore') + +// ! Target: DRAFT-07 // https://tools.ietf.org/html/draft-handrews-json-schema-01 -// Open to DRAFT 08 +// ? Open to DRAFT 08 // https://json-schema.org/draft/2019-09/json-schema-core.html // TODO logic: https://json-schema.org/draft/2019-09/json-schema-core.html#rfc.appendix.B.1 -function jschemaResolver (rootSchema, options) { - const opts = options || {} - const { externalSchemas } = opts - - // If present, the value for this keyword MUST be a string, and MUST - // represent a valid URI-reference [RFC3986]. This value SHOULD be - // normalized, and SHOULD NOT be an empty fragment <#> or an empty - // string <>. - const idExploded = URI.parse(rootSchema.$id) - idExploded.fragment = undefined // remove fragment - - const baseUri = URI.serialize(idExploded) // canonical absolute-URI - rootSchema.$id = baseUri // fix the schema $id value +function jsonSchemaResolver () { + const ee = new EventEmitter() const allIds = new Map() + let rolling = 0 + ee.on('$id', collectIds) + const allRefs = [] + ee.on('$ref', collectRefs) - if (externalSchemas) { - externalSchemas.forEach(_ => { - const ids = mapIds(idExploded, _) - ids.on('$id', collectIds) - ids.on('$ref', collectRefs) - }) + return { + resolve } - const ids = mapIds(idExploded, rootSchema) - ids.on('$id', collectIds) - ids.on('$ref', collectRefs) - // TODO attach every external schema to rootSchema/definition (or $defs for draft 08) - // TODO rewrite all the $ref pointing to /definitions + function resolve (rootSchema, opts) { + const { externalSchemas } = opts || {} - return { + // If present, the value for this keyword MUST be a string, and MUST + // represent a valid URI-reference [RFC3986]. This value SHOULD be + // normalized, and SHOULD NOT be an empty fragment <#> or an empty + // string <>. + const appUri = URI.parse(rootSchema.$id) + appUri.fragment = undefined // remove fragment + debug('Found app URI %o', appUri) + + if (externalSchemas) { + for (const es of externalSchemas) { mapIds(ee, appUri, es) } + } + debug('Processed external schemas') + + const baseUri = URI.serialize(appUri) // canonical absolute-URI + rootSchema.$id = baseUri // fix the schema $id value + rootSchema[kIgnore] = true + + mapIds(ee, appUri, rootSchema) + debug('Processed root schema') + + debug('Generating %d refs', allRefs.length) + allRefs.forEach(({ baseUri, ref, json }) => { + debug('Evaluating $ref %s', ref) + if (ref[0] === '#') { return } + + const evaluatedJson = allIds.get(baseUri) + evaluatedJson[kConsumed] = true + json.$ref = `#/definitions/${evaluatedJson[kRefToDef]}` + }) + + // TODO $def instead of definitions + allIds.forEach((json, baseUri) => { + if (json[kConsumed] === true) { + if (!rootSchema.definitions) { + rootSchema.definitions = {} + } + rootSchema.definitions[json[kRefToDef]] = json + } + }) + + return rootSchema } function collectIds (json, baseUri, relative) { + if (json[kIgnore]) { return } + const rel = (relative && URI.serialize(relative)) || '' const id = URI.serialize(baseUri) + rel - console.log(id) if (allIds.has(id)) { - console.log('WARN duplicated id .. IGNORED - ' + id) + debug('WARN duplicated id %s .. IGNORED - ', id) // TODO } else { + debug('Collected $id %s', id) + json[kRefToDef] = `def-${rolling++}` allIds.set(id, json) } } function collectRefs (json, baseUri, refVal) { const refUri = URI.parse(refVal) - console.log('Ref: ' + URI.serialize(refUri)) + debug('Pre enqueue $ref %o', refUri) + if (refUri.reference === 'same-document') { - if (refUri.reference !== 'absolute') { + } else if (refUri.reference !== 'absolute') { refUri.scheme = baseUri.scheme refUri.userinfo = baseUri.userinfo refUri.host = baseUri.host @@ -73,55 +108,51 @@ function jschemaResolver (rootSchema, options) { } } + const ref = URI.serialize(refUri) allRefs.push({ baseUri: URI.serialize(baseUri), - ref: URI.serialize(refUri), + ref, json }) + debug('Enqueue $ref %s', ref) } } -function mapIds (baseUri, schema) { - const ee = new EventEmitter() - process.nextTick(() => { search(baseUri, schema) }) - return ee - - /** +/** * * @param {URI} baseUri * @param {*} json */ - function search (baseUri, json) { - if (!(json instanceof Object)) return - - if (json.$id) { // TODO the $id should manage $anchor to support draft 08 - const $idUri = URI.parse(json.$id) - let fragment = null - - if ($idUri.reference === 'absolute') { - // "$id": "http://example.com/root.json" - baseUri = $idUri // a new baseURI for children - } else if ($idUri.reference === 'relative') { - // "$id": "other.json", - const newBaseUri = Object.assign({}, baseUri) - newBaseUri.path = $idUri.path - newBaseUri.fragment = $idUri.fragment - baseUri = newBaseUri - } else { - // { "$id": "#bar" } - fragment = $idUri - } - ee.emit('$id', json, baseUri, fragment) +function mapIds (ee, baseUri, json) { + if (!(json instanceof Object)) return + + if (json.$id) { // TODO the $id should manage $anchor to support draft 08 + const $idUri = URI.parse(json.$id) + let fragment = null + + if ($idUri.reference === 'absolute') { + // "$id": "http://example.com/root.json" + baseUri = $idUri // a new baseURI for children + } else if ($idUri.reference === 'relative') { + // "$id": "other.json", + const newBaseUri = Object.assign({}, baseUri) + newBaseUri.path = $idUri.path + newBaseUri.fragment = $idUri.fragment + baseUri = newBaseUri + } else { + // { "$id": "#bar" } + fragment = $idUri } + ee.emit('$id', json, baseUri, fragment) + } - const fields = Object.keys(json) - for (const prop of fields) { - if (prop === '$ref') { - ee.emit('$ref', json, baseUri, json[prop]) - } - search(baseUri, json[prop]) + const fields = Object.keys(json) + for (const prop of fields) { + if (prop === '$ref') { + ee.emit('$ref', json, baseUri, json[prop]) } + mapIds(ee, baseUri, json[prop]) } } -module.exports = jschemaResolver +module.exports = jsonSchemaResolver diff --git a/test/ref-resolver.test.js b/test/ref-resolver.test.js index b0fa8dd..5c0688c 100644 --- a/test/ref-resolver.test.js +++ b/test/ref-resolver.test.js @@ -2,25 +2,29 @@ const { test } = require('tap') const refResolver = require('../ref-resolver') +const clone = require('rfdc')({ circles: false }) // https://json-schema.org/draft/2019-09/json-schema-core.html#idExamples -test('', t => { +test('$ref to root', t => { + t.plan(2) + const schema = { $id: 'http://example.com/root.json', + properties: { + person: { $ref: '#/definitions/person' }, + children: { + type: 'array', + items: { $ref: '#/definitions/person' } + } + }, definitions: { - A: { $id: '#foo' }, - B: { - $id: 'other.json', - definitions: { - X: { $id: '#bar' }, - Z: { $ref: 'commonSchema#/definitions/hello' }, - Zx: { $ref: 'http://wow/commonSchema#/definitions/hello' }, - Y: { $id: 't/inner.json' } + person: { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'integer' } } - }, - C: { - $id: 'urn:uuid:ee564b8a-7a87-4125-8c96-e9f123d6766f' } } } @@ -36,6 +40,91 @@ test('', t => { } ] } - refResolver(schema, opts) - t.end() + const resolver = refResolver() + const originalSchema = clone(schema) + const out = resolver.resolve(schema, opts) + t.deepEquals(schema, originalSchema, 'the param schema should not be changed') + t.equals(schema, out, 'the output schema is the same (LINK) of the input one') +}) + +test('$ref to an external schema', t => { + t.plan(3) + const schema = { + $id: 'http://example.com/SimplePerson', + type: 'object', + properties: { + name: { type: 'string' }, + address: { $ref: '/SimpleAddress' }, + votes: { type: 'integer', minimum: 1 } + } + } + + const opts = { + externalSchemas: [ + { + $id: '/SimpleAddress', + type: 'object', + properties: { + lines: { + type: 'array', + items: { type: 'string' } + }, + zip: { type: 'string' }, + city: { type: 'string' }, + country: { type: 'string' } + }, + required: ['country'] + } + ] + } + + const resolver = refResolver() + const out = resolver.resolve(schema, opts) + t.deepEquals(schema, out, 'the output is the same input modified') + t.ok(out.definitions, 'definitions has been added') + t.deepEquals(Object.values(out.definitions), opts.externalSchemas, 'external schema has been added to definitions') +}) + +test('$ref circular', t => { + t.plan(3) + const schema = { + $id: 'http://example.com/', + $ref: 'SimplePerson' + } + + const opts = { + externalSchemas: [ + { + $id: 'SimplePerson', + type: 'object', + properties: { + name: { type: 'string' }, + address: { $ref: 'SimpleAddress#' }, + friends: { type: 'array', items: { $ref: '#' } }, + votes: { type: 'integer', minimum: 1 } + } + }, + { + $id: 'SimpleAddress', + type: 'object', + properties: { + lines: { + type: 'array', + items: { type: 'string' } + }, + zip: { type: 'string' }, + city: { type: 'string' }, + country: { type: 'string' } + }, + required: ['country'] + } + ] + } + + const resolver = refResolver() + const out = resolver.resolve(schema, opts) + t.deepEquals(schema, out, 'the output is the same input modified') + t.ok(out.definitions, 'definitions has been added') + require('fs').writeFileSync('./out.json', JSON.stringify(out, null, 2)) + t.deepEquals(Object.values(out.definitions), opts.externalSchemas, 'external schema has been added to definitions') }) From 4759f52f75005c5097f02ddff91e4a2aeb1de882 Mon Sep 17 00:00:00 2001 From: Manuel Spigolon Date: Sat, 16 May 2020 10:14:14 +0200 Subject: [PATCH 4/7] add some tests --- .gitignore | 2 + package.json | 6 +- ref-resolver.js | 61 ++++++-- test/ref-resolver.test.js | 137 ++++++++---------- test/schema-factory.js | 6 + test/schemas/absoluteId-externalRef.js | 10 ++ test/schemas/absoluteId-localRef.js | 19 +++ .../schemas/relativeId-externalAndLocalRef.js | 10 ++ test/schemas/relativeId-noRef.js | 14 ++ 9 files changed, 171 insertions(+), 94 deletions(-) create mode 100644 test/schema-factory.js create mode 100644 test/schemas/absoluteId-externalRef.js create mode 100644 test/schemas/absoluteId-localRef.js create mode 100644 test/schemas/relativeId-externalAndLocalRef.js create mode 100644 test/schemas/relativeId-noRef.js diff --git a/.gitignore b/.gitignore index cab290e..3911d3b 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,5 @@ dist .tern-port package-lock.json + +out*.json diff --git a/package.json b/package.json index a6f08c0..b4c1b33 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "lint": "standard", "lint:fix": "standard --fix", - "test": "tap test/**/*.test.js" + "test": "npm run lint && tap test/**/*.test.js --cov" }, "engines": { "node": ">=6" @@ -22,11 +22,11 @@ }, "homepage": "https://github.com/Eomm/json-schema-resolver#readme", "devDependencies": { - "rfdc": "^1.1.4", "standard": "^14.3.3", "tap": "^12.7.0" }, "dependencies": { + "rfdc": "^1.1.4", "uri-js": "^4.2.2" }, "keywords": [ @@ -36,4 +36,4 @@ "ref", "$ref" ] -} \ No newline at end of file +} diff --git a/ref-resolver.js b/ref-resolver.js index 2198735..e3ede8f 100644 --- a/ref-resolver.js +++ b/ref-resolver.js @@ -1,12 +1,13 @@ 'use strict' -const debug = require('debug')('json-schema-resolver') const URI = require('uri-js') +const cloner = require('rfdc')({ proto: true, circles: false }) const { EventEmitter } = require('events') +const debug = require('debug')('json-schema-resolver') -const kRefToDef = Symbol('json-schema-resolver.refToDef') -const kConsumed = Symbol('json-schema-resolver.consumed') -const kIgnore = Symbol('json-schema-resolver.ignore') +const kIgnore = Symbol('json-schema-resolver.ignore') // untrack a schema (usually the root one) +const kRefToDef = Symbol('json-schema-resolver.refToDef') // assign to an external json a new reference +const kConsumed = Symbol('json-schema-resolver.consumed') // when an external json has been referenced // ! Target: DRAFT-07 // https://tools.ietf.org/html/draft-handrews-json-schema-01 @@ -14,9 +15,29 @@ const kIgnore = Symbol('json-schema-resolver.ignore') // ? Open to DRAFT 08 // https://json-schema.org/draft/2019-09/json-schema-core.html -// TODO logic: https://json-schema.org/draft/2019-09/json-schema-core.html#rfc.appendix.B.1 -function jsonSchemaResolver () { +const defaultOpts = { + target: 'draft-07', + clone: false +} + +const targetSupported = ['draft-07'] // TODO , 'draft-08' +const targetCfg = { + 'draft-07': { + def: 'definitions' + }, + 'draft-08': { + def: '$defs' + } +} + +// logic: https://json-schema.org/draft/2019-09/json-schema-core.html#rfc.appendix.B.1 +function jsonSchemaResolver (options) { const ee = new EventEmitter() + const { clone, target } = Object.assign({}, defaultOpts, options) + + if (!targetSupported.includes(target)) { + throw new Error(`Unsupported JSON schema version ${target}`) + } const allIds = new Map() let rolling = 0 @@ -32,6 +53,13 @@ function jsonSchemaResolver () { function resolve (rootSchema, opts) { const { externalSchemas } = opts || {} + allIds.clear() + allRefs.length = 0 + + if (clone) { + rootSchema = cloner(rootSchema) + } + // If present, the value for this keyword MUST be a string, and MUST // represent a valid URI-reference [RFC3986]. This value SHOULD be // normalized, and SHOULD NOT be an empty fragment <#> or an empty @@ -42,8 +70,8 @@ function jsonSchemaResolver () { if (externalSchemas) { for (const es of externalSchemas) { mapIds(ee, appUri, es) } + debug('Processed external schemas') } - debug('Processed external schemas') const baseUri = URI.serialize(appUri) // canonical absolute-URI rootSchema.$id = baseUri // fix the schema $id value @@ -62,14 +90,14 @@ function jsonSchemaResolver () { json.$ref = `#/definitions/${evaluatedJson[kRefToDef]}` }) - // TODO $def instead of definitions + const defKey = targetCfg[target].def allIds.forEach((json, baseUri) => { if (json[kConsumed] === true) { - if (!rootSchema.definitions) { - rootSchema.definitions = {} + if (!rootSchema[defKey]) { + rootSchema[defKey] = {} } - rootSchema.definitions[json[kRefToDef]] = json + rootSchema[defKey][json[kRefToDef]] = json } }) @@ -81,12 +109,12 @@ function jsonSchemaResolver () { const rel = (relative && URI.serialize(relative)) || '' const id = URI.serialize(baseUri) + rel - if (allIds.has(id)) { - debug('WARN duplicated id %s .. IGNORED - ', id) // TODO - } else { + if (!allIds.has(id)) { debug('Collected $id %s', id) json[kRefToDef] = `def-${rolling++}` allIds.set(id, json) + } else { + debug('WARN duplicated id %s .. IGNORED - ', id) } } @@ -126,7 +154,7 @@ function jsonSchemaResolver () { function mapIds (ee, baseUri, json) { if (!(json instanceof Object)) return - if (json.$id) { // TODO the $id should manage $anchor to support draft 08 + if (json.$id) { const $idUri = URI.parse(json.$id) let fragment = null @@ -145,6 +173,9 @@ function mapIds (ee, baseUri, json) { } ee.emit('$id', json, baseUri, fragment) } + // else if (json.$anchor) { + // TODO the $id should manage $anchor to support draft 08 + // } const fields = Object.keys(json) for (const prop of fields) { diff --git a/test/ref-resolver.test.js b/test/ref-resolver.test.js index 5c0688c..1cfa0b4 100644 --- a/test/ref-resolver.test.js +++ b/test/ref-resolver.test.js @@ -1,46 +1,33 @@ 'use strict' const { test } = require('tap') +const clone = require('rfdc')({ proto: true, circles: false }) + const refResolver = require('../ref-resolver') -const clone = require('rfdc')({ circles: false }) +const factory = require('./schema-factory') + +// eslint-disable-next-line +const save = (out) => require('fs').writeFileSync(`./out-${Date.now()}.json`, JSON.stringify(out, null, 2)) // https://json-schema.org/draft/2019-09/json-schema-core.html#idExamples +test('wrong params', t => { + t.plan(1) + t.throws(() => refResolver({ target: 'draft-1000' })) +}) + test('$ref to root', t => { t.plan(2) - const schema = { - $id: 'http://example.com/root.json', - properties: { - person: { $ref: '#/definitions/person' }, - children: { - type: 'array', - items: { $ref: '#/definitions/person' } - } - }, - definitions: { - person: { - type: 'object', - properties: { - name: { type: 'string' }, - age: { type: 'integer' } - } - } - } - } + const schema = factory('absoluteId-localRef') const opts = { externalSchemas: [ - { - $id: 'commonSchema', - type: 'object', - definitions: { - hello: { type: 'string' } - } - } + factory('relativeId-noRef') ] } const resolver = refResolver() + const originalSchema = clone(schema) const out = resolver.resolve(schema, opts) t.deepEquals(schema, originalSchema, 'the param schema should not be changed') @@ -48,33 +35,54 @@ test('$ref to root', t => { }) test('$ref to an external schema', t => { + t.plan(3) + const schema = factory('absoluteId-externalRef') + + const opts = { + externalSchemas: [ + factory('relativeId-noRef') + ] + } + + const resolver = refResolver() + + const out = resolver.resolve(schema, opts) + t.deepEquals(schema, out, 'the output is the same input - modified') + t.ok(out.definitions, 'definitions has been added') + t.deepEquals(Object.values(out.definitions), opts.externalSchemas, 'external schema has been added to definitions') +}) + +test('$ref to an external schema without changes', t => { + t.plan(4) + const schema = factory('absoluteId-externalRef') + + const opts = { + externalSchemas: [ + factory('relativeId-noRef') + ] + } + + const resolver = refResolver({ clone: true }) + + const originalSchema = clone(schema) + const out = resolver.resolve(schema, opts) + t.deepEquals(schema, originalSchema, 'the input is unchanged') + t.notMatch(schema, out, 'the input is unchanged') + t.ok(out.definitions, 'definitions has been added') + t.deepEquals(Object.values(out.definitions), opts.externalSchemas, 'external schema has been added to definitions') +}) + +test('$ref circular', t => { t.plan(3) const schema = { - $id: 'http://example.com/SimplePerson', - type: 'object', - properties: { - name: { type: 'string' }, - address: { $ref: '/SimpleAddress' }, - votes: { type: 'integer', minimum: 1 } - } + $id: 'http://example.com/', + $ref: 'relativePerson' } const opts = { externalSchemas: [ - { - $id: '/SimpleAddress', - type: 'object', - properties: { - lines: { - type: 'array', - items: { type: 'string' } - }, - zip: { type: 'string' }, - city: { type: 'string' }, - country: { type: 'string' } - }, - required: ['country'] - } + factory('relativeId-externalAndLocalRef'), + factory('relativeId-noRef') ] } @@ -85,39 +93,17 @@ test('$ref to an external schema', t => { t.deepEquals(Object.values(out.definitions), opts.externalSchemas, 'external schema has been added to definitions') }) -test('$ref circular', t => { +test('$ref circular', { skip: true }, t => { t.plan(3) const schema = { $id: 'http://example.com/', - $ref: 'SimplePerson' + $ref: 'relativeAddress' } const opts = { externalSchemas: [ - { - $id: 'SimplePerson', - type: 'object', - properties: { - name: { type: 'string' }, - address: { $ref: 'SimpleAddress#' }, - friends: { type: 'array', items: { $ref: '#' } }, - votes: { type: 'integer', minimum: 1 } - } - }, - { - $id: 'SimpleAddress', - type: 'object', - properties: { - lines: { - type: 'array', - items: { type: 'string' } - }, - zip: { type: 'string' }, - city: { type: 'string' }, - country: { type: 'string' } - }, - required: ['country'] - } + factory('relativeId-externalAndLocalRef'), // this is not used + factory('relativeId-noRef') ] } @@ -125,6 +111,5 @@ test('$ref circular', t => { const out = resolver.resolve(schema, opts) t.deepEquals(schema, out, 'the output is the same input modified') t.ok(out.definitions, 'definitions has been added') - require('fs').writeFileSync('./out.json', JSON.stringify(out, null, 2)) - t.deepEquals(Object.values(out.definitions), opts.externalSchemas, 'external schema has been added to definitions') + t.deepEquals(Object.values(out.definitions), opts.externalSchemas[1], 'only used schema are added') }) diff --git a/test/schema-factory.js b/test/schema-factory.js new file mode 100644 index 0000000..579f5c2 --- /dev/null +++ b/test/schema-factory.js @@ -0,0 +1,6 @@ +'use strict' +const clone = require('rfdc')({ proto: true, circles: false }) + +module.exports = function giveMe (whatYouWant) { + return clone(require(`./schemas/${whatYouWant}`)) +} diff --git a/test/schemas/absoluteId-externalRef.js b/test/schemas/absoluteId-externalRef.js new file mode 100644 index 0000000..dcbf9d1 --- /dev/null +++ b/test/schemas/absoluteId-externalRef.js @@ -0,0 +1,10 @@ +module.exports = { + $id: 'http://example.com/SimplePerson', + type: 'object', + properties: { + name: { type: 'string' }, + address: { $ref: 'relativeAddress#' }, + houses: { type: 'array', items: { $ref: 'relativeAddress#' } }, + votes: { type: 'integer', minimum: 1 } + } +} diff --git a/test/schemas/absoluteId-localRef.js b/test/schemas/absoluteId-localRef.js new file mode 100644 index 0000000..b03c192 --- /dev/null +++ b/test/schemas/absoluteId-localRef.js @@ -0,0 +1,19 @@ +module.exports = { + $id: 'http://example.com/root.json', + properties: { + person: { $ref: '#/definitions/person' }, + children: { + type: 'array', + items: { $ref: '#/definitions/person' } + } + }, + definitions: { + person: { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'integer' } + } + } + } +} diff --git a/test/schemas/relativeId-externalAndLocalRef.js b/test/schemas/relativeId-externalAndLocalRef.js new file mode 100644 index 0000000..8633c54 --- /dev/null +++ b/test/schemas/relativeId-externalAndLocalRef.js @@ -0,0 +1,10 @@ +module.exports = { + $id: 'relativePerson', + type: 'object', + properties: { + name: { type: 'string' }, + address: { $ref: 'relativeAddress#' }, + friends: { type: 'array', items: { $ref: '#' } }, + votes: { type: 'integer', minimum: 1 } + } +} diff --git a/test/schemas/relativeId-noRef.js b/test/schemas/relativeId-noRef.js new file mode 100644 index 0000000..163b4cd --- /dev/null +++ b/test/schemas/relativeId-noRef.js @@ -0,0 +1,14 @@ +module.exports = { + $id: 'relativeAddress', + type: 'object', + properties: { + lines: { + type: 'array', + items: { type: 'string' } + }, + zip: { type: 'string' }, + city: { type: 'string' }, + country: { type: 'string' } + }, + required: ['country'] +} From 0c2a61da25bb9b77413fb63246e1d25f7779ba44 Mon Sep 17 00:00:00 2001 From: Manuel Spigolon Date: Sat, 16 May 2020 10:47:55 +0200 Subject: [PATCH 5/7] test and fixes --- ref-resolver.js | 19 +++++---- test/ref-resolver.test.js | 56 ++++++++++++++++++++++++++- test/schemas/absoluteId-asoluteRef.js | 10 +++++ test/schemas/multipleLocalId.js | 17 ++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 test/schemas/absoluteId-asoluteRef.js create mode 100644 test/schemas/multipleLocalId.js diff --git a/ref-resolver.js b/ref-resolver.js index e3ede8f..cee9a96 100644 --- a/ref-resolver.js +++ b/ref-resolver.js @@ -86,6 +86,10 @@ function jsonSchemaResolver (options) { if (ref[0] === '#') { return } const evaluatedJson = allIds.get(baseUri) + if (!evaluatedJson) { + debug('External $ref %s not provided', ref) + return + } evaluatedJson[kConsumed] = true json.$ref = `#/definitions/${evaluatedJson[kRefToDef]}` }) @@ -121,19 +125,20 @@ function jsonSchemaResolver (options) { function collectRefs (json, baseUri, refVal) { const refUri = URI.parse(refVal) debug('Pre enqueue $ref %o', refUri) - if (refUri.reference === 'same-document') { - } else if (refUri.reference !== 'absolute') { + // "same-document"; + // "relative"; + // "absolute"; + // "uri"; + if (refUri.reference === 'relative') { refUri.scheme = baseUri.scheme refUri.userinfo = baseUri.userinfo refUri.host = baseUri.host refUri.port = baseUri.port - if (refUri.reference === 'relative') { - const newBaseUri = Object.assign({}, baseUri) - newBaseUri.path = refUri.path - baseUri = newBaseUri - } + const newBaseUri = Object.assign({}, baseUri) + newBaseUri.path = refUri.path + baseUri = newBaseUri } const ref = URI.serialize(refUri) diff --git a/test/ref-resolver.test.js b/test/ref-resolver.test.js index 1cfa0b4..a99cc15 100644 --- a/test/ref-resolver.test.js +++ b/test/ref-resolver.test.js @@ -93,7 +93,7 @@ test('$ref circular', t => { t.deepEquals(Object.values(out.definitions), opts.externalSchemas, 'external schema has been added to definitions') }) -test('$ref circular', { skip: true }, t => { +test('$ref circular', t => { t.plan(3) const schema = { $id: 'http://example.com/', @@ -111,5 +111,57 @@ test('$ref circular', { skip: true }, t => { const out = resolver.resolve(schema, opts) t.deepEquals(schema, out, 'the output is the same input modified') t.ok(out.definitions, 'definitions has been added') - t.deepEquals(Object.values(out.definitions), opts.externalSchemas[1], 'only used schema are added') + t.deepEquals(Object.values(out.definitions), [opts.externalSchemas[1]], 'only used schema are added') +}) + +test('$ref local ids', { skip: true }, t => { + t.plan(2) + const schema = factory('multipleLocalId') + + const opts = { + externalSchemas: [ + factory('relativeId-externalAndLocalRef'), // this is not used + factory('relativeId-noRef') // this is not used + ] + } + + const resolver = refResolver() + const out = resolver.resolve(schema, opts) + t.deepEquals(schema, out, 'the output is the same input modified') + // TODO build a graph to track is an external schema is referenced by the root + t.equals(Object.values(out.definitions).length, 1, 'no external schema added') +}) + +test('skip duplicated ids', t => { + t.plan(2) + const schema = factory('multipleLocalId') + + const opts = { + externalSchemas: [ + factory('multipleLocalId') + ] + } + + const resolver = refResolver() + const out = resolver.resolve(schema, opts) + t.deepEquals(schema, out, 'the output is the same input modified') + t.equals(Object.values(out.definitions).length, 1, 'no external schema added') +}) + +test('dont resolve external schema missing', t => { + t.plan(1) + const schema = factory('absoluteId-externalRef') + + const resolver = refResolver({ clone: true }) + const out = resolver.resolve(schema) + t.deepEquals(schema, out, 'the output is the same input not modified') +}) + +test('dont resolve external schema missing #2', t => { + t.plan(1) + const schema = factory('absoluteId-asoluteRef') + + const resolver = refResolver({ clone: true }) + const out = resolver.resolve(schema) + t.deepEquals(schema, out, 'the output is the same input not modified') }) diff --git a/test/schemas/absoluteId-asoluteRef.js b/test/schemas/absoluteId-asoluteRef.js new file mode 100644 index 0000000..82b60af --- /dev/null +++ b/test/schemas/absoluteId-asoluteRef.js @@ -0,0 +1,10 @@ +module.exports = { + $id: 'http://example.com/SimplePerson', + type: 'object', + properties: { + name: { type: 'string' }, + address: { $ref: 'http://other-site.com/relativeAddress#' }, + houses: { type: 'array', items: { $ref: 'http://other-site.com/relativeAddress#' } }, + votes: { type: 'integer', minimum: 1 } + } +} diff --git a/test/schemas/multipleLocalId.js b/test/schemas/multipleLocalId.js new file mode 100644 index 0000000..a3b0977 --- /dev/null +++ b/test/schemas/multipleLocalId.js @@ -0,0 +1,17 @@ +module.exports = { + $id: 'http://example.com', + type: 'object', + properties: { + home: { $ref: '#address' }, + work: { $ref: '#address' } + }, + definitions: { + foo: { + $id: '#address', + type: 'object', + properties: { + city: { type: 'string' } + } + } + } +} From 21bdf02afe6074345493205c0df5e4ab0369f779 Mon Sep 17 00:00:00 2001 From: Manuel Spigolon Date: Sat, 16 May 2020 16:54:04 +0200 Subject: [PATCH 6/7] drop node 6 and 8 --- .github/workflows/ci.yml | 2 +- README.md | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 051d95b..c4e7774 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: - node-version: [6.x, 8.x, 10.x, 12.x, 14.x] + node-version: [10.x, 12.x, 14.x] os: [ubuntu-latest, windows-latest] steps: diff --git a/README.md b/README.md index 6cc1adc..54dee1a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Resolve all `$refs` in your [JSON schema](https://json-schema.org/specification. npm install json-schema-resolver ``` -This plugin support Node.js >=6 +This plugin support Node.js >= 10 ## Usage diff --git a/package.json b/package.json index b4c1b33..e917b89 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "test": "npm run lint && tap test/**/*.test.js --cov" }, "engines": { - "node": ">=6" + "node": ">=10" }, "repository": { "type": "git", From 4962847563ae967c71df2c2de65f481b47ff9dc6 Mon Sep 17 00:00:00 2001 From: Manuel Spigolon Date: Sat, 16 May 2020 17:06:05 +0200 Subject: [PATCH 7/7] add docs --- README.md | 67 ++++++++++++++++++++++++++++++++++++++- test/example.test.js | 67 +++++++++++++++++++++++++++++++++++++++ test/ref-resolver.test.js | 22 ++++++------- 3 files changed, 144 insertions(+), 12 deletions(-) create mode 100644 test/example.test.js diff --git a/README.md b/README.md index 54dee1a..8fe2fa3 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,72 @@ This plugin support Node.js >= 10 ## Usage -TODO +```js +const RefResolver = require('json-schema-resolver') + +const ref = RefResolver({ + clone: true // Clone the input schema without changing it. Default: false +}) + +const inputSchema = { + $id: 'http://example.com/SimplePerson', + type: 'object', + properties: { + name: { type: 'string' }, + address: { $ref: 'relativeAddress#' }, + houses: { type: 'array', items: { $ref: 'relativeAddress#' } } + } +} + +const addresSchema = { + $id: 'relativeAddress', // Note: prefer always absolute URI like: http://mysite.com + type: 'object', + properties: { + zip: { type: 'string' }, + city: { type: 'string' } + } +} + +const singleSchema = ref.resolve(inputSchema, { externalSchemas: [addresSchema] }) +// mySchema is untouched thanks to clone:true +``` + +`singleSchema` will be like: + +```json +{ + "$id": "http://example.com/SimplePerson", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "address": { + "$ref": "#/definitions/def-0" + }, + "houses": { + "type": "array", + "items": { + "$ref": "#/definitions/def-0" + } + } + }, + "definitions": { + "def-0": { + "$id": "relativeAddress", + "type": "object", + "properties": { + "zip": { + "type": "string" + }, + "city": { + "type": "string" + } + } + } + } +} +``` ## License diff --git a/test/example.test.js b/test/example.test.js new file mode 100644 index 0000000..d06771a --- /dev/null +++ b/test/example.test.js @@ -0,0 +1,67 @@ +'use strict' + +const { test } = require('tap') + +const RefResolver = require('../ref-resolver') + +test('readme example', t => { + t.plan(1) + + const ref = RefResolver({ + clone: true // Clone the input schema without changing it. Default: false + }) + + const inputSchema = { + $id: 'http://example.com/SimplePerson', + type: 'object', + properties: { + name: { type: 'string' }, + address: { $ref: 'relativeAddress#' }, + houses: { type: 'array', items: { $ref: 'relativeAddress#' } } + } + } + + const addresSchema = { + $id: 'relativeAddress', + type: 'object', + properties: { + zip: { type: 'string' }, + city: { type: 'string' } + } + } + + const singleSchema = ref.resolve(inputSchema, { externalSchemas: [addresSchema] }) + + t.deepEqual(singleSchema, { + $id: 'http://example.com/SimplePerson', + type: 'object', + properties: { + name: { + type: 'string' + }, + address: { + $ref: '#/definitions/def-0' + }, + houses: { + type: 'array', + items: { + $ref: '#/definitions/def-0' + } + } + }, + definitions: { + 'def-0': { + $id: 'relativeAddress', + type: 'object', + properties: { + zip: { + type: 'string' + }, + city: { + type: 'string' + } + } + } + } + }) +}) diff --git a/test/ref-resolver.test.js b/test/ref-resolver.test.js index a99cc15..60f7af3 100644 --- a/test/ref-resolver.test.js +++ b/test/ref-resolver.test.js @@ -3,7 +3,7 @@ const { test } = require('tap') const clone = require('rfdc')({ proto: true, circles: false }) -const refResolver = require('../ref-resolver') +const RefResolver = require('../ref-resolver') const factory = require('./schema-factory') // eslint-disable-next-line @@ -13,7 +13,7 @@ const save = (out) => require('fs').writeFileSync(`./out-${Date.now()}.json`, JS test('wrong params', t => { t.plan(1) - t.throws(() => refResolver({ target: 'draft-1000' })) + t.throws(() => RefResolver({ target: 'draft-1000' })) }) test('$ref to root', t => { @@ -26,7 +26,7 @@ test('$ref to root', t => { factory('relativeId-noRef') ] } - const resolver = refResolver() + const resolver = RefResolver() const originalSchema = clone(schema) const out = resolver.resolve(schema, opts) @@ -44,7 +44,7 @@ test('$ref to an external schema', t => { ] } - const resolver = refResolver() + const resolver = RefResolver() const out = resolver.resolve(schema, opts) t.deepEquals(schema, out, 'the output is the same input - modified') @@ -62,7 +62,7 @@ test('$ref to an external schema without changes', t => { ] } - const resolver = refResolver({ clone: true }) + const resolver = RefResolver({ clone: true }) const originalSchema = clone(schema) const out = resolver.resolve(schema, opts) @@ -86,7 +86,7 @@ test('$ref circular', t => { ] } - const resolver = refResolver() + const resolver = RefResolver() const out = resolver.resolve(schema, opts) t.deepEquals(schema, out, 'the output is the same input modified') t.ok(out.definitions, 'definitions has been added') @@ -107,7 +107,7 @@ test('$ref circular', t => { ] } - const resolver = refResolver() + const resolver = RefResolver() const out = resolver.resolve(schema, opts) t.deepEquals(schema, out, 'the output is the same input modified') t.ok(out.definitions, 'definitions has been added') @@ -125,7 +125,7 @@ test('$ref local ids', { skip: true }, t => { ] } - const resolver = refResolver() + const resolver = RefResolver() const out = resolver.resolve(schema, opts) t.deepEquals(schema, out, 'the output is the same input modified') // TODO build a graph to track is an external schema is referenced by the root @@ -142,7 +142,7 @@ test('skip duplicated ids', t => { ] } - const resolver = refResolver() + const resolver = RefResolver() const out = resolver.resolve(schema, opts) t.deepEquals(schema, out, 'the output is the same input modified') t.equals(Object.values(out.definitions).length, 1, 'no external schema added') @@ -152,7 +152,7 @@ test('dont resolve external schema missing', t => { t.plan(1) const schema = factory('absoluteId-externalRef') - const resolver = refResolver({ clone: true }) + const resolver = RefResolver({ clone: true }) const out = resolver.resolve(schema) t.deepEquals(schema, out, 'the output is the same input not modified') }) @@ -161,7 +161,7 @@ test('dont resolve external schema missing #2', t => { t.plan(1) const schema = factory('absoluteId-asoluteRef') - const resolver = refResolver({ clone: true }) + const resolver = RefResolver({ clone: true }) const out = resolver.resolve(schema) t.deepEquals(schema, out, 'the output is the same input not modified') })