Dump custom tags starting with ! as !tag instead of !<!tag>

Author: rlidwka

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `dump()` now serializes `undefined` as `null` in collections and removes keys with
   `undefined` in mappings, #571.
 - `dump()` with `skipInvalid=true` now serializes invalid items in collections as null.
+- Custom tags starting with `!` are now dumped as `!tag` instead of `!<!tag>`, #576.
 
 ### Added
 - Added `.mjs` (es modules) support.
@@ -37,12 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Custom `Tag` can now handle all tags or multiple tags with the same prefix, #385.
 
 ### Fixed
-- Astral characters are no longer encoded by dump/safeDump, #587.
+- Astral characters are no longer encoded by `dump()`, #587.
 - "duplicate mapping key" exception now points at the correct column, #452.
 - Extra commas in flow collections (e.g. `[foo,,bar]`) now throw an exception
   instead of producing null, #321.
 - `__proto__` key no longer overrides object prototype, #164.
 - Removed `bower.json`.
+- Tags are now url-decoded in `load()` and url-encoded in `dump()`
+  (previously usage of custom non-ascii tags may have led to invalid YAML that can't be parsed).
 
 
 ## [3.14.1] - 2020-12-07

examples/handle_unknown_types.js

@@ -6,13 +6,28 @@ const util = require('util');
 const yaml = require('../');
 
 
+class CustomTag {
+  constructor(type, data) {
+    this.type = type;
+    this.data = data;
+  }
+}
+
+
 const tags = [ 'scalar', 'sequence', 'mapping' ].map(function (kind) {
   // first argument here is a prefix, so this type will handle anything starting with !
   return new yaml.Type('!', {
     kind: kind,
     multi: true,
+    representName: function (object) {
+      return object.type;
+    },
+    represent: function (object) {
+      return object.data;
+    },
+    instanceOf: CustomTag,
     construct: function (data, type) {
-      return { type: type, data: data };
+      return new CustomTag(type, data);
     }
   });
 });
@@ -21,10 +36,19 @@ const SCHEMA = yaml.DEFAULT_SCHEMA.extend(tags);
 
 const data = `
 subject: Handling unknown types in JS-YAML
-scalar: !unknown_scalar_tag 123
+scalar: !unknown_scalar_tag foo bar
 sequence: !unknown_sequence_tag [ 1, 2, 3 ]
 mapping: !unknown_mapping_tag { foo: 1, bar: 2 }
 `;
 
 const loaded = yaml.load(data, { schema: SCHEMA });
+
+console.log('Parsed as:');
+console.log('-'.repeat(70));
 console.log(util.inspect(loaded, false, 20, true));
+
+console.log('');
+console.log('');
+console.log('Dumped as:');
+console.log('-'.repeat(70));
+console.log(yaml.dump(loaded, { schema: SCHEMA }));

lib/dumper.js

@@ -760,7 +760,15 @@ function detectType(state, object, explicit) {
         (!type.instanceOf || ((typeof object === 'object') && (object instanceof type.instanceOf))) &&
         (!type.predicate  || type.predicate(object))) {
 
-      state.tag = explicit ? type.tag : '?';
+      if (explicit) {
+        if (type.multi && type.representName) {
+          state.tag = type.representName(object);
+        } else {
+          state.tag = type.tag;
+        }
+      } else {
+        state.tag = '?';
+      }
 
       if (type.represent) {
         style = state.styleMap[type.tag] || type.defaultStyle;
@@ -796,6 +804,7 @@ function writeNode(state, level, object, block, compact, iskey, isblockseq) {
 
   var type = _toString.call(state.dump);
   var inblock = block;
+  var tagStr;
 
   if (block) {
     block = (state.flowLevel < 0 || state.flowLevel > level);
@@ -860,7 +869,30 @@ function writeNode(state, level, object, block, compact, iskey, isblockseq) {
     }
 
     if (state.tag !== null && state.tag !== '?') {
-      state.dump = '!<' + state.tag + '> ' + state.dump;
+      // Need to encode all characters except those allowed by the spec:
+      //
+      // [35] ns-dec-digit    ::=  [#x30-#x39] /* 0-9 */
+      // [36] ns-hex-digit    ::=  ns-dec-digit
+      //                         | [#x41-#x46] /* A-F */ | [#x61-#x66] /* a-f */
+      // [37] ns-ascii-letter ::=  [#x41-#x5A] /* A-Z */ | [#x61-#x7A] /* a-z */
+      // [38] ns-word-char    ::=  ns-dec-digit | ns-ascii-letter | “-”
+      // [39] ns-uri-char     ::=  “%” ns-hex-digit ns-hex-digit | ns-word-char | “#”
+      //                         | “;” | “/” | “?” | “:” | “@” | “&” | “=” | “+” | “$” | “,”
+      //                         | “_” | “.” | “!” | “~” | “*” | “'” | “(” | “)” | “[” | “]”
+      //
+      // Also need to encode '!' because it has special meaning (end of tag prefix).
+      //
+      tagStr = encodeURI(
+        state.tag[0] === '!' ? state.tag.slice(1) : state.tag
+      ).replace(/!/g, '%21');
+
+      if (state.tag[0] === '!') {
+        tagStr = '!' + tagStr;
+      } else {
+        tagStr = '!<' + tagStr + '>';
+      }
+
+      state.dump = tagStr + ' ' + state.dump;
     }
   }
 

lib/loader.js

@@ -248,6 +248,12 @@ var directiveHandlers = {
       throwError(state, 'ill-formed tag prefix (second argument) of the TAG directive');
     }
 
+    try {
+      prefix = decodeURIComponent(prefix);
+    } catch (err) {
+      throwError(state, 'tag prefix is malformed: ' + prefix);
+    }
+
     state.tagMap[handle] = prefix;
   }
 };
@@ -1249,6 +1255,12 @@ function readTagProperty(state) {
     throwError(state, 'tag name cannot contain such characters: ' + tagName);
   }
 
+  try {
+    tagName = decodeURIComponent(tagName);
+  } catch (err) {
+    throwError(state, 'tag name is malformed: ' + tagName);
+  }
+
   if (isVerbatim) {
     state.tag = tagName;
 

lib/type.js

@@ -10,6 +10,7 @@ var TYPE_CONSTRUCTOR_OPTIONS = [
   'instanceOf',
   'predicate',
   'represent',
+  'representName',
   'defaultStyle',
   'styleAliases'
 ];
@@ -44,16 +45,17 @@ function Type(tag, options) {
   });
 
   // TODO: Add tag format check.
-  this.tag          = tag;
-  this.kind         = options['kind']         || null;
-  this.resolve      = options['resolve']      || function () { return true; };
-  this.construct    = options['construct']    || function (data) { return data; };
-  this.instanceOf   = options['instanceOf']   || null;
-  this.predicate    = options['predicate']    || null;
-  this.represent    = options['represent']    || null;
-  this.defaultStyle = options['defaultStyle'] || null;
-  this.multi        = options['multi']        || false;
-  this.styleAliases = compileStyleAliases(options['styleAliases'] || null);
+  this.tag           = tag;
+  this.kind          = options['kind']          || null;
+  this.resolve       = options['resolve']       || function () { return true; };
+  this.construct     = options['construct']     || function (data) { return data; };
+  this.instanceOf    = options['instanceOf']    || null;
+  this.predicate     = options['predicate']     || null;
+  this.represent     = options['represent']     || null;
+  this.representName = options['representName'] || null;
+  this.defaultStyle  = options['defaultStyle']  || null;
+  this.multi         = options['multi']         || false;
+  this.styleAliases  = compileStyleAliases(options['styleAliases'] || null);
 
   if (YAML_NODE_KINDS.indexOf(this.kind) === -1) {
     throw new YAMLException('Unknown kind "' + this.kind + '" is specified for "' + tag + '" YAML type.');

test/issues/0385.js

@@ -96,4 +96,29 @@ describe('Multi tag', function () {
       schema: schema
     }), expected);
   });
+
+
+  it('should dump multi types with custom tag', function () {
+    let tags = [
+      new yaml.Type('!', {
+        kind: 'scalar',
+        multi: true,
+        predicate: function (obj) {
+          return !!obj.tag;
+        },
+        representName: function (obj) {
+          return obj.tag;
+        },
+        represent: function (obj) {
+          return obj.value;
+        }
+      })
+    ];
+
+    let schema = yaml.DEFAULT_SCHEMA.extend(tags);
+
+    assert.strictEqual(yaml.dump({ test: { tag: 'foo', value: 'bar' } }, {
+      schema: schema
+    }), 'test: !<foo> bar\n');
+  });
 });

test/issues/0576.js

@@ -0,0 +1,53 @@
+'use strict';
+
+
+const assert = require('assert');
+const yaml   = require('../../');
+
+
+describe('Custom tags', function () {
+  let tag_names = [ 'tag', '!tag', '!!tag', '!<!tag>', 'tag*-!< >{\n}', '!tagαβγ' ];
+  let encoded   = [ '!<tag>', '!tag', '!%21tag', '!%3C%21tag%3E',
+    '!<tag*-%21%3C%20%3E%7B%0A%7D>', '!tag%CE%B1%CE%B2%CE%B3' ];
+
+  let tags = tag_names.map(tag =>
+    new yaml.Type(tag, {
+      kind: 'scalar',
+      resolve: () => true,
+      construct: object => [ tag, object ],
+      predicate: object => object.tag === tag,
+      represent: () => 'value'
+    })
+  );
+
+  let schema = yaml.DEFAULT_SCHEMA.extend(tags);
+
+
+  it('Should dump tags with proper encoding', function () {
+    tag_names.forEach(function (tag, idx) {
+      assert.strictEqual(yaml.dump({ tag }, { schema }), encoded[idx] + ' value\n');
+    });
+  });
+
+
+  it('Should decode tags when loading', function () {
+    encoded.forEach(function (tag, idx) {
+      assert.deepStrictEqual(yaml.load(tag + ' value', { schema }), [ tag_names[idx], 'value' ]);
+    });
+  });
+
+
+  it('Should url-decode built-in tags', function () {
+    assert.strictEqual(yaml.load('!!%69nt 123'), 123);
+    assert.strictEqual(yaml.load('!!%73tr 123'), '123');
+  });
+
+
+  it('Should url-decode %TAG prefix', function () {
+    assert.deepStrictEqual(yaml.load(`
+%TAG !xx! %74a
+---
+!xx!g 123
+`, { schema }), [ 'tag', '123' ]);
+  });
+});