Merge branch 'master' into pep561

Author: chrisjsewell

.github/workflows/tests.yml

@@ -40,7 +40,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install .[testing]
+        pip install .[testing,linkify]
     - name: Run pytest
       run: |
         pytest --cov=markdown_it --cov-report=xml --cov-report=term-missing

.mypy.ini

@@ -0,0 +1,5 @@
+[mypy]
+warn_unused_ignores = True
+warn_redundant_casts = True
+no_implicit_optional = True
+strict_equality = True

.pre-commit-config.yaml

@@ -15,24 +15,30 @@ exclude: >
 repos:
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+    rev: v3.3.0
     hooks:
     - id: check-json
     - id: check-yaml
     - id: end-of-file-fixer
     - id: trailing-whitespace
 
   - repo: https://github.com/mgedmin/check-manifest
-    rev: "0.42"
+    rev: "0.44"
     hooks:
     - id: check-manifest
 
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.8.4
     hooks:
     - id: flake8
 
   - repo: https://github.com/psf/black
     rev: 20.8b1
     hooks:
     - id: black
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.790
+    hooks:
+    - id: mypy
+      additional_dependencies: [attrs]

CHANGELOG.md

@@ -1,5 +1,68 @@
 # Change Log
 
+## 0.5.8 - 2020-12-13
+
+✨ NEW: Add linkify, thanks to [@tsutsu3](https://github.com/tsutsu3).
+
+This extension uses [linkify-it-py](https://github.com/tsutsu3/linkify-it-py) to identify URL links within text:
+
+- `github.com` -> `<a href="http://github.com">github.com</a>`
+
+**Important:** To use this extension you must install linkify-it-py; `pip install markdown-it-py[linkify]`
+
+It can then be activated by:
+
+```python
+from markdown_it import MarkdownIt
+md = MarkdownIt().enable("linkify")
+md.options["linkify"] = True
+```
+
+## 0.5.7 - 2020-12-13
+
+✨ NEW: Add smartquotes, thanks to [@tsutsu3](https://github.com/tsutsu3).
+
+This extension will convert basic quote marks to their opening and closing variants:
+
+- 'single quotes' -> ‘single quotes’
+- "double quotes" -> “double quotes”
+
+It can be activated by:
+
+```python
+from markdown_it import MarkdownIt
+md = MarkdownIt().enable("replacements").enable("smartquotes")
+```
+
+✨ NEW: Add markdown-it-task-lists plugin, thanks to [@wna-se](https://github.com/wna-se).
+
+This is a port of the JS [markdown-it-task-lists](https://github.com/revin/markdown-it-task-lists),
+for building task/todo lists out of markdown lists with items starting with `[ ]` or `[x]`.
+For example:
+
+```markdown
+- [ ] An item that needs doing
+- [x] An item that is complete
+```
+
+This plugin can be activated by:
+
+```python
+from markdown_it import MarkdownIt
+from markdown_it.extensions.tasklists import tasklists_plugin
+md = MarkdownIt().use(tasklists_plugin)
+```
+
+🐛 Various bug fixes, thanks to [@hukkinj1](https://github.com/hukkinj1):
+
+- Do not copy empty `env` arg in `MarkdownIt.render`
+- `_Entities.__contains__` fix return data
+- Parsing of unicode ordinals
+- Handling of final character in `skipSpacesBack` and `skipCharsBack` methods
+- Avoid exception when document ends in heading/blockquote marker
+
+🧪 TESTS: Add CI for Python 3.9 and PyPy3
+
 ## 0.5.6 - 2020-10-21
 
 - ✨ NEW: Add simple typographic replacements, thanks to [@tsutsu3](https://github.com/tsutsu3):
@@ -20,7 +83,7 @@
   - ``---`` → &mdash
 
   ```python
-  md = MarkdownIt()
+  md = MarkdownIt().enable("replacements")
   md.options["typographer"] = True
   ```
 

MANIFEST.in

@@ -13,6 +13,7 @@ exclude .flake8
 exclude .circleci
 exclude .circleci/config.yml
 exclude codecov.yml
+exclude .mypy.ini
 
 include LICENSE
 include LICENSE.markdown-it

docs/conf.py

@@ -45,7 +45,7 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-nitpick_ignore = [("py:class", "Match")]
+nitpick_ignore = [("py:class", "Match"), ("py:class", "x in the interval [0, 1).")]
 
 
 # -- Options for HTML output -------------------------------------------------

docs/plugins.md

@@ -40,6 +40,9 @@ html_string = md.render("some *Markdown*")
 
 .. autofunction:: markdown_it.extensions.amsmath.amsmath_plugin
     :noindex:
+
+.. autofunction:: markdown_it.extensions.tasklists.tasklists_plugin
+    :noindex:
 ```
 
 `myst_blocks` and `myst_role` plugins are also available, for utilisation by the [MyST renderer](https://myst-parser.readthedocs.io/en/latest/using/syntax.html)

markdown_it/__init__.py

@@ -1,4 +1,4 @@
 from .main import MarkdownIt  # noqa: F401
 
 
-__version__ = "0.5.6"
+__version__ = "0.5.8"

markdown_it/cli/parse.py

@@ -40,13 +40,11 @@ def convert_file(filename):
         sys.exit('Cannot open file "{}".'.format(filename))
 
 
-def interactive(import_readline=False):
+def interactive():
     """
     Parse user input, dump to stdout, rinse and repeat.
     Python REPL style.
     """
-    if import_readline:
-        _import_readline()
     print_heading()
     contents = []
     more = False
@@ -98,13 +96,6 @@ def parse_args(args):
     return parser.parse_args(args)
 
 
-def _import_readline():
-    try:
-        import readline  # noqa: F401
-    except ImportError:
-        print("[warning] readline library not available.")
-
-
 def print_heading():
     print("{} (interactive)".format(version_str))
     print("Type Ctrl-D to complete input, or Ctrl-C to exit.")

markdown_it/common/entities.py

@@ -1,5 +1,5 @@
 """HTML5 entities map: { name -> characters }."""
-import html
+import html.entities
 
 
 class _Entities:

markdown_it/common/normalize_url.py

@@ -1,6 +1,6 @@
 import html
 import re
-from typing import Callable
+from typing import Callable, Optional
 from urllib.parse import urlparse, urlunparse, quote, unquote  # noqa: F401
 
 from .utils import ESCAPABLE
@@ -166,7 +166,7 @@ def normalizeLinkText(link):
 GOOD_DATA_RE = re.compile(r"^data:image\/(gif|png|jpeg|webp);")
 
 
-def validateLink(url: str, validator: Callable = None):
+def validateLink(url: str, validator: Optional[Callable] = None):
     """Validate URL link is allowed in output.
 
     This validator can prohibit more than really needed to prevent XSS.
@@ -180,8 +180,4 @@ def validateLink(url: str, validator: Callable = None):
     if validator is not None:
         return validator(url)
     url = url.strip().lower()
-    return (
-        (True if GOOD_DATA_RE.search(url) else False)
-        if BAD_PROTO_RE.search(url)
-        else True
-    )
+    return bool(GOOD_DATA_RE.search(url)) if BAD_PROTO_RE.search(url) else True

markdown_it/common/utils.py

@@ -53,7 +53,7 @@ def assign(obj):
     #   })
     # })
 
-    return obj
+    # return obj
 
 
 def arrayReplaceAt(src: list, pos: int, newElements: list):
@@ -92,15 +92,13 @@ def isValidEntityCode(c):
     return True
 
 
-def fromCodePoint(c):
-
-    if c > 0xFFFF:
-        c -= 0x10000
-        surrogate1 = 0xD800 + (c >> 10)
-        surrogate2 = 0xDC00 + (c & 0x3FF)
-
-        return "".join(map(chr, [surrogate1, surrogate2]))
+def fromCodePoint(c: int) -> str:
+    """Convert ordinal to unicode.
 
+    Note, in the original Javascript two string characters were required,
+    for codepoints larger than `0xFFFF`.
+    But Python 3 can represent any unicode codepoint in one character.
+    """
     return chr(c)
 
 
@@ -141,9 +139,10 @@ def replaceEntityPattern(match, name):
 
 
 def unescapeMd(string: str):
-    if "\\" in string:
-        return string
-    return string.replace(UNESCAPE_MD_RE, "$1")
+    raise NotImplementedError
+    # if "\\" in string:
+    #     return string
+    # return string.replace(UNESCAPE_MD_RE, "$1")
 
 
 def unescapeAll(string: str):
@@ -272,7 +271,7 @@ def isPunctChar(ch):
 }
 
 
-def isMdAsciiPunct(ch: str):
+def isMdAsciiPunct(ch: int):
     """Markdown ASCII punctuation characters.
 
     ::

markdown_it/extensions/anchors/index.py

@@ -1,5 +1,5 @@
 import re
-from typing import Callable, List, Optional
+from typing import Callable, List, Optional, Set
 
 from markdown_it import MarkdownIt
 from markdown_it.rules_core import StateCore
@@ -65,19 +65,20 @@ def _make_anchors_func(
     permalinkBefore: bool,
     permalinkSpace: bool,
 ):
-    slugs = set()
+    slugs: Set[str] = set()
 
     def _anchor_func(state: StateCore):
         for (idx, token) in enumerate(state.tokens):
-            token: Token
             if token.type != "heading_open":
                 continue
             level = int(token.tag[1])
             if level not in selected_levels:
                 continue
+            inline_token = state.tokens[idx + 1]
+            assert inline_token.children is not None
             title = "".join(
                 child.content
-                for child in state.tokens[idx + 1].children
+                for child in inline_token.children
                 if child.type in ["text", "code_inline"]
             )
             slug = unique_slug(slug_func(title), slugs)
@@ -95,17 +96,17 @@ def _anchor_func(state: StateCore):
                     Token("link_close", "a", -1),
                 ]
                 if permalinkBefore:
-                    state.tokens[idx + 1].children = (
+                    inline_token.children = (
                         link_tokens
                         + (
                             [Token("text", "", 0, content=" ")]
                             if permalinkSpace
                             else []
                         )
-                        + state.tokens[idx + 1].children
+                        + inline_token.children
                     )
                 else:
-                    state.tokens[idx + 1].children.extend(
+                    inline_token.children.extend(
                         ([Token("text", "", 0, content=" ")] if permalinkSpace else [])
                         + link_tokens
                     )