From 50cef5f3a6686d06b363bb5c56c9bff60f75ba3d Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 22 Oct 2020 01:06:04 +0300 Subject: [PATCH 01/12] Add mypy. Fix typing errors --- .mypy.ini | 3 +++ .pre-commit-config.yaml | 12 +++++++++--- markdown_it/common/utils.py | 9 +++++---- markdown_it/extensions/anchors/index.py | 15 ++++++++------- markdown_it/extensions/container/index.py | 1 + markdown_it/extensions/deflist/index.py | 2 +- markdown_it/extensions/footnote/index.py | 8 +++++--- markdown_it/main.py | 8 ++++---- markdown_it/renderer.py | 4 +++- markdown_it/ruler.py | 7 +++++-- markdown_it/rules_block/html_block.py | 22 +++++++++++----------- markdown_it/rules_core/replacements.py | 1 + markdown_it/rules_inline/autolink.py | 4 ++-- markdown_it/rules_inline/balance_pairs.py | 5 +++-- markdown_it/rules_inline/image.py | 6 ++++-- markdown_it/rules_inline/state_inline.py | 8 ++++---- markdown_it/rules_inline/strikethrough.py | 6 +++--- markdown_it/token.py | 16 +++++++++------- markdown_it/utils.py | 21 +++++++++++++-------- 19 files changed, 94 insertions(+), 64 deletions(-) create mode 100644 .mypy.ini diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 00000000..f10e639b --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,3 @@ +[mypy] +warn_unused_ignores = True +warn_redundant_casts = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a426013..0a91d16f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ exclude: > repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v3.3.0 hooks: - id: check-json - id: check-yaml @@ -23,12 +23,12 @@ repos: - id: trailing-whitespace - repo: https://github.com/mgedmin/check-manifest - rev: "0.42" + rev: "0.44" hooks: - id: check-manifest - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 @@ -36,3 +36,9 @@ repos: rev: 20.8b1 hooks: - id: black + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.790 + hooks: + - id: mypy + additional_dependencies: [attrs] diff --git a/markdown_it/common/utils.py b/markdown_it/common/utils.py index 1dcabc3c..8ff69e1c 100644 --- a/markdown_it/common/utils.py +++ b/markdown_it/common/utils.py @@ -53,7 +53,7 @@ def assign(obj): # }) # }) - return obj + # return obj def arrayReplaceAt(src: list, pos: int, newElements: list): @@ -139,9 +139,10 @@ def replaceEntityPattern(match, name): def unescapeMd(string: str): - if "\\" in string: - return string - return string.replace(UNESCAPE_MD_RE, "$1") + raise NotImplementedError + # if "\\" in string: + # return string + # return string.replace(UNESCAPE_MD_RE, "$1") def unescapeAll(string: str): diff --git a/markdown_it/extensions/anchors/index.py b/markdown_it/extensions/anchors/index.py index 1cfd735f..c89056ee 100644 --- a/markdown_it/extensions/anchors/index.py +++ b/markdown_it/extensions/anchors/index.py @@ -1,5 +1,5 @@ import re -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Set from markdown_it import MarkdownIt from markdown_it.rules_core import StateCore @@ -65,19 +65,20 @@ def _make_anchors_func( permalinkBefore: bool, permalinkSpace: bool, ): - slugs = set() + slugs: Set[str] = set() def _anchor_func(state: StateCore): for (idx, token) in enumerate(state.tokens): - token: Token if token.type != "heading_open": continue level = int(token.tag[1]) if level not in selected_levels: continue + inline_token = state.tokens[idx + 1] + assert inline_token.children is not None title = "".join( child.content - for child in state.tokens[idx + 1].children + for child in inline_token.children if child.type in ["text", "code_inline"] ) slug = unique_slug(slug_func(title), slugs) @@ -95,17 +96,17 @@ def _anchor_func(state: StateCore): Token("link_close", "a", -1), ] if permalinkBefore: - state.tokens[idx + 1].children = ( + inline_token.children = ( link_tokens + ( [Token("text", "", 0, content=" ")] if permalinkSpace else [] ) - + state.tokens[idx + 1].children + + inline_token.children ) else: - state.tokens[idx + 1].children.extend( + inline_token.children.extend( ([Token("text", "", 0, content=" ")] if permalinkSpace else []) + link_tokens ) diff --git a/markdown_it/extensions/container/index.py b/markdown_it/extensions/container/index.py index 6be8673d..8a31365d 100644 --- a/markdown_it/extensions/container/index.py +++ b/markdown_it/extensions/container/index.py @@ -80,6 +80,7 @@ def container_func(state: StateBlock, startLine: int, endLine: int, silent: bool markup = state.src[start:pos] params = state.src[pos:maximum] + assert validate is not None if not validate(params, markup): return False diff --git a/markdown_it/extensions/deflist/index.py b/markdown_it/extensions/deflist/index.py index 766729c3..0b353db6 100644 --- a/markdown_it/extensions/deflist/index.py +++ b/markdown_it/extensions/deflist/index.py @@ -22,7 +22,7 @@ def deflist_plugin(md: MarkdownIt): ~ Definition 2b """ - isSpace = md.utils.isSpace + isSpace = md.utils.isSpace # type: ignore def skipMarker(state: StateBlock, line: int): """Search `[:~][\n ]`, returns next pos after marker on success or -1 on fail.""" diff --git a/markdown_it/extensions/footnote/index.py b/markdown_it/extensions/footnote/index.py index 0071c8d7..8f0f46a8 100644 --- a/markdown_it/extensions/footnote/index.py +++ b/markdown_it/extensions/footnote/index.py @@ -1,6 +1,8 @@ # Process footnotes # +from typing import List, Optional + from markdown_it import MarkdownIt from markdown_it.token import Token from markdown_it.rules_inline import StateInline @@ -174,7 +176,7 @@ def footnote_inline(state: StateInline, silent: bool): refs = state.env.setdefault("footnotes", {}).setdefault("list", {}) footnoteId = len(refs) - tokens = [] + tokens: List[Token] = [] state.md.inline.parse( state.src[labelStart:labelEnd], state.md, state.env, tokens ) @@ -260,7 +262,7 @@ def footnote_tail(state: StateBlock, *args, **kwargs): if "footnotes" not in state.env: return - current = [] + current: List[Token] = [] tok_filter = [] for tok in state.tokens: @@ -320,7 +322,7 @@ def footnote_tail(state: StateBlock, *args, **kwargs): state.tokens.extend(tokens) if state.tokens[len(state.tokens) - 1].type == "paragraph_close": - lastParagraph = state.tokens.pop() + lastParagraph: Optional[Token] = state.tokens.pop() else: lastParagraph = None diff --git a/markdown_it/main.py b/markdown_it/main.py index 5e8fed95..6076aa1b 100644 --- a/markdown_it/main.py +++ b/markdown_it/main.py @@ -43,7 +43,7 @@ def __init__( self.utils = utils self.helpers = helpers - self.options = {} + self.options: dict = {} self.configure(config) self.linkify = linkify_it.LinkifyIt() if linkify_it else None @@ -193,7 +193,7 @@ def add_render_rule(self, name: str, function: Callable, fmt="html"): Only applied when ``renderer.__output__ == fmt`` """ if self.renderer.__output__ == fmt: - self.renderer.rules[name] = function.__get__(self.renderer) + self.renderer.rules[name] = function.__get__(self.renderer) # type: ignore def use(self, plugin: Callable, *params, **options) -> "MarkdownIt": """Load specified plugin with given params into current parser instance. (chainable) @@ -225,7 +225,7 @@ def parse(self, src: str, env: Optional[AttrDict] = None) -> List[Token]: and then pass updated object to renderer. """ env = AttrDict() if env is None else env - if not isinstance(env, AttrDict): + if not isinstance(env, AttrDict): # type: ignore raise TypeError(f"Input data should be an AttrDict, not {type(env)}") if not isinstance(src, str): raise TypeError(f"Input data should be a string, not {type(src)}") @@ -259,7 +259,7 @@ def parseInline(self, src: str, env: Optional[AttrDict] = None) -> List[Token]: tokens in `children` property. Also updates `env` object. """ env = AttrDict() if env is None else env - if not isinstance(env, AttrDict): + if not isinstance(env, AttrDict): # type: ignore raise TypeError(f"Input data should be an AttrDict, not {type(env)}") if not isinstance(src, str): raise TypeError(f"Input data should be a string, not {type(src)}") diff --git a/markdown_it/renderer.py b/markdown_it/renderer.py index 2a65913c..281364a4 100644 --- a/markdown_it/renderer.py +++ b/markdown_it/renderer.py @@ -64,6 +64,7 @@ def render(self, tokens: List[Token], options, env) -> str: for i, token in enumerate(tokens): if token.type == "inline": + assert token.children is not None result += self.renderInline(token.children, options, env) elif token.type in self.rules: result += self.rules[token.type](tokens, i, options, env) @@ -124,7 +125,7 @@ def renderToken( result += self.renderAttrs(token) # Add a slash for self-closing tags, e.g. ` str: if token.type == "text": result += token.content elif token.type == "image": + assert token.children is not None result += self.renderInlineAsText(token.children, options, env) return result diff --git a/markdown_it/ruler.py b/markdown_it/ruler.py index dad4ea50..3c0018b2 100644 --- a/markdown_it/ruler.py +++ b/markdown_it/ruler.py @@ -26,7 +26,9 @@ def __init__(self, src: str, md, env): self.md = md -RuleFunc = Callable[[StateBase], None] +# The first positional arg is always a `StateBase`. Other arguments are +# optional. +RuleFunc = Callable[..., None] @attr.s(slots=True) @@ -44,7 +46,7 @@ def __init__(self): # Cached rule chains. # First level - chain name, '' for default. # Second level - diginal anchor for fast filtering by charcodes. - self.__cache__: Optional[Dict[str, RuleFunc]] = None + self.__cache__: Optional[Dict[str, List[RuleFunc]]] = None def __find__(self, name: str) -> int: """Find rule index by name""" @@ -201,6 +203,7 @@ def getRules(self, chainName: str) -> List[RuleFunc]: """ if self.__cache__ is None: self.__compile__() + assert self.__cache__ is not None # Chain can be empty, if rules disabled. But we still have to return Array. return self.__cache__.get(chainName, []) or [] diff --git a/markdown_it/rules_block/html_block.py b/markdown_it/rules_block/html_block.py index 3cac032d..157f31ba 100644 --- a/markdown_it/rules_block/html_block.py +++ b/markdown_it/rules_block/html_block.py @@ -1,6 +1,7 @@ # HTML block import logging import re +from typing import List, Tuple, Pattern from .state_block import StateBlock from ..common.html_blocks import block_names @@ -10,23 +11,22 @@ # An array of opening and corresponding closing sequences for html tags, # last argument defines whether it can terminate a paragraph or not -# -HTML_SEQUENCES = [ - [ +HTML_SEQUENCES: List[Tuple[Pattern, Pattern, bool]] = [ + ( re.compile(r"^<(script|pre|style)(?=(\s|>|$))", re.IGNORECASE), re.compile(r"<\/(script|pre|style)>", re.IGNORECASE), True, - ], - [re.compile(r"^"), True], - [re.compile(r"^<\?"), re.compile(r"\?>"), True], - [re.compile(r"^"), True], - [re.compile(r"^"), True], - [ + ), + (re.compile(r"^"), True), + (re.compile(r"^<\?"), re.compile(r"\?>"), True), + (re.compile(r"^"), True), + (re.compile(r"^"), True), + ( re.compile("^|$))", re.IGNORECASE), re.compile(r"^$"), True, - ], - [re.compile(HTML_OPEN_CLOSE_TAG_STR + "\\s*$"), re.compile(r"^$"), False], + ), + (re.compile(HTML_OPEN_CLOSE_TAG_STR + "\\s*$"), re.compile(r"^$"), False), ] diff --git a/markdown_it/rules_core/replacements.py b/markdown_it/rules_core/replacements.py index 86e3c3c4..5a135277 100644 --- a/markdown_it/rules_core/replacements.py +++ b/markdown_it/rules_core/replacements.py @@ -115,6 +115,7 @@ def replace(state: StateCore): for token in state.tokens: if token.type != "inline": continue + assert token.children is not None if SCOPED_ABBR_RE.search(token.content): replace_scoped(token.children) diff --git a/markdown_it/rules_inline/autolink.py b/markdown_it/rules_inline/autolink.py index c1f8d0f1..59444092 100644 --- a/markdown_it/rules_inline/autolink.py +++ b/markdown_it/rules_inline/autolink.py @@ -1,6 +1,6 @@ # Process autolinks '' import re -from .state_inline import StateBase +from .state_inline import StateInline from ..common.normalize_url import normalizeLinkText, normalizeLink, validateLink EMAIL_RE = re.compile( @@ -9,7 +9,7 @@ AUTOLINK_RE = re.compile(r"^<([a-zA-Z][a-zA-Z0-9+.\-]{1,31}):([^<>\x00-\x20]*)>") -def autolink(state: StateBase, silent: bool): +def autolink(state: StateInline, silent: bool): pos = state.pos diff --git a/markdown_it/rules_inline/balance_pairs.py b/markdown_it/rules_inline/balance_pairs.py index 4ef8ce4d..643461b2 100644 --- a/markdown_it/rules_inline/balance_pairs.py +++ b/markdown_it/rules_inline/balance_pairs.py @@ -99,6 +99,7 @@ def link_pairs(state: StateInline): curr = 0 while curr < maximum: - if tokens_meta[curr] and "delimiters" in tokens_meta[curr]: - processDelimiters(state, tokens_meta[curr]["delimiters"]) + curr_meta = tokens_meta[curr] + if curr_meta and "delimiters" in curr_meta: + processDelimiters(state, curr_meta["delimiters"]) curr += 1 diff --git a/markdown_it/rules_inline/image.py b/markdown_it/rules_inline/image.py index 3a8d6fc3..4bba0bdf 100644 --- a/markdown_it/rules_inline/image.py +++ b/markdown_it/rules_inline/image.py @@ -1,13 +1,16 @@ # Process ![image]( "title") +from typing import List + from .state_inline import StateInline +from ..token import Token from ..common.utils import isSpace, normalizeReference from ..common.normalize_url import normalizeLink, validateLink def image(state: StateInline, silent: bool): - tokens = None + tokens: List[Token] = [] label = None href = "" oldPos = state.pos @@ -131,7 +134,6 @@ def image(state: StateInline, silent: bool): if not silent: content = state.src[labelStart:labelEnd] - tokens = [] state.md.inline.parse(content, state.md, state.env, tokens) token = state.push("image", "img", 0) diff --git a/markdown_it/rules_inline/state_inline.py b/markdown_it/rules_inline/state_inline.py index ff43d0ce..8b829253 100644 --- a/markdown_it/rules_inline/state_inline.py +++ b/markdown_it/rules_inline/state_inline.py @@ -1,5 +1,5 @@ from collections import namedtuple -from typing import List +from typing import Dict, List, Optional import attr @@ -49,7 +49,7 @@ def __init__(self, src: str, md, env, outTokens: List[Token]): self.env = env self.md = md self.tokens = outTokens - self.tokens_meta = [None] * len(outTokens) + self.tokens_meta: List[Optional[dict]] = [None] * len(outTokens) self.pos = 0 self.posMax = len(self.src) @@ -59,13 +59,13 @@ def __init__(self, src: str, md, env, outTokens: List[Token]): # Stores { start: end } pairs. Useful for backtrack # optimization of pairs parse (emphasis, strikes). - self.cache = {} + self.cache: Dict[int, int] = {} # List of emphasis-like delimiters for current tag self.delimiters: List[Delimiter] = [] # Stack of delimiter lists for upper level tags - self._prev_delimiters = [] + self._prev_delimiters: list = [] def __repr__(self): return ( diff --git a/markdown_it/rules_inline/strikethrough.py b/markdown_it/rules_inline/strikethrough.py index ccb68877..69337b07 100644 --- a/markdown_it/rules_inline/strikethrough.py +++ b/markdown_it/rules_inline/strikethrough.py @@ -123,10 +123,10 @@ def postProcess(state: StateInline): curr = 0 while curr < maximum: try: - tokens_meta[curr] + curr_meta = tokens_meta[curr] except IndexError: pass else: - if tokens_meta[curr] and "delimiters" in tokens_meta[curr]: - _postProcess(state, tokens_meta[curr]["delimiters"]) + if curr_meta and "delimiters" in curr_meta: + _postProcess(state, curr_meta["delimiters"]) curr += 1 diff --git a/markdown_it/token.py b/markdown_it/token.py index 1ba5dc98..34c6f317 100644 --- a/markdown_it/token.py +++ b/markdown_it/token.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Union import attr @@ -17,7 +17,7 @@ class Token: # Html attributes. Format: `[ [ name1, value1 ], [ name2, value2 ] ]` attrs: Optional[list] = attr.ib(default=None) # Source map info. Format: `[ line_begin, line_end ]` - map: Optional[Tuple[int, int]] = attr.ib(default=None) + map: Optional[List[int]] = attr.ib(default=None) # nesting level, the same as `state.level` level: int = attr.ib(default=0) # An array of child nodes (inline and img tokens) @@ -46,7 +46,7 @@ def attrIndex(self, name: str) -> int: return i return -1 - def attrPush(self, attrData: Tuple[str, str]): + def attrPush(self, attrData: List[str]): """Add `[ name, value ]` attribute to list. Init attrs if necessary.""" if self.attrs: self.attrs.append(attrData) @@ -59,12 +59,14 @@ def attrSet(self, name: str, value: str): if idx < 0: self.attrPush([name, value]) else: + assert self.attrs is not None self.attrs[idx] = [name, value] - def attrGet(self, name: str) -> str: + def attrGet(self, name: str) -> Optional[str]: """ Get the value of attribute `name`, or null if it does not exist.""" idx = self.attrIndex(name) if idx >= 0: + assert self.attrs is not None return self.attrs[idx][1] return None @@ -121,7 +123,7 @@ class NestedTokens: def __getattr__(self, name): return getattr(self.opening, name) - def attrGet(self, name: str) -> str: + def attrGet(self, name: str) -> Optional[str]: """ Get the value of attribute `name`, or null if it does not exist.""" return self.opening.attrGet(name) @@ -132,7 +134,7 @@ def nest_tokens(tokens: List[Token]) -> List[Union[Token, NestedTokens]]: ``NestedTokens`` contain the open and close tokens and a list of children of all tokens in between (recursively nested) """ - output = [] + output: List[Union[Token, NestedTokens]] = [] tokens = list(reversed(tokens)) while tokens: @@ -142,7 +144,7 @@ def nest_tokens(tokens: List[Token]) -> List[Union[Token, NestedTokens]]: token = token.copy() output.append(token) if token.children: - token.children = nest_tokens(token.children) + token.children = nest_tokens(token.children) # type: ignore continue assert token.nesting == 1, token.nesting diff --git a/markdown_it/utils.py b/markdown_it/utils.py index 61fbcc5c..7779351e 100644 --- a/markdown_it/utils.py +++ b/markdown_it/utils.py @@ -1,15 +1,20 @@ from pathlib import Path +from typing import TYPE_CHECKING, Any -class AttrDict(dict): - def __init__(self, *args, **kwargs): - super(AttrDict, self).__init__(*args, **kwargs) - self.__dict__ = self +if TYPE_CHECKING: + AttrDict = Any +else: - # recursively apply to all nested dictionaries - for key, item in list(self.items()): - if isinstance(item, dict): - self[key] = AttrDict(item) + class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + # recursively apply to all nested dictionaries + for key, item in list(self.items()): + if isinstance(item, dict): + self[key] = AttrDict(item) def read_fixture_file(path): From e7ae663422fb22cf838257c1afc213d8553e2a1d Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 22 Oct 2020 01:14:35 +0300 Subject: [PATCH 02/12] Exclude mypy conf in MANIFEST.in --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8ad1b1b7..26191c26 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,6 +13,7 @@ exclude .flake8 exclude .circleci exclude .circleci/config.yml exclude codecov.yml +exclude .mypy.ini include LICENSE include LICENSE.markdown-it From 2e4dbfd3b56bec08ee07049c85b28aaf37967ed2 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 22 Oct 2020 02:10:48 +0300 Subject: [PATCH 03/12] Loosen up config typing --- markdown_it/main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/markdown_it/main.py b/markdown_it/main.py index 6076aa1b..88e533fc 100644 --- a/markdown_it/main.py +++ b/markdown_it/main.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Union from . import helpers, presets # noqa F401 from .common import utils # noqa F401 @@ -28,7 +28,7 @@ class MarkdownIt: def __init__( - self, config: Union[str, AttrDict] = "commonmark", renderer_cls=RendererHTML + self, config: Union[str, Mapping] = "commonmark", renderer_cls=RendererHTML ): """Main parser class @@ -69,7 +69,7 @@ def set(self, options): """ self.options = options - def configure(self, presets: Union[str, AttrDict]): + def configure(self, presets: Union[str, Mapping]): """Batch load of all options and component settings. This is an internal method, and you probably will not need it. But if you will - see available presets and data structure @@ -87,13 +87,13 @@ def configure(self, presets: Union[str, AttrDict]): ) if not presets: raise ValueError("Wrong `markdown-it` preset, can't be empty") - presets = AttrDict(presets) + attr_presets = AttrDict(presets) - if "options" in presets: - self.set(presets.options) + if "options" in attr_presets: + self.set(attr_presets.options) - if "components" in presets: - for name, component in presets.components.items(): + if "components" in attr_presets: + for name, component in attr_presets.components.items(): rules = component.get("rules", None) if rules: self[name].ruler.enableOnly(rules) From c2b67d27b3b491793df15923e194ce01226d045a Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Sat, 31 Oct 2020 13:15:40 +0200 Subject: [PATCH 04/12] Set no_implicit_optional=True mypy conf --- .mypy.ini | 1 + markdown_it/common/normalize_url.py | 4 ++-- markdown_it/extensions/container/index.py | 4 ++-- markdown_it/parser_block.py | 11 +++++++++-- markdown_it/rules_block/state_block.py | 9 +++++++-- markdown_it/rules_core/smartquotes.py | 6 +++--- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.mypy.ini b/.mypy.ini index f10e639b..3a995cd7 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,3 +1,4 @@ [mypy] warn_unused_ignores = True warn_redundant_casts = True +no_implicit_optional = True diff --git a/markdown_it/common/normalize_url.py b/markdown_it/common/normalize_url.py index ef880249..e3c4742b 100644 --- a/markdown_it/common/normalize_url.py +++ b/markdown_it/common/normalize_url.py @@ -1,6 +1,6 @@ import html import re -from typing import Callable +from typing import Callable, Optional from urllib.parse import urlparse, urlunparse, quote, unquote # noqa: F401 from .utils import ESCAPABLE @@ -166,7 +166,7 @@ def normalizeLinkText(link): GOOD_DATA_RE = re.compile(r"^data:image\/(gif|png|jpeg|webp);") -def validateLink(url: str, validator: Callable = None): +def validateLink(url: str, validator: Optional[Callable] = None): """Validate URL link is allowed in output. This validator can prohibit more than really needed to prevent XSS. diff --git a/markdown_it/extensions/container/index.py b/markdown_it/extensions/container/index.py index 8a31365d..b6edd43a 100644 --- a/markdown_it/extensions/container/index.py +++ b/markdown_it/extensions/container/index.py @@ -1,6 +1,6 @@ """Process block-level custom containers.""" from math import floor -from typing import Callable +from typing import Callable, Optional from markdown_it import MarkdownIt from markdown_it.common.utils import charCodeAt @@ -11,7 +11,7 @@ def container_plugin( md: MarkdownIt, name: str, marker: str = ":", - validate: Callable[[str, str], bool] = None, + validate: Optional[Callable[[str, str], bool]] = None, render=None, ): """Plugin ported from diff --git a/markdown_it/parser_block.py b/markdown_it/parser_block.py index 7cb9affa..fc472fd7 100644 --- a/markdown_it/parser_block.py +++ b/markdown_it/parser_block.py @@ -1,6 +1,6 @@ """Block-level tokenizer.""" import logging -from typing import List +from typing import List, Optional from .ruler import Ruler from .token import Token @@ -92,7 +92,14 @@ def tokenize( line += 1 state.line = line - def parse(self, src: str, md, env, outTokens: List[Token], ords: List[int] = None): + def parse( + self, + src: str, + md, + env, + outTokens: List[Token], + ords: Optional[List[int]] = None, + ): """Process input string and push block tokens into `outTokens`.""" if not src: return diff --git a/markdown_it/rules_block/state_block.py b/markdown_it/rules_block/state_block.py index 47fa68c3..0b29d8a8 100644 --- a/markdown_it/rules_block/state_block.py +++ b/markdown_it/rules_block/state_block.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from ..token import Token from ..ruler import StateBase @@ -7,7 +7,12 @@ class StateBlock(StateBase): def __init__( - self, src: str, md, env, tokens: List[Token], srcCharCode: List[int] = None + self, + src: str, + md, + env, + tokens: List[Token], + srcCharCode: Optional[List[int]] = None, ): self.src = src diff --git a/markdown_it/rules_core/smartquotes.py b/markdown_it/rules_core/smartquotes.py index 65e39487..3b25d30c 100644 --- a/markdown_it/rules_core/smartquotes.py +++ b/markdown_it/rules_core/smartquotes.py @@ -1,7 +1,7 @@ """Convert straight quotation marks to typographic ones """ import re -from typing import List +from typing import Any, Dict, List from .state_core import StateCore from ..common.utils import charCodeAt @@ -22,7 +22,7 @@ def replaceAt(string: str, index: int, ch: str): def process_inlines(tokens: List[Token], state: StateCore): - stack = [] + stack: List[Dict[str, Any]] = [] for i in range(len(tokens)): token = tokens[i] @@ -198,5 +198,5 @@ def smartquotes(state: StateCore): if token.type != "inline" or not QUOTE_RE.search(token.content): continue - + assert token.children is not None process_inlines(token.children, state) From f2b762025b9df3016d701d8c229b9329dcb1ce4b Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Sun, 1 Nov 2020 09:14:42 +0200 Subject: [PATCH 05/12] Improve variable name --- markdown_it/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/markdown_it/main.py b/markdown_it/main.py index 88e533fc..03163c50 100644 --- a/markdown_it/main.py +++ b/markdown_it/main.py @@ -87,13 +87,13 @@ def configure(self, presets: Union[str, Mapping]): ) if not presets: raise ValueError("Wrong `markdown-it` preset, can't be empty") - attr_presets = AttrDict(presets) + config = AttrDict(presets) - if "options" in attr_presets: - self.set(attr_presets.options) + if "options" in config: + self.set(config.options) - if "components" in attr_presets: - for name, component in attr_presets.components.items(): + if "components" in config: + for name, component in config.components.items(): rules = component.get("rules", None) if rules: self[name].ruler.enableOnly(rules) From 7828960366995e9c50ee4a5f8b3430c1bb327a4a Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Sun, 1 Nov 2020 09:40:03 +0200 Subject: [PATCH 06/12] More accurate type annotations --- markdown_it/main.py | 2 +- markdown_it/rules_inline/state_inline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/markdown_it/main.py b/markdown_it/main.py index 03163c50..8d6dd302 100644 --- a/markdown_it/main.py +++ b/markdown_it/main.py @@ -43,7 +43,7 @@ def __init__( self.utils = utils self.helpers = helpers - self.options: dict = {} + self.options: Dict[str, Any] = {} self.configure(config) self.linkify = linkify_it.LinkifyIt() if linkify_it else None diff --git a/markdown_it/rules_inline/state_inline.py b/markdown_it/rules_inline/state_inline.py index 8b829253..224467cd 100644 --- a/markdown_it/rules_inline/state_inline.py +++ b/markdown_it/rules_inline/state_inline.py @@ -65,7 +65,7 @@ def __init__(self, src: str, md, env, outTokens: List[Token]): self.delimiters: List[Delimiter] = [] # Stack of delimiter lists for upper level tags - self._prev_delimiters: list = [] + self._prev_delimiters: List[List[Delimiter]] = [] def __repr__(self): return ( From 85d7b68004f5165c4838e23015677440f35061c7 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Wed, 4 Nov 2020 07:47:43 +0200 Subject: [PATCH 07/12] Fix `Module has no attribute "entities"` --- markdown_it/common/entities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown_it/common/entities.py b/markdown_it/common/entities.py index 2fea835d..8a3441f2 100644 --- a/markdown_it/common/entities.py +++ b/markdown_it/common/entities.py @@ -1,5 +1,5 @@ """HTML5 entities map: { name -> characters }.""" -import html +import html.entities class _Entities: From 6ad7156cefa2bc2df90e5099fbda1102450f9c59 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 12 Nov 2020 00:07:21 +0200 Subject: [PATCH 08/12] Loosen 'enable' and 'disable' methods' arg type annotations --- markdown_it/main.py | 6 +++--- markdown_it/ruler.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/markdown_it/main.py b/markdown_it/main.py index 8d6dd302..03e8ac6d 100644 --- a/markdown_it/main.py +++ b/markdown_it/main.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from typing import Any, Callable, Dict, List, Mapping, Optional, Union +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Union from . import helpers, presets # noqa F401 from .common import utils # noqa F401 @@ -122,7 +122,7 @@ def get_active_rules(self) -> Dict[str, List[str]]: return rules def enable( - self, names: Union[str, List[str]], ignoreInvalid: bool = False + self, names: Union[str, Iterable[str]], ignoreInvalid: bool = False ) -> "MarkdownIt": """Enable list or rules. (chainable) @@ -155,7 +155,7 @@ def enable( return self def disable( - self, names: Union[str, List[str]], ignoreInvalid: bool = False + self, names: Union[str, Iterable[str]], ignoreInvalid: bool = False ) -> "MarkdownIt": """The same as [[MarkdownIt.enable]], but turn specified rules off. (chainable) diff --git a/markdown_it/ruler.py b/markdown_it/ruler.py index 3c0018b2..c2bc995e 100644 --- a/markdown_it/ruler.py +++ b/markdown_it/ruler.py @@ -15,7 +15,7 @@ class Ruler rules control use [[MarkdownIt.disable]], [[MarkdownIt.enable]] and [[MarkdownIt.use]]. """ -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, Iterable, List, Optional, Union import attr @@ -135,7 +135,7 @@ def push(self, ruleName: str, fn: RuleFunc, options=None): self.__rules__.append(Rule(ruleName, True, fn, (options or {}).get("alt", []))) self.__cache__ = None - def enable(self, names: Union[str, List[str]], ignoreInvalid: bool = False): + def enable(self, names: Union[str, Iterable[str]], ignoreInvalid: bool = False): """Enable rules with given names. :param names: name or list of rule names to enable. @@ -157,7 +157,7 @@ def enable(self, names: Union[str, List[str]], ignoreInvalid: bool = False): self.__cache__ = None return result - def enableOnly(self, names: Union[str, List[str]], ignoreInvalid: bool = False): + def enableOnly(self, names: Union[str, Iterable[str]], ignoreInvalid: bool = False): """Enable rules with given names, and disable everything else. :param names: name or list of rule names to enable. @@ -171,7 +171,7 @@ def enableOnly(self, names: Union[str, List[str]], ignoreInvalid: bool = False): rule.enabled = False self.enable(names, ignoreInvalid) - def disable(self, names: Union[str, List[str]], ignoreInvalid: bool = False): + def disable(self, names: Union[str, Iterable[str]], ignoreInvalid: bool = False): """Disable rules with given names. :param names: name or list of rule names to enable. From 04ecc04cb88af802078dffb49b19109caceb654b Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 12 Nov 2020 10:11:28 +0200 Subject: [PATCH 09/12] Add `strict_equality = True` for mypy --- .mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/.mypy.ini b/.mypy.ini index 3a995cd7..ba8252ba 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -2,3 +2,4 @@ warn_unused_ignores = True warn_redundant_casts = True no_implicit_optional = True +strict_equality = True From 4cc9fba9bb58973ec76325407804f919763e657d Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Tue, 17 Nov 2020 00:20:15 +0200 Subject: [PATCH 10/12] Fix mypy error in tasklists --- markdown_it/extensions/tasklists/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/markdown_it/extensions/tasklists/__init__.py b/markdown_it/extensions/tasklists/__init__.py index fa1c1995..4a0d7ec2 100644 --- a/markdown_it/extensions/tasklists/__init__.py +++ b/markdown_it/extensions/tasklists/__init__.py @@ -89,6 +89,7 @@ def is_todo_item(tokens, index): ) def todoify(token: Token, token_constructor): + assert token.children is not None token.children.insert(0, make_checkbox(token, token_constructor)) token.children[1].content = token.children[1].content[3:] token.content = token.content[3:] From 2da1d2f44d869f5221cc6c605fcd39337cb613ab Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 19 Nov 2020 13:16:27 +0200 Subject: [PATCH 11/12] Make RuleFunc typing correct --- markdown_it/ruler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/markdown_it/ruler.py b/markdown_it/ruler.py index c2bc995e..d1db29b0 100644 --- a/markdown_it/ruler.py +++ b/markdown_it/ruler.py @@ -26,9 +26,11 @@ def __init__(self, src: str, md, env): self.md = md -# The first positional arg is always a `StateBase`. Other arguments are -# optional. -RuleFunc = Callable[..., None] +# The first positional arg is always a subtype of `StateBase`. Other +# arguments may or may not exist, based on the rule's type (block, +# core, inline). Return type is either `None` or `bool` based on the +# rule's type. +RuleFunc = Callable @attr.s(slots=True) From 8ec4ba3629be5f0c50f38d0d274512460c0df5a9 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Sun, 13 Dec 2020 13:55:41 +0200 Subject: [PATCH 12/12] Fix new mypy errors --- markdown_it/rules_block/blockquote.py | 3 ++- markdown_it/rules_block/heading.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/markdown_it/rules_block/blockquote.py b/markdown_it/rules_block/blockquote.py index 1b5aa57b..98fcdc1d 100644 --- a/markdown_it/rules_block/blockquote.py +++ b/markdown_it/rules_block/blockquote.py @@ -1,5 +1,6 @@ # Block quotes import logging +from typing import Optional from .state_block import StateBlock from ..common.utils import isSpace @@ -39,7 +40,7 @@ def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool): ) try: - second_char_code = state.srcCharCode[pos] + second_char_code: Optional[int] = state.srcCharCode[pos] except IndexError: second_char_code = None diff --git a/markdown_it/rules_block/heading.py b/markdown_it/rules_block/heading.py index 12680b99..353520a3 100644 --- a/markdown_it/rules_block/heading.py +++ b/markdown_it/rules_block/heading.py @@ -1,5 +1,6 @@ """ Atex heading (#, ##, ...) """ import logging +from typing import Optional from .state_block import StateBlock from ..common.utils import isSpace @@ -18,7 +19,7 @@ def heading(state: StateBlock, startLine: int, endLine: int, silent: bool): if state.sCount[startLine] - state.blkIndent >= 4: return False - ch = state.srcCharCode[pos] + ch: Optional[int] = state.srcCharCode[pos] # /* # */ if ch != 0x23 or pos >= maximum: