Skip to content

[Lint] Apply lint #126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 64 commits into from
Sep 23, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
64 commits
Select commit Hold shift + click to select a range
ce8a00a
modified a line in example
Sep 14, 2022
c1bf7a6
doc inside script
Sep 14, 2022
9486154
[Lint] Add flake8+back+isort
Sep 15, 2022
75e4b0e
[Lint] Add flake8+back+isort
Sep 15, 2022
57101ee
modified .flake8
titaiwangms Sep 16, 2022
7e60bf5
modified .flake8
titaiwangms Sep 16, 2022
b1feb40
CI without install
titaiwangms Sep 16, 2022
fd10719
CI install package
titaiwangms Sep 16, 2022
8763776
__version__ single-source
titaiwangms Sep 16, 2022
8d56a86
add py3.7 __version__ support
titaiwangms Sep 16, 2022
7a742a0
use pip install in CI
titaiwangms Sep 16, 2022
66d284d
importlib.metadata
titaiwangms Sep 16, 2022
0bcca9a
CI-patch1
titaiwangms Sep 16, 2022
87ffeda
CI patch-2
titaiwangms Sep 16, 2022
70d32c6
CI-patch3
titaiwangms Sep 16, 2022
89a904b
CI patch-4
titaiwangms Sep 16, 2022
1554068
CI patch-5
titaiwangms Sep 16, 2022
9d8a0a7
CI patch-6
titaiwangms Sep 16, 2022
2c61dc6
exclude tests
titaiwangms Sep 19, 2022
8dd5314
fix import bugs
titaiwangms Sep 19, 2022
d721a19
input package name
titaiwangms Sep 19, 2022
8d98b63
update README.md with lint
titaiwangms Sep 20, 2022
eb2783d
Merge branch 'main' into titaiwang/add_lint
titaiwangms Sep 20, 2022
1ded058
test on changes
titaiwangms Sep 21, 2022
bbd23c7
add pylint into toml and check importError
titaiwangms Sep 21, 2022
68fb8fa
added pylint into style.sh and test __package__
titaiwangms Sep 21, 2022
01f2312
pylint addressed
titaiwangms Sep 21, 2022
8604e3d
update .flake8 comment
titaiwangms Sep 21, 2022
744090a
modified follow warnings on UI
titaiwangms Sep 21, 2022
83c9409
typo patch-1
titaiwangms Sep 21, 2022
d39b651
format toml and update .flake8
titaiwangms Sep 21, 2022
ba52727
black and isort and lint.sh
titaiwangms Sep 21, 2022
db13552
remove relativa path
titaiwangms Sep 21, 2022
ce34d2b
remove relativa path
titaiwangms Sep 21, 2022
a223577
update azure-ci
titaiwangms Sep 21, 2022
c982c5a
Merge branch 'titaiwang/apply_lint' of github.com:AllenTiTaiWang/onnx…
titaiwangms Sep 21, 2022
e3a6753
add two pylint ignore
titaiwangms Sep 21, 2022
d402cea
f-string patch-1
titaiwangms Sep 21, 2022
cc45b6a
pylint patch-1
titaiwangms Sep 21, 2022
5ced6e2
pylint patch-2
titaiwangms Sep 21, 2022
4aef6c8
lint.sh
titaiwangms Sep 21, 2022
4193e51
format pyproject.toml
titaiwangms Sep 21, 2022
473ecd9
disable pylint on Action for now
titaiwangms Sep 21, 2022
62d0a9b
Merge branch 'main' into titaiwang/add_lint
titaiwangms Sep 22, 2022
6a3e0bd
add optional style.sh
titaiwangms Sep 22, 2022
c9bf34a
merge main
titaiwangms Sep 22, 2022
07206e9
CI adjusted
titaiwangms Sep 22, 2022
58f011f
style check patch-1
titaiwangms Sep 22, 2022
3123746
circular import
titaiwangms Sep 22, 2022
bea578c
lint
titaiwangms Sep 22, 2022
73ab472
Merge branch 'titaiwang/add_lint' into titaiwang/apply_lint
titaiwangms Sep 22, 2022
f37c795
delete additional files
titaiwangms Sep 22, 2022
1c79e88
add auto-formatter
titaiwangms Sep 22, 2022
fb87b6e
use old onnx_export.py
titaiwangms Sep 22, 2022
01f32c8
lint
titaiwangms Sep 22, 2022
8694b37
f-string
titaiwangms Sep 22, 2022
6064f6b
minor modified
titaiwangms Sep 22, 2022
85ed15f
Merge branch 'titaiwang/add_lint' into titaiwang/apply_lint
titaiwangms Sep 22, 2022
03e0203
add permission
titaiwangms Sep 22, 2022
7f4e6b3
remove permission
titaiwangms Sep 22, 2022
94f4543
minor fixes
titaiwangms Sep 23, 2022
0aa7327
Merge branch 'titaiwang/add_lint' into titaiwang/apply_lint
titaiwangms Sep 23, 2022
ccdef63
merge main
titaiwangms Sep 23, 2022
a4fe743
lint
titaiwangms Sep 23, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,4 +155,3 @@ optional check:
```

NOTE: mypy and pylint needs to be manually address

4 changes: 3 additions & 1 deletion onnxscript/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
# --------------------------------------------------------------------------

import sys
from .main import script, export_onnx_lib, OnnxFunction

from .backend.onnx_export import export2python as proto2python
from .main import export_onnx_lib, script
from .values import OnnxFunction

if sys.version_info[0:2] >= (3, 8):
import importlib.metadata as importlib_metadata
Expand Down
75 changes: 48 additions & 27 deletions onnxscript/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@

import os
from typing import Optional
import onnx
import onnx.helper as helper

import click
from onnxscript.converter import Converter
from onnxscript.backend.onnx_export import export2python
import onnx
from onnx import helper

from onnxscript import converter
from onnxscript.backend import onnx_export


@click.group()
Expand All @@ -18,18 +20,18 @@ def cli():


def convert_file(script):
converter = Converter()
return converter.convert_file(script)
convert = converter.Converter()
return convert.convert_file(script)


def to_single_model_proto(model, input_py_file: str, output_onnx_file: Optional[str] = None):
if (not output_onnx_file):
prefix, ext = os.path.splitext(input_py_file)
if not output_onnx_file:
prefix, _ = os.path.splitext(input_py_file)
output_onnx_file = prefix + ".onnx"

fnlist = convert_file(input_py_file)

if (not fnlist):
if not fnlist:
print("No functions in input.")
return

Expand All @@ -47,8 +49,9 @@ def to_single_model_proto(model, input_py_file: str, output_onnx_file: Optional[
model = onnx.helper.make_model(
graph,
functions=[f.to_function_proto() for f in fnlist],
producer_name='p2o',
opset_imports=[onnx.helper.make_opsetid("", 15)])
producer_name="p2o",
opset_imports=[onnx.helper.make_opsetid("", 15)],
)

# TODO: add options for user to specify whether to check generated model
# model = onnx.shape_inference.infer_shapes(model)
Expand All @@ -71,34 +74,52 @@ def print_ir_function(f):


@cli.command()
@click.option('--fmt', type=click.Choice(['text', 'model', 'lib'], case_sensitive=False),
help="Translate input to a single ModelProto ('model'), "
"into a LibProto ('lib'), "
"or into text 'text').")
@click.option('name', '--name', envvar='PATHS', multiple=True, type=click.Path(),
help="File or files to convert.")
def translate(fmt="text", name=None):
@click.option(
"--fmt",
type=click.Choice(["text", "model", "lib"], case_sensitive=False),
help="Translate input to a single ModelProto ('model'), "
"into a LibProto ('lib'), "
"or into text 'text').",
)
@click.option(
"name",
"--name",
envvar="PATHS",
multiple=True,
type=click.Path(),
help="File or files to convert.",
)
def translate(fmt="text", names=None):
"""Translate a file or many files into a ModelProto, a LibProto or text."""
if fmt == "text":
for name in name:
for name in names:
to_text(name)
else:
for name in name:
for name in names:
to_single_model_proto(fmt == "model", name)


@cli.command()
@click.option('name', '--name', envvar='PATHS', multiple=False, type=click.Path(),
help="filename to convert")
@click.option("--op", is_flag=True, default=False,
help="converts a numerical operator into op.Add (False) or keep it (True)")
@click.option("--rename", is_flag=True, default=False,
help="to use shorter variable name")
@click.option(
"name",
"--name",
envvar="PATHS",
multiple=False,
type=click.Path(),
help="filename to convert",
)
@click.option(
"--op",
is_flag=True,
default=False,
help="converts a numerical operator into op.Add (False) or keep it (True)",
)
@click.option("--rename", is_flag=True, default=False, help="to use shorter variable name")
def onnx2script(name, op=False, rename=False):
"""Exports an onnx graph to a script in following onnx-script syntax.
The result is printed on the standard output.
"""
code = export2python(name, use_operators=op, rename=rename)
code = onnx_export.export2python(name, use_operators=op, rename=rename)
print(code)


Expand Down
71 changes: 44 additions & 27 deletions onnxscript/analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,22 @@
# --------------------------------------------------------------------------

import ast
from .values import DebugInfo

from onnxscript import debuginfo


def get_loop_var(for_stmt, converter):
if not isinstance(for_stmt.target, ast.Name):
raise ValueError(DebugInfo(for_stmt, converter).msg(
"For loop target must be a single variable."))
raise ValueError(
debuginfo.DebugInfo(for_stmt, converter).msg(
"For loop target must be a single variable."
)
)
return for_stmt.target.id


def used_vars(expr):
''' Return set of all variables used in an expression.'''
"""Return set of all variables used in an expression."""
if isinstance(expr, ast.Name):
return set([expr.id])
if isinstance(expr, ast.Call):
Expand All @@ -31,22 +35,24 @@ def used_vars(expr):


def local_defs(lhs):
'''Utility function to return set of assigned/defined
variables in the lhs of an assignment statement.'''
"""Utility function to return set of assigned/defined
variables in the lhs of an assignment statement."""

def get_id(e):
assert isinstance(e, ast.Name), "Only simple assignments supported."
return e.id

if (isinstance(lhs, ast.Tuple)):
return set([get_id(x) for x in lhs.elts])
return set([get_id(lhs)])
if isinstance(lhs, ast.Tuple):
return {get_id(x) for x in lhs.elts}
return {get_id(lhs)}


def defs(stmt):
'''
"""
Return the set of all variables that may be defined (assigned to) in an
execution of input stmt.
'''
"""

def block_defs(block):
result = set()
for s in block:
Expand All @@ -66,7 +72,7 @@ def block_defs(block):
if isinstance(stmt, ast.Break):
return set()
try:
if stmt.value.func.id == 'print':
if stmt.value.func.id == "print":
# Any call to print function are ignored.
return set()
except (TypeError, AttributeError):
Expand All @@ -75,11 +81,12 @@ def block_defs(block):


def do_liveness_analysis(fun, converter):
'''
"""
Perform liveness analysis of the given function-ast. The results of the
analysis are stored directly with each statement-ast `s` as attributes `s.live_in`
and `s.live_out`.
'''
"""

def visit(stmt, live_out):
stmt.live_out = live_out
live = do_visit(stmt, live_out)
Expand Down Expand Up @@ -124,22 +131,25 @@ def visitBlock(block, live_out):
# Break statements in the middle of the loop, however, will require
# a generalization.
return live_out
if isinstance(stmt, ast.Expr) and hasattr(stmt, 'value'):
if isinstance(stmt, ast.Expr) and hasattr(stmt, "value"):
# docstring
if hasattr(stmt.value, 'value') and isinstance(stmt.value.value, str):
if hasattr(stmt.value, "value") and isinstance(stmt.value.value, str):
# python 3.8+
return live_out
if hasattr(stmt.value, 's') and isinstance(stmt.value.s, str):
if hasattr(stmt.value, "s") and isinstance(stmt.value.s, str):
# python 3.7
return live_out
try:
if stmt.value.func.id == 'print':
if stmt.value.func.id == "print":
# Any call to print function are ignored.
return live_out
except (TypeError, AttributeError):
pass
raise ValueError(DebugInfo(stmt, converter).msg(
f"Unsupported statement type {type(stmt)!r}."))
raise ValueError(
debuginfo.DebugInfo(stmt, converter).msg(
f"Unsupported statement type {type(stmt)!r}."
)
)

assert isinstance(fun, ast.FunctionDef)
live = set()
Expand All @@ -148,7 +158,7 @@ def visitBlock(block, live_out):


def exposed_uses(stmts, converter):
'''
"""
Return the set of variables that are used before being defined by given block.
In essence, this identifies the "inputs" to a given code-block.
For example, consider the following code-block:
Expand All @@ -163,7 +173,8 @@ def exposed_uses(stmts, converter):
the block. Even though the value of y is used within the block, it is assigned
a value before it is used. However, in contrast, the incoming value of x is used
(in the first statement). Hence x is included in the exposed_uses.
'''
"""

def visitBlock(block, live_out):
for stmt in reversed(block):
live_out = visit(stmt, live_out)
Expand All @@ -180,10 +191,13 @@ def visit(stmt, live_out):
live1 = visitBlock(stmt.body, live_out)
live2 = visitBlock(stmt.orelse, live_out)
return (live1 | live2) | used_vars(stmt.test)
if (isinstance(stmt, ast.Expr) and hasattr(stmt, 'value') and
isinstance(stmt.value, ast.Call)):
if (
isinstance(stmt, ast.Expr)
and hasattr(stmt, "value")
and isinstance(stmt.value, ast.Call)
):
f = stmt.value.func
if f.id == 'print':
if f.id == "print":
return live_out
if isinstance(stmt, ast.For):
# Analysis assumes loop may execute zero times. Results can be improved
Expand All @@ -201,7 +215,10 @@ def visit(stmt, live_out):
return used_inside_loop | used_in_loop_header | live_out
if isinstance(stmt, ast.Break):
return live_out
raise ValueError(DebugInfo(stmt, converter).msg(
f"Unsupported statement type {type(stmt)!r}."))
raise ValueError(
debuginfo.DebugInfo(stmt, converter).msg(
f"Unsupported statement type {type(stmt)!r}."
)
)

return visitBlock(stmts, set())
Loading