spectrometerHBH commented on a change in pull request #6227: URL: https://github.com/apache/incubator-tvm/pull/6227#discussion_r467448594
########## File path: python/tvm/hybrid/parser.py ########## @@ -0,0 +1,754 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Hybrid Script Parser For TIR""" +# pylint: disable=invalid-name, missing-docstring, inconsistent-return-statements, no-else-return +# pylint: disable=unnecessary-comprehension, unused-argument, import-outside-toplevel +# pylint: disable=unused-import +import json +import numbers +import operator +from typed_ast import ast3 as ast + +import tvm._ffi +from tvm import tir +from tvm._ffi.base import TVMError +from tvm.ir import GlobalVar +from tvm.tir import all as _all +from tvm.tir import expr as _expr + +from . import scope_emitter, special_stmt, scope_handler, intrin +from .meta_unparser import MetaUnparser +from .registry import Registry + + +class HybridParserError(RuntimeError): + """Hybrid Parser Runtime Error""" + + +class HybridParser(ast.NodeVisitor): + """Python AST visitor pass which finally lowers it to TIR + Notes for extension: + 1. To support new types of AST nodes. Add a function visit_xxx(). + 2. To support new functions + We divide allowed function calls in hybrid script into 3 categories, + which is scope_handler, intrin and special_stmt. + 1) scope_handler: scope_handler functions correspond to StmtNodes without body, which can be + further classified into 2 categories: with scope handler can for scope handlers + 2) intrin: intrin functions corresponds to the remaining IRNodes (StmtNodes without body, + PrimExprNodes and more) + 3) special_stmt: special_stmt functions don't correspond to an IRNode in the AST directly. + It is usually used for some information that is not suitable to be printed directly. + When visiting With node, we check with_scope registry. + When visiting For node, we check for_scope registry. + """ + + _binop_maker = { + ast.Add: tir.Add, + ast.Sub: tir.Sub, + ast.Mult: tir.Mul, + ast.Div: tir.Div, + ast.FloorDiv: tir.FloorDiv, + ast.Mod: tir.FloorMod, + ast.BitOr: operator.or_, + ast.BitAnd: operator.and_, + ast.BitXor: operator.xor, + ast.Gt: tir.GT, + ast.GtE: tir.GE, + ast.Lt: tir.LT, + ast.LtE: tir.LE, + ast.Eq: tir.EQ, + ast.NotEq: tir.NE, + ast.And: tir.And, + ast.Or: tir.Or, + } + + _unaryop_maker = { + ast.USub: operator.neg, + ast.Invert: operator.invert, + ast.Not: tir.Not + } + + def __init__(self, src, base_lienno): + self.params = None + self.buffer_map = None + self.dict_attr = None + self.scope_emitter = None + + self.src = src.split('\n') + self.base_lineno = base_lienno + self.current_lineno = 0 + self.current_col_offset = 0 + self.meta = None + + self.functions = {} + + self._in_with_func_arg = False + self._assign_target = None + + def init_function_parsing_env(self): + """Initialize function parsing environment""" + self.params = [] # parameter list + self.buffer_map = {} # buffer map + self.dict_attr = {} # dict attr + self.scope_emitter = scope_emitter.ScopeEmitter(self) # scope emitter + + @staticmethod + def is_meta(node): + """Judge whether an AST node is META""" + return isinstance(node, ast.Assign) and len(node.targets) == 1 \ + and isinstance(node.targets[0], ast.Name) and node.targets[0].id == "__tvm_meta__" + + def init_meta(self, meta_dict): + if meta_dict is not None: + self.meta = tvm.ir.load_json(json.dumps(meta_dict)) + + def visit(self, node): + """Override method in ast.NodeVisitor""" + old_lineno, old_col_offset = self.current_lineno, self.current_col_offset + + if hasattr(node, "lineno"): + self.current_lineno = self.base_lineno + node.lineno - 1 + if hasattr(node, "col_offset"): + self.current_col_offset = node.col_offset + + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + visit_res = visitor(node) + + self.current_lineno, self.current_col_offset = old_lineno, old_col_offset + + return visit_res + + def wrap_line_col(self, message, lineno, col_offset): + """Wrap the message with line number and column offset""" + src_line = self.src[lineno - self.base_lineno] + leading_space = len(src_line) - len(src_line.lstrip(' ')) + col_offset = col_offset - leading_space + src_line = src_line[leading_space:] + return "\n " + src_line + "\n " + " " * col_offset + "^\n" + "ParserError in line " \ + + str(lineno) + " : " + message + + def report_error(self, message, lineno=None, col_offset=None): + """ Report an error occur in line lineno and column col_offset + Parameters + ---------- + message : str + Error message + lineno : int + Line number of error line + col_offset : int + Column offset of error line + """ + + if lineno is None: + lineno = self.current_lineno + if col_offset is None: + col_offset = self.current_col_offset + raise HybridParserError(self.wrap_line_col(message, lineno, col_offset)) + + def get_type_name(self, vtype): + if isinstance(vtype, ast.Attribute) \ + and isinstance(vtype.value, ast.Name) and vtype.value.id == 'ty': + return vtype.attr + self.report_error("invalid type annotation") + + def get_body(self): + body = [] + while len(self.scope_emitter.node_stack[-1]) > 0: + res = self.visit(self.scope_emitter.node_stack[-1].pop()) + if res is not None: + body.append(res) + return tvm.tir.SeqStmt(body) if len(body) > 1 else body[0] + + def parse_type(self, vtype): + """ Parse type annotation AST into Type object """ + if isinstance(vtype, ast.NameConstant) and vtype.value is None: + return tvm.ir.TupleType([]) + elif isinstance(vtype, ast.Attribute): + return tvm.ir.PrimType(self.get_type_name(vtype)) + elif isinstance(vtype, ast.Subscript) and isinstance(vtype.slice, ast.Index): + type_name = self.get_type_name(vtype.value) + if isinstance(vtype.slice.value, ast.Tuple): + args = [self.parse_type(element) for element in vtype.slice.value.elts] + else: + args = [self.parse_type(vtype.slice.value)] + if type_name == "Ptr": + return tvm.ir.PointerType(*args) + elif type_name == "Tuple": + return tvm.ir.TupleType(args) + + self.report_error("invalid type annotation") + + def generic_visit(self, node): + """ Override method in ast.NodeVisitor. + To directly filter out invalidate type of stmt. + """ + + self.report_error(type(node).__name__ + " stmt is not supported now") + + def visit_Module(self, node): + """ Module visitor + AST abstract grammar: + Module(stmt* body, type_ignore* type_ignore) + By now we support two format of hybrid script shown below. + + Example + ------- + 1. Generate a Function(If the code is printed, then it may bring meta) + .. code-block:: python + + import tvm + + @tvm.hybrid.script + def A(...): + ... + + # call hybrid parser when call this function, get a Function + func = A + + 2. Generate a Module + .. code-block:: python + + import tvm + + @tvm.hybrid.script + class MyMod(): + def A(...): + ... + + def B(...): + ... + + __tvm_meta__ = ... + + # call hybrid parser during construction, get a Module + mod = MyMod + """ Review comment: updated ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
