Hello community, here is the log from the commit of package python-libcst for openSUSE:Factory checked in at 2020-09-04 10:58:35 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-libcst (Old) and /work/SRC/openSUSE:Factory/.python-libcst.new.3399 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-libcst" Fri Sep 4 10:58:35 2020 rev:2 rq:829619 version:0.3.10 Changes: -------- --- /work/SRC/openSUSE:Factory/python-libcst/python-libcst.changes 2020-08-25 12:40:11.129430507 +0200 +++ /work/SRC/openSUSE:Factory/.python-libcst.new.3399/python-libcst.changes 2020-09-04 10:59:42.726642648 +0200 @@ -1,0 +2,24 @@ +Tue Aug 25 11:33:22 UTC 2020 - Benjamin Greiner <[email protected]> + +- Update to v0.3.10 + Added + * Handle string annotations in ScopeProvider #373 + * Add is_annotation subtype for Access inreferences. #372 + Updated + * Call pyre query with noninteractive logging #371 + * Replace matchers with explicit visitation in gatherers #366 + * Include missing test data in install #365 + Fixed + * Spaces around walrus operator are not required #368 + * SaveMachedNode now matches with trailing empty wildcards #356 + * Correctly extract wildcard matchers #355 +- clean specfile multibuild +- don't require own package on test flavor +- require dataclasses for Python 3.6 distros +- despite gh#Instagram/LibCST#365 says it's fixed, we still need to + regenerate the code for tests + => don't skip tests which actually pass after codegen +- can use plain unittest again, same as upstream +- define skip_python2 for Leap repositories + +------------------------------------------------------------------- Old: ---- codemod_formatter_error_input.py.txt libcst-0.3.9.tar.gz New: ---- libcst-0.3.10.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-libcst.spec ++++++ --- /var/tmp/diff_new_pack.SAVkFh/_old 2020-09-04 10:59:46.066644438 +0200 +++ /var/tmp/diff_new_pack.SAVkFh/_new 2020-09-04 10:59:46.070644440 +0200 @@ -17,6 +17,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} +%define skip_python2 1 %global flavor @BUILD_FLAVOR@%{nil} %if "%{flavor}" == "test" %define psuffix -test @@ -26,31 +27,35 @@ %bcond_with test %endif Name: python-libcst%{psuffix} -Version: 0.3.9 +Version: 0.3.10 Release: 0 Summary: Python 3.5+ concrete syntax tree with AST-like properties License: MIT URL: https://github.com/Instagram/LibCST Source: https://files.pythonhosted.org/packages/source/l/libcst/libcst-%{version}.tar.gz -Source1: https://raw.githubusercontent.com/Instagram/LibCST/master/libcst/codemod/tests/codemod_formatter_error_input.py.txt -# all the deps over setuptools are for the generateor fix -BuildRequires: %{python_module black} +# isort needed for the code regeneration, code mod also on non test flavor BuildRequires: %{python_module isort} BuildRequires: %{python_module setuptools} -BuildRequires: %{python_module typing-inspect >= 0.4.0} -BuildRequires: %{python_module typing_extensions >= 3.7.4.2} BuildRequires: fdupes BuildRequires: python-rpm-macros Requires: python-PyYAML >= 5.2 Requires: python-typing-inspect >= 0.4.0 Requires: python-typing_extensions >= 3.7.4.2 BuildArch: noarch +%if %{python_version_nodots} < 37 +Requires: python-dataclasses +%endif %if %{with test} +%if %{python_version_nodots} < 37 +BuildRequires: %{python_module dataclasses} +%endif BuildRequires: %{python_module PyYAML >= 5.2} +# black needed for tests and the code regeneration +BuildRequires: %{python_module black} BuildRequires: %{python_module hypothesis >= 4.36.0} BuildRequires: %{python_module hypothesmith >= 0.0.4} -BuildRequires: %{python_module libcst >= %{version}} -BuildRequires: %{python_module pytest} +BuildRequires: %{python_module typing-inspect >= 0.4.0} +BuildRequires: %{python_module typing_extensions >= 3.7.4.2} %endif %python_subpackages @@ -59,10 +64,10 @@ %prep %setup -q -n libcst-%{version} -cp %{SOURCE1} libcst/codemod/tests/ +# fix executable sed -i 's/"python"/"python3"/' libcst/codemod/tests/test_codemod_cli.py # https://github.com/Instagram/LibCST/issues/331 -[ isort --version | grep "VERSION 4" ] || sed -i 's/"isort", "-y"/"isort"/' libcst/codegen/generate.py +isort --version | grep "VERSION 4" || sed -i 's/"isort", "-y"/"isort"/' libcst/codegen/generate.py # Depends on optional pyre rm \ @@ -73,30 +78,25 @@ # Test result depends on pyre sed -i 's/"error: .* stack:",/"Transformed 1 files successfully.",/' libcst/codemod/tests/test_codemod_cli.py +%if !%{with test} %build %python_build +%endif %install %if !%{with test} %python_install +%python_expand %fdupes %{buildroot}%{$python_sitelib} +%endif + +%if %{with test} +%check %{python_expand # https://github.com/Instagram/LibCST/issues/331 -export PYTHONPATH=%{buildroot}%{$python_sitelib} $python -m libcst.codegen.generate visitors $python -m libcst.codegen.generate return_types $python -m libcst.codegen.generate matchers - -rm -r %{buildroot}%{$python_sitelib}/libcst/tests/ \ - %{buildroot}%{$python_sitelib}/libcst/*/tests/ \ - %{buildroot}%{$python_sitelib}/libcst/*/*/tests/ - -%fdupes %{buildroot}%{$python_sitelib} } -%endif - -%check -%if %{with test} -# the two tests pass in non-multibuild setup, just skip them here -%pytest -k 'not test_codegen_clean_return_types and not test_codegen_clean_visitor_functions' +%pyunittest -v %endif %if !%{with test} ++++++ libcst-0.3.9.tar.gz -> libcst-0.3.10.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/PKG-INFO new/libcst-0.3.10/PKG-INFO --- old/libcst-0.3.9/PKG-INFO 2020-08-07 20:46:05.000000000 +0200 +++ new/libcst-0.3.10/PKG-INFO 2020-08-17 20:25:13.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: libcst -Version: 0.3.9 +Version: 0.3.10 Summary: A concrete syntax tree with AST-like properties for Python 3.5, 3.6, 3.7 and 3.8 programs. Home-page: https://github.com/Instagram/LibCST License: MIT diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/_nodes/expression.py new/libcst-0.3.10/libcst/_nodes/expression.py --- old/libcst-0.3.9/libcst/_nodes/expression.py 2020-08-07 20:39:26.000000000 +0200 +++ new/libcst-0.3.10/libcst/_nodes/expression.py 2020-08-14 21:02:21.000000000 +0200 @@ -3680,19 +3680,6 @@ #: Whitespace after the walrus operator, but before the value. whitespace_after_walrus: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ") - def _validate(self) -> None: - super(NamedExpr, self)._validate() - if ( - self.whitespace_before_walrus.empty - and not self.target._safe_to_use_with_word_operator(ExpressionPosition.LEFT) - ): - raise CSTValidationError("Must have at least one space after target.") - if ( - self.whitespace_after_walrus.empty - and not self.value._safe_to_use_with_word_operator(ExpressionPosition.RIGHT) - ): - raise CSTValidationError("Must have at least one space before value.") - def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "NamedExpr": return NamedExpr( lpar=visit_sequence(self, "lpar", self.lpar, visitor), diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/_nodes/tests/test_namedexpr.py new/libcst-0.3.10/libcst/_nodes/tests/test_namedexpr.py --- old/libcst-0.3.9/libcst/_nodes/tests/test_namedexpr.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/_nodes/tests/test_namedexpr.py 2020-08-14 21:02:21.000000000 +0200 @@ -87,6 +87,20 @@ "parser": _parse_statement_force_38, "expected_position": None, }, + { + "node": cst.If( + test=cst.NamedExpr( + target=cst.Name(value="x"), + value=cst.Integer(value="1"), + whitespace_before_walrus=cst.SimpleWhitespace(""), + whitespace_after_walrus=cst.SimpleWhitespace(""), + ), + body=cst.SimpleStatementSuite(body=[cst.Pass()]), + ), + "code": "if x:=1: pass\n", + "parser": _parse_statement_force_38, + "expected_position": None, + }, ) ) def test_valid(self, **kwargs: Any) -> None: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/_version.py new/libcst-0.3.10/libcst/_version.py --- old/libcst-0.3.9/libcst/_version.py 2020-08-07 20:39:26.000000000 +0200 +++ new/libcst-0.3.10/libcst/_version.py 2020-08-17 18:40:01.000000000 +0200 @@ -4,4 +4,4 @@ # LICENSE file in the root directory of this source tree. -LIBCST_VERSION: str = "0.3.9" +LIBCST_VERSION: str = "0.3.10" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/codemod/tests/codemod_formatter_error_input.py.txt new/libcst-0.3.10/libcst/codemod/tests/codemod_formatter_error_input.py.txt --- old/libcst-0.3.9/libcst/codemod/tests/codemod_formatter_error_input.py.txt 1970-01-01 01:00:00.000000000 +0100 +++ new/libcst-0.3.10/libcst/codemod/tests/codemod_formatter_error_input.py.txt 2020-08-04 22:04:41.000000000 +0200 @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# pyre-strict + +import subprocess +from contextlib import AsyncExitStack + + +def fun() -> None: + # this is an explicit syntax error to cause formatter error + async with AsyncExitStack() as stack: + stack diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/codemod/visitors/_gather_comments.py new/libcst-0.3.10/libcst/codemod/visitors/_gather_comments.py --- old/libcst-0.3.9/libcst/codemod/visitors/_gather_comments.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/codemod/visitors/_gather_comments.py 2020-08-14 21:02:21.000000000 +0200 @@ -7,7 +7,6 @@ from typing import Dict, Pattern, Union import libcst as cst -import libcst.matchers as m from libcst.codemod._context import CodemodContext from libcst.codemod._visitor import ContextAwareVisitor from libcst.metadata import PositionProvider @@ -37,11 +36,21 @@ self._comment_matcher: Pattern[str] = re.compile(comment_regex) - @m.visit(m.EmptyLine(comment=m.DoesNotMatch(None))) - @m.visit(m.TrailingWhitespace(comment=m.DoesNotMatch(None))) - def visit_comment(self, node: Union[cst.EmptyLine, cst.TrailingWhitespace]) -> None: + def visit_EmptyLine(self, node: cst.EmptyLine) -> bool: + if node.comment is not None: + self.handle_comment(node) + return False + + def visit_TrailingWhitespace(self, node: cst.TrailingWhitespace) -> bool: + if node.comment is not None: + self.handle_comment(node) + return False + + def handle_comment( + self, node: Union[cst.EmptyLine, cst.TrailingWhitespace] + ) -> None: comment = node.comment - assert comment is not None # hello, type checker + assert comment is not None # ensured by callsites above if not self._comment_matcher.match(comment.value): return line = self.get_metadata(PositionProvider, comment).start.line diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/codemod/visitors/_gather_exports.py new/libcst-0.3.10/libcst/codemod/visitors/_gather_exports.py --- old/libcst-0.3.9/libcst/codemod/visitors/_gather_exports.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/codemod/visitors/_gather_exports.py 2020-08-15 00:06:19.000000000 +0200 @@ -3,13 +3,12 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. # -from typing import Set +from typing import Set, Union -import libcst -import libcst.matchers as m +import libcst as cst from libcst.codemod._context import CodemodContext from libcst.codemod._visitor import ContextAwareVisitor -from libcst.helpers import ensure_type, get_full_name_for_node +from libcst.helpers import get_full_name_for_node class GatherExportsVisitor(ContextAwareVisitor): @@ -32,7 +31,8 @@ def __init__(self, context: CodemodContext) -> None: super().__init__(context) - # Track any re-exported objects in an __all__ reference + # Track any re-exported objects in an __all__ reference and whether + # they're defined or not self.explicit_exported_objects: Set[str] = set() # Presumably at some point in the future it would be useful to grab @@ -43,68 +43,87 @@ # that we have a reasonable place to put implicit objects in the future. # Internal bookkeeping - self._in_assignment: int = 0 - self._in_list: int = 0 + self._is_assigned_export: Set[Union[cst.Tuple, cst.List, cst.Set]] = set() + self._in_assigned_export: Set[Union[cst.Tuple, cst.List, cst.Set]] = set() - def visit_AnnAssign(self, node: libcst.AnnAssign) -> bool: - target = get_full_name_for_node(node.target) - if target == "__all__": - self._in_assignment += 1 - return True + def visit_AnnAssign(self, node: cst.AnnAssign) -> bool: + value = node.value + if value: + if self._handle_assign_target(node.target, value): + return True return False - def leave_AnnAssign(self, original_node: libcst.AnnAssign) -> None: - self._in_assignment -= 1 - - def visit_Assign(self, node: libcst.Assign) -> bool: + def visit_Assign(self, node: cst.Assign) -> bool: for target_node in node.targets: - target = get_full_name_for_node(target_node.target) - if target == "__all__": - self._in_assignment += 1 + if self._handle_assign_target(target_node.target, node.value): return True return False - def leave_Assign(self, original_node: libcst.Assign) -> None: - self._in_assignment -= 1 + def _handle_assign_target( + self, target: cst.BaseExpression, value: cst.BaseExpression + ) -> bool: + target_name = get_full_name_for_node(target) + if target_name == "__all__": + # Assignments such as `__all__ = ["os"]` + # or `__all__ = exports = ["os"]` + if isinstance(value, (cst.List, cst.Tuple, cst.Set)): + self._is_assigned_export.add(value) + return True + elif isinstance(target, cst.Tuple) and isinstance(value, cst.Tuple): + # Assignments such as `__all__, x = ["os"], []` + for element_idx, element_node in enumerate(target.elements): + element_name = get_full_name_for_node(element_node.value) + if element_name == "__all__": + element_value = value.elements[element_idx].value + if isinstance(element_value, (cst.List, cst.Tuple, cst.Set)): + self._is_assigned_export.add(value) + self._is_assigned_export.add(element_value) + return True + return False - def visit_List(self, node: libcst.List) -> bool: - self._in_list += 1 - # Only visit list/set entries when we're in an __all__ - # assignment. We gate also by internal counters, so this - # is simply an optimization. - return self._in_assignment == 1 and self._in_list == 1 - - def leave_List(self, original_node: libcst.List) -> None: - self._in_list -= 1 - - def visit_Tuple(self, node: libcst.Tuple) -> bool: - self._in_list += 1 - # Only visit list/set entries when we're in an __all__ - # assignment. We gate also by internal counters, so this - # is simply an optimization. - return self._in_assignment == 1 and self._in_list == 1 - - def leave_Tuple(self, original_node: libcst.Tuple) -> None: - self._in_list -= 1 - - def visit_Set(self, node: libcst.Set) -> bool: - # Only visit list/set entries when we're in an __all__ - # assignment. We gate also by internal counters, so this - # is simply an optimization. - self._in_list += 1 - return self._in_assignment == 1 and self._in_list == 1 - - def leave_Set(self, original_node: libcst.Set) -> None: - self._in_list -= 1 - - def visit_Element(self, node: libcst.Element) -> bool: - # See if this is a entry that is a string. - extraction = self.extract( - node, m.Element(m.SaveMatchedNode(m.SimpleString(), "string")) - ) - if extraction: - string = ensure_type(extraction["string"], libcst.SimpleString) - self.explicit_exported_objects.add(string.evaluated_value) + def visit_List(self, node: cst.List) -> bool: + if node in self._is_assigned_export: + self._in_assigned_export.add(node) + return True + return False - # Don't need to visit children + def leave_List(self, original_node: cst.List) -> None: + self._is_assigned_export.discard(original_node) + self._in_assigned_export.discard(original_node) + + def visit_Tuple(self, node: cst.Tuple) -> bool: + if node in self._is_assigned_export: + self._in_assigned_export.add(node) + return True return False + + def leave_Tuple(self, original_node: cst.Tuple) -> None: + self._is_assigned_export.discard(original_node) + self._in_assigned_export.discard(original_node) + + def visit_Set(self, node: cst.Set) -> bool: + if node in self._is_assigned_export: + self._in_assigned_export.add(node) + return True + return False + + def leave_Set(self, original_node: cst.Set) -> None: + self._is_assigned_export.discard(original_node) + self._in_assigned_export.discard(original_node) + + def visit_SimpleString(self, node: cst.SimpleString) -> bool: + self._handle_string_export(node) + return False + + def visit_ConcatenatedString(self, node: cst.ConcatenatedString) -> bool: + self._handle_string_export(node) + return False + + def _handle_string_export( + self, node: Union[cst.SimpleString, cst.ConcatenatedString] + ) -> None: + if self._in_assigned_export: + name = node.evaluated_value + if name is None: + return + self.explicit_exported_objects.add(name) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/codemod/visitors/_gather_string_annotation_names.py new/libcst-0.3.10/libcst/codemod/visitors/_gather_string_annotation_names.py --- old/libcst-0.3.9/libcst/codemod/visitors/_gather_string_annotation_names.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/codemod/visitors/_gather_string_annotation_names.py 2020-08-14 21:02:21.000000000 +0200 @@ -3,7 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from typing import Set, Union, cast +from typing import Collection, List, Set, Union, cast import libcst as cst import libcst.matchers as m @@ -13,14 +13,6 @@ FUNCS_CONSIDERED_AS_STRING_ANNOTATIONS = {"typing.TypeVar"} -ANNOTATION_MATCHER: m.BaseMatcherNode = m.Annotation() | m.Call( - metadata=m.MatchMetadataIfTrue( - QualifiedNameProvider, - lambda qualnames: any( - qn.name in FUNCS_CONSIDERED_AS_STRING_ANNOTATIONS for qn in qualnames - ), - ) -) class GatherNamesFromStringAnnotationsVisitor(ContextAwareVisitor): @@ -35,14 +27,45 @@ METADATA_DEPENDENCIES = (QualifiedNameProvider,) - def __init__(self, context: CodemodContext) -> None: + def __init__( + self, + context: CodemodContext, + typing_functions: Collection[str] = FUNCS_CONSIDERED_AS_STRING_ANNOTATIONS, + ) -> None: super().__init__(context) - + self._typing_functions: Collection[str] = typing_functions + self._annotation_stack: List[cst.CSTNode] = [] #: The set of names collected from string literals. self.names: Set[str] = set() - @m.call_if_inside(ANNOTATION_MATCHER) - @m.visit(m.ConcatenatedString()) + def visit_Annotation(self, node: cst.Annotation) -> bool: + self._annotation_stack.append(node) + return True + + def leave_Annotation(self, original_node: cst.Annotation) -> None: + self._annotation_stack.pop() + + def visit_Call(self, node: cst.Call) -> bool: + qnames = self.get_metadata(QualifiedNameProvider, node) + if any(qn.name in self._typing_functions for qn in qnames): + self._annotation_stack.append(node) + return True + return False + + def leave_Call(self, original_node: cst.Call) -> None: + if self._annotation_stack and self._annotation_stack[-1] == original_node: + self._annotation_stack.pop() + + def visit_ConcatenatedString(self, node: cst.ConcatenatedString) -> bool: + if self._annotation_stack: + self.handle_any_string(node) + return False + + def visit_SimpleString(self, node: cst.SimpleString) -> bool: + if self._annotation_stack: + self.handle_any_string(node) + return False + def handle_any_string( self, node: Union[cst.SimpleString, cst.ConcatenatedString] ) -> None: @@ -73,9 +96,3 @@ ) } self.names.update(names) - - @m.call_if_inside(ANNOTATION_MATCHER) - @m.call_if_not_inside(m.ConcatenatedString()) - @m.visit(m.SimpleString()) - def handle_simple_string(self, node: cst.SimpleString) -> None: - self.handle_any_string(node) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/codemod/visitors/_gather_unused_imports.py new/libcst-0.3.10/libcst/codemod/visitors/_gather_unused_imports.py --- old/libcst-0.3.9/libcst/codemod/visitors/_gather_unused_imports.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/codemod/visitors/_gather_unused_imports.py 2020-08-14 21:02:21.000000000 +0200 @@ -4,20 +4,23 @@ # LICENSE file in the root directory of this source tree. # -from typing import Iterable, Set, Tuple, Union +from typing import Collection, Iterable, Set, Tuple, Union import libcst as cst -import libcst.matchers as m from libcst.codemod._context import CodemodContext from libcst.codemod._visitor import ContextAwareVisitor from libcst.codemod.visitors._gather_exports import GatherExportsVisitor from libcst.codemod.visitors._gather_string_annotation_names import ( + FUNCS_CONSIDERED_AS_STRING_ANNOTATIONS, GatherNamesFromStringAnnotationsVisitor, ) from libcst.metadata import ProviderT, ScopeProvider from libcst.metadata.scope_provider import _gen_dotted_names +MODULES_IGNORED_BY_DEFAULT = {"__future__"} + + class GatherUnusedImportsVisitor(ContextAwareVisitor): """ Collects all imports from a module not directly used in the same module. @@ -37,9 +40,16 @@ ScopeProvider, ) - def __init__(self, context: CodemodContext) -> None: + def __init__( + self, + context: CodemodContext, + ignored_modules: Collection[str] = MODULES_IGNORED_BY_DEFAULT, + typing_functions: Collection[str] = FUNCS_CONSIDERED_AS_STRING_ANNOTATIONS, + ) -> None: super().__init__(context) + self._ignored_modules: Collection[str] = ignored_modules + self._typing_functions = typing_functions self._string_annotation_names: Set[str] = set() self._exported_names: Set[str] = set() #: Contains a set of (alias, parent_import) pairs that are not used @@ -52,18 +62,27 @@ export_collector = GatherExportsVisitor(self.context) node.visit(export_collector) self._exported_names = export_collector.explicit_exported_objects - annotation_visitor = GatherNamesFromStringAnnotationsVisitor(self.context) + annotation_visitor = GatherNamesFromStringAnnotationsVisitor( + self.context, typing_functions=self._typing_functions + ) node.visit(annotation_visitor) self._string_annotation_names = annotation_visitor.names return True - @m.visit( - m.Import() - | m.ImportFrom( - module=m.DoesNotMatch(m.Name("__future__")), - names=m.DoesNotMatch(m.ImportStar()), - ) - ) + def visit_Import(self, node: cst.Import) -> bool: + self.handle_import(node) + return False + + def visit_ImportFrom(self, node: cst.ImportFrom) -> bool: + module = node.module + if ( + not isinstance(node.names, cst.ImportStar) + and module is not None + and module.value not in self._ignored_modules + ): + self.handle_import(node) + return False + def handle_import(self, node: Union[cst.Import, cst.ImportFrom]) -> None: names = node.names assert not isinstance(names, cst.ImportStar) # hello, type checker diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/codemod/visitors/tests/test_gather_exports.py new/libcst-0.3.10/libcst/codemod/visitors/tests/test_gather_exports.py --- old/libcst-0.3.9/libcst/codemod/visitors/tests/test_gather_exports.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/codemod/visitors/tests/test_gather_exports.py 2020-08-15 00:06:19.000000000 +0200 @@ -26,6 +26,10 @@ list_of_str = ["foo", "bar", "baz"] + set_of_str = {"foo", "bar", "baz"} + + tuple_of_str = ("foo", "bar", "baz") + another: List[str] = ["foobar", "foobarbaz"] """ @@ -100,12 +104,34 @@ gatherer = self.gather_exports(code) self.assertEqual(gatherer.explicit_exported_objects, {"bar", "baz"}) - def test_gather_exports_ignore_invalid_3(self) -> None: + def test_gather_exports_ignore_valid_1(self) -> None: + code = """ + from foo import bar + from biz import baz + + __all__ = ["bar", "b""a""z"] + """ + + gatherer = self.gather_exports(code) + self.assertEqual(gatherer.explicit_exported_objects, {"bar", "baz"}) + + def test_gather_exports_ignore_valid_2(self) -> None: + code = """ + from foo import bar + from biz import baz + + __all__, _ = ["bar", "baz"], ["biz"] + """ + + gatherer = self.gather_exports(code) + self.assertEqual(gatherer.explicit_exported_objects, {"bar", "baz"}) + + def test_gather_exports_ignore_valid_3(self) -> None: code = """ from foo import bar from biz import baz - __all__ = ["bar", "baz", "foo""bar"] + __all__ = exported = ["bar", "baz"] """ gatherer = self.gather_exports(code) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/matchers/_matcher_base.py new/libcst-0.3.10/libcst/matchers/_matcher_base.py --- old/libcst-0.3.9/libcst/matchers/_matcher_base.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/matchers/_matcher_base.py 2020-08-14 21:02:21.000000000 +0200 @@ -7,7 +7,7 @@ import copy import inspect import re -from dataclasses import fields +from dataclasses import dataclass, fields from enum import Enum, auto from typing import ( Callable, @@ -932,6 +932,34 @@ return cast(_OtherNodeT, _ExtractMatchingNode(matcher, name)) +def _matches_zero_nodes( + matcher: Union[ + BaseMatcherNode, + _BaseWildcardNode, + MatchIfTrue[Callable[[object], bool]], + _BaseMetadataMatcher, + DoNotCareSentinel, + ] +) -> bool: + if isinstance(matcher, AtLeastN) and matcher.n == 0: + return True + if isinstance(matcher, AtMostN): + return True + if isinstance(matcher, _ExtractMatchingNode): + return _matches_zero_nodes(matcher.matcher) + return False + + +@dataclass(frozen=True) +class _SequenceMatchesResult: + sequence_capture: Optional[ + Dict[str, Union[libcst.CSTNode, Sequence[libcst.CSTNode]]] + ] + matched_nodes: Optional[ + Union[libcst.CSTNode, MaybeSentinel, Sequence[libcst.CSTNode]] + ] + + def _sequence_matches( # noqa: C901 nodes: Sequence[Union[MaybeSentinel, libcst.CSTNode]], matchers: Sequence[ @@ -944,30 +972,34 @@ ] ], metadata_lookup: Callable[[meta.ProviderT, libcst.CSTNode], object], -) -> Optional[Dict[str, Union[libcst.CSTNode, Sequence[libcst.CSTNode]]]]: +) -> _SequenceMatchesResult: if not nodes and not matchers: - # Base case, empty lists are alwatys matches - return {} + # Base case, empty lists are always matches + return _SequenceMatchesResult({}, None) if not nodes and matchers: # Base case, we have one or more matcher that wasn't matched - return ( - {} - if all( - (isinstance(m, AtLeastN) and m.n == 0) or isinstance(m, AtMostN) - for m in matchers + if all(_matches_zero_nodes(m) for m in matchers): + return _SequenceMatchesResult( + {m.name: () for m in matchers if isinstance(m, _ExtractMatchingNode)}, + (), ) - else None - ) + else: + return _SequenceMatchesResult(None, None) if nodes and not matchers: # Base case, we have nodes left that don't match any matcher - return None + return _SequenceMatchesResult(None, None) # Recursive case, nodes and matchers LHS matches node = nodes[0] matcher = matchers[0] if isinstance(matcher, DoNotCareSentinel): # We don't care about the value for this node. - return _sequence_matches(nodes[1:], matchers[1:], metadata_lookup) + return _SequenceMatchesResult( + _sequence_matches( + nodes[1:], matchers[1:], metadata_lookup + ).sequence_capture, + node, + ) elif isinstance(matcher, _BaseWildcardNode): if isinstance(matcher, AtMostN): if matcher.n > 0: @@ -977,18 +1009,24 @@ nodes[0], matcher.matcher, metadata_lookup ) if attribute_capture is not None: - sequence_capture = _sequence_matches( + result = _sequence_matches( nodes[1:], [AtMostN(matcher.matcher, n=matcher.n - 1), *matchers[1:]], metadata_lookup, ) - if sequence_capture is not None: - return {**attribute_capture, **sequence_capture} + if result.sequence_capture is not None: + return _SequenceMatchesResult( + {**attribute_capture, **result.sequence_capture}, + (node, *result.matched_nodes), + ) # Finally, assume that this does not match the current node. # Consume the matcher but not the node. - sequence_capture = _sequence_matches(nodes, matchers[1:], metadata_lookup) - if sequence_capture is not None: - return sequence_capture + return _SequenceMatchesResult( + _sequence_matches( + nodes, matchers[1:], metadata_lookup + ).sequence_capture, + (), + ) elif isinstance(matcher, AtLeastN): if matcher.n > 0: # Only match if we can consume one of the matches, since we still @@ -997,13 +1035,17 @@ nodes[0], matcher.matcher, metadata_lookup ) if attribute_capture is not None: - sequence_capture = _sequence_matches( + result = _sequence_matches( nodes[1:], [AtLeastN(matcher.matcher, n=matcher.n - 1), *matchers[1:]], metadata_lookup, ) - if sequence_capture is not None: - return {**attribute_capture, **sequence_capture} + if result.sequence_capture is not None: + return _SequenceMatchesResult( + {**attribute_capture, **result.sequence_capture}, + (node, *result.matched_nodes), + ) + return _SequenceMatchesResult(None, None) else: # First, assume that this does match a node (greedy). # Consume one node since it matched this matcher. @@ -1011,45 +1053,52 @@ nodes[0], matcher.matcher, metadata_lookup ) if attribute_capture is not None: - sequence_capture = _sequence_matches( - nodes[1:], matchers, metadata_lookup - ) - if sequence_capture is not None: - return {**attribute_capture, **sequence_capture} + result = _sequence_matches(nodes[1:], matchers, metadata_lookup) + if result.sequence_capture is not None: + return _SequenceMatchesResult( + {**attribute_capture, **result.sequence_capture}, + (node, *result.matched_nodes), + ) # Now, assume that this does not match the current node. # Consume the matcher but not the node. - sequence_capture = _sequence_matches( - nodes, matchers[1:], metadata_lookup + return _SequenceMatchesResult( + _sequence_matches( + nodes, matchers[1:], metadata_lookup + ).sequence_capture, + (), ) - if sequence_capture is not None: - return sequence_capture else: # There are no other types of wildcard consumers, but we're making # pyre happy with that fact. raise Exception(f"Logic error unrecognized wildcard {type(matcher)}!") elif isinstance(matcher, _ExtractMatchingNode): # See if the raw matcher matches. If it does, capture the sequence we matched and store it. - sequence_capture = _sequence_matches( + result = _sequence_matches( nodes, [matcher.matcher, *matchers[1:]], metadata_lookup ) - if sequence_capture is not None: - return { - # Our own match capture comes first, since we wnat to allow the same - # name later in the sequence to override us. - matcher.name: nodes, - **sequence_capture, - } - return None + if result.sequence_capture is not None: + return _SequenceMatchesResult( + { + # Our own match capture comes first, since we wnat to allow the same + # name later in the sequence to override us. + matcher.name: result.matched_nodes, + **result.sequence_capture, + }, + result.matched_nodes, + ) + return _SequenceMatchesResult(None, None) match_capture = _matches(node, matcher, metadata_lookup) if match_capture is not None: # These values match directly - sequence_capture = _sequence_matches(nodes[1:], matchers[1:], metadata_lookup) - if sequence_capture is not None: - return {**match_capture, **sequence_capture} + result = _sequence_matches(nodes[1:], matchers[1:], metadata_lookup) + if result.sequence_capture is not None: + return _SequenceMatchesResult( + {**match_capture, **result.sequence_capture}, node + ) # Failed recursive case, no match - return None + return _SequenceMatchesResult(None, None) _AttributeValueT = Optional[Union[MaybeSentinel, libcst.CSTNode, str, bool]] @@ -1110,9 +1159,9 @@ for m in matcher.options: if isinstance(m, collections.abc.Sequence): # Should match the sequence of requested nodes - sequence_capture = _sequence_matches(node, m, metadata_lookup) - if sequence_capture is not None: - return sequence_capture + result = _sequence_matches(node, m, metadata_lookup) + if result.sequence_capture is not None: + return result.sequence_capture elif isinstance(m, MatchIfTrue): return {} if matcher.func(node) else None elif isinstance(matcher, AllOf): @@ -1121,10 +1170,10 @@ for m in matcher.options: if isinstance(m, collections.abc.Sequence): # Should match the sequence of requested nodes - sequence_capture = _sequence_matches(node, m, metadata_lookup) - if sequence_capture is None: + result = _sequence_matches(node, m, metadata_lookup) + if result.sequence_capture is None: return None - all_captures = {**all_captures, **sequence_capture} + all_captures = {**all_captures, **result.sequence_capture} elif isinstance(m, MatchIfTrue): return {} if matcher.func(node) else None else: @@ -1150,7 +1199,7 @@ matcher, ), metadata_lookup, - ) + ).sequence_capture # We exhausted our possibilities, there's no match return None diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/matchers/tests/test_extract.py new/libcst-0.3.10/libcst/matchers/tests/test_extract.py --- old/libcst-0.3.9/libcst/matchers/tests/test_extract.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/matchers/tests/test_extract.py 2020-08-14 21:02:21.000000000 +0200 @@ -322,6 +322,34 @@ ) self.assertEqual(nodes, {}) + def test_extract_optional_wildcard_head(self) -> None: + expression = cst.parse_expression("[3]") + nodes = m.extract( + expression, + m.List( + elements=[ + m.SaveMatchedNode(m.ZeroOrMore(), "head1"), + m.SaveMatchedNode(m.ZeroOrMore(), "head2"), + m.Element(value=m.Integer(value="3")), + ] + ), + ) + self.assertEqual(nodes, {"head1": (), "head2": ()}) + + def test_extract_optional_wildcard_tail(self) -> None: + expression = cst.parse_expression("[3]") + nodes = m.extract( + expression, + m.List( + elements=[ + m.Element(value=m.Integer(value="3")), + m.SaveMatchedNode(m.ZeroOrMore(), "tail1"), + m.SaveMatchedNode(m.ZeroOrMore(), "tail2"), + ] + ), + ) + self.assertEqual(nodes, {"tail1": (), "tail2": ()}) + def test_extract_optional_wildcard_present(self) -> None: expression = cst.parse_expression("a + b[c], d(e, f * g, h.i.j)") nodes = m.extract( @@ -404,3 +432,25 @@ ), ) self.assertIsNone(nodes) + + def test_extract_sequence_multiple_wildcards(self) -> None: + expression = cst.parse_expression("1, 2, 3, 4") + nodes = m.extract( + expression, + m.Tuple( + elements=( + m.SaveMatchedNode(m.ZeroOrMore(), "head"), + m.SaveMatchedNode(m.Element(value=m.Integer(value="3")), "element"), + m.SaveMatchedNode(m.ZeroOrMore(), "tail"), + ) + ), + ) + tuple_elements = cst.ensure_type(expression, cst.Tuple).elements + self.assertEqual( + nodes, + { + "head": tuple(tuple_elements[:2]), + "element": tuple_elements[2], + "tail": tuple(tuple_elements[3:]), + }, + ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/metadata/scope_provider.py new/libcst-0.3.10/libcst/metadata/scope_provider.py --- old/libcst-0.3.9/libcst/metadata/scope_provider.py 2020-08-07 20:39:26.000000000 +0200 +++ new/libcst-0.3.10/libcst/metadata/scope_provider.py 2020-08-15 00:27:00.000000000 +0200 @@ -63,11 +63,14 @@ #: assignment. scope: "Scope" + is_annotation: bool + __assignments: Set["BaseAssignment"] - def __init__(self, node: cst.Name, scope: "Scope") -> None: + def __init__(self, node: cst.Name, scope: "Scope", is_annotation: bool) -> None: self.node = node self.scope = scope + self.is_annotation = is_annotation self.__assignments = set() def __hash__(self) -> int: @@ -640,6 +643,7 @@ self.scope: Scope = GlobalScope() self.__deferred_accesses: List[Tuple[Access, Optional[cst.Attribute]]] = [] self.__top_level_attribute_stack: List[Optional[cst.Attribute]] = [None] + self.__in_annotation: Set[Union[cst.Call, cst.Annotation]] = set() @contextmanager def _new_scope( @@ -695,17 +699,60 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: self.__top_level_attribute_stack.append(None) + if any( + qn.name == "typing.TypeVar" + for qn in self.scope.get_qualified_names_for(node) + ): + node.func.visit(self) + self.__in_annotation.add(node) + for arg in node.args[1:]: + arg.visit(self) + return False + return True def leave_Call(self, original_node: cst.Call) -> None: self.__top_level_attribute_stack.pop() + self.__in_annotation.discard(original_node) + + def visit_Annotation(self, node: cst.Annotation) -> Optional[bool]: + self.__in_annotation.add(node) + + def leave_Annotation(self, original_node: cst.Annotation) -> None: + self.__in_annotation.discard(original_node) + + def visit_SimpleString(self, node: cst.SimpleString) -> Optional[bool]: + self._handle_string_annotation(node) + return False + + def visit_ConcatenatedString(self, node: cst.ConcatenatedString) -> Optional[bool]: + self._handle_string_annotation(node) + return False + + def _handle_string_annotation( + self, node: Union[cst.SimpleString, cst.ConcatenatedString] + ) -> None: + if self.__in_annotation: + value = node.evaluated_value + if value: + mod = cst.parse_module(value) + mod.visit(self) + + def visit_Subscript(self, node: cst.Subscript) -> Optional[bool]: + if any( + qn.name in ("typing.Literal", "typing_extensions.Literal") + for qn in self.scope.get_qualified_names_for(node.value) + ): + node.value.visit(self) + return False + return True def visit_Name(self, node: cst.Name) -> Optional[bool]: # not all Name have ExpressionContext context = self.provider.get_metadata(ExpressionContextProvider, node, None) if context == ExpressionContext.STORE: self.scope.record_assignment(node.value, node) - elif context in (ExpressionContext.LOAD, ExpressionContext.DEL): - access = Access(node, self.scope) + elif context in (ExpressionContext.LOAD, ExpressionContext.DEL, None): + access = Access(node, self.scope, is_annotation=bool(self.__in_annotation)) self.__deferred_accesses.append( (access, self.__top_level_attribute_stack[-1]) ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/metadata/tests/test_scope_provider.py new/libcst-0.3.10/libcst/metadata/tests/test_scope_provider.py --- old/libcst-0.3.9/libcst/metadata/tests/test_scope_provider.py 2020-08-05 02:26:37.000000000 +0200 +++ new/libcst-0.3.10/libcst/metadata/tests/test_scope_provider.py 2020-08-15 00:06:19.000000000 +0200 @@ -1010,6 +1010,59 @@ self.assertEqual(len(set(scopes.values())), 3) + def test_annotation_access(self) -> None: + m, scopes = get_scope_metadata_provider( + """ + from typing import Literal, TypeVar + from a import A, B, C, D, E, F + def x(a: A): + pass + def y(b: "B"): + pass + def z(c: Literal["C"]): + pass + DType = TypeVar("DType", bound=D) + EType = TypeVar("EType", bound="E") + FType = TypeVar("F") + """ + ) + imp = ensure_type( + ensure_type(m.body[1], cst.SimpleStatementLine).body[0], cst.ImportFrom + ) + scope = scopes[imp] + + assignment = list(scope["A"])[0] + self.assertIsInstance(assignment, Assignment) + self.assertEqual(len(assignment.references), 1) + references = list(assignment.references) + self.assertTrue(references[0].is_annotation) + + assignment = list(scope["B"])[0] + self.assertIsInstance(assignment, Assignment) + self.assertEqual(len(assignment.references), 1) + references = list(assignment.references) + self.assertTrue(references[0].is_annotation) + + assignment = list(scope["C"])[0] + self.assertIsInstance(assignment, Assignment) + self.assertEqual(len(assignment.references), 0) + + assignment = list(scope["D"])[0] + self.assertIsInstance(assignment, Assignment) + self.assertEqual(len(assignment.references), 1) + references = list(assignment.references) + self.assertTrue(references[0].is_annotation) + + assignment = list(scope["E"])[0] + self.assertIsInstance(assignment, Assignment) + self.assertEqual(len(assignment.references), 1) + references = list(assignment.references) + self.assertTrue(references[0].is_annotation) + + assignment = list(scope["F"])[0] + self.assertIsInstance(assignment, Assignment) + self.assertEqual(len(assignment.references), 0) + def test_node_of_scopes(self) -> None: m, scopes = get_scope_metadata_provider( """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst/metadata/type_inference_provider.py new/libcst-0.3.10/libcst/metadata/type_inference_provider.py --- old/libcst-0.3.9/libcst/metadata/type_inference_provider.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/libcst/metadata/type_inference_provider.py 2020-08-14 21:02:21.000000000 +0200 @@ -57,7 +57,7 @@ root_path: Path, paths: List[str], timeout: Optional[int] ) -> Mapping[str, object]: params = ",".join(f"path='{root_path / path}'" for path in paths) - cmd = f'''pyre query "types({params})"''' + cmd = f'''pyre --noninteractive query "types({params})"''' try: stdout, stderr, return_code = run_command(cmd, timeout=timeout) except subprocess.TimeoutExpired as exc: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst.egg-info/PKG-INFO new/libcst-0.3.10/libcst.egg-info/PKG-INFO --- old/libcst-0.3.9/libcst.egg-info/PKG-INFO 2020-08-07 20:46:04.000000000 +0200 +++ new/libcst-0.3.10/libcst.egg-info/PKG-INFO 2020-08-17 20:25:13.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: libcst -Version: 0.3.9 +Version: 0.3.10 Summary: A concrete syntax tree with AST-like properties for Python 3.5, 3.6, 3.7 and 3.8 programs. Home-page: https://github.com/Instagram/LibCST License: MIT diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/libcst.egg-info/SOURCES.txt new/libcst-0.3.10/libcst.egg-info/SOURCES.txt --- old/libcst-0.3.9/libcst.egg-info/SOURCES.txt 2020-08-07 20:46:04.000000000 +0200 +++ new/libcst-0.3.10/libcst.egg-info/SOURCES.txt 2020-08-17 20:25:13.000000000 +0200 @@ -192,6 +192,7 @@ libcst/codemod/commands/tests/test_strip_strings_from_types.py libcst/codemod/commands/tests/test_unnecessary_format_string.py libcst/codemod/tests/__init__.py +libcst/codemod/tests/codemod_formatter_error_input.py.txt libcst/codemod/tests/test_cli.py libcst/codemod/tests/test_codemod.py libcst/codemod/tests/test_codemod_cli.py diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libcst-0.3.9/setup.py new/libcst-0.3.10/setup.py --- old/libcst-0.3.9/setup.py 2020-08-04 22:04:41.000000000 +0200 +++ new/libcst-0.3.10/setup.py 2020-08-14 21:02:21.000000000 +0200 @@ -39,7 +39,11 @@ url="https://github.com/Instagram/LibCST";, license="MIT", packages=setuptools.find_packages(), - package_data={"libcst": ["py.typed"]}, + package_data={ + "libcst": ["py.typed"], + "libcst.tests.pyre": ["*"], + "libcst.codemod.tests": ["*"], + }, test_suite="libcst", python_requires=">=3.6", install_requires=[
