bpo-16079: Add the duplicate_meth_defs.py tool as a pre-commit check on Travis

Misc/NEWS.d/next/Tools-Demos/2019-04-20-15-29-04.bpo-16079.KRYhYf.rst

@@ -0,0 +1,5 @@
+Add the `duplicate_meth_defs.py` tool as a pre-commit check on Travis. The
+`duplicate_meth_defs.py` tool reports duplicate method definitions in a
+given list of files or directories. One must add an entry in
+`Tools/scripts/duplicates_ignored.txt` to by-pass the pre-commit check when
+the reported duplicate is a false positive.

Tools/scripts/duplicate_meth_defs.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""Print duplicate method definitions in the same scope."""
+
+import sys
+import os
+import ast
+import re
+import argparse
+import traceback
+from tokenize import open as tokenize_open
+from contextlib import contextmanager
+
+class ClassFunctionVisitor(ast.NodeVisitor):
+    def __init__(self, duplicates):
+        self.duplicates = duplicates
+        self.scopes = []
+
+    @contextmanager
+    def _visit_new_scope(self, node):
+        prev_scope = self.scopes[-1] if len(self.scopes) else None
+        new_scope = Scope(node, prev_scope)
+        try:
+            yield prev_scope, new_scope
+        finally:
+            self.scopes.append(new_scope)
+            self.generic_visit(node)
+            self.scopes.pop()
+
+    def visit_FunctionDef(self, node):
+        with self._visit_new_scope(node) as (previous, new):
+            if previous is not None and previous.type == 'ClassDef':
+                new.check_duplicate_meth(self.duplicates)
+
+    def visit_ClassDef(self, node):
+        with self._visit_new_scope(node) as (previous, new):
+            pass
+
+    visit_AsyncFunctionDef = visit_ClassDef
+
+class Scope:
+    def __init__(self, node, previous):
+        self.node = node
+        self.previous = previous
+        self.type = node.__class__.__name__
+        self.methods = []
+        if previous is None:
+            self.id = [node.name]
+        else:
+            self.id = previous.id + [node.name]
+
+    def check_duplicate_meth(self, duplicates):
+        node = self.node
+        name = node.name
+        # Skip property and generic methods.
+        if 'decorator_list' in node._fields and node.decorator_list:
+            for decorator in node.decorator_list:
+                if (isinstance(decorator, ast.Attribute) and
+                        'attr' in decorator._fields and
+                        decorator.attr in
+                        ('setter', 'getter', 'deleter', 'register')):
+                    return
+                # A functools single-dispatch method.
+                elif (isinstance(decorator, ast.Call) and
+                        'func' in decorator._fields and
+                        'attr' in decorator.func._fields and
+                        decorator.func.attr == 'register'):
+                    return
+
+        if name in self.previous.methods:
+            duplicates.append((self.id, node.lineno))
+        else:
+            self.previous.methods.append(name)
+
+def get_duplicates(source, fname):
+    duplicates = []
+    tree = ast.parse(source, fname)
+    ClassFunctionVisitor(duplicates).visit(tree)
+    return duplicates
+
+def duplicates_exist(source, fname='<unknown>', dups_to_ignore=[]):
+    """Print duplicates and return True if duplicates exist.
+
+    >>> test_basic = '''
+    ... class C:
+    ...     def foo(self): pass
+    ...     def foo(self): pass
+    ... '''
+    >>> duplicates_exist(test_basic)
+    <unknown>:4 C.foo
+    True
+
+    >>> test_ignore = '''
+    ... class C:
+    ...     def foo(self): pass
+    ...     def foo(self): pass
+    ... '''
+    >>> duplicates_exist(test_ignore, dups_to_ignore=['C.foo'])
+    False
+
+    >>> test_nested = '''
+    ... def foo():
+    ...     def bar(): pass
+    ...
+    ...     class C:
+    ...         def foo(self): pass
+    ...         def bar(self): pass
+    ...         def bar(self): pass
+    ...
+    ...         class D:
+    ...             def foo(self): pass
+    ...             def bar(self): pass
+    ...             def bar(self): pass
+    ...
+    ...         def bar(self): pass
+    ...
+    ...     def bar(): pass
+    ... '''
+    >>> duplicates_exist(test_nested)
+    <unknown>:8 foo.C.bar
+    <unknown>:13 foo.C.D.bar
+    <unknown>:15 foo.C.bar
+    True
+
+    >>> test_generic = '''
+    ... class A:
+    ...     @functools.singledispatchmethod
+    ...     def t(self, arg):
+    ...         self.arg = "base"
+    ...     @t.register(int)
+    ...     def _(self, arg):
+    ...         self.arg = "int"
+    ...     @t.register(str)
+    ...     def _(self, arg):
+    ...         self.arg = "str"
+    ...
+    ... class C:
+    ...     @property
+    ...     def foo(self): pass
+    ...
+    ...     @foo.setter
+    ...     def foo(self, a, b): pass
+    ...
+    ...     def bar(self): pass
+    ...     def bar(self): pass
+    ...
+    ...     class D:
+    ...         @property
+    ...         def foo(self): pass
+    ...
+    ...         @foo.setter
+    ...         def foo(self, a, b): pass
+    ...
+    ...         def bar(self): pass
+    ...         def bar(self): pass
+    ...         def foo(self): pass
+    ... '''
+    >>> duplicates_exist(test_generic)
+    <unknown>:21 C.bar
+    <unknown>:31 C.D.bar
+    <unknown>:32 C.D.foo
+    True
+    """
+
+    cnt = 0
+    duplicates = get_duplicates(source, fname)
+    if duplicates:
+        for (id, lineno) in duplicates:
+            id = '.'.join(id)
+            if dups_to_ignore is None or id not in dups_to_ignore:
+                print('%s:%d %s' % (fname, lineno, id))
+                cnt += 1
+    return False if cnt == 0 else True
+
+def iter_modules(paths):
+    for f in paths:
+        if os.path.isdir(f):
+            for path, dirs, filenames in os.walk(f):
+                for fname in filenames:
+                    if os.path.splitext(fname)[1] == '.py':
+                        yield os.path.normpath(os.path.join(path, fname))
+        else:
+            yield os.path.normpath(f)
+
+def ignored_dups(f):
+    """Parse the ignore duplicates file."""
+
+    # Comments or empty lines.
+    comments = re.compile(r'^\s*#.*$|^\s*$')
+
+    # Mapping of filename to the list of duplicates to ignore.
+    ignored = {}
+    if f is None:
+        return ignored
+    with f:
+        for item in (l.rstrip(': \n\t').split(':') for l in f
+                if not comments.match(l)):
+            path = item[0].strip()
+            if not path:
+                continue
+            path = os.path.normpath(path)
+            if len(item) == 1:
+                ignored[path] = []
+            else:
+                dupname = item[1].strip()
+                if path in ignored:
+                    ignored[path].append(dupname)
+                else:
+                    ignored[path] = [dupname]
+    return ignored
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(description=__doc__.strip())
+    parser.add_argument('files', metavar='path', nargs='*',
+            help='python module or directory pathname - when a directory, '
+            'the python modules in the directory tree are searched '
+            'recursively for duplicates.')
+    parser.add_argument('-i', '--ignore', metavar='fname', type=open,
+            help='ignore duplicates or files listed in <fname>, see'
+            ' Tools/scripts/duplicates_ignored.txt for the'
+            ' specification of the format of the entries in this file')
+    parser.add_argument('-x', '--exclude', metavar='prefixes',
+            help="comma separated list of path names prefixes to exclude"
+            " (default: '%(default)s')")
+    parser.add_argument('-t', '--test', action='store_true',
+            help='run the doctests')
+    return parser.parse_args(argv)
+
+def _main(args):
+    ignored = ignored_dups(args.ignore)
+    if args.exclude:
+        prefixes = [p.strip() for p in args.exclude.split(',')]
+    else:
+        prefixes = []
+
+    # Find duplicates.
+    rc = 0
+    for fname in iter_modules(args.files):
+        def excluded(f):
+            for p in prefixes:
+                if f.startswith(p):
+                    return True
+
+        # Skip files whose prefix is excluded or that are configured in the
+        # '--ignore' file.
+        if excluded(fname):
+            continue
+        dups_to_ignore = ignored.get(fname)
+        if dups_to_ignore == []:
+            continue
+
+        try:
+            with tokenize_open(fname) as f:
+                if duplicates_exist(f.read(), fname, dups_to_ignore):
+                    rc = 1
+        except (UnicodeDecodeError, SyntaxError) as e:
+            print('%s: %s' % (fname, e), file=sys.stderr)
+            traceback.print_tb(sys.exc_info()[2])
+            rc = 1
+
+    return rc
+
+def main(argv):
+    args = parse_args(argv)
+    if args.test:
+        import doctest
+        doctest.testmod()
+    else:
+        return _main(args)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

Tools/scripts/duplicates_ignored.txt

@@ -0,0 +1,71 @@
+# Files or duplicates ignored by Tools/scripts/duplicate_meth_defs.py
+# when run with the '--ignore' command line option.
+#
+# Format:
+#           filename[: mfqn]
+#
+# The whole file is ignored when mfqn (method fully qualified name) is
+# missing. Use instead the '--exclude' command line option of
+# duplicate_meth_defs.py to ignore a directory tree.
+
+# Files with bad encoding or bad syntax.
+Lib/lib2to3/tests/data/bom.py
+Lib/lib2to3/tests/data/crlf.py
+Lib/lib2to3/tests/data/different_encoding.py
+Lib/lib2to3/tests/data/false_encoding.py
+Lib/lib2to3/tests/data/py2_test_grammar.py
+Lib/test/bad_coding.py
+Lib/test/bad_coding2.py
+Lib/test/badsyntax_pep3120.py
+Lib/test/badsyntax_3131.py
+Tools/test2to3/maintest.py
+Tools/test2to3/test2to3/hello.py
+Tools/test2to3/test/test_foo.py
+
+# False positives.
+Lib/multiprocessing/connection.py: Connection._close
+Lib/pathlib.py: _NormalAccessor.symlink
+Lib/socket.py: socket._sendfile_use_sendfile
+Lib/socket.py: socket.get_inheritable
+Lib/socket.py: socket.set_inheritable
+Lib/ssl.py: SSLContext.hostname_checks_common_name
+Lib/subprocess.py: Popen._get_handles
+Lib/subprocess.py: Popen._execute_child
+Lib/subprocess.py: Popen._internal_poll
+Lib/subprocess.py: Popen._wait
+Lib/subprocess.py: Popen._communicate
+Lib/subprocess.py: Popen.send_signal
+Lib/subprocess.py: Popen.terminate
+Lib/tempfile.py: _TemporaryFileCloser.close
+Lib/test/test_socket.py: InterruptedTimeoutBase.setAlarm
+Lib/test/test_pathlib.py: _BasePathTest.dirlink
+
+
+# Duplicates that must be fixed and that are temporarily ignored.
+# Issue bpo-19113
+Lib/ctypes/test/test_functions.py: FunctionTestCase.test_errors
+
+# Issue bpo-36678
+Lib/test/test_dataclasses.py: TestCase.test_helper_asdict_builtin_containers
+Lib/test/test_dataclasses.py: TestCase.test_helper_astuple_builtin_containers
+Lib/test/test_dataclasses.py: TestCase.test_not_tuple
+Lib/test/test_dataclasses.py: TestReplace.test_recursive_repr_two_attrs
+
+# Issue bpo-36679
+Lib/test/test_genericclass.py: TestClassGetitem.test_class_getitem
+
+# Issue bpo-19119
+Lib/test/test_heapq.py: TestErrorHandling.test_get_only
+
+# Issue bpo-36680
+Lib/test/test_importlib/test_util.py: PEP3147Tests.test_source_from_cache_path_like_arg
+
+# Issue bpo-36681
+Lib/test/test_logging.py: BuiltinLevelsTest.test_regression_29220
+
+# Issue bpo-36682
+Lib/test/test_sys_setprofile.py: ProfileSimulatorTestCase.test_unbound_method_invalid_args
+Lib/test/test_sys_setprofile.py: ProfileSimulatorTestCase.test_unbound_method_no_args
+
+# Issue bpo-36683
+Lib/test/test_utf8_mode.py: UTF8ModeTests.test_io_encoding