Not working with Python 3.8 (due to chances in AST module related to Python Issue 32892)

[aginiewicz]

Two test cases:

import pycode_similar
str1="""\
def f():
    return "1"
"""
pycode_similar.detect([str1, str1])

import pycode_similar
str2='{"1": 1}'
pycode_similar.detect([str2, str2])

Both give exception:

~/.local/lib/python3.8/site-packages/pycode_similar.py in detect(pycode_string_list, diff_method)
    430         root_node = ast.parse(code_str)
    431         collector = FuncNodeCollector()
--> 432         collector.visit(root_node)
    433         code_utf8_lines = code_str.splitlines(True)
    434         func_info = [FuncInfo(n, code_utf8_lines) for n in collector.get_function_nodes()]

/usr/lib/python3.8/ast.py in visit(self, node)
    358         method = 'visit_' + node.__class__.__name__
    359         visitor = getattr(self, method, self.generic_visit)
--> 360         return visitor(node)
    361 
    362     def generic_visit(self, node):

~/.local/lib/python3.8/site-packages/pycode_similar.py in generic_visit(self, node)
     74         self._last_node_lineno = max(getattr(node, 'lineno', -1), self._last_node_lineno)
     75         self._mark_docstring_sub_nodes(node)
---> 76         return super(FuncNodeCollector, self).generic_visit(node)
     77 
     78     def visit_Str(self, node):

/usr/lib/python3.8/ast.py in generic_visit(self, node)
    434                 for value in old_value:
    435                     if isinstance(value, AST):
--> 436                         value = self.visit(value)
    437                         if value is None:
    438                             continue

/usr/lib/python3.8/ast.py in visit(self, node)
    358         method = 'visit_' + node.__class__.__name__
    359         visitor = getattr(self, method, self.generic_visit)
--> 360         return visitor(node)
    361 
    362     def generic_visit(self, node):

~/.local/lib/python3.8/site-packages/pycode_similar.py in visit_FunctionDef(self, node)
    126         self._func_nodes.append(node)
    127         count = self._node_count
--> 128         self.generic_visit(node)
    129         node.endlineno = self._last_node_lineno
    130         node.nsubnodes = self._node_count - count

~/.local/lib/python3.8/site-packages/pycode_similar.py in generic_visit(self, node)
     74         self._last_node_lineno = max(getattr(node, 'lineno', -1), self._last_node_lineno)
     75         self._mark_docstring_sub_nodes(node)
---> 76         return super(FuncNodeCollector, self).generic_visit(node)
     77 
     78     def visit_Str(self, node):

/usr/lib/python3.8/ast.py in generic_visit(self, node)
    434                 for value in old_value:
    435                     if isinstance(value, AST):
--> 436                         value = self.visit(value)
    437                         if value is None:
    438                             continue

/usr/lib/python3.8/ast.py in visit(self, node)
    358         method = 'visit_' + node.__class__.__name__
    359         visitor = getattr(self, method, self.generic_visit)
--> 360         return visitor(node)
    361 
    362     def generic_visit(self, node):

~/.local/lib/python3.8/site-packages/pycode_similar.py in generic_visit(self, node)
     74         self._last_node_lineno = max(getattr(node, 'lineno', -1), self._last_node_lineno)
     75         self._mark_docstring_sub_nodes(node)
---> 76         return super(FuncNodeCollector, self).generic_visit(node)
     77 
     78     def visit_Str(self, node):

/usr/lib/python3.8/ast.py in generic_visit(self, node)
    443                 old_value[:] = new_values
    444             elif isinstance(old_value, AST):
--> 445                 new_node = self.visit(old_value)
    446                 if new_node is None:
    447                     delattr(node, field)

/usr/lib/python3.8/ast.py in visit(self, node)
    358         method = 'visit_' + node.__class__.__name__
    359         visitor = getattr(self, method, self.generic_visit)
--> 360         return visitor(node)
    361 
    362     def generic_visit(self, node):

/usr/lib/python3.8/ast.py in visit_Constant(self, node)
    388                 warnings.warn(f"{method} is deprecated; add visit_Constant",
    389                               PendingDeprecationWarning, 2)
--> 390                 return visitor(node)
    391         return self.generic_visit(node)
    392 

~/.local/lib/python3.8/site-packages/pycode_similar.py in visit_Str(self, node)
     77 
     78     def visit_Str(self, node):
---> 79         del node.s
     80         self.generic_visit(node)
     81         return node

AttributeError: can't delete attribute

[aginiewicz]

A quick look suggests that this is due to fact, that Str.s is now property referencing Constant.value after https://bugs.python.org/issue32892 was implemented. Method visit_Str is one of methods deprecated in Python 3.8 by this update. visit_Constant should be used instead. See https://github.com/python/cpython/pull/9445/files#diff-2cbfaa4a57fa843e2535720e9adaea77R324 for relevant part of code that I believe causes this exception.

[aginiewicz]

The visit_Constant is available since Python 3.6 - https://bugs.python.org/issue26146 - but looking at default visit_Constant implementation, I guess only this handler needs to be implemented (see https://bugs.python.org/issue36917 and https://github.com/python/cpython/pull/15509/files#diff-2cbfaa4a57fa843e2535720e9adaea77R363) and old ones might be left in place (they will just be unused by Python older than 3.6).

[aginiewicz]

Thanks for quick release!

Btw, I've create an Arch Linux package in AUR for this: https://aur.archlinux.org/packages/python-pycode-similar/ - so it can be installed/updated with the OS. Wasn't able to do this before, because Python 3.8 is default for some time in Arch already. Thought you might be interested.

Cheers,
Andrzej.