# Copyright (c) 2015-2024 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
"""
Creates Python source code from an decompyle3 parse tree,
and indexes fragments which can be accessed by instruction offset
address.
See https://github.com/rocky/python-decompyle3/wiki/Table-driven-semantic-actions.
for a more complete explanation, nicely marked up and with examples.
We add some format specifiers here not used in pysource
1. %x
-----
%x takes an argument (src, (dest...)) and copies all of the range attributes
from src to all nodes under dest.
For example in:
'import': ( '%|import %c%x\n', 2, (2,(0,1)), ),
node 2 range information, it in %c, is copied to nodes 0 and 1. If
1. is a nonterminal, all the nodes under it get node2 range information.
2. %r
-----
%r associates recursively location information for the string that follows
For example in:
'break': ( '%|%rbreak\n', ),
The node will be associated with the text break, excluding the trailing newline.
Note we associate the accumulated text with the node normally, but we just don't
do it recursively which is where offsets are probably located.
2. %b
-----
%b associates the text from the specified index to what we have now.
it takes an integer argument.
For example in:
'importmultiple': ( '%|import%b %c%c\n', 0, 2, 3 ),
The node position 0 will be associated with "import".
"""
# FIXME: DRY code with pysource
import re
from bisect import bisect_right
from collections import namedtuple
from typing import Optional
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG, GenericASTTraversal
from spark_parser.ast import GenericASTTraversalPruningException
from xdis import iscode
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
import decompyle3.parsers.main as python_parser
import decompyle3.parsers.parse_heads as heads
from decompyle3.parsers.main import get_python_parser
from decompyle3.parsers.treenode import SyntaxTree
from decompyle3.scanner import Code, Token, get_scanner
from decompyle3.semantics import pysource
from decompyle3.semantics.check_ast import checker
from decompyle3.semantics.consts import (
INDENT_PER_LEVEL,
MAP,
NONE,
PASS,
PRECEDENCE,
TABLE_DIRECT,
escape,
)
from decompyle3.semantics.make_function36 import make_function36
from decompyle3.semantics.pysource import (
DEFAULT_DEBUG_OPTS,
TREE_DEFAULT_DEBUG,
ParserError,
StringIO,
)
from decompyle3.show import maybe_show_tree
NodeInfo = namedtuple("NodeInfo", "node start finish")
ExtractInfo = namedtuple(
"ExtractInfo",
"lineNo lineStartOffset markerLine selectedLine selectedText nonterminal",
)
TABLE_DIRECT_FRAGMENT = {
"break": ("%|%rbreak\n",),
"continue ": ("%|%rcontinue\n",),
"pass": ("%|%rpass\n",),
"raise_stmt0": ("%|%rraise\n",),
"import": ("%|import %c%x\n", 2, (2, (0, 1))),
"import_cont": (", %c%x", (2, "alias"), (2, (0, 1))),
"import_from": ("%|from %[2]{pattr}%x import %c\n", (2, (0, 1)), (3, "importlist")),
"importfrom": ("%|from %[2]{pattr}%x import %c\n", (2, (0, 1)), 3),
# FIXME only in <= 2.4
"importmultiple": ("%|import%b %c%c\n", 0, 2, 3),
"list_for": (" for %c%x in %c%c", 2, (2, (1,)), 0, 3),
"for": ("%|for%b %c%x in %c:\n%+%c%-\n\n", 0, (3, "store"), (3, (2,)), 1, 4),
"forelsestmt": (
"%|for%b %c%x in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
0,
(3, "store"),
(3, (2,)),
1,
4,
-2,
),
"forelselaststmt": (
"%|for%b %c%x in %c:\n%+%c%-%|else:\n%+%c%-",
0,
(3, "store"),
(3, (2,)),
1,
4,
-2,
),
"forelselaststmtc": (
"%|for%b %c%x in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
0,
(3, "store"),
(3, (2,)),
1,
4,
-2,
),
"whilestmt": ("%|while%b %c:\n%+%c%-\n\n", 0, 1, 2),
"whileelsestmt": ("%|while%b %c:\n%+%c%-%|else:\n%+%c%-\n\n", 0, 1, 2, -2),
"whileelselaststmt": ("%|while%b %c:\n%+%c%-%|else:\n%+%c%-", 0, 1, 2, -2),
}
class FragmentsWalker(pysource.SourceWalker, object):
MAP_DIRECT_FRAGMENT = ()
stacked_params = ("f", "indent", "is_lambda", "_globals")
def __init__(
self,
version: tuple,
scanner,
showast=TREE_DEFAULT_DEBUG,
debug_parser=PARSER_DEFAULT_DEBUG,
compile_mode="exec",
is_pypy=IS_PYPY,
linestarts={},
tolerate_errors=True,
):
pysource.SourceWalker.__init__(
self,
version=version,
out=StringIO(),
scanner=scanner,
showast=showast,
debug_parser=debug_parser,
compile_mode=compile_mode,
is_pypy=is_pypy,
linestarts=linestarts,
tolerate_errors=tolerate_errors,
)
# hide_internal suppresses displaying the additional instructions that sometimes
# exist in code but were not written in the source code.
# An example is:
# __module__ = __name__
# If showing source code we generally don't want to show this. However in fragment
# deparsing we generally do need to see these instructions since we may be stopped
# at one. So here we do not want to suppress showing such instructions.
self.hide_internal = False
self.offsets = {}
self.last_finish = -1
self.is_pypy = is_pypy
# FIXME: is there a better way?
self.TABLE_DIRECT = TABLE_DIRECT.copy()
self.MAP_DIRECT_FRAGMENT = (dict(self.TABLE_DIRECT, **TABLE_DIRECT_FRAGMENT),)
return
f = property(
lambda s: s.params["f"],
lambda s, x: s.params.__setitem__("f", x),
lambda s: s.params.__delitem__("f"),
None,
)
indent = property(
lambda s: s.params["indent"],
lambda s, x: s.params.__setitem__("indent", x),
lambda s: s.params.__delitem__("indent"),
None,
)
is_lambda = property(
lambda s: s.params["is_lambda"],
lambda s, x: s.params.__setitem__("is_lambda", x),
lambda s: s.params.__delitem__("is_lambda"),
None,
)
_globals = property(
lambda s: s.params["_globals"],
lambda s, x: s.params.__setitem__("_globals", x),
lambda s: s.params.__delitem__("_globals"),
None,
)
def set_pos_info(self, node, start, finish, name=None):
if name is None:
name = self.name
if hasattr(node, "offset"):
node.start = start
node.finish = finish
self.offsets[name, node.offset] = node
if hasattr(node, "parent"):
assert node.parent != node
node.start = start
node.finish = finish
self.last_finish = finish
def preorder(self, node=None):
start = len(self.f.getvalue())
super(pysource.SourceWalker, self).preorder(node)
self.set_pos_info(node, start, len(self.f.getvalue()))
return
def table_r_node(self, node):
"""General pattern where the last node should should
get the text span attributes of the entire tree"""
start = len(self.f.getvalue())
try:
self.default(node)
except GenericASTTraversalPruningException:
if not hasattr(node[-1], "parent"):
node[-1].parent = node
final = len(self.f.getvalue())
self.set_pos_info(node, start, final)
self.set_pos_info(node[-1], start, final)
raise GenericASTTraversalPruningException
n_slice0 = n_slice1 = n_slice2 = n_slice3 = n_subscript = table_r_node
n_aug_assign_1 = n_print_item = exec_stmt = print_to_item = delete = table_r_node
n_classdefco1 = n_classdefco2 = except_cond1 = except_cond2 = table_r_node
def n_pass(self, node):
start = len(self.f.getvalue()) + len(self.indent)
self.set_pos_info(node, start, start + len("pass"))
self.default(node)
def n_try_except(self, node):
# Note: we could also do this via modifying the
# 5 or so template rules. That is change:
# 'try_except': ( '%|try%:\n%+%c%-%c\n\n', 1, 3 ),
# to:
# 'try_except': ( '%|try%b:\n%+%c%-%c\n\n', 0, 1, 3 ),
start = len(self.f.getvalue()) + len(self.indent)
self.set_pos_info(node[0], start, start + len("try:"))
self.default(node)
n_tryelsestmt = n_c_tryelsestmt = n_tryelsestmtl = n_tryfinallystmt = n_try_except
def n_raise_stmt0(self, node):
assert node[0] == "RAISE_VARARGS_0"
start = len(self.f.getvalue()) + len(self.indent)
try:
self.default(node)
except GenericASTTraversalPruningException:
self.set_pos_info(node[0], start, len(self.f.getvalue()))
self.prune()
def n_raise_stmt1(self, node):
assert node[1] == "RAISE_VARARGS_1"
start = len(self.f.getvalue()) + len(self.indent)
try:
self.default(node)
except GenericASTTraversalPruningException:
self.set_pos_info(node[1], start, len(self.f.getvalue()))
self.prune()
def n_raise_stmt2(self, node):
assert node[2] == "RAISE_VARARGS_2"
start = len(self.f.getvalue()) + len(self.indent)
try:
self.default(node)
except GenericASTTraversalPruningException:
self.set_pos_info(node[2], start, len(self.f.getvalue()))
self.prune()
# FIXME: Isolate: only in Python 2.x.
def n_raise_stmt3(self, node):
assert node[3] == "RAISE_VARARGS_3"
start = len(self.f.getvalue()) + len(self.indent)
try:
self.default(node)
except GenericASTTraversalPruningException:
self.set_pos_info(node[3], start, len(self.f.getvalue()))
self.prune()
def n_return(self, node):
start = len(self.f.getvalue()) + len(self.indent)
if self.params["is_lambda"] or node[0] in (
"pop_return",
"popb_return",
"pop_ex_return",
):
self.preorder(node[0])
if hasattr(node[-1], "offset"):
self.set_pos_info(node[-1], start, len(self.f.getvalue()))
self.prune()
else:
start = len(self.f.getvalue()) + len(self.indent)
self.write(self.indent, "return")
# One reason we worry over whether we use "return None" or "return"
# is that inside a generator, "return None" is illegal.
# Thank you, Python!
if self.return_none or not self.is_return_none(node):
self.write(" ")
self.last_finish = len(self.f.getvalue())
self.preorder(node[0])
if hasattr(node[-1], "offset"):
self.set_pos_info(node[-1], start, len(self.f.getvalue()))
pass
pass
else:
for n in node:
self.set_pos_info_recurse(n, start, len(self.f.getvalue()))
pass
pass
self.set_pos_info(node, start, len(self.f.getvalue()))
self.println()
self.prune() # stop recursing
def n_return_if_stmt(self, node):
start = len(self.f.getvalue()) + len(self.indent)
if self.params["is_lambda"]:
node[0].parent = node
self.preorder(node[0])
else:
start = len(self.f.getvalue()) + len(self.indent)
self.write(self.indent, "return")
if self.return_none or not self.is_return_none(node):
self.write(" ")
self.preorder(node[0])
if hasattr(node[-1], "offset"):
self.set_pos_info(node[-1], start, len(self.f.getvalue()))
self.println()
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune() # stop recursing
def n_yield(self, node):
start = len(self.f.getvalue())
try:
super(FragmentsWalker, self).n_yield(node)
except GenericASTTraversalPruningException:
pass
if node != SyntaxTree("yield", [NONE, Token("YIELD_VALUE")]):
node[0].parent = node
self.set_pos_info(node[-1], start, len(self.f.getvalue()))
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune() # stop recursing
# In Python 3.3+ only
def n_yield_from(self, node):
start = len(self.f.getvalue())
try:
super(FragmentsWalker, self).n_yield(node)
except GenericASTTraversalPruningException:
pass
self.preorder(node[0])
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune() # stop recursing
def n_buildslice3(self, node):
start = len(self.f.getvalue())
try:
super(FragmentsWalker, self).n_buildslice3(node)
except GenericASTTraversalPruningException:
pass
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune() # stop recursing
def n_buildslice2(self, node):
start = len(self.f.getvalue())
try:
super(FragmentsWalker, self).n_buildslice2(node)
except GenericASTTraversalPruningException:
pass
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune() # stop recursing
def n_expr(self, node):
start = len(self.f.getvalue())
p = self.prec
if node[0].kind.startswith("bin_op"):
n = node[0][-1][0]
else:
n = node[0]
self.prec = PRECEDENCE.get(n.kind, -2)
if n == "LOAD_CONST" and repr(n.pattr)[0] == "-":
n.parent = node
self.set_pos_info(n, start, len(self.f.getvalue()))
self.prec = 6
if p < self.prec:
self.write("(")
node[0].parent = node
self.last_finish = len(self.f.getvalue())
self.preorder(node[0])
finish = len(self.f.getvalue())
if hasattr(node[0], "offset"):
self.set_pos_info(node[0], start, len(self.f.getvalue()))
self.write(")")
self.last_finish = finish + 1
else:
node[0].parent = node
start = len(self.f.getvalue())
self.preorder(node[0])
if hasattr(node[0], "offset"):
self.set_pos_info(node[0], start, len(self.f.getvalue()))
self.prec = p
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune()
def n_return_expr(self, node):
start = len(self.f.getvalue())
super(FragmentsWalker, self).n_return_expr(node)
self.set_pos_info(node, start, len(self.f.getvalue()))
def n_bin_op(self, node):
"""bin_op (formerly "binary_expr") is the Python AST BinOp"""
start = len(self.f.getvalue())
for n in node:
n.parent = node
self.last_finish = len(self.f.getvalue())
try:
super(FragmentsWalker, self).n_bin_op(node)
except GenericASTTraversalPruningException:
pass
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune()
def n_LOAD_CONST(self, node):
start = len(self.f.getvalue())
try:
super(FragmentsWalker, self).n_LOAD_CONST(node)
except GenericASTTraversalPruningException:
pass
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune()
n_LOAD_STR = n_LOAD_CONST
def n_ifelsestmtr(self, node):
if node[2] == "COME_FROM":
return_stmts_node = node[3]
node.kind = "ifelsestmtr2"
else:
return_stmts_node = node[2]
if len(return_stmts_node) != 2:
self.default(node)
if not (
return_stmts_node[0][0][0] == "ifstmt"
and return_stmts_node[0][0][0][1][0] == "return_if_stmts"
) and not (
return_stmts_node[0][-1][0] == "ifstmt"
and return_stmts_node[0][-1][0][1][0] == "return_if_stmts"
):
self.default(node)
return
start = len(self.f.getvalue()) + len(self.indent)
self.write(self.indent, "if ")
self.preorder(node[0])
self.println(":")
self.indent_more()
node[1].parent = node
self.preorder(node[1])
self.indent_less()
if_ret_at_end = False
if len(node[2][0]) >= 3:
node[2][0].parent = node
if (
node[2][0][-1][0] == "ifstmt"
and node[2][0][-1][0][1][0] == "return_if_stmts"
):
if_ret_at_end = True
past_else = False
prev_stmt_is_if_ret = True
for n in return_stmts_node[0]:
if n[0] == "ifstmt" and n[0][1][0] == "return_if_stmts":
if prev_stmt_is_if_ret:
n[0].kind = "elifstmt"
prev_stmt_is_if_ret = True
else:
prev_stmt_is_if_ret = False
if not past_else and not if_ret_at_end:
self.println(self.indent, "else:")
self.indent_more()
past_else = True
n.parent = node
self.preorder(n)
if not past_else or if_ret_at_end:
self.println(self.indent, "else:")
self.indent_more()
node[2][1].parent = node
self.preorder(node[2][1])
self.set_pos_info(node, start, len(self.f.getvalue()))
self.indent_less()
self.prune()
def n_elifelsestmtr(self, node):
if len(node[2]) != 2:
self.default(node)
for n in node[2][0]:
if not (n[0] == "ifstmt" and n[0][1][0] == "return_if_stmts"):
self.default(node)
return
start = len(self.f.getvalue() + self.indent)
self.write(self.indent, "elif ")
node[0].parent = node
self.preorder(node[0])
self.println(":")
self.indent_more()
node[1].parent = node
self.preorder(node[1])
self.indent_less()
for n in node[2][0]:
n[0].kind = "elifstmt"
n.parent = node
self.preorder(n)
self.println(self.indent, "else:")
self.indent_more()
node[2][1].parent = node
self.preorder(node[2][1])
self.indent_less()
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune()
def n_alias(self, node):
start = len(self.f.getvalue())
iname = node[0].pattr
store_import_node = node[-1][-1]
assert store_import_node.kind.startswith("STORE_")
sname = store_import_node.pattr
self.write(iname)
finish = len(self.f.getvalue())
if iname == sname or iname.startswith(sname + "."):
self.set_pos_info_recurse(node, start, finish)
else:
self.write(" as ")
sname_start = len(self.f.getvalue())
self.write(sname)
finish = len(self.f.getvalue())
for n in node[-1]:
self.set_pos_info_recurse(n, sname_start, finish)
self.set_pos_info(node, start, finish)
self.prune() # stop recursing
def n_mkfunc(self, node):
start = len(self.f.getvalue())
if self.version >= (3, 3) or node[-2] == "kwargs":
# LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3
# MAKE_FUNCTION ..
code_node = node[-3]
elif node[-2] == "expr":
code_node = node[-2][0]
else:
# LOAD_CONST code object ..
# MAKE_FUNCTION ..
code_node = node[-2]
func_name = code_node.attr.co_name
self.write(func_name)
self.set_pos_info(code_node, start, len(self.f.getvalue()))
self.indent_more()
start = len(self.f.getvalue())
make_function36(self, node, is_lambda=False, code_node=code_node)
self.set_pos_info(node, start, len(self.f.getvalue()))
if len(self.param_stack) > 1:
self.write("\n\n")
else:
self.write("\n\n\n")
self.indent_less()
self.prune() # stop recursing
def comprehension_walk(self, node, iter_index):
p = self.prec
self.prec = 27
# FIXME: clean this up
if node == "dict_comp":
cn = node[1]
elif node in ("generator_exp", "generator_exp_async"):
if node[0] == "load_genexpr":
load_genexpr = node[0]
elif node[1] == "load_genexpr":
load_genexpr = node[1]
cn = load_genexpr[0]
else:
if len(node[1]) > 1 and hasattr(node[1][1], "attr"):
# Python 3.3+ does this
cn = node[1][1]
else:
assert False, "Can't find code for comprehension"
assert iscode(cn.attr)
code = Code(cn.attr, self.scanner, self.currentclass)
ast = self.build_ast(code._tokens, code._customize, code)
self.customize(code._customize)
# Remove single reductions as in ("stmts", "sstmt"):
while len(ast) == 1:
ast = ast[0]
n = ast[iter_index]
assert n == "comp_iter", n.kind
# Find the comprehension body. It is the inner-most
# node that is not list_.. .
while n == "comp_iter": # list_iter
n = n[0] # recurse one step
if n == "comp_for":
if n[0] == "SETUP_LOOP":
n = n[4]
else:
n = n[3]
elif n == "comp_if":
n = n[1]
elif n == "comp_if_not":
n = n[2]
assert n == "comp_body", n
self.preorder(n[0])
if node == "generator_exp_async":
self.write(" async")
iter_var_index = iter_index - 2
else:
iter_var_index = iter_index - 1
self.write(" for ")
start = len(self.f.getvalue())
store = ast[iter_var_index]
self.preorder(store)
self.set_pos_info(ast[iter_index - 1], start, len(self.f.getvalue()))
self.write(" in ")
start = len(self.f.getvalue())
if node[2] == "expr":
iter_expr = node[2]
else:
iter_expr = node[-3]
assert iter_expr == "expr"
iter_expr.parent = node
self.preorder(iter_expr)
self.set_pos_info(iter_expr, start, len(self.f.getvalue()))
start = len(self.f.getvalue())
self.preorder(ast[iter_index])
self.set_pos_info(ast[iter_index], start, len(self.f.getvalue()))
self.prec = p
def comprehension_walk3(self, node, iter_index, code_index=-5):
"""
List comprehensions the way they are done in Python3.
They're more other comprehensions, e.g. set comprehensions
See if we can combine code.
"""
p = self.prec
self.prec = 27
code = node[code_index].attr
assert iscode(code), node[code_index]
code_name = code.co_name
code = Code(code, self.scanner, self.currentclass, self.debug_opts["asm"])
ast = self.build_ast(code._tokens, code._customize, code)
self.customize(code._customize)
if ast[0] == "sstmt":
ast = ast[0]
# skip over stmt return return_expr
ast = ast[0][0][0]
store = None
if ast in ["set_comp_func", "dict_comp_func"]:
# Offset 0: BUILD_SET should have the span
# of '{'
self.gen_source(ast, code_name, {})
for k in ast:
if k == "comp_iter":
n = k
elif k == "store":
store = k
pass
pass
pass
else:
ast = ast[0][0]
n = ast[iter_index]
assert n == "list_iter", n
# FIXME: I'm not totally sure this is right.
# Find the list comprehension body. It is the inner-most
# node that is not list_.. .
if_node = None
comp_for = None
comp_store = None
if n == "comp_iter":
comp_for = n
comp_store = ast[3]
have_not = False
while n in ("list_iter", "comp_iter"):
n = n[0] # iterate one nesting deeper
if n in ("list_for", "comp_for"):
if n[2] == "store":
store = n[2]
n = n[3]
elif n in ("list_if", "list_if_not", "comp_if", "comp_ifnot"):
have_not = n in ("list_if_not", "comp_ifnot")
if_node = n[0]
if n[1] == "store":
store = n[1]
n = n[2]
pass
pass
# Python 2.7+ starts including set_comp_body
# Python 3.5+ starts including set_comp_func
assert n.kind in ("lc_body", "comp_body", "set_comp_func", "set_comp_body"), ast
assert store, "Couldn't find store in list/set comprehension"
old_name = self.name
self.name = code_name
# Issue created with later Python code generation is that there
# is a lambda set up with a dummy argument name that is then called
# So we can't just translate that as is but need to replace the
# dummy name. Below we are picking out the variable name as seen
# in the code. And trying to generate code for the other parts
# that don't have the dummy argument name in it.
# Another approach might be to be able to pass in the source name
# for the dummy argument.
self.preorder(n[0])
gen_start = len(self.f.getvalue()) + 1
self.write(" for ")
start = len(self.f.getvalue())
if comp_store:
self.preorder(comp_store)
else:
self.preorder(store)
self.set_pos_info(store, start, len(self.f.getvalue()))
# FIXME this is all merely approximate
# from trepan.api import debug; debug()
self.write(" in ")
start = len(self.f.getvalue())
node[-3].parent = node
self.preorder(node[-3])
fin = len(self.f.getvalue())
self.set_pos_info(node[-3], start, fin, old_name)
if ast == "list_comp":
list_iter = ast[1]
assert list_iter == "list_iter"
if list_iter == "list_for":
self.preorder(list_iter[3])
self.prec = p
return
pass
if comp_store:
self.preorder(comp_for)
elif if_node:
self.write(" if ")
if have_not:
self.write("not ")
self.preorder(if_node)
pass
self.prec = p
self.name = old_name
if node[-1].kind.startswith("CALL_FUNCTION"):
self.set_pos_info(node[-1], gen_start, len(self.f.getvalue()))
def listcomprehension_walk2(self, node):
"""List comprehensions the way they are done in Python 2 (and
some Python 3?).
They're more other comprehensions, e.g. set comprehensions
See if we can combine code.
"""
p = self.prec
self.prec = 27
code = Code(node[1].attr, self.scanner, self.currentclass)
ast = self.build_ast(code._tokens, code._customize, code)
self.customize(code._customize)
if node == "set_comp":
ast = ast[0][0][0]
else:
ast = ast[0][0][0][0][0]
if ast == "expr":
ast = ast[0]
n = ast[1]
collection = node[-3]
list_if = None
assert n == "list_iter"
# Find the list comprehension body. It is the inner-most
# node that is not list_.. .
while n == "list_iter":
n = n[0] # recurse one step
if n == "list_for":
store = n[2]
n = n[3]
elif n in ("list_if", "list_if_not"):
# FIXME: just a guess
if n[0].kind == "expr":
list_if = n
else:
list_if = n[1]
n = n[2]
pass
pass
assert n == "lc_body", ast
self.preorder(n[0])
self.write(" for ")
start = len(self.f.getvalue())
self.preorder(store)
self.set_pos_info(store, start, len(self.f.getvalue()))
self.write(" in ")
start = len(self.f.getvalue())
node[-3].parent = node
self.preorder(collection)
self.set_pos_info(collection, start, len(self.f.getvalue()))
if list_if:
start = len(self.f.getvalue())
self.preorder(list_if)
self.set_pos_info(list_if, start, len(self.f.getvalue()))
self.prec = p
def n_generator_exp(self, node):
start = len(self.f.getvalue())
self.write("(")
code_index = -6
self.comprehension_walk(node, iter_index=4, code_index=code_index)
self.write(")")
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune()
def n_set_comp(self, node):
start = len(self.f.getvalue())
self.write("{")
if node[0] in ["LOAD_SETCOMP", "LOAD_DICTCOMP"]:
start = len(self.f.getvalue())
self.set_pos_info(node[0], start - 1, start)
self.comprehension_walk3(node, 1, 0)
elif node[0].kind == "load_closure":
self.closure_walk(node, collection_index=4)
else:
self.comprehension_walk(node, iter_index=4)
self.write("}")
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prune()
# FIXME: Not sure if below is general. Also, add dict_comp_func.
# 'set_comp_func': ("%|lambda %c: {%c for %c in %c%c}\n", 1, 3, 3, 1, 4)
def n_set_comp_func(self, node):
setcomp_start = len(self.f.getvalue())
self.write(self.indent, "lambda ")
param_node = node[1]
start = len(self.f.getvalue())
self.preorder(param_node)
self.set_pos_info(node[0], start, len(self.f.getvalue()))
self.write(": {")
start = len(self.f.getvalue())
assert node[0].kind.startswith("BUILD_SET")
self.set_pos_info(node[0], start - 1, start)
store = node[3]
assert store == "store"
start = len(self.f.getvalue())
self.preorder(store)
fin = len(self.f.getvalue())
self.set_pos_info(store, start, fin)
for_iter_node = node[2]
assert for_iter_node.kind == "FOR_ITER"
self.set_pos_info(for_iter_node, start, fin)
self.write(" for ")
self.preorder(store)
self.write(" in ")
self.preorder(param_node)
start = len(self.f.getvalue())
self.preorder(node[4])
self.set_pos_info(node[4], start, len(self.f.getvalue()))
self.write("}")
fin = len(self.f.getvalue())
self.set_pos_info(node, setcomp_start, fin)
if node[-2] == "RETURN_VALUE":
self.set_pos_info(node[-2], setcomp_start, fin)
self.prune()
def n_list_comp(self, node):
self.write("[")
if node[0].kind == "load_closure":
self.listcomprehension_walk2(node)
else:
if node[0] == "LOAD_LISTCOMP":
start = len(self.f.getvalue())
self.set_pos_info(node[0], start - 1, start)
self.comprehension_walk_newer(node, 1, 0)
self.write("]")
self.prune()
def closure_walk(self, node, collection_index):
"""Dictionary and Set comprehensions using closures."""
p = self.prec
self.prec = 27
code = Code(node[1].attr, self.scanner, self.currentclass)
ast = self.build_ast(code._tokens, code._customize, code)
self.customize(code._customize)
ast = ast[0][0][0]
store = ast[3]
collection = node[collection_index]
n = ast[4]
list_if = None
assert n == "comp_iter"
# find inner-most node
while n == "comp_iter":
n = n[0] # recurse one step
# FIXME: adjust for set comprehension
if n == "list_for":
store = n[2]
n = n[3]
elif n in ("list_if", "list_if_not", "comp_if", "comp_if_not"):
# FIXME: just a guess
if n[0].kind == "expr":
list_if = n
else:
list_if = n[1]
n = n[2]
pass
pass
assert n == "comp_body", ast
self.preorder(n[0])
self.write(" for ")
start = len(self.f.getvalue())
self.preorder(store)
self.set_pos_info(store, start, len(self.f.getvalue()))
self.write(" in ")
start = len(self.f.getvalue())
self.preorder(collection)
self.set_pos_info(collection, start, len(self.f.getvalue()))
if list_if:
start = len(self.f.getvalue())
self.preorder(list_if)
self.set_pos_info(list_if, start, len(self.f.getvalue()))
self.prec = p
def n_classdef(self, node):
# class definition ('class X(A,B,C):')
cclass = self.currentclass
if self.version >= (3, 1):
if node == "classdefdeco2":
currentclass = node[1][2].pattr
buildclass = node
else:
currentclass = node[1][0].pattr
buildclass = node[0]
if buildclass[0] == "LOAD_BUILD_CLASS":
start = len(self.f.getvalue())
self.set_pos_info(buildclass[0], start, start + len("class") + 2)
assert "mkfunc" == buildclass[1]
mkfunc = buildclass[1]
if mkfunc[0] == "kwargs":
for n in mkfunc:
if hasattr(n, "attr") and iscode(n.attr):
subclass = n.attr
break
pass
subclass_info = node if node == "classdefdeco2" else node[0]
elif buildclass[1][0] == "load_closure":
# Python 3 with closures not functions
load_closure = buildclass[1]
if hasattr(load_closure[-3], "attr"):
# Python 3.3 classes with closures work like this.
# Note have to test before 3.2 case because
# index -2 also has an attr.
subclass = load_closure[-3].attr
elif hasattr(load_closure[-2], "attr"):
# Python 3.2 works like this
subclass = load_closure[-2].attr
else:
raise RuntimeError(
"Internal Error n_classdef: cannot find class body"
)
if hasattr(buildclass[3], "__len__"):
subclass_info = buildclass[3]
elif hasattr(buildclass[2], "__len__"):
subclass_info = buildclass[2]
else:
raise RuntimeError(
"Internal Error n_classdef: cannot superclass name"
)
else:
subclass = buildclass[1][0].attr
subclass_info = node[0]
else:
buildclass = node if (node == "classdefdeco2") else node[0]
build_list = buildclass[1][0]
if hasattr(buildclass[-3][0], "attr"):
subclass = buildclass[-3][0].attr
currentclass = buildclass[0].pattr
elif hasattr(node[0][0], "pattr"):
subclass = buildclass[-3][1].attr
currentclass = node[0][0].pattr
else:
raise RuntimeError("Internal Error n_classdef: cannot find class name")
if node == "classdefdeco2":
self.write("\n")
else:
self.write("\n\n")
self.currentclass = str(currentclass)
start = len(self.f.getvalue())
self.write(self.indent, "class ", self.currentclass)
if self.version >= (3, 1):
self.print_super_classes3(subclass_info)
else:
self.print_super_classes(build_list)
self.println(":")
# class body
self.indent_more()
self.build_class(subclass)
self.indent_less()
self.currentclass = cclass
self.set_pos_info(node, start, len(self.f.getvalue()))
if len(self.param_stack) > 1:
self.write("\n\n")
else:
self.write("\n\n\n")
self.prune()
n_classdefdeco2 = n_classdef
def gen_source(
self,
ast,
name,
customize,
is_lambda=False,
returnNone=False,
debug_opts=DEFAULT_DEBUG_OPTS,
):
"""convert parse tree to Python source code"""
rn = self.return_none
self.return_none = returnNone
old_name = self.name
self.name = name
self.debug_opts = debug_opts
# if code would be empty, append 'pass'
if len(ast) == 0:
self.println(self.indent, "pass")
else:
self.customize(customize)
self.text = self.traverse(ast, is_lambda=is_lambda)
self.name = old_name
self.return_none = rn
def build_ast(
self,
tokens,
customize,
code,
is_lambda=False,
noneInNames=False,
is_top_level_module=False,
) -> GenericASTTraversal:
# FIXME: DRY with pysource.py
# assert isinstance(tokens[0], Token)
if is_lambda:
for t in tokens:
if t.kind == "RETURN_END_IF":
t.kind = "RETURN_END_IF_LAMBDA"
elif t.kind == "RETURN_VALUE":
t.kind = "RETURN_VALUE_LAMBDA"
tokens.append(Token("LAMBDA_MARKER", optyhpe="pseudo"))
try:
if self.p_lambda is None:
self.p_lambda = get_python_parser(
self.version,
self.debug_parser,
compile_mode="lambda",
is_pypy=self.is_pypy,
)
p = self.p_lambda
p.insts = self.scanner.insts
p.offset2inst_index = self.scanner.offset2inst_index
parse_tree = python_parser.parse(p, tokens, customize, is_lambda)
self.customize(customize)
except (heads.ParserError, AssertionError) as e:
raise ParserError(e, tokens, self.debug_parser.get("reduce", False))
# FIXME: So as not to remove tokens with offsets,
# remove this phase until we have a chance to go over,
# transform_tree = self.treeTransform.transform(ast)
return parse_tree
# del ast # Save memory
# return transform_tree
# The bytecode for the end of the main routine has a
# "return None". However you can't issue a "return" statement in
# main. In the other build_ast routine we eliminate the
# return statement instructions before parsing.
# But here we want to keep these instructions at the expense of
# a fully runnable Python program because we
# my be queried about the role of one of those instructions.
#
# NOTE: this differs from behavior in pysource.py
if len(tokens) >= 2 and not noneInNames:
if tokens[-1].kind in ("RETURN_VALUE", "RETURN_VALUE_LAMBDA"):
# Python 3.4's classes can add a "return None" which is
# invalid syntax.
if tokens[-2].kind == "LOAD_CONST":
if is_top_level_module or tokens[-2].pattr is None:
del tokens[-2:]
else:
tokens.append(Token("RETURN_LAST"))
else:
tokens.append(Token("RETURN_LAST"))
if len(tokens) == 0:
return PASS
# Build a parse tree from tokenized and massaged disassembly.
try:
# FIXME: have p.insts update in a better way
# modularity is broken here
p_insts = self.p.insts
self.p.insts = self.scanner.insts
self.p.offset2inst_index = self.scanner.offset2inst_index
self.p.opc = self.scanner.opc
parse_tree = python_parser.parse(
self.p, tokens, customize, is_lambda=is_lambda
)
self.p.insts = p_insts
except (heads.ParserError, AssertionError) as e:
raise ParserError(e, tokens, self.debug_parser.get("reduce", False))
checker(parse_tree, False, self.ast_errors)
return parse_tree
# FIXME: we could provide another customized routine
# that fixes up parents along a particular path to a node that
# we are interested in.
def fixup_parents(self, node, parent):
"""Make sure each node has a parent"""
start, finish = 0, self.last_finish
# We assume anything with a start has a finish.
needs_range = not hasattr(node, "start")
if not hasattr(node, "parent"):
node.parent = parent
for n in node:
if needs_range and hasattr(n, "start"):
if n.start < start:
start = n.start
if n.finish > finish:
finish = n.finish
if hasattr(n, "offset") and not hasattr(n, "parent"):
n.parent = node
else:
self.fixup_parents(n, node)
pass
pass
if needs_range:
node.start, node.finish = start, finish
return
# FIXME: revise to do *once* over the entire tree.
# So here we should just mark that the subtree
# needs offset adjustment.
def fixup_offsets(self, new_start, node):
"""Adjust all offsets under node"""
if hasattr(node, "start"):
node.start += new_start
node.finish += new_start
for n in node:
if hasattr(n, "offset"):
if hasattr(n, "start"):
n.start += new_start
n.finish += new_start
else:
self.fixup_offsets(new_start, n)
return
def set_pos_info_recurse(self, node, start, finish, parent=None):
"""Set positions under node"""
self.set_pos_info(node, start, finish)
if parent is None:
parent = node
for n in node:
n.parent = parent
if hasattr(n, "offset"):
self.set_pos_info(n, start, finish)
else:
n.start = start
n.finish = finish
self.set_pos_info_recurse(n, start, finish, parent)
return
def node_append(self, before_str, node_text, node):
self.write(before_str)
self.last_finish = len(self.f.getvalue())
self.fixup_offsets(self.last_finish, node)
self.write(node_text)
self.last_finish = len(self.f.getvalue())
# FIXME: duplicated from pysource, since we don't find self.params
def traverse(self, node, indent=None, is_lambda=False):
"""Builds up fragment which can be used inside a larger
block of code"""
self.param_stack.append(self.params)
if indent is None:
indent = self.indent
p = self.pending_newlines
self.pending_newlines = 0
self.params = {
"_globals": {},
"_nonlocals": {}, # Python 3 has nonlocal
"f": StringIO(),
"indent": indent,
"is_lambda": is_lambda,
}
self.preorder(node)
self.f.write("\n" * self.pending_newlines)
text = self.f.getvalue()
self.last_finish = len(text)
self.params = self.param_stack.pop()
self.pending_newlines = p
return text
def extract_node_info(self, nodeInfo):
# XXX debug
# print('-' * 30)
# node = nodeInfo.node
# print(node)
# if hasattr(node, 'parent'):
# print('~' * 30)
# print(node.parent)
# else:
# print("No parent")
# print('-' * 30)
start, finish = (nodeInfo.start, nodeInfo.finish)
text = self.text
# Ignore trailing blanks
match = re.search(r"\n+$", text[start:])
if match:
text = text[: -len(match.group(0))]
# Ignore leading blanks
match = re.search(r"\s*[^ \t\n]", text[start:])
if match:
start += len(match.group(0)) - 1
at_end = False
if start >= finish:
at_end = True
selectedText = text
else:
selectedText = text[start:finish]
# Compute offsets relative to the beginning of the
# line rather than the beginning of the text.
try:
lineStart = text[:start].rindex("\n") + 1
except ValueError:
lineStart = 0
adjustedStart = start - lineStart
# If selected text is greater than a single line
# just show the first line plus ellipsis.
lines = selectedText.split("\n")
if len(lines) > 1:
adjustedEnd = len(lines[0]) - adjustedStart
selectedText = lines[0] + " ...\n" + lines[-1]
else:
adjustedEnd = len(selectedText)
if at_end:
markerLine = (" " * len(lines[-1])) + "^"
else:
markerLine = (" " * adjustedStart) + ("-" * adjustedEnd)
elided = False
if len(lines) > 1 and not at_end:
elided = True
markerLine += " ..."
# Get line that the selected text is in and
# get a line count for that.
try:
lineEnd = lineStart + text[lineStart + 1 :].index("\n") - 1
except ValueError:
lineEnd = len(text)
lines = text[:lineEnd].split("\n")
selectedLine = text[lineStart : lineEnd + 2]
if elided:
selectedLine += " ..."
if isinstance(nodeInfo, Token):
nodeInfo = nodeInfo.parent
else:
nodeInfo = nodeInfo
if isinstance(nodeInfo, SyntaxTree):
nonterminal = nodeInfo[0]
else:
nonterminal = nodeInfo.node
return ExtractInfo(
lineNo=len(lines),
lineStartOffset=lineStart,
markerLine=markerLine,
selectedLine=selectedLine,
selectedText=selectedText,
nonterminal=nonterminal,
)
def extract_line_info(self, name, offset):
if (name, offset) not in list(self.offsets.keys()):
return None
return self.extract_node_info(self.offsets[name, offset])
def prev_node(self, node):
prev = None
if not hasattr(node, "parent"):
return prev
p = node.parent
for n in p:
if node == n:
return prev
prev = n
return prev
def extract_parent_info(self, node):
if not hasattr(node, "parent"):
return None, None
p = node.parent
orig_parent = p
# If we can get different text, use that as the parent,
# otherwise we'll use the immediatate parent.
while p and (
hasattr(p, "parent") and p.start == node.start and p.finish == node.finish
):
assert p != node
node = p
p = p.parent
if p is None:
p = orig_parent
return self.extract_node_info(p), p
def print_super_classes(self, node):
if not (node == "build_list"):
return
start = len(self.f.getvalue())
self.write("(")
line_separator = ", "
sep = ""
for elem in node[:-1]:
value = self.traverse(elem)
self.node_append(sep, value, elem)
# self.write(sep, value)
sep = line_separator
self.write(")")
self.set_pos_info(node, start, len(self.f.getvalue()))
def print_super_classes3(self, node):
# FIXME: wrap superclasses onto a node
# as a custom rule
start = len(self.f.getvalue())
n = len(node) - 1
assert node[n].kind.startswith("CALL_FUNCTION")
for i in range(n - 2, 0, -1):
if not node[i].kind in ["expr", "LOAD_CLASSNAME"]:
break
pass
if i == n - 2:
return
self.write("(")
line_separator = ", "
sep = ""
i += 1
while i < n:
value = self.traverse(node[i])
self.node_append(sep, value, node[i])
i += 1
self.write(sep, value)
sep = line_separator
self.write(")")
self.set_pos_info(node, start, len(self.f.getvalue()))
def n_dict(self, node):
"""
prettyprint a dict
'dict' is something like k = {'a': 1, 'b': 42 }"
"""
p = self.prec
self.prec = 100
self.indent_more(INDENT_PER_LEVEL)
line_seperator = ",\n" + self.indent
sep = INDENT_PER_LEVEL[:-1]
start = len(self.f.getvalue())
self.write("{")
self.set_pos_info(node[0], start, start + 1)
if self.version >= (3, 0) and not self.is_pypy:
if node[0].kind.startswith("kvlist"):
# Python 3.5+ style key/value list in dict
kv_node = node[0]
ll = list(kv_node)
length = len(ll)
if kv_node[-1].kind.startswith("BUILD_MAP"):
length -= 1
i = 0
while i < length:
self.write(sep)
name = self.traverse(ll[i], indent="")
ll[i].parent = kv_node
ll[i + 1].parent = kv_node
self.write(name, ": ")
value = self.traverse(
ll[i + 1], indent=self.indent + (len(name) + 2) * " "
)
self.write(sep, name, ": ", value)
sep = line_seperator
i += 2
pass
pass
elif node[1].kind.startswith("kvlist"):
# Python 3.0..3.4 style key/value list in dict
kv_node = node[1]
ll = list(kv_node)
if len(ll) > 0 and ll[0].kind == "kv3":
# Python 3.2 does this
kv_node = node[1][0]
ll = list(kv_node)
i = 0
while i < len(ll):
ll[i].parent = kv_node
ll[i + 1].parent = kv_node
key_start = len(self.f.getvalue()) + len(sep)
name = self.traverse(ll[i + 1], indent="")
key_finish = key_start + len(name)
val_start = key_finish + 2
value = self.traverse(
ll[i], indent=self.indent + (len(name) + 2) * " "
)
self.write(sep, name, ": ", value)
self.set_pos_info_recurse(ll[i + 1], key_start, key_finish)
self.set_pos_info_recurse(ll[i], val_start, val_start + len(value))
sep = line_seperator
i += 3
pass
pass
pass
else:
# Python 2 style kvlist
assert node[-1].kind.startswith("kvlist")
kv_node = node[-1] # goto kvlist
for kv in kv_node:
assert kv in ("kv", "kv2", "kv3")
# kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
# kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
# kv3 ::= expr expr STORE_MAP
if kv == "kv":
name = self.traverse(kv[-2], indent="")
kv[1].parent = kv_node
value = self.traverse(
kv[1], indent=self.indent + (len(name) + 2) * " "
)
elif kv == "kv2":
name = self.traverse(kv[1], indent="")
kv[-3].parent = kv_node
value = self.traverse(
kv[-3], indent=self.indent + (len(name) + 2) * " "
)
elif kv == "kv3":
name = self.traverse(kv[-2], indent="")
kv[0].parent = kv_node
value = self.traverse(
kv[0], indent=self.indent + (len(name) + 2) * " "
)
self.write(sep, name, ": ", value)
sep = line_seperator
self.write("}")
finish = len(self.f.getvalue())
self.set_pos_info(node, start, finish)
self.indent_less(INDENT_PER_LEVEL)
self.prec = p
self.prune()
def n_list(self, node):
"""
prettyprint a list or tuple
"""
p = self.prec
self.prec = PRECEDENCE["yield"] - 1
n = node.pop()
lastnode = n.kind
start = len(self.f.getvalue())
if lastnode.startswith("BUILD_LIST"):
self.write("[")
endchar = "]"
elif lastnode.startswith("BUILD_TUPLE"):
self.write("(")
endchar = ")"
elif lastnode.startswith("BUILD_SET"):
self.write("{")
endchar = "}"
elif lastnode.startswith("ROT_TWO"):
self.write("(")
endchar = ")"
else:
raise RuntimeError("Internal Error: n_list expects list or tuple")
flat_elems = []
for elem in node:
if elem == "expr1024":
for subelem in elem:
for subsubelem in subelem:
flat_elems.append(subsubelem)
elif elem == "expr32":
for subelem in elem:
flat_elems.append(subelem)
else:
flat_elems.append(elem)
self.indent_more(INDENT_PER_LEVEL)
if len(node) > 3:
line_separator = ",\n" + self.indent
else:
line_separator = ", "
sep = INDENT_PER_LEVEL[:-1]
# FIXME:
# if flat_elems > some_number, then group
# do automatic wrapping
for elem in flat_elems:
if elem == "ROT_THREE":
continue
assert elem == "expr"
value = self.traverse(elem)
self.node_append(sep, value, elem)
sep = line_separator
if len(node) == 1 and lastnode.startswith("BUILD_TUPLE"):
self.write(",")
self.write(endchar)
finish = len(self.f.getvalue())
n.parent = node.parent
self.set_pos_info(n, start, finish)
self.set_pos_info(node, start, finish)
self.indent_less(INDENT_PER_LEVEL)
self.prec = p
self.prune()
return
n_set = n_tuple = n_build_set = n_list
def template_engine(self, entry, startnode):
"""The format template interpretation engine. See the comment at the
beginning of this module for how we interpret format
specifications such as %c, %C, and so on.
"""
# print("-----")
# print(startnode.kind)
# print(entry[0])
# print('======')
startnode_start = len(self.f.getvalue())
start = startnode_start
fmt = entry[0]
arg = 1
i = 0
lastC = -1
recurse_node = False
m = escape.search(fmt)
while m:
i = m.end()
self.write(m.group("prefix"))
typ = m.group("type") or "{"
node = startnode
try:
if m.group("child"):
node = node[int(m.group("child"))]
node.parent = startnode
except Exception:
print(node.__dict__)
raise
if typ == "%":
start = len(self.f.getvalue())
self.write("%")
self.set_pos_info(node, start, len(self.f.getvalue()))
elif typ == "+":
self.indent_more()
elif typ == "-":
self.indent_less()
elif typ == "|":
self.write(self.indent)
# no longer used, since BUILD_TUPLE_n is pretty printed:
elif typ == "r":
recurse_node = True
elif typ == ",":
if lastC == 1:
self.write(",")
elif typ == "b":
finish = len(self.f.getvalue())
self.set_pos_info(node[entry[arg]], start, finish)
arg += 1
elif typ == "c":
start = len(self.f.getvalue())
index = entry[arg]
if isinstance(index, tuple):
if isinstance(index[1], str):
assert (
node[index[0]] == index[1]
), "at %s[%d], expected %s node; got %s" % (
node.kind,
arg,
node[index[0]].kind,
index[1],
)
else:
assert (
node[index[0]] in index[1]
), "at %s[%d], expected to be in '%s' node; got '%s'" % (
node.kind,
arg,
index[1],
node[index[0]].kind,
)
index = index[0]
assert isinstance(
index, int
), "at %s[%d], %s should be int or tuple" % (
node.kind,
arg,
type(index),
)
try:
node[index]
except IndexError:
raise RuntimeError(
f"""
Expanding '{node.kind}' in template '{entry}[{arg}]':
{index} is invalid; has only {len(node)} entries
"""
)
self.preorder(node[index])
finish = len(self.f.getvalue())
self.set_pos_info(node, start, finish)
arg += 1
elif typ == "p":
p = self.prec
# entry[arg]
tup = entry[arg]
assert isinstance(tup, tuple)
if len(tup) == 3:
(index, nonterm_name, self.prec) = tup
if isinstance(tup[1], str):
# if node[index] != nonterm_name:
# from trepan.api import debug; debug()
assert (
node[index] == nonterm_name
), "at %s[%d], expected '%s' node; got '%s'" % (
node.kind,
arg,
nonterm_name,
node[index].kind,
)
else:
assert node[tup[0]] in tup[1], (
f"at {node.kind}[{tup[0]}], expected to be in '{tup[1]}' "
f"node; got '{node[tup[0]].kind}'"
)
else:
assert len(tup) == 2
(index, self.prec) = entry[arg]
node[index].parent = node
start = len(self.f.getvalue())
self.preorder(node[index])
self.set_pos_info(node, start, len(self.f.getvalue()))
self.prec = p
arg += 1
elif typ == "C":
low, high, sep = entry[arg]
lastC = remaining = len(node[low:high])
start = len(self.f.getvalue())
for subnode in node[low:high]:
self.preorder(subnode)
remaining -= 1
if remaining > 0:
self.write(sep)
self.set_pos_info(node, start, len(self.f.getvalue()))
arg += 1
elif typ == "D":
low, high, sep = entry[arg]
lastC = remaining = len(node[low:high])
for subnode in node[low:high]:
remaining -= 1
if len(subnode) > 0:
self.preorder(subnode)
if remaining > 0:
self.write(sep)
pass
pass
pass
arg += 1
elif typ == "x":
src, dest = entry[arg]
for d in dest:
self.set_pos_info_recurse(
node[d], node[src].start, node[src].finish
)
pass
arg += 1
elif typ == "P":
p = self.prec
low, high, sep, self.prec = entry[arg]
lastC = remaining = len(node[low:high])
start = self.last_finish
for subnode in node[low:high]:
self.preorder(subnode)
remaining -= 1
if remaining > 0:
self.write(sep)
self.prec = p
arg += 1
elif typ == "{":
d = node.__dict__
expr = m.group("expr")
# Line mapping stuff
if (
hasattr(node, "linestart")
and node.linestart
and hasattr(node, "current_line_number")
):
self.source_linemap[self.current_line_number] = node.linestart
# Additional fragment-position stuff
try:
start = len(self.f.getvalue())
self.write(eval(expr, d, d))
self.set_pos_info(node, start, len(self.f.getvalue()))
except Exception:
print(node)
raise
m = escape.search(fmt, i)
pass
self.write(fmt[i:])
fin = len(self.f.getvalue())
if recurse_node:
self.set_pos_info_recurse(startnode, startnode_start, fin)
else:
self.set_pos_info(startnode, startnode_start, fin)
# FIXME figure out how to get these cases to be table driven.
# 2. subroutine calls. It the last op is the call and for purposes of printing
# we don't need to print anything special there. However it encompasses the
# entire string of the node fn(...)
if startnode.kind == "call":
last_node = startnode[-1]
self.set_pos_info(last_node, startnode_start, self.last_finish)
return
def _get_mapping(self, node):
if (
hasattr(node, "data")
and len(node) > 0
and isinstance(node[-1], Token)
and not hasattr(node[-1], "parent")
):
node[-1].parent = node
return MAP.get(node, self.MAP_DIRECT_FRAGMENT)
pass
#
DEFAULT_DEBUG_OPTS = {"asm": False, "tree": False, "grammar": False}
# This interface is deprecated
def deparse_code(
version,
co,
out=StringIO(),
showasm=False,
showast=False,
showgrammar=False,
code_objects={},
compile_mode="exec",
is_pypy=IS_PYPY,
walker=FragmentsWalker,
):
debug_opts = {"asm": showasm, "ast": showast, "grammar": showgrammar}
return code_deparse(
co,
out,
version=version,
debug_opts=debug_opts,
code_objects=code_objects,
compile_mode=compile_mode,
is_pypy=is_pypy,
walker=walker,
)
def code_deparse(
co,
out=StringIO(),
version=None,
is_pypy=IS_PYPY,
debug_opts=DEFAULT_DEBUG_OPTS,
code_objects={},
compile_mode="exec",
walker=FragmentsWalker,
start_offset: int = 0,
stop_offset: int = -1,
):
"""
Convert the code object co into a python source fragment.
:param version: The python version this code is from as a float, for
example 2.6, 2.7, 3.2, 3.3, 3.4, 3.5 etc.
:param co: The code object to parse.
:param out: File like object to write the output to.
:param debug_opts: A dictionary with keys
'asm': value determines whether to show
mangled bytecode disassembly
'ast': value determines whether to show
'grammar': boolean determining whether to show
grammar reduction rules.
If value is a file-like object, output that object's write method will
be used rather than sys.stdout
:return: The deparsed source fragment.
"""
assert iscode(co)
if version is None:
version = PYTHON_VERSION_TRIPLE
if is_pypy is None:
is_pypy = IS_PYPY
# store final output stream for case of error
scanner = get_scanner(version, is_pypy=is_pypy, show_asm=debug_opts["asm"])
show_asm = debug_opts.get("asm", None)
tokens, customize = scanner.ingest(co, code_objects=code_objects, show_asm=show_asm)
if start_offset > 0:
for i, t in enumerate(tokens):
# If t.offset is a string, we want to skip this.
if isinstance(t.offset, int) and t.offset >= start_offset:
tokens = tokens[i:]
break
if stop_offset > -1:
for i, t in enumerate(tokens):
# In contrast to the test for start_offset If t.offset is
# a string, we want to extract the integer offset value.
if t.off2int() >= stop_offset:
tokens = tokens[:i]
break
debug_parser = dict(PARSER_DEFAULT_DEBUG)
# Build Syntax Tree from disassembly.
linestarts = dict(scanner.opc.findlinestarts(co))
deparsed = walker(
version,
scanner,
showast=debug_opts.get("tree", TREE_DEFAULT_DEBUG),
debug_parser=debug_parser,
compile_mode=compile_mode,
is_pypy=is_pypy,
linestarts=linestarts,
)
is_top_level_module = co.co_name == ""
deparsed.ast = deparsed.build_ast(
tokens, customize, co, is_top_level_module=is_top_level_module
)
assert deparsed.ast == "stmts", "Should have parsed grammar start"
# save memory
del tokens
# convert leading '__doc__ = "..." into doc string
assert deparsed.ast == "stmts"
(deparsed.mod_globs, _) = pysource.find_globals_and_nonlocals(
deparsed.ast, set(), set(), co, version
)
# Just when you think we've forgotten about what we
# were supposed to do: Generate source from the Syntax tree!
deparsed.gen_source(deparsed.ast, co.co_name, customize)
deparsed.set_pos_info(deparsed.ast, 0, len(deparsed.text))
deparsed.fixup_parents(deparsed.ast, None)
for g in sorted(deparsed.mod_globs):
deparsed.write("# global %s ## Warning: Unused global\n" % g)
if deparsed.ast_errors:
deparsed.write("# NOTE: have decompilation errors.\n")
deparsed.write("# Use -t option to show full context.")
for err in deparsed.ast_errors:
deparsed.write(err)
deparsed.ERROR = True
if deparsed.ERROR:
raise deparsed.ERROR
# To keep the API consistent with previous releases, convert
# deparse.offset values into NodeInfo items
for tup, node in deparsed.offsets.items():
deparsed.offsets[tup] = NodeInfo(
node=node, start=node.start, finish=node.finish
)
deparsed.scanner = scanner
return deparsed
def find_gt(a, x):
"Find leftmost value greater than x"
i = bisect_right(a, x)
if i != len(a):
return a[i]
raise ValueError
def code_deparse_around_offset(
name,
offset,
co,
out=StringIO(),
version: Optional[tuple] = None,
is_pypy: bool = False,
debug_opts=DEFAULT_DEBUG_OPTS,
):
"""
Like deparse_code(), but given a function/module name and
offset, finds the node closest to offset. If offset is not an instruction boundary,
we raise an IndexError.
"""
assert iscode(co)
if version is None:
version = PYTHON_VERSION_TRIPLE
if is_pypy is None:
is_pypy = IS_PYPY
deparsed = code_deparse(co, out, version, is_pypy, debug_opts)
if (name, offset) in deparsed.offsets.keys():
# This is the easy case
return deparsed
valid_offsets = [t for t in deparsed.offsets if isinstance(t[1], int)]
offset_list = sorted([t[1] for t in valid_offsets if t[0] == name])
# FIXME: should check for branching?
found_offset = find_gt(offset_list, offset)
deparsed.offsets[name, offset] = deparsed.offsets[name, found_offset]
return deparsed
# Deprecated. Here still for compatibility
def deparse_code_around_offset(
name,
offset,
version,
co,
out=StringIO(),
showasm=False,
showast=False,
showgrammar=PARSER_DEFAULT_DEBUG,
is_pypy=False,
):
debug_opts = {"asm": showasm, "ast": showast, "grammar": showgrammar}
return code_deparse(name, offset, co, out, version, is_pypy, debug_opts)
def op_at_code_loc(code, loc, opc):
"""Return the instruction name at code[loc] using
opc to look up instruction names. Returns 'got IndexError'
if code[loc] is invalid.
`code` is instruction bytecode, `loc` is an offset (integer) and
`opc` is an opcode module from `xdis`.
"""
try:
op = code[loc]
except IndexError:
return "got IndexError"
return opc.opname[op]
def deparsed_find(tup, deparsed, code):
"""Return a NodeInfo nametuple for a fragment-deparsed `deparsed` at `tup`.
`tup` is a name and offset tuple, `deparsed` is a fragment object
and `code` is instruction bytecode."""
nodeInfo = None
name, last_i = tup
if not hasattr(deparsed, "offsets"):
return None
if (name, last_i) in deparsed.offsets.keys():
nodeInfo = deparsed.offsets[name, last_i]
else:
from decompyle3.scanner import get_scanner
scanner = get_scanner(deparsed.version)
co = code.co_code
if op_at_code_loc(co, last_i, scanner.opc) == "DUP_TOP":
offset = deparsed.scanner.next_offset(co[last_i], last_i)
if (name, offset) in deparsed.offsets:
nodeInfo = deparsed.offsets[name, offset]
return nodeInfo
# if __name__ == "__main__":
# def deparse_test(co, is_pypy=IS_PYPY):
# deparsed = code_deparse(co, is_pypy=IS_PYPY)
# print("deparsed source")
# print(deparsed.text, "\n")
# print("------------------------")
# for name, offset in sorted(deparsed.offsets.keys(), key=lambda x: str(x[0])):
# print("name %s, offset %s" % (name, offset))
# nodeInfo = deparsed.offsets[name, offset]
# nodeInfo2 = deparsed_find((name, offset), deparsed, co)
# assert nodeInfo == nodeInfo2
# node = nodeInfo.node
# extractInfo = deparsed.extract_node_info(node)
# print("code: %s" % node.kind)
# # print extractInfo
# print(extractInfo.selectedText)
# print(extractInfo.selectedLine)
# print(extractInfo.markerLine)
# extractInfo, p = deparsed.extract_parent_info(node)
# if extractInfo:
# print("Contained in...")
# print(extractInfo.selectedLine)
# print(extractInfo.markerLine)
# print("code: %s" % p.kind)
# print("=" * 40)
# pass
# pass
# return
# def deparse_test_around(offset, name, co, is_pypy=IS_PYPY):
# deparsed = code_deparse_around_offset(name, offset, co)
# print("deparsed source")
# print(deparsed.text, "\n")
# print("------------------------")
# for name, offset in sorted(deparsed.offsets.keys(), key=lambda x: str(x[0])):
# print("name %s, offset %s" % (name, offset))
# nodeInfo = deparsed.offsets[name, offset]
# node = nodeInfo.node
# extractInfo = deparsed.extract_node_info(node)
# print("code: %s" % node.kind)
# # print extractInfo
# print(extractInfo.selectedText)
# print(extractInfo.selectedLine)
# print(extractInfo.markerLine)
# extractInfo, p = deparsed.extract_parent_info(node)
# if extractInfo:
# print("Contained in...")
# print(extractInfo.selectedLine)
# print(extractInfo.markerLine)
# print("code: %s" % p.kind)
# print("=" * 40)
# pass
# pass
# return
# def get_code_for_fn(fn):
# return fn.__code__
# def test():
# import os, sys
# def get_dups(li: list) -> set:
# dups = {}
# for item in li:
# dups[item] = dups.get(item, -1) + 1
# # return dups
# return {k for k in dups.keys() if dups[k] > 0}
# def div_test(a, b, c):
# return a / b / c
# def gcd(a, b):
# if a > b:
# (a, b) = (b, a)
# pass
# if a <= 0:
# return None
# if a == 1 or a == b:
# return a
# return gcd(b - a, a)
# # check_args(['3', '5'])
# # deparse_test(get_code_for_fn(gcd))
# deparse_test(get_code_for_fn(get_dups))
# # deparse_test(get_code_for_fn(test))
# # deparse_test(get_code_for_fn(FragmentsWalker.fixup_offsets))
# # deparse_test(get_code_for_fn(FragmentsWalker.n_list))
# print("=" * 30)
# # deparse_test_around(408, 'n_list',
# get_code_for_fn(FragmentsWalker.n_build_list))
# # deparse_test(inspect.currentframe().f_code)