| # -*- coding: utf-8 -*- |
| """ |
| ast |
| ~~~ |
| |
| The `ast` module helps Python applications to process trees of the Python |
| abstract syntax grammar. The abstract syntax itself might change with |
| each Python release; this module helps to find out programmatically what |
| the current grammar looks like and allows modifications of it. |
| |
| An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as |
| a flag to the `compile()` builtin function or by using the `parse()` |
| function from this module. The result will be a tree of objects whose |
| classes all inherit from `ast.AST`. |
| |
| A modified abstract syntax tree can be compiled into a Python code object |
| using the built-in `compile()` function. |
| |
| Additionally various helper functions are provided that make working with |
| the trees simpler. The main intention of the helper functions and this |
| module in general is to provide an easy to use interface for libraries |
| that work tightly with the python syntax (template engines for example). |
| |
| |
| :copyright: Copyright 2008 by Armin Ronacher. |
| :license: Python License. |
| """ |
| from _ast import * |
| from _ast import __version__ |
| |
| |
| def parse(source, filename='<unknown>', mode='exec'): |
| """ |
| Parse the source into an AST node. |
| Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). |
| """ |
| return compile(source, filename, mode, PyCF_ONLY_AST) |
| |
| |
| def literal_eval(node_or_string): |
| """ |
| Safely evaluate an expression node or a string containing a Python |
| expression. The string or node provided may only consist of the following |
| Python literal structures: strings, numbers, tuples, lists, dicts, booleans, |
| and None. |
| """ |
| _safe_names = {'None': None, 'True': True, 'False': False} |
| if isinstance(node_or_string, basestring): |
| node_or_string = parse(node_or_string, mode='eval') |
| if isinstance(node_or_string, Expression): |
| node_or_string = node_or_string.body |
| def _convert(node): |
| if isinstance(node, Str): |
| return node.s |
| elif isinstance(node, Num): |
| return node.n |
| elif isinstance(node, Tuple): |
| return tuple(map(_convert, node.elts)) |
| elif isinstance(node, List): |
| return list(map(_convert, node.elts)) |
| elif isinstance(node, Dict): |
| return dict((_convert(k), _convert(v)) for k, v |
| in zip(node.keys, node.values)) |
| elif isinstance(node, Name): |
| if node.id in _safe_names: |
| return _safe_names[node.id] |
| elif isinstance(node, BinOp) and \ |
| isinstance(node.op, (Add, Sub)) and \ |
| isinstance(node.right, Num) and \ |
| isinstance(node.right.n, complex) and \ |
| isinstance(node.left, Num) and \ |
| isinstance(node.left.n, (int, long, float)): |
| left = node.left.n |
| right = node.right.n |
| if isinstance(node.op, Add): |
| return left + right |
| else: |
| return left - right |
| raise ValueError('malformed string') |
| return _convert(node_or_string) |
| |
| |
| def dump(node, annotate_fields=True, include_attributes=False): |
| """ |
| Return a formatted dump of the tree in *node*. This is mainly useful for |
| debugging purposes. The returned string will show the names and the values |
| for fields. This makes the code impossible to evaluate, so if evaluation is |
| wanted *annotate_fields* must be set to False. Attributes such as line |
| numbers and column offsets are not dumped by default. If this is wanted, |
| *include_attributes* can be set to True. |
| """ |
| def _format(node): |
| if isinstance(node, AST): |
| fields = [(a, _format(b)) for a, b in iter_fields(node)] |
| rv = '%s(%s' % (node.__class__.__name__, ', '.join( |
| ('%s=%s' % field for field in fields) |
| if annotate_fields else |
| (b for a, b in fields) |
| )) |
| if include_attributes and node._attributes: |
| rv += fields and ', ' or ' ' |
| rv += ', '.join('%s=%s' % (a, _format(getattr(node, a))) |
| for a in node._attributes) |
| return rv + ')' |
| elif isinstance(node, list): |
| return '[%s]' % ', '.join(_format(x) for x in node) |
| return repr(node) |
| if not isinstance(node, AST): |
| raise TypeError('expected AST, got %r' % node.__class__.__name__) |
| return _format(node) |
| |
| |
| def copy_location(new_node, old_node): |
| """ |
| Copy source location (`lineno` and `col_offset` attributes) from |
| *old_node* to *new_node* if possible, and return *new_node*. |
| """ |
| for attr in 'lineno', 'col_offset': |
| if attr in old_node._attributes and attr in new_node._attributes \ |
| and hasattr(old_node, attr): |
| setattr(new_node, attr, getattr(old_node, attr)) |
| return new_node |
| |
| |
| def fix_missing_locations(node): |
| """ |
| When you compile a node tree with compile(), the compiler expects lineno and |
| col_offset attributes for every node that supports them. This is rather |
| tedious to fill in for generated nodes, so this helper adds these attributes |
| recursively where not already set, by setting them to the values of the |
| parent node. It works recursively starting at *node*. |
| """ |
| def _fix(node, lineno, col_offset): |
| if 'lineno' in node._attributes: |
| if not hasattr(node, 'lineno'): |
| node.lineno = lineno |
| else: |
| lineno = node.lineno |
| if 'col_offset' in node._attributes: |
| if not hasattr(node, 'col_offset'): |
| node.col_offset = col_offset |
| else: |
| col_offset = node.col_offset |
| for child in iter_child_nodes(node): |
| _fix(child, lineno, col_offset) |
| _fix(node, 1, 0) |
| return node |
| |
| |
| def increment_lineno(node, n=1): |
| """ |
| Increment the line number of each node in the tree starting at *node* by *n*. |
| This is useful to "move code" to a different location in a file. |
| """ |
| for child in walk(node): |
| if 'lineno' in child._attributes: |
| child.lineno = getattr(child, 'lineno', 0) + n |
| return node |
| |
| |
| def iter_fields(node): |
| """ |
| Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` |
| that is present on *node*. |
| """ |
| for field in node._fields: |
| try: |
| yield field, getattr(node, field) |
| except AttributeError: |
| pass |
| |
| |
| def iter_child_nodes(node): |
| """ |
| Yield all direct child nodes of *node*, that is, all fields that are nodes |
| and all items of fields that are lists of nodes. |
| """ |
| for name, field in iter_fields(node): |
| if isinstance(field, AST): |
| yield field |
| elif isinstance(field, list): |
| for item in field: |
| if isinstance(item, AST): |
| yield item |
| |
| |
| def get_docstring(node, clean=True): |
| """ |
| Return the docstring for the given node or None if no docstring can |
| be found. If the node provided does not have docstrings a TypeError |
| will be raised. |
| """ |
| if not isinstance(node, (FunctionDef, ClassDef, Module)): |
| raise TypeError("%r can't have docstrings" % node.__class__.__name__) |
| if node.body and isinstance(node.body[0], Expr) and \ |
| isinstance(node.body[0].value, Str): |
| if clean: |
| import inspect |
| return inspect.cleandoc(node.body[0].value.s) |
| return node.body[0].value.s |
| |
| |
| def walk(node): |
| """ |
| Recursively yield all descendant nodes in the tree starting at *node* |
| (including *node* itself), in no specified order. This is useful if you |
| only want to modify nodes in place and don't care about the context. |
| """ |
| from collections import deque |
| todo = deque([node]) |
| while todo: |
| node = todo.popleft() |
| todo.extend(iter_child_nodes(node)) |
| yield node |
| |
| |
| class NodeVisitor(object): |
| """ |
| A node visitor base class that walks the abstract syntax tree and calls a |
| visitor function for every node found. This function may return a value |
| which is forwarded by the `visit` method. |
| |
| This class is meant to be subclassed, with the subclass adding visitor |
| methods. |
| |
| Per default the visitor functions for the nodes are ``'visit_'`` + |
| class name of the node. So a `TryFinally` node visit function would |
| be `visit_TryFinally`. This behavior can be changed by overriding |
| the `visit` method. If no visitor function exists for a node |
| (return value `None`) the `generic_visit` visitor is used instead. |
| |
| Don't use the `NodeVisitor` if you want to apply changes to nodes during |
| traversing. For this a special visitor exists (`NodeTransformer`) that |
| allows modifications. |
| """ |
| |
| def visit(self, node): |
| """Visit a node.""" |
| method = 'visit_' + node.__class__.__name__ |
| visitor = getattr(self, method, self.generic_visit) |
| return visitor(node) |
| |
| def generic_visit(self, node): |
| """Called if no explicit visitor function exists for a node.""" |
| for field, value in iter_fields(node): |
| if isinstance(value, list): |
| for item in value: |
| if isinstance(item, AST): |
| self.visit(item) |
| elif isinstance(value, AST): |
| self.visit(value) |
| |
| |
| class NodeTransformer(NodeVisitor): |
| """ |
| A :class:`NodeVisitor` subclass that walks the abstract syntax tree and |
| allows modification of nodes. |
| |
| The `NodeTransformer` will walk the AST and use the return value of the |
| visitor methods to replace or remove the old node. If the return value of |
| the visitor method is ``None``, the node will be removed from its location, |
| otherwise it is replaced with the return value. The return value may be the |
| original node in which case no replacement takes place. |
| |
| Here is an example transformer that rewrites all occurrences of name lookups |
| (``foo``) to ``data['foo']``:: |
| |
| class RewriteName(NodeTransformer): |
| |
| def visit_Name(self, node): |
| return copy_location(Subscript( |
| value=Name(id='data', ctx=Load()), |
| slice=Index(value=Str(s=node.id)), |
| ctx=node.ctx |
| ), node) |
| |
| Keep in mind that if the node you're operating on has child nodes you must |
| either transform the child nodes yourself or call the :meth:`generic_visit` |
| method for the node first. |
| |
| For nodes that were part of a collection of statements (that applies to all |
| statement nodes), the visitor may also return a list of nodes rather than |
| just a single node. |
| |
| Usually you use the transformer like this:: |
| |
| node = YourTransformer().visit(node) |
| """ |
| |
| def generic_visit(self, node): |
| for field, old_value in iter_fields(node): |
| old_value = getattr(node, field, None) |
| if isinstance(old_value, list): |
| new_values = [] |
| for value in old_value: |
| if isinstance(value, AST): |
| value = self.visit(value) |
| if value is None: |
| continue |
| elif not isinstance(value, AST): |
| new_values.extend(value) |
| continue |
| new_values.append(value) |
| old_value[:] = new_values |
| elif isinstance(old_value, AST): |
| new_node = self.visit(old_value) |
| if new_node is None: |
| delattr(node, field) |
| else: |
| setattr(node, field, new_node) |
| return node |
