diff --git a/nisext/ast.py b/nisext/ast.py deleted file mode 100644 index 1984f43388..0000000000 --- a/nisext/ast.py +++ /dev/null @@ -1,387 +0,0 @@ -# -*- coding: utf-8 -*- -""" - ast - ~~~ - - The `ast` module helps Python applications to process trees of the Python - abstract syntax grammar. The abstract syntax itself might change with - each Python release; this module helps to find out programmatically what - the current grammar looks like and allows modifications of it. - - An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as - a flag to the `compile()` builtin function or by using the `parse()` - function from this module. The result will be a tree of objects whose - classes all inherit from `ast.AST`. - - A modified abstract syntax tree can be compiled into a Python code object - using the built-in `compile()` function. - - Additionally various helper functions are provided that make working with - the trees simpler. The main intention of the helper functions and this - module in general is to provide an easy to use interface for libraries - that work tightly with the python syntax (template engines for example). - - - :copyright: Copyright 2008 by Armin Ronacher. - :license: Python License. - - From: http://dev.pocoo.org/hg/sandbox -""" -from _ast import * - - -BOOLOP_SYMBOLS = { - And: 'and', - Or: 'or' -} - -BINOP_SYMBOLS = { - Add: '+', - Sub: '-', - Mult: '*', - Div: '/', - FloorDiv: '//', - Mod: '%', - LShift: '<<', - RShift: '>>', - BitOr: '|', - BitAnd: '&', - BitXor: '^' -} - -CMPOP_SYMBOLS = { - Eq: '==', - Gt: '>', - GtE: '>=', - In: 'in', - Is: 'is', - IsNot: 'is not', - Lt: '<', - LtE: '<=', - NotEq: '!=', - NotIn: 'not in' -} - -UNARYOP_SYMBOLS = { - Invert: '~', - Not: 'not', - UAdd: '+', - USub: '-' -} - -ALL_SYMBOLS = {} -ALL_SYMBOLS.update(BOOLOP_SYMBOLS) -ALL_SYMBOLS.update(BINOP_SYMBOLS) -ALL_SYMBOLS.update(CMPOP_SYMBOLS) -ALL_SYMBOLS.update(UNARYOP_SYMBOLS) - - -def parse(expr, filename='', mode='exec'): - """Parse an expression into an AST node.""" - return compile(expr, filename, mode, PyCF_ONLY_AST) - - -def literal_eval(node_or_string): - """Safe evaluate a literal. The string or node provided may include any - of the following python structures: strings, numbers, tuples, lists, - dicts, booleans or None. - """ - _safe_names = {'None': None, 'True': True, 'False': False} - if isinstance(node_or_string, basestring): - node_or_string = parse(node_or_string, mode='eval') - if isinstance(node_or_string, Expression): - node_or_string = node_or_string.body - def _convert(node): - if isinstance(node, Str): - return node.s - elif isinstance(node, Num): - return node.n - elif isinstance(node, Tuple): - return tuple(map(_convert, node.elts)) - elif isinstance(node, List): - return list(map(_convert, node.elts)) - elif isinstance(node, Dict): - return dict((_convert(k), _convert(v)) for k, v - in zip(node.keys, node.values)) - elif isinstance(node, Name): - if node.id in _safe_names: - return _safe_names[node.id] - raise ValueError('malformed string') - return _convert(node_or_string) - - -def dump(node, annotate_fields=True, include_attributes=False): - """A very verbose representation of the node passed. This is useful for - debugging purposes. Per default the returned string will show the names - and the values for fields. This makes the code impossible to evaluate, - if evaluation is wanted `annotate_fields` must be set to False. - Attributes such as line numbers and column offsets are dumped by default. - If this is wanted, `include_attributes` can be set to `True`. - """ - def _format(node): - if isinstance(node, AST): - fields = [(a, _format(b)) for a, b in iter_fields(node)] - rv = '%s(%s' % (node.__class__.__name__, ', '.join( - ('%s=%s' % field for field in fields) - if annotate_fields else - (b for a, b in fields) - )) - if include_attributes and node._attributes: - rv += fields and ', ' or ' ' - rv += ', '.join('%s=%s' % (a, _format(getattr(node, a))) - for a in node._attributes) - return rv + ')' - elif isinstance(node, list): - return '[%s]' % ', '.join(_format(x) for x in node) - return repr(node) - if not isinstance(node, AST): - raise TypeError('expected AST, got %r' % node.__class__.__name__) - return _format(node) - - -def copy_location(new_node, old_node): - """Copy the source location hint (`lineno` and `col_offset`) from the - old to the new node if possible and return the new one. - """ - for attr in 'lineno', 'col_offset': - if attr in old_node._attributes and attr in new_node._attributes \ - and hasattr(old_node, attr): - setattr(new_node, attr, getattr(old_node, attr)) - return new_node - - -def fix_missing_locations(node): - """Some nodes require a line number and the column offset. Without that - information the compiler will abort the compilation. Because it can be - a dull task to add appropriate line numbers and column offsets when - adding new nodes this function can help. It copies the line number and - column offset of the parent node to the child nodes without this - information. - - Unlike `copy_location` this works recursive and won't touch nodes that - already have a location information. - """ - def _fix(node, lineno, col_offset): - if 'lineno' in node._attributes: - if not hasattr(node, 'lineno'): - node.lineno = lineno - else: - lineno = node.lineno - if 'col_offset' in node._attributes: - if not hasattr(node, 'col_offset'): - node.col_offset = col_offset - else: - col_offset = node.col_offset - for child in iter_child_nodes(node): - _fix(child, lineno, col_offset) - _fix(node, 1, 0) - return node - - -def increment_lineno(node, n=1): - """Increment the line numbers of all nodes by `n` if they have line number - attributes. This is useful to "move code" to a different location in a - file. - """ - if 'lineno' in node._attributes: - node.lineno = getattr(node, 'lineno', 0) + n - for child in walk(node): - if 'lineno' in child._attributes: - child.lineno = getattr(child, 'lineno', 0) + n - return node - - -def iter_fields(node): - """Iterate over all fields of a node, only yielding existing fields.""" - for field in node._fields: - try: - yield field, getattr(node, field) - except AttributeError: - pass - - -def get_fields(node): - """Like `iter_fiels` but returns a dict.""" - return dict(iter_fields(node)) - - -def iter_child_nodes(node): - """Iterate over all child nodes or a node.""" - for name, field in iter_fields(node): - if isinstance(field, AST): - yield field - elif isinstance(field, list): - for item in field: - if isinstance(item, AST): - yield item - - -def get_child_nodes(node): - """Like `iter_child_nodes` but returns a list.""" - return list(iter_child_nodes(node)) - - -def get_docstring(node, trim=True): - """Return the docstring for the given node or `None` if no docstring can - be found. If the node provided does not accept docstrings a `TypeError` - will be raised. - """ - if not isinstance(node, (FunctionDef, ClassDef, Module)): - raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - doc = node.body[0].value.s - if trim: - doc = trim_docstring(doc) - return doc - - -def trim_docstring(docstring): - """Trim a docstring. This should probably go into the inspect module.""" - lines = docstring.expandtabs().splitlines() - - # Find minimum indentation of any non-blank lines after first line. - from sys import maxint - margin = maxint - for line in lines[1:]: - content = len(line.lstrip()) - if content: - indent = len(line) - content - margin = min(margin, indent) - - # Remove indentation. - if lines: - lines[0] = lines[0].lstrip() - if margin < maxint: - for i in range(1, len(lines)): - lines[i] = lines[i][margin:] - - # Remove any trailing or leading blank lines. - while lines and not lines[-1]: - lines.pop() - while lines and not lines[0]: - lines.pop(0) - return '\n'.join(lines) - - -def get_symbol(operator): - """Return the symbol of the given operator node or node type.""" - if isinstance(operator, AST): - operator = type(operator) - try: - return ALL_SYMBOLS[operator] - except KeyError: - raise LookupError('no known symbol for %r' % operator) - - -def walk(node): - """Iterate over all nodes. This is useful if you only want to modify nodes - in place and don't care about the context or the order the nodes are - returned. - """ - from collections import deque - todo = deque([node]) - while todo: - node = todo.popleft() - todo.extend(iter_child_nodes(node)) - yield node - - -class NodeVisitor(object): - """Walks the abstract syntax tree and call visitor functions for every - node found. The visitor functions may return values which will be - forwarded by the `visit` method. - - Per default the visitor functions for the nodes are ``'visit_'`` + - class name of the node. So a `TryFinally` node visit function would - be `visit_TryFinally`. This behavior can be changed by overriding - the `get_visitor` function. If no visitor function exists for a node - (return value `None`) the `generic_visit` visitor is used instead. - - Don't use the `NodeVisitor` if you want to apply changes to nodes during - traversing. For this a special visitor exists (`NodeTransformer`) that - allows modifications. - """ - - def get_visitor(self, node): - """Return the visitor function for this node or `None` if no visitor - exists for this node. In that case the generic visit function is - used instead. - """ - method = 'visit_' + node.__class__.__name__ - return getattr(self, method, None) - - def visit(self, node): - """Visit a node.""" - f = self.get_visitor(node) - if f is not None: - return f(node) - return self.generic_visit(node) - - def generic_visit(self, node): - """Called if no explicit visitor function exists for a node.""" - for field, value in iter_fields(node): - if isinstance(value, list): - for item in value: - if isinstance(item, AST): - self.visit(item) - elif isinstance(value, AST): - self.visit(value) - - -class NodeTransformer(NodeVisitor): - """Walks the abstract syntax tree and allows modifications of nodes. - - The `NodeTransformer` will walk the AST and use the return value of the - visitor functions to replace or remove the old node. If the return - value of the visitor function is `None` the node will be removed - from the previous location otherwise it's replaced with the return - value. The return value may be the original node in which case no - replacement takes place. - - Here an example transformer that rewrites all `foo` to `data['foo']`:: - - class RewriteName(NodeTransformer): - - def visit_Name(self, node): - return copy_location(Subscript( - value=Name(id='data', ctx=Load()), - slice=Index(value=Str(s=node.id)), - ctx=node.ctx - ), node) - - Keep in mind that if the node you're operating on has child nodes - you must either transform the child nodes yourself or call the generic - visit function for the node first. - - Nodes that were part of a collection of statements (that applies to - all statement nodes) may also return a list of nodes rather than just - a single node. - - Usually you use the transformer like this:: - - node = YourTransformer().visit(node) - """ - - def generic_visit(self, node): - for field, old_value in iter_fields(node): - old_value = getattr(node, field, None) - if isinstance(old_value, list): - new_values = [] - for value in old_value: - if isinstance(value, AST): - value = self.visit(value) - if value is None: - continue - elif not isinstance(value, AST): - new_values.extend(value) - continue - new_values.append(value) - old_value[:] = new_values - elif isinstance(old_value, AST): - new_node = self.visit(old_value) - if new_node is None: - delattr(node, field) - else: - setattr(node, field, new_node) - return node diff --git a/nisext/codegen.py b/nisext/codegen.py deleted file mode 100644 index 5eef41ea72..0000000000 --- a/nisext/codegen.py +++ /dev/null @@ -1,530 +0,0 @@ -# -*- coding: utf-8 -*- -""" - codegen - ~~~~~~~ - - Extension to ast that allow ast -> python code generation. - - :copyright: Copyright 2008 by Armin Ronacher. - :license: BSD. - - From: http://dev.pocoo.org/hg/sandbox -""" -# Explicit local imports to satisfy python 2.5 -from .ast import (BOOLOP_SYMBOLS, BINOP_SYMBOLS, CMPOP_SYMBOLS, UNARYOP_SYMBOLS, - NodeVisitor, If, Name) - - -def to_source(node, indent_with=' ' * 4, add_line_information=False): - """This function can convert a node tree back into python sourcecode. - This is useful for debugging purposes, especially if you're dealing with - custom asts not generated by python itself. - - It could be that the sourcecode is evaluable when the AST itself is not - compilable / evaluable. The reason for this is that the AST contains some - more data than regular sourcecode does, which is dropped during - conversion. - - Each level of indentation is replaced with `indent_with`. Per default this - parameter is equal to four spaces as suggested by PEP 8, but it might be - adjusted to match the application's styleguide. - - If `add_line_information` is set to `True` comments for the line numbers - of the nodes are added to the output. This can be used to spot wrong line - number information of statement nodes. - """ - generator = SourceGenerator(indent_with, add_line_information) - generator.visit(node) - return ''.join(generator.result) - - -class SourceGenerator(NodeVisitor): - """This visitor is able to transform a well formed syntax tree into python - sourcecode. For more details have a look at the docstring of the - `node_to_source` function. - """ - - def __init__(self, indent_with, add_line_information=False): - self.result = [] - self.indent_with = indent_with - self.add_line_information = add_line_information - self.indentation = 0 - self.new_lines = 0 - - def write(self, x): - if self.new_lines: - if self.result: - self.result.append('\n' * self.new_lines) - self.result.append(self.indent_with * self.indentation) - self.new_lines = 0 - self.result.append(x) - - def newline(self, node=None, extra=0): - self.new_lines = max(self.new_lines, 1 + extra) - if node is not None and self.add_line_information: - self.write('# line: %s' % node.lineno) - self.new_lines = 1 - - def body(self, statements): - self.new_line = True - self.indentation += 1 - for stmt in statements: - self.visit(stmt) - self.indentation -= 1 - - def body_or_else(self, node): - self.body(node.body) - if node.orelse: - self.newline() - self.write('else:') - self.body(node.orelse) - - def signature(self, node): - want_comma = [] - def write_comma(): - if want_comma: - self.write(', ') - else: - want_comma.append(True) - - padding = [None] * (len(node.args) - len(node.defaults)) - for arg, default in zip(node.args, padding + node.defaults): - write_comma() - self.visit(arg) - if default is not None: - self.write('=') - self.visit(default) - if node.vararg is not None: - write_comma() - self.write('*' + node.vararg) - if node.kwarg is not None: - write_comma() - self.write('**' + node.kwarg) - - def decorators(self, node): - for decorator in node.decorator_list: - self.newline(decorator) - self.write('@') - self.visit(decorator) - - # Statements - - def visit_Assign(self, node): - self.newline(node) - for idx, target in enumerate(node.targets): - if idx: - self.write(', ') - self.visit(target) - self.write(' = ') - self.visit(node.value) - - def visit_AugAssign(self, node): - self.newline(node) - self.visit(node.target) - self.write(BINOP_SYMBOLS[type(node.op)] + '=') - self.visit(node.value) - - def visit_ImportFrom(self, node): - self.newline(node) - self.write('from %s%s import ' % ('.' * node.level, node.module)) - for idx, item in enumerate(node.names): - if idx: - self.write(', ') - self.write(item) - - def visit_Import(self, node): - self.newline(node) - for item in node.names: - self.write('import ') - self.visit(item) - - def visit_Expr(self, node): - self.newline(node) - self.generic_visit(node) - - def visit_FunctionDef(self, node): - self.newline(extra=1) - self.decorators(node) - self.newline(node) - self.write('def %s(' % node.name) - self.signature(node.args) - self.write('):') - self.body(node.body) - - def visit_ClassDef(self, node): - have_args = [] - def paren_or_comma(): - if have_args: - self.write(', ') - else: - have_args.append(True) - self.write('(') - - self.newline(extra=2) - self.decorators(node) - self.newline(node) - self.write('class %s' % node.name) - for base in node.bases: - paren_or_comma() - self.visit(base) - # XXX: the if here is used to keep this module compatible - # with python 2.6. - if hasattr(node, 'keywords'): - for keyword in node.keywords: - paren_or_comma() - self.write(keyword.arg + '=') - self.visit(keyword.value) - if node.starargs is not None: - paren_or_comma() - self.write('*') - self.visit(node.starargs) - if node.kwargs is not None: - paren_or_comma() - self.write('**') - self.visit(node.kwargs) - self.write(have_args and '):' or ':') - self.body(node.body) - - def visit_If(self, node): - self.newline(node) - self.write('if ') - self.visit(node.test) - self.write(':') - self.body(node.body) - while True: - else_ = node.orelse - if len(else_) == 1 and isinstance(else_[0], If): - node = else_[0] - self.newline() - self.write('elif ') - self.visit(node.test) - self.write(':') - self.body(node.body) - else: - self.newline() - self.write('else:') - self.body(else_) - break - - def visit_For(self, node): - self.newline(node) - self.write('for ') - self.visit(node.target) - self.write(' in ') - self.visit(node.iter) - self.write(':') - self.body_or_else(node) - - def visit_While(self, node): - self.newline(node) - self.write('while ') - self.visit(node.test) - self.write(':') - self.body_or_else(node) - - def visit_With(self, node): - self.newline(node) - self.write('with ') - self.visit(node.context_expr) - if node.optional_vars is not None: - self.write(' as ') - self.visit(node.optional_vars) - self.write(':') - self.body(node.body) - - def visit_Pass(self, node): - self.newline(node) - self.write('pass') - - def visit_Print(self, node): - # XXX: python 2.6 only - self.newline(node) - self.write('print ') - want_comma = False - if node.dest is not None: - self.write(' >> ') - self.visit(node.dest) - want_comma = True - for value in node.values: - if want_comma: - self.write(', ') - self.visit(value) - want_comma = True - if not node.nl: - self.write(',') - - def visit_Delete(self, node): - self.newline(node) - self.write('del ') - for idx, target in enumerate(node): - if idx: - self.write(', ') - self.visit(target) - - def visit_TryExcept(self, node): - self.newline(node) - self.write('try:') - self.body(node.body) - for handler in node.handlers: - self.visit(handler) - - def visit_TryFinally(self, node): - self.newline(node) - self.write('try:') - self.body(node.body) - self.newline(node) - self.write('finally:') - self.body(node.finalbody) - - def visit_Global(self, node): - self.newline(node) - self.write('global ' + ', '.join(node.names)) - - def visit_Nonlocal(self, node): - self.newline(node) - self.write('nonlocal ' + ', '.join(node.names)) - - def visit_Return(self, node): - self.newline(node) - self.write('return ') - self.visit(node.value) - - def visit_Break(self, node): - self.newline(node) - self.write('break') - - def visit_Continue(self, node): - self.newline(node) - self.write('continue') - - def visit_Raise(self, node): - # XXX: Python 2.6 / 3.0 compatibility - self.newline(node) - self.write('raise') - if hasattr(node, 'exc') and node.exc is not None: - self.write(' ') - self.visit(node.exc) - if node.cause is not None: - self.write(' from ') - self.visit(node.cause) - elif hasattr(node, 'type') and node.type is not None: - self.visit(node.type) - if node.inst is not None: - self.write(', ') - self.visit(node.inst) - if node.tback is not None: - self.write(', ') - self.visit(node.tback) - - # Expressions - - def visit_Attribute(self, node): - self.visit(node.value) - self.write('.' + node.attr) - - def visit_Call(self, node): - want_comma = [] - def write_comma(): - if want_comma: - self.write(', ') - else: - want_comma.append(True) - - self.visit(node.func) - self.write('(') - for arg in node.args: - write_comma() - self.visit(arg) - for keyword in node.keywords: - write_comma() - self.write(keyword.arg + '=') - self.visit(keyword.value) - if node.starargs is not None: - write_comma() - self.write('*') - self.visit(node.starargs) - if node.kwargs is not None: - write_comma() - self.write('**') - self.visit(node.kwargs) - self.write(')') - - def visit_Name(self, node): - self.write(node.id) - - def visit_Str(self, node): - self.write(repr(node.s)) - - def visit_Bytes(self, node): - self.write(repr(node.s)) - - def visit_Num(self, node): - self.write(repr(node.n)) - - def visit_Tuple(self, node): - self.write('(') - idx = -1 - for idx, item in enumerate(node.elts): - if idx: - self.write(', ') - self.visit(item) - self.write(idx and ')' or ',)') - - def sequence_visit(left, right): - def visit(self, node): - self.write(left) - for idx, item in enumerate(node.elts): - if idx: - self.write(', ') - self.visit(item) - self.write(right) - return visit - - visit_List = sequence_visit('[', ']') - visit_Set = sequence_visit('{', '}') - del sequence_visit - - def visit_Dict(self, node): - self.write('{') - for idx, (key, value) in enumerate(zip(node.keys, node.values)): - if idx: - self.write(', ') - self.visit(key) - self.write(': ') - self.visit(value) - self.write('}') - - def visit_BinOp(self, node): - self.visit(node.left) - self.write(' %s ' % BINOP_SYMBOLS[type(node.op)]) - self.visit(node.right) - - def visit_BoolOp(self, node): - self.write('(') - for idx, value in enumerate(node.values): - if idx: - self.write(' %s ' % BOOLOP_SYMBOLS[type(node.op)]) - self.visit(value) - self.write(')') - - def visit_Compare(self, node): - self.write('(') - self.write(node.left) - for op, right in zip(node.ops, node.comparators): - self.write(' %s %%' % CMPOP_SYMBOLS[type(op)]) - self.visit(right) - self.write(')') - - def visit_UnaryOp(self, node): - self.write('(') - op = UNARYOP_SYMBOLS[type(node.op)] - self.write(op) - if op == 'not': - self.write(' ') - self.visit(node.operand) - self.write(')') - - def visit_Subscript(self, node): - self.visit(node.value) - self.write('[') - self.visit(node.slice) - self.write(']') - - def visit_Slice(self, node): - if node.lower is not None: - self.visit(node.lower) - self.write(':') - if node.upper is not None: - self.visit(node.upper) - if node.step is not None: - self.write(':') - if not (isinstance(node.step, Name) and node.step.id == 'None'): - self.visit(node.step) - - def visit_ExtSlice(self, node): - for idx, item in node.dims: - if idx: - self.write(', ') - self.visit(item) - - def visit_Yield(self, node): - self.write('yield ') - self.visit(node.value) - - def visit_Lambda(self, node): - self.write('lambda ') - self.signature(node.args) - self.write(': ') - self.visit(node.body) - - def visit_Ellipsis(self, node): - self.write('Ellipsis') - - def generator_visit(left, right): - def visit(self, node): - self.write(left) - self.visit(node.elt) - for comprehension in node.generators: - self.visit(comprehension) - self.write(right) - return visit - - visit_ListComp = generator_visit('[', ']') - visit_GeneratorExp = generator_visit('(', ')') - visit_SetComp = generator_visit('{', '}') - del generator_visit - - def visit_DictComp(self, node): - self.write('{') - self.visit(node.key) - self.write(': ') - self.visit(node.value) - for comprehension in node.generators: - self.visit(comprehension) - self.write('}') - - def visit_IfExp(self, node): - self.visit(node.body) - self.write(' if ') - self.visit(node.test) - self.write(' else ') - self.visit(node.orelse) - - def visit_Starred(self, node): - self.write('*') - self.visit(node.value) - - def visit_Repr(self, node): - # XXX: python 2.6 only - self.write('`') - self.visit(node.value) - self.write('`') - - # Helper Nodes - - def visit_alias(self, node): - self.write(node.name) - if node.asname is not None: - self.write(' as ' + node.asname) - - def visit_comprehension(self, node): - self.write(' for ') - self.visit(node.target) - self.write(' in ') - self.visit(node.iter) - if node.ifs: - for if_ in node.ifs: - self.write(' if ') - self.visit(if_) - - def visit_excepthandler(self, node): - self.newline(node) - self.write('except') - if node.type is not None: - self.write(' ') - self.visit(node.type) - if node.name is not None: - self.write(' as ') - self.visit(node.name) - self.write(':') - self.body(node.body) diff --git a/nisext/py3builder.py b/nisext/py3builder.py index 2efddaff1f..9435f6c60b 100644 --- a/nisext/py3builder.py +++ b/nisext/py3builder.py @@ -40,200 +40,3 @@ def log_debug(self, msg, *args): r.refactor(files, write=True) # Then doctests r.refactor(files, write=True, doctests_only=True) - # Then custom doctests markup - doctest_markup_files(files) - - -def doctest_markup_files(fnames): - """ Process simple doctest comment markup on sequence of filenames - - Parameters - ---------- - fnames : seq - sequence of filenames - - Returns - ------- - None - """ - for fname in fnames: - with open(fname, 'rt') as fobj: - res = list(fobj) - out, errs = doctest_markup(res) - for err_tuple in errs: - print('Marked line %s unchanged because "%s"' % err_tuple) - with open(fname, 'wt') as fobj: - fobj.write(''.join(out)) - - -MARK_COMMENT = re.compile('(\s*>>>\s+)(.*?)(\s*#23dt\s+)(.*?\s*)$', re.DOTALL) -PLACE_LINE_EXPRS = re.compile('\s*([\w+\- ]*):\s*(.*)$') -INDENT_SPLITTER = re.compile('(\s*)(.*?)(\s*)$', re.DOTALL) - -def doctest_markup(in_lines): - """ Process doctest comment markup on sequence of strings - - The algorithm looks for lines that start with optional whitespace followed - by ``>>>`` and ending with a comment starting with ``#23dt``. The stuff - after the ``#23dt`` marker is the *markup* and gives instructions for - modifying the corresponding line or some other line. - - The *markup* is of form : . Let's say the output - lines are in a variable ``out_lines``. - - * is an expression giving a line number. In this expression, - the two variables defined are ``here`` (giving the current line number), - and ``next == here+1``. Let's call the result of ``place``. - If is empty (only whitespace before the colon) then ``place - == here``. The result of will replace ``lines[place]``. - * is a special value (see below) or a python3 expression - returning a processed value, where ``line`` contains the line referred to - by line number ``place``, and ``lines`` is a list of all lines. If - ``place != here``, then ``line == lines[place]``. If ``place == here`` - then ``line`` will be the source line, minus the comment and markup. - - A beginning with "replace(" we take to be short for - "line.replace(". - - Special values; if ==: - - * 'bytes': make all the strings in the selected line be byte strings. This - algormithm uses the ``ast`` module, so the text in which it works must be - valid python 3 syntax. - * 'BytesIO': shorthand for ``replace('StringIO', 'BytesIO')`` - - There is also a special non-doctest comment markup - '#23dt skip rest'. If - we find that comment (with whitespace before or after) as a line in the - file, we just pass the rest of the file unchanged. This is a hack to stop - 23dt processing its own tests. - - Parameters - ---------- - in_lines : sequence of str - - Returns - ------- - out_lines : sequence of str - lines with processing applied - error_tuples : sequence of (str, str) - sequence of 2 element tuples, where the first entry in the tuple is one - line that generated an error during processing, and the second is the - explanatory message for the error. These lines remain unchanged in - `out_lines`. - - Examples - -------- - The next three lines all do the same thing: - - >> a = '1234567890' #23dt here: line.replace("'12", "b'12") - >> a = '1234567890' #23dt here: replace("'12", "b'12") - >> a = '1234567890' #23dt here: bytes - - and that is to result in the part before the comment changing to: - - >> a = b'1234567890' - - The part after the comment (including markup) stays the same. - - You might want to process the line after the comment - such as test output. - The next test replaces "'a string'" with "b'a string'" - - >> 'a string'.encode('ascii') #23dt next: bytes - 'a string' - - This might work too, to do the same thing: - - >> 'a string'.encode('ascii') #23dt here+1: bytes - 'a string' - """ - out_lines = list(in_lines)[:] - err_tuples = [] - for pos, this in enumerate(out_lines): - # Check for 'leave the rest' markup - if this.strip() == '#23dt skip rest': - break - # Check for docest line with markup - mark_match = MARK_COMMENT.search(this) - if mark_match is None: - continue - docbits, marked_line, marker, markup = mark_match.groups() - place_line_match = PLACE_LINE_EXPRS.match(markup) - if place_line_match is None: - msg = ('Found markup "%s" in line "%s" but wrong syntax' % - (markup, this)) - err_tuples.append((this, msg)) - continue - place_expr, line_expr = place_line_match.groups() - exec_globals = {'here': pos, 'next': pos+1} - if place_expr.strip() == '': - place = pos - else: - try: - place = eval(place_expr, exec_globals) - except: - msg = ('Error finding place with "%s" in line "%s"' % - (place_expr, this)) - err_tuples.append((this, msg)) - continue - # Prevent processing operating on 23dt comment part of line - if place == pos: - line = marked_line - else: - line = out_lines[place] - # Shorthand - if line_expr == 'bytes': - # Any strings on the given line are byte strings - pre, mid, post = INDENT_SPLITTER.match(line).groups() - try: - res = byter(mid) - except: - err = sys.exc_info()[1] - msg = ('Error "%s" parsing "%s"' % (err, err)) - err_tuples.append((this, msg)) - continue - res = pre + res + post - else: - exec_globals.update({'line': line, 'lines': out_lines}) - # If line_expr starts with 'replace', implies "line.replace" - if line_expr.startswith('replace('): - line_expr = 'line.' + line_expr - elif line_expr == 'BytesIO': - line_expr = "line.replace('StringIO', 'BytesIO')" - try: - res = eval(line_expr, exec_globals) - except: - err = sys.exc_info()[1] - msg = ('Error "%s" working on "%s" at line %d with "%s"' % - (err, line, place, line_expr)) - err_tuples.append((this, msg)) - continue - # Put back comment if removed - if place == pos: - res = docbits + res + marker + markup - if res != line: - out_lines[place] = res - return out_lines, err_tuples - - -def byter(src): - """ Convert strings in `src` to byte string literals - - Parameters - ---------- - src : str - source string. Must be valid python 3 source - - Returns - ------- - p_src : str - string with ``str`` literals replace by ``byte`` literals - """ - import ast - from . import codegen - class RewriteStr(ast.NodeTransformer): - def visit_Str(self, node): - return ast.Bytes(node.s.encode('ascii')) - tree = ast.parse(src) - tree = RewriteStr().visit(tree) - return codegen.to_source(tree) - diff --git a/nisext/tests/test_doctest_markup.py b/nisext/tests/test_doctest_markup.py deleted file mode 100644 index 55254a2fce..0000000000 --- a/nisext/tests/test_doctest_markup.py +++ /dev/null @@ -1,163 +0,0 @@ -""" Testing doctest markup tests -""" - -import sys -from ..py3builder import doctest_markup, byter - -from numpy.testing import (assert_array_almost_equal, assert_array_equal, dec) - -from nose.tools import assert_true, assert_equal, assert_raises - -is_2 = sys.version_info[0] < 3 -skip42 = dec.skipif(is_2) - -# Tell 23dt processing to pass the rest of this file unchanged. We don't want -# the processor to mess up the example string -#23dt skip rest - -IN_TXT = """ - -Anonymous lines, also blanks - -As all that is empty, use entropy, and endure - -# Comment, unchanged - -#23dt comment not processed without doctest marker ->>> #23dthere: no whitespace; comment not recognized even as error ->>>#23dt nor without preceding whitespace ->>> #23dt not correct syntax creates error ->>> #23dt novar: 'undefined variable creates error' ->>> #23dt here: 'OK' ->>> #23dt here : 'tolerates whitespace' ->>> #23dt here + 0 : 'OK' ->>> #23dt here -0 : 'OK' ->>> #23dt here - here + here + 0: 'OK' ->>> #23dt here *0 : 'only allowed plus or minus' ->>> #23dt : 'empty means here' ->>> #23dt : 'regardless of whitespace' ->>> #23dt 'need colon' ->>> #23dt here : 3bad syntax ->>> #23dt here : 1/0 ->>> #23dt next : line.replace('some','') -something ->>> #23dt next : replace('some','') -something ->>> #23dt next : lines[next].replace('some','') -something ->>> #23dt next + 1: line.replace('some','') -something -something ->>> #23dt next : lines[next+1].replace('some','') -this is the line where replacement happens -something - >>> whitespace #23dt : 'OK' ->>> from io import StringIO as BytesIO #23dt : replace('StringIO as ', '') ->>> from io import StringIO #23dt : BytesIO ->>> from io import StringIO #23dt : BytesIO -""" - -OUT_TXT = """ - -Anonymous lines, also blanks - -As all that is empty, use entropy, and endure - -# Comment, unchanged - -#23dt comment not processed without doctest marker ->>> #23dthere: no whitespace; comment not recognized even as error ->>>#23dt nor without preceding whitespace ->>> #23dt not correct syntax creates error ->>> #23dt novar: 'undefined variable creates error' ->>> OK#23dt here: 'OK' ->>> tolerates whitespace#23dt here : 'tolerates whitespace' ->>> OK#23dt here + 0 : 'OK' ->>> OK#23dt here -0 : 'OK' ->>> OK#23dt here - here + here + 0: 'OK' ->>> #23dt here *0 : 'only allowed plus or minus' ->>> empty means here#23dt : 'empty means here' ->>> regardless of whitespace#23dt : 'regardless of whitespace' ->>> #23dt 'need colon' ->>> #23dt here : 3bad syntax ->>> #23dt here : 1/0 ->>> #23dt next : line.replace('some','') -thing ->>> #23dt next : replace('some','') -thing ->>> #23dt next : lines[next].replace('some','') -thing ->>> #23dt next + 1: line.replace('some','') -something -thing ->>> #23dt next : lines[next+1].replace('some','') -thing -something - >>> OK #23dt : 'OK' ->>> from io import BytesIO #23dt : replace('StringIO as ', '') ->>> from io import BytesIO #23dt : BytesIO ->>> from io import BytesIO #23dt : BytesIO -""" - -ERR_TXT = \ -""">>> #23dt not correct syntax creates error ->>> #23dt novar: 'undefined variable creates error' ->>> #23dt here *0 : 'only allowed plus or minus' ->>> #23dt 'need colon' ->>> #23dt here : 3bad syntax ->>> #23dt here : 1/0 -""" - -def test_some_text(): - out_lines, err_tuples = doctest_markup(IN_TXT.splitlines(True)) - assert_equal(out_lines, OUT_TXT.splitlines(True)) - err_lines, err_msgs = zip(*err_tuples) - assert_equal(list(err_lines), ERR_TXT.splitlines(True)) - - -IN_BYTES_TXT = """\ - -Phatos lives - ->>> 'hello' #23dt : bytes ->>> (1, 'hello') #23dt : bytes ->>> 'hello' #23dt next : bytes -'TRACK' ->>> ('hello', 1, 'world') #23dt : bytes ->>> 3bad_syntax #23dt : bytes -""" - -OUT_BYTES_TXT = """\ - -Phatos lives - ->>> b'hello' #23dt : bytes ->>> (1, b'hello') #23dt : bytes ->>> 'hello' #23dt next : bytes -b'TRACK' ->>> (b'hello', 1, b'world') #23dt : bytes ->>> 3bad_syntax #23dt : bytes -""" - -ERR_BYTES_TXT = \ -""">>> 3bad_syntax #23dt : bytes -""" - -@skip42 -def test_bytes_text(): - out_lines, err_tuples = doctest_markup(IN_BYTES_TXT.splitlines(True)) - assert_equal(out_lines, OUT_BYTES_TXT.splitlines(True)) - err_lines, err_msgs = zip(*err_tuples) - assert_equal(list(err_lines), ERR_BYTES_TXT.splitlines(True)) - - -@skip42 -def test_byter(): - # Test bytes formatter - assert_equal('(b"hello \' world", b\'again\')', - byter('("hello \' world", "again")')) - line = "_ = bio.write(' ' * 10)" - assert_equal( - byter(line), - "_ = bio.write(b' ' * 10)") -