Statistics
| Branch: | Tag: | Revision:

root / snf-deploy / snfdeploy / massedit.py @ f92dffed

History | View | Annotate | Download (10.6 kB)

1 0ac84a9a Dimitris Aragiorgis
#!/usr/bin/env python
2 0ac84a9a Dimitris Aragiorgis
# encoding='cp1252'
3 0ac84a9a Dimitris Aragiorgis
4 0ac84a9a Dimitris Aragiorgis
"""A python bulk editor class to apply the same code to many files."""
5 0ac84a9a Dimitris Aragiorgis
6 0ac84a9a Dimitris Aragiorgis
# Copyright (c) 2012 Jerome Lecomte
7 0ac84a9a Dimitris Aragiorgis
#
8 0ac84a9a Dimitris Aragiorgis
# Permission is hereby granted, free of charge, to any person obtaining a copy
9 0ac84a9a Dimitris Aragiorgis
# of this software and associated documentation files (the "Software"), to deal
10 0ac84a9a Dimitris Aragiorgis
# in the Software without restriction, including without limitation the rights
11 0ac84a9a Dimitris Aragiorgis
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 0ac84a9a Dimitris Aragiorgis
# copies of the Software, and to permit persons to whom the Software is
13 0ac84a9a Dimitris Aragiorgis
# furnished to do so, subject to the following conditions:
14 0ac84a9a Dimitris Aragiorgis
#
15 0ac84a9a Dimitris Aragiorgis
# The above copyright notice and this permission notice shall be included in
16 0ac84a9a Dimitris Aragiorgis
# all copies or substantial portions of the Software.
17 0ac84a9a Dimitris Aragiorgis
#
18 0ac84a9a Dimitris Aragiorgis
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 0ac84a9a Dimitris Aragiorgis
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 0ac84a9a Dimitris Aragiorgis
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 0ac84a9a Dimitris Aragiorgis
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 0ac84a9a Dimitris Aragiorgis
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 0ac84a9a Dimitris Aragiorgis
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 0ac84a9a Dimitris Aragiorgis
# THE SOFTWARE.
25 0ac84a9a Dimitris Aragiorgis
26 0ac84a9a Dimitris Aragiorgis
27 0ac84a9a Dimitris Aragiorgis
__version__ = '0.61'  # UPDATE setup.py when changing version.
28 0ac84a9a Dimitris Aragiorgis
__author__ = 'Jerome Lecomte'
29 0ac84a9a Dimitris Aragiorgis
__license__ = 'MIT'
30 0ac84a9a Dimitris Aragiorgis
31 0ac84a9a Dimitris Aragiorgis
32 0ac84a9a Dimitris Aragiorgis
import os
33 0ac84a9a Dimitris Aragiorgis
import sys
34 0ac84a9a Dimitris Aragiorgis
import logging
35 0ac84a9a Dimitris Aragiorgis
import argparse
36 0ac84a9a Dimitris Aragiorgis
import difflib
37 0ac84a9a Dimitris Aragiorgis
# Most manip will involve re so we include it here for convenience.
38 0ac84a9a Dimitris Aragiorgis
import re  # pylint: disable=W0611
39 0ac84a9a Dimitris Aragiorgis
import fnmatch
40 0ac84a9a Dimitris Aragiorgis
41 0ac84a9a Dimitris Aragiorgis
42 0ac84a9a Dimitris Aragiorgis
logger = logging.getLogger(__name__)
43 0ac84a9a Dimitris Aragiorgis
44 0ac84a9a Dimitris Aragiorgis
45 0ac84a9a Dimitris Aragiorgis
class EditorError(RuntimeError):
46 0ac84a9a Dimitris Aragiorgis
    """Error raised by the Editor class."""
47 0ac84a9a Dimitris Aragiorgis
    pass
48 0ac84a9a Dimitris Aragiorgis
49 0ac84a9a Dimitris Aragiorgis
50 0ac84a9a Dimitris Aragiorgis
class Editor(object):
51 0ac84a9a Dimitris Aragiorgis
    """Processes input file or input line.
52 0ac84a9a Dimitris Aragiorgis

53 0ac84a9a Dimitris Aragiorgis
    Named arguments:
54 0ac84a9a Dimitris Aragiorgis
    code -- code expression to process the input with.
55 0ac84a9a Dimitris Aragiorgis
    """
56 0ac84a9a Dimitris Aragiorgis
57 0ac84a9a Dimitris Aragiorgis
    def __init__(self, **kwds):
58 0ac84a9a Dimitris Aragiorgis
        self.code_objs = dict()
59 0ac84a9a Dimitris Aragiorgis
        self._codes = []
60 0ac84a9a Dimitris Aragiorgis
        self.dry_run = None
61 0ac84a9a Dimitris Aragiorgis
        if 'module' in kwds:
62 0ac84a9a Dimitris Aragiorgis
            self.import_module(kwds['module'])
63 0ac84a9a Dimitris Aragiorgis
        if 'code' in kwds:
64 0ac84a9a Dimitris Aragiorgis
            self.append_code_expr(kwds['code'])
65 0ac84a9a Dimitris Aragiorgis
        if 'dry_run' in kwds:
66 0ac84a9a Dimitris Aragiorgis
            self.dry_run = kwds['dry_run']
67 0ac84a9a Dimitris Aragiorgis
68 0ac84a9a Dimitris Aragiorgis
    def __edit_line(self, line, code, code_obj):  # pylint: disable=R0201
69 0ac84a9a Dimitris Aragiorgis
        """Edit a line with one code object built in the ctor."""
70 0ac84a9a Dimitris Aragiorgis
        try:
71 0ac84a9a Dimitris Aragiorgis
            result = eval(code_obj, globals(), locals())
72 0ac84a9a Dimitris Aragiorgis
        except TypeError as ex:
73 0ac84a9a Dimitris Aragiorgis
            message = "failed to execute {0}: {1}".format(code, ex)
74 0ac84a9a Dimitris Aragiorgis
            logger.warning(message)
75 0ac84a9a Dimitris Aragiorgis
            raise EditorError(message)
76 0ac84a9a Dimitris Aragiorgis
        if not result:
77 0ac84a9a Dimitris Aragiorgis
            raise EditorError("cannot process line '{0}' with {1}".format(
78 0ac84a9a Dimitris Aragiorgis
                              line, code))
79 0ac84a9a Dimitris Aragiorgis
        elif isinstance(result, list) or isinstance(result, tuple):
80 0ac84a9a Dimitris Aragiorgis
            line = ' '.join([str(res_element) for res_element in result])
81 0ac84a9a Dimitris Aragiorgis
        else:
82 0ac84a9a Dimitris Aragiorgis
            line = str(result)
83 0ac84a9a Dimitris Aragiorgis
        return line
84 0ac84a9a Dimitris Aragiorgis
85 0ac84a9a Dimitris Aragiorgis
    def edit_line(self, line):
86 0ac84a9a Dimitris Aragiorgis
        """Edits a single line using the code expression."""
87 0ac84a9a Dimitris Aragiorgis
        for code, code_obj in self.code_objs.items():
88 0ac84a9a Dimitris Aragiorgis
            line = self.__edit_line(line, code, code_obj)
89 0ac84a9a Dimitris Aragiorgis
        return line
90 0ac84a9a Dimitris Aragiorgis
91 0ac84a9a Dimitris Aragiorgis
    def edit_file(self, file_name):
92 0ac84a9a Dimitris Aragiorgis
        """Edit file in place, returns a list of modifications (unified diff).
93 0ac84a9a Dimitris Aragiorgis

94 0ac84a9a Dimitris Aragiorgis
        Arguments:
95 0ac84a9a Dimitris Aragiorgis
        file_name -- The name of the file.
96 0ac84a9a Dimitris Aragiorgis
        dry_run -- only return differences, but do not edit the file.
97 0ac84a9a Dimitris Aragiorgis
        """
98 0ac84a9a Dimitris Aragiorgis
        with open(file_name, "r") as from_file:
99 0ac84a9a Dimitris Aragiorgis
            from_lines = from_file.readlines()
100 0ac84a9a Dimitris Aragiorgis
            to_lines = [self.edit_line(line) for line in from_lines]
101 0ac84a9a Dimitris Aragiorgis
            diffs = difflib.unified_diff(from_lines, to_lines,
102 0ac84a9a Dimitris Aragiorgis
                                         fromfile=file_name, tofile='<new>')
103 0ac84a9a Dimitris Aragiorgis
        if not self.dry_run:
104 0ac84a9a Dimitris Aragiorgis
            bak_file_name = file_name + ".bak"
105 0ac84a9a Dimitris Aragiorgis
            if os.path.exists(bak_file_name):
106 0ac84a9a Dimitris Aragiorgis
                raise EditorError("{0} already exists".format(bak_file_name))
107 0ac84a9a Dimitris Aragiorgis
            try:
108 0ac84a9a Dimitris Aragiorgis
                os.rename(file_name, bak_file_name)
109 0ac84a9a Dimitris Aragiorgis
                with open(file_name, "w") as new_file:
110 0ac84a9a Dimitris Aragiorgis
                    new_file.writelines(to_lines)
111 0ac84a9a Dimitris Aragiorgis
                os.unlink(bak_file_name)
112 0ac84a9a Dimitris Aragiorgis
            except:
113 0ac84a9a Dimitris Aragiorgis
                os.rename(bak_file_name, file_name)
114 0ac84a9a Dimitris Aragiorgis
                raise
115 0ac84a9a Dimitris Aragiorgis
        return list(diffs)
116 0ac84a9a Dimitris Aragiorgis
117 0ac84a9a Dimitris Aragiorgis
    def append_code_expr(self, code):
118 0ac84a9a Dimitris Aragiorgis
        """Compiles argument and adds it to the list of code objects."""
119 0ac84a9a Dimitris Aragiorgis
        assert(isinstance(code, str))  # expect a string.
120 0ac84a9a Dimitris Aragiorgis
        logger.debug("compiling code {0}...".format(code))
121 0ac84a9a Dimitris Aragiorgis
        try:
122 0ac84a9a Dimitris Aragiorgis
            code_obj = compile(code, '<string>', 'eval')
123 0ac84a9a Dimitris Aragiorgis
            self.code_objs[code] = code_obj
124 0ac84a9a Dimitris Aragiorgis
        except SyntaxError as syntax_err:
125 0ac84a9a Dimitris Aragiorgis
            logger.error("cannot compile {0}: {1}".format(
126 0ac84a9a Dimitris Aragiorgis
                code, syntax_err))
127 0ac84a9a Dimitris Aragiorgis
            raise
128 0ac84a9a Dimitris Aragiorgis
        logger.debug("compiled code {0}".format(code))
129 0ac84a9a Dimitris Aragiorgis
130 0ac84a9a Dimitris Aragiorgis
    def set_code_expr(self, codes):
131 0ac84a9a Dimitris Aragiorgis
        """Convenience: sets all the code expressions at once."""
132 0ac84a9a Dimitris Aragiorgis
        self.code_objs = dict()
133 0ac84a9a Dimitris Aragiorgis
        self._codes = []
134 0ac84a9a Dimitris Aragiorgis
        for code in codes:
135 0ac84a9a Dimitris Aragiorgis
            self.append_code_expr(code)
136 0ac84a9a Dimitris Aragiorgis
137 0ac84a9a Dimitris Aragiorgis
    def import_module(self, module):  # pylint: disable=R0201
138 0ac84a9a Dimitris Aragiorgis
        """Imports module that are needed for the code expr to compile.
139 0ac84a9a Dimitris Aragiorgis

140 0ac84a9a Dimitris Aragiorgis
        Argument:
141 0ac84a9a Dimitris Aragiorgis
        module -- can be scalar string or a list of strings.
142 0ac84a9a Dimitris Aragiorgis
        """
143 0ac84a9a Dimitris Aragiorgis
        if isinstance(module, list):
144 0ac84a9a Dimitris Aragiorgis
            all_modules = module
145 0ac84a9a Dimitris Aragiorgis
        else:
146 0ac84a9a Dimitris Aragiorgis
            all_modules = [module]
147 0ac84a9a Dimitris Aragiorgis
        for mod in all_modules:
148 0ac84a9a Dimitris Aragiorgis
            globals()[mod] = __import__(mod.strip())
149 0ac84a9a Dimitris Aragiorgis
150 0ac84a9a Dimitris Aragiorgis
151 0ac84a9a Dimitris Aragiorgis
def parse_command_line(argv):
152 0ac84a9a Dimitris Aragiorgis
    """Parses command line argument. See -h option
153 0ac84a9a Dimitris Aragiorgis

154 0ac84a9a Dimitris Aragiorgis
    argv -- arguments on the command line including the caller file.
155 0ac84a9a Dimitris Aragiorgis
    """
156 0ac84a9a Dimitris Aragiorgis
    example = """
157 0ac84a9a Dimitris Aragiorgis
    example: {0} -e "re.sub('failIf', 'assertFalse', line)" *.py
158 0ac84a9a Dimitris Aragiorgis
    """
159 0ac84a9a Dimitris Aragiorgis
    example = example.format(os.path.basename(argv[0]))
160 0ac84a9a Dimitris Aragiorgis
    if sys.version_info[0] < 3:
161 0ac84a9a Dimitris Aragiorgis
        parser = argparse.ArgumentParser(description="Python mass editor",
162 0ac84a9a Dimitris Aragiorgis
                                         version=__version__,
163 0ac84a9a Dimitris Aragiorgis
                                         epilog=example)
164 0ac84a9a Dimitris Aragiorgis
    else:
165 0ac84a9a Dimitris Aragiorgis
        parser = argparse.ArgumentParser(description="Python mass editor",
166 0ac84a9a Dimitris Aragiorgis
                                         epilog=example)
167 0ac84a9a Dimitris Aragiorgis
        parser.add_argument("-v", "--version", action="version",
168 0ac84a9a Dimitris Aragiorgis
                            version="%(prog)s {}".format(__version__))
169 0ac84a9a Dimitris Aragiorgis
    parser.add_argument("-w", "--write", dest="dry_run",
170 0ac84a9a Dimitris Aragiorgis
                        action="store_false", default=True,
171 0ac84a9a Dimitris Aragiorgis
                        help="modify target file(s) in place. "
172 0ac84a9a Dimitris Aragiorgis
                        "Shows diff otherwise.")
173 0ac84a9a Dimitris Aragiorgis
    parser.add_argument("-V", "--verbose", dest="verbose_count",
174 0ac84a9a Dimitris Aragiorgis
                        action="count", default=0,
175 0ac84a9a Dimitris Aragiorgis
                        help="increases log verbosity (can be specified "
176 0ac84a9a Dimitris Aragiorgis
                        "multiple times)")
177 0ac84a9a Dimitris Aragiorgis
    parser.add_argument('-e', "--expression", dest="expressions", nargs=1,
178 0ac84a9a Dimitris Aragiorgis
                        help="Python expressions to be applied on all files. "
179 0ac84a9a Dimitris Aragiorgis
                        "Use the line variable to reference the current line.")
180 0ac84a9a Dimitris Aragiorgis
    parser.add_argument("-s", "--start", dest="start_dir",
181 0ac84a9a Dimitris Aragiorgis
                        help="Starting directory in which to look for the "
182 0ac84a9a Dimitris Aragiorgis
                        "files. If there is one pattern only and it includes "
183 0ac84a9a Dimitris Aragiorgis
                        "a directory, the start dir will be that directory "
184 0ac84a9a Dimitris Aragiorgis
                        "and the max depth level will be set to 1.")
185 0ac84a9a Dimitris Aragiorgis
    parser.add_argument('-m', "--max-depth-level", type=int, dest="max_depth",
186 0ac84a9a Dimitris Aragiorgis
                        help="Maximum depth when walking subdirectories.")
187 0ac84a9a Dimitris Aragiorgis
    parser.add_argument('-o', '--output', metavar="output",
188 0ac84a9a Dimitris Aragiorgis
                        type=argparse.FileType('w'), default=sys.stdout,
189 0ac84a9a Dimitris Aragiorgis
                        help="redirect output to a file")
190 0ac84a9a Dimitris Aragiorgis
    parser.add_argument('patterns', metavar="pattern", nargs='+',
191 0ac84a9a Dimitris Aragiorgis
                        help="file patterns to process.")
192 0ac84a9a Dimitris Aragiorgis
    arguments = parser.parse_args(argv[1:])
193 0ac84a9a Dimitris Aragiorgis
    # Sets log level to WARN going more verbose for each new -V.
194 0ac84a9a Dimitris Aragiorgis
    logger.setLevel(max(3 - arguments.verbose_count, 0) * 10)
195 0ac84a9a Dimitris Aragiorgis
    return arguments
196 0ac84a9a Dimitris Aragiorgis
197 0ac84a9a Dimitris Aragiorgis
198 0ac84a9a Dimitris Aragiorgis
def edit_files(patterns, expressions,  # pylint: disable=R0913, R0914
199 0ac84a9a Dimitris Aragiorgis
               start_dir=None, max_depth=1, dry_run=True,
200 0ac84a9a Dimitris Aragiorgis
               output=sys.stdout):
201 0ac84a9a Dimitris Aragiorgis
    """Edits the files that match patterns with python expressions. Each
202 0ac84a9a Dimitris Aragiorgis
    expression is run (using eval()) line by line on each input file.
203 0ac84a9a Dimitris Aragiorgis

204 0ac84a9a Dimitris Aragiorgis
    Keyword Arguments:
205 0ac84a9a Dimitris Aragiorgis
    max_depth -- maximum recursion level when looking for file matches.
206 0ac84a9a Dimitris Aragiorgis
    start_dir -- directory where to start the file search.
207 0ac84a9a Dimitris Aragiorgis
    dry_run -- only display differences if True. Save modified file otherwise.
208 0ac84a9a Dimitris Aragiorgis
    output -- handle where the output should be redirected.
209 0ac84a9a Dimitris Aragiorgis
    """
210 0ac84a9a Dimitris Aragiorgis
    # Makes for a better diagnostic because str are also iterable.
211 0ac84a9a Dimitris Aragiorgis
    assert not isinstance(patterns, str), "patterns should be a list"
212 0ac84a9a Dimitris Aragiorgis
    assert not isinstance(expressions, str), "expressions should be a list"
213 0ac84a9a Dimitris Aragiorgis
214 0ac84a9a Dimitris Aragiorgis
    # Shortcut: if there is only one pattern, make sure we process just that.
215 0ac84a9a Dimitris Aragiorgis
    if len(patterns) == 1 and not start_dir:
216 0ac84a9a Dimitris Aragiorgis
        pattern = patterns[0]
217 0ac84a9a Dimitris Aragiorgis
        directory = os.path.dirname(pattern)
218 0ac84a9a Dimitris Aragiorgis
        if directory:
219 0ac84a9a Dimitris Aragiorgis
            patterns = [os.path.basename(pattern)]
220 0ac84a9a Dimitris Aragiorgis
            start_dir = directory
221 0ac84a9a Dimitris Aragiorgis
            max_depth = 1
222 0ac84a9a Dimitris Aragiorgis
223 0ac84a9a Dimitris Aragiorgis
    processed_paths = []
224 0ac84a9a Dimitris Aragiorgis
    editor = Editor(dry_run=dry_run)
225 0ac84a9a Dimitris Aragiorgis
    if expressions:
226 0ac84a9a Dimitris Aragiorgis
        editor.set_code_expr(expressions)
227 0ac84a9a Dimitris Aragiorgis
    if not start_dir:
228 0ac84a9a Dimitris Aragiorgis
        start_dir = os.getcwd()
229 0ac84a9a Dimitris Aragiorgis
    for root, dirs, files in os.walk(start_dir):  # pylint: disable=W0612
230 0ac84a9a Dimitris Aragiorgis
        if max_depth is not None:
231 0ac84a9a Dimitris Aragiorgis
            relpath = os.path.relpath(root, start=start_dir)
232 0ac84a9a Dimitris Aragiorgis
            depth = len(relpath.split(os.sep))
233 0ac84a9a Dimitris Aragiorgis
            if depth > max_depth:
234 0ac84a9a Dimitris Aragiorgis
                continue
235 0ac84a9a Dimitris Aragiorgis
        names = []
236 0ac84a9a Dimitris Aragiorgis
        for pattern in patterns:
237 0ac84a9a Dimitris Aragiorgis
            names += fnmatch.filter(files, pattern)
238 0ac84a9a Dimitris Aragiorgis
        for name in names:
239 0ac84a9a Dimitris Aragiorgis
            path = os.path.join(root, name)
240 0ac84a9a Dimitris Aragiorgis
            processed_paths.append(os.path.abspath(path))
241 0ac84a9a Dimitris Aragiorgis
            diffs = editor.edit_file(path)
242 0ac84a9a Dimitris Aragiorgis
            if dry_run:
243 0ac84a9a Dimitris Aragiorgis
                output.write("".join(diffs))
244 0ac84a9a Dimitris Aragiorgis
    if output != sys.stdout:
245 0ac84a9a Dimitris Aragiorgis
        output.close()
246 0ac84a9a Dimitris Aragiorgis
    return processed_paths
247 0ac84a9a Dimitris Aragiorgis
248 0ac84a9a Dimitris Aragiorgis
249 0ac84a9a Dimitris Aragiorgis
def command_line(argv):
250 0ac84a9a Dimitris Aragiorgis
    """Instantiate an editor and process arguments.
251 0ac84a9a Dimitris Aragiorgis

252 0ac84a9a Dimitris Aragiorgis
    Optional argument:
253 0ac84a9a Dimitris Aragiorgis
    processed_paths -- paths processed are appended to the list.
254 0ac84a9a Dimitris Aragiorgis
    """
255 0ac84a9a Dimitris Aragiorgis
    arguments = parse_command_line(argv)
256 0ac84a9a Dimitris Aragiorgis
    return edit_files(arguments.patterns, arguments.expressions,
257 0ac84a9a Dimitris Aragiorgis
                      start_dir=arguments.start_dir,
258 0ac84a9a Dimitris Aragiorgis
                      max_depth=arguments.max_depth,
259 0ac84a9a Dimitris Aragiorgis
                      dry_run=arguments.dry_run,
260 0ac84a9a Dimitris Aragiorgis
                      output=arguments.output)
261 0ac84a9a Dimitris Aragiorgis
262 0ac84a9a Dimitris Aragiorgis
263 0ac84a9a Dimitris Aragiorgis
if __name__ == "__main__":
264 0ac84a9a Dimitris Aragiorgis
    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
265 0ac84a9a Dimitris Aragiorgis
    try:
266 0ac84a9a Dimitris Aragiorgis
        command_line(sys.argv)
267 0ac84a9a Dimitris Aragiorgis
    finally:
268 0ac84a9a Dimitris Aragiorgis
        logging.shutdown()