refactor

.gitignore

@@ -0,0 +1,2 @@
+.idea
+*.pyc

app/Formatter.py

@@ -0,0 +1,64 @@
+import abc
+import json
+import os
+
+
+class Formater():
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def output(self, notebook, out_path):
+        """Outputs the notebook data in the desired form"""
+        return
+
+    @staticmethod
+    @abc.abstractmethod
+    def construct_output_path(input_path):
+        return
+
+
+class ToNotebook(Formater):
+    dry_run = None
+    overwrite = None
+
+    def __init__(self, overwrite=False, dry_run=False):
+        self.overwrite = overwrite
+        self.dry_run = dry_run
+
+    def output(self, notebook, out_path):
+        output = notebook.to_dict()
+        self.write_py_data_to_notebook(output, out_path)
+
+    @staticmethod
+    def construct_output_path(input_path):
+        input_headless, ext = os.path.splitext(input_path)
+        return input_headless + ".ipynb"
+
+    @staticmethod
+    def write_py_data_to_notebook(output, out_file_path):
+        with open(out_file_path, 'w') as outfile:
+            json.dump(output, outfile)
+
+
+class ToPy(Formater):
+    def __init__(self, overwrite=False, dry_run=False,):
+        self.overwrite = overwrite
+        self.dry_run = dry_run
+
+    def output(self, notebook, out_path):
+        with open(out_path, 'w') as output:
+            self.add_header(output, str(notebook.notebook_format))
+            for cell in notebook.cells:
+                output.write(cell.generate_field_output())
+
+    @staticmethod
+    def construct_output_path(input_path):
+        input_headless, ext = os.path.splitext(input_path)
+        return input_headless + ".py"
+
+    @staticmethod
+    def add_header(output, notebook_format):
+        assert isinstance(output, file)
+        assert isinstance(notebook_format, str)
+        output.write('# -*- coding: utf-8 -*-\n')
+        output.write('# <nbformat>' + notebook_format + '</nbformat>\n')

app/Notebook.py

@@ -0,0 +1,58 @@
+import json
+import os
+from ReadPy import ReadPy
+
+
+class Notebook(object):
+    notebook_format = None
+    cells = []
+    metadata = None
+    nbformat_minor = None
+
+    def read(self, path_to_file):
+        input_headless, ext = os.path.splitext(path_to_file)
+        if ext == ".ipynb":
+            self.read_notebook(path_to_file)
+        elif ext == ".py":
+            self.read_py(path_to_file)
+
+    def read_notebook(self, path_to_file):
+        with open(path_to_file, 'r') as notebook:
+            notebook_data = json.load(notebook)
+        self.notebook_format = notebook_data["nbformat"]
+        assert "cells" in notebook_data.keys(), "Nbformat is " + str(notebook_data['nbformat']) \
+                                                + ", try the old converter script."
+        for cell in notebook_data["cells"]:
+            self.cells.append(Cell(cell))
+
+    @staticmethod
+    def read_py(path_to_file):
+        reader = ReadPy()
+        reader.read(path_to_file)
+
+    def to_dict(self):
+        cells = {'metadata': self.metadata,
+                 'nbformat': self.notebook_format,
+                 'nbformat_minor': self.nbformat_minor,
+                 'cells': []}
+        for cell in self.cells:
+            cells['cells'].append(cell.to_dict())
+        return cells
+
+
+class Cell(object):
+    def __init__(self, cell):
+        self.type = cell['cell_type']
+        self.source = cell["source"]
+
+    def generate_field_output(self):
+        output = '\n# <' + self.type + 'cell' + '>\n\n'
+        for item in self.source:
+            if self.type == "code":
+                output += item
+            else:
+                output += "# " + item
+        return output + "\n"
+
+    def to_dict(self):
+        return {'cell_type': self.type, 'source': self.source}

app/NotebookConverter.py

@@ -0,0 +1,22 @@
+import os
+import fnmatch
+from Notebook import Notebook
+
+
+class NotebookConverter(object):
+    def convert_all(self, directory, formater):
+        for root, dirnames, filenames in os.walk(directory):
+            for filename in fnmatch.filter(filenames, '*.ipynb'):
+                filename = os.path.abspath(os.path.join(root, filename))
+                self.convert(filename, formater)
+
+    @staticmethod
+    def convert(input_file_path, formater):
+        output_file_path = formater.construct_output_path(input_file_path)
+        if not os.path.exists(output_file_path) or formater.overwrite:
+            nb = Notebook()
+            nb.read(input_file_path)
+            if not formater.dry_run:
+                formater.output(nb, output_file_path)
+            print "Created file: {}".format(output_file_path)
+

app/ReadPy.py

@@ -0,0 +1,92 @@
+from app.Notebook import Cell
+
+
+class ReadPy(object):
+    current_cell = None
+    execution_count = 1
+    _outputcells = []
+
+    def read(self, path_to_file):
+        skip_one_line = False
+        with open(path_to_file, 'r') as lines:
+            self.add_descriptive_data(lines.readlines())
+            lines.seek(0)
+            for line in lines:
+                if skip_one_line:
+                    skip_one_line = False
+                elif self.is_first_line_of_cell(line):
+                    self.close_cell()
+                    if self.current_cell == 'code':
+                        self.execution_count += 1
+                    self.open_cell(line, self.execution_count)
+                    skip_one_line = True
+                elif self.current_cell.type in ('markdown', 'code'):
+                    self.append_line_to_source(line)
+            self.close_last_cell()
+            return self._outputcells
+
+    def close_cell(self):
+        if self.current_cell.type in ('markdown', 'code'):
+            if len(self.current_cell.source) > 1:
+                del self.current_cell.source[-1:]
+            self.current_cell.source[-1] = self.current_cell.source[-1].rstrip('\n')
+            self._outputcells.append(self.current_cell)
+
+    def close_last_cell(self):
+        if self.current_cell.type in ['markdown', 'code']:
+            self.current_cell.source[-1] = self.current_cell.source[-1].rstrip('\n')
+            self._outputcells.append(self.current_cell)
+
+    def open_cell(self, line, execution_count):
+        if '<markdowncell>' in line:
+            self.current_cell = Cell({'cell_type': 'markdown', 'metadata': {}, 'source':[]})
+        else:
+            self.current_cell = Cell({'cell_type': 'code',
+                                      'execution_count': execution_count,
+                                      'metadata': {'collapsed': False},
+                                      'outputs': []})
+
+    def append_line_to_source(self, row):
+        if self.current_cell.type == 'markdown':
+            self.current_cell.source.append(row.lstrip("# "))
+        elif self.current_cell.type == 'code':
+            self.current_cell.source.append(row)
+
+    @staticmethod
+    def is_first_line_of_cell(line):
+        if line == '# <markdowncell>\n' or line == '# <codecell>\n':
+            return True
+        return False
+
+    def add_descriptive_data(self, lines):
+        self.metadata = self.create_metadata()
+        self.notebook_format = self.read_nb_format_from_py(lines)
+        self.nbformat_minor = 0
+
+    @staticmethod
+    def create_metadata():
+        kernelspec = {'display_name': 'Python 2',
+                      'language': 'python',
+                      'name': 'python2'}
+        language_info = {'codemirror_mode': {'name': 'ipython', 'version': 2},
+                         'file_extension': '.py',
+                         'mimetype': 'text/x-python',
+                         'name': 'python',
+                         'nbconvert_exporter': 'python',
+                         'pygments_lexer': 'ipython2',
+                         'version': '2.7.10'}
+        metadata = {'kernelspec': kernelspec,
+                    'language_info': language_info}
+        return metadata
+
+    @staticmethod
+    def read_nb_format_from_py(lines):
+        if '<nbformat>' in lines[1]:
+            nbformat = lines[1].split('>')[1].split('<')[0]
+            if "." in nbformat:
+                nbformat = float(nbformat)
+            else:
+                nbformat = int(nbformat)
+            return nbformat
+        else:
+            raise IOError("No or not suitable ( line[1]: "+lines[1]+") nbformat in supported lines")

notebook_v4_to_py.py → app/notebook_v4_to_py.py

@@ -45,75 +45,22 @@
    ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 """
-import sys
-import json
-import os
-import fnmatch
-
-reload(sys)
-sys.setdefaultencoding('utf8')
-THIS_FILE = os.path.abspath(__file__)
-
-def get_notebook_data(path_to_file):
-    with open(path_to_file, 'r') as notebook:
-        notebook_data = json.load(notebook)
-    return notebook_data
-
-def construct_output_py_file_path(input_file_path, skip_if_exists=True):
-    input_headless, ext = os.path.splitext(input_file_path)
-    assert ext=='.ipynb'
-    output_by_path = input_headless + '.py'
-    if os.path.exists(output_by_path) and skip_if_exists:
-        return
-    return output_by_path
-
-def write_notebook_data_to_py(notebook_data, out_file_path):
-    with open(out_file_path, 'w') as output:
-        output.write('# -*- coding: utf-8 -*-\n')
-        output.write('# <nbformat>' + str(notebook_data['nbformat']) + '</nbformat>\n')
-        try:
-            cells = notebook_data['cells']
-        except KeyError:
-            print "Nbformat is " + str(notebook_data['nbformat']) + ", try the old converter script."
-            return
-
-        for cell in cells:
-            if cell['cell_type'] in ['code', 'markdown']:
-                output.write('\n')
-                output.write('# <' + cell['cell_type'] + 'cell' + '>\n')
-                output.write('\n')
-                for item in cell['source']:
-                    if cell['cell_type']=='code':
-                        output.write(item)
-                    else:
-                        output.write('# ')
-                        output.write(item)
-                output.write('\n')
-
-def convert_notebook_to_py(input_file_path, skip_if_exists=True):
-    output_file_path = construct_output_py_file_path(input_file_path, skip_if_exists)
-    if output_file_path is not None:
-        notebook_data = get_notebook_data(input_file_path)
-        write_notebook_data_to_py(notebook_data, output_file_path)
-
-def convert_all_notebook_to_py(directory, skip_if_exists=True):
-    for root, dirnames, filenames in os.walk(directory):
-        for filename in fnmatch.filter(filenames, '*.ipynb'):
-            filename = os.path.abspath(os.path.join(root, filename))
-            print filename
-            if filename != THIS_FILE:
-                convert_notebook_to_py(filename, skip_if_exists)
+from app.Formatter import ToPy
+from app.NotebookConverter import NotebookConverter
+
 
 if __name__ == '__main__':
     import argparse
 
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('-w', '--overwrite', action='store_true', help='Overwrite existing py files', default=False)
     parser.add_argument('-f', '--file', help='Specify an Ipython notebook if you only want to convert one. '
-                                       '(This will overwrite default.)')
+                                             '(This will overwrite default.)')
     args = parser.parse_args()
+    nb2py = NotebookConverter()
 
+    overwrite=not args.overwrite
     if args.file is not None:
-        convert_notebook_to_py(args.file, skip_if_exists=not args.overwrite)
+        nb2py.convert(args.file, ToPy(args.file, overwrite))
     else:
-        convert_all_notebook_to_py(directory='.', skip_if_exists=not args.overwrite)
+        nb2py.convert_all('.', ToPy(overwrite))

app/py_to_notebook_v4.py

@@ -0,0 +1,58 @@
+"""
+   This script converts a .py file to Ipython v4 notebook format. The .py file
+   must be a result of an Ipython -> .py conversion using the notebook_v4_to_py.py
+   script or the automatic post-hook save in Ipyhon 3 based on that script.
+   In this way the version controlled .py files can be converted back to Ipython
+   notebook format.
+
+   Call this script with argument "-f" to create an .ipynb file from a .py file:
+
+   python py_to_notebook_v4.py -f filename.py
+
+   Call the script with argument "--overwrite" to overwrite existing .ipynb files.
+
+   Call the script with argument "--dry-run" to simulate (print) what would happen.
+
+   Date: 07. August 2015.
+   #############################################################################
+
+   This script is released under the MIT License
+
+   Copyright (c) 2015 Balabit SA
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy of
+   this software and associated documentation files (the "Software"), to deal in
+   the Software without restriction, including without limitation the rights to use,
+   copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+   Software, and to permit persons to whom the Software is furnished to do so,
+   subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in all
+   copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+   INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+   PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+   WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+from app.NotebookConverter import NotebookConverter
+from app.Formatter import ToNotebook
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('-w', '--overwrite', action='store_true', help='Overwrite existing py files', default=False)
+    parser.add_argument('-f', '--file', help='Specify an Ipython notebook if you only want to convert one. '
+                                             '(This will overwrite default.)')
+    parser.add_argument('--dry-run', action='store_true', help='Only prints what would happen', default=False)
+    args = parser.parse_args()
+    py2nb = NotebookConverter()
+    fm = ToNotebook()
+
+    if args.file is not None:
+        py2nb.convert(args.file, ToNotebook(overwrite=args.overwrite, dry_run=args.dry_run))
+    else:
+        py2nb.convert_all('.', ToNotebook(overwrite=args.overwrite, dry_run=args.dry_run))