diff --git a/README.md b/README.md new file mode 100644 index 0000000..a6db07f --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +A command-line utility for pulling code blocks out of markdown files. Currently only handles python code blocks. + +```sh +$ pip install -e git+https://github.com/ryneeverett/mkcodes.git#egg=mkcodes + +# For traditional markdown code-block support. +$ pip install markdown + +$ mkcodes --help +Usage: mkcodes [OPTIONS] INPUTS... + +Options: + --output TEXT + --github / --markdown Github-flavored fence blocks or pure markdown. + --safe / --unsafe Only use code blocks with language hints. + --help Show this message and exit. +``` + +Why would I want such a thing? +------------------------------ + +My purpose is testing. + +You can easily enough doctest a markdown file with `python -m doctest myfile.md`, but I don't like typing or looking at a whole bunch of `>>>` and `...`'s. Also there's no way that I know of to run linters against such code blocks. + +Instead, I include (pytest) functional tests in my codeblocks, extract the code blocks with this script, and then run my test runner and linters against the output files. diff --git a/mkcodes.py b/mkcodes.py new file mode 100644 index 0000000..d0d4d6e --- /dev/null +++ b/mkcodes.py @@ -0,0 +1,106 @@ +import os +import re +import glob +import warnings + +import click + +try: + import markdown as markdown_enabled +except ImportError: + markdown_enabled = False +else: + from markdown.extensions import Extension + from markdown.treeprocessors import Treeprocessor + + +def default_state(): + return [], True, False + + +def github_codeblocks(filename, safe): + codeblocks = [] + codeblock_re = r'^```.*' + codeblock_open_re = r'^```(py|python){0}$'.format('' if safe else '?') + + with open(filename, 'r') as f: + block = [] + python = True + in_codeblock = False + + for line in f.readlines(): + codeblock_delimiter = re.match(codeblock_re, line) + + if in_codeblock: + if codeblock_delimiter: + if python: + codeblocks.append(''.join(block)) + block = [] + python = True + in_codeblock = False + else: + block.append(line) + elif codeblock_delimiter: + in_codeblock = True + if not re.match(codeblock_open_re, line): + python = False + return codeblocks + + +def markdown_codeblocks(filename, safe): + import markdown + + codeblocks = [] + + if safe: + warnings.warn("'safe' option not available in 'markdown' mode.") + + class DoctestCollector(Treeprocessor): + def run(self, root): + nonlocal codeblocks + codeblocks = (block.text for block in root.iterfind('./pre/code')) + + class DoctestExtension(Extension): + def extendMarkdown(self, md, md_globals): + md.registerExtension(self) + md.treeprocessors.add("doctest", DoctestCollector(md), '_end') + + doctestextension = DoctestExtension() + markdowner = markdown.Markdown(extensions=[doctestextension]) + markdowner.convertFile(filename, output=os.devnull) + return codeblocks + + +def is_markdown(f): + markdown_extensions = ['.markdown', '.mdown', '.mkdn', '.mkd', '.md'] + return os.path.splitext(f)[1] in markdown_extensions + + +def get_files(inputs): + for i in inputs: + if os.path.isdir(i): + yield from filter( + is_markdown, glob.iglob(i + '/**', recursive=True)) + elif is_markdown(i): + yield i + + +@click.command() +@click.argument( + 'inputs', nargs=-1, required=True, type=click.Path(exists=True)) +@click.option('--output', default='{name}.py') +@click.option('--github/--markdown', default=bool(not markdown_enabled), + help='Github-flavored fence blocks or pure markdown.') +@click.option('--safe/--unsafe', default=True, + help='Allow code blocks without language hints.') +def main(inputs, output, github, safe): + collect_codeblocks = github_codeblocks if github else markdown_codeblocks + + for filename in get_files(inputs): + code = '\n\n'.join(collect_codeblocks(filename, safe)) + + inputname = os.path.splitext(os.path.basename(filename))[0] + outputfilename = output.format(name=inputname) + + with open(outputfilename, 'w') as outputfile: + outputfile.write(code) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4239338 --- /dev/null +++ b/setup.py @@ -0,0 +1,9 @@ +import setuptools + +setuptools.setup( + name='mkcodes', + install_requires=['click'], + extras_require={'markdown': ['markdown']}, + packages=setuptools.find_packages(), + entry_points={'console_scripts': ['mkcodes=mkcodes:main']} +) diff --git a/test b/test new file mode 100755 index 0000000..fb77b91 --- /dev/null +++ b/test @@ -0,0 +1 @@ +python -m unittest discover tests diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..0918c97 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +markdown diff --git a/tests/data/other.md b/tests/data/other.md new file mode 100644 index 0000000..367f09a --- /dev/null +++ b/tests/data/other.md @@ -0,0 +1,3 @@ +```py +qux = 4 +``` diff --git a/tests/data/some.md b/tests/data/some.md new file mode 100644 index 0000000..17e7417 --- /dev/null +++ b/tests/data/some.md @@ -0,0 +1,15 @@ +Some github-flavored code: + +``` +foo = True +``` + +Some strict github-flavored code: + +```py +bar = False +``` + +Some daring-fireball-flavored code: + + baz = None diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..e93027d --- /dev/null +++ b/tests/test.py @@ -0,0 +1,94 @@ +import os +import textwrap +import unittest +import subprocess + + +class TestBase(unittest.TestCase): + output = 'tests/output.py' + + def tearDown(self): + self.remove(self.output) + + @staticmethod + def remove(f): + try: + os.remove(f) + except FileNotFoundError: + pass + + @classmethod + def call(cls, *flags, inputfile='tests/data/some.md'): + subprocess.call([ + 'mkcodes', '--output', cls.output, *flags, inputfile]) + + def assertFileEqual(self, filename, expected): + with open(filename, 'r') as output: + self.assertEqual(output.read(), textwrap.dedent(expected)) + + def assertOutput(self, expected): + self.assertFileEqual(self.output, expected) + + +class TestMarkdown(TestBase): + @unittest.skip + def test_markdown_safe(self): + raise NotImplementedError + + def test_github_safe(self): + self.call('--github', '--safe') + self.assertOutput("""\ + bar = False + """) + + def test_markdown_unsafe(self): + self.call('--markdown', '--unsafe') + self.assertOutput("""\ + baz = None + """) + + def test_github_unsafe(self): + self.call('--github', '--unsafe') + self.assertOutput("""\ + foo = True + + + bar = False + """) + + +class TestInputs(TestBase): + @classmethod + def call(cls, **kwargs): + super().call('--github', **kwargs) + + def test_file(self): + self.call() + self.assertTrue(os.path.exists(self.output)) + + def test_directory(self): + self.call(inputfile='tests/data') + self.assertTrue(os.path.exists(self.output)) + + def test_multiple(self): + try: + subprocess.call([ + 'mkcodes', '--output', 'tests/{name}.py', '--github', + 'tests/data/some.md', 'tests/data/other.md']) + self.assertFileEqual('tests/some.py', """\ + bar = False + """) + self.assertFileEqual('tests/other.py', """\ + qux = 4 + """) + finally: + self.remove('tests/some.py') + self.remove('tests/other.py') + + @unittest.skip + def test_glob(self): + raise NotImplementedError + + +if __name__ == '__main__': + unittest.main()