cli: custom-doc: Add YAML based custom doc builder

Allows to aggregate custom documents with filtered content using human
friendly YAML files.

Use cases:
* Tutorials
* User guides
* Driver guides

Signed-off-by: Jorge Marques <[email protected]>

MANIFEST.in

@@ -1,6 +1,7 @@
 include adi_doctools/miscellaneous/*.js
 include adi_doctools/miscellaneous/*.svg
 include adi_doctools/miscellaneous/*.css
+include adi_doctools/theme/cosmic/style/*.css
 include adi_doctools/theme/cosmic/static/*.umd.js
 include adi_doctools/theme/cosmic/static/*.js.map
 include adi_doctools/theme/cosmic/static/*.min.css

adi_doctools/__init__.py

@@ -69,9 +69,7 @@ def config_inited(app, config):
             pass
         config.version = doc_version
     # Parameter to enable PDF output tweaks
-    media_print = getenv("ADOC_MEDIA_PRINT", default="0")
-    media_print = True if media_print == "1" else False
-    config.media_print = media_print
+    config.media_print = True if getenv("ADOC_MEDIA_PRINT") is not None else False
 
 def builder_inited(app):
     if app.builder.format == 'html':
@@ -82,7 +80,7 @@ def builder_inited(app):
         # Add bundled JavaScript if current theme is from this extension.
         if app.env.config.html_theme in theme_names:
             app.add_js_file("app.umd.js", priority=500, defer="")
-            app.config.html_permalinks_icon = "#"
+            app.config.html_permalinks_icon = ""
             get_pygments_theme(app)
         else:
             app.add_css_file("third-party.css", priority=500, defer="")

adi_doctools/cli/__init__.py

@@ -4,6 +4,7 @@
 from .hdl_render import hdl_render
 from .hdl_gen import hdl_gen
 from .aggregate import aggregate
+from .custom_doc import custom_doc
 
 
 @click.group()
@@ -18,7 +19,8 @@ def entry_point():
     author_mode,
     hdl_render,
     hdl_gen,
-    aggregate
+    aggregate,
+    custom_doc
 ]
 
 for cmd in commands:

adi_doctools/cli/aggregate.py

@@ -201,98 +201,6 @@ def gen_symbolic_doc(repo_dir):
     pr.wait(p)
 
 
-def gen_monolithic_doc(repo_dir):
-    def add_pyadi_iio_to_path():
-        """
-        To allow importing adi.<module> for autodoc.
-        """
-        name = os.path.join(repo_dir, 'pyadi-iio')
-        if 'PYTHONPATH' in os.environ:
-            os.environ['PYTHONPATH'] += os.pathsep + name
-        else:
-            os.environ['PYTHONPATH'] = name
-    add_pyadi_iio_to_path()
-
-    d_ = os.path.abspath(os.path.join(repo_dir, os.pardir))
-    docs_dir = os.path.join(d_, 'docs')
-    indexfile = os.path.join(docs_dir, 'index.rst')
-    if os.path.isdir(docs_dir):
-        pr.run(f"rm -r {docs_dir}")
-    pr.mkdir(docs_dir)
-
-    mk = {}
-    for r in repos:
-        cwd = os.path.join(repo_dir, f"{r}/{repos[r]['doc_folder']}")
-        mk[r] = get_sphinx_dirs(cwd)
-        if mk[r][0]:
-            continue
-        pr.mkdir(os.path.join(docs_dir, r))
-        cp_cmd = f"""\
-        for dir in */; do
-            dir="${{dir%/}}"
-            if [ "$dir" != "_build" ] && [ "$dir" != "extensions" ]; then
-                cp -r $dir {d_}/docs/{r}
-            fi
-        done
-        for file in *.rst; do
-            cp $file {d_}/docs/{r}
-        done\
-        """
-        cwd = mk[r][2]
-        pr.run(cp_cmd, cwd)
-
-        # Prefixes references with repo name, except already external
-        # references :ref:`repo:str`
-        cwd = f"{d_}/docs/{r}"
-        patch_cmd = """\
-        # Patch :ref:`str` into :ref:`{r} str`
-        find . -type f -exec sed -i -E \
-            "s/(:ref:\\`)([^<>:]+)(\\`)/\\1{r} \\2\\3/g" {{}} \\;
-        # Patch:ref:`Title <str>` into :ref:`Title <{r} str>`
-        find . -type f -exec sed -i -E \
-            "s/(:ref:\\`)([^<]+)( <)([^:>]+)(>)/\\1\\2\\3{r} \\4\\5/g" {{}} \\;
-        # Patch ^.. _str:$ into .. _{r} str:
-        find . -type f -exec sed -i -E \
-            "s/^(.. _)([^:]+)(:)\\$/\\1{r} \\2\\3/g" {{}} \\;\
-        """.format(r=r)
-        pr.run(patch_cmd, cwd)
-
-        # Patch includes outside the docs source,
-        # e.g. no-OS include README.rst
-        depth = '../' * (2 + repos[r]['doc_folder'].count('/'))
-        include_cmd = """
-        find . -type f -exec sed -i -E \
-        "s|^(.. include:: )({depth})(.*)|\\1../../../repos/{r}/\\3|g" {{}} \\;\
-        """.format(r=r, depth=depth)
-        pr.run(include_cmd, cwd)
-
-    # Convert documentation into top-level
-    cwd = f"{docs_dir}/documentation"
-    pr.run(f"mv {cwd}/* {docs_dir} ; rmdir {cwd}")
-    pr.run(f"cp -r {mk['documentation'][2]}/conf.py {docs_dir}")
-    pr.run(f"echo monolithic = True >> {docs_dir}/conf.py")
-
-    for r in repos:
-        if r != 'documentation':
-            patch_index(r, docs_dir, indexfile)
-
-    # Convert external references into local prefixed
-    cwd = docs_dir
-    for r in repos:
-        ref_cmd = """\
-        find . -type f -exec sed -i "s|ref:\\`{r}:|ref:\\`{r} |g" {{}} \\;\
-        """.format(r=r)
-        pr.run(ref_cmd, cwd)
-    ref_cmd = """\
-    find . -type f -exec sed -i "s|<|<|g" {} \\;
-    """
-    pr.run(ref_cmd, cwd)
-
-    pr.run("sphinx-build . _build", docs_dir)
-
-    cwd = d_
-
-
 @click.command()
 @click.option(
     '--directory',
@@ -303,13 +211,6 @@ def add_pyadi_iio_to_path():
     required=True,
     help="Path to create aggregated output."
 )
-@click.option(
-    '--monolithic',
-    '-m',
-    is_flag=True,
-    default=False,
-    help="Generate a single Sphinx build."
-)
 @click.option(
     '--extra',
     '-t',
@@ -341,21 +242,17 @@ def add_pyadi_iio_to_path():
     default=False,
     help="Open after generation (xdg-open)."
 )
-def aggregate(directory, monolithic, extra, no_parallel_, dry_run_, open_):
+def aggregate(directory, extra, no_parallel_, dry_run_, open_):
     """
-    Creates an aggregated documentation out of every repo documentation,
-    by deafult, generate independent Sphinx builds for each repo.
-    To resolve inter-repo-references in symbolic mode, run twice.
+    Creates a symbolic-aggregated documentation out of every repo
+    documentation.
+    To resolve interrepo-references, run the tool twice.
     """
     global dry_run, no_parallel
     no_parallel = no_parallel_
     dry_run = dry_run_
     directory = os.path.abspath(directory)
 
-    if monolithic:
-        click.echo("Currently, monolithic output is disabled")
-        return
-
     if not extra:
         click.echo("Extra features disabled, use --extra to enable.")
 
@@ -370,7 +267,7 @@ def aggregate(directory, monolithic, extra, no_parallel_, dry_run_, open_):
     for r in repos:
         cwd = os.path.join(repos_dir, r)
         if not os.path.isdir(cwd):
-            git_cmd = ["git", "clone", lut['remote'].format(r), "--depth=1", "-b",
+            git_cmd = ["git", "clone", lut['remote_ssh'].format(r), "--depth=1", "-b",
                        repos[r]['branch'], '--', cwd]
             pr.popen(git_cmd, p)
         else:
@@ -381,14 +278,9 @@ def aggregate(directory, monolithic, extra, no_parallel_, dry_run_, open_):
     if extra:
         do_extra_steps(repos_dir)
 
-    if monolithic:
-        gen_monolithic_doc(repos_dir)
-    else:
-        gen_symbolic_doc(repos_dir)
+    gen_symbolic_doc(repos_dir)
 
-    type_ = "monolithic" if monolithic else "symbolic"
-    out_ = "docs/_build" if monolithic else "html"
-    click.echo(f"Done, {type_} documentation written to {directory}/{out_}")
+    click.echo(f"Done, documentation written to {directory}/html")
 
     if open_ and not dry_run:
-        subprocess.call(f"xdg-open {directory}/{out_}/index.html", shell=True)
+        subprocess.call(f"xdg-open {directory}/html/index.html", shell=True)

adi_doctools/cli/author_mode.py

@@ -1,9 +1,13 @@
 from os import path, listdir, remove, mkdir
 from os import pardir, killpg, getpgid
+from os import environ
+from os import chdir, getcwd
 from shutil import copy
 import click
 import importlib
 
+from sphinx.application import Sphinx
+
 log = {
     'no_mk': "File Makefile not found, is {} a docs folder?",
     'inv_mk': "Failed parse Makefile, is {} a docs folder?",
@@ -109,6 +113,7 @@ def dir_assert(file, msg):
         click.echo(log['builder'].format(builder))
         return
     if builder == 'pdf':
+        environ["ADOC_MEDIA_PRINT"] = ""
         if not importlib.util.find_spec("weasyprint"):
             click.echo(log['no_weasyprint'])
             return
@@ -118,6 +123,7 @@ def dir_assert(file, msg):
 
     source_files = {'app.umd.js', 'app.umd.js.map', 'style.min.css',
                     'style.min.css.map'}
+    cwd_ = getcwd()
 
     def signal_handler(sig, frame):
         if builder == 'html':
@@ -212,6 +218,7 @@ def fetch_compiled(path_):
         click.echo(log['inv_mk'].format(directory))
         return
     builddir = path.join(directory, builddir_, builder)
+    doctreedir = path.join(builddir_, "doctrees")
     sourcedir = path.join(directory, sourcedir_)
     if dir_assert(sourcedir, log['inv_srcdir']):
         return
@@ -233,28 +240,34 @@ def generate_toctree(bookmarks, indent=0):
     if builder == 'singlehtml':
         singlehtml_file = path.join(builddir, 'index.html')
         font_config = FontConfiguration()
+        from .aux_print import sanitize_singlehtml
+
     def update_pdf():
-        html = HTML(filename=singlehtml_file)
-        document = html.render()
+        html_ = sanitize_singlehtml(singlehtml_file)
+
+        click.echo("preparing pdf styles...")
 
-        css = CSS(path.join(par_dir, cosmic_static, 'style.min.css'),
+        font_config = FontConfiguration()
+        src_dir = path.abspath(path.join(path.dirname(__file__), pardir, pardir))
+        cosmic = path.join('adi_doctools', 'theme', 'cosmic')
+        css = CSS(path.join(src_dir, cosmic, 'static', 'style.min.css'),
                   font_config=font_config)
-        contents_str = generate_toctree(document.make_bookmark_tree())
-        contents_str = f"<div class='pdf-toctree'>{contents_str}</div>"
-        contents_doc = HTML(string=contents_str)
-        contents_doc = contents_doc.render(stylesheets=[css],
-                                           font_config=font_config)
-        for page in reversed(contents_doc.pages):
-            document.pages.insert(1, page)
+        css_extra = CSS(path.join(src_dir, cosmic, 'style', 'weasyprint.css'),
+                        font_config=font_config)
 
-        document.write_pdf(path.join(builddir, '..', 'output.pdf'))
-        click.echo("The PDF is at _build/output.pdf")
+        click.echo("rendering pdf content...")
+        html = HTML(string=html_, base_url=path.dirname(singlehtml_file))
 
+        document = html.render(stylesheets=[css, css_extra])
+
+        click.echo("writing pdf...")
+        document.write_pdf(path.join(builddir, '..', 'output.pdf'))
 
     if not with_selenium and builder == 'html':
-        devpool_js = "ADOC_DEVPOOL= "
-    else:
-        devpool_js = ""
+        environ["ADOC_DEVPOOL"] = ""
+
+    app = Sphinx(directory, directory,  builddir, doctreedir, builder)
+
     watch_file_src = {}
     watch_file_rst = {}
     if dev:
@@ -276,7 +289,7 @@ def update_pdf():
                 return
 
         # Build doc the first time
-        subprocess.call(f"{devpool_js} make {builder}", shell=True, cwd=directory)
+        app.build()
         for f, s in zip(w_files, source_files):
             watch_file_src[f] = path.getctime(f)
         if not once:
@@ -286,7 +299,7 @@ def update_pdf():
                                         stdout=subprocess.DEVNULL)
     else:
         # Build doc the first time
-        subprocess.call(f"{devpool_js} make {builder}", shell=True, cwd=directory)
+        app.build()
         if builder == "singlehtml":
             update_pdf()
 
@@ -388,8 +401,7 @@ def check_files(scheduler):
             update_sphinx = False
 
         if update_sphinx:
-            subprocess.call(f"{devpool_js} make {builder}",
-                            shell=True, cwd=directory)
+            app.build()
         if update_page:
             for f, s in zip(w_files, source_files):
                 copy(f, path.join(builddir, '_static', s))

adi_doctools/cli/aux_print.py

@@ -0,0 +1,69 @@
+from packaging.version import Version
+
+from sphinx.__init__ import __version__ as sphinx_version
+from os.path import basename
+from lxml import html, etree
+
+def sanitize_singlehtml(file) -> str:
+    """
+    Alter the singlehtml etree to:
+    * Remove the embed style
+    * Extract custom doc name
+    * Covert toctree caption into "volumes" (H1 header)
+    """
+
+    root = html.parse(file).getroot()
+
+    # Remove full CSS entry to use slimer version
+    link_elements = root.xpath("//head//link[contains(@href, '_static/style.min.css')]")
+    for link in link_elements:
+        link.getparent().remove(link)
+
+    # Obtain toctree caption to use as volume titles
+    toc_tree = root.xpath("//body//div[@class='toc-tree']")[0]
+    cap_ = toc_tree.xpath("./p[@class='caption' and @role='heading']//span[@class='caption-text']")
+    volumes = []
+    for c in cap_:
+        ul_ = c.getparent().getnext()
+        i = ul_.xpath('./li//a')[0]
+        volumes.append([c.text, i.attrib['href'][1:]])
+
+    # Extract title
+    title = root.xpath("//head/title")[0].text
+
+    bwrap = root.xpath("//div[@class='bodywrapper']")[0]
+
+    # Remove first H1
+    h1_ = bwrap.xpath(".//h1")[0]
+    h1_.getparent().remove(h1_)
+
+    # Find indexes and add columes
+    for c, i in volumes:
+        e_ = bwrap.xpath(f".//span[@id='{i}']")[0]
+        ele_ = etree.Element("div")
+        ele_.attrib['class'] = "volume"
+        ele = etree.Element("h1")
+        ele.text = c
+        ele_.insert(0, ele)
+        e_p = e_.getparent()
+        e_p.insert(e_p.index(e_), ele_)
+
+    # Prior to Sphinx v8.1.0, commit 0bfaadf6c9
+    # internal references from other .rst files aggregated on the singlehtml
+    # does not have a same page anchor references:
+    #    href="#my-label"
+    # instead, have a anchored link
+    #    href="index.html#my-label"
+    # So, for < v8.1.0, patch href="my-index.html#*" -> href="#*"
+    if Version(sphinx_version) < Version("8.1.0"):
+        filename = basename(file)
+        len_fname = len(filename)
+
+        a_ = root.xpath("//a[@class='reference internal']")
+        for a__ in a_:
+            if a__.attrib['href'].startswith(filename):
+                a__.attrib['href'] = a__.attrib['href'][len_fname:]
+
+    return html.tostring(root, encoding="utf-8", method="html")
+
+