From 0e1d8b71685f48cc7003886b328f9642e257b1ab Mon Sep 17 00:00:00 2001 From: Jakub Kaczmarzyk Date: Tue, 27 Dec 2022 23:13:25 -0500 Subject: [PATCH] ADD: sphinx docs (#53) * make patchlib private: _patchlib * update code to use private patchlib * add sphinx docs + dependencies * move Weights validation to separate validate staticmethod * use map_location="cpu" when loading state dict from URL * fix pip install + add model inference example * add requirements.txt file for docs --- docs/Makefile | 20 +++++++++ docs/conf.py | 39 ++++++++++++++++++ docs/index.rst | 22 ++++++++++ docs/make.bat | 35 ++++++++++++++++ docs/requirements.txt | 2 + docs/user_guide.rst | 41 +++++++++++++++++++ setup.cfg | 4 ++ wsinfer/{patchlib => _patchlib}/README.md | 0 wsinfer/{patchlib => _patchlib}/__init__.py | 0 .../create_dense_patch_grid.py | 0 .../create_patches_fp.py | 0 .../{patchlib => _patchlib}/presets/tcga.csv | 0 .../{patchlib => _patchlib}/utils/__init__.py | 0 .../utils/file_utils.py | 0 .../{patchlib => _patchlib}/utils/utils.py | 0 .../wsi_core/WholeSlideImage.py | 0 .../wsi_core/__init__.py | 0 .../wsi_core/batch_process_utils.py | 0 .../wsi_core/util_classes.py | 0 .../wsi_core/wsi_utils.py | 0 wsinfer/cli/infer.py | 4 +- wsinfer/cli/patch.py | 2 +- wsinfer/modellib/models.py | 26 ++++++++---- 23 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/requirements.txt create mode 100644 docs/user_guide.rst rename wsinfer/{patchlib => _patchlib}/README.md (100%) rename wsinfer/{patchlib => _patchlib}/__init__.py (100%) rename wsinfer/{patchlib => _patchlib}/create_dense_patch_grid.py (100%) rename wsinfer/{patchlib => _patchlib}/create_patches_fp.py (100%) rename wsinfer/{patchlib => _patchlib}/presets/tcga.csv (100%) rename wsinfer/{patchlib => _patchlib}/utils/__init__.py (100%) rename wsinfer/{patchlib => _patchlib}/utils/file_utils.py (100%) rename wsinfer/{patchlib => _patchlib}/utils/utils.py (100%) rename wsinfer/{patchlib => _patchlib}/wsi_core/WholeSlideImage.py (100%) rename wsinfer/{patchlib => _patchlib}/wsi_core/__init__.py (100%) rename wsinfer/{patchlib => _patchlib}/wsi_core/batch_process_utils.py (100%) rename wsinfer/{patchlib => _patchlib}/wsi_core/util_classes.py (100%) rename wsinfer/{patchlib => _patchlib}/wsi_core/wsi_utils.py (100%) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..8987b2b --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,39 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "wsinfer" +copyright = "2022, Jakub Kaczmarzyk" +author = "Jakub Kaczmarzyk" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "autoapi.extension", +] + +autoapi_type = "python" +autoapi_dirs = ["../wsinfer"] +autoapi_options = [ + "members", + "undoc-members", + # "private-members", + "show-inheritance", + "show-module-summary", + "special-members", + "imported-members", +] +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "pydata_sphinx_theme" +html_static_path = ["_static"] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..1a7e19c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,22 @@ +Welcome to WSInfer! +=================== + +**WSInfer** is a program to run patch-based classification inference on whole +slide images. + +Check out the :doc:`user_guide` for further information, including how to install the +project. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + user_guide + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..69b055f --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +pydata-sphinx-theme +sphinx-autoapi diff --git a/docs/user_guide.rst b/docs/user_guide.rst new file mode 100644 index 0000000..e74d638 --- /dev/null +++ b/docs/user_guide.rst @@ -0,0 +1,41 @@ +User Guide +========== + +.. _installation: + +Installation +------------ + +To use WSInfer, first install it using pip: + +.. code-block:: console + + pip install wsinfer --find-links https://girder.github.io/large_image_wheels + +Examples +------- + +List available models +^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: console + + wsinfer list + +Run model inference +^^^^^^^^^^^^^^^^^^^ + +.. code-block:: console + + mkdir -p example-wsi-inference/sample-images + cd example-wsi-inference/sample-images + # Download a sample slide. + wget -nc https://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1.svs + cd .. + # Run inference on the slide. + CUDA_VISIBLE_DEVICES=0 wsinfer run \ + --wsi-dir sample-images/ \ + --results-dir results/ \ + --model resnet34 \ + --weights TCGA-BRCA-v1 \ + --num-workers 8 diff --git a/setup.cfg b/setup.cfg index 87c78e2..1bf0836 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,10 @@ dev = types-Pillow types-PyYAML types-tqdm +docs = + pydata-sphinx-theme + sphinx + sphinx-autoapi [options.entry_points] console_scripts = diff --git a/wsinfer/patchlib/README.md b/wsinfer/_patchlib/README.md similarity index 100% rename from wsinfer/patchlib/README.md rename to wsinfer/_patchlib/README.md diff --git a/wsinfer/patchlib/__init__.py b/wsinfer/_patchlib/__init__.py similarity index 100% rename from wsinfer/patchlib/__init__.py rename to wsinfer/_patchlib/__init__.py diff --git a/wsinfer/patchlib/create_dense_patch_grid.py b/wsinfer/_patchlib/create_dense_patch_grid.py similarity index 100% rename from wsinfer/patchlib/create_dense_patch_grid.py rename to wsinfer/_patchlib/create_dense_patch_grid.py diff --git a/wsinfer/patchlib/create_patches_fp.py b/wsinfer/_patchlib/create_patches_fp.py similarity index 100% rename from wsinfer/patchlib/create_patches_fp.py rename to wsinfer/_patchlib/create_patches_fp.py diff --git a/wsinfer/patchlib/presets/tcga.csv b/wsinfer/_patchlib/presets/tcga.csv similarity index 100% rename from wsinfer/patchlib/presets/tcga.csv rename to wsinfer/_patchlib/presets/tcga.csv diff --git a/wsinfer/patchlib/utils/__init__.py b/wsinfer/_patchlib/utils/__init__.py similarity index 100% rename from wsinfer/patchlib/utils/__init__.py rename to wsinfer/_patchlib/utils/__init__.py diff --git a/wsinfer/patchlib/utils/file_utils.py b/wsinfer/_patchlib/utils/file_utils.py similarity index 100% rename from wsinfer/patchlib/utils/file_utils.py rename to wsinfer/_patchlib/utils/file_utils.py diff --git a/wsinfer/patchlib/utils/utils.py b/wsinfer/_patchlib/utils/utils.py similarity index 100% rename from wsinfer/patchlib/utils/utils.py rename to wsinfer/_patchlib/utils/utils.py diff --git a/wsinfer/patchlib/wsi_core/WholeSlideImage.py b/wsinfer/_patchlib/wsi_core/WholeSlideImage.py similarity index 100% rename from wsinfer/patchlib/wsi_core/WholeSlideImage.py rename to wsinfer/_patchlib/wsi_core/WholeSlideImage.py diff --git a/wsinfer/patchlib/wsi_core/__init__.py b/wsinfer/_patchlib/wsi_core/__init__.py similarity index 100% rename from wsinfer/patchlib/wsi_core/__init__.py rename to wsinfer/_patchlib/wsi_core/__init__.py diff --git a/wsinfer/patchlib/wsi_core/batch_process_utils.py b/wsinfer/_patchlib/wsi_core/batch_process_utils.py similarity index 100% rename from wsinfer/patchlib/wsi_core/batch_process_utils.py rename to wsinfer/_patchlib/wsi_core/batch_process_utils.py diff --git a/wsinfer/patchlib/wsi_core/util_classes.py b/wsinfer/_patchlib/wsi_core/util_classes.py similarity index 100% rename from wsinfer/patchlib/wsi_core/util_classes.py rename to wsinfer/_patchlib/wsi_core/util_classes.py diff --git a/wsinfer/patchlib/wsi_core/wsi_utils.py b/wsinfer/_patchlib/wsi_core/wsi_utils.py similarity index 100% rename from wsinfer/patchlib/wsi_core/wsi_utils.py rename to wsinfer/_patchlib/wsi_core/wsi_utils.py diff --git a/wsinfer/cli/infer.py b/wsinfer/cli/infer.py index 4bd320a..e53a19d 100644 --- a/wsinfer/cli/infer.py +++ b/wsinfer/cli/infer.py @@ -13,8 +13,8 @@ from ..modellib.run_inference import run_inference from ..modellib import models -from ..patchlib.create_dense_patch_grid import create_grid_and_save_multi_slides -from ..patchlib.create_patches_fp import create_patches +from .._patchlib.create_dense_patch_grid import create_grid_and_save_multi_slides +from .._patchlib.create_patches_fp import create_patches PathType = typing.Union[str, Path] diff --git a/wsinfer/cli/patch.py b/wsinfer/cli/patch.py index ca1b734..6f61229 100644 --- a/wsinfer/cli/patch.py +++ b/wsinfer/cli/patch.py @@ -2,7 +2,7 @@ import click -from ..patchlib.create_patches_fp import create_patches as _create_patches +from .._patchlib.create_patches_fp import create_patches as _create_patches @click.command() diff --git a/wsinfer/modellib/models.py b/wsinfer/modellib/models.py index bd6a65f..da44d25 100644 --- a/wsinfer/modellib/models.py +++ b/wsinfer/modellib/models.py @@ -79,13 +79,12 @@ def __post_init__(self): if len(self.class_names) != self.num_classes: raise ValueError("length of class_names must be equal to num_classes") - @classmethod - def from_yaml(cls, path): - with open(path) as f: - d = yaml.safe_load(f) + @staticmethod + def _validate_input(d) -> None: + """Raise error if invalid input.""" if not isinstance(d, dict): - raise ValueError("expected YAML config to be a dictionary") + raise ValueError("expected config to be a dictionary") # Validate contents. # Validate keys. @@ -171,6 +170,14 @@ def from_yaml(cls, path): if not file.exists(): raise FileNotFoundError(f"'file' not found: {file}") + @classmethod + def from_yaml(cls, path): + """Create a new instance of Weights from a YAML file.""" + + with open(path) as f: + d = yaml.safe_load(f) + cls._validate_input(d) + transform = PatchClassification( resize_size=d["transform"]["resize_size"], mean=d["transform"]["mean"], @@ -189,13 +196,17 @@ def from_yaml(cls, path): class_names=d["class_names"], ) - def load_model(self): + def load_model(self) -> torch.nn.Module: + """Return the pytorch implementation of the architecture with weights loaded.""" model = _create_model(name=self.architecture, num_classes=self.num_classes) # Load state dict. if self.url and self.url_file_name: state_dict = load_state_dict_from_url( - url=self.url, check_hash=True, file_name=self.url_file_name + url=self.url, + map_location="cpu", + check_hash=True, + file_name=self.url_file_name, ) elif self.file: state_dict = torch.load(self.file, map_location="cpu") @@ -209,6 +220,7 @@ def load_model(self): return model def get_sha256_of_weights(self) -> str: + """Return the sha256 of the weights file.""" if self.url and self.url_file_name: p = Path(torch.hub.get_dir()) / "checkpoints" / self.url_file_name elif self.file: