🆕 Add Reading Slide-level info & Simple Plugins for Visualization Tool

docs/visualization.rst

@@ -218,6 +218,10 @@ Additional features can be added to nodes by adding extra keys to the dictionary
 
 It will be possible to color the nodes by these features in the interface, and the top 10 will appear in a tooltip when hovering over a node (you will have to turn on the hovertool in the small toolbar to the right of the main window to enable this, it is disabled by default.)
 
+Slide Level Information
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have slide-level predictions, ground truth labels, or other metadata you wish to be able to see associated with slides in the interface, this can be provided as a .csv formatted table placed in the slides folder, with "Image File" as the first column. The other columns can be anything you like. When loading a slide in the UI, if the slide name appears in the "Image File" column of the provided .csv, any other entries in that row will be displayed in the interface below the main view window when the slide is selected.
 
 .. _examples:
 
@@ -422,7 +426,7 @@ and the ability to toggle on or off specific UI elements:
 
 ::
 
-    "UI_elements_1": {              # controls which UI elements are visible
+    "ui_elements_1": {              # controls which UI elements are visible
         "slide_select": 1,          # slide select box
         "layer_drop": 1,            # overlay select drop down
         "slide_row": 1,             # slide alpha toggle and slider
@@ -437,7 +441,7 @@ and the ability to toggle on or off specific UI elements:
 
 ::
 
-    "UI_elements_2": {              # controls visible UI elements on second tab in UI
+    "ui_elements_2": {              # controls visible UI elements on second tab in UI
         "opt_buttons": 1,           # UI elements providing a few options including if annotations should be filled/outline only
         "pt_size_spinner": 1,       # control for point size and graph node size
         "edge_size_spinner": 1,     # control for edge thickness

tests/conftest.py

@@ -608,6 +608,7 @@ def data_path(tmp_path_factory: pytest.TempPathFactory) -> dict[str, object]:
     """Set up a temporary data directory for testing visualization UI."""
     tmp_path = tmp_path_factory.mktemp("data")
     (tmp_path / "slides").mkdir()
+    (tmp_path / "slides" / "CMU-1_files").mkdir()
     (tmp_path / "overlays").mkdir()
     return {"base_path": tmp_path}
 

tests/test_app_bokeh.py

@@ -18,7 +18,7 @@
 import requests
 from bokeh.application import Application
 from bokeh.application.handlers import FunctionHandler
-from bokeh.events import ButtonClick, DoubleTap, MenuItemClick
+from bokeh.events import ButtonClick, DoubleTap
 from flask_cors import CORS
 from matplotlib import colormaps
 from PIL import Image
@@ -101,6 +101,10 @@ def annotation_path(data_path: dict[str, Path]) -> dict[str, object]:
         "patch-extraction-vf",
         data_path["base_path"] / "slides",
     )
+    data_path["meta"] = _fetch_remote_sample(
+        "test_meta",
+        data_path["base_path"] / "slides",
+    )
     data_path["annotations"] = _fetch_remote_sample(
         "annotation_store_svs_1",
         data_path["base_path"] / "overlays",
@@ -129,6 +133,14 @@ def annotation_path(data_path: dict[str, Path]) -> dict[str, object]:
         "config_2",
         data_path["base_path"] / "overlays",
     )
+    data_path["plugin_img"] = _fetch_remote_sample(
+        "stainnorm-source",
+        data_path["base_path"] / "slides" / "CMU-1_files",
+    )
+    data_path["plugin_csv"] = _fetch_remote_sample(
+        "test_csv",
+        data_path["base_path"] / "slides" / "CMU-1_files",
+    )
     return data_path
 
 
@@ -181,8 +193,8 @@ def test_get_level_by_extent() -> None:
 
 def test_roots(doc: Document) -> None:
     """Test that the document has the correct number of roots."""
-    # should be 4 roots: main window, controls, slide_info, popup table
-    assert len(doc.roots) == 4
+    # should be 5 roots: main window, controls, slide_info, popup, extra_layout
+    assert len(doc.roots) == 5
 
 
 def test_config_loaded(data_path: pytest.TempPathFactory) -> None:
@@ -210,6 +222,11 @@ def test_slide_select(doc: Document, data_path: pytest.TempPathFactory) -> None:
     slide_select.value = ["CMU-1.ndpi"]
     assert main.UI["vstate"].slide_path == data_path["slide2"]
 
+    # check the slide metadata is loaded from csv
+    desc = doc.get_model_by_name("description")
+    assert "valA" in desc.text
+    assert "valB" not in desc.text
+
     # check selecting nothing has no effect
     slide_select.value = []
     assert main.UI["vstate"].slide_path == data_path["slide2"]
@@ -253,8 +270,7 @@ def test_add_annotation_layer(doc: Document, data_path: pytest.TempPathFactory)
     slide_select.value = [data_path["slide2"].name]
     layer_drop = doc.get_model_by_name("layer_drop0")
     # trigger an event to select the geojson file
-    click = MenuItemClick(layer_drop, str(data_path["geojson_anns"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["geojson_anns"])
     assert main.UI["vstate"].types == ["annotation"]
 
     # test the name2type function.
@@ -263,11 +279,10 @@ def test_add_annotation_layer(doc: Document, data_path: pytest.TempPathFactory)
     # test loading an annotation store
     slide_select.value = [data_path["slide1"].name]
     layer_drop = doc.get_model_by_name("layer_drop0")
-    assert len(layer_drop.menu) == 5
+    assert len(layer_drop.options) == 5
     n_renderers = len(doc.get_model_by_name("slide_windows").children[0].renderers)
     # trigger an event to select the annotation .db file
-    click = MenuItemClick(layer_drop, str(data_path["annotations"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["annotations"])
     # should be one more renderer now
     assert len(doc.get_model_by_name("slide_windows").children[0].renderers) == (
         n_renderers + 1
@@ -360,8 +375,7 @@ def test_load_graph(doc: Document, data_path: pytest.TempPathFactory) -> None:
     """Test loading a graph."""
     layer_drop = doc.get_model_by_name("layer_drop0")
     # trigger an event to select the graph file
-    click = MenuItemClick(layer_drop, str(data_path["graph"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["graph"])
     # we should have 2144 nodes in the node_source now
     assert len(main.UI["node_source"].data["x_"]) == 2144
 
@@ -370,8 +384,7 @@ def test_graph_with_feats(doc: Document, data_path: pytest.TempPathFactory) -> N
     """Test loading a graph with features."""
     layer_drop = doc.get_model_by_name("layer_drop0")
     # trigger an event to select the graph .json file
-    click = MenuItemClick(layer_drop, str(data_path["graph_feats"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["graph_feats"])
     # we should have keys for the features in node data source now
     for i in range(10):
         assert f"feat_{i}" in main.UI["node_source"].data
@@ -393,17 +406,15 @@ def test_graph_with_feats(doc: Document, data_path: pytest.TempPathFactory) -> N
     )
 
     # test graph overlay option remains on loading new overlay
-    click = MenuItemClick(layer_drop, str(data_path["annotations"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["annotations"])
     assert "graph_overlay" in cmap_select.options
 
 
 def test_load_img_overlay(doc: Document, data_path: pytest.TempPathFactory) -> None:
     """Test loading an image overlay."""
     layer_drop = doc.get_model_by_name("layer_drop0")
     # trigger an event to select the image overlay
-    click = MenuItemClick(layer_drop, str(data_path["img_overlay"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["img_overlay"])
     layer_slider = doc.get_model_by_name("layer2_slider")
     assert layer_slider is not None
 
@@ -465,8 +476,7 @@ def test_hovernet_on_box(doc: Document, data_path: pytest.TempPathFactory) -> No
     cprop_select = doc.get_model_by_name("cprop0")
     cprop_select.value = ["prob"]
     layer_drop = doc.get_model_by_name("layer_drop0")
-    click = MenuItemClick(layer_drop, str(data_path["dat_anns"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["dat_anns"])
     assert main.UI["vstate"].types == ["annotation"]
     # check the per-type ui controls have been updated
     assert len(main.UI["color_column"].children) == 1
@@ -507,8 +517,7 @@ def test_type_select(doc: Document, data_path: pytest.TempPathFactory) -> None:
     """Test selecting/deselecting specific types."""
     # load annotation layer
     layer_drop = doc.get_model_by_name("layer_drop0")
-    click = MenuItemClick(layer_drop, str(data_path["annotations"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["annotations"])
     time.sleep(1)
     im = get_tile("overlay", 4, 8, 4, show=False)
     _, num_before = label(np.any(im[:, :, :3], axis=2))
@@ -560,8 +569,7 @@ def test_node_and_edge_alpha(doc: Document, data_path: pytest.TempPathFactory) -
     """Test sliders for adjusting graph node and edge alpha."""
     layer_drop = doc.get_model_by_name("layer_drop0")
     # trigger an event to select the graph .db file
-    click = MenuItemClick(layer_drop, str(data_path["graph"]))
-    layer_drop._trigger_event(click)
+    layer_drop.value = str(data_path["graph"])
 
     type_column_list = doc.get_model_by_name("type_column0").children
     color_column_list = doc.get_model_by_name("color_column0").children

tests/test_json_config_bokeh.py

@@ -7,6 +7,7 @@
 from threading import Thread
 from typing import TYPE_CHECKING
 
+import pandas as pd
 import pytest
 import requests
 from bokeh.client.session import ClientSession, pull_session
@@ -29,6 +30,15 @@ def annotation_path(data_path: dict[str, Path]) -> dict[str, Path]:
         "ndpi-1",
         data_path["base_path"] / "slides",
     )
+    data_path["meta"] = _fetch_remote_sample(
+        "test_meta",
+        data_path["base_path"] / "slides",
+    )
+    meta_df = pd.read_csv(data_path["meta"])
+    # change 'Image File' column name to 'Wrong Name'
+    meta_df = meta_df.rename(columns={"Image File": "Wrong Name"})
+    # save so we can test behaviour if required column isn't there
+    meta_df.to_csv(data_path["meta"], index=False)
     data_path["annotations"] = _fetch_remote_sample(
         "annotation_store_svs_1",
         data_path["base_path"] / "overlays",
@@ -77,10 +87,14 @@ def test_slides_available(bk_session: ClientSession) -> None:
     assert slide_select.value[0] == "CMU-1-Small-Region.svs"
 
     layer_drop = doc.get_model_by_name("layer_drop0")
-    assert len(layer_drop.menu) == 2
+    assert len(layer_drop.options) == 2
     # check that the overlays are available.
     slide_select.value = ["CMU-1.ndpi"]
-    assert len(layer_drop.menu) == 2
+    assert len(layer_drop.options) == 2
+
+    # check the metadata wasnt found as the column name was wrong
+    desc = doc.get_model_by_name("description")
+    assert "Metadata:" not in desc.text
 
     bk_session.document.clear()
     assert len(bk_session.document.roots) == 0

tests/test_server_bokeh.py

@@ -85,7 +85,7 @@ def test_slides_available(bk_session: ClientSession) -> None:
     # check that the overlays are available.
     slide_select.value = ["CMU-1-Small-region.svs"]
     layer_drop = doc.get_model_by_name("layer_drop0")
-    assert len(layer_drop.menu) == 2
+    assert len(layer_drop.options) == 2
 
     bk_session.document.clear()
     assert len(bk_session.document.roots) == 0

tiatoolbox/cli/visualize.py

@@ -77,6 +77,14 @@ def run_bokeh(img_input: list[str], port: int, *, noshow: bool) -> None:
     This option must be used in conjunction with --slides.
     The --base-path option should not be used in this case.""",
 )
+@click.option(
+    "--plugin",
+    multiple=True,
+    help=r"""Path to a file to define an extra layout containing extra
+    resources such as graphs below each slide. Some pre-built plugins are
+    available in the tiatoolbox\visualization\templates folder. Can pass
+    multiple instances of this option to add multiple ui additions.""",
+)
 @click.option(
     "--port",
     type=int,
@@ -88,6 +96,7 @@ def visualize(
     base_path: str,
     slides: str,
     overlays: str,
+    plugin: list[str],
     port: int,
     *,
     noshow: bool,
@@ -101,6 +110,7 @@ def visualize(
         base_path (str): Path to base directory containing images to be displayed.
         slides (str): Path to directory containing slides to be displayed.
         overlays (str): Path to directory containing overlays to be displayed.
+        plugin (list): Paths to files containing ui plugins.
         port (int): Port to launch the visualization tool on.
         noshow (bool): Do not launch in browser (mainly intended for testing).
 
@@ -109,7 +119,7 @@ def visualize(
     if base_path is None and (slides is None or overlays is None):
         msg = "Must specify either base-path or both slides and overlays."
         raise ValueError(msg)
-    img_input = [base_path, slides, overlays]
+    img_input = [base_path, slides, overlays, *list(plugin)]
     img_input = [p for p in img_input if p is not None]
     # check that the input paths exist
     for input_path in img_input:

tiatoolbox/data/remote_samples.yaml

@@ -143,6 +143,10 @@ files:
     url: [ *testdata, "annotation/test1_config.json"]
   config_2:
     url: [ *testdata, "annotation/test2_config.json"]
+  test_meta:
+    url: [ *testdata, "annotation/test_meta.csv"]
+  test_csv:
+    url: [ *testdata, "annotation/metrics_10.csv"]
   patch_annotations:
     url: [ *testdata, "annotation/sample_wsi_patch_preds.db"]
   nuclick-output: