Enhance unit tests requirements handling

.github/workflows/test.yaml

@@ -73,6 +73,7 @@ jobs:
       MKL_NUM_THREADS: 1
       PYTHONHASHSEED: 0
       NUTILS_TENSORIAL: ${{ matrix.tensorial }}
+      NUTILS_TESTING_REQUIRES: "mod:matplotlib mod:meshio mod:PIL"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -89,7 +90,9 @@ jobs:
           path: dist/
       - name: Install Graphviz
         if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: sudo apt install -y graphviz
+        run: |
+          sudo apt install -y graphviz
+          echo "NUTILS_TESTING_REQUIRES=$NUTILS_TESTING_REQUIRES app:dot" >> $GITHUB_ENV
       - name: Install Nutils and dependencies
         id: install
         env:
@@ -98,15 +101,17 @@ jobs:
           python -um pip install --upgrade --upgrade-strategy eager wheel
           python -um pip install --upgrade --upgrade-strategy eager numpy$_numpy_version
           # Install Nutils from `dist` dir created in job `build-python-package`.
-          python -um pip install "$_wheel[import_gmsh,export_mpl]"
+          python -um pip install "$_wheel[import-gmsh,export-mpl]"
       - name: Install Scipy
         if: ${{ matrix.matrix-backend == 'scipy' }}
-        run: python -um pip install --upgrade scipy
+        run: |
+          python -um pip install --upgrade scipy
+          echo "NUTILS_TESTING_REQUIRES=$NUTILS_TESTING_REQUIRES mod:scipy" >> $GITHUB_ENV
       - name: Configure MKL
         if: ${{ matrix.matrix-backend == 'mkl' }}
         run: |
           python -um pip install --upgrade --upgrade-strategy eager mkl
-          python -um devtools.gha.configure_mkl
+          echo "NUTILS_TESTING_REQUIRES=$NUTILS_TESTING_REQUIRES lib:mkl_rt" >> $GITHUB_ENV
       - name: Test
         env:
           COVERAGE_ID: ${{ matrix.name }}
@@ -204,6 +209,7 @@ jobs:
       # Fixes https://github.com/actions/virtual-environments/issues/3080
       STORAGE_OPTS: overlay.mount_program=/usr/bin/fuse-overlayfs
       _wheel: ${{ needs.build-python-package.outputs.wheel }}
+      NUTILS_TESTING_REQUIRES: "mod:matplotlib mod:meshio mod:PIL lib:mkl_rt"
     steps:
       - name: Checkout
         uses: actions/checkout@v4

devtools/_log_default.py

@@ -38,9 +38,9 @@ def _log_msg(*msg, color: Optional[str] = None, title: Optional[str] = None, fil
 
 
 debug = functools.partial(_log_msg)
-notice = functools.partial(_log_msg, type='1;36')
+notice = functools.partial(_log_msg, color='1;36')
 warning = functools.partial(_log_msg, color='1;33')
-error = functools.partial(_log_msg, type='1;31')
+error = functools.partial(_log_msg, color='1;31')
 
 
 def info(*args: Any) -> None:

devtools/container/build.py

@@ -68,7 +68,7 @@
 
     container = stack.enter_context(Container.new_from(base, mounts=[Mount(src=wheel, dst=f'/{wheel.name}')]))
 
-    container.run('pip', 'install', '--break-system-packages', '--no-cache-dir', f'/{wheel.name}[export_mpl,import_gmsh,matrix_scipy]', env=dict(PYTHONHASHSEED='0'))
+    container.run('pip', 'install', '--break-system-packages', '--no-cache-dir', f'/{wheel.name}[export-mpl,import-gmsh,matrix-scipy]', env=dict(PYTHONHASHSEED='0'))
     container.add_label('org.opencontainers.image.url', 'https://github.com/evalf/nutils')
     container.add_label('org.opencontainers.image.source', 'https://github.com/evalf/nutils')
     container.add_label('org.opencontainers.image.authors', 'Evalf')

nutils/_util.py

@@ -18,6 +18,8 @@
 import contextlib
 import treelog
 import datetime
+import site
+import re
 from typing import Iterable, Sequence, Tuple
 
 supports_outdirfd = os.open in os.supports_dir_fd and os.listdir in os.supports_fd
@@ -172,26 +174,50 @@
         return retvals
 
 
-def loadlib(**libname):
-    '''
-    Find and load a dynamic library using :any:`ctypes.CDLL`.  For each
-    (supported) platform the name of the library should be specified as a keyword
-    argument, including the extension, where the keywords should match the
-    possible values of :any:`sys.platform`.
+def loadlib(name):
+    '''Find and load a dynamic library.
+
+    This routine will try to load the requested library from any of the
+    platform default locations. Only the unversioned name is queried, assuming
+    that this is an alias to the most recent version. If this fails then
+    site-package directories are searched for both versioned and unversioned
+    files. The library is returned upon first succes or ``None`` otherwise.
 
     Example
     -------
 
-    To load the Intel MKL runtime library, write::
+    To load the Intel MKL runtime library, write simply::
 
-        loadlib(linux='libmkl_rt.so', darwin='libmkl_rt.dylib', win32='mkl_rt.dll')
+        libmkl = loadlib('mkl_rt')
     '''
 
+    if sys.platform == 'linux':
+        libsubdir = 'lib'
+        libname = f'lib{name}.so'
+        versioned = f'lib{name}.so.(\d+)'
+    elif sys.platform == 'darwin':
+        libsubdir = 'lib'
+        libname = f'lib{name}.dylib'
+        versioned = f'lib{name}.(\d+).dylib'
+    elif sys.platform == 'win32':
+        libsubdir = r'Library\bin'
+        libname = f'{name}.dll'
+        versioned = f'{name}.(\d+).dll'
+    else:
+        return
+
     try:
-        return ctypes.CDLL(libname[sys.platform])
-    except (OSError, KeyError):
+        return ctypes.CDLL(libname)
+    except:
         pass
 
+    for prefix in dict.fromkeys(site.PREFIXES): # stable deduplication
+        if os.path.isdir(libdir := os.path.join(prefix, libsubdir)):
+            if os.path.isfile(path := os.path.join(libdir, libname)):
+                return ctypes.CDLL(path)
+            if match := max(map(re.compile(versioned).fullmatch, os.listdir(libdir)), key=lambda m: int(m.group(1)) if m else -1, default=None):
+                return ctypes.CDLL(os.path.join(libdir, match.group(0)))
+
 
 def readtext(path):
     '''Read file and return contents

nutils/matrix/_mkl.py

@@ -6,16 +6,10 @@
 import os
 import numpy
 
-libmkl_path = os.environ.get('NUTILS_MATRIX_MKL_LIB', None)
-if libmkl_path:
-    libmkl = CDLL(libmkl_path)
-else:
-    for v in '.2', '.1', '':
-        libmkl = util.loadlib(linux=f'libmkl_rt.so{v}', darwin=f'libmkl_rt{v}.dylib', win32=f'mkl_rt{v}.dll')
-        if libmkl:
-            break
-    else:
-        raise BackendNotAvailable('the Intel MKL matrix backend requires libmkl to be installed (try: pip install mkl)')
+
+libmkl = util.loadlib('mkl_rt')
+if libmkl is None:
+    raise BackendNotAvailable('the Intel MKL matrix backend requires libmkl to be installed (try: pip install mkl)')
 
 
 def assemble(data, rowptr, colidx, ncols):

nutils/testing.py

@@ -17,7 +17,9 @@
 import warnings as _builtin_warnings
 import logging
 import numpy
-from nutils import warnings, numeric
+import shutil
+import os
+from nutils import warnings, numeric, _util as util
 
 
 class PrintHandler(logging.Handler):
@@ -27,21 +29,35 @@
         print(record.msg)
 
 
-def _not_has_module(module):
-    try:
-        importlib.import_module(module)
-    except ImportError:
-        return True
-    else:
-        return False
+def _require(category, test, *items):
+    missing = [item for item in items if not test(item)]
+    if missing:
+        for item in os.getenv(f'NUTILS_TESTING_REQUIRES', '').split():
+            prefix, name = item.split(':')
+            if category.startswith(prefix) and name in missing:
+                raise RuntimeError(f'{category} {required!r} is unexpectedly missing')
+        if len(missing) > 1:
+            category += 's'
+        missing = ', '.join(missing)
+        raise unittest.SkipTest(f'missing {category}: {missing}')
 
 
-def requires(*modules):
-    missing = tuple(filter(_not_has_module, modules))
-    if missing:
-        return unittest.skip('missing module{}: {}'.format('s' if len(missing) > 1 else '', ','.join(missing)))
+def _test_decorator(test, *args):
+    try:
+        test(*args)
+    except unittest.SkipTest as e:
+        wrapper = unittest.skip(e)
+    except Exception as outer_exc:
+        inner_exc = outer_exc
+        def wrapper(f):
+            @functools.wraps(f)
+            def wrapped(self):
+                raise inner_exc
+            return wrapped
     else:
-        return lambda func: func
+        def wrapper(f):
+            return f
+    return wrapper
 
 
 class _ParametrizedCollection(type):
@@ -247,6 +263,16 @@
         status.extend(s[i:i+80] for i in range(0, len(s), 80))
         self.fail('\n'.join(status))
 
+    require_module = functools.partial(_require, 'module', importlib.util.find_spec)
+    require_application = functools.partial(_require, 'application', shutil.which)
+    require_library = functools.partial(_require, 'library', util.loadlib)
+
+
+# decorators
+requires = functools.partial(_test_decorator, TestCase.require_module)
+requires_application = functools.partial(_test_decorator, TestCase.require_application)
+requires_library = functools.partial(_test_decorator, TestCase.require_library)
+
 
 ContextTestCase = TestCase
 

pyproject.toml

@@ -21,10 +21,10 @@ classifiers = [
 
 [project.optional-dependencies]
 docs = ["Sphinx >=1.8,<8"]
-export_mpl = ["matplotlib >=3.3,<4"]
-matrix_mkl = ["mkl"]
-matrix_scipy = ["scipy >=0.13,<2"]
-import_gmsh = ["meshio >=4,<6"]
+export-mpl = ["matplotlib >=3.3,<4"]
+matrix-mkl = ["mkl"]
+matrix-scipy = ["scipy >=0.13,<2"]
+import-gmsh = ["meshio >=4,<6"]
 
 [build-system]
 requires = ["flit_core >=3.2,<4"]

tests/test_docs.py

@@ -29,9 +29,7 @@ def setUp(self):
             if blank and line[indent:].startswith('.. requires:: '):
                 requires.extend(name.strip() for name in line[indent+13:].split(','))
             blank = not line.strip()
-        missing = tuple(filter(nutils.testing._not_has_module, requires))
-        if missing:
-            self.skipTest('missing module{}: {}'.format('s' if len(missing) > 1 else '', ','.join(missing)))
+        self.require_module(*requires)
 
         if 'matplotlib' in requires:
             import matplotlib.testing

tests/test_graph.py

@@ -1,4 +1,4 @@
-from nutils.testing import TestCase
+from nutils.testing import TestCase, requires_application
 from nutils import _graph
 import itertools
 import unittest
@@ -334,8 +334,6 @@ def test_multiple_graphviz_source(self):
                          '1 -> 0;'
                          '}')
 
+    @requires_application('dot')
     def test_export_graphviz(self):
-        try:
-            self.multiple.export_graphviz()
-        except FileNotFoundError:
-            self.skipTest('graphviz not available')
+        self.multiple.export_graphviz()

tests/test_matrix.py

@@ -92,10 +92,11 @@ class backend(testing.TestCase):
 
     def setUp(self):
         super().setUp()
-        try:
-            self.enter_context(matrix.backend(self.backend))
-        except matrix.BackendNotAvailable:
-            self.skipTest('backend is unavailable')
+        if self.backend == 'scipy':
+            self.require_module('scipy')
+        elif self.backend == 'mkl':
+            self.require_library('mkl_rt')
+        self.enter_context(matrix.backend(self.backend))
         self.offdiag = -1+.5j if self.complex else -1
         self.exact = 2 * numpy.eye(self.n) + self.offdiag * numpy.eye(self.n, self.n, 1) + self.offdiag * numpy.eye(self.n, self.n, -1)
         data = sparse.prune(sparse.fromarray(self.exact), inplace=True)