Add building wheels using cibuildwheel

.circleci/config.yml

@@ -0,0 +1,30 @@
+version: 2
+
+jobs:
+  linux-aarch64-wheels:
+    working_directory: ~/linux-aarch64-wheels
+    machine:
+      image: ubuntu-2004:2022.04.1
+    # resource_class is what tells CircleCI to use an ARM worker for native arm builds
+    # https://circleci.com/product/features/resource-classes/
+    resource_class: arm.medium
+    steps:
+      - checkout
+      - run:
+          name: Build the Linux aarch64 wheels.
+          environment:
+            CIBW_SKIP: "pp* *-musllinux_aarch64"
+            CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh
+            CIBW_ENVIRONMENT_LINUX: >
+              PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH
+          command: |
+            python3 -m pip install --user cibuildwheel==2.16.5
+            python3 -m cibuildwheel --output-dir wheelhouse
+      - store_artifacts:
+          path: wheelhouse/
+
+workflows:
+  version: 2
+  all-tests:
+    jobs:
+      - linux-aarch64-wheels

.github/workflows/coverage.yml

@@ -46,7 +46,7 @@ jobs:
         working-directory: ${{runner.workspace}}/nmodl
         run: |
           pip3 install -U pip setuptools
-          pip3 install --user -r requirements.txt
+          pip3 install --user -r <(packaging/show_deps.py -b -r .)
       - name: Restore compiler cache
         uses: actions/cache@v4
         with:

.github/workflows/nmodl-ci.yml

@@ -83,7 +83,7 @@ jobs:
         working-directory: ${{runner.workspace}}/nmodl
         run: |
           python3 -m pip install -U pip setuptools
-          python3 -m pip install --user -r requirements.txt
+          python3 -m pip install --user -r <(packaging/show_deps.py -b -r .)
 
       - name: Install neuron-nightly
         if: ${{matrix.config.enable_usecases == 'On'}}

.github/workflows/nmodl-doc.yml

@@ -59,7 +59,7 @@ jobs:
         working-directory: ${{runner.workspace}}/nmodl
         run: |
           pip3 install -U pip setuptools
-          pip3 install --user -r requirements.txt
+          pip3 install --user -r <(packaging/show_deps.py -r -b .)
 
       # This step will set up an SSH connection on tmate.io for live debugging.
       # To trigger it, simply add 'live-debug-docs' to your last pushed commit message.

.github/workflows/nmodl-wheel.yml

@@ -0,0 +1,63 @@
+name: Build wheels
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest]
+
+    steps:
+      - name: Setup Flex and Bison
+        if: runner.os == 'Windows'
+        run: |
+          choco install winflexbison3 -y
+
+      - name: Setup Flex and Bison
+        if: runner.os == 'macOS'
+        run: |
+          brew install flex bison
+
+      # for multi-arch builds
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+
+      - uses: actions/checkout@v4
+
+      # Used to host cibuildwheel
+      - uses: actions/setup-python@v3
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.16.5
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        # to supply options, put them in 'env', like:
+        env:
+          # we skip PyPy, 32-bit builds, and musllinux builds
+          CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-musllinux_i686 *-musllinux_x86_64"
+          # MacOS-specific setup
+          CIBW_ENVIRONMENT_MACOS: >
+            PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH"
+            MACOSX_DEPLOYMENT_TARGET='10.15'
+          CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build"
+          # Linux-specific setup
+          CIBW_ENVIRONMENT_LINUX: >
+            PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH
+          # TODO: replace this with a custom Docker image to speed-up builds
+          CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl

.github/workflows/sonarsource.yml

@@ -32,7 +32,7 @@ jobs:
         working-directory: ${{runner.workspace}}/nmodl
         run: |
           pip3 install -U pip setuptools
-          pip3 install --user -r requirements.txt
+          pip3 install --user -r <(packaging/show_deps.py -r -b .)
       - name: Install sonar-scanner and build-wrapper
         uses: SonarSource/sonarcloud-github-c-cpp@v2
       - name: Configure project

CMakeLists.txt

@@ -25,6 +25,12 @@ set(NMODL_EXTRA_CXX_FLAGS
     ""
     CACHE STRING "Add extra compile flags for NMODL sources")
 separate_arguments(NMODL_EXTRA_CXX_FLAGS)
+option(LINK_AGAINST_PYTHON "Should the Python library be linked or not" ON)
+option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF)
+if(NMODL_BUILD_WHEEL)
+  set(LINK_AGAINST_PYTHON OFF)
+  set(NMODL_ENABLE_TESTS OFF)
+endif()
 
 # =============================================================================
 # Settings to enable project as submodule
@@ -163,9 +169,12 @@ find_package(PythonInterp 3.8 REQUIRED)
 cpp_cc_strip_python_shims(EXECUTABLE "${PYTHON_EXECUTABLE}" OUTPUT PYTHON_EXECUTABLE)
 include(cmake/hpc-coding-conventions/cpp/cmake/bbp-find-python-module.cmake)
 cpp_cc_find_python_module(jinja2 2.9.3 REQUIRED)
-cpp_cc_find_python_module(pytest 3.3.0 REQUIRED)
-cpp_cc_find_python_module(sympy 1.3 REQUIRED)
-cpp_cc_find_python_module(textwrap 0.9 REQUIRED)
+if(NMODL_ENABLE_TESTS)
+  cpp_cc_find_python_module(pytest 3.3.0 REQUIRED)
+endif()
+if(NOT NMODL_BUILD_WHEEL)
+  cpp_cc_find_python_module(sympy 1.3 REQUIRED)
+endif()
 cpp_cc_find_python_module(yaml 3.12 REQUIRED)
 
 # =============================================================================

azure-pipelines.yml

@@ -88,7 +88,7 @@ stages:
         sudo apt-get install -y python3.8 python3.8-dev python3.8-venv ninja-build
         sudo apt-get remove -y python3-importlib-metadata
         python3.8 -m pip install --upgrade pip setuptools
-        python3.8 -m pip install --user -r $(Build.Repository.LocalPath)/requirements.txt
+        python3.8 -m pip install --user -r <($(Build.Repository.LocalPath)/packaging/show_deps.py -r -b $(Build.Repository.LocalPath))
         # we manually get version 3.15.0 to make sure that changes in the cmake
         # files do not require unsupported versions of cmake in our package.
         wget --quiet --output-document=- "https://github.com/Kitware/CMake/releases/download/$CMAKE_VER/$CMAKE_PKG.tar.gz" | tar xzpf -
@@ -149,7 +149,7 @@ stages:
     - script: |
         brew install flex bison cmake python@3
         python3 -m pip install --upgrade pip setuptools
-        python3 -m pip install --user -r $(Build.Repository.LocalPath)/requirements.txt
+        python3 -m pip install --user -r <($(Build.Repository.LocalPath)/packaging/show_deps.py -r -b $(Build.Repository.LocalPath))
       displayName: 'Install Dependencies'
     - script: |
         export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH;
@@ -222,6 +222,10 @@ stages:
         else
           export TAG=""
         fi
+        # the following 2 lines are a workaround for PEP 621 not allowing a
+        # dynamic `name` in `pyproject.toml`
+        python3 -m pip install --user tomli tomli-w
+        python3 packaging/change_name.py pyproject.toml "NMODL${TAG}"
         docker run --rm \
           -w /root/nmodl \
           -v $PWD:/root/nmodl \
@@ -295,6 +299,10 @@ stages:
         else
           export NMODL_NIGHTLY_TAG=""
         fi
+        # the following 2 lines are a workaround for PEP 621 not allowing a
+        # dynamic `name` in `pyproject.toml`
+        python3 -m pip install tomli tomli-w
+        python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}"
         packaging/build_wheels.bash osx $(python.version)
       condition: succeeded()
       displayName: 'Build macos Wheel'

nmodl/__init__.py

@@ -1,7 +1,15 @@
 try:
     # Try importing but catch exception in case bindings are not available
     from ._nmodl import NmodlDriver, to_json, to_nmodl  # noqa
-    from ._nmodl import __version__
+
+    from importlib.metadata import version, PackageNotFoundError
+
+    try:
+        __version__ = version("nmodl")
+    except PackageNotFoundError:
+        # package is not installed
+        pass
+
     __all__ = ["NmodlDriver", "to_json", "to_nmodl"]
 except ImportError:
     print("[NMODL] [warning] :: Python bindings are not available")

pywheel/shim/_binwrapper.py → nmodl/_binwrapper.py

@@ -11,7 +11,7 @@
 from find_libpython import find_libpython
 
 
-def _config_exe(exe_name):
+def main():
     """Sets the environment to run the real executable (returned)"""
 
     package_name = "nmodl"
@@ -37,12 +37,7 @@ def _config_exe(exe_name):
     # set PYTHONPATH for embedded python to properly find the nmodl module
     os.environ["PYTHONPATH"] = working_set.by_key[package_name].location + ':' + os.environ.get("PYTHONPATH", "")
 
-    return os.path.join(NMODL_BIN, exe_name)
-
-
-if __name__ == "__main__":
-    """Set the pointed file as executable"""
-    exe = _config_exe(os.path.basename(sys.argv[0]))
+    exe = os.path.join(NMODL_BIN, os.path.basename(sys.argv[0]))
     st = os.stat(exe)
     os.chmod(exe, st.st_mode | stat.S_IEXEC)
     os.execv(exe, sys.argv)

packaging/build_wheels.bash

@@ -12,8 +12,8 @@ set -xe
 #  - python (>=3.8)
 #  - C/C++ compiler
 
-if [ ! -f setup.py ]; then
-    echo "Error: setup.py not found. Please launch $0 from the nmodl root dir"
+if ! [ -f setup.py ] && ! [ -f pyproject.toml ]; then
+    echo "Error: setup.py or pyproject.toml not found. Please launch $0 from the nmodl root dir"
     exit 1
 fi
 
@@ -47,14 +47,13 @@ build_wheel_linux() {
     (( $skip )) && return 0
 
     echo " - Installing build requirements"
-    pip install pip auditwheel setuptools
-    pip install -r packaging/build_requirements.txt
+    pip install pip auditwheel setuptools build
 
     echo " - Building..."
     rm -rf dist _skbuild
     # Workaround for https://github.com/pypa/manylinux/issues/1309
     git config --global --add safe.directory "*"
-    python setup.py bdist_wheel
+    python -m build --wheel -o dist/
 
     echo " - Repairing..."
     auditwheel repair dist/*.whl
@@ -70,11 +69,13 @@ build_wheel_osx() {
     (( $skip )) && return 0
 
     echo " - Installing build requirements"
-    pip install --upgrade delocate -r packaging/build_requirements.txt
+    pip install --upgrade delocate build
 
     echo " - Building..."
     rm -rf dist _skbuild
-    python setup.py bdist_wheel
+    # the custom `build-dir` is a workaround for this issue:
+    # https://gitlab.kitware.com/cmake/cmake/-/issues/20107
+    python -m build --wheel -o dist/ -C build-dir=_build
 
     echo " - Repairing..."
     delocate-wheel -w wheelhouse -v dist/*.whl  # we started clean, there's a single wheel

packaging/change_name.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""
+Barebones utility for changing the name of the package in `pyproject.toml`.
+
+Usage
+-----
+[SCRIPT] [PYPROJECT_FILE] [NAME]
+
+Notes
+-----
+Should only be used by the CI
+"""
+import re
+from argparse import ArgumentParser
+from pathlib import Path
+
+import tomli
+import tomli_w
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("pyproject_file", help="The path to the pyproject.toml file")
+    parser.add_argument("name", help="The new name of the project")
+    args = parser.parse_args()
+
+    if not Path(args.pyproject_file).exists():
+        raise FileNotFoundError(f"File {args.pyproject_file} not found")
+
+    with open(args.pyproject_file, "rb") as f:
+        toml_dict = tomli.load(f)
+
+    # check name is conforms to the naming convention
+    # see:
+    # https://packaging.python.org/en/latest/specifications/name-normalization/
+
+    if not re.match(
+        r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$",
+        args.name,
+        flags=re.IGNORECASE,
+    ):
+        raise ValueError(
+            f"The package name {args.name} does not conform to the standard Python package naming convention"
+        )
+
+    toml_dict["project"]["name"] = args.name
+
+    with open(args.pyproject_file, "wb") as f:
+        tomli_w.dump(toml_dict, f)
+
+
+if __name__ == "__main__":
+    main()

packaging/install_dependencies.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# TODO put all of this inside of a Docker image to save time
+
+set -xe
+
+yum -y install \
+    git \
+    wget \
+    make \
+    vim \
+    curl \
+    unzip \
+    autoconf \
+    automake \
+    make \
+    openssh-server \
+    libtool \
+    && yum -y clean all \
+    && rm -rf /var/cache
+
+curl -L -o flex-2.6.4.tar.gz https://github.com/westes/flex/files/981163/flex-2.6.4.tar.gz \
+    && tar -xvzf flex-2.6.4.tar.gz \
+    && cd flex-2.6.4 \
+    && ./configure --prefix=/nmodlwheel/flex \
+    && make -j 3 install \
+    && cd .. \
+    && rm -rf flex-2.6.4.tar.gz flex-2.6.4
+curl -L -o bison-3.7.3.tar.gz https://ftp.gnu.org/gnu/bison/bison-3.7.3.tar.gz \
+    && tar -xvzf bison-3.7.3.tar.gz \
+    && cd bison-3.7.3 \
+    && ./configure --prefix=/nmodlwheel/bison \
+    && make -j 3 install \
+    && cd .. \
+    && rm -rf bison-3.7.3.tar.gz bison-3.7.3
+
+set +xe