From acc71ecfe3ca397827db21e1cf64a7dff1a501b2 Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 26 Jan 2024 13:01:48 +0100 Subject: [PATCH 01/11] Update versions of github actions Current ones are deprecated now. --- .github/workflows/analyzers.yaml | 8 ++++---- .github/workflows/bigendian.yaml | 8 ++++---- .github/workflows/core.yaml | 2 +- .github/workflows/docs.yaml | 4 ++-- .github/workflows/wheels.yaml | 8 ++++---- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/analyzers.yaml b/.github/workflows/analyzers.yaml index 8f1957d86..d4aa92ee6 100644 --- a/.github/workflows/analyzers.yaml +++ b/.github/workflows/analyzers.yaml @@ -11,7 +11,7 @@ jobs: cppcheck: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: './.github/actions/build_core_dependencies' with: @@ -41,7 +41,7 @@ jobs: scan_build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: './.github/actions/build_core_dependencies' with: @@ -69,7 +69,7 @@ jobs: valgrind: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: './.github/actions/build_core_dependencies' with: @@ -96,7 +96,7 @@ jobs: debug: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: './.github/actions/build_core_dependencies' with: diff --git a/.github/workflows/bigendian.yaml b/.github/workflows/bigendian.yaml index 7a4206861..3b8e83c11 100644 --- a/.github/workflows/bigendian.yaml +++ b/.github/workflows/bigendian.yaml @@ -14,17 +14,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: builder: ${{ steps.buildx.outputs.name }} context: . diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 2ac9aa300..24485b859 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -27,7 +27,7 @@ jobs: cmake_generator: "-G \"Visual Studio 16 2019\" -A Win32" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Elevate privileges if: ${{ !contains( matrix.os, 'windows') }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b97bba824..aa8418e0d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 30ad81efa..e5d05f274 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -40,14 +40,14 @@ jobs: run: | git config --global core.autocrlf false - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Used to host cibuildwheel - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-qemu-action@v3 if: ${{ matrix.arch == 'aarch64' }} name: Set up QEMU @@ -83,7 +83,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/download-artifact@v3 From c0f1caa254cdcd83f00049930d0b5e576ed3a45b Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 26 Jul 2024 19:22:26 +0200 Subject: [PATCH 02/11] Migrate to v4 in download/upload artifact From v4 artifacts must have unique names or we get "Conflict: an artifact with this name already exists on the workflow run". --- .github/workflows/wheels.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index e5d05f274..a5a380b4b 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -73,8 +73,9 @@ jobs: run: | python -m cibuildwheel --output-dir wheelhouse python/ - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: wheel-${{ matrix.os }}-${{ matrix.arch }} path: ./wheelhouse/*.whl publish: @@ -85,7 +86,11 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 + with: + pattern: wheel-* + merge-multiple: true + path: ./wheelhouse/ - name: Publish wheels to PyPI From b209476a3a2d14ecefdf919c95bcb389215ddf89 Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 12 Jul 2024 14:22:55 +0200 Subject: [PATCH 03/11] Drop support for python 3.7 It's been more than a year since Python 3.7 EOL. --- .github/workflows/wheels.yaml | 4 ++-- README.md | 3 +-- python/dlisio/__init__.py | 20 +++++--------------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index a5a380b4b..1fe677307 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -66,9 +66,9 @@ jobs: CIBW_ARCHS: ${{ matrix.arch }} # musllinux arch skip: unlikely someone wants to run dlisio on alpine on non-standard architecture - # musllinux cp37 and cp38 skip: latest available numpy doesn't provide musslinux wheels, so it is unlikely useful + # musllinux cp38 skip: latest available numpy doesn't provide musslinux wheels, so it is unlikely useful # macosx 38 skip: cibuildwheel can't test it - CIBW_SKIP: pp* *-musllinux_i686 *-musllinux_aarch64 cp37*-musllinux* cp38*-musllinux* cp38*-macosx_*:arm64 cp36-* + CIBW_SKIP: pp* *-musllinux_i686 *-musllinux_aarch64 cp38*-musllinux* cp38*-macosx_*:arm64 cp36-* cp37-* run: | python -m cibuildwheel --output-dir wheelhouse python/ diff --git a/README.md b/README.md index ef7ed28ad..30bba91de 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,6 @@ pip install dlisio | | macOS Intel | macOS ARM | Windows 64bit | Windows 32bit | manylinux x86_64 | manylinux aarch64 | manylinux i686 | musllinux x86_64 |---------------|----|-----|-----|----|----|----|----|----| -| CPython 3.7 | ✅ | - | ✅ | ✅ | ✅ | ✅ | ✅ | - | | CPython 3.8 | ✅ | - | ✅ | ✅ | ✅ | ✅ | ✅ | - | | CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | @@ -97,7 +96,7 @@ To develop dlisio, or to build a particular revision from source, you need: * A C++11 compatible compiler (tested on gcc, clang, and msvc 2019) * [CMake](https://cmake.org/) version 3.5 or greater -* [Python](https://python.org) version 3.7 or greater +* [Python](https://python.org) version 3.8 or greater * [fmtlib](http://fmtlib.net/) tested mainly with 7.1.3 * [mpark_variant](https://github.com/mpark/variant) * [pybind11](https://github.com/pybind/pybind11) version 2.6 or greater diff --git a/python/dlisio/__init__.py b/python/dlisio/__init__.py index 3b02b06d4..31a820cfe 100644 --- a/python/dlisio/__init__.py +++ b/python/dlisio/__init__.py @@ -3,18 +3,8 @@ from . import lis from . import dlis -import sys - -# remove else once support for python 3.7 is over -if sys.version_info >= (3, 8): - try: - import importlib.metadata - __version__ = importlib.metadata.version(__name__) - except importlib.metadata.PackageNotFoundError: - pass -else: - try: - import pkg_resources - __version__ = pkg_resources.get_distribution(__name__).version - except pkg_resources.DistributionNotFound: - pass +try: + import importlib.metadata + __version__ = importlib.metadata.version(__name__) +except importlib.metadata.PackageNotFoundError: + pass From 8642698db4bf272c7228d0ad2a85a0a5d23ecb31 Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 12 Jul 2024 14:30:27 +0200 Subject: [PATCH 04/11] Unpin numpy version Looks like we are not affected by changes in numpy. --- python/requirements-dev.txt | 2 +- python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index 7582d753c..e1404f5da 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -1,6 +1,6 @@ bandit -numpy<2.0 +numpy setuptools setuptools_scm pytest diff --git a/python/setup.py b/python/setup.py index 4bbc82283..bfe63589f 100755 --- a/python/setup.py +++ b/python/setup.py @@ -36,7 +36,7 @@ def get_long_description(): packages = ['dlisio', 'dlisio.dlis', 'dlisio.lis', 'dlisio.common', 'dlisio.dlis.utils'], license = 'LGPL-3.0', platforms = 'any', - install_requires = ['numpy < 2.0'], + install_requires = ['numpy'], setup_requires = ['setuptools >= 28', 'pybind11 >= 2.3', 'setuptools_scm', From cea3efebedc93a2ed1c6a40191e7f30e5ce63bdf Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 12 Jul 2024 15:22:13 +0200 Subject: [PATCH 05/11] Update macos github runners macOS 11 support has been dropped. Updating to macos-13 as latest standard image that is Intel MacOS. Changing macos-13-xlarge to macos-14 as it is arm [1] and doesn't require xlarge runner. [1] https://github.com/actions/runner-images?tab=readme-ov-file#available-images --- .github/workflows/core.yaml | 2 +- .github/workflows/wheels.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 24485b859..13205df72 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - os: [macos-11] + os: [macos-13, macos-14] include: - os: ubuntu-20.04 diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 1fe677307..829a25d5a 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -30,9 +30,9 @@ jobs: arch: aarch64 - os: ubuntu-20.04 arch: i686 - - os: macos-11 + - os: macos-13 arch: x86_64 - - os: macos-13-xlarge + - os: macos-14 arch: arm64 steps: From fed74d3ca0e97fe750881414a8cbe2660192ddfa Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 12 Jul 2024 17:00:37 +0200 Subject: [PATCH 06/11] Add note about i686 --- .github/workflows/wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 829a25d5a..58d706cb1 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -29,7 +29,7 @@ jobs: - os: ubuntu-20.04 arch: aarch64 - os: ubuntu-20.04 - arch: i686 + arch: i686 #builds from cibuildwheel, qemu not needed - os: macos-13 arch: x86_64 - os: macos-14 From 05bfef50b72beaa6a78dd65b76783200038f5649 Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 12 Jul 2024 17:01:30 +0200 Subject: [PATCH 07/11] Remove reference to setuptools.command.test.test Originally skbuild supplied its own test command which called "develop" before calling original setuptools test command. In order to avoid calling "develop", we directly exchanged this skbuild.test command with setuptools.command.test.test Note that in setup.py we only imported setuptools, not its submodules like setuptools.command.test. In skbuild they supply own version of test command to invoke and there they imported setuptools.command.test. So we happened to use this import indirectly. In 0.18 release skbuild removed custom test command in [1]. As setuptools.command.test import went along with it, we started getting errors like module 'setuptools.command' has no attribute 'test' Our use of setuptools.command.test.test might be remnant of the past though. By defining pytest as an alias of test and running python setup.py test, we shouldn't have been running setuptools test at all. Also, setuptools test seems to use unittest internally, which we don't do. As test command is removed in setuptools 72.0.0 and reason for our own override in setup.py is gone, we just drop the line altogether. [1]https://github.com/scikit-build/scikit-build/pull/1014/commits/dde5e791f9a16705af367ddeee681d99b6e736dd --- python/setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index bfe63589f..413d0cb32 100755 --- a/python/setup.py +++ b/python/setup.py @@ -2,7 +2,6 @@ import os import skbuild -import setuptools class get_pybind_include(object): def __init__(self, user=False): @@ -54,7 +53,4 @@ def get_long_description(): # supported OS X release 10.9 '-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9', ], - # skbuild's test imples develop, which is pretty obnoxious instead, use a - # manually integrated pytest. - cmdclass = { 'test': setuptools.command.test.test }, ) From 877c8d567602245edb372d7c7ed3a5c4078eb723 Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Fri, 26 Jul 2024 18:02:32 +0200 Subject: [PATCH 08/11] Fix macos 3.8 wheel skip Wheel is still build, even though it can't be tested. Looks like wheel indication we used in cibuildwheel doesn't match the real wheel. Original intent was not to build such wheels. --- .github/workflows/wheels.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index 58d706cb1..a555b5db4 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -67,8 +67,8 @@ jobs: CIBW_ARCHS: ${{ matrix.arch }} # musllinux arch skip: unlikely someone wants to run dlisio on alpine on non-standard architecture # musllinux cp38 skip: latest available numpy doesn't provide musslinux wheels, so it is unlikely useful - # macosx 38 skip: cibuildwheel can't test it - CIBW_SKIP: pp* *-musllinux_i686 *-musllinux_aarch64 cp38*-musllinux* cp38*-macosx_*:arm64 cp36-* cp37-* + # macosx arm 38 skip: cibuildwheel can't test it + CIBW_SKIP: pp* *-musllinux_i686 *-musllinux_aarch64 cp38*-musllinux* cp38*-macosx_*arm64 cp36-* cp37-* run: | python -m cibuildwheel --output-dir wheelhouse python/ From 62eaddb2c86af2cdca8be881d17fead2c695630d Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Wed, 7 Aug 2024 19:12:06 +0200 Subject: [PATCH 09/11] Add python CI build This is not the same as other workflows: - core.yaml builds with python turned off - wheels.yaml builds cpp core separately and then builds wheels - docs.yaml uses python build system, but no tests - bigendian.yaml is the one also doing tests, but it is slow and runs on some oldish dependencies So additional workflow is the best option to test python build. --- .github/workflows/python.yaml | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/python.yaml diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml new file mode 100644 index 000000000..bce5e250a --- /dev/null +++ b/.github/workflows/python.yaml @@ -0,0 +1,62 @@ +name: Build and test python +#Build and test with python turned on + +on: + push: + branches: [master] + pull_request: + branches: [master] + workflow_dispatch: + +jobs: + build_python: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + include: + - os: ubuntu-20.04 + + steps: + - uses: actions/checkout@v4 + + - uses: "./.github/actions/build_core_dependencies" + with: + privileges: "sudo" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Create python venv + shell: bash + run: | + python -m venv venv + . ./venv/bin/activate + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + python -m pip install -r python/requirements-dev.txt + which python + + - name: Configure dlisio + shell: bash + run: | + . ./venv/bin/activate + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_PYTHON=ON + + - name: Build dlisio + shell: bash + run: | + sudo cmake \ + --build build \ + --parallel \ + --config Release + + - name: Test dlisio + shell: bash + run: | + cd build + ctest -C Release --output-on-failure --verbose From c5b76a12b0fff96b4bb9e1a83d9699e27c1ef1ad Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Wed, 7 Aug 2024 19:21:41 +0200 Subject: [PATCH 10/11] Drop setup.py test command Using setup.py to run commands is deprecated. Currently remove test command only as it is explicitly deprecated. pytest-runner was used to run pytests through setuptools, but now it is no longer needed. --- README.md | 2 +- python/CMakeLists.txt | 2 +- python/pyproject.toml | 1 - python/requirements-dev.txt | 1 - python/setup.cfg | 3 --- python/setup.py | 2 -- 6 files changed, 2 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 30bba91de..08cd94b6b 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ To develop dlisio, or to build a particular revision from source, you need: * [pybind11](https://github.com/pybind/pybind11) version 2.6 or greater * [setuptools](https://pypi.python.org/pypi/setuptools) version 28 or greater * [layered-file-protocols](https://github.com/equinor/layered-file-protocols) -* python packages pytest, pytest-runner, and numpy +* python packages pytest and numpy If you do not have pybind11 installed on your system, the easiest way to get a working copy is to `pip3 install pybind11` (NP! pybind11, not pybind) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index e3a4aa961..5c0d9562a 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -137,6 +137,6 @@ endif() # different args to setup.py, rebuilding the python lib (and wrongly so as it # either won't find dlisio or picked up on a system installed one) add_test(NAME python.unit - COMMAND ${python} ${setup.py} --skip-cmake test + COMMAND ${python} -m pytest tests WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 2f6bc6795..942b38d9a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,6 @@ requires = [ "scikit-build", "wheel", "pybind11", - "pytest-runner", ] [tool.cibuildwheel] diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index e1404f5da..51fb60538 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -4,7 +4,6 @@ numpy setuptools setuptools_scm pytest -pytest-runner pybind11 hypothesis sphinx diff --git a/python/setup.cfg b/python/setup.cfg index fa8d22139..6dd760052 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -1,5 +1,2 @@ [metadata] version = 1.0.1 - -[aliases] -test = pytest diff --git a/python/setup.py b/python/setup.py index 413d0cb32..6da649f55 100755 --- a/python/setup.py +++ b/python/setup.py @@ -39,9 +39,7 @@ def get_long_description(): setup_requires = ['setuptools >= 28', 'pybind11 >= 2.3', 'setuptools_scm', - 'pytest-runner', ], - tests_require = ['pytest'], # we're building with the pybind11 fetched from pip. Since we don't rely on # a cmake-installed pybind there's also no find_package(pybind11) - # instead, the get include dirs from the package and give directly from From 007d5e17f00fe99499ba7afe0e08c606ae929fe7 Mon Sep 17 00:00:00 2001 From: Alena Chaikouskaya Date: Thu, 8 Aug 2024 10:50:01 +0200 Subject: [PATCH 11/11] Temporary skip wheels for python 3.13 Python is in pre-release version, cibuildwheel already included it in the matrix, but not all dependencies are yet in place (looks like numpy wheels are not yet released for python 3.13). --- .github/workflows/wheels.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index a555b5db4..ed3933b47 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -68,7 +68,8 @@ jobs: # musllinux arch skip: unlikely someone wants to run dlisio on alpine on non-standard architecture # musllinux cp38 skip: latest available numpy doesn't provide musslinux wheels, so it is unlikely useful # macosx arm 38 skip: cibuildwheel can't test it - CIBW_SKIP: pp* *-musllinux_i686 *-musllinux_aarch64 cp38*-musllinux* cp38*-macosx_*arm64 cp36-* cp37-* + # 36, 37, 313 skip: python versions not supported + CIBW_SKIP: pp* *-musllinux_i686 *-musllinux_aarch64 cp38*-musllinux* cp38*-macosx_*arm64 cp36-* cp37-* cp313-* run: | python -m cibuildwheel --output-dir wheelhouse python/