diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 4285dad33..08ad2c5cd 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -1257,6 +1257,7 @@ PWOF pwrite pwritev pwsh +pybind pymdownx pytest pyw diff --git a/CMakeLists.txt b/CMakeLists.txt index a7a0f9fc3..60b8bcc9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -436,7 +436,7 @@ add_mdbx_option(MDBX_INSTALL_STATIC "Build and install libmdbx for static linkin add_mdbx_option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS}) add_mdbx_option(MDBX_BUILD_TOOLS "Build MDBX tools (mdbx_chk/stat/dump/load/copy)" ${MDBX_BUILD_TOOLS_DEFAULT}) CMAKE_DEPENDENT_OPTION(MDBX_INSTALL_MANPAGES "Install man-pages for MDBX tools (mdbx_chk/stat/dump/load/copy)" ON MDBX_BUILD_TOOLS OFF) -CMAKE_DEPENDENT_OPTION(MDBX_PYTHON_BINDINGS "Build the Python 3 bindings" ON MDBX_BUILD_SHARED_LIBRARY OFF) +CMAKE_DEPENDENT_OPTION(MDBX_PYTHON_BINDINGS "Build the Python 3 bindings" OFF MDBX_BUILD_SHARED_LIBRARY OFF) add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON) add_mdbx_option(MDBX_ENV_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO) mark_as_advanced(MDBX_ENV_CHECKPID) @@ -872,85 +872,69 @@ if(NOT MDBX_AMALGAMATED_SOURCE AND MDBX_ENABLE_TESTS) endif() ################################################################################ +# python bindings: This is temporary part and subject to replace with python's native `distutils.core` if(MDBX_PYTHON_BINDINGS) + if(CMAKE_VERSION VERSION_LESS 3.14) + message(FATAL_ERROR "Python bindings require CMake version >= 3.14") + endif() if (NOT MDBX_BUILD_SHARED_LIBRARY) message(SEND_ERROR "Python bindings require libmdbx to be built as a shared library.") endif() - if (NOT PYTEST) - find_program(PYTEST pytest) - #message(FATAL_ERROR "PYTEST: ${PYTEST}") - if (PYTEST-NOTFOUND) - find_program(PYTEST pytest-3) - if(NOT PYTEST) - set( - PYTEST - python3 -m pytest - ) - endif() - endif() - endif() - - set(MDBX_BUILD_DIR_LIB_LOCATION - $ - ) - - add_custom_target( - python-prep - COMMAND sed 's|@LIBLOCATION@|${CMAKE_INSTALL_PREFIX}/${MDBX_DLL_INSTALL_DESTINATION}/${mdbx}|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py - COMMAND sed 's|@MDBX_VERSION@|${MDBX_VERSION}|' python/libmdbx/setup.py.in > python/libmdbx/setup.py - COMMENT Prepare python bindings - ) - - add_custom_target( - python-test-prep - COMMAND sed 's|@LIBLOCATION@|${MDBX_BUILD_DIR_LIB_LOCATION}|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py - COMMENT set the DLL path to the built dll in the build dir - DEPENDS mdbx - ) + find_package(Python3 REQUIRED COMPONENTS Interpreter) # Development + + # FIXME: The SOURCE directory should not be cluttered with generated files + # that depend on the build parameters. + # E.g this will break cross-compilation and builds for different + # platforms from the single copy of code. + # + # Moreover the another problem = python code should refer to the + # installed libmdbx.so, but not to a build's temporary artifact(s). + set(MDBX_SOLIB_LOCATION "$") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in" + "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in-cmake" ESCAPE_QUOTES) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py.in" + "${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py" ESCAPE_QUOTES) + + file(GENERATE + OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py" + INPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in-cmake") + file(GENERATE + OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/tests/mdbx4test.py" + INPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in-cmake") add_custom_target( - python-test - COMMAND ${PYTEST} - COMMENT test python bindings - WORKING_DIRECTORY python - DEPENDS python-test-prep - ) + python-wheel + SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py" "${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py" + COMMAND ${Python3_EXECUTABLE} -m build + COMMENT build python wheel of the bindings + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python" + DEPENDS mdbx) add_custom_target( python-install - COMMAND python setup.py install --root="$CMAKE_INSTALL_PREFIX" + SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py" + COMMAND ${Python3_EXECUTABLE} setup.py install --root="$CMAKE_INSTALL_PREFIX" COMMENT package python bindings - WORKING_DIRECTORY python - ) - - add_custom_target( - python-wheel - COMMAND python3 -m build - COMMENT build python wheel of the bindings - WORKING_DIRECTORY python - ) + DEPENDS python-wheel + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python") add_custom_target( - python-clean - COMMAND python3 setup.py clean --all - WORKING_DIRECTORY python - ) + python-test + SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/tests/mdbx4test.py" + COMMAND ${Python3_EXECUTABLE} runme.py + COMMENT test python bindings + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/tests") add_custom_target( - python - ) - - add_dependencies( - distclean python-clean - ) + SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py" + COMMAND ${Python3_EXECUTABLE} setup.py clean --all + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python") + add_dependencies(distclean python-clean) - add_dependencies( - python - python-test - python-wheel - ) + add_custom_target(python-all) + add_dependencies(python-all python-test python-wheel) endif() set(PACKAGE "libmdbx") diff --git a/GNUmakefile b/GNUmakefile index 8437c9532..177f8e639 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -47,8 +47,6 @@ LIBS ?= $(shell uname | grep -qi SunOS && echo "-lkstat") $(shell uname | gre LDFLAGS ?= $(shell $(LD) --help 2>/dev/null | grep -q -- --gc-sections && echo '-Wl,--gc-sections,-z,relro,-O1')$(shell $(LD) --help 2>/dev/null | grep -q -- -dead_strip && echo '-Wl,-dead_strip') EXE_LDFLAGS ?= -pthread -PYTEST ?= $(shell if command -v pytest >/dev/null ; then echo pytest ; elif command -v pytest-3 >/dev/null; then echo pytest-3 ; else echo python3 -m pytest ; fi) - ################################################################################ UNAME := $(shell uname -s 2>/dev/null || echo Unknown) @@ -64,9 +62,8 @@ HEADERS := mdbx.h mdbx.h++ LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX) TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk mdbx_drop MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1 mdbx_drop.1 -BINDINGS := python -.PHONY: all help options lib tools clean install uninstall python +.PHONY: all help options lib tools clean install uninstall .PHONY: install-strip install-no-strip strip libmdbx mdbx show-options ifeq ("$(origin V)", "command line") @@ -107,7 +104,6 @@ help: @echo " make bench-triplet - run ioarena-benchmark for mdbx, lmdb, sqlite3" @echo " make bench-quartet - run ioarena-benchmark for mdbx, lmdb, rocksdb, wiredtiger" @echo " make bench-clean - remove temp database(s) after benchmark" - @echo " make python - Generate python bindings" #> dist-cutoff-begin @echo "" @echo " make test - basic test" @@ -559,38 +555,41 @@ dist/man1/mdbx_%.1: src/man1/mdbx_%.1 endif -python: python-test python-wheel +################################################################################ +# python bindings: This is temporary part and subject to replace with python's native `distutils.core` -# Python bindings generation -python/libmdbx/mdbx.py: - @echo ' GENERATING PYTHON BINDINGS' - $(QUIET)sed 's|@LIBLOCATION@|$(prefix)/libmdbx.$(SO_SUFFIX)|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py +PYTHON3 ?= python3 +.PHONY: python-test python-wheel python-clean python-install python-all +python-all: python-test python-wheel -python-test-prep: - @echo ' PREPARING PYTHON BINDINGS FOR TEST' - $(QUIET)sed 's|@LIBLOCATION@|$(abspath .)/libmdbx.$(SO_SUFFIX)|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py +python/libmdbx/mdbx.py: python/libmdbx/mdbx.py.in + @echo ' POKE solib location into $@' + $(QUIET)sed 's|@MDBX_SOLIB_LOCATION@|$(prefix)/lib$(suffix)/libmdbx.$(SO_SUFFIX)|' $< >$@ -python-test: python-test-prep libmdbx.$(SO_SUFFIX) - @echo ' TESTING PYTHON BINDINGS' - cd python/libmdbx && $(PYTEST) +python/setup.py: python/setup.py.in + @echo ' POKE version into $@' + $(QUIET)sed 's|@MDBX_VERSION@|${MDBX_GIT_VERSION}|' $< >$@ -python-prep-setup: - @echo ' SETTING WHEEL VERSION' - $(QUIET)sed 's|@MDBX_VERSION@|${MDBX_GIT_VERSION}|' python/libmdbx/setup.py.in > python/libmdbx/setup.py +python-wheel: python/libmdbx/mdbx.py python/setup.py + @echo ' CREATE python bindings wheel' + $(QUIET)cd python && $(PYTHON3) -m build >$@.log -python-wheel: python/libmdbx/mdbx.py python-prep-setup - @echo ' GENERATING PYTHON WHEEL' - cd python/libmdbx && python3 -m build +python-clean: python/libmdbx/mdbx.py python/setup.py + @echo ' CLEAN python bindings directories' + $(QUIET)cd python && $(PYTHON3) setup.py clean --all >$@.log -python-clean: python-prep-setup - @echo 'CLEANING PYTHON DIRECTORY' - cd python/libmdbx && python3 setup.py clean --all +python-install: python/libmdbx/mdbx.py python/setup.py + @echo ' INSTALLING python bindings' + $(QUIET)cd python && $(PYTHON3) setup.py install --root="$(abspath $(DESTDIR)/)" >$@.log -python-install: python/libmdbx/mdbx.py python-prep-setup - @echo 'INSTALLING PYTHON BINDINGS' - cd python/libmdbx && python3 setup.py install --root="$(DESTDIR)" +# python bindings test: This is temporary part and subject to replace with python's native `distutils.core` +python/libmdbx/tests/mdbx4test.py: python/libmdbx/mdbx.py.in libmdbx.$(SO_SUFFIX) + @echo ' POKE solib location into $@' + $(QUIET)sed 's|@MDBX_SOLIB_LOCATION@|$(abspath libmdbx.$(SO_SUFFIX))|' $< >$@ -.PHONY: python-test python-test-prep python python/libmdbx/mdbx.py python-prep-setup +python-test: python/libmdbx/tests/runme.py python/libmdbx/tests/mdbx4test.py + @echo -n ' RUN $<...' + $(QUIET)$(PYTHON3) python/libmdbx/tests/runme.py >$@.log 2>&1 && echo " Done" || echo "Fail" ################################################################################ # Cross-compilation simple test diff --git a/python/TODO.md b/python/TODO.md index 10912db37..bb655ad12 100644 --- a/python/TODO.md +++ b/python/TODO.md @@ -8,11 +8,16 @@ Incomplete TODO list to merge `python-bindings` (status: draft) branch to the `m 1. - - [ ] cmake: enable python binding only for cmake >= 3.14 and warn otherwise (but we unable change the requirement for minimal cmake version). - - [ ] cmake: use `FindPython()` instead of calling `python` directly. - - [ ] cmake: always use `python3 -m pytest` instead of finding `pytest`, since such way is fragile and unportable. - - [ ] cmake: replace using `sed` by CMake's `configure_file()`. + - [x] cmake: enable python binding only for cmake >= 3.14 and warn otherwise (but we unable change the requirement for minimal cmake version). + - [x] cmake: use `FindPython()` instead of calling `python` directly. + - [x] cmake: always use `python3 -m pytest` instead of finding `pytest`, since such way is fragile and unportable. + - [x] cmake: replace using `sed` by CMake's `configure_file()`. - [ ] cmake: seems CMake have some useful features for python than we should use instead of custom targets. + - [ ] python: no build-depend files should be created inside `${CMAKE_CURRENT_SOURCE_DIR}`, but only `${CMAKE_CURRENT_BINARY_DIR}` instead. + - [ ] python: should should refer to the installed libmdbx.so, but not to a build's temporary artifact(s). + - [ ] python: A different package name must be chosen, otherwise there is confusion with the generally accepted names of source code archives (and someone should be reserved for full C++ bindings by pybind11). + - [ ] create README.md + - [ ] egg/wheel: ability to build & install library from pip. 2. diff --git a/python/libmdbx/mdbx.py.in b/python/libmdbx/mdbx.py.in index 9899c8787..027b5f2ed 100644 --- a/python/libmdbx/mdbx.py.in +++ b/python/libmdbx/mdbx.py.in @@ -24,7 +24,7 @@ import errno import os # init lib -_lib = ctypes.CDLL("@LIBLOCATION@") +_lib = ctypes.CDLL("@MDBX_SOLIB_LOCATION@") # Names are all CamelCase because PEP 8 states class names have to be CamelCase. # Abbreviations like TXN (although they are native class names) are capitalized because of PEP 8, too. diff --git a/python/libmdbx/tests/__init__.py b/python/libmdbx/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/libmdbx/tests/mdbx_test.py b/python/libmdbx/tests/runme.py similarity index 99% rename from python/libmdbx/tests/mdbx_test.py rename to python/libmdbx/tests/runme.py index c560118dd..7455efc8a 100755 --- a/python/libmdbx/tests/mdbx_test.py +++ b/python/libmdbx/tests/runme.py @@ -1,4 +1,4 @@ -#!/bin/env python3 +#!/usr/bin/env python3 # Copyright 2021 Noel Kuntze for Contauro AG # # Redistribution and use in source and binary forms, with or without @@ -12,17 +12,18 @@ import ctypes import inspect -import libmdbx +import mdbx4test as libmdbx + import subprocess import random import unittest - import os import string import threading import time +import tempfile -MDBX_TEST_DIR="%s/MDBX_TEST" % os.path.dirname(__file__) +MDBX_TEST_DIR="%s/MDBX_TEST" % tempfile.gettempdir() MDBX_TEST_DB_NAME="MDBX_TEST_DB_NAME" MDBX_TEST_MAP_NAME="MDBX_TEST_MAP_NAME" MDBX_TEST_KEY=bytes("MDBX_TEST_KEY", "utf-8") diff --git a/python/libmdbx/pyproject.toml b/python/pyproject.toml similarity index 100% rename from python/libmdbx/pyproject.toml rename to python/pyproject.toml diff --git a/python/libmdbx/setup.py.in b/python/setup.py.in similarity index 52% rename from python/libmdbx/setup.py.in rename to python/setup.py.in index bbc49b9dc..442259d67 100644 --- a/python/libmdbx/setup.py.in +++ b/python/setup.py.in @@ -1,4 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 + +# TODO: like https://github.com/jnwatson/py-lmdb/blob/master/setup.py from distutils.core import setup @@ -8,4 +10,7 @@ setup(name='libmdbx', author='Noel Kuntze', author_email='noel.kuntze@thermi.consulting', url='https://github.com/Thermi/libmdbx/tree/python-bindings', + license='SPDX-License-Identifier: OLDAP-2.8', + keywords=['key-value', 'nosql', 'database', 'mdbx', 'libmdbx', 'lmdb'], + packages=['libmdbx'], )