Skip to content

Commit

Permalink
mdbx-cmake: some essential fixes for Python bindings (draft).
Browse files Browse the repository at this point in the history
  • Loading branch information
@erthink
erthink committed Jun 26, 2021
1 parent 2aca84b commit 11eebec
Show file tree
Hide file tree
Showing 9 changed files with 97 additions and 102 deletions.
1 change: 1 addition & 0 deletions .github/actions/spelling/expect.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1257,6 +1257,7 @@ PWOF
pwrite
pwritev
pwsh
pybind
pymdownx
pytest
pyw
Expand Down
110 changes: 47 additions & 63 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ add_mdbx_option(MDBX_INSTALL_STATIC "Build and install libmdbx for static linkin
add_mdbx_option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS})
add_mdbx_option(MDBX_BUILD_TOOLS "Build MDBX tools (mdbx_chk/stat/dump/load/copy)" ${MDBX_BUILD_TOOLS_DEFAULT})
CMAKE_DEPENDENT_OPTION(MDBX_INSTALL_MANPAGES "Install man-pages for MDBX tools (mdbx_chk/stat/dump/load/copy)" ON MDBX_BUILD_TOOLS OFF)
CMAKE_DEPENDENT_OPTION(MDBX_PYTHON_BINDINGS "Build the Python 3 bindings" ON MDBX_BUILD_SHARED_LIBRARY OFF)
CMAKE_DEPENDENT_OPTION(MDBX_PYTHON_BINDINGS "Build the Python 3 bindings" OFF MDBX_BUILD_SHARED_LIBRARY OFF)
add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON)
add_mdbx_option(MDBX_ENV_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO)
mark_as_advanced(MDBX_ENV_CHECKPID)
Expand Down Expand Up @@ -872,85 +872,69 @@ if(NOT MDBX_AMALGAMATED_SOURCE AND MDBX_ENABLE_TESTS)
endif()

################################################################################
# python bindings: This is temporary part and subject to replace with python's native `distutils.core`

if(MDBX_PYTHON_BINDINGS)
if(CMAKE_VERSION VERSION_LESS 3.14)
message(FATAL_ERROR "Python bindings require CMake version >= 3.14")
endif()
if (NOT MDBX_BUILD_SHARED_LIBRARY)
message(SEND_ERROR "Python bindings require libmdbx to be built as a shared library.")
endif()
if (NOT PYTEST)
find_program(PYTEST pytest)
#message(FATAL_ERROR "PYTEST: ${PYTEST}")
if (PYTEST-NOTFOUND)
find_program(PYTEST pytest-3)
if(NOT PYTEST)
set(
PYTEST
python3 -m pytest
)
endif()
endif()
endif()

set(MDBX_BUILD_DIR_LIB_LOCATION
$<TARGET_FILE:mdbx>
)

add_custom_target(
python-prep
COMMAND sed 's|@LIBLOCATION@|${CMAKE_INSTALL_PREFIX}/${MDBX_DLL_INSTALL_DESTINATION}/${mdbx}|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py
COMMAND sed 's|@MDBX_VERSION@|${MDBX_VERSION}|' python/libmdbx/setup.py.in > python/libmdbx/setup.py
COMMENT Prepare python bindings
)

add_custom_target(
python-test-prep
COMMAND sed 's|@LIBLOCATION@|${MDBX_BUILD_DIR_LIB_LOCATION}|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py
COMMENT set the DLL path to the built dll in the build dir
DEPENDS mdbx
)
find_package(Python3 REQUIRED COMPONENTS Interpreter) # Development

# FIXME: The SOURCE directory should not be cluttered with generated files
# that depend on the build parameters.
# E.g this will break cross-compilation and builds for different
# platforms from the single copy of code.
#
# Moreover the another problem = python code should refer to the
# installed libmdbx.so, but not to a build's temporary artifact(s).
set(MDBX_SOLIB_LOCATION "$<TARGET_FILE:mdbx>")
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in"
"${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in-cmake" ESCAPE_QUOTES)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py.in"
"${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py" ESCAPE_QUOTES)

file(GENERATE
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py"
INPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in-cmake")
file(GENERATE
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/tests/mdbx4test.py"
INPUT "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py.in-cmake")

add_custom_target(
python-test
COMMAND ${PYTEST}
COMMENT test python bindings
WORKING_DIRECTORY python
DEPENDS python-test-prep
)
python-wheel
SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/mdbx.py" "${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py"
COMMAND ${Python3_EXECUTABLE} -m build
COMMENT build python wheel of the bindings
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python"
DEPENDS mdbx)

add_custom_target(
python-install
COMMAND python setup.py install --root="$CMAKE_INSTALL_PREFIX"
SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py"
COMMAND ${Python3_EXECUTABLE} setup.py install --root="$CMAKE_INSTALL_PREFIX"
COMMENT package python bindings
WORKING_DIRECTORY python
)

add_custom_target(
python-wheel
COMMAND python3 -m build
COMMENT build python wheel of the bindings
WORKING_DIRECTORY python
)
DEPENDS python-wheel
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python")

add_custom_target(
python-clean
COMMAND python3 setup.py clean --all
WORKING_DIRECTORY python
)
python-test
SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/tests/mdbx4test.py"
COMMAND ${Python3_EXECUTABLE} runme.py
COMMENT test python bindings
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python/libmdbx/tests")

add_custom_target(
python
)

add_dependencies(
distclean
python-clean
)
SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py"
COMMAND ${Python3_EXECUTABLE} setup.py clean --all
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python")
add_dependencies(distclean python-clean)

add_dependencies(
python
python-test
python-wheel
)
add_custom_target(python-all)
add_dependencies(python-all python-test python-wheel)
endif()

set(PACKAGE "libmdbx")
Expand Down
57 changes: 28 additions & 29 deletions GNUmakefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ LIBS ?= $(shell uname | grep -qi SunOS && echo "-lkstat") $(shell uname | gre
LDFLAGS ?= $(shell $(LD) --help 2>/dev/null | grep -q -- --gc-sections && echo '-Wl,--gc-sections,-z,relro,-O1')$(shell $(LD) --help 2>/dev/null | grep -q -- -dead_strip && echo '-Wl,-dead_strip')
EXE_LDFLAGS ?= -pthread

PYTEST ?= $(shell if command -v pytest >/dev/null ; then echo pytest ; elif command -v pytest-3 >/dev/null; then echo pytest-3 ; else echo python3 -m pytest ; fi)

################################################################################

UNAME := $(shell uname -s 2>/dev/null || echo Unknown)
Expand All @@ -64,9 +62,8 @@ HEADERS := mdbx.h mdbx.h++
LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX)
TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk mdbx_drop
MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1 mdbx_drop.1
BINDINGS := python

.PHONY: all help options lib tools clean install uninstall python
.PHONY: all help options lib tools clean install uninstall
.PHONY: install-strip install-no-strip strip libmdbx mdbx show-options

ifeq ("$(origin V)", "command line")
Expand Down Expand Up @@ -107,7 +104,6 @@ help:
@echo " make bench-triplet - run ioarena-benchmark for mdbx, lmdb, sqlite3"
@echo " make bench-quartet - run ioarena-benchmark for mdbx, lmdb, rocksdb, wiredtiger"
@echo " make bench-clean - remove temp database(s) after benchmark"
@echo " make python - Generate python bindings"
#> dist-cutoff-begin
@echo ""
@echo " make test - basic test"
Expand Down Expand Up @@ -559,38 +555,41 @@ dist/man1/mdbx_%.1: src/man1/mdbx_%.1
endif
python: python-test python-wheel
################################################################################
# python bindings: This is temporary part and subject to replace with python's native `distutils.core`
# Python bindings generation
python/libmdbx/mdbx.py:
@echo ' GENERATING PYTHON BINDINGS'
$(QUIET)sed 's|@LIBLOCATION@|$(prefix)/libmdbx.$(SO_SUFFIX)|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py
PYTHON3 ?= python3
.PHONY: python-test python-wheel python-clean python-install python-all
python-all: python-test python-wheel
python-test-prep:
@echo ' PREPARING PYTHON BINDINGS FOR TEST'
$(QUIET)sed 's|@LIBLOCATION@|$(abspath .)/libmdbx.$(SO_SUFFIX)|' python/libmdbx/mdbx.py.in > python/libmdbx/mdbx.py
python/libmdbx/mdbx.py: python/libmdbx/mdbx.py.in
@echo ' POKE solib location into $@'
$(QUIET)sed 's|@MDBX_SOLIB_LOCATION@|$(prefix)/lib$(suffix)/libmdbx.$(SO_SUFFIX)|' $< >$@
python-test: python-test-prep libmdbx.$(SO_SUFFIX)
@echo ' TESTING PYTHON BINDINGS'
cd python/libmdbx && $(PYTEST)
python/setup.py: python/setup.py.in
@echo ' POKE version into $@'
$(QUIET)sed 's|@MDBX_VERSION@|${MDBX_GIT_VERSION}|' $< >$@
python-prep-setup:
@echo ' SETTING WHEEL VERSION'
$(QUIET)sed 's|@MDBX_VERSION@|${MDBX_GIT_VERSION}|' python/libmdbx/setup.py.in > python/libmdbx/setup.py
python-wheel: python/libmdbx/mdbx.py python/setup.py
@echo ' CREATE python bindings wheel'
$(QUIET)cd python && $(PYTHON3) -m build >$@.log
python-wheel: python/libmdbx/mdbx.py python-prep-setup
@echo ' GENERATING PYTHON WHEEL'
cd python/libmdbx && python3 -m build
python-clean: python/libmdbx/mdbx.py python/setup.py
@echo ' CLEAN python bindings directories'
$(QUIET)cd python && $(PYTHON3) setup.py clean --all >$@.log
python-clean: python-prep-setup
@echo 'CLEANING PYTHON DIRECTORY'
cd python/libmdbx && python3 setup.py clean --all
python-install: python/libmdbx/mdbx.py python/setup.py
@echo ' INSTALLING python bindings'
$(QUIET)cd python && $(PYTHON3) setup.py install --root="$(abspath $(DESTDIR)/)" >$@.log
python-install: python/libmdbx/mdbx.py python-prep-setup
@echo 'INSTALLING PYTHON BINDINGS'
cd python/libmdbx && python3 setup.py install --root="$(DESTDIR)"
# python bindings test: This is temporary part and subject to replace with python's native `distutils.core`
python/libmdbx/tests/mdbx4test.py: python/libmdbx/mdbx.py.in libmdbx.$(SO_SUFFIX)
@echo ' POKE solib location into $@'
$(QUIET)sed 's|@MDBX_SOLIB_LOCATION@|$(abspath libmdbx.$(SO_SUFFIX))|' $< >$@
.PHONY: python-test python-test-prep python python/libmdbx/mdbx.py python-prep-setup
python-test: python/libmdbx/tests/runme.py python/libmdbx/tests/mdbx4test.py
@echo -n ' RUN $<...'
$(QUIET)$(PYTHON3) python/libmdbx/tests/runme.py >$@.log 2>&1 && echo " Done" || echo "Fail"
################################################################################
# Cross-compilation simple test
Expand Down
13 changes: 9 additions & 4 deletions python/TODO.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,16 @@ Incomplete TODO list to merge `python-bindings` (status: draft) branch to the `m

1.

- [ ] cmake: enable python binding only for cmake >= 3.14 and warn otherwise (but we unable change the requirement for minimal cmake version).
- [ ] cmake: use `FindPython()` instead of calling `python` directly.
- [ ] cmake: always use `python3 -m pytest` instead of finding `pytest`, since such way is fragile and unportable.
- [ ] cmake: replace using `sed` by CMake's `configure_file()`.
- [x] cmake: enable python binding only for cmake >= 3.14 and warn otherwise (but we unable change the requirement for minimal cmake version).
- [x] cmake: use `FindPython()` instead of calling `python` directly.
- [x] cmake: always use `python3 -m pytest` instead of finding `pytest`, since such way is fragile and unportable.
- [x] cmake: replace using `sed` by CMake's `configure_file()`.
- [ ] cmake: seems CMake have some useful features for python than we should use instead of custom targets.
- [ ] python: no build-depend files should be created inside `${CMAKE_CURRENT_SOURCE_DIR}`, but only `${CMAKE_CURRENT_BINARY_DIR}` instead.
- [ ] python: should should refer to the installed libmdbx.so, but not to a build's temporary artifact(s).
- [ ] python: A different package name must be chosen, otherwise there is confusion with the generally accepted names of source code archives (and someone should be reserved for full C++ bindings by pybind11).
- [ ] create README.md
- [ ] egg/wheel: ability to build & install library from pip.

2.

Expand Down
2 changes: 1 addition & 1 deletion python/libmdbx/mdbx.py.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import errno
import os

# init lib
_lib = ctypes.CDLL("@LIBLOCATION@")
_lib = ctypes.CDLL("@MDBX_SOLIB_LOCATION@")

# Names are all CamelCase because PEP 8 states class names have to be CamelCase.
# Abbreviations like TXN (although they are native class names) are capitalized because of PEP 8, too.
Expand Down
Empty file removed python/libmdbx/tests/__init__.py
View file
Empty file.
9 changes: 5 additions & 4 deletions python/libmdbx/tests/mdbx_test.py → python/libmdbx/tests/runme.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/env python3
#!/usr/bin/env python3
# Copyright 2021 Noel Kuntze <[email protected]> for Contauro AG
#
# Redistribution and use in source and binary forms, with or without
Expand All @@ -12,17 +12,18 @@
import ctypes
import inspect

import libmdbx
import mdbx4test as libmdbx

import subprocess
import random
import unittest

import os
import string
import threading
import time
import tempfile

MDBX_TEST_DIR="%s/MDBX_TEST" % os.path.dirname(__file__)
MDBX_TEST_DIR="%s/MDBX_TEST" % tempfile.gettempdir()
MDBX_TEST_DB_NAME="MDBX_TEST_DB_NAME"
MDBX_TEST_MAP_NAME="MDBX_TEST_MAP_NAME"
MDBX_TEST_KEY=bytes("MDBX_TEST_KEY", "utf-8")
Expand Down
0 python/libmdbx/pyproject.toml → python/pyproject.toml
View file
File renamed without changes.
7 changes: 6 additions & 1 deletion python/libmdbx/setup.py.in → python/setup.py.in
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
#!/usr/bin/env python
#!/usr/bin/env python3

# TODO: like https://github.com/jnwatson/py-lmdb/blob/master/setup.py

from distutils.core import setup

Expand All @@ -8,4 +10,7 @@ setup(name='libmdbx',
author='Noel Kuntze',
author_email='[email protected]',
url='https://github.com/Thermi/libmdbx/tree/python-bindings',
license='SPDX-License-Identifier: OLDAP-2.8',
keywords=['key-value', 'nosql', 'database', 'mdbx', 'libmdbx', 'lmdb'],
packages=['libmdbx'],
)

1 comment on commit 11eebec

@github-actions
Copy link

@github-actions github-actions bot commented on 11eebec Jun 26, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@check-spelling-bot Report

Unrecognized words, please review:

  • Erro
  • gettempdir
  • jnwatson
  • nosql
  • OLDAP
  • runme
  • tempfile
Previously acknowledged words that are now absent LIBLOCATION prealloc
To accept these unrecognized words as correct (and remove the previously acknowledged and now absent words), run the following commands

... in a clone of the [email protected]:erthink/libmdbx.git repository
on the python-bindings branch:

update_files() {
perl -e '
my @expect_files=qw('".github/actions/spelling/expect.txt"');
@ARGV=@expect_files;
my @stale=qw('"$patch_remove"');
my $re=join "|", @stale;
my $suffix=".".time();
my $previous="";
sub maybe_unlink { unlink($_[0]) if $_[0]; }
while (<>) {
if ($ARGV ne $old_argv) { maybe_unlink($previous); $previous="$ARGV$suffix"; rename($ARGV, $previous); open(ARGV_OUT, ">$ARGV"); select(ARGV_OUT); $old_argv = $ARGV; }
next if /^(?:$re)(?:(?:\r|\n)*$| .*)/; print;
}; maybe_unlink($previous);'
perl -e '
my $new_expect_file=".github/actions/spelling/expect.txt";
use File::Path qw(make_path);
use File::Basename qw(dirname);
make_path (dirname($new_expect_file));
open FILE, q{<}, $new_expect_file; chomp(my @words = <FILE>); close FILE;
my @add=qw('"$patch_add"');
my %items; @items{@words} = @words x (1); @items{@add} = @add x (1);
@words = sort {lc($a)."-".$a cmp lc($b)."-".$b} keys %items;
open FILE, q{>}, $new_expect_file; for my $word (@words) { print FILE "$word\n" if $word =~ /\w/; };
close FILE;
system("git", "add", $new_expect_file);
'
}

comment_json=$(mktemp)
curl -L -s -S \
  --header "Content-Type: application/json" \
  "https://api.github.com/repos/erthink/libmdbx/comments/52700199" > "$comment_json"
comment_body=$(mktemp)
jq -r .body < "$comment_json" > $comment_body
rm $comment_json

patch_remove=$(perl -ne 'next unless s{^</summary>(.*)</details>$}{$1}; print' < "$comment_body")
  

patch_add=$(perl -e '$/=undef;
$_=<>;
s{<details>.*}{}s;
s{^#.*}{};
s{\n##.*}{};
s{(?:^|\n)\s*\*}{}g;
s{\s+}{ }g;
print' < "$comment_body")
  
update_files
rm $comment_body
git add -u

Please sign in to comment.