From 66d0c38e848c8f04271837fda1b2eadcce75da47 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Tue, 14 Mar 2023 09:24:35 -0400 Subject: [PATCH] Move virtual environment construction to root directory The Protege catalog XML construction design involves review of the transitive `owl:imports` closure of UCO's graph files. To do this, RDFLib needs to be available in a Python virtual environment before descending into the `/ontology` directory. Before this patch, a virtual environment with RDFLib was constructed in `/tests` for testing, but not for other ontology-related maintenance. This patch moves the virtual environment construction to the UCO repository root directory, and sets the dependency order of `all` and `check` to include `/venv` being built before descent into `/ontology` and `/tests. All paths referencing `/tests/venv` under `/tests` have been updated. As one bit of code upgrading, the `PYTHON3` selector snippet, originally written before Python 3.10's release, now looks only for the default Python 3 if not supplied when calling Make (e.g. with `make PYTHON3=python3.11 check`). This patch isolates its effects to moving the virtual environment. A follow-on patch will integrate catalog construction using the new placement. References: * https://github.com/ucoProject/UCO/issues/449 Signed-off-by: Alex Nelson --- .gitignore | 3 ++ Makefile | 45 ++++++++++++++++++-- ontology/uco/master/Makefile | 5 +-- tests/requirements.txt => requirements.txt | 0 tests/.gitignore | 3 -- tests/Makefile | 49 +++++----------------- tests/dependencies/Makefile | 10 ++--- tests/examples/Makefile | 6 +-- tests/shapes/Makefile | 8 ++-- tests/shapes/examples_uco_owl/Makefile | 8 ++-- tests/shapes/examples_uco_qc/Makefile | 8 ++-- 11 files changed, 74 insertions(+), 71 deletions(-) rename tests/requirements.txt => requirements.txt (100%) diff --git a/.gitignore b/.gitignore index d8d84865..e720787d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ .*.ttl .git_submodule_init.done.log .lib.done.log +.venv.done.log +_* +venv diff --git a/Makefile b/Makefile index 21c5d61f..31b95064 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,8 @@ SHELL := /bin/bash +PYTHON3 ?= $(shell which python3) + all: \ .lib.done.log $(MAKE) \ @@ -44,9 +46,31 @@ all: \ --directory lib touch $@ +# The two CASE-Utility... files are to trigger rebuilds based on command-line interface changes or version increments. +.venv.done.log: \ + dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/case_shacl_inheritance_reviewer/__init__.py \ + dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/setup.cfg \ + requirements.txt + rm -rf venv + $(PYTHON3) -m venv \ + venv + source venv/bin/activate \ + && pip install \ + --upgrade \ + pip \ + setuptools \ + wheel + source venv/bin/activate \ + && pip install \ + dependencies/CASE-Utility-SHACL-Inheritance-Reviewer + source venv/bin/activate \ + && pip install \ + --requirement requirements.txt + touch $@ + check: \ - .git_submodule_init.done.log \ - .lib.done.log + .lib.done.log \ + .venv.done.log $(MAKE) \ --directory ontology \ check @@ -59,7 +83,10 @@ clean: \ clean-ontology @rm -f \ .git_submodule_init.done.log \ - .lib.done.log + .lib.done.log \ + .venv.done.log + @rm -rf \ + venv clean-ontology: @$(MAKE) \ @@ -70,3 +97,15 @@ clean-tests: @$(MAKE) \ --directory tests \ clean + +# Maintain timestamp order. +dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/case_shacl_inheritance_reviewer/__init__.py: \ + .git_submodule_init.done.log + touch -c $@ + test -r $@ + +# Maintain timestamp order. +dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/setup.cfg: \ + .git_submodule_init.done.log + touch -c $@ + test -r $@ diff --git a/ontology/uco/master/Makefile b/ontology/uco/master/Makefile index d798fd0b..8bef57c6 100644 --- a/ontology/uco/master/Makefile +++ b/ontology/uco/master/Makefile @@ -18,12 +18,11 @@ top_srcdir := $(shell cd ../../.. ; pwd) all: \ catalog-v001.xml -# TODO - Move virtual environment creation up to root directory and re-order top_srcdir's descent. catalog-v001.xml: \ $(top_srcdir)/src/create-catalog-v001.xml.py \ - $(top_srcdir)/tests/.venv.done.log + $(top_srcdir)/.venv.done.log rm -f _$@ - source $(top_srcdir)/tests/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && python3 $(top_srcdir)/src/create-catalog-v001.xml.py \ _$@ \ domain_directories.tsv \ diff --git a/tests/requirements.txt b/requirements.txt similarity index 100% rename from tests/requirements.txt rename to requirements.txt diff --git a/tests/.gitignore b/tests/.gitignore index 4b1b6742..850ce0dd 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,6 +1,3 @@ .shapes.done.log -.venv.done.log -_* inheritance_review.ttl uco_monolithic.ttl -venv diff --git a/tests/Makefile b/tests/Makefile index aca7b3d8..e9c9b51f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -15,8 +15,6 @@ SHELL := /bin/bash top_srcdir := $(shell cd .. ; pwd) -PYTHON3 ?= $(shell which python3.9 2>/dev/null || which python3.8 2>/dev/null || which python3.7 2>/dev/null || which python3.6 2>/dev/null || which python3) - uco_turtle_files := $(shell /bin/ls $(top_srcdir)/ontology/*/*.ttl $(top_srcdir)/ontology/*/*/*.ttl) imported_ontology_files := \ @@ -31,7 +29,7 @@ all: # Ensure the UCO-internal shapes pass their own tests. .shapes.done.log: \ $(top_srcdir)/ontology/owl/owl.ttl \ - .venv.done.log \ + $(top_srcdir)/.venv.done.log \ shapes/uco-closure-qc.ttl \ shapes/uco-qc.ttl $(MAKE) \ @@ -39,34 +37,11 @@ all: check touch $@ -# The two CASE-Utility... files are to trigger rebuilds based on command-line interface changes or version increments. -.venv.done.log: \ - $(top_srcdir)/.git_submodule_init.done.log \ - $(top_srcdir)/dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/case_shacl_inheritance_reviewer/__init__.py \ - $(top_srcdir)/dependencies/CASE-Utility-SHACL-Inheritance-Reviewer/setup.cfg \ - requirements.txt - rm -rf venv - $(PYTHON3) -m venv \ - venv - source venv/bin/activate \ - && pip install \ - --upgrade \ - pip \ - setuptools \ - wheel - source venv/bin/activate \ - && pip install \ - $(top_srcdir)/dependencies/CASE-Utility-SHACL-Inheritance-Reviewer - source venv/bin/activate \ - && pip install \ - --requirement requirements.txt - touch $@ - check: \ check-dependencies \ inheritance_review.ttl \ uco_monolithic.ttl - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pytest \ --ignore examples \ --ignore shapes \ @@ -76,7 +51,7 @@ check: \ check check-dependencies: \ - .venv.done.log + $(top_srcdir)/.venv.done.log $(MAKE) \ --directory dependencies \ check @@ -91,14 +66,12 @@ clean: @rm -f \ .*.done.log \ uco_monolithic.ttl - @rm -rf \ - venv inheritance_review.ttl: \ $(uco_turtle_files) \ - .venv.done.log + $(top_srcdir)/.venv.done.log rm -f _$@ - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && case_shacl_inheritance_reviewer \ --strict \ _$@ \ @@ -112,19 +85,19 @@ uco_monolithic.ttl: \ $(uco_turtle_files) \ .shapes.done.log \ thing.ttl - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && rdfpipe \ --input-format ttl \ --output-format ttl \ $(uco_turtle_files) \ > __$@ # Review UCO closure for versioning consistency. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && rdfpipe \ __$@ \ $(imported_ontology_files) \ > ___$@ - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ @@ -133,7 +106,7 @@ uco_monolithic.ttl: \ --shacl-file-format turtle \ ___$@ # Review UCO closure with SHACL-SHACL. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ @@ -145,7 +118,7 @@ uco_monolithic.ttl: \ # Closure tests have passed; remove closure file. rm ___$@ # Review UCO for practice conformance. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ @@ -154,7 +127,7 @@ uco_monolithic.ttl: \ --shacl-file-format turtle \ __$@ # Review UCO for OWL 2 DL conformance. - source venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ diff --git a/tests/dependencies/Makefile b/tests/dependencies/Makefile index 2c066b86..7028d375 100644 --- a/tests/dependencies/Makefile +++ b/tests/dependencies/Makefile @@ -15,8 +15,6 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - all: check: \ @@ -29,10 +27,10 @@ clean: uco-owl-co.done.log: \ $(top_srcdir)/dependencies/collections-ontology/collections.owl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/ontology/owl/owl.ttl # Review for OWL 2 DL conformance. - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format xml \ --format turtle \ @@ -45,10 +43,10 @@ uco-owl-co.done.log: \ uco-owl-error.done.log: \ $(top_srcdir)/dependencies/error/docs/current/error.ttl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/ontology/owl/owl.ttl # Review for OWL 2 DL conformance. - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --data-file-format turtle \ --format turtle \ diff --git a/tests/examples/Makefile b/tests/examples/Makefile index 7d6bc056..81c5dfad 100644 --- a/tests/examples/Makefile +++ b/tests/examples/Makefile @@ -62,9 +62,9 @@ all: \ # characteristics. %_validation.ttl: \ %.json \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(tests_srcdir)/uco_monolithic.ttl - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --allow-warnings \ --data-file-format json-ld \ @@ -116,7 +116,7 @@ check: \ thread_PASS_validation.ttl \ thread_XFAIL_validation.ttl \ uco_thing_XFAIL_validation.ttl - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pytest \ --log-level=DEBUG diff --git a/tests/shapes/Makefile b/tests/shapes/Makefile index c53da2b5..3fbe92de 100644 --- a/tests/shapes/Makefile +++ b/tests/shapes/Makefile @@ -15,8 +15,6 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - all: \ qc_monolithic.ttl $(MAKE) \ @@ -51,7 +49,7 @@ check: \ $(MAKE) \ --directory examples_uco_qc \ check - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pytest \ --log-level=DEBUG \ --verbose \ @@ -77,12 +75,12 @@ clean: qc_monolithic.ttl qc_monolithic.ttl: \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/.lib.done.log \ uco-closure-qc.ttl \ uco-qc.ttl rm -f __$@ _$@ - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && rdfpipe \ --output-format turtle \ uco-closure-qc.ttl \ diff --git a/tests/shapes/examples_uco_owl/Makefile b/tests/shapes/examples_uco_owl/Makefile index 0b0f53ea..4f1a014e 100644 --- a/tests/shapes/examples_uco_owl/Makefile +++ b/tests/shapes/examples_uco_owl/Makefile @@ -15,9 +15,7 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - -shapes_srcdir := $(tests_srcdir)/shapes +shapes_srcdir := $(top_srcdir)/tests/shapes sample_ttls := $(wildcard *_PASS.ttl *_XFAIL.ttl) @@ -37,10 +35,10 @@ all: \ # characteristics. %_validation.ttl: \ %.ttl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(top_srcdir)/ontology/owl/owl.ttl rm -f __$@ _$@ - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --allow-warnings \ --data-file-format turtle \ diff --git a/tests/shapes/examples_uco_qc/Makefile b/tests/shapes/examples_uco_qc/Makefile index a04265f2..6e61d1c2 100644 --- a/tests/shapes/examples_uco_qc/Makefile +++ b/tests/shapes/examples_uco_qc/Makefile @@ -15,9 +15,7 @@ SHELL := /bin/bash top_srcdir := $(shell cd ../../.. ; pwd) -tests_srcdir := $(top_srcdir)/tests - -shapes_srcdir := $(tests_srcdir)/shapes +shapes_srcdir := $(top_srcdir)/tests/shapes sample_ttls := $(wildcard *_PASS.ttl *_XFAIL.ttl) @@ -37,10 +35,10 @@ all: \ # characteristics. %_validation.ttl: \ %.ttl \ - $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.venv.done.log \ $(shapes_srcdir)/qc_monolithic.ttl rm -f __$@ _$@ - source $(tests_srcdir)/venv/bin/activate \ + source $(top_srcdir)/venv/bin/activate \ && pyshacl \ --allow-warnings \ --data-file-format turtle \