diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7a2e7ef --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,66 @@ +name: Continuous Integration + +on: + push: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - '3.9' + - '3.12' + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Dependencies + run: | + pip -q install poetry pre-commit rdflib + poetry install + + - name: Pre-commit Checks + run: pre-commit run --all-files + + - name: Type Checking + run: poetry run mypy case_mapping example.py tests + + - name: Unit Tests + run: poetry run pytest --doctest-modules + + - name: Run Example + run: poetry run python example.py > case.jsonld + + # Ensure that the example output is a valid CASE JSON-LD graph + - name: CASE Export Validation + uses: kchason/case-validation-action@v2.9.0 + with: + case-path: ./ + case-version: "case-1.3.0" + extension-filter: "jsonld" + + - name: Convert example + run: | + rdfpipe --output-format turtle case.jsonld > case.ttl + test 0 -eq $(grep 'file:' case.ttl | wc -l) || (echo "ERROR:ci.yml:Some graph IRIs do not supply a resolving prefix. Look for the string 'file:///' in the file case.ttl (created by running 'make check') to see these instances." >&2 ; exit 1) + + # Always build the package as a sanity check to ensure no issues with the build system + # exist as part of the CI process + - name: Build Package + run: | + poetry build + + # Only push to PyPi when a tag is created starting with 'v' + - name: Push to PyPi + if: startsWith(github.ref, 'refs/tags/v') + run: | + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish diff --git a/.gitmodules b/.gitmodules index 6152477..a57eefa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "CASE-Mapping-Python"] - path = dependencies/CASE_Mapping_Python - url = https://github.com/casework/CASE-Mapping-Python.git + path = dependencies + url = https://github.com/casework/CASE-Mapping-Python diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a4ee4f8 --- /dev/null +++ b/Makefile @@ -0,0 +1,121 @@ +#!/usr/bin/make -f + +# Portions of this file contributed by NIST are governed by the +# following statement: +# +# This software was developed at the National Institute of Standards +# and Technology by employees of the Federal Government in the course +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. +# +# We would appreciate acknowledgement if the software is used. + +# Purpose: +# +# This Makefile assists with local development, mimicking the test suite +# run as part of continuous integration. + +SHELL := /bin/bash + +PYTHON3 ?= python3 + +top_srcdir := $(shell pwd) + +example_source_files := \ + $(wildcard case_mapping/*.py) \ + $(wildcard case_mapping/*/*.py) \ + example.py + +all: \ + .venv-pre-commit/var/.pre-commit-built.log \ + case.jsonld + +.PHONY: \ + check-mypy + +# This virtual environment is meant to be built once and then persist, even through 'make clean'. +# If a recipe is written to remove this flag file, it should first run `pre-commit uninstall`. +.venv-pre-commit/var/.pre-commit-built.log: + rm -rf .venv-pre-commit + test -r .pre-commit-config.yaml \ + || (echo "ERROR:Makefile:pre-commit is expected to install for this repository, but .pre-commit-config.yaml does not seem to exist." >&2 ; exit 1) + $(PYTHON3) -m venv \ + .venv-pre-commit + source .venv-pre-commit/bin/activate \ + && pip install \ + --upgrade \ + pip \ + setuptools \ + wheel + source .venv-pre-commit/bin/activate \ + && pip install \ + pre-commit + source .venv-pre-commit/bin/activate \ + && pre-commit install + mkdir -p \ + .venv-pre-commit/var + touch $@ + +.venv.done.log: \ + .github/workflows/ci.yml \ + pyproject.toml + rm -rf venv + $(PYTHON3) -m venv \ + venv + source venv/bin/activate \ + && pip install case-utils poetry + source venv/bin/activate && poetry install dependencies/CASE-Mapping-Python + source venv/bin/activate \ + && poetry install + touch $@ + +case.jsonld: \ + .venv.done.log \ + drafting.ttl \ + $(example_source_files) + export CDO_DEMO_NONRANDOM_UUID_BASE="$(top_srcdir)" \ + && source venv/bin/activate \ + && poetry run python example.py \ + > _$@ + source venv/bin/activate \ + && case_validate \ + --ontology-graph drafting.ttl \ + _$@ + mv _$@ $@ + +case.ttl: \ + case.jsonld + source venv/bin/activate \ + && rdfpipe \ + --output-format turtle \ + $< \ + > _$@ + source venv/bin/activate \ + && case_validate \ + _$@ + @# In instances where graph nodes omit use of a prefix, a 'default' prefix-base is used that incorporates the local directory as a file URL. This is likely to be an undesired behavior, so for the generated example JSON-LD in the top source directory, check that "file:///" doesn't start any of the graph individuals' IRIs. + test \ + 0 \ + -eq \ + $$(grep 'file:' _$@ | wc -l) \ + || ( echo "ERROR:Makefile:Some graph IRIs do not supply a resolving prefix. Look for the string 'file:///' in the file _$@ to see these instances." >&2 ; exit 1) + mv _$@ $@ + +check: \ + all \ + check-mypy \ + case.ttl + source venv/bin/activate \ + && poetry run pytest --doctest-modules + +check-mypy: \ + .venv.done.log + source venv/bin/activate \ + && mypy \ + case_mapping \ + example.py \ + tests diff --git a/UFEDtoJSON.py b/UFEDtoJSON.py index ca7c384..d900a67 100644 --- a/UFEDtoJSON.py +++ b/UFEDtoJSON.py @@ -5,7 +5,7 @@ import re import sys #from UFED_case_generator import * -from dependencies.CASE_Mapping_Python.case_mapping import base, case, drafting, uco +from dependencies.case_mapping import base, case, drafting, uco from datetime import datetime, date from typing import Dict, List, Optional, Union @@ -701,9 +701,10 @@ def __generateTraceDevice(self, deviceMAC, deviceSN, deviceModel, def __generate_trace_cookie(self, cookie_id, cookie_status, cookie_source, cookie_name, cookie_path, cookie_domain, cookie_creationTime, cookie_lastAccessedTime, cookie_expiry): - id_app = None + id_app = None if cookie_source.strip() != "": id_app = self.__check_application_name(cookie_source.strip()) + print(f"cookie application={cookie_source.strip()}, id_app={id_app}") cookie_creationTime = self.cleanDate(cookie_creationTime) cookie_lastAccessedTime = self.cleanDate(cookie_lastAccessedTime)