V0.11 Created module with content migrated from rcsb.db

.gitignore

@@ -2,7 +2,10 @@
 __pycache__/
 *.py[cod]
 *$py.class
-
+test-output/
+CACHE/
+LOGTOX
+.tox/
 # C extensions
 *.so
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "rcsb/mock-data"]
+	path = rcsb/mock-data
+	url = https://github.com/rcsb/py-rcsb_mock-data.git

HISTORY.txt

@@ -0,0 +1,3 @@
+# File: HISTORY.txt
+#
+14-Feb-2021 - V0.11 Created module with content migrated from rcsb.db

MANIFEST.in

@@ -0,0 +1,5 @@
+#
+# File: py-rcsb_utils_chemref/MANIFEST.in
+#
+include HISTORY.txt
+#

README.md

@@ -1,2 +1,43 @@
-# py-rcsb_utils_dictionary
-RCSB dictionary method utilities
+# RCSB Dictionary Method Utilities
+
+## A collection of Python Dictionary Method Utilities
+
+## Introduction
+
+This module contains a collection of utility classes for defining and managing the execution of
+dynamic dictionary methods.
+
+### Installation
+
+Install via [pip](https://pypi.python.org/pypi/pip).
+
+```bash
+pip install rcsb.utils.dictionary
+```
+
+Or, to install from the source, download the library source software from the project repository:
+
+```bash
+
+git clone --recurse-submodules https://github.com/rcsb/py-rcsb_utils_dictionary.git
+
+```
+
+Optionally, run test suite (Python versions 3.7+) using
+[setuptools](https://setuptools.readthedocs.io/en/latest/) or
+[tox](http://tox.readthedocs.io/en/latest/example/platform.html):
+
+```bash
+python setup.py test
+
+or simply run
+
+tox
+```
+
+Installation is via the program [pip](https://pypi.python.org/pypi/pip).  To run tests
+from the source tree, the package must be installed in editable mode (i.e. -e):
+
+```bash
+pip install -e .
+```

azure-pipelines.yml

@@ -0,0 +1,45 @@
+# File: azure-pipelines.yml
+# Date: 4-Jul-2019 jdw Created pipeline
+#       8-Jul-2019 jdw add macos test/publish
+#       8-Jul-2019 jdw templatize
+#       9-Jul-2019 jdw add publish jobs
+#      21-Jul-2019 jdw update to Py38
+#      10-Mar-2020 jdw py38 only
+#      10-Dec-2020 jdw py39 only
+#
+name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+trigger:
+- master
+- utilsdict-*
+
+pr:
+- master
+
+schedules:
+- cron: "0 12 * * 0"
+  displayName: Weekly Sunday build
+  branches:
+    include:
+    - master
+  always: true
+
+jobs:
+  - template: azure-template-tox-job.yml
+    parameters: {tox: 'format_pep8', python: '3.9', os: 'linux'}
+  - template: azure-template-tox-job.yml
+    parameters: {tox: 'lint_pylint', python: '3.9', os: 'linux'}
+  - template: azure-template-tox-job.yml
+    parameters: {tox: 'test_coverage', python: '3.9', os: 'linux'}
+    #
+  - template: azure-template-tox-job.yml
+    parameters: {tox: 'py39', python: '3.9', os: 'linux'}
+  #
+  - template: azure-template-tox-job.yml
+    parameters: {tox: 'py39', python: '3.9', os: 'macos'}
+  #
+  #- template: azure-template-publish-job.yml
+  #  parameters: {tox: 'py39', python: '3.9', os: 'macos'}
+  - template: azure-template-publish-job.yml
+    parameters: {tox: 'py39', python: '3.9', os: 'linux'}
+  #

azure-template-publish-job.yml

@@ -0,0 +1,61 @@
+# File: azure-template-publish-job.yml
+# Date: 8-Jul-2019 jdw split out from original pipeline
+#
+# Update:
+# 6-Aug-2019 jdw push source wheels (universal) for linux
+#
+##
+parameters:
+  tox: ""
+  python: ""
+  os: "linux"
+  fixtures: ""
+
+jobs:
+- job: ${{ format('publish_{0}_{1}', parameters.tox, parameters.os) }}
+  pool:
+    ${{ if eq(parameters.os, 'macos') }}:
+      vmImage: 'macOS-10.15'
+    ${{ if eq(parameters.os, 'linux') }}:
+      vmImage: 'ubuntu-20.04'
+  dependsOn:
+  - ${{ format('build_test_{0}_{1}', parameters.tox, parameters.os) }}
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+  #
+  steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{ parameters.python }}
+        addToPath: true
+      displayName: setup python
+    #
+    #- checkout: self
+    #  submodules: true
+    #
+    - download: current
+      artifact: ${{ format('sw_{0}_{1}', parameters.tox,  parameters.os) }}
+
+    - download: current
+      artifact: ${{ format('sw_u_{0}_{1}', parameters.tox,  parameters.os) }}
+    #
+    - script: ls -lR $(Pipeline.Workspace)/${{ format('sw_{0}_{1}', parameters.tox, parameters.os) }}
+      displayName: "Listing of downloaded artifacts"
+    #
+    - script: python -m pip install --upgrade pip twine setuptools wheel
+      displayName: 'Install packaging tools'
+    #
+    - task: DownloadSecureFile@1
+      name: pypicred
+      displayName: 'Download PyPI credentials'
+      inputs:
+        secureFile: 'PYPIRC-AZURE'
+
+    - ${{ if startsWith(parameters.os, 'linux') }}:
+      - script: twine upload --verbose --skip-existing -r pypi  --config-file $(pypicred.secureFilePath) $(Pipeline.Workspace)/${{ format('sw_u_{0}_{1}', parameters.tox, parameters.os) }}/*
+        displayName: "Linux upload sdist and source wheel to PyPi ..."
+        continueOnError: true
+    #
+    - ${{ if startsWith(parameters.os, 'macos') }}:
+      - script: twine upload --verbose --skip-existing -r pypi  --config-file $(pypicred.secureFilePath) $(Pipeline.Workspace)/${{ format('sw_{0}_{1}', parameters.tox, parameters.os) }}/*
+        displayName: "Mac upload sdist and binary wheel to PyPi ..."
+        continueOnError: true

azure-template-tox-job.yml

@@ -0,0 +1,202 @@
+# File: azure-template-tox-job.yml
+# Date: 8-Jul-2019 jdw split out from original pipeline
+# Supports: fixtures=mysql,mongodb (linux)
+#
+# Updates:
+#  6-Aug-2019  jdw build source and binary wheels by default.
+# 13-Aug-2019  jdw export config support token prior to launching tox runner
+#
+##
+parameters:
+  tox: ""
+  python: ""
+  os: "linux"
+  fixtures: ""
+
+jobs:
+- job: ${{ format('build_test_{0}_{1}', parameters.tox, parameters.os) }}
+  timeoutInMinutes: 0
+  pool:
+    ${{ if eq(parameters.os, 'macos') }}:
+      vmImage: 'macOS-10.15'
+    ${{ if eq(parameters.os, 'linux') }}:
+      vmImage: 'ubuntu-20.04'
+
+  variables:
+    - group: py-shared-variables
+
+  steps:
+    #
+    # ensure the required Python versions are available
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{ parameters.python }}
+        addToPath: true
+      displayName: setup python
+    #
+    - checkout: self
+      submodules: true
+    #
+    - ${{ if startsWith(parameters.os, 'macos') }}:
+      - bash: |
+          set -e
+          ls -la /Applications/Xcode*
+          sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
+          which g++
+          c++ --version
+        displayName: "setup Xcode"
+      #
+      - script: which brew
+        displayName: 'Check package manager'
+      - script: brew install flex
+        displayName: 'Install flex'
+      - script: which flex
+        displayName: 'Check flex'
+      - script: brew install bison
+        displayName: 'Install bison'
+      - script: which bison
+        displayName: 'Check bison'
+    # ----------------------------------------------
+    - ${{ if startsWith(parameters.os, 'linux') }}:
+      - script: lsb_release -a
+        displayName: 'Ubuntu kernal version'
+      - script: which apt
+        displayName: 'Installing OS dependencies'
+      - script: apt-cache policy | grep http | awk '{print $2 $3}' | sort -u
+        displayName: 'Checking for repos'
+      #
+      - script: sudo apt-get update
+        displayName: 'update apt'
+      #- script: sudo apt-get upgrade
+      #  displayName: 'upgrade apt'
+      #- script: sudo apt-get update
+      #  displayName: 'update apt'
+      - script: sudo apt-get install flex
+        displayName: 'Install flex'
+      - script: sudo apt-get install bison
+        displayName: 'Install bison'
+  #
+    - ${{ if and(contains(parameters.fixtures, 'mysql'), startsWith(parameters.os, 'linux')) }}:
+      - bash: |
+          sudo apt-get install python3-dev mysql-server
+          sudo apt-get install default-libmysqlclient-dev
+          sudo apt-get install python-mysqldb
+          sudo apt list --installed | grep -i mysql
+        displayName: 'Install mysql development libraries'
+      - bash: |
+          echo "Retarting mysql service"
+          sudo systemctl restart mysql.service
+          mysql -V
+          mysql --user=root --password=root -e "use mysql; select * from user;"
+          #
+          echo "Try resetting password"
+          mysqladmin --user=root --password=root password 'ChangeMeSoon'
+          #
+          # mysql -u root  -p root -e "SET PASSWORD FOR root@'localhost' = PASSWORD(‘ChangeMeSoon’);"
+          # mysql -u root  -p root -e "FLUSH PRIVILEGES; update mysql.user set password=password('ChangeMeSoon') where user='root'; FLUSH PRIVILEGES;"
+          # UPDATE mysql.user SET Password=PASSWORD('ChangeMeSoon') WHERE User='root';
+
+          echo "Running preliminary mysql setup"
+          mysql --user=root --password=ChangeMeSoon <<_EOF_
+            DELETE FROM mysql.user WHERE User='';
+            DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
+            DROP DATABASE IF EXISTS test;
+            DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%';
+            FLUSH PRIVILEGES;
+          _EOF_
+          ps -ef | grep -i my
+          mysql --user=root --password=ChangeMeSoon -e "show databases;"
+          #
+        displayName: 'Start and configure mysql ...'
+      # -----
+
+    - ${{ if and(contains(parameters.fixtures, 'mongodb'), startsWith(parameters.os, 'linux')) }}:
+      # Mongo install
+      - script: |
+          sudo apt-get install gnupg wget
+          wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | sudo apt-key add -
+          # sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4
+          sudo apt list --installed | grep mongodb
+          echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.2.list
+          # echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.2.list
+          sudo apt-get update
+          sudo apt-get install -y mongodb-org
+          sudo apt list --installed | grep mongo
+        displayName: "Installing mongodb"
+      #
+      - script:  |
+          sudo service mongod start
+          sudo ss -tulpn
+        displayName: "Start Mongo service"
+      #
+    #
+    - script: "python -c \"import sys; print(sys.version); print(sys.executable)\""
+      displayName: show python information
+    #
+    - script: python -m pip install --upgrade pip tox
+      displayName: 'Install tools'
+      #
+    - script: pip install -r requirements.txt
+      displayName: 'Install dependencies'
+    #
+    #
+    - task: DownloadSecureFile@1
+      name: oelicense
+      displayName: 'Download OE license file'
+      inputs:
+        secureFile: 'oe_license.txt'
+
+    - ${{ if startsWith(parameters.tox, 'py') }}:
+      - script: |
+          export OE_LICENSE=$(oelicense.secureFilePath)
+          export CONFIG_SUPPORT_TOKEN_ENV=$(VAR_CONFIG_SUPPORT_TOKEN_ENV)
+          ${{ format('python -m tox -e {0}', parameters.tox) }}
+        displayName: 'Running tox task'
+    - ${{ if and(not(startsWith(parameters.tox, 'py')), startsWith(parameters.python, '3.9')) }}:
+      - script: |
+          export OE_LICENSE=$(oelicense.secureFilePath)
+          export CONFIG_SUPPORT_TOKEN_ENV=$(VAR_CONFIG_SUPPORT_TOKEN_ENV)
+          ${{ format('python -m tox -e {0}-py39', parameters.tox) }}
+        displayName: 'Running tox task'
+    - ${{ if and(not(startsWith(parameters.tox, 'py')), startsWith(parameters.python, '3.8')) }}:
+      - script: |
+          export OE_LICENSE=$(oelicense.secureFilePath)
+          export CONFIG_SUPPORT_TOKEN_ENV=$(VAR_CONFIG_SUPPORT_TOKEN_ENV)
+          ${{ format('python -m tox -e {0}-py38', parameters.tox) }}
+        displayName: 'Running tox task'
+    - ${{ if and(not(startsWith(parameters.tox, 'py')), startsWith(parameters.python, '3.7')) }}:
+      - script: |
+          export OE_LICENSE=$(oelicense.secureFilePath)
+          export CONFIG_SUPPORT_TOKEN_ENV=$(VAR_CONFIG_SUPPORT_TOKEN_ENV)
+          ${{ format('python -m tox -e {0}-py37', parameters.tox) }}
+        displayName: 'Running tox task'
+    - ${{ if and(not(startsWith(parameters.tox, 'py')), startsWith(parameters.python, '2.7')) }}:
+      - script: |
+          export OE_LICENSE=$(oelicense.secureFilePath)
+          export CONFIG_SUPPORT_TOKEN_ENV=$(VAR_CONFIG_SUPPORT_TOKEN_ENV)
+          ${{ format('python -m tox -e {0}-py27', parameters.tox) }}
+        displayName: 'Runing tox task'
+    #
+    #  Build artifacts if this is a test target (i.e. labeled as py##)
+    #
+    - ${{ if startsWith(parameters.tox, 'py') }}:
+      - script: pip install --upgrade pip twine setuptools wheel
+        displayName: "Acquire build tools"
+      - script: python setup.py  sdist --dist-dir "$(System.DefaultWorkingDirectory)/dist"
+        displayName: "Build source dist"
+      - script: python setup.py  bdist_wheel --dist-dir "$(System.DefaultWorkingDirectory)/dist"
+        displayName: "Build wheel"
+    #
+      - script: python setup.py  sdist --dist-dir "$(System.DefaultWorkingDirectory)/udist"
+        displayName: "Build source dist"
+      #
+      # Check the install artifacts
+      - script: ls -lR "$(System.DefaultWorkingDirectory)/dist" "$(System.DefaultWorkingDirectory)/udist"
+        displayName: "Listing of installed software"
+      #
+      - publish: $(System.DefaultWorkingDirectory)/dist
+        artifact: ${{ format('sw_{0}_{1}', parameters.tox, parameters.os) }}
+      #
+      - publish: $(System.DefaultWorkingDirectory)/udist
+        artifact: ${{ format('sw_u_{0}_{1}', parameters.tox, parameters.os) }}
+      #