From 8ac0f689f7e111a1fc340584eede3c16485d40a2 Mon Sep 17 00:00:00 2001 From: dzaslavskiy Date: Thu, 19 May 2022 16:56:32 -0400 Subject: [PATCH] initial gpo service (#1) * initial gpo service * PR fixes refactor upload function fix requirements versions fix log level var fix file timestamp * add tzdata for dev on windows * create gpo schema if absent --- .bandit | 2 + .cfignore | 10 ++ .codeclimate.yml | 5 + .github/dependabot.yaml | 18 ++++ .github/workflows/README.md | 38 +++++++ .github/workflows/codeql-analysis.yaml | 52 +++++++++ .github/workflows/deploy.yaml | 41 ++++++++ .github/workflows/python-checks.yaml | 46 ++++++++ .github/workflows/stale-items.yaml | 23 ++++ .github/workflows/unit-tests.yaml | 36 +++++++ .gitignore | 3 + .pre-commit-config.yaml | 12 +++ .pylintrc | 2 + CONTRIBUTING.md | 37 +++++++ LICENSE.md | 33 ++++++ README.md | 82 ++++++++++++++- SECURITY.md | 32 ++++++ gpo/__init__.py | 0 gpo/crud.py | 41 ++++++++ gpo/database.py | 22 ++++ gpo/main.py | 140 +++++++++++++++++++++++++ gpo/models.py | 48 +++++++++ gpo/schemas.py | 45 ++++++++ gpo/settings.py | 26 +++++ manifest.yaml | 17 +++ requirements-dev.txt | 7 ++ requirements.txt | 7 ++ tests/test_gpo.py | 1 + vars.yaml | 3 + 29 files changed, 828 insertions(+), 1 deletion(-) create mode 100644 .bandit create mode 100644 .cfignore create mode 100644 .codeclimate.yml create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/codeql-analysis.yaml create mode 100644 .github/workflows/deploy.yaml create mode 100644 .github/workflows/python-checks.yaml create mode 100644 .github/workflows/stale-items.yaml create mode 100644 .github/workflows/unit-tests.yaml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .pylintrc create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md create mode 100644 SECURITY.md create mode 100644 gpo/__init__.py create mode 100644 gpo/crud.py create mode 100644 gpo/database.py create mode 100644 gpo/main.py create mode 100644 gpo/models.py create mode 100644 gpo/schemas.py create mode 100644 gpo/settings.py create mode 100644 manifest.yaml create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 tests/test_gpo.py create mode 100644 vars.yaml diff --git a/.bandit b/.bandit new file mode 100644 index 0000000..8787a99 --- /dev/null +++ b/.bandit @@ -0,0 +1,2 @@ +[bandit] +exclude: gpo/tests, .venv/ diff --git a/.cfignore b/.cfignore new file mode 100644 index 0000000..80fbfe5 --- /dev/null +++ b/.cfignore @@ -0,0 +1,10 @@ +*.md +.venv/ +gpo/__pycache__/ +.bandit +.codeclimate.yml +.github +.pre-commit-config.yaml +requirements-dev.txt +tests/ +vars.yaml diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..1240cdf --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,5 @@ +--- +version: "2" +plugins: + bandit: + enabled: true diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..a195221 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + labels: + - dependencies + - python + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + labels: + - dependencies + - github-actions diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..4c70917 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,38 @@ +# GitHub Actions CI/CD workflows + +## Python Checks +The Python-Checks workflow will run a series of checks on the python code +in this repository. + +### Bandit +The Bandit workflow will run the Bandit security linter tool against this +project. A failed run indicates that Bandit found at least one vulnerability. + +### Black +The workflow outlined in `black.yml` checks to ensure that the Python style +for this project is consistent and fully implemented in all Python files. +For more information about this workflow, see +https://black.readthedocs.io/en/stable/github_actions.html + +## CodeQL-Analysis +The codeql-analysis workflow the CodeQL semantic code analysis engine to help +find security issues very early on in the development process. See +[CodeQL](https://securitylab.github.com/tools/codeql) for more details. + +## Deploy +Deploys the project to the correct GIVE environment within Cloud.gov. The +deploy workflow will run unit-tests and only deploy if those test are +successful. Deployment will also only be triggered in the 18F repository. This +will prevent forks from needlessly running workflows that will always fail +(forks won't be able to authenticate into the dev environment). + +## Stale Items +The stale-items workflow will run once per day and mark issues and PR's as +stale if they have not seen any activity over the last 30 days. After being +marked stale for 5 days, the workflow will close the item. + +## Unit Tests +The unit-tests workflow will install the project runtime dependencies and run +the unit test suite against the code. This workflow is used to run unit tests +for the application against pull requests before merging takes place. Additional +unit testing will take place on merging. diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 0000000..a8906b3 --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,52 @@ +--- +name: "CodeQL" + +on: + push: + branches: [main] + paths-ignore: + - '**.md' # All markdown files in the repository + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + paths-ignore: + - '**.md' + schedule: + # weekly run at arbitrary time + - cron: '43 22 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: ['python'] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java) + # If this step fails, then remove it and run the build manually. See below + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + # - run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..9d07f7b --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,41 @@ +--- +# This workflow will run unit tests and deploy the application to a +# target environment + +name: Deploy + +on: + push: + branches: + - main + tags: + - "*" + paths-ignore: + - "**.md" # All markdown files in the repository + +jobs: + unit-test: + uses: 18F/identity-idva-gpo/.github/workflows/unit-tests.yaml@main + + deploy: + if: github.repository_owner == '18F' + needs: unit-test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: 18F/identity-idva-cf-setup@v2 + id: cf-setup + with: + cf-username: ${{ secrets.CF_USERNAME }} + cf-password: ${{ secrets.CF_PASSWORD }} + cf-org: ${{ secrets.CF_ORG }} + + - name: Deploy application + run: cf push --vars-file vars.yaml + --var ENVIRONMENT=${{ steps.cf-setup.outputs.target-environment }} + --var GPO_USERNAME=${{ secrets.GPO_USERNAME }} + --var GPO_PASSWORD=${{ secrets.GPO_PASSWORD }} + --var GPO_HOST=${{ secrets.GPO_HOST }} + --var GPO_HOSTKEY=${{ secrets.GPO_HOSTKEY }} + --strategy rolling diff --git a/.github/workflows/python-checks.yaml b/.github/workflows/python-checks.yaml new file mode 100644 index 0000000..3af476d --- /dev/null +++ b/.github/workflows/python-checks.yaml @@ -0,0 +1,46 @@ +--- +# This workflow will run the Black Python formatter as well as the +# Bandit security linter. See the following pages for details: +# See https://black.readthedocs.io/en/stable/github_actions.html +# https://github.com/PyCQA/bandit +name: Python-Checks + +on: + push: + branches: + - main + paths: + - '**.py' # All python files in the repository + pull_request: + paths: + - '**.py' + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - uses: psf/black@stable + + bandit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Scan + run: | + pip install -r requirements-dev.txt + bandit --exclude ./.venv/,./tests -r . diff --git a/.github/workflows/stale-items.yaml b/.github/workflows/stale-items.yaml new file mode 100644 index 0000000..9d2e4ea --- /dev/null +++ b/.github/workflows/stale-items.yaml @@ -0,0 +1,23 @@ +--- +name: 'Stale-Items' +on: + schedule: + # daily run at arbitrary time + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v4 + with: + stale-issue-message: >- + This issue has been automatically marked as stale because it has + not had any activity in the last 30 days. Remove stale label or + comment or this will be closed in 5 days. + stale-pr-message: >- + This issue has been automatically marked as stale because it has + not had any activity in the last 30 days. Remove stale label or + comment or this will be closed in 5 days. + days-before-stale: 30 + days-before-close: 5 diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml new file mode 100644 index 0000000..e0b2b4c --- /dev/null +++ b/.github/workflows/unit-tests.yaml @@ -0,0 +1,36 @@ +--- +# This workflow will install Python dependencies and run tests so that +# unit tests can be run against pull requests. + +name: Unit-Tests + +on: + pull_request: + paths-ignore: + - '**.md' # All markdown files in the repository + workflow_call: + +jobs: + unit-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest + - name: Test with Pytest unit tests + run: | + export DEBUG=True + python -m pytest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..213b2cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.venv +__pycache__ +.pytest_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b3998f3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +--- +repos: + - repo: https://github.com/psf/black + rev: 22.3.0 # Update with 'pre-commit autoupdate' + hooks: + - id: black + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + exclude: tests diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..063d3ad --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[MASTER] +extension-pkg-whitelist=pydantic \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..419ae8f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Welcome! + +We're so glad you're thinking about contributing to a +[open source project of the U.S. government](https://code.gov/)! If you're +unsure about anything, just ask -- or submit the issue or pull request anyway. +The worst that can happen is you'll be politely asked to change something. We +love all friendly contributions. + +We encourage you to read this project's CONTRIBUTING policy (you are here), its +[LICENSE](LICENSE.md), and its [README](README.md). + +## Policies + +We want to ensure a welcoming environment for all of our projects. Our staff +follow the [TTS Code of Conduct](https://18f.gsa.gov/code-of-conduct/) and +all contributors should do the same. + +We adhere to the +[18F Open Source Policy](https://github.com/18f/open-source-policy). If you +have any questions, just [shoot us an email](mailto:18f@gsa.gov). + +As part of a U.S. government agency, the General Services Administration +(GSA)’s Technology Transformation Services (TTS) takes seriously our +responsibility to protect the public’s information, including financial and +personal information, from unwarranted disclosure. For more information about +security and vulnerability disclosure for our projects, please read our +[18F Vulnerability Disclosure Policy](https://18f.gsa.gov/vulnerability-disclosure-policy/). + +## Public domain + +This project is in the public domain within the United States, and copyright +and related rights in the work worldwide are waived through the +[CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +All contributions to this project will be released under the CC0 dedication. By +submitting a pull request or issue, you are agreeing to comply with this waiver +of copyright interest. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..5f3ccbe --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,33 @@ +# License + +As a work of the [United States government](https://www.usa.gov/), this project +is in the public domain within the United States of America. + +Additionally, we waive copyright and related rights in the work worldwide +through the CC0 1.0 Universal public domain dedication. + +## CC0 1.0 Universal Summary + +This is a human-readable summary of the +[Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode). + +### No Copyright + +The person who associated a work with this deed has dedicated the work to the +public domain by waiving all of their rights to the work worldwide under +copyright law, including all related and neighboring rights, to the extent +allowed by law. + +You can copy, modify, distribute, and perform the work, even for commercial +purposes, all without asking permission. + +### Other Information + +In no way are the patent or trademark rights of any person affected by CC0, nor +are the rights that other persons may have in the work or in how the work is +used, such as publicity or privacy rights. + +Unless expressly stated otherwise, the person who associated a work with this +deed makes no warranties about the work, and disclaims liability for all uses +of the work, to the fullest extent permitted by applicable law. When using or +citing the work, you should not imply endorsement by the author or the affirmer. diff --git a/README.md b/README.md index 0602ebc..51cb4bd 100644 --- a/README.md +++ b/README.md @@ -1 +1,81 @@ -# identity-idva-gpo \ No newline at end of file +![Tests](https://github.com/18F/identity-idva-gpo/workflows/Unit-Tests/badge.svg) +[![Maintainability](https://api.codeclimate.com/v1/badges/7a72205acec6d179707c/maintainability)](https://codeclimate.com/github/18F/identity-idva-gpo/maintainability) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit) +![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat) + +# IDVA GPO Microservice +The GPO microservice is a Python [FastAPI](https://fastapi.tiangolo.com/) +application that exposes an API for sending letters to users using the services of the GPO. + +## CI/CD Workflows with GitHub Actions +The most up-to-date information about the CI/CD flows for this repo can be found in the +[GitHub workflows directory](https://github.com/18F/identity-idva-gpo/tree/main/.github/workflows) + +## Building Locally + +### Pre-requisites +Make sure you have the following installed if you intend to build the project locally. +- [Python 3](https://www.python.org/) (Check [runtime.txt](runtime.txt) for exact version) +- [CloudFoundry CLI](https://docs.cloudfoundry.org/cf-cli/) + +### Development Setup +To set up your environment, run the following commands (or the equivalent +commands if not using a bash-like terminal): +```shell +# Clone the project +git clone https://github.com/18F/identity-idva-gpo +cd identity-idva-gpo + +# Set up Python virtual environment +python3.9 -m venv .venv +source venv/bin/activate +# .venv\Scripts\Activate.ps1 on Windows + +# Install dependencies and pre-commit hooks +python -m pip install -r requirements-dev.txt +pre-commit install +``` + +### Running the application +After completing [development setup](#development-setup) application locally with: +```shell +python -m pytest # NOTE that without DEBUG=True, local unit tests will fail +uvicorn gpo.main:app +``` + +### Viewing API Endpoints and documentation +Documentation can be viewed locally by running the application and visiting +http://127.0.0.1:8000/redoc + +### Deploying to Cloud.gov during development +All deployments require having the correct Cloud.gov credentials in place. If +you haven't already, visit [Cloud.gov](https://cloud.gov) and set up your +account and CLI. + +*manifest.yml* file contains the deployment configuration for cloud.gov, and expects +a vars.yaml file that includes runtime variables referenced. For info, see +[cloud foundry manifest files reference](https://docs.cloudfoundry.org/devguide/deploy-apps/manifest-attributes.html) + +Running the following `cf` command will deploy the application to cloud.gov +```shell +cf push --vars-file vars.yaml \ + --var ENVIRONMENT= \ + --var GPO_USERNAME= \ + --var GPO_PASSWORD= \ + --var GPO_HOST= \ + --var GPO_HOSTKEY= \ +``` + +## Public domain + +This project is in the worldwide [public domain](LICENSE.md). As stated in +[CONTRIBUTING](CONTRIBUTING.md): + +> This project is in the public domain within the United States, and copyright +and related rights in the work worldwide are waived through the +[CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). +> +> All contributions to this project will be released under the CC0 dedication. +By submitting a pull request, you are agreeing to comply with this waiver of +copyright interest. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..aec2ce2 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,32 @@ +# Security Policy + +The GIVE team takes the security of our software seriously. If you believe +you have found a security vulnerability in any GIVE repository, please report +it to us as described below. + +## Supported Versions + +GIVE will only ever be providing security updates for the most recent +version of its software. This should always be available as the most recently +tagged version within this repository. We will not be providing security +updates to versions that are not currently released into production. + +## Reporting a Vulnerability + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them by emailing email give@gsa.gov. You should receive +a response within 72 hours. If for some reason you do not, please follow up via +email to ensure we've received your original message. + +Please include the requested information listed below, or as much as you can +provide, to help us better understand the nature and scope of the possible issue: + +* Issue type (e.g. buffer overflow, SQL injection, cross-site scripting, etc) +* Full paths of source file(s) related to the manifestation of the issue +* Location of the effected source code (direct URL or tag/branch/commit) +* Step-by-step instructions on how to reproduce the issue +* Proof-of-concept or exploit code (if possible) +* Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. diff --git a/gpo/__init__.py b/gpo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gpo/crud.py b/gpo/crud.py new file mode 100644 index 0000000..c1fa397 --- /dev/null +++ b/gpo/crud.py @@ -0,0 +1,41 @@ +""" +CRUD operations for gpo µservice +""" +from sqlalchemy import select +from sqlalchemy.orm import Session + +from . import models, schemas + + +def get_letters( + session: Session, skip: int = 0, limit: int = 1000 +) -> list[models.Letter]: + """ + get letters + """ + return ( + session.execute(select(models.Letter).offset(skip).limit(limit)).scalars().all() + ) + + +def delete_letters(session: Session, letters: list[models.Letter]): + """ + delete letters by id + """ + + for letter in letters: + session.delete(letter) + + session.commit() + return + + +def create_letter(session: Session, letter: schemas.LetterCreate) -> models.Letter: + """ + create letter + """ + db_item = models.Letter(**letter.dict()) + session.add(db_item) + session.commit() + session.refresh(db_item) + return db_item diff --git a/gpo/database.py b/gpo/database.py new file mode 100644 index 0000000..79bb2b5 --- /dev/null +++ b/gpo/database.py @@ -0,0 +1,22 @@ +""" +Db Connection for GPO +""" +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from sqlalchemy.schema import CreateSchema +from gpo import settings + +# Sqlalchemy requires 'postgresql' as the protocol +uri = settings.DB_URI.replace("postgres://", "postgresql://", 1) + +schema_name = "gpo" + +engine = create_engine(uri, connect_args={"options": f"-csearch_path={schema_name}"}) + +if not engine.dialect.has_schema(engine, schema_name): + engine.execute(CreateSchema(schema_name)) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine, future=True) + +Base = declarative_base() diff --git a/gpo/main.py b/gpo/main.py new file mode 100644 index 0000000..f25f2dd --- /dev/null +++ b/gpo/main.py @@ -0,0 +1,140 @@ +""" +GPO Microservice FastAPI Web App. +""" + +import datetime +from io import StringIO +import logging +import math +import csv +from base64 import b64decode +from zoneinfo import ZoneInfo +from fastapi import FastAPI, Depends, Response +import paramiko +from starlette_prometheus import metrics, PrometheusMiddleware +from sqlalchemy.orm import Session + +from . import settings, crud, models, schemas +from .database import SessionLocal, engine + +# pylint: disable=invalid-name + +models.Base.metadata.create_all(bind=engine) + +app = FastAPI() + +app.add_middleware(PrometheusMiddleware) +app.add_route("/metrics/", metrics) + +logging.getLogger().setLevel(settings.LOG_LEVEL) + +DEST_FILE_DIR = "gsa_order" + + +def get_db(): + """ + get db connection + """ + db = SessionLocal() + try: + yield db + finally: + db.close() + + +def write(file: StringIO | paramiko.SFTPFile, letters: list[models.Letter]): + """ + Write letter data to file + + 001|955 + 002|FAKEY MCFAKERSON|123 FAKE ST||GREAT FALLS|MT|59010|Q82GZBP71C|January 11, 2022|January 21, 2022|Example Sinatra App|https://secure.login.gov + 003|JIMMY TESTERSON|456 FAKE RD|Apt 1|FAKE CITY|CA|40323|4WVGPG0Z5Z|January 11, 2022|January 21, 2022|Example Rails App|https://secure.login.gov + 004|MIKE MCMOCKDATA|789 TEST AVE||FALLS CHURCH|VA|20943|5HVFT58WJ0|January 11, 2022|January 21, 2022|Example Java App|https://secure.login.gov + 005|... + ... + 956|... + + """ + + writer = csv.writer(file, delimiter="|") + numLines = len(letters) + 1 + + # number of digits in the row index + numIndexDigits = math.trunc(math.log(numLines, 10)) + 1 + # row index width is a least 2 and is enough to fit number of digits in the index + width = max(2, numIndexDigits) + + header = [f"{1:0{width}}", len(letters)] + writer.writerow(header) + + for i, val in enumerate(letters, start=2): + + # row with index + row = val.as_list(f"{i:0{width}}") + + # remove any pipes that might be in the data + sanitized_row = map(lambda x: x.replace("|", ""), row) + writer.writerow(sanitized_row) + + +@app.post("/upload") +def upload_batch(db: Session = Depends(get_db)): + """ + Upload letter data file to GPO server. + """ + + letters = crud.get_letters(db) + count = len(letters) + + if count == 0: + logging.info("No letters in db. Nothing to upload.") + return Response(count) + + if settings.DEBUG: + output = StringIO() + write(output, letters) + logging.debug(output.getvalue()) + crud.delete_letters(db, letters) + return Response(count) + + with paramiko.SSHClient() as ssh_client: + host_key = paramiko.RSAKey(data=b64decode(settings.GPO_HOSTKEY)) + ssh_client.get_host_keys().add(settings.GPO_HOST, "ssh-rsa", host_key) + ssh_client.connect( + settings.GPO_HOST, + username=settings.GPO_USERNAME, + password=settings.GPO_PASSWORD, + ) + with ssh_client.open_sftp() as sftp: + sftp.chdir(DEST_FILE_DIR) + date = datetime.datetime.now(ZoneInfo("US/Eastern")).strftime("%Y%m%d") + try: + with sftp.open(f"idva-{date}-0.psv", mode="wx") as file: + write(file, letters) + except PermissionError as err: + logging.error( + "Error Creating file likely because it already exists: %s", err + ) + return Response(status_code=500) + + crud.delete_letters(db, letters) + + return Response(count) + + +@app.post("/letters", response_model=schemas.Letter) +def queue_letter(letter: schemas.LetterCreate, db: Session = Depends(get_db)): + """ + Add a letter to the queue + """ + + return crud.create_letter(db, letter) + + +@app.get("/letters") +def count_letter(db: Session = Depends(get_db)): + """ + Get count of letter in the queue + """ + + return len(crud.get_letters(db)) diff --git a/gpo/models.py b/gpo/models.py new file mode 100644 index 0000000..d2b9329 --- /dev/null +++ b/gpo/models.py @@ -0,0 +1,48 @@ +""" +Models for GPO +""" + +from sqlalchemy import Column, Integer, String +from .database import Base + +# pylint: disable=too-few-public-methods +class Letter(Base): + """ + DB model for Letter + """ + + __tablename__ = "letters" + + id = Column(Integer, primary_key=True, index=True) + name = Column(String) + address = Column(String) + address2 = Column(String) + city = Column(String) + state = Column(String) + zip = Column(String) + code = Column(String) + date = Column(String) + expiry = Column(String) + app = Column(String) + url = Column(String) + + def as_list(self, index: str) -> list: + """ + Convert Letter to a list of fields + + index is a formatted line number + """ + return [ + index, + self.name, + self.address, + self.address2, + self.city, + self.state, + self.zip, + self.code, + self.date, + self.expiry, + self.app, + self.url, + ] diff --git a/gpo/schemas.py b/gpo/schemas.py new file mode 100644 index 0000000..c599e12 --- /dev/null +++ b/gpo/schemas.py @@ -0,0 +1,45 @@ +""" +REST api models for gpo µservice +""" +from pydantic import BaseModel + +# pylint: disable=too-few-public-methods + + +class LetterBase(BaseModel): + """ + base letter model + """ + + name: str + address: str + address2: str + city: str + state: str + zip: str + code: str + date: str + expiry: str + app: str + url: str + + +class LetterCreate(LetterBase): + """ + create letter model + """ + + +class Letter(LetterBase): + """ + read letter model + """ + + id: int + + class Config: + """ + config for model + """ + + orm_mode = True diff --git a/gpo/settings.py b/gpo/settings.py new file mode 100644 index 0000000..0b5becb --- /dev/null +++ b/gpo/settings.py @@ -0,0 +1,26 @@ +""" +Configuration for the GPO microservice settings. +Context is switched based on if the app is in debug mode. +""" +import logging +import os +import json +import sys + +# SECURITY WARNING: don't run with debug turned on in production! +# DEBUG set is set to True if env var is "True" +DEBUG = os.getenv("DEBUG", "False") == "True" + +LOG_LEVEL = os.getenv("LOG_LEVEL", logging.getLevelName(logging.INFO)) + +GPO_USERNAME = os.getenv("GPO_USERNAME") +GPO_PASSWORD = os.getenv("GPO_PASSWORD") +GPO_HOST = os.getenv("GPO_HOST") +GPO_HOSTKEY = os.getenv("GPO_HOSTKEY") + +try: + db_uri = json.loads(os.getenv("VCAP_SERVICES"))["aws-rds"][0]["credentials"]["uri"] +except (json.JSONDecodeError, KeyError, TypeError): + sys.exit("Failed to load the aws-rds uri from VCAP_SERVICES") + +DB_URI = os.getenv("DB_URI", db_uri) diff --git a/manifest.yaml b/manifest.yaml new file mode 100644 index 0000000..385ce1d --- /dev/null +++ b/manifest.yaml @@ -0,0 +1,17 @@ +--- +applications: + - name: gpo + routes: + - route: identity-idva-gpo-((ENVIRONMENT)).apps.internal + memory: ((MEMORY)) + instances: ((INSTANCES)) + buildpacks: + - python_buildpack + command: uvicorn gpo.main:app --host 0.0.0.0 --port $PORT + services: + - sk-postgres + env: + GPO_USERNAME: ((GPO_USERNAME)) + GPO_PASSWORD: ((GPO_PASSWORD)) + GPO_HOST: ((GPO_HOST)) + GPO_HOSTKEY: ((GPO_HOSTKEY)) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..fb885ee --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +-r requirements.txt +pre-commit +black +pylint +bandit +pytest +tzdata diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e7afcb9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.78.0 +uvicorn==0.17.6 +pydantic==1.9.0 +starlette-prometheus==0.9.0 +paramiko==2.11.0 +sqlalchemy==1.4.36 +psycopg2==2.9.3 \ No newline at end of file diff --git a/tests/test_gpo.py b/tests/test_gpo.py new file mode 100644 index 0000000..2e4e049 --- /dev/null +++ b/tests/test_gpo.py @@ -0,0 +1 @@ +""" GPO API unit tests """ diff --git a/vars.yaml b/vars.yaml new file mode 100644 index 0000000..d10ff05 --- /dev/null +++ b/vars.yaml @@ -0,0 +1,3 @@ +--- +INSTANCES: 2 +MEMORY: 128M