diff --git a/.github/workflows/python-checks.yaml b/.github/workflows/python-checks.yaml index 6d28b54..71c2a1e 100644 --- a/.github/workflows/python-checks.yaml +++ b/.github/workflows/python-checks.yaml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: '3.10' - uses: psf/black@stable @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: '3.10' - uses: actions/cache@v3 with: @@ -40,6 +40,7 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }} restore-keys: | ${{ runner.os }}-pip- + - name: Scan run: | pip install -r requirements-dev.txt diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index c4d6659..d0e4cb8 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -15,21 +15,25 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/cache@v3 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: Set up Python 3.9 + + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: "3.10" + - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements-dev.txt pip install pytest + - name: Test with Pytest unit tests run: | export DEBUG=True diff --git a/gpo/api.py b/gpo/api.py new file mode 100644 index 0000000..6edf1f0 --- /dev/null +++ b/gpo/api.py @@ -0,0 +1,79 @@ +""" +gpo rest api +""" +import logging +from datetime import datetime +import io +import zoneinfo + +import fastapi +from sqlalchemy import orm + +from . import crud, schemas, settings, sftp, database + +log = logging.getLogger(__name__) + +router = fastapi.APIRouter() + + +def get_db(): + """ + get db connection + """ + db = database.SessionLocal() + try: + yield db + finally: + db.close() + + +@router.post("/upload", response_model=schemas.Count) +def upload_batch(session: orm.Session = fastapi.Depends(get_db)): + """ + Upload letter data file to GPO server. + """ + + letters = crud.get_letters_for_update(session) + count = len(letters) + + if count == 0: + log.info("No letters in db. Nothing to upload.") + return {"count": count} + + if settings.DEBUG: + output = io.StringIO() + sftp.write(output, letters) + log.debug(output.getvalue()) + crud.delete_letters(session, letters) + return {"count": count} + + date = datetime.now(zoneinfo.ZoneInfo("US/Eastern")).strftime("%Y%m%d") + file_name = f"idva-{date}-0.psv" + + try: + sftp.write_sftp(letters, settings, file_name, settings.DEST_FILE_DIR) + except sftp.SftpError: + return fastapi.Response(status_code=400) + + crud.delete_letters(session, letters) + log.info("Uploaded %i letter(s) as %s", count, file_name) + + return {"count": count} + + +@router.post("/letters", response_model=schemas.Letter) +def queue_letter( + letter: schemas.LetterCreate, session: orm.Session = fastapi.Depends(get_db) +): + """ + Add a letter to the queue + """ + return crud.create_letter(session, letter) + + +@router.get("/letters", response_model=schemas.Count) +def count_letter(session: orm.Session = fastapi.Depends(get_db)): + """ + Get count of letter in the queue + """ + return {"count": crud.count_letters(session)} diff --git a/gpo/crud.py b/gpo/crud.py index c1fa397..529eb0d 100644 --- a/gpo/crud.py +++ b/gpo/crud.py @@ -1,36 +1,53 @@ """ CRUD operations for gpo µservice """ -from sqlalchemy import select -from sqlalchemy.orm import Session +import sqlalchemy +from sqlalchemy import orm from . import models, schemas +def count_letters(session: orm.Session) -> int: + """ + count letters + """ + return session.scalar(sqlalchemy.select(sqlalchemy.func.count(models.Letter.id))) + + def get_letters( - session: Session, skip: int = 0, limit: int = 1000 + session: orm.Session, skip: int = 0, limit: int = 1000 ) -> list[models.Letter]: """ get letters """ - return ( - session.execute(select(models.Letter).offset(skip).limit(limit)).scalars().all() + statement = sqlalchemy.select(models.Letter).offset(skip).limit(limit) + return session.execute(statement).scalars().all() + + +def get_letters_for_update( + session: orm.Session, skip: int = 0, limit: int = 1000 +) -> list[models.Letter]: + """ + get letters with lock + """ + statement = ( + sqlalchemy.select(models.Letter).with_for_update().offset(skip).limit(limit) ) + return session.execute(statement).scalars().all() -def delete_letters(session: Session, letters: list[models.Letter]): +def delete_letters(session: orm.Session, letters: list[models.Letter]): """ - delete letters by id + delete list of letters by instance """ for letter in letters: session.delete(letter) session.commit() - return -def create_letter(session: Session, letter: schemas.LetterCreate) -> models.Letter: +def create_letter(session: orm.Session, letter: schemas.LetterCreate) -> models.Letter: """ create letter """ diff --git a/gpo/database.py b/gpo/database.py index 79bb2b5..bc65b9c 100644 --- a/gpo/database.py +++ b/gpo/database.py @@ -1,22 +1,18 @@ """ Db Connection for GPO """ -from sqlalchemy import create_engine -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker -from sqlalchemy.schema import CreateSchema -from gpo import settings +import sqlalchemy +from sqlalchemy import orm, schema -# Sqlalchemy requires 'postgresql' as the protocol -uri = settings.DB_URI.replace("postgres://", "postgresql://", 1) +from . import settings -schema_name = "gpo" +engine = sqlalchemy.create_engine( + settings.DB_URI, connect_args={"options": f"-csearch_path={settings.SCHEMA_NAME}"} +) -engine = create_engine(uri, connect_args={"options": f"-csearch_path={schema_name}"}) +if not engine.dialect.has_schema(engine, settings.SCHEMA_NAME): + engine.execute(schema.CreateSchema(settings.SCHEMA_NAME)) -if not engine.dialect.has_schema(engine, schema_name): - engine.execute(CreateSchema(schema_name)) - -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine, future=True) - -Base = declarative_base() +SessionLocal = orm.sessionmaker( + autocommit=False, autoflush=False, bind=engine, future=True +) diff --git a/gpo/main.py b/gpo/main.py index f25f2dd..8d62d46 100644 --- a/gpo/main.py +++ b/gpo/main.py @@ -1,140 +1,20 @@ """ GPO Microservice FastAPI Web App. """ - -import datetime -from io import StringIO import logging -import math -import csv -from base64 import b64decode -from zoneinfo import ZoneInfo -from fastapi import FastAPI, Depends, Response -import paramiko -from starlette_prometheus import metrics, PrometheusMiddleware -from sqlalchemy.orm import Session - -from . import settings, crud, models, schemas -from .database import SessionLocal, engine - -# pylint: disable=invalid-name - -models.Base.metadata.create_all(bind=engine) - -app = FastAPI() - -app.add_middleware(PrometheusMiddleware) -app.add_route("/metrics/", metrics) - -logging.getLogger().setLevel(settings.LOG_LEVEL) - -DEST_FILE_DIR = "gsa_order" - - -def get_db(): - """ - get db connection - """ - db = SessionLocal() - try: - yield db - finally: - db.close() - - -def write(file: StringIO | paramiko.SFTPFile, letters: list[models.Letter]): - """ - Write letter data to file - - 001|955 - 002|FAKEY MCFAKERSON|123 FAKE ST||GREAT FALLS|MT|59010|Q82GZBP71C|January 11, 2022|January 21, 2022|Example Sinatra App|https://secure.login.gov - 003|JIMMY TESTERSON|456 FAKE RD|Apt 1|FAKE CITY|CA|40323|4WVGPG0Z5Z|January 11, 2022|January 21, 2022|Example Rails App|https://secure.login.gov - 004|MIKE MCMOCKDATA|789 TEST AVE||FALLS CHURCH|VA|20943|5HVFT58WJ0|January 11, 2022|January 21, 2022|Example Java App|https://secure.login.gov - 005|... - ... - 956|... - - """ - - writer = csv.writer(file, delimiter="|") - numLines = len(letters) + 1 - - # number of digits in the row index - numIndexDigits = math.trunc(math.log(numLines, 10)) + 1 - # row index width is a least 2 and is enough to fit number of digits in the index - width = max(2, numIndexDigits) - - header = [f"{1:0{width}}", len(letters)] - writer.writerow(header) - - for i, val in enumerate(letters, start=2): - - # row with index - row = val.as_list(f"{i:0{width}}") - - # remove any pipes that might be in the data - sanitized_row = map(lambda x: x.replace("|", ""), row) - writer.writerow(sanitized_row) - - -@app.post("/upload") -def upload_batch(db: Session = Depends(get_db)): - """ - Upload letter data file to GPO server. - """ - - letters = crud.get_letters(db) - count = len(letters) - - if count == 0: - logging.info("No letters in db. Nothing to upload.") - return Response(count) - - if settings.DEBUG: - output = StringIO() - write(output, letters) - logging.debug(output.getvalue()) - crud.delete_letters(db, letters) - return Response(count) - - with paramiko.SSHClient() as ssh_client: - host_key = paramiko.RSAKey(data=b64decode(settings.GPO_HOSTKEY)) - ssh_client.get_host_keys().add(settings.GPO_HOST, "ssh-rsa", host_key) - ssh_client.connect( - settings.GPO_HOST, - username=settings.GPO_USERNAME, - password=settings.GPO_PASSWORD, - ) - with ssh_client.open_sftp() as sftp: - sftp.chdir(DEST_FILE_DIR) - date = datetime.datetime.now(ZoneInfo("US/Eastern")).strftime("%Y%m%d") - try: - with sftp.open(f"idva-{date}-0.psv", mode="wx") as file: - write(file, letters) - except PermissionError as err: - logging.error( - "Error Creating file likely because it already exists: %s", err - ) - return Response(status_code=500) - - crud.delete_letters(db, letters) - return Response(count) +import fastapi +import starlette_prometheus +from . import api, database, models, settings -@app.post("/letters", response_model=schemas.Letter) -def queue_letter(letter: schemas.LetterCreate, db: Session = Depends(get_db)): - """ - Add a letter to the queue - """ +logging.basicConfig(level=settings.LOG_LEVEL) - return crud.create_letter(db, letter) +models.Base.metadata.create_all(bind=database.engine) +app = fastapi.FastAPI() -@app.get("/letters") -def count_letter(db: Session = Depends(get_db)): - """ - Get count of letter in the queue - """ +app.add_middleware(starlette_prometheus.PrometheusMiddleware) +app.add_route("/metrics/", starlette_prometheus.metrics) - return len(crud.get_letters(db)) +app.include_router(api.router) diff --git a/gpo/models.py b/gpo/models.py index d2b9329..3f54fb6 100644 --- a/gpo/models.py +++ b/gpo/models.py @@ -1,11 +1,14 @@ """ Models for GPO """ - -from sqlalchemy import Column, Integer, String -from .database import Base +import sqlalchemy as sqla +from sqlalchemy.ext import declarative # pylint: disable=too-few-public-methods + +Base = declarative.declarative_base() + + class Letter(Base): """ DB model for Letter @@ -13,18 +16,18 @@ class Letter(Base): __tablename__ = "letters" - id = Column(Integer, primary_key=True, index=True) - name = Column(String) - address = Column(String) - address2 = Column(String) - city = Column(String) - state = Column(String) - zip = Column(String) - code = Column(String) - date = Column(String) - expiry = Column(String) - app = Column(String) - url = Column(String) + id = sqla.Column(sqla.Integer, primary_key=True, index=True) + name = sqla.Column(sqla.String) + address = sqla.Column(sqla.String) + address2 = sqla.Column(sqla.String) + city = sqla.Column(sqla.String) + state = sqla.Column(sqla.String) + zip = sqla.Column(sqla.String) + code = sqla.Column(sqla.String) + date = sqla.Column(sqla.String) + expiry = sqla.Column(sqla.String) + app = sqla.Column(sqla.String) + url = sqla.Column(sqla.String) def as_list(self, index: str) -> list: """ diff --git a/gpo/schemas.py b/gpo/schemas.py index c599e12..f0d9944 100644 --- a/gpo/schemas.py +++ b/gpo/schemas.py @@ -1,12 +1,12 @@ """ REST api models for gpo µservice """ -from pydantic import BaseModel +import pydantic # pylint: disable=too-few-public-methods -class LetterBase(BaseModel): +class LetterBase(pydantic.BaseModel): """ base letter model """ @@ -43,3 +43,9 @@ class Config: """ orm_mode = True + + +class Count(pydantic.BaseModel): + """count of letters""" + + count: int diff --git a/gpo/settings.py b/gpo/settings.py index 0b5becb..a381b96 100644 --- a/gpo/settings.py +++ b/gpo/settings.py @@ -2,10 +2,11 @@ Configuration for the GPO microservice settings. Context is switched based on if the app is in debug mode. """ +import json import logging import os -import json -import sys + +log = logging.getLogger(__name__) # SECURITY WARNING: don't run with debug turned on in production! # DEBUG set is set to True if env var is "True" @@ -18,9 +19,23 @@ GPO_HOST = os.getenv("GPO_HOST") GPO_HOSTKEY = os.getenv("GPO_HOSTKEY") -try: - db_uri = json.loads(os.getenv("VCAP_SERVICES"))["aws-rds"][0]["credentials"]["uri"] -except (json.JSONDecodeError, KeyError, TypeError): - sys.exit("Failed to load the aws-rds uri from VCAP_SERVICES") -DB_URI = os.getenv("DB_URI", db_uri) +def get_db_uri(): + """get db uri""" + vcap_services = os.getenv("VCAP_SERVICES", "") + try: + db_uri = json.loads(vcap_services)["aws-rds"][0]["credentials"]["uri"] + except (json.JSONDecodeError, KeyError) as err: + log.warning("Unable to load db_uri from VCAP_SERVICES") + log.debug("Error: %s", str(err)) + db_uri = "" + + # Sqlalchemy requires 'postgresql' as the protocol + db_uri = db_uri.replace("postgres://", "postgresql://", 1) + + return os.getenv("DB_URI", db_uri) + + +DB_URI = get_db_uri() +SCHEMA_NAME = "gpo" +DEST_FILE_DIR = "gsa_order" diff --git a/gpo/sftp.py b/gpo/sftp.py new file mode 100644 index 0000000..a82e9ec --- /dev/null +++ b/gpo/sftp.py @@ -0,0 +1,88 @@ +""" +sftp letter writer service +""" +import csv +import logging +import math +import base64 +import io + +import paramiko + +from . import models + +log = logging.getLogger(__name__) + + +def write(file: io.StringIO | paramiko.SFTPFile, letters: list[models.Letter]): + """ + Write letter data to file + + 001|955 + 002|FAKEY MCFAKERSON|123 FAKE ST||GREAT FALLS|MT|59010|Q82GZBP71C|January 11, 2022|January 21, 2022|Example Sinatra App|https://secure.login.gov + 003|JIMMY TESTERSON|456 FAKE RD|Apt 1|FAKE CITY|CA|40323|4WVGPG0Z5Z|January 11, 2022|January 21, 2022|Example Rails App|https://secure.login.gov + 004|MIKE MCMOCKDATA|789 TEST AVE||FALLS CHURCH|VA|20943|5HVFT58WJ0|January 11, 2022|January 21, 2022|Example Java App|https://secure.login.gov + 005|... + ... + 956|... + + Formatting Contract: + Removed: | (delimiter), \n (lineterminator), \r + Everything else verbatim, no quoting + """ + + writer = csv.writer( + file, delimiter="|", lineterminator="\n", quotechar=None, quoting=csv.QUOTE_NONE + ) + num_lines = len(letters) + 1 + + # number of digits in the row index + num_index_digits = math.trunc(math.log(num_lines, 10)) + 1 + # row index width is a least 2 and is enough to fit number of digits in the index + width = max(2, num_index_digits) + + header = [f"{1:0{width}}", len(letters)] + writer.writerow(header) + + for i, val in enumerate(letters, start=2): + + # row with index + row = val.as_list(f"{i:0{width}}") + + # remove any | or \n that might be in the data + sanitized_row = map( + lambda x: x.replace("|", "").replace("\n", "").replace("\r", ""), row + ) + writer.writerow(sanitized_row) + + +def write_sftp( + letters: list[models.Letter], ssh_settings, file_name: str, dest_dir: str +): + """ + Write letters to sftp endpoint + """ + with paramiko.SSHClient() as ssh_client: + host_key = paramiko.RSAKey(data=base64.b64decode(ssh_settings.GPO_HOSTKEY)) + ssh_client.get_host_keys().add(ssh_settings.GPO_HOST, "ssh-rsa", host_key) + ssh_client.connect( + ssh_settings.GPO_HOST, + username=ssh_settings.GPO_USERNAME, + password=ssh_settings.GPO_PASSWORD, + ) + with ssh_client.open_sftp() as sftp: + sftp.chdir(dest_dir) + try: + with sftp.open(file_name, mode="wx") as file: + write(file, letters) + except PermissionError as err: + log.error( + "Error Creating file likely because it already exists: %s", err + ) + raise SftpError from err + + +class SftpError(PermissionError): + """ + Error with sftp operation + """ diff --git a/requirements-dev.txt b/requirements-dev.txt index fb885ee..f3aa81a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,4 @@ pylint bandit pytest tzdata +requests diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a6599e2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,51 @@ +"""fixtures""" +import sys +import typing + +import pytest +from fastapi import testclient + +import db +from db import SessionLocal + +# pylint: disable=wrong-import-position +sys.modules["gpo.database"] = db +from gpo.main import app +from gpo.models import Letter + + +@pytest.fixture +def generate_data(): + """test fixture""" + + def get_letter() -> Letter: + """get a letter object""" + return Letter( + id=5, + name="Name", + address="Address", + address2="Address 2", + city="City", + state="State", + zip="Zip", + code="Code", + date="Date", + expiry="Date", + app="App", + url="Url", + ) + + return get_letter + + +@pytest.fixture(scope="session") +def session() -> typing.Generator: + """session""" + yield SessionLocal() + + +@pytest.fixture(scope="module") +def client() -> typing.Generator: + """api test client""" + with testclient.TestClient(app) as test_client: + yield test_client diff --git a/tests/db.py b/tests/db.py new file mode 100644 index 0000000..aadc746 --- /dev/null +++ b/tests/db.py @@ -0,0 +1,12 @@ +import sqlalchemy +from sqlalchemy import orm, pool + +engine = sqlalchemy.create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=pool.StaticPool, +) + +SessionLocal = orm.sessionmaker( + autocommit=False, autoflush=False, bind=engine, future=True +) diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..10a2404 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,31 @@ +"""GPO api tests""" +from fastapi import testclient + + +def test_count_letter(client: testclient.TestClient) -> None: + """test letter count""" + response = client.get("/letters") + assert response.status_code == 200 + content = response.json() + assert isinstance(content["count"], int) + + +def test_create_letter(client: testclient.TestClient) -> None: + """ "create letter""" + letter = { + "name": "FAKEY MCFAKERSON", + "address": "123 FAKE ST", + "address2": "", + "city": "GREAT FALLS", + "state": "MT", + "zip": "59010", + "code": "Q82GZBP71C", + "date": "January 11, 2022", + "expiry": "January 21, 2022", + "app": "Example Sinatra App", + "url": "https://secure.login.gov", + } + response = client.post("/letters", json=letter) + assert response.status_code == 200 + content = response.json() + assert isinstance(content["id"], int) diff --git a/tests/test_crud.py b/tests/test_crud.py new file mode 100644 index 0000000..44083dd --- /dev/null +++ b/tests/test_crud.py @@ -0,0 +1,14 @@ +"""crud test""" +from sqlalchemy import orm + +from gpo import crud + + +def test_get_item(session: orm.Session) -> None: + """test get""" + crud.get_letters(session) + + +def test_get_item_for_update(session: orm.Session) -> None: + """test get for update""" + crud.get_letters_for_update(session) diff --git a/tests/test_gpo.py b/tests/test_gpo.py deleted file mode 100644 index 2e4e049..0000000 --- a/tests/test_gpo.py +++ /dev/null @@ -1 +0,0 @@ -""" GPO API unit tests """ diff --git a/tests/test_sftp.py b/tests/test_sftp.py new file mode 100644 index 0000000..c9374af --- /dev/null +++ b/tests/test_sftp.py @@ -0,0 +1,46 @@ +""" GPO API unit tests """ +import io + +from gpo import models, sftp + + +def test_write_letters(generate_data): + """ + write letter + """ + result = io.StringIO() + letter = [generate_data() for _ in range(5)] + + sftp.write(result, letter) + print(result.getvalue()) + assert True + + +def test_write_letters_with_special_chars(): + """ + |,\n,\r in text are elided + all others printed verbatim + """ + result = io.StringIO() + letter = [ + models.Letter( + id=5, + name="Name|", + address="Addr\ness", + address2='Ad"dress 2', + city="Ci\rty", + state="Stat\te", + zip="'Zip", + code="|Code", + date="|Date", + expiry="Dat|e", + app="App|", + url="Ur|l", + ) + ] + + sftp.write(result, letter) + expected = """01|1 +02|Name|Address|Ad"dress 2|City|Stat\te|'Zip|Code|Date|Date|App|Url +""" + assert expected == result.getvalue()