Merge pull request #12 from brighthive/feat/add-cloudwatch-logging

Feat/add cloudwatch logging

.gitignore

@@ -6,4 +6,7 @@
 **/__pycache__
 
 # pytest artifacts
-**/.pytest_cache
+**/.pytest_cache
+
+**/sandbox.py
+**/push.sh

Pipfile

@@ -18,6 +18,9 @@ flask-sqlalchemy = "*"
 sqlalchemy = "*"
 psycopg2-binary = "*"
 mci-database = {editable = true,git = "https://github.com/brighthive/mci-database.git",ref = "master"}
+watchtower = "*"
+gevent = "*"
+boto3 = "*"
 
 [requires]
 python_version = "3.7"

cmd.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 
-MAX_RETRIES=5
+if [ -z $WORKERS ]; then
+    WORKERS=4
+fi
+
 
 if [ "$APP_ENV" == "DEVELOPMENT" ] || [ -z "$APP_ENV" ]; then
-    # docker-compose.yml sets the APP_ENV
-    gunicorn -b 0.0.0.0 matching:app --reload --log-level=DEBUG --timeout 240
+    gunicorn -b 0.0.0.0 matching:app -w $WORKERS --worker-class gevent --reload --log-level=DEBUG --timeout 240
 else
-    gunicorn -b 0.0.0.0 matching:app --reload
+    gunicorn -b 0.0.0.0 matching:app -w $WORKERS --worker-class gevent 
 fi

matching/api/api.py

@@ -1,37 +1,19 @@
 '''
 This module provides the POST endpoint for finding matches.
 '''
-from flask_restful import Resource
+from flask_restful import Resource, request
 from webargs import fields
 from webargs.flaskparser import use_args
 
 from matching.common.util import compute_match_with_score
 
-individual_args = {
-    "mci_id": fields.Str(), 
-    "vendor_id": fields.Str(), 
-    "ssn": fields.Str(), 
-    "registration_date": fields.Str(), 
-    "first_name": fields.Str(), 
-    "middle_name": fields.Str(), 
-    "last_name": fields.Str(),
-    "date_of_birth": fields.Str(), 
-    "email_address": fields.Str(), 
-    "telephone": fields.Str(), 
-    "mailing_address_id": fields.Str(), 
-    "gender_id": fields.Str(), 
-    "education_level_id": fields.Str(), 
-    "employment_status_id": fields.Str(), 
-    "source_id": fields.Str()
-}
 
 class ComputeMatch(Resource):
     # Keeping this as a sanity-check placeholder, for now.
     def get(self):
         return {'hello': 'world'}
 
-    @use_args(individual_args)
-    def post(self, args):
-        mci_id, score = compute_match_with_score(args)
-
+    # @use_args(individual_args)
+    def post(self):
+        mci_id, score = compute_match_with_score(request.get_json(force=True))
         return {'mci_id': mci_id, 'score': score}, 201

matching/app/app.py

@@ -2,33 +2,94 @@
 This module houses the core Flask application.
 """
 
-from flask import Flask, g
+import os
+import json
+import logging
+import boto3.session
+import watchtower
+from flask import Flask, g, request
+from datetime import datetime
 from flask_restful import Api
 
 from matching.api import ComputeMatch
 from matching.config import ConfigurationFactory
 from matching.database import Session
 
 app = Flask(__name__)
-app.config.from_object(ConfigurationFactory.from_env())
+config = ConfigurationFactory.from_env()
+app.config.from_object(config)
 
 api = Api(app)
 api.add_resource(ComputeMatch, '/compute-match')
 
+# logger configuration
+formatter = logging.Formatter(
+    fmt='[%(asctime)s] [%(levelname)s] %(message)s', datefmt="%a, %d %b %Y %H:%M:%S")
+
+try:
+    aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
+    aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+    region_name = os.getenv('AWS_REGION_NAME')
+    logging.getLogger().setLevel(logging.INFO)
+    boto3_session = boto3.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
+                                  region_name=region_name)  
+    logger = logging.getLogger(__name__)
+    handler = watchtower.CloudWatchLogHandler(
+        boto3_session=boto3_session, log_group=os.getenv('AWS_LOG_GROUP'), stream_name=os.getenv('AWS_LOG_STREAM'))
+    formatter = logging.Formatter(
+        fmt='[%(asctime)s] [%(levelname)s] %(message)s', datefmt="%a, %d %b %Y %H:%M:%S")
+    handler.setFormatter(formatter)
+    handler.setLevel(logging.INFO)
+    logger.addHandler(handler)
+except Exception as e:
+    logging.getLogger().setLevel(logging.INFO)
+    logger = logging.getLogger(__name__)
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    handler.setLevel(logging.INFO)
+    logger.addHandler(handler)
+    logger.warning(
+        f'Failed to configure CloudWatch due to the following error: {str(e)}')
+
+
+@app.after_request
+def after_request(response):
+    info = {
+        'remote_addr': request.remote_addr,
+        'request_time': str(datetime.utcnow()),
+        'method': request.method,
+        'path': request.path,
+        'scheme': request.scheme.upper(),
+        'status_code': response.status_code,
+        'status': response.status,
+        'content_length': response.content_length,
+        'user_agent': str(request.user_agent),
+        'payload': {
+            'last_name': request.json['last_name'] if 'last_name' in request.json else '',
+            'gender_id': request.json['gender_id'] if 'gender_id' in request.json else ''
+        }
+    }
+    if info['status_code'] >= 200 and info['status_code'] < 300:
+        logger.info(info)
+    else:
+        logger.error(info)
+    return response
+
+
 @app.teardown_appcontext
 def cleanup(resp_or_exc):
     '''
     A session establishes all conversations with the database. 
     Mainly, it requests a connection with the database. 
-    
+
     A session can have a lifespan across many *short* transactions. For web applications, 
-	the scope of a session should align with the scope of a request. 
-	In other words: tear down the session at the end of a request.
+        the scope of a session should align with the scope of a request. 
+        In other words: tear down the session at the end of a request.
 
-	This decorator function ensures that Sessions are removed at the end of a request.
+        This decorator function ensures that Sessions are removed at the end of a request.
 
-	References:
-	- https://docs.sqlalchemy.org/en/13/orm/contextual.html#using-thread-local-scope-with-web-applications
-	- https://dev.to/nestedsoftware/flask-and-sqlalchemy-without-the-flask-sqlalchemy-extension-3cf8
+        References:
+        - https://docs.sqlalchemy.org/en/13/orm/contextual.html#using-thread-local-scope-with-web-applications
+        - https://dev.to/nestedsoftware/flask-and-sqlalchemy-without-the-flask-sqlalchemy-extension-3cf8
     '''
     Session.remove()

matching/config/config.py

@@ -37,11 +37,11 @@ def __init__(self):
 
     DEBUG = False
 
-    POSTGRES_USER = os.getenv('POSTGRES_USER','brighthive')
+    POSTGRES_USER = os.getenv('POSTGRES_USER', 'brighthive')
     POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD', 'test_password')
-    POSTGRES_DATABASE = os.getenv('POSTGRES_DATABASE','mci_dev')
+    POSTGRES_DATABASE = os.getenv('POSTGRES_DATABASE', 'mci_dev')
     POSTGRES_PORT = os.getenv('POSTGRES_PORT', 5432)
-    POSTGRES_HOSTNAME = os.getenv('POSTGRES_HOSTNAME','localhost')
+    POSTGRES_HOSTNAME = os.getenv('POSTGRES_HOSTNAME', 'localhost')
     SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
         POSTGRES_USER,
         POSTGRES_PASSWORD,
@@ -50,6 +50,7 @@ def __init__(self):
         POSTGRES_DATABASE
     )
 
+
 class TestConfig(Config):
     def __init__(self):
         super().__init__()
@@ -89,6 +90,13 @@ def __init__(self):
         POSTGRES_PORT,
         POSTGRES_DATABASE
     )
+    # AWS Configuration for CloudWatch
+    # AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID', None)
+    # AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY', None)
+    # AWS_REGION_NAME = os.getenv('AWS_REGION_NAME', None)
+    # AWS_LOG_GROUP = os.getenv('AWS_LOG_GROUP', None)
+    # AWS_LOG_STREAM = os.getenv('AWS_LOG_STREAM', None)
+    # AWS_LOGGER_NAME = os.getenv('AWS_LOGGER_NAME', None)
 
 
 class ConfigurationFactory(object):
@@ -115,7 +123,7 @@ def get_config(config_type: str):
             return TestConfig()
         elif config_type.upper() == 'DEVELOPMENT':
             return DevelopmentConfig()
-        elif  config_type.upper() == 'SANDBOX':
+        elif config_type.upper() == 'SANDBOX':
             return SandboxConfig()
         elif config_type.upper() == 'PRODUCTION':
             return ProductionConfig()