Add integration tests (#40)

* add integration tests

* update README

* add internal gitignore

* use modules and change the set up

* remove dep and Make from README;

.gitignore

@@ -13,4 +13,6 @@ bin/
 
 /contrib/google.golang.org/grpc.v12/vendor/
 .vscode
-coverage.txt
+coverage.txt
+
+**/.serverless

tests/integration_tests/.gitignore

@@ -0,0 +1 @@
+vendor

tests/integration_tests/README.md

@@ -0,0 +1,16 @@
+# Integration tests
+
+
+## Requirements
+
+- Node
+- Go
+- DD_API_KEY
+
+## Running
+
+```bash
+DD_API_KEY=<API_KEY> aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh
+```
+
+Use `UPDATE_SNAPSHOTS=true` to update snapshots

tests/integration_tests/go.mod

@@ -0,0 +1,10 @@
+module github.com/DataDog/datadog-lambda-go/tests/integration_tests/bin/hello
+
+go 1.12
+
+require (
+	github.com/DataDog/datadog-lambda-go v0.7.0
+	github.com/aws/aws-lambda-go v1.11.1
+)
+
+replace github.com/DataDog/datadog-lambda-go => ../../

tests/integration_tests/go.sum

@@ -0,0 +1,22 @@
+github.com/DataDog/datadog-lambda-go v0.7.0 h1:RsALFf7mhtB4k5WVduvvsDl2VUsui+eJ7oXOWB+LdnY=
+github.com/DataDog/datadog-lambda-go v0.7.0/go.mod h1:8IH+3AngDt+on4Fc7qeFAxj2h6oPuIgsXs5lEPFImto=
+github.com/aws/aws-lambda-go v1.11.1 h1:wuOnhS5aqzPOWns71FO35PtbtBKHr4MYsPVt5qXLSfI=
+github.com/aws/aws-lambda-go v1.11.1/go.mod h1:Rr2SMTLeSMKgD45uep9V/NP8tnbCcySgu04cx0k/6cw=
+github.com/aws/aws-sdk-go v1.20.2 h1:/BBeW8F4PPmvJ5jpFvgkCK4RJQXErNndVRnNhO2qEkQ=
+github.com/aws/aws-sdk-go v1.20.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-xray-sdk-go v1.0.0-rc.9 h1:MC5zypTWx5YIbWE3pgcPaG8+1ytirvfCVBkcgHbVZ5Q=
+github.com/aws/aws-xray-sdk-go v1.0.0-rc.9/go.mod h1:XtMKdBQfpVut+tJEwI7+dJFRxxRdxHDyVNp2tHXRq04=
+github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
+github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
+github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=

tests/integration_tests/input_events/api-gateway-get.json

@@ -0,0 +1,55 @@
+{
+  "path": "/test/hello",
+  "headers": {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate, lzma, sdch, br",
+    "Accept-Language": "en-US,en;q=0.8",
+    "CloudFront-Forwarded-Proto": "https",
+    "CloudFront-Is-Desktop-Viewer": "true",
+    "CloudFront-Is-Mobile-Viewer": "false",
+    "CloudFront-Is-SmartTV-Viewer": "false",
+    "CloudFront-Is-Tablet-Viewer": "false",
+    "CloudFront-Viewer-Country": "US",
+    "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48",
+    "Via": "1.1 fb7cca60f0ecd82ce07790c9c5eef16c.cloudfront.net (CloudFront)",
+    "X-Amz-Cf-Id": "nBsWBOrSHMgnaROZJK1wGCZ9PcRcSpq_oSXZNQwQ10OTZL4cimZo3g==",
+    "X-Forwarded-For": "192.168.100.1, 192.168.1.1",
+    "X-Forwarded-Port": "443",
+    "X-Forwarded-Proto": "https"
+  },
+  "pathParameters": {
+    "proxy": "hello"
+  },
+  "requestContext": {
+    "accountId": "123456789012",
+    "resourceId": "us4z18",
+    "stage": "test",
+    "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9",
+    "identity": {
+      "cognitoIdentityPoolId": "",
+      "accountId": "",
+      "cognitoIdentityId": "",
+      "caller": "",
+      "apiKey": "",
+      "sourceIp": "192.168.100.1",
+      "cognitoAuthenticationType": "",
+      "cognitoAuthenticationProvider": "",
+      "userArn": "",
+      "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48",
+      "user": ""
+    },
+    "resourcePath": "/{proxy+}",
+    "httpMethod": "GET",
+    "apiId": "wt6mne2s9k"
+  },
+  "resource": "/{proxy+}",
+  "httpMethod": "GET",
+  "queryStringParameters": {
+    "name": "me"
+  },
+  "stageVariables": {
+    "stageVarName": "stageVarValue"
+  }
+}

tests/integration_tests/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/aws/aws-lambda-go/lambda"
+
+	ddlambda "github.com/DataDog/datadog-lambda-go"
+	"github.com/aws/aws-lambda-go/events"
+)
+
+var (
+	invokeCount = 0
+)
+
+func handleRequest(ctx context.Context, ev events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
+
+	req, _ := http.NewRequest("GET", "https://www.datadoghq.com", nil)
+	ddlambda.AddTraceHeaders(ctx, req)
+	client := http.Client{}
+	client.Do(req)
+
+	headers := ddlambda.GetTraceHeaders(ctx)
+
+	ddlambda.Distribution("hello-go.dog", float64(invokeCount))
+	invokeCount++
+
+	fmt.Println("Start Logging Headers")
+	for key, value := range headers {
+		fmt.Printf("Request header: %s: %s\n", key, value)
+	}
+	fmt.Println("End Logging Headers")
+
+	return events.APIGatewayProxyResponse{
+		StatusCode: 200,
+		Body:       "hello, dog!",
+	}, nil
+}
+
+func main() {
+	lambda.Start(ddlambda.WrapHandler(handleRequest, nil))
+}

tests/integration_tests/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "lambda-sample-go",
+  "version": "1.0.0"
+}

tests/integration_tests/run_integration_tests.sh

@@ -0,0 +1,151 @@
+#!/bin/bash
+
+# Usage - run commands from repo root:
+# To check if new changes to the layer cause changes to any snapshots:
+#   DD_API_KEY=XXXX aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh
+# To regenerate snapshots:
+#   UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh
+
+set -e
+
+# These values need to be in sync with serverless.yml, where there needs to be a function
+# defined for every handler_runtime combination
+LAMBDA_HANDLERS=("hello-go")
+
+LOGS_WAIT_SECONDS=20
+
+integration_tests_dir=$(cd `dirname $0` && pwd)
+echo $integration_tests_dir
+
+script_start_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+mismatch_found=false
+
+if [ -z "$DD_API_KEY" ]; then
+    echo "No DD_API_KEY env var set, exiting"
+    exit 1
+fi
+
+if [ -n "$UPDATE_SNAPSHOTS" ]; then
+    echo "Overwriting snapshots in this execution"
+fi
+
+echo "Bulding Go binary"
+GOOS=linux go build -ldflags="-s -w" -o bin/hello
+
+echo "Deploying function"
+sls deploy --api-key $DD_API_KEY
+
+cd $integration_tests_dir
+
+input_event_files=$(ls ./input_events)
+# Sort event files by name so that snapshots stay consistent
+input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort))
+
+echo "Invoking functions"
+set +e # Don't exit this script if an invocation fails or there's a diff
+for input_event_file in "${input_event_files[@]}"; do
+    for function_name in "${LAMBDA_HANDLERS[@]}"; do
+        # Get event name without trailing ".json" so we can build the snapshot file name
+        input_event_name=$(echo "$input_event_file" | sed "s/.json//")
+        # Return value snapshot file format is snapshots/return_values/{handler}_{runtime}_{input-event}
+        snapshot_path="$integration_tests_dir/snapshots/return_values/${function_name}_${input_event_name}.json"
+
+        return_value=$(sls invoke -f $function_name --path "$integration_tests_dir/input_events/$input_event_file" --api-key=$DD_API_KEY)
+
+        if [ ! -f $snapshot_path ]; then
+            # If the snapshot file doesn't exist yet, we create it
+            echo "Writing return value to $snapshot_path because no snapshot exists yet"
+            echo "$return_value" >$snapshot_path
+        elif [ -n "$UPDATE_SNAPSHOTS" ]; then
+            # If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot
+            echo "Overwriting return value snapshot for $snapshot_path"
+            echo "$return_value" >$snapshot_path
+        else
+            # Compare new return value to snapshot
+            diff_output=$(echo "$return_value" | diff - $snapshot_path)
+            if [ $? -eq 1 ]; then
+                echo "Failed: Return value for $function_name does not match snapshot:"
+                echo "$diff_output"
+                mismatch_found=true
+            else
+                echo "Ok: Return value for $function_name with $input_event_name event matches snapshot"
+            fi
+        fi
+    done
+done
+set -e
+
+echo "Sleeping $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..."
+sleep $LOGS_WAIT_SECONDS
+
+echo "Fetching logs for invocations and comparing to snapshots"
+for function_name in "${LAMBDA_HANDLERS[@]}"; do
+    function_snapshot_path="./snapshots/logs/$function_name.log"
+
+    # Fetch logs with serverless cli
+    raw_logs=$(serverless logs -f $function_name --startTime $script_start_time)
+
+    # Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
+    logs=$(
+        echo "$raw_logs" |
+            # Filter serverless cli errors
+            sed '/Serverless: Recoverable error occurred/d' |
+            # Normalize Lambda runtime report logs
+            sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"):( )?[a-z0-9\.\-]+/\1:\2XXXX/g' |
+            # Normalize DD APM headers and AWS account ID
+            sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|account_id:) ?[0-9]+/\1XXXX/g" |
+            # Strip API key from logged requests
+            sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" |
+            # Normalize ISO combined date-time
+            sed -E "s/[0-9]{4}\-[0-9]{2}\-[0-9]{2}(T?)[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+ \(\-?[0-9:]+\))?Z/XXXX-XX-XXTXX:XX:XX.XXXZ/" |
+            # Normalize log timestamps
+            sed -E "s/[0-9]{4}(\-|\/)[0-9]{2}(\-|\/)[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+( \(\-?[0-9:]+\))?)?/XXXX-XX-XX XX:XX:XX.XXX/" |
+            # Normalize DD trace ID injection
+            sed -E "s/(dd\.trace_id=)[0-9]+ (dd\.span_id=)[0-9]+/\1XXXX \2XXXX/" |
+            # Normalize execution ID in logs prefix
+            sed -E $'s/[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\t/XXXX-XXXX-XXXX-XXXX-XXXX\t/' |
+            # Normalize minor package version tag so that these snapshots aren't broken on version bumps
+            sed -E "s/(dd_lambda_layer:datadog-go[0-9]+\.)[0-9]+\.[0-9]+/\1XX\.X/g" |
+            # Normalize data in logged traces
+            sed -E 's/"(span_id|parent_id|trace_id|start|duration|tcp\.local\.address|tcp\.local\.port|dns\.address|request_id|function_arn)":("?)[a-zA-Z0-9\.:\-]+("?)/"\1":\2XXXX\3/g' |
+            # Normalize data in logged traces
+            sed -E 's/"(points\\\":\[\[)([0-9]+)/\1XXXX/g'
+
+    )
+
+    if [ ! -f $function_snapshot_path ]; then
+        # If no snapshot file exists yet, we create one
+        echo "Writing logs to $function_snapshot_path because no snapshot exists yet"
+        echo "$logs" >$function_snapshot_path
+    elif [ -n "$UPDATE_SNAPSHOTS" ]; then
+        # If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot
+        echo "Overwriting log snapshot for $function_snapshot_path"
+        echo "$logs" >$function_snapshot_path
+    else
+        # Compare new logs to snapshots
+        set +e # Don't exit this script if there is a diff
+        diff_output=$(echo "$logs" | diff - $function_snapshot_path)
+        if [ $? -eq 1 ]; then
+            echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):"
+            echo "$diff_output"
+            mismatch_found=true
+        else
+            echo "Ok: New logs for $function_name match snapshot"
+        fi
+        set -e
+    fi
+done
+
+if [ "$mismatch_found" = true ]; then
+    echo "FAILURE: A mismatch between new data and a snapshot was found and printed above."
+    echo "If the change is expected, generate new snapshots by running 'UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX ./scripts/run_integration_tests.sh'"
+    exit 1
+fi
+
+if [ -n "$UPDATE_SNAPSHOTS" ]; then
+    echo "SUCCESS: Wrote new snapshots for all functions"
+    exit 0
+fi
+
+echo "SUCCESS: No difference found between snapshots and new return values or logs"

tests/integration_tests/serverless.yml

@@ -0,0 +1,29 @@
+service: hello-dog-go
+
+package:
+  exclude:
+    - ./**
+  include:
+    - ./bin/**
+
+provider:
+  name: aws
+  tracing:
+    lambda: true
+    apiGateway: true
+  memorySize: 128
+  timeout: 30
+  environment:
+    DD_API_KEY: ${env:DD_API_KEY}
+    DD_LOG_LEVEL: DEBUG
+    DD_INTEGRATION_TEST: true
+    DD_ENHANCED_METRICS: true
+
+functions:
+  hello-go:
+    runtime: go1.x
+    handler: bin/hello
+    events:
+      - http:
+          path: hello-go
+          method: get

tests/integration_tests/snapshots/logs/hello-go.log

@@ -0,0 +1,16 @@
+START RequestId: XXXX Version: $LATEST
+XXXX-XX-XXTXX:XX:XX.XXXZ [Info] using daemon endpoints from environment variable AWS_XRAY_DAEMON_ADDRESS: 169.254.79.2:2000
+XXXX-XX-XXTXX:XX:XX.XXXZ [Info] Emitter using address: 169.254.79.2:2000
+XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: sending metric via log forwarder"}
+{"m":"aws.lambda.enhanced.invocations","v":1,"e":XXXX,"t":["functionname:hello-dog-go-dev-hello-go","region:us-east-1","account_id:XXXX","memorysize:128","cold_start:true","resource:hello-dog-go-dev-hello-go","dd_lambda_layer:datadog-go1.XX.X"]}
+XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: adding metric \"hello-go.dog\", with value 0.000000"}
+Start Logging Headers
+Request header: x-datadog-trace-id:XXXX
+Request header: x-datadog-sampling-priority: 2
+Request header: x-datadog-parent-id:XXXX
+End Logging Headers
+XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: posting to url https://api.datadoghq.com/api/v1/distribution_points"}
+XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Sending payload with body {\"series\":[{\"metric\":\"hello-go.dog\",\"tags\":[\"dd_lambda_layer:datadog-go1.XX.X\"],\"type\":\"distribution\",\points\":[[XXXX,[0]]]}]}"}
+END RequestId: XXXX
+REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 128 MB	Max Memory Used: XXXX MB	Init Duration: XXXX ms	
+XRAY TraceId: XXXX	SegmentId: XXXX	Sampled: true	

tests/integration_tests/snapshots/return_values/hello-go_api-gateway-get.json

@@ -0,0 +1,6 @@
+{
+    "statusCode": 200,
+    "headers": null,
+    "multiValueHeaders": null,
+    "body": "hello, dog!"
+}