diff --git a/.gitignore b/.gitignore index 133f2a4b..1b2578de 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,6 @@ bin/ /contrib/google.golang.org/grpc.v12/vendor/ .vscode -coverage.txt \ No newline at end of file +coverage.txt + +**/.serverless diff --git a/tests/integration_tests/.gitignore b/tests/integration_tests/.gitignore new file mode 100644 index 00000000..5657f6ea --- /dev/null +++ b/tests/integration_tests/.gitignore @@ -0,0 +1 @@ +vendor \ No newline at end of file diff --git a/tests/integration_tests/README.md b/tests/integration_tests/README.md new file mode 100644 index 00000000..af1b0c6c --- /dev/null +++ b/tests/integration_tests/README.md @@ -0,0 +1,16 @@ +# Integration tests + + +## Requirements + +- Node +- Go +- DD_API_KEY + +## Running + +```bash +DD_API_KEY= aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh +``` + +Use `UPDATE_SNAPSHOTS=true` to update snapshots diff --git a/tests/integration_tests/go.mod b/tests/integration_tests/go.mod new file mode 100644 index 00000000..e0477b60 --- /dev/null +++ b/tests/integration_tests/go.mod @@ -0,0 +1,10 @@ +module github.com/DataDog/datadog-lambda-go/tests/integration_tests/bin/hello + +go 1.12 + +require ( + github.com/DataDog/datadog-lambda-go v0.7.0 + github.com/aws/aws-lambda-go v1.11.1 +) + +replace github.com/DataDog/datadog-lambda-go => ../../ diff --git a/tests/integration_tests/go.sum b/tests/integration_tests/go.sum new file mode 100644 index 00000000..a6cfc578 --- /dev/null +++ b/tests/integration_tests/go.sum @@ -0,0 +1,22 @@ +github.com/DataDog/datadog-lambda-go v0.7.0 h1:RsALFf7mhtB4k5WVduvvsDl2VUsui+eJ7oXOWB+LdnY= +github.com/DataDog/datadog-lambda-go v0.7.0/go.mod h1:8IH+3AngDt+on4Fc7qeFAxj2h6oPuIgsXs5lEPFImto= +github.com/aws/aws-lambda-go v1.11.1 h1:wuOnhS5aqzPOWns71FO35PtbtBKHr4MYsPVt5qXLSfI= +github.com/aws/aws-lambda-go v1.11.1/go.mod h1:Rr2SMTLeSMKgD45uep9V/NP8tnbCcySgu04cx0k/6cw= +github.com/aws/aws-sdk-go v1.20.2 h1:/BBeW8F4PPmvJ5jpFvgkCK4RJQXErNndVRnNhO2qEkQ= +github.com/aws/aws-sdk-go v1.20.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-xray-sdk-go v1.0.0-rc.9 h1:MC5zypTWx5YIbWE3pgcPaG8+1ytirvfCVBkcgHbVZ5Q= +github.com/aws/aws-xray-sdk-go v1.0.0-rc.9/go.mod h1:XtMKdBQfpVut+tJEwI7+dJFRxxRdxHDyVNp2tHXRq04= +github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= +github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= diff --git a/tests/integration_tests/input_events/api-gateway-get.json b/tests/integration_tests/input_events/api-gateway-get.json new file mode 100644 index 00000000..496553de --- /dev/null +++ b/tests/integration_tests/input_events/api-gateway-get.json @@ -0,0 +1,55 @@ +{ + "path": "/test/hello", + "headers": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, lzma, sdch, br", + "Accept-Language": "en-US,en;q=0.8", + "CloudFront-Forwarded-Proto": "https", + "CloudFront-Is-Desktop-Viewer": "true", + "CloudFront-Is-Mobile-Viewer": "false", + "CloudFront-Is-SmartTV-Viewer": "false", + "CloudFront-Is-Tablet-Viewer": "false", + "CloudFront-Viewer-Country": "US", + "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48", + "Via": "1.1 fb7cca60f0ecd82ce07790c9c5eef16c.cloudfront.net (CloudFront)", + "X-Amz-Cf-Id": "nBsWBOrSHMgnaROZJK1wGCZ9PcRcSpq_oSXZNQwQ10OTZL4cimZo3g==", + "X-Forwarded-For": "192.168.100.1, 192.168.1.1", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "pathParameters": { + "proxy": "hello" + }, + "requestContext": { + "accountId": "123456789012", + "resourceId": "us4z18", + "stage": "test", + "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9", + "identity": { + "cognitoIdentityPoolId": "", + "accountId": "", + "cognitoIdentityId": "", + "caller": "", + "apiKey": "", + "sourceIp": "192.168.100.1", + "cognitoAuthenticationType": "", + "cognitoAuthenticationProvider": "", + "userArn": "", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48", + "user": "" + }, + "resourcePath": "/{proxy+}", + "httpMethod": "GET", + "apiId": "wt6mne2s9k" + }, + "resource": "/{proxy+}", + "httpMethod": "GET", + "queryStringParameters": { + "name": "me" + }, + "stageVariables": { + "stageVarName": "stageVarValue" + } +} diff --git a/tests/integration_tests/main.go b/tests/integration_tests/main.go new file mode 100644 index 00000000..1ac4bcb1 --- /dev/null +++ b/tests/integration_tests/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "fmt" + "net/http" + + "github.com/aws/aws-lambda-go/lambda" + + ddlambda "github.com/DataDog/datadog-lambda-go" + "github.com/aws/aws-lambda-go/events" +) + +var ( + invokeCount = 0 +) + +func handleRequest(ctx context.Context, ev events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { + + req, _ := http.NewRequest("GET", "https://www.datadoghq.com", nil) + ddlambda.AddTraceHeaders(ctx, req) + client := http.Client{} + client.Do(req) + + headers := ddlambda.GetTraceHeaders(ctx) + + ddlambda.Distribution("hello-go.dog", float64(invokeCount)) + invokeCount++ + + fmt.Println("Start Logging Headers") + for key, value := range headers { + fmt.Printf("Request header: %s: %s\n", key, value) + } + fmt.Println("End Logging Headers") + + return events.APIGatewayProxyResponse{ + StatusCode: 200, + Body: "hello, dog!", + }, nil +} + +func main() { + lambda.Start(ddlambda.WrapHandler(handleRequest, nil)) +} diff --git a/tests/integration_tests/package.json b/tests/integration_tests/package.json new file mode 100644 index 00000000..83720641 --- /dev/null +++ b/tests/integration_tests/package.json @@ -0,0 +1,4 @@ +{ + "name": "lambda-sample-go", + "version": "1.0.0" +} diff --git a/tests/integration_tests/run_integration_tests.sh b/tests/integration_tests/run_integration_tests.sh new file mode 100755 index 00000000..b91f1eff --- /dev/null +++ b/tests/integration_tests/run_integration_tests.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +# Usage - run commands from repo root: +# To check if new changes to the layer cause changes to any snapshots: +# DD_API_KEY=XXXX aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh +# To regenerate snapshots: +# UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh + +set -e + +# These values need to be in sync with serverless.yml, where there needs to be a function +# defined for every handler_runtime combination +LAMBDA_HANDLERS=("hello-go") + +LOGS_WAIT_SECONDS=20 + +integration_tests_dir=$(cd `dirname $0` && pwd) +echo $integration_tests_dir + +script_start_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +mismatch_found=false + +if [ -z "$DD_API_KEY" ]; then + echo "No DD_API_KEY env var set, exiting" + exit 1 +fi + +if [ -n "$UPDATE_SNAPSHOTS" ]; then + echo "Overwriting snapshots in this execution" +fi + +echo "Bulding Go binary" +GOOS=linux go build -ldflags="-s -w" -o bin/hello + +echo "Deploying function" +sls deploy --api-key $DD_API_KEY + +cd $integration_tests_dir + +input_event_files=$(ls ./input_events) +# Sort event files by name so that snapshots stay consistent +input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort)) + +echo "Invoking functions" +set +e # Don't exit this script if an invocation fails or there's a diff +for input_event_file in "${input_event_files[@]}"; do + for function_name in "${LAMBDA_HANDLERS[@]}"; do + # Get event name without trailing ".json" so we can build the snapshot file name + input_event_name=$(echo "$input_event_file" | sed "s/.json//") + # Return value snapshot file format is snapshots/return_values/{handler}_{runtime}_{input-event} + snapshot_path="$integration_tests_dir/snapshots/return_values/${function_name}_${input_event_name}.json" + + return_value=$(sls invoke -f $function_name --path "$integration_tests_dir/input_events/$input_event_file" --api-key=$DD_API_KEY) + + if [ ! -f $snapshot_path ]; then + # If the snapshot file doesn't exist yet, we create it + echo "Writing return value to $snapshot_path because no snapshot exists yet" + echo "$return_value" >$snapshot_path + elif [ -n "$UPDATE_SNAPSHOTS" ]; then + # If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot + echo "Overwriting return value snapshot for $snapshot_path" + echo "$return_value" >$snapshot_path + else + # Compare new return value to snapshot + diff_output=$(echo "$return_value" | diff - $snapshot_path) + if [ $? -eq 1 ]; then + echo "Failed: Return value for $function_name does not match snapshot:" + echo "$diff_output" + mismatch_found=true + else + echo "Ok: Return value for $function_name with $input_event_name event matches snapshot" + fi + fi + done +done +set -e + +echo "Sleeping $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..." +sleep $LOGS_WAIT_SECONDS + +echo "Fetching logs for invocations and comparing to snapshots" +for function_name in "${LAMBDA_HANDLERS[@]}"; do + function_snapshot_path="./snapshots/logs/$function_name.log" + + # Fetch logs with serverless cli + raw_logs=$(serverless logs -f $function_name --startTime $script_start_time) + + # Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions + logs=$( + echo "$raw_logs" | + # Filter serverless cli errors + sed '/Serverless: Recoverable error occurred/d' | + # Normalize Lambda runtime report logs + sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"):( )?[a-z0-9\.\-]+/\1:\2XXXX/g' | + # Normalize DD APM headers and AWS account ID + sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|account_id:) ?[0-9]+/\1XXXX/g" | + # Strip API key from logged requests + sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" | + # Normalize ISO combined date-time + sed -E "s/[0-9]{4}\-[0-9]{2}\-[0-9]{2}(T?)[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+ \(\-?[0-9:]+\))?Z/XXXX-XX-XXTXX:XX:XX.XXXZ/" | + # Normalize log timestamps + sed -E "s/[0-9]{4}(\-|\/)[0-9]{2}(\-|\/)[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+( \(\-?[0-9:]+\))?)?/XXXX-XX-XX XX:XX:XX.XXX/" | + # Normalize DD trace ID injection + sed -E "s/(dd\.trace_id=)[0-9]+ (dd\.span_id=)[0-9]+/\1XXXX \2XXXX/" | + # Normalize execution ID in logs prefix + sed -E $'s/[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\t/XXXX-XXXX-XXXX-XXXX-XXXX\t/' | + # Normalize minor package version tag so that these snapshots aren't broken on version bumps + sed -E "s/(dd_lambda_layer:datadog-go[0-9]+\.)[0-9]+\.[0-9]+/\1XX\.X/g" | + # Normalize data in logged traces + sed -E 's/"(span_id|parent_id|trace_id|start|duration|tcp\.local\.address|tcp\.local\.port|dns\.address|request_id|function_arn)":("?)[a-zA-Z0-9\.:\-]+("?)/"\1":\2XXXX\3/g' | + # Normalize data in logged traces + sed -E 's/"(points\\\":\[\[)([0-9]+)/\1XXXX/g' + + ) + + if [ ! -f $function_snapshot_path ]; then + # If no snapshot file exists yet, we create one + echo "Writing logs to $function_snapshot_path because no snapshot exists yet" + echo "$logs" >$function_snapshot_path + elif [ -n "$UPDATE_SNAPSHOTS" ]; then + # If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot + echo "Overwriting log snapshot for $function_snapshot_path" + echo "$logs" >$function_snapshot_path + else + # Compare new logs to snapshots + set +e # Don't exit this script if there is a diff + diff_output=$(echo "$logs" | diff - $function_snapshot_path) + if [ $? -eq 1 ]; then + echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):" + echo "$diff_output" + mismatch_found=true + else + echo "Ok: New logs for $function_name match snapshot" + fi + set -e + fi +done + +if [ "$mismatch_found" = true ]; then + echo "FAILURE: A mismatch between new data and a snapshot was found and printed above." + echo "If the change is expected, generate new snapshots by running 'UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX ./scripts/run_integration_tests.sh'" + exit 1 +fi + +if [ -n "$UPDATE_SNAPSHOTS" ]; then + echo "SUCCESS: Wrote new snapshots for all functions" + exit 0 +fi + +echo "SUCCESS: No difference found between snapshots and new return values or logs" diff --git a/tests/integration_tests/serverless.yml b/tests/integration_tests/serverless.yml new file mode 100644 index 00000000..ced1da51 --- /dev/null +++ b/tests/integration_tests/serverless.yml @@ -0,0 +1,29 @@ +service: hello-dog-go + +package: + exclude: + - ./** + include: + - ./bin/** + +provider: + name: aws + tracing: + lambda: true + apiGateway: true + memorySize: 128 + timeout: 30 + environment: + DD_API_KEY: ${env:DD_API_KEY} + DD_LOG_LEVEL: DEBUG + DD_INTEGRATION_TEST: true + DD_ENHANCED_METRICS: true + +functions: + hello-go: + runtime: go1.x + handler: bin/hello + events: + - http: + path: hello-go + method: get diff --git a/tests/integration_tests/snapshots/logs/hello-go.log b/tests/integration_tests/snapshots/logs/hello-go.log new file mode 100644 index 00000000..cdef83ed --- /dev/null +++ b/tests/integration_tests/snapshots/logs/hello-go.log @@ -0,0 +1,16 @@ +START RequestId: XXXX Version: $LATEST +XXXX-XX-XXTXX:XX:XX.XXXZ [Info] using daemon endpoints from environment variable AWS_XRAY_DAEMON_ADDRESS: 169.254.79.2:2000 +XXXX-XX-XXTXX:XX:XX.XXXZ [Info] Emitter using address: 169.254.79.2:2000 +XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: sending metric via log forwarder"} +{"m":"aws.lambda.enhanced.invocations","v":1,"e":XXXX,"t":["functionname:hello-dog-go-dev-hello-go","region:us-east-1","account_id:XXXX","memorysize:128","cold_start:true","resource:hello-dog-go-dev-hello-go","dd_lambda_layer:datadog-go1.XX.X"]} +XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: adding metric \"hello-go.dog\", with value 0.000000"} +Start Logging Headers +Request header: x-datadog-trace-id:XXXX +Request header: x-datadog-sampling-priority: 2 +Request header: x-datadog-parent-id:XXXX +End Logging Headers +XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: posting to url https://api.datadoghq.com/api/v1/distribution_points"} +XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Sending payload with body {\"series\":[{\"metric\":\"hello-go.dog\",\"tags\":[\"dd_lambda_layer:datadog-go1.XX.X\"],\"type\":\"distribution\",\points\":[[XXXX,[0]]]}]}"} +END RequestId: XXXX +REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 128 MB Max Memory Used: XXXX MB Init Duration: XXXX ms +XRAY TraceId: XXXX SegmentId: XXXX Sampled: true diff --git a/tests/integration_tests/snapshots/return_values/hello-go_api-gateway-get.json b/tests/integration_tests/snapshots/return_values/hello-go_api-gateway-get.json new file mode 100644 index 00000000..61a02875 --- /dev/null +++ b/tests/integration_tests/snapshots/return_values/hello-go_api-gateway-get.json @@ -0,0 +1,6 @@ +{ + "statusCode": 200, + "headers": null, + "multiValueHeaders": null, + "body": "hello, dog!" +}