diff --git a/.changelog/9ebe24c4791541e0840da49eab6f9d97.json b/.changelog/9ebe24c4791541e0840da49eab6f9d97.json new file mode 100644 index 00000000000..e723f199e35 --- /dev/null +++ b/.changelog/9ebe24c4791541e0840da49eab6f9d97.json @@ -0,0 +1,11 @@ +{ + "id": "9ebe24c4-7915-41e0-840d-a49eab6f9d97", + "type": "feature", + "description": "S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The checksum algorithm used by default now becomes CRC32. Checksum behavior can be configured using `when_supported` and `when_required` options - in code using RequestChecksumCalculation, in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in code using ResponseChecksumValidation, in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION.", + "modules": [ + ".", + "config", + "service/internal/checksum", + "service/s3" + ] +} diff --git a/aws/checksum.go b/aws/checksum.go new file mode 100644 index 00000000000..4152caade10 --- /dev/null +++ b/aws/checksum.go @@ -0,0 +1,33 @@ +package aws + +// RequestChecksumCalculation controls request checksum calculation workflow +type RequestChecksumCalculation int + +const ( + // RequestChecksumCalculationUnset is the unset value for RequestChecksumCalculation + RequestChecksumCalculationUnset RequestChecksumCalculation = iota + + // RequestChecksumCalculationWhenSupported indicates request checksum will be calculated + // if the operation supports input checksums + RequestChecksumCalculationWhenSupported + + // RequestChecksumCalculationWhenRequired indicates request checksum will be calculated + // if required by the operation or if user elects to set a checksum algorithm in request + RequestChecksumCalculationWhenRequired +) + +// ResponseChecksumValidation controls response checksum validation workflow +type ResponseChecksumValidation int + +const ( + // ResponseChecksumValidationUnset is the unset value for ResponseChecksumValidation + ResponseChecksumValidationUnset ResponseChecksumValidation = iota + + // ResponseChecksumValidationWhenSupported indicates response checksum will be validated + // if the operation supports output checksums + ResponseChecksumValidationWhenSupported + + // ResponseChecksumValidationWhenRequired indicates response checksum will only + // be validated if the operation requires output checksum validation + ResponseChecksumValidationWhenRequired +) diff --git a/aws/config.go b/aws/config.go index 16000d79279..a015cc5b20c 100644 --- a/aws/config.go +++ b/aws/config.go @@ -165,6 +165,33 @@ type Config struct { // Controls how a resolved AWS account ID is handled for endpoint routing. AccountIDEndpointMode AccountIDEndpointMode + + // RequestChecksumCalculation determines when request checksum calculation is performed. + // + // There are two possible values for this setting: + // + // 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated + // if the operation supports it, regardless of whether the user sets an algorithm in the request. + // + // 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user + // explicitly sets a checksum algorithm in the request. + // + // This setting is sourced from the environment variable AWS_REQUEST_CHECKSUM_CALCULATION + // or the shared config profile attribute "request_checksum_calculation". + RequestChecksumCalculation RequestChecksumCalculation + + // ResponseChecksumValidation determines when response checksum validation is performed + // + // There are two possible values for this setting: + // + // 1. ResponseChecksumValidationWhenSupported (default): The checksum is always validated + // if the operation supports it, regardless of whether the user sets the validation mode to ENABLED in request. + // + // 2. ResponseChecksumValidationWhenRequired: The checksum is only validated if the user + // explicitly sets the validation mode to ENABLED in the request + // This variable is sourced from environment variable AWS_RESPONSE_CHECKSUM_VALIDATION or + // the shared config profile attribute "response_checksum_validation". + ResponseChecksumValidation ResponseChecksumValidation } // NewConfig returns a new Config pointer that can be chained with builder diff --git a/aws/middleware/user_agent.go b/aws/middleware/user_agent.go index ab4e619073a..01d758d5ff8 100644 --- a/aws/middleware/user_agent.go +++ b/aws/middleware/user_agent.go @@ -76,19 +76,28 @@ type UserAgentFeature string // Enumerates UserAgentFeature. const ( - UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types) - UserAgentFeatureWaiter = "B" - UserAgentFeaturePaginator = "C" - UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard) - UserAgentFeatureRetryModeStandard = "E" - UserAgentFeatureRetryModeAdaptive = "F" - UserAgentFeatureS3Transfer = "G" - UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external) - UserAgentFeatureS3CryptoV2 = "I" // n/a - UserAgentFeatureS3ExpressBucket = "J" - UserAgentFeatureS3AccessGrants = "K" // not yet implemented - UserAgentFeatureGZIPRequestCompression = "L" - UserAgentFeatureProtocolRPCV2CBOR = "M" + UserAgentFeatureResourceModel UserAgentFeature = "A" // n/a (we don't generate separate resource types) + UserAgentFeatureWaiter = "B" + UserAgentFeaturePaginator = "C" + UserAgentFeatureRetryModeLegacy = "D" // n/a (equivalent to standard) + UserAgentFeatureRetryModeStandard = "E" + UserAgentFeatureRetryModeAdaptive = "F" + UserAgentFeatureS3Transfer = "G" + UserAgentFeatureS3CryptoV1N = "H" // n/a (crypto client is external) + UserAgentFeatureS3CryptoV2 = "I" // n/a + UserAgentFeatureS3ExpressBucket = "J" + UserAgentFeatureS3AccessGrants = "K" // not yet implemented + UserAgentFeatureGZIPRequestCompression = "L" + UserAgentFeatureProtocolRPCV2CBOR = "M" + UserAgentFeatureRequestChecksumCRC32 = "U" + UserAgentFeatureRequestChecksumCRC32C = "V" + UserAgentFeatureRequestChecksumCRC64 = "W" + UserAgentFeatureRequestChecksumSHA1 = "X" + UserAgentFeatureRequestChecksumSHA256 = "Y" + UserAgentFeatureRequestChecksumWhenSupported = "Z" + UserAgentFeatureRequestChecksumWhenRequired = "a" + UserAgentFeatureResponseChecksumWhenSupported = "b" + UserAgentFeatureResponseChecksumWhenRequired = "c" ) // RequestUserAgent is a build middleware that set the User-Agent for the request. diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java index b73fcaa2c46..dd52460d517 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java @@ -84,6 +84,10 @@ public class AddAwsConfigFields implements GoIntegration { private static final String SDK_ACCOUNTID_ENDPOINT_MODE = "AccountIDEndpointMode"; + private static final String REQUEST_CHECKSUM_CALCULATION = "RequestChecksumCalculation"; + + private static final String RESPONSE_CHECKSUM_VALIDATION = "ResponseChecksumValidation"; + private static final List AWS_CONFIG_FIELDS = ListUtils.of( AwsConfigField.builder() .name(REGION_CONFIG_NAME) @@ -244,6 +248,18 @@ public class AddAwsConfigFields implements GoIntegration { .type(SdkGoTypes.Aws.AccountIDEndpointMode) .documentation("Indicates how aws account ID is applied in endpoint2.0 routing") .servicePredicate(AccountIDEndpointRouting::hasAccountIdEndpoints) + .build(), + AwsConfigField.builder() + .name(REQUEST_CHECKSUM_CALCULATION) + .type(SdkGoTypes.Aws.RequestChecksumCalculation) + .documentation("Indicates how user opt-in/out request checksum calculation") + .servicePredicate(AwsHttpChecksumGenerator::hasInputChecksumTrait) + .build(), + AwsConfigField.builder() + .name(RESPONSE_CHECKSUM_VALIDATION) + .type(SdkGoTypes.Aws.ResponseChecksumValidation) + .documentation("Indicates how user opt-in/out response checksum validation") + .servicePredicate(AwsHttpChecksumGenerator::hasOutputChecksumTrait) .build() ); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java index a23b71cdbc6..f21c39e665f 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java @@ -22,6 +22,7 @@ import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ServiceShape; @@ -73,9 +74,7 @@ public byte getOrder() { @Override public void processFinalizedModel(GoSettings settings, Model model) { ServiceShape service = settings.getService(model); - for (ShapeId operationId : service.getAllOperations()) { - final OperationShape operation = model.expectShape(operationId, OperationShape.class); - + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { // Create a symbol provider because one is not available in this call. SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings); @@ -128,8 +127,7 @@ public void writeAdditionalFiles( boolean supportsComputeInputChecksumsWorkflow = false; boolean supportsChecksumValidationWorkflow = false; - for (ShapeId operationID : service.getAllOperations()) { - OperationShape operation = model.expectShape(operationID, OperationShape.class); + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { if (!hasChecksumTrait(model, service, operation)) { continue; } @@ -178,11 +176,11 @@ public List getClientPlugins() { } // return true if operation shape is decorated with `httpChecksum` trait. - private boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) { return operation.hasTrait(HttpChecksumTrait.class); } - private boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { if (!hasChecksumTrait(model, service, operation)) { return false; } @@ -190,7 +188,16 @@ private boolean hasInputChecksumTrait(Model model, ServiceShape service, Operati return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent(); } - private boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + public static boolean hasInputChecksumTrait(Model model, ServiceShape service) { + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { + if (hasInputChecksumTrait(model, service, operation)) { + return true; + } + } + return false; + } + + private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { if (!hasChecksumTrait(model, service, operation)) { return false; } @@ -198,6 +205,15 @@ private boolean hasOutputChecksumTrait(Model model, ServiceShape service, Operat return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty(); } + public static boolean hasOutputChecksumTrait(Model model, ServiceShape service) { + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { + if (hasOutputChecksumTrait(model, service, operation)) { + return true; + } + } + return false; + } + private boolean isS3ServiceShape(Model model, ServiceShape service) { String serviceId = service.expectTrait(ServiceTrait.class).getSdkId(); return serviceId.equalsIgnoreCase("S3"); @@ -244,6 +260,7 @@ private void writeInputMiddlewareHelper( return $T(stack, $T{ GetAlgorithm: $L, RequireChecksum: $L, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: $L, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: $L, @@ -284,6 +301,7 @@ private void writeOutputMiddlewareHelper( writer.write(""" return $T(stack, $T{ GetValidationMode: $L, + ResponseChecksumValidation: options.ResponseChecksumValidation, ValidationAlgorithms: $L, IgnoreMultipartValidation: $L, LogValidationSkipped: true, @@ -293,7 +311,6 @@ private void writeOutputMiddlewareHelper( AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(), SymbolUtils.createValueSymbolBuilder("OutputMiddlewareOptions", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(), - getRequestValidationModeAccessorFuncName(operationName), convertToGoStringList(responseAlgorithms), ignoreMultipartChecksumValidationMap.getOrDefault( diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java index c608d6b37dd..4d58e78cb3a 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java @@ -23,6 +23,7 @@ import java.util.TreeSet; import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency; import software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill; +import software.amazon.smithy.aws.traits.HttpChecksumTrait; import software.amazon.smithy.aws.traits.ServiceTrait; import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait; import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait; @@ -67,7 +68,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration { private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware"; private static final String CONVERT_TO_PRESIGN_TYPE_NAME = "presignConverter"; private static final String NOP_HTTP_CLIENT_OPTION_FUNC_NAME = "withNopHTTPClientAPIOption"; - + private static final String NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME = "withNoDefaultChecksumAPIOption"; private static final String PRESIGN_CLIENT = "PresignClient"; private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true); @@ -218,7 +219,11 @@ public void writeAdditionalFiles( writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape); }); + boolean supportsComputeInputChecksumsWorkflow = false; for (OperationShape operationShape : TopDownIndex.of(model).getContainedOperations(serviceShape)) { + if (hasInputChecksumTrait(operationShape)) { + supportsComputeInputChecksumsWorkflow = true; + } if (!validOperations.contains(operationShape.getId())) { continue; } @@ -231,6 +236,10 @@ public void writeAdditionalFiles( writeS3AddAsUnsignedPayloadHelper(writer, model, symbolProvider, serviceShape, operationShape); }); } + + if (supportsComputeInputChecksumsWorkflow) { + writePresignRequestChecksumConfigHelpers(settings, goDelegator); + } } private void writePresignOperationFunction( @@ -263,6 +272,10 @@ private void writePresignOperationFunction( writer.write("clientOptFns := append(options.ClientOptions, $L)", NOP_HTTP_CLIENT_OPTION_FUNC_NAME); writer.write(""); + if (hasInputChecksumTrait(operationShape)) { + writer.write("clientOptFns = append(options.ClientOptions, $L)", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME); + writer.write(""); + } writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, clientOptFns,", ")", operationSymbol.getName(), () -> { @@ -572,6 +585,29 @@ private void writePresignClientHelpers( writer.write(""); } + private void writePresignRequestChecksumConfigHelpers( + GoSettings settings, + GoDelegator goDelegator + ) { + goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate(""" + func $fn:L(options *Options) { + options.RequestChecksumCalculation = $requestChecksumCalculationWhenRequired:T + }""", + Map.of( + "fn", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME, + "requestChecksumCalculationWhenRequired", + AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculationWhenRequired") + ))); + } + + private static boolean hasInputChecksumTrait(OperationShape operation) { + if (!operation.hasTrait(HttpChecksumTrait.class)) { + return false; + } + HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class); + return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent(); + } + /** * Writes the presigner interface used by the presign url client */ diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java index c91a3f4c2aa..e0158ba62af 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java @@ -39,6 +39,8 @@ public static final class Aws { public static final Symbol AccountIDEndpointModeRequired = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeRequired"); public static final Symbol AccountIDEndpointModeDisabled = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeDisabled"); + public static final Symbol RequestChecksumCalculation = AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculation"); + public static final Symbol ResponseChecksumValidation = AwsGoDependency.AWS_CORE.valueSymbol("ResponseChecksumValidation"); public static final class Middleware { public static final Symbol GetRequiresLegacyEndpoints = AwsGoDependency.AWS_MIDDLEWARE.valueSymbol("GetRequiresLegacyEndpoints"); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/ChecksumMetricsTracking.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/ChecksumMetricsTracking.java new file mode 100644 index 00000000000..693e1931723 --- /dev/null +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/ChecksumMetricsTracking.java @@ -0,0 +1,139 @@ +package software.amazon.smithy.aws.go.codegen.customization; + +import software.amazon.smithy.aws.go.codegen.AwsGoDependency; +import software.amazon.smithy.aws.traits.HttpChecksumTrait; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoDelegator; +import software.amazon.smithy.go.codegen.GoSettings; +import software.amazon.smithy.go.codegen.SmithyGoDependency; +import software.amazon.smithy.go.codegen.integration.GoIntegration; +import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; +import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; + +import java.util.List; +import java.util.Map; + +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SymbolUtils.buildPackageSymbol; + +public class ChecksumMetricsTracking implements GoIntegration { + private static final MiddlewareRegistrar RequestMIDDLEWARE = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addRequestChecksumMetricsTracking")) + .useClientOptions() + .build(); + private static final MiddlewareRegistrar ResponseMIDDLEWARE = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addResponseChecksumMetricsTracking")) + .useClientOptions() + .build(); + + @Override + public List getClientPlugins() { + return List.of( + RuntimeClientPlugin.builder() + .operationPredicate((m, s, o) -> { + if (!hasInputChecksumTrait(m, s, o)) { + return false; + } + return true; + }) + .registerMiddleware(RequestMIDDLEWARE) + .build(), + RuntimeClientPlugin.builder() + .operationPredicate((m, s, o) -> { + if (!hasOutputChecksumTrait(m, s, o)) { + return false; + } + return true; + }) + .registerMiddleware(ResponseMIDDLEWARE) + .build() + ); + } + + private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + return operation.hasTrait(HttpChecksumTrait.class); + } + + private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + if (!hasChecksumTrait(model, service, operation)) { + return false; + } + HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class); + return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent(); + } + + private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) { + if (!hasChecksumTrait(model, service, operation)) { + return false; + } + HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class); + return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty(); + } + + @Override + public void writeAdditionalFiles(GoSettings settings, Model model, SymbolProvider symbolProvider, GoDelegator goDelegator) { + ServiceShape service = settings.getService(model); + boolean supportsComputeInputChecksumsWorkflow = false; + boolean supportsChecksumValidationWorkflow = false; + + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { + if (!hasChecksumTrait(model, service, operation)) { + continue; + } + + if (hasInputChecksumTrait(model, service, operation)) { + supportsComputeInputChecksumsWorkflow = true; + } + + if (hasOutputChecksumTrait(model, service, operation)) { + supportsChecksumValidationWorkflow = true; + } + } + + if (supportsComputeInputChecksumsWorkflow) { + goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate(""" + func addRequestChecksumMetricsTracking(stack $stack:P, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&$requestMetricsTracking:P{ + RequestChecksumCalculation: options.RequestChecksumCalculation, + UserAgent: ua, + }, "UserAgent", $before:T) + }""", + Map.of( + "stack", SmithyGoDependency.SMITHY_MIDDLEWARE.struct("Stack"), + "requestMetricsTracking", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM.valueSymbol("RequestChecksumMetricsTracking"), + "before", SmithyGoDependency.SMITHY_MIDDLEWARE + .valueSymbol("Before") + ))); + } + + if (supportsChecksumValidationWorkflow) { + goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate(""" + func addResponseChecksumMetricsTracking(stack $stack:P, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&$responseMetricsTracking:P{ + ResponseChecksumValidation: options.ResponseChecksumValidation, + UserAgent: ua, + }, "UserAgent", $before:T) + }""", + Map.of( + "stack", SmithyGoDependency.SMITHY_MIDDLEWARE.struct("Stack"), + "responseMetricsTracking", AwsGoDependency.SERVICE_INTERNAL_CHECKSUM.valueSymbol("ResponseChecksumMetricsTracking"), + "before", SmithyGoDependency.SMITHY_MIDDLEWARE + .valueSymbol("Before") + ))); + } + } +} diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 14a12d710aa..bc78b9d68d7 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -88,4 +88,4 @@ software.amazon.smithy.aws.go.codegen.customization.s3.ExpressUserAgent software.amazon.smithy.aws.go.codegen.customization.BackfillRequiredTrait software.amazon.smithy.aws.go.codegen.customization.DeprecateService software.amazon.smithy.aws.go.codegen.customization.BasicUserAgentFeatures - +software.amazon.smithy.aws.go.codegen.customization.ChecksumMetricsTracking diff --git a/config/config.go b/config/config.go index d5226cb0437..0577c61869e 100644 --- a/config/config.go +++ b/config/config.go @@ -83,6 +83,12 @@ var defaultAWSConfigResolvers = []awsConfigResolver{ // Sets the AccountIDEndpointMode if present in env var or shared config profile resolveAccountIDEndpointMode, + + // Sets the RequestChecksumCalculation if present in env var or shared config profile + resolveRequestChecksumCalculation, + + // Sets the ResponseChecksumValidation if present in env var or shared config profile + resolveResponseChecksumValidation, } // A Config represents a generic configuration value or set of values. This type diff --git a/config/env_config.go b/config/env_config.go index 3a06f1412a7..a672850f53e 100644 --- a/config/env_config.go +++ b/config/env_config.go @@ -83,6 +83,9 @@ const ( awsAccountIDEnv = "AWS_ACCOUNT_ID" awsAccountIDEndpointModeEnv = "AWS_ACCOUNT_ID_ENDPOINT_MODE" + + awsRequestChecksumCalculation = "AWS_REQUEST_CHECKSUM_CALCULATION" + awsResponseChecksumValidation = "AWS_RESPONSE_CHECKSUM_VALIDATION" ) var ( @@ -296,6 +299,12 @@ type EnvConfig struct { // Indicates whether account ID will be required/ignored in endpoint2.0 routing AccountIDEndpointMode aws.AccountIDEndpointMode + + // Indicates whether request checksum should be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + + // Indicates whether response checksum should be validated + ResponseChecksumValidation aws.ResponseChecksumValidation } // loadEnvConfig reads configuration values from the OS's environment variables. @@ -400,6 +409,13 @@ func NewEnvConfig() (EnvConfig, error) { return cfg, err } + if err := setRequestChecksumCalculationFromEnvVal(&cfg.RequestChecksumCalculation, []string{awsRequestChecksumCalculation}); err != nil { + return cfg, err + } + if err := setResponseChecksumValidationFromEnvVal(&cfg.ResponseChecksumValidation, []string{awsResponseChecksumValidation}); err != nil { + return cfg, err + } + return cfg, nil } @@ -432,6 +448,14 @@ func (c EnvConfig) getAccountIDEndpointMode(context.Context) (aws.AccountIDEndpo return c.AccountIDEndpointMode, len(c.AccountIDEndpointMode) > 0, nil } +func (c EnvConfig) getRequestChecksumCalculation(context.Context) (aws.RequestChecksumCalculation, bool, error) { + return c.RequestChecksumCalculation, c.RequestChecksumCalculation > 0, nil +} + +func (c EnvConfig) getResponseChecksumValidation(context.Context) (aws.ResponseChecksumValidation, bool, error) { + return c.ResponseChecksumValidation, c.ResponseChecksumValidation > 0, nil +} + // GetRetryMaxAttempts returns the value of AWS_MAX_ATTEMPTS if was specified, // and not 0. func (c EnvConfig) GetRetryMaxAttempts(ctx context.Context) (int, bool, error) { @@ -528,6 +552,45 @@ func setAIDEndPointModeFromEnvVal(m *aws.AccountIDEndpointMode, keys []string) e return nil } +func setRequestChecksumCalculationFromEnvVal(m *aws.RequestChecksumCalculation, keys []string) error { + for _, k := range keys { + value := os.Getenv(k) + if len(value) == 0 { + continue + } + + switch strings.ToLower(value) { + case checksumWhenSupported: + *m = aws.RequestChecksumCalculationWhenSupported + case checksumWhenRequired: + *m = aws.RequestChecksumCalculationWhenRequired + default: + return fmt.Errorf("invalid value for environment variable, %s=%s, must be when_supported/when_required", k, value) + } + } + return nil +} + +func setResponseChecksumValidationFromEnvVal(m *aws.ResponseChecksumValidation, keys []string) error { + for _, k := range keys { + value := os.Getenv(k) + if len(value) == 0 { + continue + } + + switch strings.ToLower(value) { + case checksumWhenSupported: + *m = aws.ResponseChecksumValidationWhenSupported + case checksumWhenRequired: + *m = aws.ResponseChecksumValidationWhenRequired + default: + return fmt.Errorf("invalid value for environment variable, %s=%s, must be when_supported/when_required", k, value) + } + + } + return nil +} + // GetRegion returns the AWS Region if set in the environment. Returns an empty // string if not set. func (c EnvConfig) getRegion(ctx context.Context) (string, bool, error) { diff --git a/config/env_config_test.go b/config/env_config_test.go index 02c00d37aa7..870c46509bc 100644 --- a/config/env_config_test.go +++ b/config/env_config_test.go @@ -514,7 +514,6 @@ func TestNewEnvConfig(t *testing.T) { Config: EnvConfig{ AccountIDEndpointMode: aws.AccountIDEndpointModeRequired, }, - WantErr: false, }, 47: { Env: map[string]string{ @@ -523,6 +522,52 @@ func TestNewEnvConfig(t *testing.T) { Config: EnvConfig{}, WantErr: true, }, + 48: { + Env: map[string]string{ + "AWS_REQUEST_CHECKSUM_CALCULATION": "WHEN_SUPPORTED", + }, + Config: EnvConfig{ + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + }, + }, + 49: { + Env: map[string]string{ + "AWS_REQUEST_CHECKSUM_CALCULATION": "when_required", + }, + Config: EnvConfig{ + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + }, + }, + 50: { + Env: map[string]string{ + "AWS_REQUEST_CHECKSUM_CALCULATION": "blabla", + }, + Config: EnvConfig{}, + WantErr: true, + }, + 51: { + Env: map[string]string{ + "AWS_RESPONSE_CHECKSUM_VALIDATION": "WHEN_SUPPORTED", + }, + Config: EnvConfig{ + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + }, + }, + 52: { + Env: map[string]string{ + "AWS_RESPONSE_CHECKSUM_VALIDATION": "when_Required", + }, + Config: EnvConfig{ + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + }, + }, + 53: { + Env: map[string]string{ + "AWS_RESPONSE_CHECKSUM_VALIDATION": "blabla", + }, + Config: EnvConfig{}, + WantErr: true, + }, } for i, c := range cases { diff --git a/config/load_options.go b/config/load_options.go index dc6c7d29a83..0810ecf16a2 100644 --- a/config/load_options.go +++ b/config/load_options.go @@ -216,8 +216,15 @@ type LoadOptions struct { // Whether S3 Express auth is disabled. S3DisableExpressAuth *bool + // Whether account id should be built into endpoint resolution AccountIDEndpointMode aws.AccountIDEndpointMode + // Specify if request checksum should be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + + // Specifies if response checksum should be validated + ResponseChecksumValidation aws.ResponseChecksumValidation + // Service endpoint override. This value is not necessarily final and is // passed to the service's EndpointResolverV2 for further delegation. BaseEndpoint string @@ -288,6 +295,14 @@ func (o LoadOptions) getAccountIDEndpointMode(ctx context.Context) (aws.AccountI return o.AccountIDEndpointMode, len(o.AccountIDEndpointMode) > 0, nil } +func (o LoadOptions) getRequestChecksumCalculation(ctx context.Context) (aws.RequestChecksumCalculation, bool, error) { + return o.RequestChecksumCalculation, o.RequestChecksumCalculation > 0, nil +} + +func (o LoadOptions) getResponseChecksumValidation(ctx context.Context) (aws.ResponseChecksumValidation, bool, error) { + return o.ResponseChecksumValidation, o.ResponseChecksumValidation > 0, nil +} + func (o LoadOptions) getBaseEndpoint(context.Context) (string, bool, error) { return o.BaseEndpoint, o.BaseEndpoint != "", nil } @@ -357,6 +372,26 @@ func WithAccountIDEndpointMode(m aws.AccountIDEndpointMode) LoadOptionsFunc { } } +// WithRequestChecksumCalculation is a helper function to construct functional options +// that sets RequestChecksumCalculation on config's LoadOptions +func WithRequestChecksumCalculation(c aws.RequestChecksumCalculation) LoadOptionsFunc { + return func(o *LoadOptions) error { + if c > 0 { + o.RequestChecksumCalculation = c + } + return nil + } +} + +// WithResponseChecksumValidation is a helper function to construct functional options +// that sets ResponseChecksumValidation on config's LoadOptions +func WithResponseChecksumValidation(v aws.ResponseChecksumValidation) LoadOptionsFunc { + return func(o *LoadOptions) error { + o.ResponseChecksumValidation = v + return nil + } +} + // getDefaultRegion returns DefaultRegion from config's LoadOptions func (o LoadOptions) getDefaultRegion(ctx context.Context) (string, bool, error) { if len(o.DefaultRegion) == 0 { diff --git a/config/provider.go b/config/provider.go index 043781f1f77..a8ff40d846b 100644 --- a/config/provider.go +++ b/config/provider.go @@ -242,6 +242,40 @@ func getAccountIDEndpointMode(ctx context.Context, configs configs) (value aws.A return } +// requestChecksumCalculationProvider provides access to the RequestChecksumCalculation +type requestChecksumCalculationProvider interface { + getRequestChecksumCalculation(context.Context) (aws.RequestChecksumCalculation, bool, error) +} + +func getRequestChecksumCalculation(ctx context.Context, configs configs) (value aws.RequestChecksumCalculation, found bool, err error) { + for _, cfg := range configs { + if p, ok := cfg.(requestChecksumCalculationProvider); ok { + value, found, err = p.getRequestChecksumCalculation(ctx) + if err != nil || found { + break + } + } + } + return +} + +// responseChecksumValidationProvider provides access to the ResponseChecksumValidation +type responseChecksumValidationProvider interface { + getResponseChecksumValidation(context.Context) (aws.ResponseChecksumValidation, bool, error) +} + +func getResponseChecksumValidation(ctx context.Context, configs configs) (value aws.ResponseChecksumValidation, found bool, err error) { + for _, cfg := range configs { + if p, ok := cfg.(responseChecksumValidationProvider); ok { + value, found, err = p.getResponseChecksumValidation(ctx) + if err != nil || found { + break + } + } + } + return +} + // ec2IMDSRegionProvider provides access to the ec2 imds region // configuration value type ec2IMDSRegionProvider interface { diff --git a/config/resolve.go b/config/resolve.go index 41009c7da06..a68bd0993f7 100644 --- a/config/resolve.go +++ b/config/resolve.go @@ -182,6 +182,36 @@ func resolveAccountIDEndpointMode(ctx context.Context, cfg *aws.Config, configs return nil } +// resolveRequestChecksumCalculation extracts the RequestChecksumCalculation from the configs slice's +// SharedConfig or EnvConfig +func resolveRequestChecksumCalculation(ctx context.Context, cfg *aws.Config, configs configs) error { + c, found, err := getRequestChecksumCalculation(ctx, configs) + if err != nil { + return err + } + + if !found { + c = aws.RequestChecksumCalculationWhenSupported + } + cfg.RequestChecksumCalculation = c + return nil +} + +// resolveResponseValidation extracts the ResponseChecksumValidation from the configs slice's +// SharedConfig or EnvConfig +func resolveResponseChecksumValidation(ctx context.Context, cfg *aws.Config, configs configs) error { + c, found, err := getResponseChecksumValidation(ctx, configs) + if err != nil { + return err + } + + if !found { + c = aws.ResponseChecksumValidationWhenSupported + } + cfg.ResponseChecksumValidation = c + return nil +} + // resolveDefaultRegion extracts the first instance of a default region and sets `aws.Config.Region` to the default // region if region had not been resolved from other sources. func resolveDefaultRegion(ctx context.Context, cfg *aws.Config, configs configs) error { diff --git a/config/resolve_test.go b/config/resolve_test.go index e14199ce923..c8df627bbb8 100644 --- a/config/resolve_test.go +++ b/config/resolve_test.go @@ -269,6 +269,80 @@ func TestResolveAccountIDEndpointMode(t *testing.T) { } } +func TestResolveRequestChecksumCalculation(t *testing.T) { + cases := map[string]struct { + RequestChecksumCalculation aws.RequestChecksumCalculation + ExpectCalculation aws.RequestChecksumCalculation + }{ + "checksum calculation when required": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + ExpectCalculation: aws.RequestChecksumCalculationWhenRequired, + }, + "default when unset": { + ExpectCalculation: aws.RequestChecksumCalculationWhenSupported, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var options LoadOptions + optFns := []func(*LoadOptions) error{ + WithRequestChecksumCalculation(c.RequestChecksumCalculation), + } + + for _, optFn := range optFns { + optFn(&options) + } + + configs := configs{options} + var cfg aws.Config + if err := resolveRequestChecksumCalculation(context.Background(), &cfg, configs); err != nil { + t.Fatalf("expect no error, got %v", err) + } + if e, a := c.ExpectCalculation, cfg.RequestChecksumCalculation; e != a { + t.Errorf("expect RequestChecksumCalculation to be %v, got %v", e, a) + } + }) + } +} + +func TestResolveResponseChecksumValidation(t *testing.T) { + cases := map[string]struct { + ResponseChecksumValidation aws.ResponseChecksumValidation + ExpectValidation aws.ResponseChecksumValidation + }{ + "checksum validation when required": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + ExpectValidation: aws.ResponseChecksumValidationWhenRequired, + }, + "default when unset": { + ExpectValidation: aws.ResponseChecksumValidationWhenSupported, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var options LoadOptions + optFns := []func(*LoadOptions) error{ + WithResponseChecksumValidation(c.ResponseChecksumValidation), + } + + for _, optFn := range optFns { + optFn(&options) + } + + configs := configs{options} + var cfg aws.Config + if err := resolveResponseChecksumValidation(context.Background(), &cfg, configs); err != nil { + t.Fatalf("expect no error, got %v", err) + } + if e, a := c.ExpectValidation, cfg.ResponseChecksumValidation; e != a { + t.Errorf("expect ResponseChecksumValidation to be %v, got %v", e, a) + } + }) + } +} + func TestResolveCredentialsProvider(t *testing.T) { var options LoadOptions optFns := []func(options *LoadOptions) error{ diff --git a/config/shared_config.go b/config/shared_config.go index d7a2b5307ea..00b071fe6f1 100644 --- a/config/shared_config.go +++ b/config/shared_config.go @@ -118,6 +118,11 @@ const ( accountIDKey = "aws_account_id" accountIDEndpointMode = "account_id_endpoint_mode" + + requestChecksumCalculationKey = "request_checksum_calculation" + responseChecksumValidationKey = "response_checksum_validation" + checksumWhenSupported = "when_supported" + checksumWhenRequired = "when_required" ) // defaultSharedConfigProfile allows for swapping the default profile for testing @@ -346,6 +351,12 @@ type SharedConfig struct { S3DisableExpressAuth *bool AccountIDEndpointMode aws.AccountIDEndpointMode + + // RequestChecksumCalculation indicates if the request checksum should be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + + // ResponseChecksumValidation indicates if the response checksum should be validated + ResponseChecksumValidation aws.ResponseChecksumValidation } func (c SharedConfig) getDefaultsMode(ctx context.Context) (value aws.DefaultsMode, ok bool, err error) { @@ -1133,6 +1144,13 @@ func (c *SharedConfig) setFromIniSection(profile string, section ini.Section) er return fmt.Errorf("failed to load %s from shared config, %w", accountIDEndpointMode, err) } + if err := updateRequestChecksumCalculation(&c.RequestChecksumCalculation, section, requestChecksumCalculationKey); err != nil { + return fmt.Errorf("failed to load %s from shared config, %w", requestChecksumCalculationKey, err) + } + if err := updateResponseChecksumValidation(&c.ResponseChecksumValidation, section, responseChecksumValidationKey); err != nil { + return fmt.Errorf("failed to load %s from shared config, %w", responseChecksumValidationKey, err) + } + // Shared Credentials creds := aws.Credentials{ AccessKeyID: section.String(accessKeyIDKey), @@ -1207,6 +1225,42 @@ func updateAIDEndpointMode(m *aws.AccountIDEndpointMode, sec ini.Section, key st return nil } +func updateRequestChecksumCalculation(m *aws.RequestChecksumCalculation, sec ini.Section, key string) error { + if !sec.Has(key) { + return nil + } + + v := sec.String(key) + switch strings.ToLower(v) { + case checksumWhenSupported: + *m = aws.RequestChecksumCalculationWhenSupported + case checksumWhenRequired: + *m = aws.RequestChecksumCalculationWhenRequired + default: + return fmt.Errorf("invalid value for shared config profile field, %s=%s, must be when_supported/when_required", key, v) + } + + return nil +} + +func updateResponseChecksumValidation(m *aws.ResponseChecksumValidation, sec ini.Section, key string) error { + if !sec.Has(key) { + return nil + } + + v := sec.String(key) + switch strings.ToLower(v) { + case checksumWhenSupported: + *m = aws.ResponseChecksumValidationWhenSupported + case checksumWhenRequired: + *m = aws.ResponseChecksumValidationWhenRequired + default: + return fmt.Errorf("invalid value for shared config profile field, %s=%s, must be when_supported/when_required", key, v) + } + + return nil +} + func (c SharedConfig) getRequestMinCompressSizeBytes(ctx context.Context) (int64, bool, error) { if c.RequestMinCompressSizeBytes == nil { return 0, false, nil @@ -1225,6 +1279,14 @@ func (c SharedConfig) getAccountIDEndpointMode(ctx context.Context) (aws.Account return c.AccountIDEndpointMode, len(c.AccountIDEndpointMode) > 0, nil } +func (c SharedConfig) getRequestChecksumCalculation(ctx context.Context) (aws.RequestChecksumCalculation, bool, error) { + return c.RequestChecksumCalculation, c.RequestChecksumCalculation > 0, nil +} + +func (c SharedConfig) getResponseChecksumValidation(ctx context.Context) (aws.ResponseChecksumValidation, bool, error) { + return c.ResponseChecksumValidation, c.ResponseChecksumValidation > 0, nil +} + func updateDefaultsMode(mode *aws.DefaultsMode, section ini.Section, key string) error { if !section.Has(key) { return nil diff --git a/config/shared_config_test.go b/config/shared_config_test.go index ee3f0705e8b..8d71818c8d4 100644 --- a/config/shared_config_test.go +++ b/config/shared_config_test.go @@ -758,6 +758,54 @@ func TestNewSharedConfig(t *testing.T) { }, Err: fmt.Errorf("invalid value for shared config profile field, account_id_endpoint_mode=blabla, must be preferred/required/disabled"), }, + "profile with request checksum calculation when supported": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_checksum_calculation_when_supported", + Expected: SharedConfig{ + Profile: "request_checksum_calculation_when_supported", + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + }, + }, + "profile with request checksum calculation when required": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_checksum_calculation_when_required", + Expected: SharedConfig{ + Profile: "request_checksum_calculation_when_required", + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + }, + }, + "profile with invalid request checksum calculation": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_checksum_calculation_error", + Expected: SharedConfig{ + Profile: "request_checksum_calculation_error", + }, + Err: fmt.Errorf("invalid value for shared config profile field, request_checksum_calculation=blabla, must be when_supported/when_required"), + }, + "profile with response checksum validation when supported": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "response_checksum_validation_when_supported", + Expected: SharedConfig{ + Profile: "response_checksum_validation_when_supported", + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + }, + }, + "profile with response checksum validation when required": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "response_checksum_validation_when_required", + Expected: SharedConfig{ + Profile: "response_checksum_validation_when_required", + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + }, + }, + "profile with invalid response checksum validation": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "response_checksum_validation_error", + Expected: SharedConfig{ + Profile: "response_checksum_validation_error", + }, + Err: fmt.Errorf("invalid value for shared config profile field, response_checksum_validation=blabla, must be when_supported/when_required"), + }, } for name, c := range cases { diff --git a/config/testdata/shared_config b/config/testdata/shared_config index b2cbc81873b..c7159d52bff 100644 --- a/config/testdata/shared_config +++ b/config/testdata/shared_config @@ -328,3 +328,22 @@ account_id_endpoint_mode = disabled [profile account_id_endpoint_mode_error] account_id_endpoint_mode = blabla + +[profile request_checksum_calculation_when_supported] +request_checksum_calculation = when_supported + +[profile request_checksum_calculation_when_required] +request_checksum_calculation = WHEN_REQUIRED + +[profile request_checksum_calculation_error] +request_checksum_calculation = blabla + +[profile response_checksum_validation_when_supported] +response_checksum_validation = When_SUPPORTED + +[profile response_checksum_validation_when_required] +response_checksum_validation = when_required + +[profile response_checksum_validation_error] +response_checksum_validation = blabla + diff --git a/feature/s3/manager/integ_upload_test.go b/feature/s3/manager/integ_upload_test.go index 2b980fdc89c..96383c243eb 100644 --- a/feature/s3/manager/integ_upload_test.go +++ b/feature/s3/manager/integ_upload_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/aws/aws-sdk-go-v2/aws" - v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -160,9 +159,10 @@ func TestInteg_UploadPresetChecksum(t *testing.T) { expectETag string }{ "auto single part": { - "no checksum": { - payload: bytes.NewReader(singlePartBytes), - expectETag: singlePartETag, + "no checksum algorithm passed": { + payload: bytes.NewReader(singlePartBytes), + expectChecksumCRC32: singlePartCRC32, + expectETag: singlePartETag, }, "CRC32": { algorithm: s3types.ChecksumAlgorithmCrc32, @@ -215,29 +215,34 @@ func TestInteg_UploadPresetChecksum(t *testing.T) { expectETag: singlePartETag, }, "MD5": { - payload: bytes.NewReader(singlePartBytes), - contentMD5: singlePartMD5, - expectETag: singlePartETag, + payload: bytes.NewReader(singlePartBytes), + contentMD5: singlePartMD5, + expectChecksumCRC32: singlePartCRC32, + expectETag: singlePartETag, }, }, "auto multipart part": { - "no checksum": { + "no checksum algorithm passed": { payload: bytes.NewReader(multiPartBytes), expectParts: []s3types.CompletedPart{ { - ETag: aws.String(singlePartETag), - PartNumber: aws.Int32(1), + ChecksumCRC32: aws.String(singlePartCRC32), + ETag: aws.String(singlePartETag), + PartNumber: aws.Int32(1), }, { - ETag: aws.String(singlePartETag), - PartNumber: aws.Int32(2), + ChecksumCRC32: aws.String(singlePartCRC32), + ETag: aws.String(singlePartETag), + PartNumber: aws.Int32(2), }, { - ETag: aws.String(multiPartTailETag), - PartNumber: aws.Int32(3), + ChecksumCRC32: aws.String(multiPartTailCRC32), + ETag: aws.String(multiPartTailETag), + PartNumber: aws.Int32(3), }, }, - expectETag: multiPartETag, + expectChecksumCRC32: multiPartCRC32, + expectETag: multiPartETag, }, "CRC32": { algorithm: s3types.ChecksumAlgorithmCrc32, @@ -484,21 +489,21 @@ func toStringPtr(v string) *string { return &v } -type invalidateHash struct{} +type failedMultipartUpload struct{} -func (b *invalidateHash) ID() string { - return "s3manager:InvalidateHash" +func (m *failedMultipartUpload) ID() string { + return "s3manager:FailedMultipartUpload" } -func (b *invalidateHash) RegisterMiddleware(stack *middleware.Stack) error { - return stack.Serialize.Add(b, middleware.After) +func (m *failedMultipartUpload) RegisterMiddleware(stack *middleware.Stack) error { + return stack.Serialize.Add(m, middleware.After) } -func (b *invalidateHash) HandleSerialize(ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler) ( +func (m *failedMultipartUpload) HandleSerialize(ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler) ( out middleware.SerializeOutput, metadata middleware.Metadata, err error, ) { if input, ok := in.Parameters.(*s3.UploadPartInput); ok && aws.ToInt32(input.PartNumber) == 2 { - ctx = v4.SetPayloadHash(ctx, "000") + return out, metadata, fmt.Errorf("multipart upload error") } return next.HandleSerialize(ctx, in) @@ -509,7 +514,7 @@ func TestInteg_UploadFailCleanup(t *testing.T) { mgr := manager.NewUploader(client, func(u *manager.Uploader) { u.LeavePartsOnError = false u.ClientOptions = append(u.ClientOptions, func(options *s3.Options) { - options.APIOptions = append(options.APIOptions, (&invalidateHash{}).RegisterMiddleware) + options.APIOptions = append(options.APIOptions, (&failedMultipartUpload{}).RegisterMiddleware) }) }) _, err := mgr.Upload(context.Background(), &s3.PutObjectInput{ diff --git a/feature/s3/manager/upload.go b/feature/s3/manager/upload.go index 18aff6e95be..c6d04f1e99b 100644 --- a/feature/s3/manager/upload.go +++ b/feature/s3/manager/upload.go @@ -584,6 +584,8 @@ func (a completedParts) Less(i, j int) bool { // upload will perform a multipart upload using the firstBuf buffer containing // the first chunk of data. func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadOutput, error) { + u.initChecksumAlgorithm() + var params s3.CreateMultipartUploadInput awsutil.Copy(¶ms, u.in) @@ -752,6 +754,25 @@ func (u *multiuploader) send(c chunk) error { return nil } +func (u *multiuploader) initChecksumAlgorithm() { + if u.in.ChecksumAlgorithm != "" { + return + } + + switch { + case u.in.ChecksumCRC32 != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32 + case u.in.ChecksumCRC32C != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32c + case u.in.ChecksumSHA1 != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha1 + case u.in.ChecksumSHA256 != nil: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 + default: + u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32 + } +} + // geterr is a thread-safe getter for the error object func (u *multiuploader) geterr() error { u.m.Lock() diff --git a/service/internal/checksum/algorithms.go b/service/internal/checksum/algorithms.go index a17041c35d0..d241bf59fbb 100644 --- a/service/internal/checksum/algorithms.go +++ b/service/internal/checksum/algorithms.go @@ -30,6 +30,9 @@ const ( // AlgorithmSHA256 represents SHA256 hash algorithm AlgorithmSHA256 Algorithm = "SHA256" + + // AlgorithmCRC64NVME represents CRC64NVME hash algorithm + AlgorithmCRC64NVME Algorithm = "CRC64NVME" ) var supportedAlgorithms = []Algorithm{ diff --git a/service/internal/checksum/algorithms_test.go b/service/internal/checksum/algorithms_test.go index 7c821f1e2a5..05e46fb1050 100644 --- a/service/internal/checksum/algorithms_test.go +++ b/service/internal/checksum/algorithms_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum diff --git a/service/internal/checksum/aws_chunked_encoding_test.go b/service/internal/checksum/aws_chunked_encoding_test.go index 4f208b4c099..5b991b14b86 100644 --- a/service/internal/checksum/aws_chunked_encoding_test.go +++ b/service/internal/checksum/aws_chunked_encoding_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum diff --git a/service/internal/checksum/middleware_add.go b/service/internal/checksum/middleware_add.go index 1b727acbe17..11243a8048f 100644 --- a/service/internal/checksum/middleware_add.go +++ b/service/internal/checksum/middleware_add.go @@ -1,6 +1,7 @@ package checksum import ( + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/smithy-go/middleware" ) @@ -14,11 +15,16 @@ type InputMiddlewareOptions struct { // and true, or false if no algorithm is specified. GetAlgorithm func(interface{}) (string, bool) - // Forces the middleware to compute the input payload's checksum. The - // request will fail if the algorithm is not specified or unable to compute - // the checksum. + // RequireChecksum indicates whether operation model forces middleware to compute the input payload's checksum. + // If RequireChecksum is set to true, checksum will be calculated and RequestChecksumCalculation will be ignored, + // otherwise RequestChecksumCalculation will be used to indicate if checksum will be calculated RequireChecksum bool + // RequestChecksumCalculation is the user config to opt-in/out request checksum calculation. If RequireChecksum is + // set to true, checksum will be calculated and this field will be ignored, otherwise + // RequestChecksumCalculation will be used to indicate if checksum will be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation + // Enables support for wrapping the serialized input payload with a // content-encoding: aws-check wrapper, and including a trailer for the // algorithm's checksum value. @@ -72,7 +78,9 @@ func AddInputMiddleware(stack *middleware.Stack, options InputMiddlewareOptions) // Initial checksum configuration look up middleware err = stack.Initialize.Add(&setupInputContext{ - GetAlgorithm: options.GetAlgorithm, + GetAlgorithm: options.GetAlgorithm, + RequireChecksum: options.RequireChecksum, + RequestChecksumCalculation: options.RequestChecksumCalculation, }, middleware.Before) if err != nil { return err @@ -81,7 +89,6 @@ func AddInputMiddleware(stack *middleware.Stack, options InputMiddlewareOptions) stack.Build.Remove("ContentChecksum") inputChecksum := &computeInputPayloadChecksum{ - RequireChecksum: options.RequireChecksum, EnableTrailingChecksum: options.EnableTrailingChecksum, EnableComputePayloadHash: options.EnableComputeSHA256PayloadHash, EnableDecodedContentLengthHeader: options.EnableDecodedContentLengthHeader, @@ -94,7 +101,6 @@ func AddInputMiddleware(stack *middleware.Stack, options InputMiddlewareOptions) if options.EnableTrailingChecksum { trailerMiddleware := &addInputChecksumTrailer{ EnableTrailingChecksum: inputChecksum.EnableTrailingChecksum, - RequireChecksum: inputChecksum.RequireChecksum, EnableComputePayloadHash: inputChecksum.EnableComputePayloadHash, EnableDecodedContentLengthHeader: inputChecksum.EnableDecodedContentLengthHeader, } @@ -126,6 +132,9 @@ type OutputMiddlewareOptions struct { // mode and true, or false if no mode is specified. GetValidationMode func(interface{}) (string, bool) + // ResponseChecksumValidation is the user config to opt-in/out response checksum validation + ResponseChecksumValidation aws.ResponseChecksumValidation + // The set of checksum algorithms that should be used for response payload // checksum validation. The algorithm(s) used will be a union of the // output's returned algorithms and this set. @@ -134,7 +143,7 @@ type OutputMiddlewareOptions struct { ValidationAlgorithms []string // If set the middleware will ignore output multipart checksums. Otherwise - // an checksum format error will be returned by the middleware. + // a checksum format error will be returned by the middleware. IgnoreMultipartValidation bool // When set the middleware will log when output does not have checksum or @@ -150,7 +159,8 @@ type OutputMiddlewareOptions struct { // checksum. func AddOutputMiddleware(stack *middleware.Stack, options OutputMiddlewareOptions) error { err := stack.Initialize.Add(&setupOutputContext{ - GetValidationMode: options.GetValidationMode, + GetValidationMode: options.GetValidationMode, + ResponseChecksumValidation: options.ResponseChecksumValidation, }, middleware.Before) if err != nil { return err diff --git a/service/internal/checksum/middleware_add_test.go b/service/internal/checksum/middleware_add_test.go index da6efe94a26..20e8c72a483 100644 --- a/service/internal/checksum/middleware_add_test.go +++ b/service/internal/checksum/middleware_add_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum @@ -87,7 +87,6 @@ func TestAddInputMiddleware(t *testing.T) { }, }, expectFinalize: &computeInputPayloadChecksum{ - RequireChecksum: true, EnableTrailingChecksum: true, }, }, @@ -167,9 +166,6 @@ func TestAddInputMiddleware(t *testing.T) { var computeInput *computeInputPayloadChecksum if c.expectFinalize != nil && ok { computeInput = finalizeMW.(*computeInputPayloadChecksum) - if e, a := c.expectFinalize.RequireChecksum, computeInput.RequireChecksum; e != a { - t.Errorf("expect %v require checksum, got %v", e, a) - } if e, a := c.expectFinalize.EnableTrailingChecksum, computeInput.EnableTrailingChecksum; e != a { t.Errorf("expect %v enable trailing checksum, got %v", e, a) } diff --git a/service/internal/checksum/middleware_checksum_metrics_tracking.go b/service/internal/checksum/middleware_checksum_metrics_tracking.go new file mode 100644 index 00000000000..861a44293b1 --- /dev/null +++ b/service/internal/checksum/middleware_checksum_metrics_tracking.go @@ -0,0 +1,90 @@ +package checksum + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +var supportedChecksumFeatures = map[Algorithm]awsmiddleware.UserAgentFeature{ + AlgorithmCRC32: awsmiddleware.UserAgentFeatureRequestChecksumCRC32, + AlgorithmCRC32C: awsmiddleware.UserAgentFeatureRequestChecksumCRC32C, + AlgorithmSHA1: awsmiddleware.UserAgentFeatureRequestChecksumSHA1, + AlgorithmSHA256: awsmiddleware.UserAgentFeatureRequestChecksumSHA256, + AlgorithmCRC64NVME: awsmiddleware.UserAgentFeatureRequestChecksumCRC64, +} + +// RequestChecksumMetricsTracking is the middleware to track operation request's checksum usage +type RequestChecksumMetricsTracking struct { + RequestChecksumCalculation aws.RequestChecksumCalculation + UserAgent *awsmiddleware.RequestUserAgent +} + +// ID provides the middleware identifier +func (m *RequestChecksumMetricsTracking) ID() string { + return "AWSChecksum:RequestMetricsTracking" +} + +// HandleBuild checks request checksum config and checksum value sent +// and sends corresponding feature id to user agent +func (m *RequestChecksumMetricsTracking) HandleBuild( + ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler, +) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + switch m.RequestChecksumCalculation { + case aws.RequestChecksumCalculationWhenSupported: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureRequestChecksumWhenSupported) + case aws.RequestChecksumCalculationWhenRequired: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureRequestChecksumWhenRequired) + } + + for algo, feat := range supportedChecksumFeatures { + checksumHeader := AlgorithmHTTPHeader(algo) + if checksum := req.Header.Get(checksumHeader); checksum != "" { + m.UserAgent.AddUserAgentFeature(feat) + } + } + + return next.HandleBuild(ctx, in) +} + +// ResponseChecksumMetricsTracking is the middleware to track operation response's checksum usage +type ResponseChecksumMetricsTracking struct { + ResponseChecksumValidation aws.ResponseChecksumValidation + UserAgent *awsmiddleware.RequestUserAgent +} + +// ID provides the middleware identifier +func (m *ResponseChecksumMetricsTracking) ID() string { + return "AWSChecksum:ResponseMetricsTracking" +} + +// HandleBuild checks the response checksum config and sends corresponding feature id to user agent +func (m *ResponseChecksumMetricsTracking) HandleBuild( + ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler, +) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + switch m.ResponseChecksumValidation { + case aws.ResponseChecksumValidationWhenSupported: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureResponseChecksumWhenSupported) + case aws.ResponseChecksumValidationWhenRequired: + m.UserAgent.AddUserAgentFeature(awsmiddleware.UserAgentFeatureResponseChecksumWhenRequired) + } + + return next.HandleBuild(ctx, in) +} diff --git a/service/internal/checksum/middleware_checksum_metrics_tracking_test.go b/service/internal/checksum/middleware_checksum_metrics_tracking_test.go new file mode 100644 index 00000000000..cbbbb8d230c --- /dev/null +++ b/service/internal/checksum/middleware_checksum_metrics_tracking_test.go @@ -0,0 +1,123 @@ +//go:build go1.21 +// +build go1.21 + +package checksum + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + "net/http" + "strings" + "testing" +) + +func TestRequestChecksumMetricsTracking(t *testing.T) { + cases := map[string]struct { + requestChecksumCalculation aws.RequestChecksumCalculation + reqHeaders http.Header + expectedUserAgentHeader string + }{ + "default": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + reqHeaders: map[string][]string{}, + expectedUserAgentHeader: " m/Z", + }, + "calculate checksum when required": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + reqHeaders: map[string][]string{}, + expectedUserAgentHeader: " m/a", + }, + "default with crc32 checksum": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + reqHeaders: map[string][]string{ + "X-Amz-Checksum-Crc32": {"aa"}, + }, + expectedUserAgentHeader: " m/U,Z", + }, + "calculate checksum when required with sha256 checksum": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + reqHeaders: map[string][]string{ + "X-Amz-Checksum-Sha256": {"aa"}, + }, + expectedUserAgentHeader: " m/Y,a", + }, + "default with crc32c and crc64": { + requestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + reqHeaders: map[string][]string{ + "X-Amz-Checksum-Crc32c": {"aa"}, + "X-Amz-Checksum-Crc64nvme": {"aa"}, + }, + expectedUserAgentHeader: " m/V,W,Z", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ua := awsmiddleware.NewRequestUserAgent() + req := smithyhttp.NewStackRequest().(*smithyhttp.Request) + req.Header = c.reqHeaders + mw := RequestChecksumMetricsTracking{ + RequestChecksumCalculation: c.requestChecksumCalculation, + UserAgent: ua, + } + mw.HandleBuild(context.Background(), + middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + ua.HandleBuild(context.Background(), middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + if e, a := c.expectedUserAgentHeader, req.Header["User-Agent"][0]; !strings.Contains(a, e) { + t.Errorf("expected user agent header to include %s, got %s", e, a) + } + }) + } +} + +func TestResponseChecksumMetricsTracking(t *testing.T) { + cases := map[string]struct { + responseChecksumValidation aws.ResponseChecksumValidation + expectedUserAgentHeader string + }{ + "default": { + responseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + expectedUserAgentHeader: " m/b", + }, + "validate checksum when required": { + responseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + expectedUserAgentHeader: " m/c", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ua := awsmiddleware.NewRequestUserAgent() + req := smithyhttp.NewStackRequest().(*smithyhttp.Request) + mw := ResponseChecksumMetricsTracking{ + ResponseChecksumValidation: c.responseChecksumValidation, + UserAgent: ua, + } + mw.HandleBuild(context.Background(), + middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + ua.HandleBuild(context.Background(), middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, in middleware.BuildInput) (out middleware.BuildOutput, metadata middleware.Metadata, err error) { + return + })) + + if e, a := c.expectedUserAgentHeader, req.Header["User-Agent"][0]; !strings.Contains(a, e) { + t.Errorf("expected user agent header to contain %s, got %s", e, a) + } + }) + } +} diff --git a/service/internal/checksum/middleware_compute_input_checksum.go b/service/internal/checksum/middleware_compute_input_checksum.go index 7ffca33f0ef..71a51bf747a 100644 --- a/service/internal/checksum/middleware_compute_input_checksum.go +++ b/service/internal/checksum/middleware_compute_input_checksum.go @@ -7,6 +7,7 @@ import ( "hash" "io" "strconv" + "strings" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" internalcontext "github.com/aws/aws-sdk-go-v2/internal/context" @@ -16,7 +17,6 @@ import ( ) const ( - contentMD5Header = "Content-Md5" streamingUnsignedPayloadTrailerPayloadHash = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" ) @@ -49,13 +49,6 @@ type computeInputPayloadChecksum struct { // the Algorithm's header is already set on the request. EnableTrailingChecksum bool - // States that a checksum is required to be included for the operation. If - // Input does not specify a checksum, fallback to built in MD5 checksum is - // used. - // - // Replaces smithy-go's ContentChecksum middleware. - RequireChecksum bool - // Enables support for computing the SHA256 checksum of input payloads // along with the algorithm specified checksum. Prevents downstream // middleware handlers (computePayloadSHA256) re-reading the payload. @@ -110,6 +103,15 @@ func (m *computeInputPayloadChecksum) HandleFinalize( ) ( out middleware.FinalizeOutput, metadata middleware.Metadata, err error, ) { + var checksum string + algorithm, ok, err := getInputAlgorithm(ctx) + if err != nil { + return out, metadata, err + } + if !ok { + return next.HandleFinalize(ctx, in) + } + req, ok := in.Request.(*smithyhttp.Request) if !ok { return out, metadata, computeInputHeaderChecksumError{ @@ -117,8 +119,6 @@ func (m *computeInputPayloadChecksum) HandleFinalize( } } - var algorithm Algorithm - var checksum string defer func() { if algorithm == "" || checksum == "" || err != nil { return @@ -130,29 +130,14 @@ func (m *computeInputPayloadChecksum) HandleFinalize( }) }() - // If no algorithm was specified, and the operation requires a checksum, - // fallback to the legacy content MD5 checksum. - algorithm, ok, err = getInputAlgorithm(ctx) - if err != nil { - return out, metadata, err - } else if !ok { - if m.RequireChecksum { - checksum, err = setMD5Checksum(ctx, req) - if err != nil { - return out, metadata, computeInputHeaderChecksumError{ - Msg: "failed to compute stream's MD5 checksum", - Err: err, - } - } - algorithm = Algorithm("MD5") + // If any checksum header is already set nothing to do. + for header := range req.Header { + h := strings.ToUpper(header) + if strings.HasPrefix(h, "X-AMZ-CHECKSUM-") { + algorithm = Algorithm(strings.TrimPrefix(h, "X-AMZ-CHECKSUM-")) + checksum = req.Header.Get(header) + return next.HandleFinalize(ctx, in) } - return next.HandleFinalize(ctx, in) - } - - // If the checksum header is already set nothing to do. - checksumHeader := AlgorithmHTTPHeader(algorithm) - if checksum = req.Header.Get(checksumHeader); checksum != "" { - return next.HandleFinalize(ctx, in) } computePayloadHash := m.EnableComputePayloadHash @@ -217,6 +202,7 @@ func (m *computeInputPayloadChecksum) HandleFinalize( } } + checksumHeader := AlgorithmHTTPHeader(algorithm) req.Header.Set(checksumHeader, checksum) if computePayloadHash { @@ -248,7 +234,6 @@ func (e computeInputTrailingChecksumError) Unwrap() error { return e.Err } // - Trailing checksums are supported. type addInputChecksumTrailer struct { EnableTrailingChecksum bool - RequireChecksum bool EnableComputePayloadHash bool EnableDecodedContentLengthHeader bool } @@ -264,6 +249,16 @@ func (m *addInputChecksumTrailer) HandleFinalize( ) ( out middleware.FinalizeOutput, metadata middleware.Metadata, err error, ) { + algorithm, ok, err := getInputAlgorithm(ctx) + if err != nil { + return out, metadata, computeInputTrailingChecksumError{ + Msg: "failed to get algorithm", + Err: err, + } + } else if !ok { + return next.HandleFinalize(ctx, in) + } + if enabled, _ := middleware.GetStackValue(ctx, useTrailer{}).(bool); !enabled { return next.HandleFinalize(ctx, in) } @@ -281,26 +276,13 @@ func (m *addInputChecksumTrailer) HandleFinalize( } } - // If no algorithm was specified, there is nothing to do. - algorithm, ok, err := getInputAlgorithm(ctx) - if err != nil { - return out, metadata, computeInputTrailingChecksumError{ - Msg: "failed to get algorithm", - Err: err, - } - } else if !ok { - return out, metadata, computeInputTrailingChecksumError{ - Msg: "no algorithm specified", + // If any checksum header is already set nothing to do. + for header := range req.Header { + if strings.HasPrefix(strings.ToLower(header), "x-amz-checksum-") { + return next.HandleFinalize(ctx, in) } } - // If the checksum header is already set before finalize could run, there - // is nothing to do. - checksumHeader := AlgorithmHTTPHeader(algorithm) - if req.Header.Get(checksumHeader) != "" { - return next.HandleFinalize(ctx, in) - } - stream := req.GetStream() streamLength, err := getRequestStreamLength(req) if err != nil { @@ -444,39 +426,3 @@ func getRequestStreamLength(req *smithyhttp.Request) (int64, error) { return -1, nil } - -// setMD5Checksum computes the MD5 of the request payload and sets it to the -// Content-MD5 header. Returning the MD5 base64 encoded string or error. -// -// If the MD5 is already set as the Content-MD5 header, that value will be -// returned, and nothing else will be done. -// -// If the payload is empty, no MD5 will be computed. No error will be returned. -// Empty payloads do not have an MD5 value. -// -// Replaces the smithy-go middleware for httpChecksum trait. -func setMD5Checksum(ctx context.Context, req *smithyhttp.Request) (string, error) { - if v := req.Header.Get(contentMD5Header); len(v) != 0 { - return v, nil - } - stream := req.GetStream() - if stream == nil { - return "", nil - } - - if !req.IsStreamSeekable() { - return "", fmt.Errorf( - "unseekable stream is not supported for computing md5 checksum") - } - - v, err := computeMD5Checksum(stream) - if err != nil { - return "", err - } - if err := req.RewindStream(); err != nil { - return "", fmt.Errorf("failed to rewind stream after computing MD5 checksum, %w", err) - } - // set the 'Content-MD5' header - req.Header.Set(contentMD5Header, string(v)) - return string(v), nil -} diff --git a/service/internal/checksum/middleware_compute_input_checksum_test.go b/service/internal/checksum/middleware_compute_input_checksum_test.go index c8e97de22de..a2e9b02df83 100644 --- a/service/internal/checksum/middleware_compute_input_checksum_test.go +++ b/service/internal/checksum/middleware_compute_input_checksum_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum @@ -93,33 +93,96 @@ func TestComputeInputPayloadChecksum(t *testing.T) { "CRC32": "AAAAAA==", }, }, - "no algorithm": { + "http no algorithm checksum header preset": { buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("https://example.aws") + r.URL, _ = url.Parse("http://example.aws") + r.ContentLength = 11 + r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC32), "AAAAAA==") + r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + return r + }(), + }, + expectHeader: http.Header{ + "X-Amz-Checksum-Crc32": []string{"AAAAAA=="}, + }, + expectContentLength: 11, + expectPayload: []byte("hello world"), + }, + "http no algorithm set": { + buildInput: middleware.BuildInput{ + Request: func() *smithyhttp.Request { + r := smithyhttp.NewStackRequest().(*smithyhttp.Request) + r.URL, _ = url.Parse("http://example.aws") r = requestMust(r.SetStream(strings.NewReader("hello world"))) r.ContentLength = 11 return r }(), }, + expectContentLength: 11, expectHeader: http.Header{}, + expectPayload: []byte("hello world"), + }, + "https no algorithm set": { + buildInput: middleware.BuildInput{ + Request: func() *smithyhttp.Request { + r := smithyhttp.NewStackRequest().(*smithyhttp.Request) + r.URL, _ = url.Parse("https://example.aws") + r = requestMust(r.SetStream(strings.NewReader("hello world"))) + r.ContentLength = 11 + return r + }(), + }, expectContentLength: 11, + expectHeader: http.Header{}, expectPayload: []byte("hello world"), }, - "nil stream no algorithm require checksum": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true + "http crc64 checksum header preset": { + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("http://example.aws") + r.ContentLength = 11 + r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC64NVME), "S2Zv/ZHmbVs=") + r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) return r }(), }, - expectContentLength: -1, - expectHeader: http.Header{}, + expectHeader: http.Header{ + "X-Amz-Checksum-Crc64nvme": []string{"S2Zv/ZHmbVs="}, + }, + expectContentLength: 11, + expectPayload: []byte("hello world"), + expectChecksumMetadata: map[string]string{ + "CRC64NVME": "S2Zv/ZHmbVs=", + }, + }, + "https crc64 checksum header preset": { + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + }, + buildInput: middleware.BuildInput{ + Request: func() *smithyhttp.Request { + r := smithyhttp.NewStackRequest().(*smithyhttp.Request) + r.URL, _ = url.Parse("https://example.aws") + r.ContentLength = 11 + r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC64NVME), "S2Zv/ZHmbVs=") + r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + return r + }(), + }, + expectHeader: http.Header{ + "X-Amz-Checksum-Crc64nvme": []string{"S2Zv/ZHmbVs="}, + }, + expectContentLength: 11, + expectPayload: []byte("hello world"), + expectChecksumMetadata: map[string]string{ + "CRC64NVME": "S2Zv/ZHmbVs=", + }, }, }, @@ -254,94 +317,27 @@ func TestComputeInputPayloadChecksum(t *testing.T) { "CRC32": "AAAAAA==", }, }, - "http no algorithm require checksum": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true - }, - buildInput: middleware.BuildInput{ - Request: func() *smithyhttp.Request { - r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("http://example.aws") - r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) - return r - }(), - }, - expectHeader: http.Header{ - "Content-Md5": []string{"XrY7u+Ae7tCTyyK7j1rNww=="}, - }, - expectContentLength: 11, - expectPayload: []byte("hello world"), - expectChecksumMetadata: map[string]string{ - "MD5": "XrY7u+Ae7tCTyyK7j1rNww==", - }, - }, - "http no algorithm require checksum header preset": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true - }, - buildInput: middleware.BuildInput{ - Request: func() *smithyhttp.Request { - r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("http://example.aws") - r.ContentLength = 11 - r.Header.Set("Content-MD5", "XrY7u+Ae7tCTyyK7j1rNww==") - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) - return r - }(), - }, - expectHeader: http.Header{ - "Content-Md5": []string{"XrY7u+Ae7tCTyyK7j1rNww=="}, - }, - expectContentLength: 11, - expectPayload: []byte("hello world"), - expectChecksumMetadata: map[string]string{ - "MD5": "XrY7u+Ae7tCTyyK7j1rNww==", - }, - }, - "https no algorithm require checksum": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true - }, - buildInput: middleware.BuildInput{ - Request: func() *smithyhttp.Request { - r := smithyhttp.NewStackRequest().(*smithyhttp.Request) - r.URL, _ = url.Parse("https://example.aws") - r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) - return r - }(), - }, - expectHeader: http.Header{ - "Content-Md5": []string{"XrY7u+Ae7tCTyyK7j1rNww=="}, - }, - expectContentLength: 11, - expectPayload: []byte("hello world"), - expectChecksumMetadata: map[string]string{ - "MD5": "XrY7u+Ae7tCTyyK7j1rNww==", - }, - }, "http seekable": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32C)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("http://example.aws") r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + r = requestMust(r.SetStream(bytes.NewReader([]byte("Hello world")))) return r }(), }, expectHeader: http.Header{ - "X-Amz-Checksum-Crc32": []string{"DUoRhQ=="}, + "X-Amz-Checksum-Crc32c": []string{"crUfeA=="}, }, expectContentLength: 11, - expectPayload: []byte("hello world"), - expectPayloadHash: "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", + expectPayload: []byte("Hello world"), + expectPayloadHash: "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c", expectChecksumMetadata: map[string]string{ - "CRC32": "DUoRhQ==", + "CRC32C": "crUfeA==", }, }, "http payload hash already set": { @@ -474,7 +470,7 @@ func TestComputeInputPayloadChecksum(t *testing.T) { "build error": { "unknown algorithm": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string("unknown")) + return internalcontext.SetChecksumInputAlgorithm(ctx, "unknown") }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { @@ -487,9 +483,9 @@ func TestComputeInputPayloadChecksum(t *testing.T) { expectErr: "failed to parse algorithm", expectBuildErr: true, }, - "no algorithm require checksum unseekable stream": { - optionsFn: func(o *computeInputPayloadChecksum) { - o.RequireChecksum = true + "unsupported algorithm": { + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC64NVME)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { @@ -499,12 +495,12 @@ func TestComputeInputPayloadChecksum(t *testing.T) { return r }(), }, - expectErr: "unseekable stream is not supported", + expectErr: "failed to parse algorithm", expectBuildErr: true, }, "http unseekable stream": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmSHA1)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { @@ -627,54 +623,54 @@ func TestComputeInputPayloadChecksum(t *testing.T) { }, "https seekable": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmSHA1)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("https://example.aws") r.ContentLength = 11 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + r = requestMust(r.SetStream(bytes.NewReader([]byte("Hello world")))) return r }(), }, expectHeader: http.Header{ "Content-Encoding": []string{"aws-chunked"}, "X-Amz-Decoded-Content-Length": []string{"11"}, - "X-Amz-Trailer": []string{"x-amz-checksum-crc32"}, + "X-Amz-Trailer": []string{"x-amz-checksum-sha1"}, }, - expectContentLength: 52, - expectPayload: []byte("b\r\nhello world\r\n0\r\nx-amz-checksum-crc32:DUoRhQ==\r\n\r\n"), + expectContentLength: 71, + expectPayload: []byte("b\r\nHello world\r\n0\r\nx-amz-checksum-sha1:e1AsOh9IyGCa4hLN+2Od7jlnP14=\r\n\r\n"), expectPayloadHash: "STREAMING-UNSIGNED-PAYLOAD-TRAILER", expectDeferToFinalize: true, expectChecksumMetadata: map[string]string{ - "CRC32": "DUoRhQ==", + "SHA1": "e1AsOh9IyGCa4hLN+2Od7jlnP14=", }, }, "https seekable unknown length": { initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32C)) }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) r.URL, _ = url.Parse("https://example.aws") r.ContentLength = -1 - r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world")))) + r = requestMust(r.SetStream(bytes.NewReader([]byte("Hello world")))) return r }(), }, expectHeader: http.Header{ "Content-Encoding": []string{"aws-chunked"}, "X-Amz-Decoded-Content-Length": []string{"11"}, - "X-Amz-Trailer": []string{"x-amz-checksum-crc32"}, + "X-Amz-Trailer": []string{"x-amz-checksum-crc32c"}, }, - expectContentLength: 52, - expectPayload: []byte("b\r\nhello world\r\n0\r\nx-amz-checksum-crc32:DUoRhQ==\r\n\r\n"), + expectContentLength: 53, + expectPayload: []byte("b\r\nHello world\r\n0\r\nx-amz-checksum-crc32c:crUfeA==\r\n\r\n"), expectPayloadHash: "STREAMING-UNSIGNED-PAYLOAD-TRAILER", expectDeferToFinalize: true, expectChecksumMetadata: map[string]string{ - "CRC32": "DUoRhQ==", + "CRC32C": "crUfeA==", }, }, "https no compute payload hash": { @@ -706,12 +702,12 @@ func TestComputeInputPayloadChecksum(t *testing.T) { }, }, "https no decode content length": { - initContext: func(ctx context.Context) context.Context { - return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) - }, optionsFn: func(o *computeInputPayloadChecksum) { o.EnableDecodedContentLengthHeader = false }, + initContext: func(ctx context.Context) context.Context { + return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) + }, buildInput: middleware.BuildInput{ Request: func() *smithyhttp.Request { r := smithyhttp.NewStackRequest().(*smithyhttp.Request) @@ -778,7 +774,6 @@ func TestComputeInputPayloadChecksum(t *testing.T) { } trailerMiddleware := &addInputChecksumTrailer{ EnableTrailingChecksum: m.EnableTrailingChecksum, - RequireChecksum: m.RequireChecksum, EnableComputePayloadHash: m.EnableComputePayloadHash, EnableDecodedContentLengthHeader: m.EnableDecodedContentLengthHeader, } @@ -920,7 +915,7 @@ func TestComputeInputPayloadChecksum(t *testing.T) { // assert computed input checksums metadata computedMetadata, ok := GetComputedInputChecksums(metadata) - if e, a := ok, (c.expectChecksumMetadata != nil); e != a { + if e, a := (c.expectChecksumMetadata != nil), ok; e != a { t.Fatalf("expect checksum metadata %t, got %t, %v", e, a, computedMetadata) } if c.expectChecksumMetadata != nil { diff --git a/service/internal/checksum/middleware_setup_context.go b/service/internal/checksum/middleware_setup_context.go index 3db73afe7e8..7fcd29a3814 100644 --- a/service/internal/checksum/middleware_setup_context.go +++ b/service/internal/checksum/middleware_setup_context.go @@ -2,11 +2,16 @@ package checksum import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" internalcontext "github.com/aws/aws-sdk-go-v2/internal/context" "github.com/aws/smithy-go/middleware" ) +const ( + checksumValidationModeEnabled = "ENABLED" +) + // setupChecksumContext is the initial middleware that looks up the input // used to configure checksum behavior. This middleware must be executed before // input validation step or any other checksum middleware. @@ -17,6 +22,16 @@ type setupInputContext struct { // Given the input parameter value, the function must return the algorithm // and true, or false if no algorithm is specified. GetAlgorithm func(interface{}) (string, bool) + + // RequireChecksum indicates whether operation model forces middleware to compute the input payload's checksum. + // If RequireChecksum is set to true, checksum will be calculated and RequestChecksumCalculation will be ignored, + // otherwise RequestChecksumCalculation will be used to indicate if checksum will be calculated + RequireChecksum bool + + // RequestChecksumCalculation is the user config to opt-in/out request checksum calculation. If RequireChecksum is + // set to true, checksum will be calculated and this field will be ignored, otherwise + // RequestChecksumCalculation will be used to indicate if checksum will be calculated + RequestChecksumCalculation aws.RequestChecksumCalculation } // ID for the middleware @@ -31,13 +46,13 @@ func (m *setupInputContext) HandleInitialize( ) ( out middleware.InitializeOutput, metadata middleware.Metadata, err error, ) { - // Check if validation algorithm is specified. - if m.GetAlgorithm != nil { - // check is input resource has a checksum algorithm - algorithm, ok := m.GetAlgorithm(in.Parameters) - if ok && len(algorithm) != 0 { - ctx = internalcontext.SetChecksumInputAlgorithm(ctx, algorithm) - } + if algorithm, ok := m.GetAlgorithm(in.Parameters); ok { + ctx = internalcontext.SetChecksumInputAlgorithm(ctx, algorithm) + return next.HandleInitialize(ctx, in) + } + + if m.RequireChecksum || m.RequestChecksumCalculation == aws.RequestChecksumCalculationWhenSupported { + ctx = internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32)) } return next.HandleInitialize(ctx, in) @@ -50,6 +65,9 @@ type setupOutputContext struct { // Given the input parameter value, the function must return the validation // mode and true, or false if no mode is specified. GetValidationMode func(interface{}) (string, bool) + + // ResponseChecksumValidation states user config to opt-in/out checksum validation + ResponseChecksumValidation aws.ResponseChecksumValidation } // ID for the middleware @@ -64,13 +82,11 @@ func (m *setupOutputContext) HandleInitialize( ) ( out middleware.InitializeOutput, metadata middleware.Metadata, err error, ) { - // Check if validation mode is specified. - if m.GetValidationMode != nil { - // check is input resource has a checksum algorithm - mode, ok := m.GetValidationMode(in.Parameters) - if ok && len(mode) != 0 { - ctx = setContextOutputValidationMode(ctx, mode) - } + + mode, _ := m.GetValidationMode(in.Parameters) + + if m.ResponseChecksumValidation == aws.ResponseChecksumValidationWhenSupported || mode == checksumValidationModeEnabled { + ctx = setContextOutputValidationMode(ctx, checksumValidationModeEnabled) } return next.HandleInitialize(ctx, in) diff --git a/service/internal/checksum/middleware_setup_context_test.go b/service/internal/checksum/middleware_setup_context_test.go index e629ee088d7..81c7250940a 100644 --- a/service/internal/checksum/middleware_setup_context_test.go +++ b/service/internal/checksum/middleware_setup_context_test.go @@ -1,10 +1,11 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "testing" internalcontext "github.com/aws/aws-sdk-go-v2/internal/context" @@ -17,29 +18,47 @@ func TestSetupInput(t *testing.T) { } cases := map[string]struct { - inputParams interface{} - getAlgorithm func(interface{}) (string, bool) - expectValue string + inputParams interface{} + getAlgorithm func(interface{}) (string, bool) + RequireChecksum bool + RequestChecksumCalculation aws.RequestChecksumCalculation + expectValue string }{ - "nil accessor": { + "user config require checksum and algorithm unset": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + getAlgorithm: func(v interface{}) (string, bool) { + return "", false + }, expectValue: "", }, - "found empty": { - inputParams: Params{Value: ""}, + "require checksum found empty": { + RequireChecksum: true, + inputParams: Params{Value: ""}, getAlgorithm: func(v interface{}) (string, bool) { vv := v.(Params) return vv.Value, true }, expectValue: "", }, - "found not set": { - inputParams: Params{Value: ""}, + "user config require checksum found empty": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + inputParams: Params{Value: ""}, getAlgorithm: func(v interface{}) (string, bool) { - return "", false + vv := v.(Params) + return vv.Value, true }, expectValue: "", }, - "found": { + "require checksum and found": { + RequireChecksum: true, + inputParams: Params{Value: "abc123"}, + getAlgorithm: func(v interface{}) (string, bool) { + vv := v.(Params) + return vv.Value, true + }, + expectValue: "abc123", + }, + "user config support checksum and found": { inputParams: Params{Value: "abc123"}, getAlgorithm: func(v interface{}) (string, bool) { vv := v.(Params) @@ -47,12 +66,37 @@ func TestSetupInput(t *testing.T) { }, expectValue: "abc123", }, + "user config require checksum and found": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + inputParams: Params{Value: "abc123"}, + getAlgorithm: func(v interface{}) (string, bool) { + vv := v.(Params) + return vv.Value, true + }, + expectValue: "abc123", + }, + "require checksum unset and use default": { + RequireChecksum: true, + getAlgorithm: func(v interface{}) (string, bool) { + return "", false + }, + expectValue: "CRC32", + }, + "user config support checksum and use default": { + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + getAlgorithm: func(v interface{}) (string, bool) { + return "", false + }, + expectValue: "CRC32", + }, } for name, c := range cases { t.Run(name, func(t *testing.T) { m := setupInputContext{ - GetAlgorithm: c.getAlgorithm, + GetAlgorithm: c.getAlgorithm, + RequireChecksum: c.RequireChecksum, + RequestChecksumCalculation: c.RequestChecksumCalculation, } _, _, err := m.HandleInitialize(context.Background(), @@ -83,42 +127,54 @@ func TestSetupOutput(t *testing.T) { } cases := map[string]struct { - inputParams interface{} - getValidationMode func(interface{}) (string, bool) - expectValue string + inputParams interface{} + ResponseChecksumValidation aws.ResponseChecksumValidation + getValidationMode func(interface{}) (string, bool) + expectValue string }{ - "nil accessor": { - expectValue: "", + "user config support checksum found empty": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + inputParams: Params{Value: ""}, + getValidationMode: func(v interface{}) (string, bool) { + vv := v.(Params) + return vv.Value, true + }, + expectValue: "ENABLED", }, - "found empty": { - inputParams: Params{Value: ""}, + "user config support checksum found invalid value": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenSupported, + inputParams: Params{Value: "abc123"}, getValidationMode: func(v interface{}) (string, bool) { vv := v.(Params) return vv.Value, true }, - expectValue: "", + expectValue: "ENABLED", }, - "found not set": { - inputParams: Params{Value: ""}, + "user config require checksum found invalid value": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + inputParams: Params{Value: "abc123"}, getValidationMode: func(v interface{}) (string, bool) { - return "", false + vv := v.(Params) + return vv.Value, true }, expectValue: "", }, - "found": { - inputParams: Params{Value: "abc123"}, + "user config require checksum found valid value": { + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + inputParams: Params{Value: "ENABLED"}, getValidationMode: func(v interface{}) (string, bool) { vv := v.(Params) return vv.Value, true }, - expectValue: "abc123", + expectValue: "ENABLED", }, } for name, c := range cases { t.Run(name, func(t *testing.T) { m := setupOutputContext{ - GetValidationMode: c.getValidationMode, + GetValidationMode: c.getValidationMode, + ResponseChecksumValidation: c.ResponseChecksumValidation, } _, _, err := m.HandleInitialize(context.Background(), diff --git a/service/internal/checksum/middleware_validate_output.go b/service/internal/checksum/middleware_validate_output.go index 9fde12d86d7..14096a1ce31 100644 --- a/service/internal/checksum/middleware_validate_output.go +++ b/service/internal/checksum/middleware_validate_output.go @@ -55,7 +55,7 @@ func (m *validateOutputPayloadChecksum) ID() string { } // HandleDeserialize is a Deserialize middleware that wraps the HTTP response -// body with an io.ReadCloser that will validate the its checksum. +// body with an io.ReadCloser that will validate its checksum. func (m *validateOutputPayloadChecksum) HandleDeserialize( ctx context.Context, in middleware.DeserializeInput, next middleware.DeserializeHandler, ) ( @@ -66,8 +66,7 @@ func (m *validateOutputPayloadChecksum) HandleDeserialize( return out, metadata, err } - // If there is no validation mode specified nothing is supported. - if mode := getContextOutputValidationMode(ctx); mode != "ENABLED" { + if mode := getContextOutputValidationMode(ctx); mode != checksumValidationModeEnabled { return out, metadata, err } @@ -90,8 +89,6 @@ func (m *validateOutputPayloadChecksum) HandleDeserialize( algorithmToUse = algorithm } - // TODO this must validate the validation mode is set to enabled. - logger := middleware.GetLogger(ctx) // Skip validation if no checksum algorithm or checksum is available. diff --git a/service/internal/checksum/middleware_validate_output_test.go b/service/internal/checksum/middleware_validate_output_test.go index 0bf923e51e7..2b2d4bbca6d 100644 --- a/service/internal/checksum/middleware_validate_output_test.go +++ b/service/internal/checksum/middleware_validate_output_test.go @@ -1,5 +1,5 @@ -//go:build go1.16 -// +build go1.16 +//go:build go1.21 +// +build go1.21 package checksum @@ -49,7 +49,21 @@ func TestValidateOutputPayloadChecksum(t *testing.T) { expectAlgorithmsUsed: []string{"CRC32"}, expectPayload: []byte("hello world"), }, - "failure": { + "no checksum required": { + response: &smithyhttp.Response{ + Response: &http.Response{ + StatusCode: 200, + Header: func() http.Header { + h := http.Header{} + h.Set(AlgorithmHTTPHeader(AlgorithmCRC32C), "crUfeA==") + return h + }(), + Body: ioutil.NopCloser(strings.NewReader("Hello world")), + }, + }, + expectPayload: []byte("Hello world"), + }, + "checksum mismatch failure": { modifyContext: func(ctx context.Context) context.Context { return setContextOutputValidationMode(ctx, "ENABLED") }, @@ -101,19 +115,6 @@ func TestValidateOutputPayloadChecksum(t *testing.T) { expectLogged: "no supported checksum", expectPayload: []byte("hello world"), }, - "no output validation model": { - response: &smithyhttp.Response{ - Response: &http.Response{ - StatusCode: 200, - Header: func() http.Header { - h := http.Header{} - return h - }(), - Body: ioutil.NopCloser(strings.NewReader("hello world")), - }, - }, - expectPayload: []byte("hello world"), - }, "unknown output validation model": { modifyContext: func(ctx context.Context) context.Context { return setContextOutputValidationMode(ctx, "something else") @@ -189,7 +190,7 @@ func TestValidateOutputPayloadChecksum(t *testing.T) { validateOutput := validateOutputPayloadChecksum{ Algorithms: []Algorithm{ - AlgorithmSHA1, AlgorithmCRC32, AlgorithmCRC32C, + AlgorithmSHA1, AlgorithmCRC32, AlgorithmCRC32C, AlgorithmSHA256, }, LogValidationSkipped: true, LogMultipartValidationSkipped: true, diff --git a/service/internal/integrationtest/s3/checksum_test.go b/service/internal/integrationtest/s3/checksum_test.go index 02d6994f41c..1014a66225a 100644 --- a/service/internal/integrationtest/s3/checksum_test.go +++ b/service/internal/integrationtest/s3/checksum_test.go @@ -66,7 +66,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { }, getObjectChecksumMode: s3types.ChecksumModeEnabled, expectPayload: []byte("abc123"), - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "zwK7XA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -170,7 +177,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { }, getObjectChecksumMode: s3types.ChecksumModeEnabled, expectPayload: []byte("abc123"), - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "zwK7XA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -251,7 +265,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { "no checksum": { params: &s3.PutObjectInput{}, getObjectChecksumMode: s3types.ChecksumModeEnabled, - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "AAAAAA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -304,7 +325,14 @@ func TestInteg_ObjectChecksums(t *testing.T) { Body: bytes.NewBuffer([]byte{}), }, getObjectChecksumMode: s3types.ChecksumModeEnabled, - expectLogged: "Response has no supported checksum.", + expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{ + ComputedChecksums: map[string]string{ + "CRC32": "AAAAAA==", + }, + }, + expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{ + AlgorithmsUsed: []string{"CRC32"}, + }, }, "preset checksum": { params: &s3.PutObjectInput{ @@ -394,7 +422,7 @@ func TestInteg_ObjectChecksums(t *testing.T) { } // assert computed input checksums metadata computedChecksums, ok := s3.GetComputedInputChecksumsMetadata(putResult.ResultMetadata) - if e, a := ok, (c.expectComputedChecksums != nil); e != a { + if e, a := (c.expectComputedChecksums != nil), ok; e != a { t.Fatalf("expect computed checksum metadata %t, got %t, %v", e, a, computedChecksums) } if c.expectComputedChecksums != nil { @@ -442,7 +470,7 @@ func TestInteg_ObjectChecksums(t *testing.T) { // assert checksum validation metadata algorithmsUsed, ok := s3.GetChecksumValidationMetadata(getResult.ResultMetadata) - if e, a := ok, (c.expectAlgorithmsUsed != nil); e != a { + if e, a := (c.expectAlgorithmsUsed != nil), ok; e != a { t.Fatalf("expect algorithms used metadata %t, got %t, %v", e, a, algorithmsUsed) } if c.expectAlgorithmsUsed != nil { @@ -468,7 +496,7 @@ func TestInteg_RequireChecksum(t *testing.T) { expectComputedChecksums []string }{ "no algorithm": { - expectComputedChecksums: []string{"MD5"}, + expectComputedChecksums: []string{"CRC32"}, }, "with algorithm": { checksumAlgorithm: types.ChecksumAlgorithmCrc32c, diff --git a/service/s3/api_client.go b/service/s3/api_client.go index 08e432799a9..5cadcf369eb 100644 --- a/service/s3/api_client.go +++ b/service/s3/api_client.go @@ -449,15 +449,17 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - Credentials: cfg.Credentials, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + Credentials: cfg.Credentials, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + RequestChecksumCalculation: cfg.RequestChecksumCalculation, + ResponseChecksumValidation: cfg.ResponseChecksumValidation, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -845,6 +847,30 @@ func addUserAgentRetryMode(stack *middleware.Stack, options Options) error { return nil } +func addRequestChecksumMetricsTracking(stack *middleware.Stack, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&internalChecksum.RequestChecksumMetricsTracking{ + RequestChecksumCalculation: options.RequestChecksumCalculation, + UserAgent: ua, + }, "UserAgent", middleware.Before) +} + +func addResponseChecksumMetricsTracking(stack *middleware.Stack, options Options) error { + ua, err := getOrAddRequestUserAgent(stack) + if err != nil { + return err + } + + return stack.Build.Insert(&internalChecksum.ResponseChecksumMetricsTracking{ + ResponseChecksumValidation: options.ResponseChecksumValidation, + UserAgent: ua, + }, "UserAgent", middleware.Before) +} + func resolveTracerProvider(options *Options) { if options.TracerProvider == nil { options.TracerProvider = &tracing.NopTracerProvider{} @@ -1148,6 +1174,10 @@ func (c presignConverter) convertToPresignMiddleware(stack *middleware.Stack, op return nil } +func withNoDefaultChecksumAPIOption(options *Options) { + options.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired +} + func addRequestResponseLogging(stack *middleware.Stack, o Options) error { return stack.Deserialize.Add(&smithyhttp.RequestResponseLogger{ LogRequest: o.ClientLogMode.IsRequest(), diff --git a/service/s3/api_op_CreateBucketMetadataTableConfiguration.go b/service/s3/api_op_CreateBucketMetadataTableConfiguration.go index 4e61399544b..cf89007da4c 100644 --- a/service/s3/api_op_CreateBucketMetadataTableConfiguration.go +++ b/service/s3/api_op_CreateBucketMetadataTableConfiguration.go @@ -170,6 +170,9 @@ func (c *Client) addOperationCreateBucketMetadataTableConfigurationMiddlewares(s if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpCreateBucketMetadataTableConfigurationValidationMiddleware(stack); err != nil { return err } @@ -253,6 +256,7 @@ func addCreateBucketMetadataTableConfigurationInputChecksumMiddlewares(stack *mi return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getCreateBucketMetadataTableConfigurationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_DeleteObjects.go b/service/s3/api_op_DeleteObjects.go index 664a65d92d6..8509332b5de 100644 --- a/service/s3/api_op_DeleteObjects.go +++ b/service/s3/api_op_DeleteObjects.go @@ -347,6 +347,9 @@ func (c *Client) addOperationDeleteObjectsMiddlewares(stack *middleware.Stack, o if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpDeleteObjectsValidationMiddleware(stack); err != nil { return err } @@ -430,6 +433,7 @@ func addDeleteObjectsInputChecksumMiddlewares(stack *middleware.Stack, options O return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getDeleteObjectsRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_GetObject.go b/service/s3/api_op_GetObject.go index e2581124e71..50cb3eec2bf 100644 --- a/service/s3/api_op_GetObject.go +++ b/service/s3/api_op_GetObject.go @@ -720,6 +720,9 @@ func (c *Client) addOperationGetObjectMiddlewares(stack *middleware.Stack, optio if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addResponseChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpGetObjectValidationMiddleware(stack); err != nil { return err } @@ -799,6 +802,7 @@ func getGetObjectRequestValidationModeMember(input interface{}) (string, bool) { func addGetObjectOutputChecksumMiddlewares(stack *middleware.Stack, options Options) error { return internalChecksum.AddOutputMiddleware(stack, internalChecksum.OutputMiddlewareOptions{ GetValidationMode: getGetObjectRequestValidationModeMember, + ResponseChecksumValidation: options.ResponseChecksumValidation, ValidationAlgorithms: []string{"CRC32", "CRC32C", "SHA256", "SHA1"}, IgnoreMultipartValidation: true, LogValidationSkipped: true, diff --git a/service/s3/api_op_PutBucketAccelerateConfiguration.go b/service/s3/api_op_PutBucketAccelerateConfiguration.go index 737f8bfde92..0165e30f662 100644 --- a/service/s3/api_op_PutBucketAccelerateConfiguration.go +++ b/service/s3/api_op_PutBucketAccelerateConfiguration.go @@ -185,6 +185,9 @@ func (c *Client) addOperationPutBucketAccelerateConfigurationMiddlewares(stack * if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketAccelerateConfigurationValidationMiddleware(stack); err != nil { return err } @@ -265,6 +268,7 @@ func addPutBucketAccelerateConfigurationInputChecksumMiddlewares(stack *middlewa return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketAccelerateConfigurationRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketAcl.go b/service/s3/api_op_PutBucketAcl.go index eeac549e338..a123daaecef 100644 --- a/service/s3/api_op_PutBucketAcl.go +++ b/service/s3/api_op_PutBucketAcl.go @@ -329,6 +329,9 @@ func (c *Client) addOperationPutBucketAclMiddlewares(stack *middleware.Stack, op if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketAclValidationMiddleware(stack); err != nil { return err } @@ -412,6 +415,7 @@ func addPutBucketAclInputChecksumMiddlewares(stack *middleware.Stack, options Op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketAclRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketCors.go b/service/s3/api_op_PutBucketCors.go index 9eaa35fe83a..149fac241ba 100644 --- a/service/s3/api_op_PutBucketCors.go +++ b/service/s3/api_op_PutBucketCors.go @@ -207,6 +207,9 @@ func (c *Client) addOperationPutBucketCorsMiddlewares(stack *middleware.Stack, o if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketCorsValidationMiddleware(stack); err != nil { return err } @@ -290,6 +293,7 @@ func addPutBucketCorsInputChecksumMiddlewares(stack *middleware.Stack, options O return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketCorsRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketEncryption.go b/service/s3/api_op_PutBucketEncryption.go index 3edeef98383..6cd3ddc8f70 100644 --- a/service/s3/api_op_PutBucketEncryption.go +++ b/service/s3/api_op_PutBucketEncryption.go @@ -284,6 +284,9 @@ func (c *Client) addOperationPutBucketEncryptionMiddlewares(stack *middleware.St if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketEncryptionValidationMiddleware(stack); err != nil { return err } @@ -367,6 +370,7 @@ func addPutBucketEncryptionInputChecksumMiddlewares(stack *middleware.Stack, opt return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketEncryptionRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketLifecycleConfiguration.go b/service/s3/api_op_PutBucketLifecycleConfiguration.go index 6356239da93..2dba4f50905 100644 --- a/service/s3/api_op_PutBucketLifecycleConfiguration.go +++ b/service/s3/api_op_PutBucketLifecycleConfiguration.go @@ -293,6 +293,9 @@ func (c *Client) addOperationPutBucketLifecycleConfigurationMiddlewares(stack *m if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketLifecycleConfigurationValidationMiddleware(stack); err != nil { return err } @@ -376,6 +379,7 @@ func addPutBucketLifecycleConfigurationInputChecksumMiddlewares(stack *middlewar return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketLifecycleConfigurationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketLogging.go b/service/s3/api_op_PutBucketLogging.go index 01380c4bac8..bd2492cdd4d 100644 --- a/service/s3/api_op_PutBucketLogging.go +++ b/service/s3/api_op_PutBucketLogging.go @@ -214,6 +214,9 @@ func (c *Client) addOperationPutBucketLoggingMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketLoggingValidationMiddleware(stack); err != nil { return err } @@ -297,6 +300,7 @@ func addPutBucketLoggingInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketLoggingRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketOwnershipControls.go b/service/s3/api_op_PutBucketOwnershipControls.go index 7eeb45dac42..47263aa2a7c 100644 --- a/service/s3/api_op_PutBucketOwnershipControls.go +++ b/service/s3/api_op_PutBucketOwnershipControls.go @@ -156,6 +156,9 @@ func (c *Client) addOperationPutBucketOwnershipControlsMiddlewares(stack *middle if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketOwnershipControlsValidationMiddleware(stack); err != nil { return err } @@ -229,6 +232,7 @@ func addPutBucketOwnershipControlsInputChecksumMiddlewares(stack *middleware.Sta return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: nil, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketPolicy.go b/service/s3/api_op_PutBucketPolicy.go index eea55bac08f..58589436905 100644 --- a/service/s3/api_op_PutBucketPolicy.go +++ b/service/s3/api_op_PutBucketPolicy.go @@ -256,6 +256,9 @@ func (c *Client) addOperationPutBucketPolicyMiddlewares(stack *middleware.Stack, if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketPolicyValidationMiddleware(stack); err != nil { return err } @@ -339,6 +342,7 @@ func addPutBucketPolicyInputChecksumMiddlewares(stack *middleware.Stack, options return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketPolicyRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketReplication.go b/service/s3/api_op_PutBucketReplication.go index 2655ff24709..44c6e3eeb05 100644 --- a/service/s3/api_op_PutBucketReplication.go +++ b/service/s3/api_op_PutBucketReplication.go @@ -225,6 +225,9 @@ func (c *Client) addOperationPutBucketReplicationMiddlewares(stack *middleware.S if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketReplicationValidationMiddleware(stack); err != nil { return err } @@ -308,6 +311,7 @@ func addPutBucketReplicationInputChecksumMiddlewares(stack *middleware.Stack, op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketReplicationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketRequestPayment.go b/service/s3/api_op_PutBucketRequestPayment.go index 14b2a5b6940..d92624a9797 100644 --- a/service/s3/api_op_PutBucketRequestPayment.go +++ b/service/s3/api_op_PutBucketRequestPayment.go @@ -172,6 +172,9 @@ func (c *Client) addOperationPutBucketRequestPaymentMiddlewares(stack *middlewar if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketRequestPaymentValidationMiddleware(stack); err != nil { return err } @@ -255,6 +258,7 @@ func addPutBucketRequestPaymentInputChecksumMiddlewares(stack *middleware.Stack, return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketRequestPaymentRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketTagging.go b/service/s3/api_op_PutBucketTagging.go index 5d9761c0062..dbfe97cc3b6 100644 --- a/service/s3/api_op_PutBucketTagging.go +++ b/service/s3/api_op_PutBucketTagging.go @@ -204,6 +204,9 @@ func (c *Client) addOperationPutBucketTaggingMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketTaggingValidationMiddleware(stack); err != nil { return err } @@ -287,6 +290,7 @@ func addPutBucketTaggingInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketTaggingRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketVersioning.go b/service/s3/api_op_PutBucketVersioning.go index fdff8fced53..7c5738f2dbb 100644 --- a/service/s3/api_op_PutBucketVersioning.go +++ b/service/s3/api_op_PutBucketVersioning.go @@ -208,6 +208,9 @@ func (c *Client) addOperationPutBucketVersioningMiddlewares(stack *middleware.St if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketVersioningValidationMiddleware(stack); err != nil { return err } @@ -291,6 +294,7 @@ func addPutBucketVersioningInputChecksumMiddlewares(stack *middleware.Stack, opt return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketVersioningRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutBucketWebsite.go b/service/s3/api_op_PutBucketWebsite.go index be78f71af5c..78c46c9a5d6 100644 --- a/service/s3/api_op_PutBucketWebsite.go +++ b/service/s3/api_op_PutBucketWebsite.go @@ -227,6 +227,9 @@ func (c *Client) addOperationPutBucketWebsiteMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutBucketWebsiteValidationMiddleware(stack); err != nil { return err } @@ -310,6 +313,7 @@ func addPutBucketWebsiteInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutBucketWebsiteRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index d0e59738724..a49a9471cb1 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -836,6 +836,9 @@ func (c *Client) addOperationPutObjectMiddlewares(stack *middleware.Stack, optio if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectValidationMiddleware(stack); err != nil { return err } @@ -922,6 +925,7 @@ func addPutObjectInputChecksumMiddlewares(stack *middleware.Stack, options Optio return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: true, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, @@ -965,6 +969,8 @@ func (c *PresignClient) PresignPutObject(ctx context.Context, params *PutObjectI } clientOptFns := append(options.ClientOptions, withNopHTTPClientAPIOption) + clientOptFns = append(options.ClientOptions, withNoDefaultChecksumAPIOption) + result, _, err := c.client.invokeOperation(ctx, "PutObject", params, clientOptFns, c.client.addOperationPutObjectMiddlewares, presignConverter(options).convertToPresignMiddleware, diff --git a/service/s3/api_op_PutObjectAcl.go b/service/s3/api_op_PutObjectAcl.go index 6c13ee838b0..6c8b89a1bb7 100644 --- a/service/s3/api_op_PutObjectAcl.go +++ b/service/s3/api_op_PutObjectAcl.go @@ -381,6 +381,9 @@ func (c *Client) addOperationPutObjectAclMiddlewares(stack *middleware.Stack, op if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectAclValidationMiddleware(stack); err != nil { return err } @@ -464,6 +467,7 @@ func addPutObjectAclInputChecksumMiddlewares(stack *middleware.Stack, options Op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectAclRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectLegalHold.go b/service/s3/api_op_PutObjectLegalHold.go index be713b88c94..691aa6c19f0 100644 --- a/service/s3/api_op_PutObjectLegalHold.go +++ b/service/s3/api_op_PutObjectLegalHold.go @@ -196,6 +196,9 @@ func (c *Client) addOperationPutObjectLegalHoldMiddlewares(stack *middleware.Sta if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectLegalHoldValidationMiddleware(stack); err != nil { return err } @@ -279,6 +282,7 @@ func addPutObjectLegalHoldInputChecksumMiddlewares(stack *middleware.Stack, opti return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectLegalHoldRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectLockConfiguration.go b/service/s3/api_op_PutObjectLockConfiguration.go index e1f7930f2ee..82b0dc9deff 100644 --- a/service/s3/api_op_PutObjectLockConfiguration.go +++ b/service/s3/api_op_PutObjectLockConfiguration.go @@ -187,6 +187,9 @@ func (c *Client) addOperationPutObjectLockConfigurationMiddlewares(stack *middle if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectLockConfigurationValidationMiddleware(stack); err != nil { return err } @@ -270,6 +273,7 @@ func addPutObjectLockConfigurationInputChecksumMiddlewares(stack *middleware.Sta return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectLockConfigurationRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectRetention.go b/service/s3/api_op_PutObjectRetention.go index 7c5a611a245..0887dda7c2c 100644 --- a/service/s3/api_op_PutObjectRetention.go +++ b/service/s3/api_op_PutObjectRetention.go @@ -203,6 +203,9 @@ func (c *Client) addOperationPutObjectRetentionMiddlewares(stack *middleware.Sta if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectRetentionValidationMiddleware(stack); err != nil { return err } @@ -286,6 +289,7 @@ func addPutObjectRetentionInputChecksumMiddlewares(stack *middleware.Stack, opti return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectRetentionRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutObjectTagging.go b/service/s3/api_op_PutObjectTagging.go index 2703eef4eee..0877ae0d817 100644 --- a/service/s3/api_op_PutObjectTagging.go +++ b/service/s3/api_op_PutObjectTagging.go @@ -239,6 +239,9 @@ func (c *Client) addOperationPutObjectTaggingMiddlewares(stack *middleware.Stack if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutObjectTaggingValidationMiddleware(stack); err != nil { return err } @@ -322,6 +325,7 @@ func addPutObjectTaggingInputChecksumMiddlewares(stack *middleware.Stack, option return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutObjectTaggingRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_PutPublicAccessBlock.go b/service/s3/api_op_PutPublicAccessBlock.go index d1d05a6f447..d4d05ffee33 100644 --- a/service/s3/api_op_PutPublicAccessBlock.go +++ b/service/s3/api_op_PutPublicAccessBlock.go @@ -190,6 +190,9 @@ func (c *Client) addOperationPutPublicAccessBlockMiddlewares(stack *middleware.S if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpPutPublicAccessBlockValidationMiddleware(stack); err != nil { return err } @@ -273,6 +276,7 @@ func addPutPublicAccessBlockInputChecksumMiddlewares(stack *middleware.Stack, op return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getPutPublicAccessBlockRequestAlgorithmMember, RequireChecksum: true, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_RestoreObject.go b/service/s3/api_op_RestoreObject.go index 022dad38eec..697b621c4b1 100644 --- a/service/s3/api_op_RestoreObject.go +++ b/service/s3/api_op_RestoreObject.go @@ -341,6 +341,9 @@ func (c *Client) addOperationRestoreObjectMiddlewares(stack *middleware.Stack, o if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpRestoreObjectValidationMiddleware(stack); err != nil { return err } @@ -421,6 +424,7 @@ func addRestoreObjectInputChecksumMiddlewares(stack *middleware.Stack, options O return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getRestoreObjectRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: false, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, diff --git a/service/s3/api_op_UploadPart.go b/service/s3/api_op_UploadPart.go index 1f3ff357774..04f03ad2cbf 100644 --- a/service/s3/api_op_UploadPart.go +++ b/service/s3/api_op_UploadPart.go @@ -512,6 +512,9 @@ func (c *Client) addOperationUploadPartMiddlewares(stack *middleware.Stack, opti if err = addIsExpressUserAgent(stack); err != nil { return err } + if err = addRequestChecksumMetricsTracking(stack, options); err != nil { + return err + } if err = addOpUploadPartValidationMiddleware(stack); err != nil { return err } @@ -598,6 +601,7 @@ func addUploadPartInputChecksumMiddlewares(stack *middleware.Stack, options Opti return internalChecksum.AddInputMiddleware(stack, internalChecksum.InputMiddlewareOptions{ GetAlgorithm: getUploadPartRequestAlgorithmMember, RequireChecksum: false, + RequestChecksumCalculation: options.RequestChecksumCalculation, EnableTrailingChecksum: true, EnableComputeSHA256PayloadHash: true, EnableDecodedContentLengthHeader: true, @@ -642,6 +646,8 @@ func (c *PresignClient) PresignUploadPart(ctx context.Context, params *UploadPar } clientOptFns := append(options.ClientOptions, withNopHTTPClientAPIOption) + clientOptFns = append(options.ClientOptions, withNoDefaultChecksumAPIOption) + result, _, err := c.client.invokeOperation(ctx, "UploadPart", params, clientOptFns, c.client.addOperationUploadPartMiddlewares, presignConverter(options).convertToPresignMiddleware, diff --git a/service/s3/internal/customizations/presign_test.go b/service/s3/internal/customizations/presign_test.go index 5321ec99ff9..0651b7f631c 100644 --- a/service/s3/internal/customizations/presign_test.go +++ b/service/s3/internal/customizations/presign_test.go @@ -190,7 +190,7 @@ func TestPutObject_PresignURL(t *testing.T) { Bucket: aws.String("mock-bucket"), Key: aws.String("mockkey"), Body: strings.NewReader("hello world"), - ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c, + ChecksumAlgorithm: s3types.ChecksumAlgorithmSha1, ChecksumCRC32: aws.String("DUoRhQ=="), }, expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", @@ -209,6 +209,30 @@ func TestPutObject_PresignURL(t *testing.T) { "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, }, }, + "standard case with checksum algo set": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: strings.NewReader("hello world"), + ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c, + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + "X-Amz-Sdk-Checksum-Algorithm", + "X-Amz-Checksum-Crc32c", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, "standard case with checksum empty body": { input: s3.PutObjectInput{ Bucket: aws.String("mock-bucket"), diff --git a/service/s3/options.go b/service/s3/options.go index 8c67e4c6218..6b98e8802de 100644 --- a/service/s3/options.go +++ b/service/s3/options.go @@ -92,6 +92,12 @@ type Options struct { // The region to send requests to. (Required) Region string + // Indicates how user opt-in/out request checksum calculation + RequestChecksumCalculation aws.RequestChecksumCalculation + + // Indicates how user opt-in/out response checksum validation + ResponseChecksumValidation aws.ResponseChecksumValidation + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify diff --git a/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap b/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap index 4b009efa878..50ba96c834d 100644 --- a/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap +++ b/service/s3/snapshot/api_op_CreateBucketMetadataTableConfiguration.go.snap @@ -23,6 +23,7 @@ CreateBucketMetadataTableConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_DeleteObjects.go.snap b/service/s3/snapshot/api_op_DeleteObjects.go.snap index 8481825c051..58d3f780bad 100644 --- a/service/s3/snapshot/api_op_DeleteObjects.go.snap +++ b/service/s3/snapshot/api_op_DeleteObjects.go.snap @@ -23,6 +23,7 @@ DeleteObjects Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_GetObject.go.snap b/service/s3/snapshot/api_op_GetObject.go.snap index 058fe9dd341..ef8825ecc3e 100644 --- a/service/s3/snapshot/api_op_GetObject.go.snap +++ b/service/s3/snapshot/api_op_GetObject.go.snap @@ -23,6 +23,7 @@ GetObject Build stack step ClientRequestID ComputeContentLength + AWSChecksum:ResponseMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap b/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap index 32a6302623a..44bfde9eb46 100644 --- a/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap +++ b/service/s3/snapshot/api_op_PutBucketAccelerateConfiguration.go.snap @@ -23,6 +23,7 @@ PutBucketAccelerateConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketAcl.go.snap b/service/s3/snapshot/api_op_PutBucketAcl.go.snap index 61a75fe98d4..87575123558 100644 --- a/service/s3/snapshot/api_op_PutBucketAcl.go.snap +++ b/service/s3/snapshot/api_op_PutBucketAcl.go.snap @@ -23,6 +23,7 @@ PutBucketAcl Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketCors.go.snap b/service/s3/snapshot/api_op_PutBucketCors.go.snap index 62bf43b1bd8..7c4eb8d46a2 100644 --- a/service/s3/snapshot/api_op_PutBucketCors.go.snap +++ b/service/s3/snapshot/api_op_PutBucketCors.go.snap @@ -23,6 +23,7 @@ PutBucketCors Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketEncryption.go.snap b/service/s3/snapshot/api_op_PutBucketEncryption.go.snap index 8e45c4971da..bf1c5cb05b6 100644 --- a/service/s3/snapshot/api_op_PutBucketEncryption.go.snap +++ b/service/s3/snapshot/api_op_PutBucketEncryption.go.snap @@ -23,6 +23,7 @@ PutBucketEncryption Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap b/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap index e5c424450e8..583801997ef 100644 --- a/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap +++ b/service/s3/snapshot/api_op_PutBucketLifecycleConfiguration.go.snap @@ -23,6 +23,7 @@ PutBucketLifecycleConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketLogging.go.snap b/service/s3/snapshot/api_op_PutBucketLogging.go.snap index 0a076d55237..8738a13e938 100644 --- a/service/s3/snapshot/api_op_PutBucketLogging.go.snap +++ b/service/s3/snapshot/api_op_PutBucketLogging.go.snap @@ -23,6 +23,7 @@ PutBucketLogging Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap b/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap index e00b5fe5e25..0cefc18b637 100644 --- a/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap +++ b/service/s3/snapshot/api_op_PutBucketOwnershipControls.go.snap @@ -23,6 +23,7 @@ PutBucketOwnershipControls Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketPolicy.go.snap b/service/s3/snapshot/api_op_PutBucketPolicy.go.snap index af9cfdafffa..4cb4d52b26c 100644 --- a/service/s3/snapshot/api_op_PutBucketPolicy.go.snap +++ b/service/s3/snapshot/api_op_PutBucketPolicy.go.snap @@ -23,6 +23,7 @@ PutBucketPolicy Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketReplication.go.snap b/service/s3/snapshot/api_op_PutBucketReplication.go.snap index 113d84299c9..67fd7742c9d 100644 --- a/service/s3/snapshot/api_op_PutBucketReplication.go.snap +++ b/service/s3/snapshot/api_op_PutBucketReplication.go.snap @@ -23,6 +23,7 @@ PutBucketReplication Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap b/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap index c5008714204..3fea86f7735 100644 --- a/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap +++ b/service/s3/snapshot/api_op_PutBucketRequestPayment.go.snap @@ -23,6 +23,7 @@ PutBucketRequestPayment Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketTagging.go.snap b/service/s3/snapshot/api_op_PutBucketTagging.go.snap index 54855bd1cdf..d12d844b565 100644 --- a/service/s3/snapshot/api_op_PutBucketTagging.go.snap +++ b/service/s3/snapshot/api_op_PutBucketTagging.go.snap @@ -23,6 +23,7 @@ PutBucketTagging Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketVersioning.go.snap b/service/s3/snapshot/api_op_PutBucketVersioning.go.snap index 7ac57c9f441..9b04b8e15f2 100644 --- a/service/s3/snapshot/api_op_PutBucketVersioning.go.snap +++ b/service/s3/snapshot/api_op_PutBucketVersioning.go.snap @@ -23,6 +23,7 @@ PutBucketVersioning Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutBucketWebsite.go.snap b/service/s3/snapshot/api_op_PutBucketWebsite.go.snap index 57b1997313c..4d62cf8ecbd 100644 --- a/service/s3/snapshot/api_op_PutBucketWebsite.go.snap +++ b/service/s3/snapshot/api_op_PutBucketWebsite.go.snap @@ -23,6 +23,7 @@ PutBucketWebsite Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObject.go.snap b/service/s3/snapshot/api_op_PutObject.go.snap index ff941c819d8..861d7ce62aa 100644 --- a/service/s3/snapshot/api_op_PutObject.go.snap +++ b/service/s3/snapshot/api_op_PutObject.go.snap @@ -23,6 +23,7 @@ PutObject Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware S3100Continue diff --git a/service/s3/snapshot/api_op_PutObjectAcl.go.snap b/service/s3/snapshot/api_op_PutObjectAcl.go.snap index 115a13398d1..fb6f3fce035 100644 --- a/service/s3/snapshot/api_op_PutObjectAcl.go.snap +++ b/service/s3/snapshot/api_op_PutObjectAcl.go.snap @@ -23,6 +23,7 @@ PutObjectAcl Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap b/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap index c60f6008d31..5c3b59f1e69 100644 --- a/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap +++ b/service/s3/snapshot/api_op_PutObjectLegalHold.go.snap @@ -23,6 +23,7 @@ PutObjectLegalHold Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap b/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap index 51a3000e87d..d0456681261 100644 --- a/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap +++ b/service/s3/snapshot/api_op_PutObjectLockConfiguration.go.snap @@ -23,6 +23,7 @@ PutObjectLockConfiguration Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectRetention.go.snap b/service/s3/snapshot/api_op_PutObjectRetention.go.snap index b8a8d4b71f1..9273cdc017f 100644 --- a/service/s3/snapshot/api_op_PutObjectRetention.go.snap +++ b/service/s3/snapshot/api_op_PutObjectRetention.go.snap @@ -23,6 +23,7 @@ PutObjectRetention Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutObjectTagging.go.snap b/service/s3/snapshot/api_op_PutObjectTagging.go.snap index ebd21b58c8c..88f87bbee9c 100644 --- a/service/s3/snapshot/api_op_PutObjectTagging.go.snap +++ b/service/s3/snapshot/api_op_PutObjectTagging.go.snap @@ -23,6 +23,7 @@ PutObjectTagging Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap b/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap index 212f2688124..c6e2b3d6746 100644 --- a/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap +++ b/service/s3/snapshot/api_op_PutPublicAccessBlock.go.snap @@ -23,6 +23,7 @@ PutPublicAccessBlock Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_RestoreObject.go.snap b/service/s3/snapshot/api_op_RestoreObject.go.snap index 43182c060ec..830a3791363 100644 --- a/service/s3/snapshot/api_op_RestoreObject.go.snap +++ b/service/s3/snapshot/api_op_RestoreObject.go.snap @@ -23,6 +23,7 @@ RestoreObject Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware RecursionDetection diff --git a/service/s3/snapshot/api_op_UploadPart.go.snap b/service/s3/snapshot/api_op_UploadPart.go.snap index 09b15500600..0fbe9a23f05 100644 --- a/service/s3/snapshot/api_op_UploadPart.go.snap +++ b/service/s3/snapshot/api_op_UploadPart.go.snap @@ -23,6 +23,7 @@ UploadPart Build stack step ClientRequestID ComputeContentLength + AWSChecksum:RequestMetricsTracking UserAgent AddTimeOffsetMiddleware S3100Continue