diff --git a/packages/amplify-provider-awscloudformation/package.json b/packages/amplify-provider-awscloudformation/package.json index bdfef049394..03fd85e42fb 100644 --- a/packages/amplify-provider-awscloudformation/package.json +++ b/packages/amplify-provider-awscloudformation/package.json @@ -75,7 +75,7 @@ "@types/deep-diff": "^1.0.0", "@types/folder-hash": "^4.0.1", "@types/lodash.throttle": "^4.1.6", - "@types/node": "^12.12.6", + "@types/node": "^18.16.0", "@types/uuid": "^8.0.0", "jest": "^29.5.0", "typescript": "^4.9.5" diff --git a/packages/amplify-provider-awscloudformation/src/aws-utils/aws-s3.ts b/packages/amplify-provider-awscloudformation/src/aws-utils/aws-s3.ts index 1d46f63b775..b43af311007 100644 --- a/packages/amplify-provider-awscloudformation/src/aws-utils/aws-s3.ts +++ b/packages/amplify-provider-awscloudformation/src/aws-utils/aws-s3.ts @@ -10,6 +10,7 @@ import { $TSAny, $TSContext, AmplifyError, AmplifyFault, stateManager } from '@a import _ from 'lodash'; +import fs from 'fs-extra'; import ora from 'ora'; import { ListObjectVersionsOutput, ListObjectVersionsRequest, ObjectIdentifier } from 'aws-sdk/clients/s3'; import { pagedAWSCall } from './paged-call'; @@ -17,6 +18,7 @@ import { loadConfiguration } from '../configuration-manager'; import aws from './aws'; const providerName = require('../constants').ProviderName; +const consumers = require('stream/consumers'); const { fileLogger } = require('../utils/aws-logger'); @@ -126,6 +128,16 @@ export class S3 { spinner.start('Uploading files.'); } logger('uploadFile.s3.upload', [others])(); + const minChunkSize = 5 * 1024 * 1024; // 5 MB + if (augmentedS3Params.Body instanceof fs.ReadStream && fs.statSync(augmentedS3Params.Body.path).size <= minChunkSize) { + // Buffer small files to avoid memory leak. + // Previous implementation used s3.putObject for small uploads, but it didn't have retries, see https://github.com/aws-amplify/amplify-cli/pull/13493. + // On the other hand uploading small streams leads to memory leak, see https://github.com/aws/aws-sdk-js/issues/2552. + // Therefore, buffering small files ourselves seems to be middle ground between memory leak and loosing retries. + // Buffering small files brings back balance between leaking and non-leaking uploads that is matching + // the ratio from before https://github.com/aws-amplify/amplify-cli/pull/13493. + augmentedS3Params.Body = await consumers.buffer(augmentedS3Params.Body); + } uploadTask = this.s3.upload(augmentedS3Params); if (showSpinner) { uploadTask.on('httpUploadProgress', (max) => { diff --git a/packages/amplify-provider-awscloudformation/src/download-api-models.ts b/packages/amplify-provider-awscloudformation/src/download-api-models.ts index 9f37bfaf6b4..2d6e1f29831 100644 --- a/packages/amplify-provider-awscloudformation/src/download-api-models.ts +++ b/packages/amplify-provider-awscloudformation/src/download-api-models.ts @@ -50,6 +50,14 @@ const extractAPIModel = async (context: $TSContext, resource: $TSObject, framewo fs.ensureDirSync(tempDir); + // After updating node types from 12.x to 18.x the objectResult + // became not directly assignable to Buffer.from parameter type. + // However, this code has been running fine since 2022 which means that + // runtime types are compatible. + // The alternative would require multiple logical branches to handle type mismatch + // that doesn't seem to exist in runtime. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore const buff = Buffer.from(data.body); fs.writeFileSync(`${tempDir}/${apiName}.zip`, buff); await extract(`${tempDir}/${apiName}.zip`, { dir: tempDir }); diff --git a/packages/amplify-provider-awscloudformation/src/zip-util.ts b/packages/amplify-provider-awscloudformation/src/zip-util.ts index 0c15dcce7c5..1e597a1dd3b 100644 --- a/packages/amplify-provider-awscloudformation/src/zip-util.ts +++ b/packages/amplify-provider-awscloudformation/src/zip-util.ts @@ -15,6 +15,15 @@ export const downloadZip = async (s3: S3, tempDir: string, zipFileName: string, log(); const objectResult = await s3.getFile({ Key: zipFileName }, envName); fs.ensureDirSync(tempDir); + + // After updating node types from 12.x to 18.x the objectResult + // became not directly assignable to Buffer.from parameter type. + // However, this code has been running fine since 2022 which means that + // runtime types are compatible. + // The alternative would require multiple logical branches to handle type mismatch + // that doesn't seem to exist in runtime. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore const buff = Buffer.from(objectResult); const tempFile = `${tempDir}/${zipFileName}`; await fs.writeFile(tempFile, buff); diff --git a/yarn.lock b/yarn.lock index 82877b5b0bf..e0769cf0382 100644 --- a/yarn.lock +++ b/yarn.lock @@ -811,7 +811,7 @@ __metadata: "@types/deep-diff": ^1.0.0 "@types/folder-hash": ^4.0.1 "@types/lodash.throttle": ^4.1.6 - "@types/node": ^12.12.6 + "@types/node": ^18.16.0 "@types/uuid": ^8.0.0 amplify-codegen: 4.7.3 archiver: ^5.3.0 @@ -12488,10 +12488,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*, @types/node@npm:^18.16.1": - version: 18.16.1 - resolution: "@types/node@npm:18.16.1" - checksum: bd43d9e1df253955d73348ae9c8bc73f328ad3bd5481a134743142a04b0d1b74e39b90369aa0d361fd0b86a6a5c98035a226c1e1a4f67ba98e71b06734cc4f7d +"@types/node@npm:*, @types/node@npm:^18.16.0, @types/node@npm:^18.16.1": + version: 18.19.31 + resolution: "@types/node@npm:18.19.31" + dependencies: + undici-types: ~5.26.4 + checksum: bfebae8389220c0188492c82eaf328f4ba15e6e9b4abee33d6bf36d3b13f188c2f53eb695d427feb882fff09834f467405e2ed9be6aeb6ad4705509822d2ea08 languageName: node linkType: hard @@ -12503,9 +12505,9 @@ __metadata: linkType: hard "@types/node@npm:^16.9.2": - version: 16.11.14 - resolution: "@types/node@npm:16.11.14" - checksum: 017037da8387c85ea92c4ecb06dfb1f511e398ec14504a9395bc92948d58840755066eba991e308b39cf1117648be20cd3c43d184f1ec1339bb60b836dd5e4e7 + version: 16.18.96 + resolution: "@types/node@npm:16.18.96" + checksum: 05ac1c80c8d075086863f7640fd3e75c3912c4ed067bb38bb8fd5377f4e64de7a81d5be3ceae5448dc90d9802a0c7b0d3376538759b91ea652d16cc6dc7de767 languageName: node linkType: hard @@ -31466,6 +31468,13 @@ node-pty@beta: languageName: node linkType: hard +"undici-types@npm:~5.26.4": + version: 5.26.5 + resolution: "undici-types@npm:5.26.5" + checksum: bb673d7876c2d411b6eb6c560e0c571eef4a01c1c19925175d16e3a30c4c428181fb8d7ae802a261f283e4166a0ac435e2f505743aa9e45d893f9a3df017b501 + languageName: node + linkType: hard + "unfetch@npm:^4.2.0": version: 4.2.0 resolution: "unfetch@npm:4.2.0"