From cdebe7e19f98d14eaf2b2e7260082a00b363abc5 Mon Sep 17 00:00:00 2001 From: SanketD92 Date: Tue, 8 Sep 2020 14:17:54 -0400 Subject: [PATCH] adding rstudio SC product --- .../ec2-rstudio-instance.cfn.yml | 229 ++++++++++++++++++ .../steps/create-service-catalog-portfolio.js | 5 + .../environment-sc-connection-service.test.js | 110 +++++++++ .../environment-sc-connection-service.js | 28 +++ .../packages/base-raas-services/package.json | 1 + .../sidebar/common/workspaces/introduction.md | 9 +- .../config/infra/files/rstudio/set-password | 4 +- pnpm-lock.yaml | 7 +- 8 files changed, 384 insertions(+), 9 deletions(-) create mode 100644 addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/ec2-rstudio-instance.cfn.yml create mode 100644 addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/__tests__/environment-sc-connection-service.test.js diff --git a/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/ec2-rstudio-instance.cfn.yml b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/ec2-rstudio-instance.cfn.yml new file mode 100644 index 0000000000..6512e3dcac --- /dev/null +++ b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/ec2-rstudio-instance.cfn.yml @@ -0,0 +1,229 @@ +AWSTemplateFormatVersion: 2010-09-09 + +Description: Service-Workbench-on-AWS EC2-RStudio + +Parameters: + Namespace: + Type: String + Description: An environment name that will be prefixed to resource names + AmiId: + Type: String + Description: Amazon Machine Image for the EC2 instance + InstanceType: + Type: String + Description: EC2 instance type to launch + Default: t3.xlarge + KeyName: + Type: String + Description: Keypair name for SSH access + AccessFromCIDRBlock: + Type: String + Description: The CIDR used to access the ec2 instances. + S3Mounts: + Type: String + Description: A JSON array of objects with name, bucket, and prefix properties used to mount data + IamPolicyDocument: + Type: String + Description: The IAM policy to be associated with the launched workstation + VPC: + Description: The VPC in which the EC2 instance will reside + Type: AWS::EC2::VPC::Id + Subnet: + Description: The VPC subnet in which the EC2 instance will reside + Type: AWS::EC2::Subnet::Id + EnvironmentInstanceFiles: + Type: String + Description: >- + An S3 URI (starting with "s3://") that specifies the location of files to be copied to + the environment instance, including any bootstrap scripts + EncryptionKeyArn: + Type: String + Description: The ARN of the KMS encryption Key used to encrypt data in the instance + +Conditions: + IamPolicyEmpty: !Equals [!Ref IamPolicyDocument, '{}'] + +Resources: + IAMRole: + Type: 'AWS::IAM::Role' + Properties: + RoleName: !Join ['-', [Ref: Namespace, 'ec2-role']] + Path: '/' + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Principal: + Service: + - 'ec2.amazonaws.com' + Action: + - 'sts:AssumeRole' + Policies: + - !If + - IamPolicyEmpty + - !Ref 'AWS::NoValue' + - PolicyName: !Join ['-', [Ref: Namespace, 's3-studydata-policy']] + PolicyDocument: !Ref IamPolicyDocument + - PolicyName: !Join ['-', [Ref: Namespace, 's3-bootstrap-script-policy']] + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Action: 's3:GetObject' + Resource: !Sub + - 'arn:aws:s3:::${S3Location}/*' + # Remove "s3://" prefix from EnvironmentInstanceFiles + - S3Location: !Select [1, !Split ['s3://', !Ref EnvironmentInstanceFiles]] + - Effect: 'Allow' + Action: 's3:ListBucket' + Resource: !Sub + - 'arn:aws:s3:::${S3Bucket}' + - S3Bucket: !Select [2, !Split ['/', !Ref EnvironmentInstanceFiles]] + Condition: + StringLike: + s3:prefix: !Sub + - '${S3Prefix}/*' + - S3Prefix: !Select [3, !Split ['/', !Ref EnvironmentInstanceFiles]] + + InstanceProfile: + Type: 'AWS::IAM::InstanceProfile' + Properties: + InstanceProfileName: !Join ['-', [Ref: Namespace, 'ec2-profile']] + Path: '/' + Roles: + - Ref: IAMRole + + SecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: EC2 workspace security group + SecurityGroupEgress: + - IpProtocol: tcp + FromPort: 0 + ToPort: 65535 + CidrIp: 0.0.0.0/0 + - IpProtocol: icmp + FromPort: -1 + ToPort: -1 + CidrIp: !Ref AccessFromCIDRBlock + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref AccessFromCIDRBlock + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: !Ref AccessFromCIDRBlock + - IpProtocol: tcp + FromPort: 443 + ToPort: 443 + CidrIp: !Ref AccessFromCIDRBlock + Tags: + - Key: Name + Value: !Join ['-', [Ref: Namespace, 'ec2-sg']] + - Key: Description + Value: EC2 workspace security group + VpcId: !Ref VPC + + EC2Instance: + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: 'PT20M' + Properties: + ImageId: !Ref AmiId + InstanceType: !Ref InstanceType + IamInstanceProfile: !Ref InstanceProfile + KeyName: !Ref KeyName + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: 8 + Encrypted: true + KmsKeyId: !Ref EncryptionKeyArn + NetworkInterfaces: + - AssociatePublicIpAddress: 'true' + DeviceIndex: '0' + GroupSet: + - !Ref SecurityGroup + SubnetId: !Ref Subnet + Tags: + - Key: Name + Value: !Join ['-', [Ref: Namespace, 'ec2-rstudio']] + - Key: Description + Value: EC2 workspace instance + UserData: + Fn::Base64: !Sub | + #!/usr/bin/env bash + # Download and execute bootstrap script + aws s3 cp "${EnvironmentInstanceFiles}/get_bootstrap.sh" "/tmp" + chmod 500 "/tmp/get_bootstrap.sh" + /tmp/get_bootstrap.sh "${EnvironmentInstanceFiles}" '${S3Mounts}' + + # Signal result to CloudFormation + /opt/aws/bin/cfn-signal -e $? --stack "${AWS::StackName}" --resource "EC2Instance" --region "${AWS::Region}" + +Outputs: + Ec2WorkspaceDnsName: + Description: Public DNS name of the EC2 workspace instance + Value: !GetAtt [EC2Instance, PublicDnsName] + + Ec2WorkspacePublicIp: + Description: Public IP address of the EC2 workspace instance + Value: !GetAtt [EC2Instance, PublicIp] + + Ec2WorkspaceInstanceId: + Description: Instance Id for the EC2 workspace instance + Value: !Ref EC2Instance + + WorkspaceInstanceRoleArn: + Description: IAM role assumed by the EC2 workspace instance + Value: !GetAtt IAMRole.Arn + + # Connection related outputs. These outputs need to have prefix "MetaConnection" + # The "connections" are derived based on the CFN outputs as follows. + # + # CFN outputs with the OutputKey having format "MetaConnection" or "MetaConnection" + # are used for extracting connection information. + # - If the environment has only one connection then it can have outputs with "MetaConnection" format. + # - If it has multiple connections then it can have outputs with "MetaConnection" format. + # For example, MetaConnection1Name, MetaConnection2Name, etc. + # + # The expected CFN output variables used for capturing connections related information are as follows: + # + # - MetaConnectionName (or MetaConnectionName) - Provides name for connection + # + # - MetaConnectionUrl (or MetaConnectionUrl) - Provides connection url, if available + # + # - MetaConnectionScheme (or MetaConnectionScheme) - Provides connection protocol information such as http, https, ssh, jdbc, odbc etc + # + # - MetaConnectionType (or MetaConnectionType) - Provides type of the connection such as "SageMaker", "EMR", "FOO", "BAR" etc + # + # - MetaConnectionInfo (or MetaConnectionInfo) - Provides extra information required to form connection url. + # For example, in case of MetaConnectionType = SageMaker, the MetaConnectionInfo should provide SageMaker notebook + # instance name that can be used to form pre-signed SageMaker URL. + # + # - MetaConnectionInstanceId (or MetaConnectionInstanceId) - Provides AWS EC2 instanceId of the instance to connect to when applicable. + # Currently this is applicable only when ConnectionScheme = 'ssh'. + # This instanceId will be used for sending user's SSH public key using AWS EC2 Instance Connect when user wants to SSH to the instance. + # + MetaConnection1Info: + Description: The name of the RStudio notebook instance. + Value: !GetAtt [EC2Instance, PublicDnsName] + + MetaConnection1Type: + Description: Type of environment this connection is for + Value: RStudio + + MetaConnection1Name: + Description: Name for this connection + Value: RStudio Notebook + + MetaConnection1Scheme: + Description: Protocol for connection 1 + Value: https + + MetaConnection1InstanceId: + Description: EC2 Linux Instance Id + Value: !Ref EC2Instance diff --git a/addons/addon-base-raas/packages/base-raas-post-deployment/lib/steps/create-service-catalog-portfolio.js b/addons/addon-base-raas/packages/base-raas-post-deployment/lib/steps/create-service-catalog-portfolio.js index 65d4c514ce..7c3e028995 100644 --- a/addons/addon-base-raas/packages/base-raas-post-deployment/lib/steps/create-service-catalog-portfolio.js +++ b/addons/addon-base-raas/packages/base-raas-post-deployment/lib/steps/create-service-catalog-portfolio.js @@ -34,6 +34,11 @@ const productsToCreate = [ // * Product Feature 2`, // }, // DO NOT DELETE ANY ITEMS IN THIS LIST (else, you'll lose auto-create/update functionality for them) + { + filename: 'ec2-rstudio-instance', + displayName: 'EC2 RStudio', + description: `* An EC2 RStudio instance with HTTPS access`, + }, { filename: 'ec2-linux-instance', displayName: 'EC2 Linux', diff --git a/addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/__tests__/environment-sc-connection-service.test.js b/addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/__tests__/environment-sc-connection-service.test.js new file mode 100644 index 0000000000..c575812a4b --- /dev/null +++ b/addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/__tests__/environment-sc-connection-service.test.js @@ -0,0 +1,110 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +const ServicesContainer = require('@aws-ee/base-services-container/lib/services-container'); +const JsonSchemaValidationService = require('@aws-ee/base-services/lib/json-schema-validation-service'); +const Logger = require('@aws-ee/base-services/lib/logger/logger-service'); + +// Mocked dependencies +jest.mock('@aws-ee/base-api-services/lib/jwt-service'); +const JwtService = require('@aws-ee/base-api-services/lib/jwt-service'); + +jest.mock('@aws-ee/key-pair-mgmt-services/lib/key-pair/key-pair-service'); +const KeyPairServiceMock = require('@aws-ee/key-pair-mgmt-services/lib/key-pair/key-pair-service'); + +jest.mock('@aws-ee/base-services/lib/settings/env-settings-service'); +const SettingsServiceMock = require('@aws-ee/base-services/lib/settings/env-settings-service'); + +jest.mock('@aws-ee/base-services/lib/audit/audit-writer-service'); +const AuditServiceMock = require('@aws-ee/base-services/lib/audit/audit-writer-service'); + +jest.mock('@aws-ee/base-services/lib/plugin-registry/plugin-registry-service'); +const PluginRegistryServiceMock = require('@aws-ee/base-services/lib/plugin-registry/plugin-registry-service'); + +jest.mock('../../environment-dns-service.js'); +const EnvironmentDnsServiceMock = require('../../environment-dns-service.js'); + +jest.mock('../environment-sc-service'); +const EnvironmentSCServiceMock = require('../environment-sc-service'); + +jest.mock('../environment-sc-keypair-service'); +const EnvironmentScKeyPairServiceMock = require('../environment-sc-keypair-service'); + +const EnvironmentScConnectionService = require('../environment-sc-connection-service'); + +describe('EnvironmentScConnectionService', () => { + let service = null; + beforeEach(async () => { + const container = new ServicesContainer(); + container.register('jsonSchemaValidationService', new JsonSchemaValidationService()); + container.register('jwtService', new JwtService()); + container.register('log', new Logger()); + container.register('auditWriterService', new AuditServiceMock()); + container.register('pluginRegistryService', new PluginRegistryServiceMock()); + container.register('settings', new SettingsServiceMock()); + container.register('environmentDnsService', new EnvironmentDnsServiceMock()); + container.register('keyPairService', new KeyPairServiceMock()); + container.register('environmentScKeypairService', new EnvironmentScKeyPairServiceMock()); + container.register('environmentScService', new EnvironmentSCServiceMock()); + container.register('environmentScConnectionService', new EnvironmentScConnectionService()); + await container.initServices(); + + // suppress expected console errors + jest.spyOn(console, 'error').mockImplementation(); + + // Get instance of the service we are testing + service = await container.find('environmentScConnectionService'); + }); + + describe('create connection', () => { + it('should return connection if exists', async () => { + // BUILD + const connection = { url: 'www.example.com', info: 'An already existing connection' }; + service.mustFindConnection = jest.fn(() => connection); + + // OPERATE + const retConn = await service.createConnectionUrl(); + + // CHECK + expect(retConn).toBe(connection); + }); + + it('should get RStudio connection URL for RStudio connection types', async () => { + // BUILD + const connection = { type: 'RStudio' }; + service.mustFindConnection = jest.fn(() => connection); + service.getRStudioUrl = jest.fn(); + + // OPERATE + await service.createConnectionUrl(); + + // CHECK + expect(service.getRStudioUrl).toHaveBeenCalled(); + }); + + it('should NOT get RStudio connection URL for non-RStudio connection types', async () => { + // BUILD + const connection = { type: 'nonRStudio' }; + service.mustFindConnection = jest.fn(() => connection); + service.getRStudioUrl = jest.fn(); + + // OPERATE + await service.createConnectionUrl(); + + // CHECK + expect(service.getRStudioUrl).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/environment-sc-connection-service.js b/addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/environment-sc-connection-service.js index 13fdb3812a..068cc6739f 100644 --- a/addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/environment-sc-connection-service.js +++ b/addons/addon-base-raas/packages/base-raas-services/lib/environment/service-catalog/environment-sc-connection-service.js @@ -15,6 +15,9 @@ const _ = require('lodash'); const crypto = require('crypto'); +const querystring = require('querystring'); +const request = require('request-promise-native'); +const rstudioEncryptor = require('@aws-ee/base-services/lib/helpers/rstudio-encryptor'); const Service = require('@aws-ee/base-services-container/lib/service'); const sshConnectionInfoSchema = require('../../schema/ssh-connection-info-sc'); const { connectionScheme } = require('./environment-sc-connection-enum'); @@ -26,7 +29,9 @@ class EnvironmentScConnectionService extends Service { this.dependency([ 'environmentScService', 'environmentScKeypairService', + 'environmentDnsService', 'jsonSchemaValidationService', + 'jwtService', 'keyPairService', 'auditWriterService', 'pluginRegistryService', @@ -153,6 +158,10 @@ class EnvironmentScConnectionService extends Service { connection.url = _.get(sageMakerResponse, 'AuthorizedUrl'); } + if (_.toLower(_.get(connection, 'type', '')) === 'rstudio') { + connection.url = await this.getRStudioUrl(envId, connection); + } + // Give plugins chance to adjust the connection (such as connection url etc) const result = await pluginRegistryService.visitPlugins( 'env-sc-connection-url', @@ -169,6 +178,25 @@ class EnvironmentScConnectionService extends Service { return _.get(result, 'connection') || connection; } + async getRStudioUrl(id, connection) { + const environmentDnsService = await this.service('environmentDnsService'); + const rstudioDomainName = environmentDnsService.getHostname('rstudio', id); + const rstudioPublicKeyUrl = `https://${rstudioDomainName}/auth-public-key`; + const rstudioSignInUrl = `https://${rstudioDomainName}/auth-do-sign-in`; + const instanceId = connection.instanceId; + const jwtService = await this.service('jwtService'); + const jwtSecret = await jwtService.getSecret(); + const hash = crypto.createHash('sha256'); + const username = 'rstudio-user'; + const password = hash.update(`${instanceId}${jwtSecret}`).digest('hex'); + const credentials = `${username}\n${password}`; + const publicKey = await request(rstudioPublicKeyUrl); + const [exponent, modulus] = publicKey.split(':', 2); + const params = { v: rstudioEncryptor.encrypt(credentials, exponent, modulus) }; + const authorizedUrl = `${rstudioSignInUrl}?${querystring.encode(params)}`; + return authorizedUrl; + } + async sendSshPublicKey(requestContext, envId, connectionId, sshConnectionInfo) { const [environmentScService, keyPairService, validationService] = await this.service([ 'environmentScService', diff --git a/addons/addon-base-raas/packages/base-raas-services/package.json b/addons/addon-base-raas/packages/base-raas-services/package.json index 63e57d3215..64d24499c1 100644 --- a/addons/addon-base-raas/packages/base-raas-services/package.json +++ b/addons/addon-base-raas/packages/base-raas-services/package.json @@ -9,6 +9,7 @@ "@aws-ee/base-api-services": "workspace:*", "@aws-ee/base-services": "workspace:*", "@aws-ee/base-services-container": "workspace:*", + "@aws-ee/key-pair-mgmt-services": "workspace:*", "lodash": "^4.17.15", "node-cache": "^4.2.1", "request": "^2.88.2", diff --git a/docs/docs/user_guide/sidebar/common/workspaces/introduction.md b/docs/docs/user_guide/sidebar/common/workspaces/introduction.md index dc89ded997..2029033614 100644 --- a/docs/docs/user_guide/sidebar/common/workspaces/introduction.md +++ b/docs/docs/user_guide/sidebar/common/workspaces/introduction.md @@ -8,7 +8,8 @@ Once a User has found the Study or Studies that they are interested in performin A Workspace is an environment that contains a set of tools to access and integrate data. The following environments are currently provided: -* **SageMaker Notebook** - A SageMaker Jupyter Notebook with TensorFlow, Apache MXNet and Scikit-learn2 -* **EMR** - An Amazon EMR research workspace with Hail 0.2, JupyterLab, Spark 2.4.4 and Hadoop 2.8.52 -* **EC2 - Linux** - An EC2 Linux instance. -* **EC2 - Windows** - An EC2 Windows instance. +- **SageMaker Notebook** - A SageMaker Jupyter Notebook with TensorFlow, Apache MXNet and Scikit-learn2 +- **EMR** - An Amazon EMR research workspace with Hail 0.2, JupyterLab, Spark 2.4.4 and Hadoop 2.8.52 +- **EC2 - Linux** - An EC2 Linux instance. +- **EC2 - Windows** - An EC2 Windows instance. +- **EC2 - RStudio** - An EC2 RStudio instance. diff --git a/main/solution/machine-images/config/infra/files/rstudio/set-password b/main/solution/machine-images/config/infra/files/rstudio/set-password index d8eb391a5b..651dec9bf3 100755 --- a/main/solution/machine-images/config/infra/files/rstudio/set-password +++ b/main/solution/machine-images/config/infra/files/rstudio/set-password @@ -3,5 +3,5 @@ instance_id=$(curl -s "http://169.254.169.254/1.0/meta-data/instance-id") secret=$(cat "/root/secret.txt") password=$(echo -n "${instance_id}${secret}" | sha256sum | awk '{print $1}') -echo "galileo:$password" | /usr/sbin/chpasswd -echo "Set galileo password" +echo "rstudio-user:$password" | /usr/sbin/chpasswd +echo "Set rstudio-user password" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9abe760dff..27a6b79e5a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -301,6 +301,7 @@ importers: '@aws-ee/base-api-services': 'link:../../../addon-base-rest-api/packages/services' '@aws-ee/base-services': 'link:../../../addon-base/packages/services' '@aws-ee/base-services-container': 'link:../../../addon-base/packages/services-container' + '@aws-ee/key-pair-mgmt-services': 'link:../../../addon-key-pair-mgmt-api/packages/key-pair-mgmt-services' lodash: 4.17.15 moment: 2.27.0 node-cache: 4.2.1 @@ -328,6 +329,7 @@ importers: '@aws-ee/base-api-services': 'workspace:*' '@aws-ee/base-services': 'workspace:*' '@aws-ee/base-services-container': 'workspace:*' + '@aws-ee/key-pair-mgmt-services': 'workspace:*' aws-sdk-mock: ^5.1.0 eslint: ^6.8.0 eslint-config-airbnb-base: ^14.1.0 @@ -4873,7 +4875,7 @@ packages: /@typescript-eslint/typescript-estree/2.27.0: dependencies: debug: 4.1.1 - eslint-visitor-keys: 1.3.0 + eslint-visitor-keys: 1.1.0 glob: 7.1.6 is-glob: 4.0.1 lodash: 4.17.15 @@ -14220,7 +14222,6 @@ packages: resolution: integrity: sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== /moment/2.27.0: - dev: false resolution: integrity: sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ== /moo/0.5.1: @@ -19331,7 +19332,7 @@ packages: integrity: sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q== /tsutils/3.17.1: dependencies: - tslib: 1.13.0 + tslib: 1.11.1 dev: true engines: node: '>= 6'