From b8d6b2e7f83816b8d5b904458028156bab11bc93 Mon Sep 17 00:00:00 2001 From: Changshama Date: Thu, 5 Jan 2023 10:21:29 -0500 Subject: [PATCH 01/17] fix execution stepname error --- build_pipeline/build-pipeline.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_pipeline/build-pipeline.ipynb b/build_pipeline/build-pipeline.ipynb index 7ffcdc3..5732f77 100644 --- a/build_pipeline/build-pipeline.ipynb +++ b/build_pipeline/build-pipeline.ipynb @@ -249,7 +249,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_package_arn = get_execution_step(\"RegisterModel\")[0][\"RegisterModel\"][\"Arn\"]\n", + "model_package_arn = get_execution_step(\"RegisterModel-RegisterModel\")[0][\"RegisterModel\"][\"Arn\"]\n", "model_package_version = model_package_arn.split(\"/\")[-1]\n", "print(f\"Model version: {model_package_version}\")" ] From b2f13f429dc2a135b945af137be12b6eea787c2c Mon Sep 17 00:00:00 2001 From: Changshama Date: Thu, 5 Jan 2023 10:22:28 -0500 Subject: [PATCH 02/17] update sm pipeline sdk --- batch_pipeline/pipelines/pipeline.py | 7 +++---- build_pipeline/pipelines/pipeline.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/batch_pipeline/pipelines/pipeline.py b/batch_pipeline/pipelines/pipeline.py index 73da449..81b2ead 100644 --- a/batch_pipeline/pipelines/pipeline.py +++ b/batch_pipeline/pipelines/pipeline.py @@ -35,6 +35,7 @@ ParameterString, ) from sagemaker.workflow.pipeline import Pipeline +from sagemaker.workflow.pipeline_context import PipelineSession from sagemaker.workflow.steps import ( CreateModelStep, ProcessingStep, @@ -68,12 +69,10 @@ def get_session(region, default_bucket): boto_session = boto3.Session(region_name=region) sagemaker_client = boto_session.client("sagemaker") - runtime_client = boto_session.client("sagemaker-runtime") - return sagemaker.session.Session( + return PipelineSession( boto_session=boto_session, sagemaker_client=sagemaker_client, - sagemaker_runtime_client=runtime_client, - default_bucket=default_bucket, + default_bucket=default_bucket, ) diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index 7737b22..990ca09 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -35,6 +35,7 @@ from sagemaker.workflow.functions import Join from sagemaker.workflow.parameters import ParameterInteger, ParameterString from sagemaker.workflow.pipeline import Pipeline +from sagemaker.workflow.pipeline_context import PipelineSession from sagemaker.workflow.properties import PropertyFile from sagemaker.workflow.quality_check_step import ( DataQualityCheckConfig, @@ -59,12 +60,10 @@ def get_session(region, default_bucket): boto_session = boto3.Session(region_name=region) sagemaker_client = boto_session.client("sagemaker") - runtime_client = boto_session.client("sagemaker-runtime") - return sagemaker.session.Session( + return PipelineSession( boto_session=boto_session, sagemaker_client=sagemaker_client, - sagemaker_runtime_client=runtime_client, - default_bucket=default_bucket, + default_bucket=default_bucket, ) From 321619977ed5cc81e83a0de5fdb33884d185755d Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Tue, 24 Jan 2023 21:11:32 +0800 Subject: [PATCH 03/17] Refactor build SM pipelines --- build_pipeline/pipelines/pipeline.py | 169 ++++++++++++++------------- 1 file changed, 88 insertions(+), 81 deletions(-) diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index 990ca09..93296da 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -7,7 +7,6 @@ Implements a get_pipeline(**kwargs) method. """ -import json import os import boto3 @@ -22,7 +21,6 @@ from sagemaker.processing import ( ProcessingInput, ProcessingOutput, - Processor, ScriptProcessor, ) from sagemaker.s3 import S3Uploader @@ -33,16 +31,15 @@ from sagemaker.workflow.conditions import ConditionLessThanOrEqualTo from sagemaker.workflow.execution_variables import ExecutionVariables from sagemaker.workflow.functions import Join +from sagemaker.workflow.model_step import ModelStep from sagemaker.workflow.parameters import ParameterInteger, ParameterString from sagemaker.workflow.pipeline import Pipeline from sagemaker.workflow.pipeline_context import PipelineSession from sagemaker.workflow.properties import PropertyFile from sagemaker.workflow.quality_check_step import ( DataQualityCheckConfig, - ModelQualityCheckConfig, QualityCheckStep, ) -from sagemaker.workflow.step_collections import RegisterModel from sagemaker.workflow.steps import CacheConfig, ProcessingStep, TrainingStep BASE_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -63,7 +60,7 @@ def get_session(region, default_bucket): return PipelineSession( boto_session=boto_session, sagemaker_client=sagemaker_client, - default_bucket=default_bucket, + default_bucket=default_bucket, ) @@ -125,47 +122,52 @@ def get_pipeline( default_value=f"s3://{default_bucket}/{base_job_prefix}/baseline/", ) - # Create cache configuration (Unable to pass parameter for expire_after value) + # Cache configuration (Unable to pass parameter for expire_after value) cache_config = CacheConfig(enable_caching=False, expire_after="PT1H") # processing step for feature engineering + inputs = [ + ProcessingInput( + source=input_data, + destination="/opt/ml/processing/input/data", + s3_data_distribution_type="ShardedByS3Key", + ), + ProcessingInput( + source=input_zones, + destination="/opt/ml/processing/input/zones", + s3_data_distribution_type="FullyReplicated", + ), + ] + + outputs = [ + ProcessingOutput(output_name="train", source="/opt/ml/processing/train"), + ProcessingOutput( + output_name="validation", source="/opt/ml/processing/validation" + ), + ProcessingOutput(output_name="test", source="/opt/ml/processing/test"), + ProcessingOutput(output_name="baseline", source="/opt/ml/processing/baseline"), + ] + sklearn_processor = SKLearnProcessor( framework_version="0.23-1", + role=role, instance_type=processing_instance_type, - instance_count=processing_instance_count, - base_job_name=f"{base_job_prefix}/sklearn-preprocess", + instance_count=processing_instance_type, sagemaker_session=sagemaker_session, - role=role, + base_job_name=f"{base_job_prefix}/sklearn-preprocess", ) + step_process = ProcessingStep( name="PreprocessData", - processor=sklearn_processor, - inputs=[ - ProcessingInput( - source=input_data, - destination="/opt/ml/processing/input/data", - s3_data_distribution_type="ShardedByS3Key", - ), - ProcessingInput( - source=input_zones, - destination="/opt/ml/processing/input/zones", - s3_data_distribution_type="FullyReplicated", - ), - ], - outputs=[ - ProcessingOutput(output_name="train", source="/opt/ml/processing/train"), - ProcessingOutput( - output_name="validation", source="/opt/ml/processing/validation" - ), - ProcessingOutput(output_name="test", source="/opt/ml/processing/test"), - ProcessingOutput( - output_name="baseline", source="/opt/ml/processing/baseline" - ), - ], - code=os.path.join(BASE_DIR, "preprocess.py"), + step_args=sklearn_processor.run( + inputs=inputs, + outputs=outputs, + code=os.path.join(BASE_DIR, "preprocess.py"), + ), cache_config=cache_config, ) + # Data Quality Baseline step check_job_config = CheckJobConfig( role=role, instance_count=1, @@ -174,9 +176,11 @@ def get_pipeline( ) data_quality_check_config = DataQualityCheckConfig( - baseline_dataset=step_process.properties.ProcessingOutputConfig.Outputs[ - "baseline" - ].S3Output.S3Uri, + baseline_dataset=( + step_process.properties.ProcessingOutputConfig.Outputs[ + "baseline" + ].S3Output.S3Uri + ), dataset_format=DatasetFormat.csv(), output_s3_uri=Join( on="/", @@ -236,23 +240,25 @@ def get_pipeline( min_child_weight=300, subsample=0.8, ) + step_train = TrainingStep( name="TrainModel", - estimator=xgb_train, - inputs={ - "train": TrainingInput( - s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ - "train" - ].S3Output.S3Uri, - content_type="text/csv", - ), - "validation": TrainingInput( - s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ - "validation" - ].S3Output.S3Uri, - content_type="text/csv", - ), - }, + step_args=xgb_train.fit( + inputs={ + "train": TrainingInput( + s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ + "train" + ].S3Output.S3Uri, + content_type="text/csv", + ), + "validation": TrainingInput( + s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ + "validation" + ].S3Output.S3Uri, + content_type="text/csv", + ), + } + ), cache_config=cache_config, ) @@ -273,25 +279,26 @@ def get_pipeline( ) step_eval = ProcessingStep( name="EvaluateModel", - processor=script_eval, - inputs=[ - ProcessingInput( - source=step_train.properties.ModelArtifacts.S3ModelArtifacts, - destination="/opt/ml/processing/model", - ), - ProcessingInput( - source=step_process.properties.ProcessingOutputConfig.Outputs[ - "test" - ].S3Output.S3Uri, - destination="/opt/ml/processing/test", - ), - ], - outputs=[ - ProcessingOutput( - output_name="evaluation", source="/opt/ml/processing/evaluation" - ), - ], - code=os.path.join(BASE_DIR, "evaluate.py"), + step_args=script_eval.run( + inputs=[ + ProcessingInput( + source=step_train.properties.ModelArtifacts.S3ModelArtifacts, + destination="/opt/ml/processing/model", + ), + ProcessingInput( + source=step_process.properties.ProcessingOutputConfig.Outputs[ + "test" + ].S3Output.S3Uri, + destination="/opt/ml/processing/test", + ), + ], + outputs=[ + ProcessingOutput( + output_name="evaluation", source="/opt/ml/processing/evaluation" + ), + ], + code=os.path.join(BASE_DIR, "evaluate.py"), + ), property_files=[evaluation_report], cache_config=cache_config, ) @@ -327,18 +334,18 @@ def get_pipeline( ), ) - step_register = RegisterModel( + step_register = ModelStep( name="RegisterModel", - estimator=xgb_train, - model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, - content_types=["text/csv"], - response_types=["text/csv"], - inference_instances=["ml.t2.medium", "ml.t2.large", "ml.m5.large"], - transform_instances=["ml.m5.large"], - model_package_group_name=model_package_group_name, - approval_status=model_approval_status, - model_metrics=model_metrics, - drift_check_baselines=drift_check_baselines, + step_args=xgb_train.register( + content_types=["text/csv"], + response_types=["text/csv"], + inference_instances=["ml.t2.medium", "ml.t2.large", "ml.m5.large"], + transform_instances=["ml.m5.large"], + model_package_group_name=model_package_group_name, + approval_status=model_approval_status, + model_metrics=model_metrics, + drift_check_baselines=drift_check_baselines, + ), ) # condition step for evaluating model quality and branching execution From a382adb66df7bf880726dacc8f93e0e21afc9338 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Tue, 24 Jan 2023 22:58:44 +0800 Subject: [PATCH 04/17] Fix model registry step - `ModelStep` doesn't support `estimator.register()`, added intermediate `Model` - fixed variable name for the lowercase lambda --- build_pipeline/pipelines/pipeline.py | 21 +++++++++++++-------- infra/service_catalog_stack.py | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index 93296da..5e7dcb3 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -16,13 +16,10 @@ from sagemaker.drift_check_baselines import DriftCheckBaselines from sagemaker.estimator import Estimator from sagemaker.inputs import TrainingInput +from sagemaker.model import Model from sagemaker.model_metrics import MetricsSource, ModelMetrics from sagemaker.model_monitor.dataset_format import DatasetFormat -from sagemaker.processing import ( - ProcessingInput, - ProcessingOutput, - ScriptProcessor, -) +from sagemaker.processing import ProcessingInput, ProcessingOutput, ScriptProcessor from sagemaker.s3 import S3Uploader from sagemaker.sklearn.processing import SKLearnProcessor from sagemaker.utils import name_from_base @@ -152,7 +149,7 @@ def get_pipeline( framework_version="0.23-1", role=role, instance_type=processing_instance_type, - instance_count=processing_instance_type, + instance_count=processing_instance_count, sagemaker_session=sagemaker_session, base_job_name=f"{base_job_prefix}/sklearn-preprocess", ) @@ -334,9 +331,16 @@ def get_pipeline( ), ) + model = Model( + image_uri=image_uri, + model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, + sagemaker_session=sagemaker_session, + role=role, + ) + step_register = ModelStep( name="RegisterModel", - step_args=xgb_train.register( + step_args=model.register( content_types=["text/csv"], response_types=["text/csv"], inference_instances=["ml.t2.medium", "ml.t2.large", "ml.m5.large"], @@ -394,7 +398,8 @@ def upload_pipeline(pipeline: Pipeline, default_bucket, base_job_prefix): S3Uploader.upload_string_as_file_body( pipeline_definition_body, f"s3://{default_bucket}/{pipeline_name}.json" ) - # Return JSON with parameters used in Cfn Stack creation as template-configuration.json + # Return JSON with parameters used in Cfn Stack creation as + # template-configuration.json return { "PipelineDefinitionBucket": default_bucket, "PipelineDefinitionKey": f"{pipeline_name}.json", diff --git a/infra/service_catalog_stack.py b/infra/service_catalog_stack.py index 6281506..2f88902 100644 --- a/infra/service_catalog_stack.py +++ b/infra/service_catalog_stack.py @@ -95,14 +95,14 @@ def __init__( # Lambda powering the custom resource to convert names to lower case at # deploy time with open("lambda/lowercase_name.py", encoding="utf8") as fp: - lambda_start_pipeline_code = fp.read() + lambda_lowercase_code = fp.read() lowercase_lambda = lambda_.Function( self, "LowerCaseLambda", function_name="sagemaker-lowercase-names", description="Returns the lowercase version of a string", - code=lambda_.Code.from_inline(lambda_start_pipeline_code), + code=lambda_.Code.from_inline(lambda_lowercase_code), handler="index.lambda_handler", runtime=lambda_.Runtime.PYTHON_3_8, timeout=cdk.Duration.seconds(3), From a2f21699825950003074216ae2286cb6afb8b1ef Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 30 Jan 2023 08:18:35 +0800 Subject: [PATCH 05/17] Organize build pipeline outputs --- build_pipeline/.gitignore | 1 + build_pipeline/app.py | 6 + build_pipeline/cdk.json | 3 +- .../infra/sagemaker_pipeline_stack.py | 14 - build_pipeline/pipelines/pipeline.py | 142 ++-- .../pipelines/{ => preprocess}/preprocess.py | 5 +- .../pipelines/preprocess/requirements.txt | 1 + build_pipeline/poetry.lock | 724 +++++++++--------- build_pipeline/requirements.txt | 42 +- 9 files changed, 497 insertions(+), 441 deletions(-) rename build_pipeline/pipelines/{ => preprocess}/preprocess.py (97%) create mode 100644 build_pipeline/pipelines/preprocess/requirements.txt diff --git a/build_pipeline/.gitignore b/build_pipeline/.gitignore index d96dae6..3c147b5 100644 --- a/build_pipeline/.gitignore +++ b/build_pipeline/.gitignore @@ -4,6 +4,7 @@ __pycache__ .pytest_cache .venv *.egg-info +*.zip # CDK asset staging directory .cdk.staging diff --git a/build_pipeline/app.py b/build_pipeline/app.py index 06007c2..4b96874 100644 --- a/build_pipeline/app.py +++ b/build_pipeline/app.py @@ -21,6 +21,7 @@ def main( sagemaker_pipeline_description, sagemaker_pipeline_role, artifact_bucket, + commit_id=None, ): # Use project_name for pipeline and model package group name model_package_group_name = project_name @@ -31,6 +32,7 @@ def main( model_package_group_name=model_package_group_name, pipeline_name=sagemaker_pipeline_name, base_job_prefix=project_id, + commit_id=commit_id, ) # Create the pipeline definition @@ -91,6 +93,10 @@ def main( "--artifact-bucket", default=os.environ.get("ARTIFACT_BUCKET"), ) + parser.add_argument( + "--commit-id", + default=os.environ.get("COMMIT_ID"), + ) args = vars(parser.parse_args()) logger.info("args: {}".format(args)) main(**args) diff --git a/build_pipeline/cdk.json b/build_pipeline/cdk.json index 9661d79..ddbb4d5 100644 --- a/build_pipeline/cdk.json +++ b/build_pipeline/cdk.json @@ -1,4 +1,5 @@ { "app": "python3 app.py", - "context": {} + "context": {}, + "versionReporting": false } \ No newline at end of file diff --git a/build_pipeline/infra/sagemaker_pipeline_stack.py b/build_pipeline/infra/sagemaker_pipeline_stack.py index fc14baf..14ad0a0 100644 --- a/build_pipeline/infra/sagemaker_pipeline_stack.py +++ b/build_pipeline/infra/sagemaker_pipeline_stack.py @@ -25,20 +25,6 @@ def __init__( ) -> None: super().__init__(scope, construct_id, **kwargs) - # definition_bucket = cdk.CfnParameter( - # self, - # "PipelineDefinitionBucket", - # type="String", - # description="The s3 bucket for pipeline definition", - # min_length=1, - # ) - # definition_key = cdk.CfnParameter( - # self, - # "PipelineDefinitionKey", - # type="String", - # description="The s3 key for pipeline definition", - # min_length=1, - # ) sagemaker.CfnModelPackageGroup( self, diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index 5e7dcb3..a30f66c 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -19,15 +19,20 @@ from sagemaker.model import Model from sagemaker.model_metrics import MetricsSource, ModelMetrics from sagemaker.model_monitor.dataset_format import DatasetFormat -from sagemaker.processing import ProcessingInput, ProcessingOutput, ScriptProcessor +from sagemaker.processing import ( + FrameworkProcessor, + ProcessingInput, + ProcessingOutput, + ScriptProcessor, +) from sagemaker.s3 import S3Uploader -from sagemaker.sklearn.processing import SKLearnProcessor +from sagemaker.sklearn import SKLearn from sagemaker.utils import name_from_base from sagemaker.workflow.check_job_config import CheckJobConfig -from sagemaker.workflow.condition_step import ConditionStep, JsonGet +from sagemaker.workflow.condition_step import ConditionStep from sagemaker.workflow.conditions import ConditionLessThanOrEqualTo from sagemaker.workflow.execution_variables import ExecutionVariables -from sagemaker.workflow.functions import Join +from sagemaker.workflow.functions import Join, JsonGet from sagemaker.workflow.model_step import ModelStep from sagemaker.workflow.parameters import ParameterInteger, ParameterString from sagemaker.workflow.pipeline import Pipeline @@ -68,6 +73,7 @@ def get_pipeline( model_package_group_name, default_bucket, base_job_prefix, + commit_id: str = None, ): """Gets a SageMaker ML Pipeline instance working with on nyc taxi data. Args: @@ -89,36 +95,46 @@ def get_pipeline( ) input_data = ParameterString( name="InputDataUrl", - default_value=f"s3://{default_bucket}/{base_job_prefix}/input/data", + default_value=f"s3://{default_bucket}/inputs/data", ) input_zones = ParameterString( name="InputZonesUrl", - default_value=f"s3://{default_bucket}/{base_job_prefix}/input/zones/taxi_zones.zip", + default_value=f"s3://{default_bucket}/inputs/zones/taxi_zones.zip", ) processing_instance_count = ParameterInteger( - name="ProcessingInstanceCount", default_value=1 + name="ProcessingInstanceCount", + default_value=1, ) processing_instance_type = ParameterString( - name="ProcessingInstanceType", default_value="ml.m5.xlarge" + name="ProcessingInstanceType", + default_value="ml.m5.xlarge", ) baseline_instance_type = ParameterString( - name="BaselineInstanceType", default_value="ml.m5.xlarge" + name="BaselineInstanceType", + default_value="ml.m5.xlarge", ) training_instance_type = ParameterString( - name="TrainingInstanceType", default_value="ml.m5.xlarge" + name="TrainingInstanceType", + default_value="ml.m5.xlarge", ) model_approval_status = ParameterString( - name="ModelApprovalStatus", default_value="PendingManualApproval" - ) - model_output = ParameterString( - name="ModelOutputUrl", - default_value=f"s3://{default_bucket}/{base_job_prefix}/model", - ) - baseline_output = ParameterString( - name="OutputBaselineUrl", - default_value=f"s3://{default_bucket}/{base_job_prefix}/baseline/", + name="ModelApprovalStatus", + default_value="PendingManualApproval", ) + output_common_path = [ + "s3:/", + default_bucket, + "build-pipeline-runs", + Join( + on="-", + values=[ + ExecutionVariables.START_DATETIME, + ExecutionVariables.PIPELINE_EXECUTION_ID, + ], + ), + ] + # Cache configuration (Unable to pass parameter for expire_after value) cache_config = CacheConfig(enable_caching=False, expire_after="PT1H") @@ -137,21 +153,47 @@ def get_pipeline( ] outputs = [ - ProcessingOutput(output_name="train", source="/opt/ml/processing/train"), ProcessingOutput( - output_name="validation", source="/opt/ml/processing/validation" + output_name="train", + source="/opt/ml/processing/train", + destination=Join( + on="/", + values=output_common_path + ["train"], + ), + ), + ProcessingOutput( + output_name="validation", + source="/opt/ml/processing/validation", + destination=Join( + on="/", + values=output_common_path + ["validation"], + ), + ), + ProcessingOutput( + output_name="test", + source="/opt/ml/processing/test", + destination=Join( + on="/", + values=output_common_path + ["test"], + ), + ), + ProcessingOutput( + output_name="baseline", + source="/opt/ml/processing/baseline", + destination=Join( + on="/", + values=output_common_path + ["baseline"], + ), ), - ProcessingOutput(output_name="test", source="/opt/ml/processing/test"), - ProcessingOutput(output_name="baseline", source="/opt/ml/processing/baseline"), ] - sklearn_processor = SKLearnProcessor( + sklearn_processor = FrameworkProcessor( + estimator_cls=SKLearn, framework_version="0.23-1", role=role, instance_type=processing_instance_type, instance_count=processing_instance_count, sagemaker_session=sagemaker_session, - base_job_name=f"{base_job_prefix}/sklearn-preprocess", ) step_process = ProcessingStep( @@ -159,7 +201,9 @@ def get_pipeline( step_args=sklearn_processor.run( inputs=inputs, outputs=outputs, - code=os.path.join(BASE_DIR, "preprocess.py"), + code="preprocess.py", + source_dir=os.path.join(BASE_DIR, "preprocess"), + job_name=f"{commit_id}/scripts/preprocess", ), cache_config=cache_config, ) @@ -181,13 +225,7 @@ def get_pipeline( dataset_format=DatasetFormat.csv(), output_s3_uri=Join( on="/", - values=[ - "s3:/", - default_bucket, - base_job_prefix, - ExecutionVariables.PIPELINE_EXECUTION_ID, - "dataqualitycheckstep", - ], + values=output_common_path + ["dataqualitycheck"], ), ) @@ -210,15 +248,15 @@ def get_pipeline( framework="xgboost", region=region, version="1.2-2", - py_version="py3", - instance_type=training_instance_type, ) xgb_train = Estimator( image_uri=image_uri, instance_type=training_instance_type, instance_count=1, - output_path=model_output, - base_job_name=f"{base_job_prefix}/train", + output_path=Join( + on="/", + values=output_common_path + ["model"], + ), sagemaker_session=sagemaker_session, role=role, disable_profiler=False, # Profile processing job @@ -241,6 +279,7 @@ def get_pipeline( step_train = TrainingStep( name="TrainModel", step_args=xgb_train.fit( + job_name=f"{commit_id}/scripts/train", inputs={ "train": TrainingInput( s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ @@ -254,7 +293,7 @@ def get_pipeline( ].S3Output.S3Uri, content_type="text/csv", ), - } + }, ), cache_config=cache_config, ) @@ -265,7 +304,6 @@ def get_pipeline( command=["python3"], instance_type=processing_instance_type, instance_count=1, - base_job_name=f"{base_job_prefix}/script-eval", sagemaker_session=sagemaker_session, role=role, ) @@ -277,6 +315,7 @@ def get_pipeline( step_eval = ProcessingStep( name="EvaluateModel", step_args=script_eval.run( + job_name=f"{commit_id}/scripts/evaluation", inputs=[ ProcessingInput( source=step_train.properties.ModelArtifacts.S3ModelArtifacts, @@ -291,7 +330,12 @@ def get_pipeline( ], outputs=[ ProcessingOutput( - output_name="evaluation", source="/opt/ml/processing/evaluation" + output_name="evaluation", + source="/opt/ml/processing/evaluation", + destination=Join( + on="/", + values=output_common_path + ["evaluation"], + ), ), ], code=os.path.join(BASE_DIR, "evaluate.py"), @@ -303,10 +347,14 @@ def get_pipeline( # register model step that will be conditionally executed model_metrics = ModelMetrics( model_statistics=MetricsSource( - s3_uri="{}/evaluation.json".format( - step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]["S3Output"][ - "S3Uri" - ] + s3_uri=Join( + on="/", + values=[ + step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0][ + "S3Output" + ]["S3Uri"], + "evaluation.json", + ], ), content_type="application/json", ), @@ -343,8 +391,6 @@ def get_pipeline( step_args=model.register( content_types=["text/csv"], response_types=["text/csv"], - inference_instances=["ml.t2.medium", "ml.t2.large", "ml.m5.large"], - transform_instances=["ml.m5.large"], model_package_group_name=model_package_group_name, approval_status=model_approval_status, model_metrics=model_metrics, @@ -355,7 +401,7 @@ def get_pipeline( # condition step for evaluating model quality and branching execution cond_lte = ConditionLessThanOrEqualTo( left=JsonGet( - step=step_eval, + step_name=step_eval.name, property_file=evaluation_report, json_path="regression_metrics.rmse.value", ), @@ -380,8 +426,6 @@ def get_pipeline( baseline_instance_type, training_instance_type, model_approval_status, - model_output, - baseline_output, ], steps=[step_process, step_baseline, step_train, step_eval, step_cond], sagemaker_session=sagemaker_session, diff --git a/build_pipeline/pipelines/preprocess.py b/build_pipeline/pipelines/preprocess/preprocess.py similarity index 97% rename from build_pipeline/pipelines/preprocess.py rename to build_pipeline/pipelines/preprocess/preprocess.py index 0c68450..8fbfb43 100644 --- a/build_pipeline/pipelines/preprocess.py +++ b/build_pipeline/pipelines/preprocess/preprocess.py @@ -6,11 +6,8 @@ import sys from zipfile import ZipFile -# Install geopandas dependency before including pandas -subprocess.check_call([sys.executable, "-m", "pip", "install", "geopandas==0.9.0"]) - -import pandas as pd # noqa: E402 import geopandas as gpd # noqa: E402 +import pandas as pd # noqa: E402 from sklearn.model_selection import train_test_split # noqa: E402 logger = logging.getLogger() diff --git a/build_pipeline/pipelines/preprocess/requirements.txt b/build_pipeline/pipelines/preprocess/requirements.txt new file mode 100644 index 0000000..c36ae26 --- /dev/null +++ b/build_pipeline/pipelines/preprocess/requirements.txt @@ -0,0 +1 @@ +geopandas==0.9.0 \ No newline at end of file diff --git a/build_pipeline/poetry.lock b/build_pipeline/poetry.lock index 1cc83c0..ce858dd 100644 --- a/build_pipeline/poetry.lock +++ b/build_pipeline/poetry.lock @@ -1,38 +1,117 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + [[package]] name = "attrs" -version = "22.1.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" +files = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] + +[[package]] +name = "aws-cdk-asset-awscli-v1" +version = "2.2.52" +description = "A library that contains the AWS CLI for use in Lambda Layers" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-awscli-v1-2.2.52.tar.gz", hash = "sha256:ab04beec8e267e363931df2caf48a24100cb5799d7fd8db51efe881d117efa7a"}, + {file = "aws_cdk.asset_awscli_v1-2.2.52-py3-none-any.whl", hash = "sha256:6e9d686bb0b00242e869e91d57b65b619ffb42e99abe482436e3a6692485dbfe"}, +] + +[package.dependencies] +jsii = ">=1.73.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + +[[package]] +name = "aws-cdk-asset-kubectl-v20" +version = "2.1.1" +description = "A library that contains kubectl for use in Lambda Layers" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-kubectl-v20-2.1.1.tar.gz", hash = "sha256:9834cdb150c5590aea4e5eba6de2a89b4c60617451181c524810c5a75154565c"}, + {file = "aws_cdk.asset_kubectl_v20-2.1.1-py3-none-any.whl", hash = "sha256:a2fad1a5a35a94a465efe60859f91e45dacc33261fb9bbf1cf9bbc6e2f70e9d6"}, +] + +[package.dependencies] +jsii = ">=1.70.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + +[[package]] +name = "aws-cdk-asset-node-proxy-agent-v5" +version = "2.0.42" +description = "@aws-cdk/asset-node-proxy-agent-v5" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.42.tar.gz", hash = "sha256:ae1b615be42e78681e05b145460603f171c06b671a2d1caa060a159b94b06366"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.42-py3-none-any.whl", hash = "sha256:6e0174802097d558daa1be5c4e6e7f309eeba626392955e596bf967ee37865d3"}, +] + +[package.dependencies] +jsii = ">=1.73.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.47.0" +version = "2.62.2" description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "aws-cdk-lib-2.62.2.tar.gz", hash = "sha256:f85000438d849a0522ffcd8e7cb5a70be5fa34339082d4d569734169c6d37b4d"}, + {file = "aws_cdk_lib-2.62.2-py3-none-any.whl", hash = "sha256:03dfb8303b00333177b18e3b60c95d738adf4d90086a5b1e707e896fdc234d52"}, +] [package.dependencies] +"aws-cdk.asset-awscli-v1" = ">=2.2.49,<3.0.0" +"aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.38,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.69.0,<2.0.0" +jsii = ">=1.73.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "black" -version = "22.10.0" +version = "22.12.0" description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, + {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, + {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, + {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, + {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, + {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, + {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, + {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, + {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, + {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, + {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, + {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, +] [package.dependencies] click = ">=8.0.0" @@ -50,14 +129,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.24.96" +version = "1.26.59" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "boto3-1.26.59-py3-none-any.whl", hash = "sha256:34ee771a5cc84c16e75d4b9ef4672f51c2bafdce66ec457bbaac630b37d9cd5e"}, + {file = "boto3-1.26.59.tar.gz", hash = "sha256:7d9cebb507fc96e6eb429621ccb2e731b75e7bbb8d6d9f0cf0c08089ee3c1ab7"}, +] [package.dependencies] -botocore = ">=1.27.96,<1.28.0" +botocore = ">=1.29.59,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -66,11 +149,15 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.27.96" +version = "1.29.59" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "botocore-1.29.59-py3-none-any.whl", hash = "sha256:5533644ddefaccfaa98460a63eb73e61a46aad019771226d103b1054b0df6103"}, + {file = "botocore-1.29.59.tar.gz", hash = "sha256:bc75d41c5eecf624a2f9875483135aa78088a50c8d29847793f92756697cfed5"}, +] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -78,7 +165,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.14.0)"] +crt = ["awscrt (==0.15.3)"] [[package]] name = "cattrs" @@ -87,6 +174,10 @@ description = "Composable complex class support for attrs and dataclasses." category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "cattrs-22.2.0-py3-none-any.whl", hash = "sha256:bc12b1f0d000b9f9bee83335887d532a1d3e99a833d1bf0882151c97d3e68c21"}, + {file = "cattrs-22.2.0.tar.gz", hash = "sha256:f0eed5642399423cf656e7b66ce92cdc5b963ecafd041d1b24d136fdde7acf6d"}, +] [package.dependencies] attrs = ">=20" @@ -99,28 +190,40 @@ description = "Composable command line interface toolkit" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "constructs" -version = "10.1.137" +version = "10.1.234" description = "A programming model for software-defined state" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "constructs-10.1.234-py3-none-any.whl", hash = "sha256:5da8eaacb7c1c34251a9306a4b2bfae7d9be54f7803daf8684d9944cf6259e8f"}, + {file = "constructs-10.1.234.tar.gz", hash = "sha256:bb21ae76abb1c118c458d53a358ca82252b1fefd83dff9d3bd6f9f4cac161a84"}, +] [package.dependencies] -jsii = ">=1.70.0,<2.0.0" +jsii = ">=1.74.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -131,25 +234,37 @@ description = "Backports and enhancements for the contextlib module" category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, + {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, +] [[package]] name = "dill" -version = "0.3.5.1" +version = "0.3.6" description = "serialize all of python" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +python-versions = ">=3.7" +files = [ + {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, + {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, +] [package.extras] graph = ["objgraph (>=1.7.2)"] [[package]] name = "exceptiongroup" -version = "1.0.0rc9" +version = "1.1.0" description = "Backport of PEP 654 (exception groups)" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, +] [package.extras] test = ["pytest (>=6)"] @@ -161,6 +276,11 @@ description = "pasta is an AST-based Python refactoring library" category = "main" optional = false python-versions = "*" +files = [ + {file = "google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e"}, + {file = "google_pasta-0.2.0-py2-none-any.whl", hash = "sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954"}, + {file = "google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed"}, +] [package.dependencies] six = "*" @@ -172,6 +292,10 @@ description = "Read metadata from Python packages" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-4.13.0-py3-none-any.whl", hash = "sha256:8a8a81bcf996e74fee46f0d16bd3eaa382a7eb20fd82445c3ad11f4090334116"}, + {file = "importlib_metadata-4.13.0.tar.gz", hash = "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d"}, +] [package.dependencies] zipp = ">=0.5" @@ -188,14 +312,22 @@ description = "JSON Matching Expressions" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] [[package]] name = "jsii" -version = "1.70.0" +version = "1.74.0" description = "Python client for jsii runtime" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "jsii-1.74.0-py3-none-any.whl", hash = "sha256:ee76781fe66106c367fbb3bb383db4f5e9b8ff3d3c4c0f34624c050211f040be"}, + {file = "jsii-1.74.0.tar.gz", hash = "sha256:575131396ad34f8f6e9f2604953ecbf4f3368625656a828b13089e4abb81b443"}, +] [package.dependencies] attrs = ">=21.2,<23.0" @@ -207,14 +339,30 @@ typing-extensions = ">=3.7,<5.0" [[package]] name = "multiprocess" -version = "0.70.13" +version = "0.70.14" description = "better multiprocessing and multithreading in python" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +python-versions = ">=3.7" +files = [ + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:560a27540daef4ce8b24ed3cc2496a3c670df66c96d02461a4da67473685adf3"}, + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-manylinux_2_24_i686.whl", hash = "sha256:bfbbfa36f400b81d1978c940616bc77776424e5e34cb0c94974b178d727cfcd5"}, + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:89fed99553a04ec4f9067031f83a886d7fdec5952005551a896a4b6a59575bb9"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:40a5e3685462079e5fdee7c6789e3ef270595e1755199f0d50685e72523e1d2a"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-manylinux_2_24_i686.whl", hash = "sha256:44936b2978d3f2648727b3eaeab6d7fa0bedf072dc5207bf35a96d5ee7c004cf"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:e628503187b5d494bf29ffc52d3e1e57bb770ce7ce05d67c4bbdb3a0c7d3b05f"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0d5da0fc84aacb0e4bd69c41b31edbf71b39fe2fb32a54eaedcaea241050855c"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-manylinux_2_24_i686.whl", hash = "sha256:6a7b03a5b98e911a7785b9116805bd782815c5e2bd6c91c6a320f26fd3e7b7ad"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:cea5bdedd10aace3c660fedeac8b087136b4366d4ee49a30f1ebf7409bce00ae"}, + {file = "multiprocess-0.70.14-py310-none-any.whl", hash = "sha256:7dc1f2f6a1d34894c8a9a013fbc807971e336e7cc3f3ff233e61b9dc679b3b5c"}, + {file = "multiprocess-0.70.14-py37-none-any.whl", hash = "sha256:93a8208ca0926d05cdbb5b9250a604c401bed677579e96c14da3090beb798193"}, + {file = "multiprocess-0.70.14-py38-none-any.whl", hash = "sha256:6725bc79666bbd29a73ca148a0fb5f4ea22eed4a8f22fce58296492a02d18a7b"}, + {file = "multiprocess-0.70.14-py39-none-any.whl", hash = "sha256:63cee628b74a2c0631ef15da5534c8aedbc10c38910b9c8b18dcd327528d1ec7"}, + {file = "multiprocess-0.70.14.tar.gz", hash = "sha256:3eddafc12f2260d27ae03fe6069b12570ab4764ab59a75e81624fac453fbf46a"}, +] [package.dependencies] -dill = ">=0.3.5.1" +dill = ">=0.3.6" [[package]] name = "mypy-extensions" @@ -223,38 +371,103 @@ description = "Experimental type system extensions for programs checked with the category = "dev" optional = false python-versions = "*" +files = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] [[package]] name = "numpy" -version = "1.23.4" -description = "NumPy is the fundamental package for array computing with Python." +version = "1.24.1" +description = "Fundamental package for array computing in Python" category = "main" optional = false python-versions = ">=3.8" +files = [ + {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, + {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, + {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, + {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, + {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, + {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, + {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, + {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, + {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, + {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, + {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, + {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, + {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, + {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, + {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, + {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, + {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, + {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, + {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, + {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, + {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, + {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, + {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, + {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, + {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, +] [[package]] name = "packaging" -version = "21.3" +version = "23.0" description = "Core utilities for Python packages" category = "main" optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +python-versions = ">=3.7" +files = [ + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, +] [[package]] name = "pandas" -version = "1.5.1" +version = "1.5.3" description = "Powerful data structures for data analysis, time series, and statistics" category = "main" optional = false python-versions = ">=3.8" +files = [ + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, + {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, + {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, + {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, + {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, + {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, + {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, + {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, +] [package.dependencies] numpy = [ - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.1" pytz = ">=2020.1" @@ -264,59 +477,76 @@ test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] [[package]] name = "pathos" -version = "0.2.9" +version = "0.3.0" description = "parallel graph management and execution in heterogeneous computing" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +python-versions = ">=3.7" +files = [ + {file = "pathos-0.3.0-py3-none-any.whl", hash = "sha256:b1f5a79b1c79a594330d451832642ee5bb61dd77dc75ba9e5c72087c77e8994c"}, + {file = "pathos-0.3.0.tar.gz", hash = "sha256:24fa8db51fbd9284da8e191794097c4bb2aa3fce411090e57af6385e61b97e09"}, +] [package.dependencies] -dill = ">=0.3.5.1" -multiprocess = ">=0.70.13" -pox = ">=0.3.1" -ppft = ">=1.7.6.5" +dill = ">=0.3.6" +multiprocess = ">=0.70.14" +pox = ">=0.3.2" +ppft = ">=1.7.6.6" [[package]] name = "pathspec" -version = "0.10.1" +version = "0.11.0" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.0-py3-none-any.whl", hash = "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229"}, + {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, +] [[package]] name = "platformdirs" -version = "2.5.2" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "2.6.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, + {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, +] [package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pox" -version = "0.3.1" +version = "0.3.2" description = "utilities for filesystem exploration and automated builds" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +python-versions = ">=3.7" +files = [ + {file = "pox-0.3.2-py3-none-any.whl", hash = "sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61"}, + {file = "pox-0.3.2.tar.gz", hash = "sha256:e825225297638d6e3d49415f8cfb65407a5d15e56f2fb7fe9d9b9e3050c65ee1"}, +] [[package]] name = "ppft" -version = "1.7.6.5" +version = "1.7.6.6" description = "distributed and parallel python" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" - -[package.dependencies] -six = ">=1.7.3" +python-versions = ">=3.7" +files = [ + {file = "ppft-1.7.6.6-py3-none-any.whl", hash = "sha256:f355d2caeed8bd7c9e4a860c471f31f7e66d1ada2791ab5458ea7dca15a51e41"}, + {file = "ppft-1.7.6.6.tar.gz", hash = "sha256:f933f0404f3e808bc860745acb3b79cd4fe31ea19a20889a645f900415be60f1"}, +] [package.extras] -dill = ["dill (>=0.3.5)"] +dill = ["dill (>=0.3.6)"] [[package]] name = "protobuf" @@ -325,6 +555,30 @@ description = "Protocol Buffers" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "protobuf-3.20.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99"}, + {file = "protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e"}, + {file = "protobuf-3.20.3-cp310-cp310-win32.whl", hash = "sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c"}, + {file = "protobuf-3.20.3-cp310-cp310-win_amd64.whl", hash = "sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7"}, + {file = "protobuf-3.20.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469"}, + {file = "protobuf-3.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4"}, + {file = "protobuf-3.20.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4"}, + {file = "protobuf-3.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454"}, + {file = "protobuf-3.20.3-cp37-cp37m-win32.whl", hash = "sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905"}, + {file = "protobuf-3.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c"}, + {file = "protobuf-3.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7"}, + {file = "protobuf-3.20.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee"}, + {file = "protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050"}, + {file = "protobuf-3.20.3-cp38-cp38-win32.whl", hash = "sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86"}, + {file = "protobuf-3.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9"}, + {file = "protobuf-3.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b"}, + {file = "protobuf-3.20.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b"}, + {file = "protobuf-3.20.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402"}, + {file = "protobuf-3.20.3-cp39-cp39-win32.whl", hash = "sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480"}, + {file = "protobuf-3.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7"}, + {file = "protobuf-3.20.3-py2.py3-none-any.whl", hash = "sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db"}, + {file = "protobuf-3.20.3.tar.gz", hash = "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2"}, +] [[package]] name = "protobuf3-to-dict" @@ -333,6 +587,9 @@ description = "Ben Hodgson: A teeny Python library for creating Python dicts fro category = "main" optional = false python-versions = "*" +files = [ + {file = "protobuf3-to-dict-0.1.5.tar.gz", hash = "sha256:1e42c25b5afb5868e3a9b1962811077e492c17557f9c66f0fe40d821375d2b5a"}, +] [package.dependencies] protobuf = ">=2.3.0" @@ -345,17 +602,10 @@ description = "Publication helps you maintain public-api-friendly modules by pre category = "main" optional = false python-versions = "*" - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] +files = [ + {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, + {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, +] [[package]] name = "python-dateutil" @@ -364,17 +614,25 @@ description = "Extensions to the standard Python datetime module" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] [package.dependencies] six = ">=1.5" [[package]] name = "pytz" -version = "2022.5" +version = "2022.7.1" description = "World timezone definitions, modern and historical" category = "main" optional = false python-versions = "*" +files = [ + {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, + {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, +] [[package]] name = "s3transfer" @@ -383,6 +641,10 @@ description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] [package.dependencies] botocore = ">=1.12.36,<2.0a.0" @@ -392,15 +654,18 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "sagemaker" -version = "2.113.0" +version = "2.130.0" description = "Open source library for training and deploying models on Amazon SageMaker." category = "main" optional = false python-versions = ">= 3.6" +files = [ + {file = "sagemaker-2.130.0.tar.gz", hash = "sha256:ac6f80a413bd2621a5ec8ba8146cdc3b279dc1fbc513d5300caccaf177b28700"}, +] [package.dependencies] attrs = ">=20.3.0,<23" -boto3 = ">=1.20.21,<2.0" +boto3 = ">=1.26.28,<2.0" google-pasta = "*" importlib-metadata = ">=1.4.0,<5.0" numpy = ">=1.9.0,<2.0" @@ -413,10 +678,10 @@ schema = "*" smdebug_rulesconfig = "1.0.1" [package.extras] -all = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "scipy (==1.7.2)", "urllib3 (==1.26.8)"] +all = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "scipy (==1.7.3)", "urllib3 (==1.26.8)"] local = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "urllib3 (==1.26.8)"] -scipy = ["scipy (==1.7.2)"] -test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.4.1)", "apache-airflow-providers-amazon (==4.0.0)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scipy (==1.7.2)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] +scipy = ["scipy (==1.7.3)"] +test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.4.1)", "apache-airflow-providers-amazon (==4.0.0)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scikit-learn (==1.0.2)", "scipy (==1.7.3)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] [[package]] name = "schema" @@ -425,6 +690,10 @@ description = "Simple data validation library" category = "main" optional = false python-versions = "*" +files = [ + {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, + {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, +] [package.dependencies] contextlib2 = ">=0.5.5" @@ -436,6 +705,10 @@ description = "Python 2 and 3 compatibility utilities" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] [[package]] name = "smdebug-rulesconfig" @@ -444,6 +717,10 @@ description = "SMDebug RulesConfig" category = "main" optional = false python-versions = ">=2.7" +files = [ + {file = "smdebug_rulesconfig-1.0.1-py2.py3-none-any.whl", hash = "sha256:104da3e6931ecf879dfc687ca4bbb3bee5ea2bc27f4478e9dbb3ee3655f1ae61"}, + {file = "smdebug_rulesconfig-1.0.1.tar.gz", hash = "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"}, +] [[package]] name = "tomli" @@ -452,6 +729,10 @@ description = "A lil' TOML parser" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] [[package]] name = "typeguard" @@ -460,6 +741,10 @@ description = "Run-time type checker for Python" category = "main" optional = false python-versions = ">=3.5.3" +files = [ + {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, + {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, +] [package.extras] doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] @@ -472,14 +757,22 @@ description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] [[package]] name = "urllib3" -version = "1.26.12" +version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, +] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] @@ -488,296 +781,21 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "zipp" -version = "3.9.0" +version = "3.12.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "zipp-3.12.0-py3-none-any.whl", hash = "sha256:9eb0a4c5feab9b08871db0d672745b53450d7f26992fd1e4653aa43345e97b86"}, + {file = "zipp-3.12.0.tar.gz", hash = "sha256:73efd63936398aac78fd92b6f4865190119d6c91b531532e798977ea8dd402eb"}, +] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] -lock-version = "1.1" +lock-version = "2.0" python-versions = "^3.9" content-hash = "91018601b6987ea6e35ff797522671a852c9a913ac7982ea245bccf157f8ae0e" - -[metadata.files] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] -aws-cdk-lib = [ - {file = "aws-cdk-lib-2.47.0.tar.gz", hash = "sha256:c9b1626c7afc42ceaee547558efe88439dd1447b18c7837db153e03fb8186167"}, - {file = "aws_cdk_lib-2.47.0-py3-none-any.whl", hash = "sha256:1f1a2e2309625d304a14b388ee0fedb0dffc15eef14dac65e33b10f6c73bf75c"}, -] -black = [ - {file = "black-22.10.0-1fixedarch-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa"}, - {file = "black-22.10.0-1fixedarch-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef"}, - {file = "black-22.10.0-1fixedarch-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6"}, - {file = "black-22.10.0-1fixedarch-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d"}, - {file = "black-22.10.0-1fixedarch-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4"}, - {file = "black-22.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb"}, - {file = "black-22.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7"}, - {file = "black-22.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66"}, - {file = "black-22.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae"}, - {file = "black-22.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b"}, - {file = "black-22.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d"}, - {file = "black-22.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650"}, - {file = "black-22.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d"}, - {file = "black-22.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"}, - {file = "black-22.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87"}, - {file = "black-22.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395"}, - {file = "black-22.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0"}, - {file = "black-22.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383"}, - {file = "black-22.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de"}, - {file = "black-22.10.0-py3-none-any.whl", hash = "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458"}, - {file = "black-22.10.0.tar.gz", hash = "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1"}, -] -boto3 = [ - {file = "boto3-1.24.96-py3-none-any.whl", hash = "sha256:748c055214c629744c34c7f94bfa888733dfac0b92e1daef9c243e1391ea4f53"}, - {file = "boto3-1.24.96.tar.gz", hash = "sha256:6b8899542cff82becceb3498a2240bf77c96def0515b0a31f7f6a9d5b92e7a3d"}, -] -botocore = [ - {file = "botocore-1.27.96-py3-none-any.whl", hash = "sha256:e41a81a18511f2f9181b2a9ab302a55c0effecccbef846c55aad0c47bfdbefb9"}, - {file = "botocore-1.27.96.tar.gz", hash = "sha256:fc0a13ef6042e890e361cf408759230f8574409bb51f81740d2e5d8ad5d1fbea"}, -] -cattrs = [ - {file = "cattrs-22.2.0-py3-none-any.whl", hash = "sha256:bc12b1f0d000b9f9bee83335887d532a1d3e99a833d1bf0882151c97d3e68c21"}, - {file = "cattrs-22.2.0.tar.gz", hash = "sha256:f0eed5642399423cf656e7b66ce92cdc5b963ecafd041d1b24d136fdde7acf6d"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, -] -constructs = [ - {file = "constructs-10.1.137-py3-none-any.whl", hash = "sha256:1072c36c34a107809385129689b5f61ca38209b399bcc6082bc05a827d3c02f7"}, - {file = "constructs-10.1.137.tar.gz", hash = "sha256:7c1d2e17b5a594cec36e5107ceb40289776aa3f46e8742d3b2aac9ad3119574b"}, -] -contextlib2 = [ - {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, - {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, -] -dill = [ - {file = "dill-0.3.5.1-py2.py3-none-any.whl", hash = "sha256:33501d03270bbe410c72639b350e941882a8b0fd55357580fbc873fba0c59302"}, - {file = "dill-0.3.5.1.tar.gz", hash = "sha256:d75e41f3eff1eee599d738e76ba8f4ad98ea229db8b085318aa2b3333a208c86"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.0.0rc9-py3-none-any.whl", hash = "sha256:2e3c3fc1538a094aab74fad52d6c33fc94de3dfee3ee01f187c0e0c72aec5337"}, - {file = "exceptiongroup-1.0.0rc9.tar.gz", hash = "sha256:9086a4a21ef9b31c72181c77c040a074ba0889ee56a7b289ff0afb0d97655f96"}, -] -google-pasta = [ - {file = "google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e"}, - {file = "google_pasta-0.2.0-py2-none-any.whl", hash = "sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954"}, - {file = "google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed"}, -] -importlib-metadata = [ - {file = "importlib_metadata-4.13.0-py3-none-any.whl", hash = "sha256:8a8a81bcf996e74fee46f0d16bd3eaa382a7eb20fd82445c3ad11f4090334116"}, - {file = "importlib_metadata-4.13.0.tar.gz", hash = "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d"}, -] -jmespath = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] -jsii = [ - {file = "jsii-1.70.0-py3-none-any.whl", hash = "sha256:d0867c0d2f60ceda1664c026033ae34ea36178c7027315e577ded13043827664"}, - {file = "jsii-1.70.0.tar.gz", hash = "sha256:9fc57ff37868364ba3417b26dc97189cd0cc71282196a3f4765768c067354be0"}, -] -multiprocess = [ - {file = "multiprocess-0.70.13-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:b9a3be43ecee6776a9e7223af96914a0164f306affcf4624b213885172236b77"}, - {file = "multiprocess-0.70.13-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:7e6a689da3490412caa7b3e27c3385d8aaa49135f3a353ace94ca47e4c926d37"}, - {file = "multiprocess-0.70.13-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:17cb4229aa43e6973679d67c66a454cbf8b6b0d038425cba3220ea5a06d61b58"}, - {file = "multiprocess-0.70.13-cp27-cp27m-win32.whl", hash = "sha256:99bb68dd0d5b3d30fe104721bee26e4637667112d5951b51feb81479fd560876"}, - {file = "multiprocess-0.70.13-cp27-cp27m-win_amd64.whl", hash = "sha256:6cdde49defcb933062df382ebc9b5299beebcd157a98b3a65291c1c94a2edc41"}, - {file = "multiprocess-0.70.13-pp27-pypy_73-macosx_10_7_x86_64.whl", hash = "sha256:92003c247436f8699b7692e95346a238446710f078500eb364bc23bb0503dd4f"}, - {file = "multiprocess-0.70.13-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:3ec1c8015e19182bfa01b5887a9c25805c48df3c71863f48fe83803147cde5d6"}, - {file = "multiprocess-0.70.13-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b7415f61bddfffdade73396904551be8124a4a363322aa9c72d42e349c5fca39"}, - {file = "multiprocess-0.70.13-pp37-pypy37_pp73-manylinux_2_24_i686.whl", hash = "sha256:5436d1cd9f901f7ddc4f20b6fd0b462c87dcc00d941cc13eeb2401fc5bd00e42"}, - {file = "multiprocess-0.70.13-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:34e9703bd5b9fee5455c93a74e44dbabe55481c214d03be1e65f037be9d0c520"}, - {file = "multiprocess-0.70.13-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:af0a48440aa8f793d8bb100f20102c12f192de5a608638819a998f2cc59e1fcd"}, - {file = "multiprocess-0.70.13-pp38-pypy38_pp73-manylinux_2_24_i686.whl", hash = "sha256:c4a97216e8319039c69a266252cc68a392b96f9e67e3ed02ad88be9e6f2d2969"}, - {file = "multiprocess-0.70.13-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:48315eefe02c35dd7560da3fa8af66d9f4a61b9dc8f7c40801c5f972ab4604b1"}, - {file = "multiprocess-0.70.13-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5a6dca5f29f0224c855d0d5cad963476175cfc8de112d3eebe85914cb735f130"}, - {file = "multiprocess-0.70.13-pp39-pypy39_pp73-manylinux_2_24_i686.whl", hash = "sha256:5974bdad390ba466cc130288d2ef1048fdafedd01cf4641fc024f6088af70bfe"}, - {file = "multiprocess-0.70.13-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:01c1137d2f18d0cd262d0fdb7294b1fe9fc3e8dc8b126e506085434ae8eb3677"}, - {file = "multiprocess-0.70.13-py310-none-any.whl", hash = "sha256:0f4faf4811019efdb2f91db09240f893ee40cbfcb06978f3b8ed8c248e73babe"}, - {file = "multiprocess-0.70.13-py37-none-any.whl", hash = "sha256:62e556a0c31ec7176e28aa331663ac26c276ee3536b5e9bb5e850681e7a00f11"}, - {file = "multiprocess-0.70.13-py38-none-any.whl", hash = "sha256:7be9e320a41d2d0d0eddacfe693cfb07b4cb9c0d3d10007f4304255c15215778"}, - {file = "multiprocess-0.70.13-py39-none-any.whl", hash = "sha256:00ef48461d43d1e30f8f4b2e1b287ecaaffec325a37053beb5503e0d69e5a3cd"}, - {file = "multiprocess-0.70.13.tar.gz", hash = "sha256:2e096dd618a84d15aa369a9cf6695815e5539f853dc8fa4f4b9153b11b1d0b32"}, -] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] -numpy = [ - {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, - {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c237129f0e732885c9a6076a537e974160482eab8f10db6292e92154d4c67d71"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8365b942f9c1a7d0f0dc974747d99dd0a0cdfc5949a33119caf05cb314682d3"}, - {file = "numpy-1.23.4-cp310-cp310-win32.whl", hash = "sha256:2341f4ab6dba0834b685cce16dad5f9b6606ea8a00e6da154f5dbded70fdc4dd"}, - {file = "numpy-1.23.4-cp310-cp310-win_amd64.whl", hash = "sha256:d331afac87c92373826af83d2b2b435f57b17a5c74e6268b79355b970626e329"}, - {file = "numpy-1.23.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:488a66cb667359534bc70028d653ba1cf307bae88eab5929cd707c761ff037db"}, - {file = "numpy-1.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce03305dd694c4873b9429274fd41fc7eb4e0e4dea07e0af97a933b079a5814f"}, - {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8981d9b5619569899666170c7c9748920f4a5005bf79c72c07d08c8a035757b0"}, - {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a70a7d3ce4c0e9284e92285cba91a4a3f5214d87ee0e95928f3614a256a1488"}, - {file = "numpy-1.23.4-cp311-cp311-win32.whl", hash = "sha256:5e13030f8793e9ee42f9c7d5777465a560eb78fa7e11b1c053427f2ccab90c79"}, - {file = "numpy-1.23.4-cp311-cp311-win_amd64.whl", hash = "sha256:7607b598217745cc40f751da38ffd03512d33ec06f3523fb0b5f82e09f6f676d"}, - {file = "numpy-1.23.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ab46e4e7ec63c8a5e6dbf5c1b9e1c92ba23a7ebecc86c336cb7bf3bd2fb10e5"}, - {file = "numpy-1.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8aae2fb3180940011b4862b2dd3756616841c53db9734b27bb93813cd79fce6"}, - {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c053d7557a8f022ec823196d242464b6955a7e7e5015b719e76003f63f82d0f"}, - {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0882323e0ca4245eb0a3d0a74f88ce581cc33aedcfa396e415e5bba7bf05f68"}, - {file = "numpy-1.23.4-cp38-cp38-win32.whl", hash = "sha256:dada341ebb79619fe00a291185bba370c9803b1e1d7051610e01ed809ef3a4ba"}, - {file = "numpy-1.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:0fe563fc8ed9dc4474cbf70742673fc4391d70f4363f917599a7fa99f042d5a8"}, - {file = "numpy-1.23.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c67b833dbccefe97cdd3f52798d430b9d3430396af7cdb2a0c32954c3ef73894"}, - {file = "numpy-1.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f76025acc8e2114bb664294a07ede0727aa75d63a06d2fae96bf29a81747e4a7"}, - {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ac457b63ec8ded85d85c1e17d85efd3c2b0967ca39560b307a35a6703a4735"}, - {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95de7dc7dc47a312f6feddd3da2500826defdccbc41608d0031276a24181a2c0"}, - {file = "numpy-1.23.4-cp39-cp39-win32.whl", hash = "sha256:f2f390aa4da44454db40a1f0201401f9036e8d578a25f01a6e237cea238337ef"}, - {file = "numpy-1.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:f260da502d7441a45695199b4e7fd8ca87db659ba1c78f2bbf31f934fe76ae0e"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61be02e3bf810b60ab74e81d6d0d36246dbfb644a462458bb53b595791251911"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296d17aed51161dbad3c67ed6d164e51fcd18dbcd5dd4f9d0a9c6055dce30810"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4d52914c88b4930dafb6c48ba5115a96cbab40f45740239d9f4159c4ba779962"}, - {file = "numpy-1.23.4.tar.gz", hash = "sha256:ed2cc92af0efad20198638c69bb0fc2870a58dabfba6eb722c933b48556c686c"}, -] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] -pandas = [ - {file = "pandas-1.5.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0a78e05ec09731c5b3bd7a9805927ea631fe6f6cb06f0e7c63191a9a778d52b4"}, - {file = "pandas-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5b0c970e2215572197b42f1cff58a908d734503ea54b326412c70d4692256391"}, - {file = "pandas-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f340331a3f411910adfb4bbe46c2ed5872d9e473a783d7f14ecf49bc0869c594"}, - {file = "pandas-1.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8c709f4700573deb2036d240d140934df7e852520f4a584b2a8d5443b71f54d"}, - {file = "pandas-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32e3d9f65606b3f6e76555bfd1d0b68d94aff0929d82010b791b6254bf5a4b96"}, - {file = "pandas-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a52419d9ba5906db516109660b114faf791136c94c1a636ed6b29cbfff9187ee"}, - {file = "pandas-1.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66a1ad667b56e679e06ba73bb88c7309b3f48a4c279bd3afea29f65a766e9036"}, - {file = "pandas-1.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36aa1f8f680d7584e9b572c3203b20d22d697c31b71189322f16811d4ecfecd3"}, - {file = "pandas-1.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bcf1a82b770b8f8c1e495b19a20d8296f875a796c4fe6e91da5ef107f18c5ecb"}, - {file = "pandas-1.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c25e5c16ee5c0feb6cf9d982b869eec94a22ddfda9aa2fbed00842cbb697624"}, - {file = "pandas-1.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:932d2d7d3cab44cfa275601c982f30c2d874722ef6396bb539e41e4dc4618ed4"}, - {file = "pandas-1.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:eb7e8cf2cf11a2580088009b43de84cabbf6f5dae94ceb489f28dba01a17cb77"}, - {file = "pandas-1.5.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:cb2a9cf1150302d69bb99861c5cddc9c25aceacb0a4ef5299785d0f5389a3209"}, - {file = "pandas-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:81f0674fa50b38b6793cd84fae5d67f58f74c2d974d2cb4e476d26eee33343d0"}, - {file = "pandas-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:17da7035d9e6f9ea9cdc3a513161f8739b8f8489d31dc932bc5a29a27243f93d"}, - {file = "pandas-1.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:669c8605dba6c798c1863157aefde959c1796671ffb342b80fcb80a4c0bc4c26"}, - {file = "pandas-1.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:683779e5728ac9138406c59a11e09cd98c7d2c12f0a5fc2b9c5eecdbb4a00075"}, - {file = "pandas-1.5.1-cp38-cp38-win32.whl", hash = "sha256:ddf46b940ef815af4e542697eaf071f0531449407a7607dd731bf23d156e20a7"}, - {file = "pandas-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:db45b94885000981522fb92349e6b76f5aee0924cc5315881239c7859883117d"}, - {file = "pandas-1.5.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:927e59c694e039c75d7023465d311277a1fc29ed7236b5746e9dddf180393113"}, - {file = "pandas-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e675f8fe9aa6c418dc8d3aac0087b5294c1a4527f1eacf9fe5ea671685285454"}, - {file = "pandas-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:04e51b01d5192499390c0015630975f57836cc95c7411415b499b599b05c0c96"}, - {file = "pandas-1.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cee0c74e93ed4f9d39007e439debcaadc519d7ea5c0afc3d590a3a7b2edf060"}, - {file = "pandas-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b156a971bc451c68c9e1f97567c94fd44155f073e3bceb1b0d195fd98ed12048"}, - {file = "pandas-1.5.1-cp39-cp39-win32.whl", hash = "sha256:05c527c64ee02a47a24031c880ee0ded05af0623163494173204c5b72ddce658"}, - {file = "pandas-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:6bb391659a747cf4f181a227c3e64b6d197100d53da98dcd766cc158bdd9ec68"}, - {file = "pandas-1.5.1.tar.gz", hash = "sha256:249cec5f2a5b22096440bd85c33106b6102e0672204abd2d5c014106459804ee"}, -] -pathos = [ - {file = "pathos-0.2.9-py2-none-any.whl", hash = "sha256:6a6ddb514ce2719f63fb88d5ec4f4490e436b636b54f1102d952c9f7c52f18e2"}, - {file = "pathos-0.2.9-py3-none-any.whl", hash = "sha256:1c44373d8692897d5d15a8aa3b3a442ddc0814c5e848f4ff0ded5491f34b1dac"}, - {file = "pathos-0.2.9.tar.gz", hash = "sha256:a8dbddcd3d9af32ada7c6dc088d845588c513a29a0ba19ab9f64c5cd83692934"}, -] -pathspec = [ - {file = "pathspec-0.10.1-py3-none-any.whl", hash = "sha256:46846318467efc4556ccfd27816e004270a9eeeeb4d062ce5e6fc7a87c573f93"}, - {file = "pathspec-0.10.1.tar.gz", hash = "sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d"}, -] -platformdirs = [ - {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, - {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, -] -pox = [ - {file = "pox-0.3.1-py2.py3-none-any.whl", hash = "sha256:541b5c845aacb806c1364d4142003efb809d654c9ca8db82e650ee86c81e680b"}, - {file = "pox-0.3.1.tar.gz", hash = "sha256:cbb0c0acd650c0ffb620999da611e93aae5105c46a084c4ceaf2f704ed708c1e"}, -] -ppft = [ - {file = "ppft-1.7.6.5-py2.py3-none-any.whl", hash = "sha256:07166097d7dd45af7b98859654390d579d11dadf20780f6baca4bded3f55a580"}, - {file = "ppft-1.7.6.5.tar.gz", hash = "sha256:47e0dab87a516c0b9992cd5b0c908348e4c7d964304d106b227fad28ae03219e"}, -] -protobuf = [ - {file = "protobuf-3.20.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99"}, - {file = "protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e"}, - {file = "protobuf-3.20.3-cp310-cp310-win32.whl", hash = "sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c"}, - {file = "protobuf-3.20.3-cp310-cp310-win_amd64.whl", hash = "sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7"}, - {file = "protobuf-3.20.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469"}, - {file = "protobuf-3.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4"}, - {file = "protobuf-3.20.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4"}, - {file = "protobuf-3.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454"}, - {file = "protobuf-3.20.3-cp37-cp37m-win32.whl", hash = "sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905"}, - {file = "protobuf-3.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c"}, - {file = "protobuf-3.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7"}, - {file = "protobuf-3.20.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee"}, - {file = "protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050"}, - {file = "protobuf-3.20.3-cp38-cp38-win32.whl", hash = "sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86"}, - {file = "protobuf-3.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9"}, - {file = "protobuf-3.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b"}, - {file = "protobuf-3.20.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b"}, - {file = "protobuf-3.20.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402"}, - {file = "protobuf-3.20.3-cp39-cp39-win32.whl", hash = "sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480"}, - {file = "protobuf-3.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7"}, - {file = "protobuf-3.20.3-py2.py3-none-any.whl", hash = "sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db"}, - {file = "protobuf-3.20.3.tar.gz", hash = "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2"}, -] -protobuf3-to-dict = [ - {file = "protobuf3-to-dict-0.1.5.tar.gz", hash = "sha256:1e42c25b5afb5868e3a9b1962811077e492c17557f9c66f0fe40d821375d2b5a"}, -] -publication = [ - {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, - {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, -] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -pytz = [ - {file = "pytz-2022.5-py2.py3-none-any.whl", hash = "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22"}, - {file = "pytz-2022.5.tar.gz", hash = "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914"}, -] -s3transfer = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] -sagemaker = [ - {file = "sagemaker-2.113.0.tar.gz", hash = "sha256:42dac2361fe8e82db802a51a226469ce7d6db8a18d15378f45eb032e80da35aa"}, -] -schema = [ - {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, - {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -smdebug-rulesconfig = [ - {file = "smdebug_rulesconfig-1.0.1-py2.py3-none-any.whl", hash = "sha256:104da3e6931ecf879dfc687ca4bbb3bee5ea2bc27f4478e9dbb3ee3655f1ae61"}, - {file = "smdebug_rulesconfig-1.0.1.tar.gz", hash = "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -typeguard = [ - {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, - {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, -] -typing-extensions = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, -] -urllib3 = [ - {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, - {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, -] -zipp = [ - {file = "zipp-3.9.0-py3-none-any.whl", hash = "sha256:972cfa31bc2fedd3fa838a51e9bc7e64b7fb725a8c00e7431554311f180e9980"}, - {file = "zipp-3.9.0.tar.gz", hash = "sha256:3a7af91c3db40ec72dd9d154ae18e008c69efe8ca88dde4f9a731bb82fe2f9eb"}, -] diff --git a/build_pipeline/requirements.txt b/build_pipeline/requirements.txt index 484eb69..44559a4 100644 --- a/build_pipeline/requirements.txt +++ b/build_pipeline/requirements.txt @@ -1,35 +1,37 @@ -attrs==22.1.0 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-lib==2.47.0 ; python_version >= "3.9" and python_version < "4.0" -boto3==1.24.96 ; python_version >= "3.9" and python_version < "4.0" -botocore==1.27.96 ; python_version >= "3.9" and python_version < "4.0" +attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-awscli-v1==2.2.52 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-kubectl-v20==2.1.1 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-node-proxy-agent-v5==2.0.42 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-lib==2.62.2 ; python_version >= "3.9" and python_version < "4.0" +boto3==1.26.59 ; python_version >= "3.9" and python_version < "4.0" +botocore==1.29.59 ; python_version >= "3.9" and python_version < "4.0" cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -constructs==10.1.137 ; python_version >= "3.9" and python_version < "4.0" +constructs==10.1.234 ; python_version >= "3.9" and python_version < "4.0" contextlib2==21.6.0 ; python_version >= "3.9" and python_version < "4.0" -dill==0.3.5.1 ; python_version >= "3.9" and python_version < "4.0" -exceptiongroup==1.0.0rc9 ; python_version >= "3.9" and python_version < "3.11" +dill==0.3.6 ; python_version >= "3.9" and python_version < "4.0" +exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11" google-pasta==0.2.0 ; python_version >= "3.9" and python_version < "4.0" importlib-metadata==4.13.0 ; python_version >= "3.9" and python_version < "4.0" jmespath==1.0.1 ; python_version >= "3.9" and python_version < "4.0" -jsii==1.70.0 ; python_version >= "3.9" and python_version < "4.0" -multiprocess==0.70.13 ; python_version >= "3.9" and python_version < "4.0" -numpy==1.23.4 ; python_version >= "3.9" and python_version < "4.0" -packaging==21.3 ; python_version >= "3.9" and python_version < "4.0" -pandas==1.5.1 ; python_version >= "3.9" and python_version < "4.0" -pathos==0.2.9 ; python_version >= "3.9" and python_version < "4.0" -pox==0.3.1 ; python_version >= "3.9" and python_version < "4.0" -ppft==1.7.6.5 ; python_version >= "3.9" and python_version < "4.0" +jsii==1.74.0 ; python_version >= "3.9" and python_version < "4.0" +multiprocess==0.70.14 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.24.1 ; python_version < "4.0" and python_version >= "3.9" +packaging==23.0 ; python_version >= "3.9" and python_version < "4.0" +pandas==1.5.3 ; python_version >= "3.9" and python_version < "4.0" +pathos==0.3.0 ; python_version >= "3.9" and python_version < "4.0" +pox==0.3.2 ; python_version >= "3.9" and python_version < "4.0" +ppft==1.7.6.6 ; python_version >= "3.9" and python_version < "4.0" protobuf3-to-dict==0.1.5 ; python_version >= "3.9" and python_version < "4.0" protobuf==3.20.3 ; python_version >= "3.9" and python_version < "4.0" publication==0.0.3 ; python_version >= "3.9" and python_version < "4.0" -pyparsing==3.0.9 ; python_version >= "3.9" and python_version < "4.0" python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" -pytz==2022.5 ; python_version >= "3.9" and python_version < "4.0" +pytz==2022.7.1 ; python_version >= "3.9" and python_version < "4.0" s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "4.0" -sagemaker==2.113.0 ; python_version >= "3.9" and python_version < "4.0" +sagemaker==2.130.0 ; python_version >= "3.9" and python_version < "4.0" schema==0.7.5 ; python_version >= "3.9" and python_version < "4.0" six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" smdebug-rulesconfig==1.0.1 ; python_version >= "3.9" and python_version < "4.0" typeguard==2.13.3 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.4.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==1.26.12 ; python_version >= "3.9" and python_version < "4" -zipp==3.9.0 ; python_version >= "3.9" and python_version < "4.0" +urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4.0" +zipp==3.12.0 ; python_version >= "3.9" and python_version < "4.0" From f9dca77f835022fead3e7214e4119de4dbce3337 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 30 Jan 2023 08:27:57 +0800 Subject: [PATCH 06/17] Updated endpoint variant names in Deploy stack --- deployment_pipeline/app.py | 12 +- deployment_pipeline/cdk.json | 6 +- .../infra/deployment_config.py | 2 +- deployment_pipeline/infra/model_registry.py | 14 +- deployment_pipeline/infra/sagemaker_stack.py | 20 +- deployment_pipeline/poetry.lock | 364 ++++++++---------- deployment_pipeline/prod-config.json | 3 + deployment_pipeline/requirements.txt | 20 +- deployment_pipeline/staging-config.json | 7 +- 9 files changed, 217 insertions(+), 231 deletions(-) diff --git a/deployment_pipeline/app.py b/deployment_pipeline/app.py index bbf65d7..c811e86 100644 --- a/deployment_pipeline/app.py +++ b/deployment_pipeline/app.py @@ -27,7 +27,7 @@ def create_endpoint( ): # Define variables for passing down to stacks - endpoint_name = f"sagemaker-{project_name}-{stage_name}" + endpoint_name = f"{project_name}-{stage_name}" if len(endpoint_name) > 63: raise Exception( f"SageMaker endpoint: {endpoint_name} must be less than 64 characters" @@ -50,7 +50,7 @@ def create_endpoint( package_group_name = project_name # If we don't have a specific champion variant defined, get the latest approved - if deployment_config.variant_config is None: + if deployment_config.variant_config.model_package_version is None: logger.info("Selecting latest approved") p = registry.get_latest_approved_packages(package_group_name, max_results=1)[0] deployment_config.variant_config = VariantConfig( @@ -59,6 +59,8 @@ def create_endpoint( initial_variant_weight=1, instance_count=deployment_config.instance_count, instance_type=deployment_config.instance_type, + variant_name=deployment_config.variant_config.variant_name + or f"{package_group_name}-LatestApproved", ) else: # Get the versioned package and update ARN @@ -73,10 +75,10 @@ def create_endpoint( baseline_uri = registry.get_data_check_baseline_uri(p["ModelPackageArn"]) logger.info(f"Got baseline uri: {baseline_uri}") - data_capture_uri = f"s3://{artifact_bucket}/{project_id}/datacapture" + data_capture_uri = f"s3://{artifact_bucket}/monitoring/datacapture" logger.info(f"Got data capture uri: {data_capture_uri}") - reporting_uri = f"s3://{artifact_bucket}/{project_id}/monitoring" + reporting_uri = f"s3://{artifact_bucket}/monitoring/reports" logger.info(f"Got reporting uri: {reporting_uri}") stack_synthesizer = cdk.DefaultStackSynthesizer( @@ -146,5 +148,5 @@ def main( default=os.environ.get("ARTIFACT_BUCKET"), ) args = vars(parser.parse_args()) - print("args: {}".format(args)) + logger.info("args: {}".format(args)) main(**args) diff --git a/deployment_pipeline/cdk.json b/deployment_pipeline/cdk.json index 3a1004a..ddbb4d5 100644 --- a/deployment_pipeline/cdk.json +++ b/deployment_pipeline/cdk.json @@ -1,5 +1,5 @@ { "app": "python3 app.py", - "context": { - } -} + "context": {}, + "versionReporting": false +} \ No newline at end of file diff --git a/deployment_pipeline/infra/deployment_config.py b/deployment_pipeline/infra/deployment_config.py index eded421..0a4279a 100644 --- a/deployment_pipeline/infra/deployment_config.py +++ b/deployment_pipeline/infra/deployment_config.py @@ -7,7 +7,7 @@ def __init__(self, instance_count: int = 1, instance_type: str = "ml.t2.medium") class VariantConfig(InstanceConfig): def __init__( self, - model_package_version: str, + model_package_version: str = None, initial_variant_weight: float = 1.0, variant_name: str = None, instance_count: int = 1, diff --git a/deployment_pipeline/infra/model_registry.py b/deployment_pipeline/infra/model_registry.py index e0eaa0c..e7b805b 100644 --- a/deployment_pipeline/infra/model_registry.py +++ b/deployment_pipeline/infra/model_registry.py @@ -217,13 +217,17 @@ def get_pipeline_execution_arn(self, model_package_arn: str): "MetadataProperties" ]["GeneratedBy"] - def get_data_check_baseline_uri(self, model_package_arn: str): try: - model_details = self.sm_client.describe_model_package(ModelPackageName=model_package_arn) - print(model_details) - baseline_uri = model_details['DriftCheckBaselines']['ModelDataQuality']['Constraints']['S3Uri'] - baseline_uri = baseline_uri.replace('/constraints.json','') # returning the folder containing constraints and statistics + model_details = self.sm_client.describe_model_package( + ModelPackageName=model_package_arn + ) + logger.info(model_details) + baseline_uri = model_details["DriftCheckBaselines"]["ModelDataQuality"][ + "Constraints" + ]["S3Uri"] + # returning the folder containing constraints and statistics + baseline_uri = baseline_uri.replace("/constraints.json", "") return baseline_uri except ClientError as e: error_message = e.response["Error"]["Message"] diff --git a/deployment_pipeline/infra/sagemaker_stack.py b/deployment_pipeline/infra/sagemaker_stack.py index 0c5b252..648bc0f 100644 --- a/deployment_pipeline/infra/sagemaker_stack.py +++ b/deployment_pipeline/infra/sagemaker_stack.py @@ -25,15 +25,15 @@ def __init__( ) -> None: super().__init__(scope, construct_id, **kwargs) - # Select the variant config and name - needs to be same for updating Endpoint or Autoscaling deregister fails + # Select the variant config and name - needs to be same for updating Endpoint or + # Autoscaling deregister fails # see: https://docs.aws.amazon.com/sagemaker/latest/dg/endpoint-scaling.html variant_config = deployment_config.variant_config - variant_name = variant_config.variant_name or "LatestApproved" # Do not use a custom named resource for models as these get replaced model = sagemaker.CfnModel( self, - variant_name, + variant_config.variant_name, execution_role_arn=sagemaker_execution_role, primary_container=sagemaker.CfnModel.ContainerDefinitionProperty( model_package_name=variant_config.model_package_arn, @@ -46,7 +46,7 @@ def __init__( initial_variant_weight=variant_config.initial_variant_weight, instance_type=variant_config.instance_type, model_name=model.attr_model_name, - variant_name=variant_name, + variant_name=variant_config.variant_name, ) endpoint_config = sagemaker.CfnEndpointConfig( @@ -141,7 +141,7 @@ def __init__( ), tags=tags, ) - monitoring_schedule.add_depends_on(endpoint) + monitoring_schedule.add_dependency(endpoint) drift_alarm = cloudwatch.CfnAlarm( self, @@ -165,10 +165,10 @@ def __init__( datapoints_to_alarm=deployment_config.schedule_config.datapoints_to_alarm, statistic=deployment_config.schedule_config.statistic, ) - drift_alarm.add_depends_on(monitoring_schedule) + drift_alarm.add_dependency(monitoring_schedule) if deployment_config.auto_scaling is not None: - resource_id = f"endpoint/{endpoint_name}/variant/{variant_name}" + resource_id = f"endpoint/{endpoint_name}/variant/{variant_config.variant_name}" scalable_target = applicationautoscaling.CfnScalableTarget( self, @@ -180,7 +180,7 @@ def __init__( scalable_dimension="sagemaker:variant:DesiredInstanceCount", service_namespace="sagemaker", ) - scalable_target.add_depends_on(endpoint) + scalable_target.add_dependency(endpoint) scaling_policy = applicationautoscaling.CfnScalingPolicy( self, @@ -189,7 +189,7 @@ def __init__( policy_type="TargetTrackingScaling", resource_id=resource_id, scalable_dimension="sagemaker:variant:DesiredInstanceCount", - service_namespace="sagemaker", # Note: This is different to scaling above + service_namespace="sagemaker", # Note: different to scaling above target_tracking_scaling_policy_configuration=applicationautoscaling.CfnScalingPolicy.TargetTrackingScalingPolicyConfigurationProperty( target_value=deployment_config.auto_scaling.target_value, scale_in_cooldown=deployment_config.auto_scaling.scale_in_cooldown, @@ -199,7 +199,7 @@ def __init__( ), ), ) - scaling_policy.add_depends_on(scalable_target) + scaling_policy.add_dependency(scalable_target) # TODO: Add cloud watch alarm diff --git a/deployment_pipeline/poetry.lock b/deployment_pipeline/poetry.lock index 1bca843..3f2fcd4 100644 --- a/deployment_pipeline/poetry.lock +++ b/deployment_pipeline/poetry.lock @@ -1,27 +1,38 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + [[package]] name = "attrs" -version = "22.1.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" +files = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] [[package]] name = "aws-cdk-asset-awscli-v1" -version = "2.2.11" +version = "2.2.52" description = "A library that contains the AWS CLI for use in Lambda Layers" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-awscli-v1-2.2.52.tar.gz", hash = "sha256:ab04beec8e267e363931df2caf48a24100cb5799d7fd8db51efe881d117efa7a"}, + {file = "aws_cdk.asset_awscli_v1-2.2.52-py3-none-any.whl", hash = "sha256:6e9d686bb0b00242e869e91d57b65b619ffb42e99abe482436e3a6692485dbfe"}, +] [package.dependencies] -jsii = ">=1.71.0,<2.0.0" +jsii = ">=1.73.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -32,6 +43,10 @@ description = "A library that contains kubectl for use in Lambda Layers" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-kubectl-v20-2.1.1.tar.gz", hash = "sha256:9834cdb150c5590aea4e5eba6de2a89b4c60617451181c524810c5a75154565c"}, + {file = "aws_cdk.asset_kubectl_v20-2.1.1-py3-none-any.whl", hash = "sha256:a2fad1a5a35a94a465efe60859f91e45dacc33261fb9bbf1cf9bbc6e2f70e9d6"}, +] [package.dependencies] jsii = ">=1.70.0,<2.0.0" @@ -40,41 +55,63 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-asset-node-proxy-agent-v5" -version = "2.0.17" +version = "2.0.42" description = "@aws-cdk/asset-node-proxy-agent-v5" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.42.tar.gz", hash = "sha256:ae1b615be42e78681e05b145460603f171c06b671a2d1caa060a159b94b06366"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.42-py3-none-any.whl", hash = "sha256:6e0174802097d558daa1be5c4e6e7f309eeba626392955e596bf967ee37865d3"}, +] [package.dependencies] -jsii = ">=1.71.0,<2.0.0" +jsii = ">=1.73.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.51.0" +version = "2.62.2" description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "aws-cdk-lib-2.62.2.tar.gz", hash = "sha256:f85000438d849a0522ffcd8e7cb5a70be5fa34339082d4d569734169c6d37b4d"}, + {file = "aws_cdk_lib-2.62.2-py3-none-any.whl", hash = "sha256:03dfb8303b00333177b18e3b60c95d738adf4d90086a5b1e707e896fdc234d52"}, +] [package.dependencies] -"aws-cdk.asset-awscli-v1" = ">=2.2.9,<3.0.0" +"aws-cdk.asset-awscli-v1" = ">=2.2.49,<3.0.0" "aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" -"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.15,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.38,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.71.0,<2.0.0" +jsii = ">=1.73.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "black" -version = "22.10.0" +version = "22.12.0" description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, + {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, + {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, + {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, + {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, + {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, + {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, + {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, + {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, + {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, + {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, + {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, +] [package.dependencies] click = ">=8.0.0" @@ -92,14 +129,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.12" +version = "1.26.59" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "boto3-1.26.59-py3-none-any.whl", hash = "sha256:34ee771a5cc84c16e75d4b9ef4672f51c2bafdce66ec457bbaac630b37d9cd5e"}, + {file = "boto3-1.26.59.tar.gz", hash = "sha256:7d9cebb507fc96e6eb429621ccb2e731b75e7bbb8d6d9f0cf0c08089ee3c1ab7"}, +] [package.dependencies] -botocore = ">=1.29.12,<1.30.0" +botocore = ">=1.29.59,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -108,11 +149,15 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.12" +version = "1.29.59" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "botocore-1.29.59-py3-none-any.whl", hash = "sha256:5533644ddefaccfaa98460a63eb73e61a46aad019771226d103b1054b0df6103"}, + {file = "botocore-1.29.59.tar.gz", hash = "sha256:bc75d41c5eecf624a2f9875483135aa78088a50c8d29847793f92756697cfed5"}, +] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -120,7 +165,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.14.0)"] +crt = ["awscrt (==0.15.3)"] [[package]] name = "cattrs" @@ -129,6 +174,10 @@ description = "Composable complex class support for attrs and dataclasses." category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "cattrs-22.2.0-py3-none-any.whl", hash = "sha256:bc12b1f0d000b9f9bee83335887d532a1d3e99a833d1bf0882151c97d3e68c21"}, + {file = "cattrs-22.2.0.tar.gz", hash = "sha256:f0eed5642399423cf656e7b66ce92cdc5b963ecafd041d1b24d136fdde7acf6d"}, +] [package.dependencies] attrs = ">=20" @@ -141,6 +190,10 @@ description = "Composable command line interface toolkit" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -152,38 +205,54 @@ description = "Cross-platform colored terminal text." category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "constructs" -version = "10.1.163" +version = "10.1.235" description = "A programming model for software-defined state" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "constructs-10.1.235-py3-none-any.whl", hash = "sha256:7e7d56dbb75cb51b859e2e7418e4adc71da2669df53f95cd60cff4e728375563"}, + {file = "constructs-10.1.235.tar.gz", hash = "sha256:9f64f57871743722f26dad556fdc8f196d26fb5b07f36b0e4a1f4e4bf73fdcf8"}, +] [package.dependencies] -jsii = ">=1.71.0,<2.0.0" +jsii = ">=1.74.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "exceptiongroup" -version = "1.0.4" +version = "1.1.0" description = "Backport of PEP 654 (exception groups)" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, +] [package.extras] test = ["pytest (>=6)"] [[package]] name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] [[package]] name = "jmespath" @@ -192,14 +261,22 @@ description = "JSON Matching Expressions" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] [[package]] name = "jsii" -version = "1.71.0" +version = "1.74.0" description = "Python client for jsii runtime" category = "main" optional = false python-versions = "~=3.7" +files = [ + {file = "jsii-1.74.0-py3-none-any.whl", hash = "sha256:ee76781fe66106c367fbb3bb383db4f5e9b8ff3d3c4c0f34624c050211f040be"}, + {file = "jsii-1.74.0.tar.gz", hash = "sha256:575131396ad34f8f6e9f2604953ecbf4f3368625656a828b13089e4abb81b443"}, +] [package.dependencies] attrs = ">=21.2,<23.0" @@ -216,37 +293,50 @@ description = "Experimental type system extensions for programs checked with the category = "dev" optional = false python-versions = "*" +files = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] [[package]] name = "packaging" -version = "21.3" +version = "23.0" description = "Core utilities for Python packages" category = "dev" optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +python-versions = ">=3.7" +files = [ + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, +] [[package]] name = "pathspec" -version = "0.10.2" +version = "0.11.0" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.0-py3-none-any.whl", hash = "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229"}, + {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, +] [[package]] name = "platformdirs" -version = "2.5.4" +version = "2.6.2" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, + {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, +] [package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] -test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -255,6 +345,10 @@ description = "plugin and hook calling mechanisms for python" category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] [package.extras] dev = ["pre-commit", "tox"] @@ -267,25 +361,22 @@ description = "Publication helps you maintain public-api-friendly modules by pre category = "main" optional = false python-versions = "*" - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] +files = [ + {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, + {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, +] [[package]] name = "pytest" -version = "7.2.0" +version = "7.2.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, + {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, +] [package.dependencies] attrs = ">=19.2.0" @@ -306,6 +397,10 @@ description = "Extensions to the standard Python datetime module" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] [package.dependencies] six = ">=1.5" @@ -317,6 +412,10 @@ description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] [package.dependencies] botocore = ">=1.12.36,<2.0a.0" @@ -331,6 +430,10 @@ description = "Python 2 and 3 compatibility utilities" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] [[package]] name = "tomli" @@ -339,6 +442,10 @@ description = "A lil' TOML parser" category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] [[package]] name = "typeguard" @@ -347,6 +454,10 @@ description = "Run-time type checker for Python" category = "main" optional = false python-versions = ">=3.5.3" +files = [ + {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, + {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, +] [package.extras] doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] @@ -359,14 +470,22 @@ description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] [[package]] name = "urllib3" -version = "1.26.12" +version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, +] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] @@ -374,151 +493,6 @@ secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "p socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] -lock-version = "1.1" +lock-version = "2.0" python-versions = "^3.9" content-hash = "0b8ee6379573fb9ea6cad3dda51ec5f919d9c400d2ea6bb76cca9fe8f406504e" - -[metadata.files] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] -aws-cdk-asset-awscli-v1 = [ - {file = "aws-cdk.asset-awscli-v1-2.2.11.tar.gz", hash = "sha256:8eb174bbe0e98585e8dade5de42574ad1746c3b52f6b4dd1efa7fa53183f36a6"}, - {file = "aws_cdk.asset_awscli_v1-2.2.11-py3-none-any.whl", hash = "sha256:94489e30c30acc15d9371147ff44558889aa8c9b7b5d9db98ded66a34d017292"}, -] -aws-cdk-asset-kubectl-v20 = [ - {file = "aws-cdk.asset-kubectl-v20-2.1.1.tar.gz", hash = "sha256:9834cdb150c5590aea4e5eba6de2a89b4c60617451181c524810c5a75154565c"}, - {file = "aws_cdk.asset_kubectl_v20-2.1.1-py3-none-any.whl", hash = "sha256:a2fad1a5a35a94a465efe60859f91e45dacc33261fb9bbf1cf9bbc6e2f70e9d6"}, -] -aws-cdk-asset-node-proxy-agent-v5 = [ - {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.17.tar.gz", hash = "sha256:d0b039e5be3b2d1aa6b632d070c975c3a936920f182a9906e9eb7de9fa28b391"}, - {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.17-py3-none-any.whl", hash = "sha256:c622ca308959edf679452d85811e1d60afb85deb06a6446cc541cbb7fede6fe5"}, -] -aws-cdk-lib = [ - {file = "aws-cdk-lib-2.51.0.tar.gz", hash = "sha256:292db2f37062e9c19429d977c02567eb5adb310ce2c157782c8ca20d2b129e8b"}, - {file = "aws_cdk_lib-2.51.0-py3-none-any.whl", hash = "sha256:24f48a0143d2fc0712d0becfd2c09ff3e51ea609d1996a7fa1e4ea933e242bac"}, -] -black = [ - {file = "black-22.10.0-1fixedarch-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa"}, - {file = "black-22.10.0-1fixedarch-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef"}, - {file = "black-22.10.0-1fixedarch-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6"}, - {file = "black-22.10.0-1fixedarch-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d"}, - {file = "black-22.10.0-1fixedarch-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4"}, - {file = "black-22.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb"}, - {file = "black-22.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7"}, - {file = "black-22.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66"}, - {file = "black-22.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae"}, - {file = "black-22.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b"}, - {file = "black-22.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d"}, - {file = "black-22.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650"}, - {file = "black-22.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d"}, - {file = "black-22.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"}, - {file = "black-22.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87"}, - {file = "black-22.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395"}, - {file = "black-22.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0"}, - {file = "black-22.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383"}, - {file = "black-22.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de"}, - {file = "black-22.10.0-py3-none-any.whl", hash = "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458"}, - {file = "black-22.10.0.tar.gz", hash = "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1"}, -] -boto3 = [ - {file = "boto3-1.26.12-py3-none-any.whl", hash = "sha256:180b4413db1211836c622adfc16dd40a7b99b86cac894bc6e17ddea9d282ab14"}, - {file = "boto3-1.26.12.tar.gz", hash = "sha256:73da24667fe1351cef0f402ee9cd4589a0a9d97b617caca3c25b5cdc38f9d62c"}, -] -botocore = [ - {file = "botocore-1.29.12-py3-none-any.whl", hash = "sha256:3149b102e3c26c935acef6c330d0f46c717820d118886e983b6e81c306f31405"}, - {file = "botocore-1.29.12.tar.gz", hash = "sha256:fdae73306a41a30697be300bdecb1e0d560d453c6d748891856beb87e9f6573f"}, -] -cattrs = [ - {file = "cattrs-22.2.0-py3-none-any.whl", hash = "sha256:bc12b1f0d000b9f9bee83335887d532a1d3e99a833d1bf0882151c97d3e68c21"}, - {file = "cattrs-22.2.0.tar.gz", hash = "sha256:f0eed5642399423cf656e7b66ce92cdc5b963ecafd041d1b24d136fdde7acf6d"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -constructs = [ - {file = "constructs-10.1.163-py3-none-any.whl", hash = "sha256:a1587f5adf1b45b74a1e870b57164652f6a459510b8743a289cd57700058f5be"}, - {file = "constructs-10.1.163.tar.gz", hash = "sha256:1ae9819b81aa543a65ca8b9a10ebc404f398849f81df702df5755c3ceb18aa75"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, - {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, -] -iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] -jmespath = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] -jsii = [ - {file = "jsii-1.71.0-py3-none-any.whl", hash = "sha256:0f081498d9f2a12850577caedebe604e4b6e98e7fb20c3bbcca05a5ab8bd0f0b"}, - {file = "jsii-1.71.0.tar.gz", hash = "sha256:e8fedaf077c192917f1d43100b2bbb5d107e4393b713f8ca2814d7653d64a026"}, -] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] -pathspec = [ - {file = "pathspec-0.10.2-py3-none-any.whl", hash = "sha256:88c2606f2c1e818b978540f73ecc908e13999c6c3a383daf3705652ae79807a5"}, - {file = "pathspec-0.10.2.tar.gz", hash = "sha256:8f6bf73e5758fd365ef5d58ce09ac7c27d2833a8d7da51712eac6e27e35141b0"}, -] -platformdirs = [ - {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, - {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -publication = [ - {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, - {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, -] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] -pytest = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -s3transfer = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -typeguard = [ - {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, - {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, -] -typing-extensions = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, -] -urllib3 = [ - {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, - {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, -] diff --git a/deployment_pipeline/prod-config.json b/deployment_pipeline/prod-config.json index caf4874..ce54d3d 100644 --- a/deployment_pipeline/prod-config.json +++ b/deployment_pipeline/prod-config.json @@ -12,5 +12,8 @@ "metric_name": "feature_baseline_drift_fare_amount", "metric_threshold": 0.4, "period": 3600 + }, + "variant_config": { + "variant_name": "DataDrift-Prod" } } \ No newline at end of file diff --git a/deployment_pipeline/requirements.txt b/deployment_pipeline/requirements.txt index 3111f63..46ff254 100644 --- a/deployment_pipeline/requirements.txt +++ b/deployment_pipeline/requirements.txt @@ -1,19 +1,19 @@ -attrs==22.1.0 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-awscli-v1==2.2.11 ; python_version >= "3.9" and python_version < "4.0" +attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-awscli-v1==2.2.52 ; python_version >= "3.9" and python_version < "4.0" aws-cdk-asset-kubectl-v20==2.1.1 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-node-proxy-agent-v5==2.0.17 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-lib==2.51.0 ; python_version >= "3.9" and python_version < "4.0" -boto3==1.26.12 ; python_version >= "3.9" and python_version < "4.0" -botocore==1.29.12 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-node-proxy-agent-v5==2.0.42 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-lib==2.62.2 ; python_version >= "3.9" and python_version < "4.0" +boto3==1.26.59 ; python_version >= "3.9" and python_version < "4.0" +botocore==1.29.59 ; python_version >= "3.9" and python_version < "4.0" cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -constructs==10.1.163 ; python_version >= "3.9" and python_version < "4.0" -exceptiongroup==1.0.4 ; python_version >= "3.9" and python_version < "3.11" +constructs==10.1.235 ; python_version >= "3.9" and python_version < "4.0" +exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11" jmespath==1.0.1 ; python_version >= "3.9" and python_version < "4.0" -jsii==1.71.0 ; python_version >= "3.9" and python_version < "4.0" +jsii==1.74.0 ; python_version >= "3.9" and python_version < "4.0" publication==0.0.3 ; python_version >= "3.9" and python_version < "4.0" python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "4.0" six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" typeguard==2.13.3 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.4.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==1.26.12 ; python_version >= "3.9" and python_version < "4" +urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4.0" diff --git a/deployment_pipeline/staging-config.json b/deployment_pipeline/staging-config.json index a9d3136..4547a5c 100644 --- a/deployment_pipeline/staging-config.json +++ b/deployment_pipeline/staging-config.json @@ -1,5 +1,8 @@ { "stage_name": "staging", "instance_count": 1, - "instance_type": "ml.t2.medium" -} + "instance_type": "ml.t2.medium", + "variant_config": { + "variant_name": "DataDrift-Staging" + } +} \ No newline at end of file From 97c11fd8b798af83a006bddcabdf44b3b3c551ea Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 30 Jan 2023 08:32:55 +0800 Subject: [PATCH 07/17] Delete staging endpoint after approval --- batch_pipeline/buildspec.yml | 2 +- build_pipeline/buildspec.yml | 2 +- deployment_pipeline/buildspec.yml | 2 +- drift-service-catalog.yml | 404 ----------------------------- infra/build_pipeline_construct.py | 68 ++--- infra/deploy_pipeline_construct.py | 11 +- infra/pipeline_product_stack.py | 3 +- poetry.lock | 116 ++++----- requirements.txt | 12 +- 9 files changed, 113 insertions(+), 507 deletions(-) delete mode 100644 drift-service-catalog.yml diff --git a/batch_pipeline/buildspec.yml b/batch_pipeline/buildspec.yml index 2761b88..0c6d274 100644 --- a/batch_pipeline/buildspec.yml +++ b/batch_pipeline/buildspec.yml @@ -5,7 +5,7 @@ phases: nodejs: "16" python: "3.9" commands: - - npm install aws-cdk@2.46.0 + - npm install aws-cdk@2.62.2 - npm update - python -m pip install --upgrade pip - python -m pip install -r requirements.txt diff --git a/build_pipeline/buildspec.yml b/build_pipeline/buildspec.yml index e10f901..54e8bf3 100644 --- a/build_pipeline/buildspec.yml +++ b/build_pipeline/buildspec.yml @@ -5,7 +5,7 @@ phases: nodejs: '16' python: '3.9' commands: - - npm install aws-cdk@2.46.0 + - npm install aws-cdk@2.62.2 - npm update - python -m pip install -r requirements.txt build: diff --git a/deployment_pipeline/buildspec.yml b/deployment_pipeline/buildspec.yml index e10f901..54e8bf3 100644 --- a/deployment_pipeline/buildspec.yml +++ b/deployment_pipeline/buildspec.yml @@ -5,7 +5,7 @@ phases: nodejs: '16' python: '3.9' commands: - - npm install aws-cdk@2.46.0 + - npm install aws-cdk@2.62.2 - npm update - python -m pip install -r requirements.txt build: diff --git a/drift-service-catalog.yml b/drift-service-catalog.yml deleted file mode 100644 index 96c8b6a..0000000 --- a/drift-service-catalog.yml +++ /dev/null @@ -1,404 +0,0 @@ -Parameters: - ExecutionRoleArn: - Type: String - AllowedPattern: ^arn:aws[a-z\-]*:iam::\d{12}:role/?[a-zA-Z_0-9+=,.@\-_/]+$ - Description: The SageMaker Studio execution role - MinLength: 1 - PortfolioName: - Type: String - Default: SageMaker Organization Templates - Description: The name of the portfolio - MinLength: 1 - PortfolioOwner: - Type: String - Default: administrator - Description: The owner of the portfolio - MaxLength: 50 - MinLength: 1 - ProductVersion: - Type: String - Default: "1.0" - Description: The product version to deploy - MinLength: 1 -Resources: - LaunchRolePolicyA9E2E5B1: - Type: AWS::IAM::Policy - Properties: - PolicyDocument: - Statement: - - Action: - - s3:GetObject* - - s3:GetBucket* - - s3:List* - Effect: Allow - Resource: - - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - :s3:::aws-ml-blog - - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - :s3:::aws-ml-blog/* - - Action: - - ssm:DescribeParameters - - ssm:GetParameters - - ssm:GetParameter - - ssm:GetParameterHistory - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":ssm:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :parameter - - Ref: CodeCommitSeedBucket94EB6088 - - Action: - - ssm:DescribeParameters - - ssm:GetParameters - - ssm:GetParameter - - ssm:GetParameterHistory - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":ssm:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :parameter - - Ref: CodeCommitBuildKey09FC7134 - - Action: - - ssm:DescribeParameters - - ssm:GetParameters - - ssm:GetParameter - - ssm:GetParameterHistory - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":ssm:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :parameter - - Ref: CodeCommitBatchKeyB0BCA7C9 - - Action: - - ssm:DescribeParameters - - ssm:GetParameters - - ssm:GetParameter - - ssm:GetParameterHistory - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":ssm:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :parameter - - Ref: CodeCommitDeployKey5E5A6E47 - Version: "2012-10-17" - PolicyName: LaunchRolePolicyA9E2E5B1 - Roles: - - AmazonSageMakerServiceCatalogProductsLaunchRole - ProductsUseRolePolicy7FB0EE21: - Type: AWS::IAM::Policy - Properties: - PolicyDocument: - Statement: - - Action: sts:AssumeRole - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":iam::" - - Ref: AWS::AccountId - - :role/service-role/AmazonSageMakerServiceCatalogProductsUseRole - - Action: - - application-autoscaling:DeregisterScalableTarget - - application-autoscaling:DeleteScalingPolicy - - application-autoscaling:DescribeScalingPolicies - - application-autoscaling:PutScalingPolicy - - application-autoscaling:RegisterScalableTarget - - application-autoscaling:DescribeScalableTargets - - iam:CreateServiceLinkedRole - - cloudwatch:DeleteAlarms - - cloudwatch:DescribeAlarms - - cloudwatch:PutMetricAlarm - - codepipeline:PutJobSuccessResult - - codepipeline:PutJobFailureResult - Effect: Allow - Resource: "*" - - Action: iam:CreateServiceLinkedRole - Condition: - StringLike: - iam:AWSServiceName: sagemaker.application-autoscaling.amazonaws.com - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:aws:iam::" - - Ref: AWS::AccountId - - :role/aws-service-role/sagemaker.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_SageMakerEndpoint - - Action: - - codepipeline:EnableStageTransition - - codepipeline:DisableStageTransition - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:aws:codepipeline:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :sagemaker-* - - Action: - - events:EnableRule - - events:DisableRule - Effect: Allow - Resource: - Fn::Join: - - "" - - - "arn:aws:events:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - :rule/sagemaker-* - Version: "2012-10-17" - PolicyName: ProductsUseRolePolicy7FB0EE21 - Roles: - - AmazonSageMakerServiceCatalogProductsUseRole - Portfolio856A4190: - Type: AWS::ServiceCatalog::Portfolio - Properties: - DisplayName: - Ref: PortfolioName - ProviderName: - Ref: PortfolioOwner - Description: Organization templates for drift detection pipelines - PortfolioPortfolioProductAssociationafa25c91024062989E0C: - Type: AWS::ServiceCatalog::PortfolioProductAssociation - Properties: - PortfolioId: - Ref: Portfolio856A4190 - ProductId: - Ref: BatchProduct9E7E43C3 - PortfolioPortfolioProductAssociation7d95b807a437095C0A65: - Type: AWS::ServiceCatalog::PortfolioProductAssociation - Properties: - PortfolioId: - Ref: Portfolio856A4190 - ProductId: - Ref: DeployProduct78C47E65 - BatchProduct9E7E43C3: - Type: AWS::ServiceCatalog::CloudFormationProduct - Properties: - Name: Amazon SageMaker drift detection template for batch scoring - Owner: - Ref: PortfolioOwner - ProvisioningArtifactParameters: - - DisableTemplateValidation: false - Info: - LoadTemplateFromURL: - Fn::Sub: https://s3.${AWS::URLSuffix}/aws-ml-blog/artifacts/amazon-sagemaker-drift-detection/fe4e6e939c884f6be729a386942e86bec0cdfb7b27f0f8e61e3b1e33c8b679cc.json - Name: - Ref: ProductVersion - Description: This template includes a model building pipeline that includes a workflow to pre-process, train, evaluate and register a model as well as create a baseline for model monitoring. The batch pipeline creates a staging and production workflow to perform scoring, and model monitor to output metrics to automate re-training on drift detection. - Tags: - - Key: sagemaker:studio-visibility - Value: "true" - DeployProduct78C47E65: - Type: AWS::ServiceCatalog::CloudFormationProduct - Properties: - Name: Amazon SageMaker drift detection template for real-time deployment - Owner: - Ref: PortfolioOwner - ProvisioningArtifactParameters: - - DisableTemplateValidation: false - Info: - LoadTemplateFromURL: - Fn::Sub: https://s3.${AWS::URLSuffix}/aws-ml-blog/artifacts/amazon-sagemaker-drift-detection/a5523ad117867ab80983bc7cf31d32707c1483b1347bd9c1695c03e06b31440f.json - Name: - Ref: ProductVersion - Description: This template includes a model building pipeline that includes a workflow to pre-process, train, evaluate and register a model as well as create a baseline for model monitoring. The deploy pipeline creates a staging and production endpoint, and schedules model monitor to output metrics to automate re-training on drift detection. - Tags: - - Key: sagemaker:studio-visibility - Value: "true" - PortfolioPrincipalAssociation: - Type: AWS::ServiceCatalog::PortfolioPrincipalAssociation - Properties: - PortfolioId: - Ref: Portfolio856A4190 - PrincipalARN: - Ref: ExecutionRoleArn - PrincipalType: IAM - DependsOn: - - BatchProduct9E7E43C3 - - DeployProduct78C47E65 - LaunchRoleConstraint0: - Type: AWS::ServiceCatalog::LaunchRoleConstraint - Properties: - PortfolioId: - Ref: Portfolio856A4190 - ProductId: - Ref: BatchProduct9E7E43C3 - Description: - Fn::Join: - - "" - - - "Launch as arn:" - - Ref: AWS::Partition - - ":iam::" - - Ref: AWS::AccountId - - :role/service-role/AmazonSageMakerServiceCatalogProductsLaunchRole - RoleArn: - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":iam::" - - Ref: AWS::AccountId - - :role/service-role/AmazonSageMakerServiceCatalogProductsLaunchRole - DependsOn: - - PortfolioPrincipalAssociation - LaunchRoleConstraint1: - Type: AWS::ServiceCatalog::LaunchRoleConstraint - Properties: - PortfolioId: - Ref: Portfolio856A4190 - ProductId: - Ref: DeployProduct78C47E65 - Description: - Fn::Join: - - "" - - - "Launch as arn:" - - Ref: AWS::Partition - - ":iam::" - - Ref: AWS::AccountId - - :role/service-role/AmazonSageMakerServiceCatalogProductsLaunchRole - RoleArn: - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":iam::" - - Ref: AWS::AccountId - - :role/service-role/AmazonSageMakerServiceCatalogProductsLaunchRole - DependsOn: - - PortfolioPrincipalAssociation - CodeCommitSeedBucket94EB6088: - Type: AWS::SSM::Parameter - Properties: - Type: String - Value: aws-ml-blog - Name: /drift-pipeline/CodeCommitSeedBucket - CodeCommitBuildKey09FC7134: - Type: AWS::SSM::Parameter - Properties: - Type: String - Value: artifacts/amazon-sagemaker-drift-detection/52d936109e8943985b2da5ebaefc6bd41a7c6b5727626706bb58b5f4cc1edff2.zip - Name: /drift-pipeline/CodeCommitBuildKey - CodeCommitBatchKeyB0BCA7C9: - Type: AWS::SSM::Parameter - Properties: - Type: String - Value: artifacts/amazon-sagemaker-drift-detection/2f29abdac58c60544d6e912108fe61480ee7193137069b780f2327e9835d6204.zip - Name: /drift-pipeline/CodeCommitBatchKey - CodeCommitDeployKey5E5A6E47: - Type: AWS::SSM::Parameter - Properties: - Type: String - Value: artifacts/amazon-sagemaker-drift-detection/97454cab4d8ce952dd7a22409cf3789f4437f80b7449e2ad424776826bac3d2d.zip - Name: /drift-pipeline/CodeCommitDeployKey - CDKMetadata: - Type: AWS::CDK::Metadata - Properties: - Analytics: v2:deflate64:H4sIAAAAAAAAE3VQQWrDQAx8S+7rTUwg9NjG0FMPxn2BkGVHtb0qu9qGYPz37tqlTaE9ScMMo9GUtixP9rB7hGsosB32M4onO78q4GCqztXgYSIlbxoKEj2SeQqBNAl6dr2pxAX1ETWLvyVpT0TLyuIWk71nhsnOtYyMt9V33RYTyH8wEoLCKH1WeO0SJ5voL1B7adPBFEOQIZ8w1SixfRY/rfBLsKb4j7hzY4f8DuMvv869QHR4aWSk7UVgpynusYD8frBrCwnbc8SB9AyBTAhTri459j+93Ze4LIupb3oRtz/aB1sedm+BufDRKU9km21+Ai9p+LOVAQAA - Condition: CDKMetadataAvailable -Conditions: - CDKMetadataAvailable: - Fn::Or: - - Fn::Or: - - Fn::Equals: - - Ref: AWS::Region - - af-south-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-east-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-northeast-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-northeast-2 - - Fn::Equals: - - Ref: AWS::Region - - ap-south-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-southeast-1 - - Fn::Equals: - - Ref: AWS::Region - - ap-southeast-2 - - Fn::Equals: - - Ref: AWS::Region - - ca-central-1 - - Fn::Equals: - - Ref: AWS::Region - - cn-north-1 - - Fn::Equals: - - Ref: AWS::Region - - cn-northwest-1 - - Fn::Or: - - Fn::Equals: - - Ref: AWS::Region - - eu-central-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-north-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-south-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-west-1 - - Fn::Equals: - - Ref: AWS::Region - - eu-west-2 - - Fn::Equals: - - Ref: AWS::Region - - eu-west-3 - - Fn::Equals: - - Ref: AWS::Region - - me-south-1 - - Fn::Equals: - - Ref: AWS::Region - - sa-east-1 - - Fn::Equals: - - Ref: AWS::Region - - us-east-1 - - Fn::Equals: - - Ref: AWS::Region - - us-east-2 - - Fn::Or: - - Fn::Equals: - - Ref: AWS::Region - - us-west-1 - - Fn::Equals: - - Ref: AWS::Region - - us-west-2 - diff --git a/infra/build_pipeline_construct.py b/infra/build_pipeline_construct.py index e7dba64..e21da7a 100644 --- a/infra/build_pipeline_construct.py +++ b/infra/build_pipeline_construct.py @@ -133,19 +133,18 @@ def __init__( pipeline_name=code_pipeline_name, ) + source_action = codepipeline_actions.CodeCommitSourceAction( + action_name="CodeCommit_Source", + repository=code, + # trigger=codepipeline_actions.CodeCommitTrigger.NONE, # Created below + event_role=event_role, + output=source_output, + branch=branch_name, + role=code_pipeline_role, + ) source_stage = code_pipeline.add_stage( stage_name="Source", - actions=[ - codepipeline_actions.CodeCommitSourceAction( - action_name="CodeCommit_Source", - repository=code, - trigger=codepipeline_actions.CodeCommitTrigger.NONE, # Created below - event_role=event_role, - output=source_output, - branch=branch_name, - role=code_pipeline_role, - ) - ], + actions=[source_action], ) build_stage = code_pipeline.add_stage( @@ -160,6 +159,9 @@ def __init__( pipeline_build_output, ], role=code_pipeline_role, + environment_variables={ + "COMMIT_ID": source_action.variables.commit_id, + }, ), ], ) @@ -301,25 +303,25 @@ def __init__( targets=[targets.LambdaFunction(lambda_pipeline_change)], ) - events.Rule( - self, - "CodeCommitRule", - rule_name=f"sagemaker-{project_name}-codecommit-{construct_id}", - description="Rule to trigger a build when code is updated in CodeCommit.", - event_pattern=events.EventPattern( - source=["aws.codecommit"], - detail_type=["CodeCommit Repository State Change"], - detail={ - "event": ["referenceCreated", "referenceUpdated"], - "referenceType": ["branch"], - "referenceName": [branch_name], - }, - resources=[code.repository_arn], - ), - targets=[ - targets.CodePipeline( - pipeline=code_pipeline, - event_role=event_role, - ) - ], - ) + # events.Rule( + # self, + # "CodeCommitRule", + # rule_name=f"sagemaker-{project_name}-codecommit-{construct_id}", + # description="Rule to trigger a build when code is updated in CodeCommit.", + # event_pattern=events.EventPattern( + # source=["aws.codecommit"], + # detail_type=["CodeCommit Repository State Change"], + # detail={ + # "event": ["referenceCreated", "referenceUpdated"], + # "referenceType": ["branch"], + # "referenceName": [branch_name], + # }, + # resources=[code.repository_arn], + # ), + # targets=[ + # targets.CodePipeline( + # pipeline=code_pipeline, + # event_role=event_role, + # ) + # ], + # ) diff --git a/infra/deploy_pipeline_construct.py b/infra/deploy_pipeline_construct.py index ef284f4..085dae4 100644 --- a/infra/deploy_pipeline_construct.py +++ b/infra/deploy_pipeline_construct.py @@ -130,6 +130,8 @@ def __init__( ), ], ) + + staging_stack_name = f"sagemaker-{project_name}-{construct_id}-staging" staging_deploy_stage = code_pipeline.add_stage( stage_name="DeployStaging", actions=[ @@ -139,7 +141,7 @@ def __init__( template_path=cdk_build_output.at_path( "drift-deploy-staging.template.json" ), - stack_name=f"sagemaker-{project_name}-{construct_id}-staging", + stack_name=staging_stack_name, admin_permissions=False, deployment_role=cloudformation_role, role=code_pipeline_role, @@ -168,10 +170,15 @@ def __init__( role=code_pipeline_role, replace_on_failure=True, ), + codepipeline_actions.CloudFormationDeleteStackAction( + stack_name=staging_stack_name, + admin_permissions=False, + deployment_role=cloudformation_role, + role=code_pipeline_role, + ), ], ) - # Add deploy role to target the code pipeline when model package is approved events.Rule( self, "ModelRegistryRule", diff --git a/infra/pipeline_product_stack.py b/infra/pipeline_product_stack.py index a07d4b1..5de4e90 100644 --- a/infra/pipeline_product_stack.py +++ b/infra/pipeline_product_stack.py @@ -70,7 +70,8 @@ def __init__( branch_name = "main" if build_pipeline: - # Require a schedule parameter (must be cron, otherwise will trigger every time rate is enabled/disabled) + # Require a schedule parameter (must be cron, otherwise will trigger every + # time rate is enabled/disabled) # https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html retrain_schedule = cdk.CfnParameter( self, diff --git a/poetry.lock b/poetry.lock index ea3fc68..7113afe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,22 +72,22 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.59.0" +version = "2.61.1" description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.59.0.tar.gz", hash = "sha256:1faeced63e37a4caa58472ba368664e3aea935b3193ccbc3f4f55a81d93b59d1"}, - {file = "aws_cdk_lib-2.59.0-py3-none-any.whl", hash = "sha256:0f8718be6951facac8044c286e49c6459a00a833d9d8c56274f27af41ed194c6"}, + {file = "aws-cdk-lib-2.61.1.tar.gz", hash = "sha256:d2bb672be182e0cd675717648fa100e1a49f01de606ffa4f7a0b580093f31b27"}, + {file = "aws_cdk_lib-2.61.1-py3-none-any.whl", hash = "sha256:e0b5d49c73be945ca176b38ff3d2ccfc9474639000afb6af65a1eaf8b6aa3385"}, ] [package.dependencies] -"aws-cdk.asset-awscli-v1" = ">=2.2.30,<3.0.0" +"aws-cdk.asset-awscli-v1" = ">=2.2.49,<3.0.0" "aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" "aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.38,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.72.0,<2.0.0" +jsii = ">=1.73.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -129,18 +129,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.42" +version = "1.26.55" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.42-py3-none-any.whl", hash = "sha256:75c995a04723f23e35e16ea491ed91a1345e2fa6492678a216488512308dada1"}, - {file = "boto3-1.26.42.tar.gz", hash = "sha256:4cfd7e05e4033dbca2cc59bcfdafbdaef9d83dc3c0448917569b301d85766d9d"}, + {file = "boto3-1.26.55-py3-none-any.whl", hash = "sha256:e8ae1567b0a21d410f9c8a16a17a8028445a52061482272b4a516f5de5abbd92"}, + {file = "boto3-1.26.55.tar.gz", hash = "sha256:d68576dcb1c520474eafb64b996661068a369f40bbd104ccb9502cad71849e57"}, ] [package.dependencies] -botocore = ">=1.29.42,<1.30.0" +botocore = ">=1.29.55,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -149,14 +149,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.42" +version = "1.29.55" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.42-py3-none-any.whl", hash = "sha256:f52f9dbd7ad42b3528c1052086c1a7b6122a018f919afdb604f2889caefe8092"}, - {file = "botocore-1.29.42.tar.gz", hash = "sha256:d05c62f64e76194c40f598f5f7c804ec50d9820e9f03f6e0198558e4ace167c4"}, + {file = "botocore-1.29.55-py3-none-any.whl", hash = "sha256:dd8a0868b287015bed086c97b866f9bb8ca57737d15972d3ac773559f4088db8"}, + {file = "botocore-1.29.55.tar.gz", hash = "sha256:5fe6b8809292bb178a9378fea4f96bc969003d1048cf6de8ad3c5a73690fefa9"}, ] [package.dependencies] @@ -212,18 +212,18 @@ files = [ [[package]] name = "constructs" -version = "10.1.209" +version = "10.1.229" description = "A programming model for software-defined state" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "constructs-10.1.209-py3-none-any.whl", hash = "sha256:fb61b038e12f4f99550dd7537ba1f2a324e59242d0719ec5ef371b66492c7642"}, - {file = "constructs-10.1.209.tar.gz", hash = "sha256:679235f1d331f75fa170af8f6e7ed088bcd5114f91d48c6a971e266447491a6b"}, + {file = "constructs-10.1.229-py3-none-any.whl", hash = "sha256:b3479f52b0f4926c088a9c808f2cf3f4d80d7be850d5057f6a268db68b94a8e6"}, + {file = "constructs-10.1.229.tar.gz", hash = "sha256:87e9a0cffbb2573b288b1204aa9459bb21945e2c97b6d885618cd4a389954cd5"}, ] [package.dependencies] -jsii = ">=1.72.0,<2.0.0" +jsii = ">=1.73.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -331,14 +331,14 @@ files = [ [[package]] name = "jsii" -version = "1.72.0" +version = "1.73.0" description = "Python client for jsii runtime" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "jsii-1.72.0-py3-none-any.whl", hash = "sha256:4dcf65eca9400c15a6a7c9d85b27f4bfe15c96ad3e36272a502f391556858151"}, - {file = "jsii-1.72.0.tar.gz", hash = "sha256:6daf1c17362bd07c50c299e08d9a4454075550eb78e035a160ecf9ea68ded3cf"}, + {file = "jsii-1.73.0-py3-none-any.whl", hash = "sha256:13e8496c3afee70d85401ad1eef2ddedbdb88e7e7abb3e68302dd6e61527191e"}, + {file = "jsii-1.73.0.tar.gz", hash = "sha256:be6458236e787be0b02c2fe869b6f4ed906398b6cc537190d61a60d2b5c9dfbb"}, ] [package.dependencies] @@ -428,51 +428,51 @@ files = [ [[package]] name = "packaging" -version = "22.0" +version = "23.0" description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, - {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] [[package]] name = "pandas" -version = "1.5.2" +version = "1.5.3" description = "Powerful data structures for data analysis, time series, and statistics" category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, - {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e2b83abd292194f350bb04e188f9379d36b8dfac24dd445d5c87575f3beaf789"}, - {file = "pandas-1.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2552bffc808641c6eb471e55aa6899fa002ac94e4eebfa9ec058649122db5824"}, - {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fc87eac0541a7d24648a001d553406f4256e744d92df1df8ebe41829a915028"}, - {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0d8fd58df5d17ddb8c72a5075d87cd80d71b542571b5f78178fb067fa4e9c72"}, - {file = "pandas-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:4aed257c7484d01c9a194d9a94758b37d3d751849c05a0050c087a358c41ad1f"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:375262829c8c700c3e7cbb336810b94367b9c4889818bbd910d0ecb4e45dc261"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc3cd122bea268998b79adebbb8343b735a5511ec14efb70a39e7acbc11ccbdc"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4f5a82afa4f1ff482ab8ded2ae8a453a2cdfde2001567b3ca24a4c5c5ca0db3"}, - {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8092a368d3eb7116e270525329a3e5c15ae796ccdf7ccb17839a73b4f5084a39"}, - {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6257b314fc14958f8122779e5a1557517b0f8e500cfb2bd53fa1f75a8ad0af2"}, - {file = "pandas-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:82ae615826da838a8e5d4d630eb70c993ab8636f0eff13cb28aafc4291b632b5"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:457d8c3d42314ff47cc2d6c54f8fc0d23954b47977b2caed09cd9635cb75388b"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c009a92e81ce836212ce7aa98b219db7961a8b95999b97af566b8dc8c33e9519"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:71f510b0efe1629bf2f7c0eadb1ff0b9cf611e87b73cd017e6b7d6adb40e2b3a"}, - {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a40dd1e9f22e01e66ed534d6a965eb99546b41d4d52dbdb66565608fde48203f"}, - {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ae7e989f12628f41e804847a8cc2943d362440132919a69429d4dea1f164da0"}, - {file = "pandas-1.5.2-cp38-cp38-win32.whl", hash = "sha256:530948945e7b6c95e6fa7aa4be2be25764af53fba93fe76d912e35d1c9ee46f5"}, - {file = "pandas-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:73f219fdc1777cf3c45fde7f0708732ec6950dfc598afc50588d0d285fddaefc"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9608000a5a45f663be6af5c70c3cbe634fa19243e720eb380c0d378666bc7702"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:315e19a3e5c2ab47a67467fc0362cb36c7c60a93b6457f675d7d9615edad2ebe"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e18bc3764cbb5e118be139b3b611bc3fbc5d3be42a7e827d1096f46087b395eb"}, - {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0183cb04a057cc38fde5244909fca9826d5d57c4a5b7390c0cc3fa7acd9fa883"}, - {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:344021ed3e639e017b452aa8f5f6bf38a8806f5852e217a7594417fb9bbfa00e"}, - {file = "pandas-1.5.2-cp39-cp39-win32.whl", hash = "sha256:e7469271497960b6a781eaa930cba8af400dd59b62ec9ca2f4d31a19f2f91090"}, - {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, - {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, + {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, + {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, + {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, + {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, + {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, + {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, + {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, ] [package.dependencies] @@ -636,14 +636,14 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2022.7" +version = "2022.7.1" description = "World timezone definitions, modern and historical" category = "dev" optional = false python-versions = "*" files = [ - {file = "pytz-2022.7-py2.py3-none-any.whl", hash = "sha256:93007def75ae22f7cd991c84e02d434876818661f8df9ad5df9e950ff4e52cfd"}, - {file = "pytz-2022.7.tar.gz", hash = "sha256:7ccfae7b4b2c067464a6733c6261673fdb8fd1be905460396b97a073e9fa683a"}, + {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, + {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, ] [[package]] @@ -666,13 +666,13 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "sagemaker" -version = "2.127.0" +version = "2.129.0" description = "Open source library for training and deploying models on Amazon SageMaker." category = "dev" optional = false python-versions = ">= 3.6" files = [ - {file = "sagemaker-2.127.0.tar.gz", hash = "sha256:417d229513be4dafc5dc3375f94c9bf0fa4e28f03c84780ea4afcb8861b5c08d"}, + {file = "sagemaker-2.129.0.tar.gz", hash = "sha256:ede424a2091c96211f3186c66f5679ba66f0da8a4692a0095e34ad4d2f0f3ae4"}, ] [package.dependencies] @@ -862,14 +862,14 @@ files = [ [[package]] name = "urllib3" -version = "1.26.13" +version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, - {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, ] [package.extras] diff --git a/requirements.txt b/requirements.txt index 5bc3b91..3876735 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,18 +2,18 @@ attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" aws-cdk-asset-awscli-v1==2.2.49 ; python_version >= "3.9" and python_version < "4.0" aws-cdk-asset-kubectl-v20==2.1.1 ; python_version >= "3.9" and python_version < "4.0" aws-cdk-asset-node-proxy-agent-v5==2.0.38 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-lib==2.59.0 ; python_version >= "3.9" and python_version < "4.0" -boto3==1.26.42 ; python_version >= "3.9" and python_version < "4.0" -botocore==1.29.42 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-lib==2.61.1 ; python_version >= "3.9" and python_version < "4.0" +boto3==1.26.55 ; python_version >= "3.9" and python_version < "4.0" +botocore==1.29.55 ; python_version >= "3.9" and python_version < "4.0" cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -constructs==10.1.209 ; python_version >= "3.9" and python_version < "4.0" +constructs==10.1.229 ; python_version >= "3.9" and python_version < "4.0" exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11" jmespath==1.0.1 ; python_version >= "3.9" and python_version < "4.0" -jsii==1.72.0 ; python_version >= "3.9" and python_version < "4.0" +jsii==1.73.0 ; python_version >= "3.9" and python_version < "4.0" publication==0.0.3 ; python_version >= "3.9" and python_version < "4.0" python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "4.0" six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" typeguard==2.13.3 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.4.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==1.26.13 ; python_version >= "3.9" and python_version < "4.0" +urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4.0" From 28eb69700c0a3312cf04f314a18d93bf55c0ef3a Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 30 Jan 2023 11:32:22 +0800 Subject: [PATCH 08/17] Fixed CICD stage names --- build_pipeline/pipelines/pipeline.py | 6 +-- infra/build_pipeline_construct.py | 81 +++++++++++++++------------- infra/deploy_pipeline_construct.py | 18 ++++--- 3 files changed, 57 insertions(+), 48 deletions(-) diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index a30f66c..e90ecd9 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -203,7 +203,7 @@ def get_pipeline( outputs=outputs, code="preprocess.py", source_dir=os.path.join(BASE_DIR, "preprocess"), - job_name=f"{commit_id}/scripts/preprocess", + job_name=f"scripts/{commit_id[:8]}/preprocess", ), cache_config=cache_config, ) @@ -279,7 +279,7 @@ def get_pipeline( step_train = TrainingStep( name="TrainModel", step_args=xgb_train.fit( - job_name=f"{commit_id}/scripts/train", + job_name=f"scripts/{commit_id[:8]}/train", inputs={ "train": TrainingInput( s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ @@ -315,7 +315,7 @@ def get_pipeline( step_eval = ProcessingStep( name="EvaluateModel", step_args=script_eval.run( - job_name=f"{commit_id}/scripts/evaluation", + job_name=f"scripts/{commit_id[:8]}/evaluation", inputs=[ ProcessingInput( source=step_train.properties.ModelArtifacts.S3ModelArtifacts, diff --git a/infra/build_pipeline_construct.py b/infra/build_pipeline_construct.py index e21da7a..79b18f5 100644 --- a/infra/build_pipeline_construct.py +++ b/infra/build_pipeline_construct.py @@ -8,6 +8,7 @@ from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda as lambda_ from aws_cdk import aws_s3 as s3 +from aws_cdk.aws_codebuild import BuildEnvironmentVariable from constructs import Construct from infra.sagemaker_pipelines_event_target import add_sagemaker_pipeline_target @@ -99,23 +100,21 @@ def __init__( environment=codebuild.BuildEnvironment( build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_4, environment_variables={ - "SAGEMAKER_PROJECT_NAME": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PROJECT_NAME": BuildEnvironmentVariable( value=project_name ), - "SAGEMAKER_PROJECT_ID": codebuild.BuildEnvironmentVariable( - value=project_id - ), - "AWS_REGION": codebuild.BuildEnvironmentVariable(value=env.region), - "SAGEMAKER_PIPELINE_NAME": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PROJECT_ID": BuildEnvironmentVariable(value=project_id), + "AWS_REGION": BuildEnvironmentVariable(value=env.region), + "SAGEMAKER_PIPELINE_NAME": BuildEnvironmentVariable( value=pipeline_name, ), - "SAGEMAKER_PIPELINE_DESCRIPTION": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PIPELINE_DESCRIPTION": BuildEnvironmentVariable( value=pipeline_description, ), - "SAGEMAKER_PIPELINE_ROLE_ARN": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PIPELINE_ROLE_ARN": BuildEnvironmentVariable( value=sagemaker_execution_role.role_arn, ), - "ARTIFACT_BUCKET": codebuild.BuildEnvironmentVariable( + "ARTIFACT_BUCKET": BuildEnvironmentVariable( value=s3_artifact.bucket_name ), }, @@ -136,18 +135,19 @@ def __init__( source_action = codepipeline_actions.CodeCommitSourceAction( action_name="CodeCommit_Source", repository=code, - # trigger=codepipeline_actions.CodeCommitTrigger.NONE, # Created below + # Created rule below to give it a custom name + trigger=codepipeline_actions.CodeCommitTrigger.NONE, event_role=event_role, output=source_output, branch=branch_name, role=code_pipeline_role, ) - source_stage = code_pipeline.add_stage( + _ = code_pipeline.add_stage( stage_name="Source", actions=[source_action], ) - build_stage = code_pipeline.add_stage( + _ = code_pipeline.add_stage( stage_name="Build", actions=[ codepipeline_actions.CodeBuildAction( @@ -160,7 +160,9 @@ def __init__( ], role=code_pipeline_role, environment_variables={ - "COMMIT_ID": source_action.variables.commit_id, + "COMMIT_ID": BuildEnvironmentVariable( + value=source_action.variables.commit_id, + ), }, ), ], @@ -188,7 +190,8 @@ def __init__( deployment_success_rule = pipeline_deploy_stage.on_state_change( name="Start pipeline", rule_name=build_rule_name, - description="Rule to execute the Model Build pipeline once the pipeline has been deployed", + description="Rule to execute the Model Build pipeline once " + "the pipeline has been deployed", schedule=events.Schedule.expression(retrain_schedule), event_pattern=events.EventPattern( source=["aws.codepipeline"], @@ -257,7 +260,8 @@ def __init__( }, ) - # Add permissions to put job status (if we want to call this directly within CodePipeline) + # Add permissions to put job status (if we want to call this directly + # within CodePipeline) # see: https://docs.aws.amazon.com/codepipeline/latest/userguide/approvals-iam-permissions.html lambda_pipeline_change.add_to_role_policy( iam.PolicyStatement( @@ -284,7 +288,8 @@ def __init__( self, "SagemakerPipelineRule", rule_name=f"sagemaker-{project_name}-sagemakerpipeline-{construct_id}", - description="Rule to enable/disable SM pipeline triggers when a SageMaker Model Building Pipeline is in progress.", + description="Rule to enable/disable SM pipeline triggers when a " + "SageMaker Model Building Pipeline is in progress.", event_pattern=events.EventPattern( source=["aws.sagemaker"], detail_type=[ @@ -303,25 +308,25 @@ def __init__( targets=[targets.LambdaFunction(lambda_pipeline_change)], ) - # events.Rule( - # self, - # "CodeCommitRule", - # rule_name=f"sagemaker-{project_name}-codecommit-{construct_id}", - # description="Rule to trigger a build when code is updated in CodeCommit.", - # event_pattern=events.EventPattern( - # source=["aws.codecommit"], - # detail_type=["CodeCommit Repository State Change"], - # detail={ - # "event": ["referenceCreated", "referenceUpdated"], - # "referenceType": ["branch"], - # "referenceName": [branch_name], - # }, - # resources=[code.repository_arn], - # ), - # targets=[ - # targets.CodePipeline( - # pipeline=code_pipeline, - # event_role=event_role, - # ) - # ], - # ) + events.Rule( + self, + "CodeCommitRule", + rule_name=f"sagemaker-{project_name}-codecommit-{construct_id}", + description="Rule to trigger a build when code is updated in CodeCommit.", + event_pattern=events.EventPattern( + source=["aws.codecommit"], + detail_type=["CodeCommit Repository State Change"], + detail={ + "event": ["referenceCreated", "referenceUpdated"], + "referenceType": ["branch"], + "referenceName": [branch_name], + }, + resources=[code.repository_arn], + ), + targets=[ + targets.CodePipeline( + pipeline=code_pipeline, + event_role=event_role, + ) + ], + ) diff --git a/infra/deploy_pipeline_construct.py b/infra/deploy_pipeline_construct.py index 085dae4..523e569 100644 --- a/infra/deploy_pipeline_construct.py +++ b/infra/deploy_pipeline_construct.py @@ -101,13 +101,14 @@ def __init__( pipeline_name=f"sagemaker-{project_name}-{construct_id}", ) - source_stage = code_pipeline.add_stage( + _ = code_pipeline.add_stage( stage_name="Source", actions=[ codepipeline_actions.CodeCommitSourceAction( action_name="CodeCommit_Source", repository=code, - trigger=codepipeline_actions.CodeCommitTrigger.NONE, # Created below + # Created rule below to give it a custom name + trigger=codepipeline_actions.CodeCommitTrigger.NONE, event_role=event_role, output=source_output, branch=branch_name, @@ -116,7 +117,7 @@ def __init__( ], ) - build_stage = code_pipeline.add_stage( + _ = code_pipeline.add_stage( stage_name="Build", actions=[ codepipeline_actions.CodeBuildAction( @@ -132,7 +133,7 @@ def __init__( ) staging_stack_name = f"sagemaker-{project_name}-{construct_id}-staging" - staging_deploy_stage = code_pipeline.add_stage( + _ = code_pipeline.add_stage( stage_name="DeployStaging", actions=[ codepipeline_actions.CloudFormationCreateUpdateStackAction( @@ -155,7 +156,7 @@ def __init__( ), ], ) - production_deploy_stage = code_pipeline.add_stage( + _ = code_pipeline.add_stage( stage_name="DeployProd", actions=[ codepipeline_actions.CloudFormationCreateUpdateStackAction( @@ -171,6 +172,7 @@ def __init__( replace_on_failure=True, ), codepipeline_actions.CloudFormationDeleteStackAction( + action_name="Delete_CFN_Staging", stack_name=staging_stack_name, admin_permissions=False, deployment_role=cloudformation_role, @@ -183,7 +185,8 @@ def __init__( self, "ModelRegistryRule", rule_name=f"sagemaker-{project_name}-modelregistry-{construct_id}", - description="Rule to trigger a deployment when SageMaker Model registry is updated with a new model package.", + description="Rule to trigger a deployment when SageMaker Model registry " + "is updated with a new model package.", event_pattern=events.EventPattern( source=["aws.sagemaker"], detail_type=["SageMaker Model Package State Change"], @@ -206,7 +209,8 @@ def __init__( self, "CodeCommitRule", rule_name=f"sagemaker-{project_name}-codecommit-{construct_id}", - description="Rule to trigger a deployment when configuration is updated in CodeCommit.", + description="Rule to trigger a deployment when configuration is updated " + "in CodeCommit.", event_pattern=events.EventPattern( source=["aws.codecommit"], detail_type=["CodeCommit Repository State Change"], From b451238b82c9fa4bb5077bc831f97b140acc9101 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Tue, 31 Jan 2023 15:01:39 +0800 Subject: [PATCH 09/17] Moved retrain rule to deployment stack --- batch_pipeline/infra/batch_config.py | 2 +- .../infra/deployment_config.py | 8 +- .../infra/sagemaker_lineage_utils.py | 47 +++++++++ .../infra/sagemaker_pipelines_event_target.py | 45 +++++++++ ...gemaker_service_catalog_roles_construct.py | 95 +++++++++++++++++++ deployment_pipeline/infra/sagemaker_stack.py | 50 +++++++++- infra/build_pipeline_construct.py | 35 +------ infra/deploy_pipeline_construct.py | 3 +- infra/pipeline_product_stack.py | 6 +- infra/service_catalog_stack.py | 37 ++++++++ 10 files changed, 284 insertions(+), 44 deletions(-) create mode 100644 deployment_pipeline/infra/sagemaker_lineage_utils.py create mode 100644 deployment_pipeline/infra/sagemaker_pipelines_event_target.py create mode 100644 deployment_pipeline/infra/sagemaker_service_catalog_roles_construct.py diff --git a/batch_pipeline/infra/batch_config.py b/batch_pipeline/infra/batch_config.py index e636558..380e8aa 100644 --- a/batch_pipeline/infra/batch_config.py +++ b/batch_pipeline/infra/batch_config.py @@ -35,7 +35,7 @@ def __init__( self.model_package_version = model_package_version self.model_package_arn = model_package_arn self.model_monitor_enabled = model_monitor_enabled - if type(drift_config) is dict: + if isinstance(drift_config, dict): self.drift_config = DriftConfig(**drift_config) else: self.drift_config = None diff --git a/deployment_pipeline/infra/deployment_config.py b/deployment_pipeline/infra/deployment_config.py index eded421..c255edc 100644 --- a/deployment_pipeline/infra/deployment_config.py +++ b/deployment_pipeline/infra/deployment_config.py @@ -7,7 +7,7 @@ def __init__(self, instance_count: int = 1, instance_type: str = "ml.t2.medium") class VariantConfig(InstanceConfig): def __init__( self, - model_package_version: str, + model_package_version: str = None, initial_variant_weight: float = 1.0, variant_name: str = None, instance_count: int = 1, @@ -72,7 +72,7 @@ def __init__( schedule_config: ScheduleConfig = None, ): self.stage_name = stage_name - if type(variant_config) is dict: + if isinstance(variant_config, dict): self.variant_config = VariantConfig( **{ "instance_count": instance_count, @@ -82,11 +82,11 @@ def __init__( ) else: self.variant_config = None - if type(auto_scaling) is dict: + if isinstance(auto_scaling, dict): self.auto_scaling = AutoScalingConfig(**auto_scaling) else: self.auto_scaling = None - if type(schedule_config) is dict: + if isinstance(schedule_config, dict): self.schedule_config = ScheduleConfig(**schedule_config) else: self.schedule_config = None diff --git a/deployment_pipeline/infra/sagemaker_lineage_utils.py b/deployment_pipeline/infra/sagemaker_lineage_utils.py new file mode 100644 index 0000000..0463190 --- /dev/null +++ b/deployment_pipeline/infra/sagemaker_lineage_utils.py @@ -0,0 +1,47 @@ +import logging +import os + +import aws_cdk as cdk +import boto3 +import sagemaker +from sagemaker.lineage.artifact import Artifact, ModelArtifact +from sagemaker.lineage.context import Context, EndpointContext + +logger = logging.getLogger(__name__) + +region = os.getenv("AWS_REGION") +boto_session = boto3.Session(region_name=region) +sagemaker_session = sagemaker.Session( + boto_session=boto_session, +) + + +def get_pipeline_arn_from_endpoint(endpoint_name: str): + logger.info(f"Endpoint: {endpoint_name}") + endpoint_arn = f"arn:aws:sagemaker:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:endpoint/{endpoint_name}" + + contexts = Context.list( + source_uri=endpoint_arn, + sagemaker_session=sagemaker_session, + ) + context_name = list(contexts)[0].context_name + endpoint_context = EndpointContext.load(context_name=context_name) + pipeline_arn = endpoint_context.pipeline_execution_arn().split("/execution")[0] + + logger.info(f"It was created by an execution of SageMaker Pipeline {pipeline_arn}") + + return pipeline_arn + + +def get_pipeline_arn_from_model(model_package_arn: str): + logger.info(f"Model version ARN: {model_package_arn}") + model_artifact_summary = list(Artifact.list(source_uri=model_package_arn))[0] + model_artifact = ModelArtifact.load( + artifact_arn=model_artifact_summary.artifact_arn + ) + + pipeline_arn = model_artifact.pipeline_execution_arn().split("/execution")[0] + + logger.info(f"It was created by an execution of SageMaker Pipeline {pipeline_arn}") + + return pipeline_arn diff --git a/deployment_pipeline/infra/sagemaker_pipelines_event_target.py b/deployment_pipeline/infra/sagemaker_pipelines_event_target.py new file mode 100644 index 0000000..540bee0 --- /dev/null +++ b/deployment_pipeline/infra/sagemaker_pipelines_event_target.py @@ -0,0 +1,45 @@ +import aws_cdk as cdk +from aws_cdk import aws_events as events +from aws_cdk import aws_iam as iam + + +def add_sagemaker_pipeline_target( + rule: events.Rule, + event_role: iam.Role, + sagemaker_pipeline_arn: str, + pipeline_parameters: dict = None, + target_id: str = None, +) -> None: + """ + [custom target](https://github.com/aws/aws-cdk/issues/14887) + + Args: + rule (events.Rule): The event rule to add Target + event_role (iam.Role): The event role + sagemaker_pipeline_arn (str): The SageMaker Pipeline ARN + pipeline_parameters (dict): dictionary with the pipeline parameters + """ + if target_id is None: + target_id = cdk.Fn.split( + delimiter="/", + source=sagemaker_pipeline_arn, + assumed_length=2, + )[1] + + sagemaker_pipeline_target = { + "Arn": sagemaker_pipeline_arn, + "Id": target_id, + "RoleArn": event_role.role_arn, + } + if pipeline_parameters is not None: + parameters_list = [ + {"Name": k, "Value": o} for k, o in pipeline_parameters.items() + ] + + sagemaker_pipeline_target = { + **sagemaker_pipeline_target, + "SageMakerPipelineParameters": {"PipelineParameterList": parameters_list}, + } + + child = rule.node.default_child + child.add_property_override("Targets", [sagemaker_pipeline_target]) diff --git a/deployment_pipeline/infra/sagemaker_service_catalog_roles_construct.py b/deployment_pipeline/infra/sagemaker_service_catalog_roles_construct.py new file mode 100644 index 0000000..7f791f8 --- /dev/null +++ b/deployment_pipeline/infra/sagemaker_service_catalog_roles_construct.py @@ -0,0 +1,95 @@ +import aws_cdk as cdk +import aws_cdk.aws_iam as iam +from constructs import Construct + + +class SageMakerSCRoles(Construct): + def __init__( + self, scope: Construct, construct_id: str, mutable: bool = True, **kwargs + ): + super().__init__(scope, construct_id, **kwargs) + + self.execution_role = iam.Role.from_role_arn( + self, + "SMModelDeploymentRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsExecutionRole" + ), + mutable=mutable, + ) + + self.events_role = iam.Role.from_role_arn( + self, + "SMEventsRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsEventsRole" + ), + mutable=mutable, + ) + + self.code_build_role = iam.Role.from_role_arn( + self, + "SMCodeBuildRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsCodeBuildRole" + ), + mutable=mutable, + ) + + self.code_pipeline_role = iam.Role.from_role_arn( + self, + "SMCodePipelineRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsCodePipelineRole" + ), + mutable=mutable, + ) + + self.lambda_role = iam.Role.from_role_arn( + self, + "SMLambdaRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsLambdaRole" + ), + mutable=mutable, + ) + + self.api_gw_role = iam.Role.from_role_arn( + self, + "SMApiGatewayRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsApiGatewayRole" + ), + mutable=mutable, + ) + + self.firehose_role = iam.Role.from_role_arn( + self, + "SMFirehoseRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsFirehoseRole" + ), + mutable=mutable, + ) + + self.glue_role = iam.Role.from_role_arn( + self, + "SMGlueRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsGlueRole" + ), + mutable=mutable, + ) + + self.cloudformation_role = iam.Role.from_role_arn( + self, + "SMCloudformationRole", + role_arn=format_role( + role_name="AmazonSageMakerServiceCatalogProductsCloudformationRole" + ), + mutable=mutable, + ) + + +def format_role(role_name: str): + return f"arn:aws:iam::{cdk.Aws.ACCOUNT_ID}:role/service-role/{role_name}" diff --git a/deployment_pipeline/infra/sagemaker_stack.py b/deployment_pipeline/infra/sagemaker_stack.py index 0c5b252..29dd908 100644 --- a/deployment_pipeline/infra/sagemaker_stack.py +++ b/deployment_pipeline/infra/sagemaker_stack.py @@ -3,9 +3,14 @@ import aws_cdk as cdk from aws_cdk import aws_applicationautoscaling as applicationautoscaling from aws_cdk import aws_cloudwatch as cloudwatch +from aws_cdk import aws_events as events from aws_cdk import aws_sagemaker as sagemaker from constructs import Construct +from infra.endpoint_lineage import get_pipeline_arn +from infra.sagemaker_pipelines_event_target import add_sagemaker_pipeline_target +from infra.sagemaker_service_catalog_roles_construct import SageMakerSCRoles + logger = logging.getLogger(__name__) @@ -25,7 +30,8 @@ def __init__( ) -> None: super().__init__(scope, construct_id, **kwargs) - # Select the variant config and name - needs to be same for updating Endpoint or Autoscaling deregister fails + # Select the variant config and name - needs to be same for updating Endpoint + # or Autoscaling deregister fails # see: https://docs.aws.amazon.com/sagemaker/latest/dg/endpoint-scaling.html variant_config = deployment_config.variant_config variant_name = variant_config.variant_name or "LatestApproved" @@ -141,7 +147,7 @@ def __init__( ), tags=tags, ) - monitoring_schedule.add_depends_on(endpoint) + monitoring_schedule.add_dependency(endpoint) drift_alarm = cloudwatch.CfnAlarm( self, @@ -165,7 +171,41 @@ def __init__( datapoints_to_alarm=deployment_config.schedule_config.datapoints_to_alarm, statistic=deployment_config.schedule_config.statistic, ) - drift_alarm.add_depends_on(monitoring_schedule) + drift_alarm.add_dependency(monitoring_schedule) + + ### add rule to run build pipeline + # Run the pipeline if data drift is detected + drift_rule_name = f"sagemaker-{endpoint_name}-drift-{construct_id}" + drift_rule = events.Rule( + self, + "DriftRule", + enabled=True, + description="Rule to start SM pipeline when drift has been detected.", + rule_name=drift_rule_name, + event_pattern=events.EventPattern( + source=["aws.cloudwatch"], + detail_type=["CloudWatch Alarm State Change"], + detail={ + "alarmName": [ + drift_alarm.alarm_name, + ], + "state": {"value": ["ALARM"]}, + }, + ), + ) + + sm_roles = SageMakerSCRoles(self, "SmRoles", mutable=False) + event_role = sm_roles.events_role + execution_role = sm_roles.execution_role + pipeline_arn = get_pipeline_arn( + endpoint_name=endpoint.attr_endpoint_name, + sagemaker_execution_role=execution_role.role_arn, + ) + add_sagemaker_pipeline_target( + drift_rule, + event_role=event_role, + sagemaker_pipeline_arn=pipeline_arn, + ) if deployment_config.auto_scaling is not None: resource_id = f"endpoint/{endpoint_name}/variant/{variant_name}" @@ -180,7 +220,7 @@ def __init__( scalable_dimension="sagemaker:variant:DesiredInstanceCount", service_namespace="sagemaker", ) - scalable_target.add_depends_on(endpoint) + scalable_target.add_dependency(endpoint) scaling_policy = applicationautoscaling.CfnScalingPolicy( self, @@ -199,7 +239,7 @@ def __init__( ), ), ) - scaling_policy.add_depends_on(scalable_target) + scaling_policy.add_dependency(scalable_target) # TODO: Add cloud watch alarm diff --git a/infra/build_pipeline_construct.py b/infra/build_pipeline_construct.py index e7dba64..0a18946 100644 --- a/infra/build_pipeline_construct.py +++ b/infra/build_pipeline_construct.py @@ -186,7 +186,8 @@ def __init__( deployment_success_rule = pipeline_deploy_stage.on_state_change( name="Start pipeline", rule_name=build_rule_name, - description="Rule to execute the Model Build pipeline once the pipeline has been deployed", + description="Rule to execute the Model Build pipeline once " + "the pipeline has been deployed", schedule=events.Schedule.expression(retrain_schedule), event_pattern=events.EventPattern( source=["aws.codepipeline"], @@ -205,33 +206,6 @@ def __init__( sagemaker_pipeline_arn=sagemaker_pipeline_arn, ) - # Run the pipeline if data drift is detected - drift_rule = events.Rule( - self, - "DriftRule", - enabled=True, - description="Rule to start SM pipeline when drift has been detected.", - rule_name=drift_rule_name, - event_pattern=events.EventPattern( - source=["aws.cloudwatch"], - detail_type=["CloudWatch Alarm State Change"], - detail={ - "alarmName": [ - f"sagemaker-{project_name}-staging-threshold", - f"sagemaker-{project_name}-prod-threshold", - f"sagemaker-{project_name}-batch-staging-threshold", - f"sagemaker-{project_name}-batch-prod-threshold", - ], - "state": {"value": ["ALARM"]}, - }, - ), - ) - - add_sagemaker_pipeline_target( - drift_rule, - event_role=event_role, - sagemaker_pipeline_arn=sagemaker_pipeline_arn, - ) # Load the lambda pipeline change code with open("lambda/build/lambda_pipeline_change.py", encoding="utf8") as fp: @@ -255,8 +229,6 @@ def __init__( }, ) - # Add permissions to put job status (if we want to call this directly within CodePipeline) - # see: https://docs.aws.amazon.com/codepipeline/latest/userguide/approvals-iam-permissions.html lambda_pipeline_change.add_to_role_policy( iam.PolicyStatement( actions=[ @@ -282,7 +254,8 @@ def __init__( self, "SagemakerPipelineRule", rule_name=f"sagemaker-{project_name}-sagemakerpipeline-{construct_id}", - description="Rule to enable/disable SM pipeline triggers when a SageMaker Model Building Pipeline is in progress.", + description="Rule to enable/disable SM pipeline triggers when a " + "SageMaker Model Building Pipeline is in progress.", event_pattern=events.EventPattern( source=["aws.sagemaker"], detail_type=[ diff --git a/infra/deploy_pipeline_construct.py b/infra/deploy_pipeline_construct.py index ef284f4..a89b81e 100644 --- a/infra/deploy_pipeline_construct.py +++ b/infra/deploy_pipeline_construct.py @@ -176,7 +176,8 @@ def __init__( self, "ModelRegistryRule", rule_name=f"sagemaker-{project_name}-modelregistry-{construct_id}", - description="Rule to trigger a deployment when SageMaker Model registry is updated with a new model package.", + description="Rule to trigger a deployment when SageMaker Model registry " + "is updated with a new model package.", event_pattern=events.EventPattern( source=["aws.sagemaker"], detail_type=["SageMaker Model Package State Change"], diff --git a/infra/pipeline_product_stack.py b/infra/pipeline_product_stack.py index a07d4b1..b7158e3 100644 --- a/infra/pipeline_product_stack.py +++ b/infra/pipeline_product_stack.py @@ -70,13 +70,15 @@ def __init__( branch_name = "main" if build_pipeline: - # Require a schedule parameter (must be cron, otherwise will trigger every time rate is enabled/disabled) + # Require a schedule parameter (must be cron, otherwise will trigger every + # time rate is enabled/disabled) # https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html retrain_schedule = cdk.CfnParameter( self, "RetrainSchedule", type="String", - description="The expression to retrain schedule. Defaults to first day of the month.", + description="The expression to retrain schedule. Defaults to first " + "day of the month.", default="cron(0 12 1 * ? *)", # 1st of the month at 12am min_length=1, ) diff --git a/infra/service_catalog_stack.py b/infra/service_catalog_stack.py index 6281506..6b53a39 100644 --- a/infra/service_catalog_stack.py +++ b/infra/service_catalog_stack.py @@ -158,6 +158,43 @@ def __init__( resources=["*"], ) ) + cloudformation_role.add_to_principal_policy( + iam.PolicyStatement( + actions=[ + "events:DescribeRule", + "events:PutRule", + "events:DeleteRule", + "events:PutTargets", + "events:RemoveTargets", + "events:ListTargetsByRule", + "events:ListRuleNamesByTarget", + ], + resources=[ + "*", + self.format_arn( + resource="rule", service="events", resource_name="sagemaker*" + ), + ], + ) + ) + + cloudformation_role.add_to_principal_policy( + iam.PolicyStatement( + actions=[ + "iam:PassRole", + ], + resources=[ + "*", + self.format_arn( + resource="role", + service="iam", + resource_name="service-role/AmazonSageMakerServiceCatalogProductsEventsRole", + region="", + account="*", + ), + ], + ) + ) # Add permissions to start SM pipelines to the Event Role event_role = sm_roles.events_role From 39286dd9f4bb079c9d1d4fcb2e6c15dc8c9639d5 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Tue, 31 Jan 2023 15:24:12 +0800 Subject: [PATCH 10/17] refining merge --- deployment_pipeline/infra/sagemaker_stack.py | 11 ++--- infra/build_pipeline_construct.py | 48 +++++++++++--------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/deployment_pipeline/infra/sagemaker_stack.py b/deployment_pipeline/infra/sagemaker_stack.py index 29dd908..766a312 100644 --- a/deployment_pipeline/infra/sagemaker_stack.py +++ b/deployment_pipeline/infra/sagemaker_stack.py @@ -7,7 +7,7 @@ from aws_cdk import aws_sagemaker as sagemaker from constructs import Construct -from infra.endpoint_lineage import get_pipeline_arn +from infra.sagemaker_lineage_utils import get_pipeline_arn_from_model from infra.sagemaker_pipelines_event_target import add_sagemaker_pipeline_target from infra.sagemaker_service_catalog_roles_construct import SageMakerSCRoles @@ -196,10 +196,9 @@ def __init__( sm_roles = SageMakerSCRoles(self, "SmRoles", mutable=False) event_role = sm_roles.events_role - execution_role = sm_roles.execution_role - pipeline_arn = get_pipeline_arn( - endpoint_name=endpoint.attr_endpoint_name, - sagemaker_execution_role=execution_role.role_arn, + + pipeline_arn = get_pipeline_arn_from_model( + model_package_arn=variant_config.model_package_arn ) add_sagemaker_pipeline_target( drift_rule, @@ -229,7 +228,7 @@ def __init__( policy_type="TargetTrackingScaling", resource_id=resource_id, scalable_dimension="sagemaker:variant:DesiredInstanceCount", - service_namespace="sagemaker", # Note: This is different to scaling above + service_namespace="sagemaker", # Note: different to scaling above target_tracking_scaling_policy_configuration=applicationautoscaling.CfnScalingPolicy.TargetTrackingScalingPolicyConfigurationProperty( target_value=deployment_config.auto_scaling.target_value, scale_in_cooldown=deployment_config.auto_scaling.scale_in_cooldown, diff --git a/infra/build_pipeline_construct.py b/infra/build_pipeline_construct.py index 0a18946..31cf712 100644 --- a/infra/build_pipeline_construct.py +++ b/infra/build_pipeline_construct.py @@ -8,6 +8,7 @@ from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda as lambda_ from aws_cdk import aws_s3 as s3 +from aws_cdk.aws_codebuild import BuildEnvironmentVariable from constructs import Construct from infra.sagemaker_pipelines_event_target import add_sagemaker_pipeline_target @@ -99,23 +100,21 @@ def __init__( environment=codebuild.BuildEnvironment( build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_4, environment_variables={ - "SAGEMAKER_PROJECT_NAME": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PROJECT_NAME": BuildEnvironmentVariable( value=project_name ), - "SAGEMAKER_PROJECT_ID": codebuild.BuildEnvironmentVariable( - value=project_id - ), - "AWS_REGION": codebuild.BuildEnvironmentVariable(value=env.region), - "SAGEMAKER_PIPELINE_NAME": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PROJECT_ID": BuildEnvironmentVariable(value=project_id), + "AWS_REGION": BuildEnvironmentVariable(value=env.region), + "SAGEMAKER_PIPELINE_NAME": BuildEnvironmentVariable( value=pipeline_name, ), - "SAGEMAKER_PIPELINE_DESCRIPTION": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PIPELINE_DESCRIPTION": BuildEnvironmentVariable( value=pipeline_description, ), - "SAGEMAKER_PIPELINE_ROLE_ARN": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PIPELINE_ROLE_ARN": BuildEnvironmentVariable( value=sagemaker_execution_role.role_arn, ), - "ARTIFACT_BUCKET": codebuild.BuildEnvironmentVariable( + "ARTIFACT_BUCKET": BuildEnvironmentVariable( value=s3_artifact.bucket_name ), }, @@ -133,22 +132,22 @@ def __init__( pipeline_name=code_pipeline_name, ) - source_stage = code_pipeline.add_stage( + source_action = codepipeline_actions.CodeCommitSourceAction( + action_name="CodeCommit_Source", + repository=code, + # Created rule below to give it a custom name + trigger=codepipeline_actions.CodeCommitTrigger.NONE, + event_role=event_role, + output=source_output, + branch=branch_name, + role=code_pipeline_role, + ) + _ = code_pipeline.add_stage( stage_name="Source", - actions=[ - codepipeline_actions.CodeCommitSourceAction( - action_name="CodeCommit_Source", - repository=code, - trigger=codepipeline_actions.CodeCommitTrigger.NONE, # Created below - event_role=event_role, - output=source_output, - branch=branch_name, - role=code_pipeline_role, - ) - ], + actions=[source_action], ) - build_stage = code_pipeline.add_stage( + _ = code_pipeline.add_stage( stage_name="Build", actions=[ codepipeline_actions.CodeBuildAction( @@ -160,6 +159,11 @@ def __init__( pipeline_build_output, ], role=code_pipeline_role, + environment_variables={ + "COMMIT_ID": BuildEnvironmentVariable( + value=source_action.variables.commit_id, + ), + }, ), ], ) From b731c2f4ff36feb17dda16acbb4d9b101dfa18ad Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Sat, 11 Feb 2023 09:36:36 +0100 Subject: [PATCH 11/17] Updated requirements.txt --- poetry.lock | 166 +++++++++++++++++++++++------------------------ pyproject.toml | 2 +- requirements.txt | 15 ++--- 3 files changed, 89 insertions(+), 94 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7113afe..8dc3009 100644 --- a/poetry.lock +++ b/poetry.lock @@ -21,18 +21,18 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy [[package]] name = "aws-cdk-asset-awscli-v1" -version = "2.2.49" +version = "2.2.64" description = "A library that contains the AWS CLI for use in Lambda Layers" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-awscli-v1-2.2.49.tar.gz", hash = "sha256:d367da8bdc83357792b1ef16b6166d400ef15f2389cf0032b607b6327768a41a"}, - {file = "aws_cdk.asset_awscli_v1-2.2.49-py3-none-any.whl", hash = "sha256:28df4487e2fa5314d5c39c114e12d366714a1fab2de3269d55c4e544876cae44"}, + {file = "aws-cdk.asset-awscli-v1-2.2.64.tar.gz", hash = "sha256:75b290d7ad8b89ccd6fa732b0a2520c5bbdcd9ee6bf963df57d8cc310f23468f"}, + {file = "aws_cdk.asset_awscli_v1-2.2.64-py3-none-any.whl", hash = "sha256:fc64ad1ecfa2cb5a4cd9af47c676dc65c40c251d416bf4a7d61dc950989214b9"}, ] [package.dependencies] -jsii = ">=1.72.0,<2.0.0" +jsii = ">=1.74.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -55,39 +55,39 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-asset-node-proxy-agent-v5" -version = "2.0.38" +version = "2.0.53" description = "@aws-cdk/asset-node-proxy-agent-v5" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.38.tar.gz", hash = "sha256:eb80e0098899bd29e4f0938c68802cc022a8e39af8c05de4053f9a209e63bb7c"}, - {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.38-py3-none-any.whl", hash = "sha256:0add1debe24e566b8d2cf713ca85eb5e82c42de00f737dfbf02e18e22220a41b"}, + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.53.tar.gz", hash = "sha256:bb841585d2e889842bcdda49b9be9fa128058c1e1061b964dc3721a4b75080d5"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.53-py3-none-any.whl", hash = "sha256:5c5895a94f7c18de32fc3ae797578eb7bd967966cfed778b94d3574ac2b7ce33"}, ] [package.dependencies] -jsii = ">=1.72.0,<2.0.0" +jsii = ">=1.74.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.61.1" +version = "2.64.0" description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.61.1.tar.gz", hash = "sha256:d2bb672be182e0cd675717648fa100e1a49f01de606ffa4f7a0b580093f31b27"}, - {file = "aws_cdk_lib-2.61.1-py3-none-any.whl", hash = "sha256:e0b5d49c73be945ca176b38ff3d2ccfc9474639000afb6af65a1eaf8b6aa3385"}, + {file = "aws-cdk-lib-2.64.0.tar.gz", hash = "sha256:e21210765b362a0b8a7052fce7bd3a574ea5355a7e763e190c5051ee33b4868a"}, + {file = "aws_cdk_lib-2.64.0-py3-none-any.whl", hash = "sha256:e2de0a80eff201d2eb5326beffa5fb564231ae44eb08ed2b6d4da0c6a324e0e5"}, ] [package.dependencies] -"aws-cdk.asset-awscli-v1" = ">=2.2.49,<3.0.0" +"aws-cdk.asset-awscli-v1" = ">=2.2.52,<3.0.0" "aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" -"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.38,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.42,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.74.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -129,18 +129,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.55" +version = "1.26.69" description = "The AWS SDK for Python" -category = "main" +category = "dev" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.55-py3-none-any.whl", hash = "sha256:e8ae1567b0a21d410f9c8a16a17a8028445a52061482272b4a516f5de5abbd92"}, - {file = "boto3-1.26.55.tar.gz", hash = "sha256:d68576dcb1c520474eafb64b996661068a369f40bbd104ccb9502cad71849e57"}, + {file = "boto3-1.26.69-py3-none-any.whl", hash = "sha256:9a0a29179957cb26fa8c3c1fddf66b18efaeaf633e08db5fb53815ffb0421419"}, + {file = "boto3-1.26.69.tar.gz", hash = "sha256:eb8cde24a4c5755c35126e8cd460e6b51c63d04292419e7e95721232720c7e5b"}, ] [package.dependencies] -botocore = ">=1.29.55,<1.30.0" +botocore = ">=1.29.69,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -149,14 +149,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.55" +version = "1.29.69" description = "Low-level, data-driven core of boto 3." -category = "main" +category = "dev" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.55-py3-none-any.whl", hash = "sha256:dd8a0868b287015bed086c97b866f9bb8ca57737d15972d3ac773559f4088db8"}, - {file = "botocore-1.29.55.tar.gz", hash = "sha256:5fe6b8809292bb178a9378fea4f96bc969003d1048cf6de8ad3c5a73690fefa9"}, + {file = "botocore-1.29.69-py3-none-any.whl", hash = "sha256:2a4ab8bcb3177daa425019e125c09996b9a6a1a62bb0baaaeeb86ffd552719cc"}, + {file = "botocore-1.29.69.tar.gz", hash = "sha256:7e1bebca013544fbc298cb58603bfccd5f71b49c720a5c33c07cf5dfc8145a1f"}, ] [package.dependencies] @@ -165,7 +165,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.15.3)"] +crt = ["awscrt (==0.16.9)"] [[package]] name = "cattrs" @@ -212,18 +212,18 @@ files = [ [[package]] name = "constructs" -version = "10.1.229" +version = "10.1.247" description = "A programming model for software-defined state" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "constructs-10.1.229-py3-none-any.whl", hash = "sha256:b3479f52b0f4926c088a9c808f2cf3f4d80d7be850d5057f6a268db68b94a8e6"}, - {file = "constructs-10.1.229.tar.gz", hash = "sha256:87e9a0cffbb2573b288b1204aa9459bb21945e2c97b6d885618cd4a389954cd5"}, + {file = "constructs-10.1.247-py3-none-any.whl", hash = "sha256:809ed229a2404e72991698daf73f9fb6bcce6c0785e072420a82c80ecdfc7624"}, + {file = "constructs-10.1.247.tar.gz", hash = "sha256:1f99a59c1b54c4ffcee929b8b612841059e9d903a6fc5e778e9573217243842d"}, ] [package.dependencies] -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.74.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -309,7 +309,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -331,14 +331,14 @@ files = [ [[package]] name = "jsii" -version = "1.73.0" +version = "1.74.0" description = "Python client for jsii runtime" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "jsii-1.73.0-py3-none-any.whl", hash = "sha256:13e8496c3afee70d85401ad1eef2ddedbdb88e7e7abb3e68302dd6e61527191e"}, - {file = "jsii-1.73.0.tar.gz", hash = "sha256:be6458236e787be0b02c2fe869b6f4ed906398b6cc537190d61a60d2b5c9dfbb"}, + {file = "jsii-1.74.0-py3-none-any.whl", hash = "sha256:ee76781fe66106c367fbb3bb383db4f5e9b8ff3d3c4c0f34624c050211f040be"}, + {file = "jsii-1.74.0.tar.gz", hash = "sha256:575131396ad34f8f6e9f2604953ecbf4f3368625656a828b13089e4abb81b443"}, ] [package.dependencies] @@ -378,52 +378,52 @@ dill = ">=0.3.6" [[package]] name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.5" files = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] [[package]] name = "numpy" -version = "1.24.1" +version = "1.24.2" description = "Fundamental package for array computing in Python" category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, - {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, - {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, - {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, - {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, - {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, - {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, - {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, - {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, - {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, - {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, + {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, + {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, + {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, + {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, + {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, + {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, + {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, + {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, + {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, ] [[package]] @@ -507,31 +507,31 @@ ppft = ">=1.7.6.6" [[package]] name = "pathspec" -version = "0.10.3" +version = "0.11.0" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.10.3-py3-none-any.whl", hash = "sha256:3c95343af8b756205e2aba76e843ba9520a24dd84f68c22b9f93251507509dd6"}, - {file = "pathspec-0.10.3.tar.gz", hash = "sha256:56200de4077d9d0791465aa9095a01d421861e405b5096955051deefd697d6f6"}, + {file = "pathspec-0.11.0-py3-none-any.whl", hash = "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229"}, + {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, ] [[package]] name = "platformdirs" -version = "2.6.2" +version = "3.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, - {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, + {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, + {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pox" @@ -650,7 +650,7 @@ files = [ name = "s3transfer" version = "0.6.0" description = "An Amazon S3 Transfer Manager" -category = "main" +category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -666,13 +666,13 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "sagemaker" -version = "2.129.0" +version = "2.132.0" description = "Open source library for training and deploying models on Amazon SageMaker." category = "dev" optional = false python-versions = ">= 3.6" files = [ - {file = "sagemaker-2.129.0.tar.gz", hash = "sha256:ede424a2091c96211f3186c66f5679ba66f0da8a4692a0095e34ad4d2f0f3ae4"}, + {file = "sagemaker-2.132.0.tar.gz", hash = "sha256:2453ee5ae156f3fc852a81752ca548fcc3549a28028dff0e14aec6142c63274c"}, ] [package.dependencies] @@ -864,7 +864,7 @@ files = [ name = "urllib3" version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -904,21 +904,21 @@ scikit-learn = ["scikit-learn"] [[package]] name = "zipp" -version = "3.11.0" +version = "3.13.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, - {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, + {file = "zipp-3.13.0-py3-none-any.whl", hash = "sha256:e8b2a36ea17df80ffe9e2c4fda3f693c3dad6df1697d3cd3af232db680950b0b"}, + {file = "zipp-3.13.0.tar.gz", hash = "sha256:23f70e964bc11a34cef175bc90ba2914e1e4545ea1e3e2f67c079671883f9cb6"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "73c3cd44dc9cf4176865170f7d5ed76422d60ef23d2639b2057351e3c1e69c04" +content-hash = "e98bb3449cddaf12a49ec99df753cdb28bc958c51cbfaea5d55be8bd66e3a0a1" diff --git a/pyproject.toml b/pyproject.toml index 5e445be..b67590d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,6 @@ authors = ["y"] [tool.poetry.dependencies] python = "^3.9" -boto3="^1.18.14" aws-cdk-lib = "^2.38.0" constructs = "^10.1.79" @@ -18,6 +17,7 @@ black = {version = "^22.10.0", allow-prereleases = true} sagemaker = "^2.112.2" xgboost = "1.2" scikit-learn = "0.23" +boto3="^1.18.14" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/requirements.txt b/requirements.txt index 3876735..79093f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,14 @@ attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-awscli-v1==2.2.49 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-awscli-v1==2.2.64 ; python_version >= "3.9" and python_version < "4.0" aws-cdk-asset-kubectl-v20==2.1.1 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-node-proxy-agent-v5==2.0.38 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-lib==2.61.1 ; python_version >= "3.9" and python_version < "4.0" -boto3==1.26.55 ; python_version >= "3.9" and python_version < "4.0" -botocore==1.29.55 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-node-proxy-agent-v5==2.0.53 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-lib==2.64.0 ; python_version >= "3.9" and python_version < "4.0" cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -constructs==10.1.229 ; python_version >= "3.9" and python_version < "4.0" +constructs==10.1.247 ; python_version >= "3.9" and python_version < "4.0" exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11" -jmespath==1.0.1 ; python_version >= "3.9" and python_version < "4.0" -jsii==1.73.0 ; python_version >= "3.9" and python_version < "4.0" +jsii==1.74.0 ; python_version >= "3.9" and python_version < "4.0" publication==0.0.3 ; python_version >= "3.9" and python_version < "4.0" python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" -s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "4.0" six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" typeguard==2.13.3 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.4.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4.0" From cc78ebf2532400854e8b70627c8cf6532813ebe5 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 29 May 2023 13:03:14 +0800 Subject: [PATCH 12/17] Updated BatchTransformRepo --- app.py | 1 - batch_pipeline/.gitignore | 31 + batch_pipeline/app.py | 71 +- batch_pipeline/buildspec.yml | 2 +- batch_pipeline/cdk.json | 16 +- batch_pipeline/infra/model_registry.py | 27 +- .../infra/sagemaker_pipeline_stack.py | 81 +- batch_pipeline/pipelines/pipeline.py | 279 ++---- .../pipelines/postprocess_monitor_script.py | 22 +- batch_pipeline/pipelines/score.py | 3 +- batch_pipeline/poetry.lock | 934 ++++++++++++++++++ batch_pipeline/prod-config.json | 2 +- batch_pipeline/pyproject.toml | 23 + batch_pipeline/requirements.txt | 43 +- .../pipelines/preprocess/preprocess.py | 10 +- infra/batch_pipeline_construct.py | 182 ++-- infra/build_pipeline_construct.py | 56 +- infra/pipeline_product_stack.py | 1 + infra/service_catalog_stack.py | 54 +- poetry.lock | 422 +++++--- 20 files changed, 1705 insertions(+), 555 deletions(-) create mode 100644 batch_pipeline/.gitignore create mode 100644 batch_pipeline/poetry.lock create mode 100644 batch_pipeline/pyproject.toml diff --git a/app.py b/app.py index 60cc461..3418bcd 100644 --- a/app.py +++ b/app.py @@ -4,7 +4,6 @@ import aws_cdk as cdk -from infra.pipeline_product_stack import BatchPipelineStack, DeployPipelineStack from infra.service_catalog_stack import ServiceCatalogStack # Configure the logger diff --git a/batch_pipeline/.gitignore b/batch_pipeline/.gitignore new file mode 100644 index 0000000..1dec18d --- /dev/null +++ b/batch_pipeline/.gitignore @@ -0,0 +1,31 @@ +.DS_Store +*.swp +package-lock.json +__pycache__ +.pytest_cache +.env +.venv +*.egg-info +.vscode/ + +# CDK asset staging directory +.cdk.staging +cdk.out +dist/ + +# Layers +layers/python +layers/*.zip + +# Notebooks +.ipynb_checkpoints/ + +# Sample events or assets +events +assets + +# project development env +node_modules/* +node_modules +.vscode/* +notes.txt diff --git a/batch_pipeline/app.py b/batch_pipeline/app.py index cddb427..4fb4297 100644 --- a/batch_pipeline/app.py +++ b/batch_pipeline/app.py @@ -4,14 +4,11 @@ import logging import os -# Import the pipeline -from pipelines.pipeline import get_pipeline, upload_pipeline +import aws_cdk as cdk -from aws_cdk import core from infra.batch_config import BatchConfig -from infra.sagemaker_pipeline_stack import SageMakerPipelineStack from infra.model_registry import ModelRegistry - +from infra.sagemaker_pipeline_stack import SageMakerPipelineStack # Configure the logger logger = logging.getLogger(__name__) @@ -22,13 +19,11 @@ def create_pipeline( - app: core.App, + app: cdk.App, project_name: str, project_id: str, - region: str, sagemaker_pipeline_role_arn: str, artifact_bucket: str, - evaluate_drift_function_arn: str, stage_name: str, ): # Get the stage specific deployment config for sagemaker @@ -54,63 +49,25 @@ def create_pipeline( )[0] batch_config.model_package_arn = p["ModelPackageArn"] - # Set the default input data uri - data_uri = f"s3://{artifact_bucket}/{project_id}/batch/{stage_name}" - - # set the output transform uri - transform_uri = f"s3://{artifact_bucket}/{project_id}/transform/{stage_name}" - # Get the pipeline execution to get the baseline uri pipeline_execution_arn = registry.get_pipeline_execution_arn( batch_config.model_package_arn ) - logger.info(f"Got pipeline exection arn: {pipeline_execution_arn}") + logger.info(f"Got pipeline execution arn: {pipeline_execution_arn}") model_uri = registry.get_model_artifact(pipeline_execution_arn) logger.info(f"Got model uri: {model_uri}") - # Set the sagemaker pipeline name and descrption with model version + # Set the sagemaker pipeline name and description with model version sagemaker_pipeline_name = f"{project_name}-batch-{stage_name}" - sagemaker_pipeline_description = f"Batch Pipeline for {stage_name} model version: {batch_config.model_package_version}" - - # If we have drift configuration then get the baseline uri - baseline_uri = None - if batch_config.drift_config is not None: - baseline_uri = registry.get_data_check_baseline_uri(batch_config.model_package_arn) - logger.info(f"Got baseline uri: {baseline_uri}") - - # Create batch pipeline - pipeline = get_pipeline( - region=region, - role=sagemaker_pipeline_role_arn, - pipeline_name=sagemaker_pipeline_name, - default_bucket=artifact_bucket, - base_job_prefix=project_id, - evaluate_drift_function_arn=evaluate_drift_function_arn, - data_uri=data_uri, - model_uri=model_uri, - transform_uri=transform_uri, - baseline_uri=baseline_uri, - ) - - # Create the pipeline definition - logger.info("Creating/updating a SageMaker Pipeline for batch transform") - pipeline_definition_body = pipeline.definition() - parsed = json.loads(pipeline_definition_body) - logger.info(json.dumps(parsed, indent=2, sort_keys=True)) - - # Upload the pipeline to S3 bucket/key and return JSON with key/value for for Cfn Stack parameters. - # see: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sagemaker-pipeline.html - logger.info(f"Uploading {stage_name} pipeline to {artifact_bucket}") - pipeline_definition_key = upload_pipeline( - pipeline, - default_bucket=artifact_bucket, - base_job_prefix=f"{project_id}/batch-{stage_name}", + sagemaker_pipeline_description = ( + f"Batch Pipeline for {stage_name} model version:" + f"{batch_config.model_package_version}" ) tags = [ - core.CfnTag(key="sagemaker:deployment-stage", value=stage_name), - core.CfnTag(key="sagemaker:project-id", value=project_id), - core.CfnTag(key="sagemaker:project-name", value=project_name), + cdk.CfnTag(key="sagemaker:deployment-stage", value=stage_name), + cdk.CfnTag(key="sagemaker:project-id", value=project_id), + cdk.CfnTag(key="sagemaker:project-name", value=project_name), ] SageMakerPipelineStack( @@ -118,10 +75,10 @@ def create_pipeline( f"drift-batch-{stage_name}", pipeline_name=sagemaker_pipeline_name, pipeline_description=sagemaker_pipeline_description, - pipeline_definition_bucket=artifact_bucket, - pipeline_definition_key=pipeline_definition_key, sagemaker_role_arn=sagemaker_pipeline_role_arn, + default_bucket=artifact_bucket, tags=tags, + batch_config=batch_config, drift_config=batch_config.drift_config, ) @@ -135,7 +92,7 @@ def main( evaluate_drift_function_arn: str, ): # Create App and stacks - app = core.App() + app = cdk.App() create_pipeline( app=app, diff --git a/batch_pipeline/buildspec.yml b/batch_pipeline/buildspec.yml index 0c6d274..b3541f4 100644 --- a/batch_pipeline/buildspec.yml +++ b/batch_pipeline/buildspec.yml @@ -5,7 +5,7 @@ phases: nodejs: "16" python: "3.9" commands: - - npm install aws-cdk@2.62.2 + - npm install aws-cdk@2.72.1 - npm update - python -m pip install --upgrade pip - python -m pip install -r requirements.txt diff --git a/batch_pipeline/cdk.json b/batch_pipeline/cdk.json index 90e5f6e..ddbb4d5 100644 --- a/batch_pipeline/cdk.json +++ b/batch_pipeline/cdk.json @@ -1,13 +1,5 @@ { - "app": "python3 app.py", - "context": { - "@aws-cdk/core:enableStackNameDuplicates": "true", - "aws-cdk:enableDiffNoFail": "true", - "@aws-cdk/core:stackRelativeExports": "true", - "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true, - "@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true, - "@aws-cdk/aws-kms:defaultKeyPolicies": true, - "@aws-cdk/aws-s3:grantWriteWithoutAcl": true - } - } - \ No newline at end of file + "app": "python3 app.py", + "context": {}, + "versionReporting": false +} \ No newline at end of file diff --git a/batch_pipeline/infra/model_registry.py b/batch_pipeline/infra/model_registry.py index 919e532..dd0d33b 100644 --- a/batch_pipeline/infra/model_registry.py +++ b/batch_pipeline/infra/model_registry.py @@ -33,7 +33,7 @@ def create_model_package_group( ModelPackageGroupDescription=description, ) model_package_group_arn = response["ModelPackageGroupArn"] - # Add tags seperately + # Add tags separately self.sm_client.add_tags( ResourceArn=model_package_group_arn, Tags=[ @@ -84,7 +84,7 @@ def get_latest_approved_packages( "SortBy": "CreationTime", "MaxResults": max_results, } - # Add optional creationg time after + # Add optional creation time after if creation_time_after is not None: args = {**args, "CreationTimeAfter": creation_time_after} response = self.sm_client.list_model_packages(**args) @@ -165,7 +165,10 @@ def get_versioned_approved_packages( # Return error if no packages found if len(model_packages) == 0: - error_message = f"No approved packages found for: {model_package_group_name} and versions: {model_package_versions}" + error_message = ( + "No approved packages found for: " + f"{model_package_group_name} and versions: {model_package_versions}" + ) logger.error(error_message) raise Exception(error_message) @@ -182,7 +185,7 @@ def get_versioned_approved_packages( def select_versioned_packages( self, model_packages: list, model_package_versions: list ): - """Filters the model packages based on a list of model package verisons. + """Filters the model packages based on a list of model package versions. Args: model_packages: The list of packages. @@ -201,7 +204,7 @@ def select_versioned_packages( return filtered_packages def get_pipeline_execution_arn(self, model_package_arn: str): - """Geturns the execution arn for the latest approved model package + """Returns the execution arn for the latest approved model package Args: model_package_arn: The arn of the model package @@ -248,12 +251,18 @@ def get_model_artifact( def get_data_check_baseline_uri(self, model_package_arn: str): try: - model_details = self.sm_client.describe_model_package(ModelPackageName=model_package_arn) + model_details = self.sm_client.describe_model_package( + ModelPackageName=model_package_arn + ) print(model_details) - baseline_uri = model_details['DriftCheckBaselines']['ModelDataQuality']['Constraints']['S3Uri'] - baseline_uri = baseline_uri.replace('/constraints.json','') # returning the folder containing constraints and statistics + baseline_uri = model_details["DriftCheckBaselines"]["ModelDataQuality"][ + "Constraints" + ]["S3Uri"] + baseline_uri = baseline_uri.replace( + "/constraints.json", "" + ) # returning the folder containing constraints and statistics return baseline_uri except ClientError as e: error_message = e.response["Error"]["Message"] logger.error(error_message) - raise Exception(error_message) \ No newline at end of file + raise Exception(error_message) diff --git a/batch_pipeline/infra/sagemaker_pipeline_stack.py b/batch_pipeline/infra/sagemaker_pipeline_stack.py index dfcd146..f99b07f 100644 --- a/batch_pipeline/infra/sagemaker_pipeline_stack.py +++ b/batch_pipeline/infra/sagemaker_pipeline_stack.py @@ -1,15 +1,13 @@ -from aws_cdk import ( - core, - aws_cloudwatch as cloudwatch, - aws_events as events, - aws_iam as iam, - aws_sagemaker as sagemaker, -) - import logging -import os -from urllib.parse import urlparse -from batch_config import DriftConfig + +import aws_cdk as cdk +from aws_cdk import aws_cloudwatch as cloudwatch +from aws_cdk import aws_sagemaker as sagemaker +from constructs import Construct +from pipelines.pipeline import get_pipeline + + +from infra.batch_config import DriftConfig, BatchConfig logger = logging.getLogger(__name__) @@ -17,33 +15,49 @@ # see: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_sagemaker/CfnPipeline.html -class SageMakerPipelineStack(core.Stack): +class SageMakerPipelineStack(cdk.Stack): def __init__( self, - scope: core.Construct, + scope: Construct, construct_id: str, pipeline_name: str, pipeline_description: str, - pipeline_definition_bucket: str, - pipeline_definition_key: str, sagemaker_role_arn: str, + default_bucket: str, tags: list, + batch_config: BatchConfig, drift_config: DriftConfig, **kwargs, ) -> None: super().__init__(scope, construct_id, **kwargs) + # Do not use a custom named resource for models as these get replaced + model = sagemaker.CfnModel( + self, + "Model", + execution_role_arn=sagemaker_role_arn, + primary_container=sagemaker.CfnModel.ContainerDefinitionProperty( + model_package_name=batch_config.model_package_arn, + ), + model_name=f"{batch_config.model_package_arn.split('/', 1)[1].replace('/', '-')}-{batch_config.stage_name}", + ) + + pipeline_definition = get_pipeline( + role=sagemaker_role_arn, + pipeline_name=pipeline_name, + default_bucket=default_bucket, + model_package_arn=batch_config.model_package_arn, + default_model_name=model.model_name, + ).definition() + + print(model.model_name) + sagemaker.CfnPipeline( self, "Pipeline", pipeline_name=pipeline_name, pipeline_description=pipeline_description, - pipeline_definition={ - "PipelineDefinitionS3Location": { - "Bucket": pipeline_definition_bucket, - "Key": pipeline_definition_key, - } - }, + pipeline_definition={"PipelineDefinitionBody": pipeline_definition}, role_arn=sagemaker_role_arn, tags=tags, ) @@ -51,22 +65,23 @@ def __init__( if drift_config is not None: # Create a CW alarm (which will be picked up by build pipeline) alarm_name = f"sagemaker-{pipeline_name}-threshold" - cloudwatch.CfnAlarm( + + drift_metric = cloudwatch.Metric( + metric_name=drift_config.metric_name, + namespace="aws/sagemaker/ModelBuildingPipeline/data-metrics", + dimensions_map={"PipelineName": pipeline_name}, + statistic=cloudwatch.Stats.AVERAGE, + period=cdk.Duration.minutes(drift_config.period), + ) + + cloudwatch.Alarm( self, "DriftAlarm", alarm_name=alarm_name, - alarm_description=f"Batch Drift Threshold", - metric_name=drift_config.metric_name, + alarm_description="Batch Drift Threshold", + metric=drift_metric, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, threshold=drift_config.metric_threshold, - namespace="aws/sagemaker/ModelBuildingPipeline/data-metrics", - comparison_operator=drift_config.comparison_operator, - dimensions=[ - cloudwatch.CfnAlarm.DimensionProperty( - name="PipelineName", value=pipeline_name - ), - ], evaluation_periods=drift_config.evaluation_periods, - period=drift_config.period, datapoints_to_alarm=drift_config.datapoints_to_alarm, - statistic=drift_config.statistic, ) diff --git a/batch_pipeline/pipelines/pipeline.py b/batch_pipeline/pipelines/pipeline.py index 81b2ead..38e4d50 100644 --- a/batch_pipeline/pipelines/pipeline.py +++ b/batch_pipeline/pipelines/pipeline.py @@ -7,116 +7,48 @@ Implements a get_pipeline(**kwargs) method. """ -import json import os -import boto3 -import sagemaker -import sagemaker.session - -from sagemaker.inputs import CreateModelInput -from sagemaker.model import Model +# from sagemaker.model import ModelPackage from sagemaker.model_monitor.dataset_format import DatasetFormat -from sagemaker.processing import ( - ProcessingInput, - ProcessingOutput, - Processor, - ScriptProcessor, -) from sagemaker.s3 import S3Uploader -from sagemaker.workflow.lambda_step import ( - LambdaStep, - LambdaOutput, - LambdaOutputTypeEnum, -) -from sagemaker.lambda_helper import Lambda +from sagemaker.transformer import Transformer +from sagemaker.utils import name_from_base +from sagemaker.workflow.check_job_config import CheckJobConfig +from sagemaker.workflow.execution_variables import ExecutionVariables +from sagemaker.workflow.functions import Join + +# from sagemaker.workflow.model_step import ModelStep +from sagemaker.workflow.monitor_batch_transform_step import MonitorBatchTransformStep from sagemaker.workflow.parameters import ( + ParameterBoolean, ParameterInteger, ParameterString, ) from sagemaker.workflow.pipeline import Pipeline from sagemaker.workflow.pipeline_context import PipelineSession -from sagemaker.workflow.steps import ( - CreateModelStep, - ProcessingStep, - CacheConfig, -) -from sagemaker.workflow.quality_check_step import ( - DataQualityCheckConfig, - ModelQualityCheckConfig, - QualityCheckStep, -) -from sagemaker.drift_check_baselines import DriftCheckBaselines -from sagemaker.workflow.check_job_config import CheckJobConfig -from sagemaker.workflow.step_collections import RegisterModel -from sagemaker.workflow.functions import Join -from sagemaker.workflow.execution_variables import ExecutionVariables -from sagemaker.utils import name_from_base - +from sagemaker.workflow.quality_check_step import DataQualityCheckConfig +from sagemaker.workflow.steps import CacheConfig BASE_DIR = os.path.dirname(os.path.realpath(__file__)) -def get_session(region, default_bucket): - """Gets the sagemaker session based on the region. - Args: - region: the aws region to start the session - default_bucket: the bucket to use for storing the artifacts - Returns: - `sagemaker.session.Session instance - """ - - boto_session = boto3.Session(region_name=region) - - sagemaker_client = boto_session.client("sagemaker") - return PipelineSession( - boto_session=boto_session, - sagemaker_client=sagemaker_client, - default_bucket=default_bucket, - ) - - def get_pipeline( - region: str, role: str, pipeline_name: str, default_bucket: str, - base_job_prefix: str, - evaluate_drift_function_arn: str, - data_uri: str, - model_uri: str, - transform_uri: str, - baseline_uri: str = None, + model_package_arn: str, + data_uri: str = None, + default_model_name: str = None, ) -> Pipeline: - """Gets a SageMaker ML Pipeline instance working with on nyc taxi data. - Args: - region: AWS region to create and run the pipeline. - role: IAM role to create and run steps and pipeline. - pipeline_name: the bucket to use for storing the artifacts - default_bucket: the bucket to use for storing the artifacts - base_job_prefix: the prefix to include after the bucket - data_uri: the input data location - model_uri: the input model location - transform_uri: the output transform uri location - baseline_uri: optional input baseline uri for drift detection - Returns: - an instance of a pipeline - """ - sagemaker_session = get_session(region, default_bucket) + """Gets a SageMaker ML Pipeline instance.""" + sagemaker_session = PipelineSession() - # parameters for pipeline execution + # Parameters input_data_uri = ParameterString( name="DataInputUri", default_value=data_uri, ) - input_model_uri = ParameterString( - name="ModelInputUri", - default_value=model_uri, - ) - output_transform_uri = ParameterString( - name="TransformOutputUri", - default_value=transform_uri, - ) transform_instance_count = ParameterInteger( name="TransformInstanceCount", default_value=1 ) @@ -129,126 +61,99 @@ def get_pipeline( monitor_instance_type = ParameterString( name="MonitorInstanceType", default_value="ml.m5.xlarge" ) - - # Create cache configuration (Unable to pass parameter for expire_after value) + stop_if_check_fails = ParameterBoolean(name="StopIfCheckFails", default_value=False) + model_name = ParameterString(name="ModelName", default_value=default_model_name) + + parameters = [ + model_name, + input_data_uri, + transform_instance_count, + transform_instance_type, + monitor_instance_count, + monitor_instance_type, + stop_if_check_fails, + ] + + output_common_path = [ + "s3:/", + default_bucket, + "batch-transform-runs", + Join( + on="-", + values=[ + ExecutionVariables.START_DATETIME, + ExecutionVariables.PIPELINE_EXECUTION_ID, + ], + ), + ] + + model_package_group_name = model_package_arn.split("/", -1)[-2] + + # Cache configuration (Unable to pass parameter for expire_after value) cache_config = CacheConfig(enable_caching=True, expire_after="PT1H") - # Create the Model step - image_uri_inference = sagemaker.image_uris.retrieve( - framework="xgboost", - region=region, - version="1.2-2", - py_version="py3", - instance_type=transform_instance_type, - ) - - model = Model( - image_uri=image_uri_inference, - model_data=input_model_uri, - sagemaker_session=sagemaker_session, + # Data Quality Check configuration + check_job_config = CheckJobConfig( role=role, + instance_count=monitor_instance_count, + instance_type=monitor_instance_type, ) - - inputs_model = CreateModelInput(instance_type=transform_instance_type) - - step_create_model = CreateModelStep( - name="CreateModel", model=model, inputs=inputs_model + data_quality_config = DataQualityCheckConfig( + baseline_dataset=input_data_uri, + dataset_format=DatasetFormat.csv(header=False), + output_s3_uri=Join( + on="/", + values=output_common_path + ["dataqualitycheck"], + ), + post_analytics_processor_script="pipelines/postprocess_monitor_script.py" ) - # processing step for evaluation - script_eval = ScriptProcessor( - image_uri=image_uri_inference, - command=["python3"], + # Transform Step arguments + transformer = Transformer( + model_name=model_name, instance_count=transform_instance_count, instance_type=transform_instance_type, - base_job_name=f"{base_job_prefix}/script-score", + accept="text/csv", + assemble_with="Line", + output_path=Join( + on="/", + values=output_common_path + ["transformed"], + ), sagemaker_session=sagemaker_session, - role=role, ) - step_score = ProcessingStep( - name="ScoreModel", - processor=script_eval, - inputs=[ - ProcessingInput( - source=input_model_uri, - destination="/opt/ml/processing/model", - ), - ProcessingInput( - source=input_data_uri, - destination="/opt/ml/processing/input", - ), - ], - outputs=[ - ProcessingOutput(output_name="scores", source="/opt/ml/processing/output"), - ], - code=os.path.join(BASE_DIR, "score.py"), - cache_config=cache_config, + transform_arg = transformer.transform( + input_data_uri, + content_type="text/csv", + split_type="Line", + # exclude the ground truth (first column) from the validation set + # when doing inference. + input_filter="$[1:]", ) - step_score.add_depends_on([step_create_model]) - - steps = [step_create_model, step_score] - - if baseline_uri is not None: - check_job_config = CheckJobConfig( - role=role, - instance_count=monitor_instance_count, - instance_type=monitor_instance_type, - sagemaker_session=sagemaker_session, - base_job_name=f"{base_job_prefix}/monitoring", - env = { - "PipelineName": pipeline_name, - "Region": region, - } - ) - data_quality_check_config = DataQualityCheckConfig( - baseline_dataset=step_score.properties.ProcessingOutputConfig.Outputs["scores"].S3Output.S3Uri, - dataset_format=DatasetFormat.csv(header=True), - output_s3_uri=Join( - on="/", - values=[ - "s3:/", - default_bucket, - base_job_prefix, - ExecutionVariables.PIPELINE_EXECUTION_ID, - "dataqualitycheckstep", - ], - ), - post_analytics_processor_script='pipelines/postprocess_monitor_script.py', - ) - - step_monitor = QualityCheckStep( - name="ModelMonitor", - skip_check=False, - register_new_baseline=False, - quality_check_config=data_quality_check_config, - check_job_config=check_job_config, - supplied_baseline_statistics=os.path.join(baseline_uri, "statistics.json"), - supplied_baseline_constraints=os.path.join(baseline_uri, "constraints.json"), - cache_config=cache_config, - ) - - steps += [step_monitor] + transform_and_monitor_step = MonitorBatchTransformStep( + name="MonitorDataQuality", + transform_step_args=transform_arg, + monitor_configuration=data_quality_config, + check_job_configuration=check_job_config, + monitor_before_transform=True, + fail_on_violation=stop_if_check_fails, + ) + transform_and_monitor_step.steps[ + 1 + ].model_package_group_name = model_package_group_name + for step in transform_and_monitor_step.steps: + step.cache_config = cache_config - # pipeline instance - pipeline = Pipeline( + return Pipeline( name=pipeline_name, - parameters=[ - input_data_uri, - input_model_uri, - output_transform_uri, - transform_instance_count, - transform_instance_type, - monitor_instance_count, - monitor_instance_type, + parameters=parameters, + steps=[ + transform_and_monitor_step, ], - steps=steps, sagemaker_session=sagemaker_session, ) - return pipeline - def upload_pipeline(pipeline: Pipeline, default_bucket, base_job_prefix) -> str: # Get the pipeline definition diff --git a/batch_pipeline/pipelines/postprocess_monitor_script.py b/batch_pipeline/pipelines/postprocess_monitor_script.py index 8c00839..2c05e86 100644 --- a/batch_pipeline/pipelines/postprocess_monitor_script.py +++ b/batch_pipeline/pipelines/postprocess_monitor_script.py @@ -1,22 +1,24 @@ -import os import json +import logging +import os import re import subprocess import sys -import logging from datetime import datetime + def install(package): subprocess.check_call([sys.executable, "-m", "pip", "install", package]) -install('boto3') -import boto3 + +install("boto3") +import boto3 # noqa: E402 LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper() logger = logging.getLogger() logger.setLevel(LOG_LEVEL) -region = os.environ.get('Region', 'NoAWSRegionFound') -pipeline_name = os.environ.get('PipelineName', 'NoPipelineNameFound') +region = os.environ.get("AWS_REGION", "NoAWSRegionFound") +pipeline_name = os.environ.get("PipelineName", "NoPipelineNameFound") cloudwatch = boto3.client("cloudwatch", region) @@ -34,7 +36,7 @@ def get_baseline_drift(feature): "metric_threshold": float(matches.group(2)), } - + def put_cloudwatch_metric(pipeline_name: str, metrics: list): for m in metrics: logger.info(f'Putting metric: {m["metric_name"]} value: {m["metric_value"]}') @@ -52,15 +54,15 @@ def put_cloudwatch_metric(pipeline_name: str, metrics: list): ) logger.debug(response) - + def postprocess_handler(): violations_file = "/opt/ml/processing/output/constraint_violations.json" if os.path.isfile(violations_file): f = open(violations_file) violations = json.load(f) metrics = list(get_baseline_drift(violations)) - + put_cloudwatch_metric(pipeline_name, metrics) logger.info("Violation detected and added to cloudwatch") - else: + else: logger.info("No constraint_violations file found. All good!") diff --git a/batch_pipeline/pipelines/score.py b/batch_pipeline/pipelines/score.py index 178169b..ff9cffc 100644 --- a/batch_pipeline/pipelines/score.py +++ b/batch_pipeline/pipelines/score.py @@ -1,10 +1,9 @@ """Evaluation script for measuring mean squared error.""" +import glob import logging import pathlib -import glob import pickle import tarfile -from math import sqrt import pandas as pd import xgboost diff --git a/batch_pipeline/poetry.lock b/batch_pipeline/poetry.lock new file mode 100644 index 0000000..5b53bbc --- /dev/null +++ b/batch_pipeline/poetry.lock @@ -0,0 +1,934 @@ +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. + +[[package]] +name = "attrs" +version = "22.2.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] + +[[package]] +name = "aws-cdk-asset-awscli-v1" +version = "2.2.127" +description = "A library that contains the AWS CLI for use in Lambda Layers" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-awscli-v1-2.2.127.tar.gz", hash = "sha256:57898d3a4e90f2667bda238bbde20defe11a989d4a939c0f1e8d5379ee6432ec"}, + {file = "aws_cdk.asset_awscli_v1-2.2.127-py3-none-any.whl", hash = "sha256:85a0729d61b0734fd9626564ef87fb56e18249a3f8de0e4ca648d5629397f3d4"}, +] + +[package.dependencies] +jsii = ">=1.79.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + +[[package]] +name = "aws-cdk-asset-kubectl-v20" +version = "2.1.1" +description = "A library that contains kubectl for use in Lambda Layers" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-kubectl-v20-2.1.1.tar.gz", hash = "sha256:9834cdb150c5590aea4e5eba6de2a89b4c60617451181c524810c5a75154565c"}, + {file = "aws_cdk.asset_kubectl_v20-2.1.1-py3-none-any.whl", hash = "sha256:a2fad1a5a35a94a465efe60859f91e45dacc33261fb9bbf1cf9bbc6e2f70e9d6"}, +] + +[package.dependencies] +jsii = ">=1.70.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + +[[package]] +name = "aws-cdk-asset-node-proxy-agent-v5" +version = "2.0.104" +description = "@aws-cdk/asset-node-proxy-agent-v5" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.104.tar.gz", hash = "sha256:591ef90bd0fd9cedd72f9a2a93ed4db32dcc634ab6497cc5fa075846911b321c"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.104-py3-none-any.whl", hash = "sha256:18cf87249ba078f242142701abea4a6711421a6cc0c12245cd3706a714764f8b"}, +] + +[package.dependencies] +jsii = ">=1.79.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + +[[package]] +name = "aws-cdk-lib" +version = "2.72.1" +description = "Version 2 of the AWS Cloud Development Kit library" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "aws-cdk-lib-2.72.1.tar.gz", hash = "sha256:a0aeaf0e0d0dcc36fe52a1df09708028a8f71f54116bc3f2afec546b0d90c256"}, + {file = "aws_cdk_lib-2.72.1-py3-none-any.whl", hash = "sha256:0d7001b0f507dcd435c6c20688e61d6c45c297e54bae2bf36256e10520668a8a"}, +] + +[package.dependencies] +"aws-cdk.asset-awscli-v1" = ">=2.2.97,<3.0.0" +"aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.77,<3.0.0" +constructs = ">=10.0.0,<11.0.0" +jsii = ">=1.78.1,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + +[[package]] +name = "black" +version = "23.3.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, + {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, + {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, + {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, + {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, + {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, + {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, + {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, + {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, + {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, + {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, + {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, + {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, + {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "boto3" +version = "1.26.142" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "boto3-1.26.142-py3-none-any.whl", hash = "sha256:8b8ccbb42e5c4008086a187afee98f4a10d5d985892699b9e2f1e6c5b18a7754"}, + {file = "boto3-1.26.142.tar.gz", hash = "sha256:8f4b5c93a7f0c8e40ae1983cfbefd017bd625e554426544c78dceb4045648911"}, +] + +[package.dependencies] +botocore = ">=1.29.142,<1.30.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.6.0,<0.7.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.29.142" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "botocore-1.29.142-py3-none-any.whl", hash = "sha256:0677848bb8ef94d69c5d2f5c613dbab5b6710a8b7649f3fafca5172c464728b8"}, + {file = "botocore-1.29.142.tar.gz", hash = "sha256:512d2f48fc1471f169bc210eede662f8da66be3cebc1515dfb5411a18b2aeabf"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.16.9)"] + +[[package]] +name = "cattrs" +version = "22.2.0" +description = "Composable complex class support for attrs and dataclasses." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cattrs-22.2.0-py3-none-any.whl", hash = "sha256:bc12b1f0d000b9f9bee83335887d532a1d3e99a833d1bf0882151c97d3e68c21"}, + {file = "cattrs-22.2.0.tar.gz", hash = "sha256:f0eed5642399423cf656e7b66ce92cdc5b963ecafd041d1b24d136fdde7acf6d"}, +] + +[package.dependencies] +attrs = ">=20" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "constructs" +version = "10.1.300" +description = "A programming model for software-defined state" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "constructs-10.1.300-py3-none-any.whl", hash = "sha256:9688ac04201688ae1d9ddb6f7d2d3aa9d0a9dc2e1dc5a22aed10663a739283c1"}, + {file = "constructs-10.1.300.tar.gz", hash = "sha256:25fa5b871dc972d2fb2b5da1f49f1216c51da6caaf2621370fcb30755245c40a"}, +] + +[package.dependencies] +jsii = ">=1.79.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + +[[package]] +name = "contextlib2" +version = "21.6.0" +description = "Backports and enhancements for the contextlib module" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, + {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, +] + +[[package]] +name = "dill" +version = "0.3.6" +description = "serialize all of python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, + {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + +[[package]] +name = "exceptiongroup" +version = "1.1.1" +description = "Backport of PEP 654 (exception groups)" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "google-pasta" +version = "0.2.0" +description = "pasta is an AST-based Python refactoring library" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e"}, + {file = "google_pasta-0.2.0-py2-none-any.whl", hash = "sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954"}, + {file = "google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "importlib-metadata" +version = "4.13.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-4.13.0-py3-none-any.whl", hash = "sha256:8a8a81bcf996e74fee46f0d16bd3eaa382a7eb20fd82445c3ad11f4090334116"}, + {file = "importlib_metadata-4.13.0.tar.gz", hash = "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "importlib-resources" +version = "5.12.0" +description = "Read resources from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, + {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "jsii" +version = "1.79.0" +description = "Python client for jsii runtime" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "jsii-1.79.0-py3-none-any.whl", hash = "sha256:325dd57a57f57cd150bc92699b91d5414c2e042e1f81c12c97b9de53088e4ed3"}, + {file = "jsii-1.79.0.tar.gz", hash = "sha256:0a771bfb6f5775cfc0ba7b77c021d3bdf52f71fb52b75ad8ef7c038a23e49e74"}, +] + +[package.dependencies] +attrs = ">=21.2,<23.0" +cattrs = ">=1.8,<22.3" +importlib-resources = ">=5.2.0" +publication = ">=0.0.3" +python-dateutil = "*" +typeguard = ">=2.13.3,<2.14.0" +typing-extensions = ">=3.7,<5.0" + +[[package]] +name = "jsonschema" +version = "4.17.3" +description = "An implementation of JSON Schema validation for Python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "multiprocess" +version = "0.70.14" +description = "better multiprocessing and multithreading in python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:560a27540daef4ce8b24ed3cc2496a3c670df66c96d02461a4da67473685adf3"}, + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-manylinux_2_24_i686.whl", hash = "sha256:bfbbfa36f400b81d1978c940616bc77776424e5e34cb0c94974b178d727cfcd5"}, + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:89fed99553a04ec4f9067031f83a886d7fdec5952005551a896a4b6a59575bb9"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:40a5e3685462079e5fdee7c6789e3ef270595e1755199f0d50685e72523e1d2a"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-manylinux_2_24_i686.whl", hash = "sha256:44936b2978d3f2648727b3eaeab6d7fa0bedf072dc5207bf35a96d5ee7c004cf"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:e628503187b5d494bf29ffc52d3e1e57bb770ce7ce05d67c4bbdb3a0c7d3b05f"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0d5da0fc84aacb0e4bd69c41b31edbf71b39fe2fb32a54eaedcaea241050855c"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-manylinux_2_24_i686.whl", hash = "sha256:6a7b03a5b98e911a7785b9116805bd782815c5e2bd6c91c6a320f26fd3e7b7ad"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:cea5bdedd10aace3c660fedeac8b087136b4366d4ee49a30f1ebf7409bce00ae"}, + {file = "multiprocess-0.70.14-py310-none-any.whl", hash = "sha256:7dc1f2f6a1d34894c8a9a013fbc807971e336e7cc3f3ff233e61b9dc679b3b5c"}, + {file = "multiprocess-0.70.14-py37-none-any.whl", hash = "sha256:93a8208ca0926d05cdbb5b9250a604c401bed677579e96c14da3090beb798193"}, + {file = "multiprocess-0.70.14-py38-none-any.whl", hash = "sha256:6725bc79666bbd29a73ca148a0fb5f4ea22eed4a8f22fce58296492a02d18a7b"}, + {file = "multiprocess-0.70.14-py39-none-any.whl", hash = "sha256:63cee628b74a2c0631ef15da5534c8aedbc10c38910b9c8b18dcd327528d1ec7"}, + {file = "multiprocess-0.70.14.tar.gz", hash = "sha256:3eddafc12f2260d27ae03fe6069b12570ab4764ab59a75e81624fac453fbf46a"}, +] + +[package.dependencies] +dill = ">=0.3.6" + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "1.24.2" +description = "Fundamental package for array computing in Python" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, + {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, + {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, + {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, + {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, + {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, + {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, + {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, + {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, + {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, +] + +[[package]] +name = "packaging" +version = "23.0" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, +] + +[[package]] +name = "pandas" +version = "1.5.3" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, + {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, + {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, + {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, + {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, + {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, + {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, + {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + +[[package]] +name = "pathos" +version = "0.3.0" +description = "parallel graph management and execution in heterogeneous computing" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathos-0.3.0-py3-none-any.whl", hash = "sha256:b1f5a79b1c79a594330d451832642ee5bb61dd77dc75ba9e5c72087c77e8994c"}, + {file = "pathos-0.3.0.tar.gz", hash = "sha256:24fa8db51fbd9284da8e191794097c4bb2aa3fce411090e57af6385e61b97e09"}, +] + +[package.dependencies] +dill = ">=0.3.6" +multiprocess = ">=0.70.14" +pox = ">=0.3.2" +ppft = ">=1.7.6.6" + +[[package]] +name = "pathspec" +version = "0.11.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, +] + +[[package]] +name = "platformdirs" +version = "3.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, + {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, +] + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pox" +version = "0.3.2" +description = "utilities for filesystem exploration and automated builds" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pox-0.3.2-py3-none-any.whl", hash = "sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61"}, + {file = "pox-0.3.2.tar.gz", hash = "sha256:e825225297638d6e3d49415f8cfb65407a5d15e56f2fb7fe9d9b9e3050c65ee1"}, +] + +[[package]] +name = "ppft" +version = "1.7.6.6" +description = "distributed and parallel python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ppft-1.7.6.6-py3-none-any.whl", hash = "sha256:f355d2caeed8bd7c9e4a860c471f31f7e66d1ada2791ab5458ea7dca15a51e41"}, + {file = "ppft-1.7.6.6.tar.gz", hash = "sha256:f933f0404f3e808bc860745acb3b79cd4fe31ea19a20889a645f900415be60f1"}, +] + +[package.extras] +dill = ["dill (>=0.3.6)"] + +[[package]] +name = "protobuf" +version = "3.20.3" +description = "Protocol Buffers" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "protobuf-3.20.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99"}, + {file = "protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e"}, + {file = "protobuf-3.20.3-cp310-cp310-win32.whl", hash = "sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c"}, + {file = "protobuf-3.20.3-cp310-cp310-win_amd64.whl", hash = "sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7"}, + {file = "protobuf-3.20.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469"}, + {file = "protobuf-3.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4"}, + {file = "protobuf-3.20.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4"}, + {file = "protobuf-3.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454"}, + {file = "protobuf-3.20.3-cp37-cp37m-win32.whl", hash = "sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905"}, + {file = "protobuf-3.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c"}, + {file = "protobuf-3.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7"}, + {file = "protobuf-3.20.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee"}, + {file = "protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050"}, + {file = "protobuf-3.20.3-cp38-cp38-win32.whl", hash = "sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86"}, + {file = "protobuf-3.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9"}, + {file = "protobuf-3.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b"}, + {file = "protobuf-3.20.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b"}, + {file = "protobuf-3.20.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402"}, + {file = "protobuf-3.20.3-cp39-cp39-win32.whl", hash = "sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480"}, + {file = "protobuf-3.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7"}, + {file = "protobuf-3.20.3-py2.py3-none-any.whl", hash = "sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db"}, + {file = "protobuf-3.20.3.tar.gz", hash = "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2"}, +] + +[[package]] +name = "protobuf3-to-dict" +version = "0.1.5" +description = "Ben Hodgson: A teeny Python library for creating Python dicts from protocol buffers and the reverse. Useful as an intermediate step before serialisation (e.g. to JSON). Kapor: upgrade it to PB3 and PY3, rename it to protobuf3-to-dict" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "protobuf3-to-dict-0.1.5.tar.gz", hash = "sha256:1e42c25b5afb5868e3a9b1962811077e492c17557f9c66f0fe40d821375d2b5a"}, +] + +[package.dependencies] +protobuf = ">=2.3.0" +six = "*" + +[[package]] +name = "publication" +version = "0.0.3" +description = "Publication helps you maintain public-api-friendly modules by preventing unintentional access to private implementation details via introspection." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, + {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, +] + +[[package]] +name = "pyrsistent" +version = "0.19.3" +description = "Persistent/Functional/Immutable data structures" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, + {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, + {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, + {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, + {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, + {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, +] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2023.3" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, + {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, +] + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +] + +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "sagemaker" +version = "2.143.0" +description = "Open source library for training and deploying models on Amazon SageMaker." +category = "main" +optional = false +python-versions = ">= 3.6" +files = [ + {file = "sagemaker-2.143.0.tar.gz", hash = "sha256:f23871ad1e906ddfe7b51ca3e520510cebffdb08eb1a0b08cc1aef6464ec6c2a"}, +] + +[package.dependencies] +attrs = ">=20.3.0,<23" +boto3 = ">=1.26.28,<2.0" +google-pasta = "*" +importlib-metadata = ">=1.4.0,<5.0" +jsonschema = "*" +numpy = ">=1.9.0,<2.0" +packaging = ">=20.0" +pandas = "*" +pathos = "*" +platformdirs = "*" +protobuf = ">=3.1,<4.0" +protobuf3-to-dict = ">=0.1.5,<1.0" +PyYAML = "5.4.1" +schema = "*" +smdebug_rulesconfig = "1.0.1" + +[package.extras] +all = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "scipy (==1.7.3)", "urllib3 (==1.26.8)"] +local = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "urllib3 (==1.26.8)"] +scipy = ["scipy (==1.7.3)"] +test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.5.1)", "apache-airflow-providers-amazon (==7.2.1)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scikit-learn (==1.0.2)", "scipy (==1.7.3)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] + +[[package]] +name = "schema" +version = "0.7.5" +description = "Simple data validation library" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, + {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, +] + +[package.dependencies] +contextlib2 = ">=0.5.5" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "smdebug-rulesconfig" +version = "1.0.1" +description = "SMDebug RulesConfig" +category = "main" +optional = false +python-versions = ">=2.7" +files = [ + {file = "smdebug_rulesconfig-1.0.1-py2.py3-none-any.whl", hash = "sha256:104da3e6931ecf879dfc687ca4bbb3bee5ea2bc27f4478e9dbb3ee3655f1ae61"}, + {file = "smdebug_rulesconfig-1.0.1.tar.gz", hash = "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typeguard" +version = "2.13.3" +description = "Run-time type checker for Python" +category = "main" +optional = false +python-versions = ">=3.5.3" +files = [ + {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, + {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, +] + +[package.extras] +doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["mypy", "pytest", "typing-extensions"] + +[[package]] +name = "typing-extensions" +version = "4.5.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, +] + +[[package]] +name = "urllib3" +version = "1.26.15" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "zipp" +version = "3.15.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "59b08fa03a9d8f33b09667b1c30f8cedc955af19acb2de25fd6bcdf9ef53a6ad" diff --git a/batch_pipeline/prod-config.json b/batch_pipeline/prod-config.json index 566b7dc..8b515ca 100644 --- a/batch_pipeline/prod-config.json +++ b/batch_pipeline/prod-config.json @@ -5,6 +5,6 @@ "drift_config": { "metric_name": "feature_baseline_drift_fare_amount", "metric_threshold": 0.4, - "period": 3600 + "period": 60 } } \ No newline at end of file diff --git a/batch_pipeline/pyproject.toml b/batch_pipeline/pyproject.toml new file mode 100644 index 0000000..c34c6eb --- /dev/null +++ b/batch_pipeline/pyproject.toml @@ -0,0 +1,23 @@ +[tool.poetry] +name = "sagemaker-driftdemobatch-batch" +version = "0.1.0" +description = "" +authors = ["Your Name "] +readme = "README.md" +packages = [{include = "sagemaker_driftdemobatch_batch"}] + +[tool.poetry.dependencies] +python = "^3.9" +constructs = "^10.1.300" +aws-cdk-lib = "^2.72.1" +boto3 = "^1.26.104" +sagemaker = "^2.143.0" + + +[tool.poetry.group.dev.dependencies] +black = "^23.3.0" +boto3 = "^1.26.142" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/batch_pipeline/requirements.txt b/batch_pipeline/requirements.txt index d6e1198..28eb9b9 100644 --- a/batch_pipeline/requirements.txt +++ b/batch_pipeline/requirements.txt @@ -1 +1,42 @@ --e . +attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-awscli-v1==2.2.127 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-kubectl-v20==2.1.1 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-node-proxy-agent-v5==2.0.104 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-lib==2.72.1 ; python_version >= "3.9" and python_version < "4.0" +boto3==1.26.104 ; python_version >= "3.9" and python_version < "4.0" +botocore==1.29.104 ; python_version >= "3.9" and python_version < "4.0" +cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" +constructs==10.1.300 ; python_version >= "3.9" and python_version < "4.0" +contextlib2==21.6.0 ; python_version >= "3.9" and python_version < "4.0" +dill==0.3.6 ; python_version >= "3.9" and python_version < "4.0" +exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11" +google-pasta==0.2.0 ; python_version >= "3.9" and python_version < "4.0" +importlib-metadata==4.13.0 ; python_version >= "3.9" and python_version < "4.0" +importlib-resources==5.12.0 ; python_version >= "3.9" and python_version < "4.0" +jmespath==1.0.1 ; python_version >= "3.9" and python_version < "4.0" +jsii==1.79.0 ; python_version >= "3.9" and python_version < "4.0" +jsonschema==4.17.3 ; python_version >= "3.9" and python_version < "4.0" +multiprocess==0.70.14 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.24.2 ; python_version < "4.0" and python_version >= "3.9" +packaging==23.0 ; python_version >= "3.9" and python_version < "4.0" +pandas==1.5.3 ; python_version >= "3.9" and python_version < "4.0" +pathos==0.3.0 ; python_version >= "3.9" and python_version < "4.0" +platformdirs==3.2.0 ; python_version >= "3.9" and python_version < "4.0" +pox==0.3.2 ; python_version >= "3.9" and python_version < "4.0" +ppft==1.7.6.6 ; python_version >= "3.9" and python_version < "4.0" +protobuf3-to-dict==0.1.5 ; python_version >= "3.9" and python_version < "4.0" +protobuf==3.20.3 ; python_version >= "3.9" and python_version < "4.0" +publication==0.0.3 ; python_version >= "3.9" and python_version < "4.0" +pyrsistent==0.19.3 ; python_version >= "3.9" and python_version < "4.0" +python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" +pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0" +pyyaml==5.4.1 ; python_version >= "3.9" and python_version < "4.0" +s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "4.0" +sagemaker==2.143.0 ; python_version >= "3.9" and python_version < "4.0" +schema==0.7.5 ; python_version >= "3.9" and python_version < "4.0" +six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" +smdebug-rulesconfig==1.0.1 ; python_version >= "3.9" and python_version < "4.0" +typeguard==2.13.3 ; python_version >= "3.9" and python_version < "4.0" +typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "4.0" +urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0" +zipp==3.15.0 ; python_version >= "3.9" and python_version < "4.0" diff --git a/build_pipeline/pipelines/preprocess/preprocess.py b/build_pipeline/pipelines/preprocess/preprocess.py index 8fbfb43..6fe7dc2 100644 --- a/build_pipeline/pipelines/preprocess/preprocess.py +++ b/build_pipeline/pipelines/preprocess/preprocess.py @@ -2,13 +2,11 @@ import glob import logging import os -import subprocess -import sys from zipfile import ZipFile -import geopandas as gpd # noqa: E402 -import pandas as pd # noqa: E402 -from sklearn.model_selection import train_test_split # noqa: E402 +import geopandas as gpd +import pandas as pd +from sklearn.model_selection import train_test_split logger = logging.getLogger() logger.setLevel(logging.INFO) @@ -27,7 +25,7 @@ def load_zones(zones_dir: str): zone_df = gpd.read_file(os.path.join(zones_dir, "taxi_zones.shp")) # Get centroids as EPSG code of 3310 to measure distance zone_df["centroid"] = zone_df.geometry.centroid.to_crs(epsg=3310) - # Convert cordinates to the WSG84 lat/long CRS has a EPSG code of 4326. + # Convert coordinates to the WSG84 lat/long CRS has a EPSG code of 4326. zone_df["latitude"] = zone_df.centroid.to_crs(epsg=4326).x zone_df["longitude"] = zone_df.centroid.to_crs(epsg=4326).y return zone_df diff --git a/infra/batch_pipeline_construct.py b/infra/batch_pipeline_construct.py index c6b0641..a770f97 100644 --- a/infra/batch_pipeline_construct.py +++ b/infra/batch_pipeline_construct.py @@ -8,6 +8,7 @@ from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda as lambda_ from aws_cdk import aws_s3 as s3 +from aws_cdk.aws_codebuild import BuildEnvironmentVariable from constructs import Construct from infra.sagemaker_pipelines_event_target import add_sagemaker_pipeline_target @@ -36,6 +37,7 @@ def __init__( seed_bucket: str, seed_key: str, batch_schedule: str, + lowercase_lambda: lambda_.Function = None, **kwargs, ) -> None: super().__init__(scope, construct_id, **kwargs) @@ -44,7 +46,7 @@ def __init__( repo = codecommit.CfnRepository( self, "CodeRepo", - repository_name="sagemaker-{}-{}".format(project_name, construct_id), + repository_name=f"sagemaker-{project_name}-{construct_id}", repository_description=f"Amazon SageMaker Drift {construct_id} pipeline", code=codecommit.CfnRepository.CodeProperty( s3=codecommit.CfnRepository.S3Property( @@ -67,9 +69,21 @@ def __init__( # Define resource names pipeline_name = f"{project_name}-{construct_id}" + + # Use a custom resource to format the pipeline name + pipeline_name_lowercase = cdk.CustomResource( + self, + "CrPipelineNameLowercase", + service_token=lowercase_lambda.function_arn, + properties=dict(InputString=pipeline_name), + ) pipeline_description = "SageMaker Drift Detection Batch Pipeline" - sagemaker_pipeline_arn = ( - f"arn:aws:sagemaker:{env.region}:{env.account}:pipeline/{pipeline_name}" + sagemaker_pipeline_arn = cdk.Fn.join( + delimiter="/", + list_of_values=[ + f"arn:aws:sagemaker:{env.region}:{env.account}:pipeline", + pipeline_name_lowercase.get_att_string("OutputString"), + ], ) code_pipeline_name = f"sagemaker-{project_name}-{construct_id}" schedule_rule_name = f"sagemaker-{project_name}-schedule-{construct_id}" @@ -79,28 +93,26 @@ def __init__( pipeline_build = codebuild.PipelineProject( self, "PipelineBuild", - project_name="sagemaker-{}-{}".format(project_name, construct_id), + project_name=f"sagemaker-{project_name}-{construct_id}", role=code_build_role, environment=codebuild.BuildEnvironment( build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_4, environment_variables={ - "SAGEMAKER_PROJECT_NAME": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PROJECT_NAME": BuildEnvironmentVariable( value=project_name ), - "SAGEMAKER_PROJECT_ID": codebuild.BuildEnvironmentVariable( - value=project_id - ), - "AWS_REGION": codebuild.BuildEnvironmentVariable(value=env.region), - "SAGEMAKER_PIPELINE_NAME": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PROJECT_ID": BuildEnvironmentVariable(value=project_id), + "AWS_REGION": BuildEnvironmentVariable(value=env.region), + "SAGEMAKER_PIPELINE_NAME": BuildEnvironmentVariable( value=pipeline_name, ), - "SAGEMAKER_PIPELINE_DESCRIPTION": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PIPELINE_DESCRIPTION": BuildEnvironmentVariable( value=pipeline_description, ), - "SAGEMAKER_PIPELINE_ROLE_ARN": codebuild.BuildEnvironmentVariable( + "SAGEMAKER_PIPELINE_ROLE_ARN": BuildEnvironmentVariable( value=sagemaker_execution_role.role_arn, ), - "ARTIFACT_BUCKET": codebuild.BuildEnvironmentVariable( + "ARTIFACT_BUCKET": BuildEnvironmentVariable( value=s3_artifact.bucket_name ), }, @@ -116,75 +128,80 @@ def __init__( role=code_pipeline_role, artifact_bucket=s3_artifact, pipeline_name=code_pipeline_name, - stages=[ - codepipeline.StageProps( - stage_name="Source", - actions=[ - codepipeline_actions.CodeCommitSourceAction( - action_name="CodeCommit_Source", - repository=code, - trigger=codepipeline_actions.CodeCommitTrigger.NONE, # Created below - event_role=event_role, - output=source_output, - branch=branch_name, - role=code_pipeline_role, - ) + ) + + source_action = codepipeline_actions.CodeCommitSourceAction( + action_name="CodeCommit_Source", + repository=code, + # Created rule below to give it a custom name + trigger=codepipeline_actions.CodeCommitTrigger.NONE, + event_role=event_role, + output=source_output, + branch=branch_name, + role=code_pipeline_role, + ) + _ = code_pipeline.add_stage( + stage_name="Source", + actions=[source_action], + ) + + _ = code_pipeline.add_stage( + stage_name="Build", + actions=[ + codepipeline_actions.CodeBuildAction( + run_order=1, + action_name="Build_Pipeline", + project=pipeline_build, + input=source_output, + outputs=[ + pipeline_build_output, ], - ), - codepipeline.StageProps( - stage_name="Build", - actions=[ - codepipeline_actions.CodeBuildAction( - run_order=1, - action_name="Build_Pipeline", - project=pipeline_build, - input=source_output, - outputs=[ - pipeline_build_output, - ], - role=code_pipeline_role, + role=code_pipeline_role, + environment_variables={ + "COMMIT_ID": BuildEnvironmentVariable( + value=source_action.variables.commit_id, ), - ], + }, ), - codepipeline.StageProps( - stage_name="BatchStaging", - actions=[ - codepipeline_actions.CloudFormationCreateUpdateStackAction( - action_name="Batch_CFN_Staging", - run_order=1, - template_path=pipeline_build_output.at_path( - "drift-batch-staging.template.json" - ), - stack_name=f"sagemaker-{project_name}-batch-staging", - admin_permissions=False, - deployment_role=cloudformation_role, - replace_on_failure=True, - role=code_pipeline_role, - ), - codepipeline_actions.ManualApprovalAction( - action_name="Approve_Staging", - run_order=2, - additional_information="Approving deployment for production", - role=code_pipeline_role, - ), - ], + ], + ) + _ = code_pipeline.add_stage( + stage_name="BatchStaging", + actions=[ + codepipeline_actions.CloudFormationCreateUpdateStackAction( + action_name="Batch_CFN_Staging", + run_order=1, + template_path=pipeline_build_output.at_path( + "drift-batch-staging.template.json" + ), + stack_name=f"sagemaker-{project_name}-batch-staging", + admin_permissions=False, + deployment_role=cloudformation_role, + replace_on_failure=True, + role=code_pipeline_role, ), - codepipeline.StageProps( - stage_name="BatchProd", - actions=[ - codepipeline_actions.CloudFormationCreateUpdateStackAction( - action_name="Batch_CFN_Prod", - run_order=1, - template_path=pipeline_build_output.at_path( - "drift-batch-prod.template.json" - ), - stack_name=f"sagemaker-{project_name}-batch-prod", - admin_permissions=False, - deployment_role=cloudformation_role, - role=code_pipeline_role, - replace_on_failure=True, - ), - ], + codepipeline_actions.ManualApprovalAction( + action_name="Approve_Staging", + run_order=2, + additional_information="Approving deployment for production", + role=code_pipeline_role, + ), + ], + ) + _ = code_pipeline.add_stage( + stage_name="BatchProd", + actions=[ + codepipeline_actions.CloudFormationCreateUpdateStackAction( + action_name="Batch_CFN_Prod", + run_order=1, + template_path=pipeline_build_output.at_path( + "drift-batch-prod.template.json" + ), + stack_name=f"sagemaker-{project_name}-batch-prod", + admin_permissions=False, + deployment_role=cloudformation_role, + role=code_pipeline_role, + replace_on_failure=True, ), ], ) @@ -226,7 +243,8 @@ def __init__( }, ) - # Add permissions to put job status (if we want to call this directly within CodePipeline) + # Add permissions to put job status (if we want to call this directly + # within CodePipeline) # see: https://docs.aws.amazon.com/codepipeline/latest/userguide/approvals-iam-permissions.html lambda_pipeline_change.add_to_role_policy( iam.PolicyStatement( @@ -253,7 +271,8 @@ def __init__( self, "SagemakerPipelineRule", rule_name=f"sagemaker-{project_name}-sagemakerpipeline-{construct_id}", - description="Rule to enable/disable SM pipeline triggers when a SageMaker Batch Pipeline is in progress.", + description="Rule to enable/disable SM pipeline triggers when a " + "SageMaker Batch Pipeline is in progress.", event_pattern=events.EventPattern( source=["aws.sagemaker"], detail_type=[ @@ -279,7 +298,8 @@ def __init__( rule_name="sagemaker-{}-modelregistry-{}".format( project_name, construct_id ), - description="Rule to trigger a deployment when SageMaker Model registry is updated with a new model package.", + description="Rule to trigger a deployment when SageMaker Model " + "Registry is updated with a new model package.", event_pattern=events.EventPattern( source=["aws.sagemaker"], detail_type=["SageMaker Model Package State Change"], diff --git a/infra/build_pipeline_construct.py b/infra/build_pipeline_construct.py index 31cf712..9c454d2 100644 --- a/infra/build_pipeline_construct.py +++ b/infra/build_pipeline_construct.py @@ -90,6 +90,32 @@ def __init__( drift_rule_name = f"sagemaker-{project_name}-drift-{construct_id}" schedule_rule_name = f"sagemaker-{project_name}-schedule-{construct_id}" + source_output = codepipeline.Artifact() + pipeline_build_output = codepipeline.Artifact() + + code_pipeline = codepipeline.Pipeline( + self, + "Pipeline", + role=code_pipeline_role, + artifact_bucket=s3_artifact, + pipeline_name=code_pipeline_name, + ) + + source_action = codepipeline_actions.CodeCommitSourceAction( + action_name="CodeCommit_Source", + repository=code, + # Created rule below to give it a custom name + trigger=codepipeline_actions.CodeCommitTrigger.NONE, + event_role=event_role, + output=source_output, + branch=branch_name, + role=code_pipeline_role, + ) + _ = code_pipeline.add_stage( + stage_name="Source", + actions=[source_action], + ) + # Define AWS CodeBuild spec to run node.js and python # https://docs.aws.amazon.com/codebuild/latest/userguide/available-runtimes.html pipeline_build = codebuild.PipelineProject( @@ -121,32 +147,6 @@ def __init__( ), ) - source_output = codepipeline.Artifact() - pipeline_build_output = codepipeline.Artifact() - - code_pipeline = codepipeline.Pipeline( - self, - "Pipeline", - role=code_pipeline_role, - artifact_bucket=s3_artifact, - pipeline_name=code_pipeline_name, - ) - - source_action = codepipeline_actions.CodeCommitSourceAction( - action_name="CodeCommit_Source", - repository=code, - # Created rule below to give it a custom name - trigger=codepipeline_actions.CodeCommitTrigger.NONE, - event_role=event_role, - output=source_output, - branch=branch_name, - role=code_pipeline_role, - ) - _ = code_pipeline.add_stage( - stage_name="Source", - actions=[source_action], - ) - _ = code_pipeline.add_stage( stage_name="Build", actions=[ @@ -163,6 +163,9 @@ def __init__( "COMMIT_ID": BuildEnvironmentVariable( value=source_action.variables.commit_id, ), + "REPOSITORY_NAME": BuildEnvironmentVariable( + value=source_action.variables.repository_name + ), }, ), ], @@ -210,7 +213,6 @@ def __init__( sagemaker_pipeline_arn=sagemaker_pipeline_arn, ) - # Load the lambda pipeline change code with open("lambda/build/lambda_pipeline_change.py", encoding="utf8") as fp: lambda_pipeline_change_code = fp.read() diff --git a/infra/pipeline_product_stack.py b/infra/pipeline_product_stack.py index b7158e3..68c821b 100644 --- a/infra/pipeline_product_stack.py +++ b/infra/pipeline_product_stack.py @@ -128,6 +128,7 @@ def __init__( seed_bucket=seed_bucket.bucket_name, seed_key="batch_pipeline.zip", batch_schedule=batch_schedule.value_as_string, + lowercase_lambda=lowercase_lambda ) if deploy_pipeline: diff --git a/infra/service_catalog_stack.py b/infra/service_catalog_stack.py index 09236c2..1ec6570 100644 --- a/infra/service_catalog_stack.py +++ b/infra/service_catalog_stack.py @@ -2,7 +2,6 @@ from zipfile import ZipFile import aws_cdk as cdk - from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda as lambda_ from aws_cdk import aws_s3 as s3 @@ -48,7 +47,7 @@ def __init__( "PortfolioOwner", type="String", description="The owner of the portfolio", - default="administrator", + default="MLOps Admin", min_length=1, max_length=50, ) @@ -72,7 +71,7 @@ def __init__( seed_bucket = s3.Bucket( self, "SeedBucket", - bucket_name=f"sagemaker-drift-detection-template-seed-{self.account}", + bucket_name=f"sagemaker-drift-detection-seed-{self.region}-{self.account}", removal_policy=cdk.RemovalPolicy.DESTROY, auto_delete_objects=True, ) @@ -138,6 +137,53 @@ def __init__( ) sm_roles = SageMakerSCRoles(self, "SageMakerSCRoles", mutable=True) + + code_build_role = sm_roles.code_build_role + code_build_role.add_to_principal_policy( + iam.PolicyStatement( + actions=["sagemaker:DescribeTrainingJob"], + resources=[ + self.format_arn( + resource="training-job", + service="sagemaker", + resource_name="*", + ), + ], + ) + ) + + code_build_role.add_to_principal_policy( + iam.PolicyStatement( + actions=[ + "sagemaker:QueryLineage", + "sagemaker:ListArtifacts", + "sagemaker:ListTags", + ], + resources=["*"], + ) + ) + + code_build_role.add_to_principal_policy( + iam.PolicyStatement( + actions=[ + "sagemaker:DescribeArtifact", + "sagemaker:DescribeTrialComponent", + ], + resources=[ + self.format_arn( + resource="artifact", + service="sagemaker", + resource_name="*", + ), + self.format_arn( + resource="experiment-trial-component", + service="sagemaker", + resource_name="*", + ), + ], + ) + ) + # Add endpoint autoscaling policies to CloudFormation role cloudformation_role = sm_roles.cloudformation_role cloudformation_role.add_to_principal_policy( @@ -256,7 +302,7 @@ def __init__( "drift:cloudformation_role" ) ) is not None: - product_roles.code_build_role = iam.Role.from_role_arn( + product_roles.cloudformation_role = iam.Role.from_role_arn( self, "code_build_role", role_arn=cloudformation_role_arn, diff --git a/poetry.lock b/poetry.lock index 8dc3009..0bf1eea 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "attrs" @@ -21,18 +21,18 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy [[package]] name = "aws-cdk-asset-awscli-v1" -version = "2.2.64" +version = "2.2.172" description = "A library that contains the AWS CLI for use in Lambda Layers" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-awscli-v1-2.2.64.tar.gz", hash = "sha256:75b290d7ad8b89ccd6fa732b0a2520c5bbdcd9ee6bf963df57d8cc310f23468f"}, - {file = "aws_cdk.asset_awscli_v1-2.2.64-py3-none-any.whl", hash = "sha256:fc64ad1ecfa2cb5a4cd9af47c676dc65c40c251d416bf4a7d61dc950989214b9"}, + {file = "aws-cdk.asset-awscli-v1-2.2.172.tar.gz", hash = "sha256:4dcfc548d20fdde04584481102a12aa8f376fa1b4cb474dcb3abd83e04cdbe91"}, + {file = "aws_cdk.asset_awscli_v1-2.2.172-py3-none-any.whl", hash = "sha256:cac5338b82f44d134b9e82c67ec12a8923e0fdfb43ede6429d5b8c033a9797b7"}, ] [package.dependencies] -jsii = ">=1.74.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -55,39 +55,39 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-asset-node-proxy-agent-v5" -version = "2.0.53" +version = "2.0.145" description = "@aws-cdk/asset-node-proxy-agent-v5" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.53.tar.gz", hash = "sha256:bb841585d2e889842bcdda49b9be9fa128058c1e1061b964dc3721a4b75080d5"}, - {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.53-py3-none-any.whl", hash = "sha256:5c5895a94f7c18de32fc3ae797578eb7bd967966cfed778b94d3574ac2b7ce33"}, + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.145.tar.gz", hash = "sha256:b8dbacb7a9621683cfb492a7957720d69a0fd74dd93adc5f50556e5fe5adc2a3"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.145-py3-none-any.whl", hash = "sha256:9accfd5a7df2cd9b8e60c361140b5997f788ceb9bb02e6091cffd523263e4144"}, ] [package.dependencies] -jsii = ">=1.74.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.64.0" +version = "2.79.1" description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.64.0.tar.gz", hash = "sha256:e21210765b362a0b8a7052fce7bd3a574ea5355a7e763e190c5051ee33b4868a"}, - {file = "aws_cdk_lib-2.64.0-py3-none-any.whl", hash = "sha256:e2de0a80eff201d2eb5326beffa5fb564231ae44eb08ed2b6d4da0c6a324e0e5"}, + {file = "aws-cdk-lib-2.79.1.tar.gz", hash = "sha256:7d88118827ed42025c7b661547b38c54b89afef2cd54214459df51104a1034c3"}, + {file = "aws_cdk_lib-2.79.1-py3-none-any.whl", hash = "sha256:89f51117cc9c2cb2561605fdd5641ceb705ebe0185276dab799193783a70123b"}, ] [package.dependencies] -"aws-cdk.asset-awscli-v1" = ">=2.2.52,<3.0.0" +"aws-cdk.asset-awscli-v1" = ">=2.2.165,<3.0.0" "aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" -"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.42,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.139,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.74.0,<2.0.0" +jsii = ">=1.80.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -129,18 +129,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.69" +version = "1.26.133" description = "The AWS SDK for Python" category = "dev" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.69-py3-none-any.whl", hash = "sha256:9a0a29179957cb26fa8c3c1fddf66b18efaeaf633e08db5fb53815ffb0421419"}, - {file = "boto3-1.26.69.tar.gz", hash = "sha256:eb8cde24a4c5755c35126e8cd460e6b51c63d04292419e7e95721232720c7e5b"}, + {file = "boto3-1.26.133-py3-none-any.whl", hash = "sha256:62285ecee7629a4388d55ae369536f759622d68d5b9a0ced7c58a0c1a409c0f7"}, + {file = "boto3-1.26.133.tar.gz", hash = "sha256:8ff0af0b25266a01616396abc19eb34dc3d44bd867fa4158985924128b9034fb"}, ] [package.dependencies] -botocore = ">=1.29.69,<1.30.0" +botocore = ">=1.29.133,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -149,14 +149,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.69" +version = "1.29.133" description = "Low-level, data-driven core of boto 3." category = "dev" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.69-py3-none-any.whl", hash = "sha256:2a4ab8bcb3177daa425019e125c09996b9a6a1a62bb0baaaeeb86ffd552719cc"}, - {file = "botocore-1.29.69.tar.gz", hash = "sha256:7e1bebca013544fbc298cb58603bfccd5f71b49c720a5c33c07cf5dfc8145a1f"}, + {file = "botocore-1.29.133-py3-none-any.whl", hash = "sha256:b266185d7414a559952569005009a400de50af91fd3da44f05cf05b00946c4a7"}, + {file = "botocore-1.29.133.tar.gz", hash = "sha256:7b38e540f73c921d8cb0ac72794072000af9e10758c04ba7f53d5629cc52fa87"}, ] [package.dependencies] @@ -198,6 +198,18 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cloudpickle" +version = "2.2.1" +description = "Extended pickling support for Python objects" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"}, + {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, +] + [[package]] name = "colorama" version = "0.4.6" @@ -212,18 +224,18 @@ files = [ [[package]] name = "constructs" -version = "10.1.247" +version = "10.2.23" description = "A programming model for software-defined state" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "constructs-10.1.247-py3-none-any.whl", hash = "sha256:809ed229a2404e72991698daf73f9fb6bcce6c0785e072420a82c80ecdfc7624"}, - {file = "constructs-10.1.247.tar.gz", hash = "sha256:1f99a59c1b54c4ffcee929b8b612841059e9d903a6fc5e778e9573217243842d"}, + {file = "constructs-10.2.23-py3-none-any.whl", hash = "sha256:c10192e7c346d02b95eb4cf4954304b309890a074edaf30b30763acd5603db9a"}, + {file = "constructs-10.2.23.tar.gz", hash = "sha256:a8adedcf359f6d9e7c154d7d066691ac57ad33203846d6231c8ac480652b3ae2"}, ] [package.dependencies] -jsii = ">=1.74.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -256,14 +268,14 @@ graph = ["objgraph (>=1.7.2)"] [[package]] name = "exceptiongroup" -version = "1.1.0" +version = "1.1.1" description = "Backport of PEP 654 (exception groups)" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, ] [package.extras] @@ -305,6 +317,25 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker perf = ["ipython"] testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +[[package]] +name = "importlib-resources" +version = "5.12.0" +description = "Read resources from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, + {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [[package]] name = "jmespath" version = "1.0.1" @@ -331,24 +362,45 @@ files = [ [[package]] name = "jsii" -version = "1.74.0" +version = "1.81.0" description = "Python client for jsii runtime" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "jsii-1.74.0-py3-none-any.whl", hash = "sha256:ee76781fe66106c367fbb3bb383db4f5e9b8ff3d3c4c0f34624c050211f040be"}, - {file = "jsii-1.74.0.tar.gz", hash = "sha256:575131396ad34f8f6e9f2604953ecbf4f3368625656a828b13089e4abb81b443"}, + {file = "jsii-1.81.0-py3-none-any.whl", hash = "sha256:6d12cd881053bafbac19d2a28fc616497479739784e017534e4cf128ff977b62"}, + {file = "jsii-1.81.0.tar.gz", hash = "sha256:585f6bedd9b586f48ce058451d24f362ee52936179987dd897a100f5355f228f"}, ] [package.dependencies] -attrs = ">=21.2,<23.0" +attrs = ">=21.2,<24.0" cattrs = ">=1.8,<22.3" +importlib-resources = ">=5.2.0" publication = ">=0.0.3" python-dateutil = "*" typeguard = ">=2.13.3,<2.14.0" typing-extensions = ">=3.7,<5.0" +[[package]] +name = "jsonschema" +version = "4.17.3" +description = "An implementation of JSON Schema validation for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + [[package]] name = "multiprocess" version = "0.70.14" @@ -390,89 +442,87 @@ files = [ [[package]] name = "numpy" -version = "1.24.2" +version = "1.24.3" description = "Fundamental package for array computing in Python" category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, - {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, - {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, - {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, - {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, - {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, - {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, - {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, - {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, - {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, - {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, + {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, + {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, + {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, + {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, + {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, + {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, + {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, + {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, + {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, ] [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] [[package]] name = "pandas" -version = "1.5.3" +version = "2.0.1" description = "Powerful data structures for data analysis, time series, and statistics" category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, - {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, - {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, - {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, - {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, - {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, - {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, - {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, + {file = "pandas-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70a996a1d2432dadedbb638fe7d921c88b0cc4dd90374eab51bb33dc6c0c2a12"}, + {file = "pandas-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:909a72b52175590debbf1d0c9e3e6bce2f1833c80c76d80bd1aa09188be768e5"}, + {file = "pandas-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe7914d8ddb2d54b900cec264c090b88d141a1eed605c9539a187dbc2547f022"}, + {file = "pandas-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a514ae436b23a92366fbad8365807fc0eed15ca219690b3445dcfa33597a5cc"}, + {file = "pandas-2.0.1-cp310-cp310-win32.whl", hash = "sha256:12bd6618e3cc737c5200ecabbbb5eaba8ab645a4b0db508ceeb4004bb10b060e"}, + {file = "pandas-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:2b6fe5f7ce1cba0e74188c8473c9091ead9b293ef0a6794939f8cc7947057abd"}, + {file = "pandas-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:00959a04a1d7bbc63d75a768540fb20ecc9e65fd80744c930e23768345a362a7"}, + {file = "pandas-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af2449e9e984dfad39276b885271ba31c5e0204ffd9f21f287a245980b0e4091"}, + {file = "pandas-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910df06feaf9935d05247db6de452f6d59820e432c18a2919a92ffcd98f8f79b"}, + {file = "pandas-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0067f2419f933101bdc6001bcea1d50812afbd367b30943417d67fbb99678"}, + {file = "pandas-2.0.1-cp311-cp311-win32.whl", hash = "sha256:7b8395d335b08bc8b050590da264f94a439b4770ff16bb51798527f1dd840388"}, + {file = "pandas-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:8db5a644d184a38e6ed40feeb12d410d7fcc36648443defe4707022da127fc35"}, + {file = "pandas-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7bbf173d364130334e0159a9a034f573e8b44a05320995127cf676b85fd8ce86"}, + {file = "pandas-2.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6c0853d487b6c868bf107a4b270a823746175b1932093b537b9b76c639fc6f7e"}, + {file = "pandas-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25e23a03f7ad7211ffa30cb181c3e5f6d96a8e4cb22898af462a7333f8a74eb"}, + {file = "pandas-2.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e09a53a4fe8d6ae2149959a2d02e1ef2f4d2ceb285ac48f74b79798507e468b4"}, + {file = "pandas-2.0.1-cp38-cp38-win32.whl", hash = "sha256:a2564629b3a47b6aa303e024e3d84e850d36746f7e804347f64229f8c87416ea"}, + {file = "pandas-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:03e677c6bc9cfb7f93a8b617d44f6091613a5671ef2944818469be7b42114a00"}, + {file = "pandas-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3d099ecaa5b9e977b55cd43cf842ec13b14afa1cfa51b7e1179d90b38c53ce6a"}, + {file = "pandas-2.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a37ee35a3eb6ce523b2c064af6286c45ea1c7ff882d46e10d0945dbda7572753"}, + {file = "pandas-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:320b180d125c3842c5da5889183b9a43da4ebba375ab2ef938f57bf267a3c684"}, + {file = "pandas-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18d22cb9043b6c6804529810f492ab09d638ddf625c5dea8529239607295cb59"}, + {file = "pandas-2.0.1-cp39-cp39-win32.whl", hash = "sha256:90d1d365d77d287063c5e339f49b27bd99ef06d10a8843cf00b1a49326d492c1"}, + {file = "pandas-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:99f7192d8b0e6daf8e0d0fd93baa40056684e4b4aaaef9ea78dff34168e1f2f0"}, + {file = "pandas-2.0.1.tar.gz", hash = "sha256:19b8e5270da32b41ebf12f0e7165efa7024492e9513fb46fb631c5022ae5709d"}, ] [package.dependencies] @@ -481,11 +531,32 @@ numpy = [ {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] -python-dateutil = ">=2.8.1" +python-dateutil = ">=2.8.2" pytz = ">=2020.1" +tzdata = ">=2022.1" [package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] [[package]] name = "pathos" @@ -507,31 +578,31 @@ ppft = ">=1.7.6.6" [[package]] name = "pathspec" -version = "0.11.0" +version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.0-py3-none-any.whl", hash = "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229"}, - {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, ] [[package]] name = "platformdirs" -version = "3.0.0" +version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, - {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, + {file = "platformdirs-3.5.1-py3-none-any.whl", hash = "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5"}, + {file = "platformdirs-3.5.1.tar.gz", hash = "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.2.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pox" @@ -619,6 +690,43 @@ files = [ {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, ] +[[package]] +name = "pyrsistent" +version = "0.19.3" +description = "Persistent/Functional/Immutable data structures" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, + {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, + {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, + {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, + {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, + {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, +] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -636,26 +744,65 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2022.7.1" +version = "2023.3" description = "World timezone definitions, modern and historical" category = "dev" optional = false python-versions = "*" files = [ - {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, - {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, + {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, + {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, +] + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] [[package]] name = "s3transfer" -version = "0.6.0" +version = "0.6.1" description = "An Amazon S3 Transfer Manager" category = "dev" optional = false python-versions = ">= 3.7" files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, + {file = "s3transfer-0.6.1-py3-none-any.whl", hash = "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346"}, + {file = "s3transfer-0.6.1.tar.gz", hash = "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"}, ] [package.dependencies] @@ -666,34 +813,39 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "sagemaker" -version = "2.132.0" +version = "2.154.0" description = "Open source library for training and deploying models on Amazon SageMaker." category = "dev" optional = false python-versions = ">= 3.6" files = [ - {file = "sagemaker-2.132.0.tar.gz", hash = "sha256:2453ee5ae156f3fc852a81752ca548fcc3549a28028dff0e14aec6142c63274c"}, + {file = "sagemaker-2.154.0.tar.gz", hash = "sha256:4bbd773bb4252a17e0af5ff1cd68358169cc40a60a1753d619b7756ffdde8525"}, ] [package.dependencies] attrs = ">=20.3.0,<23" boto3 = ">=1.26.28,<2.0" +cloudpickle = "2.2.1" google-pasta = "*" importlib-metadata = ">=1.4.0,<5.0" +jsonschema = "*" numpy = ">=1.9.0,<2.0" packaging = ">=20.0" pandas = "*" pathos = "*" +platformdirs = "*" protobuf = ">=3.1,<4.0" protobuf3-to-dict = ">=0.1.5,<1.0" +PyYAML = "5.4.1" schema = "*" smdebug_rulesconfig = "1.0.1" +tblib = "1.7.0" [package.extras] all = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "scipy (==1.7.3)", "urllib3 (==1.26.8)"] local = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "urllib3 (==1.26.8)"] scipy = ["scipy (==1.7.3)"] -test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.4.1)", "apache-airflow-providers-amazon (==4.0.0)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scikit-learn (==1.0.2)", "scipy (==1.7.3)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] +test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.5.1)", "apache-airflow-providers-amazon (==7.2.1)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "cloudpickle (==2.2.1)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "pyvis (==0.2.1)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scikit-learn (==1.0.2)", "scipy (==1.7.3)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] [[package]] name = "schema" @@ -808,6 +960,18 @@ files = [ {file = "smdebug_rulesconfig-1.0.1.tar.gz", hash = "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"}, ] +[[package]] +name = "tblib" +version = "1.7.0" +description = "Traceback serialization library." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "tblib-1.7.0-py2.py3-none-any.whl", hash = "sha256:289fa7359e580950e7d9743eab36b0691f0310fce64dee7d9c31065b8f723e23"}, + {file = "tblib-1.7.0.tar.gz", hash = "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c"}, +] + [[package]] name = "threadpoolctl" version = "3.1.0" @@ -850,26 +1014,38 @@ test = ["mypy", "pytest", "typing-extensions"] [[package]] name = "typing-extensions" -version = "4.4.0" +version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, +] + +[[package]] +name = "tzdata" +version = "2023.3" +description = "Provider of IANA time zone data" +category = "dev" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, + {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] [[package]] name = "urllib3" -version = "1.26.14" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] [package.extras] @@ -904,19 +1080,19 @@ scikit-learn = ["scikit-learn"] [[package]] name = "zipp" -version = "3.13.0" +version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "zipp-3.13.0-py3-none-any.whl", hash = "sha256:e8b2a36ea17df80ffe9e2c4fda3f693c3dad6df1697d3cd3af232db680950b0b"}, - {file = "zipp-3.13.0.tar.gz", hash = "sha256:23f70e964bc11a34cef175bc90ba2914e1e4545ea1e3e2f67c079671883f9cb6"}, + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "2.0" From 73f8a2885d7bc289edab44662804561f95a41d76 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 29 May 2023 13:04:25 +0800 Subject: [PATCH 13/17] Updated realtime repo --- deployment_pipeline/poetry.lock | 664 ++++++++++++++++++++++++--- deployment_pipeline/pyproject.toml | 1 + deployment_pipeline/requirements.txt | 48 +- 3 files changed, 635 insertions(+), 78 deletions(-) diff --git a/deployment_pipeline/poetry.lock b/deployment_pipeline/poetry.lock index 3f2fcd4..8eb3130 100644 --- a/deployment_pipeline/poetry.lock +++ b/deployment_pipeline/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "attrs" @@ -21,18 +21,18 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy [[package]] name = "aws-cdk-asset-awscli-v1" -version = "2.2.52" +version = "2.2.173" description = "A library that contains the AWS CLI for use in Lambda Layers" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-awscli-v1-2.2.52.tar.gz", hash = "sha256:ab04beec8e267e363931df2caf48a24100cb5799d7fd8db51efe881d117efa7a"}, - {file = "aws_cdk.asset_awscli_v1-2.2.52-py3-none-any.whl", hash = "sha256:6e9d686bb0b00242e869e91d57b65b619ffb42e99abe482436e3a6692485dbfe"}, + {file = "aws-cdk.asset-awscli-v1-2.2.173.tar.gz", hash = "sha256:8fbda70421638a0cbe66386e1977c1deb9562881ab1e46b6145ad66c266e2a38"}, + {file = "aws_cdk.asset_awscli_v1-2.2.173-py3-none-any.whl", hash = "sha256:c6a0590a31284bc9fc586f00765d73526ab99817d3547e25597cd3d18bc2e9f3"}, ] [package.dependencies] -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -55,39 +55,39 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-asset-node-proxy-agent-v5" -version = "2.0.42" +version = "2.0.146" description = "@aws-cdk/asset-node-proxy-agent-v5" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.42.tar.gz", hash = "sha256:ae1b615be42e78681e05b145460603f171c06b671a2d1caa060a159b94b06366"}, - {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.42-py3-none-any.whl", hash = "sha256:6e0174802097d558daa1be5c4e6e7f309eeba626392955e596bf967ee37865d3"}, + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.146.tar.gz", hash = "sha256:9aac36a3f69e9d7d99fc403192be41369ff0d0307a3e4c028a3a70e0108ac63c"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.146-py3-none-any.whl", hash = "sha256:cec187b3e2db5f90839598511c002bb4fda1d1728b81ebcae58c13c901681583"}, ] [package.dependencies] -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.62.2" +version = "2.79.1" description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.62.2.tar.gz", hash = "sha256:f85000438d849a0522ffcd8e7cb5a70be5fa34339082d4d569734169c6d37b4d"}, - {file = "aws_cdk_lib-2.62.2-py3-none-any.whl", hash = "sha256:03dfb8303b00333177b18e3b60c95d738adf4d90086a5b1e707e896fdc234d52"}, + {file = "aws-cdk-lib-2.79.1.tar.gz", hash = "sha256:7d88118827ed42025c7b661547b38c54b89afef2cd54214459df51104a1034c3"}, + {file = "aws_cdk_lib-2.79.1-py3-none-any.whl", hash = "sha256:89f51117cc9c2cb2561605fdd5641ceb705ebe0185276dab799193783a70123b"}, ] [package.dependencies] -"aws-cdk.asset-awscli-v1" = ">=2.2.49,<3.0.0" +"aws-cdk.asset-awscli-v1" = ">=2.2.165,<3.0.0" "aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" -"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.38,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.139,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.80.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -129,18 +129,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.59" +version = "1.26.133" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.59-py3-none-any.whl", hash = "sha256:34ee771a5cc84c16e75d4b9ef4672f51c2bafdce66ec457bbaac630b37d9cd5e"}, - {file = "boto3-1.26.59.tar.gz", hash = "sha256:7d9cebb507fc96e6eb429621ccb2e731b75e7bbb8d6d9f0cf0c08089ee3c1ab7"}, + {file = "boto3-1.26.133-py3-none-any.whl", hash = "sha256:62285ecee7629a4388d55ae369536f759622d68d5b9a0ced7c58a0c1a409c0f7"}, + {file = "boto3-1.26.133.tar.gz", hash = "sha256:8ff0af0b25266a01616396abc19eb34dc3d44bd867fa4158985924128b9034fb"}, ] [package.dependencies] -botocore = ">=1.29.59,<1.30.0" +botocore = ">=1.29.133,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -149,14 +149,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.59" +version = "1.29.133" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.59-py3-none-any.whl", hash = "sha256:5533644ddefaccfaa98460a63eb73e61a46aad019771226d103b1054b0df6103"}, - {file = "botocore-1.29.59.tar.gz", hash = "sha256:bc75d41c5eecf624a2f9875483135aa78088a50c8d29847793f92756697cfed5"}, + {file = "botocore-1.29.133-py3-none-any.whl", hash = "sha256:b266185d7414a559952569005009a400de50af91fd3da44f05cf05b00946c4a7"}, + {file = "botocore-1.29.133.tar.gz", hash = "sha256:7b38e540f73c921d8cb0ac72794072000af9e10758c04ba7f53d5629cc52fa87"}, ] [package.dependencies] @@ -165,7 +165,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.15.3)"] +crt = ["awscrt (==0.16.9)"] [[package]] name = "cattrs" @@ -198,6 +198,18 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cloudpickle" +version = "2.2.1" +description = "Extended pickling support for Python objects" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"}, + {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, +] + [[package]] name = "colorama" version = "0.4.6" @@ -212,36 +224,118 @@ files = [ [[package]] name = "constructs" -version = "10.1.235" +version = "10.2.24" description = "A programming model for software-defined state" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "constructs-10.1.235-py3-none-any.whl", hash = "sha256:7e7d56dbb75cb51b859e2e7418e4adc71da2669df53f95cd60cff4e728375563"}, - {file = "constructs-10.1.235.tar.gz", hash = "sha256:9f64f57871743722f26dad556fdc8f196d26fb5b07f36b0e4a1f4e4bf73fdcf8"}, + {file = "constructs-10.2.24-py3-none-any.whl", hash = "sha256:590d2fd1f616ee027a698e479584997e3b6c206dc2746c26b8e47cceb33296c6"}, + {file = "constructs-10.2.24.tar.gz", hash = "sha256:307abe5330dc81c1120d1876a4556d98b572abe85bb8c2b68bc7b6f18e359913"}, ] [package.dependencies] -jsii = ">=1.74.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" +[[package]] +name = "contextlib2" +version = "21.6.0" +description = "Backports and enhancements for the contextlib module" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "contextlib2-21.6.0-py2.py3-none-any.whl", hash = "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f"}, + {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, +] + +[[package]] +name = "dill" +version = "0.3.6" +description = "serialize all of python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, + {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + [[package]] name = "exceptiongroup" -version = "1.1.0" +version = "1.1.1" description = "Backport of PEP 654 (exception groups)" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, ] [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "google-pasta" +version = "0.2.0" +description = "pasta is an AST-based Python refactoring library" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e"}, + {file = "google_pasta-0.2.0-py2-none-any.whl", hash = "sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954"}, + {file = "google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "importlib-metadata" +version = "4.13.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-4.13.0-py3-none-any.whl", hash = "sha256:8a8a81bcf996e74fee46f0d16bd3eaa382a7eb20fd82445c3ad11f4090334116"}, + {file = "importlib_metadata-4.13.0.tar.gz", hash = "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "importlib-resources" +version = "5.12.0" +description = "Read resources from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, + {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -268,75 +362,247 @@ files = [ [[package]] name = "jsii" -version = "1.74.0" +version = "1.81.0" description = "Python client for jsii runtime" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "jsii-1.74.0-py3-none-any.whl", hash = "sha256:ee76781fe66106c367fbb3bb383db4f5e9b8ff3d3c4c0f34624c050211f040be"}, - {file = "jsii-1.74.0.tar.gz", hash = "sha256:575131396ad34f8f6e9f2604953ecbf4f3368625656a828b13089e4abb81b443"}, + {file = "jsii-1.81.0-py3-none-any.whl", hash = "sha256:6d12cd881053bafbac19d2a28fc616497479739784e017534e4cf128ff977b62"}, + {file = "jsii-1.81.0.tar.gz", hash = "sha256:585f6bedd9b586f48ce058451d24f362ee52936179987dd897a100f5355f228f"}, ] [package.dependencies] -attrs = ">=21.2,<23.0" +attrs = ">=21.2,<24.0" cattrs = ">=1.8,<22.3" +importlib-resources = ">=5.2.0" publication = ">=0.0.3" python-dateutil = "*" typeguard = ">=2.13.3,<2.14.0" typing-extensions = ">=3.7,<5.0" +[[package]] +name = "jsonschema" +version = "4.17.3" +description = "An implementation of JSON Schema validation for Python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "multiprocess" +version = "0.70.14" +description = "better multiprocessing and multithreading in python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:560a27540daef4ce8b24ed3cc2496a3c670df66c96d02461a4da67473685adf3"}, + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-manylinux_2_24_i686.whl", hash = "sha256:bfbbfa36f400b81d1978c940616bc77776424e5e34cb0c94974b178d727cfcd5"}, + {file = "multiprocess-0.70.14-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:89fed99553a04ec4f9067031f83a886d7fdec5952005551a896a4b6a59575bb9"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:40a5e3685462079e5fdee7c6789e3ef270595e1755199f0d50685e72523e1d2a"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-manylinux_2_24_i686.whl", hash = "sha256:44936b2978d3f2648727b3eaeab6d7fa0bedf072dc5207bf35a96d5ee7c004cf"}, + {file = "multiprocess-0.70.14-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:e628503187b5d494bf29ffc52d3e1e57bb770ce7ce05d67c4bbdb3a0c7d3b05f"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0d5da0fc84aacb0e4bd69c41b31edbf71b39fe2fb32a54eaedcaea241050855c"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-manylinux_2_24_i686.whl", hash = "sha256:6a7b03a5b98e911a7785b9116805bd782815c5e2bd6c91c6a320f26fd3e7b7ad"}, + {file = "multiprocess-0.70.14-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:cea5bdedd10aace3c660fedeac8b087136b4366d4ee49a30f1ebf7409bce00ae"}, + {file = "multiprocess-0.70.14-py310-none-any.whl", hash = "sha256:7dc1f2f6a1d34894c8a9a013fbc807971e336e7cc3f3ff233e61b9dc679b3b5c"}, + {file = "multiprocess-0.70.14-py37-none-any.whl", hash = "sha256:93a8208ca0926d05cdbb5b9250a604c401bed677579e96c14da3090beb798193"}, + {file = "multiprocess-0.70.14-py38-none-any.whl", hash = "sha256:6725bc79666bbd29a73ca148a0fb5f4ea22eed4a8f22fce58296492a02d18a7b"}, + {file = "multiprocess-0.70.14-py39-none-any.whl", hash = "sha256:63cee628b74a2c0631ef15da5534c8aedbc10c38910b9c8b18dcd327528d1ec7"}, + {file = "multiprocess-0.70.14.tar.gz", hash = "sha256:3eddafc12f2260d27ae03fe6069b12570ab4764ab59a75e81624fac453fbf46a"}, +] + +[package.dependencies] +dill = ">=0.3.6" + [[package]] name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "1.24.3" +description = "Fundamental package for array computing in Python" +category = "main" +optional = false +python-versions = ">=3.8" files = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, + {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, + {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, + {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, + {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, + {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, + {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, + {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, + {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, + {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, ] [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pandas" +version = "2.0.1" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pandas-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70a996a1d2432dadedbb638fe7d921c88b0cc4dd90374eab51bb33dc6c0c2a12"}, + {file = "pandas-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:909a72b52175590debbf1d0c9e3e6bce2f1833c80c76d80bd1aa09188be768e5"}, + {file = "pandas-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe7914d8ddb2d54b900cec264c090b88d141a1eed605c9539a187dbc2547f022"}, + {file = "pandas-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a514ae436b23a92366fbad8365807fc0eed15ca219690b3445dcfa33597a5cc"}, + {file = "pandas-2.0.1-cp310-cp310-win32.whl", hash = "sha256:12bd6618e3cc737c5200ecabbbb5eaba8ab645a4b0db508ceeb4004bb10b060e"}, + {file = "pandas-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:2b6fe5f7ce1cba0e74188c8473c9091ead9b293ef0a6794939f8cc7947057abd"}, + {file = "pandas-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:00959a04a1d7bbc63d75a768540fb20ecc9e65fd80744c930e23768345a362a7"}, + {file = "pandas-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af2449e9e984dfad39276b885271ba31c5e0204ffd9f21f287a245980b0e4091"}, + {file = "pandas-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910df06feaf9935d05247db6de452f6d59820e432c18a2919a92ffcd98f8f79b"}, + {file = "pandas-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0067f2419f933101bdc6001bcea1d50812afbd367b30943417d67fbb99678"}, + {file = "pandas-2.0.1-cp311-cp311-win32.whl", hash = "sha256:7b8395d335b08bc8b050590da264f94a439b4770ff16bb51798527f1dd840388"}, + {file = "pandas-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:8db5a644d184a38e6ed40feeb12d410d7fcc36648443defe4707022da127fc35"}, + {file = "pandas-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7bbf173d364130334e0159a9a034f573e8b44a05320995127cf676b85fd8ce86"}, + {file = "pandas-2.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6c0853d487b6c868bf107a4b270a823746175b1932093b537b9b76c639fc6f7e"}, + {file = "pandas-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25e23a03f7ad7211ffa30cb181c3e5f6d96a8e4cb22898af462a7333f8a74eb"}, + {file = "pandas-2.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e09a53a4fe8d6ae2149959a2d02e1ef2f4d2ceb285ac48f74b79798507e468b4"}, + {file = "pandas-2.0.1-cp38-cp38-win32.whl", hash = "sha256:a2564629b3a47b6aa303e024e3d84e850d36746f7e804347f64229f8c87416ea"}, + {file = "pandas-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:03e677c6bc9cfb7f93a8b617d44f6091613a5671ef2944818469be7b42114a00"}, + {file = "pandas-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3d099ecaa5b9e977b55cd43cf842ec13b14afa1cfa51b7e1179d90b38c53ce6a"}, + {file = "pandas-2.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a37ee35a3eb6ce523b2c064af6286c45ea1c7ff882d46e10d0945dbda7572753"}, + {file = "pandas-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:320b180d125c3842c5da5889183b9a43da4ebba375ab2ef938f57bf267a3c684"}, + {file = "pandas-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18d22cb9043b6c6804529810f492ab09d638ddf625c5dea8529239607295cb59"}, + {file = "pandas-2.0.1-cp39-cp39-win32.whl", hash = "sha256:90d1d365d77d287063c5e339f49b27bd99ef06d10a8843cf00b1a49326d492c1"}, + {file = "pandas-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:99f7192d8b0e6daf8e0d0fd93baa40056684e4b4aaaef9ea78dff34168e1f2f0"}, + {file = "pandas-2.0.1.tar.gz", hash = "sha256:19b8e5270da32b41ebf12f0e7165efa7024492e9513fb46fb631c5022ae5709d"}, ] +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] + +[[package]] +name = "pathos" +version = "0.3.0" +description = "parallel graph management and execution in heterogeneous computing" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathos-0.3.0-py3-none-any.whl", hash = "sha256:b1f5a79b1c79a594330d451832642ee5bb61dd77dc75ba9e5c72087c77e8994c"}, + {file = "pathos-0.3.0.tar.gz", hash = "sha256:24fa8db51fbd9284da8e191794097c4bb2aa3fce411090e57af6385e61b97e09"}, +] + +[package.dependencies] +dill = ">=0.3.6" +multiprocess = ">=0.70.14" +pox = ">=0.3.2" +ppft = ">=1.7.6.6" + [[package]] name = "pathspec" -version = "0.11.0" +version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.0-py3-none-any.whl", hash = "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229"}, - {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, ] [[package]] name = "platformdirs" -version = "2.6.2" +version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, - {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, + {file = "platformdirs-3.5.1-py3-none-any.whl", hash = "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5"}, + {file = "platformdirs-3.5.1.tar.gz", hash = "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.2.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -354,6 +620,80 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "pox" +version = "0.3.2" +description = "utilities for filesystem exploration and automated builds" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pox-0.3.2-py3-none-any.whl", hash = "sha256:56fe2f099ecd8a557b8948082504492de90e8598c34733c9b1fdeca8f7b6de61"}, + {file = "pox-0.3.2.tar.gz", hash = "sha256:e825225297638d6e3d49415f8cfb65407a5d15e56f2fb7fe9d9b9e3050c65ee1"}, +] + +[[package]] +name = "ppft" +version = "1.7.6.6" +description = "distributed and parallel python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ppft-1.7.6.6-py3-none-any.whl", hash = "sha256:f355d2caeed8bd7c9e4a860c471f31f7e66d1ada2791ab5458ea7dca15a51e41"}, + {file = "ppft-1.7.6.6.tar.gz", hash = "sha256:f933f0404f3e808bc860745acb3b79cd4fe31ea19a20889a645f900415be60f1"}, +] + +[package.extras] +dill = ["dill (>=0.3.6)"] + +[[package]] +name = "protobuf" +version = "3.20.3" +description = "Protocol Buffers" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "protobuf-3.20.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99"}, + {file = "protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e"}, + {file = "protobuf-3.20.3-cp310-cp310-win32.whl", hash = "sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c"}, + {file = "protobuf-3.20.3-cp310-cp310-win_amd64.whl", hash = "sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7"}, + {file = "protobuf-3.20.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469"}, + {file = "protobuf-3.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4"}, + {file = "protobuf-3.20.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4"}, + {file = "protobuf-3.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454"}, + {file = "protobuf-3.20.3-cp37-cp37m-win32.whl", hash = "sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905"}, + {file = "protobuf-3.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c"}, + {file = "protobuf-3.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7"}, + {file = "protobuf-3.20.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee"}, + {file = "protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050"}, + {file = "protobuf-3.20.3-cp38-cp38-win32.whl", hash = "sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86"}, + {file = "protobuf-3.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9"}, + {file = "protobuf-3.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b"}, + {file = "protobuf-3.20.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b"}, + {file = "protobuf-3.20.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402"}, + {file = "protobuf-3.20.3-cp39-cp39-win32.whl", hash = "sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480"}, + {file = "protobuf-3.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7"}, + {file = "protobuf-3.20.3-py2.py3-none-any.whl", hash = "sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db"}, + {file = "protobuf-3.20.3.tar.gz", hash = "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2"}, +] + +[[package]] +name = "protobuf3-to-dict" +version = "0.1.5" +description = "Ben Hodgson: A teeny Python library for creating Python dicts from protocol buffers and the reverse. Useful as an intermediate step before serialisation (e.g. to JSON). Kapor: upgrade it to PB3 and PY3, rename it to protobuf3-to-dict" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "protobuf3-to-dict-0.1.5.tar.gz", hash = "sha256:1e42c25b5afb5868e3a9b1962811077e492c17557f9c66f0fe40d821375d2b5a"}, +] + +[package.dependencies] +protobuf = ">=2.3.0" +six = "*" + [[package]] name = "publication" version = "0.0.3" @@ -366,20 +706,56 @@ files = [ {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, ] +[[package]] +name = "pyrsistent" +version = "0.19.3" +description = "Persistent/Functional/Immutable data structures" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, + {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, + {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, + {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, + {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, + {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, +] + [[package]] name = "pytest" -version = "7.2.1" +version = "7.3.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, - {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, ] [package.dependencies] -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" @@ -388,7 +764,7 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "python-dateutil" @@ -405,16 +781,67 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytz" +version = "2023.3" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, + {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, +] + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +] + [[package]] name = "s3transfer" -version = "0.6.0" +version = "0.6.1" description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, + {file = "s3transfer-0.6.1-py3-none-any.whl", hash = "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346"}, + {file = "s3transfer-0.6.1.tar.gz", hash = "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"}, ] [package.dependencies] @@ -423,6 +850,57 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "sagemaker" +version = "2.154.0" +description = "Open source library for training and deploying models on Amazon SageMaker." +category = "main" +optional = false +python-versions = ">= 3.6" +files = [ + {file = "sagemaker-2.154.0.tar.gz", hash = "sha256:4bbd773bb4252a17e0af5ff1cd68358169cc40a60a1753d619b7756ffdde8525"}, +] + +[package.dependencies] +attrs = ">=20.3.0,<23" +boto3 = ">=1.26.28,<2.0" +cloudpickle = "2.2.1" +google-pasta = "*" +importlib-metadata = ">=1.4.0,<5.0" +jsonschema = "*" +numpy = ">=1.9.0,<2.0" +packaging = ">=20.0" +pandas = "*" +pathos = "*" +platformdirs = "*" +protobuf = ">=3.1,<4.0" +protobuf3-to-dict = ">=0.1.5,<1.0" +PyYAML = "5.4.1" +schema = "*" +smdebug_rulesconfig = "1.0.1" +tblib = "1.7.0" + +[package.extras] +all = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "scipy (==1.7.3)", "urllib3 (==1.26.8)"] +local = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "urllib3 (==1.26.8)"] +scipy = ["scipy (==1.7.3)"] +test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.5.1)", "apache-airflow-providers-amazon (==7.2.1)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "cloudpickle (==2.2.1)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "pyvis (==0.2.1)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scikit-learn (==1.0.2)", "scipy (==1.7.3)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] + +[[package]] +name = "schema" +version = "0.7.5" +description = "Simple data validation library" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "schema-0.7.5-py2.py3-none-any.whl", hash = "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c"}, + {file = "schema-0.7.5.tar.gz", hash = "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197"}, +] + +[package.dependencies] +contextlib2 = ">=0.5.5" + [[package]] name = "six" version = "1.16.0" @@ -435,6 +913,30 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "smdebug-rulesconfig" +version = "1.0.1" +description = "SMDebug RulesConfig" +category = "main" +optional = false +python-versions = ">=2.7" +files = [ + {file = "smdebug_rulesconfig-1.0.1-py2.py3-none-any.whl", hash = "sha256:104da3e6931ecf879dfc687ca4bbb3bee5ea2bc27f4478e9dbb3ee3655f1ae61"}, + {file = "smdebug_rulesconfig-1.0.1.tar.gz", hash = "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"}, +] + +[[package]] +name = "tblib" +version = "1.7.0" +description = "Traceback serialization library." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "tblib-1.7.0-py2.py3-none-any.whl", hash = "sha256:289fa7359e580950e7d9743eab36b0691f0310fce64dee7d9c31065b8f723e23"}, + {file = "tblib-1.7.0.tar.gz", hash = "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -465,26 +967,38 @@ test = ["mypy", "pytest", "typing-extensions"] [[package]] name = "typing-extensions" -version = "4.4.0" +version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, +] + +[[package]] +name = "tzdata" +version = "2023.3" +description = "Provider of IANA time zone data" +category = "main" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, + {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] [[package]] name = "urllib3" -version = "1.26.14" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] [package.extras] @@ -492,7 +1006,23 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "zipp" +version = "3.15.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "0b8ee6379573fb9ea6cad3dda51ec5f919d9c400d2ea6bb76cca9fe8f406504e" +content-hash = "4bc9f8e1fcf381abe4fa386a0cb63ed2846ea0ea25eb13861bf9439bf146782d" diff --git a/deployment_pipeline/pyproject.toml b/deployment_pipeline/pyproject.toml index f895ad1..b14e97a 100644 --- a/deployment_pipeline/pyproject.toml +++ b/deployment_pipeline/pyproject.toml @@ -11,6 +11,7 @@ python = "^3.9" aws-cdk-lib = "^2.50.0" constructs = "^10.1.153" boto3 = "^1.26.4" +sagemaker = "^2.154.0" [tool.poetry.group.dev.dependencies] diff --git a/deployment_pipeline/requirements.txt b/deployment_pipeline/requirements.txt index 46ff254..c2df074 100644 --- a/deployment_pipeline/requirements.txt +++ b/deployment_pipeline/requirements.txt @@ -1,19 +1,45 @@ attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-awscli-v1==2.2.52 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-awscli-v1==2.2.173 ; python_version >= "3.9" and python_version < "4.0" aws-cdk-asset-kubectl-v20==2.1.1 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-node-proxy-agent-v5==2.0.42 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-lib==2.62.2 ; python_version >= "3.9" and python_version < "4.0" -boto3==1.26.59 ; python_version >= "3.9" and python_version < "4.0" -botocore==1.29.59 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-node-proxy-agent-v5==2.0.146 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-lib==2.79.1 ; python_version >= "3.9" and python_version < "4.0" +boto3==1.26.133 ; python_version >= "3.9" and python_version < "4.0" +botocore==1.29.133 ; python_version >= "3.9" and python_version < "4.0" cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -constructs==10.1.235 ; python_version >= "3.9" and python_version < "4.0" -exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11" +cloudpickle==2.2.1 ; python_version >= "3.9" and python_version < "4.0" +constructs==10.2.24 ; python_version >= "3.9" and python_version < "4.0" +contextlib2==21.6.0 ; python_version >= "3.9" and python_version < "4.0" +dill==0.3.6 ; python_version >= "3.9" and python_version < "4.0" +exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11" +google-pasta==0.2.0 ; python_version >= "3.9" and python_version < "4.0" +importlib-metadata==4.13.0 ; python_version >= "3.9" and python_version < "4.0" +importlib-resources==5.12.0 ; python_version >= "3.9" and python_version < "4.0" jmespath==1.0.1 ; python_version >= "3.9" and python_version < "4.0" -jsii==1.74.0 ; python_version >= "3.9" and python_version < "4.0" +jsii==1.81.0 ; python_version >= "3.9" and python_version < "4.0" +jsonschema==4.17.3 ; python_version >= "3.9" and python_version < "4.0" +multiprocess==0.70.14 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0" +packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" +pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0" +pathos==0.3.0 ; python_version >= "3.9" and python_version < "4.0" +platformdirs==3.5.1 ; python_version >= "3.9" and python_version < "4.0" +pox==0.3.2 ; python_version >= "3.9" and python_version < "4.0" +ppft==1.7.6.6 ; python_version >= "3.9" and python_version < "4.0" +protobuf3-to-dict==0.1.5 ; python_version >= "3.9" and python_version < "4.0" +protobuf==3.20.3 ; python_version >= "3.9" and python_version < "4.0" publication==0.0.3 ; python_version >= "3.9" and python_version < "4.0" +pyrsistent==0.19.3 ; python_version >= "3.9" and python_version < "4.0" python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" -s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "4.0" +pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0" +pyyaml==5.4.1 ; python_version >= "3.9" and python_version < "4.0" +s3transfer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" +sagemaker==2.154.0 ; python_version >= "3.9" and python_version < "4.0" +schema==0.7.5 ; python_version >= "3.9" and python_version < "4.0" six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" +smdebug-rulesconfig==1.0.1 ; python_version >= "3.9" and python_version < "4.0" +tblib==1.7.0 ; python_version >= "3.9" and python_version < "4.0" typeguard==2.13.3 ; python_version >= "3.9" and python_version < "4.0" -typing-extensions==4.4.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4.0" +typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "4.0" +tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0" +urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0" +zipp==3.15.0 ; python_version >= "3.9" and python_version < "4.0" From b2ba29b164dc4c507ce6d34f2cc31536d0068825 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 29 May 2023 13:36:20 +0800 Subject: [PATCH 14/17] Updated build repo --- build_pipeline/.gitignore | 1 + build_pipeline/pipelines/pipeline.py | 41 +- .../pipelines/preprocess/preprocess.py | 12 +- .../pipelines/preprocess/requirements.txt | 3 +- build_pipeline/poetry.lock | 434 ++++++++++++------ build_pipeline/requirements.txt | 42 +- 6 files changed, 377 insertions(+), 156 deletions(-) diff --git a/build_pipeline/.gitignore b/build_pipeline/.gitignore index 3c147b5..941c482 100644 --- a/build_pipeline/.gitignore +++ b/build_pipeline/.gitignore @@ -5,6 +5,7 @@ __pycache__ .venv *.egg-info *.zip +*.csv # CDK asset staging directory .cdk.staging diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index e90ecd9..f15d5af 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -16,6 +16,7 @@ from sagemaker.drift_check_baselines import DriftCheckBaselines from sagemaker.estimator import Estimator from sagemaker.inputs import TrainingInput +from sagemaker.metadata_properties import MetadataProperties from sagemaker.model import Model from sagemaker.model_metrics import MetricsSource, ModelMetrics from sagemaker.model_monitor.dataset_format import DatasetFormat @@ -95,11 +96,11 @@ def get_pipeline( ) input_data = ParameterString( name="InputDataUrl", - default_value=f"s3://{default_bucket}/inputs/data", + default_value=f"s3://{default_bucket}/input/data", ) input_zones = ParameterString( name="InputZonesUrl", - default_value=f"s3://{default_bucket}/inputs/zones/taxi_zones.zip", + default_value=f"s3://{default_bucket}/input/zones/taxi_zones.zip", ) processing_instance_count = ParameterInteger( name="ProcessingInstanceCount", @@ -185,11 +186,19 @@ def get_pipeline( values=output_common_path + ["baseline"], ), ), + ProcessingOutput( + output_name="sample_payload", + source="/opt/ml/processing/sample_payload", + destination=Join( + on="/", + values=output_common_path + ["sample_payload"], + ), + ), ] sklearn_processor = FrameworkProcessor( estimator_cls=SKLearn, - framework_version="0.23-1", + framework_version="1.2-1", role=role, instance_type=processing_instance_type, instance_count=processing_instance_count, @@ -203,7 +212,6 @@ def get_pipeline( outputs=outputs, code="preprocess.py", source_dir=os.path.join(BASE_DIR, "preprocess"), - job_name=f"scripts/{commit_id[:8]}/preprocess", ), cache_config=cache_config, ) @@ -244,10 +252,12 @@ def get_pipeline( rules = [Rule.sagemaker(rule_configs.create_xgboost_report())] # training step for generating model artifacts + framework = "xgboost" + framework_version = "1.2-2" image_uri = sagemaker.image_uris.retrieve( - framework="xgboost", + framework=framework, region=region, - version="1.2-2", + version=framework_version, ) xgb_train = Estimator( image_uri=image_uri, @@ -279,7 +289,6 @@ def get_pipeline( step_train = TrainingStep( name="TrainModel", step_args=xgb_train.fit( - job_name=f"scripts/{commit_id[:8]}/train", inputs={ "train": TrainingInput( s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ @@ -395,6 +404,24 @@ def get_pipeline( approval_status=model_approval_status, model_metrics=model_metrics, drift_check_baselines=drift_check_baselines, + domain="MACHINE_LEARNING", + task="REGRESSION", + framework=framework.upper(), + framework_version=framework_version, + sample_payload_url=Join( + on="/", + values=[ + step_process.properties.ProcessingOutputConfig.Outputs[ + "sample_payload" + ].S3Output.S3Uri, + "payload.tar.gz", + ], + ), + metadata_properties=MetadataProperties( + commit_id=os.getenv("COMMIT_ID"), + repository=os.getenv("REPOSITORY_NAME"), + project_id=os.getenv("SAGEMAKER_PROJECT_ID"), + ), ), ) diff --git a/build_pipeline/pipelines/preprocess/preprocess.py b/build_pipeline/pipelines/preprocess/preprocess.py index 6fe7dc2..4ada8e7 100644 --- a/build_pipeline/pipelines/preprocess/preprocess.py +++ b/build_pipeline/pipelines/preprocess/preprocess.py @@ -5,7 +5,7 @@ from zipfile import ZipFile import geopandas as gpd -import pandas as pd +import pandas as pd from sklearn.model_selection import train_test_split logger = logging.getLogger() @@ -125,8 +125,16 @@ def save_files(base_dir: str, data_df: pd.DataFrame, val_size=0.2, test_size=0.0 # Save training data as baseline with header train_df.to_csv(f"{base_dir}/baseline/baseline.csv", header=True, index=False) - return train_df, val_df, test_df + # Save sample payload + train_df.sample(20).drop(columns=["fare_amount"]).to_csv( + f"{base_dir}/sample_payload/payload.tar.gz", + index=False, + header=False, + compression={'method': 'tar', "archive_name": "payload.csv"} + ) + + return train_df, val_df, test_df def main(base_dir): # Input data files diff --git a/build_pipeline/pipelines/preprocess/requirements.txt b/build_pipeline/pipelines/preprocess/requirements.txt index c36ae26..565a7d1 100644 --- a/build_pipeline/pipelines/preprocess/requirements.txt +++ b/build_pipeline/pipelines/preprocess/requirements.txt @@ -1 +1,2 @@ -geopandas==0.9.0 \ No newline at end of file +geopandas==0.9.0 +pandas==1.5.0 \ No newline at end of file diff --git a/build_pipeline/poetry.lock b/build_pipeline/poetry.lock index ce858dd..1f153c8 100644 --- a/build_pipeline/poetry.lock +++ b/build_pipeline/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "attrs" @@ -21,18 +21,18 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy [[package]] name = "aws-cdk-asset-awscli-v1" -version = "2.2.52" +version = "2.2.173" description = "A library that contains the AWS CLI for use in Lambda Layers" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-awscli-v1-2.2.52.tar.gz", hash = "sha256:ab04beec8e267e363931df2caf48a24100cb5799d7fd8db51efe881d117efa7a"}, - {file = "aws_cdk.asset_awscli_v1-2.2.52-py3-none-any.whl", hash = "sha256:6e9d686bb0b00242e869e91d57b65b619ffb42e99abe482436e3a6692485dbfe"}, + {file = "aws-cdk.asset-awscli-v1-2.2.173.tar.gz", hash = "sha256:8fbda70421638a0cbe66386e1977c1deb9562881ab1e46b6145ad66c266e2a38"}, + {file = "aws_cdk.asset_awscli_v1-2.2.173-py3-none-any.whl", hash = "sha256:c6a0590a31284bc9fc586f00765d73526ab99817d3547e25597cd3d18bc2e9f3"}, ] [package.dependencies] -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -55,39 +55,39 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-asset-node-proxy-agent-v5" -version = "2.0.42" +version = "2.0.146" description = "@aws-cdk/asset-node-proxy-agent-v5" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.42.tar.gz", hash = "sha256:ae1b615be42e78681e05b145460603f171c06b671a2d1caa060a159b94b06366"}, - {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.42-py3-none-any.whl", hash = "sha256:6e0174802097d558daa1be5c4e6e7f309eeba626392955e596bf967ee37865d3"}, + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.146.tar.gz", hash = "sha256:9aac36a3f69e9d7d99fc403192be41369ff0d0307a3e4c028a3a70e0108ac63c"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.146-py3-none-any.whl", hash = "sha256:cec187b3e2db5f90839598511c002bb4fda1d1728b81ebcae58c13c901681583"}, ] [package.dependencies] -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.62.2" +version = "2.79.1" description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.62.2.tar.gz", hash = "sha256:f85000438d849a0522ffcd8e7cb5a70be5fa34339082d4d569734169c6d37b4d"}, - {file = "aws_cdk_lib-2.62.2-py3-none-any.whl", hash = "sha256:03dfb8303b00333177b18e3b60c95d738adf4d90086a5b1e707e896fdc234d52"}, + {file = "aws-cdk-lib-2.79.1.tar.gz", hash = "sha256:7d88118827ed42025c7b661547b38c54b89afef2cd54214459df51104a1034c3"}, + {file = "aws_cdk_lib-2.79.1-py3-none-any.whl", hash = "sha256:89f51117cc9c2cb2561605fdd5641ceb705ebe0185276dab799193783a70123b"}, ] [package.dependencies] -"aws-cdk.asset-awscli-v1" = ">=2.2.49,<3.0.0" +"aws-cdk.asset-awscli-v1" = ">=2.2.165,<3.0.0" "aws-cdk.asset-kubectl-v20" = ">=2.1.1,<3.0.0" -"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.38,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.139,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.73.0,<2.0.0" +jsii = ">=1.80.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -129,18 +129,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.59" +version = "1.26.133" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.59-py3-none-any.whl", hash = "sha256:34ee771a5cc84c16e75d4b9ef4672f51c2bafdce66ec457bbaac630b37d9cd5e"}, - {file = "boto3-1.26.59.tar.gz", hash = "sha256:7d9cebb507fc96e6eb429621ccb2e731b75e7bbb8d6d9f0cf0c08089ee3c1ab7"}, + {file = "boto3-1.26.133-py3-none-any.whl", hash = "sha256:62285ecee7629a4388d55ae369536f759622d68d5b9a0ced7c58a0c1a409c0f7"}, + {file = "boto3-1.26.133.tar.gz", hash = "sha256:8ff0af0b25266a01616396abc19eb34dc3d44bd867fa4158985924128b9034fb"}, ] [package.dependencies] -botocore = ">=1.29.59,<1.30.0" +botocore = ">=1.29.133,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -149,14 +149,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.59" +version = "1.29.133" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.59-py3-none-any.whl", hash = "sha256:5533644ddefaccfaa98460a63eb73e61a46aad019771226d103b1054b0df6103"}, - {file = "botocore-1.29.59.tar.gz", hash = "sha256:bc75d41c5eecf624a2f9875483135aa78088a50c8d29847793f92756697cfed5"}, + {file = "botocore-1.29.133-py3-none-any.whl", hash = "sha256:b266185d7414a559952569005009a400de50af91fd3da44f05cf05b00946c4a7"}, + {file = "botocore-1.29.133.tar.gz", hash = "sha256:7b38e540f73c921d8cb0ac72794072000af9e10758c04ba7f53d5629cc52fa87"}, ] [package.dependencies] @@ -165,7 +165,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.15.3)"] +crt = ["awscrt (==0.16.9)"] [[package]] name = "cattrs" @@ -198,6 +198,18 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cloudpickle" +version = "2.2.1" +description = "Extended pickling support for Python objects" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"}, + {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, +] + [[package]] name = "colorama" version = "0.4.6" @@ -212,18 +224,18 @@ files = [ [[package]] name = "constructs" -version = "10.1.234" +version = "10.2.24" description = "A programming model for software-defined state" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "constructs-10.1.234-py3-none-any.whl", hash = "sha256:5da8eaacb7c1c34251a9306a4b2bfae7d9be54f7803daf8684d9944cf6259e8f"}, - {file = "constructs-10.1.234.tar.gz", hash = "sha256:bb21ae76abb1c118c458d53a358ca82252b1fefd83dff9d3bd6f9f4cac161a84"}, + {file = "constructs-10.2.24-py3-none-any.whl", hash = "sha256:590d2fd1f616ee027a698e479584997e3b6c206dc2746c26b8e47cceb33296c6"}, + {file = "constructs-10.2.24.tar.gz", hash = "sha256:307abe5330dc81c1120d1876a4556d98b572abe85bb8c2b68bc7b6f18e359913"}, ] [package.dependencies] -jsii = ">=1.74.0,<2.0.0" +jsii = ">=1.81.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -256,14 +268,14 @@ graph = ["objgraph (>=1.7.2)"] [[package]] name = "exceptiongroup" -version = "1.1.0" +version = "1.1.1" description = "Backport of PEP 654 (exception groups)" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, ] [package.extras] @@ -305,6 +317,25 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker perf = ["ipython"] testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +[[package]] +name = "importlib-resources" +version = "5.12.0" +description = "Read resources from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, + {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [[package]] name = "jmespath" version = "1.0.1" @@ -319,24 +350,45 @@ files = [ [[package]] name = "jsii" -version = "1.74.0" +version = "1.81.0" description = "Python client for jsii runtime" category = "main" optional = false python-versions = "~=3.7" files = [ - {file = "jsii-1.74.0-py3-none-any.whl", hash = "sha256:ee76781fe66106c367fbb3bb383db4f5e9b8ff3d3c4c0f34624c050211f040be"}, - {file = "jsii-1.74.0.tar.gz", hash = "sha256:575131396ad34f8f6e9f2604953ecbf4f3368625656a828b13089e4abb81b443"}, + {file = "jsii-1.81.0-py3-none-any.whl", hash = "sha256:6d12cd881053bafbac19d2a28fc616497479739784e017534e4cf128ff977b62"}, + {file = "jsii-1.81.0.tar.gz", hash = "sha256:585f6bedd9b586f48ce058451d24f362ee52936179987dd897a100f5355f228f"}, ] [package.dependencies] -attrs = ">=21.2,<23.0" +attrs = ">=21.2,<24.0" cattrs = ">=1.8,<22.3" +importlib-resources = ">=5.2.0" publication = ">=0.0.3" python-dateutil = "*" typeguard = ">=2.13.3,<2.14.0" typing-extensions = ">=3.7,<5.0" +[[package]] +name = "jsonschema" +version = "4.17.3" +description = "An implementation of JSON Schema validation for Python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + [[package]] name = "multiprocess" version = "0.70.14" @@ -366,101 +418,99 @@ dill = ">=0.3.6" [[package]] name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.5" files = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] [[package]] name = "numpy" -version = "1.24.1" +version = "1.24.3" description = "Fundamental package for array computing in Python" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, - {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, - {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, - {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, - {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, - {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, - {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, - {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, - {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, - {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, - {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, + {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, + {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, + {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, + {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, + {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, + {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, + {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, + {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, + {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, ] [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] [[package]] name = "pandas" -version = "1.5.3" +version = "2.0.1" description = "Powerful data structures for data analysis, time series, and statistics" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, - {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, - {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, - {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, - {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, - {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, - {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, - {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, + {file = "pandas-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70a996a1d2432dadedbb638fe7d921c88b0cc4dd90374eab51bb33dc6c0c2a12"}, + {file = "pandas-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:909a72b52175590debbf1d0c9e3e6bce2f1833c80c76d80bd1aa09188be768e5"}, + {file = "pandas-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe7914d8ddb2d54b900cec264c090b88d141a1eed605c9539a187dbc2547f022"}, + {file = "pandas-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a514ae436b23a92366fbad8365807fc0eed15ca219690b3445dcfa33597a5cc"}, + {file = "pandas-2.0.1-cp310-cp310-win32.whl", hash = "sha256:12bd6618e3cc737c5200ecabbbb5eaba8ab645a4b0db508ceeb4004bb10b060e"}, + {file = "pandas-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:2b6fe5f7ce1cba0e74188c8473c9091ead9b293ef0a6794939f8cc7947057abd"}, + {file = "pandas-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:00959a04a1d7bbc63d75a768540fb20ecc9e65fd80744c930e23768345a362a7"}, + {file = "pandas-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af2449e9e984dfad39276b885271ba31c5e0204ffd9f21f287a245980b0e4091"}, + {file = "pandas-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910df06feaf9935d05247db6de452f6d59820e432c18a2919a92ffcd98f8f79b"}, + {file = "pandas-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0067f2419f933101bdc6001bcea1d50812afbd367b30943417d67fbb99678"}, + {file = "pandas-2.0.1-cp311-cp311-win32.whl", hash = "sha256:7b8395d335b08bc8b050590da264f94a439b4770ff16bb51798527f1dd840388"}, + {file = "pandas-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:8db5a644d184a38e6ed40feeb12d410d7fcc36648443defe4707022da127fc35"}, + {file = "pandas-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7bbf173d364130334e0159a9a034f573e8b44a05320995127cf676b85fd8ce86"}, + {file = "pandas-2.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6c0853d487b6c868bf107a4b270a823746175b1932093b537b9b76c639fc6f7e"}, + {file = "pandas-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25e23a03f7ad7211ffa30cb181c3e5f6d96a8e4cb22898af462a7333f8a74eb"}, + {file = "pandas-2.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e09a53a4fe8d6ae2149959a2d02e1ef2f4d2ceb285ac48f74b79798507e468b4"}, + {file = "pandas-2.0.1-cp38-cp38-win32.whl", hash = "sha256:a2564629b3a47b6aa303e024e3d84e850d36746f7e804347f64229f8c87416ea"}, + {file = "pandas-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:03e677c6bc9cfb7f93a8b617d44f6091613a5671ef2944818469be7b42114a00"}, + {file = "pandas-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3d099ecaa5b9e977b55cd43cf842ec13b14afa1cfa51b7e1179d90b38c53ce6a"}, + {file = "pandas-2.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a37ee35a3eb6ce523b2c064af6286c45ea1c7ff882d46e10d0945dbda7572753"}, + {file = "pandas-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:320b180d125c3842c5da5889183b9a43da4ebba375ab2ef938f57bf267a3c684"}, + {file = "pandas-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18d22cb9043b6c6804529810f492ab09d638ddf625c5dea8529239607295cb59"}, + {file = "pandas-2.0.1-cp39-cp39-win32.whl", hash = "sha256:90d1d365d77d287063c5e339f49b27bd99ef06d10a8843cf00b1a49326d492c1"}, + {file = "pandas-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:99f7192d8b0e6daf8e0d0fd93baa40056684e4b4aaaef9ea78dff34168e1f2f0"}, + {file = "pandas-2.0.1.tar.gz", hash = "sha256:19b8e5270da32b41ebf12f0e7165efa7024492e9513fb46fb631c5022ae5709d"}, ] [package.dependencies] @@ -469,11 +519,32 @@ numpy = [ {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] -python-dateutil = ">=2.8.1" +python-dateutil = ">=2.8.2" pytz = ">=2020.1" +tzdata = ">=2022.1" [package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] [[package]] name = "pathos" @@ -495,31 +566,31 @@ ppft = ">=1.7.6.6" [[package]] name = "pathspec" -version = "0.11.0" +version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.0-py3-none-any.whl", hash = "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229"}, - {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, ] [[package]] name = "platformdirs" -version = "2.6.2" +version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, - {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, + {file = "platformdirs-3.5.1-py3-none-any.whl", hash = "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5"}, + {file = "platformdirs-3.5.1.tar.gz", hash = "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.2.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pox" @@ -607,6 +678,43 @@ files = [ {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, ] +[[package]] +name = "pyrsistent" +version = "0.19.3" +description = "Persistent/Functional/Immutable data structures" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, + {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, + {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, + {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, + {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, + {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, +] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -624,26 +732,65 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2022.7.1" +version = "2023.3" description = "World timezone definitions, modern and historical" category = "main" optional = false python-versions = "*" files = [ - {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, - {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, + {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, + {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, +] + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] [[package]] name = "s3transfer" -version = "0.6.0" +version = "0.6.1" description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, + {file = "s3transfer-0.6.1-py3-none-any.whl", hash = "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346"}, + {file = "s3transfer-0.6.1.tar.gz", hash = "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"}, ] [package.dependencies] @@ -654,34 +801,39 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "sagemaker" -version = "2.130.0" +version = "2.154.0" description = "Open source library for training and deploying models on Amazon SageMaker." category = "main" optional = false python-versions = ">= 3.6" files = [ - {file = "sagemaker-2.130.0.tar.gz", hash = "sha256:ac6f80a413bd2621a5ec8ba8146cdc3b279dc1fbc513d5300caccaf177b28700"}, + {file = "sagemaker-2.154.0.tar.gz", hash = "sha256:4bbd773bb4252a17e0af5ff1cd68358169cc40a60a1753d619b7756ffdde8525"}, ] [package.dependencies] attrs = ">=20.3.0,<23" boto3 = ">=1.26.28,<2.0" +cloudpickle = "2.2.1" google-pasta = "*" importlib-metadata = ">=1.4.0,<5.0" +jsonschema = "*" numpy = ">=1.9.0,<2.0" packaging = ">=20.0" pandas = "*" pathos = "*" +platformdirs = "*" protobuf = ">=3.1,<4.0" protobuf3-to-dict = ">=0.1.5,<1.0" +PyYAML = "5.4.1" schema = "*" smdebug_rulesconfig = "1.0.1" +tblib = "1.7.0" [package.extras] all = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "scipy (==1.7.3)", "urllib3 (==1.26.8)"] local = ["PyYAML (==5.4.1)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "urllib3 (==1.26.8)"] scipy = ["scipy (==1.7.3)"] -test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.4.1)", "apache-airflow-providers-amazon (==4.0.0)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scikit-learn (==1.0.2)", "scipy (==1.7.3)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] +test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.5.1)", "apache-airflow-providers-amazon (==7.2.1)", "attrs (==22.1.0)", "awslogs (==0.14.0)", "black (==22.3.0)", "cloudpickle (==2.2.1)", "contextlib2 (==21.6.0)", "coverage (>=5.2,<6.2)", "docker (>=5.0.2,<7.0.0)", "docker-compose (==1.29.2)", "fabric (==2.6.0)", "flake8 (==4.0.1)", "mock (==4.0.3)", "pandas (>=1.3.5,<1.5)", "pytest (==6.2.5)", "pytest-cov (==3.0.0)", "pytest-rerunfailures (==10.2)", "pytest-timeout (==2.1.0)", "pytest-xdist (==2.4.0)", "pyvis (==0.2.1)", "requests (==2.27.1)", "sagemaker-experiments (==0.1.35)", "scikit-learn (==1.0.2)", "scipy (==1.7.3)", "stopit (==1.1.2)", "tox (==3.24.5)", "urllib3 (==1.26.8)"] [[package]] name = "schema" @@ -722,6 +874,18 @@ files = [ {file = "smdebug_rulesconfig-1.0.1.tar.gz", hash = "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"}, ] +[[package]] +name = "tblib" +version = "1.7.0" +description = "Traceback serialization library." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "tblib-1.7.0-py2.py3-none-any.whl", hash = "sha256:289fa7359e580950e7d9743eab36b0691f0310fce64dee7d9c31065b8f723e23"}, + {file = "tblib-1.7.0.tar.gz", hash = "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -752,26 +916,38 @@ test = ["mypy", "pytest", "typing-extensions"] [[package]] name = "typing-extensions" -version = "4.4.0" +version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, +] + +[[package]] +name = "tzdata" +version = "2023.3" +description = "Provider of IANA time zone data" +category = "main" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, + {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] [[package]] name = "urllib3" -version = "1.26.14" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] [package.extras] @@ -781,19 +957,19 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "zipp" -version = "3.12.0" +version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "zipp-3.12.0-py3-none-any.whl", hash = "sha256:9eb0a4c5feab9b08871db0d672745b53450d7f26992fd1e4653aa43345e97b86"}, - {file = "zipp-3.12.0.tar.gz", hash = "sha256:73efd63936398aac78fd92b6f4865190119d6c91b531532e798977ea8dd402eb"}, + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "2.0" diff --git a/build_pipeline/requirements.txt b/build_pipeline/requirements.txt index 44559a4..c2df074 100644 --- a/build_pipeline/requirements.txt +++ b/build_pipeline/requirements.txt @@ -1,37 +1,45 @@ attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-awscli-v1==2.2.52 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-awscli-v1==2.2.173 ; python_version >= "3.9" and python_version < "4.0" aws-cdk-asset-kubectl-v20==2.1.1 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-asset-node-proxy-agent-v5==2.0.42 ; python_version >= "3.9" and python_version < "4.0" -aws-cdk-lib==2.62.2 ; python_version >= "3.9" and python_version < "4.0" -boto3==1.26.59 ; python_version >= "3.9" and python_version < "4.0" -botocore==1.29.59 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-asset-node-proxy-agent-v5==2.0.146 ; python_version >= "3.9" and python_version < "4.0" +aws-cdk-lib==2.79.1 ; python_version >= "3.9" and python_version < "4.0" +boto3==1.26.133 ; python_version >= "3.9" and python_version < "4.0" +botocore==1.29.133 ; python_version >= "3.9" and python_version < "4.0" cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0" -constructs==10.1.234 ; python_version >= "3.9" and python_version < "4.0" +cloudpickle==2.2.1 ; python_version >= "3.9" and python_version < "4.0" +constructs==10.2.24 ; python_version >= "3.9" and python_version < "4.0" contextlib2==21.6.0 ; python_version >= "3.9" and python_version < "4.0" dill==0.3.6 ; python_version >= "3.9" and python_version < "4.0" -exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11" +exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11" google-pasta==0.2.0 ; python_version >= "3.9" and python_version < "4.0" importlib-metadata==4.13.0 ; python_version >= "3.9" and python_version < "4.0" +importlib-resources==5.12.0 ; python_version >= "3.9" and python_version < "4.0" jmespath==1.0.1 ; python_version >= "3.9" and python_version < "4.0" -jsii==1.74.0 ; python_version >= "3.9" and python_version < "4.0" +jsii==1.81.0 ; python_version >= "3.9" and python_version < "4.0" +jsonschema==4.17.3 ; python_version >= "3.9" and python_version < "4.0" multiprocess==0.70.14 ; python_version >= "3.9" and python_version < "4.0" -numpy==1.24.1 ; python_version < "4.0" and python_version >= "3.9" -packaging==23.0 ; python_version >= "3.9" and python_version < "4.0" -pandas==1.5.3 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0" +packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" +pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0" pathos==0.3.0 ; python_version >= "3.9" and python_version < "4.0" +platformdirs==3.5.1 ; python_version >= "3.9" and python_version < "4.0" pox==0.3.2 ; python_version >= "3.9" and python_version < "4.0" ppft==1.7.6.6 ; python_version >= "3.9" and python_version < "4.0" protobuf3-to-dict==0.1.5 ; python_version >= "3.9" and python_version < "4.0" protobuf==3.20.3 ; python_version >= "3.9" and python_version < "4.0" publication==0.0.3 ; python_version >= "3.9" and python_version < "4.0" +pyrsistent==0.19.3 ; python_version >= "3.9" and python_version < "4.0" python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" -pytz==2022.7.1 ; python_version >= "3.9" and python_version < "4.0" -s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "4.0" -sagemaker==2.130.0 ; python_version >= "3.9" and python_version < "4.0" +pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0" +pyyaml==5.4.1 ; python_version >= "3.9" and python_version < "4.0" +s3transfer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" +sagemaker==2.154.0 ; python_version >= "3.9" and python_version < "4.0" schema==0.7.5 ; python_version >= "3.9" and python_version < "4.0" six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" smdebug-rulesconfig==1.0.1 ; python_version >= "3.9" and python_version < "4.0" +tblib==1.7.0 ; python_version >= "3.9" and python_version < "4.0" typeguard==2.13.3 ; python_version >= "3.9" and python_version < "4.0" -typing-extensions==4.4.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4.0" -zipp==3.12.0 ; python_version >= "3.9" and python_version < "4.0" +typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "4.0" +tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0" +urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0" +zipp==3.15.0 ; python_version >= "3.9" and python_version < "4.0" From a42f0ee01a3d2ddaba7024c84071339754f6d661 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 29 May 2023 13:47:52 +0800 Subject: [PATCH 15/17] Adding lineage and inference recommender to build repo --- build_pipeline/build-pipeline.ipynb | 624 ++++++++++++++- build_pipeline/deployment-pipeline.ipynb | 720 +++++++++++++++++- build_pipeline/lineage.ipynb | 918 +++++++++++++++++++++++ build_pipeline/visualizer.py | 166 ++++ 4 files changed, 2393 insertions(+), 35 deletions(-) create mode 100644 build_pipeline/lineage.ipynb create mode 100644 build_pipeline/visualizer.py diff --git a/build_pipeline/build-pipeline.ipynb b/build_pipeline/build-pipeline.ipynb index 5732f77..14c71eb 100644 --- a/build_pipeline/build-pipeline.ipynb +++ b/build_pipeline/build-pipeline.ipynb @@ -6,17 +6,23 @@ "source": [ "# Build Pipeline Notebook\n", "\n", - "This notebook will exercise the drift detection MLOps `build pipeline`" + "This notebook will exercise the drift detection MLOps `build pipeline`\n", + "\n", + "***\n", + "This notebook has been written for `Data Science 3.0 (Python 3) kernel`\n", + "***" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "%%capture\n", - "%pip install -U pandas seaborn" + "%pip install -U pip pandas" ] }, { @@ -31,7 +37,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "project_name = \"<>\" # << Update this drift detection project\n", @@ -48,7 +56,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import json\n", @@ -76,7 +86,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from sagemaker.s3 import S3Downloader, S3Uploader\n", @@ -88,7 +100,7 @@ "S3Downloader().download(download_uri, \"input/zones\")\n", "\n", "# Upload input to the target location\n", - "input_data_uri = f\"s3://{artifact_bucket}/{project_id}/input\"\n", + "input_data_uri = f\"s3://{artifact_bucket}/input\"\n", "S3Uploader().upload(\"input\", input_data_uri)\n", "\n", "print(\"Listing input files:\")\n", @@ -108,11 +120,12 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "from datetime import datetime\n", - "\n", + "from sagemaker.utils import name_from_base\n", "from sagemaker.workflow.pipeline import Pipeline\n", "\n", "pipeline_name = f\"{project_name}-build\"\n", @@ -120,7 +133,7 @@ "\n", "# Start pipeline\n", "execution = pipeline.start(\n", - " execution_display_name=f\"ManualExecution-{datetime.now():%Y%m%d-%H%M%S}\",\n", + " execution_display_name=name_from_base(\"ManualExecution\", short=True),\n", " execution_description=\"Execution triggered from the demo notebook\",\n", ")\n", "execution_name = execution.arn.split(\"/\")[-1]\n", @@ -141,18 +154,20 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "for step in execution.list_steps():\n", - " print(\"Step: {}, Status: {}\".format(step[\"StepName\"], step[\"StepStatus\"]))" + " print(\"Step: {StepName}, Status: {StepStatus}\".format(**step))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Evaluate\n", + "## Evaluate\n", "\n", "Get the estimator for the training job in the pipeline." ] @@ -238,7 +253,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Approve Model\n", + "## Approve Model\n", "\n", "ℹ️ Once we are happy with this training job, we can [Update the Approval Status](https://docs.aws.amazon.com/sagemaker/latest/dg/model-registry-approve.html) of a model." ] @@ -249,7 +264,9 @@ "metadata": {}, "outputs": [], "source": [ - "model_package_arn = get_execution_step(\"RegisterModel-RegisterModel\")[0][\"RegisterModel\"][\"Arn\"]\n", + "model_package_arn = get_execution_step(\"RegisterModel-RegisterModel\")[0][\n", + " \"RegisterModel\"\n", + "][\"Arn\"]\n", "model_package_version = model_package_arn.split(\"/\")[-1]\n", "print(f\"Model version: {model_package_version}\")" ] @@ -357,6 +374,581 @@ } ], "metadata": { + "availableInstances": [ + { + "_defaultOrder": 0, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.t3.medium", + "vcpuNum": 2 + }, + { + "_defaultOrder": 1, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.t3.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 2, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.t3.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 3, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.t3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 4, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 5, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 6, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 7, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 8, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 9, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 10, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 11, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 12, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5d.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 13, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5d.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 14, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5d.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 15, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5d.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 16, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5d.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 17, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5d.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 18, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5d.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 19, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 20, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": true, + "memoryGiB": 0, + "name": "ml.geospatial.interactive", + "supportedImageNames": [ + "sagemaker-geospatial-v1-0" + ], + "vcpuNum": 0 + }, + { + "_defaultOrder": 21, + "_isFastLaunch": true, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.c5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 22, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.c5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 23, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.c5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 24, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.c5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 25, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 72, + "name": "ml.c5.9xlarge", + "vcpuNum": 36 + }, + { + "_defaultOrder": 26, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 96, + "name": "ml.c5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 27, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 144, + "name": "ml.c5.18xlarge", + "vcpuNum": 72 + }, + { + "_defaultOrder": 28, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.c5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 29, + "_isFastLaunch": true, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g4dn.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 30, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g4dn.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 31, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g4dn.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 32, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g4dn.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 33, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g4dn.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 34, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g4dn.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 35, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 61, + "name": "ml.p3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 36, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 244, + "name": "ml.p3.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 37, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 488, + "name": "ml.p3.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 38, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.p3dn.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 39, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.r5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 40, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.r5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 41, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.r5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 42, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.r5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 43, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.r5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 44, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.r5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 45, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 512, + "name": "ml.r5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 46, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.r5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 47, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 48, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 49, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 50, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 51, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 52, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 53, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.g5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 54, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.g5.48xlarge", + "vcpuNum": 192 + }, + { + "_defaultOrder": 55, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 56, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4de.24xlarge", + "vcpuNum": 96 + } + ], "instance_type": "ml.t3.medium", "interpreter": { "hash": "07c1d6c68b7b22b50965762993b154aa5a1dd6aa65a365988d7d4c27c573599b" diff --git a/build_pipeline/deployment-pipeline.ipynb b/build_pipeline/deployment-pipeline.ipynb index f8d5d76..6baefa1 100644 --- a/build_pipeline/deployment-pipeline.ipynb +++ b/build_pipeline/deployment-pipeline.ipynb @@ -6,7 +6,11 @@ "source": [ "# Deployment Pipeline Notebook\n", "\n", - "This notebook will exercise the drift detection MLOps `deployment pipeline`" + "This notebook will exercise the drift detection MLOps `deployment pipeline`\n", + "\n", + "***\n", + "This notebook has been written for `Data Science 3.0 (Python 3) kernel`\n", + "***" ] }, { @@ -21,7 +25,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "%store -r project_name" @@ -37,17 +43,21 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "import sagemaker\n", "import json\n", "\n", + "import sagemaker\n", + "\n", "sess = sagemaker.session.Session()\n", "region_name = sess._region_name\n", "sm_client = sess.sagemaker_client\n", "project_id = sm_client.describe_project(ProjectName=project_name)[\"ProjectId\"]\n", "artifact_bucket = f\"sagemaker-project-{project_id}-{region_name}\"\n", + "role = sagemaker.get_execution_role()\n", "\n", "print(f\"Project: {project_name} ({project_id})\")" ] @@ -62,7 +72,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from IPython.core.display import HTML\n", @@ -84,12 +96,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ + "import random\n", + "\n", "import boto3\n", "import pandas as pd\n", - "import random\n", "from sagemaker.s3 import S3Downloader, S3Uploader\n", "\n", "\n", @@ -139,18 +154,21 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from botocore.exceptions import WaiterError\n", + "from sagemaker.deserializers import CSVDeserializer\n", "from sagemaker.predictor import Predictor\n", "from sagemaker.serializers import CSVSerializer\n", - "from sagemaker.deserializers import CSVDeserializer\n", + "\n", "\n", "# Define the predictor for staging\n", "def wait_for_predictor(stage_name):\n", " try:\n", - " endpoint_name = f\"sagemaker-{project_name}-{stage_name}\"\n", + " endpoint_name = f\"{project_name}-{stage_name}\"\n", " predictor = Predictor(\n", " endpoint_name, serializer=CSVSerializer(), deserializer=CSVDeserializer()\n", " )\n", @@ -188,13 +206,101 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "payload = \"1,-73.986114,40.685634,-73.936794,40.715370,5.318025,7,0,2\"\n", "predictor.predict(data=payload)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inference recommender (Optional)\n", + "\n", + "Inference recommender automates load testing and can help select the best instance type and configuration (such as instance count, container parameters, and model optimizations). In this example, we leverage the Staging Endpoint to base the Inference recommender job." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "Get registered model package ARN using SageMaker lineage features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from sagemaker.lineage.context import Context, EndpointContext\n", + "from sagemaker.lineage.query import (\n", + " LineageEntityEnum,\n", + " LineageFilter,\n", + " LineageQuery,\n", + " LineageQueryDirectionEnum,\n", + " LineageSourceEnum,\n", + ")\n", + "\n", + "endpoint_arn = sm_client.describe_endpoint(\n", + " EndpointName=predictor.endpoint_name,\n", + ").get(\"EndpointArn\")\n", + "\n", + "contexts = Context.list(source_uri=endpoint_arn)\n", + "context_name = list(contexts)[0].context_name\n", + "endpoint_context = EndpointContext.load(context_name=context_name)\n", + "\n", + "query_filter = LineageFilter(\n", + " entities=[LineageEntityEnum.ARTIFACT],\n", + " sources=[LineageSourceEnum.MODEL],\n", + " properties={\"ApprovalStatus\": \"Approved\"},\n", + ")\n", + "\n", + "query_result = LineageQuery(sess).query(\n", + " start_arns=[endpoint_context.context_arn],\n", + " query_filter=query_filter,\n", + " direction=LineageQueryDirectionEnum.ASCENDANTS,\n", + " include_edges=False,\n", + ")\n", + "model_package_version_arn = [\n", + " vertex.to_lineage_object().source.source_uri for vertex in query_result.vertices\n", + "][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the inference recommender job. After creation, you can follow the progress of the job and get a report on the results from the left-hand panel." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sm_client.create_inference_recommendations_job(\n", + " JobName=sagemaker.utils.name_from_base(\"DataDriftStagingJob\", short=True),\n", + " JobType=\"Default\",\n", + " RoleArn=role,\n", + " InputConfig={\n", + " \"ModelPackageVersionArn\": model_package_version_arn,\n", + " \"Endpoints\": [{\"EndpointName\": predictor.endpoint_name}],\n", + " },\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -327,8 +433,8 @@ "metadata": {}, "outputs": [], "source": [ - "from threading import Thread\n", "import time\n", + "from threading import Thread\n", "\n", "\n", "def invoke_endpoint_forever():\n", @@ -364,6 +470,7 @@ "outputs": [], "source": [ "from datetime import datetime, timedelta\n", + "\n", "from dateutil.tz import tzlocal\n", "\n", "model_monitor = predictor.list_monitors()[0]\n", @@ -373,7 +480,7 @@ "now = datetime.now(tzlocal())\n", "next_hour = (now + timedelta(hours=1)).replace(minute=0)\n", "scheduled_diff = (next_hour - now).seconds // 60\n", - "print(\"Next schedule in {} minutes\".format(scheduled_diff))" + "print(f\"Next schedule in {scheduled_diff} minutes\")" ] }, { @@ -430,11 +537,11 @@ "metadata": {}, "outputs": [], "source": [ - "import boto3\n", - "from datetime import datetime, timedelta\n", - "from dateutil.tz import tzlocal\n", "import random\n", + "from datetime import datetime, timedelta\n", "\n", + "import boto3\n", + "from dateutil.tz import tzlocal\n", "\n", "cloudwatch = boto3.client(\"cloudwatch\")\n", "\n", @@ -625,14 +732,589 @@ } ], "metadata": { + "availableInstances": [ + { + "_defaultOrder": 0, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.t3.medium", + "vcpuNum": 2 + }, + { + "_defaultOrder": 1, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.t3.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 2, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.t3.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 3, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.t3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 4, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 5, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 6, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 7, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 8, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 9, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 10, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 11, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 12, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5d.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 13, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5d.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 14, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5d.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 15, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5d.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 16, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5d.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 17, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5d.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 18, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5d.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 19, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 20, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": true, + "memoryGiB": 0, + "name": "ml.geospatial.interactive", + "supportedImageNames": [ + "sagemaker-geospatial-v1-0" + ], + "vcpuNum": 0 + }, + { + "_defaultOrder": 21, + "_isFastLaunch": true, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.c5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 22, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.c5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 23, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.c5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 24, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.c5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 25, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 72, + "name": "ml.c5.9xlarge", + "vcpuNum": 36 + }, + { + "_defaultOrder": 26, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 96, + "name": "ml.c5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 27, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 144, + "name": "ml.c5.18xlarge", + "vcpuNum": 72 + }, + { + "_defaultOrder": 28, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.c5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 29, + "_isFastLaunch": true, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g4dn.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 30, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g4dn.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 31, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g4dn.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 32, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g4dn.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 33, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g4dn.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 34, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g4dn.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 35, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 61, + "name": "ml.p3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 36, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 244, + "name": "ml.p3.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 37, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 488, + "name": "ml.p3.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 38, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.p3dn.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 39, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.r5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 40, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.r5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 41, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.r5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 42, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.r5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 43, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.r5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 44, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.r5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 45, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 512, + "name": "ml.r5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 46, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.r5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 47, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 48, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 49, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 50, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 51, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 52, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 53, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.g5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 54, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.g5.48xlarge", + "vcpuNum": 192 + }, + { + "_defaultOrder": 55, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 56, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4de.24xlarge", + "vcpuNum": 96 + } + ], "instance_type": "ml.t3.medium", "interpreter": { "hash": "07c1d6c68b7b22b50965762993b154aa5a1dd6aa65a365988d7d4c27c573599b" }, "kernelspec": { - "display_name": "Python 3 (Data Science)", + "display_name": "Python 3 (Data Science 3.0)", "language": "python", - "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-1:742091327244:image/datascience-1.0" + "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:ap-southeast-1:492261229750:image/sagemaker-data-science-310-v1" }, "language_info": { "codemirror_mode": { @@ -644,7 +1326,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/build_pipeline/lineage.ipynb b/build_pipeline/lineage.ipynb new file mode 100644 index 0000000..c6d4870 --- /dev/null +++ b/build_pipeline/lineage.ipynb @@ -0,0 +1,918 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "378d8cc7-d9bf-46a5-ba2f-6bfd4f10e202", + "metadata": {}, + "source": [ + "# Lineage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d9d9445-d313-4655-8ce2-2393a97c2f92", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%pip install -q pyvis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9671b51-ec8f-4f76-b5fe-3c6a87c31722", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import pprint\n", + "\n", + "import sagemaker\n", + "from sagemaker.lineage.action import Action\n", + "from sagemaker.lineage.artifact import Artifact, DatasetArtifact, ModelArtifact\n", + "from sagemaker.lineage.association import Association\n", + "from sagemaker.lineage.context import Context, EndpointContext\n", + "from sagemaker.lineage.query import (\n", + " LineageEntityEnum,\n", + " LineageFilter,\n", + " LineageQuery,\n", + " LineageQueryDirectionEnum,\n", + " LineageSourceEnum,\n", + ")\n", + "from visualizer import Visualizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "738fede5-d1ae-45f5-b1b1-46efce30b014", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%store -r project_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e045166-c64a-44a3-b4f5-ddf89697d791", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sagemaker_session = sagemaker.Session()\n", + "region = sagemaker_session.boto_region_name\n", + "sm_client = sagemaker_session.sagemaker_client\n", + "default_bucket = sagemaker_session.default_bucket()\n", + "role = sagemaker.get_execution_role()\n", + "project_id = sm_client.describe_project(ProjectName=project_name)[\"ProjectId\"]\n", + "\n", + "# Helper function to print query outputs\n", + "pp = pprint.PrettyPrinter()\n", + "\n", + "print(f\"Project: {project_name} ({project_id})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec785a5a-642f-40d0-af54-24921c2b2cad", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "endpoint_name = f\"{project_name}-staging\"\n", + "endpoint_arn = sm_client.describe_endpoint(\n", + " EndpointName=endpoint_name,\n", + ").get(\"EndpointArn\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "342d98b2-2f3c-400c-bbaf-74bb5828c905", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "contexts = Context.list(source_uri=endpoint_arn)\n", + "context_name = list(contexts)[0].context_name\n", + "endpoint_context = EndpointContext.load(context_name=context_name)" + ] + }, + { + "cell_type": "markdown", + "id": "ca5e1902-af14-462c-befe-56feb95775a2", + "metadata": {}, + "source": [ + "### Registered model package from Endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5fd4417b-4515-43c1-be3c-52ff4dfaeadf", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "query_filter = LineageFilter(\n", + " entities=[LineageEntityEnum.ARTIFACT],\n", + " sources=[LineageSourceEnum.MODEL],\n", + " properties={\"ApprovalStatus\": \"Approved\"},\n", + ")\n", + "\n", + "# Providing this `LineageFilter` to the `LineageQuery` constructs a query that traverses through the given context `endpoint_context`\n", + "# and find all datasets.\n", + "\n", + "query_result = LineageQuery(sagemaker_session).query(\n", + " start_arns=[endpoint_context.context_arn],\n", + " query_filter=query_filter,\n", + " direction=LineageQueryDirectionEnum.ASCENDANTS,\n", + " include_edges=False,\n", + ")\n", + "\n", + "# Parse through the query results to get the lineage objects corresponding to the model\n", + "model_artifacts = []\n", + "for vertex in query_result.vertices:\n", + " model_artifacts.append(vertex.to_lineage_object().source.source_uri)\n", + "\n", + "# The results of the `LineageQuery` API call return the ARN of the model deployed to the endpoint\n", + "# along with the S3 URI to the model.tar.gz file associated with the model\n", + "pp.pprint(model_artifacts)" + ] + }, + { + "cell_type": "markdown", + "id": "feddb6b6-0450-421e-9f1a-8a4e3effad08", + "metadata": {}, + "source": [ + "### Pipeline from Endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a818005-bc09-4b68-9418-6f14ec56f1eb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "endpoint_context.pipeline_execution_arn()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5656b55-088e-44e2-b308-699af630458b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pipeline_arn = endpoint_context.pipeline_execution_arn().split(\"/execution\")[0]\n", + "pipeline_arn" + ] + }, + { + "cell_type": "markdown", + "id": "7d987be6-7bd6-456a-a30a-e2a6a6dbb7d0", + "metadata": {}, + "source": [ + "### Training job from Endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c71960fe-0640-4797-9d6d-bb8ed96ac772", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Define the LineageFilter to look for entities of type `TRIAL_COMPONENT` and the source of type `TRAINING_JOB`.\n", + "\n", + "query_filter = LineageFilter(\n", + " entities=[LineageEntityEnum.TRIAL_COMPONENT],\n", + " sources=[LineageSourceEnum.TRAINING_JOB],\n", + ")\n", + "\n", + "# Providing this `LineageFilter` to the `LineageQuery` constructs a query that traverses through the given context `endpoint_context`\n", + "# and find all datasets.\n", + "\n", + "query_result = LineageQuery(sagemaker_session).query(\n", + " start_arns=[endpoint_context.context_arn],\n", + " query_filter=query_filter,\n", + " direction=LineageQueryDirectionEnum.ASCENDANTS,\n", + " include_edges=False,\n", + ")\n", + "\n", + "# Parse through the query results to get the ARNs of the training jobs associated with this Endpoint\n", + "trial_components = []\n", + "for vertex in query_result.vertices:\n", + " trial_components.append(vertex.arn)\n", + "\n", + "pp.pprint(trial_components)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1eedb09-d482-4ec0-930d-aed93a203e5b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "training_jobs = [training_job for training_job in endpoint_context.training_job_arns()]\n", + "print(\"Training Jobs :\\n\", pp.pformat(training_jobs))" + ] + }, + { + "cell_type": "markdown", + "id": "a1678caa-fa7c-4eeb-9b58-efab7e96d01f", + "metadata": {}, + "source": [ + "### Datasets from Endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3b08a6c-ad68-4f8b-86e4-9e1480e9294c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(\"Datasets:\")\n", + "for dataset in endpoint_context.dataset_artifacts():\n", + " print(dataset.source.source_uri)" + ] + }, + { + "cell_type": "markdown", + "id": "f1086386-6da6-4aa4-8777-09bb17d100c1", + "metadata": {}, + "source": [ + "## Visualizing the lineage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8aa7aee-b51b-47d2-bb63-f4b6990aa1e5", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "viz = Visualizer()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c32f738d-b50c-4d38-b75f-0b87fb09d527", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "query_response = sm_client.query_lineage(\n", + " StartArns=[endpoint_context.context_arn],\n", + " Direction=\"Ascendants\",\n", + " IncludeEdges=True,\n", + ")\n", + "\n", + "viz.render(query_response, \"Endpoint\", height=500, width=1200)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0275b57-899b-4d0c-bf73-290fb5646abe", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "query_response = sm_client.query_lineage(\n", + " StartArns=[endpoint_context.context_arn],\n", + " Direction=\"Both\",\n", + " IncludeEdges=True,\n", + ")\n", + "viz.render(query_response, \"Endpoint\", height=700, width=1200)" + ] + } + ], + "metadata": { + "availableInstances": [ + { + "_defaultOrder": 0, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.t3.medium", + "vcpuNum": 2 + }, + { + "_defaultOrder": 1, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.t3.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 2, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.t3.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 3, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.t3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 4, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 5, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 6, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 7, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 8, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 9, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 10, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 11, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 12, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5d.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 13, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5d.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 14, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5d.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 15, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5d.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 16, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5d.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 17, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5d.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 18, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5d.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 19, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 20, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": true, + "memoryGiB": 0, + "name": "ml.geospatial.interactive", + "supportedImageNames": [ + "sagemaker-geospatial-v1-0" + ], + "vcpuNum": 0 + }, + { + "_defaultOrder": 21, + "_isFastLaunch": true, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.c5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 22, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.c5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 23, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.c5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 24, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.c5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 25, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 72, + "name": "ml.c5.9xlarge", + "vcpuNum": 36 + }, + { + "_defaultOrder": 26, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 96, + "name": "ml.c5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 27, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 144, + "name": "ml.c5.18xlarge", + "vcpuNum": 72 + }, + { + "_defaultOrder": 28, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.c5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 29, + "_isFastLaunch": true, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g4dn.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 30, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g4dn.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 31, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g4dn.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 32, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g4dn.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 33, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g4dn.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 34, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g4dn.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 35, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 61, + "name": "ml.p3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 36, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 244, + "name": "ml.p3.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 37, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 488, + "name": "ml.p3.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 38, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.p3dn.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 39, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.r5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 40, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.r5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 41, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.r5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 42, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.r5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 43, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.r5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 44, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.r5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 45, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 512, + "name": "ml.r5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 46, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.r5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 47, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 48, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 49, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 50, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 51, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 52, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 53, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.g5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 54, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.g5.48xlarge", + "vcpuNum": 192 + }, + { + "_defaultOrder": 55, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 56, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4de.24xlarge", + "vcpuNum": 96 + } + ], + "instance_type": "ml.t3.medium", + "kernelspec": { + "display_name": "Python 3 (Data Science 3.0)", + "language": "python", + "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:ap-southeast-1:492261229750:image/sagemaker-data-science-310-v1" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/build_pipeline/visualizer.py b/build_pipeline/visualizer.py new file mode 100644 index 0000000..25ce19e --- /dev/null +++ b/build_pipeline/visualizer.py @@ -0,0 +1,166 @@ +import os + +import matplotlib.pyplot as plt +import numpy as np +import sagemaker +from matplotlib.colors import rgb2hex +from pyvis.network import Network + +sagemaker_session = sagemaker.Session() +sm_client = sagemaker_session.sagemaker_client + +cmap = plt.get_cmap("Pastel2") +names = [ + "Approval", + "DataSet", + "Endpoint", + "Image", + "Model", + "ModelDeployment", + "ProcessingJob", + "TrainingJob", +] +colors = { + name: rgb2hex(color) + for name, color in zip(names, cmap(np.linspace(0, 1, len(names)))) +} + + +def create_legend_notes(): + # Add Legend Nodes + step = 50 + x = -300 + y = -250 + legend_nodes = [ + ( + name, + { + "group": legend_node, + "label": str(legend_node), + "size": 30, + # 'fixed': True, # So that we can move the legend nodes around to arrange them better + "physics": False, + "x": x, + "y": f"{y + legend_node*step}px", + "shape": "box", + "widthConstraint": 50, + "font": {"size": 20}, + "color": color, + }, + ) + for name, color in colors.items() + ] + print(legend_nodes) + return legend_nodes + + +class Visualizer: + def __init__(self): + self.directory = "generated" + if not os.path.exists(self.directory): + os.makedirs(self.directory) + + def render(self, query_lineage_response, scenario_name, height=600, width=1000): + net = self.get_network(height=height, width=width) + for vertex in query_lineage_response["Vertices"]: + arn, label, name, title, lineage_type = self.process_vertex(vertex) + if name.startswith("s3://"): + name = f"{name.split('/')[-1]}\n{name}" + + net.add_node( + arn, + label=label + "\n" + lineage_type, + title=name if name else title, + shape="circle", + color=colors[label], + ) + + # net.add_nodes_from(create_legend_notes()) + + for edge in query_lineage_response["Edges"]: + source = edge["SourceArn"] + dest = edge["DestinationArn"] + net.add_edge(dest, source) + + return net.show(f"{self.directory}/{scenario_name}.html") + + def get_title(self, arn): + return f"Arn: {arn}" + + def get_name(self, arn, lineage_type=None): + name = arn.split("/")[1] + return name + + def process_vertex(self, vertex): + arn = vertex["Arn"] + if "Type" in vertex: + label = vertex["Type"] + else: + label = None + lineage_type = vertex["LineageType"] + name = arn.split("/")[1] + if lineage_type in ["Artifact", "DataSet"]: + name = ( + sm_client.describe_artifact(ArtifactArn=arn) + .get("Source") + .get("SourceUri") + ) + if isinstance(name, list): + name = name[0] + if name.startswith("arn"): + name = name.split("/")[1] + + title = f"Artifact Arn: {arn}" + return arn, label, name, title, lineage_type + + def get_network(self, height: int = 600, width: int = 1000): + net = Network( + height=f"{height}px", + width=f"{width}px", + directed=True, + notebook=True, + cdn_resources="in_line", + ) + net.set_options( + """ + var options = { + "nodes": { + "borderWidth": 3, + "shadow": { + "enabled": true + }, + "shapeProperties": { + "borderRadius": 3 + }, + "size": 11, + "shape": "circle" + }, + "edges": { + "arrows": { + "to": { + "enabled": true + } + }, + "color": { + "inherit": true + }, + "smooth": false + }, + "layout": { + "hierarchical": { + "enabled": false, + "direction": "LR", + "sortMethod": "directed" + } + }, + "physics": { + "hierarchicalRepulsion": { + "centralGravity": 0 + }, + "minVelocity": 0.75, + "solver": "hierarchicalRepulsion" + } +} + """ + ) + return net From c890dbdb4b42068eda7c6e2cc3927bb02c7eef93 Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 19 Jun 2023 00:55:32 +0800 Subject: [PATCH 16/17] General bugfix - Added missing SSM:GetParameters permissions to CloudFormation Role - updated `batch-pipeline` to account for CSV format requirements for Batch Transform - Removed model Package Group from base CFN templates --- batch_pipeline/app.py | 22 +- batch_pipeline/infra/model_registry.py | 64 -- batch_pipeline/pipelines/pipeline.py | 16 +- build_pipeline/app.py | 1 - build_pipeline/batch-pipeline.ipynb | 720 +++++++++++++++++- .../infra/sagemaker_pipeline_stack.py | 9 - build_pipeline/lineage.ipynb | 30 +- build_pipeline/pipelines/pipeline.py | 11 +- build_pipeline/visualizer.py | 45 +- deployment_pipeline/infra/model_registry.py | 22 +- .../infra/sagemaker_lineage_utils.py | 2 +- deployment_pipeline/infra/sagemaker_stack.py | 17 +- infra/service_catalog_stack.py | 14 +- poetry.lock | 460 +++++++++-- pyproject.toml | 1 + 15 files changed, 1168 insertions(+), 266 deletions(-) diff --git a/batch_pipeline/app.py b/batch_pipeline/app.py index 4fb4297..f1a7185 100644 --- a/batch_pipeline/app.py +++ b/batch_pipeline/app.py @@ -49,14 +49,6 @@ def create_pipeline( )[0] batch_config.model_package_arn = p["ModelPackageArn"] - # Get the pipeline execution to get the baseline uri - pipeline_execution_arn = registry.get_pipeline_execution_arn( - batch_config.model_package_arn - ) - logger.info(f"Got pipeline execution arn: {pipeline_execution_arn}") - model_uri = registry.get_model_artifact(pipeline_execution_arn) - logger.info(f"Got model uri: {model_uri}") - # Set the sagemaker pipeline name and description with model version sagemaker_pipeline_name = f"{project_name}-batch-{stage_name}" sagemaker_pipeline_description = ( @@ -89,7 +81,7 @@ def main( region: str, sagemaker_pipeline_role_arn: str, artifact_bucket: str, - evaluate_drift_function_arn: str, + # evaluate_drift_function_arn: str, ): # Create App and stacks app = cdk.App() @@ -98,10 +90,8 @@ def main( app=app, project_name=project_name, project_id=project_id, - region=region, sagemaker_pipeline_role_arn=sagemaker_pipeline_role_arn, artifact_bucket=artifact_bucket, - evaluate_drift_function_arn=evaluate_drift_function_arn, stage_name="staging", ) @@ -109,10 +99,8 @@ def main( app=app, project_name=project_name, project_id=project_id, - region=region, sagemaker_pipeline_role_arn=sagemaker_pipeline_role_arn, artifact_bucket=artifact_bucket, - evaluate_drift_function_arn=evaluate_drift_function_arn, stage_name="prod", ) @@ -131,10 +119,10 @@ def main( "--sagemaker-pipeline-role-arn", default=os.environ.get("SAGEMAKER_PIPELINE_ROLE_ARN"), ) - parser.add_argument( - "--evaluate-drift-function-arn", - default=os.environ.get("EVALUATE_DRIFT_FUNCTION_ARN"), - ) + # parser.add_argument( + # "--evaluate-drift-function-arn", + # default=os.environ.get("EVALUATE_DRIFT_FUNCTION_ARN"), + # ) parser.add_argument( "--artifact-bucket", default=os.environ.get("ARTIFACT_BUCKET"), diff --git a/batch_pipeline/infra/model_registry.py b/batch_pipeline/infra/model_registry.py index dd0d33b..b0df616 100644 --- a/batch_pipeline/infra/model_registry.py +++ b/batch_pipeline/infra/model_registry.py @@ -202,67 +202,3 @@ def select_versioned_packages( p for p in model_packages if p["ModelPackageVersion"] == version ] return filtered_packages - - def get_pipeline_execution_arn(self, model_package_arn: str): - """Returns the execution arn for the latest approved model package - - Args: - model_package_arn: The arn of the model package - - Returns: - The arn of the sagemaker pipeline that created the model package. - """ - - artifact_arn = self.sm_client.list_artifacts(SourceUri=model_package_arn)[ - "ArtifactSummaries" - ][0]["ArtifactArn"] - return self.sm_client.describe_artifact(ArtifactArn=artifact_arn)[ - "MetadataProperties" - ]["GeneratedBy"] - - def get_model_artifact( - self, - pipeline_execution_arn: str, - step_name: str = "TrainModel", - ): - """Returns the training job model artifact uri for a given step name. - - Args: - pipeline_execution_arn: The pipeline execution arn - step_name: The optional training job step name - - Returns: - The model artifact from the training job - """ - - steps = self.sm_client.list_pipeline_execution_steps( - PipelineExecutionArn=pipeline_execution_arn - )["PipelineExecutionSteps"] - training_job_arn = [ - s["Metadata"]["TrainingJob"]["Arn"] - for s in steps - if s["StepName"] == step_name - ][0] - training_job_name = training_job_arn.split("/")[-1] - outputs = self.sm_client.describe_training_job( - TrainingJobName=training_job_name - ) - return outputs["ModelArtifacts"]["S3ModelArtifacts"] - - def get_data_check_baseline_uri(self, model_package_arn: str): - try: - model_details = self.sm_client.describe_model_package( - ModelPackageName=model_package_arn - ) - print(model_details) - baseline_uri = model_details["DriftCheckBaselines"]["ModelDataQuality"][ - "Constraints" - ]["S3Uri"] - baseline_uri = baseline_uri.replace( - "/constraints.json", "" - ) # returning the folder containing constraints and statistics - return baseline_uri - except ClientError as e: - error_message = e.response["Error"]["Message"] - logger.error(error_message) - raise Exception(error_message) diff --git a/batch_pipeline/pipelines/pipeline.py b/batch_pipeline/pipelines/pipeline.py index 38e4d50..ff054a9 100644 --- a/batch_pipeline/pipelines/pipeline.py +++ b/batch_pipeline/pipelines/pipeline.py @@ -9,7 +9,6 @@ """ import os -# from sagemaker.model import ModelPackage from sagemaker.model_monitor.dataset_format import DatasetFormat from sagemaker.s3 import S3Uploader from sagemaker.transformer import Transformer @@ -17,8 +16,6 @@ from sagemaker.workflow.check_job_config import CheckJobConfig from sagemaker.workflow.execution_variables import ExecutionVariables from sagemaker.workflow.functions import Join - -# from sagemaker.workflow.model_step import ModelStep from sagemaker.workflow.monitor_batch_transform_step import MonitorBatchTransformStep from sagemaker.workflow.parameters import ( ParameterBoolean, @@ -105,7 +102,7 @@ def get_pipeline( on="/", values=output_common_path + ["dataqualitycheck"], ), - post_analytics_processor_script="pipelines/postprocess_monitor_script.py" + post_analytics_processor_script="pipelines/postprocess_monitor_script.py", ) # Transform Step arguments @@ -153,14 +150,3 @@ def get_pipeline( ], sagemaker_session=sagemaker_session, ) - - -def upload_pipeline(pipeline: Pipeline, default_bucket, base_job_prefix) -> str: - # Get the pipeline definition - pipeline_definition_body = pipeline.definition() - # Upload the pipeline to a unique location in s3 based on git commit and timestamp - pipeline_key = f"{name_from_base(base_job_prefix)}/pipeline.json" - S3Uploader.upload_string_as_file_body( - pipeline_definition_body, f"s3://{default_bucket}/{pipeline_key}" - ) - return pipeline_key diff --git a/build_pipeline/app.py b/build_pipeline/app.py index 4b96874..4f47e82 100644 --- a/build_pipeline/app.py +++ b/build_pipeline/app.py @@ -31,7 +31,6 @@ def main( default_bucket=artifact_bucket, model_package_group_name=model_package_group_name, pipeline_name=sagemaker_pipeline_name, - base_job_prefix=project_id, commit_id=commit_id, ) diff --git a/build_pipeline/batch-pipeline.ipynb b/build_pipeline/batch-pipeline.ipynb index c2588da..ffb5a71 100644 --- a/build_pipeline/batch-pipeline.ipynb +++ b/build_pipeline/batch-pipeline.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -10,6 +11,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -21,13 +23,16 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "%store -r project_name" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -37,12 +42,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "import sagemaker\n", "import json\n", "\n", + "import sagemaker\n", + "\n", "sess = sagemaker.session.Session()\n", "region_name = sess._region_name\n", "sm_client = sess.sagemaker_client\n", @@ -53,6 +61,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -62,7 +71,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from IPython.core.display import HTML\n", @@ -73,6 +84,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -84,13 +96,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ + "import random\n", + "\n", "import boto3\n", "import pandas as pd\n", - "import random\n", - "from sagemaker.s3 import S3Downloader, S3Uploader\n", "\n", "\n", "def get_latest_processed_data(pipeline_name, step_name, output_name):\n", @@ -100,14 +114,14 @@ " steps = sm_client.list_pipeline_execution_steps(\n", " PipelineExecutionArn=execution_arn, SortOrder=\"Ascending\"\n", " )[\"PipelineExecutionSteps\"]\n", - " if step_name == \"ModelMonitor\":\n", + " if \"monitoring\" in step_name:\n", " links = next(\n", " item[\"Metadata\"][\"QualityCheck\"]\n", " for item in steps\n", " if item[\"StepName\"] == step_name\n", " )\n", - " return list(filter(lambda x: str(x).startswith(\"s3:/\"),links.values()))\n", - " \n", + " return list(filter(lambda x: str(x).startswith(\"s3:/\"), links.values()))\n", + "\n", " preprocess_arn = next(\n", " item[\"Metadata\"][\"ProcessingJob\"][\"Arn\"]\n", " for item in steps\n", @@ -125,15 +139,15 @@ "\n", "pipeline_name = f\"{project_name}-build\"\n", "test_uri = get_latest_processed_data(pipeline_name, \"PreprocessData\", \"test\")\n", - "S3Downloader().download(test_uri, \"preprocessed\")\n", "\n", "# Load the test scores into a dataframe\n", - "test_df = pd.read_csv(\"preprocessed/test.csv\")\n", + "test_df = pd.read_csv(f\"{test_uri}/test.csv\")\n", "print(test_df.shape)\n", - "test_df.head()" + "test_df" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -143,14 +157,17 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "batch_staging_uri = f\"s3://{artifact_bucket}/{project_id}/batch/staging\"\n", - "S3Uploader().upload(\"preprocessed/test.csv\", batch_staging_uri);" + "batch_staging_uri = f\"s3://{artifact_bucket}/batch-input/staging/test.csv\"\n", + "test_df.to_csv(batch_staging_uri, header=False, index=False)" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -164,7 +181,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from sagemaker.workflow.pipeline import Pipeline\n", @@ -173,7 +192,7 @@ "pipeline = Pipeline(pipeline_name)\n", "\n", "# Start pipeline\n", - "execution = pipeline.start()\n", + "execution = pipeline.start(parameters={\"DataInputUri\": batch_staging_uri})\n", "execution_name = execution.arn.split(\"/\")[-1]\n", "\n", "print(f\"Waiting for execution: {execution_name} for pipeline {pipeline_name}...\")\n", @@ -183,6 +202,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -192,25 +212,57 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "staging_scores_uri = get_latest_processed_data(pipeline_name, \"ScoreModel\", \"scores\")\n", - "S3Downloader().download(staging_scores_uri, \"staging\")\n", + "from sagemaker.experiments.trial import _Trial\n", + "from sagemaker.experiments.trial_component import _TrialComponent\n", + "\n", + "latest_trial_name = list(\n", + " _Trial.list(\n", + " experiment_name=pipeline_name, sort_by=\"CreationTime\", sort_order=\"Descending\"\n", + " )\n", + ")[0].trial_name\n", + "transform_trial_component_name = [\n", + " k\n", + " for k in _TrialComponent.list(trial_name=latest_trial_name)\n", + " if k.trial_component_source.get(\"SourceType\") == \"SageMakerTransformJob\"\n", + "][0].trial_component_name\n", "\n", + "trial_component = _TrialComponent.load(\n", + " trial_component_name=transform_trial_component_name\n", + ")\n", + "transformed_data_uri = trial_component.output_artifacts[\n", + " \"SageMaker.TransformOutput\"\n", + "].value" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ "# Load the predicted scores\n", - "pred_df = pd.read_csv(\"staging/scores.csv\")\n", - "pred_df = pred_df.rename(columns={\"fare_amount\": \"fare_amount_prediction\"})\n", - "pred_df.head()" + "pred_df = pd.read_csv(\n", + " f\"{transformed_data_uri}/test.csv.out\",\n", + " names=[\"fare_amount_prediction\"],\n", + ")\n", + "pred_df" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### Evaluate\n", "\n", - "Calculate the root mean squre error (RMSE) to evaluate the performance of this model. " + "Calculate the root mean square error (RMSE) to evaluate the performance of this model. " ] }, { @@ -219,19 +271,21 @@ "metadata": {}, "outputs": [], "source": [ - "from sklearn.metrics import mean_squared_error\n", "from math import sqrt\n", "\n", + "from sklearn.metrics import mean_squared_error\n", + "\n", "mse = mean_squared_error(test_df[\"fare_amount\"], pred_df[\"fare_amount_prediction\"])\n", "rmse = sqrt(mse)\n", "print(f\"RMSE: {rmse}\")" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Plot the residules to see where the errors are relative to the fare amount." + "Plot the residues to see where the errors are relative to the fare amount." ] }, { @@ -248,6 +302,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -257,6 +312,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -280,12 +336,11 @@ ")\n", "test_df[\"geo_distance\"] = test_df[\"passenger_count\"].apply(\n", " lambda x: 70 * random.betavariate(2.5, 2)\n", - ")\n", - "\n", - "test_df.to_csv(\"preprocessed/tweaked.csv\", header=True, index=False)" + ")" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -295,14 +350,17 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "batch_prod_uri = f\"s3://{artifact_bucket}/{project_id}/batch/prod\"\n", - "S3Uploader().upload(\"preprocessed/tweaked.csv\", batch_prod_uri);" + "batch_prod_uri = f\"s3://{artifact_bucket}/batch-input/prod/tweaked.csv\"\n", + "test_df.to_csv(batch_prod_uri, header=False, index=False)" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -321,7 +379,7 @@ "pipeline = Pipeline(pipeline_name)\n", "\n", "# Start pipeline\n", - "execution = pipeline.start()\n", + "execution = pipeline.start(parameters={\"DataInputUri\": batch_prod_uri})\n", "execution_name = execution.arn.split(\"/\")[-1]\n", "\n", "print(f\"Waiting for execution: {execution_name} for pipeline {pipeline_name}...\")\n", @@ -331,6 +389,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -348,6 +407,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -363,7 +423,7 @@ "outputs": [], "source": [ "monitor_uri = get_latest_processed_data(\n", - " pipeline_name, \"ModelMonitor\", \"monitoring_output\"\n", + " pipeline_name, \"MonitorDataQuality-monitoring\", \"monitoring_output\"\n", ")\n", "\n", "print(\"Downloading monitor files:\")\n", @@ -373,6 +433,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -399,6 +460,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -425,6 +487,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -438,6 +501,7 @@ "outputs": [], "source": [ "from datetime import datetime, timedelta\n", + "\n", "from dateutil.tz import tzlocal\n", "\n", "pipeline_name = f\"{project_name}-build\"\n", @@ -454,6 +518,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -476,6 +541,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -496,6 +562,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -503,6 +570,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -534,6 +602,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -542,14 +611,589 @@ } ], "metadata": { + "availableInstances": [ + { + "_defaultOrder": 0, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.t3.medium", + "vcpuNum": 2 + }, + { + "_defaultOrder": 1, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.t3.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 2, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.t3.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 3, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.t3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 4, + "_isFastLaunch": true, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 5, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 6, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 7, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 8, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 9, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 10, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 11, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 12, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.m5d.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 13, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.m5d.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 14, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.m5d.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 15, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.m5d.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 16, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.m5d.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 17, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.m5d.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 18, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.m5d.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 19, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.m5d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 20, + "_isFastLaunch": false, + "category": "General purpose", + "gpuNum": 0, + "hideHardwareSpecs": true, + "memoryGiB": 0, + "name": "ml.geospatial.interactive", + "supportedImageNames": [ + "sagemaker-geospatial-v1-0" + ], + "vcpuNum": 0 + }, + { + "_defaultOrder": 21, + "_isFastLaunch": true, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 4, + "name": "ml.c5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 22, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 8, + "name": "ml.c5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 23, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.c5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 24, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.c5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 25, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 72, + "name": "ml.c5.9xlarge", + "vcpuNum": 36 + }, + { + "_defaultOrder": 26, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 96, + "name": "ml.c5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 27, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 144, + "name": "ml.c5.18xlarge", + "vcpuNum": 72 + }, + { + "_defaultOrder": 28, + "_isFastLaunch": false, + "category": "Compute optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.c5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 29, + "_isFastLaunch": true, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g4dn.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 30, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g4dn.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 31, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g4dn.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 32, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g4dn.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 33, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g4dn.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 34, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g4dn.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 35, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 61, + "name": "ml.p3.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 36, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 244, + "name": "ml.p3.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 37, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 488, + "name": "ml.p3.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 38, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.p3dn.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 39, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.r5.large", + "vcpuNum": 2 + }, + { + "_defaultOrder": 40, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.r5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 41, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.r5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 42, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.r5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 43, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.r5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 44, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.r5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 45, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 512, + "name": "ml.r5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 46, + "_isFastLaunch": false, + "category": "Memory Optimized", + "gpuNum": 0, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.r5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 47, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 16, + "name": "ml.g5.xlarge", + "vcpuNum": 4 + }, + { + "_defaultOrder": 48, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 32, + "name": "ml.g5.2xlarge", + "vcpuNum": 8 + }, + { + "_defaultOrder": 49, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 64, + "name": "ml.g5.4xlarge", + "vcpuNum": 16 + }, + { + "_defaultOrder": 50, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 128, + "name": "ml.g5.8xlarge", + "vcpuNum": 32 + }, + { + "_defaultOrder": 51, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 1, + "hideHardwareSpecs": false, + "memoryGiB": 256, + "name": "ml.g5.16xlarge", + "vcpuNum": 64 + }, + { + "_defaultOrder": 52, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 192, + "name": "ml.g5.12xlarge", + "vcpuNum": 48 + }, + { + "_defaultOrder": 53, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 4, + "hideHardwareSpecs": false, + "memoryGiB": 384, + "name": "ml.g5.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 54, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 768, + "name": "ml.g5.48xlarge", + "vcpuNum": 192 + }, + { + "_defaultOrder": 55, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4d.24xlarge", + "vcpuNum": 96 + }, + { + "_defaultOrder": 56, + "_isFastLaunch": false, + "category": "Accelerated computing", + "gpuNum": 8, + "hideHardwareSpecs": false, + "memoryGiB": 1152, + "name": "ml.p4de.24xlarge", + "vcpuNum": 96 + } + ], "instance_type": "ml.t3.medium", "interpreter": { "hash": "07c1d6c68b7b22b50965762993b154aa5a1dd6aa65a365988d7d4c27c573599b" }, "kernelspec": { - "display_name": "Python 3 (Data Science)", + "display_name": "Python 3 (Data Science 3.0)", "language": "python", - "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-1:742091327244:image/datascience-1.0" + "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:ap-southeast-1:492261229750:image/sagemaker-data-science-310-v1" }, "language_info": { "codemirror_mode": { @@ -561,7 +1205,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/build_pipeline/infra/sagemaker_pipeline_stack.py b/build_pipeline/infra/sagemaker_pipeline_stack.py index 14ad0a0..bfa8127 100644 --- a/build_pipeline/infra/sagemaker_pipeline_stack.py +++ b/build_pipeline/infra/sagemaker_pipeline_stack.py @@ -25,15 +25,6 @@ def __init__( ) -> None: super().__init__(scope, construct_id, **kwargs) - - sagemaker.CfnModelPackageGroup( - self, - "ModelPackageGroup", - model_package_group_name=model_package_group_name, - model_package_group_description=pipeline_description, - tags=tags, - ) - sagemaker.CfnPipeline( self, "Pipeline", diff --git a/build_pipeline/lineage.ipynb b/build_pipeline/lineage.ipynb index c6d4870..40444f2 100644 --- a/build_pipeline/lineage.ipynb +++ b/build_pipeline/lineage.ipynb @@ -1,11 +1,18 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "378d8cc7-d9bf-46a5-ba2f-6bfd4f10e202", "metadata": {}, "source": [ - "# Lineage" + "# Lineage\n", + "\n", + "This notebook shows how to use [SageMaker Lineage](https://sagemaker.readthedocs.io/en/stable/workflows/lineage/index.html) to retrieve the links between the different steps of the ML workflow.\n", + "\n", + "***\n", + "This notebook has been written for `Data Science 3.0 (Python 3) kernel`\n", + "***" ] }, { @@ -110,11 +117,12 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "ca5e1902-af14-462c-befe-56feb95775a2", "metadata": {}, "source": [ - "### Registered model package from Endpoint" + "## Registered model package from Endpoint" ] }, { @@ -153,11 +161,12 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "feddb6b6-0450-421e-9f1a-8a4e3effad08", "metadata": {}, "source": [ - "### Pipeline from Endpoint" + "## Pipeline and pipeline execution from Endpoint" ] }, { @@ -186,11 +195,12 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7d987be6-7bd6-456a-a30a-e2a6a6dbb7d0", "metadata": {}, "source": [ - "### Training job from Endpoint" + "## Training job from Endpoint" ] }, { @@ -241,11 +251,12 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "a1678caa-fa7c-4eeb-9b58-efab7e96d01f", "metadata": {}, "source": [ - "### Datasets from Endpoint" + "## Datasets from Endpoint" ] }, { @@ -263,6 +274,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "f1086386-6da6-4aa4-8777-09bb17d100c1", "metadata": {}, @@ -316,6 +328,14 @@ ")\n", "viz.render(query_response, \"Endpoint\", height=700, width=1200)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec779c5f-c257-428f-950a-26af12a37a8a", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/build_pipeline/pipelines/pipeline.py b/build_pipeline/pipelines/pipeline.py index f15d5af..c41257e 100644 --- a/build_pipeline/pipelines/pipeline.py +++ b/build_pipeline/pipelines/pipeline.py @@ -73,7 +73,6 @@ def get_pipeline( pipeline_name, model_package_group_name, default_bucket, - base_job_prefix, commit_id: str = None, ): """Gets a SageMaker ML Pipeline instance working with on nyc taxi data. @@ -83,7 +82,6 @@ def get_pipeline( default_bucket: the bucket to use for storing the artifacts pipeline_name: the bucket to use for storing the artifacts model_package_group_name: the model package group name - base_job_prefix: the prefix to include after the bucket Returns: an instance of a pipeline """ @@ -126,7 +124,7 @@ def get_pipeline( output_common_path = [ "s3:/", default_bucket, - "build-pipeline-runs", + pipeline_name, Join( on="-", values=[ @@ -324,7 +322,7 @@ def get_pipeline( step_eval = ProcessingStep( name="EvaluateModel", step_args=script_eval.run( - job_name=f"scripts/{commit_id[:8]}/evaluation", + # job_name=f"scripts/{commit_id[:8]}/evaluation", inputs=[ ProcessingInput( source=step_train.properties.ModelArtifacts.S3ModelArtifacts, @@ -397,6 +395,7 @@ def get_pipeline( step_register = ModelStep( name="RegisterModel", + display_name="RegisterModel", step_args=model.register( content_types=["text/csv"], response_types=["text/csv"], @@ -461,11 +460,11 @@ def get_pipeline( return pipeline -def upload_pipeline(pipeline: Pipeline, default_bucket, base_job_prefix): +def upload_pipeline(pipeline: Pipeline, default_bucket): # Get the pipeline definition pipeline_definition_body = pipeline.definition() # Upload the pipeline to a unique location in s3 based on git commit and timestamp - pipeline_name = name_from_base(f"{base_job_prefix}/pipeline") + pipeline_name = name_from_base(f"{pipeline.name}/pipeline-definition") S3Uploader.upload_string_as_file_body( pipeline_definition_body, f"s3://{default_bucket}/{pipeline_name}.json" ) diff --git a/build_pipeline/visualizer.py b/build_pipeline/visualizer.py index 25ce19e..8e3d70c 100644 --- a/build_pipeline/visualizer.py +++ b/build_pipeline/visualizer.py @@ -16,6 +16,7 @@ "Endpoint", "Image", "Model", + "ModelGroup", "ModelDeployment", "ProcessingJob", "TrainingJob", @@ -26,32 +27,30 @@ } -def create_legend_notes(): +def create_legend_notes(net): # Add Legend Nodes step = 50 - x = -300 + x = -500 y = -250 legend_nodes = [ - ( - name, - { - "group": legend_node, - "label": str(legend_node), - "size": 30, - # 'fixed': True, # So that we can move the legend nodes around to arrange them better - "physics": False, - "x": x, - "y": f"{y + legend_node*step}px", - "shape": "box", - "widthConstraint": 50, - "font": {"size": 20}, - "color": color, - }, - ) - for name, color in colors.items() + dict( + n_id=legend_node, + label=label, + # group=legend_node, + physics=False, + size=30, + x=x, + y=f"{y + legend_node*step}px", + shape="box", + font={"size": 20}, + color=colors[label], + ) + for legend_node, label in enumerate(names) ] - print(legend_nodes) - return legend_nodes + # print(legend_nodes) + [net.add_node(**node) for node in legend_nodes] + + return class Visualizer: @@ -74,8 +73,8 @@ def render(self, query_lineage_response, scenario_name, height=600, width=1000): shape="circle", color=colors[label], ) - - # net.add_nodes_from(create_legend_notes()) + + create_legend_notes(net) for edge in query_lineage_response["Edges"]: source = edge["SourceArn"] diff --git a/deployment_pipeline/infra/model_registry.py b/deployment_pipeline/infra/model_registry.py index e7b805b..9df3f57 100644 --- a/deployment_pipeline/infra/model_registry.py +++ b/deployment_pipeline/infra/model_registry.py @@ -165,7 +165,10 @@ def get_versioned_approved_packages( # Return error if no packages found if len(model_packages) == 0: - error_message = f"No approved packages found for: {model_package_group_name} and versions: {model_package_versions}" + error_message = ( + f"No approved packages found for: {model_package_group_name} " + f"and versions: {model_package_versions}" + ) logger.error(error_message) raise Exception(error_message) @@ -200,23 +203,6 @@ def select_versioned_packages( ] return filtered_packages - def get_pipeline_execution_arn(self, model_package_arn: str): - """Get the execution arn for the latest approved model package - - Args: - model_package_arn: The arn of the model package - - Returns: - The arn of the sagemaker pipeline that created the model package. - """ - - artifact_arn = self.sm_client.list_artifacts(SourceUri=model_package_arn)[ - "ArtifactSummaries" - ][0]["ArtifactArn"] - return self.sm_client.describe_artifact(ArtifactArn=artifact_arn)[ - "MetadataProperties" - ]["GeneratedBy"] - def get_data_check_baseline_uri(self, model_package_arn: str): try: model_details = self.sm_client.describe_model_package( diff --git a/deployment_pipeline/infra/sagemaker_lineage_utils.py b/deployment_pipeline/infra/sagemaker_lineage_utils.py index 0463190..d987a29 100644 --- a/deployment_pipeline/infra/sagemaker_lineage_utils.py +++ b/deployment_pipeline/infra/sagemaker_lineage_utils.py @@ -18,7 +18,7 @@ def get_pipeline_arn_from_endpoint(endpoint_name: str): logger.info(f"Endpoint: {endpoint_name}") - endpoint_arn = f"arn:aws:sagemaker:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:endpoint/{endpoint_name}" + endpoint_arn = f"arn:aws:sagemaker:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:endpoint/{endpoint_name}" # noqa: E501 contexts = Context.list( source_uri=endpoint_arn, diff --git a/deployment_pipeline/infra/sagemaker_stack.py b/deployment_pipeline/infra/sagemaker_stack.py index 766a312..a8c0f50 100644 --- a/deployment_pipeline/infra/sagemaker_stack.py +++ b/deployment_pipeline/infra/sagemaker_stack.py @@ -5,6 +5,7 @@ from aws_cdk import aws_cloudwatch as cloudwatch from aws_cdk import aws_events as events from aws_cdk import aws_sagemaker as sagemaker +from aws_cdk.aws_sagemaker import CfnEndpointConfig from constructs import Construct from infra.sagemaker_lineage_utils import get_pipeline_arn_from_model @@ -47,7 +48,7 @@ def __init__( ) # Create the production variant - model_variant = sagemaker.CfnEndpointConfig.ProductionVariantProperty( + model_variant = CfnEndpointConfig.ProductionVariantProperty( initial_instance_count=variant_config.instance_count, initial_variant_weight=variant_config.initial_variant_weight, instance_type=variant_config.instance_type, @@ -55,7 +56,7 @@ def __init__( variant_name=variant_name, ) - endpoint_config = sagemaker.CfnEndpointConfig( + endpoint_config = CfnEndpointConfig( self, "EndpointConfig", production_variants=[model_variant], @@ -63,19 +64,15 @@ def __init__( # Enable data capture for scheduling if deployment_config.schedule_config is not None: - endpoint_config.data_capture_config = sagemaker.CfnEndpointConfig.DataCaptureConfigProperty( + endpoint_config.data_capture_config = CfnEndpointConfig.DataCaptureConfigProperty( # noqa: E501 enable_capture=True, destination_s3_uri=data_capture_uri, initial_sampling_percentage=deployment_config.schedule_config.data_capture_sampling_percentage, capture_options=[ - sagemaker.CfnEndpointConfig.CaptureOptionProperty( - capture_mode="Input" - ), - sagemaker.CfnEndpointConfig.CaptureOptionProperty( - capture_mode="Output" - ), + CfnEndpointConfig.CaptureOptionProperty(capture_mode="Input"), + CfnEndpointConfig.CaptureOptionProperty(capture_mode="Output"), ], - capture_content_type_header=sagemaker.CfnEndpointConfig.CaptureContentTypeHeaderProperty( + capture_content_type_header=CfnEndpointConfig.CaptureContentTypeHeaderProperty( csv_content_types=["text/csv"], json_content_types=["application/json"], ), diff --git a/infra/service_catalog_stack.py b/infra/service_catalog_stack.py index 1ec6570..aaa4a01 100644 --- a/infra/service_catalog_stack.py +++ b/infra/service_catalog_stack.py @@ -223,7 +223,19 @@ def __init__( ], ) ) - + cloudformation_role.add_to_principal_policy( + iam.PolicyStatement( + actions=[ + "ssm:GetParameters", + ], + resources=[ + "*", + self.format_arn( + resource="parameter", service="ssm", resource_name="sagemaker*" + ), + ], + ) + ) cloudformation_role.add_to_principal_policy( iam.PolicyStatement( actions=[ diff --git a/poetry.lock b/poetry.lock index 0bf1eea..da4dbd4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,37 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "appnope" +version = "0.1.3" +description = "Disable App Nap on macOS >= 10.9" +optional = false +python-versions = "*" +files = [ + {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, + {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, +] + +[[package]] +name = "asttokens" +version = "2.2.1" +description = "Annotate AST trees with source code positions" +optional = false +python-versions = "*" +files = [ + {file = "asttokens-2.2.1-py2.py3-none-any.whl", hash = "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"}, + {file = "asttokens-2.2.1.tar.gz", hash = "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3"}, +] + +[package.dependencies] +six = "*" + +[package.extras] +test = ["astroid", "pytest"] [[package]] name = "attrs" version = "22.2.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -23,7 +50,6 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy name = "aws-cdk-asset-awscli-v1" version = "2.2.172" description = "A library that contains the AWS CLI for use in Lambda Layers" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -40,7 +66,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "aws-cdk-asset-kubectl-v20" version = "2.1.1" description = "A library that contains kubectl for use in Lambda Layers" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -57,7 +82,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "aws-cdk-asset-node-proxy-agent-v5" version = "2.0.145" description = "@aws-cdk/asset-node-proxy-agent-v5" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -74,7 +98,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "aws-cdk-lib" version = "2.79.1" description = "Version 2 of the AWS Cloud Development Kit library" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -91,11 +114,21 @@ jsii = ">=1.80.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" +[[package]] +name = "backcall" +version = "0.2.0" +description = "Specifications for callback functions passed in to an API" +optional = false +python-versions = "*" +files = [ + {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, + {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, +] + [[package]] name = "black" version = "22.12.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -131,7 +164,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "boto3" version = "1.26.133" description = "The AWS SDK for Python" -category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -151,7 +183,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.133" description = "Low-level, data-driven core of boto 3." -category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -171,7 +202,6 @@ crt = ["awscrt (==0.16.9)"] name = "cattrs" version = "22.2.0" description = "Composable complex class support for attrs and dataclasses." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -187,7 +217,6 @@ exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -202,7 +231,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "cloudpickle" version = "2.2.1" description = "Extended pickling support for Python objects" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -214,7 +242,6 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -226,7 +253,6 @@ files = [ name = "constructs" version = "10.2.23" description = "A programming model for software-defined state" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -243,7 +269,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "contextlib2" version = "21.6.0" description = "Backports and enhancements for the contextlib module" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -251,11 +276,21 @@ files = [ {file = "contextlib2-21.6.0.tar.gz", hash = "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869"}, ] +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + [[package]] name = "dill" version = "0.3.6" description = "serialize all of python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -270,7 +305,6 @@ graph = ["objgraph (>=1.7.2)"] name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -281,11 +315,24 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "executing" +version = "1.2.0" +description = "Get the currently executing AST node of a frame, and other information" +optional = false +python-versions = "*" +files = [ + {file = "executing-1.2.0-py2.py3-none-any.whl", hash = "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc"}, + {file = "executing-1.2.0.tar.gz", hash = "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"}, +] + +[package.extras] +tests = ["asttokens", "littleutils", "pytest", "rich"] + [[package]] name = "google-pasta" version = "0.2.0" description = "pasta is an AST-based Python refactoring library" -category = "dev" optional = false python-versions = "*" files = [ @@ -301,7 +348,6 @@ six = "*" name = "importlib-metadata" version = "4.13.0" description = "Read metadata from Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -321,7 +367,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -336,11 +381,85 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "ipython" +version = "8.14.0" +description = "IPython: Productive Interactive Computing" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ipython-8.14.0-py3-none-any.whl", hash = "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"}, + {file = "ipython-8.14.0.tar.gz", hash = "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1"}, +] + +[package.dependencies] +appnope = {version = "*", markers = "sys_platform == \"darwin\""} +backcall = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +pickleshare = "*" +prompt-toolkit = ">=3.0.30,<3.0.37 || >3.0.37,<3.1.0" +pygments = ">=2.4.0" +stack-data = "*" +traitlets = ">=5" +typing-extensions = {version = "*", markers = "python_version < \"3.10\""} + +[package.extras] +all = ["black", "curio", "docrepr", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.21)", "pandas", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] +black = ["black"] +doc = ["docrepr", "ipykernel", "matplotlib", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["ipywidgets", "notebook"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["pytest (<7.1)", "pytest-asyncio", "testpath"] +test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pandas", "pytest (<7.1)", "pytest-asyncio", "testpath", "trio"] + +[[package]] +name = "jedi" +version = "0.18.2" +description = "An autocompletion tool for Python that can be used for text editors." +optional = false +python-versions = ">=3.6" +files = [ + {file = "jedi-0.18.2-py2.py3-none-any.whl", hash = "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e"}, + {file = "jedi-0.18.2.tar.gz", hash = "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"}, +] + +[package.dependencies] +parso = ">=0.8.0,<0.9.0" + +[package.extras] +docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -352,7 +471,6 @@ files = [ name = "joblib" version = "1.2.0" description = "Lightweight pipelining with Python functions" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -364,7 +482,6 @@ files = [ name = "jsii" version = "1.81.0" description = "Python client for jsii runtime" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -381,11 +498,26 @@ python-dateutil = "*" typeguard = ">=2.13.3,<2.14.0" typing-extensions = ">=3.7,<5.0" +[[package]] +name = "jsonpickle" +version = "3.0.1" +description = "Python library for serializing any arbitrary object graph into JSON" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonpickle-3.0.1-py2.py3-none-any.whl", hash = "sha256:130d8b293ea0add3845de311aaba55e6d706d0bb17bc123bd2c8baf8a39ac77c"}, + {file = "jsonpickle-3.0.1.tar.gz", hash = "sha256:032538804795e73b94ead410800ac387fdb6de98f8882ac957fcd247e3a85200"}, +] + +[package.extras] +docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] +testing-libs = ["simplejson", "ujson"] + [[package]] name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -401,11 +533,83 @@ pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +description = "Inline Matplotlib backend for Jupyter" +optional = false +python-versions = ">=3.5" +files = [ + {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, + {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, +] + +[package.dependencies] +traitlets = "*" + [[package]] name = "multiprocess" version = "0.70.14" description = "better multiprocessing and multithreading in python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -432,7 +636,6 @@ dill = ">=0.3.6" name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -440,11 +643,28 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "networkx" +version = "3.1" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.8" +files = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "numpy" version = "1.24.3" description = "Fundamental package for array computing in Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -482,7 +702,6 @@ files = [ name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -494,7 +713,6 @@ files = [ name = "pandas" version = "2.0.1" description = "Powerful data structures for data analysis, time series, and statistics" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -558,11 +776,25 @@ sql-other = ["SQLAlchemy (>=1.4.16)"] test = ["hypothesis (>=6.34.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.6.3)"] +[[package]] +name = "parso" +version = "0.8.3" +description = "A Python Parser" +optional = false +python-versions = ">=3.6" +files = [ + {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, + {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, +] + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + [[package]] name = "pathos" version = "0.3.0" description = "parallel graph management and execution in heterogeneous computing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -580,7 +812,6 @@ ppft = ">=1.7.6.6" name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -588,11 +819,35 @@ files = [ {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, ] +[[package]] +name = "pexpect" +version = "4.8.0" +description = "Pexpect allows easy control of interactive console applications." +optional = false +python-versions = "*" +files = [ + {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, + {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, +] + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "pickleshare" +version = "0.7.5" +description = "Tiny 'shelve'-like database with concurrency support" +optional = false +python-versions = "*" +files = [ + {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, + {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, +] + [[package]] name = "platformdirs" version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -608,7 +863,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pox" version = "0.3.2" description = "utilities for filesystem exploration and automated builds" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -620,7 +874,6 @@ files = [ name = "ppft" version = "1.7.6.6" description = "distributed and parallel python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -631,11 +884,24 @@ files = [ [package.extras] dill = ["dill (>=0.3.6)"] +[[package]] +name = "prompt-toolkit" +version = "3.0.38" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.38-py3-none-any.whl", hash = "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"}, + {file = "prompt_toolkit-3.0.38.tar.gz", hash = "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b"}, +] + +[package.dependencies] +wcwidth = "*" + [[package]] name = "protobuf" version = "3.20.3" description = "Protocol Buffers" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -667,7 +933,6 @@ files = [ name = "protobuf3-to-dict" version = "0.1.5" description = "Ben Hodgson: A teeny Python library for creating Python dicts from protocol buffers and the reverse. Useful as an intermediate step before serialisation (e.g. to JSON). Kapor: upgrade it to PB3 and PY3, rename it to protobuf3-to-dict" -category = "dev" optional = false python-versions = "*" files = [ @@ -678,11 +943,21 @@ files = [ protobuf = ">=2.3.0" six = "*" +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +optional = false +python-versions = "*" +files = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] + [[package]] name = "publication" version = "0.0.3" description = "Publication helps you maintain public-api-friendly modules by preventing unintentional access to private implementation details via introspection." -category = "main" optional = false python-versions = "*" files = [ @@ -690,11 +965,38 @@ files = [ {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, ] +[[package]] +name = "pure-eval" +version = "0.2.2" +description = "Safely evaluate AST nodes without side effects" +optional = false +python-versions = "*" +files = [ + {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, + {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, +] + +[package.extras] +tests = ["pytest"] + +[[package]] +name = "pygments" +version = "2.15.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, + {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, +] + +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -731,7 +1033,6 @@ files = [ name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -746,7 +1047,6 @@ six = ">=1.5" name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "dev" optional = false python-versions = "*" files = [ @@ -754,11 +1054,26 @@ files = [ {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] +[[package]] +name = "pyvis" +version = "0.3.2" +description = "A Python network graph visualization library" +optional = false +python-versions = ">3.6" +files = [ + {file = "pyvis-0.3.2-py3-none-any.whl", hash = "sha256:5720c4ca8161dc5d9ab352015723abb7a8bb8fb443edeb07f7a322db34a97555"}, +] + +[package.dependencies] +ipython = ">=5.3.0" +jinja2 = ">=2.9.6" +jsonpickle = ">=1.4.1" +networkx = ">=1.11" + [[package]] name = "pyyaml" version = "5.4.1" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -797,7 +1112,6 @@ files = [ name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" -category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -815,7 +1129,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "sagemaker" version = "2.154.0" description = "Open source library for training and deploying models on Amazon SageMaker." -category = "dev" optional = false python-versions = ">= 3.6" files = [ @@ -851,7 +1164,6 @@ test = ["Jinja2 (==3.0.3)", "PyYAML (==5.4.1)", "apache-airflow (==2.5.1)", "apa name = "schema" version = "0.7.5" description = "Simple data validation library" -category = "dev" optional = false python-versions = "*" files = [ @@ -866,7 +1178,6 @@ contextlib2 = ">=0.5.5" name = "scikit-learn" version = "0.23.0" description = "A set of python modules for machine learning and data mining" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -901,7 +1212,6 @@ alldeps = ["numpy (>=1.13.3)", "scipy (>=0.19.1)"] name = "scipy" version = "1.9.3" description = "Fundamental algorithms for scientific computing in Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -940,7 +1250,6 @@ test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "sciki name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -952,7 +1261,6 @@ files = [ name = "smdebug-rulesconfig" version = "1.0.1" description = "SMDebug RulesConfig" -category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -960,11 +1268,29 @@ files = [ {file = "smdebug_rulesconfig-1.0.1.tar.gz", hash = "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"}, ] +[[package]] +name = "stack-data" +version = "0.6.2" +description = "Extract data from python stack frames and tracebacks for informative displays" +optional = false +python-versions = "*" +files = [ + {file = "stack_data-0.6.2-py3-none-any.whl", hash = "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"}, + {file = "stack_data-0.6.2.tar.gz", hash = "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815"}, +] + +[package.dependencies] +asttokens = ">=2.1.0" +executing = ">=1.2.0" +pure-eval = "*" + +[package.extras] +tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] + [[package]] name = "tblib" version = "1.7.0" description = "Traceback serialization library." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -976,7 +1302,6 @@ files = [ name = "threadpoolctl" version = "3.1.0" description = "threadpoolctl" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -988,7 +1313,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -996,11 +1320,25 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "traitlets" +version = "5.9.0" +description = "Traitlets Python configuration system" +optional = false +python-versions = ">=3.7" +files = [ + {file = "traitlets-5.9.0-py3-none-any.whl", hash = "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8"}, + {file = "traitlets-5.9.0.tar.gz", hash = "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"}, +] + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] +test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] + [[package]] name = "typeguard" version = "2.13.3" description = "Run-time type checker for Python" -category = "main" optional = false python-versions = ">=3.5.3" files = [ @@ -1016,7 +1354,6 @@ test = ["mypy", "pytest", "typing-extensions"] name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1028,7 +1365,6 @@ files = [ name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" -category = "dev" optional = false python-versions = ">=2" files = [ @@ -1040,7 +1376,6 @@ files = [ name = "urllib3" version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -1053,11 +1388,21 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "wcwidth" +version = "0.2.6" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, + {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, +] + [[package]] name = "xgboost" version = "1.2.0" description = "XGBoost Python Package" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1082,7 +1427,6 @@ scikit-learn = ["scikit-learn"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1097,4 +1441,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "e98bb3449cddaf12a49ec99df753cdb28bc958c51cbfaea5d55be8bd66e3a0a1" +content-hash = "c39f54f0f5989a553139d830192f9fbabfc4fca0f97ecac47e443992ef825845" diff --git a/pyproject.toml b/pyproject.toml index b67590d..8b72850 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ sagemaker = "^2.112.2" xgboost = "1.2" scikit-learn = "0.23" boto3="^1.18.14" +pyvis = "^0.3.2" [build-system] requires = ["poetry-core>=1.0.0"] From 8ae766652747a8bf040b2b9adaf916111afeac5c Mon Sep 17 00:00:00 2001 From: Alessandro Cere Date: Mon, 19 Jun 2023 01:56:15 +0800 Subject: [PATCH 17/17] Fix - missing import in `batch-pipeline` --- build_pipeline/batch-pipeline.ipynb | 58 +++++++++++++---------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/build_pipeline/batch-pipeline.ipynb b/build_pipeline/batch-pipeline.ipynb index ffb5a71..c04aca9 100644 --- a/build_pipeline/batch-pipeline.ipynb +++ b/build_pipeline/batch-pipeline.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -11,7 +10,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -32,7 +30,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -61,7 +58,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -84,7 +80,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -147,7 +142,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -167,7 +161,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -202,7 +195,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -256,7 +248,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -268,7 +259,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "from math import sqrt\n", @@ -281,7 +274,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -291,7 +283,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import seaborn as sns\n", @@ -302,7 +296,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -312,7 +305,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -328,7 +320,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "test_df[\"passenger_count\"] = random.choices(\n", @@ -340,7 +334,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -360,7 +353,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -372,7 +364,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "pipeline_name = f\"{project_name}-batch-prod\"\n", @@ -389,7 +383,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -399,7 +392,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "for step in execution.list_steps():\n", @@ -407,7 +402,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -419,9 +413,13 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ + "from sagemaker.s3 import S3Downloader\n", + "\n", "monitor_uri = get_latest_processed_data(\n", " pipeline_name, \"MonitorDataQuality-monitoring\", \"monitoring_output\"\n", ")\n", @@ -433,7 +431,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -443,7 +440,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "import json\n", @@ -460,7 +459,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -476,7 +474,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "alarm_name = f\"sagemaker-{pipeline_name}-threshold\"\n", @@ -487,7 +487,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -518,7 +517,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -541,7 +539,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -562,7 +559,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -570,7 +566,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -602,7 +597,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [