From edb0659310ee0ed8117a291857b0a8596b9caaec Mon Sep 17 00:00:00 2001 From: JordonPhillips Date: Wed, 29 Jan 2025 15:42:06 +0100 Subject: [PATCH] Enable shapes-only codegen This adds an alternative generation mode that only generates "data shapes". --- README.md | 153 +++- ...thon.codegen-plugin-conventions.gradle.kts | 35 + ....java => DirectedPythonClientCodegen.java} | 170 +--- .../codegen/PythonClientCodegenPlugin.java | 2 +- .../python/codegen/PythonFormatter.java | 88 ++ .../smithy/python/codegen/PythonSettings.java | 31 +- ...Visitor.java => PythonSymbolProvider.java} | 10 +- .../codegen/generators/InitGenerator.java | 44 + .../codegen/generators/SchemaGenerator.java | 26 + .../generators/ServiceErrorGenerator.java | 67 ++ .../codegen/integrations/HttpApiKeyAuth.java | 4 + codegen/plugins/build.gradle.kts | 14 + codegen/plugins/types/README.md | 0 codegen/plugins/types/build.gradle.kts | 8 + .../codegen/types/PythonTypeCodegenTest.java | 64 ++ .../it/resources/META-INF/smithy/main.smithy | 790 ++++++++++++++++++ .../src/it/resources/META-INF/smithy/manifest | 3 + .../META-INF/smithy/more-nesting.smithy | 8 + .../resources/META-INF/smithy/nested.smithy | 8 + .../types/DirectedPythonTypeCodegen.java | 172 ++++ .../types/PythonTypeCodegenPlugin.java | 105 +++ .../types/PythonTypeCodegenSettings.java | 138 +++ ...ware.amazon.smithy.build.SmithyBuildPlugin | 6 + codegen/settings.gradle.kts | 2 + 24 files changed, 1774 insertions(+), 174 deletions(-) create mode 100644 codegen/buildSrc/src/main/kotlin/smithy-python.codegen-plugin-conventions.gradle.kts rename codegen/core/src/main/java/software/amazon/smithy/python/codegen/{DirectedPythonCodegen.java => DirectedPythonClientCodegen.java} (55%) create mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java rename codegen/core/src/main/java/software/amazon/smithy/python/codegen/{SymbolVisitor.java => PythonSymbolProvider.java} (97%) create mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java create mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java create mode 100644 codegen/plugins/build.gradle.kts create mode 100644 codegen/plugins/types/README.md create mode 100644 codegen/plugins/types/build.gradle.kts create mode 100644 codegen/plugins/types/src/it/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenTest.java create mode 100644 codegen/plugins/types/src/it/resources/META-INF/smithy/main.smithy create mode 100644 codegen/plugins/types/src/it/resources/META-INF/smithy/manifest create mode 100644 codegen/plugins/types/src/it/resources/META-INF/smithy/more-nesting.smithy create mode 100644 codegen/plugins/types/src/it/resources/META-INF/smithy/nested.smithy create mode 100644 codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/DirectedPythonTypeCodegen.java create mode 100644 codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenPlugin.java create mode 100644 codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenSettings.java create mode 100644 codegen/plugins/types/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin diff --git a/README.md b/README.md index e904b4eec..aa6fca6f4 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ This repository does *not* contain any generated clients, such as for S3 or othe AWS services. Rather, these are the tools that facilitate the generation of those clients and non-AWS Smithy clients. -### How do I use this? +### How do I use this to create a client? The first step is to create a Smithy package. If this is your first time working with Smithy, follow [this quickstart guide](https://smithy.io/2.0/quickstart.html) @@ -82,8 +82,8 @@ this file, see the "sources": ["model"], "maven": { "dependencies": [ - "software.amazon.smithy:smithy-model:[1.34.0,2.0)", - "software.amazon.smithy:smithy-aws-traits:[1.34.0,2.0)", + "software.amazon.smithy:smithy-model:[1.54.0,2.0)", + "software.amazon.smithy:smithy-aws-traits:[1.54.0,2.0)", "software.amazon.smithy.python:smithy-python-codegen:0.1.0" ] }, @@ -175,6 +175,153 @@ Only for now. Once the generator has been published, the Smithy CLI will be able to run it without a separate Java installation. Similarly, once the python helper libraries have been published you won't need to install them locally. +### How do I generate types for shapes without a client? + +If all you want are concrete Python classes for the shapes in your Smithy model, +all you need to do is replace `python-client-codegen` with +`python-type-codegen` when following the steps above. Your `smithy-build.json` +would now look like: + +```json +{ + "version": "1.0", + "sources": ["model"], + "maven": { + "dependencies": [ + "software.amazon.smithy:smithy-model:[1.54.0,2.0)", + "software.amazon.smithy.python:smithy-python-codegen:0.1.0" + ] + }, + "projections": { + "shapes": { + "plugins": { + "python-type-codegen": { + "service": "com.example#EchoService", + "module": "echo", + "moduleVersion": "0.0.1" + } + } + } + } +} +``` + +The module with the generated shape classes can be found in +`build/smithy/client/python-type-codegen` after you run `smithy-build`. + +Unlike when generating a client, a service shape is not required for shape +generation. If a service is not provided then every shape found in the model +will be generated. Any naming conflicts may be resolved by using the +[`renameShapes` transform](https://smithy.io/2.0/guides/smithy-build-json.html#renameshapes) +(or renaming the shapes in the model of course). + +The set of shapes generated can also be constrained by using the +[`includeShapesBySelector` transform](https://smithy.io/2.0/guides/smithy-build-json.html#includeshapesbyselector). +For example, to generate only shapes within the `com.example` namespace: + +```json +{ + "version": "1.0", + "sources": ["model"], + "maven": { + "dependencies": [ + "software.amazon.smithy:smithy-model:[1.54.0,2.0)", + "software.amazon.smithy.python:smithy-python-codegen:0.1.0" + ] + }, + "projections": { + "shapes": { + "transforms": [ + { + "name": "includeShapesBySelector", + "args": { + "selector": "[id|namespace = 'com.example']" + } + } + ], + "plugins": { + "python-type-codegen": { + "module": "echo", + "moduleVersion": "0.0.1" + } + } + } + } +} +``` + +Input and output shapes (shapes with the `@input` or `@output` traits and +operation inputs / outputs created as part of an operation definition) are not +generated by default. To generate these shapes anyway, remove the traits with +the +[`excludeTraits` transform](https://smithy.io/2.0/guides/smithy-build-json.html#excludetraits): + +```json +{ + "version": "1.0", + "sources": ["model"], + "maven": { + "dependencies": [ + "software.amazon.smithy:smithy-model:[1.54.0,2.0)", + "software.amazon.smithy.python:smithy-python-codegen:0.1.0" + ] + }, + "projections": { + "shapes": { + "transforms": [ + { + "name": "excludeTraits", + "args": { + "traits": ["input", "output"] + } + } + ], + "plugins": { + "python-type-codegen": { + "module": "echo", + "moduleVersion": "0.0.1" + } + } + } + } +} +``` + +You can also generate both a client package and a shape package in one build, +but they won't depend on each other. To do this, just add both plugins in the +projection, or create a projection for each plugin. Below is an example showing +both plugins in one projection: + +```json +{ + "version": "1.0", + "sources": ["model"], + "maven": { + "dependencies": [ + "software.amazon.smithy:smithy-model:[1.54.0,2.0)", + "software.amazon.smithy:smithy-aws-traits:[1.54.0,2.0)", + "software.amazon.smithy.python:smithy-python-codegen:0.1.0" + ] + }, + "projections": { + "client": { + "plugins": { + "python-client-codegen": { + "service": "com.example#EchoService", + "module": "echo", + "moduleVersion": "0.0.1" + }, + "python-type-codegen": { + "service": "com.example#EchoService", + "module": "echo", + "moduleVersion": "0.0.1" + } + } + } + } +} +``` + ### Core Modules and Interfaces * `smithy-core` provides transport-agnostic core modules and interfaces diff --git a/codegen/buildSrc/src/main/kotlin/smithy-python.codegen-plugin-conventions.gradle.kts b/codegen/buildSrc/src/main/kotlin/smithy-python.codegen-plugin-conventions.gradle.kts new file mode 100644 index 000000000..a6ffc4e46 --- /dev/null +++ b/codegen/buildSrc/src/main/kotlin/smithy-python.codegen-plugin-conventions.gradle.kts @@ -0,0 +1,35 @@ +import org.gradle.api.Project + +plugins { + id("smithy-python.module-conventions") + id("smithy-python.integ-test-conventions") +} + +// Workaround per: https://github.com/gradle/gradle/issues/15383 +val Project.libs get() = the() + +group = "software.amazon.smithy.python.codegen.plugins" + +dependencies { + implementation(libs.smithy.codegen) + implementation(project(":core")) + + // Avoid circular dependency in codegen core + if (project.name != "core") { + api(project(":core")) + } +} + +val generatedSrcDir = layout.buildDirectory.dir("generated-src").get() + +// Add generated sources to integration test sources +sourceSets { + named("it") { + java { + srcDir(generatedSrcDir) + } + } +} + +// Ensure integ tests are executed as part of test suite +tasks["test"].finalizedBy("integ") diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java similarity index 55% rename from codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java rename to codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java index a52cb082e..285576845 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java @@ -4,22 +4,14 @@ */ package software.amazon.smithy.python.codegen; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import software.amazon.smithy.build.FileManifest; -import software.amazon.smithy.codegen.core.CodegenException; import software.amazon.smithy.codegen.core.SymbolProvider; import software.amazon.smithy.codegen.core.TopologicalIndex; -import software.amazon.smithy.codegen.core.WriterDelegator; import software.amazon.smithy.codegen.core.directed.CreateContextDirective; import software.amazon.smithy.codegen.core.directed.CreateSymbolProviderDirective; import software.amazon.smithy.codegen.core.directed.CustomizeDirective; @@ -29,40 +21,37 @@ import software.amazon.smithy.codegen.core.directed.GenerateIntEnumDirective; import software.amazon.smithy.codegen.core.directed.GenerateListDirective; import software.amazon.smithy.codegen.core.directed.GenerateMapDirective; -import software.amazon.smithy.codegen.core.directed.GenerateResourceDirective; import software.amazon.smithy.codegen.core.directed.GenerateServiceDirective; import software.amazon.smithy.codegen.core.directed.GenerateStructureDirective; import software.amazon.smithy.codegen.core.directed.GenerateUnionDirective; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.knowledge.ServiceIndex; -import software.amazon.smithy.model.loader.Prelude; import software.amazon.smithy.model.shapes.ServiceShape; -import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.python.codegen.generators.ConfigGenerator; import software.amazon.smithy.python.codegen.generators.EnumGenerator; +import software.amazon.smithy.python.codegen.generators.InitGenerator; import software.amazon.smithy.python.codegen.generators.IntEnumGenerator; import software.amazon.smithy.python.codegen.generators.ListGenerator; import software.amazon.smithy.python.codegen.generators.MapGenerator; import software.amazon.smithy.python.codegen.generators.ProtocolGenerator; import software.amazon.smithy.python.codegen.generators.SchemaGenerator; -import software.amazon.smithy.python.codegen.generators.SetupGenerator; +import software.amazon.smithy.python.codegen.generators.ServiceErrorGenerator; import software.amazon.smithy.python.codegen.generators.StructureGenerator; import software.amazon.smithy.python.codegen.generators.UnionGenerator; import software.amazon.smithy.python.codegen.integrations.PythonIntegration; import software.amazon.smithy.python.codegen.writer.PythonDelegator; -import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyUnstableApi; @SmithyUnstableApi -final class DirectedPythonCodegen implements DirectedCodegen { +final class DirectedPythonClientCodegen + implements DirectedCodegen { - private static final Logger LOGGER = Logger.getLogger(DirectedPythonCodegen.class.getName()); - private static final int PYTHON_MINOR_VERSION = 12; // 3.12 + private static final Logger LOGGER = Logger.getLogger(DirectedPythonClientCodegen.class.getName()); @Override public SymbolProvider createSymbolProvider(CreateSymbolProviderDirective directive) { - return new SymbolVisitor(directive.model(), directive.settings()); + return new PythonSymbolProvider(directive.model(), directive.settings()); } @Override @@ -113,11 +102,10 @@ private ProtocolGenerator resolveProtocolGenerator( @Override public void customizeBeforeShapeGeneration(CustomizeDirective directive) { - generateServiceErrors(directive.settings(), directive.context().writerDelegator()); - new ConfigGenerator(directive.settings(), directive.context()).run(); - - generateSchemas(directive.context(), directive.connectedShapes().values()); + new ServiceErrorGenerator(directive.settings(), directive.context().writerDelegator()).run(); + SchemaGenerator.generateAll(directive.context(), directive.connectedShapes().values()); + new ConfigGenerator(directive.settings(), directive.context()).run(); var serviceIndex = ServiceIndex.of(directive.model()); if (directive.context().applicationProtocol().isHttpProtocol() && !serviceIndex.getAuthSchemes(directive.service()).isEmpty()) { @@ -125,19 +113,6 @@ public void customizeBeforeShapeGeneration(CustomizeDirective shapes) { - var schemaGenerator = new SchemaGenerator(context); - var index = TopologicalIndex.of(context.model()); - Stream.concat(index.getOrderedShapes().stream(), index.getRecursiveShapes().stream()) - .filter(shapes::contains) - .filter(shape -> !shape.isOperationShape() && !shape.isResourceShape() - && !shape.isServiceShape() - && !shape.isMemberShape() - && !Prelude.isPreludeShape(shape)) - .forEach(schemaGenerator); - schemaGenerator.finalizeRecursiveShapes(); - } - @Override public void generateService(GenerateServiceDirective directive) { new ClientGenerator(directive.context(), directive.service()).run(); @@ -156,48 +131,6 @@ public void generateService(GenerateServiceDirective writers) { - var serviceError = CodegenUtils.getServiceError(settings); - writers.useFileWriter(serviceError.getDefinitionFile(), serviceError.getNamespace(), writer -> { - writer.addDependency(SmithyPythonDependency.SMITHY_CORE); - writer.addImport("smithy_core.exceptions", "SmithyException"); - writer.write(""" - class $L(SmithyException): - ""\"Base error for all errors in the service.""\" - pass - """, serviceError.getName()); - }); - - var apiError = CodegenUtils.getApiError(settings); - writers.useFileWriter(apiError.getDefinitionFile(), apiError.getNamespace(), writer -> { - writer.addStdlibImports("typing", Set.of("Literal", "ClassVar")); - var unknownApiError = CodegenUtils.getUnknownApiError(settings); - - writer.write(""" - @dataclass - class $1L($2T): - ""\"Base error for all API errors in the service.""\" - code: ClassVar[str] - fault: ClassVar[Literal["client", "server"]] - - message: str - - def __post_init__(self) -> None: - super().__init__(self.message) - - - @dataclass - class $3L($1L): - ""\"Error representing any unknown api errors""\" - code: ClassVar[str] = 'Unknown' - fault: ClassVar[Literal["client", "server"]] = "client" - """, apiError.getName(), serviceError, unknownApiError.getName()); - }); - } - - @Override - public void generateResource(GenerateResourceDirective directive) {} - @Override public void generateStructure(GenerateStructureDirective directive) { directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { @@ -273,92 +206,11 @@ public void generateIntEnumShape(GenerateIntEnumDirective directive) { - generateInits(directive); - } - - /** - * Creates __init__.py files where not already present. - */ - private void generateInits(CustomizeDirective directive) { - var directories = directive.context() - .writerDelegator() - .getWriters() - .keySet() - .stream() - .map(Paths::get) - .filter(path -> !path.getParent().equals(directive.fileManifest().getBaseDir())) - .collect(Collectors.groupingBy(Path::getParent, Collectors.toSet())); - for (var entry : directories.entrySet()) { - var initPath = entry.getKey().resolve("__init__.py"); - if (!entry.getValue().contains(initPath)) { - directive.fileManifest() - .writeFile(initPath, - "# Code generated by smithy-python-codegen DO NOT EDIT.\n"); - } - } + new InitGenerator(directive.context()).run(); } @Override public void customizeAfterIntegrations(CustomizeDirective directive) { - Pattern versionPattern = Pattern.compile("Python \\d\\.(?\\d+)\\.(?\\d+)"); - FileManifest fileManifest = directive.fileManifest(); - SetupGenerator.generateSetup(directive.settings(), directive.context()); - - LOGGER.info("Flushing writers in preparation for formatting and linting."); - directive.context().writerDelegator().flushWriters(); - - String output; - try { - LOGGER.info("Attempting to discover python version"); - output = CodegenUtils.runCommand("python3 --version", fileManifest.getBaseDir()).strip(); - } catch (CodegenException e) { - LOGGER.warning("Unable to find python on the path. Skipping formatting and type checking."); - return; - } - var matcher = versionPattern.matcher(output); - if (!matcher.find()) { - LOGGER.warning("Unable to parse python version string. Skipping formatting and type checking."); - } - int minorVersion = Integer.parseInt(matcher.group("minor")); - if (minorVersion < PYTHON_MINOR_VERSION) { - LOGGER.warning(String.format(""" - Found incompatible python version 3.%s.%s, expected 3.12.0 or greater. \ - Skipping formatting and type checking.""", - matcher.group("minor"), - matcher.group("patch"))); - return; - } - LOGGER.info("Verifying python files"); - for (var file : fileManifest.getFiles()) { - var fileName = file.getFileName(); - if (fileName == null || !fileName.endsWith(".py")) { - continue; - } - CodegenUtils.runCommand("python3 " + file, fileManifest.getBaseDir()); - } - format(fileManifest); - check(fileManifest); - } - - private void format(FileManifest fileManifest) { - try { - CodegenUtils.runCommand("python3 -m black -h", fileManifest.getBaseDir()); - } catch (CodegenException e) { - LOGGER.warning("Unable to find the python package black. Skipping formatting."); - return; - } - LOGGER.info("Running code formatter on generated code"); - CodegenUtils.runCommand("python3 -m black . --exclude \"\"", fileManifest.getBaseDir()); - } - - private void check(FileManifest fileManifest) { - try { - CodegenUtils.runCommand("python3 -m pyright -h", fileManifest.getBaseDir()); - } catch (CodegenException e) { - LOGGER.warning("Unable to find the pyright package. Skipping type checking."); - return; - } - LOGGER.info("Running mypy on generated code"); - CodegenUtils.runCommand("python3 -m pyright .", fileManifest.getBaseDir()); + new PythonFormatter(directive.context()).run(); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java index 567e8efd8..eeaf2e5d1 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java @@ -28,7 +28,7 @@ public void execute(PluginContext context) { PythonSettings settings = PythonSettings.fromNode(context.getSettings()); runner.settings(settings); - runner.directedCodegen(new DirectedPythonCodegen()); + runner.directedCodegen(new DirectedPythonClientCodegen()); runner.fileManifest(context.getFileManifest()); runner.service(settings.service()); runner.model(context.getModel()); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java new file mode 100644 index 000000000..5b5efc2e5 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java @@ -0,0 +1,88 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen; + +import java.util.logging.Logger; +import java.util.regex.Pattern; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.python.codegen.generators.SetupGenerator; +import software.amazon.smithy.utils.SmithyInternalApi; + +@SmithyInternalApi +public final class PythonFormatter implements Runnable { + private static final Logger LOGGER = Logger.getLogger(PythonFormatter.class.getName()); + private static final int PYTHON_MINOR_VERSION = 12; // 3.12 + + private final GenerationContext context; + + public PythonFormatter(GenerationContext context) { + this.context = context; + } + + @Override + public void run() { + Pattern versionPattern = Pattern.compile("Python \\d\\.(?\\d+)\\.(?\\d+)"); + FileManifest fileManifest = context.fileManifest(); + SetupGenerator.generateSetup(context.settings(), context); + + LOGGER.info("Flushing writers in preparation for formatting and linting."); + context.writerDelegator().flushWriters(); + + String output; + try { + LOGGER.info("Attempting to discover python version"); + output = CodegenUtils.runCommand("python3 --version", fileManifest.getBaseDir()).strip(); + } catch (CodegenException e) { + LOGGER.warning("Unable to find python on the path. Skipping formatting and type checking."); + return; + } + var matcher = versionPattern.matcher(output); + if (!matcher.find()) { + LOGGER.warning("Unable to parse python version string. Skipping formatting and type checking."); + } + int minorVersion = Integer.parseInt(matcher.group("minor")); + if (minorVersion < PYTHON_MINOR_VERSION) { + LOGGER.warning(String.format(""" + Found incompatible python version 3.%s.%s, expected 3.12.0 or greater. \ + Skipping formatting and type checking.""", + matcher.group("minor"), + matcher.group("patch"))); + return; + } + LOGGER.info("Verifying python files"); + for (var file : fileManifest.getFiles()) { + var fileName = file.getFileName(); + if (fileName == null || !fileName.endsWith(".py")) { + continue; + } + CodegenUtils.runCommand("python3 " + file, fileManifest.getBaseDir()); + } + format(fileManifest); + check(fileManifest); + } + + private void format(FileManifest fileManifest) { + try { + CodegenUtils.runCommand("python3 -m black -h", fileManifest.getBaseDir()); + } catch (CodegenException e) { + LOGGER.warning("Unable to find the python package black. Skipping formatting."); + return; + } + LOGGER.info("Running code formatter on generated code"); + CodegenUtils.runCommand("python3 -m black . --exclude \"\"", fileManifest.getBaseDir()); + } + + private void check(FileManifest fileManifest) { + try { + CodegenUtils.runCommand("python3 -m pyright -h", fileManifest.getBaseDir()); + } catch (CodegenException e) { + LOGGER.warning("Unable to find the pyright package. Skipping type checking."); + return; + } + LOGGER.info("Running mypy on generated code"); + CodegenUtils.runCommand("python3 -m pyright .", fileManifest.getBaseDir()); + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSettings.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSettings.java index f152f2d9c..5c9cdbbb0 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSettings.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSettings.java @@ -5,6 +5,7 @@ package software.amazon.smithy.python.codegen; import java.util.Arrays; +import java.util.Locale; import java.util.Objects; import software.amazon.smithy.codegen.core.CodegenException; import software.amazon.smithy.model.Model; @@ -24,13 +25,15 @@ * @param moduleName The name of the module to generate. * @param moduleVersion The version of the module to generate. * @param moduleDescription The optional module description for the module that will be generated. + * @param artifactType The type of artifact being generated (e.g. client/shapes/server) */ @SmithyUnstableApi public record PythonSettings( ShapeId service, String moduleName, String moduleVersion, - String moduleDescription) implements ToSmithyBuilder { + String moduleDescription, + ArtifactType artifactType) implements ToSmithyBuilder { private static final String SERVICE = "service"; private static final String MODULE_NAME = "module"; @@ -48,14 +51,15 @@ public record PythonSettings( } } - public PythonSettings(Builder builder) { + private PythonSettings(Builder builder) { this( builder.service, builder.moduleName, builder.moduleVersion, StringUtils.isBlank(builder.moduleDescription) - ? builder.moduleName + " client" - : builder.moduleDescription); + ? builder.moduleName + " " + builder.artifactType.name().toLowerCase(Locale.ENGLISH) + : builder.moduleDescription, + builder.artifactType); } /** @@ -92,13 +96,22 @@ public static PythonSettings fromNode(ObjectNode config) { return builder.build(); } + /** + * The type of artifact being generated. + */ + public enum ArtifactType { + CLIENT, + TYPES; + } + @Override - public SmithyBuilder toBuilder() { + public Builder toBuilder() { return builder() .service(service) .moduleName(moduleName) .moduleVersion(moduleVersion) - .moduleDescription(moduleDescription); + .moduleDescription(moduleDescription) + .artifactType(artifactType); } public static Builder builder() { @@ -111,6 +124,7 @@ public static class Builder implements SmithyBuilder { private String moduleName; private String moduleVersion; private String moduleDescription; + private ArtifactType artifactType = ArtifactType.CLIENT; @Override public PythonSettings build() { @@ -139,5 +153,10 @@ public Builder moduleDescription(String moduleDescription) { this.moduleDescription = moduleDescription; return this; } + + public Builder artifactType(ArtifactType artifactType) { + this.artifactType = artifactType; + return this; + } } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java similarity index 97% rename from codegen/core/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java rename to codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java index 4adc041e7..10394bc7d 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java @@ -60,9 +60,9 @@ * may be attached to symbols. */ @SmithyInternalApi -public final class SymbolVisitor implements SymbolProvider, ShapeVisitor { +public final class PythonSymbolProvider implements SymbolProvider, ShapeVisitor { - private static final Logger LOGGER = Logger.getLogger(SymbolVisitor.class.getName()); + private static final Logger LOGGER = Logger.getLogger(PythonSymbolProvider.class.getName()); private static final String SHAPES_FILE = "models"; private static final String SCHEMAS_FILE = "_private/schemas"; @@ -72,17 +72,17 @@ public final class SymbolVisitor implements SymbolProvider, ShapeVisitor private final PythonSettings settings; private final ServiceShape service; - SymbolVisitor(Model model, PythonSettings settings) { + public PythonSymbolProvider(Model model, PythonSettings settings) { this.model = model; this.settings = settings; this.service = model.expectShape(settings.service(), ServiceShape.class); // Load reserved words from new-line delimited files. var reservedClassNames = new ReservedWordsBuilder() - .loadWords(SymbolVisitor.class.getResource("reserved-class-names.txt"), this::escapeWord) + .loadWords(PythonSymbolProvider.class.getResource("reserved-class-names.txt"), this::escapeWord) .build(); var reservedMemberNamesBuilder = new ReservedWordsBuilder() - .loadWords(SymbolVisitor.class.getResource("reserved-member-names.txt"), this::escapeWord); + .loadWords(PythonSymbolProvider.class.getResource("reserved-member-names.txt"), this::escapeWord); escaper = ReservedWordSymbolProvider.builder() .nameReservedWords(reservedClassNames) diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java new file mode 100644 index 000000000..d90e56818 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.generators; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Collectors; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Creates __init__.py files where not already present. + */ +@SmithyInternalApi +public final class InitGenerator implements Runnable { + + private final GenerationContext context; + + public InitGenerator(GenerationContext context) { + this.context = context; + } + + @Override + public void run() { + var directories = context + .writerDelegator() + .getWriters() + .keySet() + .stream() + .map(Paths::get) + .filter(path -> !path.getParent().equals(context.fileManifest().getBaseDir())) + .collect(Collectors.groupingBy(Path::getParent, Collectors.toSet())); + for (var entry : directories.entrySet()) { + var initPath = entry.getKey().resolve("__init__.py"); + if (!entry.getValue().contains(initPath)) { + context.fileManifest() + .writeFile(initPath, + "# Code generated by smithy-python-codegen DO NOT EDIT.\n"); + } + } + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SchemaGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SchemaGenerator.java index 413cf5e0f..e0a0f5425 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SchemaGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SchemaGenerator.java @@ -4,15 +4,19 @@ */ package software.amazon.smithy.python.codegen.generators; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.TopologicalIndex; import software.amazon.smithy.model.loader.Prelude; import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.shapes.MemberShape; @@ -70,6 +74,28 @@ public void accept(Shape shape) { generatedShapes.add(shape.getId()); } + public static void generateAll( + GenerationContext context, + Collection shapes, + Function filter + ) { + var schemaGenerator = new SchemaGenerator(context); + var index = TopologicalIndex.of(context.model()); + Stream.concat(index.getOrderedShapes().stream(), index.getRecursiveShapes().stream()) + .filter(shapes::contains) + .filter(shape -> !shape.isOperationShape() && !shape.isResourceShape() + && !shape.isServiceShape() + && !shape.isMemberShape() + && !Prelude.isPreludeShape(shape)) + .filter(filter::apply) + .forEach(schemaGenerator); + schemaGenerator.finalizeRecursiveShapes(); + } + + public static void generateAll(GenerationContext context, Collection shapes) { + SchemaGenerator.generateAll(context, shapes, s -> true); + } + private void writeShapeSchema(PythonWriter writer, Shape shape) { writer.addImport("smithy_core.schemas", "Schema"); writer.addImports("smithy_core.shapes", Set.of("ShapeID", "ShapeType")); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java new file mode 100644 index 000000000..f5b64c5b7 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java @@ -0,0 +1,67 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.generators; + +import java.util.Set; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.python.codegen.writer.PythonWriter; +import software.amazon.smithy.utils.SmithyInternalApi; + +@SmithyInternalApi +public final class ServiceErrorGenerator implements Runnable { + private final PythonSettings settings; + private final WriterDelegator writers; + + public ServiceErrorGenerator( + PythonSettings settings, + WriterDelegator writers + ) { + this.settings = settings; + this.writers = writers; + } + + @Override + public void run() { + var serviceError = CodegenUtils.getServiceError(settings); + writers.useFileWriter(serviceError.getDefinitionFile(), serviceError.getNamespace(), writer -> { + writer.addDependency(SmithyPythonDependency.SMITHY_CORE); + writer.addImport("smithy_core.exceptions", "SmithyException"); + writer.write(""" + class $L(SmithyException): + ""\"Base error for all errors in the service.""\" + pass + """, serviceError.getName()); + }); + + var apiError = CodegenUtils.getApiError(settings); + writers.useFileWriter(apiError.getDefinitionFile(), apiError.getNamespace(), writer -> { + writer.addStdlibImports("typing", Set.of("Literal", "ClassVar")); + var unknownApiError = CodegenUtils.getUnknownApiError(settings); + + writer.write(""" + @dataclass + class $1L($2T): + ""\"Base error for all API errors in the service.""\" + code: ClassVar[str] + fault: ClassVar[Literal["client", "server"]] + + message: str + + def __post_init__(self) -> None: + super().__init__(self.message) + + + @dataclass + class $3L($1L): + ""\"Error representing any unknown api errors""\" + code: ClassVar[str] = 'Unknown' + fault: ClassVar[Literal["client", "server"]] = "client" + """, apiError.getName(), serviceError, unknownApiError.getName()); + }); + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java index f8034d0ab..ede2dd479 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java @@ -12,6 +12,7 @@ import software.amazon.smithy.python.codegen.CodegenUtils; import software.amazon.smithy.python.codegen.ConfigProperty; import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; import software.amazon.smithy.python.codegen.SmithyPythonDependency; import software.amazon.smithy.utils.SmithyInternalApi; @@ -101,6 +102,9 @@ public void customize(GenerationContext context) { } private boolean hasApiKeyAuth(GenerationContext context) { + if (context.settings().artifactType() == PythonSettings.ArtifactType.TYPES) { + return false; + } var service = context.settings().service(context.model()); return service.hasTrait(HttpApiKeyAuthTrait.class); } diff --git a/codegen/plugins/build.gradle.kts b/codegen/plugins/build.gradle.kts new file mode 100644 index 000000000..46ec8d81f --- /dev/null +++ b/codegen/plugins/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + id("smithy-python.module-conventions") + id("smithy-python.publishing-conventions") +} + +description = "This module provides Python code generation plugins for Smithy" +group = "software.amazon.smithy.python.codegen" + +extra["displayName"] = "Smithy :: Python :: Codegen :: Plugins" +extra["moduleName"] = "software.amazon.smithy.python.codegen.plugins" + +dependencies { + subprojects.forEach { api(project(it.path)) } +} diff --git a/codegen/plugins/types/README.md b/codegen/plugins/types/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/codegen/plugins/types/build.gradle.kts b/codegen/plugins/types/build.gradle.kts new file mode 100644 index 000000000..e4c577992 --- /dev/null +++ b/codegen/plugins/types/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + id("smithy-python.codegen-plugin-conventions") +} + +description = "This module provides the codegen plugin for Smithy java type codegen" + +extra["displayName"] = "Smithy :: Java :: Codegen :: Plugins :: Types" +extra["moduleName"] = "software.amazon.smithy.java.codegen.types" diff --git a/codegen/plugins/types/src/it/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenTest.java b/codegen/plugins/types/src/it/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenTest.java new file mode 100644 index 000000000..9eb26d1e1 --- /dev/null +++ b/codegen/plugins/types/src/it/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenTest.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.types; + +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.node.ObjectNode; + +/** + * Simple test that executes the Python type codegen plugin. Currently, this is about as much "testing" as + * we can do, aside from the protocol tests. JUnit will set up and tear down a tempdir to house the codegen artifacts. + */ +public class PythonTypeCodegenTest { + + @Test + public void testCodegen(@TempDir Path tempDir) { + PythonTypeCodegenPlugin plugin = new PythonTypeCodegenPlugin(); + Model model = Model.assembler(PythonTypeCodegenTest.class.getClassLoader()) + .discoverModels(PythonTypeCodegenTest.class.getClassLoader()) + .assemble() + .unwrap(); + PluginContext context = PluginContext.builder() + .fileManifest(FileManifest.create(tempDir)) + .settings( + ObjectNode.builder() + .withMember("service", "example.weather#Weather") + .withMember("module", "types_test") + .withMember("moduleVersion", "0.0.1") + .build()) + .model(model) + .build(); + plugin.execute(context); + } + + @Test + public void testCodegenWithoutService(@TempDir Path tempDir) { + PythonTypeCodegenPlugin plugin = new PythonTypeCodegenPlugin(); + Model model = Model.assembler(PythonTypeCodegenTest.class.getClassLoader()) + .discoverModels(PythonTypeCodegenTest.class.getClassLoader()) + .assemble() + .unwrap(); + PluginContext context = PluginContext.builder() + .fileManifest(FileManifest.create(tempDir)) + .settings( + ObjectNode.builder() + .withMember("selector", """ + :test([id|namespace = 'example.weather'], + [id|namespace = 'example.weather.nested'], + [id|namespace = 'example.weather.nested.more']) + """) + .withMember("module", "types_test") + .withMember("moduleVersion", "0.0.1") + .build()) + .model(model) + .build(); + plugin.execute(context); + } +} diff --git a/codegen/plugins/types/src/it/resources/META-INF/smithy/main.smithy b/codegen/plugins/types/src/it/resources/META-INF/smithy/main.smithy new file mode 100644 index 000000000..d79556b83 --- /dev/null +++ b/codegen/plugins/types/src/it/resources/META-INF/smithy/main.smithy @@ -0,0 +1,790 @@ +$version: "2.0" + +namespace example.weather + +use aws.protocols#restJson1 +use smithy.test#httpRequestTests +use smithy.test#httpResponseTests +use smithy.waiters#waitable + +/// Provides weather forecasts. +@restJson1( + http: ["h2", "http/1.1"] + eventStreamHttp: ["h2"] +) +@fakeProtocol +@paginated(inputToken: "nextToken", outputToken: "nextToken", pageSize: "pageSize") +@httpApiKeyAuth(name: "weather-auth", in: "header") +service Weather { + version: "2006-03-01" + resources: [ + City + ] + operations: [ + GetCurrentTime + TestUnionListOperation + StreamAtmosphericConditions + ] +} + +resource City { + identifiers: { + cityId: CityId + } + read: GetCity + list: ListCities + resources: [ + Forecast + CityImage + ] + operations: [ + GetCityAnnouncements + ] +} + +@http(code: 200, method: "POST", uri: "/test-union-list") +operation TestUnionListOperation { + input := { + inputList: UnionList + } + output := { + response: String + } +} + +list UnionList { + member: UnionListMember +} + +union UnionListMember { + field1: String + field2: Integer +} + +resource Forecast { + identifiers: { + cityId: CityId + } + read: GetForecast +} + +resource CityImage { + identifiers: { + cityId: CityId + } + read: GetCityImage +} + +// "pattern" is a trait. +@pattern("^[A-Za-z0-9 ]+$") +string CityId + +@readonly +@suppress(["WaitableTraitInvalidErrorType"]) +@waitable( + CityExists: { + acceptors: [ + // Fail-fast if the thing transitions to a "failed" state. + { + state: "failure" + matcher: { errorType: "NoSuchResource" } + } + // Fail-fast if the thing transitions to a "failed" state. + { + state: "failure" + matcher: { errorType: "UnModeledError" } + } + // Succeed when the city image value is not empty i.e. enters into a "success" state. + { + state: "success" + matcher: { success: true } + } + // Retry if city id input is of same length as city name in output + { + state: "retry" + matcher: { + inputOutput: { + path: "length(input.cityId) == length(output.name)" + comparator: "booleanEquals" + expected: "true" + } + } + } + // Success if city name in output is seattle + { + state: "success" + matcher: { + output: { path: "name", comparator: "stringEquals", expected: "seattle" } + } + } + ] + } +) +@http(method: "GET", uri: "/cities/{cityId}") +operation GetCity { + input := { + // "cityId" provides the identifier for the resource and + // has to be marked as required. + @required + @httpLabel + cityId: CityId + } + + output := { + // "required" is used on output to indicate if the service + // will always provide a value for the member. + @required + name: String + + @required + coordinates: CityCoordinates + + city: CitySummary + + cityData: JsonString + + binaryCityData: JsonBlob + } + + errors: [ + NoSuchResource + EmptyError + ] +} + +// Tests that HTTP protocol tests are generated. +apply GetCity @httpRequestTests([ + { + id: "WriteGetCityAssertions" + documentation: "Does something" + protocol: "example.weather#fakeProtocol" + method: "GET" + uri: "/cities/123" + body: "" + params: { cityId: "123" } + } +]) + +apply GetCity @httpResponseTests([ + { + id: "WriteGetCityResponseAssertions" + documentation: "Does something" + protocol: "example.weather#fakeProtocol" + code: 200 + body: """ + { + "name": "Seattle", + "coordinates": { + "latitude": 12.34, + "longitude": -56.78 + }, + "city": { + "cityId": "123", + "name": "Seattle", + "number": "One", + "case": "Upper" + } + }""" + bodyMediaType: "application/json" + params: { + name: "Seattle" + coordinates: { latitude: 12.34, longitude: -56.78 } + city: { cityId: "123", name: "Seattle", number: "One", case: "Upper" } + } + } +]) + +@mediaType("application/json") +string JsonString + +@mediaType("application/json") +blob JsonBlob + +// This structure is nested within GetCityOutput. +structure CityCoordinates { + @required + latitude: Float + + @required + longitude: Float +} + +/// Error encountered when no resource could be found. +@error("client") +@httpError(404) +structure NoSuchResource { + /// The type of resource that was not found. + @required + resourceType: String + + message: String +} + +apply NoSuchResource @httpResponseTests([ + { + id: "WriteNoSuchResourceAssertions" + documentation: "Does something" + protocol: "example.weather#fakeProtocol" + code: 404 + body: """ + { + "resourceType": "City", + "message": "Your custom message" + }""" + bodyMediaType: "application/json" + params: { resourceType: "City", message: "Your custom message" } + } +]) + +// This will have a synthetic message member added to it, even +// though it doesn't actually have one. +@error("client") +@httpError(400) +structure EmptyError {} + +// The paginated trait indicates that the operation may +// return truncated results. +@readonly +@paginated(items: "items") +@waitable( + ListContainsCity: { + acceptors: [ + // failure in case all items returned match to seattle + { + state: "failure" + matcher: { + output: { path: "items[].name", comparator: "allStringEquals", expected: "seattle" } + } + } + // success in case any items returned match to NewYork + { + state: "success" + matcher: { + output: { path: "items[].name", comparator: "anyStringEquals", expected: "NewYork" } + } + } + ] + } +) +@http(method: "GET", uri: "/cities") +operation ListCities { + input := { + @httpQuery("nextToken") + nextToken: String + + @httpQuery("aString") + aString: String + + @httpQuery("someEnum") + someEnum: StringYesNo + + @httpQuery("pageSize") + pageSize: Integer + } + + output := { + nextToken: String + + someEnum: StringYesNo + + aString: String + + defaults: Defaults + + escaping: MemberEscaping + + escapeTrue: True + + escapeFalse: False + + escapeNone: None + + @required + items: CitySummaries + + sparseItems: SparseCitySummaries + + mutual: MutuallyRecursiveA + } +} + +apply ListCities @httpRequestTests([ + { + id: "WriteListCitiesAssertions" + documentation: "Does something" + protocol: "example.weather#fakeProtocol" + method: "GET" + uri: "/cities" + body: "" + queryParams: ["pageSize=50"] + forbidQueryParams: ["nextToken"] + params: { pageSize: 50 } + } +]) + +structure Defaults { + @required + requiredBool: Boolean + + optionalBool: Boolean + + defaultTrue: Boolean = true + + defaultFalse: Boolean = false + + @required + requiredDefaultBool: Boolean = true + + @required + requiredStr: String + + optionalStr: String + + defaultString: String = "spam" + + @required + requiredDefaultStr: String = "eggs" + + @required + requiredInt: Integer + + optionalInt: Integer + + defaultInt: Integer = 42 + + @required + requiredDefaultInt: Integer = 42 + + @required + requiredFloat: Float + + optionalFloat: Float + + defaultFloat: Float = 4.2 + + @required + requiredDefaultFloat: Float = 4.2 + + @required + requiredBlob: Blob + + optionalBlob: Blob + + defaultBlob: Blob = "c3BhbQ==" + + @required + requiredDefaultBlob: Blob = "c3BhbQ==" + + // timestamp + @required + requiredTimestamp: Timestamp + + optionalTimestamp: Timestamp + + defaultImplicitDateTime: Timestamp = "2011-12-03T10:15:30Z" + + defaultImplicitEpochTime: Timestamp = 4.2 + + defaultExplicitDateTime: DateTime = "2011-12-03T10:15:30Z" + + defaultExplicitEpochTime: EpochSeconds = 4.2 + + defaultExplicitHttpTime: HttpDate = "Tue, 3 Jun 2008 11:05:30 GMT" + + @required + requiredDefaultTimestamp: Timestamp = 4.2 + + @required + requiredList: StringList + + optionalList: StringList + + defaultList: StringList = [] + + @required + requiredDefaultList: StringList = [] + + @required + requiredMap: StringMap + + optionalMap: StringMap + + defaultMap: StringMap = {} + + @required + requiredDefaultMap: StringMap = {} + + @required + requiredDocument: Document + + optionalDocument: Document + + defaultNullDocument: Document = null + + defaultNumberDocument: Document = 42 + + defaultStringDocument: Document = "spam" + + defaultBooleanDocument: Document = true + + defaultListDocument: Document = [] + + defaultMapDocument: Document = {} + + @required + requiredDefaultDocument: Document = "eggs" +} + +// This structure has members that need to be escaped. +structure MemberEscaping { + // This first set of member names are all reserved words that are a syntax + // error to use as identifiers. A full list of these can be found here: + // https://docs.python.org/3/reference/lexical_analysis.html#keywords + and: String + + as: String + + assert: String + + async: String + + await: String + + break: String + + class: String + + continue: String + + def: String + + del: String + + elif: String + + else: String + + except: String + + finally: String + + for: String + + from: String + + global: String + + if: String + + import: String + + in: String + + is: String + + lambda: String + + nonlocal: String + + not: String + + or: String + + pass: String + + raise: String + + return: String + + try: String + + while: String + + with: String + + yield: String + + // These are built-in types, but not reserved words. They can be shadowed, + // but the shadowing naturally makes it impossible to use them later in + // scope. A listing of these can be found here: + // https://docs.python.org/3/library/stdtypes.html + bool: Boolean + + dict: StringMap + + float: Float + + int: Integer + + list: StringList + + str: String + + bytes: Blob + + bytearray: Blob + + // We don't actually use these, but they're here for completeness. + complex: Float + + tuple: StringList + + range: StringList + + memoryview: Blob + + set: StringList + + frozenset: StringList +} + +// These would result in class names that produce syntax errors since they're +// reserved words. +structure True {} + +structure False {} + +structure None {} + +@timestampFormat("date-time") +timestamp DateTime + +@timestampFormat("epoch-seconds") +timestamp EpochSeconds + +@timestampFormat("http-date") +timestamp HttpDate + +list StringList { + member: String +} + +structure MutuallyRecursiveA { + mutual: MutuallyRecursiveB +} + +structure MutuallyRecursiveB { + mutual: MutuallyRecursiveA +} + +// CitySummaries is a list of CitySummary structures. +list CitySummaries { + member: CitySummary +} + +// CitySummaries is a sparse list of CitySummary structures. +@sparse +list SparseCitySummaries { + member: CitySummary +} + +// CitySummary contains a reference to a City. +@references([ + { + resource: City + } +]) +structure CitySummary { + @required + cityId: CityId + + @required + name: String + + number: String + + case: String +} + +@readonly +@http(method: "GET", uri: "/current-time") +operation GetCurrentTime { + output := { + @required + time: Timestamp + } +} + +@readonly +@http(method: "GET", uri: "/cities/{cityId}/forecast") +operation GetForecast { + input := { + @required + @httpLabel + cityId: CityId + } + + output := { + chanceOfRain: Float + precipitation: Precipitation + } +} + +union Precipitation { + rain: PrimitiveBoolean + sleet: PrimitiveBoolean + hail: StringMap + snow: StringYesNo + mixed: IntYesNo + other: OtherStructure + blob: Blob + foo: example.weather.nested#Foo + baz: example.weather.nested.more#Baz +} + +@http(method: "POST", uri: "/cities/{cityId}/atmosphere") +operation StreamAtmosphericConditions { + input := { + @required + @httpLabel + cityId: CityId + + @required + @httpPayload + stream: AtmosphericConditions + } + + output := { + @required + @httpHeader("x-initial-sample-rate") + initialSampleRate: Double + + @required + @httpPayload + stream: CollectionDirectives + } +} + +@streaming +union AtmosphericConditions { + humidity: HumiditySample + pressure: PressureSample + temperature: TemperatureSample +} + +@mixin +structure Sample { + @required + collectionTime: Timestamp +} + +structure HumiditySample with [Sample] { + @required + humidity: Double +} + +structure PressureSample with [Sample] { + @required + pressure: Double +} + +structure TemperatureSample with [Sample] { + @required + temperature: Double +} + +@streaming +union CollectionDirectives { + sampleRate: SampleRate +} + +structure SampleRate { + @required + samplesPerMinute: Double +} + +structure OtherStructure {} + +enum StringYesNo { + YES + NO +} + +intEnum IntYesNo { + YES = 1 + NO = 2 +} + +map StringMap { + key: String + value: String +} + +@readonly +@suppress(["HttpMethodSemantics"]) +@http(method: "POST", uri: "/cities/{cityId}/image") +operation GetCityImage { + input := { + @required + @httpLabel + cityId: CityId + + @required + imageType: ImageType + } + + output := { + @httpPayload + @required + image: CityImageData + } + + errors: [ + NoSuchResource + ] +} + +union ImageType { + raw: Boolean + png: PNGImage +} + +structure PNGImage { + @required + height: Integer + + @required + width: Integer +} + +@streaming +blob CityImageData + +@readonly +@http(method: "GET", uri: "/cities/{cityId}/announcements") +operation GetCityAnnouncements { + input := { + @required + @httpLabel + cityId: CityId + } + + output := { + @httpHeader("x-last-updated") + lastUpdated: Timestamp + + @httpPayload + announcements: Announcements + } + + errors: [ + NoSuchResource + ] +} + +@streaming +union Announcements { + police: Message + fire: Message + health: Message +} + +structure Message { + message: String + author: String +} + +// Define a fake protocol trait for use. +@trait +@protocolDefinition +structure fakeProtocol {} diff --git a/codegen/plugins/types/src/it/resources/META-INF/smithy/manifest b/codegen/plugins/types/src/it/resources/META-INF/smithy/manifest new file mode 100644 index 000000000..36913fb59 --- /dev/null +++ b/codegen/plugins/types/src/it/resources/META-INF/smithy/manifest @@ -0,0 +1,3 @@ +main.smithy +more-nesting.smithy +nested.smithy diff --git a/codegen/plugins/types/src/it/resources/META-INF/smithy/more-nesting.smithy b/codegen/plugins/types/src/it/resources/META-INF/smithy/more-nesting.smithy new file mode 100644 index 000000000..fd5827d5d --- /dev/null +++ b/codegen/plugins/types/src/it/resources/META-INF/smithy/more-nesting.smithy @@ -0,0 +1,8 @@ +$version: "2.0" + +namespace example.weather.nested.more + +structure Baz { + baz: String + bar: String +} diff --git a/codegen/plugins/types/src/it/resources/META-INF/smithy/nested.smithy b/codegen/plugins/types/src/it/resources/META-INF/smithy/nested.smithy new file mode 100644 index 000000000..ef9a0398e --- /dev/null +++ b/codegen/plugins/types/src/it/resources/META-INF/smithy/nested.smithy @@ -0,0 +1,8 @@ +$version: "2.0" + +namespace example.weather.nested + +structure Foo { + baz: String + bar: String +} diff --git a/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/DirectedPythonTypeCodegen.java b/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/DirectedPythonTypeCodegen.java new file mode 100644 index 000000000..c107267d3 --- /dev/null +++ b/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/DirectedPythonTypeCodegen.java @@ -0,0 +1,172 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.types; + +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.codegen.core.TopologicalIndex; +import software.amazon.smithy.codegen.core.directed.CreateContextDirective; +import software.amazon.smithy.codegen.core.directed.CreateSymbolProviderDirective; +import software.amazon.smithy.codegen.core.directed.CustomizeDirective; +import software.amazon.smithy.codegen.core.directed.DirectedCodegen; +import software.amazon.smithy.codegen.core.directed.GenerateEnumDirective; +import software.amazon.smithy.codegen.core.directed.GenerateErrorDirective; +import software.amazon.smithy.codegen.core.directed.GenerateIntEnumDirective; +import software.amazon.smithy.codegen.core.directed.GenerateListDirective; +import software.amazon.smithy.codegen.core.directed.GenerateMapDirective; +import software.amazon.smithy.codegen.core.directed.GenerateServiceDirective; +import software.amazon.smithy.codegen.core.directed.GenerateStructureDirective; +import software.amazon.smithy.codegen.core.directed.GenerateUnionDirective; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.traits.InputTrait; +import software.amazon.smithy.model.traits.OutputTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonFormatter; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonSymbolProvider; +import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.generators.EnumGenerator; +import software.amazon.smithy.python.codegen.generators.InitGenerator; +import software.amazon.smithy.python.codegen.generators.IntEnumGenerator; +import software.amazon.smithy.python.codegen.generators.ListGenerator; +import software.amazon.smithy.python.codegen.generators.MapGenerator; +import software.amazon.smithy.python.codegen.generators.SchemaGenerator; +import software.amazon.smithy.python.codegen.generators.ServiceErrorGenerator; +import software.amazon.smithy.python.codegen.generators.StructureGenerator; +import software.amazon.smithy.python.codegen.generators.UnionGenerator; +import software.amazon.smithy.python.codegen.integrations.PythonIntegration; +import software.amazon.smithy.python.codegen.writer.PythonDelegator; + +public class DirectedPythonTypeCodegen + implements DirectedCodegen { + @Override + public SymbolProvider createSymbolProvider(CreateSymbolProviderDirective directive) { + return new PythonSymbolProvider(directive.model(), directive.settings()); + } + + @Override + public GenerationContext createContext(CreateContextDirective directive) { + return GenerationContext.builder() + .model(directive.model()) + .settings(directive.settings()) + .symbolProvider(directive.symbolProvider()) + .fileManifest(directive.fileManifest()) + .writerDelegator(new PythonDelegator( + directive.fileManifest(), + directive.symbolProvider(), + directive.settings())) + .integrations(directive.integrations()) + .build(); + } + + @Override + public void customizeBeforeShapeGeneration(CustomizeDirective directive) { + new ServiceErrorGenerator(directive.settings(), directive.context().writerDelegator()).run(); + SchemaGenerator.generateAll(directive.context(), directive.connectedShapes().values(), shape -> { + if (shape.isStructureShape()) { + return shouldGenerateStructure(directive.settings(), shape); + } + return true; + }); + } + + @Override + public void generateService(GenerateServiceDirective directive) {} + + @Override + public void generateStructure(GenerateStructureDirective directive) { + // If we're only generating data shapes, there's no need to generate input or output shapes. + if (!shouldGenerateStructure(directive.settings(), directive.shape())) { + return; + } + + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + StructureGenerator generator = new StructureGenerator( + directive.context(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes()); + generator.run(); + }); + } + + private boolean shouldGenerateStructure(PythonSettings settings, Shape shape) { + if (shape.getId().getNamespace().equals("smithy.synthetic")) { + return false; + } + return !(settings.artifactType().equals(PythonSettings.ArtifactType.TYPES) + && (shape.hasTrait(InputTrait.class) || shape.hasTrait(OutputTrait.class))); + } + + @Override + public void generateError(GenerateErrorDirective directive) { + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + StructureGenerator generator = new StructureGenerator( + directive.context(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes()); + generator.run(); + }); + } + + @Override + public void generateUnion(GenerateUnionDirective directive) { + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + UnionGenerator generator = new UnionGenerator( + directive.context(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes()); + generator.run(); + }); + } + + @Override + public void generateList(GenerateListDirective directive) { + var serSymbol = directive.context() + .symbolProvider() + .toSymbol(directive.shape()) + .expectProperty(SymbolProperties.SERIALIZER); + var delegator = directive.context().writerDelegator(); + delegator.useFileWriter(serSymbol.getDefinitionFile(), serSymbol.getNamespace(), writer -> { + new ListGenerator(directive.context(), writer, directive.shape()).run(); + }); + } + + @Override + public void generateMap(GenerateMapDirective directive) { + var serSymbol = directive.context() + .symbolProvider() + .toSymbol(directive.shape()) + .expectProperty(SymbolProperties.SERIALIZER); + var delegator = directive.context().writerDelegator(); + delegator.useFileWriter(serSymbol.getDefinitionFile(), serSymbol.getNamespace(), writer -> { + new MapGenerator(directive.context(), writer, directive.shape()).run(); + }); + } + + @Override + public void generateEnumShape(GenerateEnumDirective directive) { + if (!directive.shape().isEnumShape()) { + return; + } + new EnumGenerator(directive.context(), directive.shape().asEnumShape().get()).run(); + } + + @Override + public void generateIntEnumShape(GenerateIntEnumDirective directive) { + new IntEnumGenerator(directive).run(); + } + + @Override + public void customizeBeforeIntegrations(CustomizeDirective directive) { + new InitGenerator(directive.context()).run(); + } + + @Override + public void customizeAfterIntegrations(CustomizeDirective directive) { + new PythonFormatter(directive.context()).run(); + } +} diff --git a/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenPlugin.java b/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenPlugin.java new file mode 100644 index 000000000..454136120 --- /dev/null +++ b/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenPlugin.java @@ -0,0 +1,105 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.types; + +import java.util.Collection; +import java.util.Set; +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.build.SmithyBuildPlugin; +import software.amazon.smithy.codegen.core.directed.CodegenDirector; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.loader.Prelude; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.InputTrait; +import software.amazon.smithy.model.traits.MixinTrait; +import software.amazon.smithy.model.traits.OutputTrait; +import software.amazon.smithy.model.transform.ModelTransformer; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.integrations.PythonIntegration; +import software.amazon.smithy.python.codegen.writer.PythonWriter; + +public final class PythonTypeCodegenPlugin implements SmithyBuildPlugin { + private static final String SYNTHETIC_NAMESPACE = "smithy.synthetic"; + private static final ShapeId SYNTHETIC_SERVICE_ID = ShapeId.fromParts(SYNTHETIC_NAMESPACE, "TypesGenService"); + private static final ShapeId SYNTHETIC_OPERATION_ID = ShapeId.fromParts(SYNTHETIC_NAMESPACE, "TypesGenOperation"); + private static final ShapeId SYNTHETIC_INPUT_ID = ShapeId.fromParts(SYNTHETIC_NAMESPACE, "TypesGenOperationInput"); + private static final Set GENERATED_TYPES = Set.of( + ShapeType.STRUCTURE, + ShapeType.UNION, + ShapeType.ENUM, + ShapeType.INT_ENUM); + + @Override + public String getName() { + return "python-type-codegen"; + } + + @Override + public void execute(PluginContext context) { + CodegenDirector runner = + new CodegenDirector<>(); + + var typeSettings = PythonTypeCodegenSettings.fromNode(context.getSettings()); + var service = typeSettings.service().orElse(SYNTHETIC_SERVICE_ID); + var pythonSettings = typeSettings.toPythonSettings(service); + + var model = context.getModel(); + if (typeSettings.service().isEmpty()) { + model = addSyntheticService(model, typeSettings.selector().select(model)); + } + + runner.settings(pythonSettings); + runner.directedCodegen(new DirectedPythonTypeCodegen()); + runner.fileManifest(context.getFileManifest()); + runner.service(pythonSettings.service()); + runner.model(model); + runner.integrationClass(PythonIntegration.class); + runner.performDefaultCodegenTransforms(); + runner.run(); + } + + private Model addSyntheticService(Model model, Collection shapes) { + StructureShape.Builder inputBuilder = StructureShape.builder() + .id(SYNTHETIC_INPUT_ID) + .addTrait(new InputTrait()); + + OperationShape.Builder operationBuilder = OperationShape.builder() + .id(SYNTHETIC_OPERATION_ID) + .input(SYNTHETIC_INPUT_ID); + + var index = 0; + for (Shape shape : shapes) { + if (!GENERATED_TYPES.contains(shape.getType()) + || shape.hasTrait(InputTrait.class) + || shape.hasTrait(OutputTrait.class) + || shape.hasTrait(MixinTrait.class) + || Prelude.isPreludeShape(shape)) { + continue; + } + + if (shape.hasTrait(ErrorTrait.class)) { + operationBuilder.addError(shape.getId()); + } else { + inputBuilder.addMember("member" + index, shape.getId()); + index++; + } + } + + ServiceShape service = ServiceShape.builder() + .id(SYNTHETIC_SERVICE_ID) + .addOperation(SYNTHETIC_OPERATION_ID) + .build(); + + ModelTransformer transformer = ModelTransformer.create(); + return transformer.replaceShapes(model, Set.of(inputBuilder.build(), operationBuilder.build(), service)); + } +} diff --git a/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenSettings.java b/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenSettings.java new file mode 100644 index 000000000..337e3d60f --- /dev/null +++ b/codegen/plugins/types/src/main/java/software/amazon/smithy/python/codegen/types/PythonTypeCodegenSettings.java @@ -0,0 +1,138 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.types; + +import java.util.Arrays; +import java.util.Optional; +import software.amazon.smithy.model.node.ObjectNode; +import software.amazon.smithy.model.node.StringNode; +import software.amazon.smithy.model.selector.Selector; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.SmithyUnstableApi; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * Settings used by {@link PythonTypeCodegenPlugin}. + * + * @param service The id of the service that is being generated. + * @param moduleName The name of the module to generate. + * @param moduleVersion The version of the module to generate. + * @param moduleDescription The optional module description for the module that will be generated. + * @param selector An optional selector to reduce the set of shapes to be generated. + */ +@SmithyUnstableApi +public record PythonTypeCodegenSettings( + Optional service, + String moduleName, + String moduleVersion, + String moduleDescription, + Selector selector) implements ToSmithyBuilder { + + private static final String SERVICE = "service"; + private static final String MODULE_NAME = "module"; + private static final String MODULE_DESCRIPTION = "moduleDescription"; + private static final String MODULE_VERSION = "moduleVersion"; + private static final String SELECTOR = "selector"; + + private PythonTypeCodegenSettings(Builder builder) { + this( + Optional.ofNullable(builder.service), + builder.moduleName, + builder.moduleVersion, + builder.moduleDescription, + builder.selector); + } + + @Override + public Builder toBuilder() { + Builder builder = builder() + .moduleName(moduleName) + .moduleVersion(moduleVersion) + .moduleDescription(moduleDescription) + .selector(selector); + service.ifPresent(builder::service); + return builder; + } + + public PythonSettings toPythonSettings(ShapeId service) { + return PythonSettings.builder() + .service(service) + .moduleName(moduleName) + .moduleVersion(moduleVersion) + .moduleDescription(moduleDescription) + .artifactType(PythonSettings.ArtifactType.TYPES) + .build(); + } + + public PythonSettings toPythonSettings() { + return toPythonSettings(service.get()); + } + + /** + * Create a settings object from a configuration object node. + * + * @param config Config object to load. + * @return Returns the extracted settings. + */ + public static PythonTypeCodegenSettings fromNode(ObjectNode config) { + config.warnIfAdditionalProperties(Arrays.asList(SERVICE, MODULE_NAME, MODULE_DESCRIPTION, MODULE_VERSION)); + + String moduleName = config.expectStringMember(MODULE_NAME).getValue(); + Builder builder = builder() + .moduleName(moduleName) + .moduleVersion(config.expectStringMember(MODULE_VERSION).getValue()); + config.getStringMember(SERVICE).map(StringNode::expectShapeId).ifPresent(builder::service); + config.getStringMember(MODULE_DESCRIPTION).map(StringNode::getValue).ifPresent(builder::moduleDescription); + config.getStringMember(SELECTOR).map(node -> Selector.parse(node.getValue())).ifPresent(builder::selector); + return builder.build(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder implements SmithyBuilder { + + private ShapeId service; + private String moduleName; + private String moduleVersion; + private String moduleDescription; + private Selector selector = Selector.IDENTITY; + + @Override + public PythonTypeCodegenSettings build() { + SmithyBuilder.requiredState("moduleName", moduleName); + SmithyBuilder.requiredState("moduleVersion", moduleVersion); + return new PythonTypeCodegenSettings(this); + } + + public Builder service(ShapeId service) { + this.service = service; + return this; + } + + public Builder moduleName(String moduleName) { + this.moduleName = moduleName; + return this; + } + + public Builder moduleVersion(String moduleVersion) { + this.moduleVersion = moduleVersion; + return this; + } + + public Builder moduleDescription(String moduleDescription) { + this.moduleDescription = moduleDescription; + return this; + } + + public Builder selector(Selector selector) { + this.selector = selector == null ? Selector.IDENTITY : selector; + return this; + } + } +} diff --git a/codegen/plugins/types/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin b/codegen/plugins/types/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin new file mode 100644 index 000000000..34f8ddd24 --- /dev/null +++ b/codegen/plugins/types/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin @@ -0,0 +1,6 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.python.codegen.PythonTypeCodegenPlugin diff --git a/codegen/settings.gradle.kts b/codegen/settings.gradle.kts index 6591b98d4..4932387b8 100644 --- a/codegen/settings.gradle.kts +++ b/codegen/settings.gradle.kts @@ -17,3 +17,5 @@ rootProject.name = "smithy-python" include(":core") include(":protocol-test") include(":aws:core") +include(":plugins") +include(":plugins:types")