fix: improve the handling of Avro schemas (#155)

* fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * Avro schemas are now named based on their name and namespace. * Removed file that wasn't supposed to be checked in. * fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * Avro schemas are now named based on their name and namespace. * Removed file that wasn't supposed to be checked in. * Fixed linter problems. * Add a technical requirements section to the README to note the required generator version. * feat: add a comment to methods listing which message types they use. (#163) Co-authored-by: Michael Davis <[email protected]> * chore(release): v0.11.0 * Updated package.json with the minimum version of generator. * fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * Avro schemas are now named based on their name and namespace. * Add a technical requirements section to the README to note the required generator version. * Updated package.json with the minimum version of generator. Co-authored-by: Michael Davis <[email protected]> Co-authored-by: asyncapi-bot <[email protected]>
asyncapi · Sep 13, 2021 · e1338fc · e1338fc
1 parent 830a803
commit e1338fc
Show file tree

Hide file tree

Showing 8 changed files with 269 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -8,6 +8,11 @@ The Spring Cloud Stream microservice generated using this template will be an _a
 
 Note that this template ignores the 'Servers' section of AsyncAPI documents. The main reason for this is because SCSt does not directly work with messaging protocols. Protocols are implementation details specific to binders, and SCSt applications need not know or care which protocol is being used.
 
+## Technical requirements
+
+- 1.8.6 =< [Generator](https://github.com/asyncapi/generator/)
+- Generator specific [requirements](https://github.com/asyncapi/generator/#requirements)
+
 ## Specification Conformance
 Note that this template interprets the AsyncAPI document in conformance with the [AsyncAPI Specification](https://www.asyncapi.com/docs/specifications/2.0.0/).
 This means that when the template sees a subscribe operation, it will generate code to publish to that operation's channel.

diff --git a/hooks/post-process.js b/hooks/post-process.js
@@ -1,7 +1,8 @@
 // vim: set ts=2 sw=2 sts=2 expandtab :
 const fs = require('fs');
 const path = require('path');
-const _ = require('lodash');
+const ScsLib = require('../lib/scsLib.js');
+const scsLib = new ScsLib();
 // To enable debug logging, set the env var DEBUG="postProcess" with whatever things you want to see.
 const debugPostProcess = require('debug')('postProcess');
 
@@ -14,14 +15,22 @@ module.exports = {
     const info = asyncapi.info();
     let javaPackage = generator.templateParams['javaPackage'];
     const extensions = info.extensions();
+    let overridePath;
 
     if (!javaPackage && info && extensions) {
       javaPackage = extensions['x-java-package'];
     }
 
     if (javaPackage) {
       debugPostProcess(`package: ${javaPackage}`);
-      const overridePath = `${generator.targetDir + sourceHead + javaPackage.replace(/\./g, '/')}/`;
+      overridePath = `${generator.targetDir + sourceHead + javaPackage.replace(/\./g, '/')}/`;
+    }
+
+    asyncapi.allSchemas().forEach((value, key, map) => {
+      processSchema(key, value);
+    });
+
+    if (javaPackage) {
       debugPostProcess(`Moving files from ${sourcePath} to ${overridePath}`);
       let first = true;
       fs.readdirSync(sourcePath).forEach(file => {
@@ -33,8 +42,7 @@ module.exports = {
           }
 
           debugPostProcess(`Copying ${file}`);
-          fs.copyFileSync(path.resolve(sourcePath, file), path.resolve(overridePath, file));
-          fs.unlinkSync(path.resolve(sourcePath, file));
+          moveFile(sourcePath, overridePath, file);
         }
       });
       sourcePath = overridePath;
@@ -54,29 +62,63 @@ module.exports = {
 
     // This renames schema objects ensuring they're proper Java class names. It also removes files that are schemas of simple types.
 
-    const schemas = asyncapi.components().schemas();
-    debugPostProcess('schemas:');
-    debugPostProcess(schemas);
+    function processSchema(schemaName, schema) {
+      if (schemaName.startsWith('<')) {
+        debugPostProcess(`found an anonymous schema ${schemaName}`);
+        schemaName = schemaName.replace('<', '');
+        schemaName = schemaName.replace('>', '');    
+      }
+
+      // First see if we need to move it to a different package based on its namespace.
+      // This mainly applies to Avro files which have the fully qualified name.
+      let newSourceDir = sourcePath;
+      const generatedFileName = `${schemaName}.java`;
+      let desiredClassName = scsLib.getClassName(schemaName);
 
-    for (const schemaName in asyncapi.components().schemas()) {
-      const schema = schemas[schemaName];
-      const type = schema.type();
-      debugPostProcess(`postprocess schema ${schemaName} ${type}`);
-      const oldPath = path.resolve(sourcePath, `${schemaName}.java`);
+      const indexOfDot = schemaName.lastIndexOf('.');
+      if (indexOfDot > 0) {
+        const newPackage = schemaName.substring(0, indexOfDot);
+        const className = schemaName.substring(indexOfDot + 1);
+        debugPostProcess(`package: ${newPackage} className: ${className}`);
+        newSourceDir = `${generator.targetDir + sourceHead + newPackage.replace(/\./g, '/')}/`;
+        moveFile(sourcePath, newSourceDir, generatedFileName);
+        desiredClassName = scsLib.getClassName(className);
+      }
 
-      if (type === 'object' || type === 'enum') {
-        let javaName = _.camelCase(schemaName);
-        javaName = _.upperFirst(javaName);
+      const oldPath = path.resolve(newSourceDir, generatedFileName);
+      debugPostProcess(`old path: ${oldPath}`);
 
-        if (javaName !== schemaName) {
-          const newPath = path.resolve(sourcePath, `${javaName}.java`);
-          fs.renameSync(oldPath, newPath);
-          debugPostProcess(`Renamed class file ${schemaName} to ${javaName}`);
+      if (fs.existsSync(oldPath)) {
+        const schemaType = schema.type();
+        debugPostProcess(`Old path exists. schemaType: ${schemaType}`);
+        if (schemaType === 'object' || schemaType === 'enum') {
+          const javaName = scsLib.getClassName(schemaName);
+          debugPostProcess(`desiredClassName: ${desiredClassName} schemaName: ${schemaName}`);
+
+          if (javaName !== schemaName) {
+            const newPath = path.resolve(newSourceDir, `${desiredClassName}.java`);
+            fs.renameSync(oldPath, newPath);
+            debugPostProcess(`Renamed class file ${schemaName} to ${desiredClassName}`);
+          }
+        } else {
+          // In this case it's an anonymous schema for a primitive type or something.
+          debugPostProcess(`deleting ${oldPath}`);
+          fs.unlinkSync(oldPath);
         }
-      } else {
-        fs.unlinkSync(oldPath);
       }
     }
+
+    function moveFile(oldDirectory, newDirectory, fileName) {
+      if (!fs.existsSync(newDirectory)) {
+        fs.mkdirSync(newDirectory, { recursive: true });
+        debugPostProcess(`Made directory ${newDirectory}`);
+      }
+      const oldPath = path.resolve(oldDirectory, fileName);
+      const newPath = path.resolve(newDirectory, fileName);
+      fs.copyFileSync(oldPath, newPath);
+      fs.unlinkSync(oldPath);
+      debugPostProcess(`Moved ${fileName} from ${oldPath} to ${newPath}`);
+    }
   }
 };
 
diff --git a/lib/scsLib.js b/lib/scsLib.js
@@ -4,7 +4,7 @@ const _ = require('lodash');
 class ScsLib {
   // This returns a valid Java class name.
   getClassName(name) {
-    const ret = this.getIdentifierName(name);
+    const ret = _.camelCase(name);
     return _.upperFirst(ret);
   }
 

diff --git a/package.json b/package.json
@@ -65,7 +65,7 @@
     ]
   },
   "generator": {
-    "generator": ">=0.50.0 <2.0.0",
+    "generator": ">=1.8.6 <2.0.0",
     "parameters": {
       "actuator": {
         "description": "If present, it adds the dependencies for spring-boot-starter-web, spring-boot-starter-actuator and micrometer-registry-prometheus.",

diff --git a/template/src/main/java/$$schema$$.java → template/src/main/java/$$objectSchema$$.java b/template/src/main/java/$$schema$$.java → template/src/main/java/$$objectSchema$$.java
diff --git a/test/__snapshots__/integration.test.js.snap b/test/__snapshots__/integration.test.js.snap
@@ -1,5 +1,139 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`template integration tests using the generator avro schemas should appear in a package based on their namespace, if any. 1`] = `
+"package com.acme;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class User {
+
+	public User () {
+	}
+	public User (
+		String displayName, 
+		String email, 
+		Integer age) {
+		this.displayName = displayName;
+		this.email = email;
+		this.age = age;
+	}
+
+
+	private String displayName;
+	private String email;
+	private Integer age;
+
+	public String getDisplayName() {
+		return displayName;
+	}
+
+	public User setDisplayName(String displayName) {
+		this.displayName = displayName;
+		return this;
+	}
+
+
+	public String getEmail() {
+		return email;
+	}
+
+	public User setEmail(String email) {
+		this.email = email;
+		return this;
+	}
+
+
+	public Integer getAge() {
+		return age;
+	}
+
+	public User setAge(Integer age) {
+		this.age = age;
+		return this;
+	}
+
+
+	public String toString() {
+		return \\"User [\\"
+		+ \\" displayName: \\" + displayName
+		+ \\" email: \\" + email
+		+ \\" age: \\" + age
+		+ \\" ]\\";
+	}
+}
+
+"
+`;
+
+exports[`template integration tests using the generator avro schemas should appear in a package based on their namespace, if any. 2`] = `
+"package com.acme;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class UserpublisherUser {
+
+	public UserpublisherUser () {
+	}
+	public UserpublisherUser (
+		String displayName, 
+		String email, 
+		Integer age) {
+		this.displayName = displayName;
+		this.email = email;
+		this.age = age;
+	}
+
+
+	private String displayName;
+	private String email;
+	private Integer age;
+
+	public String getDisplayName() {
+		return displayName;
+	}
+
+	public UserpublisherUser setDisplayName(String displayName) {
+		this.displayName = displayName;
+		return this;
+	}
+
+
+	public String getEmail() {
+		return email;
+	}
+
+	public UserpublisherUser setEmail(String email) {
+		this.email = email;
+		return this;
+	}
+
+
+	public Integer getAge() {
+		return age;
+	}
+
+	public UserpublisherUser setAge(Integer age) {
+		this.age = age;
+		return this;
+	}
+
+
+	public String toString() {
+		return \\"UserpublisherUser [\\"
+		+ \\" displayName: \\" + displayName
+		+ \\" email: \\" + email
+		+ \\" age: \\" + age
+		+ \\" ]\\";
+	}
+}
+
+"
+`;
+
 exports[`template integration tests using the generator should generate a comment for a consumer receiving multiple messages 1`] = `
 "
 import java.util.function.Consumer;

diff --git a/test/integration.test.js b/test/integration.test.js
@@ -100,4 +100,29 @@ describe('template integration tests using the generator', () => {
       expect(file).toMatchSnapshot();
     }
   });
-});
+
+  it('avro schemas should appear in a package based on their namespace, if any.', async () => {
+    // Note that this file has 2 Avro schemas named User, but one has the namespace 'userpublisher.'
+    const OUTPUT_DIR = generateFolderName();
+    const PACKAGE = 'com.acme';
+    const PACKAGE_PATH = path.join(...PACKAGE.split('.'));
+    const AVRO_PACKAGE_PATH = 'userpublisher';
+    const params = {
+      binder: 'kafka',
+      javaPackage: PACKAGE,
+      artifactId: 'asyncApiFileName'
+    };
+
+    const generator = new Generator(path.normalize('./'), OUTPUT_DIR, { forceWrite: true, templateParams: params });
+    await generator.generateFromFile(path.resolve('test', 'mocks/kafka-avro.yaml'));
+
+    const expectedFiles = [
+      `src/main/java/${PACKAGE_PATH}/User.java`,
+      `src/main/java/${AVRO_PACKAGE_PATH}/User.java`,
+    ];
+    for (const index in expectedFiles) {
+      const file = await readFile(path.join(OUTPUT_DIR, expectedFiles[index]), 'utf8');
+      expect(file).toMatchSnapshot();
+    }
+  });
+});
diff --git a/test/mocks/kafka-avro.yaml b/test/mocks/kafka-avro.yaml
@@ -0,0 +1,39 @@
+asyncapi: '2.0.0'
+info:
+  title: Avro Test
+  version: '1.0.0'
+  description: Tests Avro schema generation
+channels:
+  userUpdates:
+    publish:
+      bindings:
+        kafka:
+          groupId: my-group
+      message:
+        schemaFormat: 'application/vnd.apache.avro;version=1.9.0'
+        payload:
+          name: User
+          namespace: userpublisher
+          type: record
+          doc: User information
+          fields:
+            - name: displayName
+              type: string
+            - name: email
+              type: string
+            - name: age
+              type: int
+    subscribe:
+      message:
+        schemaFormat: 'application/vnd.apache.avro;version=1.9.0'
+        payload:
+          name: User
+          type: record
+          doc: User information
+          fields:
+            - name: displayName
+              type: string
+            - name: email
+              type: string
+            - name: age
+              type: int