feat(translator): Provide a module containing a translator aiming spe…

…cifically at Spark generated queries. Spark will wrap most queries in a subquery, using it as a derived table. This will often lead to the outer query being a SQL statement, containing possible Cypher as the derived table. The SQL to Cypher translator will be unhappy with that. So this additional translator might be configured to run _before_ the SQL to Cypher translator, so that it can unwrap the inner query if it's cypher, wrap in in a `CALL {}` statement and than skip SQL to Cypher translation. Signed-off-by: Michael Simons <[email protected]>
neo4j · Jan 20, 2025 · b0416d8 · b0416d8
1 parent b1e3435
commit b0416d8
Show file tree

Hide file tree

Showing 17 changed files with 627 additions and 13 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -58,7 +58,7 @@ jobs:
         run: >
           ./mvnw --no-transfer-progress
           -P$USE_SONAR -Dsonar.projectKey=neo4j-jdbc -Dsonar.projectName='neo4j-jdbc'
-          -am -pl neo4j-jdbc -pl neo4j-jdbc-bom -pl bundles/neo4j-jdbc-bundle -pl bundles/neo4j-jdbc-full-bundle          
+          -am -pl neo4j-jdbc -pl neo4j-jdbc-bom -pl bundles/neo4j-jdbc-bundle -pl bundles/neo4j-jdbc-full-bundle -pl neo4j-jdbc-translator/sparkcleaner
           clean install
 
   integration_tests:

diff --git a/neo4j-jdbc-bom/pom.xml b/neo4j-jdbc-bom/pom.xml
@@ -56,6 +56,11 @@
 				<artifactId>neo4j-jdbc-translator-impl</artifactId>
 				<version>${project.version}</version>
 			</dependency>
+			<dependency>
+				<groupId>org.neo4j</groupId>
+				<artifactId>neo4j-jdbc-translator-sparkcleaner</artifactId>
+				<version>${project.version}</version>
+			</dependency>
 			<dependency>
 				<groupId>org.neo4j</groupId>
 				<artifactId>neo4j-jdbc-translator-spi</artifactId>

diff --git a/neo4j-jdbc-it/neo4j-jdbc-it-cp/pom.xml b/neo4j-jdbc-it/neo4j-jdbc-it-cp/pom.xml
@@ -57,6 +57,11 @@
 			<artifactId>neo4j-jdbc-translator-impl</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.neo4j</groupId>
+			<artifactId>neo4j-jdbc-translator-sparkcleaner</artifactId>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-simple</artifactId>

diff --git a/neo4j-jdbc-it/neo4j-jdbc-it-cp/src/test/java/org/neo4j/jdbc/it/cp/TranslationIT.java b/neo4j-jdbc-it/neo4j-jdbc-it-cp/src/test/java/org/neo4j/jdbc/it/cp/TranslationIT.java
@@ -399,6 +399,56 @@ void multipleHops() throws SQLException {
 
 	}
 
+	@SuppressWarnings({ "SqlNoDataSourceInspection", "unchecked" })
+	@Test
+	void sparkUnwrapRewrap() throws Exception {
+
+		var title = "JDBC The SQL strikes back";
+
+		try (var connection = getConnection(true, false)) {
+			try (var statement = ((Neo4jPreparedStatement) connection.prepareStatement("""
+					/*+ NEO4J FORCE_CYPHER */
+					CREATE (m:Movie {title:  $title, released: $released})
+					CREATE (d:Person {name: 'Donald D. Chamberlin'})
+					CREATE (r:Person {name: 'Raymond F. Boyce'})
+					CREATE (e:Person {name: 'Edgar F. Codd'})
+					CREATE (r)-[:ACTED_IN {roles: ['Researcher']}]->(m)
+					CREATE (d)-[:ACTED_IN {roles: ['Designer']}]->(m)
+					CREATE (e)-[:ACTED_IN {roles: ['Influencer']}]->(m)
+					"""))) {
+				statement.setString("title", title);
+				statement.setInt("released", 1974);
+				statement.execute();
+			}
+
+			try (var statement = connection.createStatement(); var rs = statement.executeQuery("""
+					SELECT * FROM (
+					MATCH (m:Movie)<-[:ACTED_IN]-(p:Person)
+					RETURN m.title AS title, collect(p.name) AS actors
+					ORDER BY m.title
+					) SPARK_GEN_SUBQ_0 WHERE 1=0
+					""")) {
+
+				assertThat(rs.next()).isTrue();
+				assertThat(rs.getString("title")).isEqualTo(title);
+				assertThat((List<String>) rs.getObject("actors")).containsExactlyInAnyOrder("Donald D. Chamberlin",
+						"Raymond F. Boyce", "Edgar F. Codd");
+				assertThat(rs.next()).isFalse();
+			}
+
+			try (var statement = connection.createStatement(); var rs = statement.executeQuery("""
+					SELECT * FROM (
+					SELECT name FROM Person SPARK_GEN_SUBQ_0 ORDER BY name DESC
+					) SPARK_GEN_SUBQ_0 WHERE 1=0
+					""")) {
+				assertThat(rs.next()).isTrue();
+				assertThat(rs.getString(1)).isEqualTo("Raymond F. Boyce");
+				assertThat(rs.next()).isFalse();
+			}
+
+		}
+	}
+
 	record PersonAndTitle(String name, String title) {
 	}
 

diff --git a/neo4j-jdbc-it/neo4j-jdbc-it-mp/pom.xml b/neo4j-jdbc-it/neo4j-jdbc-it-mp/pom.xml
@@ -48,6 +48,11 @@
 			<artifactId>neo4j-jdbc-translator-impl</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.neo4j</groupId>
+			<artifactId>neo4j-jdbc-translator-sparkcleaner</artifactId>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 	<build>

diff --git a/neo4j-jdbc-it/neo4j-jdbc-it-mp/src/test/java/org/neo4j/jdbc/it/mp/SmokeIT.java b/neo4j-jdbc-it/neo4j-jdbc-it-mp/src/test/java/org/neo4j/jdbc/it/mp/SmokeIT.java
@@ -43,6 +43,7 @@ void driverShouldBeLoaded() {
 		Assertions.assertThatNoException().isThrownBy(() -> DriverManager.getDriver(url));
 	}
 
+	@SuppressWarnings("SqlNoDataSourceInspection")
 	@Test
 	void shouldConfigureConnectionToUseSqlTranslator() throws SQLException {
 
@@ -51,6 +52,18 @@ void shouldConfigureConnectionToUseSqlTranslator() throws SQLException {
 		assertThat(connection).isNotNull();
 		assertThat(connection.nativeSQL("SELECT * FROM FooBar"))
 			.isEqualTo("MATCH (foobar:FooBar) RETURN elementId(foobar) AS `v$id`");
+		assertThat(connection.nativeSQL("""
+				SELECT * FROM (
+				MATCH (m:Movie)<-[:ACTED_IN]-(p:Person)
+				RETURN m.title AS title, collect(p.name) AS actors
+				ORDER BY m.title
+				) SPARK_GEN_SUBQ_0 WHERE 1=0
+				""")).isEqualTo("""
+				/*+ NEO4J FORCE_CYPHER */
+				CALL {MATCH (m:Movie)<-[:ACTED_IN]-(p:Person)
+				RETURN m.title AS title, collect(p.name) AS actors
+				ORDER BY m.title} RETURN * LIMIT 1
+				""".trim());
 	}
 
 	static boolean boltPortIsReachable() {

diff --git a/neo4j-jdbc-translator/impl/src/main/java/org/neo4j/jdbc/translator/impl/SqlToCypher.java b/neo4j-jdbc-translator/impl/src/main/java/org/neo4j/jdbc/translator/impl/SqlToCypher.java
@@ -59,10 +59,7 @@
 import org.jooq.Table;
 import org.jooq.TableField;
 import org.jooq.conf.ParamType;
-import org.jooq.conf.ParseUnknownFunctions;
-import org.jooq.conf.ParseWithMetaLookups;
 import org.jooq.impl.DSL;
-import org.jooq.impl.DefaultConfiguration;
 import org.jooq.impl.ParserException;
 import org.jooq.impl.QOM;
 import org.jooq.impl.QOM.TableAlias;
@@ -183,14 +180,7 @@ private String translate0(Query query, DatabaseMetaData databaseMetaData) {
 	@SuppressWarnings("ResultOfMethodCallIgnored")
 	private DSLContext createDSLContext() {
 
-		var settings = new DefaultConfiguration().settings()
-			.withParseNameCase(this.config.getParseNameCase())
-			.withRenderNameCase(this.config.getRenderNameCase())
-			.withParseWithMetaLookups(ParseWithMetaLookups.IGNORE_ON_FAILURE)
-			.withDiagnosticsLogging(true)
-			.withParseUnknownFunctions(ParseUnknownFunctions.IGNORE)
-			.withParseDialect(this.config.getSqlDialect());
-
+		var settings = this.config.asSettings();
 		Optional.ofNullable(this.config.getParseNamedParamPrefix())
 			.filter(Predicate.not(String::isBlank))
 			.map(String::trim)

diff --git a/...-jdbc-translator/impl/src/main/java/org/neo4j/jdbc/translator/impl/SqlToCypherConfig.java b/...-jdbc-translator/impl/src/main/java/org/neo4j/jdbc/translator/impl/SqlToCypherConfig.java
@@ -30,7 +30,11 @@
 
 import org.jooq.SQLDialect;
 import org.jooq.conf.ParseNameCase;
+import org.jooq.conf.ParseUnknownFunctions;
+import org.jooq.conf.ParseWithMetaLookups;
 import org.jooq.conf.RenderNameCase;
+import org.jooq.conf.Settings;
+import org.jooq.impl.DefaultConfiguration;
 import org.neo4j.jdbc.translator.spi.Translator;
 
 /**
@@ -349,6 +353,20 @@ public Integer getPrecedence() {
 		return this.precedence;
 	}
 
+	public Settings asSettings() {
+		return asSettings(ParseWithMetaLookups.IGNORE_ON_FAILURE);
+	}
+
+	public Settings asSettings(ParseWithMetaLookups withMetaLookups) {
+		return new DefaultConfiguration().settings()
+			.withParseNameCase(getParseNameCase())
+			.withRenderNameCase(getRenderNameCase())
+			.withParseWithMetaLookups(withMetaLookups)
+			.withDiagnosticsLogging(isJooqDiagnosticLogging())
+			.withParseUnknownFunctions(ParseUnknownFunctions.IGNORE)
+			.withParseDialect(getSqlDialect());
+	}
+
 	/**
 	 * A builder to create new instances of {@link SqlToCypherConfig configurations}.
 	 */

diff --git a/neo4j-jdbc-translator/pom.xml b/neo4j-jdbc-translator/pom.xml
@@ -36,6 +36,7 @@
 	<modules>
 		<module>spi</module>
 		<module>impl</module>
+		<module>sparkcleaner</module>
 		<module>text2cypher</module>
 	</modules>
 

diff --git a/neo4j-jdbc-translator/sparkcleaner/pom.xml b/neo4j-jdbc-translator/sparkcleaner/pom.xml
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright (c) 2023-2025 "Neo4j,"
+    Neo4j Sweden AB [https://neo4j.com]
+
+    This file is part of Neo4j.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        https://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>org.neo4j</groupId>
+		<artifactId>neo4j-jdbc-translator</artifactId>
+		<version>6.1.2-SNAPSHOT</version>
+	</parent>
+
+	<artifactId>neo4j-jdbc-translator-sparkcleaner</artifactId>
+
+	<packaging>jar</packaging>
+	<name>Neo4j JDBC Driver (Spark preparing Translator)</name>
+	<description>A specialized translator unwrapping Spark subqueries</description>
+
+	<properties>
+		<sonar.coverage.jacoco.xmlReportPaths>${basedir}/../${aggregate.report.dir}</sonar.coverage.jacoco.xmlReportPaths>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.neo4j</groupId>
+			<artifactId>cypher-v5-antlr-parser</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.neo4j</groupId>
+			<artifactId>neo4j-jdbc-translator-spi</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.assertj</groupId>
+			<artifactId>assertj-core</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.mockito</groupId>
+			<artifactId>mockito-core</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>com.github.siom79.japicmp</groupId>
+				<artifactId>japicmp-maven-plugin</artifactId>
+			</plugin>
+			<plugin>
+				<!-- First shade the things we won't definitely not have as dependency -->
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<phase>package</phase>
+						<configuration>
+							<artifactSet>
+								<includes>
+									<include>org.neo4j:cypher-antlr-common</include>
+									<include>org.neo4j:cypher-v5-antlr-parser</include>
+									<include>org.antlr:antlr4-runtime</include>
+								</includes>
+							</artifactSet>
+							<relocations>
+								<relocation>
+									<pattern>org.antlr</pattern>
+									<shadedPattern>org.neo4j.jdbc.translator.sparkcleaner.internal.shaded.antlr</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.neo4j.cypher.internal.parser</pattern>
+									<shadedPattern>org.neo4j.jdbc.translator.sparkcleaner.internal.shaded.parser.common</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.neo4j.cypher.internal.parser.v5</pattern>
+									<shadedPattern>org.neo4j.jdbc.translator.sparkcleaner.internal.shaded.parser.v5</shadedPattern>
+								</relocation>
+							</relocations>
+							<transformers>
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+							</transformers>
+							<filters>
+								<filter>
+									<artifact>*:*</artifact>
+									<excludes>
+										<exclude>LICENSE.txt</exclude>
+										<exclude>META-INF/LICENSE*.txt</exclude>
+										<exclude>META-INF/MANIFEST.MF</exclude>
+										<exclude>META-INF/NOTICE.txt</exclude>
+									</excludes>
+								</filter>
+							</filters>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<!-- Than add the module info -->
+				<groupId>org.moditect</groupId>
+				<artifactId>moditect-maven-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>add-module-infos</id>
+						<goals>
+							<goal>add-module-info</goal>
+						</goals>
+						<phase>package</phase>
+						<configuration>
+							<overwriteExistingFiles>true</overwriteExistingFiles>
+							<module>
+								<moduleInfoSource>module org.neo4j.jdbc.translator.sparkcleaner {
+										provides org.neo4j.jdbc.translator.spi.TranslatorFactory with org.neo4j.jdbc.translator.sparkcleaner.SparkSubqueryCleaningTranslatorFactory;
+										requires org.neo4j.jdbc.translator.spi;
+									}</moduleInfoSource>
+							</module>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>