Updates to latest Zipkin supporting Elasticsearch 8.x (#215)

This drops support of Elasticsearch 6.x, including writing to old index
templates, as it is no longer possible with the driver that supports 7.x
and 8.x.

See elastic/elasticsearch-hadoop#2188

This also updates to the latest Spark 3.x version possible, currently
3.3.x. Note: we can't update to Scala 2.13 due to a conflict between
connectors: (DataStax) Cassandra requires Spark 3.4 on Scala 2.13, but
Elastic requires Spark 3.3.

See elastic/elasticsearch-hadoop#2187

Signed-off-by: Adrian Cole <[email protected]>

README.md

@@ -13,13 +13,13 @@ are supported, including Cassandra, MySQL and Elasticsearch.
 
 ## Versions
 
-* `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 3.11
-* `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 5.6
-* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 5+; tested against last minor release of 6.x and 7.x
+* `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 4.0
+* `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 10.11
+* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 7+; tested against last minor release of 7.x and 8.x
 
 ## Quick-start
 
-Due to [SPARK-26134](https://issues.apache.org/jira/browse/SPARK-26134), Zipkin Dependencies currently requires Java 1.8 or 9 to run.
+Until [SPARK-43831](https://issues.apache.org/jira/browse/SPARK-43831), Zipkin Dependencies requires Java 11 to run.
 
 The quickest way to get started is to fetch the [latest released job](https://search.maven.org/remote_content?g=io.zipkin.dependencies&a=zipkin-dependencies&v=LATEST) as a self-contained jar. For example:
 
@@ -144,7 +144,7 @@ You can use the following command to display what this project is built against:
 ```bash
 $ SPARK_VERSION=$(./mvnw help:evaluate -Dexpression=spark.version -q -DforceStdout)
 $ echo $SPARK_VERSION
-2.4.0
+3.3.4
 ```
 
 Once you've verified your setup is on the correct version, set the `SPARK_MASTER` variable:

cassandra3/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>io.zipkin.dependencies</groupId>
     <artifactId>zipkin-dependencies-parent</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>zipkin-dependencies-cassandra3</artifactId>

cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraExtension.java

@@ -169,7 +169,7 @@ static boolean poolInFlight(CqlSession session) {
   // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
   static final class CassandraContainer extends GenericContainer<CassandraContainer> {
     CassandraContainer() {
-      super(parse("ghcr.io/openzipkin/zipkin-cassandra:2.25.2"));
+      super(parse("ghcr.io/openzipkin/zipkin-cassandra:2.26.0"));
       if ("true".equals(System.getProperty("docker.skip"))) {
         throw new TestAbortedException("${docker.skip} == true");
       }

docker/Dockerfile

@@ -17,7 +17,9 @@
 # Use latest version here: https://github.com/orgs/openzipkin/packages/container/package/java
 # This is defined in many places because Docker has no "env" script functionality unless you use
 # docker-compose: When updating, update everywhere.
-ARG java_version=21.0.1_p12
+#
+# TODO: temporarily 11 until we can update to spark on scala 2.13
+ARG java_version=11.0.21_p9
 
 # We copy files from the context into a scratch container first to avoid a problem where docker and
 # docker-compose don't share layer hashes https://github.com/docker/compose/issues/883 normally.

elasticsearch/pom.xml

@@ -14,13 +14,15 @@
     the License.
 
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
     <groupId>io.zipkin.dependencies</groupId>
     <artifactId>zipkin-dependencies-parent</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>zipkin-dependencies-elasticsearch</artifactId>
@@ -35,7 +37,7 @@
     <dependency>
       <groupId>org.elasticsearch</groupId>
       <artifactId>elasticsearch-spark-30_${scala.binary.version}</artifactId>
-      <version>7.13.4</version>
+      <version>${elasticsearch-spark.version}</version>
     </dependency>
 
     <dependency>

elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java

@@ -168,12 +168,7 @@ private static String getSystemPropertyAsFileResource(String key) {
   }
 
   public void run() {
-    run( // single-type index
-      index + ":span-" + dateStamp + "/span",
-      index + ":dependency-" + dateStamp + "/dependency",
-      SpanBytesDecoder.JSON_V2);
-
-    run( // single-type index with ES 7+
+    run(
       index + "-span-" + dateStamp,
       index + "-dependency-" + dateStamp,
       SpanBytesDecoder.JSON_V2);

elasticsearch/src/test/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJobTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2020 The OpenZipkin Authors
+ * Copyright 2016-2023 The OpenZipkin Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
  * in compliance with the License. You may obtain a copy of the License at
@@ -14,6 +14,7 @@
 package zipkin2.dependencies.elasticsearch;
 
 import java.io.IOException;
+import java.util.Base64;
 import okhttp3.mockwebserver.MockResponse;
 import okhttp3.mockwebserver.MockWebServer;
 import org.elasticsearch.hadoop.EsHadoopException;
@@ -24,7 +25,6 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static okhttp3.mockwebserver.SocketPolicy.DISCONNECT_AT_START;
 import static okhttp3.tls.internal.TlsUtil.localhost;
-import static org.apache.commons.net.util.Base64.encodeBase64String;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
@@ -65,8 +65,9 @@ class ElasticsearchDependenciesJobTest {
     assertThatThrownBy(job::run)
       .isInstanceOf(EsHadoopException.class);
 
+    String encoded = Base64.getEncoder().encodeToString("foo:bar".getBytes(UTF_8));
     assertThat(es.takeRequest().getHeader("Authorization"))
-      .isEqualTo("Basic " + encodeBase64String("foo:bar".getBytes(UTF_8)).trim());
+      .isEqualTo("Basic " + encoded.trim());
   }
 
   @Test void authWorksWithSsl() throws Exception {
@@ -88,8 +89,9 @@ class ElasticsearchDependenciesJobTest {
     assertThatThrownBy(job::run)
       .isInstanceOf(EsHadoopException.class);
 
+    String encoded = Base64.getEncoder().encodeToString("foo:bar".getBytes(UTF_8));
     assertThat(es.takeRequest().getHeader("Authorization"))
-      .isEqualTo("Basic " + encodeBase64String("foo:bar".getBytes("UTF-8")).trim());
+      .isEqualTo("Basic " + encoded.trim());
   }
 
   @Test void parseHosts_default() {

elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchExtension.java

@@ -126,7 +126,7 @@ String hostPort() {
   // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
   static final class ElasticsearchContainer extends GenericContainer<ElasticsearchContainer> {
     ElasticsearchContainer(int majorVersion) {
-      super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":2.25.2"));
+      super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":2.26.0"));
       if ("true".equals(System.getProperty("docker.skip"))) {
         throw new TestAbortedException("${docker.skip} == true");
       }

elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV6Dependencies.java → elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV8Dependencies.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2020 The OpenZipkin Authors
+ * Copyright 2016-2023 The OpenZipkin Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
  * in compliance with the License. You may obtain a copy of the License at
@@ -17,8 +17,8 @@
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
-class ITElasticsearchV6Dependencies extends ITElasticsearchDependencies {
-  @RegisterExtension ElasticsearchExtension elasticsearch = new ElasticsearchExtension(6);
+class ITElasticsearchV8Dependencies extends ITElasticsearchDependencies {
+  @RegisterExtension ElasticsearchExtension elasticsearch = new ElasticsearchExtension(8);
 
   @Override ElasticsearchExtension elasticsearch() {
     return elasticsearch;

main/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>io.zipkin.dependencies</groupId>
     <artifactId>zipkin-dependencies-parent</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>zipkin-dependencies</artifactId>

mysql/pom.xml

@@ -20,20 +20,14 @@
   <parent>
     <groupId>io.zipkin.dependencies</groupId>
     <artifactId>zipkin-dependencies-parent</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>zipkin-dependencies-mysql</artifactId>
   <name>Zipkin Dependencies: MySQL</name>
 
   <properties>
     <main.basedir>${project.basedir}/..</main.basedir>
-    <!-- MySQL connector is GPL, even if it has an OSS exception.
-         https://www.mysql.com/about/legal/licensing/foss-exception/
-
-         MariaDB has a friendlier license, LGPL, which is less scary in audits.
-    -->
-    <mariadb-java-client.version>2.7.11</mariadb-java-client.version>
   </properties>
 
   <dependencies>

mysql/src/main/java/zipkin2/dependencies/mysql/MySQLDependenciesJob.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2020 The OpenZipkin Authors
+ * Copyright 2016-2023 The OpenZipkin Authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
  * in compliance with the License. You may obtain a copy of the License at
@@ -159,7 +159,8 @@ public MySQLDependenciesJob build() {
     this.url = new StringBuilder("jdbc:mysql://")
         .append(builder.host).append(":").append(builder.port)
         .append("/").append(builder.db)
-        .append("?autoReconnect=true")
+        .append("?permitMysqlScheme") // spark doesn't understand "mariadb"
+        .append("&autoReconnect=true")
         .append("&useSSL=").append(builder.useSsl).toString();
     this.user = builder.user;
     this.password = builder.password;

mysql/src/test/java/zipkin2/storage/mysql/v1/MySQLExtension.java

@@ -54,10 +54,11 @@ MySQLStorage.Builder computeStorageBuilder() {
     final MariaDbDataSource dataSource;
 
     try {
-      dataSource = new MariaDbDataSource(host(), port(), "zipkin");
+      dataSource = new MariaDbDataSource(String.format(
+        "jdbc:mysql://%s:%s/zipkin?permitMysqlScheme&autoReconnect=true&useUnicode=yes&characterEncoding=UTF-8",
+        host(), port()));
       dataSource.setUser("zipkin");
       dataSource.setPassword("zipkin");
-      dataSource.setProperties("autoReconnect=true&useUnicode=yes&characterEncoding=UTF-8");
     } catch (SQLException e) {
       throw new AssertionError(e);
     }
@@ -78,7 +79,7 @@ int port() {
   // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
   static final class MySQLContainer extends GenericContainer<MySQLContainer> {
     MySQLContainer() {
-      super(parse("ghcr.io/openzipkin/zipkin-mysql:2.25.2"));
+      super(parse("ghcr.io/openzipkin/zipkin-mysql:2.26.0"));
       if ("true".equals(System.getProperty("docker.skip"))) {
         throw new TestAbortedException("${docker.skip} == true");
       }

pom.xml

@@ -19,7 +19,7 @@
 
   <groupId>io.zipkin.dependencies</groupId>
   <artifactId>zipkin-dependencies-parent</artifactId>
-  <version>2.7.4-SNAPSHOT</version>
+  <version>3.0.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <modules>
@@ -36,35 +36,55 @@
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 
     <main.basedir>${project.basedir}</main.basedir>
-    <main.java.version>1.8</main.java.version>
-    <main.signature.artifact>java18</main.signature.artifact>
+
+    <!-- All source is Java 11+ -->
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
+    <!-- We don't use animal-sniffer anymore as release obviates it.
+         See https://github.com/mojohaus/animal-sniffer/issues/62 -->
+    <maven.compiler.release>11</maven.compiler.release>
 
     <!-- override to set exclusions per-project -->
     <errorprone.args />
-    <errorprone.version>2.23.0</errorprone.version>
+    <errorprone.version>2.24.0</errorprone.version>
 
-    <!-- TODO: can't update to 2.13 without updating ES driver -->
+    <!-- Use a common scala binary version, and the highest spark patch version.
+         Scala 2.13 is held back due to spark version conflicts until Elastic
+         updates their version. Once 2.13 works, we can also use JDK 17 per
+         https://issues.apache.org/jira/browse/SPARK-33772 and possibly 12 per
+         https://issues.apache.org/jira/browse/SPARK-43831 -->
     <scala.binary.version>2.12</scala.binary.version>
-    <!-- TODO: can't update further without updating ES driver -->
-    <spark.version>3.0.3</spark.version>
+    <!-- Latest patch of highest common minor. 3.3, not 3.4 until ES updates.
+         See https://github.com/elastic/elasticsearch-hadoop/issues/2187 -->
+    <spark.version>3.3.4</spark.version>
+
+    <!-- This dependency is in elasticsearch-hadoop -->
+    <elasticsearch-spark.version>8.11.3</elasticsearch-spark.version>
+
+    <!-- Use latest Cassandra connector matching minor versioned of spark. -->
+    <spark-cassandra-connector.version>3.3.0</spark-cassandra-connector.version>
+    <!-- Note: Zipkin uses the Apache java-driver. Watch for the connector to
+         change from DataStax to this. -->
+    <java-driver.version>4.17.0</java-driver.version>
 
-    <spark-cassandra-connector.version>2.5.1</spark-cassandra-connector.version>
-    <java-driver.version>4.9.0</java-driver.version>
+    <!-- Use zipkin's version of MariaDB client -->
+    <mariadb-java-client.version>3.3.2</mariadb-java-client.version>
 
     <junit-jupiter.version>5.10.1</junit-jupiter.version>
     <testcontainers.version>1.19.3</testcontainers.version>
 
     <!-- These need to match zipkin, so that tests don't have classpath
          conflicts. When updating, also update IT*.java -->
-    <zipkin.version>2.25.2</zipkin.version>
+    <zipkin.version>2.26.0</zipkin.version>
     <armeria.version>1.26.4</armeria.version>
-    <jackson.version>2.16.0</jackson.version>
+    <!-- Match Armeria version to avoid conflicts including running tests in the IDE -->
+    <netty.version>4.1.100.Final</netty.version>
+    <jackson.version>2.16.1</jackson.version>
     <assertj.version>3.24.2</assertj.version>
     <slf4j.version>1.7.36</slf4j.version>
 
     <license.skip>${skipTests}</license.skip>
 
-    <animal-sniffer-maven-plugin.version>1.23</animal-sniffer-maven-plugin.version>
     <go-offline-maven-plugin.version>1.2.8</go-offline-maven-plugin.version>
     <!-- TODO: cleanup any redundant ignores now also in the 4.0 release (once final) -->
     <license-maven-plugin.version>4.3</license-maven-plugin.version>
@@ -137,6 +157,14 @@
 
   <dependencyManagement>
     <dependencies>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-bom</artifactId>
+        <version>${netty.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+
       <dependency>
         <groupId>org.apache.spark</groupId>
         <artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -213,13 +241,6 @@
           <configuration>
             <!-- Add dependencies indirectly referenced by build plugins -->
             <dynamicDependencies>
-              <DynamicDependency>
-                <groupId>org.codehaus.mojo.signature</groupId>
-                <artifactId>${main.signature.artifact}</artifactId>
-                <version>1.0</version>
-                <type>signature</type>
-                <repositoryType>MAIN</repositoryType>
-              </DynamicDependency>
               <DynamicDependency>
                 <groupId>com.mycila</groupId>
                 <artifactId>license-maven-plugin-git</artifactId>
@@ -299,10 +320,7 @@
         <version>${maven-compiler-plugin.version}</version>
         <inherited>true</inherited>
         <configuration>
-          <source>${main.java.version}</source>
-          <target>${main.java.version}</target>
           <fork>true</fork>
-          <release>8</release>
           <showWarnings>true</showWarnings>
         </configuration>
       </plugin>
@@ -335,29 +353,6 @@
         </configuration>
       </plugin>
 
-      <!-- Ensure no Java 9+ api usage -->
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>animal-sniffer-maven-plugin</artifactId>
-        <version>${animal-sniffer-maven-plugin.version}</version>
-        <configuration>
-          <checkTestClasses>false</checkTestClasses>
-          <skip>${skipTests}</skip>
-          <signature>
-            <groupId>org.codehaus.mojo.signature</groupId>
-            <artifactId>${main.signature.artifact}</artifactId>
-            <version>1.0</version>
-          </signature>
-        </configuration>
-        <executions>
-          <execution>
-            <goals>
-              <goal>check</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
         <version>${maven-enforcer-plugin.version}</version>
@@ -481,8 +476,6 @@
             <version>${maven-compiler-plugin.version}</version>
             <inherited>true</inherited>
             <configuration>
-              <source>${main.java.version}</source>
-              <target>${main.java.version}</target>
               <fork>true</fork>
               <showWarnings>true</showWarnings>
             </configuration>