diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml index dbff6665fabf..c6353fa6ffb1 100644 --- a/plugin/trino-clickhouse/pom.xml +++ b/plugin/trino-clickhouse/pom.xml @@ -46,14 +46,7 @@ com.clickhouse clickhouse-jdbc - 0.3.2-patch3 all - - - * - * - - diff --git a/pom.xml b/pom.xml index 75898736d45d..ba6d70b4de7b 100644 --- a/pom.xml +++ b/pom.xml @@ -988,6 +988,19 @@ 2.9.5 + + com.clickhouse + clickhouse-jdbc + 0.3.2-patch3 + all + + + * + * + + + + com.esri.geometry esri-geometry-api diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java index 1f9dce8c7874..ad0175f1869d 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Standard.java @@ -186,6 +186,7 @@ private static void enablePrestoJavaDebugger(DockerContainer container, String c try { FileAttribute> rwx = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwxrwx")); Path script = Files.createTempFile("enable-java-debugger", ".sh", rwx); + script.toFile().deleteOnExit(); Files.writeString( script, format( diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java new file mode 100644 index 000000000000..695d9b77f22a --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java @@ -0,0 +1,172 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.DockerContainer; +import io.trino.tests.product.launcher.env.Environment.Builder; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.StandardMultinode; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; +import io.trino.tests.product.launcher.testcontainers.PortBinder; +import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; + +import javax.inject.Inject; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Duration; +import java.util.Set; + +import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.WORKER; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; +import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC; +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public class EnvMultinodeClickhouse + extends EnvironmentProvider +{ + private static final String ZOOKEEPER = "zookeeper"; + + private static final String CLICKHOUSE = "clickhouse"; + private static final String CLICKHOUSE_NTH = CLICKHOUSE + "-"; + private static final String CONTAINER_CLICKHOUSE_CONFIG_DIR = "/etc/clickhouse-server/"; + private static final String CONTAINER_CLICKHOUSE_USERS_D = CONTAINER_CLICKHOUSE_CONFIG_DIR + "users.d/"; + private static final String CONTAINER_CLICKHOUSE_CONFIG_D = CONTAINER_CLICKHOUSE_CONFIG_DIR + "config.d/"; + private static final int CLICKHOUSE_DEFAULT_HTTP_PORT = 8123; + private static final int CLICKHOUSE_DEFAULT_NATIVE_PORT = 9000; + + private final DockerFiles dockerFiles; + private final ResourceProvider configDir; + private final PortBinder portBinder; + + @Inject + public EnvMultinodeClickhouse(StandardMultinode standardMultinode, DockerFiles dockerFiles, PortBinder portBinder) + { + super(standardMultinode); + this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); + this.configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/multinode-clickhouse/"); + this.portBinder = requireNonNull(portBinder, "portBinder is null"); + } + + @Override + public void extendEnvironment(Builder builder) + { + builder.configureContainer(COORDINATOR, this::addCatalogs); + builder.configureContainer(WORKER, this::addCatalogs); + + builder.addContainers( + createZookeeper(portBinder), + createClickHouse(1, dockerFiles, portBinder), + createClickHouse(2, dockerFiles, portBinder), + createClickHouse(3, dockerFiles, portBinder)) + .containerDependsOn(logicalName(1), ZOOKEEPER) + .containerDependsOn(logicalName(2), ZOOKEEPER) + .containerDependsOn(logicalName(3), ZOOKEEPER); + + builder.configureContainer(ZOOKEEPER, container -> container.withNetworkAliases(container.getLogicalName(), "localhost")); + builder.configureContainer(logicalName(1), container -> container.withNetworkAliases(CLICKHOUSE, container.getLogicalName(), "localhost")); + builder.configureContainer(logicalName(2), container -> container.withNetworkAliases(container.getLogicalName(), "localhost")); + builder.configureContainer(logicalName(3), container -> container.withNetworkAliases(container.getLogicalName(), "localhost")); + + configureTempto(builder, configDir); + } + + private void addCatalogs(DockerContainer container) + { + container + .withCopyFileToContainer( + forHostPath(configDir.getPath("clickhouse.properties")), + CONTAINER_PRESTO_ETC + "/catalog/clickhouse.properties"); + } + + private static DockerContainer createZookeeper(PortBinder portBinder) + { + DockerContainer container = new DockerContainer("zookeeper:3.7.0", ZOOKEEPER) + .withEnv("ZOOKEEPER_CLIENT_PORT", "2181") + .withEnv("ZOOKEEPER_TICK_TIME", "2000") + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(forSelectedPorts(2181)) + .withStartupTimeout(Duration.ofMinutes(5)); + + portBinder.exposePort(container, 2181); + + return container; + } + + private static DockerContainer createClickHouse(int number, DockerFiles dockerFiles, PortBinder portBinder) + { + int httpPort = CLICKHOUSE_DEFAULT_HTTP_PORT + number; + int nativePort = CLICKHOUSE_DEFAULT_NATIVE_PORT + number; + + DockerContainer container = new DockerContainer("yandex/clickhouse-server:21.3.2.5", logicalName(number)) + .withCopyFileToContainer( + forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-clickhouse/test.xml")), + CONTAINER_CLICKHOUSE_USERS_D + "test.xml") + .withCopyFileToContainer( + forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-clickhouse/metrika.xml")), + CONTAINER_CLICKHOUSE_CONFIG_D + "metrika.xml") + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(forSelectedPorts(httpPort, nativePort)) + .withStartupTimeout(Duration.ofMinutes(5)); + + modifyDefaultPorts(container, httpPort, nativePort); + + portBinder.exposePort(container, httpPort); + portBinder.exposePort(container, nativePort); + + return container; + } + + private static String logicalName(int number) + { + return CLICKHOUSE_NTH + number; + } + + private static void modifyDefaultPorts(DockerContainer container, int httpPort, int nativePort) + { + try { + FileAttribute> rwx = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwxrwx")); + Path customConfig = Files.createTempFile("custom", ".xml", rwx); + customConfig.toFile().deleteOnExit(); + Files.writeString( + customConfig, + format( + "\n" + + "\n" + + " %s\n" + + " %s\n" + + "\n", + httpPort, + nativePort), + UTF_8); + container.withCopyFileToContainer(forHostPath(customConfig), CONTAINER_CLICKHOUSE_CONFIG_D + "custom.xml"); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java index e0a7faf3792f..e40a90de9438 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite7NonGeneric.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableList; import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentDefaults; +import io.trino.tests.product.launcher.env.environment.EnvMultinodeClickhouse; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHiveIcebergRedirections; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeKerberosHdfsImpersonationCrossRealm; import io.trino.tests.product.launcher.env.environment.EnvSinglenodeMysql; @@ -45,6 +46,7 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeMysql.class).withGroups("mysql").build(), testOnEnvironment(EnvSinglenodePostgresql.class).withGroups("postgresql").build(), testOnEnvironment(EnvSinglenodeSqlserver.class).withGroups("sqlserver").build(), + testOnEnvironment(EnvMultinodeClickhouse.class).withGroups("clickhouse").build(), testOnEnvironment(EnvSinglenodeSparkHive.class).withGroups("hive_spark").build(), testOnEnvironment(EnvSinglenodeSparkIceberg.class).withGroups("iceberg").withExcludedGroups("storage_formats").build(), testOnEnvironment(EnvSinglenodeHiveIcebergRedirections.class).withGroups("hive_iceberg_redirections").build(), diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/clickhouse.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/clickhouse.properties new file mode 100644 index 000000000000..ce3f28cf6f48 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/clickhouse.properties @@ -0,0 +1,4 @@ +connector.name=clickhouse +connection-url=jdbc:clickhouse://clickhouse:8124/ +connection-user=test +connection-password=test diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/metrika.xml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/metrika.xml new file mode 100644 index 000000000000..79f23db9bd3e --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/metrika.xml @@ -0,0 +1,37 @@ + + + + + + + clickhouse-1 + 9001 + + + + + clickhouse-2 + 9002 + + + + + clickhouse-3 + 9003 + + + + + + + + zookeeper + 2181 + + + + + 01 + 01 + + diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/tempto-configuration.yaml new file mode 100644 index 000000000000..eecb457a79ec --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/tempto-configuration.yaml @@ -0,0 +1,3 @@ +databases: + presto: + jdbc_url: "jdbc:trino://${databases.presto.host}:${databases.presto.port}/clickhouse/default" diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/test.xml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/test.xml new file mode 100644 index 000000000000..2ca298d0ff5b --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-clickhouse/test.xml @@ -0,0 +1,13 @@ + + + + + test + + ::/0 + + default + default + + + diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index f0d7f6d7a483..4e18a8bbf04e 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -167,6 +167,13 @@ runtime + + com.clickhouse + clickhouse-jdbc + all + runtime + + com.microsoft.sqlserver mssql-jdbc diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java index 2bc06ecf4853..f5e605af1507 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java @@ -67,6 +67,7 @@ public final class TestGroups public static final String ICEBERG = "iceberg"; public static final String AVRO = "avro"; public static final String PHOENIX = "phoenix"; + public static final String CLICKHOUSE = "clickhouse"; private TestGroups() {} } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/clickhouse/TestClickHouse.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/clickhouse/TestClickHouse.java new file mode 100644 index 000000000000..5a26f4b803ae --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/clickhouse/TestClickHouse.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.clickhouse; + +import io.trino.tempto.ProductTest; +import io.trino.tempto.query.QueryResult; +import org.testng.annotations.Test; + +import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.tempto.assertions.QueryAssert.assertThat; +import static io.trino.tests.product.TestGroups.CLICKHOUSE; +import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; + +public class TestClickHouse + extends ProductTest +{ + @Test(groups = {CLICKHOUSE, PROFILE_SPECIFIC_TESTS}) + public void testCreateTableAsSelect() + { + QueryResult result = onTrino().executeQuery("CREATE TABLE nation AS SELECT * FROM tpch.tiny.nation"); + try { + assertThat(result).updatedRowsCountIsEqualTo(25); + assertThat(onTrino().executeQuery("SELECT COUNT(*) FROM nation")) + .containsOnly(row(25)); + } + finally { + onTrino().executeQuery("DROP TABLE nation"); + } + } +}