From 3a552a76b901a9b97a667ad182d199ae92156aca Mon Sep 17 00:00:00 2001 From: Joseph Grogan Date: Wed, 8 Jan 2025 10:07:18 -0500 Subject: [PATCH] Add ConfigProvider SPI and K8s implementation to load ConfigMap resources (#80) * Load configMap properties at start up * Add config provider to clean up dependencies * Address feedback - change service loader interface, removing system props, and load configs per driver * Move field expansion out of SPI --- Makefile | 2 +- deploy/config/hoptimator-configmap.yaml | 19 ++++++++ .../linkedin/hoptimator/ConfigProvider.java | 8 ++++ hoptimator-cli/build.gradle | 1 - hoptimator-jdbc/build.gradle | 27 +++++++++--- .../hoptimator/jdbc/HoptimatorDriver.java | 2 +- .../jdbc/SystemPropertiesConfigProvider.java | 12 +++++ .../com.linkedin.hoptimator.ConfigProvider | 1 + .../hoptimator/k8s/K8sApiEndpoints.java | 5 +++ .../hoptimator/k8s/K8sConfigProvider.java | 29 ++++++++++++ .../com.linkedin.hoptimator.ConfigProvider | 1 + hoptimator-kafka/build.gradle | 5 ++- .../hoptimator/kafka/KafkaDriver.java | 6 ++- hoptimator-util/build.gradle | 4 +- .../hoptimator/util/ConfigService.java | 44 +++++++++++++++++++ .../hoptimator/util/DataTypeUtils.java | 3 +- hoptimator-venice/build.gradle | 4 +- .../hoptimator/venice/VeniceDriver.java | 8 +++- test-model.yaml | 16 ------- 19 files changed, 163 insertions(+), 34 deletions(-) create mode 100644 deploy/config/hoptimator-configmap.yaml create mode 100644 hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java create mode 100644 hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java create mode 100644 hoptimator-jdbc/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider create mode 100644 hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java create mode 100644 hoptimator-k8s/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider create mode 100644 hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java delete mode 100644 test-model.yaml diff --git a/Makefile b/Makefile index d79b101..0367741 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ clean: ./gradlew clean deploy-config: - kubectl create configmap hoptimator-configmap --from-file=model.yaml=test-model.yaml --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f ./deploy/config/hoptimator-configmap.yaml undeploy-config: kubectl delete configmap hoptimator-configmap || echo "skipping" diff --git a/deploy/config/hoptimator-configmap.yaml b/deploy/config/hoptimator-configmap.yaml new file mode 100644 index 0000000..053f763 --- /dev/null +++ b/deploy/config/hoptimator-configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hoptimator-configmap +data: + # https://kubernetes.io/docs/concepts/configuration/configmap/ + + # property-like keys; each key maps to a simple value + player_initial_lives: "3" + ui_properties_file_name: "user-interface.properties" + + # file-like keys + game.properties: | + enemy.types=aliens,monsters + player.maximum-lives=5 + user-interface.properties: | + color.good=purple + color.bad=yellow + allow.textmode=true \ No newline at end of file diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java new file mode 100644 index 0000000..bc210c0 --- /dev/null +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java @@ -0,0 +1,8 @@ +package com.linkedin.hoptimator; + +import java.util.Properties; + +public interface ConfigProvider { + + Properties loadConfig() throws Exception; +} diff --git a/hoptimator-cli/build.gradle b/hoptimator-cli/build.gradle index ad460fd..0702e8e 100644 --- a/hoptimator-cli/build.gradle +++ b/hoptimator-cli/build.gradle @@ -6,7 +6,6 @@ plugins { } dependencies { - implementation project(':hoptimator-jdbc') implementation project(':hoptimator-api') implementation project(':hoptimator-avro') implementation project(':hoptimator-demodb') diff --git a/hoptimator-jdbc/build.gradle b/hoptimator-jdbc/build.gradle index 474145b..efc66db 100644 --- a/hoptimator-jdbc/build.gradle +++ b/hoptimator-jdbc/build.gradle @@ -14,23 +14,40 @@ dependencies { testFixturesImplementation libs.calcite.core testFixturesImplementation project(':hoptimator-api') testFixturesImplementation project(':hoptimator-util') + testFixturesImplementation(platform('org.junit:junit-bom:5.11.3')) + testFixturesImplementation 'org.junit.jupiter:junit-jupiter' testRuntimeOnly project(':hoptimator-demodb') - testFixturesImplementation(platform('org.junit:junit-bom:5.11.3')) - testImplementation(platform('org.junit:junit-bom:5.11.3')) - testImplementation 'org.junit.jupiter:junit-jupiter' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation(platform('org.junit:junit-bom:5.11.3')) + testImplementation 'org.junit.jupiter:junit-jupiter' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } test { - useJUnitPlatform() + useJUnitPlatform { + excludeTags 'integration' + } testLogging { events "passed", "skipped", "failed" } } +tasks.register('intTest', Test) { + description = 'Runs integration tests.' + group = 'verification' + + shouldRunAfter test + + useJUnitPlatform { + includeTags 'integration' + } + + testLogging { + events "passed", "skipped", "failed" + } +} + java { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java index 16a11d1..cedf9ac 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java @@ -23,7 +23,7 @@ public class HoptimatorDriver extends Driver { public HoptimatorDriver() { - super(() -> new Prepare()); + super(Prepare::new); } static { diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java new file mode 100644 index 0000000..2158ee5 --- /dev/null +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java @@ -0,0 +1,12 @@ +package com.linkedin.hoptimator.jdbc; + +import java.util.Properties; + +import com.linkedin.hoptimator.ConfigProvider; + +public class SystemPropertiesConfigProvider implements ConfigProvider { + + public Properties loadConfig() { + return System.getProperties(); + } +} diff --git a/hoptimator-jdbc/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider b/hoptimator-jdbc/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider new file mode 100644 index 0000000..267d377 --- /dev/null +++ b/hoptimator-jdbc/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider @@ -0,0 +1 @@ +com.linkedin.hoptimator.jdbc.SystemPropertiesConfigProvider diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java index c92655b..f2f3f46 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sApiEndpoints.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.k8s; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ConfigMapList; import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1NamespaceList; import io.kubernetes.client.openapi.models.V1Secret; @@ -24,6 +26,9 @@ public final class K8sApiEndpoints { new K8sApiEndpoint<>("Namespace", "", "v1", "namespaces", true, V1Namespace.class, V1NamespaceList.class); public static final K8sApiEndpoint SECRETS = new K8sApiEndpoint<>("Secret", "", "v1", "secrets", false, V1Secret.class, V1SecretList.class); + public static final K8sApiEndpoint CONFIG_MAPS = + new K8sApiEndpoint<>("ConfigMap", "", "v1", "configmaps", false, + V1ConfigMap.class, V1ConfigMapList.class); // Hoptimator custom resources public static final K8sApiEndpoint DATABASES = diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java new file mode 100644 index 0000000..54f4c18 --- /dev/null +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java @@ -0,0 +1,29 @@ +package com.linkedin.hoptimator.k8s; + +import java.sql.SQLException; +import java.util.Map; +import java.util.Properties; + +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ConfigMapList; + +import com.linkedin.hoptimator.ConfigProvider; + + +public class K8sConfigProvider implements ConfigProvider { + + public static final String HOPTIMATOR_CONFIG_MAP = "hoptimator-configmap"; + + public Properties loadConfig() throws SQLException { + Map topLevelConfigs = loadTopLevelConfig(HOPTIMATOR_CONFIG_MAP); + Properties p = new Properties(); + p.putAll(topLevelConfigs); + return p; + } + + // Load top-level config map properties + private Map loadTopLevelConfig(String configMapName) throws SQLException { + K8sApi configMapApi = new K8sApi<>(K8sContext.currentContext(), K8sApiEndpoints.CONFIG_MAPS); + return configMapApi.get(configMapName).getData(); + } +} diff --git a/hoptimator-k8s/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider b/hoptimator-k8s/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider new file mode 100644 index 0000000..b436d56 --- /dev/null +++ b/hoptimator-k8s/src/main/resources/META-INF/services/com.linkedin.hoptimator.ConfigProvider @@ -0,0 +1 @@ +com.linkedin.hoptimator.k8s.K8sConfigProvider diff --git a/hoptimator-kafka/build.gradle b/hoptimator-kafka/build.gradle index df2442d..685a28b 100644 --- a/hoptimator-kafka/build.gradle +++ b/hoptimator-kafka/build.gradle @@ -3,6 +3,7 @@ plugins { } dependencies { + implementation project(':hoptimator-util') implementation libs.calcite.core implementation libs.kafka.clients @@ -15,8 +16,8 @@ dependencies { testRuntimeOnly project(':hoptimator-k8s') testRuntimeOnly project(':hoptimator-kafka') testImplementation(testFixtures(project(':hoptimator-jdbc'))) - testImplementation(platform('org.junit:junit-bom:5.11.3')) - testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation(platform('org.junit:junit-bom:5.11.3')) + testImplementation 'org.junit.jupiter:junit-jupiter' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } diff --git a/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDriver.java b/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDriver.java index 34da42e..1ce292c 100644 --- a/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDriver.java +++ b/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDriver.java @@ -11,6 +11,8 @@ import org.apache.calcite.jdbc.Driver; import org.apache.calcite.schema.SchemaPlus; +import com.linkedin.hoptimator.util.ConfigService; + /** JDBC driver for Kafka topics. */ public class KafkaDriver extends Driver { @@ -34,7 +36,9 @@ public Connection connect(String url, Properties props) throws SQLException { if (!url.startsWith(getConnectStringPrefix())) { return null; } - Properties properties = ConnectStringParser.parse(url.substring(getConnectStringPrefix().length())); + // Connection string properties are given precedence over config properties + Properties properties = ConfigService.config(); + properties.putAll(ConnectStringParser.parse(url.substring(getConnectStringPrefix().length()))); try { Connection connection = super.connect(url, props); if (connection == null) { diff --git a/hoptimator-util/build.gradle b/hoptimator-util/build.gradle index c55c112..5aa75a0 100644 --- a/hoptimator-util/build.gradle +++ b/hoptimator-util/build.gradle @@ -8,8 +8,8 @@ dependencies { implementation libs.calcite.core testImplementation(testFixtures(project(':hoptimator-jdbc'))) - testImplementation(platform('org.junit:junit-bom:5.11.3')) - testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation(platform('org.junit:junit-bom:5.11.3')) + testImplementation 'org.junit.jupiter:junit-jupiter' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java new file mode 100644 index 0000000..4b4c4d1 --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java @@ -0,0 +1,44 @@ +package com.linkedin.hoptimator.util; + +import java.io.StringReader; +import java.util.Properties; +import java.util.ServiceLoader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linkedin.hoptimator.ConfigProvider; + +public final class ConfigService { + + private static final Logger log = LoggerFactory.getLogger(ConfigService.class); + + private ConfigService() { + } + + // Loads top level configs and expands input fields as file-like properties + // Ex: + // log.properties: | + // level=INFO + public static Properties config(String... expansionFields) { + ServiceLoader loader = ServiceLoader.load(ConfigProvider.class); + Properties properties = new Properties(); + for (ConfigProvider provider : loader) { + try { + Properties loadedProperties = provider.loadConfig(); + log.debug("Loaded properties={} from provider={}", loadedProperties, provider); + properties.putAll(loadedProperties); + for (String expansionField : expansionFields) { + if (loadedProperties == null || !loadedProperties.containsKey(expansionField)) { + log.warn("provider={} does not contain field={}", provider, expansionField); + continue; + } + properties.load(new StringReader(loadedProperties.getProperty(expansionField))); + } + } catch (Exception e) { + log.warn("Could not load properties for provider={}", provider, e); + } + } + return properties; + } +} diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DataTypeUtils.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DataTypeUtils.java index 2add458..1ab6c8c 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DataTypeUtils.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DataTypeUtils.java @@ -3,6 +3,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -80,7 +81,7 @@ private static RelDataType buildRecord(Node node, RelDataTypeFactory typeFactory return node.dataType; } RelDataTypeFactory.Builder builder = new RelDataTypeFactory.Builder(typeFactory); - for (LinkedHashMap.Entry child : node.children.entrySet()) { + for (Map.Entry child : node.children.entrySet()) { builder.add(child.getKey(), buildRecord(child.getValue(), typeFactory)); } return builder.build(); diff --git a/hoptimator-venice/build.gradle b/hoptimator-venice/build.gradle index 6e4cb11..0d32a03 100644 --- a/hoptimator-venice/build.gradle +++ b/hoptimator-venice/build.gradle @@ -21,8 +21,8 @@ dependencies { testRuntimeOnly project(':hoptimator-k8s') testImplementation(testFixtures(project(':hoptimator-jdbc'))) - testImplementation(platform('org.junit:junit-bom:5.11.3')) - testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation(platform('org.junit:junit-bom:5.11.3')) + testImplementation 'org.junit.jupiter:junit-jupiter' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } diff --git a/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDriver.java b/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDriver.java index 3a24429..fbf9bcc 100644 --- a/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDriver.java +++ b/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDriver.java @@ -12,11 +12,13 @@ import org.apache.calcite.jdbc.Driver; import org.apache.calcite.schema.SchemaPlus; +import com.linkedin.hoptimator.util.ConfigService; + /** JDBC driver for Venice stores. */ public class VeniceDriver extends Driver { - public static final String CATALOG_NAME = "VENICE"; + public static final String CONFIG_NAME = "venice.config"; static { new VeniceDriver().register(); @@ -37,7 +39,9 @@ public Connection connect(String url, Properties props) throws SQLException { if (!url.startsWith(getConnectStringPrefix())) { return null; } - Properties properties = ConnectStringParser.parse(url.substring(getConnectStringPrefix().length())); + // Connection string properties are given precedence over config properties + Properties properties = ConfigService.config(CONFIG_NAME); + properties.putAll(ConnectStringParser.parse(url.substring(getConnectStringPrefix().length()))); String cluster = properties.getProperty("cluster"); if (cluster == null) { throw new IllegalArgumentException("Missing required cluster property. Need: jdbc:venice://cluster=..."); diff --git a/test-model.yaml b/test-model.yaml deleted file mode 100644 index 6c20686..0000000 --- a/test-model.yaml +++ /dev/null @@ -1,16 +0,0 @@ -version: 1.0 -defaultSchema: DATAGEN -schemas: - -- name: DATAGEN - type: custom - factory: com.linkedin.hoptimator.catalog.builtin.DatagenSchemaFactory - -- name: RAWKAFKA - type: custom - factory: com.linkedin.hoptimator.catalog.kafka.RawKafkaSchemaFactory - operand: - clientConfig: - bootstrap.servers: one-kafka-bootstrap.kafka.svc:9092 - group.id: hoptimator-test - auto.offset.reset: earliest