From 00b3b5b9321df966f2ae8c2e812e95103ea35e8c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 13 Aug 2019 01:04:41 +0200 Subject: [PATCH 1/2] Add a way to use shards in ClickHouse --- .../graphouse/config/MetricsConfig.java | 5 ++-- .../graphouse/data/MetricDataService.java | 28 +++++++++++++++---- .../resources/graphouse-default.properties | 1 + 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/main/java/ru/yandex/market/graphouse/config/MetricsConfig.java b/src/main/java/ru/yandex/market/graphouse/config/MetricsConfig.java index f47dcd76..f477e26b 100644 --- a/src/main/java/ru/yandex/market/graphouse/config/MetricsConfig.java +++ b/src/main/java/ru/yandex/market/graphouse/config/MetricsConfig.java @@ -68,9 +68,10 @@ public RetentionProvider retentionProvider() { @Bean public MetricDataService dataService(@Value("${graphouse.clickhouse.data-read-table}") String graphiteDataReadTable, - @Value("${graphouse.metric-data.max-points-per-metric}") int maxPointsPerMetric) { + @Value("${graphouse.metric-data.max-points-per-metric}") int maxPointsPerMetric, + @Value("${graphouse.clickhouse.use-sharding}") boolean useSharding) { return new MetricDataService( - metricSearch(), clickHouseJdbcTemplate, graphiteDataReadTable, maxPointsPerMetric + metricSearch(), clickHouseJdbcTemplate, graphiteDataReadTable, maxPointsPerMetric, useSharding ); } diff --git a/src/main/java/ru/yandex/market/graphouse/data/MetricDataService.java b/src/main/java/ru/yandex/market/graphouse/data/MetricDataService.java index 53cea708..ab5ce172 100644 --- a/src/main/java/ru/yandex/market/graphouse/data/MetricDataService.java +++ b/src/main/java/ru/yandex/market/graphouse/data/MetricDataService.java @@ -28,13 +28,16 @@ public class MetricDataService { private final JdbcTemplate clickHouseJdbcTemplate; private final String graphiteDataReadTable; private final int maxPointsPerMetric; + private final boolean useSharding; public MetricDataService(MetricSearch metricSearch, JdbcTemplate clickHouseJdbcTemplate, - String graphiteDataReadTable, int maxPointsPerMetric) { + String graphiteDataReadTable, int maxPointsPerMetric, + boolean useSharding) { this.metricSearch = metricSearch; this.clickHouseJdbcTemplate = clickHouseJdbcTemplate; this.graphiteDataReadTable = graphiteDataReadTable; this.maxPointsPerMetric = maxPointsPerMetric; + this.useSharding = useSharding; } @@ -68,13 +71,26 @@ private void appendData(List metrics, String function, MetricDataRowCallbackHandler handler = new MetricDataRowCallbackHandler(jsonWriter, queryParams, metricsSet); - clickHouseJdbcTemplate.query( - "SELECT metric, ts, " + function + "(value) as value FROM (" + - " SELECT metric, ts, argMax(value, updated) as value FROM " + graphiteDataReadTable + + String query; + if (this.useSharding) { + query = "SELECT metric, ts, " + function + "(value) as value FROM (" + + " SELECT metric, timestamp as ts, value FROM " + graphiteDataReadTable + + " WHERE metric IN (" + metricString + ")" + + " AND ts >= ? AND ts < ? AND date >= toDate(?) AND date <= toDate(?)" + + ") GROUP BY metric, intDiv(toUInt32(ts), ?) * ? as ts ORDER BY metric, ts"; + } else { + query = "SELECT metric, ts, " + function + "(value) as value FROM (" + + " SELECT metric, timestamp as ts, argMax(value, updated) as value FROM " + graphiteDataReadTable + " WHERE metric IN (" + metricString + ")" + " AND ts >= ? AND ts < ? AND date >= toDate(?) AND date <= toDate(?)" + - " GROUP BY metric, timestamp as ts" + - ") GROUP BY metric, intDiv(toUInt32(ts), ?) * ? as ts ORDER BY metric, ts", + " GROUP BY metric, ts" + + ") GROUP BY metric, intDiv(toUInt32(ts), ?) * ? as ts ORDER BY metric, ts"; + } + + + + clickHouseJdbcTemplate.query( + query, handler, queryParams.getStartTimeSeconds(), queryParams.getEndTimeSeconds(), queryParams.getStartTimeSeconds(), queryParams.getEndTimeSeconds(), diff --git a/src/main/resources/graphouse-default.properties b/src/main/resources/graphouse-default.properties index ea806235..c75b36bb 100644 --- a/src/main/resources/graphouse-default.properties +++ b/src/main/resources/graphouse-default.properties @@ -10,6 +10,7 @@ graphouse.clickhouse.user= graphouse.clickhouse.password= graphouse.clickhouse.compress=false graphouse.clickhouse.host-ping-rate-seconds=10 +graphouse.clickhouse.use-sharding=false graphouse.clickhouse.data-table=data graphouse.clickhouse.data-write-table=${graphouse.clickhouse.data-table} From 17526cb1a599b90dc1b1177687b68bd11f0a2ebf Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 13 Aug 2019 01:08:25 +0200 Subject: [PATCH 2/2] Update documentation --- doc/cluster.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/doc/cluster.md b/doc/cluster.md index c2d805fd..c1e7bd8f 100644 --- a/doc/cluster.md +++ b/doc/cluster.md @@ -75,7 +75,7 @@ ORDER BY (metric, timestamp) SETTINGS index_granularity = 8192; -CREATE TABLE graphite.data +CREATE TABLE graphite.data_write ( metric String, value Float64, @@ -84,8 +84,37 @@ CREATE TABLE graphite.data updated UInt32 ) ENGINE Distributed(CLICKHOUSE_CLUSTER_NAME, 'graphite', 'data_lr', sipHash64(metric)); + +CREATE VIEW graphite.data_view +( + `metric` String, + `value` Float64, + `timestamp` UInt32, + `date` Date +) AS +SELECT + metric, + timestamp, + argMax(value, updated) AS value, + date +FROM graphite.data_lr +GROUP BY + metric, + timestamp, + date; + +CREATE TABLE graphite.data_view_distributed +( + `metric` String, + `value` Float64, + `timestamp` UInt32, + `date` Date +) +ENGINE = Distributed(CLICKHOUSE_CLUSTER_NAME, 'graphite', 'data_view', sipHash64(metric)) ``` Don't forget to replace ```CLICKHOUSE_CLUSTER_NAME``` with name from you config. +You have to set the property graphouse.clickhouse.use-sharding=true and use different tables for reads and writes. + **Notice**: We use sharding only for ```data```, couse ```metrics``` is small and contains only metric names.