WIP: Select view

doc/cluster.md

@@ -75,7 +75,7 @@ ORDER BY (metric, timestamp)
 SETTINGS index_granularity = 8192;
 
 
-CREATE TABLE graphite.data
+CREATE TABLE graphite.data_write
 (
     metric String,
     value Float64,
@@ -84,8 +84,37 @@ CREATE TABLE graphite.data
     updated UInt32
 )
 ENGINE Distributed(CLICKHOUSE_CLUSTER_NAME, 'graphite', 'data_lr', sipHash64(metric));
+
+CREATE VIEW graphite.data_view
+(
+    `metric` String,
+    `value` Float64,
+    `timestamp` UInt32,
+    `date` Date
+) AS
+SELECT
+    metric,
+    timestamp,
+    argMax(value, updated) AS value,
+    date
+FROM graphite.data_lr
+GROUP BY
+    metric,
+    timestamp,
+    date;
+
+CREATE TABLE graphite.data_view_distributed
+(
+    `metric` String,
+    `value` Float64,
+    `timestamp` UInt32,
+    `date` Date
+)
+ENGINE = Distributed(CLICKHOUSE_CLUSTER_NAME, 'graphite', 'data_view', sipHash64(metric))
 ```
 
 Don't forget to replace ```CLICKHOUSE_CLUSTER_NAME``` with name from you config.
 
+You have to set the property graphouse.clickhouse.use-sharding=true and use different tables for reads and writes.
+
 **Notice**: We use sharding only for ```data```, couse ```metrics``` is small and contains only metric names.

src/main/java/ru/yandex/market/graphouse/config/MetricsConfig.java

@@ -68,9 +68,10 @@ public RetentionProvider retentionProvider() {
 
     @Bean
     public MetricDataService dataService(@Value("${graphouse.clickhouse.data-read-table}") String graphiteDataReadTable,
-                                         @Value("${graphouse.metric-data.max-points-per-metric}") int maxPointsPerMetric) {
+                                         @Value("${graphouse.metric-data.max-points-per-metric}") int maxPointsPerMetric,
+                                         @Value("${graphouse.clickhouse.use-sharding}") boolean useSharding) {
         return new MetricDataService(
-            metricSearch(), clickHouseJdbcTemplate, graphiteDataReadTable, maxPointsPerMetric
+            metricSearch(), clickHouseJdbcTemplate, graphiteDataReadTable, maxPointsPerMetric, useSharding
         );
     }
 

src/main/java/ru/yandex/market/graphouse/data/MetricDataService.java

@@ -28,13 +28,16 @@ public class MetricDataService {
     private final JdbcTemplate clickHouseJdbcTemplate;
     private final String graphiteDataReadTable;
     private final int maxPointsPerMetric;
+    private final boolean useSharding;
 
     public MetricDataService(MetricSearch metricSearch, JdbcTemplate clickHouseJdbcTemplate,
-                             String graphiteDataReadTable, int maxPointsPerMetric) {
+                             String graphiteDataReadTable, int maxPointsPerMetric,
+                             boolean useSharding) {
         this.metricSearch = metricSearch;
         this.clickHouseJdbcTemplate = clickHouseJdbcTemplate;
         this.graphiteDataReadTable = graphiteDataReadTable;
         this.maxPointsPerMetric = maxPointsPerMetric;
+        this.useSharding = useSharding;
     }
 
 
@@ -68,13 +71,26 @@ private void appendData(List<MetricName> metrics, String function,
 
         MetricDataRowCallbackHandler handler = new MetricDataRowCallbackHandler(jsonWriter, queryParams, metricsSet);
 
-        clickHouseJdbcTemplate.query(
-            "SELECT metric, ts, " + function + "(value) as value FROM (" +
-                "   SELECT metric, ts, argMax(value, updated) as value FROM " + graphiteDataReadTable +
+        String query;
+        if (this.useSharding) {
+            query = "SELECT metric, ts, " + function + "(value) as value FROM (" +
+                "   SELECT metric, timestamp as ts, value FROM " + graphiteDataReadTable +
+                "       WHERE metric IN (" + metricString + ")" +
+                "           AND ts >= ? AND ts < ? AND date >= toDate(?) AND date <= toDate(?)" +
+                ") GROUP BY metric, intDiv(toUInt32(ts), ?) * ? as ts ORDER BY metric, ts";
+        } else {
+            query = "SELECT metric, ts, " + function + "(value) as value FROM (" +
+                "   SELECT metric, timestamp as ts, argMax(value, updated) as value FROM " + graphiteDataReadTable +
                 "       WHERE metric IN (" + metricString + ")" +
                 "           AND ts >= ? AND ts < ? AND date >= toDate(?) AND date <= toDate(?)" +
-                "       GROUP BY metric, timestamp as ts" +
-                ") GROUP BY metric, intDiv(toUInt32(ts), ?) * ? as ts ORDER BY metric, ts",
+                "       GROUP BY metric, ts" +
+                ") GROUP BY metric, intDiv(toUInt32(ts), ?) * ? as ts ORDER BY metric, ts";
+        }
+
+
+
+        clickHouseJdbcTemplate.query(
+            query,
             handler,
             queryParams.getStartTimeSeconds(), queryParams.getEndTimeSeconds(),
             queryParams.getStartTimeSeconds(), queryParams.getEndTimeSeconds(),

src/main/resources/graphouse-default.properties

@@ -10,6 +10,7 @@ graphouse.clickhouse.user=
 graphouse.clickhouse.password=
 graphouse.clickhouse.compress=false
 graphouse.clickhouse.host-ping-rate-seconds=10
+graphouse.clickhouse.use-sharding=false
 
 graphouse.clickhouse.data-table=data
 graphouse.clickhouse.data-write-table=${graphouse.clickhouse.data-table}