feat: docker/swarm deployment restructuring and improvements

.dockerignore

@@ -3,4 +3,7 @@
 .vscode
 README.md
 deploy
-sample-apps
+
+# frontend
+.env
+node_modules

.github/workflows/build.yaml → .github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: build-pipeline
+name: ci
 
 on:
   pull_request:
@@ -14,7 +14,7 @@ jobs:
       - name: Run check
         run: make check-no-ee-references
 
-  build-frontend:
+  test-frontend:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -28,35 +28,8 @@ jobs:
       - name: TSC
         run: yarn tsc
         working-directory: ./frontend
-      - name: Build frontend docker image
-        shell: bash
-        run: |
-          make build-frontend-amd64
-
-  build-frontend-ee:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Create .env file
-        run: |
-          echo 'INTERCOM_APP_ID="${{ secrets.INTERCOM_APP_ID }}"' > frontend/.env
-          echo 'SEGMENT_ID="${{ secrets.SEGMENT_ID }}"' >> frontend/.env
-      - name: Install dependencies
-        run: cd frontend && yarn install
-      - name: Run ESLint
-        run: cd frontend && npm run lint
-      - name: Run Jest
-        run: cd frontend && npm run jest
-      - name: TSC
-        run: yarn tsc
-        working-directory: ./frontend
-      - name: Build frontend docker image
-        shell: bash
-        run: |
-          make build-frontend-amd64
 
-  build-query-service:
+  test-query-service:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -74,7 +47,7 @@ jobs:
         run: |
           make build-query-service-amd64
 
-  build-ee-query-service:
+  test-ee-query-service:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -83,6 +56,10 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: "1.21"
+      - name: Run tests
+        shell: bash
+        run: |
+          make test-ee
       - name: Build EE query-service image
         shell: bash
         run: |

.github/workflows/push.yaml

@@ -106,102 +106,3 @@ jobs:
           sudo apt-get install -y gcc-aarch64-linux-gnu musl-tools
       - name: Build and push docker image
         run: make build-push-ee-query-service
-
-  image-build-and-push-frontend:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        working-directory: frontend
-        run: yarn install
-      - name: Run Prettier
-        working-directory: frontend
-        run: npm run prettify
-        continue-on-error: true
-      - name: Run ESLint
-        working-directory: frontend
-        run: npm run lint
-        continue-on-error: true
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: benjlevesque/[email protected]
-        id: short-sha
-      - name: Get branch name
-        id: branch-name
-        uses: tj-actions/[email protected]
-      - name: Set docker tag environment
-        run: |
-          if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then
-            tag="${{ steps.branch-name.outputs.tag }}"
-            tag="${tag:1}"
-            echo "DOCKER_TAG=$tag" >> $GITHUB_ENV
-          elif [ '${{ steps.branch-name.outputs.current_branch }}' == 'main' ]; then
-            echo "DOCKER_TAG=latest" >> $GITHUB_ENV
-          else
-            echo "DOCKER_TAG=${{ steps.branch-name.outputs.current_branch }}" >> $GITHUB_ENV
-          fi
-      - name: Build and push docker image
-        run: make build-push-frontend
-
-  image-build-and-push-frontend-ee:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Create .env file
-        run: |
-          echo 'INTERCOM_APP_ID="${{ secrets.INTERCOM_APP_ID }}"' > frontend/.env
-          echo 'SEGMENT_ID="${{ secrets.SEGMENT_ID }}"' >> frontend/.env
-          echo 'SENTRY_AUTH_TOKEN="${{ secrets.SENTRY_AUTH_TOKEN }}"' >> frontend/.env
-          echo 'SENTRY_ORG="${{ secrets.SENTRY_ORG }}"' >> frontend/.env
-          echo 'SENTRY_PROJECT_ID="${{ secrets.SENTRY_PROJECT_ID }}"' >> frontend/.env
-          echo 'SENTRY_DSN="${{ secrets.SENTRY_DSN }}"' >> frontend/.env
-          echo 'TUNNEL_URL="${{ secrets.TUNNEL_URL }}"' >> frontend/.env
-          echo 'TUNNEL_DOMAIN="${{ secrets.TUNNEL_DOMAIN }}"' >> frontend/.env
-          echo 'POSTHOG_KEY="${{ secrets.POSTHOG_KEY }}"' >> frontend/.env
-      - name: Install dependencies
-        working-directory: frontend
-        run: yarn install
-      - name: Run Prettier
-        working-directory: frontend
-        run: npm run prettify
-        continue-on-error: true
-      - name: Run ESLint
-        working-directory: frontend
-        run: npm run lint
-        continue-on-error: true
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: benjlevesque/[email protected]
-        id: short-sha
-      - name: Get branch name
-        id: branch-name
-        uses: tj-actions/[email protected]
-      - name: Set docker tag environment
-        run: |
-          if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then
-            tag="${{ steps.branch-name.outputs.tag }}"
-            tag="${tag:1}"
-            echo "DOCKER_TAG=${tag}-ee" >> $GITHUB_ENV
-          elif [ '${{ steps.branch-name.outputs.current_branch }}' == 'main' ]; then
-            echo "DOCKER_TAG=latest-ee" >> $GITHUB_ENV
-          else
-            echo "DOCKER_TAG=${{ steps.branch-name.outputs.current_branch }}-ee" >> $GITHUB_ENV
-          fi
-      - name: Build and push docker image
-        run: make build-push-frontend

Makefile

@@ -191,3 +191,6 @@ check-no-ee-references:
 
 test:
 	go test ./pkg/...
+
+test-ee:
+	go test ./ee/...

deploy/docker/clickhouse-setup/clickhouse-cluster.xml → deploy/common/clickhouse/cluster.ha.xml

@@ -10,14 +10,14 @@
             <host>zookeeper-1</host>
             <port>2181</port>
         </node>
-        <!-- <node index="2">
+        <node index="2">
             <host>zookeeper-2</host>
             <port>2181</port>
         </node>
         <node index="3">
             <host>zookeeper-3</host>
             <port>2181</port>
-        </node> -->
+        </node>
     </zookeeper>
 
     <!-- Configuration of clusters that could be used in Distributed tables.
@@ -58,7 +58,7 @@
                     <!-- <priority>1</priority> -->
                 </replica>
             </shard>
-            <!-- <shard>
+            <shard>
                 <replica>
                     <host>clickhouse-2</host>
                     <port>9000</port>
@@ -69,7 +69,7 @@
                     <host>clickhouse-3</host>
                     <port>9000</port>
                 </replica>
-            </shard> -->
+            </shard>
         </cluster>
     </remote_servers>
 </clickhouse>

deploy/docker/clickhouse-setup/clickhouse-config.xml → deploy/common/clickhouse/config.xml

@@ -716,7 +716,7 @@
         asynchronous_metrics - send data from table system.asynchronous_metrics
         status_info - send data from different component from CH, ex: Dictionaries status
     -->
-    <!--
+
     <prometheus>
         <endpoint>/metrics</endpoint>
         <port>9363</port>
@@ -726,7 +726,6 @@
         <asynchronous_metrics>true</asynchronous_metrics>
         <status_info>true</status_info>
     </prometheus>
-    -->
 
     <!-- Query log. Used only for queries with setting log_queries = 1. -->
     <query_log>

deploy/docker-swarm/clickhouse-setup/prometheus.yml → deploy/common/signoz/prometheus.yml

@@ -12,10 +12,10 @@ alerting:
       - alertmanager:9093
 
 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
-rule_files:
+rule_files: []
   # - "first_rules.yml"
   # - "second_rules.yml"
-  - 'alerts.yml'
+  # - 'alerts.yml'
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.

deploy/docker-swarm/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=clickhouse-setup