feat(interactive): Decouple ingestor and store, remote direct RPC con…

…nections. (#3525)

charts/gie-standalone/templates/_helpers.tpl

@@ -31,10 +31,6 @@ If release name contains chart name it will be used as a full name.
 {{- printf "%s-%s" (include "graphscope-store.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
-{{- define "graphscope-store.ingestor.fullname" -}}
-{{- printf "%s-%s" (include "graphscope-store.fullname" .) "ingestor" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
 {{- define "graphscope-store.store.fullname" -}}
 {{- printf "%s-%s" (include "graphscope-store.fullname" .) "store" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

charts/graphscope-store-one-pod/templates/_helpers.tpl

@@ -31,10 +31,6 @@ If release name contains chart name it will be used as a full name.
 {{- printf "%s-%s" (include "graphscope-store.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
-{{- define "graphscope-store.ingestor.fullname" -}}
-{{- printf "%s-%s" (include "graphscope-store.fullname" .) "ingestor" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
 {{- define "graphscope-store.store.fullname" -}}
 {{- printf "%s-%s" (include "graphscope-store.fullname" .) "store" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

charts/graphscope-store-one-pod/templates/configmap.yaml

@@ -21,21 +21,14 @@ data:
     rpc.max.bytes.mb={{ .Values.rpcMaxBytesMb }}
     store.node.count={{ .Values.store.replicaCount }}
     frontend.node.count=1
-    ingestor.node.count={{ .Values.ingestor.replicaCount }}
     coordinator.node.count=1
-    ingestor.queue.count={{ .Values.ingestor.replicaCount }}
     partition.count={{ .Values.store.replicaCount | mul 16 }}
     discovery.mode=zookeeper
 
     ## Frontend Config
     frontend.server.num=1
     enable.hash.generate.eid={{ .Values.enableHashGenerateEid }}
 
-    ## Ingestor Config
-    ingestor.queue.buffer.max.count={{ .Values.ingestorQueueBufferMaxCount }}
-    ingestor.sender.buffer.max.count={{ .Values.ingestorSenderBufferMaxCount }}
-    ingestor.sender.operation.max.count={{ .Values.ingestorSenderOperationMaxCount }}
-
     ## Coordinator Config
     snapshot.increase.interval.ms={{ .Values.snapshotIncreaseIntervalMs }}
     offsets.persist.interval.ms={{ .Values.offsetsPersistIntervalMs }}
@@ -53,7 +46,6 @@ data:
     neo4j.bolt.server.disabled=true
 
     dns.name.prefix.frontend=localhost
-    dns.name.prefix.ingestor=localhost
     dns.name.prefix.coordinator=localhost
     dns.name.prefix.store=localhost
 

charts/graphscope-store-one-pod/values.yaml

@@ -122,36 +122,6 @@ frontend:
     ##
     annotations: {}
 
-ingestor:
-  replicaCount: 2
-
-  ## @param hostAliases pods host aliases
-  ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/
-  ##
-  hostAliases: []
-  ## @param hostNetwork Specify if host network should be enabled for pods
-  ##
-  hostNetwork: false
-  ## @param hostIPC Specify if host IPC should be enabled for pods
-  ##
-  hostIPC: false
-  ## @param podLabels Extra labels for pods
-  ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
-  ##
-  podLabels: {}
-  ## @param podAnnotations Extra annotations for pods
-  ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
-  ##
-  podAnnotations: {}
-  ## @param podAffinityPreset Pod affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard`
-  ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity
-  ##
-
-  ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
-  ##
-  schedulerName: ""
-
-
 ## @section Common parameters
 ##
 
@@ -350,11 +320,6 @@ javaOpts: ""
 
 rpcMaxBytesMb: 20
 
-## Ingestor Config
-ingestorQueueBufferMaxCount: 102400
-ingestorSenderBufferMaxCount: 102400
-ingestorSenderOperationMaxCount: 102400
-
 ## Coordinator Config
 snapshotIncreaseIntervalMs: 1000
 offsetsPersistIntervalMs: 3000

charts/graphscope-store/README.md

@@ -99,7 +99,6 @@ Here we give a list of most frequently used parameters.
 | store.replicaCount | Number of nodes | 2 |
 | frontend.replicaCount | Number of nodes | 1 |
 | frontend.service.type| Kubernetes Service type| NodePort |
-| ingestor.replicaCount | Number of nodes | 1 |
 | coordinator.replicaCount | Number of nodes | 1 |
 
 ### Kafka chart parameters

charts/graphscope-store/templates/_helpers.tpl

@@ -31,10 +31,6 @@ If release name contains chart name it will be used as a full name.
 {{- printf "%s-%s" (include "graphscope-store.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
-{{- define "graphscope-store.ingestor.fullname" -}}
-{{- printf "%s-%s" (include "graphscope-store.fullname" .) "ingestor" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
 {{- define "graphscope-store.store.fullname" -}}
 {{- printf "%s-%s" (include "graphscope-store.fullname" .) "store" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

charts/graphscope-store/templates/configmap.yaml

@@ -21,9 +21,7 @@ data:
     rpc.max.bytes.mb={{ .Values.rpcMaxBytesMb }}
     store.node.count={{ .Values.store.replicaCount }}
     frontend.node.count={{ .Values.frontend.replicaCount }}
-    ingestor.node.count={{ .Values.ingestor.replicaCount }}
     coordinator.node.count={{ .Values.coordinator.replicaCount }}
-    ingestor.queue.count={{ .Values.ingestor.replicaCount }}
     partition.count={{ .Values.store.replicaCount | mul 16 }}
     discovery.mode={{ .Values.discoveryMode }}
 
@@ -33,11 +31,6 @@ data:
     frontend.server.num={{ .Values.frontend.replicaCount }}
     enable.hash.generate.eid={{ .Values.enableHashGenerateEid }}
 
-    ## Ingestor Config
-    ingestor.queue.buffer.max.count={{ .Values.ingestorQueueBufferMaxCount }}
-    ingestor.sender.buffer.max.count={{ .Values.ingestorSenderBufferMaxCount }}
-    ingestor.sender.operation.max.count={{ .Values.ingestorSenderOperationMaxCount }}
-
     ## Coordinator Config
     snapshot.increase.interval.ms={{ .Values.snapshotIncreaseIntervalMs }}
     offsets.persist.interval.ms={{ .Values.offsetsPersistIntervalMs }}
@@ -61,7 +54,6 @@ data:
     neo4j.bolt.server.disabled=true
 
     dns.name.prefix.frontend=FRONTEND
-    dns.name.prefix.ingestor=INGESTOR
     dns.name.prefix.coordinator=COORDINATOR
     dns.name.prefix.store=STORE
 
@@ -118,7 +110,6 @@ data:
               -e "s/INDEX/${ordinal}/g" \
               -e "s/KAFKA_SERVERS/${KAFKA_SERVERS}/g" \
               -e "s/FRONTEND/${DNS_NAME_PREFIX_FRONTEND}/g" \
-              -e "s/INGESTOR/${DNS_NAME_PREFIX_INGESTOR}/g" \
               -e "s/COORDINATOR/${DNS_NAME_PREFIX_COORDINATOR}/g" \
               -e "s/STORE/${DNS_NAME_PREFIX_STORE}/g" \
               -e "s/PEGASUS_HOSTS/${pegasus_hosts}/g" \

charts/graphscope-store/templates/coordinator/statefulset.yaml

@@ -1,5 +1,4 @@
 {{- $frontendFullname := include "graphscope-store.frontend.fullname" . }}
-{{- $ingestorFullname := include "graphscope-store.ingestor.fullname" . }}
 {{- $coordinatorFullname := include "graphscope-store.coordinator.fullname" . }}
 {{- $storeFullname := include "graphscope-store.store.fullname" . }}
 {{- $kafkaFullname := include "graphscope-store.kafka.fullname" . -}}
@@ -110,16 +109,12 @@ spec:
               value: "coordinator"
             - name: FRONTEND_COUNT
               value: {{ .Values.frontend.replicaCount | quote }}
-            - name: INGESTOR_COUNT
-              value: {{ .Values.ingestor.replicaCount | quote }}
             - name: COORDINATOR_COUNT
               value: {{ .Values.coordinator.replicaCount | quote }}
             - name: STORE_COUNT
               value: {{ .Values.store.replicaCount | quote }}
             - name: DNS_NAME_PREFIX_FRONTEND
               value: {{ $frontendFullname }}-{}.{{ $frontendFullname }}-headless
-            - name: DNS_NAME_PREFIX_INGESTOR
-              value: {{ $ingestorFullname }}-{}.{{ $ingestorFullname }}-headless
             - name: DNS_NAME_PREFIX_COORDINATOR
               value: {{ $coordinatorFullname }}-{}.{{ $coordinatorFullname }}-headless
             - name: DNS_NAME_PREFIX_STORE

charts/graphscope-store/templates/frontend/statefulset.yaml

@@ -1,5 +1,4 @@
 {{- $frontendFullname := include "graphscope-store.frontend.fullname" . }}
-{{- $ingestorFullname := include "graphscope-store.ingestor.fullname" . }}
 {{- $coordinatorFullname := include "graphscope-store.coordinator.fullname" . }}
 {{- $storeFullname := include "graphscope-store.store.fullname" . }}
 {{- $kafkaFullname := include "graphscope-store.kafka.fullname" . -}}
@@ -110,16 +109,12 @@ spec:
               value: "frontend"
             - name: FRONTEND_COUNT
               value: {{ .Values.frontend.replicaCount | quote }}
-            - name: INGESTOR_COUNT
-              value: {{ .Values.ingestor.replicaCount | quote }}
             - name: COORDINATOR_COUNT
               value: {{ .Values.coordinator.replicaCount | quote }}
             - name: STORE_COUNT
               value: {{ .Values.store.replicaCount | quote }}
             - name: DNS_NAME_PREFIX_FRONTEND
               value: {{ $frontendFullname }}-{}.{{ $frontendFullname }}-headless
-            - name: DNS_NAME_PREFIX_INGESTOR
-              value: {{ $ingestorFullname }}-{}.{{ $ingestorFullname }}-headless
             - name: DNS_NAME_PREFIX_COORDINATOR
               value: {{ $coordinatorFullname }}-{}.{{ $coordinatorFullname }}-headless
             - name: DNS_NAME_PREFIX_STORE