feat(interactive): Implement admin http service for hqps engine (#3322)

1. Implement an HTTP service that provides admin management services for the HQPS engine. Including - CreateGraph - LoadGraph - DeleteGraph - GetGraphSchema - CreateProcedure - GetAllProcedure - GetProcedureByName - UpdateProcedure - RemoveProcedure - StartService - ServiceStatus - NodeStatus 2. Reorganize the `engines/http_server` directory. Put all `actors` in actor directory, all `handlers` in handler direcotry, all `services` in service directory. 3. Introduce a thirdpary dependency to `nlohmann/json`, since we need to parse the http request body as json string, and also return the results in json format. 4. Introduce a new binary `interactive_server`, which start the both the admin service and the query service. 5. Move some commonly used function to `service_utils.h`. 6. Add `LoadFromDataDirectory` method for `GraphDB`, support deserialized graph from data directory without `bulk_load` and `schema` given. GraphDB can also switch between graphs via this method. 7. Support add plugin and remove plugin at runtime. 8. Introduce `GraphManager` to provide graph management and procedure management. Coarse-grained lock will be utilized for stage-changing operations. 9. Add End-2-End CI test. 10. Add document for admin service. 11. Some minor fix. 12. Merge the diverged procedure management of hqps engine to graph_db.
alibaba · Nov 27, 2023 · eb9e8b5 · eb9e8b5
1 parent c237328
commit eb9e8b5
Show file tree

Hide file tree

Showing 85 changed files with 6,227 additions and 1,326 deletions.
diff --git a/.github/workflows/flex.yml b/.github/workflows/flex.yml
@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8
     steps:
     - uses: actions/checkout@v3
 
@@ -44,6 +44,8 @@ jobs:
       env:
         HOME: /home/graphscope/
       run: |
+        cd ${GITHUB_WORKSPACE}/
+        git submodule update --init
         cd ${GITHUB_WORKSPACE}/flex
         mkdir build && cd build
         cmake .. && sudo make -j$(nproc)
@@ -74,10 +76,10 @@ jobs:
         GLOG_v=10 ./bin/graph_loader  ${SCHEMA_FILE} ${BULK_LOAD_FILE} /tmp/csr-data-dir/
 
     - name: Test Graph Loading on type_test graph
-    env:
+      env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest/
         FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/type_test/
-    run: |
+      run: |
         # remove modern graph indices
         rm -rf /tmp/csr-data-dir/
 

diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml
@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.8
     steps:
     - uses: actions/checkout@v3
 
@@ -68,6 +68,8 @@ jobs:
         GIE_HOME: ${{ github.workspace }}/interactive_engine/
         HOME: /home/graphscope/
       run: |
+        cd ${GITHUB_WORKSPACE}/
+        git submodule update --init
         cd ${GITHUB_WORKSPACE}/flex
         mkdir build && cd build
         cmake .. && sudo make -j$(nproc)
@@ -90,9 +92,27 @@ jobs:
         git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
         mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc
         GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
+        BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
         cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
+        cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml
         mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies
         cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml
+        cp ${GS_TEST_DIR}/flex/movies/movies_import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml
+
+        # load graph
+        cd ${GITHUB_WORKSPACE}/flex/build
+        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
+        GLOG_v=10 ./bin/graph_loader ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml ${INTERACTIVE_WORKSPACE}/data/ldbc/indices/
+        export FLEX_DATA_DIR=../interactive/examples/movies
+        GLOG_v=10 ./bin/graph_loader ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/indices/
+
+    - name: Test HQPS admin http service
+      env: 
+        GS_TEST_DIR: ${{ github.workspace }}/gstest
+        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
+      run:
+        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
+        bash hqps_admin_test.sh /tmp/temp_workspace ./engine_config_test.yaml ${GS_TEST_DIR}
 
     - name: Sample Query test
       env:
@@ -110,13 +130,12 @@ jobs:
         HOME : /home/graphscope/
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
-        GIE_HOME=${GITHUB_WORKSPACE}/interactive_engine
         cd ${GITHUB_WORKSPACE}/flex/bin
 
         for i in 1 2 3 4 5 6 7 8 9 10 11 12;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
           eval ${cmd}
@@ -125,19 +144,19 @@ jobs:
         for i in 1 2 3 4 5 6 7 8 9 10 11 12;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codgen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
           echo $cmd
           eval ${cmd}
         done
 
         # test movie graph, 8,9,10 are not supported now
-        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies
-        sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
+        # change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies
+        sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml
         for i in 1 2 3 4 5 6 7 11 12 13 14 15;
         do
           cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codgen/"
-          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
+          cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
           cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
           echo $cmd
           eval ${cmd}
@@ -150,12 +169,11 @@ jobs:
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
-        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
         export ENGINE_TYPE=hiactor
-        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to ldbc
-        sed -i 's/default_graph: movies/default_graph: ldbc/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
-        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml \
-              ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
+        # change the default_graph config in ./engine_config_test.yaml to ldbc
+        sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml
+        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
+              ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
 
     - name: Run End-to-End cypher adhoc movie query test
       env:
@@ -164,11 +182,10 @@ jobs:
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
       run: |
         cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
-        export FLEX_DATA_DIR=../../interactive/examples/movies/
         export ENGINE_TYPE=hiactor
-        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies
-        sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
-        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies ${GS_TEST_DIR}/flex/movies/movies_import.yaml \
-          ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
+        # change the default_graph config in ./engine_config_test.yaml to movies
+        sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml
+        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies \
+          ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
 
 
diff --git a/.gitmodules b/.gitmodules
@@ -9,3 +9,7 @@
 [submodule "learning_engine/graphlearn-for-pytorch"]
 	path = learning_engine/graphlearn-for-pytorch
 	url = https://github.com/alibaba/graphlearn-for-pytorch.git
+
+[submodule "flex/third_party/nlohmann-json"]
+	path = flex/third_party/nlohmann-json
+	url = https://github.com/nlohmann/json.git
diff --git a/docs/flex/interactive/configuration.md b/docs/flex/interactive/configuration.md
@@ -44,6 +44,10 @@ compiler:
     - FilterIntoJoinRule
     - NotExistToAntiJoinRule
   query_timeout: 20000  # query timeout in milliseconds, default 20000
+http_service:
+  default_listen_address: localhost
+  admin_port: 7777
+  query_port: 10000
 ```
 
 
@@ -65,5 +69,8 @@ In this following table, we use the `.` notation to represent the hierarchy with
 | compiler.planner.rules.FilterMatchRule | N/A | An optimization rule that pushes filter (`Where`) conditions into the `Match` clause | 0.0.1 |
 | compiler.planner.rules.FilterIntoJoinRule | N/A | A native Calcite optimization rule that pushes filter conditions to the Join participants before performing the join | 0.0.1 |
 | compiler.planner.rules.NotMatchToAntiJoinRule | N/A | An optimization rule that transforms a "not exist" pattern into an anti-join operation  | 0.0.1 |
+| http_service.default_listen_address | localhost | The address for http service to bind | 0.0.2 |
+| http_service.admin_port | 7777 | The port for admin service to listen on | 0.0.2 |
+| http_service.query_port | 10000 | The port for query service to listen on, for stored procedure queries, user can directory submit queries to query_port without compiler involved | 0.0.2 |
 
 
diff --git a/docs/flex/interactive/dev_guide.md b/docs/flex/interactive/dev_guide.md
@@ -1,6 +1,7 @@
 # GraphScope Interactive Development guide
 
 Users can develop C++ stored procedures and Cypher stored procedures based on `GraphScope Interactive`.
+Interactive also provides an Admin Service for managing graph data and stored procedures at runtime, and it can also retrieve the running status of the service.
 
 ```{toctree} arguments
 ---
@@ -9,4 +10,5 @@ maxdepth: 2
 ---
 development/cypher_procedure
 development/cpp_procedure
+development/admin_service
 ```