From d29516b9dfc208c58e8e3a0e9aa454c1ac1273b1 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 2 Feb 2024 19:34:08 +0800 Subject: [PATCH] fix(interactive): support non-blocking data loading interface in coordinator and fix failure during flexbuild process (#3530) --- flex/coordinator/.openapi-generator-ignore | 1 + flex/coordinator/.openapi-generator/FILES | 2 +- .../controllers/job_controller.py | 68 ++ .../controllers/legacy_controller.py | 17 - .../core/alert/message_collector.py | 2 +- .../core/client_wrapper.py | 40 +- .../core/interactive/hqps.py | 75 +- .../core/interactive/hqps_client/__init__.py | 6 +- .../interactive/hqps_client/api/__init__.py | 3 +- .../hqps_client/api/dataloading_api.py | 331 ----- .../interactive/hqps_client/api/job_api.py | 1082 +++++++++++++++++ .../interactive/hqps_client/api/node_api.py | 288 ----- .../hqps_client/models/__init__.py | 3 +- .../{node_status.py => job_response.py} | 23 +- .../hqps_client/models/job_status.py | 111 ++ .../gs_flex_coordinator/models/__init__.py | 1 + .../models/job_response.py | 61 + .../gs_flex_coordinator/models/job_status.py | 225 ++++ .../gs_flex_coordinator/openapi/openapi.yaml | 123 +- .../test/test_job_controller.py | 80 ++ flex/coordinator/requirements.txt | 12 +- flex/coordinator/setup.py | 7 +- flex/interactive/docker/Makefile | 13 +- .../docker/interactive-base.Dockerfile | 3 +- .../docker/interactive-runtime.Dockerfile | 36 +- flex/openapi/openapi_coordinator.yaml | 81 +- flex/openapi/openapi_interactive.yaml | 104 +- python/graphscope/flex/rest/__init__.py | 2 + python/graphscope/flex/rest/api/__init__.py | 1 + python/graphscope/flex/rest/api/job_api.py | 1078 ++++++++++++++++ python/graphscope/flex/rest/api/legacy_api.py | 283 ----- .../graphscope/flex/rest/models/__init__.py | 1 + .../graphscope/flex/rest/models/job_status.py | 111 ++ python/graphscope/gsctl/commands/common.py | 2 +- python/graphscope/gsctl/commands/dev.py | 44 + .../graphscope/gsctl/commands/interactive.py | 101 +- python/graphscope/gsctl/impl/__init__.py | 5 +- python/graphscope/gsctl/impl/job.py | 64 + python/graphscope/gsctl/impl/legacy.py | 11 - python/requirements.txt | 1 + python/setup_flex.py | 24 +- 41 files changed, 3456 insertions(+), 1070 deletions(-) create mode 100644 flex/coordinator/gs_flex_coordinator/controllers/job_controller.py delete mode 100644 flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/dataloading_api.py create mode 100644 flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/job_api.py delete mode 100644 flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/node_api.py rename flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/{node_status.py => job_response.py} (74%) create mode 100644 flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/job_status.py create mode 100644 flex/coordinator/gs_flex_coordinator/models/job_response.py create mode 100644 flex/coordinator/gs_flex_coordinator/models/job_status.py create mode 100644 flex/coordinator/gs_flex_coordinator/test/test_job_controller.py create mode 100644 python/graphscope/flex/rest/api/job_api.py create mode 100644 python/graphscope/flex/rest/models/job_status.py create mode 100644 python/graphscope/gsctl/impl/job.py diff --git a/flex/coordinator/.openapi-generator-ignore b/flex/coordinator/.openapi-generator-ignore index a81ae113a81a..b215b2c874ac 100644 --- a/flex/coordinator/.openapi-generator-ignore +++ b/flex/coordinator/.openapi-generator-ignore @@ -24,3 +24,4 @@ gs_flex_coordinator/controllers/* setup.py +requirements.txt diff --git a/flex/coordinator/.openapi-generator/FILES b/flex/coordinator/.openapi-generator/FILES index 1358357d3e69..ec8eef479551 100644 --- a/flex/coordinator/.openapi-generator/FILES +++ b/flex/coordinator/.openapi-generator/FILES @@ -34,6 +34,7 @@ gs_flex_coordinator/models/groot_graph_gremlin_interface.py gs_flex_coordinator/models/groot_property.py gs_flex_coordinator/models/groot_schema.py gs_flex_coordinator/models/groot_vertex_type.py +gs_flex_coordinator/models/job_status.py gs_flex_coordinator/models/model_property.py gs_flex_coordinator/models/model_schema.py gs_flex_coordinator/models/node_status.py @@ -54,6 +55,5 @@ gs_flex_coordinator/openapi/openapi.yaml gs_flex_coordinator/test/__init__.py gs_flex_coordinator/typing_utils.py gs_flex_coordinator/util.py -requirements.txt test-requirements.txt tox.ini diff --git a/flex/coordinator/gs_flex_coordinator/controllers/job_controller.py b/flex/coordinator/gs_flex_coordinator/controllers/job_controller.py new file mode 100644 index 000000000000..f48cec0e291f --- /dev/null +++ b/flex/coordinator/gs_flex_coordinator/controllers/job_controller.py @@ -0,0 +1,68 @@ +import connexion +from typing import Dict +from typing import Tuple +from typing import Union + +from gs_flex_coordinator.core import client_wrapper +from gs_flex_coordinator.core import handle_api_exception +from gs_flex_coordinator.models.job_status import JobStatus # noqa: E501 +from gs_flex_coordinator.models.schema_mapping import SchemaMapping # noqa: E501 +from gs_flex_coordinator import util + + +@handle_api_exception() +def create_dataloading_job(graph_name, schema_mapping): # noqa: E501 + """create_dataloading_job + + # noqa: E501 + + :param graph_name: + :type graph_name: str + :param schema_mapping: + :type schema_mapping: dict | bytes + + :rtype: Union[str, Tuple[str, int], Tuple[str, int, Dict[str, str]] + """ + if connexion.request.is_json: + schema_mapping = SchemaMapping.from_dict(connexion.request.get_json()) # noqa: E501 + return client_wrapper.create_dataloading_job(graph_name, schema_mapping) + + +@handle_api_exception() +def delete_job_by_id(job_id): # noqa: E501 + """delete_job_by_id + + # noqa: E501 + + :param job_id: + :type job_id: str + + :rtype: Union[str, Tuple[str, int], Tuple[str, int, Dict[str, str]] + """ + return client_wrapper.delete_job_by_id(job_id) + + +@handle_api_exception() +def get_job_by_id(job_id): # noqa: E501 + """get_job_by_id + + # noqa: E501 + + :param job_id: + :type job_id: str + + :rtype: Union[JobStatus, Tuple[JobStatus, int], Tuple[JobStatus, int, Dict[str, str]] + """ + return client_wrapper.get_job_by_id(job_id) + + +@handle_api_exception() +def list_jobs(): # noqa: E501 + """list_jobs + + # noqa: E501 + + + :rtype: Union[List[JobStatus], Tuple[List[JobStatus], int], Tuple[List[JobStatus], int, Dict[str, str]] + """ + return client_wrapper.list_jobs() diff --git a/flex/coordinator/gs_flex_coordinator/controllers/legacy_controller.py b/flex/coordinator/gs_flex_coordinator/controllers/legacy_controller.py index 30ad3a637d6c..35f4dc52748d 100644 --- a/flex/coordinator/gs_flex_coordinator/controllers/legacy_controller.py +++ b/flex/coordinator/gs_flex_coordinator/controllers/legacy_controller.py @@ -10,23 +10,6 @@ from gs_flex_coordinator.models.schema_mapping import SchemaMapping # noqa: E501 from gs_flex_coordinator import util -@handle_api_exception() -def data_import(graph_name, schema_mapping): # noqa: E501 - """data_import - - # noqa: E501 - - :param graph_name: - :type graph_name: str - :param schema_mapping: - :type schema_mapping: dict | bytes - - :rtype: Union[str, Tuple[str, int], Tuple[str, int, Dict[str, str]] - """ - if connexion.request.is_json: - schema_mapping = SchemaMapping.from_dict(connexion.request.get_json()) # noqa: E501 - return client_wrapper.data_import(graph_name, schema_mapping) - @handle_api_exception() def get_groot_schema(graph_name): # noqa: E501 diff --git a/flex/coordinator/gs_flex_coordinator/core/alert/message_collector.py b/flex/coordinator/gs_flex_coordinator/core/alert/message_collector.py index 434f42f2ae11..5076a2d578ef 100644 --- a/flex/coordinator/gs_flex_coordinator/core/alert/message_collector.py +++ b/flex/coordinator/gs_flex_coordinator/core/alert/message_collector.py @@ -171,7 +171,7 @@ def collect(self, message: AlertMessage): message.trigger_time ) - logger.info("Alert message generated: %s", str(message.to_dict())) + # logger.info("Alert message generated: %s", str(message.to_dict())) # add message to current collector self._current_message_collector.add_message(message) diff --git a/flex/coordinator/gs_flex_coordinator/core/client_wrapper.py b/flex/coordinator/gs_flex_coordinator/core/client_wrapper.py index f38a22a89026..a52a71107f6a 100644 --- a/flex/coordinator/gs_flex_coordinator/core/client_wrapper.py +++ b/flex/coordinator/gs_flex_coordinator/core/client_wrapper.py @@ -16,6 +16,7 @@ # limitations under the License. # +import datetime import itertools import logging import socket @@ -23,17 +24,14 @@ from typing import List, Union import psutil -from gs_flex_coordinator.core.config import CLUSTER_TYPE, INSTANCE_NAME, SOLUTION +from gs_flex_coordinator.core.config import (CLUSTER_TYPE, INSTANCE_NAME, + SOLUTION) from gs_flex_coordinator.core.interactive import init_hqps_client -from gs_flex_coordinator.models import ( - DeploymentInfo, - Graph, - ModelSchema, - NodeStatus, - Procedure, - ServiceStatus, - StartServiceRequest, -) +from gs_flex_coordinator.core.utils import encode_datetime +from gs_flex_coordinator.models import (DeploymentInfo, Graph, JobStatus, + ModelSchema, NodeStatus, Procedure, + SchemaMapping, ServiceStatus, + StartServiceRequest) from gs_flex_coordinator.version import __version__ logger = logging.getLogger("graphscope") @@ -132,7 +130,23 @@ def restart_service(self) -> str: def start_service(self, request: StartServiceRequest) -> str: return self._client.start_service(request) - def data_import(self, graph_name, schema_mapping) -> str: + def list_jobs(self) -> List[JobStatus]: + # transfer + rlt = [] + for job_status_dict in self._client.list_jobs(): + rlt.append(JobStatus.from_dict(job_status_dict)) + return rlt + + def get_job_by_id(self, job_id: str) -> JobStatus: + job_status_dict = self._client.get_job_by_id(job_id) + return JobStatus.from_dict(job_status_dict) + + def delete_job_by_id(self, job_id: str) -> str: + return self._client.delete_job_by_id(job_id) + + def create_dataloading_job( + self, graph_name: str, schema_mapping: SchemaMapping + ) -> str: # there are some tricks here, since property is a keyword of openapi # specification, so it will be converted into the _property field. schema_mapping_dict = schema_mapping.to_dict() @@ -142,8 +156,8 @@ def data_import(self, graph_name, schema_mapping) -> str: for column_mapping in mapping["column_mappings"]: if "_property" in column_mapping: column_mapping["property"] = column_mapping.pop("_property") - print(schema_mapping_dict) - return self._client.data_import(graph_name, schema_mapping_dict) + job_id = self._client.create_dataloading_job(graph_name, schema_mapping_dict) + return job_id client_wrapper = ClientWrapper() diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps.py index 2d7ba8cda265..ac2015cb0e1c 100644 --- a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps.py +++ b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps.py @@ -16,27 +16,20 @@ # limitations under the License. # +import datetime import logging import os from typing import List, Union -from gs_flex_coordinator.core.config import ( - CLUSTER_TYPE, - HQPS_ADMIN_SERVICE_PORT, - WORKSPACE, -) -from gs_flex_coordinator.core.utils import get_internal_ip -from gs_flex_coordinator.models import StartServiceRequest - import hqps_client -from hqps_client import ( - Graph, - ModelSchema, - NodeStatus, - Procedure, - SchemaMapping, - Service, -) +from hqps_client import (Graph, JobResponse, JobStatus, ModelSchema, Procedure, + SchemaMapping, Service) + +from gs_flex_coordinator.core.config import (CLUSTER_TYPE, + HQPS_ADMIN_SERVICE_PORT, + WORKSPACE) +from gs_flex_coordinator.core.utils import encode_datetime, get_internal_ip +from gs_flex_coordinator.models import StartServiceRequest logger = logging.getLogger("graphscope") @@ -170,15 +163,57 @@ def start_service(self, request: StartServiceRequest) -> str: Service.from_dict({"graph_name": request.graph_name}) ) - def data_import(self, graph_name: str, schema_mapping: dict) -> str: - print(graph_name, schema_mapping) + def list_jobs(self) -> List[dict]: + with hqps_client.ApiClient( + hqps_client.Configuration(self._hqps_endpoint) + ) as api_client: + api_instance = hqps_client.JobApi(api_client) + rlt = [] + for s in api_instance.list_jobs(): + job_status = s.to_dict() + job_status["start_time"] = encode_datetime( + datetime.datetime.fromtimestamp(job_status["start_time"] / 1000) + ) + if "end_time" in job_status: + job_status["end_time"] = encode_datetime( + datetime.datetime.fromtimestamp(job_status["end_time"] / 1000) + ) + rlt.append(job_status) + return rlt + + def get_job_by_id(self, job_id: str) -> dict: + with hqps_client.ApiClient( + hqps_client.Configuration(self._hqps_endpoint) + ) as api_client: + api_instance = hqps_client.JobApi(api_client) + job_status = api_instance.get_job_by_id(job_id).to_dict() + job_status["start_time"] = encode_datetime( + datetime.datetime.fromtimestamp(job_status["start_time"] / 1000) + ) + if "end_time" in job_status: + job_status["end_time"] = encode_datetime( + datetime.datetime.fromtimestamp(job_status["end_time"] / 1000) + ) + return job_status + + def delete_job_by_id(self, job_id: str) -> str: + with hqps_client.ApiClient( + hqps_client.Configuration(self._hqps_endpoint) + ) as api_client: + api_instance = hqps_client.JobApi(api_client) + return api_instance.delete_job_by_id(job_id) + + def create_dataloading_job( + self, graph_name: str, schema_mapping: dict + ) -> JobResponse: with hqps_client.ApiClient( hqps_client.Configuration(self._hqps_endpoint) ) as api_client: - api_instance = hqps_client.DataloadingApi(api_client) - return api_instance.create_dataloading_job( + api_instance = hqps_client.JobApi(api_client) + response = api_instance.create_dataloading_job( graph_name, SchemaMapping.from_dict(schema_mapping) ) + return response.job_id def init_hqps_client(): diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/__init__.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/__init__.py index f3e42ad091c0..1ca63d2e1849 100644 --- a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/__init__.py +++ b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/__init__.py @@ -18,9 +18,8 @@ __version__ = "1.0.0" # import apis into sdk package -from hqps_client.api.dataloading_api import DataloadingApi from hqps_client.api.graph_api import GraphApi -from hqps_client.api.node_api import NodeApi +from hqps_client.api.job_api import JobApi from hqps_client.api.procedure_api import ProcedureApi from hqps_client.api.service_api import ServiceApi @@ -47,9 +46,10 @@ from hqps_client.models.edge_type_vertex_type_pair_relations_inner_x_csr_params import EdgeTypeVertexTypePairRelationsInnerXCsrParams from hqps_client.models.graph import Graph from hqps_client.models.graph_stored_procedures import GraphStoredProcedures +from hqps_client.models.job_response import JobResponse +from hqps_client.models.job_status import JobStatus from hqps_client.models.model_property import ModelProperty from hqps_client.models.model_schema import ModelSchema -from hqps_client.models.node_status import NodeStatus from hqps_client.models.procedure import Procedure from hqps_client.models.procedure_params_inner import ProcedureParamsInner from hqps_client.models.property_property_type import PropertyPropertyType diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/__init__.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/__init__.py index 04a9f52f241e..5fbd916816d1 100644 --- a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/__init__.py +++ b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/__init__.py @@ -1,9 +1,8 @@ # flake8: noqa # import apis into api package -from hqps_client.api.dataloading_api import DataloadingApi from hqps_client.api.graph_api import GraphApi -from hqps_client.api.node_api import NodeApi +from hqps_client.api.job_api import JobApi from hqps_client.api.procedure_api import ProcedureApi from hqps_client.api.service_api import ServiceApi diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/dataloading_api.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/dataloading_api.py deleted file mode 100644 index e2e8c0494696..000000000000 --- a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/dataloading_api.py +++ /dev/null @@ -1,331 +0,0 @@ -# coding: utf-8 - -""" - GraphScope Interactive API - - This is a specification for GraphScope Interactive based on the OpenAPI 3.0 specification. You can find out more details about specification at [doc](https://swagger.io/specification/v3/). Some useful links: - [GraphScope Repository](https://github.com/alibaba/GraphScope) - [The Source API definition for GraphScope Interactive](#) - - The version of the OpenAPI document: 0.9.1 - Contact: graphscope@alibaba-inc.com - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -import io -import warnings - -from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt -from typing import Dict, List, Optional, Tuple, Union, Any - -try: - from typing import Annotated -except ImportError: - from typing_extensions import Annotated - -from pydantic import StrictStr - -from hqps_client.models.schema_mapping import SchemaMapping - -from hqps_client.api_client import ApiClient -from hqps_client.api_response import ApiResponse -from hqps_client.rest import RESTResponseType - - -class DataloadingApi: - """NOTE: This class is auto generated by OpenAPI Generator - Ref: https://openapi-generator.tech - - Do not edit the class manually. - """ - - def __init__(self, api_client=None) -> None: - if api_client is None: - api_client = ApiClient.get_default() - self.api_client = api_client - - - @validate_call - def create_dataloading_job( - self, - graph_name: StrictStr, - schema_mapping: SchemaMapping, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> str: - """create_dataloading_job - - Create a dataloading job (Synchronous) - - :param graph_name: (required) - :type graph_name: str - :param schema_mapping: (required) - :type schema_mapping: SchemaMapping - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._create_dataloading_job_serialize( - graph_name=graph_name, - schema_mapping=schema_mapping, - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "str", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - response_data.read() - return self.api_client.response_deserialize( - response_data=response_data, - response_types_map=_response_types_map, - ).data - - - @validate_call - def create_dataloading_job_with_http_info( - self, - graph_name: StrictStr, - schema_mapping: SchemaMapping, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> ApiResponse[str]: - """create_dataloading_job - - Create a dataloading job (Synchronous) - - :param graph_name: (required) - :type graph_name: str - :param schema_mapping: (required) - :type schema_mapping: SchemaMapping - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._create_dataloading_job_serialize( - graph_name=graph_name, - schema_mapping=schema_mapping, - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "str", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - response_data.read() - return self.api_client.response_deserialize( - response_data=response_data, - response_types_map=_response_types_map, - ) - - - @validate_call - def create_dataloading_job_without_preload_content( - self, - graph_name: StrictStr, - schema_mapping: SchemaMapping, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> RESTResponseType: - """create_dataloading_job - - Create a dataloading job (Synchronous) - - :param graph_name: (required) - :type graph_name: str - :param schema_mapping: (required) - :type schema_mapping: SchemaMapping - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._create_dataloading_job_serialize( - graph_name=graph_name, - schema_mapping=schema_mapping, - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "str", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - return response_data.response - - - def _create_dataloading_job_serialize( - self, - graph_name, - schema_mapping, - _request_auth, - _content_type, - _headers, - _host_index, - ) -> Tuple: - - _host = None - - _collection_formats: Dict[str, str] = { - } - - _path_params: Dict[str, str] = {} - _query_params: List[Tuple[str, str]] = [] - _header_params: Dict[str, Optional[str]] = _headers or {} - _form_params: List[Tuple[str, str]] = [] - _files: Dict[str, str] = {} - _body_params: Optional[bytes] = None - - # process the path parameters - if graph_name is not None: - _path_params['graph_name'] = graph_name - # process the query parameters - # process the header parameters - # process the form parameters - # process the body parameter - if schema_mapping is not None: - _body_params = schema_mapping - - - # set the HTTP header `Accept` - _header_params['Accept'] = self.api_client.select_header_accept( - [ - 'application/json' - ] - ) - - # set the HTTP header `Content-Type` - if _content_type: - _header_params['Content-Type'] = _content_type - else: - _default_content_type = ( - self.api_client.select_header_content_type( - [ - 'application/json' - ] - ) - ) - if _default_content_type is not None: - _header_params['Content-Type'] = _default_content_type - - # authentication setting - _auth_settings: List[str] = [ - ] - - return self.api_client.param_serialize( - method='POST', - resource_path='/v1/graph/{graph_name}/dataloading', - path_params=_path_params, - query_params=_query_params, - header_params=_header_params, - body=_body_params, - post_params=_form_params, - files=_files, - auth_settings=_auth_settings, - collection_formats=_collection_formats, - _host=_host, - _request_auth=_request_auth - ) - - diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/job_api.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/job_api.py new file mode 100644 index 000000000000..4b035cb84aee --- /dev/null +++ b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/job_api.py @@ -0,0 +1,1082 @@ +# coding: utf-8 + +""" + GraphScope Interactive API + + This is a specification for GraphScope Interactive based on the OpenAPI 3.0 specification. You can find out more details about specification at [doc](https://swagger.io/specification/v3/). Some useful links: - [GraphScope Repository](https://github.com/alibaba/GraphScope) - [The Source API definition for GraphScope Interactive](#) + + The version of the OpenAPI document: 0.9.1 + Contact: graphscope@alibaba-inc.com + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import io +import warnings + +from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt +from typing import Dict, List, Optional, Tuple, Union, Any + +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + +from pydantic import StrictStr + +from typing import List + +from hqps_client.models.job_response import JobResponse +from hqps_client.models.job_status import JobStatus +from hqps_client.models.schema_mapping import SchemaMapping + +from hqps_client.api_client import ApiClient +from hqps_client.api_response import ApiResponse +from hqps_client.rest import RESTResponseType + + +class JobApi: + """NOTE: This class is auto generated by OpenAPI Generator + Ref: https://openapi-generator.tech + + Do not edit the class manually. + """ + + def __init__(self, api_client=None) -> None: + if api_client is None: + api_client = ApiClient.get_default() + self.api_client = api_client + + + @validate_call + def create_dataloading_job( + self, + graph_name: StrictStr, + schema_mapping: SchemaMapping, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> JobResponse: + """create_dataloading_job + + Create a dataloading job + + :param graph_name: (required) + :type graph_name: str + :param schema_mapping: (required) + :type schema_mapping: SchemaMapping + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._create_dataloading_job_serialize( + graph_name=graph_name, + schema_mapping=schema_mapping, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobResponse", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def create_dataloading_job_with_http_info( + self, + graph_name: StrictStr, + schema_mapping: SchemaMapping, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[JobResponse]: + """create_dataloading_job + + Create a dataloading job + + :param graph_name: (required) + :type graph_name: str + :param schema_mapping: (required) + :type schema_mapping: SchemaMapping + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._create_dataloading_job_serialize( + graph_name=graph_name, + schema_mapping=schema_mapping, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobResponse", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def create_dataloading_job_without_preload_content( + self, + graph_name: StrictStr, + schema_mapping: SchemaMapping, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """create_dataloading_job + + Create a dataloading job + + :param graph_name: (required) + :type graph_name: str + :param schema_mapping: (required) + :type schema_mapping: SchemaMapping + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._create_dataloading_job_serialize( + graph_name=graph_name, + schema_mapping=schema_mapping, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobResponse", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _create_dataloading_job_serialize( + self, + graph_name, + schema_mapping, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + if graph_name is not None: + _path_params['graph_name'] = graph_name + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + if schema_mapping is not None: + _body_params = schema_mapping + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + # set the HTTP header `Content-Type` + if _content_type: + _header_params['Content-Type'] = _content_type + else: + _default_content_type = ( + self.api_client.select_header_content_type( + [ + 'application/json' + ] + ) + ) + if _default_content_type is not None: + _header_params['Content-Type'] = _default_content_type + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='POST', + resource_path='/v1/graph/{graph_name}/dataloading', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + + + + @validate_call + def delete_job_by_id( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> str: + """delete_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._delete_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def delete_job_by_id_with_http_info( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[str]: + """delete_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._delete_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def delete_job_by_id_without_preload_content( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """delete_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._delete_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _delete_job_by_id_serialize( + self, + job_id, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + if job_id is not None: + _path_params['job_id'] = job_id + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='DELETE', + resource_path='/v1/job/{job_id}', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + + + + @validate_call + def get_job_by_id( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> JobStatus: + """get_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._get_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobStatus", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def get_job_by_id_with_http_info( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[JobStatus]: + """get_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._get_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobStatus", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def get_job_by_id_without_preload_content( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """get_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._get_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobStatus", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _get_job_by_id_serialize( + self, + job_id, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + if job_id is not None: + _path_params['job_id'] = job_id + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='GET', + resource_path='/v1/job/{job_id}', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + + + + @validate_call + def list_jobs( + self, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> List[JobStatus]: + """list_jobs + + + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._list_jobs_serialize( + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "List[JobStatus]", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def list_jobs_with_http_info( + self, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[List[JobStatus]]: + """list_jobs + + + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._list_jobs_serialize( + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "List[JobStatus]", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def list_jobs_without_preload_content( + self, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """list_jobs + + + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._list_jobs_serialize( + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "List[JobStatus]", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _list_jobs_serialize( + self, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='GET', + resource_path='/v1/job', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/node_api.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/node_api.py deleted file mode 100644 index 0103d6a2acec..000000000000 --- a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/api/node_api.py +++ /dev/null @@ -1,288 +0,0 @@ -# coding: utf-8 - -""" - GraphScope Interactive API - - This is a specification for GraphScope Interactive based on the OpenAPI 3.0 specification. You can find out more details about specification at [doc](https://swagger.io/specification/v3/). Some useful links: - [GraphScope Repository](https://github.com/alibaba/GraphScope) - [The Source API definition for GraphScope Interactive](#) - - The version of the OpenAPI document: 0.9.1 - Contact: graphscope@alibaba-inc.com - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" # noqa: E501 - - -import io -import warnings - -from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt -from typing import Dict, List, Optional, Tuple, Union, Any - -try: - from typing import Annotated -except ImportError: - from typing_extensions import Annotated - -from typing import List - -from hqps_client.models.node_status import NodeStatus - -from hqps_client.api_client import ApiClient -from hqps_client.api_response import ApiResponse -from hqps_client.rest import RESTResponseType - - -class NodeApi: - """NOTE: This class is auto generated by OpenAPI Generator - Ref: https://openapi-generator.tech - - Do not edit the class manually. - """ - - def __init__(self, api_client=None) -> None: - if api_client is None: - api_client = ApiClient.get_default() - self.api_client = api_client - - - @validate_call - def get_node_status( - self, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> List[NodeStatus]: - """Get node status - - Get node status - - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._get_node_status_serialize( - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "List[NodeStatus]", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - response_data.read() - return self.api_client.response_deserialize( - response_data=response_data, - response_types_map=_response_types_map, - ).data - - - @validate_call - def get_node_status_with_http_info( - self, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> ApiResponse[List[NodeStatus]]: - """Get node status - - Get node status - - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._get_node_status_serialize( - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "List[NodeStatus]", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - response_data.read() - return self.api_client.response_deserialize( - response_data=response_data, - response_types_map=_response_types_map, - ) - - - @validate_call - def get_node_status_without_preload_content( - self, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> RESTResponseType: - """Get node status - - Get node status - - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._get_node_status_serialize( - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "List[NodeStatus]", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - return response_data.response - - - def _get_node_status_serialize( - self, - _request_auth, - _content_type, - _headers, - _host_index, - ) -> Tuple: - - _host = None - - _collection_formats: Dict[str, str] = { - } - - _path_params: Dict[str, str] = {} - _query_params: List[Tuple[str, str]] = [] - _header_params: Dict[str, Optional[str]] = _headers or {} - _form_params: List[Tuple[str, str]] = [] - _files: Dict[str, str] = {} - _body_params: Optional[bytes] = None - - # process the path parameters - # process the query parameters - # process the header parameters - # process the form parameters - # process the body parameter - - - # set the HTTP header `Accept` - _header_params['Accept'] = self.api_client.select_header_accept( - [ - 'application/json' - ] - ) - - - # authentication setting - _auth_settings: List[str] = [ - ] - - return self.api_client.param_serialize( - method='GET', - resource_path='/v1/node/status', - path_params=_path_params, - query_params=_query_params, - header_params=_header_params, - body=_body_params, - post_params=_form_params, - files=_files, - auth_settings=_auth_settings, - collection_formats=_collection_formats, - _host=_host, - _request_auth=_request_auth - ) - - diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/__init__.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/__init__.py index 74d6794c3735..6eeaab32b362 100644 --- a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/__init__.py +++ b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/__init__.py @@ -26,9 +26,10 @@ from hqps_client.models.edge_type_vertex_type_pair_relations_inner_x_csr_params import EdgeTypeVertexTypePairRelationsInnerXCsrParams from hqps_client.models.graph import Graph from hqps_client.models.graph_stored_procedures import GraphStoredProcedures +from hqps_client.models.job_response import JobResponse +from hqps_client.models.job_status import JobStatus from hqps_client.models.model_property import ModelProperty from hqps_client.models.model_schema import ModelSchema -from hqps_client.models.node_status import NodeStatus from hqps_client.models.procedure import Procedure from hqps_client.models.procedure_params_inner import ProcedureParamsInner from hqps_client.models.property_property_type import PropertyPropertyType diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/node_status.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/job_response.py similarity index 74% rename from flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/node_status.py rename to flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/job_response.py index b28dd08ad9c0..78292eeb21c5 100644 --- a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/node_status.py +++ b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/job_response.py @@ -20,21 +20,18 @@ from typing import Any, ClassVar, Dict, List, Optional -from pydantic import BaseModel, StrictInt, StrictStr +from pydantic import BaseModel, StrictStr try: from typing import Self except ImportError: from typing_extensions import Self -class NodeStatus(BaseModel): +class JobResponse(BaseModel): """ - NodeStatus + JobResponse """ # noqa: E501 - node: Optional[StrictStr] = None - cpu_usage: Optional[StrictInt] = None - memory_usage: Optional[StrictInt] = None - disk_usage: Optional[StrictInt] = None - __properties: ClassVar[List[str]] = ["node", "cpu_usage", "memory_usage", "disk_usage"] + job_id: Optional[StrictStr] = None + __properties: ClassVar[List[str]] = ["job_id"] model_config = { "populate_by_name": True, @@ -49,12 +46,11 @@ def to_str(self) -> str: def to_json(self) -> str: """Returns the JSON representation of the model using alias""" - # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead return json.dumps(self.to_dict()) @classmethod def from_json(cls, json_str: str) -> Self: - """Create an instance of NodeStatus from a JSON string""" + """Create an instance of JobResponse from a JSON string""" return cls.from_dict(json.loads(json_str)) def to_dict(self) -> Dict[str, Any]: @@ -77,7 +73,7 @@ def to_dict(self) -> Dict[str, Any]: @classmethod def from_dict(cls, obj: Dict) -> Self: - """Create an instance of NodeStatus from a dict""" + """Create an instance of JobResponse from a dict""" if obj is None: return None @@ -85,10 +81,7 @@ def from_dict(cls, obj: Dict) -> Self: return cls.model_validate(obj) _obj = cls.model_validate({ - "node": obj.get("node"), - "cpu_usage": obj.get("cpu_usage"), - "memory_usage": obj.get("memory_usage"), - "disk_usage": obj.get("disk_usage") + "job_id": obj.get("job_id") }) return _obj diff --git a/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/job_status.py b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/job_status.py new file mode 100644 index 000000000000..36a93a011e56 --- /dev/null +++ b/flex/coordinator/gs_flex_coordinator/core/interactive/hqps_client/models/job_status.py @@ -0,0 +1,111 @@ +# coding: utf-8 + +""" + GraphScope Interactive API + + This is a specification for GraphScope Interactive based on the OpenAPI 3.0 specification. You can find out more details about specification at [doc](https://swagger.io/specification/v3/). Some useful links: - [GraphScope Repository](https://github.com/alibaba/GraphScope) - [The Source API definition for GraphScope Interactive](#) + + The version of the OpenAPI document: 0.9.1 + Contact: graphscope@alibaba-inc.com + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + + +from typing import Any, ClassVar, Dict, List, Optional +from pydantic import BaseModel, StrictInt, StrictStr, field_validator +from pydantic import Field +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +class JobStatus(BaseModel): + """ + JobStatus + """ # noqa: E501 + job_id: Optional[StrictStr] = None + type: Optional[StrictStr] = None + status: Optional[StrictStr] = None + start_time: Optional[StrictInt] = None + end_time: Optional[StrictInt] = None + log: Optional[StrictStr] = Field(default=None, description="URL or log string") + detail: Optional[Dict[str, Any]] = None + __properties: ClassVar[List[str]] = ["job_id", "type", "status", "start_time", "end_time", "log", "detail"] + + @field_validator('status') + def status_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in ('RUNNING', 'SUCCESS', 'FAILED', 'CANCELLED', 'WAITING'): + raise ValueError("must be one of enum values ('RUNNING', 'SUCCESS', 'FAILED', 'CANCELLED', 'WAITING')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of JobStatus from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={ + }, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of JobStatus from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "job_id": obj.get("job_id"), + "type": obj.get("type"), + "status": obj.get("status"), + "start_time": obj.get("start_time"), + "end_time": obj.get("end_time"), + "log": obj.get("log"), + "detail": obj.get("detail") + }) + return _obj + + diff --git a/flex/coordinator/gs_flex_coordinator/models/__init__.py b/flex/coordinator/gs_flex_coordinator/models/__init__.py index bb6b8e8fbdd4..4d718329279b 100644 --- a/flex/coordinator/gs_flex_coordinator/models/__init__.py +++ b/flex/coordinator/gs_flex_coordinator/models/__init__.py @@ -25,6 +25,7 @@ from gs_flex_coordinator.models.groot_property import GrootProperty from gs_flex_coordinator.models.groot_schema import GrootSchema from gs_flex_coordinator.models.groot_vertex_type import GrootVertexType +from gs_flex_coordinator.models.job_status import JobStatus from gs_flex_coordinator.models.model_property import ModelProperty from gs_flex_coordinator.models.model_schema import ModelSchema from gs_flex_coordinator.models.node_status import NodeStatus diff --git a/flex/coordinator/gs_flex_coordinator/models/job_response.py b/flex/coordinator/gs_flex_coordinator/models/job_response.py new file mode 100644 index 000000000000..0ff619af3936 --- /dev/null +++ b/flex/coordinator/gs_flex_coordinator/models/job_response.py @@ -0,0 +1,61 @@ +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from gs_flex_coordinator.models.base_model import Model +from gs_flex_coordinator import util + + +class JobResponse(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, job_id=None): # noqa: E501 + """JobResponse - a model defined in OpenAPI + + :param job_id: The job_id of this JobResponse. # noqa: E501 + :type job_id: str + """ + self.openapi_types = { + 'job_id': str + } + + self.attribute_map = { + 'job_id': 'job_id' + } + + self._job_id = job_id + + @classmethod + def from_dict(cls, dikt) -> 'JobResponse': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The JobResponse of this JobResponse. # noqa: E501 + :rtype: JobResponse + """ + return util.deserialize_model(dikt, cls) + + @property + def job_id(self) -> str: + """Gets the job_id of this JobResponse. + + + :return: The job_id of this JobResponse. + :rtype: str + """ + return self._job_id + + @job_id.setter + def job_id(self, job_id: str): + """Sets the job_id of this JobResponse. + + + :param job_id: The job_id of this JobResponse. + :type job_id: str + """ + + self._job_id = job_id diff --git a/flex/coordinator/gs_flex_coordinator/models/job_status.py b/flex/coordinator/gs_flex_coordinator/models/job_status.py new file mode 100644 index 000000000000..c4894ec41752 --- /dev/null +++ b/flex/coordinator/gs_flex_coordinator/models/job_status.py @@ -0,0 +1,225 @@ +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from gs_flex_coordinator.models.base_model import Model +from gs_flex_coordinator import util + + +class JobStatus(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, job_id=None, type=None, status=None, start_time=None, end_time=None, log=None, detail=None): # noqa: E501 + """JobStatus - a model defined in OpenAPI + + :param job_id: The job_id of this JobStatus. # noqa: E501 + :type job_id: str + :param type: The type of this JobStatus. # noqa: E501 + :type type: str + :param status: The status of this JobStatus. # noqa: E501 + :type status: str + :param start_time: The start_time of this JobStatus. # noqa: E501 + :type start_time: str + :param end_time: The end_time of this JobStatus. # noqa: E501 + :type end_time: str + :param log: The log of this JobStatus. # noqa: E501 + :type log: str + :param detail: The detail of this JobStatus. # noqa: E501 + :type detail: Dict[str, object] + """ + self.openapi_types = { + 'job_id': str, + 'type': str, + 'status': str, + 'start_time': str, + 'end_time': str, + 'log': str, + 'detail': Dict[str, object] + } + + self.attribute_map = { + 'job_id': 'job_id', + 'type': 'type', + 'status': 'status', + 'start_time': 'start_time', + 'end_time': 'end_time', + 'log': 'log', + 'detail': 'detail' + } + + self._job_id = job_id + self._type = type + self._status = status + self._start_time = start_time + self._end_time = end_time + self._log = log + self._detail = detail + + @classmethod + def from_dict(cls, dikt) -> 'JobStatus': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The JobStatus of this JobStatus. # noqa: E501 + :rtype: JobStatus + """ + return util.deserialize_model(dikt, cls) + + @property + def job_id(self) -> str: + """Gets the job_id of this JobStatus. + + + :return: The job_id of this JobStatus. + :rtype: str + """ + return self._job_id + + @job_id.setter + def job_id(self, job_id: str): + """Sets the job_id of this JobStatus. + + + :param job_id: The job_id of this JobStatus. + :type job_id: str + """ + + self._job_id = job_id + + @property + def type(self) -> str: + """Gets the type of this JobStatus. + + + :return: The type of this JobStatus. + :rtype: str + """ + return self._type + + @type.setter + def type(self, type: str): + """Sets the type of this JobStatus. + + + :param type: The type of this JobStatus. + :type type: str + """ + + self._type = type + + @property + def status(self) -> str: + """Gets the status of this JobStatus. + + + :return: The status of this JobStatus. + :rtype: str + """ + return self._status + + @status.setter + def status(self, status: str): + """Sets the status of this JobStatus. + + + :param status: The status of this JobStatus. + :type status: str + """ + allowed_values = ["RUNNING", "SUCCESS", "FAILED", "CANCELLED", "WAITING"] # noqa: E501 + if status not in allowed_values: + raise ValueError( + "Invalid value for `status` ({0}), must be one of {1}" + .format(status, allowed_values) + ) + + self._status = status + + @property + def start_time(self) -> str: + """Gets the start_time of this JobStatus. + + + :return: The start_time of this JobStatus. + :rtype: str + """ + return self._start_time + + @start_time.setter + def start_time(self, start_time: str): + """Sets the start_time of this JobStatus. + + + :param start_time: The start_time of this JobStatus. + :type start_time: str + """ + + self._start_time = start_time + + @property + def end_time(self) -> str: + """Gets the end_time of this JobStatus. + + + :return: The end_time of this JobStatus. + :rtype: str + """ + return self._end_time + + @end_time.setter + def end_time(self, end_time: str): + """Sets the end_time of this JobStatus. + + + :param end_time: The end_time of this JobStatus. + :type end_time: str + """ + + self._end_time = end_time + + @property + def log(self) -> str: + """Gets the log of this JobStatus. + + URL or log string # noqa: E501 + + :return: The log of this JobStatus. + :rtype: str + """ + return self._log + + @log.setter + def log(self, log: str): + """Sets the log of this JobStatus. + + URL or log string # noqa: E501 + + :param log: The log of this JobStatus. + :type log: str + """ + + self._log = log + + @property + def detail(self) -> Dict[str, object]: + """Gets the detail of this JobStatus. + + + :return: The detail of this JobStatus. + :rtype: Dict[str, object] + """ + return self._detail + + @detail.setter + def detail(self, detail: Dict[str, object]): + """Sets the detail of this JobStatus. + + + :param detail: The detail of this JobStatus. + :type detail: Dict[str, object] + """ + + self._detail = detail diff --git a/flex/coordinator/gs_flex_coordinator/openapi/openapi.yaml b/flex/coordinator/gs_flex_coordinator/openapi/openapi.yaml index 17a1698ad16f..f5ea71fa0569 100644 --- a/flex/coordinator/gs_flex_coordinator/openapi/openapi.yaml +++ b/flex/coordinator/gs_flex_coordinator/openapi/openapi.yaml @@ -409,6 +409,33 @@ paths: tags: - graph x-openapi-router-controller: gs_flex_coordinator.controllers.graph_controller + /api/v1/graph/{graph_name}/dataloading: + post: + operationId: create_dataloading_job + parameters: + - explode: false + in: path + name: graph_name + required: true + schema: + type: string + style: simple + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/SchemaMapping' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/ApiResponse' + description: successful operation + tags: + - job + x-openapi-router-controller: gs_flex_coordinator.controllers.job_controller /api/v1/graph/{graph_name}/procedure: get: description: List stored procedures on a certain graph @@ -737,33 +764,62 @@ paths: tags: - legacy x-openapi-router-controller: gs_flex_coordinator.controllers.legacy_controller - /api/v1/interactive/graph/{graph_name}/dataloading: - post: - operationId: data_import + /api/v1/job: + get: + operationId: list_jobs + responses: + "200": + content: + application/json: + schema: + items: + $ref: '#/components/schemas/JobStatus' + type: array + description: successful operation + tags: + - job + x-openapi-router-controller: gs_flex_coordinator.controllers.job_controller + /api/v1/job/{job_id}: + delete: + operationId: delete_job_by_id parameters: - explode: false in: path - name: graph_name + name: job_id required: true schema: type: string style: simple - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/SchemaMapping' - required: true responses: "200": content: application/json: schema: $ref: '#/components/schemas/ApiResponse' + description: Successful operation + tags: + - job + x-openapi-router-controller: gs_flex_coordinator.controllers.job_controller + get: + operationId: get_job_by_id + parameters: + - explode: false + in: path + name: job_id + required: true + schema: + type: string + style: simple + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/JobStatus' description: successful operation tags: - - legacy - x-openapi-router-controller: gs_flex_coordinator.controllers.legacy_controller + - job + x-openapi-router-controller: gs_flex_coordinator.controllers.job_controller /api/v1/node/status: get: description: "Get node status (cpu/memory/disk, local only)" @@ -1727,6 +1783,49 @@ components: title: ColumnMapping type: object x-body-name: column_mapping + JobStatus: + example: + start_time: start_time + log: log + job_id: job_id + end_time: end_time + detail: + key: "" + type: type + status: RUNNING + properties: + job_id: + title: job_id + type: string + type: + title: type + type: string + status: + enum: + - RUNNING + - SUCCESS + - FAILED + - CANCELLED + - WAITING + title: status + type: string + start_time: + title: start_time + type: string + end_time: + title: end_time + type: string + log: + description: URL or log string + title: log + type: string + detail: + additionalProperties: true + title: detail + type: object + title: JobStatus + type: object + x-body-name: job_status GrootGraph: example: creation_time: creation_time diff --git a/flex/coordinator/gs_flex_coordinator/test/test_job_controller.py b/flex/coordinator/gs_flex_coordinator/test/test_job_controller.py new file mode 100644 index 000000000000..1cbd07019ce7 --- /dev/null +++ b/flex/coordinator/gs_flex_coordinator/test/test_job_controller.py @@ -0,0 +1,80 @@ +import unittest + +from flask import json + +from gs_flex_coordinator.models.job_response import JobResponse # noqa: E501 +from gs_flex_coordinator.models.job_status import JobStatus # noqa: E501 +from gs_flex_coordinator.models.schema_mapping import SchemaMapping # noqa: E501 +from gs_flex_coordinator.test import BaseTestCase + + +class TestJobController(BaseTestCase): + """JobController integration test stubs""" + + def test_create_dataloading_job(self): + """Test case for create_dataloading_job + + + """ + schema_mapping = {"loading_config":{"format":{"metadata":{"key":""},"type":"type"},"import_option":"init","data_source":{"scheme":"file"}},"edge_mappings":[{"inputs":["inputs","inputs"],"source_vertex_mappings":[{"column":{"name":"name","index":0}},{"column":{"name":"name","index":0}}],"destination_vertex_mappings":[{"column":{"name":"name","index":0}},{"column":{"name":"name","index":0}}],"column_mappings":[{"column":{"name":"name","index":0},"property":"property"},{"column":{"name":"name","index":0},"property":"property"}],"type_triplet":{"edge":"edge","source_vertex":"source_vertex","destination_vertex":"destination_vertex"}},{"inputs":["inputs","inputs"],"source_vertex_mappings":[{"column":{"name":"name","index":0}},{"column":{"name":"name","index":0}}],"destination_vertex_mappings":[{"column":{"name":"name","index":0}},{"column":{"name":"name","index":0}}],"column_mappings":[{"column":{"name":"name","index":0},"property":"property"},{"column":{"name":"name","index":0},"property":"property"}],"type_triplet":{"edge":"edge","source_vertex":"source_vertex","destination_vertex":"destination_vertex"}}],"graph":"graph","vertex_mappings":[{"type_name":"type_name","inputs":["inputs","inputs"],"column_mappings":[{"column":{"name":"name","index":0},"property":"property"},{"column":{"name":"name","index":0},"property":"property"}]},{"type_name":"type_name","inputs":["inputs","inputs"],"column_mappings":[{"column":{"name":"name","index":0},"property":"property"},{"column":{"name":"name","index":0},"property":"property"}]}]} + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + } + response = self.client.open( + '/api/v1/graph/{graph_name}/dataloading'.format(graph_name='graph_name_example'), + method='POST', + headers=headers, + data=json.dumps(schema_mapping), + content_type='application/json') + self.assert200(response, + 'Response body is : ' + response.data.decode('utf-8')) + + def test_delete_job_by_id(self): + """Test case for delete_job_by_id + + + """ + headers = { + 'Accept': 'application/json', + } + response = self.client.open( + '/api/v1/job/{job_id}'.format(job_id='job_id_example'), + method='DELETE', + headers=headers) + self.assert200(response, + 'Response body is : ' + response.data.decode('utf-8')) + + def test_get_job_by_id(self): + """Test case for get_job_by_id + + + """ + headers = { + 'Accept': 'application/json', + } + response = self.client.open( + '/api/v1/job/{job_id}'.format(job_id='job_id_example'), + method='GET', + headers=headers) + self.assert200(response, + 'Response body is : ' + response.data.decode('utf-8')) + + def test_list_jobs(self): + """Test case for list_jobs + + + """ + headers = { + 'Accept': 'application/json', + } + response = self.client.open( + '/api/v1/job', + method='GET', + headers=headers) + self.assert200(response, + 'Response body is : ' + response.data.decode('utf-8')) + + +if __name__ == '__main__': + unittest.main() diff --git a/flex/coordinator/requirements.txt b/flex/coordinator/requirements.txt index 666385a4642e..cd62f37bfeb0 100644 --- a/flex/coordinator/requirements.txt +++ b/flex/coordinator/requirements.txt @@ -1,11 +1,11 @@ -connexion[swagger-ui] >= 2.6.0; python_version>="3.6" -# 2.3 is the last version that supports python 3.4-3.5 -connexion[swagger-ui] <= 2.3.0; python_version=="3.5" or python_version=="3.4" -# connexion requires werkzeug but connexion < 2.4.0 does not install werkzeug -# we must peg werkzeug versions below to fix connexion -# https://github.com/zalando/connexion/pull/1044 +connexion == 2.14.2 werkzeug == 2.3.8; python_version=="3.5" or python_version=="3.4" swagger-ui-bundle >= 0.0.2 python_dateutil >= 2.6.0 setuptools >= 21.0.0 Flask == 2.2.5 +urllib3 >= 1.25.3, < 2.1.0 +pydantic >= 2 +typing-extensions >= 4.7.1 +psutil +schedule diff --git a/flex/coordinator/setup.py b/flex/coordinator/setup.py index f4459a4fffe8..2efb969adb61 100644 --- a/flex/coordinator/setup.py +++ b/flex/coordinator/setup.py @@ -30,7 +30,10 @@ pkg_root = os.path.dirname(os.path.abspath(__file__)) -REQUIRES = ["connexion>=2.0.2", "swagger-ui-bundle>=0.0.2", "python_dateutil>=2.6.0"] + +def parsed_reqs(): + with open(os.path.join(pkg_root, "requirements.txt"), "r", encoding="utf-8") as fp: + return fp.read().splitlines() class GenerateFlexServer(Command): @@ -120,7 +123,7 @@ def run(self): author_email="graphscope@alibaba-inc.com", url="", keywords=["OpenAPI", "GraphScope FLEX HTTP SERVICE API"], - install_requires=REQUIRES, + install_requires=parsed_reqs(), packages=find_packages(), package_data={"": ["openapi/openapi.yaml"]}, cmdclass={ diff --git a/flex/interactive/docker/Makefile b/flex/interactive/docker/Makefile index 50cadecf1b87..2bdf64e48cb0 100644 --- a/flex/interactive/docker/Makefile +++ b/flex/interactive/docker/Makefile @@ -1,6 +1,7 @@ MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) WORKING_DIR := $(dir $(MKFILE_PATH)) -DOCKERFILES_DIR := $(WORKING_DIR) +DOCKERFILES_DIR := $(WORKING_DIR) +SHORT_SHA := $(shell git rev-parse --short HEAD) ifeq ($(REGISTRY),) REGISTRY := registry.cn-hongkong.aliyuncs.com @@ -11,7 +12,8 @@ ARCH := $(shell uname -m) VERSION ?= latest -BUILD_PROGRESS ?= auto +BUILD_PROGRESS ?= auto +ENABLE_COORDINATOR ?= false .PHONY: all interactive-base interactive-runtime @@ -23,9 +25,10 @@ interactive-base: interactive-runtime: cd $(WORKING_DIR) && \ + cd ../../../ && \ docker build \ - -f interactive-runtime.Dockerfile \ + -f ${WORKING_DIR}/interactive-runtime.Dockerfile \ --target final_image \ --build-arg ARCH=$(ARCH) \ - -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${VERSION}-${ARCH} --push . - + --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ + -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${SHORT_SHA}-${ARCH} --push . diff --git a/flex/interactive/docker/interactive-base.Dockerfile b/flex/interactive/docker/interactive-base.Dockerfile index 53baac903723..5c94f7da30ea 100644 --- a/flex/interactive/docker/interactive-base.Dockerfile +++ b/flex/interactive/docker/interactive-base.Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:20.04 -ARG CI=false +ENV DEBIAN_FRONTEND=noninteractive + # change bash as default SHELL ["/bin/bash", "-c"] diff --git a/flex/interactive/docker/interactive-runtime.Dockerfile b/flex/interactive/docker/interactive-runtime.Dockerfile index 516c9cec382a..693e3781524c 100644 --- a/flex/interactive/docker/interactive-runtime.Dockerfile +++ b/flex/interactive/docker/interactive-runtime.Dockerfile @@ -1,8 +1,10 @@ ARG ARCH=x86_64 FROM registry.cn-hongkong.aliyuncs.com/graphscope/interactive-base:latest AS builder -ARG CI=false ARG ARCH +ARG ENABLE_COORDINATOR="false" + +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope # change bash as default SHELL ["/bin/bash", "-c"] @@ -21,7 +23,7 @@ RUN curl -sf -L https://static.rust-lang.org/rustup.sh | \ cargo --version # install flex -RUN . ${HOME}/.cargo/env && cd ${HOME} && git clone https://github.com/alibaba/GraphScope.git -b main --single-branch && cd GraphScope/flex && \ +RUN . ${HOME}/.cargo/env && cd ${HOME}/GraphScope/flex && \ git submodule update --init && mkdir build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=/opt/flex -DBUILD_DOC=OFF && make -j && make install && \ cd ~/GraphScope/interactive_engine/ && mvn clean package -Pexperimental -DskipTests && \ cd ~/GraphScope/interactive_engine/compiler && cp target/compiler-0.0.1-SNAPSHOT.jar /opt/flex/lib/ && \ @@ -29,8 +31,18 @@ RUN . ${HOME}/.cargo/env && cd ${HOME} && git clone https://github.com/alibaba/ ls ~/GraphScope/interactive_engine/executor/ir && \ cp ~/GraphScope/interactive_engine/executor/ir/target/release/libir_core.so /opt/flex/lib/ +# build coordinator +RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ + cd ${HOME}/GraphScope/flex/coordinator && \ + python3 setup.py bdist_wheel && \ + mkdir -p /opt/flex/wheel && cp dist/*.whl /opt/flex/wheel/; \ + fi + from ubuntu:20.04 as final_image ARG ARCH +ARG ENABLE_COORDINATOR="false" + +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y sudo @@ -41,6 +53,13 @@ RUN useradd -m graphscope -u 1001 && \ # g++ + jre 500MB RUN apt-get update && apt-get -y install locales g++-9 cmake openjdk-11-jre-headless && \ ln -sf /usr/bin/g++-9 /usr/bin/g++ && locale-gen en_US.UTF-8 && apt-get clean -y && sudo rm -rf /var/lib/apt/lists/* + +# python3 +RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ + apt-get update && apt-get -y install python3 python3-pip && \ + apt-get clean -y && sudo rm -rf /var/lib/apt/lists/*; \ + fi + ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 @@ -54,7 +73,7 @@ COPY --from=builder /opt/flex /opt/flex # copy the builtin graph, modern_graph RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/ -COPY --from=builder ${HOME}/GraphScope/flex/interactive/examples/modern_graph/* /opt/flex/share/gs_interactive_default_graph/ +COPY --from=builder /home/graphscope/GraphScope/flex/interactive/examples/modern_graph/* /opt/flex/share/gs_interactive_default_graph/ # remove bin/run_app RUN rm -rf /opt/flex/bin/run_app @@ -71,7 +90,6 @@ COPY --from=builder /usr/lib/$ARCH-linux-gnu/libcrypto*.so* /usr/lib/$ARCH-linux COPY --from=builder /usr/lib/$ARCH-linux-gnu/libopen-rte*.so* /usr/lib/$ARCH-linux-gnu/ COPY --from=builder /usr/lib/$ARCH-linux-gnu/libhwloc*.so* /usr/lib/$ARCH-linux-gnu/ - # libunwind for arm64 seems not installed here, and seems not needed for aarch64(tested) COPY --from=builder /usr/lib/$ARCH-linux-gnu/libunwind*.so* /usr/lib/$ARCH-linux-gnu/ COPY --from=builder /usr/lib/$ARCH-linux-gnu/libarrow.so* /usr/lib/$ARCH-linux-gnu/ @@ -101,5 +119,15 @@ RUN sudo ln -sf /opt/flex/bin/* /usr/local/bin/ \ RUN chmod +x /opt/flex/bin/* +RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ + pip3 install /opt/flex/wheel/*.whl; \ + fi + ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/flex/lib/:/usr/lib/:/usr/local/lib/ +# flex solution +ENV SOLUTION=INTERACTIVE +# set home to graphscope user +ENV HOME=/home/graphscope +USER graphscope +WORKDIR /home/graphscope diff --git a/flex/openapi/openapi_coordinator.yaml b/flex/openapi/openapi_coordinator.yaml index 4774da856c17..00d3eef8e9d6 100644 --- a/flex/openapi/openapi_coordinator.yaml +++ b/flex/openapi/openapi_coordinator.yaml @@ -287,11 +287,60 @@ paths: application/json: schema: $ref: '#/components/schemas/GrootSchema' - /api/v1/interactive/graph/{graph_name}/dataloading: + /api/v1/job: + get: + tags: + - job + operationId: list_jobs + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/JobStatus' + /api/v1/job/{job_id}: + get: + tags: + - job + operationId: get_job_by_id + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/JobStatus' + delete: + tags: + - job + operationId: delete_job_by_id + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/ApiResponse' + /api/v1/graph/{graph_name}/dataloading: post: tags: - - legacy - operationId: data_import + - job + operationId: create_dataloading_job parameters: - name: graph_name in: path @@ -1225,6 +1274,32 @@ components: property: type: string description: must align with the schema + JobStatus: + type: object + x-body-name: job_status + properties: + job_id: + type: string + type: + type: string + status: + type: string + enum: + - RUNNING + - SUCCESS + - FAILED + - CANCELLED + - WAITING + start_time: + type: string + end_time: + type: string + log: + type: string + description: URL or log string + detail: + type: object + additionalProperties: true GrootGraph: x-body-name: groot_graph type: object diff --git a/flex/openapi/openapi_interactive.yaml b/flex/openapi/openapi_interactive.yaml index 471ef5e65c13..39c4e5af0464 100644 --- a/flex/openapi/openapi_interactive.yaml +++ b/flex/openapi/openapi_interactive.yaml @@ -99,8 +99,8 @@ paths: /v1/graph/{graph_name}/dataloading: post: tags: - - dataloading - description: Create a dataloading job (Synchronous) + - job + description: Create a dataloading job operationId: create_dataloading_job parameters: - name: graph_name @@ -117,10 +117,59 @@ paths: responses: '200': description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/JobResponse' + /v1/job/{job_id}: + get: + tags: + - job + operationId: get_job_by_id + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/JobStatus' + delete: + tags: + - job + operationId: delete_job_by_id + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Successful operation content: application/json: schema: $ref: '#/components/schemas/ApiResponse' + /v1/job: + get: + tags: + - job + operationId: list_jobs + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/JobStatus' /v1/graph/{graph_name}/procedure: post: tags: @@ -299,22 +348,6 @@ paths: application/json: schema: $ref: '#/components/schemas/ServiceStatus' - /v1/node/status: - get: - tags: - - node - summary: Get node status - description: Get node status - operationId: get_node_status - responses: - '200': - description: successful operation - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/NodeStatus' components: schemas: Graph: @@ -602,20 +635,39 @@ components: hqps_port: type: integer format: int32 - NodeStatus: - x-body-name: node_status + JobResponse: type: object + x-body-name: job_response properties: - node: + job_id: type: string - cpu_usage: - type: integer - format: int32 - memory_usage: + JobStatus: + type: object + x-body-name: job_status + properties: + job_id: + type: string + type: + type: string + status: + type: string + enum: + - RUNNING + - SUCCESS + - FAILED + - CANCELLED + - WAITING + start_time: type: integer format: int32 - disk_usage: + end_time: type: integer format: int32 + log: + type: string + description: URL or log string + detail: + type: object + additionalProperties: true ApiResponse: type: string diff --git a/python/graphscope/flex/rest/__init__.py b/python/graphscope/flex/rest/__init__.py index 87b370448ebc..33f5ad9cc0d0 100644 --- a/python/graphscope/flex/rest/__init__.py +++ b/python/graphscope/flex/rest/__init__.py @@ -22,6 +22,7 @@ from graphscope.flex.rest.api.connection_api import ConnectionApi from graphscope.flex.rest.api.deployment_api import DeploymentApi from graphscope.flex.rest.api.graph_api import GraphApi +from graphscope.flex.rest.api.job_api import JobApi from graphscope.flex.rest.api.legacy_api import LegacyApi from graphscope.flex.rest.api.procedure_api import ProcedureApi from graphscope.flex.rest.api.service_api import ServiceApi @@ -63,6 +64,7 @@ from graphscope.flex.rest.models.groot_property import GrootProperty from graphscope.flex.rest.models.groot_schema import GrootSchema from graphscope.flex.rest.models.groot_vertex_type import GrootVertexType +from graphscope.flex.rest.models.job_status import JobStatus from graphscope.flex.rest.models.model_property import ModelProperty from graphscope.flex.rest.models.model_schema import ModelSchema from graphscope.flex.rest.models.node_status import NodeStatus diff --git a/python/graphscope/flex/rest/api/__init__.py b/python/graphscope/flex/rest/api/__init__.py index 3b4b92b64cec..80ff0415eb1d 100644 --- a/python/graphscope/flex/rest/api/__init__.py +++ b/python/graphscope/flex/rest/api/__init__.py @@ -5,6 +5,7 @@ from graphscope.flex.rest.api.connection_api import ConnectionApi from graphscope.flex.rest.api.deployment_api import DeploymentApi from graphscope.flex.rest.api.graph_api import GraphApi +from graphscope.flex.rest.api.job_api import JobApi from graphscope.flex.rest.api.legacy_api import LegacyApi from graphscope.flex.rest.api.procedure_api import ProcedureApi from graphscope.flex.rest.api.service_api import ServiceApi diff --git a/python/graphscope/flex/rest/api/job_api.py b/python/graphscope/flex/rest/api/job_api.py new file mode 100644 index 000000000000..4e7ae5b6fc6e --- /dev/null +++ b/python/graphscope/flex/rest/api/job_api.py @@ -0,0 +1,1078 @@ +# coding: utf-8 + +""" + GraphScope FLEX HTTP SERVICE API + + This is a specification for GraphScope FLEX HTTP service based on the OpenAPI 3.0 specification. You can find out more details about specification at [doc](https://swagger.io/specification/v3/). Some useful links: - [GraphScope Repository](https://github.com/alibaba/GraphScope) - [The Source API definition for GraphScope Interactive](https://github.com/GraphScope/portal/tree/main/httpservice) + + The version of the OpenAPI document: 0.9.1 + Contact: graphscope@alibaba-inc.com + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import io +import warnings + +from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt +from typing import Dict, List, Optional, Tuple, Union, Any + +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + +from pydantic import StrictStr + +from typing import List + +from graphscope.flex.rest.models.job_status import JobStatus +from graphscope.flex.rest.models.schema_mapping import SchemaMapping + +from graphscope.flex.rest.api_client import ApiClient +from graphscope.flex.rest.api_response import ApiResponse +from graphscope.flex.rest.rest import RESTResponseType + + +class JobApi: + """NOTE: This class is auto generated by OpenAPI Generator + Ref: https://openapi-generator.tech + + Do not edit the class manually. + """ + + def __init__(self, api_client=None) -> None: + if api_client is None: + api_client = ApiClient.get_default() + self.api_client = api_client + + + @validate_call + def create_dataloading_job( + self, + graph_name: StrictStr, + schema_mapping: SchemaMapping, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> str: + """create_dataloading_job + + + :param graph_name: (required) + :type graph_name: str + :param schema_mapping: (required) + :type schema_mapping: SchemaMapping + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._create_dataloading_job_serialize( + graph_name=graph_name, + schema_mapping=schema_mapping, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def create_dataloading_job_with_http_info( + self, + graph_name: StrictStr, + schema_mapping: SchemaMapping, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[str]: + """create_dataloading_job + + + :param graph_name: (required) + :type graph_name: str + :param schema_mapping: (required) + :type schema_mapping: SchemaMapping + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._create_dataloading_job_serialize( + graph_name=graph_name, + schema_mapping=schema_mapping, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def create_dataloading_job_without_preload_content( + self, + graph_name: StrictStr, + schema_mapping: SchemaMapping, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """create_dataloading_job + + + :param graph_name: (required) + :type graph_name: str + :param schema_mapping: (required) + :type schema_mapping: SchemaMapping + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._create_dataloading_job_serialize( + graph_name=graph_name, + schema_mapping=schema_mapping, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _create_dataloading_job_serialize( + self, + graph_name, + schema_mapping, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + if graph_name is not None: + _path_params['graph_name'] = graph_name + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + if schema_mapping is not None: + _body_params = schema_mapping + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + # set the HTTP header `Content-Type` + if _content_type: + _header_params['Content-Type'] = _content_type + else: + _default_content_type = ( + self.api_client.select_header_content_type( + [ + 'application/json' + ] + ) + ) + if _default_content_type is not None: + _header_params['Content-Type'] = _default_content_type + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='POST', + resource_path='/api/v1/graph/{graph_name}/dataloading', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + + + + @validate_call + def delete_job_by_id( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> str: + """delete_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._delete_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def delete_job_by_id_with_http_info( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[str]: + """delete_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._delete_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def delete_job_by_id_without_preload_content( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """delete_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._delete_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "str", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _delete_job_by_id_serialize( + self, + job_id, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + if job_id is not None: + _path_params['job_id'] = job_id + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='DELETE', + resource_path='/api/v1/job/{job_id}', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + + + + @validate_call + def get_job_by_id( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> JobStatus: + """get_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._get_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobStatus", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def get_job_by_id_with_http_info( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[JobStatus]: + """get_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._get_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobStatus", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def get_job_by_id_without_preload_content( + self, + job_id: StrictStr, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """get_job_by_id + + + :param job_id: (required) + :type job_id: str + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._get_job_by_id_serialize( + job_id=job_id, + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "JobStatus", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _get_job_by_id_serialize( + self, + job_id, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + if job_id is not None: + _path_params['job_id'] = job_id + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='GET', + resource_path='/api/v1/job/{job_id}', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + + + + @validate_call + def list_jobs( + self, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> List[JobStatus]: + """list_jobs + + + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._list_jobs_serialize( + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "List[JobStatus]", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ).data + + + @validate_call + def list_jobs_with_http_info( + self, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> ApiResponse[List[JobStatus]]: + """list_jobs + + + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._list_jobs_serialize( + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "List[JobStatus]", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + response_data.read() + return self.api_client.response_deserialize( + response_data=response_data, + response_types_map=_response_types_map, + ) + + + @validate_call + def list_jobs_without_preload_content( + self, + _request_timeout: Union[ + None, + Annotated[StrictFloat, Field(gt=0)], + Tuple[ + Annotated[StrictFloat, Field(gt=0)], + Annotated[StrictFloat, Field(gt=0)] + ] + ] = None, + _request_auth: Optional[Dict[StrictStr, Any]] = None, + _content_type: Optional[StrictStr] = None, + _headers: Optional[Dict[StrictStr, Any]] = None, + _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, + ) -> RESTResponseType: + """list_jobs + + + :param _request_timeout: timeout setting for this request. If one + number provided, it will be total request + timeout. It can also be a pair (tuple) of + (connection, read) timeouts. + :type _request_timeout: int, tuple(int, int), optional + :param _request_auth: set to override the auth_settings for an a single + request; this effectively ignores the + authentication in the spec for a single request. + :type _request_auth: dict, optional + :param _content_type: force content-type for the request. + :type _content_type: str, Optional + :param _headers: set to override the headers for a single + request; this effectively ignores the headers + in the spec for a single request. + :type _headers: dict, optional + :param _host_index: set to override the host_index for a single + request; this effectively ignores the host_index + in the spec for a single request. + :type _host_index: int, optional + :return: Returns the result object. + """ # noqa: E501 + + _param = self._list_jobs_serialize( + _request_auth=_request_auth, + _content_type=_content_type, + _headers=_headers, + _host_index=_host_index + ) + + _response_types_map: Dict[str, Optional[str]] = { + '200': "List[JobStatus]", + } + response_data = self.api_client.call_api( + *_param, + _request_timeout=_request_timeout + ) + return response_data.response + + + def _list_jobs_serialize( + self, + _request_auth, + _content_type, + _headers, + _host_index, + ) -> Tuple: + + _host = None + + _collection_formats: Dict[str, str] = { + } + + _path_params: Dict[str, str] = {} + _query_params: List[Tuple[str, str]] = [] + _header_params: Dict[str, Optional[str]] = _headers or {} + _form_params: List[Tuple[str, str]] = [] + _files: Dict[str, str] = {} + _body_params: Optional[bytes] = None + + # process the path parameters + # process the query parameters + # process the header parameters + # process the form parameters + # process the body parameter + + + # set the HTTP header `Accept` + _header_params['Accept'] = self.api_client.select_header_accept( + [ + 'application/json' + ] + ) + + + # authentication setting + _auth_settings: List[str] = [ + ] + + return self.api_client.param_serialize( + method='GET', + resource_path='/api/v1/job', + path_params=_path_params, + query_params=_query_params, + header_params=_header_params, + body=_body_params, + post_params=_form_params, + files=_files, + auth_settings=_auth_settings, + collection_formats=_collection_formats, + _host=_host, + _request_auth=_request_auth + ) + + diff --git a/python/graphscope/flex/rest/api/legacy_api.py b/python/graphscope/flex/rest/api/legacy_api.py index e78c56b2ecc9..2ec721504cbc 100644 --- a/python/graphscope/flex/rest/api/legacy_api.py +++ b/python/graphscope/flex/rest/api/legacy_api.py @@ -30,7 +30,6 @@ from graphscope.flex.rest.models.groot_graph import GrootGraph from graphscope.flex.rest.models.groot_schema import GrootSchema -from graphscope.flex.rest.models.schema_mapping import SchemaMapping from graphscope.flex.rest.api_client import ApiClient from graphscope.flex.rest.api_response import ApiResponse @@ -50,288 +49,6 @@ def __init__(self, api_client=None) -> None: self.api_client = api_client - @validate_call - def data_import( - self, - graph_name: StrictStr, - schema_mapping: SchemaMapping, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> str: - """data_import - - - :param graph_name: (required) - :type graph_name: str - :param schema_mapping: (required) - :type schema_mapping: SchemaMapping - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._data_import_serialize( - graph_name=graph_name, - schema_mapping=schema_mapping, - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "str", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - response_data.read() - return self.api_client.response_deserialize( - response_data=response_data, - response_types_map=_response_types_map, - ).data - - - @validate_call - def data_import_with_http_info( - self, - graph_name: StrictStr, - schema_mapping: SchemaMapping, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> ApiResponse[str]: - """data_import - - - :param graph_name: (required) - :type graph_name: str - :param schema_mapping: (required) - :type schema_mapping: SchemaMapping - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._data_import_serialize( - graph_name=graph_name, - schema_mapping=schema_mapping, - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "str", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - response_data.read() - return self.api_client.response_deserialize( - response_data=response_data, - response_types_map=_response_types_map, - ) - - - @validate_call - def data_import_without_preload_content( - self, - graph_name: StrictStr, - schema_mapping: SchemaMapping, - _request_timeout: Union[ - None, - Annotated[StrictFloat, Field(gt=0)], - Tuple[ - Annotated[StrictFloat, Field(gt=0)], - Annotated[StrictFloat, Field(gt=0)] - ] - ] = None, - _request_auth: Optional[Dict[StrictStr, Any]] = None, - _content_type: Optional[StrictStr] = None, - _headers: Optional[Dict[StrictStr, Any]] = None, - _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0, - ) -> RESTResponseType: - """data_import - - - :param graph_name: (required) - :type graph_name: str - :param schema_mapping: (required) - :type schema_mapping: SchemaMapping - :param _request_timeout: timeout setting for this request. If one - number provided, it will be total request - timeout. It can also be a pair (tuple) of - (connection, read) timeouts. - :type _request_timeout: int, tuple(int, int), optional - :param _request_auth: set to override the auth_settings for an a single - request; this effectively ignores the - authentication in the spec for a single request. - :type _request_auth: dict, optional - :param _content_type: force content-type for the request. - :type _content_type: str, Optional - :param _headers: set to override the headers for a single - request; this effectively ignores the headers - in the spec for a single request. - :type _headers: dict, optional - :param _host_index: set to override the host_index for a single - request; this effectively ignores the host_index - in the spec for a single request. - :type _host_index: int, optional - :return: Returns the result object. - """ # noqa: E501 - - _param = self._data_import_serialize( - graph_name=graph_name, - schema_mapping=schema_mapping, - _request_auth=_request_auth, - _content_type=_content_type, - _headers=_headers, - _host_index=_host_index - ) - - _response_types_map: Dict[str, Optional[str]] = { - '200': "str", - } - response_data = self.api_client.call_api( - *_param, - _request_timeout=_request_timeout - ) - return response_data.response - - - def _data_import_serialize( - self, - graph_name, - schema_mapping, - _request_auth, - _content_type, - _headers, - _host_index, - ) -> Tuple: - - _host = None - - _collection_formats: Dict[str, str] = { - } - - _path_params: Dict[str, str] = {} - _query_params: List[Tuple[str, str]] = [] - _header_params: Dict[str, Optional[str]] = _headers or {} - _form_params: List[Tuple[str, str]] = [] - _files: Dict[str, str] = {} - _body_params: Optional[bytes] = None - - # process the path parameters - if graph_name is not None: - _path_params['graph_name'] = graph_name - # process the query parameters - # process the header parameters - # process the form parameters - # process the body parameter - if schema_mapping is not None: - _body_params = schema_mapping - - - # set the HTTP header `Accept` - _header_params['Accept'] = self.api_client.select_header_accept( - [ - 'application/json' - ] - ) - - # set the HTTP header `Content-Type` - if _content_type: - _header_params['Content-Type'] = _content_type - else: - _default_content_type = ( - self.api_client.select_header_content_type( - [ - 'application/json' - ] - ) - ) - if _default_content_type is not None: - _header_params['Content-Type'] = _default_content_type - - # authentication setting - _auth_settings: List[str] = [ - ] - - return self.api_client.param_serialize( - method='POST', - resource_path='/api/v1/interactive/graph/{graph_name}/dataloading', - path_params=_path_params, - query_params=_query_params, - header_params=_header_params, - body=_body_params, - post_params=_form_params, - files=_files, - auth_settings=_auth_settings, - collection_formats=_collection_formats, - _host=_host, - _request_auth=_request_auth - ) - - - - @validate_call def get_groot_schema( self, diff --git a/python/graphscope/flex/rest/models/__init__.py b/python/graphscope/flex/rest/models/__init__.py index 02524abb807b..7e9609b5acf0 100644 --- a/python/graphscope/flex/rest/models/__init__.py +++ b/python/graphscope/flex/rest/models/__init__.py @@ -40,6 +40,7 @@ from graphscope.flex.rest.models.groot_property import GrootProperty from graphscope.flex.rest.models.groot_schema import GrootSchema from graphscope.flex.rest.models.groot_vertex_type import GrootVertexType +from graphscope.flex.rest.models.job_status import JobStatus from graphscope.flex.rest.models.model_property import ModelProperty from graphscope.flex.rest.models.model_schema import ModelSchema from graphscope.flex.rest.models.node_status import NodeStatus diff --git a/python/graphscope/flex/rest/models/job_status.py b/python/graphscope/flex/rest/models/job_status.py new file mode 100644 index 000000000000..4071df51c975 --- /dev/null +++ b/python/graphscope/flex/rest/models/job_status.py @@ -0,0 +1,111 @@ +# coding: utf-8 + +""" + GraphScope FLEX HTTP SERVICE API + + This is a specification for GraphScope FLEX HTTP service based on the OpenAPI 3.0 specification. You can find out more details about specification at [doc](https://swagger.io/specification/v3/). Some useful links: - [GraphScope Repository](https://github.com/alibaba/GraphScope) - [The Source API definition for GraphScope Interactive](https://github.com/GraphScope/portal/tree/main/httpservice) + + The version of the OpenAPI document: 0.9.1 + Contact: graphscope@alibaba-inc.com + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + + +from typing import Any, ClassVar, Dict, List, Optional +from pydantic import BaseModel, StrictStr, field_validator +from pydantic import Field +try: + from typing import Self +except ImportError: + from typing_extensions import Self + +class JobStatus(BaseModel): + """ + JobStatus + """ # noqa: E501 + job_id: Optional[StrictStr] = None + type: Optional[StrictStr] = None + status: Optional[StrictStr] = None + start_time: Optional[StrictStr] = None + end_time: Optional[StrictStr] = None + log: Optional[StrictStr] = Field(default=None, description="URL or log string") + detail: Optional[Dict[str, Any]] = None + __properties: ClassVar[List[str]] = ["job_id", "type", "status", "start_time", "end_time", "log", "detail"] + + @field_validator('status') + def status_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in ('RUNNING', 'SUCCESS', 'FAILED', 'CANCELLED', 'WAITING'): + raise ValueError("must be one of enum values ('RUNNING', 'SUCCESS', 'FAILED', 'CANCELLED', 'WAITING')") + return value + + model_config = { + "populate_by_name": True, + "validate_assignment": True, + "protected_namespaces": (), + } + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of JobStatus from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + _dict = self.model_dump( + by_alias=True, + exclude={ + }, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Dict) -> Self: + """Create an instance of JobStatus from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "job_id": obj.get("job_id"), + "type": obj.get("type"), + "status": obj.get("status"), + "start_time": obj.get("start_time"), + "end_time": obj.get("end_time"), + "log": obj.get("log"), + "detail": obj.get("detail") + }) + return _obj + + diff --git a/python/graphscope/gsctl/commands/common.py b/python/graphscope/gsctl/commands/common.py index 35adae39ab84..f5540257d49a 100644 --- a/python/graphscope/gsctl/commands/common.py +++ b/python/graphscope/gsctl/commands/common.py @@ -49,7 +49,7 @@ def connect(coordinator_endpoint): if context is None: click.secho( "No available context found, try to connect to coordinator with --coordinator-endpoint", - fg="blug", + fg="blue", ) return coordinator_endpoint = context.coordinator_endpoint diff --git a/python/graphscope/gsctl/commands/dev.py b/python/graphscope/gsctl/commands/dev.py index 8df7e70b59ae..a03377887043 100644 --- a/python/graphscope/gsctl/commands/dev.py +++ b/python/graphscope/gsctl/commands/dev.py @@ -29,6 +29,13 @@ make_script = os.path.join(scripts_dir, "make_command.sh") make_image_script = os.path.join(scripts_dir, "make_image_command.sh") test_script = os.path.join(scripts_dir, "test_command.sh") +default_graphscope_repo_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "..", + "..", + "..", + "..", +) def run_shell_cmd(cmd, workingdir): @@ -47,6 +54,43 @@ def cli(): pass +@cli.group() +def flexbuild(): + """Build a customized stack using specific components.""" + pass + + +@flexbuild.command() +@click.option( + "--app", + type=click.Choice(["docker"]), + required=True, + help="Applicatin type of the built artifacts you want to build", +) +@click.option( + "--graphscope-repo", + required=False, + help="GraphScope code repo location.", +) +def interactive(app, graphscope_repo): + """Build Interactive for high throughput scenarios""" + if graphscope_repo is None: + graphscope_repo = default_graphscope_repo_path + interactive_build_dir = os.path.join( + graphscope_repo, "flex", "interactive", "docker" + ) + if not os.path.exists(interactive_build_dir) or not os.path.isdir( + interactive_build_dir + ): + click.secho( + f"No such file or directory {interactive_build_dir}, try --graphscope-repo param.", + fg="red", + ) + return + cmd = ["make", "interactive-runtime", "ENABLE_COORDINATOR=true"] + run_shell_cmd(cmd, os.path.join(graphscope_repo, interactive_build_dir)) + + @click.command() @click.argument( "type", diff --git a/python/graphscope/gsctl/commands/interactive.py b/python/graphscope/gsctl/commands/interactive.py index ba1f883c8444..9d5461095c02 100644 --- a/python/graphscope/gsctl/commands/interactive.py +++ b/python/graphscope/gsctl/commands/interactive.py @@ -20,22 +20,25 @@ import yaml from graphscope.gsctl.impl import connect_coordinator +from graphscope.gsctl.impl import create_dataloading_job from graphscope.gsctl.impl import create_graph from graphscope.gsctl.impl import create_procedure from graphscope.gsctl.impl import delete_alert_receiver_by_id from graphscope.gsctl.impl import delete_alert_rule_by_name from graphscope.gsctl.impl import delete_graph_by_name +from graphscope.gsctl.impl import delete_job_by_id from graphscope.gsctl.impl import delete_procedure_by_name from graphscope.gsctl.impl import disconnect_coordinator from graphscope.gsctl.impl import get_deployment_info +from graphscope.gsctl.impl import get_job_by_id from graphscope.gsctl.impl import get_node_status from graphscope.gsctl.impl import get_schema_by_name from graphscope.gsctl.impl import get_service_status -from graphscope.gsctl.impl import import_data_to_interactive_graph from graphscope.gsctl.impl import list_alert_messages from graphscope.gsctl.impl import list_alert_receivers from graphscope.gsctl.impl import list_alert_rules from graphscope.gsctl.impl import list_graphs +from graphscope.gsctl.impl import list_jobs from graphscope.gsctl.impl import list_procedures from graphscope.gsctl.impl import register_receiver from graphscope.gsctl.impl import restart_service @@ -103,27 +106,6 @@ def restart(): pass -@cli.command() -@click.option( - "-f", - "--filename", - required=True, - help="Path of yaml file to use to import data", -) -def dataimport(filename): - """Import data to Interactive graph (deprecated)""" - if not is_valid_file_path(filename): - click.secho("Invalid file: {0}".format(filename), fg="blue") - return - try: - config = read_yaml_file(filename) - import_data_to_interactive_graph(config) - except Exception as e: - click.secho(f"Failed to import data: {str(e)}", fg="red") - else: - click.secho("Import data successfully.", fg="green") - - @get.command() def graph(): """Display graphs in database""" @@ -211,6 +193,81 @@ def graph(graph_name): # noqa: F811 click.secho(f"Delete graph {graph_name} successfully.", fg="green") +@get.command() +def job(): + """Display jobs in database""" + + def _construct_and_display_data(jobs): + if not jobs: + click.secho("no job found in database.", fg="blue") + return + head = ["JOBID", "TYPE", "STATUS", "START_TIME", "END_TIME"] + data = [head] + for j in jobs: + data.append( + [ + j.job_id, + j.type, + j.status, + j.start_time, + str(j.end_time), + ] + ) + terminal_display(data) + + try: + jobs = list_jobs() + except Exception as e: + click.secho(f"Failed to list jobs: {str(e)}", fg="red") + else: + _construct_and_display_data(jobs) + + +@describe.command() +@click.argument("job_id", required=True) +def job(job_id): # noqa: F811 + """Show details of job""" + try: + job = get_job_by_id(job_id) + except Exception as e: + click.secho(f"Failed to get job: {str(e)}", fg="red") + else: + click.secho(yaml.dump(job.to_dict())) + + +@create.command() +@click.option( + "-f", + "--filename", + required=True, + help="Path of yaml file to use to create a graph", +) +def job(filename): # noqa: F811 + """Create a dataloading job in database""" + if not is_valid_file_path(filename): + click.secho("Invalid file: {0}".format(filename), fg="blue") + return + try: + config = read_yaml_file(filename) + jobid = create_dataloading_job(config["graph"], config) + except Exception as e: + click.secho(f"Failed to create a job: {str(e)}", fg="red") + else: + click.secho(f"Create job {jobid} successfully.", fg="green") + + +@delete.command() +@click.argument("job_id", required=True) +def job(job_id): # noqa: F811 + """Cancel a job by id in database""" + try: + delete_job_by_id(job_id) + except Exception as e: + click.secho(f"Failed to delete job {job_id}: {str(e)}", fg="red") + else: + click.secho(f"Delete job {job_id} successfully.", fg="green") + + @create.command() @click.option( "-g", diff --git a/python/graphscope/gsctl/impl/__init__.py b/python/graphscope/gsctl/impl/__init__.py index 662af6f8d4b5..3267a26b510d 100644 --- a/python/graphscope/gsctl/impl/__init__.py +++ b/python/graphscope/gsctl/impl/__init__.py @@ -33,7 +33,10 @@ from graphscope.gsctl.impl.graph import delete_graph_by_name from graphscope.gsctl.impl.graph import get_schema_by_name from graphscope.gsctl.impl.graph import list_graphs -from graphscope.gsctl.impl.legacy import import_data_to_interactive_graph +from graphscope.gsctl.impl.job import create_dataloading_job +from graphscope.gsctl.impl.job import delete_job_by_id +from graphscope.gsctl.impl.job import get_job_by_id +from graphscope.gsctl.impl.job import list_jobs from graphscope.gsctl.impl.procedure import create_procedure from graphscope.gsctl.impl.procedure import delete_procedure_by_name from graphscope.gsctl.impl.procedure import list_procedures diff --git a/python/graphscope/gsctl/impl/job.py b/python/graphscope/gsctl/impl/job.py new file mode 100644 index 000000000000..a4cede4583f6 --- /dev/null +++ b/python/graphscope/gsctl/impl/job.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2024 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List + +import graphscope.flex.rest +from graphscope.flex.rest import JobStatus +from graphscope.flex.rest import SchemaMapping +from graphscope.gsctl.config import get_current_context + + +def list_jobs() -> List[JobStatus]: + context = get_current_context() + with graphscope.flex.rest.ApiClient( + graphscope.flex.rest.Configuration(context.coordinator_endpoint) + ) as api_client: + api_instance = graphscope.flex.rest.JobApi(api_client) + jobs = api_instance.list_jobs() + return jobs + + +def get_job_by_id(job_id: str) -> JobStatus: + context = get_current_context() + with graphscope.flex.rest.ApiClient( + graphscope.flex.rest.Configuration(context.coordinator_endpoint) + ) as api_client: + api_instance = graphscope.flex.rest.JobApi(api_client) + job = api_instance.get_job_by_id(job_id) + return job + + +def create_dataloading_job(graph_name: str, job_config: dict) -> str: + context = get_current_context() + with graphscope.flex.rest.ApiClient( + graphscope.flex.rest.Configuration(context.coordinator_endpoint) + ) as api_client: + api_instance = graphscope.flex.rest.JobApi(api_client) + return api_instance.create_dataloading_job( + graph_name, SchemaMapping.from_dict(job_config) + ) + + +def delete_job_by_id(job_id: str) -> str: + context = get_current_context() + with graphscope.flex.rest.ApiClient( + graphscope.flex.rest.Configuration(context.coordinator_endpoint) + ) as api_client: + api_instance = graphscope.flex.rest.JobApi(api_client) + return api_instance.delete_job_by_id(job_id) diff --git a/python/graphscope/gsctl/impl/legacy.py b/python/graphscope/gsctl/impl/legacy.py index 0d1fe36b1c00..7df2b57425cf 100644 --- a/python/graphscope/gsctl/impl/legacy.py +++ b/python/graphscope/gsctl/impl/legacy.py @@ -17,15 +17,4 @@ # import graphscope.flex.rest -from graphscope.flex.rest import SchemaMapping from graphscope.gsctl.config import get_current_context - - -def import_data_to_interactive_graph(config: dict) -> str: - context = get_current_context() - with graphscope.flex.rest.ApiClient( - graphscope.flex.rest.Configuration(context.coordinator_endpoint) - ) as api_client: - graph_name = config["graph"] - api_instance = graphscope.flex.rest.LegacyApi(api_client) - return api_instance.data_import(graph_name, SchemaMapping.from_dict(config)) diff --git a/python/requirements.txt b/python/requirements.txt index aeecca8269b6..a3bc47bded4d 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -24,3 +24,4 @@ tqdm click vineyard>=0.16.3;sys_platform!="win32" simple-parsing +graphscope-flex>=0.27.0 diff --git a/python/setup_flex.py b/python/setup_flex.py index 0dc6ffb285b5..fb6733e989ae 100644 --- a/python/setup_flex.py +++ b/python/setup_flex.py @@ -28,10 +28,29 @@ Do not edit the class manually. """ # noqa: E501 +import os from setuptools import find_packages # noqa: H301 from setuptools import setup +pkg_root = os.path.dirname(os.path.abspath(__file__)) + + +def parse_version(root, **kwargs): + """ + Parse function for setuptools_scm that first tries to read '../VERSION' file + to get a version number. + """ + from setuptools_scm.git import parse + from setuptools_scm.version import meta + + version_file = os.path.join(pkg_root, "..", "VERSION") + if os.path.isfile(version_file): + with open(version_file, "r", encoding="utf-8") as fp: + return meta(fp.read().strip()) + return parse(root, **kwargs) + + # To install the library, run the following # # python setup.py install @@ -50,12 +69,15 @@ setup( name=NAME, - version=VERSION, description="GraphScope FLEX HTTP SERVICE API", author="GraphScope", author_email="graphscope@alibaba-inc.com", url="", keywords=["OpenAPI", "OpenAPI-Generator", "GraphScope FLEX HTTP SERVICE API"], + use_scm_version={ + "root": pkg_root, + "parse": parse_version, + }, install_requires=REQUIRES, packages=find_packages(include=["graphscope.flex.rest"]), include_package_data=True,