Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fdedup::transform() return 0 for success or error code #1041

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Fdedup::transform() return 0 for success or error code #1041

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions transforms/universal/fdedup/dpk_fdedup/transform_python.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@
import dpk_fdedup.data_cleaning.transform
import dpk_fdedup.get_duplicate_list.transform
import dpk_fdedup.signature_calc.transform
from data_processing.runtime.pure_python import PythonTransformLauncher
from data_processing.utils import ParamsUtils, get_logger, str2bool
from dpk_fdedup.cluster_analysis.transform_python import (
ClusterAnalysisPythonTransformConfiguration,
)
from dpk_fdedup.data_cleaning.transform_python import DataCleaningPythonTransformConfiguration
from data_processing.runtime.pure_python import PythonTransformLauncher
from data_processing.utils import ParamsUtils, get_logger, str2bool
from dpk_fdedup.data_cleaning.transform_python import (
DataCleaningPythonTransformConfiguration,
)
from dpk_fdedup.get_duplicate_list.transform_python import (
GetDuplicateListPythonTransformConfiguration,
)
Expand Down Expand Up @@ -59,8 +61,9 @@ def __init__(self, global_params: argparse.Namespace = None):
self.global_params = global_params
self.logger = get_logger(__name__)

def orchestrate(self):
def orchestrate(self) -> int:
service_list = self.global_params.services.split(",")
status = 0
for service in service_list:
self.logger.info(f"Starting {service} step")
if service not in SERVICE_DICT:
Expand All @@ -76,6 +79,7 @@ def orchestrate(self):
else:
self.logger.error(f"{service} failed with status {status}, aborting ...")
break
return status

def get_arguments(self, in_args: argparse.Namespace, service_name: str) -> list:
sys_argv = ["python"]
Expand Down Expand Up @@ -261,8 +265,6 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()




# Class used by the notebooks to ingest binary files and create parquet files
class Fdedup:
def __init__(self, **kwargs):
Expand Down