handle utils

Test-tag: pr CPUUsage DAOSVersion DmgStorageScanSCMTest Ecodtruncate NetworkFailureTest POSIXStatTest SSDSocketTest SuperBlockVersioning soak_smoke MacsioTest daos_racer OSAOnlineExtend OSAOnlineParallelTest DdbTest ContainerListConsolidationTest performance,-manual NvmeEnospace
Skip-unit-tests: true
Skip-fault-injection-test: true

Signed-off-by: Dalton Bohning <[email protected]>

src/tests/ftest/daos_racer/parallel.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 """
 (C) Copyright 2021-2022 Intel Corporation.
+(C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
 SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -52,7 +53,8 @@ def test_daos_racer_parallel(self):
             job_manager.run()
 
         except CommandFailure as error:
-            self.log.error("DAOS Racer Failed: %s", str(error))
-            self.fail("Test was expected to pass but it failed.\n")
+            msg = f"daos_racer failed: {error}"
+            self.log.error(msg)
+            self.fail(msg)
 
         self.log.info("Test passed!")

src/tests/ftest/nvme/enospace.py

@@ -340,7 +340,7 @@ def err_to_str(err_no):
             return pydaos.DaosErrorCode(err_no).name
 
         logfile_glob = log_file + r".*[0-9]"
-        errors_count = get_errors_count(self.hostlist_clients, logfile_glob)
+        errors_count = get_errors_count(self.log, self.hostlist_clients, logfile_glob)
         for error in self.expected_errors:
             if error not in errors_count:
                 errors_count[error] = 0

src/tests/ftest/recovery/container_list_consolidation.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -74,9 +75,8 @@ def test_orphan_container(self):
                 server_host=NodeSet(self.hostlist_servers[0]), path=self.bin,
                 mount_point=scm_mount, pool_uuid=pool.uuid, vos_file=vos_file)
             cmd_result = ddb_command.list_component()
-            ls_out = "\n".join(cmd_result[0]["stdout"])
             uuid_regex = r"([0-f]{8}-[0-f]{4}-[0-f]{4}-[0-f]{4}-[0-f]{12})"
-            match = re.search(uuid_regex, ls_out)
+            match = re.search(uuid_regex, cmd_result.joined_stdout)
             if match is None:
                 self.fail("Unexpected output from ddb command, unable to parse.")
             self.log.info("Container UUID from ddb ls = %s", match.group(1))
@@ -133,9 +133,8 @@ def test_orphan_container(self):
                    "(PMEM only).")
             self.log_step(msg)
             cmd_result = ddb_command.list_component()
-            ls_out = "\n".join(cmd_result[0]["stdout"])
             uuid_regex = r"([0-f]{8}-[0-f]{4}-[0-f]{4}-[0-f]{4}-[0-f]{12})"
-            match = re.search(uuid_regex, ls_out)
+            match = re.search(uuid_regex, cmd_result.joined_stdout)
             if match:
                 errors.append("Container UUID is found in shard! Checker didn't remove it.")
 

src/tests/ftest/recovery/ddb.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2022-2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -176,10 +177,9 @@ def test_recovery_ddb_ls(self):
         #   CONT: (/[0]) /3082b7d3-32f9-41ea-bcbf-5d6450c1b34f
         # stdout is a list which contains each line as separate element. Concatenate them
         # to single string so that we can apply regex.
-        ls_out = "\n".join(cmd_result[0]["stdout"])
         # Matches the container uuid
         uuid_regex = r"([0-f]{8}-[0-f]{4}-[0-f]{4}-[0-f]{4}-[0-f]{12})"
-        match = re.search(uuid_regex, ls_out)
+        match = re.search(uuid_regex, cmd_result.joined_stdout)
         if match is None:
             self.fail("Unexpected output from ddb command, unable to parse.")
         self.log.info("Container UUID from ddb ls = %s", match.group(1))
@@ -198,10 +198,9 @@ def test_recovery_ddb_ls(self):
         #   OBJ: (/[0]/[0]) /3082b7d3-32f9-41ea-bcbf-5d6450c1b34f/937030214649643008.1.0.1
         #   OBJ: (/[0]/[1]) /3082b7d3-32f9-41ea-bcbf-5d6450c1b34f/937030214649643009.1.0.1
         #   OBJ: (/[0]/[2]) /3082b7d3-32f9-41ea-bcbf-5d6450c1b34f/937030214649643016.1.0.1
-        ls_out = "\n".join(cmd_result[0]["stdout"])
         # Matches an object id. (4 digits separated by a period '.')
         object_id_regex = r"\d+\.\d+\.\d+\.\d+"
-        match = re.findall(object_id_regex, ls_out)
+        match = re.findall(object_id_regex, cmd_result.joined_stdout)
         self.log.info("List objects match = %s", match)
 
         actual_object_count = len(match)
@@ -217,12 +216,11 @@ def test_recovery_ddb_ls(self):
         for obj_index in range(object_count):
             component_path = "[0]/[{}]".format(obj_index)
             cmd_result = ddb_command.list_component(component_path=component_path)
-            ls_out = "\n".join(cmd_result[0]["stdout"])
             # Sample output.
             # /d4e0c836-17bd-4df3-b255-929732486bab/281479271677953.0.0/
             # [0] 'Sample dkey 0 0' (15)
             # [1] 'Sample dkey 0 1' (15)
-            match = re.findall(dkey_regex, ls_out)
+            match = re.findall(dkey_regex, cmd_result.joined_stdout)
 
             actual_dkey_count += len(match)
 
@@ -248,7 +246,7 @@ def test_recovery_ddb_ls(self):
             for dkey_index in range(dkey_count):
                 component_path = "[0]/[{}]/[{}]".format(obj_index, dkey_index)
                 cmd_result = ddb_command.list_component(component_path=component_path)
-                ls_out = "\n".join(cmd_result[0]["stdout"])
+                ls_out = cmd_result.joined_stdout
                 msg = "List akeys obj_index = {}, dkey_index = {}, stdout = {}".format(
                     obj_index, dkey_index, ls_out)
                 self.log.info(msg)
@@ -356,7 +354,7 @@ def test_recovery_ddb_rm(self):
 
         # 4. Call ddb rm to remove the akey.
         cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]/[0]")
-        self.log.info("rm akey stdout = %s", cmd_result[0]["stdout"])
+        self.log.info("rm akey stdout = %s", cmd_result.joined_stdout)
 
         # 5. Restart the server to use the API.
         dmg_command.system_start()
@@ -386,7 +384,7 @@ def test_recovery_ddb_rm(self):
 
         # 9. Call ddb rm to remove the dkey.
         cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]")
-        self.log.info("rm dkey stdout = %s", cmd_result[0]["stdout"])
+        self.log.info("rm dkey stdout = %s", cmd_result.joined_stdout)
 
         # 10. Restart the server to use the API.
         dmg_command.system_start()
@@ -415,7 +413,7 @@ def test_recovery_ddb_rm(self):
 
         # 14. Call ddb rm to remove the object.
         cmd_result = ddb_command.remove_component(component_path="[0]/[0]")
-        self.log.info("rm object stdout = %s", cmd_result[0]["stdout"])
+        self.log.info("rm object stdout = %s", cmd_result.joined_stdout)
 
         # 15. Restart the server to use daos command.
         dmg_command.system_start()

src/tests/ftest/util/agent_utils.py

@@ -1,10 +1,10 @@
 """
   (C) Copyright 2019-2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
 import os
-import re
 import socket
 
 from agent_utils_params import DaosAgentTransportCredentials, DaosAgentYamlParameters
@@ -13,7 +13,7 @@
 from command_utils_base import (CommandWithParameters, CommonConfig, EnvironmentVariables,
                                 FormattedParameter)
 from exception_utils import CommandFailure
-from general_utils import get_default_config_file, get_log_file, run_pcmd
+from general_utils import get_default_config_file, get_log_file
 from run_utils import run_remote
 
 
@@ -293,12 +293,17 @@ def start(self):
         super().start()
 
     def dump_attachinfo(self):
-        """Run dump-attachinfo on the daos_agent."""
+        """Run dump-attachinfo on the daos_agent.
+
+        Raises:
+            CommandFailure: if the daos_agent command fails.
+
+        Returns:
+            CommandResult: groups of command results from the same hosts with the same return status
+        """
         cmd = self.manager.job.copy()
         cmd.set_sub_command("dump-attachinfo")
-        self.attachinfo = run_pcmd(self.hosts,
-                                   str(self.manager.job))[0]["stdout"]
-        self.log.info("Agent attachinfo: %s", self.attachinfo)
+        return run_remote(self.log, self.hosts, cmd.with_exports)
 
     def support_collect_log(self, **kwargs):
         """Collect logs for debug purpose.
@@ -309,6 +314,7 @@ def support_collect_log(self, **kwargs):
             archive (bool, optional): Archive the log/config files
             extra_logs_dir (str, optional): Collect the Logs from given custom directory
             target-host (str, optional): R sync all the logs to target system
+
         Raises:
             CommandFailure: if the daos_agent command fails.
 
@@ -324,36 +330,6 @@ def support_collect_log(self, **kwargs):
         self.log.info("Support collect-log on clients: %s", str(cmd))
         return run_remote(self.log, self.hosts, cmd.with_exports)
 
-    def get_attachinfo_file(self):
-        """Run dump-attachinfo on the daos_agent.
-
-        Returns:
-            str: the attach info file path
-
-        """
-        server_name = self.get_config_value("name")
-
-        self.dump_attachinfo()
-
-        attach_info = self.attachinfo
-
-        # Filter log messages from attachinfo content
-        messages = [x for x in attach_info if re.match(r"^(name\s|size\s|all|\d+\s)", x)]
-        attach_info_contents = "\n".join(messages)
-        attach_info_filename = f"{server_name}.attach_info_tmp"
-
-        if len(messages) < 4:
-            self.log.info("Malformed attachinfo file: %s", attach_info_contents)
-            return None
-
-        # Write an attach_info_tmp file in this directory for cart_ctl to use
-        attachinfo_file_path = os.path.join(self.outputdir, attach_info_filename)
-
-        with open(attachinfo_file_path, 'w', encoding='utf-8') as file_handle:
-            file_handle.write(attach_info_contents)
-
-        return attachinfo_file_path
-
     def stop(self):
         """Stop the agent through the job manager.
 

src/tests/ftest/util/daos_racer_utils.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2020-2023 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -10,7 +11,8 @@
 from command_utils_base import BasicParameter, FormattedParameter
 from env_modules import load_mpi
 from exception_utils import CommandFailure, MPILoadError
-from general_utils import get_log_file, pcmd
+from general_utils import get_log_file
+from run_utils import run_remote
 
 
 class DaosRacerCommand(ExecutableCommand):
@@ -109,15 +111,14 @@ def run(self, raise_exception=None):
             str(self), self.host,
             "no" if self.clush_timeout.value is None else
             "a {}s".format(self.clush_timeout.value))
-        return_codes = pcmd(self.host, self.with_exports, True, self.clush_timeout.value)
-        if 0 not in return_codes or len(return_codes) > 1:
-            # Kill the daos_racer process if the remote command timed out
-            if 255 in return_codes:
-                self.log.info(
-                    "Stopping timed out daos_racer process on %s", self.host)
-                pcmd(self.host, "pkill daos_racer", True)
+        result = run_remote(
+            self.log, self.host, self.with_exports, timeout=self.clush_timeout.value)
+        if not result.passed:
+            if result.timeout:
+                self.log.info("Stopping timed out daos_racer process on %s", result.timeout_hosts)
+                run_remote(self.log, result.timeout_hosts, "pkill daos_racer", True)
 
             if raise_exception:
-                raise CommandFailure("Error running '{}'".format(self._command))
+                raise CommandFailure(f"Error running '{self._command}'")
 
         self.log.info("Test passed!")

src/tests/ftest/util/ddb_utils.py

@@ -1,12 +1,13 @@
 """
   (C) Copyright 2022 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
 import os
 
 from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter
-from general_utils import run_pcmd
+from run_utils import run_remote
 
 
 class DdbCommandBase(CommandWithParameters):
@@ -18,9 +19,9 @@ def __init__(self, server_host, path, verbose=True, timeout=None, sudo=True):
         Args:
             server_host (NodeSet): Server host to run the command.
             path (str): path to the ddb command.
-            verbose (bool, optional): Display command output when run_pcmd is called.
+            verbose (bool, optional): Display command output in run.
                 Defaults to True.
-            timeout (int, optional): Command timeout (sec) used in run_pcmd. Defaults to
+            timeout (int, optional): Command timeout (sec) used in run. Defaults to
                 None.
             sudo (bool, optional): Whether to run ddb with sudo. Defaults to True.
         """
@@ -40,7 +41,7 @@ def __init__(self, server_host, path, verbose=True, timeout=None, sudo=True):
         # VOS file path.
         self.vos_path = BasicParameter(None, position=1)
 
-        # Members needed for run_pcmd().
+        # Members needed for run().
         self.verbose = verbose
         self.timeout = timeout
 
@@ -60,13 +61,11 @@ def run(self):
         """Run the command.
 
         Returns:
-            list: A list of dictionaries with each entry containing output, exit status,
-                and interrupted status common to each group of hosts.
+            CommandResult: groups of command results from the same hosts with the same return status
 
         """
-        return run_pcmd(
-            hosts=self.host, command=str(self), verbose=self.verbose,
-            timeout=self.timeout)
+        return run_remote(
+            self.log, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout)
 
 
 class DdbCommand(DdbCommandBase):

src/tests/ftest/util/general_utils.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2018-2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -788,10 +789,11 @@ def get_remote_file_size(host, file_name):
     return int(result.stdout_text)
 
 
-def get_errors_count(hostlist, file_glob):
+def get_errors_count(log, hostlist, file_glob):
     """Count the number of errors found in log files.
 
     Args:
+        log (logger): logger for the messages produced by this method
         hostlist (list): System list to looks for an error.
         file_glob (str): Glob pattern of the log file to parse.
 
@@ -803,9 +805,9 @@ def get_errors_count(hostlist, file_glob):
     cmd = "cat {} | sed -n -E -e ".format(get_log_file(file_glob))
     cmd += r"'/^.+[[:space:]]ERR[[:space:]].+[[:space:]]DER_[^(]+\([^)]+\).+$/"
     cmd += r"s/^.+[[:space:]]DER_[^(]+\((-[[:digit:]]+)\).+$/\1/p'"
-    results = run_pcmd(hostlist, cmd, False, None, None)
+    result = run_remote(log, hostlist, cmd, verbose=False)
     errors_count = {}
-    for error_str in sum([result["stdout"] for result in results], []):
+    for error_str in sum([data.stdout for data in result.output], []):
         error = int(error_str)
         if error not in errors_count:
             errors_count[error] = 0