commit

src/fabric_cicd/_common/_fabric_endpoint.py

@@ -37,23 +37,24 @@ def invoke(self, method, url, body="{}", files=None):
         :param files: The file path to be included in the request. Defaults to None.
         :return: A dictionary containing the response headers, body, and status code.
         """
-        max_retries = 5
-        base_delay = 0.5
+        exit_loop = False
+        iteration_count = 0
         long_running = False
 
-        for attempt in range(max_retries):
+        while not exit_loop:
             try:
                 if files is None:
                     headers = {
                         "Authorization": f"Bearer {self.aad_token}",
                         "Content-Type": "application/json; charset=utf-8",
                     }
                     response = requests.request(method=method, url=url, headers=headers, json=body)
-
                 else:
                     headers = {"Authorization": f"Bearer {self.aad_token}"}
                     response = requests.request(method=method, url=url, headers=headers, files=files)
 
+                iteration_count += 1
+
                 invoke_log_message = _format_invoke_log(response, method, url, body)
 
                 # Handle long-running operations
@@ -68,31 +69,44 @@ def invoke(self, method, url, body="{}", files=None):
                         status = response_json.get("status")
                         if status == "Succeeded":
                             long_running = False
-                            break
-                        if status == "Failed":
+                            exit_loop = True
+                        elif status == "Failed":
                             response_error = response_json["error"]
-                            msg = f"Operation failed. Error Code: {response_error['errorCode']}. Error Message: {response_error['message']}"
+                            msg = (
+                                f"Operation failed. Error Code: {response_error['errorCode']}. "
+                                f"Error Message: {response_error['message']}"
+                            )
                             raise Exception(msg)
-                        if status == "Undefined":
+                        elif status == "Undefined":
                             msg = f"Operation is in an undefined state. Full Body: {response_json}"
                             raise Exception(msg)
-                        retry_after = float(response.headers.get("Retry-After", base_delay))
-                        delay = min(retry_after, base_delay * (2**attempt))
-                        logger.info(f"Operation in progress. Checking again in {delay} seconds.")
-                        time.sleep(delay)
+                        else:
+                            # Handle retry for long-running operations
+                            if "Retry-After" in response.headers:
+                                self._handle_retry(response, invoke_log_message, attempt=iteration_count - 1)
+                            else:
+                                retry_after = 0.5
+                                logger.info(f"Operation in progress. Checking again in {retry_after} seconds.")
+                                time.sleep(retry_after)
                     else:
                         time.sleep(1)
                         long_running = True
 
                 # Handle successful responses
                 elif response.status_code in {200, 201}:
-                    break
+                    exit_loop = True
 
                 # Handle API throttling
                 elif response.status_code == 429:
-                    retry_after = float(response.headers.get("Retry-After", 5)) + 5
-                    logger.info(f"API Overloaded: Retrying in {retry_after} seconds")
-                    time.sleep(retry_after)
+                    if "Retry-After" in response.headers:
+                        self._handle_retry(response, invoke_log_message, attempt=iteration_count - 1)
+                    else:
+                        retry_after = 5 + 5  # Default delay if Retry-After is not provided
+                        logger.info(f"API Overloaded: Retrying in {retry_after} seconds")
+                        time.sleep(retry_after)
+                        if iteration_count >= 5:
+                            msg = "Maximum retry attempts exceeded due to API throttling."
+                            raise InvokeError(msg, logger, invoke_log_message)
 
                 # Handle expired authentication token
                 elif (
@@ -113,34 +127,37 @@ def invoke(self, method, url, body="{}", files=None):
                     response.status_code == 400
                     and response.headers.get("x-ms-public-api-error-code") == "ItemDisplayNameAlreadyInUse"
                 ):
-                    if attempt < max_retries - 1:
+                    if iteration_count <= 6:
                         logger.info("Item name is reserved. Retrying in 60 seconds.")
                         time.sleep(60)
                     else:
-                        msg = f"Item name still in use after 5 attempts. Description: {response.reason}"
+                        msg = f"Item name still in use after 6 attempts. Description: {response.reason}"
                         raise Exception(msg)
 
                 # Handle scenario where library removed from environment before being removed from repo
                 elif response.status_code == 400 and "is not present in the environment." in response.json().get(
                     "message", "No message provided"
                 ):
-                    msg = f"Deployment attempted to remove a library that is not present in the environment. Description: {response.json().get('message')}"
+                    msg = (
+                        f"Deployment attempted to remove a library that is not present in the environment. "
+                        f"Description: {response.json().get('message')}"
+                    )
                     raise Exception(msg)
 
                 # Handle no environment libraries on GET request
                 elif (
                     response.status_code == 404
                     and response.headers.get("x-ms-public-api-error-code") == "EnvironmentLibrariesNotFound"
                 ):
-                    logger.info("Live environment doesnt have any libraries, continuing")
-                    break
+                    logger.info("Live environment doesn't have any libraries, continuing")
+                    exit_loop = True
 
                 # Handle unsupported principal type
                 elif (
                     response.status_code == 400
                     and response.headers.get("x-ms-public-api-error-code") == "PrincipalTypeNotSupported"
                 ):
-                    msg = f"The executing principal type is not supported to call {method} on '{url}'"
+                    msg = f"The executing principal type is not supported to call {method} on '{url}'."
                     raise Exception(msg)
 
                 # Handle unsupported item types
@@ -172,6 +189,27 @@ def invoke(self, method, url, body="{}", files=None):
             "status_code": response.status_code,
         }
 
+    def _handle_retry(self, response, invoke_log_message, attempt, max_retries=5, base_delay=0.5):
+        """
+        Handles retry logic with exponential backoff based on the response.
+
+        :param response: The HTTP response object.
+        :param invoke_log_message: Log message for the current invocation.
+        :param attempt: The current attempt number.
+        :param max_retries: Maximum number of retry attempts.
+        :param base_delay: Base delay in seconds for backoff.
+        :raises InvokeError: If maximum retries are exceeded.
+        """
+        if attempt < max_retries:
+            retry_after = float(response.headers.get("Retry-After", base_delay))
+            delay = min(retry_after, base_delay * (2**attempt))
+            logger.info(f"Retrying in {delay} seconds (Attempt {attempt + 1}/{max_retries})...")
+            time.sleep(delay)
+        else:
+            msg = f"Maximum retry attempts ({max_retries}) exceeded."
+            logger.debug(msg)
+            raise InvokeError(msg, logger, invoke_log_message)
+
     def _refresh_token(self):
         """Refreshes the AAD token if empty or expiration has passed"""
         if (