Add Logic to Handle if Key Encounters GitHub API Rate Limit

scripts/metricsLib/constants.py

@@ -7,6 +7,7 @@
 from enum import Enum
 
 TIMEOUT_IN_SECONDS = 120
+REQUEST_RETRIES = 5
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 # Folder Names to send over our projects tracked data
 PATH_TO_METRICS_DATA = (Path(__file__).parent /

scripts/metricsLib/metrics_data_structures.py

@@ -4,10 +4,11 @@
 import json
 from json.decoder import JSONDecodeError
 import datetime
+from time import sleep, mktime, gmtime, time, localtime
 from functools import reduce
 import operator
 import requests
-from metricsLib.constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT
+from metricsLib.constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT, REQUEST_RETRIES
 
 # Simple metric that can be represented by a count or value.
 
@@ -48,7 +49,7 @@
         before returning it.
     """
 
     def __init__(self, name, needed_params, endpoint_url, return_values, token=None, method='GET'):
         self.name = name
         self.return_values = return_values
         self.url = endpoint_url
@@ -75,25 +76,52 @@
             endpoint_to_hit = self.url.format(**params)
             request_params = None
 
-        if self.headers:
-            _args_ = (self.method, endpoint_to_hit)
-            _kwargs_ = {
-                "params": request_params,
-                "headers": self.headers,
-                "timeout": TIMEOUT_IN_SECONDS
-            }
-            response = requests.request(*_args_, **_kwargs_)
-        else:
-            response = requests.request(
-                self.method, endpoint_to_hit, params=request_params, timeout=TIMEOUT_IN_SECONDS)
-
-        try:
-            if response.status_code == 200:
-                response_json = json.loads(response.text)
+        attempts = 0
+
+        while attempts < REQUEST_RETRIES:
+            if self.headers:
+                _args_ = (self.method, endpoint_to_hit)
+                _kwargs_ = {
+                    "params": request_params,
+                    "headers": self.headers,
+                    "timeout": TIMEOUT_IN_SECONDS
+                }
+                response = requests.request(*_args_, **_kwargs_)
             else:
-                raise ConnectionError(f"Non valid status code {response.status_code}!")
-        except JSONDecodeError:
-            response_json = {}
+                response = requests.request(
+                    self.method, endpoint_to_hit, params=request_params, timeout=TIMEOUT_IN_SECONDS)
+
+            try:
+                if response.status_code == 200:
+                    response_json = json.loads(response.text)
+                    break
+                elif response.status_code in (403,429):
+                    #rate limit was triggered.
+                    wait_until = int(response.headers.get("x-ratelimit-reset"))
+                    wait_in_seconds = int(
+                        mktime(gmtime(wait_until)) -
+                        mktime(gmtime(time()))
+                    )
+                    wait_until_time = localtime(wait_until)
+
+                    print(f"Ran into rate limit sleeping for {self.name}!")
+                    print(
+                        f"sleeping until {wait_until_time.tm_hour}:{wait_until_time.tm_min} ({wait_in_seconds} seconds)"
+                    )
+                    sleep(wait_in_seconds)
+
+                    response_json = {}
+                    attempts += 1
+
+                    if attempts >= REQUEST_RETRIES:
+                        raise ConnectionError(
+                            f"Rate limit was reached and couldn't be rectified after {attempts} tries"
+                        )
+                else:
+                    raise ConnectionError(f"Non valid status code {response.status_code}!")
+            except JSONDecodeError:
+                response_json = {}
+                attempts += 1
 
         return response_json
 
@@ -143,7 +171,7 @@
         Fetch data and save it in desired path and format
     """
 
     def __init__(self, name, needed_params, url, fmt='png', token=None):
         super().__init__(name, needed_params, url, {}, token=token)
         self.format = fmt
 
@@ -224,7 +252,7 @@
     """
     # Return value is a dict of lists of strings that match to the keys of the dict.
 
     def __init__(self, name, needed_params, query, return_vals, token=None, url=GH_GQL_ENDPOINT):
         super().__init__(name, needed_params, url, return_vals, token=token)
         self.query = query
 
@@ -263,10 +291,10 @@
 
         if "data" not in response_json.keys():
             if "message" not in response_json.keys():
                 raise requests.exceptions.InvalidJSONError(
                     response_json['errors'][0]['message'])
 
             raise requests.exceptions.InvalidJSONError(
                 response_json['message'])
 
         # print(f"Response_JSON: {response_json}")
@@ -298,7 +326,7 @@
         before returning it.
     """
 
     def __init__(self, name, needed_params, endpoint_url, return_val, token=None, method='GET'):
         super().__init__(name, needed_params, endpoint_url,
                          return_val, token=token, method=method)