Tolerate missing X-RateLimit-Remaining for custom base URLs (GHE)

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-github',
-      version='2.0.15',
+      version='2.0.16',
       description='Singer.io tap for extracting data from the GitHub API',
       author='Stitch',
       url='http://singer.io',

tap_github/client.py

@@ -167,7 +167,7 @@ def calculate_seconds(epoch):
     current = time.time()
     return max(0, int(ceil(epoch - current)))
 
-def rate_throttling(response, max_sleep_seconds, min_remain_rate_limit):
+def rate_throttling(response, max_sleep_seconds, min_remain_rate_limit, base_url=DEFAULT_DOMAIN):
     """
     For rate limit errors, get the remaining time before retrying and calculate the time to sleep before making a new request.
     """
@@ -186,9 +186,10 @@ def rate_throttling(response, max_sleep_seconds, min_remain_rate_limit):
 
             LOGGER.info("API rate limit exceeded. Tap will retry the data collection after %s seconds.", seconds_to_sleep)
             time.sleep(seconds_to_sleep)
-    else:
-        # Raise an exception if `X-RateLimit-Remaining` is not found in the header.
-        # API does include this key header if provided base URL is not a valid github custom domain.
+    elif base_url == DEFAULT_DOMAIN:
+        # On github.com a missing `X-RateLimit-Remaining` header means the base URL is not a
+        # valid GitHub domain. GitHub Enterprise (a custom base_url) can omit the header even on
+        # a successful response, so its absence there is not an error -- just skip throttling.
         raise GithubException("The API call using the specified base url was unsuccessful. Please double-check the provided base URL.")
 
 class GithubClient:
@@ -246,7 +247,7 @@ def authed_get_single_page(self, source, url, headers={}, stream="", should_skip
             if resp.status_code != 200:
                 raise_for_error(resp, source, stream, self, should_skip_404)
             timer.tags[metrics.Tag.http_status_code] = resp.status_code
-            rate_throttling(resp, self.max_sleep_seconds, self.min_remain_rate_limit)
+            rate_throttling(resp, self.max_sleep_seconds, self.min_remain_rate_limit, self.base_url)
             if resp.status_code == 404 or resp.status_code == 422:
                 # Return an empty response body since we're not raising a NotFoundException
                 resp._content = b'{}'  # pylint: disable=protected-access

tests/unittests/test_rate_limit.py

@@ -96,3 +96,15 @@ def test_rate_limt_header_not_found(self, mocked_sleep):
 
         # Verifying the message formed for the invalid base URL
         self.assertEqual(str(e.exception), "The API call using the specified base url was unsuccessful. Please double-check the provided base URL.")
+
+    def test_rate_limit_header_not_found_custom_base_url(self, mocked_sleep):
+        """
+        Test that `rate_throttling` does not raise when `X-RateLimit-Remaining` is missing for a
+        custom base URL: GitHub Enterprise can omit the header even on a successful response.
+        """
+        resp = api_call()
+        resp.headers = {}
+
+        # Should neither raise nor sleep -- throttling is simply skipped.
+        rate_throttling(resp, DEFAULT_SLEEP_SECONDS, DEFAULT_MIN_REMAIN_RATE_LIMIT, base_url="https://github.example.com/api/v3")
+        self.assertFalse(mocked_sleep.called)