diff --git a/lib/kennel/api.rb b/lib/kennel/api.rb index 607dd341..4c4e264f 100644 --- a/lib/kennel/api.rb +++ b/lib/kennel/api.rb @@ -5,6 +5,8 @@ module Kennel class Api CACHE_FILE = ENV.fetch("KENNEL_API_CACHE_FILE", "tmp/cache/details") + RateLimitParams = Data.define(:limit, :period, :remaining, :reset, :name) + def self.tag(api_resource, reply) klass = Models::Record.api_resource_map[api_resource] return reply unless klass # do not blow up on unknown models @@ -122,6 +124,23 @@ def request(method, path, body: nil, params: {}, ignore_404: false) end end + rate_limit = RateLimitParams.new( + limit: response.headers["x-ratelimit-limit"], + period: response.headers["x-ratelimit-period"], + remaining: response.headers["x-ratelimit-remaining"], + reset: response.headers["x-ratelimit-reset"], + name: response.headers["x-ratelimit-name"] + ) + + if response.status == 429 + message = "Datadog rate limit #{rate_limit.name.inspect} hit" + message += " (#{rate_limit.limit} requests per #{rate_limit.period} seconds)" + message += "; sleeping #{rate_limit.reset} seconds before trying again" + Kennel.err.puts message + sleep rate_limit.reset.to_f + redo + end + break if i == tries - 1 || method != :get || response.status < 500 Kennel.err.puts "Retrying on server error #{response.status} for #{path}" end diff --git a/test/kennel/api_test.rb b/test/kennel/api_test.rb index 3c638b63..afbacbc1 100644 --- a/test/kennel/api_test.rb +++ b/test/kennel/api_test.rb @@ -255,6 +255,28 @@ def tracking(id) end end + describe "rate limiting" do + capture_all + + it "retries on a rate-limited response" do + request = stub_datadog_request(:get, "monitor/1234").to_return( + [ + { status: 429, headers: { + "X-RateLimit-Name": "too many secrets", + "X-RateLimit-Limit": "1000", + "X-RateLimit-Period": "60", + "X-RateLimit-Remaining": "-5", + "X-RateLimit-Reset": "1.1" + } }, + { status: 200, body: { foo: "bar" }.to_json } + ] + ) + api.show("monitor", 1234).must_equal(foo: "bar", klass: Kennel::Models::Monitor, tracking_id: nil) + assert_requested request, times: 2 + stderr.string.must_equal "Datadog rate limit \"too many secrets\" hit (1000 requests per 60 seconds); sleeping 1.1 seconds before trying again\n" + end + end + describe "retries" do capture_all