mirror of
https://akkoma.dev/AkkomaGang/akkoma.git
synced 2024-11-09 09:52:13 +00:00
Add ratelimit backoff to HTTP get
This commit is contained in:
parent
2437a3e9ba
commit
3c384c1b76
|
@ -9,14 +9,24 @@ defmodule Pleroma.HTTP.Backoff do
|
|||
# figure out from the 429 response when we can make the next request
|
||||
# mastodon uses the x-ratelimit-reset header, so we will use that!
|
||||
# other servers may not, so we'll default to 5 minutes from now if we can't find it
|
||||
default_5_minute_backoff =
|
||||
DateTime.utc_now()
|
||||
|> Timex.shift(seconds: 5 * 60)
|
||||
|
||||
case Enum.find_value(headers, fn {"x-ratelimit-reset", value} -> value end) do
|
||||
nil ->
|
||||
DateTime.utc_now()
|
||||
|> Timex.shift(seconds: 5 * 60)
|
||||
Logger.error("Rate limited, but couldn't find timestamp! Using default 5 minute backoff until #{default_5_minute_backoff}")
|
||||
default_5_minute_backoff
|
||||
|
||||
value ->
|
||||
{:ok, stamp} = DateTime.from_iso8601(value)
|
||||
stamp
|
||||
with {:ok, stamp, _} <- DateTime.from_iso8601(value) do
|
||||
Logger.error("Rate limited until #{stamp}")
|
||||
stamp
|
||||
else
|
||||
_ ->
|
||||
Logger.error("Rate limited, but couldn't parse timestamp! Using default 5 minute backoff until #{default_5_minute_backoff}")
|
||||
default_5_minute_backoff
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
@ -28,7 +38,8 @@ defmodule Pleroma.HTTP.Backoff do
|
|||
# this ensures that we don't hammer the server with requests, and instead wait for the backoff to expire
|
||||
# this is a very simple implementation, and can be improved upon!
|
||||
%{host: host} = URI.parse(url)
|
||||
case @cachex.get(@backoff_cache, host) do
|
||||
|
||||
case @cachex.get(@backoff_cache, host) do
|
||||
{:ok, nil} ->
|
||||
case HTTP.get(url, headers, options) do
|
||||
{:ok, env} ->
|
||||
|
|
|
@ -9,6 +9,7 @@ defmodule Pleroma.HTTP.BackoffTest do
|
|||
%Tesla.Env{url: "https://akkoma.dev/api/v1/instance"} ->
|
||||
{:ok, %Tesla.Env{status: 200, body: "ok"}}
|
||||
end)
|
||||
|
||||
assert {:ok, env} = Backoff.get("https://akkoma.dev/api/v1/instance")
|
||||
assert env.status == 200
|
||||
end
|
||||
|
@ -29,6 +30,25 @@ defmodule Pleroma.HTTP.BackoffTest do
|
|||
assert {:error, env} = Backoff.get("https://ratelimited.dev/api/v1/instance")
|
||||
assert env.status == 429
|
||||
assert {:ok, true} = Cachex.get(@backoff_cache, "ratelimited.dev")
|
||||
end
|
||||
end
|
||||
|
||||
test "should parse the value of x-ratelimit-reset, if present" do
|
||||
ten_minutes_from_now =
|
||||
DateTime.utc_now() |> Timex.shift(minutes: 10) |> DateTime.to_iso8601()
|
||||
|
||||
Tesla.Mock.mock_global(fn
|
||||
%Tesla.Env{url: "https://ratelimited.dev/api/v1/instance"} ->
|
||||
{:ok,
|
||||
%Tesla.Env{
|
||||
status: 429,
|
||||
body: "Rate limited",
|
||||
headers: [{"x-ratelimit-reset", ten_minutes_from_now}]
|
||||
}}
|
||||
end)
|
||||
|
||||
assert {:error, env} = Backoff.get("https://ratelimited.dev/api/v1/instance")
|
||||
assert env.status == 429
|
||||
assert {:ok, true} = Cachex.get(@backoff_cache, "ratelimited.dev")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Reference in a new issue