I've built a helper function and custom Retry class so I can set BACKOFF_MAX for a requests session as per this solution:
from requests import Session
from requests.adapters import HTTPAdapter, Retry
class RetryRequest(Retry):
def __init__(self, backoff_max: int, **kwargs):
super().__init__(**kwargs)
self.BACKOFF_MAX = backoff_max
def create_session(
retries: int,
backoff_factor: float,
backoff_max: int,
user_agent: str = "*",
referer: str = None,
) -> Session:
session = Session()
retries_spec = RetryRequest(
total=retries,
backoff_factor=backoff_factor,
backoff_max=backoff_max,
)
session.mount("https://", HTTPAdapter(max_retries=retries_spec))
headers = {"User-Agent": user_agent, "Referer": referer}
session.headers.update(headers)
return session
The session has been working fine for weeks but today it raised the following exception after it had been online for a few hours:
File "/Users/username/GitHub/polgara/apis/base_endpoint.py", line 54, in _get
response = self.client.session.get(url=url, headers=headers, params=params)
File "/Users/username/miniconda3/envs/capra_production/lib/python3.10/site-packages/requests/sessions.py", line 600, in get
return self.request("GET", url, **kwargs)
File "/Users/username/miniconda3/envs/capra_production/lib/python3.10/site-packages/requests/sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "/Users/username/miniconda3/envs/capra_production/lib/python3.10/site-packages/requests/sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "/Users/username/miniconda3/envs/capra_production/lib/python3.10/site-packages/requests/adapters.py", line 489, in send
resp = conn.urlopen(
File "/Users/username/miniconda3/envs/capra_production/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
retries = retries.increment(
File "/Users/username/miniconda3/envs/capra_production/lib/python3.10/site-packages/urllib3/util/retry.py", line 581, in increment
new_retry = self.new(
File "/Users/username/miniconda3/envs/capra_production/lib/python3.10/site-packages/urllib3/util/retry.py", line 338, in new
return type(self)(**params)
TypeError: RetryRequest.__init__() missing 1 required positional argument: 'backoff_max'
It looks like something has gone awry in requests and it has tried to create a new session with the RetryRequest class but because it usually only deals with a Retry class it didn't pass the custom parameter of backoff_max.
I could get around this by setting a default argument for backoff_max but this defeats the object of being able to customise it when I create the session to begin with.
Would anyone have any idea on how I'd be able to solve this?
requests version is 2.28.2
from Exception with custom Retry class to set BACKOFF_MAX
No comments:
Post a Comment