I am using PyGithub to scrape some repositories, though I get some error while iterating through the search pages.
def scrape_interval(self, interval):
for repo_number, repo in self.search(interval):
code...
def search(self, interval):
try:
iterator = enumerate(self.github.search_repositories(query="Laravel created:" + interval))
except:
print.warning("Going to sleep for 1 hour. The search API hit the limit")
time.sleep(3600)
iterator = self.search(interval)
return iterator
As you can see I try to catch errors when creating the iterator in def search. But the error is thrown on line for repo_number, repo in self.search(interval): so that is at some point when the iterator get next item?
What are my options to make these errors catchable? I would preferably avoid wrapping the whole for loop in a try clause, rather manage it during the iteration itself.
For reference about the error in itself:
for repo_number, repo in self.search(interval): File "/anaconda3/envs/laravel-ai/lib/python3.7/site-packages/github/PaginatedList.py", line 58, in iter newElements = self._grow() File "/anaconda3/envs/laravel-ai/lib/python3.7/site-packages/github/PaginatedList.py", line 70, in _grow newElements = self._fetchNextPage() File "/anaconda3/envs/laravel-ai/lib/python3.7/site-packages/github/PaginatedList.py", line 172, in _fetchNextPage headers=self.__headers File "/anaconda3/envs/laravel-ai/lib/python3.7/site-packages/github/Requester.py", line 185, in requestJsonAndCheck return self.__check(*self.requestJson(verb, url, parameters, headers, input, cnx)) File "/anaconda3/envs/laravel-ai/lib/python3.7/site-packages/github/Requester.py", line 231, in requestJson return self.__requestEncode(cnx, verb, url, parameters, headers, input, encode) File "/anaconda3/envs/laravel-ai/lib/python3.7/site-packages/github/Requester.py", line 284, in __requestEncode status, responseHeaders, output = self.__requestRaw(cnx, verb, url, requestHeaders, encoded_input) File "/anaconda3/envs/laravel-ai/lib/python3.7/site-packages/github/Requester.py", line 309, in __requestRaw requestHeaders File "/anaconda3/envs/laravel-ai/lib/python3.7/http/client.py", line 1229, in request self._send_request(method, url, body, headers, encode_chunked) File "/anaconda3/envs/laravel-ai/lib/python3.7/http/client.py", line 1275, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/anaconda3/envs/laravel-ai/lib/python3.7/http/client.py", line 1224, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/anaconda3/envs/laravel-ai/lib/python3.7/http/client.py", line 1016, in _send_output self.send(msg) File "/anaconda3/envs/laravel-ai/lib/python3.7/http/client.py", line 956, in send self.connect() File "/anaconda3/envs/laravel-ai/lib/python3.7/http/client.py", line 1384, in connect super().connect() File "/anaconda3/envs/laravel-ai/lib/python3.7/http/client.py", line 928, in connect (self.host,self.port), self.timeout, self.source_address) File "/anaconda3/envs/laravel-ai/lib/python3.7/socket.py", line 707, in create_connection for res in getaddrinfo(host, port, 0, SOCK_STREAM): File "/anaconda3/envs/laravel-ai/lib/python3.7/socket.py", line 748, in getaddrinfo for res in _socket.getaddrinfo(host, port, family, type, proto, flags): socket.gaierror: [Errno 8] nodename nor servname provided, or not known
from Gracefully catch python Exception in Iterator next
No comments:
Post a Comment