I am going to scrape the Pararius.nl as a practice with scrapy but when I start crawling it returns the fairlane protection, how can I pass it? Do I need any tools? please help with one example
def parse(self, response):
url = 'https://www.pararius.nl/{deal_type}/nederland/p-{page}/'
for deal_type in ['huurwoningen', 'koopwoningen']:
for i in range(1, 2):
yield scrapy.Request(url.format(deal_type=deal_type, page=i), callback=self.parse_pages,cookies=self.cookies,
headers=self.h, method='GET', cb_kwargs={'deal_type': deal_type})
def parse_pages(self, response, deal_type):
print(response.url)
return
from Scrape data with scrapy
No comments:
Post a Comment