I am just trying to get deals items from this amazon URL :
when I open this link in browser and write the query in console, it works: document.querySelectorAll('div[class*="DealItem-module__dealItem_"]')
but when I try to fetch this through this phantomjs
script, it seems to always returning nothing:
var page = require('webpage').create();
page.viewportSize = { height: 800, width: 1920 }; // BRODIE : CHROME
page.customHeaders = {
accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
// 'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9',
dnt: '1',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="90", "Microsoft Edge";v="90"',
'sec-ch-ua-mobile': '?0',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Edg/90.0.818.66',
};
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;
//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
page.onConsoleMessage = function (message) {
console.log('console.log() -- ', message);
}; // BUBBLE UP LOGS FROM BROWSER CONSOLE TO PHANTOM CONSOLE
page.onLoadStarted = function () {
loadInProgress = true;
console.log('page loading started');
};
page.onLoadFinished = function () {
loadInProgress = false;
console.log('page loading finished');
};
page.onError = function (msg, trace) {
console.log(msg);
trace.forEach(function (item) {
console.log(' ', item.file, ':', item.line);
});
};
// OPEN PAGE
console.log('page.open()');
page.open(
'https://www.amazon.com/gp/goldbox/ref=gbps_ftr_s-5_cd34_wht_26179410?gb_f_deals1=sortOrder:BY_SCORE,includedAccessTypes:GIVEAWAY_DEAL,enforcedCategories:2617941011&pf_rd_p=fd51d8cf-b5df-4144-8086-80096db8cd34&pf_rd_s=slot-5&pf_rd_t=701&pf_rd_i=gb_main&pf_rd_m=ATVPDKIKX0DER&pf_rd_r=A89BX6V6RQRQ94NFA0DP&ie=UTF8',
function (status) {
if (status !== 'success')
console.log('U N A B L E T O O P E N P A G E . . .');
else console.log(' P A G E O P E N E D . . .');
var selector = 'div[class*="DealItem-module__dealItem_"]'
var findAll = setInterval(function () {
console.log('trying to fetch deals...');
var deals = page.evaluate(function (sel) {
return document.querySelectorAll(
'div[class*="DealItem-module__dealItem_"]'
);
}, selector);
if(deals.length) {
console.log('deals.length', deals.length);
clearInterval(findAll);
}
}, 1000);
}
);
Also, when I try to take screenshot using page.render()
, it shows page with unloaded/unfinished JS (which is different from when we type that URL in browser and search:):
Also, I noticed that when I run this script in terminal, I get some JS errors of webpage:
Any help will be greatly appriciated
from phantomjs: document.querySelectorAll() not working for dynamic page
No comments:
Post a Comment