Here is my code so far:
const allRows = [];
async function fileToLines(file) {
return new Promise((resolve, reject) => {
reader = new FileReader();
reader.onload = function(e) {
parsedLines = e.target.result.split(/\r|\n|\r\n/);
resolve(parsedLines);
};
reader.readAsText(file);
});
}
document
.getElementById('fileInput')
.addEventListener('change', async function(e) {
var file = e.target.files[0];
if (file != undefined) {
fileToLines(file).then( async id => {
console.log(id)
console.log(parsedLines)
console.log(typeof id);
var idInt = id.map(Number);
var idFiltered = id.filter(function(v){return v!==''});
console.log(idFiltered)
for(let id of idFiltered) {
const row = await getRelease(id);
allRows.push(row);
}
download();
});
}
});
function getRelease(idFiltered) {
return fetch(`https://api.***.com/releases/${idFiltered}`, {
headers: {
'User-Agent': '***/0.1',
},
})
.then(response => response.json())
.then(data => {
if (data.message === 'Release not found.') {
return { error: `Release with ID ${idFiltered} does not exist` };
} else {
const id = data.id;
const delimiter = document.getElementById("delimiter").value || "|";
const artists = data.artists ? data.artists.map(artist => artist.name) : [];
const barcode = data.identifiers.filter(id => id.type === 'Barcode')
.map(barcode => barcode.value);
var formattedBarcode = barcode.join(delimiter);
const country = data.country || 'Unknown';
const genres = data.genres || [];
const formattedGenres = genres.join(delimiter);
const labels = data.labels ? data.labels.map(label => label.name) : [];
const formattedLabels = labels.join(delimiter);
const catno = data.labels ? data.labels.map(catno => catno.catno) : [];
const formattedCatNo = catno.join(delimiter);
const styles = data.styles || [];
const formattedStyles = styles.join(delimiter);
const tracklist = data.tracklist ? data.tracklist
.map(track => track.title) : [];
const formattedTracklist = tracklist.join(delimiter);
const year = data.year || 'Unknown';
const format = data.formats ? data.formats.map(format => format.name) : [];
const qty = data.formats ? data.formats.map(format => format.qty) : [];
const descriptions = data.formats ? data.formats
.map(descriptions => descriptions.descriptions) : [];
const preformattedDescriptions = descriptions.toString()
.replace('"','""').replace(/,/g, ', ');
const formattedDescriptions = '"' + preformattedDescriptions + '"';
return [idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
];
}
});
}
function download() {
const ROW_NAMES = [
"release_id",
"artist",
"format",
"qty",
"format descriptions",
"label",
"catno",
"country",
"year",
"genres",
"styles",
"barcode",
"tracklist"
];
var csvContent = "data:text/csv;charset=utf-8,"
+ ROW_NAMES + "\n" + allRows.map(e => e.join(",")).join("\n");
console.log(csvContent);
var encodedUri = encodeURI(csvContent);
var link = document.createElement("a");
link.setAttribute("href", encodedUri);
link.setAttribute("download", "my_data.csv");
document.body.appendChild(link); // Required for FF
link.click();
}
When I was previously trying to crack this problem 2.5 years ago (!) someone told me the easiest way "is to maintain a chain of promises to keep track of the requests", like this...
const timer = ms => new Promise(resolve => setTimeout(resolve, ms));
let requests = Promise.resolve();
function getRelease(id) {
const apiCall = requests.then(() =>
fetch(`https://api.***.com/releases/${id}`, {
headers: {
'User-Agent': '***/0.1',
}
})
);
// add to chain / queue
requests = apiCall.then(response =>
+response.headers.get("X-***-Ratelimit-Remaining") <= 1 && timer(60 * 1000)
);
return apiCall
.then(response => response.json())
.then(parseReleaseData);
}
The person who suggested this code commented...
Now one request will be done after another, and if the rate limit gets reached it waits a minute.
You might want to retry in case of a rate limiting error. You could also add multiple promise queues to allow for higher throughput.
It seems that when I tried that previously, it set a 60s delay before making any calls? I think I would like to try this method again, but I'm not sure how to code it. Like, I'm not sure how const apiCall = requests.then(() => would fit in with my current code. I can see that the suggested code actually returns 'apiCall', whereas my method is set up to return all the individual data fields, so I'm not sure how to proceed there. It seems like a good method to get the Ratelimit from the host and set a timeout as needed, but I'm just not sure where to start really. Any help please?
from How to throttle my JS API fetch requests, using the rate-limit supplied by the host?
No comments:
Post a Comment