Saturday 26 September 2020

Inconsistency downloading multiple files with nodejs but not with php

Difference downloading multiple files with nodejs and php

I coded a script on both nodejs and php which achieve the same thing, ping an API, retrieve a list of files, loop through each file and download them to disk to a specified location.

On the left is by nodejs, and on the right by php.

I observe that, certain files randomly fail to download on each attempt in nodejs. And on a certain attempt, all files would succeed as well. While on php, each attempt is consistent and all files get downloaded fine.

Is there something am missing in nodejs, i.e. a configuration/header that isn't included by default through requests to download file? Or does downloading multiple files needs to be handled differently in nodejs?

Nodejs code:

const http = require('https');
const fs = require('fs');

function getResponse(url, callback) {
    http.get(url, response => {
        let body = '';

        response.on('data', data => {
            body += data
        })

        response.on('end', () => {
            callback(JSON.parse(body))
        })
    })
}


var download = function (url, dest, callback) {
    http.get(url, response => {
        response.on('error', function (err) {
            console.log(err)
        })
            .pipe(fs.createWriteStream(dest))
            .on('close', callback)
    });
};

getResponse('https://wallhaven.cc/api/v1/search?page=1', json => {
    json.data.forEach((item, index) => {
        download(item.path, `files/file-${index}.jpg`, function () {
            console.log('Finished Downloading' + `file-${index}.jpg`)
        });
    })
})

PHP code

    $client = new \GuzzleHttp\Client();

    $response = $client->get('https://wallhaven.cc/api/v1/search?page=1');

    $json = json_decode((string)$response->getBody());

    $rows = $json->data;

    foreach ($rows as $index => $row) {
        $content = file_get_contents($row->path);

        Storage::put("files/file-$index.jpg", $content);
    }

    return 'done';


from Inconsistency downloading multiple files with nodejs but not with php

No comments:

Post a Comment