Monday, 28 January 2019

Google Cloud Function Node JS SFTP To Cloud Storage Transfer File Error

I'm trying to run the following code using node library ssh2-sftp but I don't get any error from GCP Console. What i'm trying to do here is to pipe the data that I receive from the SFTP into an output file on GCS.

Here is the code:

  /**
 * Generic background Cloud Function to be triggered by Cloud Storage.
 *
 * @param {object} event The Cloud Functions event.
 * @param {function} callback The callback function.
 */

let request = require("request");
let Storage = require('@google-cloud/storage');
let Client = require('ssh2-sftp-client');
let sftp = new Client();
const fs = require('fs');
const path = require('path');
const storage = new Storage({projectId: process.env.PROJECT_ID});

exports.retrieveFilesFromSFTP = (event, callback) => {


    let pubsubMessage = event.data;
    let fileName = pubsubMessage.data
        ? Buffer.from(pubsubMessage.data, 'base64').toString()
        : 'No Files in pubsub';


    console.log(fileName, 'File name from Pub/Sub Message was received');


    fs.readdir(__dirname, (err, files) => {
        if (err) {
            console.error(err);
            res.sendStatus(500);
        } else {
            console.log('Files', files);
            res.sendStatus(200);
        }
    });

    getFilesFromSFTP(fileName);
    callback();
};

function getFilesFromSFTP(fileName) {

    const bucket = storage.bucket('dataflowdirectory');

    const outFile = bucket.file("relex_output/"+fileName);

    //TODO create a file on the bucket , get the correct file path and then pipe it to the file

    sftp.connect({
        host: process.env.SFTP_SERVER,
        port: process.env.SFTP_PORT,
        username: process.env.SFTP_USER,
        password: process.env.SFTP_PASSWORD
    }).then(() => {


        sftp.list(process.env.SFTP_OUTBOUND_PATH)
            .then((data) => {

                sftp.get(process.env.SFTP_OUTBOUND_PATH + "/" + fileName).then((data) => {

                    data.pipe((outFile.createWriteStream({gzip: true})
                        .on('error', function (err) {
                            console.error("error",err);
                        })
                        .on('finish', function () {
                            // The file upload is complete.
                            outFile.close();
                        })));

                });

            }).catch((err) => {
            console.log(err);

        });

    })

}

The only thing that I see from the log is the following: Ignoring exception from a finished function

Any ideas how to solve this?

Thanks,



from Google Cloud Function Node JS SFTP To Cloud Storage Transfer File Error

No comments:

Post a Comment