Monday, 24 September 2018

Amazon S3 Remote File Upload with Axios

I am trying to write a function that would:

  1. Take a remote URL as a parameter,
  2. Get the file using axios
  3. Upload the stream to amazon s3
  4. And finally, return the uploaded url

I found help here on stackoverflow. So far, I have this:

/* 
 * Method to pipe the stream 
 */
const uploadFromStream = (file_name, content_type) => {
  const pass = new stream.PassThrough();

  const obj_key = generateObjKey(file_name);
  const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };

  s3.upload(params, function(err, data) {
    if(!err){
        return data.Location;
    } else {
        console.log(err, data);
    }
  });

  return pass;
}


/*
 * Method to upload remote file to s3
 */
const uploadRemoteFileToS3 = async (remoteAddr) => {
    axios({
        method: 'get',
        url: remoteAddr,
        responseType: 'stream'
    }).then( (response) => {
        if(response.status===200){
            const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
            const content_type = response.headers['content-type'];
            response.data.pipe(uploadFromStream(file_name, content_type));
        }
    });
}

But uploadRemoteFileToS3 does not return anything (because it's a asynchronous function). How can I get the uploaded url?

UPDATE

I have further improved upon the code and wrote a class. Here is what I have now:

const config = require('../config.json');

const stream = require('stream');
const axios = require('axios');
const AWS = require('aws-sdk');

class S3RemoteUploader {
    constructor(remoteAddr){
        this.remoteAddr = remoteAddr;
        this.stream = stream;
        this.axios = axios;
        this.config = config;
        this.AWS = AWS;
        this.AWS.config.update({
            accessKeyId: this.config.api_key,
            secretAccessKey: this.config.api_secret
        });
        this.spacesEndpoint = new this.AWS.Endpoint(this.config.endpoint);
        this.s3 = new this.AWS.S3({endpoint: this.spacesEndpoint});

        this.file_name = this.remoteAddr.substring(this.remoteAddr.lastIndexOf('/')+1);
        this.obj_key = this.config.subfolder+'/'+this.file_name;
        this.content_type = 'application/octet-stream';

        this.uploadStream();
    }

    uploadStream(){
        const pass = new this.stream.PassThrough();
        this.promise = this.s3.upload({
            Bucket: this.config.bucket,
            Key: this.obj_key,
            ACL: this.config.acl,
            Body: pass,
            ContentType: this.content_type
        }).promise();
        return pass;
    }

    initiateAxiosCall() {
        axios({
            method: 'get',
            url: this.remoteAddr,
            responseType: 'stream'
        }).then( (response) => {
            if(response.status===200){
                this.content_type = response.headers['content-type'];
                response.data.pipe(this.uploadStream());
            }
        });
    }

    dispatch() {
        this.initiateAxiosCall();
    }

    async finish(){
        //console.log(this.promise); /* return Promise { Pending } */
        return this.promise.then( (r) => {
            console.log(r.Location);
            return r.Location;
        }).catch( (e)=>{
            console.log(e);
        });
    }

    run() {
        this.dispatch();
        this.finish();
    }
}

But still have no clue how to catch the result when the promise is resolved. So far, I tried these:

testUpload = new S3RemoteUploader('https://avatars2.githubusercontent.com/u/41177');
testUpload.run();
//console.log(testUpload.promise); /* Returns Promise { Pending } */
testUpload.promise.then(r => console.log); // does nothing

But none of the above works. I have a feeling I am missing something very subtle. Any clue, anyone?



from Amazon S3 Remote File Upload with Axios

No comments:

Post a Comment