Thursday, 14 November 2019

Trouble extracting data from a NodeJS stream

I am working with streams for the first time and I am having some trouble extracting data from my readable stream.

I am using the pg and pg-copy-streams to extract a large amount of data as a stream from a PSQL DB with the aim of creating a CSV file with the data from the Database.

Here is my code:

const aws = require('aws-sdk');
const {Client} = require('pg'); //  Needs the nodePostgres Lambda Layer
const copyTo = require('pg-copy-streams').to;

exports.handler = async (event) => {
let response = {};

console.log(JSON.stringify(event));

const client = new Client();

const deviceId = event.deviceId;
const fromDate = event.fromDate;
const toDate = event.toDate;

if (!deviceId) { // if we do not have a device id, just bail.
    return response = {
        statusCode: 400,
        body: "No device Id",
    };
}

const tempTableQuery = getQuery(deviceId, fromDate, toDate);
console.log("Search query: " + tempTableQuery);
try {

    await client.connect();

    await client.query(tempTableQuery);

    const q = `COPY temp_csv_table to STDOUT with csv DELIMITER ';'`;
    const dataStream = client.query(copyTo(q));

    // dataStream.pipe(console.log(process.stdout));
    dataStream.on('readable', function() {
        // There is some data to read now.
        let data;

        while (data = this.read()) {
            console.log(data); //<- this dosent print anything :(
        }
    });

    dataStream.on('error', async function (err) {
        // Here we can control stream errors
        await client.end();
    });
    dataStream.on('end', async function () {
        await client.end();
    });


} catch (e) {
    response = {
        statusCode: 500,
        result: "Error: " + e
    };
} finally {
    client.end();
}
};

function getQuery(deviceId, fromDate, toDate) {
return `CREATE TEMPORARY TABLE temp_csv_table AS
            SELECT * 
            FROM sensor_data_v2 
            WHERE device_id = '${deviceId}' and 
                  time_stamp between '${fromDate}' and '${toDate}' 
            LIMIT 10;`;
}

Question: How do I extract the rows from the data stream?

Notes:

  1. Running this on an AWS Lambda NodeJS 10.x runtime.
  2. I know there is data in the table for the filters I have specified.
  3. I have set the LIMIT 10 just for this test, those conditions will return 2600 rows of data.
  4. I will be using the csv-write-stream package to make a CSV file with the data from the DB. Not really attached to this package, happy to use another CSV writer if it would be easier to wok with.


from Trouble extracting data from a NodeJS stream

No comments:

Post a Comment