I have fetched the data from table tb_project_milestones and want to insert this projectMilestoneRow in a table tb_xyz using streams. I checked the documentation https://github.com/mysqljs/mysql#streaming-query-rows but couldn't find how to implement it. Has anyone implemented reading through streams and inserting through streams in MySQL.
let insertProjectMilestones = [];
const getProjectMilestones = executeQueryStream.query('SELECT * FROM tb_project_milestones WHERE project_id = ? ');
getProjectMilestones
.on('error', function(err) {
// Handle error, an 'end' event will be emitted after this as well
})
.on('result', function(projectMilestoneRow) {
// Pausing the connnection is useful if your processing involves I/O
connection.pause();
processRow(projectMilestoneRow, function() {
_.each(payload.projects, (project_id)=> {
_.each(projectMilestoneRow, (el)=> {
insertProjectMilestones.push([el.project_milestone_id, el.name, el.prefix, el.short_name, el.description, el.pre_requisites, project_id,
el.milestone_template_id, el.generic_milestone_id, el.planned_date, el.actual_date, el.forecast_date,
el.planned_date_only, el.forecast_date_only, el.actual_date_only, el.planned_time_only, el.forecast_time_only, el.actual_time_only,
el.planned_date_formula, el.actual_date_formula, el.forecast_date_formula, el.planned_date_is_active, el.forecast_date_is_active,
el.actual_date_is_active, el.creation_datetime, el.allow_notes, el.forecast_date_allow_notes, el.actual_date_allow_notes,
el.planned_date_allow_notes, 0, el.requires_approval]);
});
});
connection.resume();
});
})
.on('end', function() {
// all rows have been received
});
EDIT
I used streams in this case because millions of records are fetched from tb_project_milestones and then inserted into an array(after a manipulation) and then pushed into another table.
Considering the fact that pushing these many rows in the array would increase the memory of node I thought of using stream here.
Is stream better choice or could I just implement a batch insert in DB using transactions?
from Implementing reading through streams and inserting through streams in mysql
No comments:
Post a Comment