Monday 9 September 2019

Depending on completion of multiple parallel gulp tasks

I have a complex NodeJS application, composed from several yarn workspace packages. The following gulpconfig.ts file defines a gulp build task for the entire set of packages:

import { series, parallel, TaskFunction, src, dest } from 'gulp';
import gulpTslint from 'gulp-tslint';
import { createProject } from 'gulp-typescript';
import tslint, { Linter } from 'tslint';
import { resolve } from 'path';
import { DestOptions } from 'vinyl-fs';
import { exec } from 'child_process';
import { promisify } from 'util';

const execPromise = promisify(exec);

interface GulpTasks {
  [key: string]: TaskFunction;
}

type TaskFunctionCallback = (error?: any) => void;

class Project {
  public projectFiles: NodeJS.ReadWriteStream;
  public constructor(
    public readonly projectDirectory: string,
    public readonly sourcemaps: boolean = false,
  ) {
    this.projectFiles = src('./src/**/*.ts', { cwd: projectDirectory, sourcemaps });
  }

  public lint(): this {
    this.projectFiles = this.projectFiles.pipe(
      gulpTslint({
        tslint,
        configuration: resolve(__dirname, 'tslint.json'),
        program: Linter.createProgram(
          resolve(this.projectDirectory, 'tsconfig.json'),
          this.projectDirectory,
        ),
      }),
    );

    return this;
  }

  public build(): this {
    const opts: DestOptions = {};
    if (this.sourcemaps) {
      opts.sourcemaps = true;
    }
    this.projectFiles = this.projectFiles
      .pipe(createProject(resolve(this.projectDirectory, 'tsconfig.build.json'))())
      .pipe(dest(resolve(this.projectDirectory, 'dist'), opts));

    return this;
  }
}

const buildDatabase = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'server/database')).build().projectFiles;

const buildSharedCommon = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'shared/common')).lint().build().projectFiles;
const buildSharedFrontoffice = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'shared/frontoffice')).lint().build().projectFiles;
const buildSharedBackoffice = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'shared/backoffice')).lint().build().projectFiles;

const buildClientCommon = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'client/common')).lint().build().projectFiles;
const buildClientFrontoffice = async (cb: TaskFunctionCallback): Promise<unknown> =>
  execPromise('yarn run build', { cwd: 'client/frontoffice' });
const buildClientBackoffice = async (cb: TaskFunctionCallback): Promise<unknown> =>
  execPromise('yarn run build', { cwd: 'client/backoffice' });

const buildServerCommon = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'server/common')).lint().build().projectFiles;
const buildServerFrontoffice = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'server/frontoffice')).lint().build().projectFiles;
const buildServerBackoffice = (cb: TaskFunctionCallback): NodeJS.ReadWriteStream =>
  new Project(resolve(__dirname, 'server/backoffice')).lint().build().projectFiles;

const tasks: GulpTasks = {
  build: parallel(
    buildDatabase,
    series(
      buildSharedCommon,
      parallel(buildSharedBackoffice, buildSharedFrontoffice),
      parallel(
        series(buildClientCommon, parallel(buildClientFrontoffice, buildClientBackoffice)),
        series(buildServerCommon, parallel(buildServerFrontoffice, buildServerBackoffice)),
      ),
    ),
  ),
};

export = tasks;

Because of how my application is structured, the buildServerCommon depends on buildDatabase. The buildDatabase however doesn't depend on anything, and takes a long time, which is why I start it in parallel with the rest.

With the setup above though, buildDatabase may (and does) finish later than buildServerCommon starts.

How can I make buildServerCommon start only after buildDatabase and buildSharedCommon, but still have everything else start as early as possible?

Basically, in my dependency tree:

  • I have "client", "server" and "shared" package folders.
  • "{client,server,shared}/common" is required by the "frontoffice" and "backoffice" packages in that same package folder.
  • "shared/{common,frontoffice,backoffice}" is required by both the "client" and "server" of the respective package.
  • "server/database" is special, in that it only exists in "server", and is required by "server/common".

I tried to use

const tasks: GulpTasks = {
  build: series(
    parallel(buildSharedCommon, buildDatabase),
    parallel(buildServerCommon, buildSharedFrontoffice, buildSharedBackoffice),
    parallel(
      series(buildClientCommon, parallel(buildClientFrontoffice, buildClientBackoffice)),
      parallel(buildServerFrontoffice, buildServerBackoffice),
    ),
  ),
};

but that isn't optimal, since buildSharedFrontoffice and buildSharedBackoffice are not being compiled until buildDatabase is done, and from there the client is also needlessly delayed.



from Depending on completion of multiple parallel gulp tasks

No comments:

Post a Comment