Controlling the Zoom out view port in FabricJS

I am trying out the examples given here on the pan and zoom in fabric js and got it working. I am using the zoom to point to zoom where the cursor was. However, the problem I face is that if I zoomIn in one place and zoomOut in another, there is an offset created between the border and background. Even though the code works for restricting the pan area to 1000x400 px, while zooming out, the boundaries are not restricted.

How can I restrict the boundaries even while zooming out, such that there is no offset?

Code here:

(function(){
var canvas = new fabric.Canvas('canvas1',{backgroundImage:"https://images.unsplash.com/photo-1520299607509-dcd935f9a839?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=1489&q=80"});
 viewportTransform=canvas.viewportTransform;
    canvas.add(new fabric.Rect({ width: 50, height: 50, fill: 'blue', angle: 10 }))
    canvas.add(new fabric.Circle({ radius: 50, fill: 'red', top: 44, left: 80 }))
    canvas.add(new fabric.Ellipse({ rx: 50, ry: 10, fill: 'yellow', top: 80, left: 35 }))
    canvas.add(new fabric.Rect({ width: 50, height: 50, fill: 'purple', angle: -19, top: 70, left: 70 }))
    canvas.add(new fabric.Circle({ radius: 50, fill: 'green', top: 110, left: 30 }))
    canvas.add(new fabric.Ellipse({ rx: 50, ry: 10, fill: 'orange', top: 12, left: 100, angle: 30 }))
    canvas.on('mouse:wheel', (opt) => {
      var delta = opt.e.deltaY;
      var zoom = canvas.getZoom();
      zoom *= 0.999 ** delta;
      if (zoom > 20) zoom = 20;
      if (zoom < 0.01) zoom = 0.01;
      canvas.zoomToPoint(new fabric.Point(opt.e.offsetX, opt.e.offsetY), zoom);
      opt.e.preventDefault();
      opt.e.stopPropagation();
      var vpt = viewportTransform;
      if (zoom < 0.4) {
        vpt[4] = 200 - 1000 * zoom / 2;
        vpt[5] = 200 - 400 * zoom / 2;
      } else {
        if (vpt[4] >= 0) {
          vpt[4] = 0;
        } else if (vpt[4] < canvas.getWidth() - 1000 * zoom) {
          vpt[4] = canvas.getWidth() - 1000 * zoom;
        }
        if (vpt[5] >= 0) {
          vpt[5] = 0;
        } else if (vpt[5] < canvas.getHeight() - 400 * zoom) {
          vpt[5] = canvas.getHeight() - 400 * zoom;
        }
      }
    });
    canvas.on('mouse:down', function (opt) {
      var evt= opt.e;
      if (evt.altKey === true) {
        this.isDragging = true;
        this.selection = false;
        this.lastPosX = evt.clientX;
        this.lastPosY = evt.clientY;
      }
    });
    canvas.on('mouse:move', function (opt) {
      if (this.isDragging) {
        var e = opt.e;
        var zoom = canvas.getZoom();
        var vpt = this.viewportTransform;
        if (zoom < 0.4) {
          vpt[4] = 200 - 1000 * zoom / 2;
          vpt[5] = 200 - 400 * zoom / 2;
        } else {
          vpt[4] += e.clientX - this.lastPosX;
          vpt[5] += e.clientY - this.lastPosY;
          if (vpt[4] >= 0) {
            vpt[4] = 0;
          } else if (vpt[4] < canvas.getWidth() - 1000 * zoom) {
            vpt[4] = canvas.getWidth() - 1000 * zoom;
          }
          if (vpt[5] >= 0) {
            vpt[5] = 0;
          } else if (vpt[5] < canvas.getHeight() - 400 * zoom) {
            vpt[5] = canvas.getHeight() - 400 * zoom;
          }
        }
        this.requestRenderAll();
        this.lastPosX = e.clientX;
        this.lastPosY = e.clientY;
      }
    });
    canvas.on('mouse:up', function (opt) {
      this.setViewportTransform(this.viewportTransform);
      this.isDragging = false;
      this.selection = true;
    });
})()

<script src="https://cdnjs.cloudflare.com/ajax/libs/fabric.js/3.6.6/fabric.min.js"></script>
  <canvas id="canvas1" height="300" width="400" style="border:1px solid"></canvas>

Issue Screenshot:

from Controlling the Zoom out view port in FabricJS