Move Camera to the User-Clicked Location Using Tween for Three.js Panorama

My goal is to move Camera to the user-clicked location so the clicked location is centered for my equirectangular panorama. I am using tween.js to achieve this and the Camera does span but I cannot get the new coordinates centered correctly.

I tried to calculate the difference between the screen center and clicked location but it is not behaving right

   var handlePanoramaClick = function (event) {
      var center={x: window.innerWidth / 2, y: window.innerHeight / 2}  
      var click={x: event.clientX, y: event.clientY}

      var diffx = click.x - center.x;
      var diffy = click.y - center.y;

      var x = coords.latLon.current.x + diffy;
      var y = coords.latLon.current.y + diffx;
      
      var tween = new TWEEN.Tween(coords.latLon.current)
      .to({x:x,y:y}, 1000)
      .start();
   };

   var update = function update() {
     var distX = coords.latLon.current.x - coords.latLon.delta.x;
     var distY = coords.latLon.current.y - coords.latLon.delta.y;

     coords.latLon.delta.x += distX / 7;
     coords.latLon.delta.y += distY / 7;

     coords.phi = THREE.Math.degToRad(90 - coords.latLon.delta.x);
     coords.theta = THREE.Math.degToRad(coords.latLon.delta.y);
 
     coords.target.x = 500 * Math.sin(coords.phi) * Math.cos(coords.theta);
     coords.target.y = 500 * Math.cos(coords.phi);
     coords.target.z = 500 * Math.sin(coords.phi) * Math.sin(coords.theta);

     Cam.camera.lookAt(coords.target);
     renderer.render(Scene, Camera);
   };
 
   var animate = function animate() {
     update();
     requestAnimationFrame(animate);
     TWEEN.update();
   };

from Move Camera to the User-Clicked Location Using Tween for Three.js Panorama