Recentre the other stereo camera on the same object when synchronised panning settles (#1934)

* Recentre the other stereo camera on the same object when synchronised panning settles

* Invalidate outstanding stereo pan lookups when settings or views change

* accounting for resizing and not triggering the schedule of linked viewing

---------

Co-authored-by: Bryon Lewis <Bryon.Lewis@kitware.com>

Author: mattdawkins

client/dive-common/components/Viewer.vue

@@ -100,6 +100,13 @@ import AlignedViewToggle from './AlignedViewToggle.vue';
 import PrimaryAttributeTrackFilter from './PrimaryAttributeTrackFilter.vue';
 import UserSettingsDialog from './UserSettingsDialog.vue';
 
+export interface StereoViewLinkParams {
+  camera: string;
+  frameNum: number;
+  point: [number, number];
+}
+export type StereoViewLinkFunc = (params: StereoViewLinkParams) => Promise<[number, number] | null>;
+
 export interface ImageDataItem {
   url: string;
   filename: string;
@@ -185,6 +192,15 @@ export default defineComponent({
       type: Number as PropType<number | undefined>,
       default: undefined,
     },
+    /**
+     * Where a point on one stereo camera lands on the other, using the loaded
+     * stereo matcher; null when it cannot be found. Lets synchronised panning
+     * follow the same object on both cameras.
+     */
+    stereoViewLink: {
+      type: Function as PropType<StereoViewLinkFunc | undefined>,
+      default: undefined,
+    },
   },
   setup(props, { emit }) {
     const { prompt, visible } = usePrompt();
@@ -801,6 +817,32 @@ export default defineComponent({
     provideAutoRegisterJob(autoRegisterJob);
     onBeforeUnmount(() => autoRegisterJob.dispose());
 
+    // Linked panning: with camera controls synchronised and auto-compute on,
+    // the other pane recentres on where this pane's centre is on its camera.
+    const stereoViewLinkResolver = async (camera: string, point: [number, number]) => {
+      if (!props.stereoViewLink) return null;
+      let frameNum: number;
+      try {
+        frameNum = aggregateController.value.getController(camera).frame.value;
+      } catch {
+        return null;
+      }
+      return props.stereoViewLink({ camera, frameNum, point });
+    };
+    watch(
+      [
+        () => clientSettings.stereoSettings.autoComputeOtherCamera,
+        () => props.stereoViewLink,
+        () => multiCamList.value.length,
+      ],
+      ([autoCompute, link, cameras]) => {
+        aggregateController.value.setViewLinkResolver(
+          autoCompute && link && cameras === 2 ? stereoViewLinkResolver : null,
+        );
+      },
+      { immediate: true },
+    );
+
     // Provides wrappers for actions to integrate with settings
     const {
       linkingTrack,

client/dive-common/use/stereo/useStereoOnnxTransfer.ts

@@ -568,10 +568,28 @@ export default function useStereoOnnxTransfer(config: StereoOnnxTransferConfig)
     return counts;
   }
 
+  /**
+   * Where one point on `sourceCamera` lands on the other camera at `frameNum`,
+   * or null when the match is rejected or the stereo setup is incomplete.
+   * Used for linked panning, so it never reports errors to the user.
+   */
+  async function warpPoint(point: Point, sourceCamera: string, frameNum: number): Promise<Point | null> {
+    const otherCamera = getMultiCamList().find((c) => c !== sourceCamera);
+    if (!otherCamera || getMultiCamList().length !== 2) return null;
+    try {
+      const [result] = await warp([point], sourceCamera, otherCamera, frameNum);
+      return result?.accepted && Number.isFinite(result.x) && Number.isFinite(result.y)
+        ? [result.x, result.y] : null;
+    } catch {
+      return null;
+    }
+  }
+
   return {
     handleStereoAnnotationComplete,
     handleStereoTrackLinked,
     warpAllFromCamera,
+    warpPoint,
     measureAtFrame,
     precomputeFrame,
   };

client/platform/desktop/frontend/components/ViewerLoader.vue

@@ -9,7 +9,7 @@ import {
 import { ANNOTATION_SOURCE_QUERY } from 'dive-common/scoring/viewerNavigation';
 import { parseViewerFocus } from 'dive-common/review/viewerNavigation';
 import { useRoute, useRouter } from 'vue-router/composables';
-import Viewer from 'dive-common/components/Viewer.vue';
+import Viewer, { StereoViewLinkParams } from 'dive-common/components/Viewer.vue';
 import RunPipelineMenu from 'dive-common/components/RunPipelineMenu.vue';
 import ImportAnnotations from 'dive-common//components/ImportAnnotations.vue';
 import CalibrationMenu from 'dive-common/components/CalibrationMenu.vue';
@@ -1508,6 +1508,36 @@ export default defineComponent({
      *   result status instead — used by bulk import "Warp to All" so failures
      *   can be aggregated rather than cleared by the next job.
      */
+    /**
+     * Where `point` on `camera` lands on the other stereo camera, for linked
+     * panning. Uses whatever matcher the stereo service loaded; null when the
+     * service is off or the match is rejected.
+     */
+    async function stereoViewLink(params: StereoViewLinkParams): Promise<[number, number] | null> {
+      if (!stereoEnabled.value) return null;
+      const cameras = Object.keys(stereoImagePathGetters.value);
+      if (cameras.length !== 2 || !cameras.includes(params.camera)) return null;
+      if (!(await ensureStereoFrame(params.frameNum))) return null;
+      const fps = stereoCameraFps.value[cameras[0]] || stereoDatasetFps || Object.values(stereoCameraFps.value)[0];
+      try {
+        const response = await stereoTransferPoints({
+          points: [params.point],
+          strict: true,
+          sourceCamera: params.camera === cameras[0] ? 'left' : 'right',
+          leftImagePath: stereoImagePathGetters.value[cameras[0]](params.frameNum),
+          rightImagePath: stereoImagePathGetters.value[cameras[1]](params.frameNum),
+          frameTime: fps ? params.frameNum / fps : undefined,
+        });
+        const point = response.transferredPoints?.[0];
+        if (!response.success || response.validMatches?.[0] !== true || !point?.every(Number.isFinite)) {
+          return null;
+        }
+        return [point[0], point[1]];
+      } catch {
+        return null;
+      }
+    }
+
     async function handleStereoAnnotationComplete(
       params: StereoAnnotationCompleteParams,
       forceAutoCompute = false,
@@ -2188,6 +2218,7 @@ export default defineComponent({
       stereoLengthMessage,
       closeStereoLoadingDialog,
       handleStereoAnnotationComplete,
+      stereoViewLink,
       handleStereoWarpImported,
       handleStereoAnnotationReset,
       handleStereoSegmentationFinalize,
@@ -2215,6 +2246,7 @@ export default defineComponent({
       :initial-track-id="viewerFocus.trackId"
       :text-query-enabled="true"
       :text-query-available="textQueryAvailable"
+      :stereo-view-link="stereoViewLink"
       @return-to-current-annotations="returnToCurrentAnnotations"
       @change-camera="changeCamera"
       @large-image-warning="largeImageWarning()"

client/platform/web-girder/useStereoOnnxWeb.ts

@@ -424,10 +424,16 @@ export default function useStereoOnnxWeb(opts: StereoOnnxWebOptions) {
     return getTransfer()?.warpAllFromCamera(cameraName);
   }
 
+  async function stereoViewLink(params: { camera: string; frameNum: number; point: [number, number] }) {
+    if (!clientSettings.stereoSettings.autoComputeOtherCamera) return null;
+    return (await getTransfer()?.warpPoint(params.point, params.camera, params.frameNum)) ?? null;
+  }
+
   return {
     handleStereoAnnotationComplete,
     handleStereoTrackLinked,
     warpAllFromCamera,
+    stereoViewLink,
     precomputeCurrentFrame,
     invalidateCalibration,
   };

client/platform/web-girder/views/ViewerLoader.vue

@@ -177,7 +177,7 @@ export default defineComponent({
 
     const {
       handleStereoAnnotationComplete, handleStereoTrackLinked, warpAllFromCamera,
-      invalidateCalibration,
+      invalidateCalibration, stereoViewLink,
     } = useStereoOnnxWeb({
       getViewer: () => viewerRef.value,
       getDatasetId: () => parentDatasetId(props.id),
@@ -518,6 +518,7 @@ export default defineComponent({
       exportDatasetIds,
       handleStereoAnnotationComplete,
       handleStereoTrackLinked,
+      stereoViewLink,
       stereoBusyMessage,
       stereoDownloadProgress,
       stereoDownloadPercent,
@@ -550,6 +551,7 @@ export default defineComponent({
       :annotation-source-returnable="annotationSourceReturnable"
       :initial-frame="viewerFocus.frame"
       :initial-track-id="viewerFocus.trackId"
+      :stereo-view-link="stereoViewLink"
       @return-to-current-annotations="returnToCurrentAnnotations"
       @large-image-warning="largeImageWarning()"
       @update:set="routeSet"

client/src/components/annotators/mediaControllerType.ts

@@ -52,9 +52,10 @@ export interface AggregateMediaController {
    * True only while onResize is applying its programmatic size()/resetZoom()
    * to the panes. resetZoom emits GeoJS pan/zoom events synchronously; the
    * linked-viewer navigation (useAlignedNavigation / useRegistrationNavigation)
-   * must ignore those so one pane's native-space reset isn't broadcast to the
-   * others as if it were a shared-space move (which parks warped panes on an
-   * empty corner). The resizeTrigger bump that follows re-snaps every pane from
+   * and stereo view-link must ignore those so one pane's native-space reset
+   * isn't broadcast to the others as if it were a shared-space move (which
+   * parks warped panes on an empty corner, or recentres the stereo pair on the
+   * wrong object). The resizeTrigger bump that follows re-snaps every pane from
    * the reference once the reset has settled.
    */
   resizing: Readonly<Ref<boolean>>;
@@ -75,6 +76,15 @@ export interface AggregateMediaController {
   setSpeed: (speed: number) => void;
   getController: (cameraName: string) => MediaController;
   toggleSynchronizeCameras: (sync: boolean) => void;
+  /**
+   * While cameras are synchronised, recentre the other panes on where the
+   * source pane's centre lands on their camera (null to disable). The
+   * resolver returns null when the point cannot be found there.
+   */
+  setViewLinkResolver: (
+    resolver: ((sourceCamera: string, point: [number, number]) =>
+      Promise<[number, number] | null>) | null,
+  ) => void;
   /**
    * Seeks so that `camera` lands on its own local frame `localFrame` (e.g.
    * jumping to a track's stored begin/end, which is in local-frame units).

client/src/components/annotators/useMediaController.ts

@@ -7,6 +7,7 @@ import Vue, {
   ref, shallowRef, reactive, provide, toRef, Ref, UnwrapRef, computed, watch,
 } from 'vue';
 import { map, over } from 'lodash';
+import createViewLink from './viewLink';
 
 import { use } from '../../provides';
 import type {
@@ -113,10 +114,27 @@ export function useMediaController() {
   let state: Record<string, UnwrapRef<MediaControllerReactiveData>> = {};
   let cameraControllerSymbols: Record<string, symbol> = {};
   const synchronizeCameras: Ref<boolean> = ref(false);
+
+  // Installed by the viewer while stereo auto-compute is on; otherwise
+  // synchronised panes only copy each other's screen motion.
+  const viewLink = createViewLink({
+    center: (key) => geoViewers[key]?.value?.center(),
+    cameraName: (key) => state[key]?.cameraName,
+    synced: () => synchronizeCameras.value,
+    recenter: (sourceKey, point) => {
+      allowCameraTrigger = false;
+      Object.entries(geoViewers).forEach(([camera, geoViewer]) => {
+        if (geoViewer.value && camera !== sourceKey) {
+          geoViewer.value.center({ x: point[0], y: point[1] });
+        }
+      });
+      allowCameraTrigger = true;
+    },
+  });
   const resizeTrigger: Ref<number> = ref(0);
   // Raised only while onResize applies its programmatic resetZoom, so the
-  // linked-viewer navigation ignores the resulting pan/zoom events (see
-  // AggregateMediaController.resizing).
+  // linked-viewer navigation and stereo view-link ignore the resulting
+  // pan/zoom events (see AggregateMediaController.resizing).
   const resizing: Ref<boolean> = ref(false);
   // shallowRef: an AlignedFrameResolver carries nested Refs (slotCount, frameRate)
   // that must NOT be deep-reactive-converted/auto-unwrapped by a plain ref().
@@ -150,6 +168,7 @@ export function useMediaController() {
     currentTime: emptyControllerCurrentTime,
     getController,
     toggleSynchronizeCameras,
+    setViewLinkResolver,
     cameraSync: synchronizeCameras,
     resizeTrigger,
     resizing,
@@ -287,6 +306,12 @@ export function useMediaController() {
     synchronizeCameras.value = val;
   }
 
+  function setViewLinkResolver(
+    resolver: Parameters<typeof viewLink.setResolver>[0],
+  ) {
+    viewLink.setResolver(resolver);
+  }
+
   /**
    * Optional replacement for the aggregate "reset pan and zoom" behavior,
    * installed by the aligned-view navigation link (useAlignedNavigation).
@@ -320,6 +345,12 @@ export function useMediaController() {
         }
       });
       allowCameraTrigger = true;
+      // onResize's resetZoom emits pan/zoom in native space; skip the stereo
+      // lookup so a pane's native center isn't warped onto the other camera
+      // after the resize settles (same guard as aligned/registration nav).
+      if (!resizing.value) {
+        viewLink.schedule(camEvent.camera);
+      }
     }
   });
 
@@ -333,6 +364,9 @@ export function useMediaController() {
         }
       });
       allowCameraTrigger = true;
+      if (!resizing.value) {
+        viewLink.schedule(camEvent.camera);
+      }
     }
   });
   /**
@@ -654,6 +688,7 @@ export function useMediaController() {
       getController,
       resetMapDimensions,
       toggleSynchronizeCameras,
+      setViewLinkResolver,
       cameraSync: synchronizeCameras,
       resizeTrigger,
       resizing,
@@ -820,6 +855,7 @@ export function useMediaController() {
       currentTime: defaultController.currentTime,
       getController,
       toggleSynchronizeCameras,
+      setViewLinkResolver,
       cameraSync: synchronizeCameras,
       resizeTrigger,
       resizing,