diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index f09531a56..962802881 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -73,8 +73,9 @@ import type { } from 'dive-common/use/useModeManager'; import clientSettingsSetup, { clientSettings, isStereoInteractiveModeEnabled } from 'dive-common/store/settings'; import { - useApi, FrameImage, DatasetType, GlobalStyleSettings, + useApi, FrameImage, DatasetType, GlobalStyleSettings, DatasetStereoCalibration, } from 'dive-common/apispec'; +import { parentDatasetId } from 'dive-common/compositeDatasetId'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; import { buildAlignedTimeline, buildInverseAlignedIndex, computeGapSlots, TimelineResult, @@ -94,6 +95,12 @@ import { import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; +import TrackViewer from 'vue-media-annotator/components/track_3d_viewer/TrackViewer.vue'; +import TrackViewerSettings from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettings.vue'; +import TrackViewerSettingsStore from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettingsStore'; +import { + displayOptions, displayValue, parseDisplayValue, Viewer3dMode, +} from 'dive-common/stereoDisplay'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; import RegistrationToolsVue from './CameraRegistration/RegistrationTools.vue'; @@ -133,6 +140,7 @@ export default defineComponent({ UnsavedChangesDialog, EditorMenu, MultiCamToolbar, + TrackViewer, AlignedViewToggle, PrimaryAttributeTrackFilter, TrackList, @@ -255,7 +263,7 @@ export default defineComponent({ const { loadDetections, loadConfig, saveConfig, getTiles, getTileURL, getTileHistogram, loadGlobalStyleSettings, saveGlobalStyleSettings, - getPipelineList, runPipeline, watchPipelineJob, + getPipelineList, runPipeline, watchPipelineJob, getDatasetCalibration, } = useApi(); const progress = reactive({ // Loaded flag prevents annotator window from populating @@ -2105,6 +2113,18 @@ export default defineComponent({ } } } + viewer3dRequested.value = 'off'; + if (isStereoPair.value) { + context.register({ + component: TrackViewerSettings, + description: '3D Viewer Settings', + }); + } else { + context.unregister({ + component: TrackViewerSettings, + description: '3D Viewer Settings', + }); + } annotationUndo.start(); progress.loaded = true; fetchSelectedCameraHistogram().catch(() => {}); @@ -2272,6 +2292,49 @@ export default defineComponent({ }; + const trackViewerSettingsStore = new TrackViewerSettingsStore(); + const viewer3dRequested = ref('off' as Viewer3dMode); + const isStereoPair = computed(() => subType.value === 'stereo' && multiCamList.value.length === 2); + const viewer3dMode = computed(() => (isStereoPair.value ? viewer3dRequested.value : 'off')); + const viewer3dActive = computed(() => viewer3dMode.value !== 'off'); + const displayItems = computed(() => displayOptions( + multiCamList.value, + isStereoPair.value, + defaultCamera.value, + )); + const selectedDisplay = computed(() => displayValue(selectedCamera.value, viewer3dMode.value)); + function changeDisplay(value: string) { + const { camera, mode } = parseDisplayValue(value); + viewer3dRequested.value = mode; + changeCamera(camera); + } + /** In 'replace' mode the 3D viewer takes the pane of the camera that is not selected. */ + function isCameraReplacedBy3dViewer(camera: string) { + return viewer3dMode.value === 'replace' && camera !== selectedCamera.value; + } + const viewer3dPaneOrder = computed(() => ( + viewer3dMode.value === 'replace' && multiCamList.value.indexOf(selectedCamera.value) !== 0 + ? -1 : 1 + )); + const rigCalibration = ref(null as DatasetStereoCalibration | null); + watch(viewer3dActive, async (active) => { + if (!active) { + return; + } + try { + const result = await getDatasetCalibration(parentDatasetId(datasetId.value)); + rigCalibration.value = result?.calibration ?? null; + } catch { + rigCalibration.value = null; + } + }); + watch([viewer3dMode, selectedCamera], async () => { + await nextTick(); + handleResize(); + // The bottom layout has no controls container for handleResize to act on. + onResize(); + }); + provideAnnotator( { annotatorPreferences: toRef(clientSettings, 'annotatorPreferences'), @@ -2279,6 +2342,7 @@ export default defineComponent({ cameraStore, cameraRegistration, alignedView, + trackViewerSettingsStore, datasetId, editingMode, groupFilters, @@ -2599,6 +2663,14 @@ export default defineComponent({ defaultCamera, selectedCamera, changeCamera, + isStereoPair, + viewer3dActive, + viewer3dPaneOrder, + rigCalibration, + displayItems, + selectedDisplay, + changeDisplay, + isCameraReplacedBy3dViewer, noteRightMouseDown, // For Navigation Guarding unsavedChangesDialog, @@ -2809,19 +2881,20 @@ export default defineComponent({ @@ -2978,7 +3051,11 @@ export default defineComponent({
+
+ +
+
+ +
{ + it('offers each stereo camera selected, beside the 3D viewer, or with it as a third pane', () => { + expect(displayOptions(['left', 'right'], true, 'left').map((option) => option.text)).toEqual([ + 'Stereo (Left)', 'Stereo (Right)', + 'Left + 3D', 'Right + 3D', + 'Stereo (Left) + 3D', 'Stereo (Right) + 3D', + ]); + }); + + it('lists plain cameras for other multi-camera datasets', () => { + expect(displayOptions(['eo', 'ir'], false, 'ir')).toEqual([ + { value: 'camera:eo', text: 'eo', menuText: 'eo' }, + { value: 'camera:ir', text: 'ir', menuText: 'ir (Default)' }, + ]); + }); + + it('round-trips a display value', () => { + expect(parseDisplayValue(displayValue('right', 'replace'))).toEqual({ camera: 'right', mode: 'replace' }); + expect(parseDisplayValue(displayValue('left', 'beside'))).toEqual({ camera: 'left', mode: 'beside' }); + expect(parseDisplayValue(displayValue('viewer3d:odd', 'off'))) + .toEqual({ camera: 'viewer3d:odd', mode: 'off' }); + }); +}); diff --git a/client/dive-common/stereoDisplay.ts b/client/dive-common/stereoDisplay.ts new file mode 100644 index 000000000..ea3c98ebe --- /dev/null +++ b/client/dive-common/stereoDisplay.ts @@ -0,0 +1,58 @@ +/** + * Entries of the viewer's camera selector. A stereo pair offers, per selected + * camera: both cameras, that camera beside the 3D viewer (which takes the other + * camera's pane), or both cameras with the 3D viewer as a third pane. + */ +export type Viewer3dMode = 'off' | 'replace' | 'beside'; + +export interface DisplayOption { + value: string; + text: string; + menuText: string; +} + +const PREFIXES: Record = { + off: 'camera:', + replace: 'viewer3d:', + beside: 'stereo3d:', +}; +const MODES = Object.keys(PREFIXES) as Viewer3dMode[]; + +export function displayValue(camera: string, mode: Viewer3dMode) { + return `${PREFIXES[mode]}${camera}`; +} + +export function parseDisplayValue(value: string): { camera: string; mode: Viewer3dMode } { + const mode = MODES.find((candidate) => value.startsWith(PREFIXES[candidate])) ?? 'off'; + return { camera: value.slice(PREFIXES[mode].length), mode }; +} + +function capitalize(name: string) { + return name.charAt(0).toUpperCase() + name.slice(1); +} + +function stereoLabel(camera: string, mode: Viewer3dMode) { + const name = capitalize(camera); + if (mode === 'replace') { + return `${name} + 3D`; + } + return mode === 'beside' ? `Stereo (${name}) + 3D` : `Stereo (${name})`; +} + +export function displayOptions( + cameras: readonly string[], + stereo: boolean, + defaultCamera: string, +): DisplayOption[] { + if (!stereo) { + return cameras.map((camera) => ({ + value: displayValue(camera, 'off'), + text: camera, + menuText: camera === defaultCamera ? `${camera} (Default)` : camera, + })); + } + return MODES.flatMap((mode) => cameras.map((camera) => { + const text = stereoLabel(camera, mode); + return { value: displayValue(camera, mode), text, menuText: text }; + })); +} diff --git a/client/dive-common/store/context.ts b/client/dive-common/store/context.ts index f2b805c74..d4e1807d7 100644 --- a/client/dive-common/store/context.ts +++ b/client/dive-common/store/context.ts @@ -7,6 +7,7 @@ import AttributesSideBar from 'dive-common/components/Attributes/AttributesSideB import MultiCamTools from 'dive-common/components/MultiCamTools.vue'; import RegistrationTools from 'dive-common/components/CameraRegistration/RegistrationTools.vue'; import AttributeTrackFilters from 'vue-media-annotator/components/AttributeTrackFilters.vue'; +import TrackViewerSettings from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettings.vue'; import DatasetInfo from 'dive-common/components/DatasetInfo/DatasetInfo.vue'; interface ContextState { @@ -62,6 +63,10 @@ const componentMapEntries: ComponentMapItem[] = [ description: 'Attribute Track Filters', component: AttributeTrackFilters, }, + { + description: '3D Viewer Settings', + component: TrackViewerSettings, + }, ]; const componentMap: Record = Object.fromEntries( diff --git a/client/package-lock.json b/client/package-lock.json index 48aee6f02..0be48652d 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -12,6 +12,7 @@ "@flatten-js/interval-tree": "^1.0.11", "@girder/components": "github:girder/girder_web_components#girder-5-axios-upgrade", "@huggingface/transformers": "^4.2.0", + "@kitware/vtk.js": "^25.6.0", "@mdi/font": "^6.2.95", "@sentry/browser": "^5.24.2", "@sentry/integrations": "^5.24.2", @@ -2941,9 +2942,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2960,9 +2958,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2979,9 +2974,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2998,9 +2990,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3017,9 +3006,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3036,9 +3022,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3055,9 +3038,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3074,9 +3054,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3093,9 +3070,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3118,9 +3092,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3143,9 +3114,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3168,9 +3136,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3193,9 +3158,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3218,9 +3180,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3243,9 +3202,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3268,9 +3224,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3548,6 +3501,155 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@kitware/vtk.js": { + "version": "25.15.2", + "resolved": "https://registry.npmjs.org/@kitware/vtk.js/-/vtk.js-25.15.2.tgz", + "integrity": "sha512-bekuiKfB2O9IRJBGXScnWDqA1nT9yAoJJ9t/GTritdrhx4B9kwtcj8sc4MTX000UN7sVruyHoM/NF5qeKvx+fQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/runtime": "7.17.9", + "commander": "9.2.0", + "d3-scale": "4.0.2", + "fast-deep-equal": "^3.1.3", + "fflate": "0.7.3", + "gl-matrix": "3.4.3", + "globalthis": "1.0.3", + "seedrandom": "3.0.5", + "shader-loader": "1.3.1", + "shelljs": "0.8.5", + "spark-md5": "3.0.2", + "stream-browserify": "3.0.0", + "webworker-promise": "0.5.0", + "worker-loader": "3.0.8", + "xmlbuilder2": "3.0.2" + }, + "bin": { + "vtkDataConverter": "Utilities/DataGenerator/convert-cli.js", + "xml2json": "Utilities/XMLConverter/xml2json-cli.js" + }, + "peerDependencies": { + "wslink": "^1.1.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@babel/runtime": { + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", + "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.13.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/dom": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@oozcitak/dom/-/dom-1.15.10.tgz", + "integrity": "sha512-0JT29/LaxVgRcGKvHmSrUTEvZ8BXvZhGl2LASRUgHqDTC1M5g1pLmVv56IYNyt3bG2CUjDkc67wnyZC14pbQrQ==", + "license": "MIT", + "dependencies": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/url": "1.0.4", + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/infra": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@oozcitak/infra/-/infra-1.0.8.tgz", + "integrity": "sha512-JRAUc9VR6IGHOL7OGF+yrvs0LO8SlqGnPAMqyzOuFZPSZSXI7Xf2O9+awQPSMXgIWGtgUf/dA6Hs6X6ySEaWTg==", + "license": "MIT", + "dependencies": { + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/url": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@oozcitak/url/-/url-1.0.4.tgz", + "integrity": "sha512-kDcD8y+y3FCSOvnBI6HJgl00viO/nGbQoCINmQ0h98OhnGITrWR3bOGfwYCthgcrV8AnTJz8MzslTQbC3SOAmw==", + "license": "MIT", + "dependencies": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/util": { + "version": "8.3.8", + "resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.3.8.tgz", + "integrity": "sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==", + "license": "MIT", + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/commander": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.2.0.tgz", + "integrity": "sha512-e2i4wANQiSXgnrBlIatyHtP1odfUp0BbV5Y5nEGbxtIrStkEOAAzCUirvLBNXHLr7kwLvJl6V+4V3XV9x7Wd9w==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || >=14" + } + }, + "node_modules/@kitware/vtk.js/node_modules/globalthis": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz", + "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/@kitware/vtk.js/node_modules/js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@kitware/vtk.js/node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, + "node_modules/@kitware/vtk.js/node_modules/xmlbuilder2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/xmlbuilder2/-/xmlbuilder2-3.0.2.tgz", + "integrity": "sha512-h4MUawGY21CTdhV4xm3DG9dgsqyhDkZvVJBx88beqX8wJs3VgyGQgAn5VreHuae6unTQxh115aMK5InCVmOIKw==", + "license": "MIT", + "dependencies": { + "@oozcitak/dom": "1.15.10", + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8", + "@types/node": "*", + "js-yaml": "3.14.0" + }, + "engines": { + "node": ">=12.0" + } + }, "node_modules/@malept/cross-spawn-promise": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz", @@ -5164,7 +5266,6 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, "license": "MIT" }, "node_modules/@types/json5": { @@ -6321,7 +6422,6 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -6376,6 +6476,15 @@ "dev": true, "license": "MIT" }, + "node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, "node_modules/alien-signals": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/alien-signals/-/alien-signals-0.2.2.tgz", @@ -6799,6 +6908,15 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -7369,6 +7487,15 @@ "require-from-string": "^2.0.2" } }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -7971,7 +8098,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "devOptional": true, "license": "MIT" }, "node_modules/condense-newlines": { @@ -9442,6 +9568,15 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, + "node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -10271,6 +10406,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", @@ -10469,7 +10617,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "devOptional": true, "license": "MIT" }, "node_modules/fast-fifo": { @@ -10512,7 +10659,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, "license": "MIT" }, "node_modules/fast-levenshtein": { @@ -10571,8 +10717,7 @@ "version": "0.7.3", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", "integrity": "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/ffmpeg-ffprobe-static": { "version": "6.1.2-rc.1", @@ -10818,7 +10963,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "devOptional": true, "license": "ISC" }, "node_modules/fsevents": { @@ -11043,8 +11187,7 @@ "version": "3.4.3", "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.3.tgz", "integrity": "sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/gl-vec3": { "version": "1.1.3", @@ -11062,7 +11205,6 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "devOptional": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -11565,7 +11707,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "devOptional": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -11614,7 +11755,6 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.10" } @@ -11729,7 +11869,6 @@ "version": "2.16.2", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", - "devOptional": true, "license": "MIT", "dependencies": { "hasown": "^2.0.3" @@ -12441,7 +12580,6 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { @@ -12463,7 +12601,6 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, "license": "MIT", "bin": { "json5": "lib/cli.js" @@ -12644,6 +12781,32 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/loader-utils": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.2.tgz", + "integrity": "sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg==", + "license": "MIT", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^1.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/loader-utils/node_modules/json5": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, "node_modules/localforage": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.8.1.tgz", @@ -12923,7 +13086,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "devOptional": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -12936,7 +13098,6 @@ "version": "1.1.18", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", - "devOptional": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -12947,7 +13108,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -13424,7 +13584,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "devOptional": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -13643,7 +13802,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "devOptional": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -13653,7 +13811,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "devOptional": true, "license": "MIT" }, "node_modules/path-scurry": { @@ -14079,7 +14236,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -14208,7 +14364,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -14257,7 +14412,6 @@ "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "optional": true, "dependencies": { "resolve": "^1.1.6" }, @@ -14413,7 +14567,6 @@ "version": "1.22.12", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", - "devOptional": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -14712,12 +14865,29 @@ "node": ">=v12.22.7" } }, + "node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/seedrandom": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/semver": { "version": "7.7.4", @@ -14885,6 +15055,15 @@ "dev": true, "license": "ISC" }, + "node_modules/shader-loader": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/shader-loader/-/shader-loader-1.3.1.tgz", + "integrity": "sha512-dt8F9K0x4rjmaFyHh7rNDfpt4LUiR64zhNIEwp2WbE99B3z4ALuvvmhftkElg93dUD6sTmv/aXa/z9SJiEddcA==", + "license": "MIT", + "dependencies": { + "loader-utils": "^1.1.0" + } + }, "node_modules/sharp": { "version": "0.35.4", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", @@ -14951,7 +15130,6 @@ "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "license": "BSD-3-Clause", - "optional": true, "dependencies": { "glob": "^7.0.0", "interpret": "^1.0.0", @@ -15106,8 +15284,13 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/spark-md5/-/spark-md5-3.0.2.tgz", "integrity": "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==", - "license": "(WTFPL OR MIT)", - "optional": true + "license": "(WTFPL OR MIT)" + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" }, "node_modules/stackback": { "version": "0.0.2", @@ -15143,6 +15326,16 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, "node_modules/streamx": { "version": "2.28.0", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.0.tgz", @@ -15349,7 +15542,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -16168,7 +16360,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" @@ -16871,8 +17062,7 @@ "version": "0.5.0", "resolved": "https://registry.npmjs.org/webworker-promise/-/webworker-promise-0.5.0.tgz", "integrity": "sha512-14iR79jHAV7ozwvbfif+3wCaApT3I1g8Lo0rJZrwAu6wxZGx/08Y8KXz6as6ZLNUEEufeiEBBYrqyDBClXOsEw==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/whatwg-mimetype": { "version": "5.0.0", @@ -17041,6 +17231,40 @@ "node": ">=0.10.0" } }, + "node_modules/worker-loader": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.8.tgz", + "integrity": "sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==", + "license": "MIT", + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/worker-loader/node_modules/loader-utils": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", + "license": "MIT", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -17081,7 +17305,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "devOptional": true, "license": "ISC" }, "node_modules/wslink": { diff --git a/client/package.json b/client/package.json index b608ff827..38243c403 100644 --- a/client/package.json +++ b/client/package.json @@ -35,6 +35,7 @@ "@flatten-js/interval-tree": "^1.0.11", "@girder/components": "github:girder/girder_web_components#girder-5-axios-upgrade", "@huggingface/transformers": "^4.2.0", + "@kitware/vtk.js": "^25.6.0", "@mdi/font": "^6.2.95", "@sentry/browser": "^5.24.2", "@sentry/integrations": "^5.24.2", diff --git a/client/src/components/track_3d_viewer/TrackManager.ts b/client/src/components/track_3d_viewer/TrackManager.ts new file mode 100644 index 000000000..747ba38c1 --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackManager.ts @@ -0,0 +1,134 @@ +import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import { RGBColor } from './lookupTable'; + +export type FrameNumber = number; + +export type TrackTracker = { + trackActor: vtkActor; + hidden: boolean; // true if the track should remain hidden whatever happens + detectionsMap: Map; + trackColor: RGBColor; + trackType: string; +}; + +export type TracksMap = Map; + +export type FramesMap = Map; + +export default class TrackManager { + tracksMap: TracksMap; + + framesMap: FramesMap; + + constructor() { + this.tracksMap = new Map(); + this.framesMap = new Map(); + } + + registerTrack( + trackId: AnnotationId, + trackActor: vtkActor, + frameDetections: Array<[FrameNumber, vtkActor]>, + trackColor: RGBColor, + trackType: string, + ) { + const detectionsMap = new Map(); + + frameDetections.forEach(([frameNumber, actor]) => { + actor.getProperty().setDiffuse(0.6); + actor.getProperty().setAmbient(0.6); + detectionsMap.set(frameNumber, actor); + + const frameTracker = this.framesMap.get(frameNumber); + + if (frameTracker) { + frameTracker.trackIds.push(trackId); + frameTracker.detectionActors.push(actor); + } else { + this.framesMap.set(frameNumber, { + trackIds: [trackId], + detectionActors: [actor], + }); + } + }); + + this.tracksMap.set(trackId, { + trackActor, + detectionsMap, + hidden: false, // track should not be hidden by default + trackColor, + trackType, + }); + } + + unregisterTrack(trackId: AnnotationId) { + const trackTracker = this.tracksMap.get(trackId); + if (!trackTracker) { + return undefined; + } + trackTracker.detectionsMap.forEach((actor, frameNumber) => { + const frameTracker = this.framesMap.get(frameNumber); + if (!frameTracker) { + return; + } + const index = frameTracker.detectionActors.indexOf(actor); + if (index !== -1) { + frameTracker.trackIds.splice(index, 1); + frameTracker.detectionActors.splice(index, 1); + } + if (frameTracker.trackIds.length === 0) { + this.framesMap.delete(frameNumber); + } + }); + this.tracksMap.delete(trackId); + return trackTracker; + } + + getFrameTracker(frameNumber: FrameNumber) { + return this.framesMap.get(frameNumber); + } + + getTrackFrameDetection(trackId: AnnotationId, frameNumber: FrameNumber) { + return this.tracksMap.get(trackId)?.detectionsMap.get(frameNumber); + } + + forEachTrack(cb: (track: TrackTracker, trackId: AnnotationId) => void) { + this.tracksMap.forEach(cb); + } + + getTrack(trackId: AnnotationId) { + return this.tracksMap.get(trackId); + } + + getAllTracks() { + return this.tracksMap; + } + + hasTrack(trackId: AnnotationId) { + return this.tracksMap.has(trackId); + } + + getTrackActorsInFrameRange(frameStart: number, frameCount: number) { + const trackActors: Set = new Set(); + + // eslint-disable-next-line no-plusplus + for (let frame = frameStart; frame < frameStart + frameCount; frame++) { + const frameTracker = this.framesMap.get(frame); + + if (frameTracker && frameTracker.trackIds.length > 0) { + frameTracker.trackIds.forEach((trackId) => { + trackActors.add( + + this.tracksMap.get(trackId)!.trackActor, + ); + }); + } + } + + return trackActors; + } +} diff --git a/client/src/components/track_3d_viewer/TrackViewer.vue b/client/src/components/track_3d_viewer/TrackViewer.vue new file mode 100644 index 000000000..608b79bf8 --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackViewer.vue @@ -0,0 +1,527 @@ + + + + + diff --git a/client/src/components/track_3d_viewer/TrackViewerSettings.vue b/client/src/components/track_3d_viewer/TrackViewerSettings.vue new file mode 100644 index 000000000..038a77c88 --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackViewerSettings.vue @@ -0,0 +1,142 @@ + + + diff --git a/client/src/components/track_3d_viewer/TrackViewerSettingsStore.ts b/client/src/components/track_3d_viewer/TrackViewerSettingsStore.ts new file mode 100644 index 000000000..ac5ba871f --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackViewerSettingsStore.ts @@ -0,0 +1,33 @@ +import { ref, Ref } from 'vue'; + +export default class TrackViewerSettingsStore { + cameraParallelProjection: Ref; + + onlyShowSelectedTrack: Ref; + + detectionGlyphSize: Ref; + + cubeAxesBounds: Ref<{ + xrange: [number, number]; + yrange: [number, number]; + zrange: [number, number]; + }>; + + adjustCubeAxesBoundsManually: Ref; + + showAxesBox: Ref; + + constructor() { + this.cameraParallelProjection = ref(false); + this.onlyShowSelectedTrack = ref(false); + this.detectionGlyphSize = ref(2); + + this.cubeAxesBounds = ref({ + xrange: [-1, 1], + yrange: [-1, 1], + zrange: [-1, 1], + }); + this.adjustCubeAxesBoundsManually = ref(false); + this.showAxesBox = ref(false); + } +} diff --git a/client/src/components/track_3d_viewer/lookupTable.ts b/client/src/components/track_3d_viewer/lookupTable.ts new file mode 100644 index 000000000..77324b737 --- /dev/null +++ b/client/src/components/track_3d_viewer/lookupTable.ts @@ -0,0 +1,44 @@ +/* eslint-disable no-plusplus */ + +import vtkLookupTable from '@kitware/vtk.js/Common/Core/LookupTable'; +import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray'; +import { float2rgb } from './utils'; + +export type RGBColor = [number, number, number]; + +export function buildLookupTable(color: RGBColor, currentFrame: number, frameCount = 10) { + // if outside of range, opacity = 0 + const lookupTable = vtkLookupTable.newInstance(); + + lookupTable.setUseBelowRangeColor(true); + lookupTable.setBelowRangeColor([...color, 0]); + lookupTable.setUseAboveRangeColor(true); + lookupTable.setAboveRangeColor([...color, 0]); + + const size = Math.min(currentFrame + 1, frameCount); + + const range = { + min: currentFrame - size + 1, + max: currentFrame, + }; + + const dataArray = vtkDataArray.newInstance({ + numberOfComponents: 4, + size: 4 * size, + dataType: 'Uint8Array', + }); + + // if range.min === range.max there is some weird things happening + // in the mapper during texture map generation. adding 0.01 is a hack to prevent this. + lookupTable.setRange(range.min, range.max + 0.01); + + let alpha = 255; + for (let i = size - 1; i >= 0; i--) { + const tuple = [...float2rgb(color), alpha]; + dataArray.setTuple(i, tuple); + alpha -= 20; + } + lookupTable.setTable(dataArray); + + return lookupTable; +} diff --git a/client/src/components/track_3d_viewer/picking.spec.ts b/client/src/components/track_3d_viewer/picking.spec.ts new file mode 100644 index 000000000..076404f0d --- /dev/null +++ b/client/src/components/track_3d_viewer/picking.spec.ts @@ -0,0 +1,24 @@ +import { ballAt } from './picking'; + +describe('ballAt', () => { + const balls = [ + { id: 7, center: [100, 100] as [number, number], radius: 20 }, + { id: 0, center: [130, 100] as [number, number], radius: 20 }, + { id: 3, center: [300, 300] as [number, number], radius: 2 }, + ]; + + it('picks the ball under the point, the nearest center where they overlap', () => { + expect(ballAt(balls, [95, 100], 6)).toBe(7); + expect(ballAt(balls, [118, 100], 6)).toBe(0); + }); + + it('gives small balls a minimum size to hit', () => { + expect(ballAt(balls, [305, 300], 6)).toBe(3); + expect(ballAt(balls, [308, 300], 6)).toBeNull(); + }); + + it('ignores points outside every ball', () => { + expect(ballAt(balls, [100, 125], 6)).toBeNull(); + expect(ballAt([], [0, 0], 6)).toBeNull(); + }); +}); diff --git a/client/src/components/track_3d_viewer/picking.ts b/client/src/components/track_3d_viewer/picking.ts new file mode 100644 index 000000000..da11a6775 --- /dev/null +++ b/client/src/components/track_3d_viewer/picking.ts @@ -0,0 +1,26 @@ +/** + * Picking a track by the ball drawn at its current position: a click inside a + * ball selects it, the nearest center winning where balls overlap. + */ +export type Point2 = [number, number]; + +export interface Ball { + id: Id; + /** In screen pixels. */ + center: Point2; + radius: number; +} + +export function ballAt(balls: Ball[], point: Point2, minimumRadius: number): Id | null { + let picked: Id | null = null; + let best = Infinity; + balls.forEach(({ id, center, radius }) => { + const distance = Math.hypot(point[0] - center[0], point[1] - center[1]); + // Far away balls shrink to a few pixels, too small to hit without some allowance + if (distance <= Math.max(radius, minimumRadius) && distance < best) { + best = distance; + picked = id; + } + }); + return picked; +} diff --git a/client/src/components/track_3d_viewer/positions.spec.ts b/client/src/components/track_3d_viewer/positions.spec.ts new file mode 100644 index 000000000..d607a818b --- /dev/null +++ b/client/src/components/track_3d_viewer/positions.spec.ts @@ -0,0 +1,47 @@ +import { featurePosition, robustBounds, Vec3 } from './positions'; + +describe('featurePosition', () => { + it('reads the midpoint, from numbers or numeric strings', () => { + expect(featurePosition({ midpoint_x: 1, midpoint_y: '2.5', midpoint_z: -3 })) + .toEqual({ x: 1, y: 2.5, z: -3 }); + }); + + it('includes head and tail only when both are complete', () => { + const midpoint = { midpoint_x: 1, midpoint_y: 1, midpoint_z: 1 }; + const head = { head_x: 0, head_y: 0, head_z: 0 }; + expect(featurePosition({ + ...midpoint, ...head, tail_x: 2, tail_y: 2, tail_z: 2, + })).toEqual({ + x: 1, y: 1, z: 1, head: [0, 0, 0], tail: [2, 2, 2], + }); + expect(featurePosition({ ...midpoint, ...head, tail_x: 2 })).toEqual({ x: 1, y: 1, z: 1 }); + }); + + it('falls back to the middle of head and tail without a midpoint', () => { + expect(featurePosition({ + head_x: 0, head_y: 0, head_z: 10, tail_x: 4, tail_y: 2, tail_z: 20, + })).toMatchObject({ x: 2, y: 1, z: 15 }); + }); + + it('is null without a usable position', () => { + expect(featurePosition(undefined)).toBeNull(); + expect(featurePosition({ length: 10 })).toBeNull(); + expect(featurePosition({ midpoint_x: 1, midpoint_y: '', midpoint_z: 3 })).toBeNull(); + }); +}); + +describe('robustBounds', () => { + it('spans all points when none is an outlier', () => { + expect(robustBounds([[0, 0, 10], [2, 4, 20], [1, 2, 15]])).toEqual([0, 2, 0, 4, 10, 20]); + }); + + it('leaves out points far from the rest', () => { + const points: Vec3[] = Array.from({ length: 20 }, (_, i) => [i, 0, 1000 + i]); + points.push([5, 0, 684541]); + expect(robustBounds(points)).toEqual([0, 19, 0, 0, 1000, 1019]); + }); + + it('is null without points', () => { + expect(robustBounds([])).toBeNull(); + }); +}); diff --git a/client/src/components/track_3d_viewer/positions.ts b/client/src/components/track_3d_viewer/positions.ts new file mode 100644 index 000000000..3c92afc2e --- /dev/null +++ b/client/src/components/track_3d_viewer/positions.ts @@ -0,0 +1,67 @@ +/** + * 3D positions of a detection, read from the attributes stereo measurement + * writes: the head-tail midpoint, and optionally the head and tail themselves. + */ +export type Vec3 = [number, number, number]; +export type Bounds3 = [number, number, number, number, number, number]; + +export interface Position { + x: number; + y: number; + z: number; + head?: Vec3; + tail?: Vec3; +} + +const AXES = ['x', 'y', 'z'] as const; + +function readPoint(attributes: Record, prefix: string): Vec3 | null { + const point = AXES.map((axis) => { + const value = attributes[`${prefix}_${axis}`]; + return value === undefined || value === null || value === '' ? NaN : Number(value); + }); + return point.every(Number.isFinite) ? point as Vec3 : null; +} + +export function featurePosition(attributes?: Record): Position | null { + if (!attributes) { + return null; + } + const head = readPoint(attributes, 'head'); + const tail = readPoint(attributes, 'tail'); + const ends = head && tail ? { head, tail } : {}; + const midpoint = readPoint(attributes, 'midpoint') + ?? (head && tail ? head.map((value, axis) => (value + tail[axis]) / 2) as Vec3 : null); + if (!midpoint) { + return null; + } + const [x, y, z] = midpoint; + return { + x, y, z, ...ends, + }; +} + +function quantile(sorted: number[], q: number) { + const index = (sorted.length - 1) * q; + const lower = Math.floor(index); + const upper = Math.ceil(index); + return sorted[lower] + (sorted[upper] - sorted[lower]) * (index - lower); +} + +/** + * Per-axis range of the points inside Tukey's fences, so a few mismatched + * detections triangulated far away do not shrink everything else to a dot. + */ +export function robustBounds(points: readonly Vec3[]): Bounds3 | null { + if (points.length === 0) { + return null; + } + return AXES.flatMap((_, axis) => { + const values = points.map((point) => point[axis]).sort((a, b) => a - b); + const q1 = quantile(values, 0.25); + const q3 = quantile(values, 0.75); + const reach = 1.5 * (q3 - q1); + const inside = values.filter((value) => value >= q1 - reach && value <= q3 + reach); + return [inside[0], inside[inside.length - 1]]; + }) as Bounds3; +} diff --git a/client/src/components/track_3d_viewer/sceneGuides.spec.ts b/client/src/components/track_3d_viewer/sceneGuides.spec.ts new file mode 100644 index 000000000..c5ce02c48 --- /dev/null +++ b/client/src/components/track_3d_viewer/sceneGuides.spec.ts @@ -0,0 +1,92 @@ +import { + floorGrid, niceStep, rigBaseline, rigCameras, +} from './sceneGuides'; + +const identity = [1, 0, 0, 0, 1, 0, 0, 0, 1]; + +describe('rigCameras', () => { + it('places only the left camera at the origin without a calibration', () => { + const cameras = rigCameras(null, 2); + expect(cameras.map((camera) => camera.name)).toEqual(['left']); + expect(cameras[0].center).toEqual([0, 0, 0]); + // The square front rests at depth 0, with the tip behind it + expect(cameras[0].corners.every((corner) => corner[2] === 0)).toBe(true); + expect(cameras[0].tip).toEqual([0, 0, -2]); + }); + + it('places the right camera at -Rt * T', () => { + const [, right] = rigCameras({ R: identity, T: [-300, 0, 0] }, 100); + expect(right.center).toEqual([300, 0, 0]); + // 90 degrees about y: Xr = R * Xl puts the left x axis on the right -z axis + const [, turned] = rigCameras({ R: [0, 0, 1, 0, 1, 0, -1, 0, 0], T: [0, 0, -300] }, 100); + expect(turned.center.map(Math.round)).toEqual([-300, 0, 0]); + }); + + it('sizes the frustum from the intrinsics', () => { + const [left] = rigCameras({ + R: identity, + T: [-300, 0, 0], + imageWidth: 2000, + imageHeight: 1000, + calibrations: { + left: { + fx: 1000, fy: 1000, cx: 1000, cy: 500, + }, + }, + }, 10); + expect(left.corners).toEqual([[-10, -5, 0], [10, -5, 0], [10, 5, 0], [-10, 5, 0]]); + }); + + it('matches VIAME for a measured rig', () => { + // A real calibration; VIAME places the right camera at -Rt * T + const [, right] = rigCameras({ + R: [ + 0.999887, 0.007012, 0.013281, + -0.006903, 0.999942, -0.008226, + -0.013337, 0.008134, 0.999878, + ], + T: [-288.926, -176.928, -17.388], + }, 100); + expect(right.center[0]).toBeCloseTo(287.4, 0); + expect(right.center[1]).toBeCloseTo(179.1, 0); + expect(right.center[2]).toBeCloseTo(19.8, 0); + }); + + it('reads R given as three rows and T as a column', () => { + const [, right] = rigCameras({ + R: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], + T: [[-300], [0], [0]], + }, 100); + expect(right.center).toEqual([300, 0, 0]); + }); + + it('ignores a malformed calibration', () => { + expect(rigCameras({ R: [1, 0, 0], T: [1, 2, 3] }, 1)).toHaveLength(1); + expect(rigBaseline({ R: identity, T: [3, 4, 0] })).toBe(5); + expect(rigBaseline(null)).toBe(0); + }); +}); + +describe('floorGrid', () => { + it('uses round spacing', () => { + expect(niceStep(6000)).toBe(1000); + expect(niceStep(3100)).toBe(500); + expect(niceStep(1.2)).toBeCloseTo(0.2); + expect(niceStep(0)).toBe(1); + }); + + it('spans the rig and the data, below the lowest point', () => { + const grid = floorGrid([-400, 1100, -100, 260, 1500, 3100], [[0, 0, 0], [300, 170, -200]]); + expect(grid.zs).toEqual([0, 500, 1000, 1500, 2000, 2500, 3000, 3500]); + expect(grid.xs).toEqual([-500, 0, 500, 1000, 1500]); + expect(grid.step).toBe(500); + // From above the highest point (y grows downward) to the last mark over the floor + expect(grid.ys).toEqual([-200, 0, 200]); + expect(grid.y).toBeCloseTo(260 + 0.02 * 3100); + }); + + it('marks at least three heights however flat the scene', () => { + expect(floorGrid([0, 5000, 10, 12, 1000, 9000], []).ys.length).toBeGreaterThanOrEqual(3); + expect(floorGrid([0, 5000, 40, 40, 1000, 9000], []).ys.length).toBeGreaterThanOrEqual(3); + }); +}); diff --git a/client/src/components/track_3d_viewer/sceneGuides.ts b/client/src/components/track_3d_viewer/sceneGuides.ts new file mode 100644 index 000000000..2e41c921a --- /dev/null +++ b/client/src/components/track_3d_viewer/sceneGuides.ts @@ -0,0 +1,148 @@ +/** + * Geometry of the 3D view's reference furniture: the stereo rig's cameras and + * a floor grid receding along the depth axis. Everything is in the left + * camera's frame (x right, y down, z forward), like the plotted positions. + */ +import { Bounds3, Vec3 } from './positions'; + +export interface RigCalibration { + /** Left-to-right rotation, row-major, and translation: Xr = R * Xl + T. */ + R: readonly number[] | readonly number[][]; + T: readonly number[] | readonly number[][]; + imageWidth?: number; + imageHeight?: number; + calibrations?: Record; +} + +export interface CameraFrustum { + name: string; + /** The optical center, where depth is measured from. */ + center: Vec3; + /** + * The diagram opens toward the scene, and its square front rests at the + * camera's depth, around the optical center; the tip sits behind it. Drawn + * this way it ends at depth 0 instead of reaching into the scene. + */ + tip: Vec3; + /** Top-left, top-right, bottom-right, bottom-left, as seen from the camera. */ + corners: [Vec3, Vec3, Vec3, Vec3]; +} + +export interface FloorGrid { + y: number; + step: number; + xs: number[]; + /** Heights marked on the vertical axis, from the top of the scene down to the floor. */ + ys: number[]; + zs: number[]; +} + +// Used when the calibration carries no intrinsics: about a 60 x 45 degree view. +const DEFAULT_HALF_EXTENT = [0.58, 0.41]; + +/** Calibration files store R flat or as three rows, and T flat or as a column. */ +function numbers(values: unknown, length: number): number[] | null { + const flat = Array.isArray(values) ? (values as unknown[]).flat(Infinity).map(Number) : []; + return flat.length === length && flat.every(Number.isFinite) ? flat : null; +} + +/** Distance between the two camera centers, or 0 without a usable calibration. */ +export function rigBaseline(calibration: RigCalibration | null) { + const T = numbers(calibration?.T, 3); + return T ? Math.hypot(...T) : 0; +} + +function cornerRays(calibration: RigCalibration | null, name: string): [number, number][] { + const intrinsics = calibration?.calibrations?.[name]; + const width = calibration?.imageWidth; + const height = calibration?.imageHeight; + const { + fx, fy, cx, cy, + } = intrinsics ?? {}; + if (fx && fy && width && height && cx !== undefined && cy !== undefined) { + return [[0, 0], [width, 0], [width, height], [0, height]] + .map(([u, v]) => [(u - cx) / fx, (v - cy) / fy]); + } + const [x, y] = DEFAULT_HALF_EXTENT; + return [[-x, -y], [x, -y], [x, y], [-x, y]]; +} + +/** The left camera sits at the origin; the right one only shows with a calibration. */ +export function rigCameras(calibration: RigCalibration | null, depth: number): CameraFrustum[] { + const frustum = (name: string, toLeft: (point: Vec3) => Vec3): CameraFrustum => ({ + name, + center: toLeft([0, 0, 0]), + tip: toLeft([0, 0, -depth]), + corners: cornerRays(calibration, name) + .map(([x, y]) => toLeft([x * depth, y * depth, 0])) as CameraFrustum['corners'], + }); + const cameras = [frustum('left', (point) => point)]; + const R = numbers(calibration?.R, 9); + const T = numbers(calibration?.T, 3); + if (R && T) { + // Xl = Rt * (Xr - T) + cameras.push(frustum('right', (point) => { + const shifted = point.map((value, axis) => value - T[axis]); + return [0, 1, 2].map((column) => ( + R[column] * shifted[0] + R[3 + column] * shifted[1] + R[6 + column] * shifted[2] + )) as Vec3; + })); + } + return cameras; +} + +/** A round spacing (1, 2 or 5 times a power of ten) giving about `count` intervals. */ +export function niceStep(extent: number, count = 6) { + if (!(extent > 0)) { + return 1; + } + const raw = extent / count; + const magnitude = 10 ** Math.floor(Math.log10(raw)); + const fraction = raw / magnitude; + let factor = 10; + if (fraction < 1.5) { + factor = 1; + } else if (fraction < 3.5) { + factor = 2; + } else if (fraction < 7.5) { + factor = 5; + } + return factor * magnitude; +} + +function ticks(min: number, max: number, step: number) { + const first = Math.floor(min / step); + const last = Math.ceil(max / step); + return Array.from({ length: last - first + 1 }, (_, i) => (first + i) * step); +} + +/** + * A grid under the data, reaching from the rig to the farthest position. + * y grows downward, so "under" is the largest y. Depth starts at the cameras: + * the grid is widened and lowered to take in `include`, never pulled back. + */ +export function floorGrid(bounds: Bounds3, include: readonly Vec3[]): FloorGrid { + const xs = [bounds[0], bounds[1], ...include.map((point) => point[0])]; + const ys = [bounds[2], bounds[3], ...include.map((point) => point[1])]; + const zs = [bounds[4], bounds[5], 0]; + const [xMin, xMax, zMin, zMax] = [Math.min(...xs), Math.max(...xs), Math.min(...zs), Math.max(...zs)]; + const extent = Math.max(xMax - xMin, zMax - zMin); + const step = niceStep(extent); + const y = Math.max(...ys) + 0.02 * extent; + // Heights span far less than the floor, so they get a spacing of their own, + // fine enough for the axis to carry at least three marks + const yMin = Math.min(...ys); + let heights: number[] = []; + for (let count = 2; heights.length < 3 && count <= 20; count += 1) { + heights = ticks(yMin, y, niceStep(y - yMin, count)).filter((value) => value <= y); + } + return { + step, + y, + xs: ticks(xMin, xMax, step), + ys: heights, + zs: ticks(zMin, zMax, step), + }; +} diff --git a/client/src/components/track_3d_viewer/trackUtils.ts b/client/src/components/track_3d_viewer/trackUtils.ts new file mode 100644 index 000000000..2672aca27 --- /dev/null +++ b/client/src/components/track_3d_viewer/trackUtils.ts @@ -0,0 +1,34 @@ +import { TrackWithContext } from 'vue-media-annotator/BaseFilterControls'; +import StyleManager from 'vue-media-annotator/StyleManager'; +import * as vtkMath from '@kitware/vtk.js/Common/Core/Math'; +import { Bounds3, Position } from './positions'; + +export interface ViewUtils { + rerender: (resetCamera?: boolean) => void; + /** Extent of the plotted positions, ignoring outliers; null when there are none. */ + sceneBounds: () => Bounds3 | null; +} + +export interface Feature extends Position { + frameNumber: number; +} + +export type TrackType = string; + +export function getTrackType(trackWithContext: TrackWithContext) { + const track = trackWithContext.annotation; + return track.getType( + trackWithContext.context.confidencePairIndex, + ); +} + +export function getTrackTypeColor(trackType: TrackType, styleManager: StyleManager) { + return vtkMath.hex2float( + styleManager.typeStyling.value.color(trackType), + ) as [number, number, number]; +} + +export function getTrackColor(trackWithContext: TrackWithContext, styleManager: StyleManager) { + const trackType = getTrackType(trackWithContext); + return getTrackTypeColor(trackType, styleManager); +} diff --git a/client/src/components/track_3d_viewer/useLabelDrawer.ts b/client/src/components/track_3d_viewer/useLabelDrawer.ts new file mode 100644 index 000000000..bd62f1d43 --- /dev/null +++ b/client/src/components/track_3d_viewer/useLabelDrawer.ts @@ -0,0 +1,93 @@ +/* eslint-disable import/prefer-default-export */ + +import vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; +import vtkOpenGLRenderWindow from '@kitware/vtk.js/Rendering/OpenGL/RenderWindow'; +import { Ref } from 'vue'; + +interface LabelDrawerParams { + renderer: Ref; + viewportDimensions: Ref<{ + width: number; + height: number; + }>; + openglRenderWindow: Ref; +} + +export function useLabelDrawer({ + renderer, + viewportDimensions, + openglRenderWindow, +}: LabelDrawerParams) { + let labelTextCanvas: HTMLCanvasElement | null = null; + let labelTextContext: CanvasRenderingContext2D | null = null; + + const initialize = function initialize() { + labelTextCanvas = document.createElement('canvas'); + labelTextCanvas.style.position = 'absolute'; + labelTextCanvas.style.top = '0'; + labelTextCanvas.style.right = '0'; + labelTextCanvas.style.bottom = '0'; + labelTextCanvas.style.left = '0'; + labelTextContext = labelTextCanvas.getContext('2d'); + + return { + labelTextCanvas, + }; + }; + const clearLabelContext = function clearLabelContext() { + if (!labelTextContext) { + throw new Error('Please initialize the drawer first.'); + } + + labelTextContext.clearRect( + 0, + 0, + viewportDimensions.value.width, + viewportDimensions.value.height, + ); + }; + /** Draws on top of what is already there; callers clear the canvas first. */ + const drawLabels = function drawLabels( + positionToLabel: Map<[number, number, number], string>, + { color, font }: { color: string; font: string }, + ) { + if (!labelTextContext) { + throw new Error('Please initialize the drawer first.'); + } + + const camera = renderer.value!.getActiveCamera(); + const eye = camera.getPosition(); + const direction = camera.getDirectionOfProjection(); + + positionToLabel.forEach((label, position) => { + // Points behind the viewer would otherwise project mirrored onto the view + const ahead = position.reduce( + (sum, value, axis) => sum + (value - eye[axis]) * direction[axis], + 0, + ); + if (ahead <= 0) { + return; + } + const displayCoordinates = openglRenderWindow.value!.worldToDisplay( + ...position, + renderer.value!, + ); + const x = displayCoordinates[0]; + const y = viewportDimensions.value.height - displayCoordinates[1]; + labelTextContext!.font = font; + labelTextContext!.textAlign = 'center'; + labelTextContext!.textBaseline = 'middle'; + labelTextContext!.lineWidth = 3; + labelTextContext!.strokeStyle = 'black'; + labelTextContext!.strokeText(label, x, y); + labelTextContext!.fillStyle = color; + labelTextContext!.fillText(label, x, y); + }); + }; + + return { + initialize, + drawLabels, + clearLabelContext, + }; +} diff --git a/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts new file mode 100644 index 000000000..bc2e76cb4 --- /dev/null +++ b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts @@ -0,0 +1,164 @@ +/* eslint-disable import/prefer-default-export */ +import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import vtkAppendPolyData from '@kitware/vtk.js/Filters/General/AppendPolyData'; +import vtkArrowSource from '@kitware/vtk.js/Filters/Sources/ArrowSource'; +import type vtkAxesActor from '@kitware/vtk.js/Rendering/Core/AxesActor'; +import vtkCylinderSource from '@kitware/vtk.js/Filters/Sources/CylinderSource'; +import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray'; +import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; +import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; +import vtkCamera from '@kitware/vtk.js/Rendering/Core/Camera'; +import vtkInteractiveOrientationWidget from '@kitware/vtk.js/Widgets/Widgets3D/InteractiveOrientationWidget'; +import vtkOrientationMarkerWidget from '@kitware/vtk.js/Interaction/Widgets/OrientationMarkerWidget'; +import vtkRenderWindowInteractor from '@kitware/vtk.js/Rendering/Core/RenderWindowInteractor'; +import vtkWidgetManager from '@kitware/vtk.js/Widgets/Core/WidgetManager'; +import * as vtkMath from '@kitware/vtk.js/Common/Core/Math'; +import type { Vector3 } from '@kitware/vtk.js/types'; +import { ref } from 'vue'; + +const getMajorAxisFromViewUp = function getMajorAxisFromViewUp( + viewUp: Vector3, + idxA: number, + idxB: number, +): Vector3 { + const axis: Vector3 = [0, 0, 0]; + const idx = Math.abs(viewUp[idxA]) > Math.abs(viewUp[idxB]) ? idxA : idxB; + const value = viewUp[idx] > 0 ? 1 : -1; + axis[idx] = value; + return axis; +}; + +const AXIS_RADIUS = 0.03; + +function colored(polyData: vtkPolyData, color: [number, number, number]) { + const count = polyData.getPoints().getNumberOfPoints(); + const values = new Uint8Array(3 * count); + for (let i = 0; i < count; i += 1) { + values.set(color, 3 * i); + } + polyData.getPointData().setScalars(vtkDataArray.newInstance({ + name: 'color', numberOfComponents: 3, values, + })); + return polyData; +} + +/** + * The three axes as plain bars through the center: which way x and z run says + * nothing about the scene. Only y carries an arrow, pointing up, because y + * grows downward in the cameras' frame and "up" is worth marking. + */ +function buildAxes() { + const bar = (direction: Vector3) => vtkCylinderSource.newInstance({ + direction, height: 1, radius: AXIS_RADIUS, resolution: 60, + }).getOutputData(); + const up = vtkArrowSource.newInstance({ + direction: [0, -1, 0], + tipResolution: 60, + tipRadius: 0.1, + tipLength: 0.2, + shaftResolution: 60, + shaftRadius: AXIS_RADIUS, + }).getOutputData(); + // The tip makes the arrow lopsided; center its length on the origin so the + // three axes cross in the middle + const points = up.getPoints().getData(); + const [, , yMin, yMax] = up.getPoints().getBounds(); + for (let i = 1; i < points.length; i += 3) { + points[i] -= (yMin + yMax) / 2; + } + + const source = vtkAppendPolyData.newInstance(); + source.setInputData(colored(bar([1, 0, 0]), [255, 0, 0])); + source.addInputData(colored(up, [255, 255, 0])); + source.addInputData(colored(bar([0, 0, 1]), [0, 128, 0])); + const mapper = vtkMapper.newInstance(); + mapper.setInputConnection(source.getOutputPort()); + const actor = vtkActor.newInstance(); + actor.setMapper(mapper); + return actor; +} + +export function useOrientationMarkerWidget() { + const previousVisibility = ref(false); + + const widgetManager = vtkWidgetManager.newInstance(); + const axes = buildAxes(); + const orientationMarkerWidget = vtkOrientationMarkerWidget.newInstance({ + // The widget only needs something to draw; its typings name two actors + actor: axes as unknown as vtkAxesActor, + }); + + const interactiveOrientationWidget = vtkInteractiveOrientationWidget.newInstance(); + + const enable = function enable( + interactor: vtkRenderWindowInteractor, + camera: vtkCamera, + onOrientationChanged: () => void, + ) { + orientationMarkerWidget.setInteractor(interactor); + orientationMarkerWidget.setEnabled(true); + // Clear of the axes, which meet at the bottom left of the opening view + orientationMarkerWidget.setViewportCorner(vtkOrientationMarkerWidget.Corners.TOP_LEFT); + orientationMarkerWidget.setViewportSize(0.15); + orientationMarkerWidget.setMinPixelSize(100); + orientationMarkerWidget.setMaxPixelSize(300); + + const widgetRenderer = orientationMarkerWidget.getRenderer(); + widgetManager.setRenderer(widgetRenderer); + + interactiveOrientationWidget.placeWidget(axes.getBounds()); + interactiveOrientationWidget.setBounds(axes.getBounds()); + interactiveOrientationWidget.setPlaceFactor(1); + + const viewWidget = widgetManager.addWidget(interactiveOrientationWidget); + + if (!viewWidget) { + throw new Error('Failed to add widget to view.'); + } + + // @ts-expect-error it exists + viewWidget.onOrientationChange(({ direction }) => { + const focalPoint = camera.getFocalPoint(); + const position = camera.getPosition(); + const viewUp = camera.getViewUp(); + + const distance = Math.sqrt(vtkMath.distance2BetweenPoints(position, focalPoint)); + + // Put the camera orthogonally compared with the clicked cube's face + camera.setPosition( + focalPoint[0] + direction[0] * distance, + focalPoint[1] + direction[1] * distance, + focalPoint[2] + direction[2] * distance, + ); + + // Set the viewup on the camera so the cube appears aligned with the ground + if (direction[0]) { + camera.setViewUp(getMajorAxisFromViewUp(viewUp, 1, 2)); + } + if (direction[1]) { + camera.setViewUp(getMajorAxisFromViewUp(viewUp, 0, 2)); + } + if (direction[2]) { + camera.setViewUp(getMajorAxisFromViewUp(viewUp, 0, 1)); + } + + orientationMarkerWidget.updateMarkerOrientation(); + widgetManager.enablePicking(); + onOrientationChanged(); + }); + + previousVisibility.value = true; + }; + + const disable = function disable() { + orientationMarkerWidget.delete(); + widgetManager.removeWidget(interactiveOrientationWidget); + widgetManager.delete(); + axes.delete(); + }; + + return { + enable, + disable, + }; +} diff --git a/client/src/components/track_3d_viewer/useSceneGuides.ts b/client/src/components/track_3d_viewer/useSceneGuides.ts new file mode 100644 index 000000000..7b91ecdfa --- /dev/null +++ b/client/src/components/track_3d_viewer/useSceneGuides.ts @@ -0,0 +1,161 @@ +import { Ref } from 'vue'; +import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import vtkCellArray from '@kitware/vtk.js/Common/Core/CellArray'; +import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; +import vtkPoints from '@kitware/vtk.js/Common/Core/Points'; +import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; +import vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; +import { Bounds3, Vec3 } from './positions'; +import { + floorGrid, rigBaseline, rigCameras, RigCalibration, +} from './sceneGuides'; + +export interface GuideLabel { + position: Vec3; + text: string; + color: string; +} + +type Segment = [Vec3, Vec3]; + +const GRID_COLOR: Vec3 = [0.3, 0.3, 0.3]; +const RIG_COLOR: Vec3 = [1, 1, 1]; +// The orientation cube's colors, so the two read as the same axes +const AXIS_COLORS: Record<'x' | 'y' | 'z', Vec3> = { + x: [1, 0.25, 0.25], + y: [1, 0.9, 0.2], + z: [0.3, 0.85, 0.35], +}; +const OPTICAL_AXIS_COLOR: Vec3 = [0.15, 0.4, 0.18]; +const css = (color: Vec3) => `rgb(${color.map((value) => Math.round(255 * value)).join(',')})`; +// Shown before any position exists, in units of the rig's baseline. +const EMPTY_SCENE_DEPTHS = 10; + +function lineActor(segments: Segment[], color: Vec3, width: number) { + const points = vtkPoints.newInstance(); + points.setNumberOfPoints(segments.length * 2); + const cells = new Uint32Array(segments.length * 3); + segments.forEach(([from, to], i) => { + points.setPoint(2 * i, ...from); + points.setPoint(2 * i + 1, ...to); + cells.set([2, 2 * i, 2 * i + 1], 3 * i); + }); + const polyData = vtkPolyData.newInstance(); + polyData.setPoints(points); + polyData.setLines(vtkCellArray.newInstance({ values: cells })); + const mapper = vtkMapper.newInstance(); + mapper.setInputData(polyData); + const actor = vtkActor.newInstance(); + actor.setMapper(mapper); + actor.getProperty().setColor(...color); + actor.getProperty().setLineWidth(width); + actor.getProperty().setLighting(false); + return actor; +} + +/** Draws the stereo rig and a floor grid, and lists the labels that go with them. */ +export default function useSceneGuides(renderer: Ref) { + let actors: vtkActor[] = []; + let labels: GuideLabel[] = []; + let extent: Bounds3 = [-1, 1, -1, 1, 0, 1]; + + const clear = function clear() { + actors.forEach((actor) => { + renderer.value?.removeActor(actor); + actor.delete(); + }); + actors = []; + labels = []; + }; + + const build = function build(sceneBounds: Bounds3 | null, calibration: RigCalibration | null) { + clear(); + const baseline = rigBaseline(calibration) || 1; + const bounds: Bounds3 = sceneBounds ?? [ + -baseline, baseline, -baseline, baseline, 0, EMPTY_SCENE_DEPTHS * baseline, + ]; + const size = Math.max(bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4]); + const cameras = rigCameras(calibration, Math.max(0.6 * baseline, 0.04 * size)); + const rigPoints = cameras.flatMap(({ center, tip, corners }) => [center, tip, ...corners]); + + const grid = floorGrid(bounds, rigPoints); + const [xFirst, xLast] = [grid.xs[0], grid.xs[grid.xs.length - 1]]; + const [zFirst, zLast] = [grid.zs[0], grid.zs[grid.zs.length - 1]]; + const yTop = grid.ys[0] ?? grid.y; + const gridSegments: Segment[] = [ + ...grid.xs.map((x): Segment => [[x, grid.y, zFirst], [x, grid.y, zLast]]), + ...grid.zs.map((z): Segment => [[xFirst, grid.y, z], [xLast, grid.y, z]]), + ]; + // The line on the floor under the left camera's optical axis + const opticalAxis: Segment[] = xFirst < 0 && xLast > 0 + ? [[[0, grid.y, zFirst], [0, grid.y, zLast]]] : []; + const rigSegments = cameras.flatMap(({ tip, corners }) => corners.flatMap( + (corner, i): Segment[] => [[tip, corner], [corner, corners[(i + 1) % corners.length]]], + )); + // The three axes meet at the grid's near left corner, out of the way of the data + const tick = 0.15 * grid.step; + const axes: Record<'x' | 'y' | 'z', Segment[]> = { + x: [[[xFirst, grid.y, zFirst], [xLast, grid.y, zFirst]]], + y: [ + [[xFirst, grid.y, zFirst], [xFirst, yTop, zFirst]], + ...grid.ys.map((y): Segment => [[xFirst - tick, y, zFirst], [xFirst + tick, y, zFirst]]), + ], + z: [[[xFirst, grid.y, zFirst], [xFirst, grid.y, zLast]]], + }; + + actors = [ + lineActor(gridSegments, GRID_COLOR, 1), + lineActor(opticalAxis, OPTICAL_AXIS_COLOR, 2), + lineActor(rigSegments, RIG_COLOR, 2), + lineActor(axes.x, AXIS_COLORS.x, 3), + lineActor(axes.y, AXIS_COLORS.y, 3), + lineActor(axes.z, AXIS_COLORS.z, 3), + ]; + actors.forEach((actor) => renderer.value?.addActor(actor)); + + // Labels sit just outside the grid so they stay clear of its lines and of each other + const gap = 0.4 * grid.step; + const number = (value: number) => value.toLocaleString(); + labels = [ + ...grid.xs.map((x) => ({ + position: [x, grid.y, zFirst - gap] as Vec3, text: number(x), color: css(AXIS_COLORS.x), + })), + { position: [xLast + 2 * gap, grid.y, zFirst] as Vec3, text: 'X', color: css(AXIS_COLORS.x) }, + ...grid.ys.map((y) => ({ + position: [xFirst - gap, y, zFirst] as Vec3, text: number(y), color: css(AXIS_COLORS.y), + })), + { + position: [xFirst, yTop - gap, zFirst] as Vec3, text: 'Y', color: css(AXIS_COLORS.y), + }, + ...grid.zs.filter((z) => z !== zFirst).map((z) => ({ + position: [xFirst - gap, grid.y, z] as Vec3, text: number(z), color: css(AXIS_COLORS.z), + })), + { + position: [xFirst, grid.y, zLast + 2 * gap] as Vec3, + text: 'Z (depth)', + color: css(AXIS_COLORS.z), + }, + ...cameras.map(({ name, center }) => ({ + position: center, + text: name.charAt(0).toUpperCase() + name.slice(1), + color: css(RIG_COLOR), + })), + ]; + + const ys = [bounds[2], yTop, grid.y, ...rigPoints.map((point) => point[1])]; + // Room for the labels around the grid, and for the rig behind its near edge + extent = [ + xFirst - 3 * gap, xLast + 3 * gap, + Math.min(...ys) - gap, Math.max(...ys), + Math.min(zFirst - 2 * gap, ...rigPoints.map((point) => point[2])), zLast + 3 * gap, + ]; + }; + + return { + build, + clear, + labels: () => labels, + /** Everything worth framing: the data, the grid under it and the rig. */ + extent: () => extent, + }; +} diff --git a/client/src/components/track_3d_viewer/useTrackDrawer.ts b/client/src/components/track_3d_viewer/useTrackDrawer.ts new file mode 100644 index 000000000..e51c5d482 --- /dev/null +++ b/client/src/components/track_3d_viewer/useTrackDrawer.ts @@ -0,0 +1,591 @@ +import { Ref, watch } from 'vue'; +import { + useTrackFilters, + useSelectedTrackId, + useTrackStyleManager, + useCameraStore, +} from 'vue-media-annotator/provides'; +import * as vtkMath from '@kitware/vtk.js/Common/Core/Math'; +import { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import { TrackWithContext } from 'vue-media-annotator/BaseFilterControls'; +import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray'; +import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; +import vtkPoints from '@kitware/vtk.js/Common/Core/Points'; +import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; +import vtkSphereSource from '@kitware/vtk.js/Filters/Sources/SphereSource'; +import vtkConeSource from '@kitware/vtk.js/Filters/Sources/ConeSource'; +import vtkLineSource from '@kitware/vtk.js/Filters/Sources/LineSource'; +import vtkAppendPolyData from '@kitware/vtk.js/Filters/General/AppendPolyData'; +import vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; +import vtkCellArray from '@kitware/vtk.js/Common/Core/CellArray'; +import vtkLookupTable from '@kitware/vtk.js/Common/Core/LookupTable'; +import TrackManager, { TrackTracker } from './TrackManager'; +import { injectAggregateController } from '../annotators/useMediaController'; +import { + getTrackColor, getTrackType, getTrackTypeColor, ViewUtils, Feature, +} from './trackUtils'; +import { buildLookupTable } from './lookupTable'; +import { + Bounds3, featurePosition, robustBounds, Vec3, +} from './positions'; + +export interface TrackDrawerParams { + trackManager: TrackManager; + onlyShowSelectedTrack: Ref; + detectionGlyphSize: Ref; + viewUtils: ViewUtils; + renderer: Ref; +} + +interface TrackEntry { + trackWithContext: TrackWithContext; + revision: string; +} + +interface HeadMarker { + cone: vtkConeSource; + // Relative to the detection's midpoint, where its actor is positioned. + head: Vec3; + direction: Vec3; +} + +export default function useTrackDrawer({ + trackManager, + onlyShowSelectedTrack, + detectionGlyphSize, + viewUtils, + renderer, +}: TrackDrawerParams) { + const selectedTrackIdRef = useSelectedTrackId(); + const mediaController = injectAggregateController(); + const trackStyleManager = useTrackStyleManager(); + const filteredTracksRef = useTrackFilters(); + const cameraStore = useCameraStore(); + const { frame: frameRef } = mediaController.value; + const trackTypes: string[] = []; + const trackKeys = new Map(); + const trackPositions = new Map(); + const headMarkers = new Map(); + let sceneBounds: Bounds3 | null = null; + + // We uses a sphere to represents a detection in space + const detectionGlyphSource = vtkSphereSource.newInstance(); + + /** The cone's tip sits on the head, pointing the way the animal faces. */ + const sizeHeadMarker = function sizeHeadMarker({ cone, head, direction }: HeadMarker) { + const radius = detectionGlyphSource.getRadius(); + const height = 3 * radius; + cone.setRadius(radius); + cone.setHeight(height); + cone.setCenter(...head.map((value, axis) => value - (direction[axis] * height) / 2) as Vec3); + }; + + /** + * Positions are in the calibration's units, so the glyph is sized relative + * to the extent of the data rather than by an absolute radius. + */ + const updateGlyphRadius = function updateGlyphRadius() { + const points: Vec3[] = []; + trackPositions.forEach((features) => features.forEach(({ + x, y, z, head, tail, + }) => { + points.push([x, y, z], ...(head && tail ? [head, tail] : [])); + })); + sceneBounds = robustBounds(points); + const extent = sceneBounds + ? Math.max(...[0, 2, 4].map((axis) => (sceneBounds as Bounds3)[axis + 1] - (sceneBounds as Bounds3)[axis])) + : 0; + const distance = sceneBounds ? Math.max(...sceneBounds.map(Math.abs)) : 0; + let scale = 1; + if (extent > 0) { + scale = extent; + } else if (distance > 0) { + scale = distance; + } + detectionGlyphSource.setRadius((Number(detectionGlyphSize.value) / 100) * scale); + headMarkers.forEach(sizeHeadMarker); + }; + // eslint-disable-next-line no-param-reassign + viewUtils.sceneBounds = () => sceneBounds; + + watch(detectionGlyphSize, () => { + updateGlyphRadius(); + viewUtils.rerender(); + }, { + immediate: true, + }); + + detectionGlyphSource.setThetaResolution(20); + + const drawFeature = function drawFeature( + points: vtkPoints, + lines: vtkCellArray, + trackColor: [number, number, number], + { + x, y, z, head, tail, + }: Feature, + idx: number, + frameDataArray: vtkDataArray, + frameNumber: number, + ) { + points.setPoint( + idx, + x, + y, + z, + ); + + frameDataArray.setTuple(idx, [frameNumber]); + + if (idx > 0) { + // eslint-disable-next-line no-param-reassign + lines.getData()[idx] = idx - 1; + // eslint-disable-next-line no-param-reassign + lines.getData()[idx + 1] = idx; + } + + const pointMapper = vtkMapper.newInstance(); + const pointActor = vtkActor.newInstance(); + const length = head && tail ? Math.hypot(...head.map((value, axis) => value - tail[axis])) : 0; + + if (head && tail && length > 0) { + // Head and tail are known: draw the body as a segment through the midpoint + const middle = [x, y, z]; + const localHead = head.map((value, axis) => value - middle[axis]) as Vec3; + const localTail = tail.map((value, axis) => value - middle[axis]) as Vec3; + const marker: HeadMarker = { + cone: vtkConeSource.newInstance({ resolution: 12 }), + head: localHead, + direction: head.map((value, axis) => (value - tail[axis]) / length) as Vec3, + }; + marker.cone.setDirection(...marker.direction); + sizeHeadMarker(marker); + headMarkers.set(pointActor, marker); + + const body = vtkLineSource.newInstance({ point1: localHead, point2: localTail }); + const glyph = vtkAppendPolyData.newInstance(); + glyph.setInputConnection(detectionGlyphSource.getOutputPort()); + glyph.addInputConnection(body.getOutputPort()); + glyph.addInputConnection(marker.cone.getOutputPort()); + pointMapper.setInputConnection(glyph.getOutputPort()); + pointActor.getProperty().setLineWidth(3); + } else { + pointMapper.setInputConnection(detectionGlyphSource.getOutputPort()); + } + + pointActor.setPosition(Number(x), Number(y), Number(z)); + pointActor.setMapper(pointMapper); + pointActor.getProperty().setColor(...trackColor); + pointActor.setVisibility(false); + + if (renderer.value) { + renderer.value.addActor(pointActor); + } + + return pointActor; + }; + + /** + * A 3D track is vtkPolyData made of one single line. + * Each point in this line represents a single detection. + * The points are ordered by frame number. + * We also store a point data field array to store the exact frame number of each point. + */ + const drawTrack = function drawTrack( + mapper: vtkMapper, + trackColor: [number, number, number], + features: Array<{x: number; y: number; z: number; frameNumber: number}>, + ) { + const numberOfFeatures = features.length; + // One point stands for one detection + const points = vtkPoints.newInstance(); + const trackPolyData = vtkPolyData.newInstance(); + + const frameDataArray = vtkDataArray.newInstance({ + numberOfComponents: 1, + size: numberOfFeatures, + dataType: 'Uint32Array', + name: 'frames', + }); + + const trackActor = vtkActor.newInstance(); + const trackActorProperty = trackActor.getProperty(); + points.setNumberOfPoints(numberOfFeatures); + + const lines = vtkCellArray.newInstance({ size: numberOfFeatures + 1 }); + lines.getData()[0] = numberOfFeatures; + + const frameDetections: [number, vtkActor][] = features + .map((feature, idx) => { + const featureActor = drawFeature( + points, + lines, + trackColor, + feature, + idx, + frameDataArray, + feature.frameNumber, + ); + return [feature.frameNumber, featureActor]; + }); + + trackPolyData.setPoints(points); + + // store the frame number of each point + trackPolyData.getPointData().setScalars(frameDataArray); + + mapper.setInputData(trackPolyData); + trackPolyData.setLines(lines); + + // trackActorProperty.setColor(...trackColor); + trackActorProperty.setLineWidth(3); + trackActor.setMapper(mapper); + + if (renderer.value) { + renderer.value.addActor(trackActor); + } + + return { + trackActor, + frameDetections, + }; + }; + + /** + * Emphasize on a specific track: make sure it is visible, increase its line width and + * set its color to selected + */ + const emphasizeTrack = function emphasizeTrack(trackId: AnnotationId) { + const trackActor = trackManager.getTrack(trackId)?.trackActor; + + if (!trackActor) { + return; + } + + trackActor.setVisibility(true); + trackActor.getProperty().setLineWidth(6); + + const selectedColor = vtkMath.hex2float( + trackStyleManager.stateStyles.selected.color, + ) as [number, number, number]; + + trackActor.getProperty().setColor(...selectedColor); + trackManager.getTrack(trackId)?.detectionsMap.forEach((actor) => { + actor.getProperty().setColor(...selectedColor); + actor.getProperty().setLineWidth(5); + }); + }; + + const deEmphasizeTrack = function deEmphasizeTrack(trackId: AnnotationId, trackActor: vtkActor) { + // Restore line width + trackActor.getProperty().setLineWidth(3); + const trackTracker = trackManager.getTrack(trackId); + trackTracker?.detectionsMap.forEach((actor) => { + actor.getProperty().setColor(...trackTracker.trackColor); + actor.getProperty().setLineWidth(3); + }); + + const trackWithContext = filteredTracksRef + .filteredAnnotations + .value + .find((track) => track.annotation.id === trackId); + + // If we do not find the track anymore in the list, + // this is because the track has been filtered out in the meantime + // In this case we cannot get the color, so we do nothing as the track will be hidden anyway + if (trackWithContext) { + const trackColor = getTrackColor(trackWithContext, trackStyleManager); + trackActor.getProperty().setColor(...trackColor); + } + }; + + const hideTrack = function hideTrack(trackMap: TrackTracker) { + trackMap.trackActor.setVisibility(false); + trackMap.detectionsMap.forEach((actor) => actor.setVisibility(false)); + }; + + /** + * Hide all tracks except the one matching the track id passed in the except param. + */ + const hideAllTracks = function hideAllTracks(except?: AnnotationId | null) { + trackManager.forEachTrack(({ trackActor, detectionsMap }, trackId) => { + if (trackId !== except) { + trackActor.setVisibility(false); + detectionsMap.forEach((sphereActor) => { + sphereActor.setVisibility(false); + }); + } + }); + }; + + /** + * Show all tracks, except the ones that are explicitly flagged as hidden + */ + const showAllTracks = function showAllTracks() { + trackManager.forEachTrack(({ trackActor, detectionsMap, hidden }) => { + if (hidden) { + return; + } + + trackActor.setVisibility(true); + detectionsMap.forEach((sphereActor, frameNumber) => { + if (frameNumber === frameRef.value) { + sphereActor.setVisibility(true); + } + }); + }); + }; + + /** + * One position per frame, from whichever camera's replica holds the + * measurement, ordered by frame. + */ + const trackFeatures = function trackFeatures(trackId: AnnotationId) { + const byFrame = new Map(); + cameraStore.getTrackAll(trackId).forEach((track) => { + track.features.forEach((feature) => { + if (!feature || byFrame.has(feature.frame)) { + return; + } + const position = featurePosition(feature.attributes); + if (position) { + byFrame.set(feature.frame, { ...position, frameNumber: feature.frame }); + } + }); + }); + return Array.from(byFrame.values()).sort((a, b) => a.frameNumber - b.frameNumber); + }; + + const removeTrack = function removeTrack(trackId: AnnotationId) { + const trackTracker = trackManager.unregisterTrack(trackId); + trackPositions.delete(trackId); + if (!trackTracker) { + return; + } + [trackTracker.trackActor, ...trackTracker.detectionsMap.values()].forEach((actor) => { + headMarkers.delete(actor); + if (renderer.value) { + renderer.value.removeActor(actor); + } + actor.delete(); + }); + }; + + const initializeTrack = function initializeTrack(trackWithContext: TrackWithContext) { + const trackId = trackWithContext.annotation.id; + const features = trackFeatures(trackId); + + if (features.length > 0) { + const trackColor = getTrackColor(trackWithContext, trackStyleManager); + const mapper = vtkMapper.newInstance({ + useLookupTableScalarRange: true, + }); + mapper.setColorByArrayName('frames'); + mapper.setScalarVisibility(true); + mapper.setColorModeToMapScalars(); + mapper.setInterpolateScalarsBeforeMapping(true); + + const { trackActor, frameDetections } = drawTrack(mapper, trackColor, features); + + const trackType = getTrackType(trackWithContext); + trackTypes.push(trackType); + trackManager.registerTrack(trackId, trackActor, frameDetections, trackColor, trackType); + trackPositions.set(trackId, features); + } + }; + + /** + * Build one lookup table per track type. + * Then, assign the correct lookup table to the track actor mapper. + */ + const updateLookupTables = function updateLookupTables(currentFrame: number) { + const trackTypeToLut: Map = new Map(); + trackTypes.forEach((trackType) => { + if (!trackTypeToLut.get(trackType)) { + const trackTypeColor = getTrackTypeColor(trackType, trackStyleManager); + trackTypeToLut.set(trackType, buildLookupTable(trackTypeColor, currentFrame)); + } + }); + + // The trail is colored through its lookup table, not the actor's color + const selectedLut = buildLookupTable( + vtkMath.hex2float(trackStyleManager.stateStyles.selected.color) as [number, number, number], + currentFrame, + ); + trackManager.forEachTrack((trackTracker, trackId) => { + const { trackType } = trackTracker; + const lut = trackId === selectedTrackIdRef.value ? selectedLut : trackTypeToLut.get(trackType); + const mapper = trackTracker.trackActor.getMapper(); + if (mapper) { mapper.setLookupTable(lut); } + }); + }; + + /** + * Show the detections representation for the current frame and hide others. + * If onlyShowSelectedTrack is true, only detections from this track will be shown. + */ + const onFrameChange = function onFrameChange( + newFrameNumber: number, + oldFrameNumber?: number, + ) { + updateLookupTables(newFrameNumber); + + if (onlyShowSelectedTrack.value && selectedTrackIdRef.value !== null) { + // Draw only detection from this frame and the selected track + const actor = trackManager.getTrackFrameDetection(selectedTrackIdRef.value, newFrameNumber); + + if (actor) { + actor.setVisibility(true); + } + } else { + // Draw all frame actors if there are some. + const frameTracker = trackManager.getFrameTracker(newFrameNumber); + + if (frameTracker) { + frameTracker.detectionActors.forEach((detectionActor) => { + detectionActor.setVisibility(true); + }); + } + } + + if (oldFrameNumber !== undefined) { + const frameTracker = trackManager.getFrameTracker(oldFrameNumber); + + if (frameTracker) { + frameTracker.detectionActors.forEach((sphereActor) => sphereActor.setVisibility(false)); + } + } + viewUtils.rerender(); + }; + + /** + * Emphasize on the new selected track and de-emphasize the previous one. + */ + const onSelectedTrackChange = function onSelectedTrackChange( + newTrackId: number|null, + oldTrackId: number|null, + ) { + if (oldTrackId !== null && oldTrackId !== newTrackId) { + const trackTracker = trackManager.getTrack(oldTrackId); + + if (trackTracker) { + deEmphasizeTrack(oldTrackId, trackTracker.trackActor); + + if (onlyShowSelectedTrack.value) { + hideTrack(trackTracker); + } + } + } + + if (newTrackId !== null) { + emphasizeTrack(newTrackId); + } + updateLookupTables(frameRef.value); + + viewUtils.rerender(); + }; + + const onOnlyShowSelectedTrackChange = function onOnlyShowSelectedTrackChange( + newOnlyShowSelectedTrack: boolean, + ) { + if (newOnlyShowSelectedTrack) { + // Hide all tracks except the one that is selected (if there is one) + hideAllTracks(selectedTrackIdRef.value); + } else { + showAllTracks(); + } + + viewUtils.rerender(true); + }; + + /** Reading each replica's revision re-runs the watcher when a track is edited. */ + const trackEntries = function trackEntries(): TrackEntry[] { + return filteredTracksRef.filteredAnnotations.value.map((trackWithContext) => ({ + trackWithContext, + revision: cameraStore.getTrackAll(trackWithContext.annotation.id) + .map((track) => track.revision.value).join(','), + })); + }; + + const onTracksChange = function onTracksChange(entries: TrackEntry[]) { + if (!renderer.value) { + return; + } + const trackIds = new Set(); + let rebuilt = false; + + entries.forEach(({ trackWithContext, revision }) => { + const trackId = trackWithContext.annotation.id; + trackIds.add(trackId); + + const key = `${revision}|${getTrackType(trackWithContext)}`; + if (trackKeys.get(trackId) !== key) { + removeTrack(trackId); + initializeTrack(trackWithContext); + trackKeys.set(trackId, key); + rebuilt = true; + } + + const trackTracker = trackManager.getTrack(trackId); + if (trackTracker) { + trackTracker.hidden = false; + trackTracker.trackActor.setVisibility( + !onlyShowSelectedTrack.value || selectedTrackIdRef.value === trackId, + ); + } + }); + + Array.from(trackKeys.keys()).forEach((trackId) => { + if (!trackIds.has(trackId)) { + removeTrack(trackId); + trackKeys.delete(trackId); + rebuilt = true; + } + }); + + if (rebuilt) { + updateGlyphRadius(); + onFrameChange(frameRef.value, undefined); + if (selectedTrackIdRef.value !== null) { + emphasizeTrack(selectedTrackIdRef.value); + } + } + viewUtils.rerender(); + }; + + const initializeTracks = function drawTracks() { + onTracksChange(trackEntries()); + onSelectedTrackChange(selectedTrackIdRef.value, null); + }; + + watch(selectedTrackIdRef, onSelectedTrackChange); + watch(frameRef, onFrameChange); + watch(onlyShowSelectedTrack, onOnlyShowSelectedTrackChange); + watch(trackEntries, onTracksChange); + + /** The ball at each shown track's position on the current frame, for picking. */ + const visibleBalls = function visibleBalls() { + const frame = frameRef.value; + const radius = detectionGlyphSource.getRadius(); + const balls: { trackId: AnnotationId; center: Vec3; radius: number }[] = []; + trackPositions.forEach((features, trackId) => { + const trackTracker = trackManager.getTrack(trackId); + const shown = trackTracker && !trackTracker.hidden + && (!onlyShowSelectedTrack.value || selectedTrackIdRef.value === trackId); + const current = shown && features.find(({ frameNumber }) => frameNumber === frame); + if (current) { + balls.push({ trackId, center: [current.x, current.y, current.z], radius }); + } + }); + return balls; + }; + + return { + onSelectedTrackChange, + onFrameChange, + initializeTracks, + initialize: initializeTracks, + visibleBalls, + }; +} diff --git a/client/src/components/track_3d_viewer/utils.spec.ts b/client/src/components/track_3d_viewer/utils.spec.ts new file mode 100644 index 000000000..c028b2e2b --- /dev/null +++ b/client/src/components/track_3d_viewer/utils.spec.ts @@ -0,0 +1,12 @@ +import { smoothBounds } from './utils'; + +describe('smoothBounds', () => { + it('centers a cube of the largest extent on each axis', () => { + expect(smoothBounds([-400, 600, 0, 200, 1500, 3500])) + .toEqual([-900, 1100, -900, 1100, 1500, 3500]); + }); + + it('falls back to a unit cube when nothing is visible', () => { + expect(smoothBounds([1, -1, 1, -1, 1, -1])).toEqual([-1, 1, -1, 1, -1, 1]); + }); +}); diff --git a/client/src/components/track_3d_viewer/utils.ts b/client/src/components/track_3d_viewer/utils.ts new file mode 100644 index 000000000..cad4d74f3 --- /dev/null +++ b/client/src/components/track_3d_viewer/utils.ts @@ -0,0 +1,25 @@ +import { Bounds, RGBColor } from '@kitware/vtk.js/types'; + +export const noOp = () => undefined; + +export function float2rgb(color: RGBColor): RGBColor { + return [color[0] * 255, color[1] * 255, color[2] * 255]; +} + +/** + * A cube around the data: every axis spans the largest extent, centered on its + * own range, so positions far from the origin still fill the view. + */ +export function smoothBounds(bounds: Bounds): Bounds { + const [xmin, xmax, ymin, ymax, zmin, zmax] = bounds; + const half = Math.max(xmax - xmin, ymax - ymin, zmax - zmin) / 2; + if (!(half > 0)) { + return [-1, 1, -1, 1, -1, 1]; + } + const [x, y, z] = [(xmin + xmax) / 2, (ymin + ymax) / 2, (zmin + zmax) / 2]; + return [ + x - half, x + half, + y - half, y + half, + z - half, z + half, + ]; +} diff --git a/client/src/provides.ts b/client/src/provides.ts index 449894b9b..87c7a708a 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -26,6 +26,7 @@ import GroupFilterControls from './GroupFilterControls'; import CameraStore from './CameraStore'; import CameraRegistrationStore from './alignedView/CameraRegistrationStore'; import AlignedViewStore from './alignedView/AlignedViewStore'; +import TrackViewerSettingsStore from './components/track_3d_viewer/TrackViewerSettingsStore'; /** * Type definitions are read only because injectors may mutate internal state, @@ -128,6 +129,7 @@ type PercentileHistogramLoadingType = Readonly>; const CameraStoreSymbol = Symbol('cameraStore'); const CameraRegistrationSymbol = Symbol('cameraRegistration'); const AlignedViewSymbol = Symbol('alignedView'); +const TrackViewerSettingsStoreSymbol = Symbol('trackViewerSettingsStore'); const TrackStyleManagerSymbol = Symbol('trackTypeStyling'); const GroupStyleManagerSymbol = Symbol('groupTypeStyling'); @@ -307,6 +309,7 @@ export interface State { cameraStore: CameraStore; cameraRegistration: CameraRegistrationStore; alignedView: AlignedViewStore; + trackViewerSettingsStore: TrackViewerSettingsStore; datasetId: DatasetIdType; editingMode: EditingModeType; groupFilters: GroupFilterControls; @@ -382,6 +385,7 @@ function dummyState(): State { cameraStore, cameraRegistration: new CameraRegistrationStore(), alignedView: new AlignedViewStore(), + trackViewerSettingsStore: new TrackViewerSettingsStore(), datasetId: ref(''), editingMode: ref(false), multiSelectList: ref([]), @@ -438,6 +442,7 @@ function provideAnnotator(state: State, handler: Handler, attributesFilters: Att provide(CameraStoreSymbol, state.cameraStore); provide(CameraRegistrationSymbol, state.cameraRegistration); provide(AlignedViewSymbol, state.alignedView); + provide(TrackViewerSettingsStoreSymbol, state.trackViewerSettingsStore); provide(DatasetIdSymbol, state.datasetId); provide(EditingModeSymbol, state.editingMode); provide(GroupFilterControlsSymbol, state.groupFilters); @@ -502,6 +507,9 @@ function useCameraRegistration() { function useAlignedView() { return use(AlignedViewSymbol); } +function useTrackViewerSettingsStore() { + return use(TrackViewerSettingsStoreSymbol); +} function useDatasetId() { return use(DatasetIdSymbol); } @@ -627,6 +635,7 @@ export { useCameraStore, useCameraRegistration, useAlignedView, + useTrackViewerSettingsStore, useDatasetId, useEditingMode, useHandler, diff --git a/client/src/track.ts b/client/src/track.ts index cab160d1b..44bb851ac 100644 --- a/client/src/track.ts +++ b/client/src/track.ts @@ -43,6 +43,11 @@ interface TrackParams extends BaseAnnotationParams { set?: string; } +/** Head and tail 3D locations a measurement pipeline may record with the midpoint. */ +const STEREO_END_LOCATION_ATTRS = [ + 'head_x', 'head_y', 'head_z', 'tail_x', 'tail_y', 'tail_z', +]; + /** * Track manages the state of a track, its * frame data, and all metadata. @@ -345,7 +350,8 @@ export default class Track extends BaseAnnotation { if (current.attributes.length_method !== 'user_set') { current.fishLength = undefined; ['length', 'curved_length', 'straight_length', 'curvature_ratio', 'avg_length', - 'midpoint_x', 'midpoint_y', 'midpoint_z', 'midpoint_range', 'stereo_rms'] + 'midpoint_x', 'midpoint_y', 'midpoint_z', 'midpoint_range', 'stereo_rms', + ...STEREO_END_LOCATION_ATTRS] .forEach((key) => { delete current.attributes?.[key]; }); } } @@ -375,7 +381,8 @@ export default class Track extends BaseAnnotation { const attributes = { ...feature.attributes, measurement_stale: true } as StringKeyObject; const locked = attributes.length_method === 'user_set'; ['curved_length', 'straight_length', 'curvature_ratio', 'avg_length', - 'midpoint_x', 'midpoint_y', 'midpoint_z', 'midpoint_range', 'stereo_rms'] + 'midpoint_x', 'midpoint_y', 'midpoint_z', 'midpoint_range', 'stereo_rms', + ...STEREO_END_LOCATION_ATTRS] .forEach((key) => { delete attributes[key]; }); if (!locked) delete attributes.length; this.setFeature({ frame, attributes, fishLength: locked ? feature.fishLength : undefined }); diff --git a/docker/girder.Dockerfile b/docker/girder.Dockerfile index 0e1a52f62..38d2d9d2e 100644 --- a/docker/girder.Dockerfile +++ b/docker/girder.Dockerfile @@ -5,7 +5,7 @@ FROM node:24.15.0 AS client-builder WORKDIR /app # Install dependencies -COPY client/package.json client/package-lock.json /app/ +COPY client/package.json client/package-lock.json client/.npmrc /app/ RUN npm install # Build COPY .git/ /app/.git/