From 29d8edafb1daa9e7a10a0a2cdc0b1ed81b4917f0 Mon Sep 17 00:00:00 2001 From: Jules Bourdais Date: Mon, 28 Sep 2026 17:22:08 -0400 Subject: [PATCH 01/19] Add a 3D viewer for stereo tracks and detections Viewer taken from #1348 as resolved on dev/port-ifremer-features. --- client/package-lock.json | 391 +++++++++++---- client/package.json | 1 + .../track_3d_viewer/TrackManager.ts | 111 +++++ .../track_3d_viewer/TrackViewer.vue | 282 +++++++++++ .../track_3d_viewer/TrackViewerSettings.vue | 132 ++++++ .../TrackViewerSettingsStore.ts | 30 ++ .../components/track_3d_viewer/lookupTable.ts | 44 ++ client/src/components/track_3d_viewer/misc.ts | 33 ++ .../components/track_3d_viewer/trackUtils.ts | 34 ++ .../track_3d_viewer/useLabelDrawer.ts | 78 +++ .../useOrientationMarkerWidget.ts | 105 +++++ .../track_3d_viewer/useTrackDrawer.ts | 446 ++++++++++++++++++ .../src/components/track_3d_viewer/utils.ts | 18 + client/src/provides.ts | 9 + 14 files changed, 1630 insertions(+), 84 deletions(-) create mode 100644 client/src/components/track_3d_viewer/TrackManager.ts create mode 100644 client/src/components/track_3d_viewer/TrackViewer.vue create mode 100644 client/src/components/track_3d_viewer/TrackViewerSettings.vue create mode 100644 client/src/components/track_3d_viewer/TrackViewerSettingsStore.ts create mode 100644 client/src/components/track_3d_viewer/lookupTable.ts create mode 100644 client/src/components/track_3d_viewer/misc.ts create mode 100644 client/src/components/track_3d_viewer/trackUtils.ts create mode 100644 client/src/components/track_3d_viewer/useLabelDrawer.ts create mode 100644 client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts create mode 100644 client/src/components/track_3d_viewer/useTrackDrawer.ts create mode 100644 client/src/components/track_3d_viewer/utils.ts diff --git a/client/package-lock.json b/client/package-lock.json index 48aee6f02..0be48652d 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -12,6 +12,7 @@ "@flatten-js/interval-tree": "^1.0.11", "@girder/components": "github:girder/girder_web_components#girder-5-axios-upgrade", "@huggingface/transformers": "^4.2.0", + "@kitware/vtk.js": "^25.6.0", "@mdi/font": "^6.2.95", "@sentry/browser": "^5.24.2", "@sentry/integrations": "^5.24.2", @@ -2941,9 +2942,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2960,9 +2958,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2979,9 +2974,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2998,9 +2990,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3017,9 +3006,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3036,9 +3022,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3055,9 +3038,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3074,9 +3054,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3093,9 +3070,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3118,9 +3092,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3143,9 +3114,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3168,9 +3136,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3193,9 +3158,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3218,9 +3180,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3243,9 +3202,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3268,9 +3224,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3548,6 +3501,155 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@kitware/vtk.js": { + "version": "25.15.2", + "resolved": "https://registry.npmjs.org/@kitware/vtk.js/-/vtk.js-25.15.2.tgz", + "integrity": "sha512-bekuiKfB2O9IRJBGXScnWDqA1nT9yAoJJ9t/GTritdrhx4B9kwtcj8sc4MTX000UN7sVruyHoM/NF5qeKvx+fQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/runtime": "7.17.9", + "commander": "9.2.0", + "d3-scale": "4.0.2", + "fast-deep-equal": "^3.1.3", + "fflate": "0.7.3", + "gl-matrix": "3.4.3", + "globalthis": "1.0.3", + "seedrandom": "3.0.5", + "shader-loader": "1.3.1", + "shelljs": "0.8.5", + "spark-md5": "3.0.2", + "stream-browserify": "3.0.0", + "webworker-promise": "0.5.0", + "worker-loader": "3.0.8", + "xmlbuilder2": "3.0.2" + }, + "bin": { + "vtkDataConverter": "Utilities/DataGenerator/convert-cli.js", + "xml2json": "Utilities/XMLConverter/xml2json-cli.js" + }, + "peerDependencies": { + "wslink": "^1.1.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@babel/runtime": { + "version": "7.17.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.9.tgz", + "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.13.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/dom": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@oozcitak/dom/-/dom-1.15.10.tgz", + "integrity": "sha512-0JT29/LaxVgRcGKvHmSrUTEvZ8BXvZhGl2LASRUgHqDTC1M5g1pLmVv56IYNyt3bG2CUjDkc67wnyZC14pbQrQ==", + "license": "MIT", + "dependencies": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/url": "1.0.4", + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/infra": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@oozcitak/infra/-/infra-1.0.8.tgz", + "integrity": "sha512-JRAUc9VR6IGHOL7OGF+yrvs0LO8SlqGnPAMqyzOuFZPSZSXI7Xf2O9+awQPSMXgIWGtgUf/dA6Hs6X6ySEaWTg==", + "license": "MIT", + "dependencies": { + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/url": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@oozcitak/url/-/url-1.0.4.tgz", + "integrity": "sha512-kDcD8y+y3FCSOvnBI6HJgl00viO/nGbQoCINmQ0h98OhnGITrWR3bOGfwYCthgcrV8AnTJz8MzslTQbC3SOAmw==", + "license": "MIT", + "dependencies": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/@oozcitak/util": { + "version": "8.3.8", + "resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.3.8.tgz", + "integrity": "sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==", + "license": "MIT", + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@kitware/vtk.js/node_modules/commander": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.2.0.tgz", + "integrity": "sha512-e2i4wANQiSXgnrBlIatyHtP1odfUp0BbV5Y5nEGbxtIrStkEOAAzCUirvLBNXHLr7kwLvJl6V+4V3XV9x7Wd9w==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || >=14" + } + }, + "node_modules/@kitware/vtk.js/node_modules/globalthis": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz", + "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/@kitware/vtk.js/node_modules/js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@kitware/vtk.js/node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, + "node_modules/@kitware/vtk.js/node_modules/xmlbuilder2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/xmlbuilder2/-/xmlbuilder2-3.0.2.tgz", + "integrity": "sha512-h4MUawGY21CTdhV4xm3DG9dgsqyhDkZvVJBx88beqX8wJs3VgyGQgAn5VreHuae6unTQxh115aMK5InCVmOIKw==", + "license": "MIT", + "dependencies": { + "@oozcitak/dom": "1.15.10", + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8", + "@types/node": "*", + "js-yaml": "3.14.0" + }, + "engines": { + "node": ">=12.0" + } + }, "node_modules/@malept/cross-spawn-promise": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz", @@ -5164,7 +5266,6 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, "license": "MIT" }, "node_modules/@types/json5": { @@ -6321,7 +6422,6 @@ "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -6376,6 +6476,15 @@ "dev": true, "license": "MIT" }, + "node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, "node_modules/alien-signals": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/alien-signals/-/alien-signals-0.2.2.tgz", @@ -6799,6 +6908,15 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -7369,6 +7487,15 @@ "require-from-string": "^2.0.2" } }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -7971,7 +8098,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "devOptional": true, "license": "MIT" }, "node_modules/condense-newlines": { @@ -9442,6 +9568,15 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, + "node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -10271,6 +10406,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", @@ -10469,7 +10617,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "devOptional": true, "license": "MIT" }, "node_modules/fast-fifo": { @@ -10512,7 +10659,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, "license": "MIT" }, "node_modules/fast-levenshtein": { @@ -10571,8 +10717,7 @@ "version": "0.7.3", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.7.3.tgz", "integrity": "sha512-0Zz1jOzJWERhyhsimS54VTqOteCNwRtIlh8isdL0AXLo0g7xNTfTL7oWrkmCnPhZGocKIkWHBistBrrpoNH3aw==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/ffmpeg-ffprobe-static": { "version": "6.1.2-rc.1", @@ -10818,7 +10963,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "devOptional": true, "license": "ISC" }, "node_modules/fsevents": { @@ -11043,8 +11187,7 @@ "version": "3.4.3", "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.3.tgz", "integrity": "sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/gl-vec3": { "version": "1.1.3", @@ -11062,7 +11205,6 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "devOptional": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -11565,7 +11707,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "devOptional": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -11614,7 +11755,6 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", "license": "MIT", - "optional": true, "engines": { "node": ">= 0.10" } @@ -11729,7 +11869,6 @@ "version": "2.16.2", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", - "devOptional": true, "license": "MIT", "dependencies": { "hasown": "^2.0.3" @@ -12441,7 +12580,6 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { @@ -12463,7 +12601,6 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, "license": "MIT", "bin": { "json5": "lib/cli.js" @@ -12644,6 +12781,32 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/loader-utils": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.2.tgz", + "integrity": "sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg==", + "license": "MIT", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^1.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/loader-utils/node_modules/json5": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, "node_modules/localforage": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.8.1.tgz", @@ -12923,7 +13086,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "devOptional": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -12936,7 +13098,6 @@ "version": "1.1.18", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", - "devOptional": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -12947,7 +13108,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -13424,7 +13584,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "devOptional": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -13643,7 +13802,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "devOptional": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -13653,7 +13811,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "devOptional": true, "license": "MIT" }, "node_modules/path-scurry": { @@ -14079,7 +14236,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -14208,7 +14364,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -14257,7 +14412,6 @@ "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "optional": true, "dependencies": { "resolve": "^1.1.6" }, @@ -14413,7 +14567,6 @@ "version": "1.22.12", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", - "devOptional": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -14712,12 +14865,29 @@ "node": ">=v12.22.7" } }, + "node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/seedrandom": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/semver": { "version": "7.7.4", @@ -14885,6 +15055,15 @@ "dev": true, "license": "ISC" }, + "node_modules/shader-loader": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/shader-loader/-/shader-loader-1.3.1.tgz", + "integrity": "sha512-dt8F9K0x4rjmaFyHh7rNDfpt4LUiR64zhNIEwp2WbE99B3z4ALuvvmhftkElg93dUD6sTmv/aXa/z9SJiEddcA==", + "license": "MIT", + "dependencies": { + "loader-utils": "^1.1.0" + } + }, "node_modules/sharp": { "version": "0.35.4", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", @@ -14951,7 +15130,6 @@ "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "license": "BSD-3-Clause", - "optional": true, "dependencies": { "glob": "^7.0.0", "interpret": "^1.0.0", @@ -15106,8 +15284,13 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/spark-md5/-/spark-md5-3.0.2.tgz", "integrity": "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==", - "license": "(WTFPL OR MIT)", - "optional": true + "license": "(WTFPL OR MIT)" + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" }, "node_modules/stackback": { "version": "0.0.2", @@ -15143,6 +15326,16 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, "node_modules/streamx": { "version": "2.28.0", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.0.tgz", @@ -15349,7 +15542,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -16168,7 +16360,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" @@ -16871,8 +17062,7 @@ "version": "0.5.0", "resolved": "https://registry.npmjs.org/webworker-promise/-/webworker-promise-0.5.0.tgz", "integrity": "sha512-14iR79jHAV7ozwvbfif+3wCaApT3I1g8Lo0rJZrwAu6wxZGx/08Y8KXz6as6ZLNUEEufeiEBBYrqyDBClXOsEw==", - "license": "MIT", - "optional": true + "license": "MIT" }, "node_modules/whatwg-mimetype": { "version": "5.0.0", @@ -17041,6 +17231,40 @@ "node": ">=0.10.0" } }, + "node_modules/worker-loader": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.8.tgz", + "integrity": "sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==", + "license": "MIT", + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/worker-loader/node_modules/loader-utils": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", + "license": "MIT", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -17081,7 +17305,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "devOptional": true, "license": "ISC" }, "node_modules/wslink": { diff --git a/client/package.json b/client/package.json index b608ff827..38243c403 100644 --- a/client/package.json +++ b/client/package.json @@ -35,6 +35,7 @@ "@flatten-js/interval-tree": "^1.0.11", "@girder/components": "github:girder/girder_web_components#girder-5-axios-upgrade", "@huggingface/transformers": "^4.2.0", + "@kitware/vtk.js": "^25.6.0", "@mdi/font": "^6.2.95", "@sentry/browser": "^5.24.2", "@sentry/integrations": "^5.24.2", diff --git a/client/src/components/track_3d_viewer/TrackManager.ts b/client/src/components/track_3d_viewer/TrackManager.ts new file mode 100644 index 000000000..3d3552080 --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackManager.ts @@ -0,0 +1,111 @@ +import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import { RGBColor } from './lookupTable'; + +export type FrameNumber = number; + +export type TrackTracker = { + trackActor: vtkActor; + hidden: boolean; // true if the track should remain hidden whatever happens + detectionsMap: Map; + trackColor: RGBColor; + trackType: string; +}; + +export type TracksMap = Map; + +export type FramesMap = Map; + +export default class TrackManager { + tracksMap: TracksMap; + + framesMap: FramesMap; + + constructor() { + this.tracksMap = new Map(); + this.framesMap = new Map(); + } + + registerTrack( + trackId: AnnotationId, + trackActor: vtkActor, + frameDetections: Array<[FrameNumber, vtkActor]>, + trackColor: RGBColor, + trackType: string, + ) { + const detectionsMap = new Map(); + + frameDetections.forEach(([frameNumber, actor]) => { + actor.getProperty().setDiffuse(0.6); + actor.getProperty().setAmbient(0.6); + detectionsMap.set(frameNumber, actor); + + const frameTracker = this.framesMap.get(frameNumber); + + if (frameTracker) { + frameTracker.trackIds.push(trackId); + frameTracker.detectionActors.push(actor); + } else { + this.framesMap.set(frameNumber, { + trackIds: [trackId], + detectionActors: [actor], + }); + } + }); + + this.tracksMap.set(trackId, { + trackActor, + detectionsMap, + hidden: false, // track should not be hidden by default + trackColor, + trackType, + }); + } + + getFrameTracker(frameNumber: FrameNumber) { + return this.framesMap.get(frameNumber); + } + + getTrackFrameDetection(trackId: AnnotationId, frameNumber: FrameNumber) { + return this.tracksMap.get(trackId)?.detectionsMap.get(frameNumber); + } + + forEachTrack(cb: (track: TrackTracker, trackId: AnnotationId) => void) { + this.tracksMap.forEach(cb); + } + + getTrack(trackId: AnnotationId) { + return this.tracksMap.get(trackId); + } + + getAllTracks() { + return this.tracksMap; + } + + hasTrack(trackId: AnnotationId) { + return this.tracksMap.has(trackId); + } + + getTrackActorsInFrameRange(frameStart: number, frameCount: number) { + const trackActors: Set = new Set(); + + // eslint-disable-next-line no-plusplus + for (let frame = frameStart; frame < frameStart + frameCount; frame++) { + const frameTracker = this.framesMap.get(frame); + + if (frameTracker && frameTracker.trackIds.length > 0) { + frameTracker.trackIds.forEach((trackId) => { + trackActors.add( + + this.tracksMap.get(trackId)!.trackActor, + ); + }); + } + } + + return trackActors; + } +} diff --git a/client/src/components/track_3d_viewer/TrackViewer.vue b/client/src/components/track_3d_viewer/TrackViewer.vue new file mode 100644 index 000000000..af9038370 --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackViewer.vue @@ -0,0 +1,282 @@ + + + + + diff --git a/client/src/components/track_3d_viewer/TrackViewerSettings.vue b/client/src/components/track_3d_viewer/TrackViewerSettings.vue new file mode 100644 index 000000000..cc72c1edc --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackViewerSettings.vue @@ -0,0 +1,132 @@ + + + diff --git a/client/src/components/track_3d_viewer/TrackViewerSettingsStore.ts b/client/src/components/track_3d_viewer/TrackViewerSettingsStore.ts new file mode 100644 index 000000000..4c5247cde --- /dev/null +++ b/client/src/components/track_3d_viewer/TrackViewerSettingsStore.ts @@ -0,0 +1,30 @@ +import { ref, Ref } from 'vue'; + +export default class TrackViewerSettingsStore { + cameraParallelProjection: Ref; + + onlyShowSelectedTrack: Ref; + + detectionGlyphSize: Ref; + + cubeAxesBounds: Ref<{ + xrange: [number, number]; + yrange: [number, number]; + zrange: [number, number]; + }>; + + adjustCubeAxesBoundsManually: Ref; + + constructor() { + this.cameraParallelProjection = ref(true); + this.onlyShowSelectedTrack = ref(false); + this.detectionGlyphSize = ref(0.003); + + this.cubeAxesBounds = ref({ + xrange: [-1, 1], + yrange: [-1, 1], + zrange: [-1, 1], + }); + this.adjustCubeAxesBoundsManually = ref(false); + } +} diff --git a/client/src/components/track_3d_viewer/lookupTable.ts b/client/src/components/track_3d_viewer/lookupTable.ts new file mode 100644 index 000000000..77324b737 --- /dev/null +++ b/client/src/components/track_3d_viewer/lookupTable.ts @@ -0,0 +1,44 @@ +/* eslint-disable no-plusplus */ + +import vtkLookupTable from '@kitware/vtk.js/Common/Core/LookupTable'; +import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray'; +import { float2rgb } from './utils'; + +export type RGBColor = [number, number, number]; + +export function buildLookupTable(color: RGBColor, currentFrame: number, frameCount = 10) { + // if outside of range, opacity = 0 + const lookupTable = vtkLookupTable.newInstance(); + + lookupTable.setUseBelowRangeColor(true); + lookupTable.setBelowRangeColor([...color, 0]); + lookupTable.setUseAboveRangeColor(true); + lookupTable.setAboveRangeColor([...color, 0]); + + const size = Math.min(currentFrame + 1, frameCount); + + const range = { + min: currentFrame - size + 1, + max: currentFrame, + }; + + const dataArray = vtkDataArray.newInstance({ + numberOfComponents: 4, + size: 4 * size, + dataType: 'Uint8Array', + }); + + // if range.min === range.max there is some weird things happening + // in the mapper during texture map generation. adding 0.01 is a hack to prevent this. + lookupTable.setRange(range.min, range.max + 0.01); + + let alpha = 255; + for (let i = size - 1; i >= 0; i--) { + const tuple = [...float2rgb(color), alpha]; + dataArray.setTuple(i, tuple); + alpha -= 20; + } + lookupTable.setTable(dataArray); + + return lookupTable; +} diff --git a/client/src/components/track_3d_viewer/misc.ts b/client/src/components/track_3d_viewer/misc.ts new file mode 100644 index 000000000..262e75f20 --- /dev/null +++ b/client/src/components/track_3d_viewer/misc.ts @@ -0,0 +1,33 @@ +import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; + +const EXPECTED_ATTRIBUTE_NAMES = [ + 'stereo3d_x', + 'stereo3d_y', + 'stereo3d_z', +]; + +export const noOp = () => undefined; + +export function isStereo3dReady(attrs: Attribute[]) { + const attributeNamesToFind = [...EXPECTED_ATTRIBUTE_NAMES]; + + // eslint-disable-next-line no-restricted-syntax + for (const attr of attrs) { + // eslint-disable-next-line no-plusplus + for (let i = 0; i < attributeNamesToFind.length; i++) { + if (attr.name === attributeNamesToFind[i]) { + if (attr.belongs === 'detection' && attr.datatype === 'number') { + attributeNamesToFind.splice(i, 1); + } else { + return false; + } + } + } + + if (attributeNamesToFind.length === 0) { + return true; + } + } + + return false; +} diff --git a/client/src/components/track_3d_viewer/trackUtils.ts b/client/src/components/track_3d_viewer/trackUtils.ts new file mode 100644 index 000000000..1de0542ce --- /dev/null +++ b/client/src/components/track_3d_viewer/trackUtils.ts @@ -0,0 +1,34 @@ +import { TrackWithContext } from 'vue-media-annotator/BaseFilterControls'; +import StyleManager from 'vue-media-annotator/StyleManager'; +import * as vtkMath from '@kitware/vtk.js/Common/Core/Math'; + +export interface ViewUtils { + rerender: (resetCamera?: boolean) => void; +} + +export interface Feature { + x: number; + y: number; + z: number; + frameNumber: number; +} + +export type TrackType = string; + +export function getTrackType(trackWithContext: TrackWithContext) { + const track = trackWithContext.annotation; + return track.getType( + trackWithContext.context.confidencePairIndex, + ); +} + +export function getTrackTypeColor(trackType: TrackType, styleManager: StyleManager) { + return vtkMath.hex2float( + styleManager.typeStyling.value.color(trackType), + ) as [number, number, number]; +} + +export function getTrackColor(trackWithContext: TrackWithContext, styleManager: StyleManager) { + const trackType = getTrackType(trackWithContext); + return getTrackTypeColor(trackType, styleManager); +} diff --git a/client/src/components/track_3d_viewer/useLabelDrawer.ts b/client/src/components/track_3d_viewer/useLabelDrawer.ts new file mode 100644 index 000000000..3602c924c --- /dev/null +++ b/client/src/components/track_3d_viewer/useLabelDrawer.ts @@ -0,0 +1,78 @@ +/* eslint-disable import/prefer-default-export */ + +import vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; +import vtkOpenGLRenderWindow from '@kitware/vtk.js/Rendering/OpenGL/RenderWindow'; +import { Ref } from 'vue'; + +interface LabelDrawerParams { + renderer: Ref; + viewportDimensions: Ref<{ + width: number; + height: number; + }>; + openglRenderWindow: Ref; +} + +export function useLabelDrawer({ + renderer, + viewportDimensions, + openglRenderWindow, +}: LabelDrawerParams) { + let labelTextCanvas: HTMLCanvasElement | null = null; + let labelTextContext: CanvasRenderingContext2D | null = null; + + const initialize = function initialize() { + labelTextCanvas = document.createElement('canvas'); + labelTextCanvas.style.position = 'absolute'; + labelTextCanvas.style.top = '0'; + labelTextCanvas.style.right = '0'; + labelTextCanvas.style.bottom = '0'; + labelTextCanvas.style.left = '0'; + labelTextContext = labelTextCanvas.getContext('2d'); + + return { + labelTextCanvas, + }; + }; + const clearLabelContext = function clearLabelContext() { + if (!labelTextContext) { + throw new Error('Please initialize the drawer first.'); + } + + labelTextContext.clearRect( + 0, + 0, + viewportDimensions.value.width, + viewportDimensions.value.height, + ); + }; + const drawLabels = function drawLabels(positionToLabel: Map<[number, number, number], string>) { + if (!labelTextContext) { + throw new Error('Please initialize the drawer first.'); + } + + clearLabelContext(); + + positionToLabel.forEach((label, position) => { + const displayCoordinates = openglRenderWindow.value!.worldToDisplay( + ...position, + renderer.value!, + ); + labelTextContext!.font = 'bold 14px serif'; + labelTextContext!.textAlign = 'center'; + labelTextContext!.textBaseline = 'middle'; + + labelTextContext!.fillText( + label, + displayCoordinates[0], + viewportDimensions.value.height - displayCoordinates[1], + ); + }); + }; + + return { + initialize, + drawLabels, + clearLabelContext, + }; +} diff --git a/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts new file mode 100644 index 000000000..3295d08fe --- /dev/null +++ b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts @@ -0,0 +1,105 @@ +/* eslint-disable import/prefer-default-export */ +import vtkAxesActor from '@kitware/vtk.js/Rendering/Core/AxesActor'; +import vtkCamera from '@kitware/vtk.js/Rendering/Core/Camera'; +import vtkInteractiveOrientationWidget from '@kitware/vtk.js/Widgets/Widgets3D/InteractiveOrientationWidget'; +import vtkOrientationMarkerWidget from '@kitware/vtk.js/Interaction/Widgets/OrientationMarkerWidget'; +import vtkRenderWindowInteractor from '@kitware/vtk.js/Rendering/Core/RenderWindowInteractor'; +import vtkWidgetManager from '@kitware/vtk.js/Widgets/Core/WidgetManager'; +import * as vtkMath from '@kitware/vtk.js/Common/Core/Math'; +import type { Vector3 } from '@kitware/vtk.js/types'; +import { ref } from 'vue'; + +const getMajorAxisFromViewUp = function getMajorAxisFromViewUp( + viewUp: Vector3, + idxA: number, + idxB: number, +): Vector3 { + const axis: Vector3 = [0, 0, 0]; + const idx = Math.abs(viewUp[idxA]) > Math.abs(viewUp[idxB]) ? idxA : idxB; + const value = viewUp[idx] > 0 ? 1 : -1; + axis[idx] = value; + return axis; +}; + +export function useOrientationMarkerWidget() { + const previousVisibility = ref(false); + + const widgetManager = vtkWidgetManager.newInstance(); + const axes = vtkAxesActor.newInstance(); + const orientationMarkerWidget = vtkOrientationMarkerWidget.newInstance({ + actor: axes, + }); + + const interactiveOrientationWidget = vtkInteractiveOrientationWidget.newInstance(); + + const enable = function enable( + interactor: vtkRenderWindowInteractor, + camera: vtkCamera, + onOrientationChanged: () => void, + ) { + orientationMarkerWidget.setInteractor(interactor); + orientationMarkerWidget.setEnabled(true); + orientationMarkerWidget.setViewportCorner(vtkOrientationMarkerWidget.Corners.BOTTOM_LEFT); + orientationMarkerWidget.setViewportSize(0.15); + orientationMarkerWidget.setMinPixelSize(100); + orientationMarkerWidget.setMaxPixelSize(300); + + const widgetRenderer = orientationMarkerWidget.getRenderer(); + widgetManager.setRenderer(widgetRenderer); + + interactiveOrientationWidget.placeWidget(axes.getBounds()); + interactiveOrientationWidget.setBounds(axes.getBounds()); + interactiveOrientationWidget.setPlaceFactor(1); + + const viewWidget = widgetManager.addWidget(interactiveOrientationWidget); + + if (!viewWidget) { + throw new Error('Failed to add widget to view.'); + } + + // @ts-expect-error it exists + viewWidget.onOrientationChange(({ direction }) => { + const focalPoint = camera.getFocalPoint(); + const position = camera.getPosition(); + const viewUp = camera.getViewUp(); + + const distance = Math.sqrt(vtkMath.distance2BetweenPoints(position, focalPoint)); + + // Put the camera orthogonally compared with the clicked cube's face + camera.setPosition( + focalPoint[0] + direction[0] * distance, + focalPoint[1] + direction[1] * distance, + focalPoint[2] + direction[2] * distance, + ); + + // Set the viewup on the camera so the cube appears aligned with the ground + if (direction[0]) { + camera.setViewUp(getMajorAxisFromViewUp(viewUp, 1, 2)); + } + if (direction[1]) { + camera.setViewUp(getMajorAxisFromViewUp(viewUp, 0, 2)); + } + if (direction[2]) { + camera.setViewUp(getMajorAxisFromViewUp(viewUp, 0, 1)); + } + + orientationMarkerWidget.updateMarkerOrientation(); + widgetManager.enablePicking(); + onOrientationChanged(); + }); + + previousVisibility.value = true; + }; + + const disable = function disable() { + orientationMarkerWidget.delete(); + widgetManager.removeWidget(interactiveOrientationWidget); + widgetManager.delete(); + axes.delete(); + }; + + return { + enable, + disable, + }; +} diff --git a/client/src/components/track_3d_viewer/useTrackDrawer.ts b/client/src/components/track_3d_viewer/useTrackDrawer.ts new file mode 100644 index 000000000..bedec8d29 --- /dev/null +++ b/client/src/components/track_3d_viewer/useTrackDrawer.ts @@ -0,0 +1,446 @@ +/* eslint-disable consistent-return */ + +import { Ref, watch } from 'vue'; +import { + useTrackFilters, + useSelectedTrackId, + useTrackStyleManager, + useCameraStore, +} from 'vue-media-annotator/provides'; +import * as vtkMath from '@kitware/vtk.js/Common/Core/Math'; +import { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import { AnnotationWithContext, TrackWithContext } from 'vue-media-annotator/BaseFilterControls'; +import Track from 'vue-media-annotator/track'; +import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray'; +import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; +import vtkPoints from '@kitware/vtk.js/Common/Core/Points'; +import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; +import vtkSphereSource from '@kitware/vtk.js/Filters/Sources/SphereSource'; +import vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; +import vtkCellArray from '@kitware/vtk.js/Common/Core/CellArray'; +import vtkLookupTable from '@kitware/vtk.js/Common/Core/LookupTable'; +import TrackManager, { TrackTracker } from './TrackManager'; +import { injectAggregateController } from '../annotators/useMediaController'; +import { + getTrackColor, getTrackType, getTrackTypeColor, ViewUtils, Feature, +} from './trackUtils'; +import { buildLookupTable } from './lookupTable'; + +export interface TrackDrawerParams { + trackManager: TrackManager; + onlyShowSelectedTrack: Ref; + detectionGlyphSize: Ref; + viewUtils: ViewUtils; + renderer: Ref; +} + +export default function useTrackDrawer({ + trackManager, + onlyShowSelectedTrack, + detectionGlyphSize, + viewUtils, + renderer, +}: TrackDrawerParams) { + const selectedTrackIdRef = useSelectedTrackId(); + const mediaController = injectAggregateController(); + const trackStyleManager = useTrackStyleManager(); + const filteredTracksRef = useTrackFilters(); + const cameraStore = useCameraStore(); + const { frame: frameRef } = mediaController.value; + const trackTypes: string[] = []; + + // We uses a sphere to represents a detection in space + const detectionGlyphSource = vtkSphereSource.newInstance(); + + watch(detectionGlyphSize, (newSize) => { + detectionGlyphSource.setRadius(newSize); + viewUtils.rerender(); + }, { + immediate: true, + }); + + detectionGlyphSource.setThetaResolution(20); + + const drawFeature = function drawFeature( + points: vtkPoints, + lines: vtkCellArray, + trackColor: [number, number, number], + { x, y, z }: { x: number; y: number; z: number}, + idx: number, + frameDataArray: vtkDataArray, + frameNumber: number, + ) { + points.setPoint( + idx, + x, + y, + z, + ); + + frameDataArray.setTuple(idx, [frameNumber]); + + if (idx > 0) { + // eslint-disable-next-line no-param-reassign + lines.getData()[idx] = idx - 1; + // eslint-disable-next-line no-param-reassign + lines.getData()[idx + 1] = idx; + } + + const pointMapper = vtkMapper.newInstance(); + pointMapper.setInputConnection(detectionGlyphSource.getOutputPort()); + + const pointActor = vtkActor.newInstance(); + + pointActor.setPosition(Number(x), Number(y), Number(z)); + pointActor.setMapper(pointMapper); + pointActor.getProperty().setColor(...trackColor); + pointActor.setVisibility(false); + + if (renderer.value) { + renderer.value.addActor(pointActor); + } + + return pointActor; + }; + + /** + * A 3D track is vtkPolyData made of one single line. + * Each point in this line represents a single detection. + * The points are ordered by frame number. + * We also store a point data field array to store the exact frame number of each point. + */ + const drawTrack = function drawTrack( + mapper: vtkMapper, + trackColor: [number, number, number], + features: Array<{x: number; y: number; z: number; frameNumber: number}>, + ) { + const numberOfFeatures = features.length; + // One point stands for one detection + const points = vtkPoints.newInstance(); + const trackPolyData = vtkPolyData.newInstance(); + + const frameDataArray = vtkDataArray.newInstance({ + numberOfComponents: 1, + size: numberOfFeatures, + dataType: 'Uint32Array', + name: 'frames', + }); + + const trackActor = vtkActor.newInstance(); + const trackActorProperty = trackActor.getProperty(); + points.setNumberOfPoints(numberOfFeatures); + + const lines = vtkCellArray.newInstance({ size: numberOfFeatures + 1 }); + lines.getData()[0] = numberOfFeatures; + + const frameDetections: [number, vtkActor][] = features + .map((feature, idx) => { + const featureActor = drawFeature( + points, + lines, + trackColor, + feature, + idx, + frameDataArray, + feature.frameNumber, + ); + return [feature.frameNumber, featureActor]; + }); + + trackPolyData.setPoints(points); + + // store the frame number of each point + trackPolyData.getPointData().setScalars(frameDataArray); + + mapper.setInputData(trackPolyData); + trackPolyData.setLines(lines); + + // trackActorProperty.setColor(...trackColor); + trackActorProperty.setLineWidth(3); + trackActor.setMapper(mapper); + + if (renderer.value) { + renderer.value.addActor(trackActor); + } + + return { + trackActor, + frameDetections, + }; + }; + + /** + * Emphasize on a specific track: make sure it is visible, increase its line width and + * set its color to selected + */ + const emphasizeTrack = function emphasizeTrack(trackId: AnnotationId) { + const trackActor = trackManager.getTrack(trackId)?.trackActor; + + if (!trackActor) { + return; + } + + trackActor.setVisibility(true); + trackActor.getProperty().setLineWidth(5); + + const selectedColor = vtkMath.hex2float( + trackStyleManager.stateStyles.selected.color, + ) as [number, number, number]; + + trackActor.getProperty().setColor(...selectedColor); + }; + + const deEmphasizeTrack = function deEmphasizeTrack(trackId: AnnotationId, trackActor: vtkActor) { + // Restore line width + trackActor.getProperty().setLineWidth(3); + + const trackWithContext = filteredTracksRef + .filteredAnnotations + .value + .find((track) => track.annotation.id === trackId); + + // If we do not find the track anymore in the list, + // this is because the track has been filtered out in the meantime + // In this case we cannot get the color, so we do nothing as the track will be hidden anyway + if (trackWithContext) { + const trackColor = getTrackColor(trackWithContext, trackStyleManager); + trackActor.getProperty().setColor(...trackColor); + } + }; + + const hideTrack = function hideTrack(trackMap: TrackTracker) { + trackMap.trackActor.setVisibility(false); + trackMap.detectionsMap.forEach((actor) => actor.setVisibility(false)); + }; + + /** + * Hide all tracks except the one matching the track id passed in the except param. + */ + const hideAllTracks = function hideAllTracks(except?: AnnotationId | null) { + trackManager.forEachTrack(({ trackActor, detectionsMap }, trackId) => { + if (trackId !== except) { + trackActor.setVisibility(false); + detectionsMap.forEach((sphereActor) => { + sphereActor.setVisibility(false); + }); + } + }); + }; + + /** + * Show all tracks, except the ones that are explicitly flagged as hidden + */ + const showAllTracks = function showAllTracks() { + trackManager.forEachTrack(({ trackActor, detectionsMap, hidden }) => { + if (hidden) { + return; + } + + trackActor.setVisibility(true); + detectionsMap.forEach((sphereActor, frameNumber) => { + if (frameNumber === frameRef.value) { + sphereActor.setVisibility(true); + } + }); + }); + }; + + const initializeTrack = function initializeTrack(trackWithContext: TrackWithContext) { + const annotation = cameraStore.getTrackProjection(trackWithContext.annotation.id); + + // pre-process features in order to represent them + const features: Feature[] = annotation.features + .filter((feature) => feature !== undefined) + .map((feature) => { + const { attributes } = feature; + + if (!attributes) { + return; + } + + const { stereo3d_x: x, stereo3d_y: y, stereo3d_z: z } = attributes; + + if (x === undefined || y === undefined || z === undefined) { + return; + } + + return { + x: Number(x), + y: Number(y), + z: Number(z), + frameNumber: feature.frame, + }; + }) + .filter((feature) => feature !== undefined) as Feature[]; + + if (features.length > 0) { + const trackColor = getTrackColor(trackWithContext, trackStyleManager); + const mapper = vtkMapper.newInstance({ + useLookupTableScalarRange: true, + }); + mapper.setColorByArrayName('frames'); + mapper.setScalarVisibility(true); + mapper.setColorModeToMapScalars(); + mapper.setInterpolateScalarsBeforeMapping(true); + + const { trackActor, frameDetections } = drawTrack(mapper, trackColor, features); + + const trackType = getTrackType(trackWithContext); + trackTypes.push(trackType); + trackManager.registerTrack(annotation.id, trackActor, frameDetections, trackColor, trackType); + } + }; + + /** + * Build one lookup table per track type. + * Then, assign the correct lookup table to the track actor mapper. + */ + const updateLookupTables = function updateLookupTables(currentFrame: number) { + const trackTypeToLut: Map = new Map(); + trackTypes.forEach((trackType) => { + if (!trackTypeToLut.get(trackType)) { + const trackTypeColor = getTrackTypeColor(trackType, trackStyleManager); + trackTypeToLut.set(trackType, buildLookupTable(trackTypeColor, currentFrame)); + } + }); + + trackManager.forEachTrack((trackTracker) => { + const { trackType } = trackTracker; + const lut = trackTypeToLut.get(trackType); + const mapper = trackTracker.trackActor.getMapper(); + if (mapper) { mapper.setLookupTable(lut); } + }); + }; + + /** + * Show the detections representation for the current frame and hide others. + * If onlyShowSelectedTrack is true, only detections from this track will be shown. + */ + const onFrameChange = function onFrameChange( + newFrameNumber: number, + oldFrameNumber?: number, + ) { + updateLookupTables(newFrameNumber); + + if (onlyShowSelectedTrack.value && selectedTrackIdRef.value !== null) { + // Draw only detection from this frame and the selected track + const actor = trackManager.getTrackFrameDetection(selectedTrackIdRef.value, newFrameNumber); + + if (actor) { + actor.setVisibility(true); + } + } else { + // Draw all frame actors if there are some. + const frameTracker = trackManager.getFrameTracker(newFrameNumber); + + if (frameTracker) { + frameTracker.detectionActors.forEach((detectionActor) => { + detectionActor.setVisibility(true); + }); + } + } + + if (oldFrameNumber !== undefined) { + const frameTracker = trackManager.getFrameTracker(oldFrameNumber); + + if (frameTracker) { + frameTracker.detectionActors.forEach((sphereActor) => sphereActor.setVisibility(false)); + } + } + viewUtils.rerender(); + }; + + /** + * Emphasize on the new selected track and de-emphasize the previous one. + */ + const onSelectedTrackChange = function onSelectedTrackChange( + newTrackId: number|null, + oldTrackId: number|null, + ) { + if (newTrackId) { + emphasizeTrack(newTrackId); + } + + if (oldTrackId) { + const trackTracker = trackManager.getTrack(oldTrackId); + + if (!trackTracker) { + return; + } + + deEmphasizeTrack(oldTrackId, trackTracker.trackActor); + + if (onlyShowSelectedTrack.value) { + hideTrack(trackTracker); + } + } + + viewUtils.rerender(); + }; + + const onOnlyShowSelectedTrackChange = function onOnlyShowSelectedTrackChange( + newOnlyShowSelectedTrack: boolean, + ) { + if (newOnlyShowSelectedTrack) { + // Hide all tracks except the one that is selected (if there is one) + hideAllTracks(selectedTrackIdRef.value); + } else { + showAllTracks(); + } + + viewUtils.rerender(true); + }; + + const onFilteredAnnotationsChange = function onFilteredAnnotationsChange( + annotations: AnnotationWithContext[], + ) { + const trackIds: AnnotationId[] = []; + + annotations.forEach((trackWithContext) => { + const trackId = trackWithContext.annotation.id; + trackIds.push(trackId); + + const trackTracker = trackManager.getTrack(trackId); + + // New track, so initialize it + if (!trackTracker) { + initializeTrack(trackWithContext); + } else { + // The track exists, so just show it + trackTracker.hidden = false; + trackTracker.trackActor.setVisibility(true); + } + }); + + // All tracks that we have should be hidden + trackManager.forEachTrack((track, trackId) => { + if (!trackIds.includes(trackId)) { + // eslint-disable-next-line no-param-reassign + track.hidden = true; + track.trackActor.setVisibility(false); + } + }); + + viewUtils.rerender(); + }; + + const initializeTracks = function drawTracks() { + filteredTracksRef.filteredAnnotations.value.forEach(initializeTrack); + + onFrameChange(frameRef.value, undefined); + onSelectedTrackChange(selectedTrackIdRef.value, null); + }; + + watch(selectedTrackIdRef, onSelectedTrackChange); + watch(frameRef, onFrameChange); + watch(onlyShowSelectedTrack, onOnlyShowSelectedTrackChange); + watch(filteredTracksRef.filteredAnnotations, onFilteredAnnotationsChange); + + return { + onSelectedTrackChange, + onFrameChange, + initializeTracks, + initialize: initializeTracks, + }; +} diff --git a/client/src/components/track_3d_viewer/utils.ts b/client/src/components/track_3d_viewer/utils.ts new file mode 100644 index 000000000..f5338551f --- /dev/null +++ b/client/src/components/track_3d_viewer/utils.ts @@ -0,0 +1,18 @@ +import { Bounds, RGBColor } from '@kitware/vtk.js/types'; + +export function float2rgb(color: RGBColor): RGBColor { + return [color[0] * 255, color[1] * 255, color[2] * 255]; +} + +export function smoothBounds(bounds: Bounds): Bounds { + const [xmin, xmax, ymin, ymax, zmin, zmax] = bounds; + + const min = Math.min(xmin, ymin, zmin); + const max = Math.max(xmax, ymax, zmax); + + return [ + min, max, + min, max, + min, max, + ]; +} diff --git a/client/src/provides.ts b/client/src/provides.ts index 449894b9b..87c7a708a 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -26,6 +26,7 @@ import GroupFilterControls from './GroupFilterControls'; import CameraStore from './CameraStore'; import CameraRegistrationStore from './alignedView/CameraRegistrationStore'; import AlignedViewStore from './alignedView/AlignedViewStore'; +import TrackViewerSettingsStore from './components/track_3d_viewer/TrackViewerSettingsStore'; /** * Type definitions are read only because injectors may mutate internal state, @@ -128,6 +129,7 @@ type PercentileHistogramLoadingType = Readonly>; const CameraStoreSymbol = Symbol('cameraStore'); const CameraRegistrationSymbol = Symbol('cameraRegistration'); const AlignedViewSymbol = Symbol('alignedView'); +const TrackViewerSettingsStoreSymbol = Symbol('trackViewerSettingsStore'); const TrackStyleManagerSymbol = Symbol('trackTypeStyling'); const GroupStyleManagerSymbol = Symbol('groupTypeStyling'); @@ -307,6 +309,7 @@ export interface State { cameraStore: CameraStore; cameraRegistration: CameraRegistrationStore; alignedView: AlignedViewStore; + trackViewerSettingsStore: TrackViewerSettingsStore; datasetId: DatasetIdType; editingMode: EditingModeType; groupFilters: GroupFilterControls; @@ -382,6 +385,7 @@ function dummyState(): State { cameraStore, cameraRegistration: new CameraRegistrationStore(), alignedView: new AlignedViewStore(), + trackViewerSettingsStore: new TrackViewerSettingsStore(), datasetId: ref(''), editingMode: ref(false), multiSelectList: ref([]), @@ -438,6 +442,7 @@ function provideAnnotator(state: State, handler: Handler, attributesFilters: Att provide(CameraStoreSymbol, state.cameraStore); provide(CameraRegistrationSymbol, state.cameraRegistration); provide(AlignedViewSymbol, state.alignedView); + provide(TrackViewerSettingsStoreSymbol, state.trackViewerSettingsStore); provide(DatasetIdSymbol, state.datasetId); provide(EditingModeSymbol, state.editingMode); provide(GroupFilterControlsSymbol, state.groupFilters); @@ -502,6 +507,9 @@ function useCameraRegistration() { function useAlignedView() { return use(AlignedViewSymbol); } +function useTrackViewerSettingsStore() { + return use(TrackViewerSettingsStoreSymbol); +} function useDatasetId() { return use(DatasetIdSymbol); } @@ -627,6 +635,7 @@ export { useCameraStore, useCameraRegistration, useAlignedView, + useTrackViewerSettingsStore, useDatasetId, useEditingMode, useHandler, From 721fe74c5fe84eb62d4aa4fa0f2e57e90039cb34 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 17:39:01 -0400 Subject: [PATCH 02/19] Show the 3D viewer from the stereo Display selector Stereo pairs get a Display selector: either camera selected, or a camera beside the 3D viewer. The viewer plots midpoint_x/y/z and follows edits. --- client/dive-common/components/Viewer.vue | 99 +++++++++- client/dive-common/stereoDisplay.spec.ts | 22 +++ client/dive-common/stereoDisplay.ts | 47 +++++ client/dive-common/store/context.ts | 5 + .../track_3d_viewer/TrackManager.ts | 23 +++ .../track_3d_viewer/TrackViewer.vue | 28 ++- .../track_3d_viewer/TrackViewerSettings.vue | 5 +- .../TrackViewerSettingsStore.ts | 2 +- client/src/components/track_3d_viewer/misc.ts | 33 ---- .../track_3d_viewer/useTrackDrawer.ts | 176 +++++++++++++----- .../components/track_3d_viewer/utils.spec.ts | 12 ++ .../src/components/track_3d_viewer/utils.ts | 21 ++- 12 files changed, 369 insertions(+), 104 deletions(-) create mode 100644 client/dive-common/stereoDisplay.spec.ts create mode 100644 client/dive-common/stereoDisplay.ts delete mode 100644 client/src/components/track_3d_viewer/misc.ts create mode 100644 client/src/components/track_3d_viewer/utils.spec.ts diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index f09531a56..568f931c3 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -94,6 +94,10 @@ import { import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; +import TrackViewer from 'vue-media-annotator/components/track_3d_viewer/TrackViewer.vue'; +import TrackViewerSettings from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettings.vue'; +import TrackViewerSettingsStore from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettingsStore'; +import { displayOptions, displayValue, parseDisplayValue } from 'dive-common/stereoDisplay'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; import RegistrationToolsVue from './CameraRegistration/RegistrationTools.vue'; @@ -133,6 +137,7 @@ export default defineComponent({ UnsavedChangesDialog, EditorMenu, MultiCamToolbar, + TrackViewer, AlignedViewToggle, PrimaryAttributeTrackFilter, TrackList, @@ -2105,6 +2110,18 @@ export default defineComponent({ } } } + viewer3dRequested.value = false; + if (isStereoPair.value) { + context.register({ + component: TrackViewerSettings, + description: '3D Viewer Settings', + }); + } else { + context.unregister({ + component: TrackViewerSettings, + description: '3D Viewer Settings', + }); + } annotationUndo.start(); progress.loaded = true; fetchSelectedCameraHistogram().catch(() => {}); @@ -2272,6 +2289,33 @@ export default defineComponent({ }; + const trackViewerSettingsStore = new TrackViewerSettingsStore(); + const viewer3dRequested = ref(false); + const isStereoPair = computed(() => subType.value === 'stereo' && multiCamList.value.length === 2); + const viewer3dActive = computed(() => viewer3dRequested.value && isStereoPair.value); + const displayItems = computed(() => displayOptions( + multiCamList.value, + isStereoPair.value, + defaultCamera.value, + )); + const selectedDisplay = computed(() => displayValue(selectedCamera.value, viewer3dActive.value)); + function changeDisplay(value: string) { + const { camera, viewer3d } = parseDisplayValue(value); + viewer3dRequested.value = viewer3d; + changeCamera(camera); + } + /** The 3D viewer takes the pane of the camera that is not selected. */ + function isCameraReplacedBy3dViewer(camera: string) { + return viewer3dActive.value && camera !== selectedCamera.value; + } + const viewer3dPaneOrder = computed(() => ( + multiCamList.value.indexOf(selectedCamera.value) === 0 ? 1 : -1 + )); + watch([viewer3dActive, selectedCamera], async () => { + await nextTick(); + handleResize(); + }); + provideAnnotator( { annotatorPreferences: toRef(clientSettings, 'annotatorPreferences'), @@ -2279,6 +2323,7 @@ export default defineComponent({ cameraStore, cameraRegistration, alignedView, + trackViewerSettingsStore, datasetId, editingMode, groupFilters, @@ -2599,6 +2644,13 @@ export default defineComponent({ defaultCamera, selectedCamera, changeCamera, + isStereoPair, + viewer3dActive, + viewer3dPaneOrder, + displayItems, + selectedDisplay, + changeDisplay, + isCameraReplacedBy3dViewer, noteRightMouseDown, // For Navigation Guarding unsavedChangesDialog, @@ -2809,19 +2861,20 @@ export default defineComponent({ @@ -2978,7 +3031,11 @@ export default defineComponent({
+
+ +
+
+ +
{ + it('offers each stereo camera selected or beside the 3D viewer', () => { + expect(displayOptions(['left', 'right'], true, 'left').map((option) => option.text)).toEqual([ + 'Left (Selected)', 'Right (Selected)', 'Left + 3D Viewer', 'Right + 3D Viewer', + ]); + }); + + it('lists plain cameras for other multi-camera datasets', () => { + expect(displayOptions(['eo', 'ir'], false, 'ir')).toEqual([ + { value: 'camera:eo', text: 'eo', menuText: 'eo' }, + { value: 'camera:ir', text: 'ir', menuText: 'ir (Default)' }, + ]); + }); + + it('round-trips a display value', () => { + expect(parseDisplayValue(displayValue('right', true))).toEqual({ camera: 'right', viewer3d: true }); + expect(parseDisplayValue(displayValue('viewer3d:odd', false))) + .toEqual({ camera: 'viewer3d:odd', viewer3d: false }); + }); +}); diff --git a/client/dive-common/stereoDisplay.ts b/client/dive-common/stereoDisplay.ts new file mode 100644 index 000000000..994206e9c --- /dev/null +++ b/client/dive-common/stereoDisplay.ts @@ -0,0 +1,47 @@ +/** + * Entries of the viewer's camera selector. A stereo pair offers, per camera, + * both cameras side by side with that camera selected, or that camera beside + * the 3D viewer, which takes the other camera's pane. + */ +export interface DisplayOption { + value: string; + text: string; + menuText: string; +} + +const CAMERA_PREFIX = 'camera:'; +const VIEWER_3D_PREFIX = 'viewer3d:'; + +export function displayValue(camera: string, viewer3d: boolean) { + return `${viewer3d ? VIEWER_3D_PREFIX : CAMERA_PREFIX}${camera}`; +} + +export function parseDisplayValue(value: string) { + const viewer3d = value.startsWith(VIEWER_3D_PREFIX); + return { + camera: value.slice((viewer3d ? VIEWER_3D_PREFIX : CAMERA_PREFIX).length), + viewer3d, + }; +} + +function capitalize(name: string) { + return name.charAt(0).toUpperCase() + name.slice(1); +} + +export function displayOptions( + cameras: readonly string[], + stereo: boolean, + defaultCamera: string, +): DisplayOption[] { + if (!stereo) { + return cameras.map((camera) => ({ + value: displayValue(camera, false), + text: camera, + menuText: camera === defaultCamera ? `${camera} (Default)` : camera, + })); + } + return [false, true].flatMap((viewer3d) => cameras.map((camera) => { + const text = `${capitalize(camera)} ${viewer3d ? '+ 3D Viewer' : '(Selected)'}`; + return { value: displayValue(camera, viewer3d), text, menuText: text }; + })); +} diff --git a/client/dive-common/store/context.ts b/client/dive-common/store/context.ts index f2b805c74..d4e1807d7 100644 --- a/client/dive-common/store/context.ts +++ b/client/dive-common/store/context.ts @@ -7,6 +7,7 @@ import AttributesSideBar from 'dive-common/components/Attributes/AttributesSideB import MultiCamTools from 'dive-common/components/MultiCamTools.vue'; import RegistrationTools from 'dive-common/components/CameraRegistration/RegistrationTools.vue'; import AttributeTrackFilters from 'vue-media-annotator/components/AttributeTrackFilters.vue'; +import TrackViewerSettings from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettings.vue'; import DatasetInfo from 'dive-common/components/DatasetInfo/DatasetInfo.vue'; interface ContextState { @@ -62,6 +63,10 @@ const componentMapEntries: ComponentMapItem[] = [ description: 'Attribute Track Filters', component: AttributeTrackFilters, }, + { + description: '3D Viewer Settings', + component: TrackViewerSettings, + }, ]; const componentMap: Record = Object.fromEntries( diff --git a/client/src/components/track_3d_viewer/TrackManager.ts b/client/src/components/track_3d_viewer/TrackManager.ts index 3d3552080..747ba38c1 100644 --- a/client/src/components/track_3d_viewer/TrackManager.ts +++ b/client/src/components/track_3d_viewer/TrackManager.ts @@ -65,6 +65,29 @@ export default class TrackManager { }); } + unregisterTrack(trackId: AnnotationId) { + const trackTracker = this.tracksMap.get(trackId); + if (!trackTracker) { + return undefined; + } + trackTracker.detectionsMap.forEach((actor, frameNumber) => { + const frameTracker = this.framesMap.get(frameNumber); + if (!frameTracker) { + return; + } + const index = frameTracker.detectionActors.indexOf(actor); + if (index !== -1) { + frameTracker.trackIds.splice(index, 1); + frameTracker.detectionActors.splice(index, 1); + } + if (frameTracker.trackIds.length === 0) { + this.framesMap.delete(frameNumber); + } + }); + this.tracksMap.delete(trackId); + return trackTracker; + } + getFrameTracker(frameNumber: FrameNumber) { return this.framesMap.get(frameNumber); } diff --git a/client/src/components/track_3d_viewer/TrackViewer.vue b/client/src/components/track_3d_viewer/TrackViewer.vue index af9038370..be314c3e5 100644 --- a/client/src/components/track_3d_viewer/TrackViewer.vue +++ b/client/src/components/track_3d_viewer/TrackViewer.vue @@ -22,8 +22,7 @@ import { import { ViewUtils } from './trackUtils'; import TrackManager from './TrackManager'; import useTrackDrawer from './useTrackDrawer'; -import { smoothBounds } from './utils'; -import { noOp } from './misc'; +import { noOp, smoothBounds } from './utils'; import { useOrientationMarkerWidget } from './useOrientationMarkerWidget'; import { useLabelDrawer } from './useLabelDrawer'; import { injectAggregateController } from '../annotators/useMediaController'; @@ -51,6 +50,7 @@ export default defineComponent({ } = trackViewerSettingsStore; const renderer = ref(); + const positionedTrackCount = ref(0); const viewportDimensions = ref({ height: 0, width: 0, @@ -79,6 +79,7 @@ export default defineComponent({ trackManager, onlyShowSelectedTrack, detectionGlyphSize, + positionedTrackCount, viewUtils, renderer, }); @@ -261,6 +262,7 @@ export default defineComponent({ return { vtkContainer, + positionedTrackCount, }; }, }); @@ -271,12 +273,32 @@ export default defineComponent({ ref="vtkContainer" class="vtk-container" :style="`--controls-height: ${controlsHeight}px`" - /> + > +
+ No detections have a 3D position yet. Stereo measurement adds one. +
+ diff --git a/client/src/components/track_3d_viewer/TrackViewerSettings.vue b/client/src/components/track_3d_viewer/TrackViewerSettings.vue index cc72c1edc..84be3726e 100644 --- a/client/src/components/track_3d_viewer/TrackViewerSettings.vue +++ b/client/src/components/track_3d_viewer/TrackViewerSettings.vue @@ -55,8 +55,9 @@ export default defineComponent({ undefined; - -export function isStereo3dReady(attrs: Attribute[]) { - const attributeNamesToFind = [...EXPECTED_ATTRIBUTE_NAMES]; - - // eslint-disable-next-line no-restricted-syntax - for (const attr of attrs) { - // eslint-disable-next-line no-plusplus - for (let i = 0; i < attributeNamesToFind.length; i++) { - if (attr.name === attributeNamesToFind[i]) { - if (attr.belongs === 'detection' && attr.datatype === 'number') { - attributeNamesToFind.splice(i, 1); - } else { - return false; - } - } - } - - if (attributeNamesToFind.length === 0) { - return true; - } - } - - return false; -} diff --git a/client/src/components/track_3d_viewer/useTrackDrawer.ts b/client/src/components/track_3d_viewer/useTrackDrawer.ts index bedec8d29..191b714c8 100644 --- a/client/src/components/track_3d_viewer/useTrackDrawer.ts +++ b/client/src/components/track_3d_viewer/useTrackDrawer.ts @@ -9,8 +9,7 @@ import { } from 'vue-media-annotator/provides'; import * as vtkMath from '@kitware/vtk.js/Common/Core/Math'; import { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; -import { AnnotationWithContext, TrackWithContext } from 'vue-media-annotator/BaseFilterControls'; -import Track from 'vue-media-annotator/track'; +import { TrackWithContext } from 'vue-media-annotator/BaseFilterControls'; import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray'; import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; @@ -31,14 +30,24 @@ export interface TrackDrawerParams { trackManager: TrackManager; onlyShowSelectedTrack: Ref; detectionGlyphSize: Ref; + positionedTrackCount: Ref; viewUtils: ViewUtils; renderer: Ref; } +interface TrackEntry { + trackWithContext: TrackWithContext; + revision: string; +} + +/** Stereo measurement attributes holding a detection's position in the rig's frame. */ +const POSITION_KEYS = ['midpoint_x', 'midpoint_y', 'midpoint_z'] as const; + export default function useTrackDrawer({ trackManager, onlyShowSelectedTrack, detectionGlyphSize, + positionedTrackCount, viewUtils, renderer, }: TrackDrawerParams) { @@ -49,12 +58,38 @@ export default function useTrackDrawer({ const cameraStore = useCameraStore(); const { frame: frameRef } = mediaController.value; const trackTypes: string[] = []; + const trackKeys = new Map(); + const trackPositions = new Map(); // We uses a sphere to represents a detection in space const detectionGlyphSource = vtkSphereSource.newInstance(); - watch(detectionGlyphSize, (newSize) => { - detectionGlyphSource.setRadius(newSize); + /** + * Positions are in the calibration's units, so the glyph is sized relative + * to the extent of the data rather than by an absolute radius. + */ + const updateGlyphRadius = function updateGlyphRadius() { + const min = [Infinity, Infinity, Infinity]; + const max = [-Infinity, -Infinity, -Infinity]; + trackPositions.forEach((features) => features.forEach(({ x, y, z }) => { + [x, y, z].forEach((value, axis) => { + min[axis] = Math.min(min[axis], value); + max[axis] = Math.max(max[axis], value); + }); + })); + const extent = Math.max(...max.map((value, axis) => value - min[axis])); + const distance = Math.max(...max.map(Math.abs), ...min.map(Math.abs)); + let scale = 1; + if (Number.isFinite(extent) && extent > 0) { + scale = extent; + } else if (Number.isFinite(distance) && distance > 0) { + scale = distance; + } + detectionGlyphSource.setRadius((Number(detectionGlyphSize.value) / 100) * scale); + }; + + watch(detectionGlyphSize, () => { + updateGlyphRadius(); viewUtils.rerender(); }, { immediate: true, @@ -246,33 +281,48 @@ export default function useTrackDrawer({ }); }; - const initializeTrack = function initializeTrack(trackWithContext: TrackWithContext) { - const annotation = cameraStore.getTrackProjection(trackWithContext.annotation.id); - - // pre-process features in order to represent them - const features: Feature[] = annotation.features - .filter((feature) => feature !== undefined) - .map((feature) => { - const { attributes } = feature; - - if (!attributes) { + /** + * One position per frame, from whichever camera's replica holds the + * measurement, ordered by frame. + */ + const trackFeatures = function trackFeatures(trackId: AnnotationId) { + const byFrame = new Map(); + cameraStore.getTrackAll(trackId).forEach((track) => { + track.features.forEach((feature) => { + if (!feature || byFrame.has(feature.frame)) { return; } - - const { stereo3d_x: x, stereo3d_y: y, stereo3d_z: z } = attributes; - - if (x === undefined || y === undefined || z === undefined) { - return; + const [x, y, z] = POSITION_KEYS.map((key) => { + const value = feature.attributes?.[key]; + return value === undefined || value === null || value === '' ? NaN : Number(value); + }); + if ([x, y, z].every(Number.isFinite)) { + byFrame.set(feature.frame, { + x, y, z, frameNumber: feature.frame, + }); } + }); + }); + return Array.from(byFrame.values()).sort((a, b) => a.frameNumber - b.frameNumber); + }; + + const removeTrack = function removeTrack(trackId: AnnotationId) { + const trackTracker = trackManager.unregisterTrack(trackId); + trackPositions.delete(trackId); + if (!trackTracker) { + return; + } + [trackTracker.trackActor, ...trackTracker.detectionsMap.values()].forEach((actor) => { + if (renderer.value) { + renderer.value.removeActor(actor); + } + actor.delete(); + }); + }; - return { - x: Number(x), - y: Number(y), - z: Number(z), - frameNumber: feature.frame, - }; - }) - .filter((feature) => feature !== undefined) as Feature[]; + const initializeTrack = function initializeTrack(trackWithContext: TrackWithContext) { + const trackId = trackWithContext.annotation.id; + const features = trackFeatures(trackId); if (features.length > 0) { const trackColor = getTrackColor(trackWithContext, trackStyleManager); @@ -288,7 +338,8 @@ export default function useTrackDrawer({ const trackType = getTrackType(trackWithContext); trackTypes.push(trackType); - trackManager.registerTrack(annotation.id, trackActor, frameDetections, trackColor, trackType); + trackManager.registerTrack(trackId, trackActor, frameDetections, trackColor, trackType); + trackPositions.set(trackId, features); } }; @@ -392,50 +443,73 @@ export default function useTrackDrawer({ viewUtils.rerender(true); }; - const onFilteredAnnotationsChange = function onFilteredAnnotationsChange( - annotations: AnnotationWithContext[], - ) { - const trackIds: AnnotationId[] = []; + /** Reading each replica's revision re-runs the watcher when a track is edited. */ + const trackEntries = function trackEntries(): TrackEntry[] { + return filteredTracksRef.filteredAnnotations.value.map((trackWithContext) => ({ + trackWithContext, + revision: cameraStore.getTrackAll(trackWithContext.annotation.id) + .map((track) => track.revision.value).join(','), + })); + }; - annotations.forEach((trackWithContext) => { - const trackId = trackWithContext.annotation.id; - trackIds.push(trackId); + const onTracksChange = function onTracksChange(entries: TrackEntry[]) { + if (!renderer.value) { + return; + } + const trackIds = new Set(); + let rebuilt = false; - const trackTracker = trackManager.getTrack(trackId); + entries.forEach(({ trackWithContext, revision }) => { + const trackId = trackWithContext.annotation.id; + trackIds.add(trackId); - // New track, so initialize it - if (!trackTracker) { + const key = `${revision}|${getTrackType(trackWithContext)}`; + if (trackKeys.get(trackId) !== key) { + removeTrack(trackId); initializeTrack(trackWithContext); - } else { - // The track exists, so just show it + trackKeys.set(trackId, key); + rebuilt = true; + } + + const trackTracker = trackManager.getTrack(trackId); + if (trackTracker) { trackTracker.hidden = false; - trackTracker.trackActor.setVisibility(true); + trackTracker.trackActor.setVisibility( + !onlyShowSelectedTrack.value || selectedTrackIdRef.value === trackId, + ); } }); - // All tracks that we have should be hidden - trackManager.forEachTrack((track, trackId) => { - if (!trackIds.includes(trackId)) { - // eslint-disable-next-line no-param-reassign - track.hidden = true; - track.trackActor.setVisibility(false); + Array.from(trackKeys.keys()).forEach((trackId) => { + if (!trackIds.has(trackId)) { + removeTrack(trackId); + trackKeys.delete(trackId); + rebuilt = true; } }); + if (rebuilt) { + updateGlyphRadius(); + onFrameChange(frameRef.value, undefined); + if (selectedTrackIdRef.value !== null) { + emphasizeTrack(selectedTrackIdRef.value); + } + } + // eslint-disable-next-line no-param-reassign + positionedTrackCount.value = trackManager.getAllTracks().size; + viewUtils.rerender(); }; const initializeTracks = function drawTracks() { - filteredTracksRef.filteredAnnotations.value.forEach(initializeTrack); - - onFrameChange(frameRef.value, undefined); + onTracksChange(trackEntries()); onSelectedTrackChange(selectedTrackIdRef.value, null); }; watch(selectedTrackIdRef, onSelectedTrackChange); watch(frameRef, onFrameChange); watch(onlyShowSelectedTrack, onOnlyShowSelectedTrackChange); - watch(filteredTracksRef.filteredAnnotations, onFilteredAnnotationsChange); + watch(trackEntries, onTracksChange); return { onSelectedTrackChange, diff --git a/client/src/components/track_3d_viewer/utils.spec.ts b/client/src/components/track_3d_viewer/utils.spec.ts new file mode 100644 index 000000000..c028b2e2b --- /dev/null +++ b/client/src/components/track_3d_viewer/utils.spec.ts @@ -0,0 +1,12 @@ +import { smoothBounds } from './utils'; + +describe('smoothBounds', () => { + it('centers a cube of the largest extent on each axis', () => { + expect(smoothBounds([-400, 600, 0, 200, 1500, 3500])) + .toEqual([-900, 1100, -900, 1100, 1500, 3500]); + }); + + it('falls back to a unit cube when nothing is visible', () => { + expect(smoothBounds([1, -1, 1, -1, 1, -1])).toEqual([-1, 1, -1, 1, -1, 1]); + }); +}); diff --git a/client/src/components/track_3d_viewer/utils.ts b/client/src/components/track_3d_viewer/utils.ts index f5338551f..cad4d74f3 100644 --- a/client/src/components/track_3d_viewer/utils.ts +++ b/client/src/components/track_3d_viewer/utils.ts @@ -1,18 +1,25 @@ import { Bounds, RGBColor } from '@kitware/vtk.js/types'; +export const noOp = () => undefined; + export function float2rgb(color: RGBColor): RGBColor { return [color[0] * 255, color[1] * 255, color[2] * 255]; } +/** + * A cube around the data: every axis spans the largest extent, centered on its + * own range, so positions far from the origin still fill the view. + */ export function smoothBounds(bounds: Bounds): Bounds { const [xmin, xmax, ymin, ymax, zmin, zmax] = bounds; - - const min = Math.min(xmin, ymin, zmin); - const max = Math.max(xmax, ymax, zmax); - + const half = Math.max(xmax - xmin, ymax - ymin, zmax - zmin) / 2; + if (!(half > 0)) { + return [-1, 1, -1, 1, -1, 1]; + } + const [x, y, z] = [(xmin + xmax) / 2, (ymin + ymax) / 2, (zmin + zmax) / 2]; return [ - min, max, - min, max, - min, max, + x - half, x + half, + y - half, y + half, + z - half, z + half, ]; } From 44798e441a7f823e8aa7172d7a44e782864c08c0 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 18:07:51 -0400 Subject: [PATCH 03/19] Keep the panels in place and add stereo + 3D display modes The 3D pane no longer sizes the layout, so the bottom track list and bars stay put. Adds both cameras plus the 3D viewer as a third pane, and narrows the Display selector. --- client/dive-common/components/Viewer.vue | 54 ++++++++++++++----- client/dive-common/stereoDisplay.spec.ts | 13 +++-- client/dive-common/stereoDisplay.ts | 45 ++++++++++------ .../track_3d_viewer/TrackViewer.vue | 11 ++++ 4 files changed, 87 insertions(+), 36 deletions(-) diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 568f931c3..69fad4f4c 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -97,7 +97,9 @@ import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls import TrackViewer from 'vue-media-annotator/components/track_3d_viewer/TrackViewer.vue'; import TrackViewerSettings from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettings.vue'; import TrackViewerSettingsStore from 'vue-media-annotator/components/track_3d_viewer/TrackViewerSettingsStore'; -import { displayOptions, displayValue, parseDisplayValue } from 'dive-common/stereoDisplay'; +import { + displayOptions, displayValue, parseDisplayValue, Viewer3dMode, +} from 'dive-common/stereoDisplay'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; import RegistrationToolsVue from './CameraRegistration/RegistrationTools.vue'; @@ -2110,7 +2112,7 @@ export default defineComponent({ } } } - viewer3dRequested.value = false; + viewer3dRequested.value = 'off'; if (isStereoPair.value) { context.register({ component: TrackViewerSettings, @@ -2290,30 +2292,34 @@ export default defineComponent({ }; const trackViewerSettingsStore = new TrackViewerSettingsStore(); - const viewer3dRequested = ref(false); + const viewer3dRequested = ref('off' as Viewer3dMode); const isStereoPair = computed(() => subType.value === 'stereo' && multiCamList.value.length === 2); - const viewer3dActive = computed(() => viewer3dRequested.value && isStereoPair.value); + const viewer3dMode = computed(() => (isStereoPair.value ? viewer3dRequested.value : 'off')); + const viewer3dActive = computed(() => viewer3dMode.value !== 'off'); const displayItems = computed(() => displayOptions( multiCamList.value, isStereoPair.value, defaultCamera.value, )); - const selectedDisplay = computed(() => displayValue(selectedCamera.value, viewer3dActive.value)); + const selectedDisplay = computed(() => displayValue(selectedCamera.value, viewer3dMode.value)); function changeDisplay(value: string) { - const { camera, viewer3d } = parseDisplayValue(value); - viewer3dRequested.value = viewer3d; + const { camera, mode } = parseDisplayValue(value); + viewer3dRequested.value = mode; changeCamera(camera); } - /** The 3D viewer takes the pane of the camera that is not selected. */ + /** In 'replace' mode the 3D viewer takes the pane of the camera that is not selected. */ function isCameraReplacedBy3dViewer(camera: string) { - return viewer3dActive.value && camera !== selectedCamera.value; + return viewer3dMode.value === 'replace' && camera !== selectedCamera.value; } const viewer3dPaneOrder = computed(() => ( - multiCamList.value.indexOf(selectedCamera.value) === 0 ? 1 : -1 + viewer3dMode.value === 'replace' && multiCamList.value.indexOf(selectedCamera.value) !== 0 + ? -1 : 1 )); - watch([viewer3dActive, selectedCamera], async () => { + watch([viewer3dMode, selectedCamera], async () => { await nextTick(); handleResize(); + // The bottom layout has no controls container for handleResize to act on. + onResize(); }); provideAnnotator( @@ -2866,7 +2872,7 @@ export default defineComponent({ :label="isStereoPair ? 'Display' : 'Camera'" class="mx-1 shrink camera-select" :class="{ 'display-select': isStereoPair }" - :menu-props="{ minWidth: 140 }" + :menu-props="{ minWidth: isStereoPair ? 200 : 140 }" outlined hide-details dense @@ -3344,8 +3350,28 @@ html { } .camera-select.display-select { - width: 200px; - max-width: 200px; + width: 148px; + max-width: 148px; + font-size: 0.8em; + + .v-input__slot { + padding: 0 6px 0 8px !important; + } + + .v-select__selection--comma { + max-width: 100%; + margin-right: 0; + } + + .v-select__selections input { + flex: 0 0 0; + width: 0; + padding: 0; + } + + .v-input__append-inner { + padding-left: 0; + } } .viewer-3d-split { diff --git a/client/dive-common/stereoDisplay.spec.ts b/client/dive-common/stereoDisplay.spec.ts index 028783ce4..9e333ef87 100644 --- a/client/dive-common/stereoDisplay.spec.ts +++ b/client/dive-common/stereoDisplay.spec.ts @@ -1,9 +1,11 @@ import { displayOptions, displayValue, parseDisplayValue } from './stereoDisplay'; describe('stereoDisplay', () => { - it('offers each stereo camera selected or beside the 3D viewer', () => { + it('offers each stereo camera selected, beside the 3D viewer, or with it as a third pane', () => { expect(displayOptions(['left', 'right'], true, 'left').map((option) => option.text)).toEqual([ - 'Left (Selected)', 'Right (Selected)', 'Left + 3D Viewer', 'Right + 3D Viewer', + 'Left (Selected)', 'Right (Selected)', + 'Left + 3D Viewer', 'Right + 3D Viewer', + 'Stereo (Left Sel) + 3D', 'Stereo (Right Sel) + 3D', ]); }); @@ -15,8 +17,9 @@ describe('stereoDisplay', () => { }); it('round-trips a display value', () => { - expect(parseDisplayValue(displayValue('right', true))).toEqual({ camera: 'right', viewer3d: true }); - expect(parseDisplayValue(displayValue('viewer3d:odd', false))) - .toEqual({ camera: 'viewer3d:odd', viewer3d: false }); + expect(parseDisplayValue(displayValue('right', 'replace'))).toEqual({ camera: 'right', mode: 'replace' }); + expect(parseDisplayValue(displayValue('left', 'beside'))).toEqual({ camera: 'left', mode: 'beside' }); + expect(parseDisplayValue(displayValue('viewer3d:odd', 'off'))) + .toEqual({ camera: 'viewer3d:odd', mode: 'off' }); }); }); diff --git a/client/dive-common/stereoDisplay.ts b/client/dive-common/stereoDisplay.ts index 994206e9c..fc07ecd29 100644 --- a/client/dive-common/stereoDisplay.ts +++ b/client/dive-common/stereoDisplay.ts @@ -1,33 +1,44 @@ /** - * Entries of the viewer's camera selector. A stereo pair offers, per camera, - * both cameras side by side with that camera selected, or that camera beside - * the 3D viewer, which takes the other camera's pane. + * Entries of the viewer's camera selector. A stereo pair offers, per selected + * camera: both cameras, that camera beside the 3D viewer (which takes the other + * camera's pane), or both cameras with the 3D viewer as a third pane. */ +export type Viewer3dMode = 'off' | 'replace' | 'beside'; + export interface DisplayOption { value: string; text: string; menuText: string; } -const CAMERA_PREFIX = 'camera:'; -const VIEWER_3D_PREFIX = 'viewer3d:'; +const PREFIXES: Record = { + off: 'camera:', + replace: 'viewer3d:', + beside: 'stereo3d:', +}; +const MODES = Object.keys(PREFIXES) as Viewer3dMode[]; -export function displayValue(camera: string, viewer3d: boolean) { - return `${viewer3d ? VIEWER_3D_PREFIX : CAMERA_PREFIX}${camera}`; +export function displayValue(camera: string, mode: Viewer3dMode) { + return `${PREFIXES[mode]}${camera}`; } -export function parseDisplayValue(value: string) { - const viewer3d = value.startsWith(VIEWER_3D_PREFIX); - return { - camera: value.slice((viewer3d ? VIEWER_3D_PREFIX : CAMERA_PREFIX).length), - viewer3d, - }; +export function parseDisplayValue(value: string): { camera: string; mode: Viewer3dMode } { + const mode = MODES.find((candidate) => value.startsWith(PREFIXES[candidate])) ?? 'off'; + return { camera: value.slice(PREFIXES[mode].length), mode }; } function capitalize(name: string) { return name.charAt(0).toUpperCase() + name.slice(1); } +function stereoLabel(camera: string, mode: Viewer3dMode) { + const name = capitalize(camera); + if (mode === 'replace') { + return `${name} + 3D Viewer`; + } + return mode === 'beside' ? `Stereo (${name} Sel) + 3D` : `${name} (Selected)`; +} + export function displayOptions( cameras: readonly string[], stereo: boolean, @@ -35,13 +46,13 @@ export function displayOptions( ): DisplayOption[] { if (!stereo) { return cameras.map((camera) => ({ - value: displayValue(camera, false), + value: displayValue(camera, 'off'), text: camera, menuText: camera === defaultCamera ? `${camera} (Default)` : camera, })); } - return [false, true].flatMap((viewer3d) => cameras.map((camera) => { - const text = `${capitalize(camera)} ${viewer3d ? '+ 3D Viewer' : '(Selected)'}`; - return { value: displayValue(camera, viewer3d), text, menuText: text }; + return MODES.flatMap((mode) => cameras.map((camera) => { + const text = stereoLabel(camera, mode); + return { value: displayValue(camera, mode), text, menuText: text }; })); } diff --git a/client/src/components/track_3d_viewer/TrackViewer.vue b/client/src/components/track_3d_viewer/TrackViewer.vue index be314c3e5..fad6d2d2c 100644 --- a/client/src/components/track_3d_viewer/TrackViewer.vue +++ b/client/src/components/track_3d_viewer/TrackViewer.vue @@ -284,8 +284,19 @@ export default defineComponent({ diff --git a/client/src/components/track_3d_viewer/sceneGuides.spec.ts b/client/src/components/track_3d_viewer/sceneGuides.spec.ts index 18ec70a0e..9e6657415 100644 --- a/client/src/components/track_3d_viewer/sceneGuides.spec.ts +++ b/client/src/components/track_3d_viewer/sceneGuides.spec.ts @@ -1,5 +1,5 @@ import { - floorGrid, niceStep, rigBaseline, rigCameras, + depthGrid, niceStep, rigBaseline, rigCameras, } from './sceneGuides'; const identity = [1, 0, 0, 0, 1, 0, 0, 0, 1]; @@ -66,7 +66,7 @@ describe('rigCameras', () => { }); }); -describe('floorGrid', () => { +describe('depthGrid', () => { it('uses round spacing', () => { expect(niceStep(6000)).toBe(1000); expect(niceStep(3100)).toBe(500); @@ -74,18 +74,18 @@ describe('floorGrid', () => { expect(niceStep(0)).toBe(1); }); - it('spans the rig and the data, below the lowest point', () => { - const grid = floorGrid([-400, 1100, -100, 260, 1500, 3100], [[0, 0, 0], [300, 170, -200]]); + it('spans the rig and the data, starting at the cameras', () => { + const grid = depthGrid([-400, 1100, -100, 260, 1500, 3100], [[0, 0, 0], [300, 170, -200]]); expect(grid.zs).toEqual([0, 500, 1000, 1500, 2000, 2500, 3000, 3500]); expect(grid.xs).toEqual([-500, 0, 500, 1000, 1500]); expect(grid.step).toBe(500); - // From above the highest point (y grows downward) to the last mark over the floor - expect(grid.ys).toEqual([-200, 0, 200]); - expect(grid.y).toBeCloseTo(260 + 0.02 * 3100); + // Heights run both ways from the grid at 0 (y grows downward) + expect(grid.ys).toEqual([-200, 0, 200, 400]); }); it('marks at least three heights however flat the scene', () => { - expect(floorGrid([0, 5000, 10, 12, 1000, 9000], []).ys.length).toBeGreaterThanOrEqual(3); - expect(floorGrid([0, 5000, 40, 40, 1000, 9000], []).ys.length).toBeGreaterThanOrEqual(3); + expect(depthGrid([0, 5000, 10, 12, 1000, 9000], []).ys).toEqual([0, 5, 10, 15]); + expect(depthGrid([0, 5000, 40, 40, 1000, 9000], []).ys).toEqual([0, 20, 40]); + expect(depthGrid([0, 5000, 0, 0, 1000, 9000], []).ys).toEqual([-2000, 0, 2000]); }); }); diff --git a/client/src/components/track_3d_viewer/sceneGuides.ts b/client/src/components/track_3d_viewer/sceneGuides.ts index 3e73531cd..23697d8e9 100644 --- a/client/src/components/track_3d_viewer/sceneGuides.ts +++ b/client/src/components/track_3d_viewer/sceneGuides.ts @@ -1,6 +1,6 @@ /** * Geometry of the 3D view's reference furniture: the stereo rig's cameras and - * a floor grid receding along the depth axis. Everything is in the left + * a grid receding along the depth axis. Everything is in the left * camera's frame (x right, y down, z forward), like the plotted positions. */ import { Bounds3, Vec3 } from './positions'; @@ -28,11 +28,11 @@ export interface CameraFrustum { corners: [Vec3, Vec3, Vec3, Vec3]; } -export interface FloorGrid { - y: number; +export interface DepthGrid { + /** Spacing of the x and z lines. */ step: number; xs: number[]; - /** Heights marked on the vertical axis, from the top of the scene down to the floor. */ + /** Heights marked on the vertical axis, above and below the grid. */ ys: number[]; zs: number[]; } @@ -116,30 +116,28 @@ function ticks(min: number, max: number, step: number) { } /** - * A grid under the data, reaching from the rig to the farthest position. - * y grows downward, so "under" is the largest y. Depth starts at the cameras: - * the grid is widened and lowered to take in `include`, never pulled back. + * The grid lies in the plane y = 0, level with the left camera's optical + * center, so positions read as above or below the cameras. It reaches from the + * rig to the farthest position; depth starts at the cameras, and the grid is + * widened to take in `include`, never pulled back behind them. */ -export function floorGrid(bounds: Bounds3, include: readonly Vec3[]): FloorGrid { +export function depthGrid(bounds: Bounds3, include: readonly Vec3[]): DepthGrid { const xs = [bounds[0], bounds[1], ...include.map((point) => point[0])]; - const ys = [bounds[2], bounds[3], ...include.map((point) => point[1])]; + const ys = [bounds[2], bounds[3], 0, ...include.map((point) => point[1])]; const zs = [bounds[4], bounds[5], 0]; const [xMin, xMax, zMin, zMax] = [Math.min(...xs), Math.max(...xs), Math.min(...zs), Math.max(...zs)]; - const extent = Math.max(xMax - xMin, zMax - zMin); - const step = niceStep(extent); - const y = Math.max(...ys) + 0.02 * extent; - // Heights span far less than the floor, so they get a spacing of their own, + const step = niceStep(Math.max(xMax - xMin, zMax - zMin)); + // Heights span far less than the grid, so they get a spacing of their own, // fine enough for the axis to carry at least three marks - const yMin = Math.min(...ys); + const [yMin, yMax] = [Math.min(...ys), Math.max(...ys)]; let heights: number[] = []; - for (let count = 2; heights.length < 3 && count <= 20; count += 1) { - heights = ticks(yMin, y, niceStep(y - yMin, count)).filter((value) => value <= y); + for (let count = 2; heights.length < 3 && count <= 20 && yMax > yMin; count += 1) { + heights = ticks(yMin, yMax, niceStep(yMax - yMin, count)); } return { step, - y, xs: ticks(xMin, xMax, step), - ys: heights, + ys: heights.length < 3 ? [-step, 0, step] : heights, zs: ticks(zMin, zMax, step), }; } diff --git a/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts index 48fb670f9..4e900f049 100644 --- a/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts +++ b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts @@ -1,5 +1,12 @@ /* eslint-disable import/prefer-default-export */ -import vtkAxesActor from '@kitware/vtk.js/Rendering/Core/AxesActor'; +import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; +import vtkAppendPolyData from '@kitware/vtk.js/Filters/General/AppendPolyData'; +import vtkArrowSource from '@kitware/vtk.js/Filters/Sources/ArrowSource'; +import type vtkAxesActor from '@kitware/vtk.js/Rendering/Core/AxesActor'; +import vtkCylinderSource from '@kitware/vtk.js/Filters/Sources/CylinderSource'; +import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray'; +import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; +import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; import vtkCamera from '@kitware/vtk.js/Rendering/Core/Camera'; import vtkInteractiveOrientationWidget from '@kitware/vtk.js/Widgets/Widgets3D/InteractiveOrientationWidget'; import vtkOrientationMarkerWidget from '@kitware/vtk.js/Interaction/Widgets/OrientationMarkerWidget'; @@ -21,13 +28,62 @@ const getMajorAxisFromViewUp = function getMajorAxisFromViewUp( return axis; }; +const AXIS_RADIUS = 0.03; + +function colored(polyData: vtkPolyData, color: [number, number, number]) { + const count = polyData.getPoints().getNumberOfPoints(); + const values = new Uint8Array(3 * count); + for (let i = 0; i < count; i += 1) { + values.set(color, 3 * i); + } + polyData.getPointData().setScalars(vtkDataArray.newInstance({ + name: 'color', numberOfComponents: 3, values, + })); + return polyData; +} + +/** + * The three axes as plain bars through the center: which way x and z run says + * nothing about the scene. Only y carries an arrow, pointing up, because y + * grows downward in the cameras' frame and "up" is worth marking. + */ +function buildAxes() { + const bar = (direction: Vector3) => vtkCylinderSource.newInstance({ + direction, height: 1, radius: AXIS_RADIUS, resolution: 60, + }).getOutputData(); + const up = vtkArrowSource.newInstance({ + direction: [0, -1, 0], + tipResolution: 60, + tipRadius: 0.1, + tipLength: 0.2, + shaftResolution: 60, + shaftRadius: AXIS_RADIUS, + }).getOutputData(); + // The arrow starts at the origin; center it like the bars + const points = up.getPoints().getData(); + for (let i = 1; i < points.length; i += 3) { + points[i] += 0.5; + } + + const source = vtkAppendPolyData.newInstance(); + source.setInputData(colored(bar([1, 0, 0]), [255, 0, 0])); + source.addInputData(colored(up, [255, 255, 0])); + source.addInputData(colored(bar([0, 0, 1]), [0, 128, 0])); + const mapper = vtkMapper.newInstance(); + mapper.setInputConnection(source.getOutputPort()); + const actor = vtkActor.newInstance(); + actor.setMapper(mapper); + return actor; +} + export function useOrientationMarkerWidget() { const previousVisibility = ref(false); const widgetManager = vtkWidgetManager.newInstance(); - const axes = vtkAxesActor.newInstance(); + const axes = buildAxes(); const orientationMarkerWidget = vtkOrientationMarkerWidget.newInstance({ - actor: axes, + // The widget only needs something to draw; its typings name two actors + actor: axes as unknown as vtkAxesActor, }); const interactiveOrientationWidget = vtkInteractiveOrientationWidget.newInstance(); diff --git a/client/src/components/track_3d_viewer/useSceneGuides.ts b/client/src/components/track_3d_viewer/useSceneGuides.ts index 90a163f26..7a6b008bb 100644 --- a/client/src/components/track_3d_viewer/useSceneGuides.ts +++ b/client/src/components/track_3d_viewer/useSceneGuides.ts @@ -7,7 +7,7 @@ import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; import vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; import { Bounds3, Vec3 } from './positions'; import { - floorGrid, rigBaseline, rigCameras, RigCalibration, + depthGrid, rigBaseline, rigCameras, RigCalibration, } from './sceneGuides'; export interface GuideLabel { @@ -53,7 +53,7 @@ function lineActor(segments: Segment[], color: Vec3, width: number) { return actor; } -/** Draws the stereo rig and a floor grid, and lists the labels that go with them. */ +/** Draws the stereo rig, a grid at camera height and the axes, and lists their labels. */ export default function useSceneGuides(renderer: Ref) { let actors: vtkActor[] = []; let labels: GuideLabel[] = []; @@ -78,29 +78,29 @@ export default function useSceneGuides(renderer: Ref) { const cameras = rigCameras(calibration, Math.max(0.6 * baseline, 0.04 * size)); const rigPoints = cameras.flatMap(({ center, corners }) => [center, ...corners]); - const grid = floorGrid(bounds, rigPoints); + const grid = depthGrid(bounds, rigPoints); const [xFirst, xLast] = [grid.xs[0], grid.xs[grid.xs.length - 1]]; const [zFirst, zLast] = [grid.zs[0], grid.zs[grid.zs.length - 1]]; - const yTop = grid.ys[0] ?? grid.y; + const [yTop, yBottom] = [grid.ys[0], grid.ys[grid.ys.length - 1]]; const gridSegments: Segment[] = [ - ...grid.xs.map((x): Segment => [[x, grid.y, zFirst], [x, grid.y, zLast]]), - ...grid.zs.map((z): Segment => [[xFirst, grid.y, z], [xLast, grid.y, z]]), + ...grid.xs.map((x): Segment => [[x, 0, zFirst], [x, 0, zLast]]), + ...grid.zs.map((z): Segment => [[xFirst, 0, z], [xLast, 0, z]]), ]; - // The line on the floor under the left camera's optical axis + // The left camera's optical axis, which lies in the grid const opticalAxis: Segment[] = xFirst < 0 && xLast > 0 - ? [[[0, grid.y, zFirst], [0, grid.y, zLast]]] : []; + ? [[[0, 0, zFirst], [0, 0, zLast]]] : []; const rigSegments = cameras.flatMap(({ center, corners }) => corners.flatMap( (corner, i): Segment[] => [[center, corner], [corner, corners[(i + 1) % corners.length]]], )); // The three axes meet at the grid's near left corner, out of the way of the data const tick = 0.15 * grid.step; const axes: Record<'x' | 'y' | 'z', Segment[]> = { - x: [[[xFirst, grid.y, zFirst], [xLast, grid.y, zFirst]]], + x: [[[xFirst, 0, zFirst], [xLast, 0, zFirst]]], y: [ - [[xFirst, grid.y, zFirst], [xFirst, yTop, zFirst]], + [[xFirst, yBottom, zFirst], [xFirst, yTop, zFirst]], ...grid.ys.map((y): Segment => [[xFirst - tick, y, zFirst], [xFirst + tick, y, zFirst]]), ], - z: [[[xFirst, grid.y, zFirst], [xFirst, grid.y, zLast]]], + z: [[[xFirst, 0, zFirst], [xFirst, 0, zLast]]], }; actors = [ @@ -118,9 +118,9 @@ export default function useSceneGuides(renderer: Ref) { const number = (value: number) => value.toLocaleString(); labels = [ ...grid.xs.map((x) => ({ - position: [x, grid.y, zFirst - gap] as Vec3, text: number(x), color: css(AXIS_COLORS.x), + position: [x, 0, zFirst - gap] as Vec3, text: number(x), color: css(AXIS_COLORS.x), })), - { position: [xLast + 2 * gap, grid.y, zFirst] as Vec3, text: 'X', color: css(AXIS_COLORS.x) }, + { position: [xLast + 2 * gap, 0, zFirst] as Vec3, text: 'X', color: css(AXIS_COLORS.x) }, ...grid.ys.map((y) => ({ position: [xFirst - gap, y, zFirst] as Vec3, text: number(y), color: css(AXIS_COLORS.y), })), @@ -128,10 +128,10 @@ export default function useSceneGuides(renderer: Ref) { position: [xFirst, yTop - gap, zFirst] as Vec3, text: 'Y', color: css(AXIS_COLORS.y), }, ...grid.zs.filter((z) => z !== zFirst).map((z) => ({ - position: [xFirst - gap, grid.y, z] as Vec3, text: number(z), color: css(AXIS_COLORS.z), + position: [xFirst - gap, 0, z] as Vec3, text: number(z), color: css(AXIS_COLORS.z), })), { - position: [xFirst, grid.y, zLast + 2 * gap] as Vec3, + position: [xFirst, 0, zLast + 2 * gap] as Vec3, text: 'Z (depth)', color: css(AXIS_COLORS.z), }, @@ -142,11 +142,11 @@ export default function useSceneGuides(renderer: Ref) { })), ]; - const ys = [bounds[2], yTop, grid.y, ...rigPoints.map((point) => point[1])]; + const ys = [bounds[2], bounds[3], yTop, yBottom, ...rigPoints.map((point) => point[1])]; // Room for the labels around the grid, and for the rig behind its near edge extent = [ xFirst - 3 * gap, xLast + 3 * gap, - Math.min(...ys) - gap, Math.max(...ys), + Math.min(...ys) - gap, Math.max(...ys) + gap, Math.min(zFirst - 2 * gap, ...rigPoints.map((point) => point[2])), zLast + 3 * gap, ]; }; @@ -155,7 +155,7 @@ export default function useSceneGuides(renderer: Ref) { build, clear, labels: () => labels, - /** Everything worth framing: the data, the grid under it and the rig. */ + /** Everything worth framing: the data, the grid through it and the rig. */ extent: () => extent, }; } diff --git a/client/src/components/track_3d_viewer/useTrackDrawer.ts b/client/src/components/track_3d_viewer/useTrackDrawer.ts index 160bedde2..8fa20bf65 100644 --- a/client/src/components/track_3d_viewer/useTrackDrawer.ts +++ b/client/src/components/track_3d_viewer/useTrackDrawer.ts @@ -36,7 +36,6 @@ export interface TrackDrawerParams { trackManager: TrackManager; onlyShowSelectedTrack: Ref; detectionGlyphSize: Ref; - positionedTrackCount: Ref; viewUtils: ViewUtils; renderer: Ref; } @@ -57,7 +56,6 @@ export default function useTrackDrawer({ trackManager, onlyShowSelectedTrack, detectionGlyphSize, - positionedTrackCount, viewUtils, renderer, }: TrackDrawerParams) { @@ -555,9 +553,6 @@ export default function useTrackDrawer({ emphasizeTrack(selectedTrackIdRef.value); } } - // eslint-disable-next-line no-param-reassign - positionedTrackCount.value = trackManager.getAllTracks().size; - viewUtils.rerender(); }; From 9c45e536bdeda441e9197d68ab82195f4bc2a308 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 20:52:33 -0400 Subject: [PATCH 13/19] Rest the camera diagrams on depth 0 and return the grid to the floor The diagrams open toward the scene again, with the square front at the camera's depth and the tip behind it. The grid goes back under the data. --- .../track_3d_viewer/sceneGuides.spec.ts | 25 ++++++----- .../components/track_3d_viewer/sceneGuides.ts | 45 ++++++++++--------- .../track_3d_viewer/useSceneGuides.ts | 42 ++++++++--------- 3 files changed, 59 insertions(+), 53 deletions(-) diff --git a/client/src/components/track_3d_viewer/sceneGuides.spec.ts b/client/src/components/track_3d_viewer/sceneGuides.spec.ts index 9e6657415..c5ce02c48 100644 --- a/client/src/components/track_3d_viewer/sceneGuides.spec.ts +++ b/client/src/components/track_3d_viewer/sceneGuides.spec.ts @@ -1,5 +1,5 @@ import { - depthGrid, niceStep, rigBaseline, rigCameras, + floorGrid, niceStep, rigBaseline, rigCameras, } from './sceneGuides'; const identity = [1, 0, 0, 0, 1, 0, 0, 0, 1]; @@ -9,8 +9,9 @@ describe('rigCameras', () => { const cameras = rigCameras(null, 2); expect(cameras.map((camera) => camera.name)).toEqual(['left']); expect(cameras[0].center).toEqual([0, 0, 0]); - // The diagram lies behind the optical center, ending at depth 0 - expect(cameras[0].corners.every((corner) => corner[2] === -2)).toBe(true); + // The square front rests at depth 0, with the tip behind it + expect(cameras[0].corners.every((corner) => corner[2] === 0)).toBe(true); + expect(cameras[0].tip).toEqual([0, 0, -2]); }); it('places the right camera at -Rt * T', () => { @@ -33,7 +34,7 @@ describe('rigCameras', () => { }, }, }, 10); - expect(left.corners).toEqual([[10, 5, -10], [-10, 5, -10], [-10, -5, -10], [10, -5, -10]]); + expect(left.corners).toEqual([[-10, -5, 0], [10, -5, 0], [10, 5, 0], [-10, 5, 0]]); }); it('matches VIAME for a measured rig', () => { @@ -66,7 +67,7 @@ describe('rigCameras', () => { }); }); -describe('depthGrid', () => { +describe('floorGrid', () => { it('uses round spacing', () => { expect(niceStep(6000)).toBe(1000); expect(niceStep(3100)).toBe(500); @@ -74,18 +75,18 @@ describe('depthGrid', () => { expect(niceStep(0)).toBe(1); }); - it('spans the rig and the data, starting at the cameras', () => { - const grid = depthGrid([-400, 1100, -100, 260, 1500, 3100], [[0, 0, 0], [300, 170, -200]]); + it('spans the rig and the data, below the lowest point', () => { + const grid = floorGrid([-400, 1100, -100, 260, 1500, 3100], [[0, 0, 0], [300, 170, -200]]); expect(grid.zs).toEqual([0, 500, 1000, 1500, 2000, 2500, 3000, 3500]); expect(grid.xs).toEqual([-500, 0, 500, 1000, 1500]); expect(grid.step).toBe(500); - // Heights run both ways from the grid at 0 (y grows downward) - expect(grid.ys).toEqual([-200, 0, 200, 400]); + // From above the highest point (y grows downward) to the last mark over the floor + expect(grid.ys).toEqual([-200, 0, 200]); + expect(grid.y).toBeCloseTo(260 + 0.02 * 3100); }); it('marks at least three heights however flat the scene', () => { - expect(depthGrid([0, 5000, 10, 12, 1000, 9000], []).ys).toEqual([0, 5, 10, 15]); - expect(depthGrid([0, 5000, 40, 40, 1000, 9000], []).ys).toEqual([0, 20, 40]); - expect(depthGrid([0, 5000, 0, 0, 1000, 9000], []).ys).toEqual([-2000, 0, 2000]); + expect(floorGrid([0, 5000, 10, 12, 1000, 9000], []).ys.length).toBeGreaterThanOrEqual(3); + expect(floorGrid([0, 5000, 40, 40, 1000, 9000], []).ys.length).toBeGreaterThanOrEqual(3); }); }); diff --git a/client/src/components/track_3d_viewer/sceneGuides.ts b/client/src/components/track_3d_viewer/sceneGuides.ts index 23697d8e9..2e41c921a 100644 --- a/client/src/components/track_3d_viewer/sceneGuides.ts +++ b/client/src/components/track_3d_viewer/sceneGuides.ts @@ -1,6 +1,6 @@ /** * Geometry of the 3D view's reference furniture: the stereo rig's cameras and - * a grid receding along the depth axis. Everything is in the left + * a floor grid receding along the depth axis. Everything is in the left * camera's frame (x right, y down, z forward), like the plotted positions. */ import { Bounds3, Vec3 } from './positions'; @@ -21,18 +21,20 @@ export interface CameraFrustum { /** The optical center, where depth is measured from. */ center: Vec3; /** - * Corners of the image plane, drawn behind the optical center as in a - * pinhole camera, so the diagram ends at the camera's position rather than - * reaching into the scene. + * The diagram opens toward the scene, and its square front rests at the + * camera's depth, around the optical center; the tip sits behind it. Drawn + * this way it ends at depth 0 instead of reaching into the scene. */ + tip: Vec3; + /** Top-left, top-right, bottom-right, bottom-left, as seen from the camera. */ corners: [Vec3, Vec3, Vec3, Vec3]; } -export interface DepthGrid { - /** Spacing of the x and z lines. */ +export interface FloorGrid { + y: number; step: number; xs: number[]; - /** Heights marked on the vertical axis, above and below the grid. */ + /** Heights marked on the vertical axis, from the top of the scene down to the floor. */ ys: number[]; zs: number[]; } @@ -72,8 +74,9 @@ export function rigCameras(calibration: RigCalibration | null, depth: number): C const frustum = (name: string, toLeft: (point: Vec3) => Vec3): CameraFrustum => ({ name, center: toLeft([0, 0, 0]), + tip: toLeft([0, 0, -depth]), corners: cornerRays(calibration, name) - .map(([x, y]) => toLeft([-x * depth, -y * depth, -depth])) as CameraFrustum['corners'], + .map(([x, y]) => toLeft([x * depth, y * depth, 0])) as CameraFrustum['corners'], }); const cameras = [frustum('left', (point) => point)]; const R = numbers(calibration?.R, 9); @@ -116,28 +119,30 @@ function ticks(min: number, max: number, step: number) { } /** - * The grid lies in the plane y = 0, level with the left camera's optical - * center, so positions read as above or below the cameras. It reaches from the - * rig to the farthest position; depth starts at the cameras, and the grid is - * widened to take in `include`, never pulled back behind them. + * A grid under the data, reaching from the rig to the farthest position. + * y grows downward, so "under" is the largest y. Depth starts at the cameras: + * the grid is widened and lowered to take in `include`, never pulled back. */ -export function depthGrid(bounds: Bounds3, include: readonly Vec3[]): DepthGrid { +export function floorGrid(bounds: Bounds3, include: readonly Vec3[]): FloorGrid { const xs = [bounds[0], bounds[1], ...include.map((point) => point[0])]; - const ys = [bounds[2], bounds[3], 0, ...include.map((point) => point[1])]; + const ys = [bounds[2], bounds[3], ...include.map((point) => point[1])]; const zs = [bounds[4], bounds[5], 0]; const [xMin, xMax, zMin, zMax] = [Math.min(...xs), Math.max(...xs), Math.min(...zs), Math.max(...zs)]; - const step = niceStep(Math.max(xMax - xMin, zMax - zMin)); - // Heights span far less than the grid, so they get a spacing of their own, + const extent = Math.max(xMax - xMin, zMax - zMin); + const step = niceStep(extent); + const y = Math.max(...ys) + 0.02 * extent; + // Heights span far less than the floor, so they get a spacing of their own, // fine enough for the axis to carry at least three marks - const [yMin, yMax] = [Math.min(...ys), Math.max(...ys)]; + const yMin = Math.min(...ys); let heights: number[] = []; - for (let count = 2; heights.length < 3 && count <= 20 && yMax > yMin; count += 1) { - heights = ticks(yMin, yMax, niceStep(yMax - yMin, count)); + for (let count = 2; heights.length < 3 && count <= 20; count += 1) { + heights = ticks(yMin, y, niceStep(y - yMin, count)).filter((value) => value <= y); } return { step, + y, xs: ticks(xMin, xMax, step), - ys: heights.length < 3 ? [-step, 0, step] : heights, + ys: heights, zs: ticks(zMin, zMax, step), }; } diff --git a/client/src/components/track_3d_viewer/useSceneGuides.ts b/client/src/components/track_3d_viewer/useSceneGuides.ts index 7a6b008bb..7b91ecdfa 100644 --- a/client/src/components/track_3d_viewer/useSceneGuides.ts +++ b/client/src/components/track_3d_viewer/useSceneGuides.ts @@ -7,7 +7,7 @@ import vtkPolyData from '@kitware/vtk.js/Common/DataModel/PolyData'; import vtkRenderer from '@kitware/vtk.js/Rendering/Core/Renderer'; import { Bounds3, Vec3 } from './positions'; import { - depthGrid, rigBaseline, rigCameras, RigCalibration, + floorGrid, rigBaseline, rigCameras, RigCalibration, } from './sceneGuides'; export interface GuideLabel { @@ -53,7 +53,7 @@ function lineActor(segments: Segment[], color: Vec3, width: number) { return actor; } -/** Draws the stereo rig, a grid at camera height and the axes, and lists their labels. */ +/** Draws the stereo rig and a floor grid, and lists the labels that go with them. */ export default function useSceneGuides(renderer: Ref) { let actors: vtkActor[] = []; let labels: GuideLabel[] = []; @@ -76,31 +76,31 @@ export default function useSceneGuides(renderer: Ref) { ]; const size = Math.max(bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4]); const cameras = rigCameras(calibration, Math.max(0.6 * baseline, 0.04 * size)); - const rigPoints = cameras.flatMap(({ center, corners }) => [center, ...corners]); + const rigPoints = cameras.flatMap(({ center, tip, corners }) => [center, tip, ...corners]); - const grid = depthGrid(bounds, rigPoints); + const grid = floorGrid(bounds, rigPoints); const [xFirst, xLast] = [grid.xs[0], grid.xs[grid.xs.length - 1]]; const [zFirst, zLast] = [grid.zs[0], grid.zs[grid.zs.length - 1]]; - const [yTop, yBottom] = [grid.ys[0], grid.ys[grid.ys.length - 1]]; + const yTop = grid.ys[0] ?? grid.y; const gridSegments: Segment[] = [ - ...grid.xs.map((x): Segment => [[x, 0, zFirst], [x, 0, zLast]]), - ...grid.zs.map((z): Segment => [[xFirst, 0, z], [xLast, 0, z]]), + ...grid.xs.map((x): Segment => [[x, grid.y, zFirst], [x, grid.y, zLast]]), + ...grid.zs.map((z): Segment => [[xFirst, grid.y, z], [xLast, grid.y, z]]), ]; - // The left camera's optical axis, which lies in the grid + // The line on the floor under the left camera's optical axis const opticalAxis: Segment[] = xFirst < 0 && xLast > 0 - ? [[[0, 0, zFirst], [0, 0, zLast]]] : []; - const rigSegments = cameras.flatMap(({ center, corners }) => corners.flatMap( - (corner, i): Segment[] => [[center, corner], [corner, corners[(i + 1) % corners.length]]], + ? [[[0, grid.y, zFirst], [0, grid.y, zLast]]] : []; + const rigSegments = cameras.flatMap(({ tip, corners }) => corners.flatMap( + (corner, i): Segment[] => [[tip, corner], [corner, corners[(i + 1) % corners.length]]], )); // The three axes meet at the grid's near left corner, out of the way of the data const tick = 0.15 * grid.step; const axes: Record<'x' | 'y' | 'z', Segment[]> = { - x: [[[xFirst, 0, zFirst], [xLast, 0, zFirst]]], + x: [[[xFirst, grid.y, zFirst], [xLast, grid.y, zFirst]]], y: [ - [[xFirst, yBottom, zFirst], [xFirst, yTop, zFirst]], + [[xFirst, grid.y, zFirst], [xFirst, yTop, zFirst]], ...grid.ys.map((y): Segment => [[xFirst - tick, y, zFirst], [xFirst + tick, y, zFirst]]), ], - z: [[[xFirst, 0, zFirst], [xFirst, 0, zLast]]], + z: [[[xFirst, grid.y, zFirst], [xFirst, grid.y, zLast]]], }; actors = [ @@ -118,9 +118,9 @@ export default function useSceneGuides(renderer: Ref) { const number = (value: number) => value.toLocaleString(); labels = [ ...grid.xs.map((x) => ({ - position: [x, 0, zFirst - gap] as Vec3, text: number(x), color: css(AXIS_COLORS.x), + position: [x, grid.y, zFirst - gap] as Vec3, text: number(x), color: css(AXIS_COLORS.x), })), - { position: [xLast + 2 * gap, 0, zFirst] as Vec3, text: 'X', color: css(AXIS_COLORS.x) }, + { position: [xLast + 2 * gap, grid.y, zFirst] as Vec3, text: 'X', color: css(AXIS_COLORS.x) }, ...grid.ys.map((y) => ({ position: [xFirst - gap, y, zFirst] as Vec3, text: number(y), color: css(AXIS_COLORS.y), })), @@ -128,10 +128,10 @@ export default function useSceneGuides(renderer: Ref) { position: [xFirst, yTop - gap, zFirst] as Vec3, text: 'Y', color: css(AXIS_COLORS.y), }, ...grid.zs.filter((z) => z !== zFirst).map((z) => ({ - position: [xFirst - gap, 0, z] as Vec3, text: number(z), color: css(AXIS_COLORS.z), + position: [xFirst - gap, grid.y, z] as Vec3, text: number(z), color: css(AXIS_COLORS.z), })), { - position: [xFirst, 0, zLast + 2 * gap] as Vec3, + position: [xFirst, grid.y, zLast + 2 * gap] as Vec3, text: 'Z (depth)', color: css(AXIS_COLORS.z), }, @@ -142,11 +142,11 @@ export default function useSceneGuides(renderer: Ref) { })), ]; - const ys = [bounds[2], bounds[3], yTop, yBottom, ...rigPoints.map((point) => point[1])]; + const ys = [bounds[2], yTop, grid.y, ...rigPoints.map((point) => point[1])]; // Room for the labels around the grid, and for the rig behind its near edge extent = [ xFirst - 3 * gap, xLast + 3 * gap, - Math.min(...ys) - gap, Math.max(...ys) + gap, + Math.min(...ys) - gap, Math.max(...ys), Math.min(zFirst - 2 * gap, ...rigPoints.map((point) => point[2])), zLast + 3 * gap, ]; }; @@ -155,7 +155,7 @@ export default function useSceneGuides(renderer: Ref) { build, clear, labels: () => labels, - /** Everything worth framing: the data, the grid through it and the rig. */ + /** Everything worth framing: the data, the grid under it and the rig. */ extent: () => extent, }; } From 4f692817695baaf408d5f38779cf55137b4c6389 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 20:56:17 -0400 Subject: [PATCH 14/19] Center the orientation marker's axes on each other --- .../track_3d_viewer/useOrientationMarkerWidget.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts index 4e900f049..bc2e76cb4 100644 --- a/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts +++ b/client/src/components/track_3d_viewer/useOrientationMarkerWidget.ts @@ -59,10 +59,12 @@ function buildAxes() { shaftResolution: 60, shaftRadius: AXIS_RADIUS, }).getOutputData(); - // The arrow starts at the origin; center it like the bars + // The tip makes the arrow lopsided; center its length on the origin so the + // three axes cross in the middle const points = up.getPoints().getData(); + const [, , yMin, yMax] = up.getPoints().getBounds(); for (let i = 1; i < points.length; i += 3) { - points[i] += 0.5; + points[i] -= (yMin + yMax) / 2; } const source = vtkAppendPolyData.newInstance(); From 43d8d9d1752368913c3cbd76f1729bcaec3c3613 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 21:02:13 -0400 Subject: [PATCH 15/19] Keep the stereo Display box a fixed width --- client/dive-common/components/Viewer.vue | 29 ++++-------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 040533f73..129a7f625 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -2303,27 +2303,6 @@ export default defineComponent({ defaultCamera.value, )); const selectedDisplay = computed(() => displayValue(selectedCamera.value, viewer3dMode.value)); - /** - * The stereo selector is as wide as its current label, leaving no dead - * space: estimated from the label's length, then measured once rendered. - */ - const DISPLAY_SELECT_CHROME = 42; // padding, border and the dropdown arrow - const displaySelect = ref(null as Vue | null); - const displayWidth = ref(undefined as string | undefined); - watch([selectedDisplay, isStereoPair, () => progress.loaded], async () => { - if (!isStereoPair.value) { - displayWidth.value = undefined; - return; - } - const text = displayItems.value - .find((item) => item.value === selectedDisplay.value)?.text ?? ''; - displayWidth.value = `${Math.ceil(6.3 * text.length) + DISPLAY_SELECT_CHROME}px`; - await nextTick(); - const label = displaySelect.value?.$el.querySelector('.v-select__selection--comma'); - if (label) { - displayWidth.value = `${label.scrollWidth + DISPLAY_SELECT_CHROME}px`; - } - }, { immediate: true }); function changeDisplay(value: string) { const { camera, mode } = parseDisplayValue(value); viewer3dRequested.value = mode; @@ -2689,8 +2668,6 @@ export default defineComponent({ viewer3dPaneOrder, rigCalibration, displayItems, - displayWidth, - displaySelect, selectedDisplay, changeDisplay, isCameraReplacedBy3dViewer, @@ -2904,13 +2881,11 @@ export default defineComponent({ Date: Mon, 28 Sep 2026 21:31:41 -0400 Subject: [PATCH 16/19] Lock the Display box width and add rotate buttons to the 3D view The box had no minimum width, so a crowded toolbar squeezed it to its label. The 3D view's reset uses the timeline's pan-and-zoom icon, and two corner buttons turn the view left and right. --- client/dive-common/components/Viewer.vue | 21 ++++- .../track_3d_viewer/TrackViewer.vue | 80 +++++++++++++++---- 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 129a7f625..0d32b1220 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -3370,14 +3370,27 @@ html { } // Fixed, so choosing another display never moves the buttons beside it; a -// label too long for the box is cut short +// label too long for the box is cut short. Without a minimum the toolbar +// squeezes the box to its label when space runs short, which is what let its +// width follow the selection. .camera-select.display-select { - width: 120px; - max-width: 120px; + flex: 0 0 116px !important; + width: 116px !important; + min-width: 116px !important; + max-width: 116px !important; font-size: 0.8em; + .v-input__control, + .v-input__slot, + .v-select__slot, + .v-select__selections { + min-width: 0; + max-width: 100%; + overflow: hidden; + } + .v-input__slot { - padding: 0 6px 0 8px !important; + padding: 0 4px 0 6px !important; } .v-select__selection--comma { diff --git a/client/src/components/track_3d_viewer/TrackViewer.vue b/client/src/components/track_3d_viewer/TrackViewer.vue index 9340d6aa2..608b79bf8 100644 --- a/client/src/components/track_3d_viewer/TrackViewer.vue +++ b/client/src/components/track_3d_viewer/TrackViewer.vue @@ -40,6 +40,8 @@ const GUIDE_FONT = '12px sans-serif'; // may travel before it counts as a drag of the view const MINIMUM_PICK_RADIUS = 6; const CLICK_TRAVEL = 4; +// Degrees the view turns for each press of a rotate button +const ROTATE_STEP = 15; export default defineComponent({ name: 'TrackViewer', @@ -93,6 +95,21 @@ export default defineComponent({ frameView(); viewUtils.rerender(); }; + /** + * Turns the scene about the vertical through what the view looks at: + * -1 counter-clockwise as seen from above, 1 clockwise. + */ + const rotateView = (direction: number) => { + if (!renderer.value) { + return; + } + const camera = renderer.value.getActiveCamera(); + // The scene turning one way is the viewer circling it the other way + camera.azimuth(direction * ROTATE_STEP); + camera.orthogonalizeViewUp(); + renderer.value.resetCameraClippingRange(); + viewUtils.rerender(); + }; const mediaController = injectAggregateController(); @@ -416,6 +433,7 @@ export default defineComponent({ return { vtkContainer, resetView, + rotateView, }; }, }); @@ -428,21 +446,36 @@ export default defineComponent({ class="vtk-container" :style="`--controls-height: ${controlsHeight}px`" /> - - - Reset view - + + mdi-image-filter-center-focus + + + mdi-rotate-left + + + mdi-rotate-right + @@ -464,11 +497,24 @@ export default defineComponent({ overflow: hidden; } -.vtk-reset { +.vtk-button { position: absolute; + background: rgba(0, 0, 0, 0.6); +} + +.vtk-reset { top: 8px; right: 8px; - background: rgba(0, 0, 0, 0.6); +} + +.vtk-rotate-left { + bottom: 8px; + left: 8px; +} + +.vtk-rotate-right { + bottom: 8px; + right: 8px; } .vtk-container > canvas { From 2683dc8abdf42bf37c66e4cfcd1ed1cf7fe5930e Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 21:39:44 -0400 Subject: [PATCH 17/19] Drop an unused lint exemption from the track drawer --- client/src/components/track_3d_viewer/useTrackDrawer.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/src/components/track_3d_viewer/useTrackDrawer.ts b/client/src/components/track_3d_viewer/useTrackDrawer.ts index 8fa20bf65..e51c5d482 100644 --- a/client/src/components/track_3d_viewer/useTrackDrawer.ts +++ b/client/src/components/track_3d_viewer/useTrackDrawer.ts @@ -1,5 +1,3 @@ -/* eslint-disable consistent-return */ - import { Ref, watch } from 'vue'; import { useTrackFilters, From 0bd0bc2bb3c83f957c8e55f6d948d7afa0d9c91d Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 22:50:14 -0400 Subject: [PATCH 18/19] Stop the Display box from clipping its label --- client/dive-common/components/Viewer.vue | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 0d32b1220..962802881 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -3374,23 +3374,31 @@ html { // squeezes the box to its label when space runs short, which is what let its // width follow the selection. .camera-select.display-select { - flex: 0 0 116px !important; - width: 116px !important; - min-width: 116px !important; - max-width: 116px !important; + flex: 0 0 120px !important; + width: 120px !important; + min-width: 120px !important; + max-width: 120px !important; font-size: 0.8em; + // Let the wrappers shrink to the box. Only the selected text is clipped: + // the label floats above the wrappers and clipping them would cut it off. .v-input__control, .v-input__slot, - .v-select__slot, + .v-select__slot { + min-width: 0; + max-width: 100%; + } + .v-select__selections { min-width: 0; max-width: 100%; overflow: hidden; } + // The left padding stays as Vuetify sets it, so the label lines up with + // the gap left for it in the outline .v-input__slot { - padding: 0 4px 0 6px !important; + padding-right: 4px !important; } .v-select__selection--comma { From f577a7caaa8150eba88d23f8b6aaea597d35999e Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Tue, 29 Sep 2026 09:23:53 -0400 Subject: [PATCH 19/19] Copy .npmrc into the web image build so the client installs The 3D viewer's @kitware/vtk.js only resolves with legacy-peer-deps. --- docker/girder.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/girder.Dockerfile b/docker/girder.Dockerfile index 0e1a52f62..38d2d9d2e 100644 --- a/docker/girder.Dockerfile +++ b/docker/girder.Dockerfile @@ -5,7 +5,7 @@ FROM node:24.15.0 AS client-builder WORKDIR /app # Install dependencies -COPY client/package.json client/package-lock.json /app/ +COPY client/package.json client/package-lock.json client/.npmrc /app/ RUN npm install # Build COPY .git/ /app/.git/