From 1a3577c9da1bf0d378b61b62d85ea27a19562848 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Thu, 24 Sep 2026 16:00:47 -0400 Subject: [PATCH 1/4] Take training output from the trained_model.zip pack VIAME now packs its output into trained_model.zip (folder renamed from category_models); desktop unpacks it into DIVE_Pipelines and the web task into the upload folder, both still accepting an unpacked folder. --- .../desktop/backend/native/common.spec.ts | 18 +++---- .../platform/desktop/backend/native/common.ts | 52 ++++++++++++++----- .../desktop/backend/native/modelPack.spec.ts | 26 +++++++++- .../desktop/backend/native/modelPack.ts | 32 ++++++++++++ server/dive_tasks/run_training.py | 15 +++++- 5 files changed, 117 insertions(+), 26 deletions(-) diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 47cc94305..f3a364520 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -539,7 +539,7 @@ beforeEach(() => { '/home/user/viamedata': { DIVE_Jobs: { goodTrainingJob: { - category_models: { + trained_model: { 'detector.pipe': '', 'trained_detector.zip': '', }, @@ -548,12 +548,12 @@ beforeEach(() => { missingModelFolder: {}, }, missingPipeTrainingJob: { - category_models: { + trained_model: { 'trained_detector.zip': '', }, }, detectorAndTrackerTrainingJob: { - category_models: { + trained_model: { 'detector.pipe': '', 'tracker.pipe': '', 'trained_detector.zip': '', @@ -2869,7 +2869,7 @@ describe('native.common', () => { const contents = await common.processTrainedPipeline(settings, trainingArgs, '/home/user/viamedata/DIVE_Jobs/goodTrainingJob/'); expect(contents).toEqual(['detector.pipe', 'trained_detector.zip']); //Data should be moved out of the current folder - const sourceFolder = fs.readdirSync('/home/user/viamedata/DIVE_Jobs/goodTrainingJob/category_models'); + const sourceFolder = fs.readdirSync('/home/user/viamedata/DIVE_Jobs/goodTrainingJob/trained_model'); expect(sourceFolder.length).toBe(0); //Folders hould be created for new pipeline const pipelineFolder = '/home/user/viamedata/DIVE_Pipelines/trainedPipelineName'; @@ -2888,10 +2888,10 @@ describe('native.common', () => { annotatedFramesOnly: false, }; await expect(common.processTrainedPipeline(settings, trainingArgs, '/home/user/viamedata/DIVE_Jobs/badTrainingJob/')).rejects.toThrow( - 'Path: /home/user/viamedata/DIVE_Jobs/badTrainingJob/category_models does not exist', + 'Path: /home/user/viamedata/DIVE_Jobs/badTrainingJob/trained_model does not exist', ); await expect(common.processTrainedPipeline(settings, trainingArgs, '/home/user/viamedata/DIVE_Jobs/missingPipeTrainingJob/')).rejects.toThrow( - 'Could not located trained pipe file inside of /home/user/viamedata/DIVE_Jobs/missingPipeTrainingJob/category_models', + 'Could not located trained pipe file inside of /home/user/viamedata/DIVE_Jobs/missingPipeTrainingJob/trained_model', ); }); @@ -2906,7 +2906,7 @@ describe('native.common', () => { const contents = await common.processTrainedPipeline(settings, trainingArgs, '/home/user/viamedata/DIVE_Jobs/goodTrainingJob/'); expect(contents).toEqual(['detector.pipe', 'trained_detector.zip']); //Data should be moved out of the current folder - const sourceFolder = fs.readdirSync('/home/user/viamedata/DIVE_Jobs/goodTrainingJob/category_models'); + const sourceFolder = fs.readdirSync('/home/user/viamedata/DIVE_Jobs/goodTrainingJob/trained_model'); expect(sourceFolder.length).toBe(0); //Folders hould be created for new pipeline const pipelineFolder = '/home/user/viamedata/DIVE_Pipelines/trainedPipelineName'; @@ -3161,12 +3161,12 @@ describe('resumable training jobs', () => { expect(found.map((j) => j.workingDir)).toEqual([npath.join(jobsDir, 'interrupted')]); }); - it('excludes legacy successful runs with an emptied category_models', async () => { + it('excludes legacy successful runs with an emptied trained_model', async () => { mockJobsFolder({ legacySuccess: jobDirConfig( 'legacySuccess', {}, - ['deep_training', 'category_models', 'input_folder_list.txt', 'input_truth_list.txt'], + ['deep_training', 'trained_model', 'input_folder_list.txt', 'input_truth_list.txt'], ), legacyInterrupted: jobDirConfig('legacyInterrupted', {}), }); diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index d33d9737e..73a0798e9 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -77,6 +77,7 @@ import processTrackAttributes from './attributeProcessor'; import { upgrade } from './migrations'; // TODO: Check to Refactor this import { getMultiCamUrls, transcodeMultiCam } from './multiCamUtils'; +import { extractModelPackTo } from './modelPack'; import { loadRegistrationFiles, referenceCameraName, saveRegistrationToDatasetDir, } from './cameraRegistration'; @@ -1901,13 +1902,45 @@ async function ingestDataFiles( processedFiles, annotationsSaved, meta, warnings: outwarnings, speciesStyling, }; } +const TrainedModelPackName = 'trained_model.zip'; +// Newest first; the legacy name is what releases before the pack wrote +const TrainedModelFolderNames = ['trained_model', 'category_models']; + +async function findTrainedModelDir(workingDir: string): Promise { + const candidates = TrainedModelFolderNames.map((name) => npath.join(workingDir, name)); + const present = await Promise.all(candidates.map((candidate) => fs.pathExists(candidate))); + return candidates.find((_, index) => present[index]); +} + /** * Need to take the trained pipeline if it exists and place it in the DIVE_Pipelines folder */ async function processTrainedPipeline(settings: Settings, args: RunTraining, workingDir: string) { - //Look for trained_detector.zip and detector.pipe and move them to DIVE_Pipelines folder const allowedPatterns = /^detector.+|^tracker.+|^generate.+/; - const trainedDir = npath.join(workingDir, '/category_models'); + const baseFolder = npath.join(settings.dataPath, PipelinesFolderName); + if (!fs.existsSync(baseFolder)) { + await fs.mkdir(baseFolder); + } + const folderName = npath.join(baseFolder, args.pipelineName); + if (!fs.existsSync(folderName)) { + await fs.mkdir(folderName); + } + + // Current VIAME packs the whole output into one zip + const pack = npath.join(workingDir, TrainedModelPackName); + if (await fs.pathExists(pack)) { + await extractModelPackTo(pack, folderName); + const packContents = await fs.readdir(folderName); + if (!packContents.some((p) => p.match(allowedPatterns))) { + throw new Error(`Could not located trained pipe file inside of ${pack}`); + } + await fs.remove(pack); + return packContents; + } + + // An unpacked folder: SVM trainers keep one, as did older VIAME releases + const trainedDir = await findTrainedModelDir(workingDir) + ?? npath.join(workingDir, TrainedModelFolderNames[0]); const exists = await fs.pathExists(trainedDir); if (!exists) { throw new Error(`Path: ${trainedDir} does not exist`); @@ -1918,15 +1951,6 @@ async function processTrainedPipeline(settings: Settings, args: RunTraining, wor if (!pipes.length) { throw new Error(`Could not located trained pipe file inside of ${trainedDir}`); } - const baseFolder = npath.join(settings.dataPath, PipelinesFolderName); - if (!fs.existsSync(baseFolder)) { - await fs.mkdir(baseFolder); - } - - const folderName = npath.join(baseFolder, args.pipelineName); - if (!fs.existsSync(folderName)) { - await fs.mkdir(folderName); - } //Move detector and model to the new folder await Promise.all(folderContents.map(async (item) => { const abspath = npath.join(trainedDir, item); @@ -1974,10 +1998,10 @@ async function findResumableTrainingJobs(settings: Settings): Promise fs.existsSync(npath.join(workingDir, f)))) return; // Manifests predating final-status recording never carry an end time; a - // successful run's models were moved out of category_models, leaving it empty + // successful run's models were moved out of the model folder, leaving it empty if (manifest.endTime === undefined) { - const modelsDir = npath.join(workingDir, 'category_models'); - if (fs.existsSync(modelsDir) && (await fs.readdir(modelsDir)).length === 0) return; + const modelsDir = await findTrainedModelDir(workingDir); + if (modelsDir && (await fs.readdir(modelsDir)).length === 0) return; } // The jobs folder may have been relocated since the manifest was written results.push({ ...manifest, workingDir }); diff --git a/client/platform/desktop/backend/native/modelPack.spec.ts b/client/platform/desktop/backend/native/modelPack.spec.ts index 258e52d28..6ecf1b9ea 100644 --- a/client/platform/desktop/backend/native/modelPack.spec.ts +++ b/client/platform/desktop/backend/native/modelPack.spec.ts @@ -4,7 +4,10 @@ import fs from 'fs-extra'; import archiver from 'archiver'; import type { Pipe } from 'dive-common/apispec'; import { PipelinesFolderName, Settings } from 'platform/desktop/constants'; -import { exportModelPack, importModelPack, modelPackPaths } from './modelPack'; +import * as common from './common'; +import { + exportModelPack, extractModelPackTo, importModelPack, modelPackPaths, +} from './modelPack'; const layouts: { name: string; files: string[]; expected: string[] }[] = JSON.parse( fs.readFileSync(path.resolve(__dirname, '../../../../../testutils/model-pack-layouts.json'), 'utf8'), @@ -31,6 +34,27 @@ async function makeZip(files: string[], destination: string) { }); } +it('extracts a training pack as it is', async () => { + const source = path.join(temp, 'trained_model.zip'); + await makeZip(['detector.pipe', 'trained_detector.pth', 'test_results/plots/pr.csv'], source); + const written = await extractModelPackTo(source, path.join(temp, 'out')); + expect(written.sort()).toEqual(['detector.pipe', 'test_results/plots/pr.csv', 'trained_detector.pth']); + expect(await fs.readFile(path.join(temp, 'out', 'test_results', 'plots', 'pr.csv'), 'utf8')).toBe('contents of test_results/plots/pr.csv'); +}); + +it('moves a trained pack into the pipelines folder and drops the archive', async () => { + const jobDir = path.join(temp, 'DIVE_Jobs', 'job'); + await fs.ensureDir(jobDir); + await makeZip(['detector.pipe', 'trained_detector.pth', 'MODEL_CARD.md'], path.join(jobDir, 'trained_model.zip')); + const contents = await common.processTrainedPipeline(settings, { pipelineName: 'fish' } as never, jobDir); + expect(contents.sort()).toEqual(['MODEL_CARD.md', 'detector.pipe', 'trained_detector.pth']); + expect(await fs.pathExists(path.join(jobDir, 'trained_model.zip'))).toBe(false); + expect(await fs.pathExists(path.join(temp, PipelinesFolderName, 'fish', 'detector.pipe'))).toBe(true); + await makeZip(['notes.txt'], path.join(jobDir, 'trained_model.zip')); + await expect(common.processTrainedPipeline(settings, { pipelineName: 'nopipe' } as never, jobDir)) + .rejects.toThrow('Could not located trained pipe'); +}); + it.each(layouts)('imports $name layout with paths and contents intact', async ({ files, expected }) => { expect([...modelPackPaths(files).values()]).toEqual(expected); const source = path.join(temp, 'fish.zip'); diff --git a/client/platform/desktop/backend/native/modelPack.ts b/client/platform/desktop/backend/native/modelPack.ts index 4958cb34b..014b2c166 100644 --- a/client/platform/desktop/backend/native/modelPack.ts +++ b/client/platform/desktop/backend/native/modelPack.ts @@ -121,6 +121,38 @@ export async function importModelPack(settings: Settings, filename: string): Pro } } +/** + * Unpack an archive as it is, every entry under destination. Used for the + * pack `viame train` writes, whose layout is already the pipeline folder's. + */ +export async function extractModelPackTo(filename: string, destination: string): Promise { + let zip: yauzl.ZipFile | undefined; + const written: string[] = []; + try { + zip = await openArchive(filename); + const entries = (await readEntries(zip)).filter((entry) => !entry.fileName.endsWith('/')); + for (const entry of entries) { + const relative = entry.fileName.replace(/\\/g, '/'); + const parts = relative.split('/'); + if (relative.startsWith('/') || parts.some((p) => p === '..' || p === '' || p.includes(':'))) { + throw new Error(`Unsafe model archive path: ${entry.fileName}`); + } + const target = path.join(destination, ...parts); + await fs.ensureDir(path.dirname(target)); + const source = await new Promise((resolve, reject) => { + zip!.openReadStream(entry, (error, stream) => { + if (error || !stream) reject(error || new Error('Unable to read ZIP entry')); else resolve(stream); + }); + }); + await streamPipeline(source, fs.createWriteStream(target)); + written.push(relative); + } + } finally { + zip?.close(); + } + return written; +} + export async function exportModelPack(settings: Settings, model: Pipe, destination: string): Promise { const root = await fs.realpath(path.join(settings.dataPath, PipelinesFolderName)); const folder = await fs.realpath(path.dirname(model.pipe)); diff --git a/server/dive_tasks/run_training.py b/server/dive_tasks/run_training.py index 57e7f11ec..96c655aca 100644 --- a/server/dive_tasks/run_training.py +++ b/server/dive_tasks/run_training.py @@ -12,7 +12,7 @@ from dive_tasks import utils from dive_tasks.manager import patch_manager from dive_tasks.viame_config import Config -from dive_utils import constants +from dive_utils import constants, model_pack from dive_utils.types import ExportTrainedPipelineJob, TrainingJob @@ -138,7 +138,7 @@ def train_pipeline(self: Task, params: TrainingJob): data_list.write(f"{folder_path}\n") truth_list.write(f"{groundtruth_path}\n") - training_results_path = utils.make_directory(output_path / "category_models") + training_results_path = utils.make_directory(output_path / "trained_model") command = [ f". {shlex.quote(str(conf.viame_setup_script))} &&", @@ -187,6 +187,17 @@ def train_pipeline(self: Task, params: TrainingJob): } utils.stream_subprocess(self, context, manager, popen_kwargs) + # Current VIAME packs the whole output into one zip; older releases and + # SVM trainers leave a folder (category_models before the rename) + pack_path = output_path / "trained_model.zip" + if pack_path.exists(): + training_results_path = utils.make_directory(output_path / "trained_model") + model_pack.extract_model_pack(pack_path, training_results_path) + elif len(list(training_results_path.glob("*"))) == 0: + legacy_path = output_path / "category_models" + if legacy_path.exists() and any(legacy_path.iterdir()): + training_results_path = legacy_path + # Check that there are results in the output path if len(list(training_results_path.glob("*"))) == 0: raise RuntimeError("Training output didn't produce results, discarding...") From 56403a5d187fbaf7a9922493069af769dbf762ac Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Thu, 24 Sep 2026 16:05:10 -0400 Subject: [PATCH 2/4] Add an optional 3D descriptor-space view of search results Grid / 3D toggle on the results toolbar (grid stays the default): the query sits at the center with the top N results hung off it on lines, placed by the service's new layout_results command (VIAME side). Co-Authored-By: Claude Fable 5.1 --- client/dive-common/apispec.ts | 18 + client/platform/desktop/backend/ipcService.ts | 5 + .../desktop/backend/native/videoSearch.ts | 8 + client/platform/desktop/frontend/api.ts | 7 + .../desktop/frontend/components/QueryPage.vue | 5 +- .../components/VideoSearchResultsGrid.spec.ts | 139 +++++ .../components/VideoSearchResultsGrid.vue | 194 ++++++- .../components/VideoSearchResultsSpace.vue | 543 ++++++++++++++++++ .../platform/desktop/frontend/querySession.ts | 4 + .../desktop/frontend/resultSpace.spec.ts | 72 +++ .../platform/desktop/frontend/resultSpace.ts | 126 ++++ .../desktop/frontend/useVideoSearch.ts | 14 +- docs/Query.md | 2 + 13 files changed, 1127 insertions(+), 10 deletions(-) create mode 100644 client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts create mode 100644 client/platform/desktop/frontend/components/VideoSearchResultsSpace.vue create mode 100644 client/platform/desktop/frontend/resultSpace.spec.ts create mode 100644 client/platform/desktop/frontend/resultSpace.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 43c146267..2b726576c 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -888,6 +888,24 @@ export interface VideoSearchQueryResponse { feedback_requests?: VideoSearchResult[]; } +/** One result placed in descriptor space around the query (at the origin). */ +export interface VideoSearchLayoutPoint { + ref: string; + /** Offset from the query along the three principal axes of the result set. */ + position: [number, number, number]; + /** Full-dimensional descriptor distance from the query. */ + distance: number; +} + +export interface VideoSearchLayoutResponse { + success: boolean; + error?: string; + dimensions?: number; + points?: VideoSearchLayoutPoint[]; + /** Results with no stored descriptor vector, left out of the layout. */ + missing?: string[]; +} + export { provideApi, useApi, diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index bbde40635..7dcb3b259 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -717,6 +717,11 @@ export default function register() { return manager.refine(args.positiveIds, args.negativeIds); }); + ipcMain.handle('video-search-layout', async (_, args: { refs: string[] }) => { + const manager = videoSearch.getQueryServiceManager(); + return manager.layoutResults(args.refs); + }); + ipcMain.handle('video-search-export-model', async (_, args: { name: string }) => { const outputDir = await videoSearch.exportSearchModel(settings.get(), args.name); return { success: true, outputDir }; diff --git a/client/platform/desktop/backend/native/videoSearch.ts b/client/platform/desktop/backend/native/videoSearch.ts index 8301bf8fd..1e892ad1c 100644 --- a/client/platform/desktop/backend/native/videoSearch.ts +++ b/client/platform/desktop/backend/native/videoSearch.ts @@ -770,6 +770,14 @@ export class QueryServiceManager extends EventEmitter { return QueryServiceManager.check(response, 'Query refinement'); } + async layoutResults(refs: string[]): Promise { + const response = await this.sendRequest({ + command: 'layout_results', + refs, + }, 'Result layout'); + return QueryServiceManager.check(response, 'Result layout'); + } + async exportModel(outputPath?: string): Promise { const response = await this.sendRequest({ command: 'export_model', diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 74d87a1aa..f0e57d99b 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -12,6 +12,7 @@ import type { PipelineJobResult, ScoringDatasetSummary, ScoringJobArgs, ScoringResult, ScoringResultSummary, ScoringSourceOptions, VideoSearchIndexStatus, VideoSearchIndexMethod, VideoSearchQueryResponse, VideoSearchIndexInfo, + VideoSearchLayoutResponse, } from 'dive-common/apispec'; import axios, { AxiosInstance } from 'axios'; import { watch } from 'vue'; @@ -686,6 +687,11 @@ async function videoSearchRefine(positiveIds: string[], negativeIds: string[]): return window.diveDesktop.invoke('video-search-refine', { positiveIds, negativeIds }); } +/** Place the given results of the open query in 3D descriptor space around it. */ +async function videoSearchLayout(refs: string[]): Promise { + return window.diveDesktop.invoke('video-search-layout', { refs }); +} + async function videoSearchExportModel(name: string): Promise<{ success: boolean; outputDir: string }> { return window.diveDesktop.invoke('video-search-export-model', { name }); } @@ -1148,6 +1154,7 @@ export { videoSearchFormulate, videoSearchQuery, videoSearchRefine, + videoSearchLayout, videoSearchExportModel, videoSearchClose, videoSearchExtractFrame, diff --git a/client/platform/desktop/frontend/components/QueryPage.vue b/client/platform/desktop/frontend/components/QueryPage.vue index 565143b42..12dc404cd 100644 --- a/client/platform/desktop/frontend/components/QueryPage.vue +++ b/client/platform/desktop/frontend/components/QueryPage.vue @@ -87,7 +87,9 @@ export default defineComponent({ hidden: (result) => searchReview.isRemoved(result), }); const textChips = resumed?.textChips ?? createItemChips(page.textItems); - const resultsMemory = resumed?.results ?? reactive({ page: 0, hideReviewed: false }); + const resultsMemory = resumed?.results ?? reactive({ + page: 0, hideReviewed: false, space: false, spaceCount: 25, + }); const gridSettings = usePersistentGridSettings(); const textGridActive = computed(() => view.value === 'query' && page.mode.value === 'text'); const textGrid = useReviewGrid({ @@ -764,6 +766,7 @@ export default defineComponent({ :search-chips="searchChips" :search-review="searchReview" :memory="resultsMemory" + :exemplar-url="page.mode.value === 'video' ? videoFrameUrl : imageUrl" @open-result="openViewer" @save-model="openSaveModel" /> diff --git a/client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts new file mode 100644 index 000000000..5b95fe654 --- /dev/null +++ b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts @@ -0,0 +1,139 @@ +// @vitest-environment jsdom +import { shallowMount } from '@vue/test-utils'; +import { + computed, markRaw, reactive, ref, +} from 'vue'; +import type { ReviewItem } from 'dive-common/review/types'; +import VideoSearchResultsGrid from './VideoSearchResultsGrid.vue'; + +const mocks = vi.hoisted(() => ({ search: null as unknown })); +vi.mock('../useVideoSearch', () => ({ useVideoSearch: () => mocks.search })); +vi.mock('dive-common/vue-utilities/prompt-service', () => ({ usePrompt: () => ({ prompt: vi.fn() }) })); +vi.mock('dive-common/use/useReview', () => ({ useReview: () => null })); +vi.mock('vue-media-annotator/provides', () => ({ useHandler: () => null })); +vi.mock('./VideoSearchResultsSpace.vue', () => ({ + default: { + name: 'VideoSearchResultsSpace', + props: ['points', 'cells', 'exemplarUrl', 'loading', 'error', 'missingCount'], + render: (h: (tag: string) => unknown) => h('div'), + }, +})); + +function item(key: string, frame: number): ReviewItem { + return { + key, + datasetId: 'a', + trackId: frame, + primary: { frame, bounds: [0, 0, 10, 10] }, + frames: [{ frame, bounds: [0, 0, 10, 10] }], + keyframeCount: 1, + type: '', + confidence: 1, + }; +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function mount(refs: string[]) { + const results = refs.map((ref, i) => ({ ref, stream_id: 's', relevancy_score: 1 - i * 0.1 })); + const search = { + state: reactive({ + results, adjudications: {} as Record, busy: null, error: null, iteration: 0, queryGeneration: 1, modelAvailable: false, + }), + resultDatasetName: () => null, + resultIsLocal: () => true, + resultDatasetId: () => 'a', + mark: vi.fn(), + refine: vi.fn(), + layoutResults: vi.fn(async (wanted: string[]) => ({ + success: true, + points: wanted.map((ref, i) => ({ ref, position: [i, 0, 0] as [number, number, number], distance: i * 2 })), + missing: [], + })), + }; + mocks.search = search; + const items = computed(() => search.state.results.map((r, i) => item(r.ref, i))); + // Raw: the root instance would otherwise observe the prop and unwrap its refs. + const searchChips = markRaw({ + items, + itemsByRef: computed(() => new Map(items.value.map((i) => [i.key, i]))), + chips: ref>({ [refs[0]]: 'data:chip' }), + store: { + chips: ref({}), + sequences: ref({}), + transforms: ref({}), + sequenceTransforms: ref({}), + failures: ref({}), + ensurePrimary: vi.fn(), + ensureSequences: vi.fn(), + trimQueues: vi.fn(), + setOptions: vi.fn(), + }, + dispose: vi.fn(), + }); + const memory = reactive({ + page: 0, hideReviewed: false, space: false, spaceCount: 2, + }); + const wrapper = shallowMount(VideoSearchResultsGrid, { + propsData: { + inline: true, searchChips, memory, exemplarUrl: 'file:///exemplar.jpg', + }, + stubs: ['v-card', 'v-toolbar', 'v-toolbar-title', 'v-btn', 'v-btn-toggle', 'v-icon', 'v-spacer', 'v-select', 'v-progress-linear', 'v-alert'], + }); + return { + wrapper, search, searchChips, memory, + }; +} + +it('only places results in descriptor space once the 3D view is chosen, remembering the choice', async () => { + const { + wrapper, search, searchChips, memory, + } = mount(['0:1', '0:2', '0:3']); + await flush(); + expect(search.layoutResults).not.toHaveBeenCalled(); + expect(wrapper.findComponent({ name: 'VideoSearchResultsSpace' }).exists()).toBe(false); + + (wrapper.vm as unknown as { space: boolean }).space = true; + await flush(); + expect(search.layoutResults).toHaveBeenCalledWith(['0:1', '0:2']); + expect(searchChips.store.ensurePrimary).toHaveBeenCalledWith(expect.arrayContaining([expect.objectContaining({ key: '0:1' })])); + expect(memory).toMatchObject({ space: true, spaceCount: 2 }); + const space = wrapper.findComponent({ name: 'VideoSearchResultsSpace' }); + expect(space.exists()).toBe(true); + expect(space.props('points')).toEqual([ + { key: '0:1', position: [0, 0, 0] }, { key: '0:2', position: [1, 0, 0] }, + ]); + expect(space.props('cells')).toMatchObject([ + { + key: '0:1', rank: 1, chip: 'data:chip', distance: 0, score: 1, + }, + { + key: '0:2', rank: 2, chip: null, distance: 2, + }, + ]); + expect(space.props('exemplarUrl')).toBe('file:///exemplar.jpg'); + + (wrapper.vm as unknown as { spaceCount: number }).spaceCount = 3; + await flush(); + expect(search.layoutResults).toHaveBeenLastCalledWith(['0:1', '0:2', '0:3']); + expect(memory.spaceCount).toBe(3); + + // Refinement re-ranks under the same refs: the layout is fetched again. + search.state.results = [...search.state.results].reverse(); + search.state.iteration = 1; + await flush(); + expect(search.layoutResults).toHaveBeenLastCalledWith(['0:3', '0:2', '0:1']); + wrapper.destroy(); +}); + +it('marks and opens results from the 3D view through the shared session', async () => { + const { wrapper, search } = mount(['0:1', '0:2']); + (wrapper.vm as unknown as { space: boolean }).space = true; + await flush(); + const space = wrapper.findComponent({ name: 'VideoSearchResultsSpace' }); + space.vm.$emit('mark', '0:2', 'negative'); + expect(search.mark).toHaveBeenCalledWith('0:2', 'negative'); + space.vm.$emit('open', '0:2'); + expect(wrapper.emitted('open-result')?.[0]).toEqual(['a', 1, undefined]); + wrapper.destroy(); +}); diff --git a/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue index 29c06b8d2..1b9ba4235 100644 --- a/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue +++ b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue @@ -1,6 +1,6 @@ + + + + diff --git a/client/platform/desktop/frontend/querySession.ts b/client/platform/desktop/frontend/querySession.ts index be7035170..0f07dab6a 100644 --- a/client/platform/desktop/frontend/querySession.ts +++ b/client/platform/desktop/frontend/querySession.ts @@ -14,6 +14,10 @@ import type { SearchReview } from './useSearchReview'; export interface ResultsGridMemory { page: number; hideReviewed: boolean; + /** Showing the 3D descriptor-space view instead of the grid. */ + space: boolean; + /** How many top results the 3D view places. */ + spaceCount: number; } export interface QuerySession { diff --git a/client/platform/desktop/frontend/resultSpace.spec.ts b/client/platform/desktop/frontend/resultSpace.spec.ts new file mode 100644 index 000000000..d4fe415b3 --- /dev/null +++ b/client/platform/desktop/frontend/resultSpace.spec.ts @@ -0,0 +1,72 @@ +import { + DEFAULT_ORBIT, normalizePositions, orbitDrag, orbitZoom, paintOrder, pick, project, rotate, +} from './resultSpace'; + +const viewport = { width: 400, height: 300 }; + +it('normalizes the farthest point onto the unit sphere and keeps the origin fixed', () => { + const points = normalizePositions([ + { key: 'a', position: [0, 0, 0] }, + { key: 'b', position: [3, 0, 4] }, + { key: 'c', position: [0, 2.5, 0] }, + ]); + expect(points[0].position).toEqual([0, 0, 0]); + expect(points[1].position).toEqual([0.6, 0, 0.8]); + expect(points[2].position).toEqual([0, 0.5, 0]); + expect(normalizePositions([{ key: 'a', position: [0, 0, 0] }])[0].position).toEqual([0, 0, 0]); +}); + +it('projects the origin to the viewport center and nearer points larger', () => { + const orbit = { yaw: 0, pitch: 0, zoom: 1 }; + const [origin, near, far] = project([ + { key: 'o', position: [0, 0, 0] }, + { key: 'near', position: [0.5, 0, 0.5] }, + { key: 'far', position: [0.5, 0, -0.5] }, + ], orbit, viewport); + expect(origin.x).toBe(200); + expect(origin.y).toBe(150); + expect(origin.scale).toBe(1); + expect(near.depth).toBeLessThan(far.depth); + expect(near.scale).toBeGreaterThan(1); + expect(far.scale).toBeLessThan(1); + expect(near.x - 200).toBeGreaterThan(far.x - 200); + expect(paintOrder([near, far, origin]).map((p) => p.key)).toEqual(['far', 'o', 'near']); +}); + +it('rotates with yaw and pitch and zooms toward the origin', () => { + const [x, , z] = rotate([1, 0, 0], { yaw: Math.PI / 2, pitch: 0, zoom: 1 }); + expect(x).toBeCloseTo(0); + expect(z).toBeCloseTo(-1); + const [, y, z2] = rotate([0, 0, 1], { yaw: 0, pitch: Math.PI / 2, zoom: 1 }); + expect(y).toBeCloseTo(-1); + expect(z2).toBeCloseTo(0); + const [side] = project([{ key: 's', position: [0.5, 0, 0] }], DEFAULT_ORBIT, viewport); + const [closer] = project([{ key: 's', position: [0.5, 0, 0] }], orbitZoom(DEFAULT_ORBIT, -100), viewport); + expect(Math.abs(closer.x - 200)).toBeGreaterThan(Math.abs(side.x - 200)); + expect(orbitZoom(DEFAULT_ORBIT, 100000).zoom).toBe(0.4); + expect(orbitZoom(DEFAULT_ORBIT, -100000).zoom).toBe(4); +}); + +it('clamps the pitch while dragging', () => { + const dragged = orbitDrag(DEFAULT_ORBIT, 50, 100000); + expect(dragged.yaw).toBeCloseTo(DEFAULT_ORBIT.yaw + 0.4); + expect(dragged.pitch).toBeLessThan(Math.PI / 2); + expect(orbitDrag(DEFAULT_ORBIT, 0, -100000).pitch).toBeGreaterThan(-Math.PI / 2); +}); + +it('picks the nearest billboard under the cursor', () => { + const projected = [ + { + key: 'behind', x: 100, y: 100, depth: 4, scale: 0.8, + }, + { + key: 'front', x: 104, y: 98, depth: 2, scale: 1.5, + }, + { + key: 'elsewhere', x: 300, y: 100, depth: 1, scale: 1, + }, + ]; + expect(pick(projected, 101, 101, 20)?.key).toBe('front'); + expect(pick(projected, 100, 100, 1)?.key).toBe('behind'); + expect(pick(projected, 200, 200, 20)).toBeNull(); +}); diff --git a/client/platform/desktop/frontend/resultSpace.ts b/client/platform/desktop/frontend/resultSpace.ts new file mode 100644 index 000000000..f66747564 --- /dev/null +++ b/client/platform/desktop/frontend/resultSpace.ts @@ -0,0 +1,126 @@ +/** + * Geometry behind the 3D view of search results: the query sits at the + * origin, results float around it at their descriptor-space offsets, and an + * orbiting perspective camera projects them onto the canvas. + */ + +export type Vec3 = [number, number, number]; + +export interface SpacePoint { + key: string; + position: Vec3; +} + +/** Camera orbit: yaw about the vertical axis, pitch above the horizon, zoom factor. */ +export interface Orbit { + yaw: number; + pitch: number; + zoom: number; +} + +export interface Viewport { + width: number; + height: number; +} + +/** A point on the canvas, with its depth from the camera and billboard scale. */ +export interface Projected { + key: string; + x: number; + y: number; + /** Distance from the camera along its view axis; larger is farther away. */ + depth: number; + /** Size multiplier relative to a billboard at the origin. */ + scale: number; +} + +export const DEFAULT_ORBIT: Orbit = { yaw: 0.6, pitch: 0.35, zoom: 1 }; + +/** Camera distance from the origin at zoom 1; points are normalized inside radius 1. */ +const CAMERA_DISTANCE = 3.2; +const MAX_PITCH = Math.PI / 2 - 0.05; +const MIN_ZOOM = 0.4; +const MAX_ZOOM = 4; +const DRAG_RADIANS_PER_PX = 0.008; +/** Fraction of the shorter viewport side the unit sphere spans at zoom 1. */ +const FIT_FRACTION = 0.38; + +/** Scale positions uniformly so the farthest point sits at radius 1. */ +export function normalizePositions(points: SpacePoint[]): SpacePoint[] { + const radius = Math.max(...points.map((p) => Math.hypot(...p.position)), 0); + if (!(radius > 0)) return points.map((p) => ({ key: p.key, position: [0, 0, 0] })); + return points.map((p) => ({ + key: p.key, + position: p.position.map((v) => v / radius) as Vec3, + })); +} + +/** Rotate a point into camera-aligned coordinates (camera looks down -z). */ +export function rotate([x, y, z]: Vec3, orbit: Orbit): Vec3 { + const cy = Math.cos(orbit.yaw); + const sy = Math.sin(orbit.yaw); + const x1 = cy * x + sy * z; + const z1 = -sy * x + cy * z; + const cp = Math.cos(orbit.pitch); + const sp = Math.sin(orbit.pitch); + const y2 = cp * y - sp * z1; + const z2 = sp * y + cp * z1; + return [x1, y2, z2]; +} + +export function cameraDistance(orbit: Orbit): number { + return CAMERA_DISTANCE / orbit.zoom; +} + +/** Perspective projection of every point; the origin projects to the viewport center. */ +export function project(points: SpacePoint[], orbit: Orbit, viewport: Viewport): Projected[] { + const distance = cameraDistance(orbit); + const focal = FIT_FRACTION * Math.min(viewport.width, viewport.height) * CAMERA_DISTANCE; + const cx = viewport.width / 2; + const cy = viewport.height / 2; + return points.map((point) => { + const [x, y, z] = rotate(point.position, orbit); + const depth = distance - z; + const s = focal / depth; + return { + key: point.key, + x: cx + x * s, + y: cy - y * s, + depth, + scale: distance / depth, + }; + }); +} + +/** Far to near, so nearer billboards paint over farther ones. */ +export function paintOrder(projected: Projected[]): Projected[] { + return [...projected].sort((a, b) => b.depth - a.depth); +} + +/** + * The nearest billboard under a canvas position, given each billboard's + * half size at scale 1 (they are squares centered on their point). + */ +export function pick(projected: Projected[], x: number, y: number, halfSize: number): Projected | null { + let best: Projected | null = null; + projected.forEach((p) => { + const half = halfSize * p.scale; + if (Math.abs(p.x - x) > half || Math.abs(p.y - y) > half) return; + if (!best || p.depth < best.depth) best = p; + }); + return best; +} + +export function orbitDrag(orbit: Orbit, dx: number, dy: number): Orbit { + return { + ...orbit, + yaw: orbit.yaw + dx * DRAG_RADIANS_PER_PX, + pitch: Math.max(-MAX_PITCH, Math.min(MAX_PITCH, orbit.pitch + dy * DRAG_RADIANS_PER_PX)), + }; +} + +/** Wheel zoom: each 100 units of deltaY scales by ~1.25x. */ +export function orbitZoom(orbit: Orbit, deltaY: number): Orbit { + const zoom = orbit.zoom * (1.25 ** (-deltaY / 100)); + return { ...orbit, zoom: Math.max(MIN_ZOOM, Math.min(MAX_ZOOM, zoom)) }; +} diff --git a/client/platform/desktop/frontend/useVideoSearch.ts b/client/platform/desktop/frontend/useVideoSearch.ts index e5d4d2da3..b976941aa 100644 --- a/client/platform/desktop/frontend/useVideoSearch.ts +++ b/client/platform/desktop/frontend/useVideoSearch.ts @@ -11,12 +11,12 @@ import { reactive, provide, inject } from 'vue'; import type { VideoSearchIndexStatus, VideoSearchIndexMethod, VideoSearchIndexInfo, - VideoSearchResult, VideoSearchQueryResponse, + VideoSearchResult, VideoSearchQueryResponse, VideoSearchLayoutResponse, } from 'dive-common/apispec'; import { videoSearchInstalled, videoSearchIndexStatus, videoSearchBuildIndex, videoSearchRemoveIndex, videoSearchOpenIndex, - videoSearchFormulate, videoSearchQuery, videoSearchRefine, + videoSearchFormulate, videoSearchQuery, videoSearchRefine, videoSearchLayout, videoSearchExportModel, videoSearchClose, videoSearchExtractFrame, loadConfig, } from 'platform/desktop/frontend/api'; @@ -317,6 +317,15 @@ export function createVideoSearch( }); } + /** + * Positions of the given results in descriptor space around the query. + * Read-only on the service, so it neither takes nor waits for the busy + * flag (the service answers requests in order anyway). + */ + async function layoutResults(refs: string[]): Promise { + return videoSearchLayout(refs); + } + async function saveModel(name: string): Promise { return guarded('Saving model...', async () => { const { outputDir } = await videoSearchExportModel(name); @@ -354,6 +363,7 @@ export function createVideoSearch( resultDatasetName, mark, refine, + layoutResults, saveModel, closeSession, }; diff --git a/docs/Query.md b/docs/Query.md index 0cdc125a3..bff799104 100644 --- a/docs/Query.md +++ b/docs/Query.md @@ -29,6 +29,8 @@ Results can be turned into annotations without leaving the page: accepted result **Text.** Type what to find. Sampled frames of every listed dataset (every N frames, up to a per-dataset cap) are searched with the SAM3 text model, which must be installed as a VIAME add-on. Hits show in the grid with their label and score. Each hit offers **Search the index for objects like this one**, which turns it into an image query, and **Open in the annotation viewer**. +The results toolbar can switch from the grid to a **3D view** (the cube button; off by default) that places the top results around the query in descriptor space: the query exemplar sits at the center, each result hangs off it on a line, and results with similar descriptors cluster together, with the more similar ones closer to the center. Positions are the results' descriptor offsets from the query projected onto the three principal axes of the shown set. Choose how many top results to place, drag to orbit, use the wheel to zoom, click a result to mark it correct or incorrect, and double click to open it. + Double clicking any result opens its dataset in the viewer at that frame. Coming back to Query afterwards resumes the page as it was left: the exemplar, the results and their marks, unsaved annotation edits, and the grid page, so adjudication carries on where it stopped. Index builds appear in **Jobs** while preparing and running, with their dataset, indexing method, live stdout/stderr, and final result. Expand the job's output to diagnose failures; the process log is also saved as `runlog.txt` in its working directory. Builds waiting for the GPU appear under **Queued Jobs** as indexing jobs. Startup failures remain in job history. From 4f5d72353203bd0fdbf5a210d9fcf3c4466e0f2b Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 22:18:49 -0400 Subject: [PATCH 3/4] Pan, box-select and bulk-mark results in the 3D results view Up to 1000 results can be placed. A middle drag slides the view and the reset button re-centers it; a drawn box selects results to mark, retype or delete together. Chips here are cropped to the box alone. --- .../dive-common/review/chipRenderer.spec.ts | 14 +- client/dive-common/review/chipRenderer.ts | 24 +- client/dive-common/review/chipStore.ts | 5 +- .../components/VideoSearchResultsGrid.spec.ts | 36 ++- .../components/VideoSearchResultsGrid.vue | 47 ++- .../components/VideoSearchResultsSpace.vue | 293 +++++++++++++++--- .../desktop/frontend/resultSpace.spec.ts | 53 +++- .../platform/desktop/frontend/resultSpace.ts | 77 ++++- .../desktop/frontend/useSearchChips.ts | 9 + docs/Query.md | 2 +- 10 files changed, 484 insertions(+), 76 deletions(-) diff --git a/client/dive-common/review/chipRenderer.spec.ts b/client/dive-common/review/chipRenderer.spec.ts index b7a77cada..c500eabd9 100644 --- a/client/dive-common/review/chipRenderer.spec.ts +++ b/client/dive-common/review/chipRenderer.spec.ts @@ -1,5 +1,5 @@ import { - chipRegion, chipScale, chipSizeFor, frameRegion, toChipPoint, toImagePoint, + boxRegion, chipRegion, chipScale, chipSizeFor, frameRegion, toChipPoint, toImagePoint, } from './chipRenderer'; describe('chipRegion', () => { @@ -30,6 +30,18 @@ describe('chipRegion', () => { }); }); +describe('boxRegion', () => { + it('is the box itself, with no context and no squaring', () => { + expect(boxRegion([10, 20, 30, 60])).toEqual({ + x: 10, y: 20, width: 20, height: 40, + }); + expect(boxRegion([30, 60, 10, 20])).toEqual({ + x: 10, y: 20, width: 20, height: 40, + }); + expect(boxRegion([5, 5, 5, 5])).toMatchObject({ width: 1, height: 1 }); + }); +}); + describe('chipSizeFor', () => { it('rounds cell sizes up to a bucket and caps at the largest', () => { expect(chipSizeFor(100)).toBe(128); diff --git a/client/dive-common/review/chipRenderer.ts b/client/dive-common/review/chipRenderer.ts index a842e82bc..a6f33ba8a 100644 --- a/client/dive-common/review/chipRenderer.ts +++ b/client/dive-common/review/chipRenderer.ts @@ -16,6 +16,11 @@ export interface ChipRenderOptions { aspect?: number; /** Box outline colour; omit to draw no outline. */ outline?: string; + /** + * Crop to the box itself: no context and no squaring, so the chip takes the + * box's own shape. Overrides padding and aspect. + */ + tight?: boolean; /** JPEG quality. */ quality?: number; } @@ -71,6 +76,17 @@ export function chipRegion(bounds: RectBounds, padding: number, aspect = 1): Chi }; } +/** Exactly the box, whatever its shape. */ +export function boxRegion(bounds: RectBounds): ChipRegion { + const [x1, y1, x2, y2] = bounds; + return { + x: Math.min(x1, x2), + y: Math.min(y1, y2), + width: Math.max(1, Math.abs(x2 - x1)), + height: Math.max(1, Math.abs(y2 - y1)), + }; +} + /** Pixel sizes chips are rendered at; cells pick the smallest that covers them. */ export const CHIP_SIZE_BUCKETS = [128, 192, 256, 384, 512, 768]; @@ -108,9 +124,11 @@ export function renderChip( bounds: RectBounds | null, options: ChipRenderOptions, ): RenderedChip { - const region = bounds - ? chipRegion(bounds, options.padding, options.aspect) - : frameRegion(frame.width, frame.height, options.aspect); + let region = frameRegion(frame.width, frame.height, options.aspect); + if (bounds) { + region = options.tight + ? boxRegion(bounds) : chipRegion(bounds, options.padding, options.aspect); + } const scale = chipScale(region, options.size); const canvas = document.createElement('canvas'); canvas.width = Math.max(1, Math.round(region.width * scale)); diff --git a/client/dive-common/review/chipStore.ts b/client/dive-common/review/chipStore.ts index be19fbba4..db61d7f55 100644 --- a/client/dive-common/review/chipStore.ts +++ b/client/dive-common/review/chipStore.ts @@ -19,6 +19,8 @@ export interface ChipStoreOptions { /** Chip width / height, matching the cells the chips are shown in. */ aspect: number; outline: string; + /** Crop to the box itself, ignoring padding and aspect. */ + tight?: boolean; } export interface ChipStoreDeps { @@ -72,7 +74,8 @@ export function createChipStore(deps: ChipStoreDeps, initial: ChipStoreOptions) /** Re-render everything when the crop or resolution changes. */ function setOptions(next: ChipStoreOptions) { if (next.padding === options.padding && next.size === options.size - && next.aspect === options.aspect && next.outline === options.outline) { + && next.aspect === options.aspect && next.outline === options.outline + && !!next.tight === !!options.tight) { return; } options = { ...next }; diff --git a/client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts index 161f2a1d1..c9348a316 100644 --- a/client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts +++ b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.spec.ts @@ -15,7 +15,7 @@ vi.mock('vue-media-annotator/provides', () => ({ useHandler: () => null })); vi.mock('./VideoSearchResultsSpace.vue', () => ({ default: { name: 'VideoSearchResultsSpace', - props: ['points', 'cells', 'exemplarUrl', 'loading', 'error', 'missingCount'], + props: ['points', 'cells', 'exemplarUrl', 'loading', 'error', 'missingCount', 'editable', 'typeOptions'], render: (h: (tag: string) => unknown) => h('div'), }, })); @@ -59,6 +59,12 @@ function mount(refs: string[]) { items, itemsByRef: computed(() => new Map(items.value.map((i) => [i.key, i]))), chips: ref>({ [refs[0]]: 'data:chip' }), + tightChips: ref>({ [refs[0]]: 'data:tight' }), + tightStore: { + chips: ref({}), + ensurePrimary: vi.fn(), + reset: vi.fn(), + }, store: { chips: ref({}), sequences: ref({}), @@ -97,7 +103,9 @@ it('only places results in descriptor space once the 3D view is chosen, remember (wrapper.vm as unknown as { space: boolean }).space = true; await flush(); expect(search.layoutResults).toHaveBeenCalledWith(['0:1', '0:2']); - expect(searchChips.store.ensurePrimary).toHaveBeenCalledWith(expect.arrayContaining([expect.objectContaining({ key: '0:1' })])); + // The 3D view asks for its own chips, cropped to the box alone + expect(searchChips.tightStore.ensurePrimary).toHaveBeenCalledWith(expect.arrayContaining([expect.objectContaining({ key: '0:1' })])); + expect(searchChips.store.ensurePrimary).not.toHaveBeenCalled(); expect(memory).toMatchObject({ space: true, spaceCount: 2 }); const space = wrapper.findComponent({ name: 'VideoSearchResultsSpace' }); expect(space.exists()).toBe(true); @@ -106,7 +114,7 @@ it('only places results in descriptor space once the 3D view is chosen, remember ]); expect(space.props('cells')).toMatchObject([ { - key: '0:1', rank: 1, chip: 'data:chip', distance: 0, score: 1, + key: '0:1', rank: 1, chip: 'data:tight', distance: 0, score: 1, }, { key: '0:2', rank: 2, chip: null, distance: 2, @@ -132,9 +140,29 @@ it('marks and opens results from the 3D view through the shared session', async (wrapper.vm as unknown as { space: boolean }).space = true; await flush(); const space = wrapper.findComponent({ name: 'VideoSearchResultsSpace' }); - space.vm.$emit('mark', '0:2', 'negative'); + space.vm.$emit('mark', ['0:2'], 'negative'); expect(search.mark).toHaveBeenCalledWith('0:2', 'negative'); space.vm.$emit('open', '0:2'); expect(wrapper.emitted('open-result')?.[0]).toEqual(['a', 1, undefined]); wrapper.destroy(); }); + +it('sets a mark on every selected result, leaving alone those that already carry it', async () => { + const { wrapper, search } = mount(['0:1', '0:2', '0:3']); + (wrapper.vm as unknown as { space: boolean; spaceCount: number }).spaceCount = 3; + (wrapper.vm as unknown as { space: boolean }).space = true; + search.state.adjudications = { '0:2': 'positive' }; + await flush(); + const space = wrapper.findComponent({ name: 'VideoSearchResultsSpace' }); + // A mark toggles, so marking '0:2' again would have cleared it + space.vm.$emit('mark', ['0:1', '0:2', '0:3'], 'positive'); + expect(search.mark.mock.calls).toEqual([['0:1', 'positive'], ['0:3', 'positive']]); + wrapper.destroy(); +}); + +it('offers up to a thousand results in the 3D view', () => { + const { wrapper } = mount(['0:1']); + expect((wrapper.vm as unknown as { spaceCountChoices: number[] }).spaceCountChoices) + .toEqual([10, 25, 50, 100, 250, 1000]); + wrapper.destroy(); +}); diff --git a/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue index 1b9ba4235..87c8e4e06 100644 --- a/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue +++ b/client/platform/desktop/frontend/components/VideoSearchResultsGrid.vue @@ -23,7 +23,7 @@ import VideoSearchResultsSpace, { SpaceCell } from './VideoSearchResultsSpace.vu /** Footer height of a search cell (type field and caption), for the chip aspect ratio. */ const SearchCellFooterPx = 48; /** How many top results the 3D view may place. */ -const SpaceCountChoices = [10, 25, 50, 100]; +const SpaceCountChoices = [10, 25, 50, 100, 250, 1000]; const DefaultSpaceCount = 25; /** @@ -130,7 +130,7 @@ export default defineComponent({ spaceLoading.value = false; return; } - props.searchChips.store.ensurePrimary(spaceItems.value); + props.searchChips.tightStore.ensurePrimary(spaceItems.value); spaceLoading.value = true; spaceError.value = ''; try { @@ -149,14 +149,22 @@ export default defineComponent({ const spacePoints = computed(() => (spaceLayout.value?.points ?? []) .map((point) => ({ key: point.ref, position: point.position }))); const spaceCells = computed(() => { + // Read so type changes and deletions re-render, as in the grid's cells + const revision = review?.dataRevision.value ?? 0; const distances = new Map((spaceLayout.value?.points ?? []).map((point) => [point.ref, point.distance])); return spaceItems.value.map((item, index): SpaceCell => { const result = resultsByRef.value.get(item.key); const datasetName = search && result ? search.resultDatasetName(result) : null; + const adopted = result && props.searchReview + ? props.searchReview.itemOf(result) : undefined; return { key: item.key, rank: index + 1, - chip: props.searchChips.chips.value[item.key] || null, + // Cropped to the box alone; the grid's chips carry context around it + chip: props.searchChips.tightChips.value[item.key] || null, + revision, + type: (adopted && review ? review.currentType(adopted)?.type : '') ?? '', + adopted: adopted !== undefined, adjudication: (result && adjudications.value[result.ref]) || '', title: `${datasetName || 'This dataset'} · frame ${item.primary.frame}`, subtitle: [`Frame ${item.primary.frame}`, datasetName].filter(Boolean).join(' · '), @@ -269,8 +277,23 @@ export default defineComponent({ if (item) openItem(item); } - function markSpaceItem(key: string, adjudication: 'positive' | 'negative') { - search?.mark(key, adjudication); + /** One result toggles its mark, as in the grid; several are all set to it. */ + function markSpaceItems(keys: string[], adjudication: 'positive' | 'negative') { + if (!search) return; + if (keys.length === 1) { + search.mark(keys[0], adjudication); + return; + } + keys.filter((key) => adjudications.value[key] !== adjudication) + .forEach((key) => search.mark(key, adjudication)); + } + + function assignSpaceItems(keys: string[], type: string) { + keys.forEach((key) => assign(resultsByRef.value.get(key), type)); + } + + function removeSpaceItems(keys: string[]) { + keys.forEach((key) => remove(resultsByRef.value.get(key))); } function assign(result: VideoSearchResult | undefined, type: string) { @@ -329,7 +352,9 @@ export default defineComponent({ spaceError, spaceMissingCount, openSpaceItem, - markSpaceItem, + markSpaceItems, + assignSpaceItems, + removeSpaceItems, close, openItem, mark, @@ -514,7 +539,9 @@ export default defineComponent({ >({{ reviewedCount }}) - Drag to orbit · wheel to zoom · double click to open + + Drag to orbit · middle drag to slide · wheel to zoom · shift drag to select +
, @@ -74,6 +92,15 @@ export default defineComponent({ type: String, default: '', }, + /** Whether results can be given a type and deleted as annotations. */ + editable: { + type: Boolean, + default: false, + }, + typeOptions: { + type: Array as PropType, + default: () => [], + }, }, setup(props, { emit }) { const container = ref(null); @@ -81,11 +108,19 @@ export default defineComponent({ const orbit = ref({ ...DEFAULT_ORBIT }); const spin = ref(false); const hovered = ref(null); - const selected = ref(null); + const selected = ref([]); + const selectedSet = computed(() => new Set(selected.value)); + const boxTool = ref(false); + const drawnBox = ref(null); const cursor = ref({ x: 0, y: 0 }); const size = ref({ width: 0, height: 0 }); const cellsByKey = computed(() => new Map(props.cells.map((cell) => [cell.key, cell]))); + const halfSize = computed(() => billboardHalfSize( + props.points.length, + HALF_SIZE, + SMALLEST_HALF_SIZE, + )); const normalized = computed(() => normalizePositions(props.points)); const projected = shallowRef([]); @@ -106,7 +141,7 @@ export default defineComponent({ if (frame) return; frame = window.requestAnimationFrame((now) => { frame = 0; - if (spin.value && !dragging) { + if (spin.value && !gesture) { const elapsed = lastTick ? now - lastTick : 0; orbit.value = { ...orbit.value, yaw: orbit.value.yaw + elapsed * SPIN_RADIANS_PER_MS }; } @@ -123,6 +158,18 @@ export default defineComponent({ return COLORS.neutral; } + /** + * Chips are cropped to their box, so each billboard takes its chip's + * shape: the longer side spans the full size, the other in proportion. + */ + function shapeOf(key: string): [number, number] { + const src = cellsByKey.value.get(key)?.chip; + const image = src ? images.get(src) : undefined; + if (!image || !image.naturalWidth || !image.naturalHeight) return [1, 1]; + const longest = Math.max(image.naturalWidth, image.naturalHeight); + return [image.naturalWidth / longest, image.naturalHeight / longest]; + } + function drawBillboard( ctx: CanvasRenderingContext2D, p: { x: number; y: number; scale: number }, @@ -131,27 +178,29 @@ export default defineComponent({ stroke: string, lineWidth: number, label?: string, + shape: [number, number] = [1, 1], ) { - const h = half * p.scale; + const w = half * p.scale * shape[0]; + const h = half * p.scale * shape[1]; const image = src ? imageFor(src) : null; ctx.save(); if (image) { - ctx.drawImage(image, p.x - h, p.y - h, 2 * h, 2 * h); + ctx.drawImage(image, p.x - w, p.y - h, 2 * w, 2 * h); } else { ctx.fillStyle = 'rgba(255, 255, 255, 0.12)'; - ctx.fillRect(p.x - h, p.y - h, 2 * h, 2 * h); + ctx.fillRect(p.x - w, p.y - h, 2 * w, 2 * h); } ctx.strokeStyle = stroke; ctx.lineWidth = lineWidth; - ctx.strokeRect(p.x - h, p.y - h, 2 * h, 2 * h); - if (label && h >= 14) { + ctx.strokeRect(p.x - w, p.y - h, 2 * w, 2 * h); + if (label && half * p.scale >= 14) { ctx.font = `${Math.max(9, Math.round(10 * p.scale))}px sans-serif`; const width = ctx.measureText(label).width + 6; ctx.fillStyle = COLORS.label; - ctx.fillRect(p.x - h, p.y - h, width, 13 * Math.max(0.8, p.scale)); + ctx.fillRect(p.x - w, p.y - h, width, 13 * Math.max(0.8, p.scale)); ctx.fillStyle = '#fff'; ctx.textBaseline = 'top'; - ctx.fillText(label, p.x - h + 3, p.y - h + 2); + ctx.fillText(label, p.x - w + 3, p.y - h + 2); } ctx.restore(); } @@ -178,6 +227,7 @@ export default defineComponent({ { key: 'y', position: [0, 1, 0] }, { key: '-y', position: [0, -1, 0] }, { key: 'z', position: [0, 0, 1] }, { key: '-z', position: [0, 0, -1] }, ], orbit.value, viewport); + const isFocused = (key: string) => key === hovered.value || selectedSet.value.has(key); ctx.strokeStyle = COLORS.axis; ctx.lineWidth = 1; for (let i = 0; i < axisEnds.length; i += 2) { @@ -191,7 +241,7 @@ export default defineComponent({ projected.value = ordered; ordered.forEach((p) => { const cell = cellsByKey.value.get(p.key); - const focused = p.key === hovered.value || p.key === selected.value; + const focused = isFocused(p.key); ctx.strokeStyle = focused || cell?.adjudication ? strokeFor(cell, focused) : COLORS.line; ctx.lineWidth = focused ? 2 : 1; ctx.globalAlpha = focused ? 1 : Math.min(1, 0.35 + p.scale * 0.5); @@ -210,17 +260,39 @@ export default defineComponent({ exemplarDrawn = true; } const cell = cellsByKey.value.get(p.key); - const focused = p.key === hovered.value || p.key === selected.value; - drawBillboard(ctx, p, HALF_SIZE, cell?.chip ?? null, strokeFor(cell, focused), focused ? 3 : 1.5, cell ? `#${cell.rank}` : undefined); + const focused = isFocused(p.key); + drawBillboard( + ctx, + p, + halfSize.value, + cell?.chip ?? null, + strokeFor(cell, focused), + focused ? 3 : 1.5, + cell ? `#${cell.rank}` : undefined, + shapeOf(p.key), + ); }); if (!exemplarDrawn) { drawBillboard(ctx, center, EXEMPLAR_HALF_SIZE, props.exemplarUrl || null, COLORS.exemplar, 2, 'Query'); } + + const box = drawnBox.value; + if (box) { + const [x, y] = [Math.min(box.x1, box.x2), Math.min(box.y1, box.y2)]; + const [w, h] = [Math.abs(box.x2 - box.x1), Math.abs(box.y2 - box.y1)]; + ctx.fillStyle = COLORS.boxFill; + ctx.fillRect(x, y, w, h); + ctx.strokeStyle = COLORS.box; + ctx.lineWidth = 1; + ctx.setLineDash([4, 3]); + ctx.strokeRect(x, y, w, h); + ctx.setLineDash([]); + } } // ---- interaction ------------------------------------------------------ - let dragging = false; + let gesture: Gesture = null; let dragMoved = false; let last = { x: 0, y: 0 }; @@ -229,44 +301,80 @@ export default defineComponent({ return { x: event.clientX - (rect?.left ?? 0), y: event.clientY - (rect?.top ?? 0) }; } + function hitAt(x: number, y: number) { + return pick(projected.value, x, y, halfSize.value, shapeOf); + } + function onMouseDown(event: MouseEvent) { - if (event.button !== 0) return; - dragging = true; + if (event.button === 1) { + // Keep the browser from starting its own middle-button scroll + event.preventDefault(); + gesture = 'pan'; + } else if (event.button === 0) { + gesture = boxTool.value || event.shiftKey ? 'box' : 'orbit'; + } else { + return; + } dragMoved = false; last = { x: event.clientX, y: event.clientY }; + if (gesture === 'box') { + const { x, y } = localPoint(event); + drawnBox.value = { + x1: x, y1: y, x2: x, y2: y, + }; + } } function onMouseMove(event: MouseEvent) { cursor.value = localPoint(event); - if (dragging) { + if (gesture) { const dx = event.clientX - last.x; const dy = event.clientY - last.y; if (Math.abs(dx) + Math.abs(dy) > 2) dragMoved = true; last = { x: event.clientX, y: event.clientY }; - orbit.value = orbitDrag(orbit.value, dx, dy); + if (gesture === 'pan') { + orbit.value = orbitPan(orbit.value, dx, dy); + } else if (gesture === 'orbit') { + orbit.value = orbitDrag(orbit.value, dx, dy); + } else if (drawnBox.value) { + drawnBox.value = { ...drawnBox.value, x2: cursor.value.x, y2: cursor.value.y }; + } return; } - const hit = pick(projected.value, cursor.value.x, cursor.value.y, HALF_SIZE); - hovered.value = hit?.key ?? null; + hovered.value = hitAt(cursor.value.x, cursor.value.y)?.key ?? null; } function onMouseUp(event: MouseEvent) { - if (!dragging) return; - dragging = false; + const finished = gesture; + const box = drawnBox.value; + gesture = null; + drawnBox.value = null; + if (!finished || finished === 'pan') return; + if (finished === 'box' && box && dragMoved) { + selected.value = pickInBox(projected.value, box, halfSize.value, shapeOf); + return; + } if (dragMoved) return; const { x, y } = localPoint(event); - const hit = pick(projected.value, x, y, HALF_SIZE); - selected.value = hit?.key ?? null; + const hit = hitAt(x, y)?.key ?? null; + if (event.shiftKey && hit) { + // Shift adds to, or takes from, what is already selected + selected.value = selectedSet.value.has(hit) + ? selected.value.filter((key) => key !== hit) : [...selected.value, hit]; + } else { + selected.value = hit ? [hit] : []; + } } function onMouseLeave() { hovered.value = null; - dragging = false; + gesture = null; + drawnBox.value = null; } function onDoubleClick(event: MouseEvent) { const { x, y } = localPoint(event); - const hit = pick(projected.value, x, y, HALF_SIZE); + const hit = hitAt(x, y); if (hit) emit('open', hit.key); } @@ -275,25 +383,57 @@ export default defineComponent({ orbit.value = orbitZoom(orbit.value, event.deltaY); } + /** Back to the opening view: the query centered, at the opening angle and zoom. */ function resetView() { orbit.value = { ...DEFAULT_ORBIT }; } - const selectedCell = computed(() => (selected.value ? cellsByKey.value.get(selected.value) ?? null : null)); - const hoveredCell = computed(() => (hovered.value && hovered.value !== selected.value + const selectedCells = computed(() => selected.value + .map((key) => cellsByKey.value.get(key)) + .filter((cell): cell is SpaceCell => cell !== undefined)); + /** The type the selection shares, or none when it is mixed. */ + const selectedType = computed(() => { + const types = new Set(selectedCells.value.map((cell) => cell.type)); + return types.size === 1 ? [...types][0] : ''; + }); + const selectedAdopted = computed(() => selectedCells.value.filter((cell) => cell.adopted)); + /** Whether every selected result already carries this mark. */ + function allMarked(adjudication: Adjudication) { + return selectedCells.value.length > 0 + && selectedCells.value.every((cell) => cell.adjudication === adjudication); + } + function markSelected(adjudication: Adjudication) { + emit('mark', selectedCells.value.map((cell) => cell.key), adjudication); + } + function assignSelected(type: string) { + emit('assign', selectedCells.value.map((cell) => cell.key), type); + } + function removeSelected() { + emit('remove', selectedAdopted.value.map((cell) => cell.key)); + selected.value = []; + } + + const selectedCell = computed(() => (selectedCells.value.length === 1 + ? selectedCells.value[0] : null)); + const hoveredCell = computed(() => (hovered.value && !selectedSet.value.has(hovered.value) ? cellsByKey.value.get(hovered.value) ?? null : null)); const tooltipStyle = computed(() => ({ left: `${Math.min(cursor.value.x + 14, Math.max(0, size.value.width - 220))}px`, top: `${Math.min(cursor.value.y + 14, Math.max(0, size.value.height - 60))}px`, })); - watch([orbit, hovered, selected, normalized, () => props.cells, () => props.exemplarUrl], () => requestDraw(), { deep: true }); + watch([ + orbit, hovered, selected, drawnBox, normalized, () => props.cells, () => props.exemplarUrl, + ], () => requestDraw(), { deep: true }); watch(spin, (on) => { lastTick = 0; if (on) requestDraw(); }); watch(() => props.points, () => { - if (selected.value && !props.points.some((p) => p.key === selected.value)) selected.value = null; + const placed = new Set(props.points.map((p) => p.key)); + if (selected.value.some((key) => !placed.has(key))) { + selected.value = selected.value.filter((key) => placed.has(key)); + } }); let observer: ResizeObserver | null = null; @@ -320,8 +460,16 @@ export default defineComponent({ container, canvas, spin, + boxTool, selected, selectedCell, + selectedCells, + selectedType, + selectedAdopted, + allMarked, + markSelected, + assignSelected, + removeSelected, hoveredCell, tooltipStyle, onMouseDown, @@ -344,7 +492,7 @@ export default defineComponent({ - mdi-restore + mdi-image-filter-center-focus + + + + + mdi-select-drag
-
+
#{{ selectedCell.rank }} · {{ selectedCell.subtitle }}
score {{ selectedCell.score.toFixed(3) }} · distance {{ selectedCell.distance.toFixed(2) }}
+
+ {{ selectedCells.length }} results selected +
mdi-close
-
+
- {{ selectedCell.adjudication === 'positive' ? 'mdi-check-circle' : 'mdi-check-circle-outline' }} + {{ allMarked('positive') ? 'mdi-check-circle' : 'mdi-check-circle-outline' }} - {{ selectedCell.adjudication === 'negative' ? 'mdi-close-circle' : 'mdi-close-circle-outline' }} + {{ allMarked('negative') ? 'mdi-close-circle' : 'mdi-close-circle-outline' }} + + + + + mdi-delete-outline
+
+ Type + +
{ expect(pick(projected, 100, 100, 1)?.key).toBe('behind'); expect(pick(projected, 200, 200, 20)).toBeNull(); }); + +it('slides the view with a pan, and the default orbit undoes it', () => { + const panned = orbitPan(orbitPan(DEFAULT_ORBIT, 30, -10), 5, 5); + expect(panned).toMatchObject({ panX: 35, panY: -5, yaw: DEFAULT_ORBIT.yaw }); + const [origin] = project([{ key: 'o', position: [0, 0, 0] }], panned, viewport); + expect(origin.x).toBe(235); + expect(origin.y).toBe(145); + const [reset] = project([{ key: 'o', position: [0, 0, 0] }], { ...DEFAULT_ORBIT }, viewport); + expect([reset.x, reset.y]).toEqual([200, 150]); +}); + +it('picks by each billboard\'s own shape', () => { + const projected = [{ + key: 'thin', x: 100, y: 100, depth: 2, scale: 1, + }]; + const shape = () => [1, 0.25] as [number, number]; + expect(pick(projected, 115, 100, 20, shape)?.key).toBe('thin'); + expect(pick(projected, 100, 110, 20, shape)).toBeNull(); + expect(pick(projected, 100, 110, 20)?.key).toBe('thin'); +}); + +it('gathers every billboard a drawn box touches, whichever way it was drawn', () => { + const projected = [ + { + key: 'a', x: 100, y: 100, depth: 2, scale: 1, + }, + { + key: 'b', x: 160, y: 120, depth: 3, scale: 1, + }, + { + key: 'c', x: 300, y: 100, depth: 1, scale: 1, + }, + ]; + const box = { + x1: 170, y1: 130, x2: 90, y2: 90, + }; + expect(pickInBox(projected, box, 10)).toEqual(['a', 'b']); + // 'c' only touches the box with its edge + expect(pickInBox(projected, { ...box, x1: 291 }, 10)).toEqual(['a', 'b', 'c']); + expect(pickInBox(projected, { + x1: 0, y1: 0, x2: 20, y2: 20, + }, 10)).toEqual([]); +}); + +it('shrinks billboards once more than a hundred results are shown', () => { + expect(billboardHalfSize(25, 26, 8)).toBe(26); + expect(billboardHalfSize(100, 26, 8)).toBe(26); + expect(billboardHalfSize(250, 26, 8)).toBe(16); + expect(billboardHalfSize(1000, 26, 8)).toBe(8); +}); diff --git a/client/platform/desktop/frontend/resultSpace.ts b/client/platform/desktop/frontend/resultSpace.ts index f66747564..18a0f0b86 100644 --- a/client/platform/desktop/frontend/resultSpace.ts +++ b/client/platform/desktop/frontend/resultSpace.ts @@ -11,13 +11,31 @@ export interface SpacePoint { position: Vec3; } -/** Camera orbit: yaw about the vertical axis, pitch above the horizon, zoom factor. */ +/** + * Camera orbit: yaw about the vertical axis, pitch above the horizon, zoom + * factor, and how far the query has been slid from the viewport's center. + */ export interface Orbit { yaw: number; pitch: number; zoom: number; + /** Canvas pixels. */ + panX?: number; + panY?: number; +} + +/** A rectangle on the canvas, by two opposite corners in any order. */ +export interface CanvasBox { + x1: number; + y1: number; + x2: number; + y2: number; } +/** A billboard's half width and half height, as fractions of the square half size. */ +export type Shape = (key: string) => [number, number]; +const SQUARE: Shape = () => [1, 1]; + export interface Viewport { width: number; height: number; @@ -34,7 +52,9 @@ export interface Projected { scale: number; } -export const DEFAULT_ORBIT: Orbit = { yaw: 0.6, pitch: 0.35, zoom: 1 }; +export const DEFAULT_ORBIT: Orbit = { + yaw: 0.6, pitch: 0.35, zoom: 1, panX: 0, panY: 0, +}; /** Camera distance from the origin at zoom 1; points are normalized inside radius 1. */ const CAMERA_DISTANCE = 3.2; @@ -72,12 +92,15 @@ export function cameraDistance(orbit: Orbit): number { return CAMERA_DISTANCE / orbit.zoom; } -/** Perspective projection of every point; the origin projects to the viewport center. */ +/** + * Perspective projection of every point; the origin projects to the + * viewport center, moved by the orbit's pan. + */ export function project(points: SpacePoint[], orbit: Orbit, viewport: Viewport): Projected[] { const distance = cameraDistance(orbit); const focal = FIT_FRACTION * Math.min(viewport.width, viewport.height) * CAMERA_DISTANCE; - const cx = viewport.width / 2; - const cy = viewport.height / 2; + const cx = viewport.width / 2 + (orbit.panX ?? 0); + const cy = viewport.height / 2 + (orbit.panY ?? 0); return points.map((point) => { const [x, y, z] = rotate(point.position, orbit); const depth = distance - z; @@ -98,19 +121,50 @@ export function paintOrder(projected: Projected[]): Projected[] { } /** - * The nearest billboard under a canvas position, given each billboard's - * half size at scale 1 (they are squares centered on their point). + * The nearest billboard under a canvas position, given the billboards' half + * size at scale 1. They are centered on their point, square unless `shape` + * says otherwise. */ -export function pick(projected: Projected[], x: number, y: number, halfSize: number): Projected | null { +export function pick( + projected: Projected[], + x: number, + y: number, + halfSize: number, + shape: Shape = SQUARE, +): Projected | null { let best: Projected | null = null; projected.forEach((p) => { + const [wide, tall] = shape(p.key); const half = halfSize * p.scale; - if (Math.abs(p.x - x) > half || Math.abs(p.y - y) > half) return; + if (Math.abs(p.x - x) > half * wide || Math.abs(p.y - y) > half * tall) return; if (!best || p.depth < best.depth) best = p; }); return best; } +/** Every billboard a box drawn on the canvas touches. */ +export function pickInBox( + projected: Projected[], + box: CanvasBox, + halfSize: number, + shape: Shape = SQUARE, +): string[] { + const [left, right] = [Math.min(box.x1, box.x2), Math.max(box.x1, box.x2)]; + const [top, bottom] = [Math.min(box.y1, box.y2), Math.max(box.y1, box.y2)]; + return projected.filter((p) => { + const [wide, tall] = shape(p.key); + const half = halfSize * p.scale; + return p.x + half * wide >= left && p.x - half * wide <= right + && p.y + half * tall >= top && p.y - half * tall <= bottom; + }).map((p) => p.key); +} + +/** Billboards shrink as more results are shown, so a thousand still read as separate. */ +export function billboardHalfSize(count: number, halfSize: number, smallest: number): number { + const crowding = count > 100 ? Math.sqrt(100 / count) : 1; + return Math.max(smallest, Math.round(halfSize * crowding)); +} + export function orbitDrag(orbit: Orbit, dx: number, dy: number): Orbit { return { ...orbit, @@ -119,6 +173,11 @@ export function orbitDrag(orbit: Orbit, dx: number, dy: number): Orbit { }; } +/** Slide the whole view across the canvas. */ +export function orbitPan(orbit: Orbit, dx: number, dy: number): Orbit { + return { ...orbit, panX: (orbit.panX ?? 0) + dx, panY: (orbit.panY ?? 0) + dy }; +} + /** Wheel zoom: each 100 units of deltaY scales by ~1.25x. */ export function orbitZoom(orbit: Orbit, deltaY: number): Orbit { const zoom = orbit.zoom * (1.25 ** (-deltaY / 100)); diff --git a/client/platform/desktop/frontend/useSearchChips.ts b/client/platform/desktop/frontend/useSearchChips.ts index 85400e59a..83683e387 100644 --- a/client/platform/desktop/frontend/useSearchChips.ts +++ b/client/platform/desktop/frontend/useSearchChips.ts @@ -60,6 +60,11 @@ function createScopedSearchChips(search: VideoSearchContextType, options: Search padding: DEFAULT_REVIEW_GRID.padding, size: PanelChipSize, aspect: 1, outline: '#00e5ff', }); + // The 3D results view shows each result as its box alone, with no context + const tightStore = createChipStore({ frameSourceFor: registry.frameSourceFor }, { + padding: 0, size: PanelChipSize, aspect: 1, outline: '', tight: true, + }); + /** Every current result as a grid item, in rank order. */ const items = computed(() => (options.results?.value ?? search.state.results) .filter((result) => !options.hidden?.(result)) @@ -84,6 +89,7 @@ function createScopedSearchChips(search: VideoSearchContextType, options: Search // New query: refs may repeat with different content, so start over. generation = search.state.queryGeneration; store.reset(); + tightStore.reset(); } // Refinement re-ranks within the same query; cached chips stay valid. store.ensurePrimary(current.slice(0, EagerLoadCount)); @@ -91,14 +97,17 @@ function createScopedSearchChips(search: VideoSearchContextType, options: Search function dispose() { store.reset(); + tightStore.reset(); registry.dispose(); } return { store, + tightStore, items, itemsByRef, chips: store.chips, + tightChips: tightStore.chips, dispose, }; } diff --git a/docs/Query.md b/docs/Query.md index bff799104..091c3ec8d 100644 --- a/docs/Query.md +++ b/docs/Query.md @@ -29,7 +29,7 @@ Results can be turned into annotations without leaving the page: accepted result **Text.** Type what to find. Sampled frames of every listed dataset (every N frames, up to a per-dataset cap) are searched with the SAM3 text model, which must be installed as a VIAME add-on. Hits show in the grid with their label and score. Each hit offers **Search the index for objects like this one**, which turns it into an image query, and **Open in the annotation viewer**. -The results toolbar can switch from the grid to a **3D view** (the cube button; off by default) that places the top results around the query in descriptor space: the query exemplar sits at the center, each result hangs off it on a line, and results with similar descriptors cluster together, with the more similar ones closer to the center. Positions are the results' descriptor offsets from the query projected onto the three principal axes of the shown set. Choose how many top results to place, drag to orbit, use the wheel to zoom, click a result to mark it correct or incorrect, and double click to open it. +The results toolbar can switch from the grid to a **3D view** (the cube button; off by default) that places the top results around the query in descriptor space: the query exemplar sits at the center, each result hangs off it on a line, and results with similar descriptors cluster together, with the more similar ones closer to the center. Positions are the results' descriptor offsets from the query projected onto the three principal axes of the shown set. Choose how many top results to place (up to 1000; this needs a VIAME whose query service returns that many), drag to orbit, drag with the middle button to slide the view, use the wheel to zoom, and press the reset button to bring the query back to the center. Results are shown cropped to their box alone. Click a result to mark it correct or incorrect, and double click to open it. To work on several at once, hold shift and drag a box over them (or turn on the box tool), then mark them all correct or incorrect, give them all a type, or delete them together; shift click adds or removes a single result. Double clicking any result opens its dataset in the viewer at that frame. Coming back to Query afterwards resumes the page as it was left: the exemplar, the results and their marks, unsaved annotation edits, and the grid page, so adjudication carries on where it stopped. From 406bc9a98b02afbd3b11e8c4beb15a7e3aabe9a0 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 28 Sep 2026 22:49:12 -0400 Subject: [PATCH 4/4] Show the queried box at the 3D view's center and fix the box drawing offset The exemplar in the 3D results view is cropped to the queried box. The box drawn on the exemplar was placed against a container that also held the caption, which shifted it away from the pointer. --- .../frontend/components/QueryExemplar.vue | 38 +++++++----- .../desktop/frontend/components/QueryPage.vue | 2 + .../components/VideoSearchResultsGrid.spec.ts | 6 +- .../components/VideoSearchResultsGrid.vue | 6 ++ .../components/VideoSearchResultsSpace.vue | 62 ++++++++++++++++--- 5 files changed, 91 insertions(+), 23 deletions(-) diff --git a/client/platform/desktop/frontend/components/QueryExemplar.vue b/client/platform/desktop/frontend/components/QueryExemplar.vue index b392c9a1a..03337bfb9 100644 --- a/client/platform/desktop/frontend/components/QueryExemplar.vue +++ b/client/platform/desktop/frontend/components/QueryExemplar.vue @@ -100,20 +100,23 @@ export default defineComponent({