From 869bae2efb98ceba3d7a88c7659abb60a3d6d00a Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Tue, 29 Sep 2026 00:27:48 -0400 Subject: [PATCH 1/2] Limit web segmentation polygons to the desktop vertex maximum Rings are cut to 25 points; point-click masks grow to 100 past a 5% area error. --- .../dive-common/use/segmentation/SamOnnx.ts | 3 +- .../use/segmentation/maskGeometry.spec.ts | 38 ++++++++++++ .../use/segmentation/maskGeometry.ts | 61 ++++++++++++++----- 3 files changed, 85 insertions(+), 17 deletions(-) diff --git a/client/dive-common/use/segmentation/SamOnnx.ts b/client/dive-common/use/segmentation/SamOnnx.ts index c9b6dba78..dc2f5f6ab 100644 --- a/client/dive-common/use/segmentation/SamOnnx.ts +++ b/client/dive-common/use/segmentation/SamOnnx.ts @@ -313,6 +313,7 @@ export default class SamOnnx { // Copy the prompts: the recipe may receive another click while queued. const points = request.points.map((p) => [...p]); const labels = [...request.pointLabels]; + const pointClick = !request.line && !request.box && !labels.some((label) => label > 1); return this.run(async () => { if (version !== this.version) throw new Error('Segmentation model changed.'); if (!points.length || points.length !== labels.length @@ -377,7 +378,7 @@ export default class SamOnnx { if (!this.postprocessor) this.postprocessor = await SamMaskPostprocessor.create(); const { mask, score } = await this.postprocessor.run(output.pred_masks, output.iou_scores, frame.original[0], frame.reshaped[0], frame.padded); if (version !== this.version) throw new Error('Segmentation model changed.'); - return { ...maskGeometry(mask, image.width, image.height), score }; + return { ...maskGeometry(mask, image.width, image.height, pointClick), score }; } finally { output.pred_masks.dispose(); output.iou_scores.dispose(); diff --git a/client/dive-common/use/segmentation/maskGeometry.spec.ts b/client/dive-common/use/segmentation/maskGeometry.spec.ts index 64c829266..3ee260df2 100644 --- a/client/dive-common/use/segmentation/maskGeometry.spec.ts +++ b/client/dive-common/use/segmentation/maskGeometry.spec.ts @@ -32,6 +32,44 @@ it('derives hull-extremes head/tail across all components and clips to their bou expect(maskKeypoints([a, b])).toEqual({ success: true, head: [20, 2], tail: [0, 2] }); }); +describe('polygon vertex budget', () => { + const size = 400; + const draw = (inside: (x: number, y: number) => boolean) => { + const mask = new Uint8Array(size * size); + for (let y = 0; y < size; y += 1) { + for (let x = 0; x < size; x += 1) if (inside(x - size / 2, y - size / 2)) mask[y * size + x] = 1; + } + return mask; + }; + const disk = draw((x, y) => Math.hypot(x, y) < 150); + const star = draw((x, y) => Math.hypot(x, y) < 110 + 70 * Math.cos(16 * Math.atan2(y, x))); + const ring = draw((x, y) => Math.hypot(x, y) < 150 && Math.hypot(x, y) > 60); + + it('keeps every ring within the desktop maximum', () => { + [disk, star, ring].forEach((mask) => { + const { polygons } = maskGeometry(mask, size, size); + polygons!.flatMap((p) => [p.exterior, ...p.holes]).forEach((points) => { + expect(points.length).toBeLessThanOrEqual(25); + expect(points.length).toBeGreaterThanOrEqual(4); + expect(points[0]).toEqual(points[points.length - 1]); + }); + }); + expect(maskGeometry(ring, size, size).polygons![0].holes).toHaveLength(1); + }); + + it('keeps the area of a simple outline', () => { + const area = maskArea(maskGeometry(disk, size, size).polygons!); + expect(Math.abs(area - Math.PI * 150 ** 2) / (Math.PI * 150 ** 2)).toBeLessThan(0.05); + }); + + it('grows the budget for a point-click mask too complex for it, up to the limit', () => { + expect(maskGeometry(disk, size, size, true).polygons![0].exterior.length).toBeLessThanOrEqual(25); + const points = maskGeometry(star, size, size, true).polygons![0].exterior; + expect(points.length).toBeGreaterThan(25); + expect(points.length).toBeLessThanOrEqual(100); + }); +}); + it('returns a failure for an empty mask rather than an infinite bounding box', () => { expect(maskGeometry(new Uint8Array(100), 10, 10).success).toBe(false); }); diff --git a/client/dive-common/use/segmentation/maskGeometry.ts b/client/dive-common/use/segmentation/maskGeometry.ts index 74651634c..e50818f9e 100644 --- a/client/dive-common/use/segmentation/maskGeometry.ts +++ b/client/dive-common/use/segmentation/maskGeometry.ts @@ -10,35 +10,64 @@ function nearest(p: Point, a: Point, b: Point): Point { return [a[0] + t * dx, a[1] + t * dy]; } -/** Remove redundant contour vertices with a sub-pixel error bound. */ -function simplify(points: Point[]): Point[] { - const keep = new Set([0, points.length - 1]); - const pending = [[0, points.length - 1]]; - while (pending.length) { - const [start, end] = pending.pop()!; - let farthest = -1; let distance = 0.25; +// The desktop segmentation service's max_polygon_points, +// max_polygon_points_limit and area error bound. +const MaxPolygonPoints = 25; +const MaxPolygonPointsLimit = 100; +const MaxAreaError = 0.05; + +/** Douglas-Peucker split vertices, most significant first. Past a closed + * ring's first four points, only deviations over half a pixel count. */ +function splitOrder(points: Point[], limit: number): number[] { + const farthest = (start: number, end: number) => { + let index = -1; let distance = 0; for (let i = start + 1; i < end; i += 1) { const q = nearest(points[i], points[start], points[end]); const d = (q[0] - points[i][0]) ** 2 + (q[1] - points[i][1]) ** 2; - if (d > distance) { distance = d; farthest = i; } - } - if (farthest >= 0) { - keep.add(farthest); - pending.push([start, farthest], [farthest, end]); + if (d > distance) { distance = d; index = i; } } + return { + start, end, index, distance, + }; + }; + const pending = [farthest(0, points.length - 1)]; + const order: number[] = []; + while (order.length < limit) { + let next = -1; + pending.forEach((segment, i) => { + if (segment.index >= 0 && (next < 0 || segment.distance > pending[next].distance)) next = i; + }); + if (next < 0 || (order.length >= 2 && pending[next].distance <= 0.25)) break; + const { start, end, index } = pending[next]; + order.push(index); + pending.splice(next, 1, farthest(start, index), farthest(index, end)); } - const result = [...keep].sort((a, b) => a - b).map((i) => points[i]); - return result.length >= 4 ? result : points; + return order; +} + +/** Reduce a closed contour ring to MaxPolygonPoints. With grow, that budget + * doubles, up to MaxPolygonPointsLimit, while the area cut off or added + * exceeds MaxAreaError of the ring's area. */ +function simplify(points: Point[], grow: boolean): Point[] { + const order = splitOrder(points, (grow ? MaxPolygonPointsLimit : MaxPolygonPoints) - 2); + const area = Math.abs(polygonArea(points)); + const ring = (budget: number) => [0, ...order.slice(0, budget - 2).sort((a, b) => a - b), points.length - 1]; + const error = (kept: number[]) => kept.slice(1).reduce((sum, index, i) => sum + + Math.abs(polygonArea(points.slice(kept[i], index + 1))), 0); + let budget = MaxPolygonPoints; + while (budget - 2 < order.length && error(ring(budget)) > MaxAreaError * area) budget *= 2; + const kept = ring(budget); + return kept.length >= 4 ? kept.map((i) => points[i]) : points; } /** All mask components and holes, in original image coordinates. */ -export function maskGeometry(mask: Uint8Array, width: number, height: number): SegmentationPredictResponse { +export function maskGeometry(mask: Uint8Array, width: number, height: number, grow = false): SegmentationPredictResponse { if (mask.length !== width * height) throw new Error('Invalid mask dimensions.'); const geometry = contours().size([width, height]).thresholds([0.5])(mask as unknown as number[])[0]; const polygons = geometry.coordinates.map((rings) => { const scaled = rings.map((ring) => simplify(ring.map(([x, y]): Point => [ Math.max(0, Math.min(width, x)), Math.max(0, Math.min(height, y)), - ]))); + ]), grow)); return { exterior: scaled[0], holes: scaled.slice(1) }; }); if (!polygons.length) return { success: false, error: 'No object found for these prompts.' }; From 6bdc1e9080a5959525cf3f0dbc897cf365819860 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 29 Sep 2026 09:57:40 -0400 Subject: [PATCH 2/2] align segment constants for desktop/web --- .../dive-common/use/segmentation/constants.ts | 14 +++++++++++ .../use/segmentation/maskGeometry.spec.ts | 19 +++++++++++---- .../use/segmentation/maskGeometry.ts | 23 +++++++++---------- .../desktop/backend/native/segmentation.ts | 12 ++++++++++ 4 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 client/dive-common/use/segmentation/constants.ts diff --git a/client/dive-common/use/segmentation/constants.ts b/client/dive-common/use/segmentation/constants.ts new file mode 100644 index 000000000..38d0ffef3 --- /dev/null +++ b/client/dive-common/use/segmentation/constants.ts @@ -0,0 +1,14 @@ +/** + * Interactive segmentation polygon limits shared by web mask geometry and desktop + * VIAME `service:*` settings (see `service:max_polygon_points` in + * `interactive_segmenter_sam2.conf` / `interactive_segmenter_sam3.conf`). + */ + +/** Default `service:max_polygon_points` when the config omits it. */ +export const SegmentationMaxPolygonPoints = 25; + +/** Maximum vertices when web point-click masks grow the budget for area error. */ +export const SegmentationMaxPolygonPointsLimit = 100; + +/** Relative ring area error (5%) that triggers doubling the vertex budget. */ +export const SegmentationMaxPolygonAreaError = 0.05; diff --git a/client/dive-common/use/segmentation/maskGeometry.spec.ts b/client/dive-common/use/segmentation/maskGeometry.spec.ts index 3ee260df2..75363d3f8 100644 --- a/client/dive-common/use/segmentation/maskGeometry.spec.ts +++ b/client/dive-common/use/segmentation/maskGeometry.spec.ts @@ -1,6 +1,11 @@ import { polygonContains } from 'd3'; import type { SegmentationPolygon } from 'dive-common/apispec'; import { readFileSync } from 'fs'; +import { + SegmentationMaxPolygonAreaError, + SegmentationMaxPolygonPoints, + SegmentationMaxPolygonPointsLimit, +} from './constants'; import { maskGeometry, maskKeypoints, maskSeeds, consistentMaskSeeds, maskArea, } from './maskGeometry'; @@ -49,7 +54,7 @@ describe('polygon vertex budget', () => { [disk, star, ring].forEach((mask) => { const { polygons } = maskGeometry(mask, size, size); polygons!.flatMap((p) => [p.exterior, ...p.holes]).forEach((points) => { - expect(points.length).toBeLessThanOrEqual(25); + expect(points.length).toBeLessThanOrEqual(SegmentationMaxPolygonPoints); expect(points.length).toBeGreaterThanOrEqual(4); expect(points[0]).toEqual(points[points.length - 1]); }); @@ -59,14 +64,18 @@ describe('polygon vertex budget', () => { it('keeps the area of a simple outline', () => { const area = maskArea(maskGeometry(disk, size, size).polygons!); - expect(Math.abs(area - Math.PI * 150 ** 2) / (Math.PI * 150 ** 2)).toBeLessThan(0.05); + expect(Math.abs(area - Math.PI * 150 ** 2) / (Math.PI * 150 ** 2)).toBeLessThan( + SegmentationMaxPolygonAreaError, + ); }); it('grows the budget for a point-click mask too complex for it, up to the limit', () => { - expect(maskGeometry(disk, size, size, true).polygons![0].exterior.length).toBeLessThanOrEqual(25); + expect(maskGeometry(disk, size, size, true).polygons![0].exterior.length).toBeLessThanOrEqual( + SegmentationMaxPolygonPoints, + ); const points = maskGeometry(star, size, size, true).polygons![0].exterior; - expect(points.length).toBeGreaterThan(25); - expect(points.length).toBeLessThanOrEqual(100); + expect(points.length).toBeGreaterThan(SegmentationMaxPolygonPoints); + expect(points.length).toBeLessThanOrEqual(SegmentationMaxPolygonPointsLimit); }); }); diff --git a/client/dive-common/use/segmentation/maskGeometry.ts b/client/dive-common/use/segmentation/maskGeometry.ts index e50818f9e..612df8fe6 100644 --- a/client/dive-common/use/segmentation/maskGeometry.ts +++ b/client/dive-common/use/segmentation/maskGeometry.ts @@ -1,6 +1,11 @@ import { contours, polygonHull, polygonArea } from 'd3'; import type { SegmentationPolygon, SegmentationPredictResponse } from 'dive-common/apispec'; import { componentsBounds } from 'dive-common/recipes/segmentationPolygons'; +import { + SegmentationMaxPolygonAreaError, + SegmentationMaxPolygonPoints, + SegmentationMaxPolygonPointsLimit, +} from './constants'; type Point = [number, number]; @@ -10,12 +15,6 @@ function nearest(p: Point, a: Point, b: Point): Point { return [a[0] + t * dx, a[1] + t * dy]; } -// The desktop segmentation service's max_polygon_points, -// max_polygon_points_limit and area error bound. -const MaxPolygonPoints = 25; -const MaxPolygonPointsLimit = 100; -const MaxAreaError = 0.05; - /** Douglas-Peucker split vertices, most significant first. Past a closed * ring's first four points, only deviations over half a pixel count. */ function splitOrder(points: Point[], limit: number): number[] { @@ -45,17 +44,17 @@ function splitOrder(points: Point[], limit: number): number[] { return order; } -/** Reduce a closed contour ring to MaxPolygonPoints. With grow, that budget - * doubles, up to MaxPolygonPointsLimit, while the area cut off or added - * exceeds MaxAreaError of the ring's area. */ +/** Reduce a closed contour ring to SegmentationMaxPolygonPoints. With grow, that budget + * doubles, up to SegmentationMaxPolygonPointsLimit, while the area cut off or added + * exceeds SegmentationMaxPolygonAreaError of the ring's area. */ function simplify(points: Point[], grow: boolean): Point[] { - const order = splitOrder(points, (grow ? MaxPolygonPointsLimit : MaxPolygonPoints) - 2); + const order = splitOrder(points, (grow ? SegmentationMaxPolygonPointsLimit : SegmentationMaxPolygonPoints) - 2); const area = Math.abs(polygonArea(points)); const ring = (budget: number) => [0, ...order.slice(0, budget - 2).sort((a, b) => a - b), points.length - 1]; const error = (kept: number[]) => kept.slice(1).reduce((sum, index, i) => sum + Math.abs(polygonArea(points.slice(kept[i], index + 1))), 0); - let budget = MaxPolygonPoints; - while (budget - 2 < order.length && error(ring(budget)) > MaxAreaError * area) budget *= 2; + let budget = SegmentationMaxPolygonPoints; + while (budget - 2 < order.length && error(ring(budget)) > SegmentationMaxPolygonAreaError * area) budget *= 2; const kept = ring(budget); return kept.length >= 4 ? kept.map((i) => points[i]) : points; } diff --git a/client/platform/desktop/backend/native/segmentation.ts b/client/platform/desktop/backend/native/segmentation.ts index 5c586a5e3..cfb47ede4 100644 --- a/client/platform/desktop/backend/native/segmentation.ts +++ b/client/platform/desktop/backend/native/segmentation.ts @@ -7,11 +7,23 @@ */ import type { SegmentationPolygon } from 'dive-common/apispec'; +import { + SegmentationMaxPolygonAreaError, + SegmentationMaxPolygonPoints, + SegmentationMaxPolygonPointsLimit, +} from 'dive-common/use/segmentation/constants'; import { StereoMeasurement } from './stereo'; /** Error message shown to users when segmentation model process fails to load */ export const SEGMENTATION_LOAD_ERROR_MESSAGE = "Model failed to load. If you haven't downloaded the SAM2 model pack from the VIAME Add-On wiki, please do so."; +/** Re-exported for desktop callers; same as VIAME `service:max_polygon_points`. */ +export { + SegmentationMaxPolygonAreaError, + SegmentationMaxPolygonPoints, + SegmentationMaxPolygonPointsLimit, +}; + /** Request to the segmentation service */ export interface SegmentationInternalPredictRequest { /** Unique request ID for correlation */