Limit web segmentation polygons to the desktop vertex maximum

client/dive-common/use/segmentation/SamOnnx.ts

@@ -313,6 +313,7 @@ export default class SamOnnx {
     // Copy the prompts: the recipe may receive another click while queued.
     const points = request.points.map((p) => [...p]);
     const labels = [...request.pointLabels];
+    const pointClick = !request.line && !request.box && !labels.some((label) => label > 1);
     return this.run(async () => {
       if (version !== this.version) throw new Error('Segmentation model changed.');
       if (!points.length || points.length !== labels.length
@@ -377,7 +378,7 @@ export default class SamOnnx {
             if (!this.postprocessor) this.postprocessor = await SamMaskPostprocessor.create();
             const { mask, score } = await this.postprocessor.run(output.pred_masks, output.iou_scores, frame.original[0], frame.reshaped[0], frame.padded);
             if (version !== this.version) throw new Error('Segmentation model changed.');
-            return { ...maskGeometry(mask, image.width, image.height), score };
+            return { ...maskGeometry(mask, image.width, image.height, pointClick), score };
           } finally {
             output.pred_masks.dispose();
             output.iou_scores.dispose();

client/dive-common/use/segmentation/constants.ts

@@ -0,0 +1,14 @@
+/**
+ * Interactive segmentation polygon limits shared by web mask geometry and desktop
+ * VIAME `service:*` settings (see `service:max_polygon_points` in
+ * `interactive_segmenter_sam2.conf` / `interactive_segmenter_sam3.conf`).
+ */
+
+/** Default `service:max_polygon_points` when the config omits it. */
+export const SegmentationMaxPolygonPoints = 25;
+
+/** Maximum vertices when web point-click masks grow the budget for area error. */
+export const SegmentationMaxPolygonPointsLimit = 100;
+
+/** Relative ring area error (5%) that triggers doubling the vertex budget. */
+export const SegmentationMaxPolygonAreaError = 0.05;

client/dive-common/use/segmentation/maskGeometry.spec.ts

@@ -1,6 +1,11 @@
 import { polygonContains } from 'd3';
 import type { SegmentationPolygon } from 'dive-common/apispec';
 import { readFileSync } from 'fs';
+import {
+  SegmentationMaxPolygonAreaError,
+  SegmentationMaxPolygonPoints,
+  SegmentationMaxPolygonPointsLimit,
+} from './constants';
 import {
   maskGeometry, maskKeypoints, maskSeeds, consistentMaskSeeds, maskArea,
 } from './maskGeometry';
@@ -32,6 +37,48 @@ it('derives hull-extremes head/tail across all components and clips to their bou
   expect(maskKeypoints([a, b])).toEqual({ success: true, head: [20, 2], tail: [0, 2] });
 });
 
+describe('polygon vertex budget', () => {
+  const size = 400;
+  const draw = (inside: (x: number, y: number) => boolean) => {
+    const mask = new Uint8Array(size * size);
+    for (let y = 0; y < size; y += 1) {
+      for (let x = 0; x < size; x += 1) if (inside(x - size / 2, y - size / 2)) mask[y * size + x] = 1;
+    }
+    return mask;
+  };
+  const disk = draw((x, y) => Math.hypot(x, y) < 150);
+  const star = draw((x, y) => Math.hypot(x, y) < 110 + 70 * Math.cos(16 * Math.atan2(y, x)));
+  const ring = draw((x, y) => Math.hypot(x, y) < 150 && Math.hypot(x, y) > 60);
+
+  it('keeps every ring within the desktop maximum', () => {
+    [disk, star, ring].forEach((mask) => {
+      const { polygons } = maskGeometry(mask, size, size);
+      polygons!.flatMap((p) => [p.exterior, ...p.holes]).forEach((points) => {
+        expect(points.length).toBeLessThanOrEqual(SegmentationMaxPolygonPoints);
+        expect(points.length).toBeGreaterThanOrEqual(4);
+        expect(points[0]).toEqual(points[points.length - 1]);
+      });
+    });
+    expect(maskGeometry(ring, size, size).polygons![0].holes).toHaveLength(1);
+  });
+
+  it('keeps the area of a simple outline', () => {
+    const area = maskArea(maskGeometry(disk, size, size).polygons!);
+    expect(Math.abs(area - Math.PI * 150 ** 2) / (Math.PI * 150 ** 2)).toBeLessThan(
+      SegmentationMaxPolygonAreaError,
+    );
+  });
+
+  it('grows the budget for a point-click mask too complex for it, up to the limit', () => {
+    expect(maskGeometry(disk, size, size, true).polygons![0].exterior.length).toBeLessThanOrEqual(
+      SegmentationMaxPolygonPoints,
+    );
+    const points = maskGeometry(star, size, size, true).polygons![0].exterior;
+    expect(points.length).toBeGreaterThan(SegmentationMaxPolygonPoints);
+    expect(points.length).toBeLessThanOrEqual(SegmentationMaxPolygonPointsLimit);
+  });
+});
+
 it('returns a failure for an empty mask rather than an infinite bounding box', () => {
   expect(maskGeometry(new Uint8Array(100), 10, 10).success).toBe(false);
 });

client/dive-common/use/segmentation/maskGeometry.ts

@@ -1,6 +1,11 @@
 import { contours, polygonHull, polygonArea } from 'd3';
 import type { SegmentationPolygon, SegmentationPredictResponse } from 'dive-common/apispec';
 import { componentsBounds } from 'dive-common/recipes/segmentationPolygons';
+import {
+  SegmentationMaxPolygonAreaError,
+  SegmentationMaxPolygonPoints,
+  SegmentationMaxPolygonPointsLimit,
+} from './constants';
 
 type Point = [number, number];
 
@@ -10,35 +15,58 @@ function nearest(p: Point, a: Point, b: Point): Point {
   return [a[0] + t * dx, a[1] + t * dy];
 }
 
-/** Remove redundant contour vertices with a sub-pixel error bound. */
-function simplify(points: Point[]): Point[] {
-  const keep = new Set([0, points.length - 1]);
-  const pending = [[0, points.length - 1]];
-  while (pending.length) {
-    const [start, end] = pending.pop()!;
-    let farthest = -1; let distance = 0.25;
+/** Douglas-Peucker split vertices, most significant first. Past a closed
+ * ring's first four points, only deviations over half a pixel count. */
+function splitOrder(points: Point[], limit: number): number[] {
+  const farthest = (start: number, end: number) => {
+    let index = -1; let distance = 0;
     for (let i = start + 1; i < end; i += 1) {
       const q = nearest(points[i], points[start], points[end]);
       const d = (q[0] - points[i][0]) ** 2 + (q[1] - points[i][1]) ** 2;
-      if (d > distance) { distance = d; farthest = i; }
-    }
-    if (farthest >= 0) {
-      keep.add(farthest);
-      pending.push([start, farthest], [farthest, end]);
+      if (d > distance) { distance = d; index = i; }
     }
+    return {
+      start, end, index, distance,
+    };
+  };
+  const pending = [farthest(0, points.length - 1)];
+  const order: number[] = [];
+  while (order.length < limit) {
+    let next = -1;
+    pending.forEach((segment, i) => {
+      if (segment.index >= 0 && (next < 0 || segment.distance > pending[next].distance)) next = i;
+    });
+    if (next < 0 || (order.length >= 2 && pending[next].distance <= 0.25)) break;
+    const { start, end, index } = pending[next];
+    order.push(index);
+    pending.splice(next, 1, farthest(start, index), farthest(index, end));
   }
-  const result = [...keep].sort((a, b) => a - b).map((i) => points[i]);
-  return result.length >= 4 ? result : points;
+  return order;
+}
+
+/** Reduce a closed contour ring to SegmentationMaxPolygonPoints. With grow, that budget
+ * doubles, up to SegmentationMaxPolygonPointsLimit, while the area cut off or added
+ * exceeds SegmentationMaxPolygonAreaError of the ring's area. */
+function simplify(points: Point[], grow: boolean): Point[] {
+  const order = splitOrder(points, (grow ? SegmentationMaxPolygonPointsLimit : SegmentationMaxPolygonPoints) - 2);
+  const area = Math.abs(polygonArea(points));
+  const ring = (budget: number) => [0, ...order.slice(0, budget - 2).sort((a, b) => a - b), points.length - 1];
+  const error = (kept: number[]) => kept.slice(1).reduce((sum, index, i) => sum
+    + Math.abs(polygonArea(points.slice(kept[i], index + 1))), 0);
+  let budget = SegmentationMaxPolygonPoints;
+  while (budget - 2 < order.length && error(ring(budget)) > SegmentationMaxPolygonAreaError * area) budget *= 2;
+  const kept = ring(budget);
+  return kept.length >= 4 ? kept.map((i) => points[i]) : points;
 }
 
 /** All mask components and holes, in original image coordinates. */
-export function maskGeometry(mask: Uint8Array, width: number, height: number): SegmentationPredictResponse {
+export function maskGeometry(mask: Uint8Array, width: number, height: number, grow = false): SegmentationPredictResponse {
   if (mask.length !== width * height) throw new Error('Invalid mask dimensions.');
   const geometry = contours().size([width, height]).thresholds([0.5])(mask as unknown as number[])[0];
   const polygons = geometry.coordinates.map((rings) => {
     const scaled = rings.map((ring) => simplify(ring.map(([x, y]): Point => [
       Math.max(0, Math.min(width, x)), Math.max(0, Math.min(height, y)),
-    ])));
+    ]), grow));
     return { exterior: scaled[0], holes: scaled.slice(1) };
   });
   if (!polygons.length) return { success: false, error: 'No object found for these prompts.' };

client/platform/desktop/backend/native/segmentation.ts

@@ -7,11 +7,23 @@
  */
 
 import type { SegmentationPolygon } from 'dive-common/apispec';
+import {
+  SegmentationMaxPolygonAreaError,
+  SegmentationMaxPolygonPoints,
+  SegmentationMaxPolygonPointsLimit,
+} from 'dive-common/use/segmentation/constants';
 import { StereoMeasurement } from './stereo';
 
 /** Error message shown to users when segmentation model process fails to load */
 export const SEGMENTATION_LOAD_ERROR_MESSAGE = "Model failed to load. If you haven't downloaded the SAM2 model pack from the VIAME Add-On wiki, please do so.";
 
+/** Re-exported for desktop callers; same as VIAME `service:max_polygon_points`. */
+export {
+  SegmentationMaxPolygonAreaError,
+  SegmentationMaxPolygonPoints,
+  SegmentationMaxPolygonPointsLimit,
+};
+
 /** Request to the segmentation service */
 export interface SegmentationInternalPredictRequest {
   /** Unique request ID for correlation */