[Improvement] Per-model coordinate spaces via coord_scale

config/agent/default.yaml

@@ -54,6 +54,15 @@ save_som: False # Add a set of marks to the screenshot.
 # extract_visible_tag: False # Add a "visible" tag to visible elements in the AXTree.
 # extract_clickable_tag: False #  Add a "clickable" tag to clickable elements in the AXTree.
 extract_coords: False # Add the coordinates of the elements.
+
+# --- coordinate space ---
+# What an (x, y) in the model's output means. null = use the action_parser
+# family's default coordinate space (uitars: raw viewport pixels; qwen3vl: 0-1000).
+# An integer N = a normalized [0, N) grid converted to pixels by rescale_xy.
+# Qwen-VL / GLM-VL use 1000; the PaliGemma/Gemma lineage bins to 1024.
+# Leave null unless you've measured the model's convention; a wrong scale clicks
+# somewhere plausible and scores 0 silently.
+coord_scale: null
 # filter_visible_elements_only: False  # filter elements that are not visible
 # use_focused_element: False  # use focused element
 

src/open_apps/agent/README.md

@@ -67,7 +67,52 @@ The `vllm_prompt.py` has several key components:
     - Improve the prompt such that the model can output according to instruction.
     - Improve the response parser to be more lenient in parsing. Note that if you change the prompt format, you might need to change the parser!
     - Enable multi-action? (for example, don't some tasks require fill and click at the same time step?)
-    - Figure out how to map coordinates correctly! Vision models use coordinates and it's unclear when a coordinate is correct but just off by a factor or a coordinate is wrong.
+
+## Coordinate Spaces
+
+Vision models disagree about what an `(x, y)` in their output means, and getting
+it wrong is silent: the click lands somewhere plausible and the episode scores 0
+without an error. `action_parsers/coords.py::rescale_xy` is the single place that
+conversion happens; every parser goes through `ActionParser.rescale`.
+
+| convention | `coord_scale` | who |
+| --- | --- | --- |
+| raw viewport pixels | `null` | UI-TARS 1.5, GPT-4o-style computer use |
+| normalized 0-1000 | `1000` | Qwen-VL, GLM-VL |
+| normalized [0, N) | `N` | PaliGemma/Gemma-lineage `<locNNNN>` bins are 0-1024 |
+
+Each parser family carries a default (`uitars`: raw pixels, `qwen3vl`: 1000). In the
+agent yaml, leaving `coord_scale` unset (or `null`) means "use the family default";
+set `coord_scale: N` to override. Note this is one scalar applied
+cannot express a model predicting in its own non-square resized image space.
+
+The most reliable setup is to *declare* the grid in the prompt and set
+`coord_scale` to match, rather than reverse-engineering a checkpoint's native
+convention — then the conversion is correct by construction as long as the model
+complies. `config/agent/Qwen3.6-VL-computer-use.yaml` does this with a 1000x1000
+grid.
+
+Under the `uitars` grammar, rescaling applies to UI-TARS-native forms
+(`click(point=)`, `click(start_box=)`, `click(x=)`, `right_single(point=)`, and
+the `scroll(direction=, point=)` magnitude) *and* to models prompted directly in
+browsergym syntax (`mouse_click(x=, y=)` and friends). Bare `scroll(dx, dy)` is
+left alone — say so in the prompt, since a model on a normalized grid will
+otherwise not know what units to scroll in.
+
+### Calibrating a new model
+
+Don't guess the scale, measure it:
+
+1. Run a few episodes with `save_dir` set. Each step writes ground-truth element
+   boxes to `<exp_dir>/set_of_marks_coordinates.json` (see `utils.save_som_coordinates`).
+2. For a step where the model clearly intended a particular element, compare the
+   raw predicted `(x, y)` (kept verbatim in `displayed_action`) against that
+   element's `bbox`.
+3. `predicted / actual` should come out near a constant ratio per axis. A ratio
+   of ~0.52 on a 1920-wide viewport means the model is emitting 0-1000; ~0.53
+   means 0-1024. A ratio near 1.0 with scattered error means the model is
+   grounding badly, not scaling wrong — no `coord_scale` will fix that. Fall back
+   to targeting elements by set-of-marks bid (`save_som: true`) instead.
 
 ## Configuration Options
 

src/open_apps/agent/action_parsers/__init__.py

@@ -3,6 +3,7 @@
 Add a family: write an ``ActionParser`` subclass and register it below.
 """
 from .base import ActionParser, ActionParserResult
+from .coords import rescale_xy
 from .uitars import UITarsActionParser
 from .qwen3vl import Qwen3VLActionParser
 
@@ -12,12 +13,28 @@
 }
 
 
-def get_action_parser(name: str | None) -> ActionParser:
+def get_action_parser(
+    name: str | None, coord_scale: int | None = None
+) -> ActionParser:
+    """Build the action_parser for ``name``.
+
+    ``coord_scale`` overrides the family's default coordinate space when given;
+    None keeps the subclass default (uitars: raw pixels, qwen3vl: 0-1000).
+    """
     if not name:
         name = "uitars"
     if name not in REGISTRY:
         raise ValueError(f"Unknown action_parser {name!r}. Available: {sorted(REGISTRY)}")
-    return REGISTRY[name]()
+    cls = REGISTRY[name]
+    if coord_scale is None:
+        return cls()
+    return cls(coord_scale=coord_scale)
 
 
-__all__ = ["ActionParser", "ActionParserResult", "REGISTRY", "get_action_parser"]
+__all__ = [
+    "ActionParser",
+    "ActionParserResult",
+    "REGISTRY",
+    "get_action_parser",
+    "rescale_xy",
+]

src/open_apps/agent/action_parsers/base.py

@@ -7,6 +7,8 @@
 from dataclasses import dataclass
 from typing import TypedDict
 
+from .coords import rescale_xy
+
 
 class ActionParserResult(TypedDict, total=False):
     action: str
@@ -16,9 +18,18 @@ class ActionParserResult(TypedDict, total=False):
 
 @dataclass
 class ActionParser:
+    # Coordinate convention of this model family: None = raw viewport pixels,
+    # N = normalized [0, N) grid. Subclasses override the default; an agent yaml
+    # can override the subclass via ``agent.coord_scale``. See coords.rescale_xy.
+    coord_scale: int | None = None
+
     def default_prompts(self) -> dict:
         return {}
 
     def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult:
         """``viewport`` is (width, height) px. Raise ParseError on bad output."""
         raise NotImplementedError
+
+    def rescale(self, x: float, y: float, viewport: tuple[int, int]) -> tuple[int, int]:
+        """Convert one model-space (x, y) pair into viewport pixels."""
+        return rescale_xy(x, y, self.coord_scale, viewport)

src/open_apps/agent/action_parsers/coords.py

@@ -0,0 +1,41 @@
+"""Coordinate-space conversion shared by every action_parser.
+
+Models disagree on what an (x, y) in their output means. Three conventions are
+in play here:
+
+* raw viewport pixels -- UI-TARS 1.5, GPT-4o-style computer use. ``coord_scale=None``.
+* 0-1000 normalized -- Qwen-VL, GLM-VL. ``coord_scale=1000``.
+* any other normalized [0, N) grid -- e.g. PaliGemma/Gemma-lineage ``<locNNNN>``
+  bins are 0-1024. ``coord_scale=1024``.
+
+Getting this wrong is silent: the click lands somewhere plausible-looking on the
+page and the episode just scores 0. Set ``coord_scale`` per model family (see
+``ActionParser.coord_scale``), never per call site.
+"""
+from __future__ import annotations
+
+
+def rescale_xy(
+    x: float,
+    y: float,
+    coord_scale: int | None,
+    viewport: tuple[int, int],
+) -> tuple[int, int]:
+    """Map (x, y) into viewport pixels.
+
+    coord_scale=None: raw pixels (UI-TARS, GPT-4o). coord_scale=1000:
+    Qwen-VL / GLM-VL 0-1000 normalized. coord_scale=N: any normalized
+    [0, N) convention.
+
+    ``coord_scale`` is a single scalar applied against each viewport axis
+    independently, which is what a square normalized grid means. It cannot
+    express a model that predicts in its own non-square resized image space
+    (that needs a separate scale per axis).
+    """
+    vw, vh = viewport
+    if coord_scale:
+        return (
+            int(round(float(x) * vw / coord_scale)),
+            int(round(float(y) * vh / coord_scale)),
+        )
+    return int(round(float(x))), int(round(float(y)))

src/open_apps/agent/action_parsers/qwen3vl.py

@@ -8,22 +8,24 @@
 
 import json
 import re
+from dataclasses import dataclass
 
 from agentlab.llm.llm_utils import ParseError
 
 from .base import ActionParser, ActionParserResult
 
-# The model is prompted with a fictional 1000x1000 screen, so all coordinates
-# and scroll deltas it emits are in [0, 1000).
-_COORD_SPACE = 1000
-
 # Closing tag may be absent on truncation; json.loads decides well-formedness.
 _TOOL_CALL_RE = re.compile(r"<tool_call>\s*(\{.*?\})\s*(?:</tool_call>|$)", re.DOTALL)
 _THINK_RE = re.compile(r"<think>(.*?)</think>", re.DOTALL | re.IGNORECASE)
 _ACTION_LINE_RE = re.compile(r"^\s*Action:\s*(.+?)\s*$", re.MULTILINE | re.IGNORECASE)
 
 
+@dataclass
 class Qwen3VLActionParser(ActionParser):
+    # The model is prompted with a fictional 1000x1000 screen, so all
+    # coordinates and scroll deltas it emits are in [0, 1000).
+    coord_scale: int | None = 1000
+
     def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult:
         response = (response or "").strip()
         if not response:
@@ -83,7 +85,7 @@ def _to_browsergym(self, action_name, args: dict, viewport: tuple[int, int]) ->
                 raise ParseError(
                     f"scroll 'delta' must be a [dx, dy] list, got {delta!r}."
                 )
-            dx, dy = self._rescale(delta, viewport)
+            dx, dy = self.rescale(delta[0], delta[1], viewport)
             return f"scroll({dx}, {dy})"
 
         if action_name == "wait":
@@ -103,11 +105,4 @@ def _xy(self, args: dict, viewport: tuple[int, int]) -> tuple[int, int]:
         coord = args.get("coordinate")
         if not (isinstance(coord, (list, tuple)) and len(coord) == 2):
             raise ParseError(f"'coordinate' must be a [x, y] list, got {coord!r}.")
-        return self._rescale(coord, viewport)
-
-    def _rescale(self, xy, viewport: tuple[int, int]) -> tuple[int, int]:
-        vw, vh = viewport
-        return (
-            int(round(float(xy[0]) * vw / _COORD_SPACE)),
-            int(round(float(xy[1]) * vh / _COORD_SPACE)),
-        )
+        return self.rescale(coord[0], coord[1], viewport)

src/open_apps/agent/action_parsers/uitars.py

@@ -1,11 +1,29 @@
-"""UI-TARS action_parser (default): <think>/<action> ReAct, raw pixel coordinates."""
+"""UI-TARS action_parser (default): <think>/<action> ReAct grammar.
+
+UI-TARS grounds in raw viewport pixels, so ``coord_scale`` defaults to None and
+coordinates pass through untouched. Other models reuse this grammar with a
+normalized coordinate grid (Gemma emits a 0-N box); set ``agent.coord_scale``
+in their yaml and the same parser converts to pixels.
+"""
 from __future__ import annotations
 
+from dataclasses import dataclass
+
 from open_apps.agent.utils import flexible_parser
 
 from .base import ActionParser, ActionParserResult
 
 
+@dataclass
 class UITarsActionParser(ActionParser):
+    coord_scale: int | None = None
+
     def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult:
-        return flexible_parser(response)
+        # Pass no hook at all when there is nothing to convert: flexible_parser
+        # treats "no rescale" as "leave every coordinate exactly as written",
+        # which is stricter than converting through an identity (that would
+        # still round floats and rewrite already-valid browsergym calls).
+        rescale = None
+        if self.coord_scale:
+            rescale = lambda x, y: self.rescale(x, y, viewport)  # noqa: E731
+        return flexible_parser(response, rescale=rescale)

src/open_apps/agent/utils.py

@@ -187,9 +187,12 @@ def retry(
     raise ParseError(f"Could not parse a valid value after {n_retry} retries.")
 
 
-def flexible_parser(response: str) -> dict:
+def flexible_parser(response: str, rescale=None) -> dict:
     """
     A parser that tries to correct or interpret the LLMs output into a valid policy, e.g. if it did not close a parenthesis or tag.
+
+    ``rescale`` is an optional ``(x, y) -> (x, y)`` callable mapping the model's
+    coordinate space onto viewport pixels; None leaves coordinates untouched.
     """
     response = response.strip()
     result = {"action": None, "think": None}
@@ -257,7 +260,7 @@ def flexible_parser(response: str) -> dict:
 
     # HACK to help UI TARS: remap UI TARS native actions to browser gym actions
     result["displayed_action"] = result["action"]  # store model native actions
-    result = uitars_parser(result)
+    result = uitars_parser(result, rescale=rescale)
 
     return result
 
@@ -321,11 +324,48 @@ def _normalize_hotkey_key(key: str) -> "str | None":
     return "+".join(out)
 
 
-def uitars_parser(result):
-    "Translates UITARS actions to browser gym actions"
+# mouse_click(x=870, y=940) / mouse_dblclick(x=1.5, y=2) / mouse_move(...),
+# capturing the two numbers and whatever trailing kwargs follow.
+_BG_MOUSE_XY_RE = re.compile(
+    r"^(?P<fn>mouse_click|mouse_dblclick|mouse_move|mouse_down|mouse_up)"
+    r"\(\s*x\s*=\s*(?P<x>-?\d+(?:\.\d+)?)\s*,"
+    r"\s*y\s*=\s*(?P<y>-?\d+(?:\.\d+)?)\s*(?P<rest>[,)].*)$",
+    re.DOTALL,
+)
+
+
+def _rescale_browsergym_mouse_action(action: str, rescale) -> str:
+    """Convert x=/y= in an already-browsergym-form mouse action to pixels."""
+    match = _BG_MOUSE_XY_RE.match(action.strip())
+    if not match:
+        return action
+    x, y = rescale(match.group("x"), match.group("y"))
+    return f"{match.group('fn')}(x={x}, y={y}{match.group('rest')}"
+
+
+def uitars_parser(result, rescale=None):
+    """Translates UITARS actions to browser gym actions.
+
+    ``rescale`` is an optional ``(x, y) -> (x, y)`` callable converting the
+    model's coordinate space to viewport pixels (see
+    ``open_apps.agent.action_parsers.coords.rescale_xy``). UI-TARS itself emits
+    raw pixels, so the default is a no-op; models reusing this grammar with a
+    normalized grid (Gemma) pass one in.
+    """
     # note karenu: I am not sure if the translation is perfect
     # in particular if the coord are just transferable like that, but looks reasonable in practice
     # also both browsergym and uitars docs are ass, so i have to guess
+    scaling = rescale is not None
+    if not scaling:
+        def rescale(x, y):
+            return int(round(float(x))), int(round(float(y)))
+
+    # A model prompted in browsergym syntax emits mouse_click(x=, y=) directly,
+    # so it never hits the UI-TARS remaps below and would otherwise skip
+    # rescaling entirely. Rewrite in place, preserving any other kwargs
+    # (button=...). Guarded on ``scaling`` so the default path is untouched.
+    if scaling:
+        result["action"] = _rescale_browsergym_mouse_action(result["action"], rescale)
 
     # UITARS API -> BrowserGym API
 
@@ -341,7 +381,8 @@ def uitars_parser(result):
                 f"Could not parse two integer coordinates from click action: {result['action']!r}. "
                 "Expected format like click(point='(x,y)'), click(start_box='(x,y)'), or click(x=X, y=Y)."
             )
-        result["action"] = f"mouse_click(x={int(coords[0])}, y={int(coords[1])})"
+        x, y = rescale(coords[0], coords[1])
+        result["action"] = f"mouse_click(x={x}, y={y})"
     # type(content=text) -> keyboard_type(text=text)
     if result["action"].startswith("type(content="):
         result["action"] = translate_uitars_type_action(result["action"])
@@ -355,7 +396,9 @@ def uitars_parser(result):
         nums = re.findall(r"-?\d+", result["action"])
         if dir_match and len(nums) >= 2:
             direction = dir_match.group(1).lower()
-            x, y = int(nums[0]), int(nums[1])
+            # The magnitude is read off the point, so it is in the same space as
+            # a click coordinate and needs the same conversion.
+            x, y = rescale(nums[0], nums[1])
             if direction == "down":
                 dx, dy = 0, y
             elif direction == "up":
@@ -373,9 +416,8 @@ def uitars_parser(result):
                 f"Could not parse two integer coordinates from right_single action: {result['action']!r}. "
                 "Expected format like right_single(point='(x,y)')."
             )
-        result["action"] = (
-            f"mouse_click(x={int(coords[0])}, y={int(coords[1])}, button='right')"
-        )
+        x, y = rescale(coords[0], coords[1])
+        result["action"] = f"mouse_click(x={x}, y={y}, button='right')"
     # hotkey(key='ctrl alt e') -> keyboard_press(key='Control+Alt+e')
     if result["action"].startswith("hotkey(key="):
         key_comb = re.findall(r"hotkey\(key='(.*?)'\)", result["action"])