Skip to content

Commit 7fa104d

Browse files
karenchuukarenchuu
authored
refactor(control-plane): single owner for private-text classification (#5245)
1 parent da5aac1 commit 7fa104d

6 files changed

Lines changed: 555 additions & 58 deletions

File tree

‎loopx/control_plane/goals/artifact_lifecycle.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@
1919

2020
from typing import Any
2121

22-
from ...public_safe_text import find_private_text_match
22+
from ...public_safe_text import (
23+
ARTIFACT_LIFECYCLE_CATEGORIES,
24+
classify_private_text,
25+
)
2326
from ..runtime.public_safety import (
24-
SECRET_LIKE_SURFACE_PATTERN,
2527
public_safe_compact_text,
2628
validate_public_safe_value,
2729
)
@@ -60,9 +62,14 @@ def _compact_text(value: Any, *, limit: int = 240) -> str | None:
6062
validate_public_safe_value(value)
6163
except ValueError:
6264
return None
63-
# Preserve the stricter existing private-text/provider-token contract too;
64-
# these checks supplement, never replace, the shared public-safety owner.
65-
if find_private_text_match(value) or SECRET_LIKE_SURFACE_PATTERN.search(value):
65+
# One policy-aware call into the shared classifier replaces OR-ing the
66+
# text-owner detector with the credential shape detector. The named policy
67+
# (ARTIFACT_LIFECYCLE_CATEGORIES) preserves this projection's historical
68+
# verdict exactly: every category but a raw remote location.
69+
if (
70+
classify_private_text(value, categories=ARTIFACT_LIFECYCLE_CATEGORIES)
71+
is not None
72+
):
6673
return None
6774
return public_safe_compact_text(value, limit=limit)
6875

‎loopx/control_plane/runtime/public_safety.py‎

Lines changed: 11 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,42 +4,21 @@
44
from collections.abc import Mapping
55
from typing import Any, Callable, Optional
66

7+
# Refs #5136: the text-shape definitions live in one owner now
8+
# (loopx/public_safe_text.py). This module consumes them for recursive payload
9+
# validation and public-output policy instead of restating a competing set. The
10+
# redundant-alias form makes each an explicit re-export (house style under
11+
# --no-implicit-reexport), so the existing importers of these names from this
12+
# module are unchanged.
13+
from ...public_safe_text import (
14+
LOCAL_PATH_SURFACE_PATTERN as LOCAL_PATH_SURFACE_PATTERN,
15+
REMOTE_LOCATION_SURFACE_PATTERN as REMOTE_LOCATION_SURFACE_PATTERN,
16+
SECRET_LIKE_SURFACE_PATTERN as SECRET_LIKE_SURFACE_PATTERN,
17+
)
718

819
NormalizeText = Callable[..., str]
920
CompactText = Callable[..., Optional[str]]
1021
DEFAULT_PUBLIC_SAFE_LIST_LIMIT = 4
11-
LOCAL_PATH_SURFACE_PATTERN = re.compile(
12-
r"(?<![:/A-Za-z0-9])(?:"
13-
r"/(?:Users|home|Volumes|private|tmp|var|etc|opt|srv|mnt|root|data|workspace|workspaces)/"
14-
r"[^\s`'\"<>]+|"
15-
r"[A-Za-z]:[\\/][^\s`'\"<>]+|"
16-
r"\\\\[A-Za-z0-9_.-]+\\[^\s`'\"<>]+"
17-
r")",
18-
re.IGNORECASE,
19-
)
20-
# Refs #5136: one definition for "this string carries a raw remote location".
21-
# Three validators each restated the same scheme list, and the canonical
22-
# public-safety owner had no counterpart, so a fourth caller had to invent one.
23-
REMOTE_LOCATION_SURFACE_PATTERN = re.compile(r"(?i)\b(?:https?|file|s3|gs|tos|hdfs)://")
24-
SECRET_LIKE_SURFACE_PATTERN = re.compile(
25-
r"(?i)(?:\bbearer\s+[a-z0-9._~+/=-]{16,}|"
26-
r"\b(?:access|api|secret)[_-]?key[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|"
27-
r"\b(?:ak|sk)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|"
28-
r"(?<![a-z0-9_])(?:ak|sk)[-_=:][a-z0-9_=-]{10,}|"
29-
r"\bgh[pousr]_[a-z0-9]{16,}\b|"
30-
r"\bgithub_pat_[a-z0-9_]{20,}|"
31-
r"\b(?:akia|asia)[a-z0-9]{16}\b|"
32-
r"\bxox[baprs]-[a-z0-9-]{10,}|"
33-
r"\baiza[a-z0-9_-]{20,}|"
34-
r"\b(?:sk|rk)_(?:live|test)_[a-z0-9]{12,}|"
35-
r"\bnpm_[a-z0-9]{20,}|"
36-
r"\bpypi-[a-z0-9_-]{20,}|"
37-
r"\beyj[a-z0-9_-]{10,}\.[a-z0-9_-]{10,}\.[a-z0-9_-]{10,}\b|"
38-
r"\b(?:access|refresh)[_-]?token[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,}|"
39-
r"\b(?:password|secret)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,}|"
40-
r"-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY|"
41-
r"\btoken[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,})"
42-
)
4322
_CREDENTIAL_FIELD_FAMILIES = frozenset(
4423
{
4524
"accesskey",

‎loopx/public_safe_text.py‎

Lines changed: 265 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,17 @@
11
"""Canonical private-looking-text rules for public-safe control-plane fields.
22
3-
Four real validator owners enforce the same contract: `feedback`,
3+
This module is the single owner of "does this string look private?" for the
4+
control plane. It answers two questions that used to be conflated:
5+
6+
* **Detection** -- recognize a credential shape, a local path, a raw remote
7+
location, or an internal organizational marker, and return an explicit
8+
*category* plus a *reason* so a caller can decide what to do.
9+
* **Permission** -- a named *policy* selects which categories a given surface
10+
rejects. Detection recognizing a value never implies every surface must
11+
reject it: owner-private operational state and repository/PR publication have
12+
different disclosure boundaries (Refs #5136).
13+
14+
Four real validator owners enforce the same text contract: `feedback`,
415
`authority`, `boundary_authority`, and the TypeScript Vision checkpoint. The
516
rule set used to be copied into each owner, and the copies drifted: one still
617
rejected the ordinary English word "authorization" while another accepted a
@@ -9,13 +20,39 @@
920
`tests/fixtures/public_safe_text_corpus.json` pins both runtimes to one
1021
contract.
1122
23+
`control_plane/runtime/public_safety.py` also consumes the shape definitions
24+
here (`SECRET_LIKE_SURFACE_PATTERN`, `LOCAL_PATH_SURFACE_PATTERN`,
25+
`REMOTE_LOCATION_SURFACE_PATTERN`) instead of owning a competing set, so a
26+
caller such as `artifact_lifecycle` can make one policy-aware call rather than
27+
OR-ing independent detectors.
28+
1229
Each owner keeps its own error message and guidance, because those describe
1330
the owning surface, not the shared rule.
1431
"""
1532

1633
from __future__ import annotations
1734

1835
import re
36+
from dataclasses import dataclass
37+
38+
# ---------------------------------------------------------------------------
39+
# Explicit categories. A caller names a policy (a set of categories) rather
40+
# than reaching for a bare regex, so "recognized" and "rejected here" stay
41+
# separate decisions (Refs #5136, direction 2).
42+
# ---------------------------------------------------------------------------
43+
CATEGORY_CREDENTIAL = "credential"
44+
CATEGORY_LOCAL_PATH = "local_path"
45+
CATEGORY_REMOTE_LOCATION = "remote_location"
46+
CATEGORY_ORG_MARKER = "org_marker"
47+
48+
ALL_CATEGORIES: frozenset[str] = frozenset(
49+
{
50+
CATEGORY_CREDENTIAL,
51+
CATEGORY_LOCAL_PATH,
52+
CATEGORY_REMOTE_LOCATION,
53+
CATEGORY_ORG_MARKER,
54+
}
55+
)
1956

2057

2158
# Credential shape, not the plain English word. LoopX governance prose says
@@ -38,27 +75,241 @@
3875
r"[A-Za-z0-9+/=]{16,}",
3976
)
4077

41-
PRIVATE_TEXT_PATTERNS: tuple[re.Pattern[str], ...] = (
42-
re.compile(r"/" + r"Users/"),
43-
re.compile(r"/" + r"ext_data/"),
44-
re.compile("la" + "rk" + "office", re.I),
45-
re.compile("docs" + r"\." + "internal", re.I),
46-
re.compile(r"\bt-20\d{12}-[a-z0-9]+\b"),
47-
re.compile(r"\b" + "Bear" + r"er\b", re.I),
48-
_AUTHORIZATION_CREDENTIAL_SHAPE,
49-
_BASIC_CREDENTIAL_VALUE,
50-
re.compile(r"\b" + "tok" + r"en\s*=", re.I),
51-
re.compile(r"\b" + "pass" + r"word\b", re.I),
52-
re.compile(r"\b" + "sec" + r"ret\b", re.I),
78+
# Refs #5136: relocated here from control_plane/runtime/public_safety.py so a
79+
# single owner defines each shape. public_safety re-exports these names, so its
80+
# ~8 direct importers and 30+ recursive-validation callers are unchanged. This
81+
# pattern is byte-identical to the one public_safety enforced before the move:
82+
# slice A is a behavior-preserving consolidation, so no existing consumer's
83+
# verdict changes.
84+
LOCAL_PATH_SURFACE_PATTERN = re.compile(
85+
r"(?<![:/A-Za-z0-9])(?:"
86+
r"/(?:Users|home|Volumes|private|tmp|var|etc|opt|srv|mnt|root|data|workspace|workspaces)/"
87+
r"[^\s`'\"<>]+|"
88+
r"[A-Za-z]:[\\/][^\s`'\"<>]+|"
89+
r"\\\\[A-Za-z0-9_.-]+\\[^\s`'\"<>]+"
90+
r")",
91+
re.IGNORECASE,
92+
)
93+
# Refs #5136, direction 3: the shared classifier can *recognize* the local-path
94+
# shapes the legacy surface pattern misses -- a home-relative `~/...` path and a
95+
# local path behind an explicit `path:` prefix. Recognition is opt-in
96+
# (`include_path_gaps`) so this consolidation does not silently tighten the 30+
97+
# consumers of LOCAL_PATH_SURFACE_PATTERN; wiring these into a surface's
98+
# enforcement policy is the disclosed behavior change tracked separately.
99+
# `file://` is not added to the gap set here: direction 3 does classify it as a
100+
# local path, but acting on that means a public projection stops carrying a
101+
# location it accepts today, which is a disclosed tightening rather than part of
102+
# this relocation. It is applied with the enforcement policy in the follow-up.
103+
HOME_RELATIVE_PATH_PATTERN = re.compile(r"(?<![\w~])~[\\/][^\s`'\"<>]+")
104+
PATH_PREFIX_LOCAL_PATTERN = re.compile(
105+
r"(?<![\w:])path:[\\/][^\s`'\"<>]+", re.IGNORECASE
106+
)
107+
LOCAL_PATH_GAP_PATTERNS: tuple[re.Pattern[str], ...] = (
108+
HOME_RELATIVE_PATH_PATTERN,
109+
PATH_PREFIX_LOCAL_PATTERN,
110+
)
111+
# Refs #5136: one definition for "this string carries a raw remote location".
112+
# Three validators each restated the same scheme list, and the canonical
113+
# public-safety owner had no counterpart, so a fourth caller had to invent one.
114+
REMOTE_LOCATION_SURFACE_PATTERN = re.compile(r"(?i)\b(?:https?|file|s3|gs|tos|hdfs)://")
115+
SECRET_LIKE_SURFACE_PATTERN = re.compile(
116+
r"(?i)(?:\bbearer\s+[a-z0-9._~+/=-]{16,}|"
117+
r"\b(?:access|api|secret)[_-]?key[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|"
118+
r"\b(?:ak|sk)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|"
119+
r"(?<![a-z0-9_])(?:ak|sk)[-_=:][a-z0-9_=-]{10,}|"
120+
r"\bgh[pousr]_[a-z0-9]{16,}\b|"
121+
r"\bgithub_pat_[a-z0-9_]{20,}|"
122+
r"\b(?:akia|asia)[a-z0-9]{16}\b|"
123+
r"\bxox[baprs]-[a-z0-9-]{10,}|"
124+
r"\baiza[a-z0-9_-]{20,}|"
125+
r"\b(?:sk|rk)_(?:live|test)_[a-z0-9]{12,}|"
126+
r"\bnpm_[a-z0-9]{20,}|"
127+
r"\bpypi-[a-z0-9_-]{20,}|"
128+
r"\beyj[a-z0-9_-]{10,}\.[a-z0-9_-]{10,}\.[a-z0-9_-]{10,}\b|"
129+
r"\b(?:access|refresh)[_-]?token[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,}|"
130+
r"\b(?:password|secret)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,}|"
131+
r"-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY|"
132+
r"\btoken[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,})"
133+
)
134+
135+
136+
# The text-owner rule set (feedback / authority / boundary_authority / the
137+
# TypeScript Vision checkpoint). Each entry carries an explicit category and a
138+
# stable reason so `classify_private_text` can hand a caller a named verdict
139+
# instead of a bare regex object. Order is significant: `find_private_text_match`
140+
# returns the first match, and the shared corpus pins that first-match contract.
141+
@dataclass(frozen=True)
142+
class _CategorizedPattern:
143+
pattern: re.Pattern[str]
144+
category: str
145+
reason: str
146+
147+
148+
_CATEGORIZED_PRIVATE_TEXT_PATTERNS: tuple[_CategorizedPattern, ...] = (
149+
_CategorizedPattern(
150+
re.compile(r"/" + r"Users/"), CATEGORY_LOCAL_PATH, "absolute home-directory path"
151+
),
152+
_CategorizedPattern(
153+
re.compile(r"/" + r"ext_data/"), CATEGORY_ORG_MARKER, "internal ext_data path"
154+
),
155+
_CategorizedPattern(
156+
re.compile("la" + "rk" + "office", re.I),
157+
CATEGORY_ORG_MARKER,
158+
"internal Lark/Feishu office marker",
159+
),
160+
_CategorizedPattern(
161+
re.compile("docs" + r"\." + "internal", re.I),
162+
CATEGORY_ORG_MARKER,
163+
"internal docs host marker",
164+
),
165+
_CategorizedPattern(
166+
re.compile(r"\bt-20\d{12}-[a-z0-9]+\b"),
167+
CATEGORY_ORG_MARKER,
168+
"internal ticket identifier",
169+
),
170+
_CategorizedPattern(
171+
re.compile(r"\b" + "Bear" + r"er\b", re.I),
172+
CATEGORY_CREDENTIAL,
173+
"bearer auth scheme word",
174+
),
175+
_CategorizedPattern(
176+
_AUTHORIZATION_CREDENTIAL_SHAPE,
177+
CATEGORY_CREDENTIAL,
178+
"authorization header/assignment shape",
179+
),
180+
_CategorizedPattern(
181+
_BASIC_CREDENTIAL_VALUE, CATEGORY_CREDENTIAL, "basic-auth credential value"
182+
),
183+
_CategorizedPattern(
184+
re.compile(r"\b" + "tok" + r"en\s*=", re.I),
185+
CATEGORY_CREDENTIAL,
186+
"token assignment shape",
187+
),
188+
_CategorizedPattern(
189+
re.compile(r"\b" + "pass" + r"word\b", re.I),
190+
CATEGORY_CREDENTIAL,
191+
"password word",
192+
),
193+
_CategorizedPattern(
194+
re.compile(r"\b" + "sec" + r"ret\b", re.I),
195+
CATEGORY_CREDENTIAL,
196+
"secret word",
197+
),
198+
)
199+
200+
# Kept as the plain pattern tuple so `find_private_text_match` and every
201+
# existing importer see byte-identical behavior (same patterns, same order).
202+
PRIVATE_TEXT_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
203+
entry.pattern for entry in _CATEGORIZED_PRIVATE_TEXT_PATTERNS
204+
)
205+
206+
207+
# ---------------------------------------------------------------------------
208+
# Named policies. A policy is the set of categories a surface rejects. Keeping
209+
# these named (rather than inline regex ORs at each caller) is what lets one
210+
# detection owner serve surfaces with different disclosure boundaries.
211+
# ---------------------------------------------------------------------------
212+
# The four text owners reject every recognized category (their historical
213+
# behavior): credential shapes, local paths, remote locations, org markers.
214+
TEXT_OWNER_CATEGORIES: frozenset[str] = ALL_CATEGORIES
215+
# artifact_lifecycle historically OR-ed find_private_text_match (the text-owner
216+
# set) with SECRET_LIKE_SURFACE_PATTERN, downstream of a validate_public_safe_value
217+
# call that already rejected the local-path and credential shapes. That union
218+
# covers every category *except* a raw remote location: this projection has
219+
# always let an ordinary http(s) URL through. The policy preserves that exactly
220+
# rather than silently tightening it; widening it to remote_location is a
221+
# separate, disclosed decision (Refs #5136, direction 2).
222+
ARTIFACT_LIFECYCLE_CATEGORIES: frozenset[str] = frozenset(
223+
{CATEGORY_CREDENTIAL, CATEGORY_LOCAL_PATH, CATEGORY_ORG_MARKER}
53224
)
54225

55226

227+
@dataclass(frozen=True)
228+
class PrivateTextMatch:
229+
"""An explicit, categorized private-text detection result."""
230+
231+
category: str
232+
reason: str
233+
pattern: re.Pattern[str]
234+
235+
56236
def find_private_text_match(value: str | None) -> re.Pattern[str] | None:
57-
"""Return the first matching private-text pattern, or None when clean."""
237+
"""Return the first matching private-text pattern, or None when clean.
238+
239+
Preserved verbatim for the four text owners and the shared corpus parity
240+
test; `classify_private_text` is the category-aware successor.
241+
"""
58242

59243
if not value:
60244
return None
61245
for pattern in PRIVATE_TEXT_PATTERNS:
62246
if pattern.search(value):
63247
return pattern
64248
return None
249+
250+
251+
# The shape-based detectors, categorized. These supplement the text-owner
252+
# patterns so a single call can cover both owners that artifact_lifecycle used
253+
# to OR together.
254+
_SHAPE_DETECTORS: tuple[tuple[re.Pattern[str], str, str], ...] = (
255+
(SECRET_LIKE_SURFACE_PATTERN, CATEGORY_CREDENTIAL, "credential-like value shape"),
256+
(LOCAL_PATH_SURFACE_PATTERN, CATEGORY_LOCAL_PATH, "local filesystem path"),
257+
(
258+
REMOTE_LOCATION_SURFACE_PATTERN,
259+
CATEGORY_REMOTE_LOCATION,
260+
"raw remote location URL",
261+
),
262+
)
263+
264+
265+
def classify_private_text(
266+
value: str | None,
267+
*,
268+
categories: frozenset[str] = ALL_CATEGORIES,
269+
include_path_gaps: bool = False,
270+
) -> PrivateTextMatch | None:
271+
"""Return the first recognized private-text match within ``categories``.
272+
273+
Detection only: recognizing a value does not decide whether a given surface
274+
may publish it. Callers pass the named policy (category set) for their
275+
destination. The text-owner patterns are checked first, in their pinned
276+
order, then the relocated shape detectors, so a value that both owners used
277+
to flag still resolves to a single explicit category and reason.
278+
279+
``include_path_gaps`` opts a surface into the direction-3 recognition of
280+
home-relative (``~/``) and ``path:``-prefixed local references. It defaults
281+
to False so this consolidation does not silently tighten any surface that
282+
has not chosen the wider policy.
283+
"""
284+
285+
if not value:
286+
return None
287+
for entry in _CATEGORIZED_PRIVATE_TEXT_PATTERNS:
288+
if entry.category in categories and entry.pattern.search(value):
289+
return PrivateTextMatch(entry.category, entry.reason, entry.pattern)
290+
for pattern, category, reason in _SHAPE_DETECTORS:
291+
if category in categories and pattern.search(value):
292+
return PrivateTextMatch(category, reason, pattern)
293+
if include_path_gaps and CATEGORY_LOCAL_PATH in categories:
294+
for pattern in LOCAL_PATH_GAP_PATTERNS:
295+
if pattern.search(value):
296+
return PrivateTextMatch(
297+
CATEGORY_LOCAL_PATH, "local path behind a relative/prefixed form", pattern
298+
)
299+
return None
300+
301+
302+
def matches_private_text_policy(
303+
value: str | None,
304+
*,
305+
categories: frozenset[str] = ALL_CATEGORIES,
306+
include_path_gaps: bool = False,
307+
) -> bool:
308+
"""True when ``value`` is recognized within the named policy's categories."""
309+
310+
return (
311+
classify_private_text(
312+
value, categories=categories, include_path_gaps=include_path_gaps
313+
)
314+
is not None
315+
)

0 commit comments

Comments
 (0)