feat: report unresolved local flags on evaluate_flags snapshots

.sampo/changesets/unresolved-local-flags.md

@@ -0,0 +1,14 @@
+---
+pypi/posthog: minor
+---
+
+Report flags that local evaluation could not resolve on `evaluate_flags()` snapshots. `FeatureFlagEvaluations.unresolved_flags` maps each such flag to an `UnresolvedFlagReason` (`experience_continuity`, `unsupported_definition`, `unresolved_dependency` or `missing_context`), and reading one reports the `local_evaluation_inconclusive` error instead of `flag_missing`. When definitions load, log one warning per flag that local evaluation can never resolve (experience continuity, a static cohort, an unknown operator, a malformed value, or a missing or circular flag dependency), and again only when its definition changes. `AsyncClient` evaluates only remotely, so its snapshots have no unresolved flags.
+
+Behavior changes:
+
+- A condition with a filter that local evaluation can't check and a filter that does not match now resolves locally as no match. Before, such a flag fell back to a remote request, or stayed unresolved with `only_evaluate_locally=True`.
+- An inactive flag with experience continuity enabled now resolves to `false` locally instead of falling back to a remote request.
+- A flag that depends on a flag with experience continuity enabled is no longer evaluated locally. It falls back to a remote request, or stays unresolved with `only_evaluate_locally=True`.
+- A flag that depends on a flag that needs the server (for example, a static cohort) now tries its other conditions locally. Before, the dependency always forced a remote request.
+- On `$feature_flag_called` events from an `evaluate_flags()` snapshot, `$feature_flag_error` is now `local_evaluation_inconclusive` instead of `flag_missing` for a flag that has a loaded definition but that local evaluation could not resolve.
+- `get_all_flags()`, `get_all_flags_and_payloads()` and `evaluate_flags()` no longer log an error with a traceback when a flag needs the server (for example, a static cohort). They fall back to a remote request without logging, as `get_feature_flag()` already did.

posthog/__init__.py

@@ -83,6 +83,7 @@
     FeatureFlagEvaluationRuntime as FeatureFlagEvaluationRuntime,
     FlagValue as FlagValue,
     FlagsAndPayloads as FlagsAndPayloads,
+    UnresolvedFlagReason as UnresolvedFlagReason,
 )
 from posthog.types import (
     FeatureFlagResult as FeatureFlagResult,
@@ -1104,7 +1105,8 @@ def evaluate_flags(
         person_properties: Person properties to use for evaluation.
         group_properties: Group properties keyed by group type.
         only_evaluate_locally: If ``True``, never fall back to remote evaluation and
-            omit flags that cannot be evaluated locally.
+            omit flags that cannot be evaluated locally. The snapshot's
+            ``unresolved_flags`` lists them with a reason.
         disable_geoip: Whether to disable GeoIP lookup.
         flag_keys: Optional list that scopes local evaluation, the underlying ``/flags``
             request, and the returned snapshot. When omitted or ``None``, all flags are evaluated.

posthog/async_client.py

@@ -893,6 +893,8 @@ async def evaluate_flags(
 
         The returned snapshot has synchronous in-memory accessors. Pass it to
         ``capture(flags=...)`` to attach the exact values used for branching.
+        This client does not evaluate flags locally, so the snapshot's
+        ``unresolved_flags`` is always empty.
         """
         host = self._get_feature_flag_evaluations_host()
         if distinct_id is None:

posthog/client.py

@@ -11,7 +11,7 @@
 import weakref
 from contextvars import ContextVar
 from datetime import datetime, timedelta, timezone
-from typing import Any, Callable, Dict, List, Mapping, Optional, Union, cast
+from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union, cast
 from uuid import UUID, uuid4
 
 from typing_extensions import Unpack
@@ -77,6 +77,7 @@
 from posthog.feature_flags import (
     InconclusiveMatchError,
     RequiresServerEvaluation,
+    _get_definition_unresolved_reason,
     match_feature_flag_properties,
     resolve_bucketing_value,
 )
@@ -115,6 +116,7 @@
     FlagsResponse,
     FlagValue,
     SendFeatureFlagsOptions,
+    UnresolvedFlagReason,
     normalize_flags_response,
     to_flags_and_payloads,
     to_payloads,
@@ -926,6 +928,11 @@ def __init__(
         self._flag_definition_cache_generation = 0
         self._flag_definition_publication_lock = threading.RLock()
         self._flag_definition_cache_write_lock = threading.RLock()
+        # Flag key -> definition fingerprint of flags already warned about as never
+        # resolvable locally, so each warning fires once until the definition changes.
+        self._warned_unresolvable_flags: Dict[str, str] = {}
+        # Definitions fingerprint of the last scan for never-resolvable flags.
+        self._unresolvable_flags_fingerprint: Optional[str] = None
         self._flag_definition_cache_provider = flag_definition_cache_provider
         self._flag_definition_cache_provider_async_runner: Optional[
             _BackgroundEventLoopRunner
@@ -3262,6 +3269,101 @@ def _update_flag_state(
                 return old_fingerprint
         return None
 
+    def _warn_unresolvable_flags(self) -> None:
+        """Log flags the published definitions can never resolve locally.
+
+        Call it after publication, outside the publication lock: classifying every
+        flag walks its cohorts, and log handlers can be slow."""
+        with self._flag_definition_publication_lock:
+            definitions_fingerprint = self._flag_definition_fingerprint
+            if definitions_fingerprint == self._unresolvable_flags_fingerprint:
+                return
+            snapshot = self._local_evaluation_snapshot()
+
+        unresolvable = self._classify_unresolvable_flags(snapshot)
+
+        with self._flag_definition_publication_lock:
+            # Newer definitions or another caller own the warning for this state.
+            if (
+                definitions_fingerprint != self._flag_definition_fingerprint
+                or definitions_fingerprint == self._unresolvable_flags_fingerprint
+            ):
+                return
+            self._unresolvable_flags_fingerprint = definitions_fingerprint
+            new_keys_by_reason: Dict[UnresolvedFlagReason, List[str]] = {}
+            for key, (reason, fingerprint) in unresolvable.items():
+                if self._warned_unresolvable_flags.get(key) != fingerprint:
+                    new_keys_by_reason.setdefault(reason, []).append(key)
+            # Rebuilt from the current definitions, so a flag that is deleted or
+            # becomes resolvable is forgotten and warns again if it later becomes
+            # unresolvable.
+            self._warned_unresolvable_flags = {
+                key: fingerprint for key, (_, fingerprint) in unresolvable.items()
+            }
+
+        self._log_unresolvable_flags(new_keys_by_reason)
+
+    @staticmethod
+    def _classify_unresolvable_flags(
+        snapshot: _LocalEvaluationSnapshot,
+    ) -> Dict[str, Tuple[UnresolvedFlagReason, str]]:
+        """Map each never-resolvable flag key to its reason and a fingerprint of its
+        definition."""
+        unresolvable: Dict[str, Tuple[UnresolvedFlagReason, str]] = {}
+        for flag in snapshot["flags"]:
+            try:
+                key = flag["key"]
+                reason = _get_definition_unresolved_reason(
+                    flag,
+                    snapshot["flags_by_key"],
+                    snapshot["cohorts"],
+                    snapshot["group_type_mapping"],
+                )
+            except Exception:
+                # Evaluation reports a definition this malformed when the flag is used.
+                continue
+            if reason is None:
+                continue
+            fingerprint = _hashlib.sha256(
+                json.dumps([reason.value, flag], sort_keys=True, default=str).encode(
+                    "utf-8"
+                )
+            ).hexdigest()
+            unresolvable[str(key)] = (reason, fingerprint)
+        return unresolvable
+
+    def _log_unresolvable_flags(
+        self, new_keys_by_reason: Dict[UnresolvedFlagReason, List[str]]
+    ) -> None:
+        experience_continuity_keys = new_keys_by_reason.get(
+            UnresolvedFlagReason.EXPERIENCE_CONTINUITY
+        )
+        if experience_continuity_keys:
+            self.log.warning(
+                f"[FEATURE FLAGS] {len(experience_continuity_keys)} flag(s) have "
+                "experience continuity enabled and can't be evaluated locally "
+                f"(reason: {UnresolvedFlagReason.EXPERIENCE_CONTINUITY.value}): "
+                f"{', '.join(experience_continuity_keys)}. Evaluating them makes a "
+                "remote request, or leaves them unresolved with "
+                "only_evaluate_locally=True. To evaluate them locally, disable "
+                "experience continuity on these flags in PostHog."
+            )
+        unsupported_definition_keys = new_keys_by_reason.get(
+            UnresolvedFlagReason.UNSUPPORTED_DEFINITION
+        )
+        if unsupported_definition_keys:
+            self.log.warning(
+                f"[FEATURE FLAGS] {len(unsupported_definition_keys)} flag(s) use "
+                "something local evaluation doesn't support, such as a static "
+                "cohort, an unknown operator, a malformed value, or a missing or "
+                "circular flag dependency "
+                f"(reason: {UnresolvedFlagReason.UNSUPPORTED_DEFINITION.value}): "
+                f"{', '.join(unsupported_definition_keys)}. Evaluating them makes a "
+                "remote request when evaluation reaches that part, or leaves them "
+                "unresolved with only_evaluate_locally=True. To evaluate them "
+                "locally, change these flags in PostHog or upgrade posthog."
+            )
+
     def _local_evaluation_snapshot(self) -> _LocalEvaluationSnapshot:
         # Capture references together. Publication replaces these collections, so
         # recursive and multi-flag evaluations can finish on their original rules.
@@ -3301,6 +3403,7 @@ def _load_feature_flags(self):
                         "[FEATURE FLAGS] Using cached flag definitions from external cache"
                     )
                     old_fingerprint = self._update_flag_state(cached_data)
+                    self._warn_unresolvable_flags()
                     if isinstance(self.flag_cache, RedisFlagCache):
                         self.flag_cache._invalidate_snapshot(old_fingerprint)
                     self._last_feature_flag_poll = datetime.now(tz=timezone.utc)
@@ -3406,6 +3509,8 @@ def _fetch_feature_flags_from_api(self):
                 self._flag_definition_published_generation = fetch_generation
                 self._flag_definition_cache_generation = fetch_generation
 
+            self._warn_unresolvable_flags()
+
             if cache_data_to_store and self._flag_definition_cache_provider:
                 # Keep provider I/O out of the publication lock. The separate lock
                 # preserves cache write order without delaying newer API fetches or
@@ -3567,12 +3672,15 @@ def _compute_flag_locally(
         # Create evaluation cache for flag dependencies
         evaluation_cache: dict[str, Optional[FlagValue]] = {}
 
-        if feature_flag.get("ensure_experience_continuity", False):
-            raise InconclusiveMatchError("Flag has experience continuity enabled")
-
         if not feature_flag.get("active"):
             return False
 
+        if feature_flag.get("ensure_experience_continuity", False):
+            raise InconclusiveMatchError(
+                "Flag has experience continuity enabled",
+                reason=UnresolvedFlagReason.EXPERIENCE_CONTINUITY,
+            )
+
         flag_filters = feature_flag.get("filters") or {}
         aggregation_group_type_index = flag_filters.get("aggregation_group_type_index")
         group_type_mapping = snapshot["group_type_mapping"]
@@ -3585,7 +3693,10 @@ def _compute_flag_locally(
                     f"[FEATURE FLAGS] Unknown group type index {aggregation_group_type_index} for feature flag {feature_flag['key']}"
                 )
                 # failover to `/flags`
-                raise InconclusiveMatchError("Flag has unknown group type index")
+                raise InconclusiveMatchError(
+                    "Flag has unknown group type index",
+                    reason=UnresolvedFlagReason.UNSUPPORTED_DEFINITION,
+                )
 
             if group_name not in groups:
                 # Group flags are never enabled in `groups` aren't passed in
@@ -3602,7 +3713,8 @@ def _compute_flag_locally(
 
             if group_name not in group_properties:
                 raise InconclusiveMatchError(
-                    f"Flag has no group properties for group '{group_name}'"
+                    f"Flag has no group properties for group '{group_name}'",
+                    reason=UnresolvedFlagReason.MISSING_CONTEXT,
                 )
             focused_group_properties = group_properties[group_name]
             group_key = groups[group_name]
@@ -3621,9 +3733,23 @@ def _compute_flag_locally(
                 group_properties=group_properties,
             )
         else:
-            bucketing_value = resolve_bucketing_value(
-                feature_flag, distinct_id, device_id
-            )
+            try:
+                bucketing_value = resolve_bucketing_value(
+                    feature_flag, distinct_id, device_id
+                )
+            except InconclusiveMatchError as e:
+                # No condition is checked without a bucketing value, so a cause in
+                # the definition also applies and outranks missing_context.
+                e.reason = (
+                    _get_definition_unresolved_reason(
+                        feature_flag,
+                        flags_by_key,
+                        snapshot["cohorts"],
+                        group_type_mapping,
+                    )
+                    or e.reason
+                )
+                raise
             return match_feature_flag_properties(
                 feature_flag,
                 distinct_id,
@@ -4539,7 +4665,8 @@ def evaluate_flags(
             person_properties: Person properties to use for evaluation.
             group_properties: Group properties keyed by group type.
             only_evaluate_locally: If True, never fall back to remote evaluation —
-                flags that can't be evaluated locally are simply omitted from the snapshot.
+                flags that can't be evaluated locally are omitted from the snapshot and
+                listed with a reason in its ``unresolved_flags``.
             disable_geoip: Whether to disable GeoIP lookup.
             flag_keys: Optional list that scopes local evaluation, the underlying
                 ``/flags`` request, and the returned snapshot. When omitted or ``None``, all
@@ -4610,6 +4737,7 @@ def evaluate_flags(
         errors_while_computing = False
         quota_limited = False
         locally_evaluated_keys: set[str] = set()
+        unresolved: Dict[str, UnresolvedFlagReason] = {}
         # Try local evaluation first when the poller has loaded definitions.
         local_person_properties = self._person_properties_for_local_evaluation(
             distinct_id, person_properties
@@ -4622,6 +4750,7 @@ def evaluate_flags(
                 group_properties=group_properties,
                 flag_keys_to_evaluate=flag_keys,
                 device_id=device_id,
+                unresolved=unresolved,
             )
         )
 
@@ -4732,6 +4861,9 @@ def evaluate_flags(
             errors_while_computing=errors_while_computing,
             quota_limited=quota_limited,
             minimal_flag_called_events=minimal_flag_called_events,
+            unresolved_flags={
+                key: reason for key, reason in unresolved.items() if key not in records
+            },
         )
 
     _feature_flag_evaluations_host_cache: Optional[_FeatureFlagEvaluationsHost] = None
@@ -4754,6 +4886,7 @@ def _get_all_flags_and_payloads_locally(
         warn_on_unknown_groups=False,
         flag_keys_to_evaluate: Optional[list[str]] = None,
         device_id: Optional[str] = None,
+        unresolved: Optional[Dict[str, UnresolvedFlagReason]] = None,
     ) -> tuple[FlagsAndPayloads, bool, _LocalEvaluationSnapshot]:
         person_properties = person_properties or {}
         group_properties = group_properties or {}
@@ -4794,9 +4927,11 @@ def _get_all_flags_and_payloads_locally(
                     )
                     if matched_payload is not None:
                         payloads[flag["key"]] = matched_payload
-                except InconclusiveMatchError:
+                except (InconclusiveMatchError, RequiresServerEvaluation) as e:
                     # No need to log this, since it's just telling us to fall back to `/flags`
                     fallback_to_flags = True
+                    if unresolved is not None:
+                        unresolved[flag["key"]] = e.reason
                 except Exception as e:
                     self.log.exception(
                         f"[FEATURE FLAGS] Error while computing variant and payload: {e}"