fix(analytics): distinguish fallback reasons + forward backend error message (SDK-79, SDK-83)

mixpanel/flags/local_feature_flags.py

@@ -15,9 +15,11 @@
 from .types import (
     ExperimentationFlag,
     ExperimentationFlags,
+    FallbackReason,
     LocalFlagsConfig,
     Rollout,
     SelectedVariant,
+    VariantSource,
 )
 from .utils import (
     EXPOSURE_EVENT,
@@ -228,15 +230,17 @@ def get_variant(
 
         if not flag_definition:
             logger.warning("Cannot find flag definition for key: '%s'", flag_key)
-            return fallback_value
+            return fallback_value.as_fallback(FallbackReason.flag_not_found())
 
         if not (context_value := context.get(flag_definition.context)):
             logger.warning(
                 "The rollout context, '%s' for flag, '%s' is not present in the supplied context dictionary",
                 flag_definition.context,
                 flag_key,
             )
-            return fallback_value
+            return fallback_value.as_fallback(
+                FallbackReason.missing_context_key(flag_definition.context)
+            )
 
         selected_variant: SelectedVariant | None = None
 
@@ -257,15 +261,15 @@ def get_variant(
                 self._track_exposure(
                     flag_key, selected_variant, context, end_time - start_time
                 )
-            return selected_variant
+            return selected_variant.with_source(VariantSource.LOCAL)
 
         logger.debug(
             "%s context %s not eligible for any rollout for flag: %s",
             flag_definition.context,
             context_value,
             flag_key,
         )
-        return fallback_value
+        return fallback_value.as_fallback(FallbackReason.no_rollout_match())
 
     def track_exposure_event(
         self, flag_key: str, variant: SelectedVariant, context: dict[str, Any]

mixpanel/flags/remote_feature_flags.py

@@ -11,7 +11,13 @@
 
 from mixpanel.credentials import ServiceAccountCredentials
 
-from .types import RemoteFlagsConfig, RemoteFlagsResponse, SelectedVariant
+from .types import (
+    FallbackReason,
+    RemoteFlagsConfig,
+    RemoteFlagsResponse,
+    SelectedVariant,
+    VariantSource,
+)
 from .utils import (
     EXPOSURE_EVENT,
     REQUEST_HEADERS,
@@ -94,6 +100,8 @@ async def aget_all_variants(
         except Exception:
             logger.exception("Failed to get remote variants")
 
+        if flags is not None:
+            flags = {k: v.with_source(VariantSource.REMOTE) for k, v in flags.items()}
         return flags
 
     async def aget_variant_value(
@@ -151,9 +159,15 @@ async def aget_variant(
                         distinct_id, EXPOSURE_EVENT, properties
                     )
                 )
-        except Exception:
+        except Exception as exc:
             logger.exception("Failed to get remote variant for flag '%s'", flag_key)
-            return fallback_value
+            # SDK-83: attach the exception message so the OpenFeature wrapper
+            # can forward it as error_message. Without this the caller sees
+            # a bare GENERAL error and has to dig through logs to find out
+            # the backend rejected the request.
+            return fallback_value.as_fallback(
+                FallbackReason.backend_error(self._describe_backend_error(exc))
+            )
         else:
             return selected_variant
 
@@ -210,6 +224,8 @@ def get_all_variants(
         except Exception:
             logger.exception("Failed to get remote variants")
 
+        if flags is not None:
+            flags = {k: v.with_source(VariantSource.REMOTE) for k, v in flags.items()}
         return flags
 
     def get_variant_value(
@@ -265,9 +281,13 @@ def get_variant(
                 )
                 self._tracker(distinct_id, EXPOSURE_EVENT, properties)
 
-        except Exception:
+        except Exception as exc:
             logger.exception("Failed to get remote variant for flag '%s'", flag_key)
-            return fallback_value
+            # SDK-83: attach the exception message so the OpenFeature wrapper
+            # can forward it as error_message.
+            return fallback_value.as_fallback(
+                FallbackReason.backend_error(self._describe_backend_error(exc))
+            )
         else:
             return selected_variant
 
@@ -355,20 +375,46 @@ def _handle_response(self, response: httpx.Response) -> dict[str, SelectedVarian
         flags_response = RemoteFlagsResponse.model_validate(response.json())
         return flags_response.flags
 
+    @staticmethod
+    def _describe_backend_error(exc: Exception) -> str:
+        """Best-effort backend message for FallbackReason.backend_error.
+
+        For HTTP errors the response body usually contains the actionable
+        detail (e.g. "distinct_id must be provided in evalContext as a
+        string") — httpx's default str(exc) only carries the status line,
+        so reach into exc.response.text when available.
+
+        Never fall back to ``str(exc)`` for HTTPStatusError: httpx's default
+        formatting includes the full request URL, which carries the project
+        token and distinct_id in the query string. Since this message is
+        forwarded verbatim into ``FlagResolutionDetails.error_message`` by
+        the OpenFeature wrapper, leaking those into user-visible output
+        would be a real regression (SDK-83 security review).
+        """
+        if isinstance(exc, httpx.HTTPStatusError):
+            body = exc.response.text.strip() if exc.response is not None else ""
+            status = exc.response.status_code if exc.response is not None else "?"
+            return f"HTTP {status}: {body}" if body else f"HTTP {status}"
+        return str(exc)
+
     def _lookup_flag_in_response(
         self,
         flag_key: str,
         flags: dict[str, SelectedVariant],
         fallback_value: SelectedVariant,
     ) -> tuple[SelectedVariant, bool]:
         if flag_key in flags:
-            return flags[flag_key], False
+            return flags[flag_key].with_source(VariantSource.REMOTE), False
         logger.debug(
             "Flag '%s' not found in remote response. Returning fallback, '%s'",
             flag_key,
             fallback_value,
         )
-        return fallback_value, True
+        # The /flags endpoint only returns variants the user is enrolled in,
+        # so a missing key could mean the flag doesn't exist OR the user
+        # isn't in any rollout. The remote SDK can't tell them apart without
+        # server-side help — surface as FLAG_NOT_FOUND for now.
+        return fallback_value.as_fallback(FallbackReason.flag_not_found()), True
 
     def shutdown(self):
         self._sync_client.close()

mixpanel/flags/test_local_feature_flags.py

@@ -23,6 +23,7 @@
     SelectedVariant,
     Variant,
     VariantOverride,
+    VariantSource,
 )
 
 TEST_FLAG_KEY = "test_flag"
@@ -723,6 +724,45 @@ async def test_is_enabled_returns_true_for_true_variant_value(self):
         result = self._flags.is_enabled(TEST_FLAG_KEY, USER_CONTEXT)
         assert result is True
 
+    @respx.mock
+    async def test_get_variant_tags_match_as_local(self):
+        flag = create_test_flag(rollout_percentage=100.0)
+        await self.setup_flags([flag])
+        fallback = SelectedVariant(variant_value="fb")
+        result = self._flags.get_variant(TEST_FLAG_KEY, fallback, USER_CONTEXT)
+        assert result.variant_source == VariantSource.LOCAL
+        assert result.fallback_reason is None
+        assert result.variant_key is not None
+
+    @respx.mock
+    async def test_get_variant_tags_missing_flag(self):
+        await self.setup_flags([])
+        fallback = SelectedVariant(variant_value="fb")
+        result = self._flags.get_variant("missing", fallback, USER_CONTEXT)
+        assert result.variant_source == VariantSource.FALLBACK
+        assert result.fallback_reason.kind == "FLAG_NOT_FOUND"
+        assert result.fallback_reason.message is None
+        assert result.variant_value == "fb"
+
+    @respx.mock
+    async def test_get_variant_tags_missing_context(self):
+        flag = create_test_flag(context="distinct_id")
+        await self.setup_flags([flag])
+        fallback = SelectedVariant(variant_value="fb")
+        result = self._flags.get_variant(TEST_FLAG_KEY, fallback, {})
+        assert result.variant_source == VariantSource.FALLBACK
+        assert result.fallback_reason.kind == "MISSING_CONTEXT_KEY"
+        assert result.fallback_reason.message == "distinct_id"
+
+    @respx.mock
+    async def test_get_variant_tags_no_rollout_match(self):
+        flag = create_test_flag(rollout_percentage=0.0)
+        await self.setup_flags([flag])
+        fallback = SelectedVariant(variant_value="fb")
+        result = self._flags.get_variant(TEST_FLAG_KEY, fallback, USER_CONTEXT)
+        assert result.variant_source == VariantSource.FALLBACK
+        assert result.fallback_reason.kind == "NO_ROLLOUT_MATCH"
+
     @respx.mock
     async def test_get_variant_value_uses_most_recent_polled_flag(self):
         polling_iterations = 0

mixpanel/flags/test_remote_feature_flags.py

@@ -10,7 +10,12 @@
 from mixpanel.credentials import ServiceAccountCredentials
 
 from .remote_feature_flags import RemoteFeatureFlagsProvider
-from .types import RemoteFlagsConfig, RemoteFlagsResponse, SelectedVariant
+from .types import (
+    RemoteFlagsConfig,
+    RemoteFlagsResponse,
+    SelectedVariant,
+    VariantSource,
+)
 
 ENDPOINT = "https://api.mixpanel.com/flags"
 
@@ -154,7 +159,11 @@ async def test_aget_all_variants_returns_all_variants_from_api(self):
 
         result = await self._flags.aget_all_variants({"distinct_id": "user123"})
 
-        assert result == variants
+        assert set(result.keys()) == {"flag1", "flag2"}
+        assert result["flag1"].variant_value == "value1"
+        assert result["flag2"].variant_value == "value2"
+        # Every returned variant must be tagged with variant_source=REMOTE.
+        assert all(v.variant_source == VariantSource.REMOTE for v in result.values())
 
     @respx.mock
     async def test_aget_all_variants_returns_none_on_network_error(self):
@@ -265,6 +274,47 @@ def test_get_variant_value_is_fallback_if_call_fails(self):
         )
         assert result == "control"
 
+    @respx.mock
+    def test_get_variant_tags_fallback_with_backend_message_on_http_error(self):
+        """SDK-83: the backend's response message must propagate through
+        FallbackReason.message so the OpenFeature wrapper can forward it
+        as error_message instead of swallowing it into a bare GENERAL."""
+        respx.get(ENDPOINT).mock(
+            return_value=httpx.Response(
+                400, text="distinct_id must be provided in evalContext as a string"
+            )
+        )
+
+        fallback = SelectedVariant(variant_value="control")
+        result = self._flags.get_variant(
+            "test_flag", fallback, {"distinct_id": "user123"}, reportExposure=False
+        )
+
+        assert result.variant_source == VariantSource.FALLBACK
+        assert result.fallback_reason.kind == "BACKEND_ERROR"
+        assert "distinct_id must be provided" in result.fallback_reason.message
+
+    @respx.mock
+    def test_backend_error_message_never_leaks_token_or_distinct_id(self):
+        """Empty-body HTTP error must NOT expose the request URL — httpx's
+        default str(exc) formatting would include the query string, which
+        carries the project token and JSON-encoded context (SDK-83 security
+        review). Only the status code should surface downstream."""
+        respx.get(ENDPOINT).mock(return_value=httpx.Response(500, text=""))
+
+        fallback = SelectedVariant(variant_value="control")
+        result = self._flags.get_variant(
+            "test_flag", fallback, {"distinct_id": "user123"}, reportExposure=False
+        )
+
+        assert result.variant_source == VariantSource.FALLBACK
+        assert result.fallback_reason.kind == "BACKEND_ERROR"
+        message = result.fallback_reason.message
+        assert message == "HTTP 500"
+        assert "test-token" not in message
+        assert "distinct_id" not in message
+        assert "user123" not in message
+
     @respx.mock
     def test_get_variant_value_is_fallback_if_bad_response_format(self):
         respx.get(ENDPOINT).mock(return_value=httpx.Response(200, text="invalid json"))
@@ -365,7 +415,10 @@ def test_get_all_variants_returns_all_variants_from_api(self):
 
         result = self._flags.get_all_variants({"distinct_id": "user123"})
 
-        assert result == variants
+        assert set(result.keys()) == {"flag1", "flag2"}
+        assert result["flag1"].variant_value == "value1"
+        assert result["flag2"].variant_value == "value2"
+        assert all(v.variant_source == VariantSource.REMOTE for v in result.values())
 
     @respx.mock
     def test_get_all_variants_returns_none_on_network_error(self):