diff --git a/.sampo/changesets/experiment-exposure.md b/.sampo/changesets/experiment-exposure.md new file mode 100644 index 00000000..5636ed78 --- /dev/null +++ b/.sampo/changesets/experiment-exposure.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +Send `$experiment_exposure` when a remotely evaluated config version 2 flag assigns an experiment variant. The SDK sends it together with `$feature_flag_called`, from `get_feature_flag()`, `feature_enabled()`, `get_feature_flag_result()`, and the `is_enabled()` and `get_flag()` reads of an `evaluate_flags()` snapshot. It carries the flag key, the same `$feature_flag_response` as its `$feature_flag_called`, the variant key and rule ID, `locally_evaluated: false`, and the flag ID, version, request ID and evaluation time when the response has them. It is sent once per distinct ID, groups, flag, rule and variant, whatever value or request it came from. `$feature_flag_called` for a config version 2 flag gets `$feature_flag_config_version` and `$feature_flag_reason_code`, which replaces the human-readable `$feature_flag_reason`. A targeting match, rollout miss, paused experiment or holdout adds the `$feature_flag_rule_id` and `$feature_flag_rule_type` it names, and a complete experiment split adds these and `$feature_flag_variant`. Other outcomes, including failed flags, add no rule properties. Minimal `$feature_flag_called` events keep these properties, and their dedupe includes them, so equal-valued arms and successive experiments of one flag are each reported. A split that lacks its rule type, rule ID or variant key sends no exposure and reports no rule or variant. A `get_feature_flag_payload()` lookup with `match_value` reports the caller's value, so it sends no exposure. `send_feature_flag_events=False` suppresses both events. Events of config version 1 flags, flags from servers without typed values and locally evaluated flags are unchanged, including their `$feature_flag_reason`, and `$feature_flag_response` is unchanged on every event. diff --git a/posthog/async_client.py b/posthog/async_client.py index 323b16b8..d2c6baf8 100644 --- a/posthog/async_client.py +++ b/posthog/async_client.py @@ -38,8 +38,8 @@ from .capture_mode import CaptureMode, _resolve_capture_mode from .client import ( MAX_DICT_SIZE as _MAX_DICT_SIZE, - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES, Client as _SyncClient, + _capture_flag_events, _metadata_has_experiment, _parse_flag_payload, add_context_tags as _add_context_tags, @@ -78,6 +78,7 @@ from .release_id import _resolve_release_id from .request import QuotaLimitError, determine_server_host, normalize_host from .types import ( + FeatureFlag, FlagMetadata, FlagValue, _has_untyped_flag_records, @@ -186,6 +187,9 @@ def __init__( 0, feature_flags_request_max_retries ) self.distinct_ids_feature_flags_reported = SizeLimitedDict(_MAX_DICT_SIZE, set) + # $experiment_exposure dedupe, kept apart from $feature_flag_called's so that a + # reported call never suppresses its exposure. Same bound and eviction. + self._experiment_exposures_reported = SizeLimitedDict(_MAX_DICT_SIZE, set) self._feature_flag_evaluations_host_cache: Optional[ _FeatureFlagEvaluationsHost ] = None @@ -1054,38 +1058,29 @@ def _capture_feature_flag_called_if_needed( disable_geoip: Optional[bool] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, + details: Optional[FeatureFlag] = None, + value_from_details: bool = False, ) -> None: - groups_key = ( - tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () - ) - reported_key = (key, response, groups_key) - reported_flags = self.distinct_ids_feature_flags_reported.get(distinct_id) - if reported_flags is None: - reported_flags = set() - self.distinct_ids_feature_flags_reported[distinct_id] = reported_flags - if reported_key in reported_flags: - return - - if has_experiment is not None: - properties["$feature_flag_has_experiment"] = has_experiment - capture_kwargs: dict[str, Any] = {} - if minimal_flag_called_events and has_experiment is False: - capture_kwargs["_property_allowlist"] = ( - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES - ) capture_groups = { str(group_type): _stringify_id(group_key) for group_type, group_key in (groups or {}).items() } - self.capture( - "$feature_flag_called", + _capture_flag_events( + self.capture, + self.distinct_ids_feature_flags_reported, + self._experiment_exposures_reported, distinct_id=distinct_id, + key=key, + response=response, properties=properties, - groups=capture_groups, + groups=groups, + capture_groups=capture_groups, disable_geoip=disable_geoip, - **capture_kwargs, + has_experiment=has_experiment, + minimal_flag_called_events=minimal_flag_called_events, + details=details, + value_from_details=value_from_details, ) - reported_flags.add(reported_key) def _pending_queue_items(self) -> int: return int(getattr(self._queue, "_unfinished_tasks", self._queue.qsize())) @@ -1240,6 +1235,7 @@ async def shutdown(self) -> None: errors.append(error) self.distinct_ids_feature_flags_reported.clear() + self._experiment_exposures_reported.clear() self._closed = True if errors and self.debug: raise errors[0] diff --git a/posthog/client.py b/posthog/client.py index 4c090029..d31cea2c 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -11,7 +11,18 @@ import weakref from contextvars import ContextVar from datetime import datetime, timedelta, timezone -from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + Mapping, + NamedTuple, + Optional, + Tuple, + Union, + cast, +) from uuid import UUID, uuid4 from typing_extensions import Unpack @@ -330,6 +341,12 @@ def wrapper(self, *args, **kwargs): "$feature_flag_evaluated_at", "$feature_flag_error", "locally_evaluated", + # Config version 2 attribution, which minimal events must keep + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant", # Correctness-required "$groups", "$process_person_profile", @@ -362,6 +379,236 @@ def _metadata_has_experiment(metadata: Any) -> Optional[bool]: return metadata.has_experiment if isinstance(metadata, FlagMetadata) else None +# The reason code and rule type of the one outcome that sends ``$experiment_exposure``: +# a variant assigned by an experiment rule. +_EXPERIMENT_SPLIT_REASON = "experiment_split" +_EXPERIMENT_RULE_TYPE = "experiment" + +# The reason codes of a config version 2 record that report the rule that produced +# the result. Any other reason code, and any failed record, reports no rule, even if +# the record names one. +_RULE_REASON_CODES = frozenset( + { + "targeting_match", + _EXPERIMENT_SPLIT_REASON, + "rollout_miss", + "experiment_paused", + "holdout", + } +) + +# ``$experiment_exposure`` copies these from the ``$feature_flag_called`` of the same +# read, when it has them. +_EXPOSURE_PROPERTIES_FROM_FLAG_CALLED = ( + "$feature_flag_id", + "$feature_flag_version", + "$feature_flag_request_id", + "$feature_flag_evaluated_at", +) + + +def _v4_field(record: Any, name: str) -> Any: + """Read a ``/flags?v=4`` field of a parsed flag record or its metadata, such as + ``failed`` or ``config_version``. + + These fields stay private, as ``_failed``, ``_config_version`` and so on, until + the typed flag accessors make them public under their wire names. Either spelling + is read, so the two changes can land in either order. + """ + return getattr(record, name, getattr(record, f"_{name}", None)) + + +class _RuleAttribution(NamedTuple): + """What a config version 2 flag record reports on its flag events.""" + + reason_code: Optional[str] + rule_id: Optional[str] + rule_type: Optional[str] + # Set only for a complete experiment split, the outcome that sends an exposure. + variant_key: Optional[str] + + def flag_called_properties(self) -> Dict[str, Any]: + properties: Dict[str, Any] = {"$feature_flag_config_version": 2} + if self.reason_code is not None: + properties["$feature_flag_reason_code"] = self.reason_code + if self.rule_id is not None: + properties["$feature_flag_rule_id"] = self.rule_id + if self.rule_type is not None: + properties["$feature_flag_rule_type"] = self.rule_type + if self.variant_key is not None: + properties["$feature_flag_variant"] = self.variant_key + return properties + + +def _rule_attribution(details: Optional[FeatureFlag]) -> Optional[_RuleAttribution]: + """Read the event attribution of a config version 2 flag record. + + Returns ``None`` for anything else: no record, an unreadable record, a config + version 1 record, or a record from a server without typed values. Only the + evaluation details are read, never the value a getter returned. + + The config version and reason code are always reported. The rule ID and rule + type are reported, when the record has them, only for a targeting match, a + rollout miss, a paused experiment, a holdout and a complete experiment split. A + failed record, or any other reason code, reports neither. The variant key is + reported only for a complete experiment split: reason ``experiment_split``, rule + type ``experiment``, a rule ID and a variant key. A split without all of them is + malformed. It keeps its reason but reports no rule ID, rule type or variant, and + sends no exposure. Its identity is never guessed from the value, the key or an + earlier result. + """ + if details is None or not isinstance(details.metadata, FlagMetadata): + return None + metadata = details.metadata + if _v4_field(metadata, "config_version") != 2: + return None + reason_code = (details.reason.code if details.reason else None) or None + if _v4_field(details, "failed") is True or reason_code not in _RULE_REASON_CODES: + return _RuleAttribution(reason_code, None, None, None) + rule_id = _v4_field(metadata, "rule_id") or None + rule_type = _v4_field(metadata, "rule_type") or None + if reason_code != _EXPERIMENT_SPLIT_REASON: + # Only an experiment split assigns a variant. + return _RuleAttribution(reason_code, rule_id, rule_type, None) + variant_key = _v4_field(metadata, "variant_key") or None + if rule_type != _EXPERIMENT_RULE_TYPE or rule_id is None or variant_key is None: + return _RuleAttribution(reason_code, None, None, None) + return _RuleAttribution(reason_code, rule_id, rule_type, variant_key) + + +def _capture_flag_events( + capture: Callable[..., Any], + flag_calls_reported: Dict[Any, Any], + exposures_reported: Dict[Any, Any], + *, + distinct_id: ID_TYPES, + key: str, + response: Optional[FlagValue], + properties: Dict[str, Any], + groups: Optional[Mapping[str, Union[str, int]]], + capture_groups: Mapping[str, Any], + disable_geoip: Optional[bool], + has_experiment: Optional[bool], + minimal_flag_called_events: bool, + details: Optional[FeatureFlag], + value_from_details: bool, +) -> None: + """Send the flag events of one flag read: ``$feature_flag_called`` and, for an + experiment split, ``$experiment_exposure``. + + Every read that sends flag events comes here, on both clients: the single-flag + getters and the reads of an ``evaluate_flags()`` snapshot. Both events are built + from ``details``, the parsed ``/flags`` record that supplied the value, and the + call's ``properties``; the flag is never evaluated again. + + ``$feature_flag_called`` is sent if the (distinct_id, flag, response, groups) + tuple hasn't already been reported on this client. Group context is included so + that group-scoped flags fire a separate event for each group a user is evaluated + under. A config version 2 record adds its attribution to the event and to that + tuple, so equal-valued A/A arms, successive experiments of one flag, and + holdouts are each reported. Its reason code replaces the human-readable + ``$feature_flag_reason``. A config version 1 event and tuple are unchanged. + + ``has_experiment`` is the server-reported signal for whether the flag is linked + to an experiment (``None`` when the server did not report it). When the + server-controlled gate is on and the flag is known non-experiment, the event is + trimmed to a strict allowlist; any missing signal sends the full legacy shape, + and experiment-linked flags keep the full set for exposure analysis. + + ``minimal_flag_called_events`` is the gate as observed by the evaluation that + produced ``response``, not a fresh read of client-wide state: both callers + resolve it themselves (the snapshot pins it at construction; the single-flag + path reads it from the specific local/remote source that produced the value) so + a concurrent poller refresh or another ``/flags`` call can't reshape an event + after the fact. + + ``$experiment_exposure`` is sent for a complete config version 2 experiment + split whose record supplied the reported value (``value_from_details``), once + per distinct ID, groups, flag, rule and variant. It has its own dedupe, so a + ``$feature_flag_called`` already reported never suppresses it. + """ + groups_key = ( + tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () + ) + attribution = _rule_attribution(details) + feature_flag_reported_key: Tuple[Any, ...] = (key, response, groups_key) + if attribution is not None: + feature_flag_reported_key += (2, *attribution) + + reported_flags = flag_calls_reported.get(distinct_id) + if reported_flags is None: + reported_flags = set() + flag_calls_reported[distinct_id] = reported_flags + + if feature_flag_reported_key not in reported_flags: + if attribution is not None: + properties.pop("$feature_flag_reason", None) + properties.update(attribution.flag_called_properties()) + # Record the server's experiment signal when known, so minimization's impact + # can be measured by segmenting on it. + if has_experiment is not None: + properties["$feature_flag_has_experiment"] = has_experiment + + # Minimize iff the server-controlled gate is on AND the flag is known to have + # no linked experiment. Any missing signal (gate absent, has_experiment + # missing) fails safe to the full legacy shape. + should_minimize = minimal_flag_called_events and has_experiment is False + # Only thread the internal allowlist through when minimizing, so the + # full-property path's capture() call signature stays unchanged. + extra_capture_kwargs: Dict[str, Any] = {} + if should_minimize: + extra_capture_kwargs["_property_allowlist"] = ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + ) + + capture( + "$feature_flag_called", + distinct_id=distinct_id, + properties=properties, + groups=capture_groups, + disable_geoip=disable_geoip, + **extra_capture_kwargs, + ) + reported_flags.add(feature_flag_reported_key) + + if attribution is None or attribution.variant_key is None or not value_from_details: + return + + # The value, request ID, evaluation time and evaluation source are left out, so + # one assignment is one exposure however often it is read. + exposure_key = (key, groups_key, 2, attribution.rule_id, attribution.variant_key) + normalized_distinct_id = str(distinct_id) + reported_exposures = exposures_reported.get(normalized_distinct_id) + if reported_exposures is None: + reported_exposures = set() + exposures_reported[normalized_distinct_id] = reported_exposures + if exposure_key in reported_exposures: + return + + exposure_properties: Dict[str, Any] = { + name: properties[name] + for name in _EXPOSURE_PROPERTIES_FROM_FLAG_CALLED + if name in properties + } + exposure_properties.update( + { + "$feature_flag": key, + "$feature_flag_response": response, + "$feature_flag_variant": attribution.variant_key, + "$feature_flag_rule_id": attribution.rule_id, + "locally_evaluated": properties.get("locally_evaluated") is True, + } + ) + capture( + "$experiment_exposure", + distinct_id=distinct_id, + properties=exposure_properties, + groups=capture_groups, + disable_geoip=disable_geoip, + ) + reported_exposures.add(exposure_key) + + class _Lane: """A capture lane: a queue drained by its own consumer pool, posting to one endpoint. @@ -926,6 +1173,9 @@ def __init__( ) self.poller: Optional[Poller] = None self.distinct_ids_feature_flags_reported = SizeLimitedDict(MAX_DICT_SIZE, set) + # $experiment_exposure dedupe, kept apart from $feature_flag_called's so that a + # reported call never suppresses its exposure. Same bound and eviction. + self._experiment_exposures_reported = SizeLimitedDict(MAX_DICT_SIZE, set) self.flag_fallback_cache_url = flag_fallback_cache_url self.flag_cache = self._initialize_flag_cache(flag_fallback_cache_url) self.flag_definition_version = 0 @@ -3162,6 +3412,11 @@ def _shutdown_once(self, errors: list[Exception]) -> None: self.distinct_ids_feature_flags_reported.clear, errors, ) + self._run_lifecycle_cleanup( + "Failed to clear experiment exposure deduplication state on shutdown", + self._experiment_exposures_reported.clear, + errors, + ) if self.exception_capture: self._run_lifecycle_cleanup( @@ -4030,6 +4285,9 @@ def _get_feature_flag_result( flag_result = None flag_details = None + # Whether flag_details supplied the value this read reports. Only then may the + # read send $experiment_exposure. + value_from_details = False request_id = None evaluated_at = None feature_flag_error: Optional[str] = None @@ -4129,6 +4387,11 @@ def _get_feature_flag_result( self.log.debug( f"Successfully computed flag remotely: #{key} -> #{flag_result}" ) + # A payload lookup for a caller's match_value reports that value, not + # the record's. A failure above falls back to a stale cached value. + value_from_details = ( + flag_details is not None and override_match_value is None + ) except QuotaLimitError as e: self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}") feature_flag_error = FeatureFlagError.QUOTA_LIMITED @@ -4179,6 +4442,7 @@ def _get_feature_flag_result( request_id, evaluated_at, flag_details, + value_from_details, feature_flag_error, has_experiment, minimal_flag_called_events, @@ -4202,7 +4466,9 @@ def get_feature_flag_result( """ Get a FeatureFlagResult object which contains the flag result and payload for a key by evaluating locally or remotely depending on whether local evaluation is enabled and the flag can be locally evaluated. - This also captures the `$feature_flag_called` event unless `send_feature_flag_events` is `False`. + This also captures the `$feature_flag_called` event unless `send_feature_flag_events` is `False`, + and, when a remotely evaluated config version 2 flag assigns an experiment variant, the + `$experiment_exposure` event. Examples: ```python @@ -4468,6 +4734,7 @@ def _capture_feature_flag_called( request_id: Optional[str], evaluated_at: Optional[int], flag_details: Optional[FeatureFlag], + value_from_details: bool, feature_flag_error: Optional[str] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, @@ -4507,6 +4774,8 @@ def _capture_feature_flag_called( disable_geoip=disable_geoip, has_experiment=has_experiment, minimal_flag_called_events=minimal_flag_called_events, + details=flag_details, + value_from_details=value_from_details, ) def _capture_feature_flag_called_if_needed( @@ -4520,66 +4789,34 @@ def _capture_feature_flag_called_if_needed( disable_geoip: Optional[bool] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, + details: Optional[FeatureFlag] = None, + value_from_details: bool = False, ) -> None: - """Fire a ``$feature_flag_called`` event if the (distinct_id, flag, response, - groups) tuple hasn't already been reported on this client. Group context is - included so that group-scoped flags fire a separate event for each group a user - is evaluated under. Shared by the single-flag evaluation path and - ``FeatureFlagEvaluations.is_enabled() / get_flag()`` so both paths dedupe - identically. - - ``has_experiment`` is the server-reported signal for whether the flag is linked - to an experiment (``None`` when the server did not report it). When the - server-controlled gate is on and the flag is known non-experiment, the event is - trimmed to a strict allowlist; any missing signal sends the full legacy shape, - and experiment-linked flags keep the full set for exposure analysis. - - ``minimal_flag_called_events`` is the gate as observed by the evaluation that - produced ``response``, not a fresh read of client-wide state: both callers - resolve it themselves (the snapshot pins it at construction; the single-flag - path reads it from the specific local/remote source that produced the value) - so a concurrent poller refresh or another ``/flags`` call can't reshape an - event after the fact. + """Send the flag events of a flag read: ``$feature_flag_called`` and, for an + experiment split, ``$experiment_exposure``. See ``_capture_flag_events``. + + Shared by the single-flag evaluation path and the reads of a + ``FeatureFlagEvaluations`` snapshot, so every read reports and dedupes + identically. ``details`` is the parsed ``/flags`` record of the read, or + ``None`` for a locally evaluated flag. ``value_from_details`` says whether that + record supplied the reported value; only then can the read send an exposure. """ - groups_key = ( - tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () - ) - feature_flag_reported_key = (key, response, groups_key) - - reported_flags = self.distinct_ids_feature_flags_reported.get(distinct_id) - if reported_flags is None: - reported_flags = set() - self.distinct_ids_feature_flags_reported[distinct_id] = reported_flags - - if feature_flag_reported_key in reported_flags: - return - - # Record the server's experiment signal when known, so minimization's impact - # can be measured by segmenting on it. - if has_experiment is not None: - properties["$feature_flag_has_experiment"] = has_experiment - - # Minimize iff the server-controlled gate is on AND the flag is known to have - # no linked experiment. Any missing signal (gate absent, has_experiment - # missing) fails safe to the full legacy shape. - should_minimize = minimal_flag_called_events and has_experiment is False - # Only thread the internal allowlist through when minimizing, so the - # full-property path's capture() call signature stays unchanged. - extra_capture_kwargs: dict[str, Any] = {} - if should_minimize: - extra_capture_kwargs["_property_allowlist"] = ( - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES - ) - - self.capture( - "$feature_flag_called", + _capture_flag_events( + self.capture, + self.distinct_ids_feature_flags_reported, + self._experiment_exposures_reported, distinct_id=distinct_id, + key=key, + response=response, properties=properties, - groups=groups or {}, + groups=groups, + capture_groups=groups or {}, disable_geoip=disable_geoip, - **extra_capture_kwargs, + has_experiment=has_experiment, + minimal_flag_called_events=minimal_flag_called_events, + details=details, + value_from_details=value_from_details, ) - reported_flags.add(feature_flag_reported_key) def get_remote_config_payload(self, key: str): """ diff --git a/posthog/feature_flag_evaluations.py b/posthog/feature_flag_evaluations.py index ef4e401d..1723b85f 100644 --- a/posthog/feature_flag_evaluations.py +++ b/posthog/feature_flag_evaluations.py @@ -62,6 +62,11 @@ class FeatureFlagEvaluations: To narrow the set of flags that get attached to a captured event, use the in-memory helpers :meth:`only` and :meth:`only_accessed`. To narrow the set of flags requested from the server in the first place, pass ``flag_keys`` to :meth:`Client.evaluate_flags`. + + Reading a flag sends ``$feature_flag_called``, deduplicated as :meth:`is_enabled` + describes. When the flag was evaluated remotely and its config version 2 record + assigns an experiment variant, the read also sends ``$experiment_exposure``, once + per assignment. """ def __init__( @@ -305,4 +310,8 @@ def _record_access(self, key: str) -> None: properties=properties, has_experiment=flag.has_experiment if flag else None, minimal_flag_called_events=self._minimal_flag_called_events, + # The record that supplied the value. A remotely evaluated experiment + # split also sends $experiment_exposure, whatever this read returned. + details=flag.details if flag else None, + value_from_details=True, ) diff --git a/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json b/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json index df7f1c7e..f7b015d0 100644 --- a/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json +++ b/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json @@ -1,10 +1,10 @@ { "description": "Copied unchanged from the released feature flag rules v2 contract in PostHog/posthog-sdk-test-harness. Do not edit these files; copy a newer release instead.", "repository": "https://github.com/PostHog/posthog-sdk-test-harness", - "release": "1.22.0", - "revision": "17d861f8e230fe943934d70e3eebf5cff6bf3aa4", + "release": "1.27.0", + "revision": "75fb00092c6045daef60c30acc10ac83afefbfe9", "contract_directory": "contracts/feature_flag_rules_v2", - "contract_version": "7.0.0", + "contract_version": "8.0.0", "files": { "readers.json": { "source_path": "fixtures/wire/readers.json", @@ -21,6 +21,14 @@ "v2_value_evaluation.json": { "source_path": "corpus/v2_value_evaluation.json", "sha256": "aaddd05e47fc5a4ae07ea42ada5a659798f4b5fc8d0c0827f6a68e4daf68c128" + }, + "feature_flag_called_context.schema.json": { + "source_path": "schemas/feature_flag_called_context.schema.json", + "sha256": "816acfe33391d38471a4530ddac8b147c2bdc2e3c4ddbd8f384368faf57ee3b6" + }, + "experiment_exposure_properties.schema.json": { + "source_path": "schemas/experiment_exposure_properties.schema.json", + "sha256": "6d33707de086537abb90bb6b649da49bd4aca359fcf2c7877eceffb0a86dee63" } } } diff --git a/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json b/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json new file mode 100644 index 00000000..9f50ebe7 --- /dev/null +++ b/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://posthog.com/contracts/feature_flag_rules_v2/experiment_exposure_properties.schema.json", + "title": "SDK-sent $experiment_exposure properties", + "type": "object", + "additionalProperties": false, + "required": [ + "$feature_flag", + "$feature_flag_response", + "$feature_flag_variant", + "$feature_flag_rule_id", + "locally_evaluated" + ], + "properties": { + "$feature_flag": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_response": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "$feature_flag_variant": { + "type": "string" + }, + "$feature_flag_rule_id": { + "type": "string", + "format": "uuid" + }, + "locally_evaluated": { + "type": "boolean" + }, + "$feature_flag_forced_variant": { + "const": true + } + }, + "description": "Closed projection of the properties of an SDK-sent exposure. The SDK sends it only for a config version 2 experiment split, so it carries no config version, rule type, reason code or source marker; the rule ID marks it as SDK-sent. Experiment ID and forced-variant context are forbidden. Capture identity and existing optional transport properties are outside this projection. Scan the entire event recursively for assignment seeds.", + "not": { + "required": [ + "$feature_flag_forced_variant" + ] + } +} diff --git a/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json b/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json new file mode 100644 index 00000000..e55fd2fa --- /dev/null +++ b/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json @@ -0,0 +1,491 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://posthog.com/contracts/feature_flag_rules_v2/feature_flag_called_context.schema.json", + "title": "$feature_flag_called context properties", + "type": "object", + "additionalProperties": false, + "required": [ + "$feature_flag", + "$feature_flag_response" + ], + "properties": { + "$feature_flag": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_response": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "$feature_flag_config_version": { + "enum": [ + 1, + 2 + ] + }, + "$feature_flag_variant": { + "type": "string" + }, + "$feature_flag_rule_id": { + "type": "string", + "format": "uuid" + }, + "$feature_flag_rule_type": { + "enum": [ + "targeted_release", + "percentage_rollout", + "experiment" + ] + }, + "$feature_flag_experiment_id": { + "type": "integer" + }, + "$feature_flag_reason_code": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_reason": {}, + "$feature_flag_forced_variant": { + "const": true + } + }, + "description": "Closed projection of the flag-access context properties. Other existing capture fields are outside this projection. Complete v2 producer context follows the terminal outcome; tolerant-reader diagnostics may omit an incomplete attribution tuple. A config version 2 call carries $feature_flag_reason_code and never $feature_flag_reason, the reason description that only version 1 calls carry.", + "allOf": [ + { + "if": { + "required": [ + "$feature_flag_config_version" + ], + "properties": { + "$feature_flag_config_version": { + "const": 2 + } + } + }, + "then": { + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_forced_variant" + ] + }, + { + "required": [ + "$feature_flag_reason" + ] + } + ] + }, + "oneOf": [ + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + }, + "$feature_flag_rule_type": { + "const": "targeted_release" + } + }, + "required": [ + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + }, + "$feature_flag_rule_type": { + "const": "percentage_rollout" + } + }, + "required": [ + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_split" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_rule_type", + "$feature_flag_rule_id", + "$feature_flag_variant" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + }, + "$feature_flag_rule_type": { + "const": "percentage_rollout" + } + }, + "required": [ + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_paused" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "holdout" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "no_rule_match" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "missing_group_key" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "dependency_error" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "error" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "description": "A local-only outcome with no matrix row: the server never returns flag_disabled, and the call carries no rule or experiment attribution.", + "properties": { + "$feature_flag_reason_code": { + "const": "flag_disabled" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "description": "A tolerant reader may emit a diagnostic call for a malformed split after omitting all rule/experiment attribution. The config marker and reason remain available for diagnostics.", + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_split" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + } + ] + }, + "else": { + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_rule_type" + ] + } + ] + } + } + } + ] +} diff --git a/posthog/test/test_experiment_exposure.py b/posthog/test/test_experiment_exposure.py new file mode 100644 index 00000000..ba0dc3f8 --- /dev/null +++ b/posthog/test/test_experiment_exposure.py @@ -0,0 +1,1218 @@ +"""``$experiment_exposure`` and the attribution of ``$feature_flag_called``. + +A remotely evaluated config version 2 record that assigns an experiment variant sends +``$experiment_exposure`` next to its ``$feature_flag_called``, and every config version +2 record adds its rule attribution to ``$feature_flag_called``. The reader fixtures and +event schemas are copied unchanged from a released ``PostHog/posthog-sdk-test-harness`` +contract; ``fixtures/feature_flag_rules_v2/SOURCE.json`` records which. +""" + +from __future__ import annotations + +import copy +import json +import warnings +from dataclasses import dataclass +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional +from unittest import mock + +import jsonschema +import pytest + +from posthog import AsyncPosthog +from posthog.client import _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES, Client +from posthog.test.test_flags_v4_response import FIXTURES, READERS, _build +from posthog.test.test_utils import FAKE_TEST_API_KEY +from posthog.types import FeatureFlagResult + +FLAG = "pricing-page" +DISTINCT_ID = "visitor-42" +RULE = "7f3c2a10-8d4e-4b6a-9c21-5e0f1a2b3c4d" +OTHER_RULE = "2b9d4e61-0c7a-4f35-8e12-6a4b3c2d1e0f" +REQUEST_ID = "5d1e2f30-4a5b-4c6d-8e7f-90a1b2c3d4e5" +OTHER_REQUEST_ID = "6e2f3a41-5b6c-4d7e-9f80-a1b2c3d4e5f6" +EVALUATED_AT = 1_800_000_000_000 + +EXPOSURE = "$experiment_exposure" +FLAG_CALLED = "$feature_flag_called" +ATTRIBUTION_PROPERTIES = ( + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant", +) + + +# $experiment_exposure leaves these out. The config version, rule type and reason code +# stay on the $feature_flag_called of the same read. The schema projection drops +# undeclared properties, so their absence is asserted separately. +EXPOSURE_LEAVES_OUT = ( + "$feature_flag_config_version", + "$feature_flag_rule_type", + "$feature_flag_reason_code", + "$experiment_exposure_source", +) + + +def _schema(name: str) -> jsonschema.Draft202012Validator: + schema = json.loads((FIXTURES / name).read_text(encoding="utf-8")) + return jsonschema.Draft202012Validator(schema) + + +FLAG_CALLED_SCHEMA = _schema("feature_flag_called_context.schema.json") +EXPOSURE_SCHEMA = _schema("experiment_exposure_properties.schema.json") + + +def _projection( + validator: jsonschema.Draft202012Validator, properties: Mapping[str, Any] +) -> Dict[str, Any]: + """The properties a closed contract projection covers.""" + names = validator.schema["properties"] + return {key: value for key, value in properties.items() if key in names} + + +# --- Responses ---------------------------------------------------------------- + + +def _record( + code: str, + value: Any = True, + *, + key: str = FLAG, + rule_type: Optional[str] = None, + rule_id: Optional[str] = None, + variant_key: Optional[str] = None, + config_version: int = 2, + has_experiment: bool = False, + description: Optional[str] = None, + failed: bool = False, +) -> Dict[str, Any]: + metadata: Dict[str, Any] = { + "id": 77, + "version": 3, + "config_version": config_version, + "description": None, + "payload": None, + "has_experiment": has_experiment, + } + for field, field_value in ( + ("rule_type", rule_type), + ("rule_id", rule_id), + ("variant_key", variant_key), + ): + if field_value is not None: + metadata[field] = field_value + reason: Dict[str, Any] = { + "code": code, + "condition_index": 0 if rule_id else None, + } + if description is not None: + reason["description"] = description + record = {"key": key, "value": value, "reason": reason, "metadata": metadata} + if failed: + record["failed"] = True + return record + + +def _split( + value: Any = "annual", + variant_key: str = "annual-first", + rule_id: str = RULE, + **overrides: Any, +) -> Dict[str, Any]: + fields: Dict[str, Any] = { + "rule_type": "experiment", + "rule_id": rule_id, + "variant_key": variant_key, + "has_experiment": True, + **overrides, + } + return _record("experiment_split", value, **fields) + + +def _response( + *records: Dict[str, Any], + request_id: str = REQUEST_ID, + evaluated_at: int = EVALUATED_AT, + **envelope: Any, +) -> Dict[str, Any]: + return { + "flags": {record["key"]: record for record in records}, + "errorsWhileComputingFlags": False, + "requestId": request_id, + "evaluatedAt": evaluated_at, + **envelope, + } + + +# --- Recording events ----------------------------------------------------------- + + +@dataclass +class _Event: + name: str + distinct_id: Any + properties: Dict[str, Any] + groups: Any + property_allowlist: Any + uuid: Optional[str] + + +def _record_events(client: Any) -> tuple[Any, List[_Event]]: + """Patch ``client.capture`` to record each call, then capture as usual.""" + events: List[_Event] = [] + capture = client.capture + + def record(event: str, **kwargs: Any) -> Optional[str]: + uuid = capture(event, **kwargs) + events.append( + _Event( + name=event, + distinct_id=kwargs.get("distinct_id"), + properties=copy.deepcopy(kwargs.get("properties") or {}), + groups=kwargs.get("groups"), + property_allowlist=kwargs.get("_property_allowlist"), + uuid=uuid, + ) + ) + return uuid + + return mock.patch.object(client, "capture", side_effect=record), events + + +def _named(events: List[_Event], name: str, key: str = FLAG) -> List[_Event]: + return [ + event + for event in events + if event.name == name and event.properties.get("$feature_flag") == key + ] + + +def _attribution(properties: Mapping[str, Any]) -> Dict[str, Any]: + return { + name: properties[name] for name in ATTRIBUTION_PROPERTIES if name in properties + } + + +# --- Reading through each path ------------------------------------------------------ + +# Every sync read that sends flag events. Each takes the client, the key, the +# distinct ID and the groups. +SYNC_READS: Dict[str, Callable[..., Any]] = { + "get_feature_flag": lambda client, key, distinct_id, groups: ( + client.get_feature_flag(key, distinct_id, groups=groups) + ), + "feature_enabled": lambda client, key, distinct_id, groups: ( + client.feature_enabled(key, distinct_id, groups=groups) + ), + "get_feature_flag_result": lambda client, key, distinct_id, groups: ( + client.get_feature_flag_result(key, distinct_id, groups=groups) + ), + "snapshot.get_flag": lambda client, key, distinct_id, groups: ( + client.evaluate_flags(distinct_id, groups=groups).get_flag(key) + ), + "snapshot.is_enabled": lambda client, key, distinct_id, groups: ( + client.evaluate_flags(distinct_id, groups=groups).is_enabled(key) + ), +} +SINGLE_FLAG_READS = ["get_feature_flag", "feature_enabled", "get_feature_flag_result"] +SNAPSHOT_READS = ["snapshot.get_flag", "snapshot.is_enabled"] + + +@pytest.fixture +def client() -> Iterator[Client]: + client = Client(FAKE_TEST_API_KEY, send=False) + yield client + client.shutdown() + + +def _serving(*responses: Dict[str, Any]) -> Any: + """Patch ``/flags`` to answer with each response in turn, the last one after that.""" + remaining = list(responses) + + def answer(*args: Any, **kwargs: Any) -> Dict[str, Any]: + response = remaining.pop(0) if len(remaining) > 1 else remaining[0] + return copy.deepcopy(response) + + return mock.patch("posthog.client.flags", side_effect=answer) + + +def _read( + client: Client, + path: str, + key: str = FLAG, + distinct_id: Any = DISTINCT_ID, + groups: Optional[Dict[str, Any]] = None, +) -> Any: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return SYNC_READS[path](client, key, distinct_id, groups or {}) + + +def _read_events( + path: str, + *responses: Dict[str, Any], + reads: int = 1, + client: Optional[Client] = None, + **read_kwargs: Any, +) -> List[_Event]: + """Read ``FLAG`` ``reads`` times through ``path``, one response per read.""" + own_client = client is None + client = client or Client(FAKE_TEST_API_KEY, send=False) + recording, events = _record_events(client) + try: + with _serving(*responses), recording: + for _ in range(reads): + _read(client, path, **read_kwargs) + finally: + if own_client: + client.shutdown() + return events + + +async def _read_async_events( + responses: List[Dict[str, Any]], + reads: List[Callable[[Any], Any]], +) -> List[_Event]: + """Run the reads on one async client, answering each evaluation with the next + response, and the last one after that.""" + remaining = list(responses) + + def answer(*args: Any, **kwargs: Any) -> Dict[str, Any]: + response = remaining.pop(0) if len(remaining) > 1 else remaining[0] + return copy.deepcopy(response) + + with mock.patch( + "posthog.async_client._async_flags", new=mock.AsyncMock(side_effect=answer) + ): + client = AsyncPosthog("project-key", send=False) + recording, events = _record_events(client) + with recording: + for read in reads: + await read(client) + await client.shutdown() + return events + + +# --- The contract's reader cases -------------------------------------------------------- + +READER_CASES = [case for case in READERS["cases"] if case["expected"] == "reader"] + + +def _check_reader_events(case: Dict[str, Any], events: List[_Event]) -> None: + reader = case["reader"] + expectations = {reader["requested_key"]: reader} + expectations.update(case.get("sibling_expectations", {})) + for key, expected in expectations.items(): + exposures = _named(events, EXPOSURE, key) + assert len(exposures) == (1 if expected["experiment_exposure"] else 0), key + [called] = _named(events, FLAG_CALLED, key) + config_version = expected.get("config_version") + if expected.get("error_code") == "PARSE_ERROR": + config_version = None + assert called.properties.get("$feature_flag_config_version") == ( + 2 if config_version == 2 else None + ), key + if called.properties["$feature_flag_response"] is not None: + FLAG_CALLED_SCHEMA.validate( + _projection(FLAG_CALLED_SCHEMA, called.properties) + ) + for exposure in exposures: + EXPOSURE_SCHEMA.validate(_projection(EXPOSURE_SCHEMA, exposure.properties)) + assert not set(EXPOSURE_LEAVES_OUT) & set(exposure.properties), key + assert ( + exposure.properties["$feature_flag_response"] + == (called.properties["$feature_flag_response"]) + ) + assert exposure.uuid is not None and exposure.uuid != called.uuid + for event in [called, *exposures]: + # A flag may be named anything, so `$feature/` is left out. + for name in event.properties: + if not name.startswith("$feature/"): + for part in ("experiment_id", "holdout", "seed"): + assert part not in name, name + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize("case", READER_CASES, ids=lambda case: case["id"]) +def test_reader_cases_send_an_exposure_exactly_when_the_contract_expects(case, path): + response = _build(READERS, case) + client = Client(FAKE_TEST_API_KEY, send=False) + recording, events = _record_events(client) + try: + with _serving(response), recording: + _read(client, path, case["reader"]["requested_key"]) + for sibling in case.get("sibling_expectations", {}): + _read(client, path, sibling) + finally: + client.shutdown() + + _check_reader_events(case, events) + + +@pytest.mark.parametrize("method", ["get_flag", "is_enabled"]) +@pytest.mark.parametrize("case", READER_CASES, ids=lambda case: case["id"]) +async def test_async_reader_cases_send_an_exposure_exactly_when_the_contract_expects( + case, method +): + keys = [case["reader"]["requested_key"], *case.get("sibling_expectations", {})] + + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + for key in keys: + getattr(snapshot, method)(key) + + events = await _read_async_events([_build(READERS, case)], [read]) + + _check_reader_events(case, events) + + +# --- Attribution on $feature_flag_called -------------------------------------------------- + +_RULE_ID = "10000000-0000-4000-8000-000000000001" +ATTRIBUTION_CASES = [ + # (reader template or case id, attribution on $feature_flag_called) + ( + "experiment_split", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + "$feature_flag_variant": "arm_a", + }, + ), + ( + "targeting_match_targeted_release", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "targeted_release", + }, + ), + ( + "targeting_match_percentage_rollout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "percentage_rollout", + }, + ), + ( + "rollout_miss_percentage_rollout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "rollout_miss", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "percentage_rollout", + }, + ), + ( + "rollout_miss_experiment", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "rollout_miss", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "experiment_paused", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_paused", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "holdout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "holdout", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + # A holdout assigns no variant, even when its record names one. + ( + "readers.holdout_variant_key", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "holdout", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "no_rule_match", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "no_rule_match", + }, + ), + ( + "missing_group_key", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "missing_group_key", + }, + ), + ( + "dependency_error", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "dependency_error", + }, + ), + ( + "error", + {"$feature_flag_config_version": 2, "$feature_flag_reason_code": "error"}, + ), + # A malformed split keeps its config version and reason, and loses the whole rule + # tuple. + *( + ( + case_id, + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + }, + ) + for case_id in ( + "readers.partial_split_missing_rule_type", + "readers.partial_split_missing_rule_id", + "readers.partial_split_missing_variant_key", + "readers.wrong_rule_type_literal", + "readers.partial_split_string_value", + ) + ), + # Config version 1, from either kind of server, is unchanged. + ("v1", {}), + ("old_server", {}), + ("readers.partial_split_missing_config_version", {}), +] + + +def _attribution_response(source: str) -> Dict[str, Any]: + if source.startswith("readers."): + [case] = [case for case in READERS["cases"] if case["id"] == source] + return _build(READERS, case) + return _build(READERS, {"template": source}) + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "source, expected", ATTRIBUTION_CASES, ids=[c[0] for c in ATTRIBUTION_CASES] +) +def test_flag_called_attribution_follows_the_record(source, expected, path): + events = _read_events(path, _attribution_response(source), key="garden-layout") + + [called] = _named(events, FLAG_CALLED, "garden-layout") + assert _attribution(called.properties) == expected + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + exposures = _named(events, EXPOSURE, "garden-layout") + assert len(exposures) == (1 if "$feature_flag_variant" in expected else 0) + + +# A record whose reason names no rule, or that failed, reports no rule even when it +# carries one. Neither sends an exposure. The last element says whether the record is +# a valid server outcome, which the contract's event schema then accepts. +STRAY_RULE_CASES = [ + ("no_rule_match", {}, True), + ("missing_group_key", {}, True), + ("flag_disabled", {}, True), + ("dependency_error", {"failed": True, "value": None}, True), + ("error", {"failed": True, "value": None}, True), + # A failed record reports no rule, whatever its reason code. + ( + "experiment_split", + {"failed": True, "value": None, "variant_key": "annual-first"}, + True, + ), + ("holdout", {"failed": True, "value": None}, False), + # A reason code this SDK does not know. + ("future_reason", {}, False), +] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, fields, valid_outcome", + STRAY_RULE_CASES, + ids=[f"{c}-{sorted(f)}" for c, f, _ in STRAY_RULE_CASES], +) +def test_rule_fields_are_sent_only_for_reasons_that_name_a_rule( + code, fields, valid_outcome, path +): + fields = {"value": False, **fields} + record = _record( + code, rule_type="experiment", rule_id=RULE, has_experiment=True, **fields + ) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + } + assert _named(events, EXPOSURE) == [] + if valid_outcome: + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, rule_type", + [ + ("targeting_match", "targeted_release"), + ("targeting_match", "percentage_rollout"), + ("rollout_miss", "percentage_rollout"), + ("rollout_miss", "experiment"), + ("experiment_paused", "experiment"), + ("holdout", "experiment"), + ], +) +def test_rule_matches_misses_pauses_and_holdouts_name_their_rule(code, rule_type, path): + record = _record(code, False, rule_type=rule_type, rule_id=RULE) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + "$feature_flag_rule_id": RULE, + "$feature_flag_rule_type": rule_type, + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +def test_an_older_server_record_reports_no_attribution(): + # A record without a typed value comes from a server before /flags?v=4. Its + # metadata is not read as config version 2, whatever it says. + response = _response( + { + "key": FLAG, + "enabled": True, + "variant": "annual", + "reason": {"code": "condition_match", "condition_index": 0}, + "metadata": { + "id": 77, + "version": 3, + "config_version": 2, + "rule_type": "experiment", + "rule_id": RULE, + "variant_key": "annual-first", + }, + } + ) + for path in SYNC_READS: + events = _read_events(path, response) + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == {} + assert _named(events, EXPOSURE) == [] + + +# --- $feature_flag_reason ------------------------------------------------------------------ + +REASON_DESCRIPTION = "Matched rule 1" + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize( + "record", + [ + _record( + "targeting_match", + rule_type="targeted_release", + rule_id=RULE, + description=REASON_DESCRIPTION, + ), + _record("no_rule_match", False, description=REASON_DESCRIPTION), + _record("error", None, failed=True, description=REASON_DESCRIPTION), + _split(description=REASON_DESCRIPTION), + ], + ids=["targeting_match", "no_rule_match", "failed", "experiment_split"], +) +def test_a_config_version_2_call_reports_its_reason_by_code_only(record, path): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_reason_code"] == record["reason"]["code"] + assert "$feature_flag_reason" not in called.properties + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize( + "record", + [ + _record( + "condition_match", + "annual", + config_version=1, + description=REASON_DESCRIPTION, + ), + # A record from a server before /flags?v=4, which has no config version. + { + "key": FLAG, + "enabled": True, + "variant": "annual", + "reason": { + "code": "condition_match", + "condition_index": 0, + "description": REASON_DESCRIPTION, + }, + "metadata": {"id": 77, "version": 3}, + }, + ], + ids=["config_version_1", "older_server"], +) +def test_a_call_without_config_version_2_keeps_its_reason_description(record, path): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_reason"] == REASON_DESCRIPTION + assert _attribution(called.properties) == {} + + +@pytest.mark.parametrize( + "config_version, code, reason", + [(1, "condition_match", REASON_DESCRIPTION), (2, "targeting_match", None)], +) +def test_minimal_calls_keep_the_reason_description_only_without_config_version_2( + config_version, code, reason +): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + record = _record( + code, config_version=config_version, description=REASON_DESCRIPTION + ) + try: + with _serving(_response(record, minimalFlagCalledEvents=True)): + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + client.get_feature_flag_result(FLAG, "visitor-43") + finally: + client.shutdown() + + assert [message["event"] for message in messages] == [FLAG_CALLED, FLAG_CALLED] + for message in messages: + assert set(message["properties"]) <= _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + assert message["properties"].get("$feature_flag_reason") == reason + + +# --- The exact events of a split --------------------------------------------------------- + + +# The attribution of a split's $feature_flag_called. +EXPECTED_ATTRIBUTION = { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + "$feature_flag_rule_id": RULE, + "$feature_flag_rule_type": "experiment", + "$feature_flag_variant": "annual-first", +} +EXPECTED_EXPOSURE = { + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "$feature_flag_variant": "annual-first", + "$feature_flag_rule_id": RULE, + "locally_evaluated": False, + "$feature_flag_id": 77, + "$feature_flag_version": 3, + "$feature_flag_request_id": REQUEST_ID, + "$feature_flag_evaluated_at": EVALUATED_AT, +} + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_split_sends_both_events_with_exact_properties(path): + response = _response(_split(description="Assigned variant 'annual-first'")) + + events = _read_events(path, response, groups={"company": "company-7"}) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + called, exposure = events + assert called.properties == { + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "locally_evaluated": False, + f"$feature/{FLAG}": "annual", + "$feature_flag_request_id": REQUEST_ID, + "$feature_flag_evaluated_at": EVALUATED_AT, + # The reason code replaces the record's reason description. + "$feature_flag_version": 3, + "$feature_flag_id": 77, + "$feature_flag_has_experiment": True, + **EXPECTED_ATTRIBUTION, + } + assert exposure.properties == EXPECTED_EXPOSURE + assert exposure.distinct_id == called.distinct_id == DISTINCT_ID + assert exposure.groups == called.groups == {"company": "company-7"} + assert exposure.property_allowlist is None + + +async def test_async_split_sends_both_events_with_exact_properties(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags( + DISTINCT_ID, groups={"company": 7}, disable_geoip=True + ) + snapshot.get_flag(FLAG) + + response = _response(_split(description="Assigned variant 'annual-first'")) + events = await _read_async_events([response], [read]) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + called, exposure = events + assert exposure.properties == EXPECTED_EXPOSURE + assert _attribution(called.properties) == EXPECTED_ATTRIBUTION + assert "$feature_flag_reason" not in called.properties + # The async client sends group keys as strings. + assert exposure.groups == called.groups == {"company": "7"} + assert exposure.distinct_id == called.distinct_id == DISTINCT_ID + + +def test_the_pair_shares_capture_identity_but_not_its_uuid(): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + try: + with _serving(_response(_split())): + snapshot = client.evaluate_flags( + DISTINCT_ID, groups={"company": "company-7"}, disable_geoip=True + ) + assert snapshot.get_flag(FLAG) == "annual" + finally: + client.shutdown() + + called, exposure = messages + assert (called["event"], exposure["event"]) == (FLAG_CALLED, EXPOSURE) + assert called["uuid"] and exposure["uuid"] and called["uuid"] != exposure["uuid"] + assert exposure["distinct_id"] == called["distinct_id"] == DISTINCT_ID + for name in ("$groups", "$lib", "$lib_version", "$geoip_disable"): + assert exposure["properties"][name] == called["properties"][name], name + assert exposure["properties"]["$groups"] == {"company": "company-7"} + + +@pytest.mark.parametrize("path", SNAPSHOT_READS) +def test_an_assignment_is_reported_whatever_the_read_returns(path): + # An experiment arm whose value is false still assigned the visitor to that arm. + response = _response(_split(value=False, variant_key="no-banner")) + + events = _read_events(path, response) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_response"] is False + assert exposure.properties["$feature_flag_variant"] == "no-banner" + + +# --- What sends no exposure ------------------------------------------------------------ + + +@pytest.mark.parametrize("path", SINGLE_FLAG_READS) +def test_opting_out_of_flag_events_suppresses_both(client, path): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + getattr(client, path)(FLAG, DISTINCT_ID, send_feature_flag_events=False) + + assert events == [] + + +def test_a_payload_read_sends_no_flag_events(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload(FLAG, DISTINCT_ID) + snapshot = client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag_payload(FLAG) + + assert events == [] + + +def test_a_payload_read_that_sends_flag_events_also_sends_the_exposure(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload( + FLAG, DISTINCT_ID, send_feature_flag_events=True + ) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +MATCH_VALUE_CASES = [ + ("another value", _split(value="annual"), "monthly"), + ("the record's own value", _split(value="annual"), "annual"), + # Object and number arms both read as True, so equal values prove nothing about + # where a value came from. + ("an object arm", _split(value={"layout": "grid"}), True), + ("a number arm", _split(value=12.5), True), +] + + +@pytest.mark.parametrize("later_read", list(SYNC_READS)) +@pytest.mark.parametrize( + "record, match_value", + [case[1:] for case in MATCH_VALUE_CASES], + ids=[case[0] for case in MATCH_VALUE_CASES], +) +def test_a_payload_lookup_for_a_match_value_sends_no_exposure( + client, record, match_value, later_read +): + # The lookup reports the caller's match_value, not the record's value, so it + # sends no exposure, whatever the value. A later read of the record does. + recording, events = _record_events(client) + with _serving(_response(record)), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload( + FLAG, DISTINCT_ID, match_value=match_value, send_feature_flag_events=True + ) + [called] = events + assert called.name == FLAG_CALLED + assert called.properties["$feature_flag_response"] == match_value + # The call still carries the record's attribution. + assert _attribution(called.properties) == EXPECTED_ATTRIBUTION + + _read(client, later_read) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_variant"] == "annual-first" + + +def test_a_stale_cached_value_sends_no_exposure(client): + # The record arrived, but caching it failed, so the read reports the stale cached + # value instead. That value did not come from the record, even though it is equal. + cache = mock.Mock() + cache._set_cached_remote_flag.side_effect = RuntimeError("cache unavailable") + cache.get_stale_cached_flag.return_value = FeatureFlagResult( + key=FLAG, enabled=True, variant="annual", payload=None, reason=None + ) + client.flag_cache = cache + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + result = client.get_feature_flag_result(FLAG, DISTINCT_ID) + + assert result is not None and result.get_value() == "annual" + cache.get_stale_cached_flag.assert_called_once() + assert [event.name for event in events] == [FLAG_CALLED] + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_locally_evaluated_flag_is_unchanged(path): + client = Client(FAKE_TEST_API_KEY, send=False, secret_key="secret-key") + client.feature_flags = [ + { + "id": 9, + "name": "Pricing page", + "key": FLAG, + "active": True, + "filters": { + "groups": [{"properties": [], "rollout_percentage": 100}], + "multivariate": { + "variants": [{"key": "annual", "rollout_percentage": 100}] + }, + }, + } + ] + recording, events = _record_events(client) + try: + with _serving(_response(_split())) as remote, recording: + _read(client, path) + finally: + client.shutdown() + + assert remote.call_count == 0 + [called] = events + assert called.name == FLAG_CALLED + assert called.properties["locally_evaluated"] is True + assert _attribution(called.properties) == {} + + +def test_an_empty_snapshot_sends_nothing(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + snapshot = client.evaluate_flags() + snapshot.get_flag(FLAG) + snapshot.is_enabled(FLAG) + + assert events == [] + + +def test_a_failed_exposure_capture_does_not_change_the_read(): + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + if message["event"] == EXPOSURE: + raise ValueError("rejected") + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + try: + with _serving(_response(_split())), warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + assert client.get_feature_flag(FLAG, DISTINCT_ID) == "annual" + assert client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) == "annual" + finally: + client.shutdown() + + +# --- Dedupe ------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + "first, second", + [ + ("get_feature_flag", "get_feature_flag"), + ("snapshot.get_flag", "snapshot.is_enabled"), + ("feature_enabled", "snapshot.get_flag"), + ("snapshot.get_flag", "get_feature_flag_result"), + ], +) +def test_a_repeated_read_sends_nothing_new(client, first, second): + # The second evaluation has a new request ID and time, which neither event's + # dedupe reads. + later = _response( + _split(), request_id=OTHER_REQUEST_ID, evaluated_at=EVALUATED_AT + 1 + ) + recording, events = _record_events(client) + with _serving(_response(_split()), later), recording: + _read(client, first) + _read(client, second) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_equal_valued_arms_are_each_reported(path): + # An A/A experiment: both arms return the same value. + events = _read_events( + path, + _response(_split(value=True, variant_key="control")), + _response(_split(value=True, variant_key="test")), + reads=2, + ) + + exposures = _named(events, EXPOSURE) + assert [e.properties["$feature_flag_variant"] for e in exposures] == [ + "control", + "test", + ] + assert len(_named(events, FLAG_CALLED)) == 2 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_later_experiment_on_the_flag_is_reported_again(path): + events = _read_events( + path, + _response(_split(rule_id=RULE)), + _response(_split(rule_id=OTHER_RULE)), + reads=2, + ) + + exposures = _named(events, EXPOSURE) + assert [e.properties["$feature_flag_rule_id"] for e in exposures] == [ + RULE, + OTHER_RULE, + ] + assert len(_named(events, FLAG_CALLED)) == 2 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_the_exposure_dedupe_leaves_out_the_value(path): + # The arm's value changed, but the visitor is still in the same arm. + events = _read_events( + path, + _response(_split(value="annual")), + _response(_split(value="annual-v2")), + reads=2, + ) + + assert len(_named(events, FLAG_CALLED)) == 2 + assert len(_named(events, EXPOSURE)) == 1 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_holdout_and_a_split_of_one_rule_are_both_reported(path): + holdout = _record( + "holdout", False, rule_type="experiment", rule_id=RULE, has_experiment=True + ) + events = _read_events( + path, _response(holdout), _response(_split(value=False)), reads=2 + ) + + called = _named(events, FLAG_CALLED) + assert [e.properties["$feature_flag_reason_code"] for e in called] == [ + "holdout", + "experiment_split", + ] + assert len(_named(events, EXPOSURE)) == 1 + + +def test_groups_are_part_of_the_exposure_identity(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "snapshot.get_flag", groups={"company": "a", "team": "b"}) + _read(client, "get_feature_flag", groups={"team": "b", "company": "a"}) + _read(client, "snapshot.get_flag", groups={"company": "c"}) + + exposures = _named(events, EXPOSURE) + assert [e.groups for e in exposures] == [ + {"company": "a", "team": "b"}, + {"company": "c"}, + ] + + +def test_the_exposure_dedupe_normalizes_the_distinct_id(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "get_feature_flag_result", distinct_id=42) + _read(client, "snapshot.get_flag", distinct_id="42") + + assert len(_named(events, EXPOSURE)) == 1 + + +def test_a_reported_call_never_suppresses_its_exposure(client): + # A read whose value the record did not supply reports the call without an + # exposure. A later read of the same record still sends the exposure, although + # its call is a duplicate. + with _serving(_response(_split())): + details = client.evaluate_flags(DISTINCT_ID)._flags[FLAG].details + recording, events = _record_events(client) + with recording: + for value_from_details in (False, True): + client._capture_feature_flag_called_if_needed( + distinct_id=DISTINCT_ID, + key=FLAG, + response="annual", + properties={"$feature_flag": FLAG, "$feature_flag_response": "annual"}, + details=details, + value_from_details=value_from_details, + ) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +async def test_async_a_reported_call_never_suppresses_its_exposure(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + details = snapshot._flags[FLAG].details + for value_from_details in (False, True): + client._capture_feature_flag_called_if_needed( + distinct_id=DISTINCT_ID, + key=FLAG, + response="annual", + properties={"$feature_flag": FLAG, "$feature_flag_response": "annual"}, + details=details, + value_from_details=value_from_details, + ) + + events = await _read_async_events([_response(_split())], [read]) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +async def test_async_snapshots_dedupe_both_events(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag(FLAG) + snapshot.is_enabled(FLAG) + snapshot.only([FLAG]).get_flag(FLAG) + + events = await _read_async_events( + [ + _response(_split(variant_key="control", value=True)), + _response(_split(variant_key="control", value=True)), + _response(_split(variant_key="test", value=True)), + ], + [read, read, read], + ) + + assert [e.properties["$feature_flag_variant"] for e in events] == [ + "control", + "control", + "test", + "test", + ] + assert [e.name for e in events] == [FLAG_CALLED, EXPOSURE] * 2 + + +def test_the_v1_dedupe_key_is_unchanged(client): + with _serving(_build(READERS, {"template": "v1"})): + _read(client, "get_feature_flag", "garden-layout") + + assert client.distinct_ids_feature_flags_reported[DISTINCT_ID] == { + ("garden-layout", "compact", ()) + } + assert len(client._experiment_exposures_reported) == 0 + + +def test_shutdown_clears_the_exposure_dedupe(): + client = Client(FAKE_TEST_API_KEY, send=False) + with _serving(_response(_split())): + _read(client, "snapshot.get_flag") + assert len(client._experiment_exposures_reported) == 1 + + client.shutdown() + + assert len(client._experiment_exposures_reported) == 0 + + +async def test_async_shutdown_clears_the_exposure_dedupe(): + with mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=_response(_split())), + ): + client = AsyncPosthog("project-key", send=False) + (await client.evaluate_flags(DISTINCT_ID)).get_flag(FLAG) + assert len(client._experiment_exposures_reported) == 1 + await client.shutdown() + + assert len(client._experiment_exposures_reported) == 0 + + +# --- Minimal events ------------------------------------------------------------------- + + +def test_minimal_flag_called_events_keep_the_attribution(): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client( + FAKE_TEST_API_KEY, + send=False, + before_send=before_send, + super_properties={"plan": "team"}, + ) + # A split of an experiment rule no experiment links to: the flag reports no + # experiment, so its $feature_flag_called is minimal. + response = _response(_split(has_experiment=False), minimalFlagCalledEvents=True) + try: + with _serving(response): + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + finally: + client.shutdown() + + called, exposure = messages + assert set(called["properties"]) <= _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + assert "plan" not in called["properties"] + assert _attribution(called["properties"]) == EXPECTED_ATTRIBUTION + # The exposure is a full event. + assert exposure["event"] == EXPOSURE + assert exposure["properties"]["plan"] == "team" + assert _attribution(exposure["properties"]) == _attribution(EXPECTED_EXPOSURE) diff --git a/posthog/test/test_flags_v4_response.py b/posthog/test/test_flags_v4_response.py index a0b3011a..fa228562 100644 --- a/posthog/test/test_flags_v4_response.py +++ b/posthog/test/test_flags_v4_response.py @@ -82,8 +82,8 @@ def _evaluate(client: Client, response: Dict[str, Any]) -> FeatureFlagEvaluation def test_vendored_fixtures_match_the_recorded_release_digests(): source = _load("SOURCE.json") - assert source["release"] == "1.22.0" - assert source["contract_version"] == "7.0.0" + assert source["release"] == "1.27.0" + assert source["contract_version"] == "8.0.0" for name, entry in source["files"].items(): digest = hashlib.sha256((FIXTURES / name).read_bytes()).hexdigest() assert digest == entry["sha256"], name @@ -875,11 +875,27 @@ def _parity_response(shape: str) -> Dict[str, Any]: } +# A config version 2 record adds these to its flag events, and leaves out +# $feature_flag_reason. A v2-shaped response has no config version 2 records, so they +# are left out of the comparison of the v2-* flags and checked in +# test_experiment_exposure.py. +_CONFIG_VERSION_2_EVENT_PROPERTIES = { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant", +} + + def _flag_event_properties(message: Dict[str, Any]) -> Dict[str, Any]: + left_out = set(_CONFIG_VERSION_2_EVENT_PROPERTIES) + if str(message["properties"].get("$feature_flag", "")).startswith("v2-"): + left_out.add("$feature_flag_reason") return { key: value for key, value in message["properties"].items() - if key.startswith("$feature") + if (key.startswith("$feature") and key not in left_out) or key in ("$active_feature_flags", "locally_evaluated") }