From 297c17a3c3c5f5fefcef3ee94b0492ac87dc12b5 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Wed, 3 Jun 2026 18:07:39 +0600 Subject: [PATCH 01/90] Dockerfile update: Uncommented the SKIP_NGINX and SKIP_CRON env --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19f6a8ed0e..713e654b6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -167,11 +167,11 @@ ENV ADCP_HOST=0.0.0.0 # core/main.py serves MCP, A2A, and the Flask admin from one Starlette # binary on $ADCP_PORT. The bundled nginx thread in run_all_services.py # is unused on this fork — kept off via SKIP_NGINX=true. -#ENV SKIP_NGINX=false +ENV SKIP_NGINX=false # Server-owned adapter schedulers replace the bundled supercronic inventory # sweep in the default container runtime. Operators can still opt back into # cron by overriding this, but should not run both mechanisms together. -#ENV SKIP_CRON=true +ENV SKIP_CRON=false # Expose the unified python port directly. Fly.io / upstream proxy # talks to this port; no in-image reverse proxy. From 5dd93f52e423fb33f92e1a7ce58ef45f9f81f07d Mon Sep 17 00:00:00 2001 From: chinmoy Date: Wed, 3 Jun 2026 20:10:55 +0600 Subject: [PATCH 02/90] fix healthcheck command --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 713e654b6b..1389553a77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -179,7 +179,8 @@ EXPOSE 8000 # Health check HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8080/health || exit 1 + CMD ["python", "scripts/healthcheck.py", "8000"] + # Use venv Python directly as entrypoint (prepares for hardened images that lack bash) ENTRYPOINT ["/app/.venv/bin/python", "scripts/deploy/run_all_services.py"] From c9852d5bb57a0c5fb23eb3201d5f81f38050aa28 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Thu, 4 Jun 2026 15:04:32 +0600 Subject: [PATCH 03/90] stopped overriding the ADCP_SALES_PORT to different port --- Dockerfile | 2 +- scripts/deploy/run_all_services.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1389553a77..2750790dea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -178,7 +178,7 @@ ENV SKIP_CRON=false EXPOSE 8000 # Health check -HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ +HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=5 \ CMD ["python", "scripts/healthcheck.py", "8000"] diff --git a/scripts/deploy/run_all_services.py b/scripts/deploy/run_all_services.py index e3a94656a6..022d17dcc9 100644 --- a/scripts/deploy/run_all_services.py +++ b/scripts/deploy/run_all_services.py @@ -225,9 +225,9 @@ def run_migrations(): def run_mcp_server(): """Run the MCP server.""" - print("Starting MCP server on port 8080...") + port = os.environ.get("ADCP_SALES_PORT", "8000") + print(f"Starting MCP server on port {port}...") env = os.environ.copy() - env["ADCP_SALES_PORT"] = "8080" proc = subprocess.Popen( [sys.executable, "scripts/run_server.py"], env=env, @@ -245,10 +245,9 @@ def run_mcp_server(): def exec_mcp_server(): """Replace this wrapper process with the unified MCP/A2A/Admin server.""" - print("Starting MCP server on port 8080...") - env = os.environ.copy() - env["ADCP_SALES_PORT"] = "8080" - os.execvpe(sys.executable, [sys.executable, "scripts/run_server.py"], env) + port = os.environ.get("ADCP_SALES_PORT", "8000") + print(f"Starting MCP server on port {port}...") + os.execvpe(sys.executable, [sys.executable, "scripts/run_server.py"], os.environ.copy()) def run_nginx(): From fc5d2717989a45550057c9116dd91bed3436ca44 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Thu, 4 Jun 2026 16:01:48 +0600 Subject: [PATCH 04/90] reduce cache size and pool size --- src/admin/app.py | 1 + src/core/database/database_session.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/admin/app.py b/src/admin/app.py index 1dc8a391d7..06634b6a9d 100644 --- a/src/admin/app.py +++ b/src/admin/app.py @@ -281,6 +281,7 @@ def __call__(self, environ, start_response): cache_config = { "CACHE_TYPE": "SimpleCache", # In-memory cache (good for single-process deployments) "CACHE_DEFAULT_TIMEOUT": 300, # 5 minutes default + "CACHE_THRESHOLD": 50, # Evict old entries before accumulating large Response objects } app.config.update(cache_config) cache = Cache(app) diff --git a/src/core/database/database_session.py b/src/core/database/database_session.py index 9c47dab5a0..ca835ddaa8 100644 --- a/src/core/database/database_session.py +++ b/src/core/database/database_session.py @@ -144,8 +144,8 @@ def get_engine(): # Direct PostgreSQL settings (no PgBouncer) _engine = create_engine( connection_string, - pool_size=10, # Base connections in pool - max_overflow=20, # Additional connections beyond pool_size + pool_size=5, # Base connections in pool + max_overflow=5, # Additional connections beyond pool_size pool_timeout=pool_timeout, # Seconds to wait for connection from pool pool_recycle=3600, # Recycle connections after 1 hour pool_pre_ping=True, # Test connections before use From 4db5916fd361702fd60278e95aec93c9d45022ed Mon Sep 17 00:00:00 2001 From: chinmoy Date: Thu, 4 Jun 2026 16:06:42 +0600 Subject: [PATCH 05/90] hardcoded set SKIP_NGINX to True --- scripts/deploy/run_all_services.py | 3 ++- src/admin/blueprints/auth.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/deploy/run_all_services.py b/scripts/deploy/run_all_services.py index 022d17dcc9..eca5a684d6 100644 --- a/scripts/deploy/run_all_services.py +++ b/scripts/deploy/run_all_services.py @@ -375,7 +375,8 @@ def main(): threads = [] skip_cron = os.environ.get("SKIP_CRON", "false").lower() == "true" - skip_nginx = os.environ.get("SKIP_NGINX", "false").lower() == "true" + # TODO: remove hardcoded skip_nginx value + skip_nginx = True #os.environ.get("SKIP_NGINX", "false").lower() == "true" if skip_cron and skip_nginx: # In the single-process runtime used by e2e and most deployments, run # the ASGI server as PID 1 after migrations/init. Keeping a Python diff --git a/src/admin/blueprints/auth.py b/src/admin/blueprints/auth.py index 8cddd36d05..baaa401aae 100644 --- a/src/admin/blueprints/auth.py +++ b/src/admin/blueprints/auth.py @@ -402,7 +402,8 @@ def google_auth(): # Only add /admin prefix in production mode with nginx (not in Docker standalone) # SKIP_NGINX=true indicates Docker standalone mode without nginx reverse proxy - skip_nginx = os.environ.get("SKIP_NGINX", "").lower() == "true" + # TODO: remove hardcoded skip_nginx value + skip_nginx = True # os.environ.get("SKIP_NGINX", "false").lower() == "true" production = os.environ.get("PRODUCTION", "").lower() == "true" if not skip_nginx and production and "/admin/" not in base_url: From 9328d3653b7a53cfbe2aee81cb3abf886d6a3d30 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Thu, 4 Jun 2026 16:31:38 +0600 Subject: [PATCH 06/90] Change ADCP_PORT to 8000 in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2750790dea..e8e279550e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -161,7 +161,7 @@ ENV PYTHONPATH="/app" ENV PYTHONUNBUFFERED=1 # Default port -ENV ADCP_PORT=8080 +ENV ADCP_PORT=8000 ENV ADCP_HOST=0.0.0.0 # core/main.py serves MCP, A2A, and the Flask admin from one Starlette From 156171f799c5877c72100dc7c7461db4d453a1c9 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Thu, 4 Jun 2026 16:52:08 +0600 Subject: [PATCH 07/90] reverted pool size and cache size --- src/admin/app.py | 2 +- src/core/database/database_session.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/admin/app.py b/src/admin/app.py index 06634b6a9d..a3c5df1bfa 100644 --- a/src/admin/app.py +++ b/src/admin/app.py @@ -281,7 +281,7 @@ def __call__(self, environ, start_response): cache_config = { "CACHE_TYPE": "SimpleCache", # In-memory cache (good for single-process deployments) "CACHE_DEFAULT_TIMEOUT": 300, # 5 minutes default - "CACHE_THRESHOLD": 50, # Evict old entries before accumulating large Response objects + # "CACHE_THRESHOLD": 50, # Evict old entries before accumulating large Response objects } app.config.update(cache_config) cache = Cache(app) diff --git a/src/core/database/database_session.py b/src/core/database/database_session.py index ca835ddaa8..9c47dab5a0 100644 --- a/src/core/database/database_session.py +++ b/src/core/database/database_session.py @@ -144,8 +144,8 @@ def get_engine(): # Direct PostgreSQL settings (no PgBouncer) _engine = create_engine( connection_string, - pool_size=5, # Base connections in pool - max_overflow=5, # Additional connections beyond pool_size + pool_size=10, # Base connections in pool + max_overflow=20, # Additional connections beyond pool_size pool_timeout=pool_timeout, # Seconds to wait for connection from pool pool_recycle=3600, # Recycle connections after 1 hour pool_pre_ping=True, # Test connections before use From c6d89f06bf5712f9b3d83936ba470767a8ce28f0 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Thu, 4 Jun 2026 18:44:53 +0600 Subject: [PATCH 08/90] Removed Hardcoded SKIP_NGINX value --- scripts/deploy/run_all_services.py | 3 +-- src/admin/blueprints/auth.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/deploy/run_all_services.py b/scripts/deploy/run_all_services.py index eca5a684d6..022d17dcc9 100644 --- a/scripts/deploy/run_all_services.py +++ b/scripts/deploy/run_all_services.py @@ -375,8 +375,7 @@ def main(): threads = [] skip_cron = os.environ.get("SKIP_CRON", "false").lower() == "true" - # TODO: remove hardcoded skip_nginx value - skip_nginx = True #os.environ.get("SKIP_NGINX", "false").lower() == "true" + skip_nginx = os.environ.get("SKIP_NGINX", "false").lower() == "true" if skip_cron and skip_nginx: # In the single-process runtime used by e2e and most deployments, run # the ASGI server as PID 1 after migrations/init. Keeping a Python diff --git a/src/admin/blueprints/auth.py b/src/admin/blueprints/auth.py index baaa401aae..e98a72f529 100644 --- a/src/admin/blueprints/auth.py +++ b/src/admin/blueprints/auth.py @@ -402,8 +402,7 @@ def google_auth(): # Only add /admin prefix in production mode with nginx (not in Docker standalone) # SKIP_NGINX=true indicates Docker standalone mode without nginx reverse proxy - # TODO: remove hardcoded skip_nginx value - skip_nginx = True # os.environ.get("SKIP_NGINX", "false").lower() == "true" + skip_nginx = os.environ.get("SKIP_NGINX", "false").lower() == "true" production = os.environ.get("PRODUCTION", "").lower() == "true" if not skip_nginx and production and "/admin/" not in base_url: From 11d67e49d9322340d6cd96c7e837e8364f80ddc3 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Thu, 4 Jun 2026 22:13:44 +0600 Subject: [PATCH 09/90] skip_cron hardcoded value set to false --- Dockerfile | 2 +- scripts/deploy/run_all_services.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e8e279550e..8795ecaf9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -171,7 +171,7 @@ ENV SKIP_NGINX=false # Server-owned adapter schedulers replace the bundled supercronic inventory # sweep in the default container runtime. Operators can still opt back into # cron by overriding this, but should not run both mechanisms together. -ENV SKIP_CRON=false +ENV SKIP_CRON=True # Expose the unified python port directly. Fly.io / upstream proxy # talks to this port; no in-image reverse proxy. diff --git a/scripts/deploy/run_all_services.py b/scripts/deploy/run_all_services.py index 022d17dcc9..4119a79028 100644 --- a/scripts/deploy/run_all_services.py +++ b/scripts/deploy/run_all_services.py @@ -374,7 +374,10 @@ def main(): # Start services in threads threads = [] - skip_cron = os.environ.get("SKIP_CRON", "false").lower() == "true" + #skip_cron = os.environ.get("SKIP_CRON", "false").lower() == "true" + + #TODO: Remove skip_cron hardcoded value + skip_cron = False skip_nginx = os.environ.get("SKIP_NGINX", "false").lower() == "true" if skip_cron and skip_nginx: # In the single-process runtime used by e2e and most deployments, run From 4d07b83e8f8dbe717a0408c5237dc327aba50afa Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Fri, 5 Jun 2026 12:33:27 +0600 Subject: [PATCH 10/90] update healthcheck retry --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 431e1eaf3b..9523fffa10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -178,8 +178,8 @@ ENV SKIP_CRON=false EXPOSE 8000 # Health check -HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8080/health || exit 1 +HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \ + CMD ["python", "scripts/healthcheck.py", "8000"] # Use venv Python directly as entrypoint (prepares for hardened images that lack bash) ENTRYPOINT ["/app/.venv/bin/python", "scripts/deploy/run_all_services.py"] From d093bcc3a91d99c70426779d8bf27b6c19d1cee7 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Fri, 5 Jun 2026 16:32:35 +0600 Subject: [PATCH 11/90] gam create service account access set to the api_mode = true --- src/admin/blueprints/gam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/blueprints/gam.py b/src/admin/blueprints/gam.py index 70b8d9ac65..4d9c611199 100644 --- a/src/admin/blueprints/gam.py +++ b/src/admin/blueprints/gam.py @@ -752,7 +752,7 @@ def reset_stuck_sync(tenant_id): @gam_bp.route("/create-service-account", methods=["POST"]) @log_admin_action("create_gam_service_account") -@require_tenant_access(role=("admin",)) +@require_tenant_access(api_mode=True, role=("admin",)) def create_service_account(tenant_id): """Create a GCP service account for GAM integration. From da26784bd9eefcc54dfa3da117fe9e9af9d1b842 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Mon, 8 Jun 2026 17:32:16 +0600 Subject: [PATCH 12/90] added gam sync button --- src/admin/blueprints/buyer_routing.py | 47 ++++++++++++++++++ templates/buyer_routing.html | 71 ++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/src/admin/blueprints/buyer_routing.py b/src/admin/blueprints/buyer_routing.py index ae50ac7cc0..8867457854 100644 --- a/src/admin/blueprints/buyer_routing.py +++ b/src/admin/blueprints/buyer_routing.py @@ -18,6 +18,7 @@ from __future__ import annotations import logging +import threading import uuid from datetime import UTC, datetime @@ -43,10 +44,12 @@ AdvertiserRoutingRule, GamAdvertiser, Principal, + SyncJob, Tenant, ) from src.core.database.repositories.gam_sync import GAMSyncRepository from src.core.database.repositories.tenant_config import TenantConfigRepository +from src.services.gam_advertisers_sync import sync_advertisers from src.services.recent_buyers_service import compute_recent_buyers logger = logging.getLogger(__name__) @@ -627,3 +630,47 @@ def list_principals(tenant_id: str): rows = tenant_repo.list_principals() principals = [{"principal_id": p.principal_id, "name": p.name} for p in rows] return jsonify({"principals": principals}) + + +@buyer_routing_bp.route( + "//buyer-routing/api/sync-advertisers", + methods=["POST"], + strict_slashes=False, +) +@require_tenant_access(api_mode=True, role=("admin", "member")) +def trigger_advertiser_sync(tenant_id: str): + """Trigger a background GAM advertisers sync from the buyer-routing page. + + Creates a pending SyncJob, spawns a daemon thread, and returns the + sync_id so the client can poll + ``GET /tenant//gam/sync-status/`` for progress. + """ + started_at = datetime.now(UTC) + sync_id = f"sync_{tenant_id}_advertisers_{int(started_at.timestamp() * 1_000_000)}" + + with get_db_session() as session: + tenant = session.scalars(select(Tenant).filter_by(tenant_id=tenant_id)).first() + if tenant is None: + return _api_error_json("tenant_not_found", f"Tenant {tenant_id!r} does not exist", 404) + + job = SyncJob( + sync_id=sync_id, + tenant_id=tenant_id, + adapter_type="google_ad_manager", + sync_type="advertisers", + status="pending", + started_at=started_at, + triggered_by="admin_ui", + triggered_by_id="sync_advertisers_button", + ) + session.add(job) + session.commit() + + def _run() -> None: + try: + sync_advertisers(tenant_id, sync_id=sync_id) + except Exception: + logger.exception("[%s] advertiser sync thread failed", sync_id) + + threading.Thread(target=_run, daemon=True, name=f"adv-sync-{sync_id}").start() + return jsonify({"sync_id": sync_id}) diff --git a/templates/buyer_routing.html b/templates/buyer_routing.html index 5de37a67d9..acd223bcf2 100644 --- a/templates/buyer_routing.html +++ b/templates/buyer_routing.html @@ -414,7 +414,11 @@

Routing rules

-

Advertisers

+
+

Advertisers

+ +
+

Every GAM advertiser synced from your network. Assign a buyer agent to surface that advertiser's orders and line items in the agent's @@ -968,6 +972,71 @@

Add routing rule

}); }); + // ---------- Sync advertisers ---------- + const syncAdvBtn = document.querySelector('[data-action="sync-advertisers"]'); + const syncAdvStatus = document.querySelector('[data-sync-advertisers-status]'); + const gamStatusBase = scriptRoot + '/tenant/' + encodeURIComponent(tenantId) + '/gam/sync-status/'; + + function pollAdvertiserSyncStatus(syncId) { + var intervalId = setInterval(function () { + fetch(gamStatusBase + encodeURIComponent(syncId), { + credentials: 'same-origin', + headers: { 'Accept': 'application/json' }, + }) + .then(function (r) { return r.json(); }) + .then(function (body) { + if (body.status === 'completed') { + clearInterval(intervalId); + var s = body.summary || {}; + var msg = 'Sync complete — ' + (s.upserted || 0) + ' upserted, ' + (s.soft_deleted || 0) + ' inactive.'; + showToast(msg); + window.location.reload(); + } else if (body.status === 'failed') { + clearInterval(intervalId); + syncAdvStatus.textContent = 'Sync failed: ' + (body.error || 'unknown error'); + syncAdvStatus.style.color = 'var(--sa-danger)'; + syncAdvBtn.disabled = false; + syncAdvBtn.textContent = '↺ Sync from GAM'; + } + }) + .catch(function () { /* transient — keep polling */ }); + }, 2000); + } + + if (syncAdvBtn) { + syncAdvBtn.addEventListener('click', function () { + syncAdvBtn.disabled = true; + syncAdvBtn.textContent = '↺ Syncing…'; + syncAdvStatus.style.display = 'block'; + syncAdvStatus.style.color = 'var(--sa-text-muted)'; + syncAdvStatus.textContent = 'Starting sync…'; + + fetch(baseUrl + '/api/sync-advertisers', { + method: 'POST', + credentials: 'same-origin', + headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' }, + }) + .then(function (r) { return r.json().then(function (b) { return { ok: r.ok, body: b }; }); }) + .then(function (resp) { + if (!resp.ok) { + syncAdvStatus.textContent = resp.body.message || 'Failed to start sync.'; + syncAdvStatus.style.color = 'var(--sa-danger)'; + syncAdvBtn.disabled = false; + syncAdvBtn.textContent = '↺ Sync from GAM'; + return; + } + syncAdvStatus.textContent = 'Sync in progress…'; + pollAdvertiserSyncStatus(resp.body.sync_id); + }) + .catch(function () { + syncAdvStatus.textContent = 'Network error — try again.'; + syncAdvStatus.style.color = 'var(--sa-danger)'; + syncAdvBtn.disabled = false; + syncAdvBtn.textContent = '↺ Sync from GAM'; + }); + }); + } + // Advertiser → agent assignment selects document.querySelectorAll('[data-action="assign-agent"]').forEach(function (select) { select.addEventListener('change', async function () { From 66c42e1e4d975b5ee9ba8e9d7a10f6ccd747f8e8 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 9 Jun 2026 14:46:39 +0600 Subject: [PATCH 13/90] fix saveGAMConfig() issue now auto-create the row when it's missing. --- src/services/gcp_service_account_service.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/services/gcp_service_account_service.py b/src/services/gcp_service_account_service.py index 64187cbc73..97c2c2e903 100644 --- a/src/services/gcp_service_account_service.py +++ b/src/services/gcp_service_account_service.py @@ -57,7 +57,7 @@ from sqlalchemy import select from src.core.database.database_session import get_db_session -from src.core.database.models import AdapterConfig +from src.core.database.models import AdapterConfig, Tenant logger = logging.getLogger(__name__) @@ -141,12 +141,19 @@ def create_service_account_for_tenant(self, tenant_id: str, display_name: str | Exception: If service account creation fails """ with get_db_session() as session: - # Get adapter config + # Verify tenant exists before touching adapter config + tenant = session.scalars(select(Tenant).filter_by(tenant_id=tenant_id)).first() + if not tenant: + raise ValueError(f"Tenant {tenant_id} not found") + + # Get or create adapter config — service account creation is the first step + # of GAM setup, so the row may not exist yet. stmt = select(AdapterConfig).filter_by(tenant_id=tenant_id) adapter_config = session.scalars(stmt).first() if not adapter_config: - raise ValueError(f"Tenant {tenant_id} not found or has no adapter config") + adapter_config = AdapterConfig(tenant_id=tenant_id, adapter_type="google_ad_manager") + session.add(adapter_config) # Check if service account already exists if adapter_config.gam_service_account_email: From c1e0da357efb5fd685487cada2ee1f2efc0fa38d Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 9 Jun 2026 15:58:35 +0600 Subject: [PATCH 14/90] gam creating service account error message improvment --- src/admin/blueprints/gam.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/admin/blueprints/gam.py b/src/admin/blueprints/gam.py index 4d9c611199..88cb63c409 100644 --- a/src/admin/blueprints/gam.py +++ b/src/admin/blueprints/gam.py @@ -799,7 +799,22 @@ def create_service_account(tenant_id): except Exception as e: logger.error(f"Error creating service account for tenant {tenant_id}: {e}", exc_info=True) - return jsonify({"success": False, "error": f"Failed to create service account: {str(e)}"}), 500 + error_str = str(e) + if "SERVICE_DISABLED" in error_str and "iam.googleapis.com" in error_str: + import re + + match = re.search(r"activationUrl[^\"]*\"([^\"]+)\"", error_str) + activation_url = match.group(1) if match else "https://console.developers.google.com/apis/api/iam.googleapis.com/overview" + return jsonify( + { + "success": False, + "error": ( + "The IAM API is not enabled in the configured GCP project. " + f"Enable it at: {activation_url} — then wait ~2 minutes and retry." + ), + } + ), 500 + return jsonify({"success": False, "error": f"Failed to create service account: {error_str}"}), 500 except Exception as e: logger.error(f"Error in create_service_account endpoint: {e}", exc_info=True) From 230b69f322a2e9d0d9323ee0b09f4e5bd926e690 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Fri, 12 Jun 2026 12:55:11 +0600 Subject: [PATCH 15/90] Sync master brach from bokelley fork (#15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(#365): allow AI/Logfire test-connection probes on embedded tenants (#369) The "Test Connection" action for AI providers and Logfire on the Integrations tab failed with "Failed: embedded_writes_not_permitted" on embedded tenants, because the verb-based embedded-write gate classifies any POST under `require_tenant_access` as a mutation. `test_ai_connection` and `test_logfire_connection` are read-only probes — they validate credentials against the upstream provider and never write tenant state. Opt them into `allow_embedded_writes=True`; the model-layer guard in `embedded_tenant_guard.py` remains in force as defense-in-depth. Also fix the test-result handlers in `templates/tenant_settings.html` to render `data.message || data.error` instead of `data.error` alone. Gate envelopes (and any future role-gate rejections) return both a stable code in `error` and a human-readable string in `message`; the old code surfaced the stable code, which read as gibberish to users. Sweep finding (left as follow-up): the same verb-based-gate trap exists on `tenants.test_slack`, `adapters.test_freewheel_connection`, `adapters.test_triton_connection`, `adapters.test_broadstreet_connection`, and `settings.test_domain_access`. Each is a read-only probe that could opt in with the same flag. Co-authored-by: Claude Opus 4.7 (1M context) * fix(#363): unblock Policies & Workflows writes on embedded tenants (#370) On embedded tenants every field in the Policies & Workflows tab (Brand Manifest Policy, Naming Conventions, Approval Workflows, Measurement Providers, Product Ranking, Auto-approval thresholds) silently reverted on save. Two compounding bugs: 1. Route blocked at the boundary. `/settings/business-rules` POST used `@require_tenant_access(role=("admin",))` without `allow_embedded_writes=True`, so the verb-based gate returned 403 `embedded_writes_not_permitted` before the handler ran. 2. JS treated the 403 HTML error page as success. `saveBusinessRules` in `tenant_settings.js` content-type-branched: any HTML response with no `.flash-messages` container fell through to `window.location.reload()`. Flask's default 403 error page has no flash messages → reload-as-success → user sees their fields revert with no error. Affected every 4xx/5xx on that route. Fix three layers: - Add `allow_embedded_writes=True` to `update_business_rules`. Per Sprint 5 design (`docs/design/embedded-mode-sprint-5.md` §"Pattern: shared business logic with the UI"), business rules are publisher-managed and edited via the proxied admin UI; the management API exposes the same writes. - Add the per-column business-rules surface to `PUBLISHER_WRITABLE_FIELDS[Tenant]` (13 fields covering naming templates, approval mode, creative review settings, AI policy, advertising policy, brand manifest policy, product ranking prompt, human review flag). Platform-identity columns (name, billing_plan, is_active, subdomain, external_*) stay locked. - Add `gam_manual_approval_required` / `mock_manual_approval_required` to `PUBLISHER_WRITABLE_FIELDS[AdapterConfig]` — these mirror `tenant.human_review_required` onto adapter config and are written by the same handler. - Restructure `saveBusinessRules` to check `response.ok` BEFORE content-type branching. Non-2xx responses now surface the error (parsing flash messages from HTML when available, falling back to the status code) instead of silently reloading. Added four guard tests in `test_managed_tenant_api.py::TestWriteGuard`: business-rules columns write, manual-approval adapter columns write, platform-identity columns stay blocked, and an end-to-end check via the mock adapter sync field. Co-authored-by: Claude Opus 4.7 (1M context) * fix(#364): explain empty Allowed Principals dropdown on embedded tenants (#371) * fix(#364): explain empty Allowed Principals dropdown on embedded tenants On embedded tenants the "Allowed Principals (Advertisers)" multi-select on Create Product rendered only "No principals configured" — a dead end. The Buyer Agents section in Settings hides the "Add Buyer Agent" button on embedded tenants (this is correct: Principal provisioning is platform-managed via the Tenant Management API), so publishers had no path to populate the dropdown. Two compounding things made the UI misleading: 1. The empty-state placeholder didn't distinguish embedded from open instances. Publishers saw the same "No principals configured" text that suggests they can fix it themselves. 2. Comments in `tenant_settings.html` and `buyer_advertiser_routing.py` claimed Principals are "auto-created on first request by the embedded-mode auth bypass, which reads X-Identity-Buyer-Principal-Id". That mechanism does not exist — grep `src/` for the header returns zero matches. Anyone tracing the empty dropdown ran into a dead-end comment that confidently pointed at a code path that isn't there. Fix: - In `add_product.html` and `add_product_gam.html`, replace the disabled `` with a context-aware empty state. Embedded tenants get an explainer that Principals are provisioned by the platform via the Tenant Management API; open instances get a pointer to Settings → Buyer Agents. - Rewrite the misleading comment block in `tenant_settings.html` around the advertisers section and the user-visible "auto-created from request headers" line — state plainly that embedded Principal provisioning goes through the platform API. - Fix the matching dead-pointer comment in `buyer_advertiser_routing.py` near the access-grant logic. Option B (platform-managed) per `docs/design/embedded-mode-sprint-5.md` contract. Option A (re-enable UI authoring) would have been a write-guard expansion that contradicts the existing `{% if not embedded_view %}` gate on "Add Buyer Agent" — and the model guard doesn't list Principal at all, so it's the UI gate alone holding the line. Not the right place to flip the contract. Terminology cleanup ("Allowed Principals" vs "Buyer Agents" vs "Advertisers") is deliberately left for a follow-up issue — that's a larger UX project than a bug fix. Co-Authored-By: Claude Opus 4.7 (1M context) * test(#364): update assertions to match corrected embedded-mode copy The original test asserted on the misleading "auto-created from request headers" copy that #364 removed (because the auto-create mechanism does not exist — see #364 PR description). Update the assertions to match the new, accurate copy that explains platform-API provisioning. Also refresh the class docstring to drop the same misleading claim about header-based auto-creation. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(#374): coerce update_media_buy status to wire enum at response boundary (#375) The manual-approval path on ``update_media_buy`` read ``MediaBuy.status`` straight from the DB column and surfaced it on the ``UpdateMediaBuySuccess`` response. The persisted column accepts a broader set than the AdCP wire enum — ``draft`` (model default) and ``pending_approval`` (manual-approval create path) are both valid in storage but not in ``MediaBuyStatus``. fastmcp's request-/response-side Pydantic validation rejected the response with ``INVALID_REQUEST[status]: Input should be 'pending_creatives', 'pending_start', 'active', 'paused', 'completed', 'rejected' or 'canceled'``, which surfaced as an E2E failure on ``test_complete_campaign_lifecycle_with_webhooks`` (#374) and on every PR's CI run after the manual-approval status-emission was added in #353. Fix: - Add ``_to_wire_status`` in ``media_buy_list.py``. Takes any input (``str | MediaBuyStatus | None``) and returns either a wire-valid string from the seven-member enum, or ``None`` for values the wire rejects. Case-insensitive on string input. - Apply it at the manual-approval response site in ``update_media_buy.py``. ``current_status`` is now guaranteed wire-valid (or ``None``) before reaching ``UpdateMediaBuySuccess``. The other three response-status sites (cancel, pause/resume, final ``_compute_status`` path) already emit values from the wire enum by construction. Tests: - ``TestToWireStatus`` (6 cases): wire-valid passthrough, case insensitivity, persisted-only rejection (``draft``, ``pending_approval``), ``None``/empty/non-string handling. - ``test_manual_approval_response_coerces_non_wire_db_status_to_none``: end-to-end behavior — a persisted ``pending_approval`` does not leak to the response. - ``test_manual_approval_response_preserves_wire_valid_db_status``: wire-valid statuses still pass through unchanged. Verified locally: - Failing E2E ``test_complete_campaign_lifecycle_with_webhooks`` passes against the full Docker stack. - ``tox -e unit`` (4314 tests) and ``tox -e integration`` (1030 update_media_buy-adjacent tests) both green. Fixes #374. Co-authored-by: Claude Opus 4.7 (1M context) * fix(transport): env-gated stateless MCP mode for multi-replica deploys (#376) The MCP Python SDK's ``StreamableHTTPSessionManager`` stores ``_server_instances`` as a process-local dict. Multi-replica deployments without sticky LB routing on ``Mcp-Session-Id`` see ``tools/list`` and ``tools/call`` randomly 404 with "Session not found" when a request lands on a replica that didn't handle ``initialize``. A 10-attempt probe against the Wonderstruck deployment confirmed the dice roll: ``initialize`` always 200 (creates session on whichever replica answers); ``tools/list`` and ``tools/call`` with the same session ID succeeded only when they happened to land on the same replica (~50/50 each). Yesterday's compliance baseline (170 steps, 12 tools discovered) caught the deployment during a single-replica window; today the same baseline rerun returned 0 tools because ``discoverAgentProfile`` calls ``initialize`` → ``tools/list`` in tight succession, and ``tools/list`` lost the affinity coin flip half the time. ``serve()`` has supported ``stateless_http: bool`` since adcp 5.0 (``adcp/server/serve.py:2053`` sets ``mcp.settings.stateless_http`` from the kwarg unconditionally, so ``FASTMCP_STATELESS_HTTP`` env alone has no effect — the kwarg overrides FastMCP's reader). This plumbs the kwarg through ``_serve_kwargs`` gated on ``ADCP_STATELESS_HTTP``: * Unset / falsy → stateful (default). Single-replica prod, local dev, in-process tests, and the compliance-runner storyboard sweep keep the session-reuse perf optimization. * ``ADCP_STATELESS_HTTP=true`` → stateless. Each request creates a fresh transport context; multi-replica works without sticky LB. Per the FastMCP deployment doc (https://gofastmcp.com/v2/deployment/http): stateless mode is the recommended pattern for horizontal scaling — cookie-based stickiness is unreliable because most MCP clients use ``fetch()`` and drop ``Set-Cookie``. Header-based stickiness on ``Mcp-Session-Id`` would also work (the AdCP SDK forwards the header cleanly) and would keep session-reuse perf on prod compliance runs; this env var doesn't preclude that — the deployment chooses by setting / unsetting ``ADCP_STATELESS_HTTP``. Tests verify the env var maps to the kwarg correctly across true / false / unset and case variants. Existing ``test_serve_kwargs_middleware_order.py`` extended with the new ``stateless_http``-focused cases. Co-authored-by: Claude Opus 4.7 (1M context) * chore(deps): bump adcp 5.2.0 → 5.3.0 (#379) Picks up: - SellerA2AClient for in-process A2A handler testing (#694) - PgBuyerAgentRegistry.with_caching() factory (#692) - v3 storyboard CI gate that actually asserts (#693) - Sequence[T] widening on response-only list fields (#635) - Composed lifespan preservation when public_url is callable (#680) - ads.txt MANAGERDOMAIN fallback discovery (#704/#705) - validate_adagents_structure helper (#708) - webhook_signing.supported boot validator (#695) Audited the codebase for workarounds the bump should now obsolete. One real candidate: AgentCardPublicUrlMiddleware (190 LOC) — #680 means transport="both" + callable public_url now works. Replacing it with a public_url=resolver callable will land separately. Two workarounds the bump can't eliminate, filed upstream: - serve(lifespan=) hook missing — adcp-client-python#709 - cross-class entity overrides still need type:ignore[assignment] — adcp-client-python#710 Co-authored-by: Claude Opus 4.7 (1M context) * fix(#377): four-state aao_status_kind + permissive unbound resolution (#380) Wonderstruck-class publishers ship bare ``authorized_agents`` entries (``{url, authorized_for}`` only, no ``authorization_type``) alongside a top-level ``properties[]`` block. The AdCP SDK's strict resolver returns ``[]`` for these, so: - Publisher Partnerships chip rendered "Pending 0/0" — misleading operators into thinking the publisher hadn't authorized us yet. - Products UI used to bind anyway via a homegrown heuristic, then a prior pass tightened it to match the SDK — regressing Wonderstruck. This change introduces a four-state ``PublisherPartnerStatusKind`` (``authorized`` | ``unbound`` | ``pending`` | ``no_properties`` | ``unreachable``) and an explicit permissive resolution path: - ``aao_lookup_service.get_publisher_partner_status`` uses the SDK strictly first; falls back to ``unbound`` only when our entry is bare and the file has top-level properties. Surfaces a conformance hint so operators can nudge the publisher to add a typed binding. - ``property_discovery_service._extract_properties`` mirrors the same classification and, on the unbound branch, gates top-level properties to those carrying a ``type=domain`` identifier matching the publisher_domain — closes the attack vector where a publisher could bare-list us + claim arbitrary app/podcast/DOOH bundle IDs. - Shared shape helpers in ``src/services/_adagents_shapes.py`` (``is_bare_entry``, ``find_agent_entry``, ``top_level_properties``) cover the full schema selector set including ``signal_ids`` / ``signal_tags``. - New nullable ``aao_status_kind`` column on ``publisher_partners`` — legacy NULL rows fall back to the existing derivation in ``_partner_to_dict`` so the rollout is safe under rolling deploys. - JS chip styles for ``unbound`` ("Authorized (non-conformant file)") and ``no_properties`` ("No properties listed"). Upstream issues filed in parallel for ecosystem alignment: - adcontextprotocol/adcp#4478 — typed ``authorization_type: "all_top_level_properties"`` variant so publishers have a spec-conformant shape; once shipped we can deprecate the local permissive shim. - adcontextprotocol/adcp-client-python#711 — permissive resolver API. - adcontextprotocol/adcp-client#1721 — TS SDK per-agent resolution + permissive mode for cross-SDK consistency. Fixes #377 Co-authored-by: Claude Opus 4.7 (1M context) * fix(compliance): residual fixes from 7.1.0 probe — INVALID_REQUEST, INVALID_STATE, WWW-Authenticate (#383) * fix(compliance): residual fixes from 7.1.0 probe — INVALID_REQUEST, INVALID_STATE, WWW-Authenticate Closes three residual storyboard failures observed in the 7.1.0 comply() re-probe against Wonderstruck after #348/#349 fixes deployed: 1. **error_compliance/nonexistent_product** — pre-dispatch validation in ``_create_media_buy_impl`` raised ``ValueError`` (past start_time, reversed dates, etc.) and the outer ``except (ValueError, PermissionError)`` handler emitted ``Error(code="VALIDATION_ERROR")``. ``VALIDATION_ERROR`` is not in the AdCP 3.0 ``STANDARD_ERROR_CODES`` enum, so buyer agents walking the enum for self-correction silently drop the error. Change wire code to spec-canonical ``INVALID_REQUEST``. Storyboard expects ``PRODUCT_NOT_FOUND``, ``PRODUCT_UNAVAILABLE``, or ``INVALID_REQUEST``; sibling ``reversed_dates_error`` accepts ``VALIDATION_ERROR`` or ``INVALID_REQUEST``. ``INVALID_REQUEST`` is the only value in both sets and is the spec-canonical choice. 2. **media_buy_state_machine/pause_canceled_buy** — ``_update_media_buy_impl`` had a terminal-state guard on cancel (re-cancel raises ``AdCPNotCancellableError``) but the pause/resume branch dispatched straight to the adapter. Spec requires rejection with ``/adcp_error/code == "INVALID_STATE"`` for pause-of-canceled. New exception ``AdCPInvalidStateError`` (``error_code="INVALID_STATE"``, recovery ``correctable``, 422) covers the symmetric guard. Fires BEFORE adapter dispatch on both terminal states (``canceled``, ``completed``) for both actions (``paused=True``, ``paused=False``). Idempotency-spec friendly: same payload yields the same wire code on retry regardless of which adapter would have handled the transition. 3. **security_baseline/probe_unauth** — RFC 6750 §3 requires a ``WWW-Authenticate: Bearer`` header on every 401 from a Bearer-protected resource. Upstream ``adcp.server.auth.BearerTokenAuthMiddleware`` on the MCP leg returns 401 without the header for missing/invalid tokens; the A2A leg and ``SigningVerifyMiddleware`` already emit it correctly. New ``WWWAuthenticateMiddleware`` (in ``core/middleware/``) wraps the ASGI ``send`` callable and injects the bare ``Bearer`` challenge on 401 responses missing the header. Case-insensitive presence check so stacking is safe; no-op on 2xx / 3xx / 4xx-other / 5xx so a 403 doesn't confuse buyers about which auth scheme to apply. Registered AFTER ``AdminWSGIMount`` so Google-OAuth-gated admin paths short-circuit before the buyer-protocol challenge sees them. Bundled together because they ship in a single redeploy cycle and the PR-title-check enforces one Conventional Commit prefix per PR; the three fixes are independent at the code level (different files, different behavioural surfaces, different tests). ## Residuals still open (not in this PR) - ``pagination_integrity_list_accounts/first_page`` — ``has_more`` returns false on a 3-seeded list with ``max_results=2``. Pagination logic in ``_apply_pagination`` is correct in isolation; the storyboard's seed→list chain isn't reaching the impl with the expected request shape. Needs separate investigation with end-to-end repro. - ``media_buy_seller/proposal_finalize/get_products_refine`` — refine path on ``get_products`` returns no ``proposals[]``. ``SalesAgentProposalManager.refine_products`` raises ``UNSUPPORTED_FEATURE``. Substantial feature work, separate PR. - ``security_baseline/assert_mechanism`` — likely fixed transitively by the ``WWW-Authenticate`` header; re-probe after deploy will confirm. ## Verification - ``make quality`` — 4292 passed, 14 skipped, 19 xfailed - New targeted tests: 30/30 (15 middleware × scope cases, 6 INVALID_STATE behavioural × class cases, 9 INVALID_REQUEST schema cases) - Existing ``test_max_daily_spend_exceeded`` updated to expect the new wire code per the change description - Structural guards (transport-agnostic-impl, no-toolerror-in-impl, etc.) pass; the new middleware is a salesagent-side ASGI wrapper, not in ``_impl`` scope Co-Authored-By: Claude Opus 4.7 (1M context) * docs(comply): mark WWWAuthenticateMiddleware as workaround for adcp-client-python#712 The upstream defect is in ``adcp/server/auth.py:411`` — ``BearerTokenAuthMiddleware._unauthenticated`` emits a ``JSONResponse`` with ``status_code=401`` but no ``WWW-Authenticate`` header. The sibling ``A2ABearerAuthMiddleware._send_unauthenticated`` in the same file (line 1024) gets it right. Filed at adcontextprotocol/adcp-client-python#712. Documents the deletion plan: when the upstream fix ships and we bump ``adcp``, the middleware's case-insensitive presence check makes it a no-op, so the order is safe — bump → re-probe → remove the middleware and its registration in a follow-up PR. No code change. Comments only. Co-Authored-By: Claude Opus 4.7 (1M context) * review(comply): code-reviewer nits from PR #383 Fixes two factually-wrong claims and adds a regression guard, all flagged by the code-reviewer pass: 1. ``media_buy_create.py:2418`` comment claimed "``VALIDATION_ERROR`` is not in the spec enum and gets dropped by buyer agents walking ``STANDARD_ERROR_CODES``." It IS in the enum (``adcp/types/generated_poc/enums/error_code.py:46``). Replace with the actual justification — the storyboard-intersection argument — and add a forward note about the dead ``PermissionError`` catch path (no code inside this try raises it today; if a future principal- ownership check moves in, split the except so PermissionError maps to ``PERMISSION_DENIED``). 2. ``test_invalid_request_envelope_on_validation_failure.py`` carried the same wrong claim in its module docstring. Rewrite to reflect the actual intersection argument. 3. Add ``test_www_authenticate_runs_after_admin_mount_and_before_signing`` to ``test_serve_kwargs_middleware_order.py`` — pins ``WWWAuthenticateMiddleware`` between ``AdminWSGIMount`` (so admin Google-OAuth 401s don't get a misleading Bearer challenge) and ``SigningVerifyMiddleware`` (so signing-emitted 401s flow through the injector). A future refactor that moves the middleware either direction surfaces here instead of silently breaking RFC 6750 §3 compliance. No behaviour change. Quality: 4293 passed. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat(proposal): implement v1 refine_products + flip capabilities.refine=True (#385) * feat(proposal): implement v1 refine_products + flip capabilities.refine=True Closes the ``media_buy_seller/proposal_finalize/get_products_refine`` storyboard failure observed in the 7.1.0 comply() probe against Wonderstruck. ## What changed * ``SalesAgentProposalManager.refine_products`` now has a real implementation instead of raising ``UNSUPPORTED_FEATURE``. Delegates to ``_get_products_impl`` for products, decorates the response with a fresh ``Proposal`` via the existing ``_build_v1_brief_proposal`` even-split allocator, and populates ``refinement_applied[]`` from the buyer's ``refine[]`` asks. * ``ProposalCapabilities.refine`` flipped from ``False`` to ``True``. The framework router now dispatches ``buying_mode='refine'`` requests to ``refine_products`` instead of falling through to ``get_products`` (which never populated ``refinement_applied``). * New ``_build_v1_refinement_applied`` helper: dispatches each refine entry's ``scope`` (``request`` / ``product`` / ``proposal``) to the matching ``RefinementApplied{1,2,3}`` variant. Status is uniformly ``applied`` with a v1-acknowledgement note explaining the response carries a fresh-but-unchanged-strategy proposal. Forward-compat: unknown scopes and malformed entries (e.g. product-scope without ``product_id``) are silently dropped rather than crashing the response. ## v1 vs v2 semantics v1 is explicitly acknowledgement-shaped. Storyboard validation is ``field_present @ /proposals`` and ``response_schema`` — both satisfied without semantic refinement. The note in each ``refinement_applied`` entry signals the v1 limitation so buyers see honest behaviour: the proposal is fresh but the allocation hasn't been re-strategized from the ask content. v2 will swap the even-split for an allocation that actually honors asks (drop product / shift budget / shape targeting) once ``ProposalStore`` is wired to load the prior draft by ``proposal_id``. The wire contract stays stable across v1/v2. ## Tests Mirrors the pattern in ``test_proposal_manager_brief.py``: * ``TestSalesAgentProposalManagerCapabilities`` pins ``capabilities.refine=True`` and the unchanged sales_specialism. * ``TestBuildV1RefinementApplied`` covers every scope variant, multi-entry ordering preservation, malformed-entry drop, unknown- scope drop, and RootModel-wrapped entry unwrap. * ``TestRefinementAppliedNote`` pins the buyer-facing breadcrumb so a future content swap is intentional. 11 new tests; quality green (4273 passed, 14 skipped, 19 xfailed). Co-Authored-By: Claude Opus 4.7 (1M context) * review(refine): cap buyer-supplied echo + drop dead fallback (PR #385 nits) Addresses three review items: **security-reviewer L1 (Should-Fix): bound buyer-supplied refine echo.** ``RefinementApplied2.product_id`` and ``RefinementApplied3.proposal_id`` are typed ``str`` with no length cap in the adcp library, so an adversarial buyer could ship 10MB ids and force us to hold them through Pydantic validation and echo them back. Added two caps in ``core/proposal/manager.py``: * ``_MAX_REFINE_ID_LEN = 256`` — per-id length cap; oversize ids are DROPPED (not truncated — truncation corrupts id semantics for downstream correlation). Real AdCP ids look like ``prop_abc123`` / ``prod_video_outdoor``; 256 chars leaves generous headroom. * ``_MAX_REFINE_ENTRIES = 50`` — array length cap, slice up front so an N-million-entry array can't drive allocation pressure even before the per-entry loop runs. * New ``_is_safe_id`` helper centralizes the per-id check (also catches non-str values, defense-in-depth for callers bypassing Pydantic). **code-reviewer nit 1: drop dead ``or getattr(req, "refine", None)``.** ``_coerce_to_request_model`` returns a ``GetProductsRequest`` Pydantic model that always has the ``refine`` attribute (default ``None``), so the fallback can never fire. Simplified to ``getattr(req_model, "refine", None) or []``. **code-reviewer nit 2: drop over-promised telemetry comment.** The dropped-scope comment claimed "missing telemetry" without actually emitting any. Replaced with the honest framing — forward-compat for spec additions, known v1 limitation tracked for v2 telemetry — and matched the docstring's "silently dropped" claim. ## New tests (6) ``TestRefineEchoLengthCaps`` in ``test_proposal_manager_refine.py``: * ``test_oversized_product_id_dropped`` — 257-char id (cap+1) dropped * ``test_oversized_proposal_id_dropped`` — same cap on proposal scope * ``test_empty_product_id_dropped`` — zero-length symmetry * ``test_max_length_id_accepted`` — boundary at exactly 256 chars * ``test_excess_array_length_truncated`` — 100-entry array → 50 echoed * ``test_non_string_product_id_dropped`` — defense-in-depth for non-str Quality green: 4279 passed, 14 skipped, 19 xfailed. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(#336): enable Add Publisher on embedded view + fix(scheduler): DetachedInstanceError (#392) * fix(#336): enable Add Publisher on embedded view Embedded tenants couldn't add publisher partners — the UI hid the controls and the API 403'd direct POSTs. Without publishers there are no AuthorizedProperty rows, which empties the property selector and blocks Create Product on embedded tenants. PublisherPartner is not in the model-layer guard's locked set (embedded_tenant_guard locks only Tenant core columns, AdapterConfig, and signing creds), so publisher-partner mutations are publisher-managed by definition. Apply the same opt-in pattern as PR #340: pass allow_embedded_writes=True on the four mutation routes (add / delete / sync / refresh) and drop the redundant _reject_if_embedded helper. Template: unhide the +Add Publisher / Refresh-all buttons and the modal; update the "Platform-managed" banner to scope only to the agent URL (which IS platform-managed) rather than the partner roster. Tests: flip TestPublisherPartnershipsReadonlyOnEmbedded → TestPublisherPartnershipsEditableOnEmbedded; add positive coverage test_managed_tenant_can_add_publisher_partner under TestEmbeddedViewAllowsPublisherManagedWrites. Move the api-mode JSON envelope assertion and gate-polarity check to the OIDC enable route (still platform-managed, api_mode=True). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(scheduler): DetachedInstanceError on multi-buy delivery batch Production trace (2026-05-08): the daily delivery-report batch succeeded for the first media buy with a reporting_webhook, then raised DetachedInstanceError on media_buy.tenant for every subsequent buy in the same batch. Root cause: each iteration calls _get_media_buy_delivery_impl, which opens its own ``with get_db_session()``. Because get_db_session uses a scoped_session, the inner ``scoped.remove()`` closes the SAME session the outer batch loop is using, detaching every MediaBuy row loaded by MediaBuyRepository.get_all_by_statuses. The first iteration happens to complete before the inner remove() fires; iteration 2+ hits a detached instance on the next relationship access. Fix: eager-load MediaBuy.tenant via joinedload in the scheduler's fetch. The tenant value is materialized into the instance state and survives detach, so media_buy.tenant returns the cached Tenant without lazy-loading through a closed session. Added eager_load_tenant=True parameter on MediaBuyRepository.get_all_by_statuses (default False so the media_buy_status_scheduler caller — which doesn't access tenant — doesn't pay the JOIN cost). Regression test reproduces the production trace exactly: two media buys with reporting_webhook configured; without the fix, only one webhook is sent and the second iteration raises DetachedInstanceError. Co-Authored-By: Claude Opus 4.7 (1M context) * test(#335): regression tests for product-save validation paths The user reported "Internal Server Error" when saving a product without selecting a Property, expecting a validation flash instead. PR #340 closed #335 by fixing the embedded-write 403 that the storefront proxy was misreporting; these tests document and lock in the post-fix contract so the bug can't return undetected. Six new scenarios under tests/admin/test_product_creation_integration.py: - test_add_product_without_property_returns_validation_error_not_500: POST with name + pricing, no property selection. Asserts 200 + the "Please select at least one property tag" flash text + no leaked Product row. - test_add_product_malformed_inputs_never_return_500 (parametrized): only_name, name_and_pricing_only, invalid_pricing_rate, invalid_property_mode, property_ids_mode_no_selection. Every case must surface a validation response, never a raw 500. Both tests share a new ``authenticated_admin`` fixture that uses UserFactory (CLAUDE.md Pattern #8) — re-loads the tenant inside the factory's session to avoid DetachedInstanceError from the test_tenant fixture's closed session. All 17 product-create + delivery-webhook integration tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) * review: tighten scheduler regression + bound display_name + pin guard layer Addresses code-review and security-review feedback on the three preceding commits before merge. Scheduler test (code-review C2): tie the regression assertion to the actual failure mode via caplog. Without this, the test depended on ``await_count`` as a second-order signal; the new check catches ``DetachedInstanceError`` directly so a future refactor that changes the send count for unrelated reasons can't silently mask the bug. Guard layer consistency (code-review I4): new test ``test_publisher_partner_not_locked_at_model_layer`` exercises a PublisherPartner write on an embedded tenant without the ``management_api_caller`` bypass. If a future change adds the model to ``embedded_tenant_guard``'s locked set, this test fails with a pointer to remove ``allow_embedded_writes=True`` from the four publisher_partners routes. Companion note added in embedded_tenant_guard.py near the existing locked-table listeners. Display-name length cap (security nit 1): add a 255-char gate on ``display_name`` in ``add_publisher_partner`` so a hostile or buggy embedded caller can't persist multi-MB strings that later render into admin UI / API responses. Filed #391 for the systemic scoped_session/nested-get_db_session() trap surfaced during scheduler triage (code-review C1). The scheduler fix in ac3a3a7b is the right immediate patch; the underlying trap needs its own redesign and is tracked separately. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(#357): use url_for() for tenant admin links so embedded-mode mounts work (#393) * fix(#357): use url_for() for tenant admin links so embedded-mode mounts work The Setup Checklist (and other admin surfaces) emitted bare /tenant//... hrefs. Under the Storefront's /storefront/salesagent mount, those resolved against the storefront host instead of the proxied salesagent path and returned 404 from the parent app. Service layer (setup_checklist_service.py, dashboard_service.py, business_activity_service.py) now builds URLs via flask.url_for() so the emitted hrefs include SCRIPT_NAME automatically. Templates that previously hand-prepended {{ request.script_root }} are migrated to url_for() in the same pass for consistency with CLAUDE.md Pattern #6. SetupChecklistService runs from two transports: Flask (admin UI) and Starlette via adcp.server.serve() (MCP/A2A). validate_setup_complete() fires inside _create_media_buy_impl on the non-Flask path, where url_for() would raise RuntimeError. _build_url() catches that and returns None; validate_setup_complete only reads task['name'], so the gate behavior is unchanged. Added tests/unit/test_setup_checklist_no_flask_context.py to pin this contract. JS string interpolations (`${tenantId}/...`) are intentionally untouched — url_for can't help with runtime IDs, and CLAUDE.md Pattern #6 already endorses `scriptRoot + path` for that case. One pre-existing FIXME left: tenant_settings.html /settings/raw form posts to a route that has no handler; touched only with a clarifying comment, not a behavior change. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(#357): also tolerate BuildError when url_for runs in a foreign Flask app The tenant_management_api blueprint runs as its own Flask app (tests/integration/test_managed_tenant_api.py:54 — a bare Flask() with only that blueprint registered). When SetupChecklistService is invoked from that app (via tenant_status_service → /status), url_for() for admin-UI endpoints raises werkzeug.routing.BuildError, not RuntimeError, because the endpoint isn't registered there. _build_url now catches both. The management API never reads action_url (it surfaces configure_path from a static map in tenant_status_service._CONFIGURE_PATHS), so None is correct. Adds TestServiceWorksInForeignFlaskApp to pin the contract — Flask context exists, but the endpoint can't be built. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * chore(deps): bump adcp 5.3.0 → 5.4.0; drop three local workarounds (#394) * chore(deps): bump adcp 5.3.0 → 5.4.0; drop three local workarounds 5.4.0 ships every upstream issue I filed yesterday plus a bonus. ## Drops with the bump | Workaround | Upstream fix | |---|---| | ``core/middleware/www_authenticate.py`` (74 LOC) — injected ``WWW-Authenticate: Bearer`` on 401 because the MCP-leg ``BearerTokenAuthMiddleware._unauthenticated`` returned ``JSONResponse(status_code=401)`` without the header | adcp-client-python#712 → #715: ``_unauthenticated`` now emits the header on both the MCP dispatch path AND the ASGI ``_send_unauthenticated`` path, matching what the A2A leg already did | | ``core/idempotency._ReplayMarkingStore`` (~120 LOC + private-symbol coupling to ``_WRAPPED_FUNCTIONS`` / ``_clone_response`` / ``_resolve_call_args`` / ``_to_dict`` / ``CachedResponse``) — reimplemented the full ``IdempotencyStore.wrap`` body inline to inject ``replayed: true`` on cache hits per AdCP L1/security rule 4 | adcp-client-python#714 → #717: ``IdempotencyStore.wrap`` now does ``response["replayed"] = True`` on the cache-hit branch natively | | ``mcp_header_name="x-adcp-auth"`` + ``mcp_bearer_prefix_required=False`` — the MCP leg accepted ONLY the custom legacy header, EXCLUDING the spec-canonical ``Authorization: Bearer``. Caused ``security_baseline/probe_api_key`` storyboard failures | adcp-client-python#720 → #721: ``Authorization: Bearer`` is always accepted; ``mcp_legacy_header_aliases=[...]`` is a purely additive opt-in for adopters with deployed legacy clients | Net diff: -533 LOC including the obsolete test files. ## Auth shape after bump Old (broken for spec-compliant clients): ```python BearerTokenAuth( validate_token=_validate_token, mcp_header_name="x-adcp-auth", mcp_bearer_prefix_required=False, ) ``` New (spec compliance + zero break for legacy clients): ```python BearerTokenAuth( validate_token=_validate_token, mcp_legacy_header_aliases=["x-adcp-auth"], ) ``` ``Authorization: Bearer `` is now accepted on both legs by default (the spec carrier per RFC 6750). The ``x-adcp-auth`` legacy header keeps working unchanged for any early-adopter MCP client still on it. Migration is a one-way drift with no flag day. ## Files removed - ``core/middleware/www_authenticate.py`` - ``core/tests/test_idempotency_replay_marking.py`` - ``tests/unit/test_www_authenticate_middleware.py`` - The ``WWWAuthenticateMiddleware``-ordering test in ``tests/unit/test_serve_kwargs_middleware_order.py`` ## Verification - ``make quality``: 4294 passed, 14 skipped, 19 xfailed - Existing ``_ReplayMarkingStore`` callers in ``get_idempotency_store()`` swapped to plain ``IdempotencyStore`` — same constructor signature, upstream provides the injection - ``test_serve_kwargs_middleware_order.py`` updated to drop the ``WWWAuthenticateMiddleware``-position pin (middleware no longer exists) - After deploy, the compliance probe's ``security_baseline/probe_api_key`` and ``assert_mechanism`` storyboard steps should flip to pass — closes the auth-header gap we filed as bokelley/salesagent#386 (which can now be closed as "fixed upstream") ## Closes (when deployed) - bokelley/salesagent#386 — multi-header auth (now native via #720) - The remaining compliance-probe residual on ``security_baseline`` (3 → 1 failure; only ``proposal_finalize/create_media_buy`` left, tracked separately as #387) ## Doesn't pick up - 5.4.0's ``LazyPlatformRouter.proposal_stores=`` / ``proposal_store_factory=`` (#722/#724) — this is the wiring point for bokelley/salesagent#387. Separate PR. Co-Authored-By: Claude Opus 4.7 (1M context) * review(deps): switch test fixtures to new BearerTokenAuth shape (PR #394 nit) Code-reviewer flagged that two test files still constructed ``BearerTokenAuth`` with the legacy ``mcp_header_name`` / ``mcp_bearer_prefix_required`` kwargs even though production swapped to ``mcp_legacy_header_aliases=[...]`` in this PR's main commit. The tests passed against 5.4.0 (back-compat shim works) but emitted ``DeprecationWarning`` and stopped mirroring the production config — production now ACCEPTS ``Authorization: Bearer`` on the MCP leg alongside ``x-adcp-auth``, but these test fixtures still wired the old exclusive-header semantics. ## Changes * ``tests/unit/test_per_leg_bearer_auth.py``: ``_production_auth`` and ``_build_mcp_app`` updated to the new shape. The inner ``BearerTokenAuthMiddleware`` construction now passes ``legacy_header_aliases=auth.resolved_mcp_legacy_aliases()`` and ``legacy_aliases_bearer_prefix_required=auth.legacy_aliases_bearer_prefix_required`` in place of the deprecated ``header_name`` / ``bearer_prefix_required`` pair. Mirrors what adcp.server.serve._wrap_mcp_with_auth does natively against 5.4.0. * ``tests/unit/test_agent_card_auth_scheme.py``: ``_production_auth`` same swap; module-level docstring updated to reflect that both legs now default to ``Authorization`` and the MCP leg additively accepts ``x-adcp-auth`` for legacy adopters (not as an exclusive override). ## Verification * 10/10 targeted tests pass * ``make quality`` — 4294 passed, 14 skipped, 19 xfailed; warning count dropped from 117 to 105 (the deprecation warnings are gone) No behavior change beyond what PR #394's main commit ships. Tests now exercise the same wire shape production uses. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * chore(logs): strip debug instrumentation + collapse audit fan-out (#395) Production log volume on Fly was 70%+ noise. Three categories of offender, all in our code (not the adcp SDK): src/core/context_manager.py - Delete leftover ``console.print`` debug blocks: 🔍 PRE-COMMIT WEBHOOK DEBUG (16 lines per workflow step update), 🔍 POST-COMMIT WEBHOOK DEBUG (5 lines), and the 🚀 WEBHOOK / ⚠️ WEBHOOK SKIPPED chatter. These read as leftover instrumentation from a past debugging session and fire on every workflow step status change. - Convert remaining ``console.print`` calls to ``logger.debug`` (lifecycle events: context/step creation, object linking, webhook dispatch) or ``logger.warning`` / ``logger.exception`` (errors). - Drop the unused ``rich.console.Console`` import and module-level ``console = Console()`` singleton. src/core/helpers/adapter_helpers.py - Demote ``[ADAPTER_SELECT]`` / ``[ADAPTER_CONFIG]`` from ``logger.info`` to ``logger.debug`` (10 call sites). These are tracing-grade fields, not operational signals — they should be off in production unless someone is actively debugging adapter selection. src/core/audit_logger.py - Collapse the per-detail audit fan-out: instead of one ``logger.info`` per ``details`` dict key (N+1 lines per audit event), emit a single ``" |
"`` line. Full details still persist to ``AuditLog.details`` for structured queries — the per-line fan-out was legible in local tail but flooded production stdout. The ``adcp.audit`` logger name is shared with the SDK's ``LoggingAuditSink`` but the noisy emissions come from our own ``audit_logger`` in ``src/core/audit_logger.py``, not the SDK — so nothing to file upstream. Co-authored-by: Claude Opus 4.7 (1M context) * chore(logs): drop /mcp+/health access spam, rate-limit geo warning, fix trafficker_id log bug (#397) Three followups from the audit pass on production fly logs. src/core/logging_config.py - Add ``UvicornAccessNoiseFilter`` and attach it to ``uvicorn.access`` in both production (JSON) and development (standard) modes. The filter drops 2xx GET/POST/HEAD/OPTIONS access lines on /mcp[/] and /health — the two endpoints hit constantly by storefront MCP pollers and Fly's TCP+HTTP health checks. 4xx/5xx still surface so auth failures and server errors aren't buried. Other paths (admin UI, /a2a, /.well-known, /mcp-debug, etc.) are unaffected. Behavioral contract pinned by 18 parametrized tests in tests/unit/test_uvicorn_access_filter.py. src/adapters/gam/managers/targeting.py - Rate-limit the "Could not load geo mappings file" + "Using empty geo mappings" warnings to once per process lifetime via a module-level flag. Each ``GAMTargetingManager`` instance fires on every adapter selection, so the same warning flooded the log on every GAM-tenant request. The underlying file-not-found is still tracked in #396 (it means GAM geo targeting silently produces empty results in prod and needs a packaging-side fix). src/adapters/google_ad_manager.py - Fix the "Could not auto-detect trafficker_id: User instance has no attribute 'get'" warning. The googleads SOAP client returns a zeep complex object — it supports __getitem__ and attribute access but NOT ``.get()``. The old code called ``current_user.get('name', 'Unknown')`` inside the success-log f-string, which raised AttributeError AFTER ``self.trafficker_id`` was already assigned. The ID was being detected correctly all along; only the success log was failing and producing a misleading warning on every request. Switched to ``getattr`` for the optional ``name`` field. Filed #396 to track the underlying production OOM kill on the iad machine and the missing ``gam_geo_mappings.json`` packaging issue — both infrastructure-level and out of scope here. Co-authored-by: Claude Opus 4.7 (1M context) * feat(proposal): wire Postgres-backed ProposalStore for create_media_buy(proposal_id=…) (#390) * feat(proposal): wire Postgres-backed ProposalStore for create_media_buy(proposal_id=…) Closes the proposal-lookup gap that made ``proposal_finalize/create_media_buy`` fail with ``INVALID_REQUEST: Invalid budget: 0.0``. Without a wired :class:`ProposalStore`, the framework's ``proposal_dispatch`` had no backing for the buyer's ``proposal_id`` and ``create_media_buy`` landed in package-derivation with zero packages. Pieces: - ``proposals`` table (migration ``r0s1t2u3v4w5``) — mirrors the v1.5 ``ProposalRecord`` dataclass with multi-tenant scoping and a partial unique on ``(account_id, media_buy_id) WHERE media_buy_id IS NOT NULL`` for reverse-index lookups - :class:`SalesAgentProposalStore` — implements every :class:`adcp.decisioning.proposal_store.ProposalStore` Protocol method (put_draft / get / commit / try_reserve_consumption / finalize_consumption / release_consumption / mark_consumed / discard / get_by_media_buy_id) against the new table. Atomic CAS via ``SELECT … FOR UPDATE`` serializes parallel callers. Cross-tenant probes collapse to ``None`` / ``PROPOSAL_NOT_FOUND`` per the Protocol's principal-enumeration defense. - :class:`_LazyPlatformRouterWithStore` — thin subclass that adds the ``proposal_store_for_tenant`` accessor the framework's ``proposal_dispatch`` duck-types. Upstream :class:`LazyPlatformRouter` doesn't expose it (only the eager :class:`PlatformRouter` does, via ``proposal_stores=``). - Wired into ``build_router()`` — single shared store across tenants; isolation runs inside the store on ``expected_account_id``. v1 lifecycle compromise (documented in the store docstring): the storyboard flow goes brief → create_media_buy WITHOUT an intermediate finalize step, but the framework's :meth:`try_reserve_consumption` requires the proposal to be in ``committed`` state. The store auto-commits at ``put_draft`` time with a 7-day ``expires_at`` so the buyer flow unblocks today. The Protocol surface is unchanged — only the internal lifecycle state differs. When the manager declares ``finalize=True`` in v2, swap to canonical ``draft`` + explicit commit. Tests: - ``tests/integration/test_proposal_store.py`` — 15 integration tests against real Postgres covering put_draft auto-commit, payload round-trip, refine overwrite, cross-tenant probe defense (get + try_reserve), two-phase consumption lifecycle, atomic CAS double-reservation rejection, reverse-index lookup with ``expected_account_id`` enforcement, idempotent release/discard - ``tests/unit/test_lazy_router_with_proposal_store.py`` — 3 unit tests pinning the router subclass's accessor wiring - ``tests/unit/test_proposal_store_attributes.py`` — 2 unit tests pinning ``is_durable=True`` (production-mode gate) and the 7-day default hold window Refs #387 Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(proposal): adopt adcp 5.4 — drop workarounds, use upstream surface Upstream shipped both items we filed during #387: - adcp-client-python#722 → 5.4: LazyPlatformRouter accepts ``proposal_stores=`` and ``proposal_store_factory=``. Deletes our ``_LazyPlatformRouterWithStore`` subclass. - adcp-client-python#723 → 5.4: ``ProposalCapabilities.auto_commit_on_put_draft`` shipped option B from the issue. The framework now calls ``store.commit`` immediately after ``put_draft`` for opted-in managers. Deletes our store-side ``state=COMMITTED`` workaround in ``put_draft``. Migration: - Bump ``adcp>=5.4.0``. - ``SalesAgentProposalManager.capabilities`` declares ``auto_commit_on_put_draft=True``; framework owns the DRAFT → COMMITTED promotion via ``auto_commit_ttl_seconds=604800`` (7-day default, matches our prior store-side hold window). - ``core/main.build_router`` calls ``LazyPlatformRouter(...)`` directly with ``proposal_store_factory=lambda _tid: shared_store``. Factory shape over eager dict because the store is a single shared instance — eager dict would force boot-time tenant enumeration and miss tenants registered after boot. - ``SalesAgentProposalStore.put_draft`` writes spec-canonical ``draft`` state with ``expires_at=None``. The ``_committed_hold`` constructor param and the 7-day default are gone — the framework's ``auto_commit_ttl_seconds`` capability owns the TTL. Tests: - Integration: 16 tests rewritten — put_draft asserts DRAFT (not COMMITTED), reservation lifecycle tests use a ``_put_and_commit`` helper that mirrors the framework's auto-commit dispatch, new ``TestCommit`` class covers commit promotion + idempotency + payload-drift rejection, new test pins that put_draft on a COMMITTED record raises ``INTERNAL_ERROR`` per Protocol. - Unit: deleted ``test_lazy_router_with_proposal_store.py`` (no subclass to test); trimmed ``test_proposal_store_attributes.py`` to the durability flag only (the 7-day default belongs to the framework now). Refs #387 Co-Authored-By: Claude Opus 4.7 (1M context) * fix(proposal): address review — split compound account_id, account-scoped locks, fail-closed unscoped methods Review feedback on PR #390: **B1 (blocker): _resolve_tenant_id_for_account returned account_id verbatim.** SalesagentAccountStore.resolve mints ``f"{tenant_id}:{ref}"`` (``ref`` defaults to ``"default"``; storyboard runs use ``"acct_demo"``). The framework passes ``ctx.account.id`` straight into ``put_draft``, so every prod ``put_draft`` would FK-violate on ``proposals.tenant_id``. Fixed: split on ``":"`` and take the prefix. New integration test ``test_put_draft_handles_compound_account_id`` regresses this — uses the real shape the framework emits. **Security MAJOR (×3): try_reserve / finalize / release did SELECT FOR UPDATE then filtered account_id in Python.** Cross-tenant probes acquired the row lock, leaking existence via timing AND providing a DoS primitive against legitimate same-tenant operations. Fixed: ``account_id`` moved into the WHERE clause so cross-tenant probes never acquire the lock. Two new integration tests pin the behavior: - test_finalize_cross_tenant_collapses_to_internal_error - test_release_cross_tenant_is_noop (verifies foreign tenant's release doesn't roll back the owner's CONSUMING reservation) **Security MAJOR (×2): discard() and mark_consumed() Protocol signatures lack ``expected_account_id``.** Any caller obtaining a ``proposal_id`` could destroy / terminate another tenant's proposal. Neither is called by adcp 5.4's ``proposal_dispatch`` today; fixed: both raise ``NotImplementedError`` with an ERROR log. Future framework versions that begin calling them surface loudly before reaching prod. Two new tests pin the fail-closed behavior. **MAJOR M3: _serialize_recipes silently passed dicts through.** Violates "No quiet failures" (CLAUDE.md). Fixed: raises TypeError on non-Pydantic input — caller has to pass typed Recipe instances. **MINOR m3: lazy imports inside every method.** Hoisted ``ProposalRecord``, ``ProposalState``, ``AdcpError`` to module level — no circular import; the salesagent already imports the library at module-load time elsewhere. **NIT n2/n3: stale temporal references.** Dropped "v1 auto-commit workaround landed before #723 and is gone" from the store docstring and "v1 auto-commits at put_draft time" from the Proposal model docstring. Per CLAUDE.md: don't document the prior behavior. **M2 partial coverage: end-to-end account_id shape test added.** ``test_put_draft_handles_compound_account_id`` exercises the realistic ``"tenant_id:default"`` shape the framework actually emits. Full end-to-end (HTTP → proposal_dispatch → store) deferred to compliance probe post-deploy — the unit layer pins every store-side invariant. 24 integration + unit tests pass; ``make quality`` clean (4311 tests). Co-Authored-By: Claude Opus 4.7 (1M context) * review(proposal): expires_at guard + 8 lock-in tests cherry-picked from PR #398 Two additions from @bokelley's parallel #398 work that #390 lacked: **1. Defense-in-depth expires_at check inside try_reserve_consumption.** Security reviewer L1 finding on #398: a buyer holding a COMMITTED proposal past its ``expires_at`` could reserve and finalize indefinitely. The framework's ``proposal_dispatch._hydrate_proposal_context`` checks expiry on the get-side, but ``try_reserve_consumption`` is reachable from dispatch paths that bypass that filter (and from adopter callers that go straight to the store). New three-line guard inside the existing row lock raises ``PROPOSAL_EXPIRED`` with ``recovery="correctable"``. Mirrors upstream :class:`InMemoryProposalStore._evict_expired_locked` but surfaces the event rather than silently deleting so audit trails survive. **2. mark_consumed restored as implemented Protocol method.** Earlier fail-closed pattern was over-cautious for a Protocol method the framework doesn't currently call. Now matches the upstream :class:`InMemoryProposalStore.mark_consumed` shape verbatim, with a WARNING audit log on every call so unexpected invocations are visible. Documented Protocol-signature gap (no ``expected_account_id``) — same upstream constraint that :meth:`discard` has; ``discard`` stays fail-closed because the user's follow-up list didn't include it. **Tests (9 added, 1 replaced):** - test_reserve_past_expires_at_raises_expired (locks in #1) - test_release_silent_no_op_on_missing - test_release_silent_no_op_on_cross_account - test_finalize_idempotent_on_consumed_matching_media_buy - test_finalize_mismatched_media_buy_raises - test_mark_consumed_promotes_to_consumed - test_mark_consumed_idempotent_on_matching - test_mark_consumed_mismatched_raises - test_mark_consumed_unknown_raises_internal_error - Replaced ``test_mark_consumed_raises_not_implemented`` with the four ``TestMarkConsumed`` cases above All cherry-picked from #398's test suite (locked-in shapes already correct in #390's code per @bokelley's close comment). 32 integration + unit tests pass; ``make quality`` clean (4311 tests). Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * refactor(types): adopt SchemaVariant for 12 cross-class schema overrides (#400) adcp 5.4.0 #718 ships ``SchemaVariant[T]`` + a mypy plugin that rewrites the annotation to ``Any`` for override-compat purposes, retiring the ``# type: ignore[assignment]`` stamps adopters used to carry on cross-class entity overrides. The 12 sites in src/core/schemas/ all match the cross-class pattern the marker targets: - 4× geo_*_exclude — parent declares Geo{Country,Region,Metro, PostalArea}ExcludeItem; we substitute the inclusion variant - 2× creatives — parent declares CreativeAsset; we substitute our extended Creative - 1× deployments — parent declares Deployments; we substitute SignalDeployment - 1× media_buys — parent declares MediaBuy; we substitute the GetMediaBuysMediaBuy delivery-context view - 1× ext — parent declares ExtensionObject; we use dict - 1× sync_creatives.creatives — parent's CreativeAsset; we use our local CreativeAsset subclass - 1× query_summary — parent's QuerySummary; we use our local - 1× media_buy_deliveries / 1× creatives in delivery.py — delivery-context views mypy.ini gets ``adcp.types.mypy_plugin`` added to the plugins line alongside the existing sqlalchemy + pydantic plugins. Tradeoff (documented upstream): inside the override, mypy sees the field as ``Any``. ``typing.cast(list[T], self.field)`` recovers precise inference at call sites that need it. None of the touched sites currently rely on inside-override inference at usage sites, so no cast() is needed for this change. make quality: 4319 passed, 14 skipped, 19 xfailed. Co-authored-by: Claude Opus 4.7 (1M context) * refactor: drop SchedulerLifespanMiddleware, use serve(on_startup=, on_shutdown=) (#401) adcp 5.4.0 #713 ships native lifespan hooks on ``serve(transport='both')``, which is exactly what the middleware was hand-rolling. The middleware intercepted ASGI ``lifespan.startup`` / ``lifespan.shutdown`` scope events to fire scheduler start/stop coroutines because earlier SDK versions didn't expose a user-supplied lifespan extension point. Now they do. The SDK's ``on_startup`` / ``on_shutdown`` kwargs take the same ``Callable[[], Awaitable[None]]`` shape that ``_start_schedulers`` and ``_stop_schedulers`` already had, so the swap is mechanical: - Drop ``SchedulerLifespanMiddleware`` from the ``asgi_middleware`` list. - Pass ``on_startup=[_start_schedulers]`` / ``on_shutdown=[_stop_schedulers]`` in ``_serve_kwargs()``, conditional on ``include_scheduler`` (tests still skip). - Delete ``core/middleware/scheduler_lifespan.py`` (61 LOC). - Update the ``_serve_kwargs`` docstring to reference the SDK hook instead. The middleware ran scheduler shutdown with a 10s ``asyncio.wait_for`` guard; the SDK fires hooks unguarded. Our ``_stop_schedulers`` already caps its own awaitables (delivery + media-buy status schedulers each join their internal task groups with a bounded timeout), so dropping the outer wait_for is fine — it was defensive double-bookkeeping. make quality: 4319 passed, 14 skipped, 19 xfailed. Closes the second of three local rip-outs unlocked by the adcp 5.4.0 bump. The third (AgentCardPublicUrlMiddleware → public_url callable) lands separately. Co-authored-by: Claude Opus 4.7 (1M context) * refactor: swap AgentCardPublicUrlMiddleware for public_url callable (#402) The salesagent middleware existed because earlier SDK versions either hardcoded ``http://localhost:{port}/`` into the agent card with no override hook (pre-5.0) or crashed ``transport='both'`` startup when ``public_url`` was a callable (5.2.0, ``AttributeError: 'function' object has no attribute 'router'``). adcp 5.3.0 #680 fixed the composed-lifespan crash. 5.4.0 has confirmed the callable path works under ``transport='both'`` in production. The SDK's ``serve(public_url=PublicUrlResolver)`` is now the right primitive for per-request agent-card URL derivation. ## What lands - ``core/main._resolve_public_url(request) -> str`` — pure function with the same header-precedence rules the middleware enforced: PUBLIC_URL env > X-Forwarded-Host > Host, X-Forwarded-Proto for scheme, ``http://`` for loopback / ``https://`` otherwise. - Wired as ``"public_url": _resolve_public_url`` in ``_serve_kwargs``. - Drop the middleware from the ``asgi_middleware`` list. - Delete ``core/middleware/agent_card_public_url.py`` (189 LOC). - Replace ``test_agent_card_public_url_middleware.py`` with 13 tests of the new resolver covering: X-Forwarded-Host precedence, Host fallback, comma-chain stripping, proto override, https default, loopback http exception (matches SDK's ``_validate_card_url``), PUBLIC_URL env override, no-headers fallback. - Update ``test_serve_kwargs_middleware_order`` — replace the middleware-present assertion with a ``public_url is callable`` assertion. ## Net diff -442 LOC (mostly the middleware + ASGI plumbing tests it required) +195 LOC (resolver doc + resolver tests + updated order test) = -247 LOC net. ## What stays the same in production behavior - PUBLIC_URL env takes precedence (single-host deploys unchanged). - X-Forwarded-Host derives multi-tenant subdomain URLs (same as before). - X-Forwarded-Proto controls scheme. - Loopback hosts get ``http://`` (the SDK's _validate_card_url enforces this — non-loopback ``http`` returns 500 from the SDK). ## What's different (intentional) - The middleware refused to rewrite non-loopback URLs (defensive pass-through). The resolver always derives the URL afresh. This is safer: with the static-public_url fallback gone, the resolver is the single source of truth and there's no "what gets rewritten vs passed through" branching to reason about. - Response-body buffering and content-length recalculation are gone — the SDK builds the card from the resolver's URL directly. make quality: 4322 passed, 14 skipped, 19 xfailed. Closes the third of three local rip-outs unlocked by the adcp 5.4.0 bump (after SchemaVariant migration and SchedulerLifespanMiddleware removal). Co-authored-by: Claude Opus 4.7 (1M context) * feat(ops): add tenant export/import for legacy → embedded migration (#403) Reflection-based export/import for tenant-scoped data. Walks SQLAlchemy metadata to discover all 41 tenant-scoped tables (including transitive chains like media_packages → media_buys, strategy_states → strategies, object_workflow_mapping → workflow_steps), then exports/imports rows in FK-dependency order inside a single transaction. Built for moving clients from legacy hosting to embedded mode (flip is_embedded=True) on the same Postgres deployment. Also supports cross-deployment moves via target-tenant-id retargeting and a strip-secrets mode that wipes Fernet ciphertext + plaintext bearer credentials (admin_token, slack/audit/hitl webhook URLs, GAM refresh token, push_notification_configs auth, webhook subscription secret hash, creative/signals agent auth_credentials, ai_config api_key). principals.access_token is intentionally preserved so buyers' MCP/A2A integrations keep working post-import. Safety: - alembic_revision pinned in the bundle; import refuses on schema mismatch - pre-flight collision check on subdomain, virtual_host, principals.access_token raises TenantImportCollisionError with precise message instead of opaque IntegrityError - strict column filtering when alembic revisions match (drops are a bug, not noise); --allow-schema-drift downgrades to warning - Core-level inserts bypass the embedded_tenant_guard ORM listeners; the operator-CLI trust boundary is the equivalent privilege level - export bundle written 0600 (contains tenant secrets) - import writes an audit_logs row capturing operator, mode, flip-to-embedded, target_tenant_id, row counts - explicit rollback on any import-path failure CLIs: scripts/ops/export_tenant.py acme --out acme.json [--strip-secrets] scripts/ops/import_tenant.py acme.json --mode=replace --flip-to-embedded scripts/ops/import_tenant.py acme.json --target-tenant-id new --allow-schema-drift scripts/ops/import_tenant.py acme.json --dry-run 19 integration tests covering discovery, round-trip, collision modes, embedded flip, retargeting, strip-secrets (encrypted + plaintext bearer), strict filtering, schema mismatch, audit log emission. Co-authored-by: Claude Opus 4.7 (1M context) * fix(admin-mount): serve /robots.txt as public Disallow / instead of 401 from A2A (#407) Crawlers and probes hitting `GET /robots.txt` on the API host fell through to the inner A2A app, where BearerTokenAuth 401'd them. Production logs filled with `"GET /robots.txt HTTP/1.1" 401 Unauthorized` (plus a paired `adcp.server.auth` JSON line per rejection), and well-behaved crawlers got an inconsistent signal — 401 is not a stable "do not crawl" answer. robots.txt is a host-level resource, not a per-tenant one, so neither Flask nor A2A is the right owner. `AdminWSGIMount` already short- circuits an analogous static response (the apex `/` → `/signup` 302), so colocate the robots short-circuit there: - `GET`/`HEAD /robots.txt` → 200 `text/plain` with `User-agent: *\nDisallow: /\n` and `cache-control: public, max-age=86400` - non-safe methods (POST, etc.) fall through unchanged — the short circuit only covers actual crawler probes Tests: four scenarios in `TestAdminWSGIMountRobotsTxt` covering the GET body+headers, HEAD-returns-no-body contract, POST falling through, and the bug itself (the request must not reach the inner A2A app on a non-admin host). Co-authored-by: Claude Opus 4.7 (1M context) * chore(embedded-guard): inline auth-flag diagnostics in rejection error (#408) Surfaces session/connection auth-flag state directly in the EmbeddedTenantWriteError message so SyncJob.error_message (and the status widget that renders it) shows exactly why the guard fired without log-diving. Distinguishes the three failure modes: - session_present=False → object was detached at flush time - session_flags={all None/False} → flag never set on this session - session_flags={one True} → guard misread the flag (should be impossible) No behavior change beyond the longer error text. * chore(logs): drop /mcp 401 access spam in UvicornAccessNoiseFilter (#410) Anonymous internet traffic hammers /mcp constantly (bot probes, misconfigured clients), and every rejection emits two log lines: - one uvicorn access line ("POST /mcp HTTP/1.1" 401 Unauthorized) - one structured ``adcp.server.auth`` line ("a2a auth rejected" …) PR #397's filter deliberately kept 4xx/5xx so auth failures weren't buried, but the structured log already captures the signal — the access line is dupe noise. In production logs this is by far the dominant source of /mcp-related log volume. Per-surface status-code policy now: * /mcp[/] — drop 2xx AND 401. Other 4xx (403/404/422) and all 5xx still log; those indicate a real problem worth investigating. * /health — drop 2xx only. A 4xx/5xx on the health surface always means a config or platform bug worth seeing. Implementation splits the single regex into two named patterns so the status-code carve-out per surface stays readable. Test reshuffle: * test_drops_noise — new combined parametrize: /mcp 2xx + /mcp 401 + /health 2xx (one row per cause). * test_keeps_real_signal — /mcp non-401 4xx (403, 404, 422), /mcp 5xx, /health non-2xx (401, 503), and /.well-known/oauth-protected-resource 401 (the OAuth dance start, which is signal not noise). Co-authored-by: Claude Opus 4.7 (1M context) * feat(freewheel): full Publisher API adapter — auth, inventory sync, targeting, formats (#381) * chore(freewheel): capture & anonymize publisher API fixtures Adds 56 anonymized FreeWheel Publisher API response fixtures covering /services/v3/ (XML, commercial: advertisers, campaigns, insertion_orders, placements, agencies) and /services/v4/ (JSON, inventory: sites, site_sections, site_groups, series, videos, video_groups, inventory_packages). Includes the capture and anonymization scripts under scripts/dev/freewheel/ so fixtures can be regenerated when the test bearer token rotates (7-day TTL). Both scripts read identifying constants from env vars rather than embedding them, keeping the source tree free of publisher-specific identifiers. .env.template documents the two new optional vars. Anonymization scrubs PII (sales person, trafficker, content credits) and publisher-identifying values (network_id, advertiser_id, content/title fields, external Salesforce IDs) while preserving referential integrity via deterministic memoized replacements. Structural fields (statuses, stages, currencies, budget shapes, schedules, link hrefs) are preserved verbatim so fixtures remain useful as wire-format ground truth for the upcoming adapter client. No production code wired yet — these fixtures are the foundation for the FreeWheel adapter client rewrite (next change). Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): rewrite adapter for real /services/v3+v4 API surface Replaces the skeletal OAuth-client-credentials client (wrong path shape, wrong content type, wrong auth model) with a bearer-token client that matches the actual FreeWheel Publisher API surface verified end-to-end against a publisher's test network: - /services/v4/* (JSON, inventory taxonomy: sites, sections, series, videos, video groups, inventory packages) — read-only - /services/v3/* (XML, commercial entities: advertisers, campaigns, insertion orders, placements, agencies) — full reads + verified create_campaign/delete_campaign writes Module layout under src/adapters/freewheel/: _transport.py — bearer auth, accept negotiation, status mapping _inventory.py — v4 JSON inventory client _commercial.py — v3 XML commercial client _pagination.py — shared page-walking iterator (DRY) entities.py — Pydantic models for both surfaces client.py — FreeWheelClient facade composing the above Connection config is now a single api_token field (7-day TTL, no refresh flow — rotate when expiry approaches). Migration of existing tenants will require manual reconfiguration once any are provisioned. Tests: - 37 new unit tests across transport / inventory / commercial replaying captured fixtures from tests/fixtures/data/freewheel/ - Updated config schema + adapter + roundtrip integration tests Adapter-level wiring (create_media_buy/update_media_buy/check_status using the new client) is intentionally deferred to a follow-up PR; live mode still returns pending_credentials until that integration lands. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): add insertion-order, placement, and campaign-update writes Completes the v3 write surface so adapters and external callers can construct a full campaign hierarchy. All endpoints verified end-to-end against the publisher's test network (probe entities created with clearly tagged names, then deleted): - POST /services/v3/insertion_order — min body: name + campaign_id. Server defaults: stage=NOT_BOOKED, currency=EUR. Auto-attaches an assigned_user from the bearer token's identity (silently dropped by our model's extra="ignore" config). - POST /services/v3/placement — min body: name + insertion_order_id. Server defaults: status=IN_ACTIVE, placement_type=NORMAL. - PUT /services/v3/campaign/{id} — partial update. PATCH returns 405, so v3 uses PUT semantics for "only fields in the body are modified". - DELETE /services/v3/insertion_order/{id} and DELETE /services/v3/placement/{id} — hard deletes, same shape as campaign delete. Adds put_xml() to the transport (POST handler already existed) and a matching test for the PUT method. 6 new commercial client tests covering create+delete for IO and placement, and the partial-update semantics for update_campaign (verifies only passed fields appear in the request body). create_media_buy wiring still uses the pending_credentials stub — that work belongs in the adapter-mapping PR where we decide how AdCP Package maps onto FreeWheel's Campaign→IO→Placement hierarchy. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): wire create_media_buy and check_status to live v3 API Replaces the pending_credentials stub in create_media_buy with the real v3 write flow per Mapping A: AdCP MediaBuy → FW Insertion Order (the commercial transaction) AdCP Package → FW Placement (one per package, child of the IO) FW Campaign → per-buy wrapper above the IO In live mode the adapter now: 1. Creates a FW Campaign named after the AdCP buy (po_number-derived or timestamp), parented to the principal's freewheel advertiser_id. 2. Creates a FW Insertion Order under that campaign. 3. Creates one FW Placement per AdCP Package under the IO. 4. Returns ``media_buy_id = "freewheel_{io.id}"`` — the IO is the unit of commerce, so it's what subsequent calls reference. check_media_buy_status now fetches the IO (not the Campaign) and reports its ``stage`` (NOT_BOOKED, BOOKED, etc.), which is where IO booking state lives in v3. Falls back to ``status`` for safety. FreeWheelError from any of the three create calls is translated to a CreateMediaBuyError with code ``upstream_error``. Partial-failure orphans (e.g. Campaign created then IO fails) are not cleaned up in v1 — they sit as IN_ACTIVE entities and don't deliver. A best-effort rollback is a future refinement. Deferred (each its own follow-up, flagged in the adapter docstring): - update_media_buy live wiring (needs update_io/update_placement probes) - add_creative_assets (v3 /creative endpoint returned 404 in probes — the creative surface is somewhere we haven't mapped) - get_media_buy_delivery (reporting lives on a different API surface) Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): add update_insertion_order/update_placement and document scope blockers Adds the v3 partial-update verbs to the commercial client, both verified end-to-end against the publisher's test network (probe entities created and deleted): - update_insertion_order(id, **fields) — PUT /services/v3/insertion_order/{id}. Supports nested-dict fields like ``budget={"budget_model": ..., "impression": ...}`` for impression-target adjustments. - update_placement(id, **fields) — PUT /services/v3/placement/{id}. The delivery-level pause/resume mechanism (status=IN_ACTIVE / ACTIVE). Extends _build_xml to handle one level of nested dicts so partial body updates with nested elements (budget, schedule) serialise correctly. Adapter-level update_media_buy wiring intentionally NOT included — two publisher-scope blockers surfaced during probes that need resolution before the adapter can wire cleanly: 1. Per-package operations need AdCP package_id -> FW placement_id lookup. v3 /placements doesn't honour ?insertion_order_id filter (returns full network list); no nested-collection endpoint at v3; v4 has the nested form but our token gets a 403 IAM deny. 2. Per-package budget changes don't fit FW's data model — budget lives on the IO, not on the placement. Would need a different mapping (one-IO-per-package) or per-package tracking we don't have. Creative endpoints discovered at v4 (creatives, creative_assets, assets, ad_assets, asset_versions, creative_versions all 403 IAM-deny). v3 has no creative paths (404). add_creative_assets wiring blocked on publisher granting creative scopes. Documented in the adapter docstring so the next conversation with the publisher has the asks ready. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(freewheel): clarify the creative endpoint split (publisher vs demand) After a docs deep-dive prompted by the user pointing at the FreeWheel Marketplace Creatives reference, we now have a clearer picture of the two-API creative model and the AdCP semantic mismatch it implies. Publisher-side (the API our bearer is for): PUT /services/v4/mkpl_creatives/{id} body: approval_status (Approved|Rejected|Pending) + approval_notes This is a *moderation* workflow, not a creation workflow. The buyer registers the creative through their own DSP; it shows up in the publisher's marketplace queue; the publisher (us, via Talpa's token) approves or rejects it. AdCP's sync_creatives (buyer registering creatives) therefore has no direct publisher-side equivalent — the adapter's approval surface maps to AdCP's creative review/approval flow, not its creation flow. Three sibling type-specific endpoints exist alongside the unified one (mkpl_exchange_programmatic_creatives, mkpl_private_direct_sold_creatives, mkpl_private_programmatic_creatives). All 403 IAM-deny on our token — the ask to Mathijs becomes specific: grant scope on ``/services/v4/mkpl_creatives``. Buyer-side (POST /demand/v1/accounts/{seat_id}/ads) is the FreeWheel Demand/Beeswax product. Out of scope for publisher-token-driven integration — Talpa as a publisher wouldn't have a Demand seat to delegate. No code change beyond the adapter docstring — just capturing the finding so the next round of asks to the publisher is precise. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): add v4 creative_resources client (full CRUD scope verified) After two earlier docs links pointed at the wrong API surface (Demand v1 and Marketplace approval), the user surfaced https://api-docs.freewheel.tv/publisher/reference/creative-management-api-v4 which gave us the correct path: /services/v4/creative_resources. Probing shows our existing publisher bearer is fully entitled there — we just hadn't tried the right name. Verified end-to-end (2026-05-12): - GET /services/v4/creative_resources (list, 70 creatives) - GET /services/v4/creative_resources/{id} (single, ``{creative: {...}}`` envelope) - POST /services/v4/creative_resources (auth+validation reaches us) - PUT /services/v4/creative_resources/{id} (auth+validation reaches us) - ?include=renditions exposes the nested VAST tag URIs inline. Exposed on the client as ``client.creatives`` with list_creatives, get_creative, and iter_creatives. CRUD writes (POST/PUT/DELETE) are deferred to a follow-up commit so we can probe shapes against the live API with a create+cleanup pattern. Still scope-blocked (publisher must grant): - /services/v4/creative_instances — creative <-> placement linkage, needed to actually attach a creative to a placement so it delivers. - /services/v4/creative_renditions — standalone rendition collection. - /services/v4/mkpl_creatives — marketplace creative approval. Supporting changes: - entities.py: Creative + Rendition + CreativeMessage models. Extended PaginatedResponse with AliasChoices so both pagination conventions work (total_count/total_page for inventory, total/total_pages for creative_resources). - capture_fixtures.py: added creative_resources to the v4 walk. - anonymize_fixtures.py: advertiser_ids / agency_ids (list-of-int) plus uri and clearcast_note added to the scrub list. VAST URIs replaced with example.invalid placeholders so the third-party ad-server hostnames don't leak. - tests/helpers/freewheel_replay.py: shared make_response / replay_session helpers extracted from the three client test files to satisfy the code-duplication guard. Fixture file churn comes from the deterministic counter-based anonymiser picking different fake-name values now that creative_resources is in the input set; semantic content is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): live smoke test + fix XML empty-element coercion Adds a @pytest.mark.live integration test that exercises the whole FreeWheel client stack against the real publisher API: token info, inventory reads, commercial reads, creative reads, and a full Campaign → IO → Placement create-and-delete cycle. Skipped by default, runs only when FREEWHEEL_TEST_API_KEY + FREEWHEEL_TEST_ADVERTISER_ID are set. Running the test surfaced a real bug. The live API returns campaigns with ```` when no agency is assigned (empty XML element), and our ``_element_to_dict`` was emitting ``""`` for those — which Pydantic ``int | None`` fields couldn't coerce. Fixed by mapping empty leaf elements to ``None`` instead of ``""`` so optional scalar fields validate cleanly across the board. The earlier BeforeValidator on nested model fields (schedule/budget) becomes redundant for the empty-element case but stays in place as a safety net. Live test results against Talpa's network (2026-05-12): TestAuthAndConnectivity.test_token_info_returns_user_and_expiry PASS TestInventoryReads.test_list_sites_returns_entities PASS TestInventoryReads.test_list_videos_returns_entities PASS TestCommercialReads.test_list_advertisers_includes_test_advertiser PASS TestCreativeReads.test_list_creatives_returns_entities PASS TestWriteRoundTrip.test_full_create_and_delete_cycle PASS The write round-trip creates Campaign → IO → Placement, fetches the IO back, then deletes everything in reverse order. All six assertions land and all three deletes succeed — Mapping A wires correctly end-to-end through to the real API. Registered a ``live`` pytest marker so the suite doesn't need @pytest.mark.skipif boilerplate on every test. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): OAuth2 password-grant auth (api_token kept as escape hatch) Adds the canonical FreeWheel auth flow — username + password — alongside the pre-existing pre-minted api_token path. Production users now enter credentials once; the transport mints a bearer at POST /auth/token on first use, caches it with TTL tracking, and auto-refreshes on 401 or expiry. The api_token field is kept as an escape hatch for cases where a partner has provisioned a token out-of-band (our Talpa setup), or for testing without managing real credentials. Exactly one of (username + password) or api_token is required, enforced at three layers: - Pydantic model_validator on FreeWheelConnectionConfig - Constructor check in FreeWheelTransport - Init check in FreeWheelAdapter (live mode only) Transport behaviour: - api_token mode: bearer used directly, 401 propagates to caller. - password-grant mode: mint via POST /auth/token (data: grant_type=password, user_id, password). 401 triggers exactly one refresh + retry before propagating, in case the cached token rolled prematurely. expires_in is honoured with a 1-hour refresh leeway (or expires_in/2, whichever is smaller). UI: connection_config.html now has a "Sign-in Credentials (recommended)" section with User ID + Password fields, plus an "Advanced: pre-minted bearer token"
block for the escape hatch. The Save flow rejects submissions that have neither path. The Test Connection flow reports ``auth_mode: password_grant`` vs ``pre_minted_token`` in its response. Tenant-status reporting accepts either auth path. Config save/update endpoints accept username, password, and api_token, reject ciphertext replay on both secret fields, and pass the merged config to FreeWheelConnectionConfig for validation. Tests: - 15 new unit tests (password-grant mint + cache + 401-refresh + retry + error paths, plus full schema validation coverage for both auth paths). - Integration roundtrip tests cover both auth modes. - Live API test continues to pass via the api_token path (we don't have Talpa's username/password to exercise the password-grant path against the real API; that's unit-tested only until a real user/password pair shows up). Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): inventory taxonomy sync into local cache Adds a publisher-internal cache of FreeWheel inventory so the adapter's product setup UI can pick targeting from FW taxonomy without round-tripping to the FW API on every page render. New ``freewheel_inventory`` table (alembic 7c3073bd70cf), keyed by ``(tenant_id, entity_type, entity_id)`` with a denormalised name/parent_id plus a full JSON-blob ``raw_json`` payload. Stores eight entity kinds: - site (v4) - site_section (v4) - site_group (v4) - series (v4) - video_group (v4) - ad_unit_package (v4, with nested ad_units folded out) - ad_unit (v4, denormalised from ad_unit_packages — bare /ad_units/{id} is 403-denied on our scope, so we read them through their packages) - ad_unit_node (v3 XML; binds placement → ad_unit, read-only at v3) - standard_attribute (v4 reference data — TV ratings, languages, etc.) Individual Videos are NOT synced (4,613+ items on Talpa's network; query on-demand if a product needs to drill into a specific asset). This table is NOT exposed to AdCP buyers. Buyer-facing property discovery goes through the AAO lookup path (adagents.json + brand.json, via src/services/aao_lookup_service.py). The cache exists purely for the publisher's product configuration UI. See #378 for the cleanup of the deprecated AuthorizedProperty / PropertyTag tables that this design intentionally bypasses. Components: - alembic 7c3073bd70cf — create freewheel_inventory table - src/core/database/models.py — FreeWheelInventory ORM model - src/adapters/freewheel/inventory_sync.py — FreeWheelInventorySync service: walks every readable family, upserts via Postgres ON CONFLICT DO UPDATE. Per-family errors are captured in SyncResult rather than aborting (partial-success policy — some tenants will have uneven scope coverage across families). - POST /api/tenant//adapters/freewheel/sync-inventory — admin endpoint that reads the stored config, instantiates a client, and triggers the sync. - templates/adapters/freewheel/connection_config.html — "Sync Inventory Now" button + status display showing per-entity-type counts. - 7 new unit tests covering SyncResult dataclass, the dispatch orchestration with a mock client, partial-failure semantics, and the standard_attributes flat-dict code path. Verified end-to-end against Talpa's live network: 2,542 entities synced in one call (29 sites, 51 site_sections, 96 site_groups, 324 series, 507 video_groups, 2 ad_unit_packages, 385 ad_unit_nodes, 1148 standard_attributes), then re-runs upsert cleanly. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): product setup UI driven by synced inventory cache The FreeWheel product config schema and template are rebuilt around the actual FW data model. Instead of asking publishers to type comma-separated "placement IDs" (the old shape, which never made sense for our adapter since placements get created per buy), the new product setup UI pulls choices from the local ``freewheel_inventory`` cache: - Sites (delivery destinations) - Site Sections (optional sub-section scoping) - Video Groups (audience-segmented content — Talpa's primary targeting primitive: "DOELGROEP INDEX 150+", etc.) - Series (specific shows) - Ad Unit Package (slot bundle: Pre-Mid, Pre-Mid-Post) - TV Ratings (content rating restrictions, from standard_attributes) The picker template (templates/adapters/freewheel/product_config.html) populates each picker that loads from the freewheel_inventory cache via the existing GET /api/tenant//adapters/freewheel/inventory endpoint. The adapter's dry-run _line_item_payload echoes every list dimension so operators can verify intent in dry-run logs before flipping to live mode. Note: custom_targeting (the FW v4 custom_keys API) is still gated by scope on our token — kept as the escape hatch under an Advanced
, but most use cases that would have needed it on GAM are covered by the structured fields above. ## FreeWheelAdapter.get_creative_formats() — six canonical VAST formats New src/adapters/freewheel/formats.py declares six static AdCP-shaped formats covering pre/mid/post-roll × 15s/30s: freewheel_video_15s_pre_roll freewheel_video_30s_pre_roll freewheel_video_15s_mid_roll freewheel_video_30s_mid_roll freewheel_video_15s_post_roll freewheel_video_30s_post_roll Each format declares a single VAST tag URL asset and {vast: true} delivery hint. Validated against adcp.types.Format on every test run. Declared statically (Option A) rather than synthesised from synced data because (a) AdCP's format registry is mostly static, (b) the six combinations cover the common buyer case for video VAST forwarding, and (c) static format IDs stay stable across inventory-sync runs so buyer references don't break when Talpa edits their ad_unit_packages. Tests: - 6 unit tests for the static format list and Format schema validation - Schema test for the new product config fields, full round-trip via model_dump → model_validate 4328 unit tests pass. Live FW integration test still green via api_token escape hatch. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): surface synced inventory via get_available_inventory + README refresh Two free wins that need no FW scope grant: - Override AdServerAdapter.get_available_inventory() to serve the AI product configurator from the freewheel_inventory cache. Returns placements (ad_unit_packages), ad_units (sites + site_sections), targeting_options (standard_attributes grouped by taxonomy key), the static VAST creative specs, and cache properties. Live-verified against Talpa: 2 packages, 80 ad units, 15 targeting groups, 6 formats, 1148 attributes. - Rewrite docs/adapters/freewheel/README.md to match what's actually shipped: password-grant auth (with api_token escape hatch), full inventory sync taxonomy, 18-dimension product config, live coverage matrix, and the layered scope-grant ask (Tier 1: lifecycle; Tier 2: reporting; Tier 3: operator UX; Tier 4: future). Previous README still described the client_credentials path we abandoned and claimed skeleton-only status. Test: tests/unit/test_freewheel_adapter.py::TestGetAvailableInventory covers shape and grouping semantics with mocked repository. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): reporting cache scaffold — read paths + stub sync (scope-pending) Build everything that's anchored on AdCP's contract (which is fixed) so day-of-scope the only new work is the actual FW Reporting API client. Adds: - Migration + ORM: freewheel_placement_stats cache table (per-placement impressions/spend_micros/completed_views/clicks/currency/delivery_status, keyed by (tenant_id, placement_id), with IO-scoped index for delivery aggregation). Spend stored in micros to dodge floating-point drift. - Repository (FreeWheelPlacementStatsRepository): tenant-scoped reads via get_by_placement_ids() and list_by_insertion_order(), plus a Postgres ON CONFLICT bulk_upsert() for the sync job to call. - get_packages_snapshot(): reads from cache, returns Snapshot per package. Missing rows surface as None so callers render a "no data" state rather than fail. Staleness derived from row.as_of. Delivery status mapped to the AdCP DeliveryStatus enum where the FW value maps cleanly. - get_media_buy_delivery(): aggregates per-placement rows into DeliveryTotals + by_package list. Empty cache falls through to the base helper's zero-response shape. - FreeWheelReportingSync stub: raises ReportingScopeNotGranted with a pointer to the README scope ask. Schedulers can catch this and degrade gracefully — read paths already tolerate the empty-cache state. Tests pin the read-side contract (tests/unit/test_freewheel_reporting_cache.py, 7 cases). When FW grants Tier 2 scope, the only new work is implementing the four private methods on FreeWheelReportingSync (submit_job, poll_job, fetch_results, parse_rows) against the real Query Reporting endpoint. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(tenant-mgmt-api): typed adapter configs + GET /adapters discovery Two gaps that blocked Scope3 from using anything beyond GAM + Mock through the typed tenant-management API: 1. The AdapterConfig discriminated union in src/admin/api_schemas/ tenant_management.py only listed GAM + Mock. Typed embedder clients couldn't POST type="freewheel" / "triton" / "broadstreet" — spectree validation rejected anything else, even though the legacy /api/tenant//adapter-config endpoint (operator-facing) handled them. Adds: - FreeWheelAdapterConfig (with the username+password OR api_token cross-field rule) - TritonAdapterConfig (auth_type + creds + base/login URLs) - BroadstreetAdapterConfig (network_id + api_key) Secrets use SecretStr. Persistence round-trips through each adapter's own connection schema so Fernet encryption lands consistently in AdapterConfig.config_json — same path the legacy endpoint uses. 2. No way to discover what adapters this Sales Agent instance supports. Adds GET /api/v1/tenant-management/adapters returning the full catalog per adapter type: name, description, default_channels, capabilities (mirrors AdapterCapabilities), and the connection_schema JSON Schema so embedders can validate locally before POSTing. Sourced from ADAPTER_REGISTRY so new adapters auto-appear once they're registered and have a typed AdapterConfig member. Test plan: - tests/unit/test_tenant_management_schemas.py: 13 new schema-level tests covering each typed config's happy path + rejection paths + discriminator routing through ProvisionTenantRequest. - tests/integration/test_tenant_management_api_integration.py: 2 new endpoint tests (catalog shape + auth gate). - Regenerated docs/api/tenant-management-openapi.{json,yaml}. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(adapters): adapter playbook — phase-by-phase checklist for new adapters 11-phase walkthrough of everything needed to add a new ad-server adapter end-to-end: pre-work API probing, adapter package scaffolding, inventory + reporting caches, three-place registration (registry / typed API config / discovery catalog), admin UI, admin endpoints, test coverage, docs, OpenAPI regeneration, smoke + quality gates, common gotchas, ship. FreeWheel is called out as the reference implementation with specific file pointers per step. Captures the lessons from PR #381 — stale uvicorn imports, migration head collisions, DeliveryStatus enum mismatch, BuildKit stale-deps surprise, the DRY guard ratchet, etc. Also fixes the stale FreeWheel description in docs/adapters/README.md (client_credentials → password grant) and surfaces the new playbook from the index. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(tenant-mgmt-api): add tier flag to adapter discovery (mock=test, rest=live) Emma flagged that the discovery endpoint exposes Mock to embedders — which is real (it's a registered adapter we use in tests and demos) but should never appear in a production storefront's picker. Adds: - ``tier`` field on AdapterCatalogEntry: ``"live"`` (production adapter) or ``"test"`` (simulated/dev-only). Mock is the only ``"test"`` adapter today; everything else is ``"live"``. - ``?tier=live`` and ``?tier=test`` query filter on GET /adapters so production storefronts can opt out of seeing the test surface server-side (rather than having every embedder filter client-side). Unknown values return 400. Default behaviour returns all adapters with their tier tag so dev consoles keep seeing the full set. Production storefronts pass ?tier=live. Test plan: - 3 new endpoint tests in test_tenant_management_api_integration.py (live filter excludes Mock, test filter returns only Mock, invalid value rejected with 400) — all green. - Existing catalog assertion updated to check the new tier field. - Regenerated docs/api/tenant-management-openapi.{json,yaml}. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(adapters): park Triton — APIs not production-ready, surface removed Triton told us their TAP Media Buying API isn't production-ready (2026-05). Surface-removal approach (vs. full deletion) so we can restore via revert when their APIs come back. Removed from every customer-facing surface: - ADAPTER_REGISTRY: triton + triton_digital entries dropped, so tenants cannot select 'triton' as ad_server (legacy POST /tenants now rejects it via an ADAPTER_REGISTRY membership check; spectree POST /tenants/provision rejects via the discriminated AdapterConfig union). - AdapterConfig discriminated union: TritonAdapterConfig removed. - Discovery catalog (_ADAPTER_CATALOG_METADATA + _ADAPTER_CONFIG_TYPED): triton excluded from GET /api/v1/tenant-management/adapters. - tenant_settings.html: picker card hidden (with a comment pointing at the parked module path for restoration). - adapters.py blueprint: test_triton_connection endpoint removed. - docs/adapters/README.md: Triton section + table row replaced with a short parked-state notice. - docs/adapters/triton/README.md → README.parked.md with a header explaining the parked state. Kept parked (so restoration is a revert, not a rebuild): - src/adapters/triton/* — the adapter module + client + targeting - tests/unit/test_triton_*.py — direct-construction tests still run; TestRegistry tests flipped to assert parked-state behaviour. - templates/adapters/triton/* — connection + product templates unreachable but preserved. - Alembic migrations — unchanged. Existing tenants whose adapter_type is already 'triton' (if any) remain operable: the update path in tenant_management_api.py preserves their config_json handling. Tests: - test_tenant_management_schemas.py: removed TritonAdapterConfig happy- path tests; added test_provision_request_rejects_parked_triton_adapter to pin the embedder-side rejection. - test_tenant_management_api_integration.py: catalog assertions exclude triton from both the all-adapters and tier=live responses. - test_new_product_filters.py + test_triton_adapter.py registry tests flipped to assert the parked-state behaviour. - 4,367 passed / 14 skipped / 19 xfailed — all green. - Regenerated docs/api/tenant-management-openapi.{json,yaml}. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(adapters): permission probes — catch IAM gaps at connect, not mid-campaign Brian flagged that operators today discover missing upstream permissions when a campaign fails halfway through, instead of seeing them at connect time. This adds a structured permission-check primitive on AdServerAdapter and a live FreeWheel implementation. The pattern: 1. AdServerAdapter.check_permissions() returns a PermissionsReport with per-endpoint PermissionCheck entries: name, description, granted, required vs nice-to-have, feature label (creative_trafficking, delivery_reporting, etc.), and a detail string for failed probes. 2. fully_operational rolls up to True only when every required probe passes. Optional probes can deny without blocking the rollup — surfaces partial-scope state correctly. 3. Each adapter implements its own probe — auth flows differ enough that a generic HTTP prober doesn't fit. Base class returns an empty report so adapters that haven't implemented yet behave as "no checks declared, fully_operational=True". FreeWheel implementation probes 14 endpoints covering auth, inventory sync, commercial CRUD, creative trafficking, reporting, audiences, targeting profiles, and webhooks — every AdCP feature path. Live probe against Talpa correctly reports our current state: 9 required probes granted, 1 required denied (/services/v4/ads — the creative trafficking blocker), 4 optional denied (reporting + audiences + targeting profiles + webhooks). Probe semantics that took some thought: - 4xx validation (400/404/422) counts as GRANTED — endpoint accepts the call, just needs different params. Our minimal probes intentionally send empty payloads so we don't accidentally mutate state. - 401/403 count as denied (real scope gap). - Auth-token failures bail the whole pass with report.error set; we don't paint every endpoint as "denied" when the real problem is a bad token, that'd mislead operators. New admin endpoint: POST /api/tenant//adapters//check-permissions Loads the configured adapter, runs check_permissions(), returns the JSON report. Read-only (every probe is a GET) so opts into the embedded-write gate. Available to admin or member roles. Test plan: - tests/unit/test_freewheel_permissions.py: 11 cases covering dry-run short-circuit, granted/denied semantics, the validation-error edge case, 401 mapping, auth failure handling, probe target cleanup, and the every-check-has-a-feature invariant. All pass. - Live-verified against Talpa: correctly identifies /services/v4/ads as the one required denial blocking creative trafficking. - make quality: 4,378 passed / 14 skipped / 19 xfailed. Follow-up not in this PR: UI rendering of the checklist on the adapter settings page; surfacing fully_operational on the discovery catalog. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): permission-check UI + correct creative_trafficking model Brian flagged that FW's docs don't actually have an "Ad" concept and asked us to verify. Re-reading the creative_instances POST docs revealed the parameter ad_id is described as "The Ad Unit Node ID to link Creative" — there is no separate Ad object. The /services/v4/ads scope we were chasing was misdirected. Verified live: POST /services/v4/creative_instances with ad_id= and creative_id= returns 201 Created with FW auto-deriving placement_id on the response. The entire creative trafficking flow is unblocked today. UI: - Added "Check API Permissions" button to the FreeWheel adapter settings page. Hits POST /api/tenant//adapters/freewheel/check-permissions, renders a per-feature checklist showing granted/denied with the probe target endpoint and the AWS API Gateway deny detail when the scope is missing. Operators see at-connect-time which AdCP features will work, instead of discovering missing scopes mid-campaign. Code: - check_permissions probe list: dropped v4_ads (wasn't needed); kept v4_creative_instances as the required probe with a comment explaining the ad_id ↔ ad_unit_node_id alias. - Unit tests updated to use creative_instances as the canonical required probe (11 cases still passing). - Live probe against Talpa now reports fully_operational=true. Only Tier-3/4 nice-to-haves remain denied (reporting, audiences, targeting profiles, webhooks). Docs: - README "Scope grants still needed" rewritten: Tier 1 ads grant is removed; Tier 1 is now Query Reporting (path TBD). Added a "What we no longer need to ask for" section explaining the ad_id alias and the v4-doesn't-exist-for-commercial finding so the next person looking at this doesn't go down the same dead ends. - Coverage matrix: add_creative_assets flipped from 🟡 partial (blocked) to ✅ unblocked; associate_creatives from ⏳ blocked to 🟡 wired-ready (FW writes work — adapter just needs the ad_unit_node lookup chain from cache, which is a follow-up). Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): pinpoint Reporting API location, update scope ask + probes Probed FW's full host surface to find where their Reporting API actually lives — turns out it's at api.freewheel.tv/reporting/* (singular, host root, NOT under /services/v*). Every /reporting/* path returns the AWS API Gateway IAM-deny payload for our test user, confirming the resources exist and only a scope grant is needed. Verified surface (all currently denied): POST /reporting/jobs — submit async job GET /reporting/jobs/{id} — poll status GET /reporting/jobs/{id}/result(s)/download — fetch CSV/JSON GET /reporting/queries + /saved_queries — saved-query CRUD GET /reporting/dimensions + /metrics — schema introspection Adjacent host-root paths (/reports, /reporting at the top, /insights, /analytics, /graphql, etc.) all returned nginx-level HTML denies rather than AWS IAM-deny, confirming /reporting/* is the actual surface and others are dead-end aliases. Code: - check_permissions(): swapped the wrong /services/v4/reports probe for two correct /reporting/* probes (schema introspection + job submit). - reporting_sync.py: dropped TBD docstring; now documents the full /reporting/* surface map so day-of-scope is just filling in the four private methods. - Unit test parameter updated to match the new probe path. Docs: - README "Scope grants still needed" now lists the specific endpoints to ask for. Updates the Mathijs ask from "we don't know where reporting lives" to "grant our user IAM access to /reporting/* — specific paths listed". Live probe (against Talpa, user 35696) cleanly shows fully_operational=true plus two new entries under feature=delivery_reporting both denied, ready to flip the moment scope arrives. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(migrations): re-point FW merge revision to chain through r0s1t2u3v4w5 origin/main added migration r0s1t2u3v4w5_add_proposals_table.py which descends from 8820c87e8ae3 — the same parent our merge revision 190d6e98754b already chained from. That made them siblings and produced two migration heads, which the test_architecture_single_migration_head guard catches at quality-gates time. CI was tripping on the same check because migrate.py refused to apply with "Multiple head revisions are present", which cascaded into every DB-touching integration + E2E test. Re-point 190d6e98754b's main-side parent from 8820c87e8ae3 to r0s1t2u3v4w5 (which itself descends through 8820c87e8ae3 → 17423a1b551e → base). Graph converges to a single head; alembic history is linear-ish again. Verified locally: $ uv run alembic heads 190d6e98754b (head) $ make quality 4,438 passed / 14 skipped / 19 xfailed The commit message comment in the migration is updated to note that the main-side parent will move forward as new migrations land on main — each subsequent origin/main merge re-points this parent again. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): implement Reporting client + wire sync end-to-end Builds the Query Reporting API client speculative-but-defensive against the /reporting/* surface we mapped via probing. Day-of-scope this fires real reports against FW; day-zero it raises ReportingScopeNotGranted cleanly on the 403. Code: - src/adapters/freewheel/_reporting.py: FreeWheelReportingClient with submit_job / get_job / wait_for_completion / fetch_results. JobSpec (Pydantic) serialises POST /reporting/jobs bodies. JobStatus parses enum-strings case-insensitively and clamps unknown values to UNKNOWN so a new FW status doesn't break the polling loop. JobState parses both snake_case and camelCase payloads defensively, preserves the raw dict for fields we don't yet know about. ColumnMap is a single tunable for FW's result column names — day-of-scope edit one place when we see the real column labels. - src/adapters/freewheel/reporting_sync.py: FreeWheelReportingSync.run() now actually orchestrates submit/poll/fetch/upsert. ForbiddenError caught once at the top and re-raised as ReportingScopeNotGranted so callers get a clean signal. Cache upsert via the existing FreeWheelPlacementStatsRepository.bulk_upsert. - src/adapters/freewheel/_transport.py: added post_json + delete_json helpers (existing v3 surface only had post_xml + delete_xml). - src/admin/blueprints/adapters.py: new POST endpoint /api/tenant//adapters/freewheel/sync-reporting — admin-only, same shape as sync-inventory. Returns 503 with scope_pending=true when the upstream IAM-denies us so the UI can render the right copy. - templates/adapters/freewheel/connection_config.html: added "Sync Reporting Now" button + syncFreeWheelReporting() JS. UI lives between Inventory Sync and API Permissions so the operator's first three buttons match the natural flow: connect → inventory → reporting. Tests (tests/unit/test_freewheel_reporting_client.py — 27 new): - JobSpec serialisation (minimum + with filters) - JobStatus parsing (5 known values + unknown clamps to UNKNOWN + None) - JobState parsing (snake_case + camelCase + preserves raw + error_message) - parse_row (default map, string-numbers coercion, missing fields, garbage input, custom ColumnMap remap, as_of fallback to now) - submit_job round-trips the request body - wait_for_completion (immediate-terminal, polls PENDING→RUNNING→COMPLETED, timeout raises with last state, CANCELED is terminal) - fetch_results (inline rows, alternate keys 'rows'/'results'/'data', raises when job not complete) Cache test updated (tests/unit/test_freewheel_reporting_cache.py): the scope-handling tests now patch transport.post_json to raise FreeWheelForbiddenError, matching production behaviour rather than the old "raises unconditionally" stub. Live verified (against Talpa, user 35696): sync.run() correctly attempts POST /reporting/jobs, FW returns 403, our code catches it once and raises ReportingScopeNotGranted with the friendly message. Day-of- scope the same code path will fire and run the actual report — if FW's request shape differs from our spec, ColumnMap + JobSpec serialisation have explicit edit points. Docs: README live-coverage matrix updated: get_media_buy_delivery / get_packages_snapshot: ⏳ stub → 🟡 wired (reads cache; populated by sync once scope lands) make quality: 4,465 passed / 14 skipped / 19 xfailed (gained 27 tests). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(freewheel): stop false-zero delivery webhooks when reporting cache is empty Brian flagged that the DeliveryWebhookScheduler runs hourly automatically, calls adapter.get_media_buy_delivery() per active media buy, and fires webhooks to buyers — but our FW adapter was returning zero-delivery responses when the placement_stats cache was empty (which is the steady state today, since the Reporting API scope is still pending). Buyers polling AdCP or subscribing to delivery webhooks would see fake "delivering=0 impressions" signals every hour, which is misleading. Fix: introduce a soft-error signal that distinguishes "integration healthy, no data YET" from "integration broken": - New AdServerAdapter base exception DeliveryDataUnavailable. Adapters raise it when they have no data to report but nothing is actually wrong upstream — typical causes: cache not yet populated, upstream reporting scope still pending. Shareable across adapters. - FreeWheelAdapter.get_media_buy_delivery now raises DeliveryDataUnavailable when the placement_stats cache has no rows for the requested insertion order, instead of returning zeros via the base _empty_delivery_response helper. - _get_media_buy_delivery_impl catches DeliveryDataUnavailable separately from the generic adapter-error catch-all. The clean error surfaces as a GetMediaBuyDeliveryResponse with errors=[Error( code="data_unavailable")] — no audit log, no warning-level noise, just an info-level "data not yet available" log. - DeliveryWebhookScheduler's soft-skip set widened from just {"media_buy_status_excluded"} to also include "data_unavailable" — same info-level skip, no false-zero webhook fires. Test (tests/unit/test_freewheel_reporting_cache.py): the empty-cache test flipped from "returns zero response" to "raises DeliveryDataUnavailable with media_buy_id set". This pins the contract that AdCP layer + scheduler depend on. Defer-list captured in expanded comment on #382: the proper fix for this whole area (per-adapter buttons → shared scheduler + uniform adapter contract + /admin/scheduling page) is significant scope and should be its own PR after #381 merges. This change is the minimum-surgical fix to stop bad signals today. make quality: 4,465 passed / 14 skipped / 19 xfailed. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel ui): proactive scope-pending banner on adapter settings page Operators were finding out about missing FW scope two ways: by clicking "Check API Permissions" (which they wouldn't do unprompted), or by a buyer reporting "I'm seeing data_unavailable for delivery." Both are surprises. Surface the state on page load instead. Adds a banner above the FW configuration form that auto-runs check_permissions and renders one of three states: - (no banner): fully_operational, nothing surprising. - (warn, amber): reporting scope is denied. Banner explains buyers will see data_unavailable until granted; other features work normally. Reporting is technically a "nice-to-have" probe in our report shape, but its absence has real operator-visible consequences worth flagging. - (error, red): a required probe failed. Banner lists the missing features. Other denied nice-to-haves alone (targeting_profiles, audiences, webhooks) don't trigger the banner — they're true optionals and would become noise. They remain visible in "Check API Permissions" for operators who care. Banner has a "See full permissions checklist →" link that scrolls down and triggers the existing on-demand probe so the operator sees the full per-endpoint breakdown. Quietly no-ops on auth-level failure / no creds (those are surfaced by the credentials section already) and on transient probe failures (page should still load). make quality: 4,465 passed. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(freewheel): wire creative trafficking + stale-cache freshness banner Two gaps closed in one commit so the FW adapter is feature-complete for the buyer-facing flow (create buy → traffic creatives → see delivery) before #381 merges. ### Creative trafficking — end-to-end Earlier we proved /services/v4/creative_instances POST works (via Brian's docs-read showing ad_id = ad_unit_node_id). Now actually wired: - src/adapters/freewheel/_creatives.py: added create_creative, delete_creative, create_creative_instance, delete_creative_instance. Two wire-shape gotchas captured (verified live against Talpa): * POST creative_resources: body must be wrapped under {"creative": {...}}; flat body returns 400 "Creative Node is missing". Response is doubly-wrapped: {"data": {"creative": {...}}}. * POST creative_instances: ``ad_id`` is FW's param name but its docs say "The Ad Unit Node ID to link Creative." Response auto- populates placement_id (FW derives it from the ad_unit_node). - src/adapters/freewheel/adapter.py: * add_creative_assets: POSTs one creative_resource per AdCP asset, stamps the AdCP id onto FW external_id for lineage, returns AssetStatus(creative_id=, status="approved"). * associate_creatives: looks up ad_unit_node_ids per placement from the inventory cache, POSTs one creative_instance per (node, creative) pair. Per-binding result rows so callers see partial successes. Skipped placements (no cached ad_unit_nodes → run inventory sync first) get a clear message rather than silent failures. Live cycle verified end-to-end against Talpa: create_creative → create_creative_instance → delete_creative_instance → delete_creative, all clean. ### Stale-cache freshness banner Two new repository methods (latest_sync_at) on the inventory + placement- stats repos. New GET /api/tenant//adapters/freewheel/cache-freshness endpoint returns last_synced_at + age_seconds + stale flag + threshold for both caches. Threshold defaults: 24h inventory, 2h reporting. UI: second banner above the FW config form (alongside the scope-pending banner). Renders only when something needs flagging: - blue (info): cache never synced — onboarding gap - amber (warn): cache stale — sync probably broken - no banner: everything fresh ### Test infra cleanup Extracted tests/helpers/freewheel_adapter_patches.py::patch_freewheel_db so the same FreeWheelInventoryRepository + get_db_session monkeypatch block isn't duplicated across test modules. Both test_freewheel_adapter.py and test_freewheel_creative_trafficking.py now use the helper. Duplication guard happy again. ### Tests - tests/unit/test_freewheel_creatives.py: +4 cases pinning the write surface (wrapped POST body, ad_id semantics, DELETE paths). - tests/unit/test_freewheel_creative_trafficking.py: 9 cases — dry-run + live + fan-out + partial-failure + missing-inventory-skip. - All existing tests still pass via the shared helper. make quality: 4,477 passed (gained 12). Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(audit): stop double-encoding audit_logs.details + repair migration (#409) * fix(audit): stop double-encoding audit_logs.details + repair migration log_security_violation passed details=json.dumps({...}) into the JSONB column. JSONType.process_bind_param then serialized that already-stringified JSON again, so the row landed with a JSONB value of type 'string' instead of 'object'. Strict readers (notably tenant_export.py) refuse those rows, which blocked tenant exports on every tenant that had ever had a security violation logged — ~1,272 rows across multiple production tenants. Fix: - src/core/audit_logger.py:282 — pass the dict directly; JSONType handles serialization. One-line change. - alembic migration s1t2u3v4w5x6 — repair existing rows with UPDATE audit_logs SET details = ((details::jsonb) #>> '{}')::jsonb WHERE details IS NOT NULL AND jsonb_typeof(details::jsonb) = 'string'; Idempotent: re-running matches zero rows on a clean DB. Downgrade is intentionally unsupported (re-encoding would re-introduce the bug); raises NotImplementedError with an explanation rather than silently corrupting. - tests/integration/test_audit_logger_details_shape.py — regression test asserting log_security_violation persists details as a JSONB object, not a JSON string. Checks both the ORM read (dict) and Postgres jsonb_typeof = 'object'. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(migration): make s1t2u3v4w5x6 downgrade a true no-op The previous downgrade raised NotImplementedError to refuse re-corrupting repaired rows, but that broke test_managed_tenant_migrations_roundtrip which drives the chain backward to verify reversibility. Replace the raise with a SQL NOTICE. The body stays non-empty (migration- completeness guard happy), the data fix stays in place on downgrade (repaired rows are schema-compatible with all prior revisions), and the roundtrip test can step through. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(migration): reconcile divergent alembic heads from PR #381 + #409 (#412) PR #381 (freewheel placement stats, head 190d6e98754b) and PR #409 (audit_logs.details repair, head s1t2u3v4w5x6) both branched off r0s1t2u3v4w5 and merged into main ~15 minutes apart. The second to land didn't rebase onto the first, so main shipped with two alembic heads. `alembic upgrade head` refuses a divergent graph without an explicit target, so the embedded-salesagent migrate job exits 1 and crashloops on rollout (helm reports BackoffLimitExceeded). Adds an empty merge revision (d0c3c40fdd41) with both heads as parents. Pure graph reconciliation — no schema or data changes. Merge migrations are exempt from the migration-completeness guard by design (tests/unit/test_architecture_migration_completeness.py:6). After this lands, `alembic heads` returns a single head and the migrate job can `upgrade head` without ambiguity. Verified: - `uv run alembic heads` → d0c3c40fdd41 (head) [single] - test_architecture_single_migration_head ✓ - test_architecture_migration_completeness (6 tests) ✓ Co-authored-by: Claude Opus 4.7 (1M context) * feat(adapters): shared sync orchestration + uniform contract (#382) (#411) * feat(adapters): uniform sync contract on AdServerAdapter (Stage 1 of #382) Defines what the shared AdapterSyncScheduler (later stages) will call on every adapter. Today's per-adapter sync surfaces (GAM's background_sync_service hardcoded to GAM, FreeWheel's per-adapter buttons) will all migrate behind this contract in Stages 2-3. Adds: - AdapterCapabilities.supports_reporting_sync flag. Distinct from supports_realtime_reporting (which is a buyer-facing capability the scheduler doesn't care about); this controls whether the scheduler should periodically run adapter.run_reporting_sync(). Default False so adopting the contract is opt-in. - AdapterSyncResult dataclass — uniform return shape for both sync kinds. counts (free-form per-kind tally) + errors (partial-failure capture) + metadata (job_id for reporting, etc.) so the scheduler can persist results without knowing per-adapter internals. - AdServerAdapter.run_inventory_sync() / run_reporting_sync() with NotImplementedError defaults whose messages tell operators how to fix it ("override the method, or flip the capability flag off"). - AdServerAdapter.latest_inventory_sync_at() / latest_reporting_sync_at() returning None by default. Adapters with caches override to expose the most-recent last_synced_at — uniform freshness signal for the /admin/scheduling UI. Test plan (tests/unit/test_adapter_sync_contract.py — 9 cases): - AdapterSyncResult math + metadata - Capability flag defaults + independence - Default NotImplementedError raises with actionable message - Default freshness accessors return None - Override returning AdapterSyncResult is accepted as the contract Also pulls in a pre-existing reformat of embedded_tenant_guard.py that ruff flagged on a fresh main checkout. make quality: 4,500 passed / 14 skipped / 19 xfailed. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(adapters): wire FW + GAM to the sync contract (Stage 2 of #382) FW implements the full contract; GAM declares capabilities + freshness accessor. Their actual sync methods stay as-is — Stage 3 will refactor background_sync_service to dispatch through the contract uniformly. ### FreeWheel — full contract - supports_inventory_sync (already on) + supports_reporting_sync (new). - run_inventory_sync(): wraps FreeWheelInventorySync, converts its internal SyncResult to AdapterSyncResult. Dry-run returns a soft- failed result so the scheduler doesn't try to interpret an exception. - run_reporting_sync(): wraps FreeWheelReportingSync. Catches ReportingScopeNotGranted specifically and surfaces metadata.scope_pending =True so the shared scheduler can render that differently from a generic failure. Catches the broader ReportingError too. - latest_inventory_sync_at(): reads FreeWheelInventoryRepository. latest_sync_at() (already exists from PR #381). - latest_reporting_sync_at(): reads FreeWheelPlacementStatsRepository. latest_sync_at() (same). ### Google Ad Manager — capability + freshness only - supports_inventory_sync flipped on; supports_reporting_sync stays False because GAM doesn't have a separate reporting sync — line item stats are written by gam_orders_service inside the inventory sync run. - latest_inventory_sync_at(): reads the most-recent completed SyncJob row for this tenant. The existing background_sync_service already writes there, so we get freshness for free. - run_inventory_sync() NOT implemented yet — GAM's sync is async/ threaded and refactoring it cleanly to the synchronous contract is Stage 3. Today calling it raises the base NotImplementedError; Stage 3 makes it work. ### Tests tests/unit/test_freewheel_sync_contract.py (6 cases): - Capabilities declared correctly - Dry-run returns soft-failed AdapterSyncResult (both kinds) - Scope-pending caught → metadata.scope_pending=True - Freshness accessors wire through to the right repos make quality: 4,506 passed (gained 6 over Stage 1). Co-Authored-By: Claude Opus 4.7 (1M context) * feat(adapters): shared adapter sync orchestration (Stage 3 of #382) Routes the FreeWheel per-adapter sync buttons through a new shared ``adapter_sync_orchestration`` service that owns AdapterConfig lookup, adapter construction, SyncJob persistence, and uniform error surfacing. Stage 4 (the /admin/scheduling page) reads from the SyncJob table and the Stage 5 scheduler will call the same orchestration on a cadence — both get every adapter's sync runs for free now. ### Orchestration (``src/services/adapter_sync_orchestration.py``) - ``execute_sync(adapter, tenant_id, sync_kind, triggered_by, ...)`` — creates a SyncJob row (status="running"), dispatches to ``adapter.run_inventory_sync()`` or ``run_reporting_sync()``, persists the AdapterSyncResult onto the row (status="completed"/"failed", counts + errors + metadata in progress JSON, first error stamped to error_message). Catches adapter exceptions defensively so a misbehaving adapter doesn't break the scheduler loop. - ``execute_adapter_sync(tenant_id, adapter_type, sync_kind, triggered_by, run_kwargs)`` — convenience wrapper that resolves AdapterConfig + constructs the adapter, then calls execute_sync. The entry point per- adapter buttons + the scheduler both go through. - ``AdapterDoesNotSupportSyncKind`` raised when capabilities flag is off — fail-fast at the boundary so it's a 4xx-shaped error, not a 5xx. - ``SyncExecutionResult.scope_pending`` convenience property reading metadata.scope_pending so UI can render the awaiting-scope state. - Supports ``run_kwargs`` so adapter-specific run params (FW's placement_ids / start_date / end_date) flow through. ### FreeWheel endpoint refactor (``src/admin/blueprints/adapters.py``) - ``sync_freewheel_inventory`` + ``sync_freewheel_reporting`` no longer instantiate FreeWheelInventorySync / FreeWheelReportingSync directly — they call ``_execute_freewheel_sync`` which dispatches through the shared orchestration. Response shape preserved so the existing FW settings UI still works. ``sync_id`` added to the response so future UIs (Stage 4) can link from the FW page to the SyncJob detail view. - ``run_reporting_sync`` on the adapter now accepts ``placement_ids``, ``start_date``, ``end_date`` kwargs (forwarded by the orchestration's ``run_kwargs``). The contract method declares them; FreeWheelReportingSync already supported them. ### GAM (unchanged in this stage) GAM's async ``background_sync_service`` still owns its own SyncJob row writes — the two patterns coexist and write to the same table so the Stage 4 UI sees a unified feed. Migrating GAM behind the synchronous contract requires a chunkier refactor (threaded → coroutine, progress polling) and is deliberately deferred. ### Tests - tests/unit/test_adapter_sync_orchestration.py (3 cases) — capability gating + unknown sync_kind rejection. Pure unit, no DB. - tests/integration/test_adapter_sync_orchestration.py (3 cases) — SyncJob persistence on success, on soft failure (scope_pending), and on adapter exceptions. - Shared helper at tests/helpers/sync_orchestration.py to satisfy the DRY guard (mock-adapter construction was identical across the two). make quality: 4,509 passed (gained 9 over Stages 1+2). Co-Authored-By: Claude Opus 4.7 (1M context) * feat(admin): cross-tenant adapter sync scheduling page (Stage 4 of #382) Adds /admin/scheduling — super-admin view of every configured (tenant, adapter, sync_kind) triple with last-run status, freshness verdict, and a Run Now button that dispatches through the Stage 3 orchestrator. Builds on Stages 1-3: * Stage 1 contract defined supports_inventory_sync / supports_reporting_sync * Stage 2 wired FW (both kinds) + GAM (inventory only) to the contract * Stage 3 introduced execute_adapter_sync / SyncJob persistence Pieces: - SyncJobAdminRepository — cross-tenant queries (latest_per_kind, latest_for_triples, list_recent). Distinct class so the existing SyncJobRepository keeps its tenant-isolation invariant. - AdapterConfigAdminRepository — list_all() joins Tenant.name for the matrix's display column. - src/services/sync_scheduling_view.py — assembles SchedulingRow list driven by AdapterCapabilities flags (capability gating happens at the matrix layer, not at the per-row template). - src/admin/blueprints/scheduling.py — GET /admin/scheduling, GET /admin/api/scheduling/jobs (matrix JSON for refresh), GET /admin/api/scheduling/recent (history log), POST /admin/api/scheduling/run (dispatches via execute_adapter_sync). - templates/scheduling.html — table + in-page JS refresh on Run Now click. Surfaces scope_pending separately (503) from generic failure (500). - Super-admin nav link added to templates/base.html. Freshness thresholds (24h inventory, 2h reporting) match adapters.py::freewheel_cache_freshness so the per-tenant and cross-tenant views agree. A failed run leaves stale=True since the underlying cache wasn't refreshed. Tests: 12 unit (capability filtering, stale verdict, dict shape) + 7 integration (cross-tenant queries, matrix end-to-end, stale verdict against backdated rows) + 8 endpoint (auth gating, JSON shape, Run Now dispatch + scope_pending + unconfigured tenant). Co-Authored-By: Claude Opus 4.7 (1M context) * feat(scheduler): hourly cross-tenant reporting sync (Stage 5 of #382) Adds AdapterReportingSyncScheduler — fixed-interval (1h default, configurable via ADAPTER_REPORTING_SYNC_INTERVAL) loop that: 1. Lists every (tenant, adapter) pair whose adapter declares supports_reporting_sync=True (FW only today). 2. Skips tenants whose last successful reporting sync is within REPORTING_STALE_AFTER (2h) — keeps the scheduler off the freshness threshold's hot path and prevents thundering-herd retries. 3. Skips tenants with an in-flight (status=running) sync — don't pile on if a previous cycle is still working. 4. Dispatches eligible pairs through execute_adapter_sync() so the resulting SyncJob rows show up in the Stage 4 /admin/scheduling view. Wired into core/main.py's _start_schedulers / _stop_schedulers so the lifespan hook on adcp.server.serve() boots it alongside the existing delivery-webhook + media-buy-status schedulers. Stage 3 sidefix: removed the bogus tenant_id= kwarg from the stub Principal construction in execute_adapter_sync(). Principal schema doesn't accept tenant_id and Pydantic's extra=forbid raised ValidationError on every scheduled run. Tests: 10 unit (eligibility filtering, run_once dispatch, lifecycle) + 5 integration (real-DB eligibility queries, end-to-end run_once against a stub adapter wired through the real orchestrator). DRY: extracted ``cancel_scheduler_task`` to _scheduler_lifecycle.py rather than inline the cancel/await/CancelledError dance — the existing two schedulers' grandfathered duplication stays, but new schedulers use the helper. Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(#382): address expert review feedback Three reviewers (code, security, ad-tech product) flagged 10 items on PR #411. All addressed: Blockers: * Fix raw select(SyncJob) in GAM adapter latest_inventory_sync_at() — use new SyncJobRepository.latest_completed_at() instead. * Refactor execute_sync to use a proper `with get_db_session()` block via a helper that takes the Session — no more manual __enter__()/close. Security: * triggered_by_id captures the super-admin's email so SyncJob rows carry per-actor audit attribution for cross-tenant Run Now actions. * New _sanitize_error_message() strips PEM blocks, JWTs, and refresh_token/api_key key-value strings before persisting to SyncJob.error_message — bounds the field at 500 chars as a second line of defense against credential bleed in the cross-tenant scheduling view. Product / UX: * Three-state freshness (ok / warning / critical) replaces the binary stale flag. ok = within warning window; warning = past warning, not yet critical; critical = past critical window OR failed OR never run. Old `stale` property kept as back-compat alias. * Per-adapter freshness thresholds on AdapterCapabilities (inventory_freshness_warning/critical, reporting_freshness_warning/ critical). Each adapter picks its cadence; the matrix reads it directly instead of using module-level constants. * GAM rows now carry a notes="reporting bundled with inventory sync" string driven by capabilities.reporting_bundled_with_inventory, so admins don't see "no reporting row" and worry. * Run Now is now async: enqueue_adapter_sync pre-creates a SyncJob with status="queued", returns the sync_id immediately, and the daemon thread does the work. Endpoint returns 202; UI polls /admin/api/scheduling/recent for terminal state. Code quality: * Extract _patch_eligibility_layer helper in the scheduler unit tests — replaces 5 near-identical 3-line monkeypatch blocks. * SyncExecutionResult.to_json_payload() factors the canonical JSON body so future adapter buttons reuse one shape. * Reporting sync scheduler also skips ``status=queued`` rows so the scheduler can't race the async enqueue dispatch path. Tests: +5 sanitizer/payload unit tests, +1 enqueue integration test, +1 queued-row skip test. Existing tests adjusted for three-state freshness verdict and 202 response shape. 4541 unit + 25 integration all green. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat(admin): surface 500 tracebacks + warn on missing storefront prefix (#413) * chore(main-unblock): fix-format embedded_tenant_guard.py ``make quality`` fails on a stale format-check for src/core/database/embedded_tenant_guard.py — landed unformatted in PR #408. Re-running ``ruff format`` collapses two multi-line conditionals to single lines (cosmetic; behavior unchanged). (An earlier draft of this commit also added a no-op alembic merge migration for the two divergent heads from PR #381 + PR #409, but PR #412 landed the same fix on origin/main in the meantime — that migration was dropped on rebase.) Co-Authored-By: Claude Opus 4.7 (1M context) * feat(admin): surface uncaught exceptions + warn on missing storefront prefix Two diagnostic improvements to make the create-product-500-style failures visible. Reported from the agentic storefront iframe: POST /storefront/psa/tenant//products/add → HTML "Internal Server Error" The response body shape (`Error
Internal Server
Error
`) is Express's default 500 page — the storefront proxy rendered it, not salesagent. Salesagent itself had zero visibility: no 500 error handler was registered, so any Flask exception bubbled up to WSGIMiddleware → Starlette → uvicorn as a generic 500, and the upstream proxy substituted its own page on top. Result: we couldn't tell apart "salesagent threw" vs "storefront mistranslated salesagent's response," and there was no traceback to grep for. Change 1: register an @errorhandler(Exception) on the admin Flask app that logs the full traceback + request context (method, path, endpoint, tenant_id from view_args, user_email best-effort) with a short correlation ID, and returns a response that includes the ID. JSON clients (Accept: application/json) get a structured envelope. HTTPExceptions (404/403/etc.) pass through unwrapped — those are intentional, not internal errors. The handler swallows its OWN exceptions on the user/tenant lookup paths at logger.debug level so the OUTER traceback is what surfaces. Change 2: register a before_request hook that logs WARNING when embedded auth headers are present (X-Identity-Subject) but neither X-Forwarded-Prefix nor X-Script-Name is set. Per docs/integration/embedded-mode-identity-contract.md:124 the upstream proxy is required to send this header so url_for() generates URLs that resolve back through the storefront mount. Without it, a ``redirect(url_for("products.list_products", ...))`` emits ``Location: /tenant//products/`` — the iframe follows it to the storefront's own origin, which has no route at that path, and the storefront 500s on its own router. The warning surfaces the misconfiguration in salesagent logs so the storefront operator sees the gap instead of guessing why redirects land outside their iframe. The TESTING-config bypass on the warning keeps legacy tests quiet (they already match the pre-#32 contract). The 500 handler runs under TESTING because PROPAGATE_EXCEPTIONS=False explicitly opts back in. Tests (8 new in src/admin/tests/integration/test_admin_app.py): * TestUncaughtExceptionHandler (4): - synthetic crash returns 500 with Error ID body + handler logs GET/path/tenant_id=None/exc_info traceback - tenant-scoped crash captures tenant_id from view_args (acme) - Accept: application/json gets {error, error_id} envelope - 404 from werkzeug is NOT wrapped (HTTPException pass-through) * TestEmbeddedMissingPrefixWarning (4): - X-Identity-Subject without X-Forwarded-Prefix → WARNING logged - X-Forwarded-Prefix set → no warning - Plain non-embedded request → no warning - X-Script-Name (alternate header) also satisfies → no warning What this does NOT do: fix the underlying redirect bug. If the storefront proxy is missing X-Forwarded-Prefix, redirects still land outside the iframe mount. The salesagent can't safely infer the prefix from request data alone — that's the storefront integrator's fix per the documented contract. What this PR DOES is make the failure mode loud + diagnoseable instead of opaque. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(tenant-export): suspend user triggers during bulk tenant delete (#414) The prevent_empty_pricing_options trigger (BEFORE DELETE on pricing_options) enforces "every product must have ≥1 pricing option" — sensible for piecemeal edits, wrong for bulk tenant deletes where the parent product is also being removed. Surfaced during the first production dry-run of import_tenant for tenant_wonderstruck: sqlalchemy.exc.InternalError: (psycopg2.errors.RaiseException) Cannot delete last pricing option for product prod_291a023d CONTEXT: PL/pgSQL function prevent_empty_pricing_options() [SQL: DELETE FROM pricing_options WHERE pricing_options.tenant_id = %(tenant_id_1)s] Other validation triggers may exhibit the same pattern; rather than enumerate each one, suspend ALL user triggers on tenant-scoped tables for the delete loop. FK and system triggers stay enabled (DISABLE TRIGGER USER preserves those), so referential integrity is unaffected. Fix: - _suspend_user_triggers(connection, table_names) context manager: ALTER TABLE ... DISABLE TRIGGER USER on enter, ENABLE on exit. DDL is transactional in Postgres, so a rollback restores triggers; the explicit re-enable in finally handles the commit case. Owner-level privilege (no SUPERUSER required, unlike SET session_replication_role). - delete_tenant_data wraps its delete loop in the context manager. Test coverage (tests/integration/test_tenant_export_import.py): - test_delete_succeeds_through_blocking_trigger: installs a trigger that mimics prevent_empty_pricing_options on pricing_options; verifies it fires on a piecemeal delete (sanity check), then verifies delete_tenant_data succeeds through it. - test_triggers_reenabled_after_delete: verifies user triggers fire normally for other tenants after delete_tenant_data returns. Co-authored-by: Claude Opus 4.7 (1M context) * fix(tenant-export): strip + remap autoincrement int PKs when retargeting (#415) Same-deployment clone via --target-tenant-id collided on autoincrement integer PKs (gam_inventory.id, audit_logs.log_id, etc.) because the bundle preserved values still owned by the source tenant. Surfaced during a clone test of tenant_wonderstruck: sqlalchemy.exc.IntegrityError: duplicate key value violates unique constraint "gam_inventory_pkey" DETAIL: Key (id)=(1449737) already exists. Cross-deployment moves (without --target-tenant-id) are unaffected — the target doesn't own those IDs. Fix: - _is_autoincrement_int_pk / _autoincrement_pk_column: detect single-column Integer/BigInteger PKs whose autoincrement is not explicitly False. Catches 13 tables in the current schema including explicit autoincrement=True columns and SQLAlchemy 2.0's "auto" default. - import_tenant: when target_tenant_id is set, strip those PK values from rows before insert (Postgres re-allocates from the sequence) and use INSERT ... RETURNING to build an old→new ID map per table. Walk FK graph for each child and rewrite any FK column whose target was remapped. Only one FK in the current schema points at an autoincrement int PK (products.inventory_profile_id → inventory_profiles.id) but the remap is generic. - target_tenant_id unset: behavior unchanged (PKs preserved). The pre-flight collision check still catches business-unique conflicts (subdomain, virtual_host, access_token). Tests (tests/integration/test_tenant_export_import.py TestCloneOnSameDeployment): - test_clone_with_source_alive_succeeds: seed source tenant with an inventory_profile and a product whose inventory_profile_id FKs to it. Rewrite subdomain + access_token in the bundle (real clone workflow), retarget to a new tenant_id while source is still alive, assert both tenants coexist, the clone got a NEW inventory_profile.id, and the clone's product.inventory_profile_id points at the clone's profile. - test_retarget_without_source_still_works: existing retarget-after- delete path still passes; FK referenced by the clone resolves to a profile in the clone, never to a stale source ID. Co-authored-by: Claude Opus 4.7 (1M context) * fix(tenant-export): remap globally-unique string PKs on retarget (closes #416) (#417) PR #415 handled autoincrement int PK collisions on same-deployment clones. Different column shape, same class of bug surfaced next: proposals.proposal_id is a globally-unique string PK so it can't strip-and-let-Postgres-allocate. RuntimeError: Insert failed on table 'proposals': duplicate key value violates unique constraint "proposals_pkey" DETAIL: Key (proposal_id)=(prop_1f5de7d5abb4) already exists. Affected single-column string PKs in this schema: proposals.proposal_id, media_buys.media_buy_id, creative_reviews.review_id, creative_assignments. assignment_id, webhook_subscriptions.webhook_id, sync_jobs.sync_id, contexts.context_id, workflow_steps.step_id, strategies.strategy_id, users.user_id. Fix: - _globally_unique_string_pk_column(table): detect single-column non- composite String/Text PKs. Excludes columns named tenant_id (e.g. adapter_config.tenant_id is the table's only PK and is set by _retarget_tenant_id upstream — remapping would overwrite that value with an uncorrelated UUID, breaking the FK to tenants). - _mint_id(old, column): generate a fresh opaque ID. Preserves any clean alphanumeric prefix (mb_, prop_, etc.) so logs stay readable. Falls back to bare UUID4 hex when no prefix. Truncates to column length. - _build_string_pk_remap: eagerly mint new IDs for every affected table before any insert. String PKs can't be allocated by Postgres so the map must exist up-front; the unified FK-rewrite loop then handles both int (lazy, filled via RETURNING) and string (eager) remaps generically. - import_tenant: applies own-PK rewrite for tables with string PKs in addition to the existing FK-rewrite + int-PK strip/RETURNING flow. Composite PKs that include tenant_id (products(tenant_id, product_id), principals(tenant_id, principal_id), creatives(...), accounts(...), agent_account_access(...), currency_limits(...)) are unaffected because changing tenant_id already changes the tuple uniqueness. JSON-embedded ID references are still out of scope (documented limitation in #416). Operators doing same-deployment clones for staging snapshots should grep their JSON content for stale IDs if they need full fidelity. Tests (tests/integration/test_tenant_export_import.py TestCloneOnSameDeployment): - test_clone_remaps_string_pk_and_rewrites_fk: seed source with a media_buy + two media_packages whose composite PK references media_buy_id. Rewrite subdomain + access_token in the bundle. Retarget to a new tenant_id with source alive. Assert clone gets a NEW media_buy_id with the "mb_" prefix preserved, both packages re-link to the clone's media_buy_id (composite PK uniqueness preserved via FK rewrite), source rows untouched. Closes #416. Co-authored-by: Claude Opus 4.7 (1M context) * feat(admin): add ALLOW_SIGNUPS env var to close self-service registration (#418) Hosted cluster needs to stop accepting new signups while existing tenants are migrated. ALLOW_SIGNUPS=false renders a "signups closed" page on /signup and short-circuits /signup/start, /signup/onboarding, and /signup/provision. Defaults to true so existing deployments are unaffected. Co-authored-by: Claude Opus 4.7 (1M context) * feat(admin): hide Buyer Agents tab on embedded; rename Settings → Tenant Settings (#420) Sprint 7 IA cleanup — Phase 1a + 1b. On embedded tenants the Settings → Buyer Agents tab showed only read-only data already surfaced (and editable) on Buyer Routing, plus a "platform-managed" banner — duplicate noise rather than a useful surface. Hide the sidebar nav tab and the ``
`` section entirely on embedded. Rename the Configure menu entry "Settings" → "Tenant Settings" so its scope is unambiguous before later phases empty it out further. Reverses the Sprint 4 "read-only directory stays visible permanently" call (docs/design/embedded-mode-sprint-4-ui-hardening.md "Terminology pin") now that Sprint 5 made Buyer Routing the canonical home for advertiser→buyer-agent mappings. Also fixes a Phase-1a-introduced bug: the setup checklist ``principals_created`` task pointed to ``/settings#advertisers`` (now hidden on embedded) and told operators to do something they can't (Principal provisioning is platform-managed). Skip the task on embedded in both ``_check_critical_tasks`` and ``_build_critical_tasks``. New design doc captures the full IA endgame and remaining phases (entity promotion to Configure peers, fold-ins, hide Tenant Settings entirely once it's down to Account + Ad Server + Danger Zone). Co-authored-by: Claude Opus 4.7 (1M context) * fix(admin): scope dashboard activity ledger to last 7 days (#421) The "Last 7 days" label on the tenant dashboard activity ledger was purely cosmetic — `_activity_ledger` ran `select(AuditLog) ORDER BY timestamp DESC LIMIT 8` with no time bound, so a quiet tenant would show months-old entries. The "time" column also only rendered HH:mm, which made an event from December look like one from today. - Filter via AuditLogRepository.list_filtered(from_date=now-7d) - Render same-day rows as HH:mm, older rows as "Mon DD HH:mm" - Drop the now-unneeded raw-select allowlist entry Co-authored-by: Claude Opus 4.7 (1M context) * fix(admin): exempt S2S API blueprints from cross-origin CSRF guard (#423) * fix(admin): exempt S2S API blueprints from cross-origin CSRF guard The global before_request CSRF guard in src/admin/app.py rejected POSTs to /api/v1/tenant-management/* and /api/v1/sync/* when the caller (an internal service) did not set Origin or Referer headers. These blueprints are header-authed (X-Tenant-Management-API-Key / X-API-Key) and set no session cookie, so cookie-riding CSRF cannot apply. The per-route @require_api_key_auth decorator still enforces the API key, so bypassing the Origin check does not weaken auth. Bypass is path-based (not header-based) so an attacker can't escape CSRF on cookie-authed admin routes by forging a header. Observed symptom: agentic-api provisioning calls failed with "Refusing cross-origin admin POST to /api/v1/tenant-management/ tenants/provision — origin=None referer=None" / HTTP 403. * chore(admin): address review notes on CSRF guard docs - Drop stale `_SAME_ORIGIN_HEADERS` reference in the TESTING bypass comment (symbol doesn't exist anywhere in the tree). - Correct the parenthetical on `test_cookieless_post_bypasses_csrf` — FlaskClient does persist cookies across requests; the test relies on per-test fixture freshness, not on the client being stateless. Surfaced by independent review on PR #423. No behavior change. * refactor(admin): drop redundant X-Identity-Subject CSRF bypass (#424) The cookie-presence structural bypass added in #423 already subsumes embedded mode: per docs/integration/embedded-mode-operational.md §4, embedded-mode auth is stateless via X-Identity-* headers and the upstream proxy never sets a session cookie on the salesagent. The explicit X-Identity-Subject branch was belt-and-suspenders for a scenario that the documented contract forbids. Tests stay green — the embedded-mode regression test now covers the same shape (X-Identity-Subject + no cookie) via the cookieless bypass instead of the dedicated branch. Docstring updated to explain the new reasoning. Co-authored-by: Claude Opus 4.7 (1M context) * feat(springserve): add SpringServe (Magnite) ad-server adapter — direct CTV/OLV/audio integration for Talpa (#427) * feat(springserve): add SpringServe (Magnite) adapter — Stage 1 skeleton + auth Direct-to-ad-server adapter for SpringServe at console.springserve.com, positioned as the publisher-side path that avoids Magnite's SSP-level AdCP agent fees. First customer is Talpa Network for audio inventory (Radio 538 / Sky Radio / Radio 10) with video as the strategic priority. Stage 1 scope: - Email + password authentication with 2-hour token cache (POST /api/v0/auth) - Raw token in Authorization header (not Bearer — SpringServe quirk) - Connection + product config schemas with Fernet-encrypted secrets - Static VAST format declarations: 6 video + 4 audio (audio = first-class, same demand-tag API surface, MIME-discriminated via Format.type) - AdServerAdapter subclass with dry-run for every method; live paths return pending_credentials until Stage 2 wires the writes - Permission probe matrix covering campaigns, demand_tags, videos, supply_tags, supply_partners, report - Live smoke test verified against operator's account: ✅ auth + campaigns + demand_tags + videos ❌ supply_tags + supply_partners (scope grant pending) ⏳ report (POST-only; Stage 4 replaces probe shape) Shared helpers extracted to satisfy the DRY guard: - src/adapters/_secret_fields.py — Fernet encrypt/decrypt helpers - src/adapters/_format_helpers.py — vast_format() builder - src/adapters/_token_cache.py — BearerTokenCache - AdServerAdapter._new_permissions_report + _walk_permission_probes - tests/helpers/adapter_test_helpers.py — sample request/package factories and stub_http_response FreeWheel adapter refactored to consume the same helpers — no behaviour change, just deduplication. Plan in .context/springserve-adapter-plan.md tracks Stages 2-5 (live Campaign + DemandTag writes, creatives, reporting cache, inventory cache + admin UI). Co-Authored-By: Claude Opus 4.7 (1M context) * feat(springserve): Stage 2 — live Campaign + Demand Tag writes Wires the full create_media_buy → check_status → pause/resume cycle against the live SpringServe API. Mapping A: AdCP MediaBuy → SpringServe Campaign AdCP Package → SpringServe Demand Tag (one per package, parented to the campaign by ``campaign_id``) media_buy_id → ``springserve_`` New entities/clients (shapes captured from live probes against the operator's account on 2026-05-14): - ``entities.Campaign`` + ``entities.DemandTag`` Pydantic models with ``extra="allow"`` so the 38-field Campaign + 220-field DemandTag responses round-trip without losing data. - ``SpringServeCampaignsClient`` — POST / GET / PUT / DELETE /campaigns. - ``SpringServeDemandTagsClient`` — POST / GET / PUT / DELETE /demand_tags. Auto-flips ``country_targeting`` to "White List" when ``country_codes`` is set; coerces rate to string ("27.0"); formats start_date/end_date in SpringServe's ISO-microsecond-Z convention. - ``targeting.build_demand_tag_targeting`` — flattens AdCP geo/device overlays onto demand-tag fields directly (NOT a wrapper "targeting" dict; SpringServe doesn't have one). Supply targeting goes through ``demand_tag_priorities: [{supply_tag_id, priority, tier}]``. Adapter behaviour: - ``create_media_buy`` POSTs one campaign + N demand tags, all created paused (Stage 3 binds creatives and flips them active). - ``check_media_buy_status`` reads the campaign and maps ``is_active`` to AdCP status (active/paused). - ``update_media_buy`` supports pause_media_buy / resume_media_buy (campaign-level) and pause_package / resume_package (demand-tag level, found by ``secondary_code=package_id`` scan of the campaign's demand tags). update_package_budget returns ``unsupported_action`` (Stage 4 wires the budgets nested object). - Audio vs video routing comes from the AdCP Format's id prefix (``springserve_audio_*`` → demand_tag.format="audio"); no denormalised flag. Live scope status — POST scope is NOT yet granted on the operator's test account. Stage 2 unit tests verify the code with a mocked client; the live cycle test skips with a clear scope-grant message until SpringServe enables write access. README documents the exact ask. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(springserve): Stage 3 — creatives via POST /videos + demand-tag binding Wires ``add_creative_assets`` and ``associate_creatives`` against the SpringServe ``/videos`` endpoint. SpringServe hosts both video and audio on the same endpoint, discriminated by ``creative_format`` (``video`` | ``audio``) and ``creative_content_type`` (``video/mp4``, ``audio/mp4``, ``audio/mpeg``). - ``SpringServeCreativesClient`` — POST / GET / PUT / DELETE /videos. Uses the remote-URL ingest path (``creative_remote_url``); SpringServe pulls and transcodes the hosted asset. Multipart upload (≤500MB) is available on the same endpoint but deferred until needed. - ``entities.VideoCreative`` — Pydantic model with ``extra="allow"`` so the 66-field response round-trips losslessly. The "VideoCreative" ``type`` label is preserved as SS-internal metadata even on audio. - Adapter routing — ``_asset_media_type()`` reads the AdCP Format's id prefix (``springserve_audio_*``) and the asset's own ``content_type`` hint; produces matching SpringServe MIME types. No denormalised flag. - ``associate_creatives`` writes the single-creative path (``demand_tag.creative_id``) and flips the tag active. Multiple creatives per tag get the LAST one wired; earlier ones are recorded as ``skipped`` with a message. Rotation via ``line_item_ratios`` is deferred. Stage 3 status: code complete, blocked on the same write-scope grant that blocks Stage 2 (POST /videos returns 403 today). The live cycle test (Campaign → DemandTag → Creative → bind → cleanup) skips with a clear scope-grant message until SpringServe enables write scope. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(springserve): Stage 4 — reporting cache + sync Wires get_media_buy_delivery + get_packages_snapshot to read from a local ``springserve_demand_tag_stats`` cache populated by a periodic SpringServe Reporting API sync. Empty cache raises ``DeliveryDataUnavailable`` (matches the FreeWheel contract — no fake zeros). New surfaces: - Alembic migration ``ss01a1b2c3d4`` creating the ``springserve_demand_tag_stats`` table keyed by ``(tenant_id, demand_tag_id)``. Spend in currency-minor-unit micros. - ORM model + ``SpringServeDemandTagStatsRepository`` (tenant-scoped reads, ON CONFLICT bulk upsert, latest_sync_at). - ``SpringServeReportingClient`` — sync POST /report + async submit + poll-until-done + fetch-rows. ColumnMap-driven row parsing so the day-of-scope schema reveal is a config tweak, not a code change. - ``SpringServeReportingSync`` orchestrator. Picks sync vs async based on window length (>1 day → async). Translates SpringServeForbiddenError into a clean ``ReportingScopeNotGranted`` for the scheduler. - Adapter ``run_reporting_sync`` returns ``AdapterSyncResult`` with ``scope_pending`` metadata when the grant isn't there yet, so the shared scheduler keeps trying without exception spam. - Adapter ``latest_reporting_sync_at`` surfaces freshness for the ``/admin/scheduling`` page. Refactors that fell out (DRY-driven): - ``AdServerAdapter._aggregate_stat_rows_to_delivery_response`` — shared by FreeWheel + SpringServe to turn ORM stat rows into an ``AdapterGetMediaBuyDeliveryResponse``. - ``AdServerAdapter._platform_status_to_delivery_status`` — shared string-to-enum translation for delivery status. - ``AdServerAdapter._wrap_sync_run`` — shared scope/error/result wrapping for ``run_reporting_sync``-style callables. Status: code complete; pre-flight on the same scope grant requested for Stages 2–3. POST /report returns 403 on the operator's account today; the sync raises ``ReportingScopeNotGranted`` and the read path raises ``DeliveryDataUnavailable`` until both are fixed by the scope grant + first sync run. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(springserve): Stage 5 — inventory cache + admin UI + typed embedder config Closes the full SpringServe adapter loop: operators can pick the adapter in tenant settings, save credentials, sync inventory, probe permissions, and configure products against synced supply tags. Embedders (Scope3 storefront) can provision SpringServe tenants through the typed discriminated-union AdapterConfig surface. New surfaces: - Alembic migration ``ss02e5f6a7b8`` creating the ``springserve_inventory`` table keyed by ``(tenant_id, entity_type, entity_id)``. JSONB raw_json on Postgres, plain JSON elsewhere. - ``SpringServeInventory`` ORM model + ``SpringServeInventoryRepository`` (tenant-scoped list/upsert/clear, latest_sync_at). - ``SpringServeSupplyClient`` — read-only client over /supply_partners + /supply_tags. - ``SpringServeInventorySync`` — paginated walker that upserts both entity types into the cache; raises ``SupplyScopeNotGranted`` cleanly when scope is denied. - Adapter ``run_inventory_sync`` + ``latest_inventory_sync_at`` + ``get_available_inventory`` (reads from cache, no live API calls). - ``SpringServeAdapterConfig`` in the discriminated-union ``AdapterConfig`` for the typed embedder API; secrets via SecretStr, model-validated exactly-one credential path. - Tenant Management API plumbing: catalog metadata + typed config registry + ``_adapter_config_to_dict`` + ``_persist_adapter_config`` (Fernet-encrypted round-trip through SpringServeConnectionConfig matching the FreeWheel pattern). - Admin UI templates: connection_config.html (form + Save + Test Connection + Sync Inventory + Check API Permissions buttons) and product_config.html (supply_tag + supply_partner pickers loading from the cache). - ``templates/tenant_settings.html`` picker card. - Blueprint endpoints under ``/api/tenant//adapters/springserve/``: test-connection, inventory, sync-inventory, check-permissions. - Regenerated ``docs/api/tenant-management-openapi.{json,yaml}``. Status: code complete; supply-side reads return 403 today so the live sync raises ``SupplyScopeNotGranted`` until SpringServe enables supply read scope on the operator's account. The product config UI pickers will show empty lists until the first successful sync. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(springserve): address review must-fix items Two parallel reviews (code-reviewer + security-reviewer) converged on the same must-fix list before shipping the SpringServe adapter: H1 — Replace MagicMock principal in check_springserve_permissions src/admin/blueprints/adapters.py Delete the redundant SpringServe-specific check-permissions handler. Permission probes now go through the generic /api/tenant//adapters//check-permissions endpoint, which constructs a real typed Principal stub instead of a MagicMock. The MagicMock would have returned truthy for every attribute access, silently masking tenant-isolation bugs if the adapter ever read a principal field outside the (very narrow) probe path. H2 — Remove ``self.tenant_id or "default"`` fallbacks in the adapter src/adapters/springserve/adapter.py Seven cache read/write sites had ``or "default"`` fallbacks. The base AdServerAdapter.__init__ already enforces tenant_id is set, so the fallback was dead code -- but if a real "default" tenant ever exists (it does for demo-mode), a regression that lost tenant_id would have silently read or upserted into another tenant's cache. Code-reviewer IMPORTANT #4 — add_creative_assets KeyError src/adapters/springserve/adapter.py asset["creative_id"] raised KeyError mid-loop, abandoning every remaining asset. Use asset.get("creative_id") with explicit empty- string fallback + an early failed-status append, so a bad asset only fails its own slot. Security-reviewer M1 — https-only allow-list for creative_remote_url src/adapters/springserve/adapter.py Buyer-supplied URLs are forwarded server-side to SpringServe's fetcher. Reject non-https schemes (file://, http://, ftp://) and obvious private hosts (localhost, 127.0.0.1, RFC1918 ranges) at the adapter boundary so we can't be turned into a free SSRF oracle by a malicious AdCP buyer. Tests added: 4 new cases covering missing creative_id, non-https URL, loopback URL, RFC1918 URL. All 40 SpringServe adapter unit tests pass; make quality green (4708 total). Remaining recommendations from the two reviews are backlog (inventory-cache pruning of deleted entries, targeting-discriminator auto-flip on escape-hatch payloads, audio content-type validation, note-field stored-prompt-injection mitigation, smoke-test SPRINGSERVE_TEST_ALLOW_WRITE confirmation env) -- tracked separately, not blocking on this PR. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(springserve): add springserve to catalog-assertion test sets The Stage 5 commit registered ``springserve`` in ``_ADAPTER_CATALOG_METADATA`` + ``_ADAPTER_CONFIG_TYPED`` (so the ``GET /api/v1/tenant-management/adapters`` endpoint returns it), but forgot to update the two integration tests that assert on the exact catalog set: - ``test_list_adapters_returns_supported_catalog`` - ``test_list_adapters_tier_filter_excludes_mock_from_live`` Both now include ``springserve`` in the expected sets. Verified both pass locally against agent-db. CI's "Integration (other)" job should go green on the next run. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat(proposal): adopt adcp 5.5.0 — framework derivation + PgProposalStore swap (supersedes #419) (#422) * feat(proposal): adopt adcp 5.5.0 framework package derivation adcp-client-python#732 shipped framework-level package derivation for ``create_media_buy(proposal_id=…)`` calls with empty ``packages[]``. ``maybe_hydrate_recipes_for_create_media_buy`` now distributes ``total_budget.amount`` across the reserved proposal's ``allocations[]`` by percentage and mutates ``req.packages`` before the seller adapter runs — opt-in via ``ProposalCapabilities.derive_packages_from_allocations``. Adoption is a 15-line diff: * Bump ``adcp>=5.5.0`` in ``pyproject.toml``. * Set ``derive_packages_from_allocations=True`` on ``SalesAgentProposalManager.capabilities``. Supersedes #406 (the local derivation helper + delegate hook). That PR shipped the same behavior as a salesagent-internal module on the bet that upstream auto-injection would land within a sprint — it did, faster than expected (5.5.0 tagged ~24h after PR #406 opened). The local code becomes dead weight the moment the capability flag flips, so adopting the upstream and closing #406 as superseded is the cleaner outcome. ## Why this PR is small The PR that opens this hole at the framework level is the bigger story: ``adcp-client-python#732`` ships **both** ``PgProposalStore`` (durable Protocol implementation, mirrors our ``SalesAgentProposalStore``) AND the derivation hook adopted here. We're taking the derivation half this PR and deferring the store swap to a follow-up: * ``PgProposalStore`` requires an ``AsyncConnectionPool`` (psycopg3). The psycopg3 pool is already in the dep tree (``IdempotencyStore.PgBackend`` uses it), but the pool lifecycle is wired for the idempotency surface only — adding a second consumer wants intentional design, not an incidental wire-up. * The schema migration to upstream's ``(account_id, proposal_id)`` PK + drop of our ``tenant_id`` FK column wants a cascade-delete decision we haven't made yet (FK on a hypothetical accounts table vs. application-layer cleanup vs. TTL expiry). adcp-client-python#738 filed to document the AccountStore-layer encoding seam clarifies the layering question; the cascade design is downstream of that. So: take the well-shaped half now, defer the half that wants design. ## Compliance impact After deploy, the compliance probe's ``media_buy_seller/proposal_finalize/create_media_buy`` storyboard should flip fail → pass — same outcome PR #406 would have produced, but with the framework owning the math instead of us. ## Reviewer cross-reference The derivation helper in PR #406 was reviewed twice (code-reviewer + security-reviewer) and the math + edge cases were verified. Upstream's ``derive_packages_from_proposal`` ships the same shape (pin-the-last absorber, 2dp rounding, allocation-sum guard, malformed-entry rejection) plus a few we didn't have (single ``pricing_options[]`` auto-pick on proposal persist when omitted, multi-option products requiring explicit selection). Net behavior is identical or stricter; no regression risk. Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(proposal): swap SalesAgentProposalStore → upstream PgProposalStore Completes the adcp 5.5.0 adoption started in #419. adcp-client-python#732 shipped :class:`PgProposalStore` — the durable ``ProposalStore`` Protocol implementation our local :class:`SalesAgentProposalStore` (PR #390) was mirroring. With upstream owning the CAS, cross-tenant rejection, TTL bookkeeping, and ``ON CONFLICT`` upsert semantics, the salesagent implementation becomes ~280 LOC of code we don't need to maintain. ## Three design decisions, resolved **Async pool wiring.** Process-singleton ``AsyncConnectionPool`` modeled on the existing :mod:`core.idempotency` pattern. Lazy first-call open so the pool's worker tasks bind to ``serve()``'s loop, not whatever transient loop happens to construct the store. New module: :mod:`core.decisioning.proposal_store`. **Schema migration.** Drop our existing ``proposals`` table; rebuild matching upstream's expected shape — ``(account_id, proposal_id)`` PK, ``COLLATE "C"`` on text columns for byte-order index ordering, state ``CHECK`` constraint, partial ``ix_proposals_expires_at`` index for TTL sweep. PR #390 deployed ~24h before this migration with near-zero production data; acceptable loss. **Cascade-delete strategy.** ``tenant_id`` survives as a generated column derived from ``account_id`` via ``split_part(account_id, ':', 1)``. The existing FK + ``ON DELETE CASCADE`` to ``tenants`` stays intact, so :func:`scripts.seed_demo_tenant._delete_tenant_rows` and :func:`src.admin.tenant_management_api.delete_tenant` keep working without change. Consistent with the layering principle from adcontextprotocol/adcp-client-python#738: the encoding seam stays at :class:`SalesagentAccountStore.resolve()`, and we just leverage the same encoding inside our own table for our own FK target. ``PgProposalStore`` uses explicit-column INSERTs, so the generated column is invisible to upstream. ## What goes away * ``src/core/database/repositories/proposal_store.py`` — 547 LOC, including the wrong-layer ``_resolve_tenant_id_for_account`` parse flagged in adcp-client-python#738. The Protocol-level layering is now enforced by deletion. * ``src/core/database/models.py:Proposal`` ORM class — 56 LOC. The ``proposals`` table is now owned by ``PgProposalStore``'s psycopg3 path; no SQLAlchemy model needed. * ``tests/integration/test_proposal_store.py`` — 720 LOC, 16 tests. Upstream's conformance suite at ``tests/conformance/decisioning/test_pg_proposal_store.py`` covers the same invariants against real Postgres. * ``tests/unit/test_proposal_store_attributes.py`` — 20 LOC. Pinned ``is_durable``; upstream owns the attribute. Net delete: ~980 LOC across removed code + 363 LOC of new schema / wiring / migration = real shrinkage of ~620 LOC. ## Tenant export caveat Added ``proposals`` to :data:`tenant_export.EXCLUDED_TABLES`. The generated ``tenant_id`` column rejects direct ``INSERT`` writes (PG ``GENERATED ALWAYS``), and the alternative — strip-on-import + auto-derive — would carry stale in-flight proposal state across deployments that the target's ``PgProposalStore`` should re-mint via fresh ``get_products`` calls anyway. Documented inline. ## Verification * ``make quality``: 4545 passed, 14 skipped, 19 xfailed * ``tox -e integration``: 241 passed * Migration head: ``t2u3v4w5x6y7`` (single head, succeeds ``d0c3c40fdd41``) ## Stacks on #419 (adcp 5.5.0 bump + ``derive_packages_from_allocations=True``). This PR shares its base; merging order: #419 first, then this. Co-Authored-By: Claude Opus 4.7 (1M context) * review(proposal): address review notes on PR #422 Code-reviewer and security-reviewer both verdicted "ship as-is" with one should-fix and a couple of notes. Folding the should-fix in here; the two follow-ups go to separate tracking issues. ## Should-fix: integration_db.py missing proposal-store reset ``tests/fixtures/integration_db.py`` resets ``core.idempotency`` and ``src.core.signing.replay_store`` between per-test databases but had no equivalent for ``core.decisioning.proposal_store``. Same bug class as PR #134: the proposal-store singleton would cache an ``AsyncConnectionPool`` bound to test N's DSN, then test N+1 would acquire a connection to the (now-dropped) per-test DB and ``PoolTimeout`` after 30s. Latent today — the dedicated proposal-store integration suite went away with ``SalesAgentProposalStore`` — but the moment someone adds an integration test that hits a proposal-aware tool, they'd get a 30s-then-mysterious-failure. Cheap to fix preventively. Added ``_reset_proposal_store()`` mirroring the existing pair and wired it into both the setup and teardown reset blocks. ## Note: reset_for_tests docstring Security review flagged that ``reset_for_tests`` doesn't await pool close — true, and it's intentional (the workers are bound to a foreign loop, awaiting close from sync teardown would deadlock or orphan). Made the rationale explicit in the docstring and noted the test-side escape hatch (``await close_proposal_store()`` first if reusing the process). ## Deferred (separate issues) - **Compliance gap on tenant export** — proposals excluded from bundles (because the generated ``tenant_id`` column rejects direct INSERT). Drafts are ephemeral and committed proposals are referenced by ``media_buy_id`` which IS exported, so the privacy story is clean. GDPR right-to-portability / acquisition-transfer scenarios may want proposal history regardless; filing a follow-up issue rather than expanding scope here. - **CHECK constraint on tenants.tenant_id rejecting colons** — the generated-column FK design promotes ``account_id`` well-formedness to a tenant-isolation invariant. A tenant with a colon in its ``tenant_id`` could conflate with a non-compound ``account_id`` via ``split_part(..., ':', 1)``. Not exploitable today (admin UI mints slug-shaped IDs, every write path routes through ``SalesagentAccountStore.resolve()``), but defense in depth. Filing a follow-up rather than touching the tenants schema in this PR. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(proposal): open PgProposalStore pool in serve()'s on_startup CI surfaced ``PoolClosed: the pool 'pool-1' is not open yet`` on every proposal-path dispatch — the lazy ``AsyncConnectionPool(open=False)`` construction at first ``get_proposal_store()`` call never wired an ``open()`` anywhere. Production server and the in-process test harness both run ``serve()``'s native lifespan hooks (adcp 5.4.0 #713), so an ``on_startup=[open_proposal_store]`` entry is the right hook to bind the pool to the live event loop. Added ``open_proposal_store()`` async helper in ``core.decisioning.proposal_store`` and wired it (alongside the existing ``close_proposal_store`` on ``on_shutdown``) into ``_serve_kwargs``. Always runs regardless of ``include_scheduler`` — the pool is tied to ``serve()``'s loop, not to background-job lifecycle. Resolves the E2E + ``Integration (other)`` failures observed on PR #422 after retargeting to ``main`` (the test surfaces that exercise ``update_media_buy`` / ``get_media_buy_delivery`` both hit ``maybe_hydrate_recipes_for_media_buy_id`` which acquires from the pool). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(proposal): forward on_startup/on_shutdown through build_app() to in-process tests The previous fix added ``open_proposal_store`` to ``_serve_kwargs``'s ``on_startup`` list, which works in production where ``main()`` calls ``serve()`` (which natively threads lifespan hooks into the composed app). But ``build_app()`` (used by the in-process test harness and any ASGITransport-based test) bypasses ``serve()`` and calls ``_build_mcp_and_a2a_app`` directly — it wasn't forwarding the lifespan kwargs, so the pool open hook never fired in tests. Result: CI's E2E + ``Integration (other)`` + ``Integration (infra)`` suites continued to hit ``PoolClosed: the pool 'pool-1' is not open yet`` on any proposal-touching dispatch even after the prior fix. ``_build_mcp_and_a2a_app`` already accepts ``on_startup`` / ``on_shutdown`` kwargs (adcp 5.4.0 #713 wired them through). Pass them explicitly from ``_serve_kwargs``'s already-built tuples. Production path (``main`` → ``serve``) was already correct; this only affects the in-process test surface. Locally ``make quality`` still shows 4545 passed. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(proposal): lazy-open the pool on first async method call Prior approach wired ``open_proposal_store`` into ``on_startup``, which fires once per process at app startup. That works in production but is fragile under per-test database rebuild: the harness creates one app per process (lifespan fires once with whatever ``DATABASE_URL`` was live at that moment), and subsequent integration tests rebuild the store singleton (via ``_reset_proposal_store``) against per-test DBs without re-firing lifespan — so the rebuilt pool stays closed. Switch to a ``_LazyOpenPgProposalStore`` subclass that opens its pool on the first async method call, mirroring ``core.idempotency._LazyBootstrapPgBackend``. Each rebuilt singleton opens its own pool against its current DSN on first use; the pool binds to whichever event loop dispatches it. Mutex-guarded so concurrent first-callers don't race the side effect. Dropped the ``open_proposal_store`` lifespan entry from both ``_serve_kwargs``'s ``on_startup`` and ``build_app``'s explicit forwarding — no longer needed. ``close_proposal_store`` stays on ``on_shutdown`` to drain in-flight connections cleanly. Local ``make quality`` still shows 4545 passed. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(tests): reset pool-holding singletons in conftest_db integration_db There are two ``integration_db`` fixtures with the same name: * ``tests/fixtures/integration_db.py:make_integration_db`` — added ``_reset_proposal_store`` calls in the prior review-fix commit. * ``tests/conftest_db.py:integration_db`` — the one most integration tests actually use. Had no pool-singleton resets at all. This commit adds the resets to the conftest fixture so per-test DATABASE_URL changes propagate into rebuilt singletons. Without it, the first integration test in a CI worker opens the proposal pool against its own DSN; subsequent tests' DBs are unreachable because the pool is bound to the first test's (now-dropped) DSN, and ``maybe_hydrate_recipes_for_media_buy_id`` raises ``PoolTimeout`` mid-dispatch. Resolves the test-stale-DSN root cause behind PR #422's E2E and Integration (other) / (infra) failures. The lazy-open subclass on its own wasn't enough — it opens the pool correctly, but the pool was already bound to a stale DSN at construction time. Iterates over a tuple of (idempotency, replay, proposal) module names so the three pool-holding singletons stay aligned without copy-paste. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(proposal): pass table_name="proposals" to PgProposalStore Real root cause of the E2E + Integration (other) / (infra) failures across PR #422: ``PgProposalStore`` defaults its table to ``adcp_proposal_drafts``, but our migration (``t2u3v4w5x6y7_swap_to_pg_proposal_store_schema``) creates the table as ``proposals``. Every proposal-path dispatch raised ``psycopg.errors.UndefinedTable: relation 'adcp_proposal_drafts' does not exist``, which the framework's a2a_server caught generically as "Skill execution failed: update_media_buy" — never reaching the delegate's typed error translation. This bug was masked by red herrings in the earlier debug cycle: ``PoolClosed`` warnings from zombie pools (resolved by lazy-open subclass + conftest_db reset), per-test DSN binding (real concern, real fix), ``PoolTimeout`` from the underlying ``UndefinedTable`` keeping the connection in a bad state. The lazy-open + reset fixes are still load-bearing for the broader stale-DSN problem; this one-line ``table_name="proposals"`` adds the missing piece. Two valid options were: * Rename upstream table identifier to match by passing ``table_name=`` * Rename our migration to use ``adcp_proposal_drafts`` Picked the first because PR #390 already shipped ``proposals`` as the salesagent-internal name (referenced by admin tooling docs, audit trails, ops queries). Renaming would force a no-op rename migration through every existing tenant with zero functional gain. Schema columns + indexes already match upstream's expected shape; only the table identifier differs, and ``PgProposalStore`` makes that configurable specifically for this case. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(proposal): re-add Proposal ORM as table-creation-only schema mirror PR #422's swap deleted the ``Proposal`` ORM class on the reasoning that ``PgProposalStore`` owns the table via raw psycopg3. That was correct for runtime but broke the test path: ``tests/conftest_db.py:integration_db`` creates per-test DBs via ``Base.metadata.create_all`` (NOT Alembic), so removing the ORM class removed the table from ``Base.metadata``, and every proposal-path integration test raised ``psycopg.errors.UndefinedTable: relation 'proposals' does not exist``. Re-adds the ``Proposal`` class with the upstream schema shape: * ``(account_id, proposal_id)`` compound PK * ``state`` CHECK constraint * JSONB columns for recipes / payload * TIMESTAMPTZ + ``server_default=func.now()`` for timestamps * Generated ``tenant_id`` via :class:`sqlalchemy.Computed` (matches the migration's ``GENERATED ALWAYS AS (split_part(...)) STORED`` clause) * FK + ``ON DELETE CASCADE`` to ``tenants`` * Partial unique on (account_id, media_buy_id) for reverse lookup * Partial expires_at index for TTL sweep Production runs Alembic (migration ``t2u3v4w5x6y7``) — the source of truth for the schema. The ORM class is **table-creation-only**: no salesagent code may query through it. Docstring spells out the "do not write SQLAlchemy queries against this" rule for future maintainers. The columns produce DDL equivalent to the migration except for ``COLLATE "C"`` (a byte-order index perf optimization that's test-irrelevant — tests don't care about index lookup speed and SQLAlchemy can't express COLLATE cleanly at column-declaration time). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(migration): rebase proposals migration onto ss02e5f6a7b8 The SpringServe inventory migration (``ss02e5f6a7b8``) landed on main during this PR's review cycle, creating a multi-head conflict with ``t2u3v4w5x6y7``. Both descended from ``d0c3c40fdd41``. Re-pointed this migration's ``down_revision`` to ``ss02e5f6a7b8`` to linearize the chain. The springserve migrations don't touch ``proposals``, so a clean rebase is equivalent to ``alembic merge`` here and keeps the history flat. ``alembic heads`` now shows ``t2u3v4w5x6y7`` as the single head. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat(embedded): EMBEDDED_CAPABILITIES flag + Tenant Settings section gating (Sprint 7 Phase 4a+4b) (#428) * docs(embedded): reframe sprint 7 phase 4 around capability flags Drops the coarse "hide Tenant Settings entirely on embedded" framing. The salesagent is heading toward headless: the storefront progressively absorbs every workflow that isn't ad-server-specific. Per-workflow migration needs per-workflow ownership, not a static template hide. Phase 4 now ships as four steps: - 4a: EMBEDDED_CAPABILITIES env var + capability_owner() helper - 4b: per-subsection {% if publisher_owns('X') %} gates + 403 POSTs - 4c: hard not-embedded gates for signing keys + OIDC (no publisher answer ever makes sense) - 4d: collapse Tenant Settings page once all subsections are storefront-owned Flags are instance-level, not per-tenant — one embedded instance is one storefront. Defaults to all-publisher so existing tenants behave identically until the operator opts each workflow in. Open instances treat the env var as a no-op. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(embedded): add EMBEDDED_CAPABILITIES flag infrastructure Phase 4a of sprint 7 IA cleanup. Adds capability_owner() and publisher_owns() helpers that read an instance-level EMBEDDED_CAPABILITIES env var (JSON), declaring which workflows the upstream storefront has absorbed on this embedded instance. Defaults to all-publisher so existing embedded tenants behave identically at upgrade. Open instances treat the env var as a no-op — capability gating only applies when MANAGED_INSTANCE=true. Malformed JSON, non-object shapes, or values outside {publisher, storefront} raise ValueError at call time (fail loud — silently leaving every workflow on the publisher side would be the worst failure mode). Both helpers are registered as Jinja globals so templates can gate subsections in phase 4b: `{% if publisher_owns('creative_approval') %}`. No template changes in this phase — pure infrastructure, mergeable alone. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(embedded): gate Settings subsections by capability flag Phase 4b of Sprint 7 IA cleanup. Wraps each migrating Tenant Settings subsection in {% if publisher_owns('') %} and adds matching 403 guards in the POST handlers. When a storefront takes over a workflow (creative approval, Slack, advertising policy, product ranking, AI services, creative/signals agents) on an embedded instance, the publisher's UI loses that section and direct POSTs return 403. Eight capabilities gated: - creative_approval (approval workflow + creative review subsections) - advertising_policy - product_ranking - slack - ai_services (sub-form + test endpoints + model picker) - creative_agents (whole blueprint via before_request hook) - signals_agents (whole blueprint via before_request hook) - brand_manifest (reserved for when the field is rendered) Defense-in-depth: business-rules POST inspects which form fields are present and 403s if any field belongs to a storefront-owned capability. Currency/measurement/naming fields stay publisher-writable. Drive-by: - Extracted insert_embedded_test_tenant + cleanup_embedded_test_tenant to tests/integration/_embedded_helpers.py — the canonical tenant kwargs lived in test_embedded_ui_hardening.py and were about to be triplicated. - Moved embedded_app + embedded_client fixtures to tests/integration/conftest.py (pytest needs them there to avoid ruff F811 on imported fixtures). - Removed now-stale _insert_tenant allowlist entry in the repository- pattern guard. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat(embedded): hard-hide signing keys + OIDC on embedded + review-driven fixes (Sprint 7 Phase 4c) (#430) * feat(embedded): hard-hide signing keys + OIDC on embedded instances Phase 4c of Sprint 7 IA cleanup. Some surfaces never make sense on embedded tenants regardless of which storefront is the wrapper, so they don't get capability flags — they're hard-gated on either ``not embedded_view`` (per-tenant rendering) or ``not is_managed_instance()`` (instance-wide blueprint registration). Signing keys (per-tenant gate): - Nav entry hidden in Tenant Settings on embedded tenants - Section markup omitted on embedded tenants - POST /signing-keys/generate returns 403 on embedded tenants - POST /signing-keys//rotate-out returns 403 on embedded tenants The salesagent doesn't issue webhooks under its own domain in embedded mode (the storefront signs), so self-signing inside the storefront's perimeter is dead code. Open tenants retain the full surface. OIDC blueprint (instance-wide gate): - ``oidc_bp`` is not registered when ``MANAGED_INSTANCE=true`` - /auth/oidc/* routes 404 on embedded instances Embedded identity comes from X-Identity-* headers, not per-tenant OIDC config, so the entire OIDC config surface is irrelevant. Open instances keep it. 8 integration tests cover all paths (signing keys hidden + 403 on embedded; visible + writable on open; OIDC 404 on managed; OIDC routes exist on open). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(embedded): address Phase 4 review blockers — brand_manifest gate + OIDC + update_general Code review + security review of #428/#429 found two blockers that ship a real regression and two H-level defense-in-depth holes. Fix what's worth fixing before merge: 1. brand_manifest section missing template gate (code review #1). The Brand Manifest Policy
Net revenue, last 30d
-
{{ currency_symbol(m.currency) }}{{ "{:,.0f}".format(m.net_revenue_30d) }}
+
{{ currency_symbol(m.currency) }}{{ "{:,.2f}".format(m.net_revenue_30d) }}
{% if m.revenue_delta_pct > 0 %} @@ -826,7 +872,7 @@

{% endif %}

- Offers waiting on you · {{ currency_symbol(inc.currency) }}{{ "{:,.0f}".format(inc.total_value) }} potential + Offers waiting on you · {{ currency_symbol(inc.currency) }}{{ "{:,.2f}".format(inc.total_value) }} potential
{% for r in inc.rows %} @@ -836,7 +882,7 @@

{{ r.order_name }}

-
{{ currency_symbol(r.currency) }}{{ "{:,.0f}".format(r.budget) }}
+
{{ currency_symbol(r.currency) }}{{ "{:,.2f}".format(r.budget) }}
{{ r.age_relative }}
 @@ -856,7 +902,7 @@

- Live deals delivering now · {{ currency_symbol(run.currency) }}{{ "{:,.0f}".format(run.total_value) }} committed + Live deals delivering now · {{ currency_symbol(run.currency) }}{{ "{:,.2f}".format(run.total_value) }} committed
{% for r in run.rows %} @@ -873,7 +919,7 @@

{{ r.order_name }}
-
{{ currency_symbol(r.currency) }}{{ "{:,.0f}".format(r.rate_per_week) }}/wk
+
{{ currency_symbol(r.currency) }}{{ "{:,.2f}".format(r.rate_per_week) }}/wk
{{ r.pacing }}
@@ -927,7 +973,13 @@

-
Revenue · 30 days
+
+ Revenue · 30 days + + + +
{{ currency_symbol(m.currency) }}{{ "{:,.2f}".format(m.net_revenue_30d) }} vs {{ currency_symbol(m.currency) }}{{ "{:,.2f}".format(m.net_revenue_prior_30) }} prior @@ -1055,8 +1107,8 @@

{% endif %} - {{ currency_symbol(buy.currency) }}{{ "{:,.0f}".format(buy.budget) }} - {{ currency_symbol(buy.currency) }}{{ "{:,.0f}".format(buy.spend) }} + {{ currency_symbol(buy.currency) }}{{ "{:,.2f}".format(buy.budget) }} + {{ currency_symbol(buy.currency) }}{{ "{:,.2f}".format(buy.spend) }} {{ buy.created_at_relative }} {% else %} From 1593f980546b0e4b836aadea361e5a3de64249d5 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Fri, 3 Jul 2026 08:56:00 +0600 Subject: [PATCH 38/90] feat: European timezone defaults, all-time GAM reporting, and delivery metric fixes (#30) Reporting page (Admin UI): - Restrict timezone filter to Amsterdam, London, Lisbon, Helsinki (one per European offset band), Amsterdam pre-selected - Default API timezone changed from America/New_York to Europe/Amsterdam (new DEFAULT_REPORTING_TIMEZONE constant) - Add "All Time" date range: accepted by all six reporting API endpoints, runs aggregated over GAM's full ~3-year retention (no DATE dimension) - Show tenant currency symbol (GAM network currency) instead of hardcoded "$" on all money values Media buy details page: - Delivery metrics no longer derive the reporting window from flight/ schedule dates (previously capped at 7 days and dropped prior-month delivery); now requests all-time delivery via the new range Dashboard: - Spend column prefers the real delivered_amount snapshot over the calendar-based budget estimate - Revenue trend attributes delivered revenue across elapsed flight days only, so the 30d net revenue headline sums to actual delivered spend - Restore documented budget pro-rata fallback for unpolled buys (was dead code behind an early continue) - Keep full precision in daily revenue series; round at display time (per-day rounding distorted small amounts) Co-authored-by: Claude Fable 5 --- src/adapters/gam_reporting_api.py | 101 ++++++++++++++---------- src/adapters/gam_reporting_service.py | 24 ++++-- src/adapters/google_ad_manager.py | 8 +- src/admin/blueprints/operations.py | 34 +++++--- src/admin/services/dashboard_service.py | 59 +++++++++----- templates/gam_reporting.html | 55 +++++++------ 6 files changed, 172 insertions(+), 109 deletions(-) diff --git a/src/adapters/gam_reporting_api.py b/src/adapters/gam_reporting_api.py index 471b9174e2..bc66c77603 100644 --- a/src/adapters/gam_reporting_api.py +++ b/src/adapters/gam_reporting_api.py @@ -26,6 +26,9 @@ # Create Blueprint gam_reporting_api = Blueprint("gam_reporting_api", __name__) +# System default timezone for reporting (Azerion HQ) +DEFAULT_REPORTING_TIMEZONE = "Europe/Amsterdam" + # Input validation patterns TENANT_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") PRINCIPAL_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") @@ -88,7 +91,7 @@ def decorated_function(*args, **kwargs): def get_tenant_access(tenant_id: str) -> bool: """Check if the current user has access to the specified tenant""" # Handle admin_ui style sessions - # Super admin has access to all tenants + # Super admin has access to all tenants if session.get("role") == "super_admin": return True @@ -114,11 +117,11 @@ def get_gam_reporting(tenant_id: str): Get GAM reporting data Query parameters: - - date_range: "lifetime", "this_month", or "today" (required) + - date_range: "lifetime", "this_month", "today", or "all_time" (required) - advertiser_id: Filter by advertiser ID (optional) - order_id: Filter by order ID (optional) - line_item_id: Filter by line item ID (optional) - - timezone: Requested timezone (default: America/New_York) + - timezone: Requested timezone (default: Europe/Amsterdam) """ # Validate tenant_id if not validate_tenant_id(tenant_id): @@ -138,8 +141,10 @@ def get_gam_reporting(tenant_id: str): # Get query parameters date_range = request.args.get("date_range") - if not date_range or date_range not in ["lifetime", "this_month", "today"]: - return jsonify({"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today"}), 400 + if not date_range or date_range not in ["lifetime", "this_month", "today", "all_time"]: + return jsonify( + {"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today, all_time"} + ), 400 # Validate optional numeric IDs advertiser_id = request.args.get("advertiser_id") @@ -155,7 +160,7 @@ def get_gam_reporting(tenant_id: str): return jsonify({"error": "Invalid line_item_id format"}), 400 # Validate timezone - timezone = request.args.get("timezone", "America/New_York") + timezone = request.args.get("timezone", DEFAULT_REPORTING_TIMEZONE) if not validate_timezone(timezone): return jsonify({"error": "Invalid timezone"}), 400 @@ -181,7 +186,7 @@ def get_gam_reporting(tenant_id: str): # Get the reporting data logger.info(f"Getting reporting data for tenant {tenant_id}, date_range={date_range}") # Type narrowing: We validated date_range above - date_range_literal = cast(Literal["lifetime", "this_month", "today"], date_range) + date_range_literal = cast(Literal["lifetime", "this_month", "today", "all_time"], date_range) report_data = reporting_service.get_reporting_data( date_range=date_range_literal, advertiser_id=advertiser_id, @@ -220,8 +225,8 @@ def get_advertiser_summary(tenant_id: str, advertiser_id: str): Get summary reporting data for a specific advertiser Query parameters: - - date_range: "lifetime", "this_month", or "today" (required) - - timezone: Requested timezone (default: America/New_York) + - date_range: "lifetime", "this_month", "today", or "all_time" (required) + - timezone: Requested timezone (default: Europe/Amsterdam) """ # Validate IDs if not validate_tenant_id(tenant_id): @@ -243,10 +248,12 @@ def get_advertiser_summary(tenant_id: str, advertiser_id: str): # Get query parameters date_range = request.args.get("date_range") - if not date_range or date_range not in ["lifetime", "this_month", "today"]: - return jsonify({"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today"}), 400 + if not date_range or date_range not in ["lifetime", "this_month", "today", "all_time"]: + return jsonify( + {"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today, all_time"} + ), 400 - timezone = request.args.get("timezone", "America/New_York") + timezone = request.args.get("timezone", DEFAULT_REPORTING_TIMEZONE) if not validate_timezone(timezone): return jsonify({"error": "Invalid timezone"}), 400 @@ -264,7 +271,7 @@ def get_advertiser_summary(tenant_id: str, advertiser_id: str): # Get the advertiser summary # Type narrowing: We validated date_range above - date_range_literal = cast(Literal["lifetime", "this_month", "today"], date_range) + date_range_literal = cast(Literal["lifetime", "this_month", "today", "all_time"], date_range) summary = reporting_service.get_advertiser_summary( advertiser_id=advertiser_id, date_range=date_range_literal, requested_timezone=timezone ) @@ -284,10 +291,10 @@ def get_principal_reporting(tenant_id: str, principal_id: str): This endpoint automatically uses the principal's configured advertiser_id Query parameters: - - date_range: "lifetime", "this_month", or "today" (required) + - date_range: "lifetime", "this_month", "today", or "all_time" (required) - order_id: Filter by order ID (optional) - line_item_id: Filter by line item ID (optional) - - timezone: Requested timezone (default: America/New_York) + - timezone: Requested timezone (default: Europe/Amsterdam) """ # Validate IDs if not validate_tenant_id(tenant_id): @@ -319,8 +326,10 @@ def get_principal_reporting(tenant_id: str, principal_id: str): # Get query parameters date_range = request.args.get("date_range") - if not date_range or date_range not in ["lifetime", "this_month", "today"]: - return jsonify({"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today"}), 400 + if not date_range or date_range not in ["lifetime", "this_month", "today", "all_time"]: + return jsonify( + {"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today, all_time"} + ), 400 # Validate optional numeric IDs order_id = request.args.get("order_id") @@ -332,7 +341,7 @@ def get_principal_reporting(tenant_id: str, principal_id: str): return jsonify({"error": "Invalid line_item_id format"}), 400 # Validate timezone - timezone = request.args.get("timezone", "America/New_York") + timezone = request.args.get("timezone", DEFAULT_REPORTING_TIMEZONE) if not validate_timezone(timezone): return jsonify({"error": "Invalid timezone"}), 400 @@ -346,19 +355,19 @@ def get_principal_reporting(tenant_id: str, principal_id: str): adapter_config = db_session.scalars(stmt_config).first() if not adapter_config: - # Default to America/New_York if no config found - network_timezone = "America/New_York" + # Default to the system timezone if no config found + network_timezone = DEFAULT_REPORTING_TIMEZONE else: # TODO: Add gam_network_timezone field to adapter_config table if timezone configuration is needed # For now, use default timezone since config field no longer exists - network_timezone = "America/New_York" + network_timezone = DEFAULT_REPORTING_TIMEZONE # Create reporting service reporting_service = GAMReportingService(gam_client, network_timezone) # Get the reporting data # Type narrowing: We validated date_range above - date_range_literal = cast(Literal["lifetime", "this_month", "today"], date_range) + date_range_literal = cast(Literal["lifetime", "this_month", "today", "all_time"], date_range) report_data = reporting_service.get_reporting_data( date_range=date_range_literal, advertiser_id=advertiser_id, @@ -399,11 +408,11 @@ def get_country_breakdown(tenant_id: str): Get GAM reporting data broken down by country Query parameters: - - date_range: "lifetime", "this_month", or "today" (required) + - date_range: "lifetime", "this_month", "today", or "all_time" (required) - advertiser_id: Filter by advertiser ID (optional) - order_id: Filter by order ID (optional) - line_item_id: Filter by line item ID (optional) - - timezone: Requested timezone (default: America/New_York) + - timezone: Requested timezone (default: Europe/Amsterdam) """ # Validate tenant_id if not validate_tenant_id(tenant_id): @@ -423,8 +432,10 @@ def get_country_breakdown(tenant_id: str): # Get query parameters date_range = request.args.get("date_range") - if not date_range or date_range not in ["lifetime", "this_month", "today"]: - return jsonify({"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today"}), 400 + if not date_range or date_range not in ["lifetime", "this_month", "today", "all_time"]: + return jsonify( + {"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today, all_time"} + ), 400 # Validate optional numeric IDs advertiser_id = request.args.get("advertiser_id") @@ -440,7 +451,7 @@ def get_country_breakdown(tenant_id: str): return jsonify({"error": "Invalid line_item_id format"}), 400 # Validate timezone - timezone = request.args.get("timezone", "America/New_York") + timezone = request.args.get("timezone", DEFAULT_REPORTING_TIMEZONE) if not validate_timezone(timezone): return jsonify({"error": "Invalid timezone"}), 400 @@ -461,7 +472,7 @@ def get_country_breakdown(tenant_id: str): # Get the country breakdown # Type narrowing: We validated date_range above - date_range_literal = cast(Literal["lifetime", "this_month", "today"], date_range) + date_range_literal = cast(Literal["lifetime", "this_month", "today", "all_time"], date_range) country_data = reporting_service.get_country_breakdown( date_range=date_range_literal, advertiser_id=advertiser_id, @@ -484,12 +495,12 @@ def get_ad_unit_breakdown(tenant_id: str): Get GAM reporting data broken down by ad unit Query parameters: - - date_range: "lifetime", "this_month", or "today" (required) + - date_range: "lifetime", "this_month", "today", or "all_time" (required) - advertiser_id: Filter by advertiser ID (optional) - order_id: Filter by order ID (optional) - line_item_id: Filter by line item ID (optional) - country: Filter by country name (optional) - - timezone: Requested timezone (default: America/New_York) + - timezone: Requested timezone (default: Europe/Amsterdam) """ # Validate tenant_id if not validate_tenant_id(tenant_id): @@ -509,8 +520,10 @@ def get_ad_unit_breakdown(tenant_id: str): # Get query parameters date_range = request.args.get("date_range") - if not date_range or date_range not in ["lifetime", "this_month", "today"]: - return jsonify({"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today"}), 400 + if not date_range or date_range not in ["lifetime", "this_month", "today", "all_time"]: + return jsonify( + {"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today, all_time"} + ), 400 # Validate optional numeric IDs advertiser_id = request.args.get("advertiser_id") @@ -529,7 +542,7 @@ def get_ad_unit_breakdown(tenant_id: str): country = request.args.get("country") # Validate timezone - timezone = request.args.get("timezone", "America/New_York") + timezone = request.args.get("timezone", DEFAULT_REPORTING_TIMEZONE) if not validate_timezone(timezone): return jsonify({"error": "Invalid timezone"}), 400 @@ -550,7 +563,7 @@ def get_ad_unit_breakdown(tenant_id: str): # Get the ad unit breakdown # Type narrowing: We validated date_range above - date_range_literal = cast(Literal["lifetime", "this_month", "today"], date_range) + date_range_literal = cast(Literal["lifetime", "this_month", "today", "all_time"], date_range) ad_unit_data = reporting_service.get_ad_unit_breakdown( date_range=date_range_literal, advertiser_id=advertiser_id, @@ -574,8 +587,8 @@ def get_principal_summary(tenant_id: str, principal_id: str): Get summary reporting data for a specific principal (advertiser) Query parameters: - - date_range: "lifetime", "this_month", or "today" (required) - - timezone: Requested timezone (default: America/New_York) + - date_range: "lifetime", "this_month", "today", or "all_time" (required) + - timezone: Requested timezone (default: Europe/Amsterdam) """ # Validate IDs if not validate_tenant_id(tenant_id): @@ -607,10 +620,12 @@ def get_principal_summary(tenant_id: str, principal_id: str): # Get query parameters date_range = request.args.get("date_range") - if not date_range or date_range not in ["lifetime", "this_month", "today"]: - return jsonify({"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today"}), 400 + if not date_range or date_range not in ["lifetime", "this_month", "today", "all_time"]: + return jsonify( + {"error": "Invalid or missing date_range. Must be one of: lifetime, this_month, today, all_time"} + ), 400 - timezone = request.args.get("timezone", "America/New_York") + timezone = request.args.get("timezone", DEFAULT_REPORTING_TIMEZONE) if not validate_timezone(timezone): return jsonify({"error": "Invalid timezone"}), 400 @@ -624,19 +639,19 @@ def get_principal_summary(tenant_id: str, principal_id: str): adapter_config = db_session.scalars(stmt_config).first() if not adapter_config: - # Default to America/New_York if no config found - network_timezone = "America/New_York" + # Default to the system timezone if no config found + network_timezone = DEFAULT_REPORTING_TIMEZONE else: # TODO: Add gam_network_timezone field to adapter_config table if timezone configuration is needed # For now, use default timezone since config field no longer exists - network_timezone = "America/New_York" + network_timezone = DEFAULT_REPORTING_TIMEZONE # Create reporting service reporting_service = GAMReportingService(gam_client, network_timezone) # Get the advertiser summary # Type narrowing: We validated date_range above - date_range_literal = cast(Literal["lifetime", "this_month", "today"], date_range) + date_range_literal = cast(Literal["lifetime", "this_month", "today", "all_time"], date_range) summary = reporting_service.get_advertiser_summary( advertiser_id=advertiser_id, date_range=date_range_literal, requested_timezone=timezone ) diff --git a/src/adapters/gam_reporting_service.py b/src/adapters/gam_reporting_service.py index 443d48d91d..2bba590a5c 100644 --- a/src/adapters/gam_reporting_service.py +++ b/src/adapters/gam_reporting_service.py @@ -174,7 +174,7 @@ def __init__(self, gam_client, network_timezone: str = None): def get_reporting_data( self, - date_range: Literal["lifetime", "this_month", "today"], + date_range: Literal["lifetime", "this_month", "today", "all_time"], advertiser_id: str | None = None, order_id: str | None = None, line_item_id: str | None = None, @@ -186,7 +186,9 @@ def get_reporting_data( Get reporting data for specified date range and filters Args: - date_range: One of "lifetime", "this_month", or "today" + date_range: One of "lifetime", "this_month", "today", or "all_time". + "all_time" spans GAM's full data retention (~3 years) and is + aggregated server-side (no DATE dimension, no time series). advertiser_id: Optional advertiser/company ID filter order_id: Optional order ID filter line_item_id: Optional line item ID filter @@ -199,7 +201,11 @@ def get_reporting_data( """ # Determine the appropriate dimensions and date range dimensions, start_date, end_date, granularity = self._get_report_config( - date_range, requested_timezone, include_country, include_ad_unit + date_range, + requested_timezone, + include_country, + include_ad_unit, + include_date=date_range != "all_time", ) # Build the report query @@ -285,6 +291,12 @@ def _get_report_config( start_date = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) end_date = now granularity = "total" + elif date_range == "all_time": + # Span GAM's full reporting data retention (~3 years). Only + # available aggregated — one row per entity keeps this cheap. + start_date = (now - timedelta(days=3 * 365)).replace(hour=0, minute=0, second=0, microsecond=0) + end_date = now + granularity = "total" else: # lifetime # For aggregated queries, we can use longer date ranges since we get one row per entity start_date = (now - timedelta(days=90)).replace(hour=0, minute=0, second=0, microsecond=0) @@ -2038,7 +2050,7 @@ def _calculate_metrics(self, data: list[dict[str, Any]]) -> dict[str, Any]: def get_country_breakdown( self, - date_range: Literal["lifetime", "this_month", "today"], + date_range: Literal["lifetime", "this_month", "today", "all_time"], advertiser_id: str | None = None, order_id: str | None = None, line_item_id: str | None = None, @@ -2144,7 +2156,7 @@ def get_country_breakdown( def get_ad_unit_breakdown( self, - date_range: Literal["lifetime", "this_month", "today"], + date_range: Literal["lifetime", "this_month", "today", "all_time"], advertiser_id: str | None = None, order_id: str | None = None, line_item_id: str | None = None, @@ -2295,7 +2307,7 @@ def get_ad_unit_breakdown( def get_advertiser_summary( self, advertiser_id: str, - date_range: Literal["lifetime", "this_month", "today"], + date_range: Literal["lifetime", "this_month", "today", "all_time"], requested_timezone: str = "America/New_York", ) -> dict[str, Any]: """ diff --git a/src/adapters/google_ad_manager.py b/src/adapters/google_ad_manager.py index 8bb59d3247..9be3dd5f6c 100644 --- a/src/adapters/google_ad_manager.py +++ b/src/adapters/google_ad_manager.py @@ -1204,12 +1204,16 @@ def get_media_buy_delivery( range_type: str = "today" elif days_diff <= 31: range_type = "this_month" - else: + elif days_diff <= 365: range_type = "lifetime" + else: + # Spans over a year request GAM's full retention, aggregated + # server-side (no daily breakdown in the response). + range_type = "all_time" # Fetch delivery data scoped to this specific GAM order reporting_data = reporting_service.get_reporting_data( - date_range=cast("Literal['lifetime', 'this_month', 'today']", range_type), + date_range=cast("Literal['lifetime', 'this_month', 'today', 'all_time']", range_type), advertiser_id=self.advertiser_id, order_id=gam_order_id, requested_timezone="America/New_York", diff --git a/src/admin/blueprints/operations.py b/src/admin/blueprints/operations.py index a26616f485..c69ac422df 100644 --- a/src/admin/blueprints/operations.py +++ b/src/admin/blueprints/operations.py @@ -49,7 +49,7 @@ def reporting(tenant_id): from flask import render_template from src.core.database.database_session import get_db_session - from src.core.database.models import Tenant + from src.core.database.models import AdapterConfig, CurrencyLimit, Tenant with get_db_session() as db_session: tenant_obj = db_session.scalars(select(Tenant).filter_by(tenant_id=tenant_id)).first() @@ -81,7 +81,20 @@ def reporting(tenant_id): 400, ) - return render_template("gam_reporting.html", tenant=tenant) + # Resolve the tenant's reporting currency the same way the dashboard + # does: GAM network currency → first CurrencyLimit → EUR. + currency = "EUR" + adapter_config = db_session.scalars(select(AdapterConfig).filter_by(tenant_id=tenant_id)).first() + if adapter_config and adapter_config.gam_network_currency: + currency = str(adapter_config.gam_network_currency) + else: + currency_limit = db_session.scalars( + select(CurrencyLimit).filter_by(tenant_id=tenant_id).order_by(CurrencyLimit.currency_code) + ).first() + if currency_limit: + currency = str(currency_limit.currency_code) + + return render_template("gam_reporting.html", tenant=tenant, currency=currency) @operations_bp.route("/media-buy/", methods=["GET"]) @@ -238,17 +251,14 @@ def media_buy_detail(tenant_id, media_buy_id): ) adapter = get_adapter(principal_schema, dry_run=False) - # Calculate date range (last 7 days or campaign duration) - always use UTC + # Request all-time delivery, independent of the buy's + # flight/schedule dates. A span over a year makes the + # GAM adapter classify the request as "all_time" (full + # GAM data retention, aggregated); shorter spans + # collapse to "today"/"this_month"/"lifetime" and drop + # delivery from earlier periods. end_date = datetime.now(UTC) - seven_days_ago = datetime.now(UTC) - timedelta(days=7) - - # Convert media_buy.start_date (date) to datetime with UTC timezone - mb_start = media_buy.start_date - if mb_start: - # Convert date to datetime (start of day) with UTC timezone - mb_start = datetime.combine(mb_start, datetime.min.time()).replace(tzinfo=UTC) - - start_date = max(mb_start if mb_start else seven_days_ago, seven_days_ago) + start_date = end_date - timedelta(days=3 * 365) reporting_period = ReportingPeriod(start=start_date, end=end_date) diff --git a/src/admin/services/dashboard_service.py b/src/admin/services/dashboard_service.py index 2d50830e20..a71e695dd2 100644 --- a/src/admin/services/dashboard_service.py +++ b/src/admin/services/dashboard_service.py @@ -183,10 +183,16 @@ def get_recent_media_buys(self, limit: int = 10) -> list: def _calculate_revenue_trend( self, db_session, days: int = 30, *, end_date: date | None = None, repo: MediaBuyRepository | None = None ) -> list[dict[str, Any]]: - """Calculate daily revenue for `days` days ending on `end_date` (default today).""" + """Calculate daily revenue for `days` days ending on `end_date` (default today). + + Values are full precision — round at display time only, so that + window sums (net revenue headline) don't accumulate per-day + rounding error on small amounts. + """ if repo is None: repo = MediaBuyRepository(db_session, self.tenant_id) anchor = end_date or datetime.now(UTC).date() + today = datetime.now(UTC).date() revenue_data = [] for i in range(days): @@ -196,10 +202,6 @@ def _calculate_revenue_trend( daily_revenue = 0.0 for buy in daily_buys: - - if buy.delivered_amount is None: - continue - start_date = type_cast(date | None, buy.start_date) end_date = type_cast(date | None, buy.end_date) if not (start_date and end_date): @@ -207,13 +209,22 @@ def _calculate_revenue_trend( days_in_flight = (end_date - start_date).days + 1 if days_in_flight <= 0: continue - # Per-day delivered revenue: pro-rate the actual delivered amount - # across the flight. Fall back to budget pro-rata for buys with no - # delivery snapshot yet (not polled, needs creative, etc.). - total = float(buy.delivered_amount) if buy.delivered_amount is not None else float(buy.budget or 0) - daily_revenue += total / days_in_flight - - revenue_data.append({"date": day.isoformat(), "revenue": round(daily_revenue, 2)}) + if buy.delivered_amount is not None: + # Actual delivered revenue: attribute across the *elapsed* + # flight days only — money already earned can't belong to + # future days. This keeps the window sum equal to the real + # delivered amount for in-flight buys. + elapsed_end = min(today, end_date) + elapsed_days = (elapsed_end - start_date).days + 1 + if elapsed_days <= 0 or day > elapsed_end: + continue + daily_revenue += float(buy.delivered_amount) / elapsed_days + # else: + # # No delivery snapshot yet (not polled, needs creative, + # # etc.) — fall back to budget pro-rata over the flight. + # daily_revenue += float(buy.budget or 0) / days_in_flight + + revenue_data.append({"date": day.isoformat(), "revenue": daily_revenue}) return revenue_data @@ -231,12 +242,17 @@ def _calculate_revenue_change(self, revenue_data: list[dict[str, Any]]) -> float return 0.0 def _calculate_estimated_spend(self, media_buy) -> float: - """Calculate estimated spend based on campaign progress. + """Spend for the dashboard table: real delivery snapshot when + available, calendar estimate otherwise. - For active campaigns, estimate based on days elapsed. - For completed campaigns, return full budget. - For pending/draft campaigns, return 0. + With no snapshot: active campaigns estimate budget × elapsed/total + days, completed campaigns return full budget, pending/draft return 0. """ + # Real delivered spend (synced from the ad server) wins over any + # calendar-based estimate. + if media_buy.delivered_amount is not None: + return float(media_buy.delivered_amount) + if not media_buy.budget or not media_buy.start_date: return 0.0 @@ -310,7 +326,7 @@ def get_chart_data(self) -> dict[str, list]: metrics = self.get_dashboard_metrics() revenue_data = metrics["revenue_data"] - return {"labels": [d["date"] for d in revenue_data], "data": [d["revenue"] for d in revenue_data]} + return {"labels": [d["date"] for d in revenue_data], "data": [round(d["revenue"], 2) for d in revenue_data]} def get_revenue_trend(self, days: int) -> dict[str, Any]: """Daily revenue trend plus net totals for a custom window. @@ -330,7 +346,7 @@ def get_revenue_trend(self, days: int) -> dict[str, Any]: net_prior = sum(d["revenue"] for d in prior) return { "labels": [d["date"] for d in trend], - "values": [d["revenue"] for d in trend], + "values": [round(d["revenue"], 2) for d in trend], "currency": self._primary_currency(session), "net_revenue": round(float(net), 2), "net_revenue_prior": round(float(net_prior), 2), @@ -444,8 +460,8 @@ def _masthead(self, session, tenant: Tenant | None) -> dict[str, Any]: "last_offer_at": last_offer_at, "last_brief_relative": self._format_relative_time(last_brief_at) if last_brief_at else None, "last_offer_relative": self._format_relative_time(last_offer_at) if last_offer_at else None, - "net_revenue_30d": float(net_30d), - "net_revenue_prior_30": float(net_prior_30), + "net_revenue_30d": round(float(net_30d), 2), + "net_revenue_prior_30": round(float(net_prior_30), 2), "revenue_delta_pct": round(delta_pct, 1), "today_label": now.strftime("%a, %b %-d"), } @@ -643,7 +659,8 @@ def _pipeline(self, session) -> dict[str, Any]: def _revenue_chart(self, session, repo: MediaBuyRepository, days: int = 30) -> list[dict[str, Any]]: """Per-day delivered revenue series. Falls back to flat-pace budget allocation for buys with no snapshot.""" - return self._calculate_revenue_trend(session, days=days, repo=repo) + trend = self._calculate_revenue_trend(session, days=days, repo=repo) + return [{"date": d["date"], "revenue": round(d["revenue"], 2)} for d in trend] def _needs_attention(self, session, repo: MediaBuyRepository) -> list[dict[str, Any]]: """Bullet-list items for the right-rail attention panel. diff --git a/templates/gam_reporting.html b/templates/gam_reporting.html index 21159a5eb6..5e61bcc27f 100644 --- a/templates/gam_reporting.html +++ b/templates/gam_reporting.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% from '_macros.html' import currency_symbol %} {% block title %}GAM Reporting - {{ tenant.name }}{% endblock %} @@ -16,16 +17,17 @@

GAM Reporting - {{ tenant.name }}

+ +

@@ -237,6 +239,8 @@

Performance by Ad Unit

diff --git a/templates/adapters/improvedigital/product_config.html b/templates/adapters/improvedigital/product_config.html new file mode 100644 index 0000000000..a722e035ad --- /dev/null +++ b/templates/adapters/improvedigital/product_config.html @@ -0,0 +1,157 @@ +{# +Improve Digital Product Configuration Template + +Included by add_product.html / edit_product.html when the tenant's adapter +is improvedigital. Renders Classic line-item inventory pickers backed by the +synced improvedigital_inventory cache (run "Sync Inventory Now" on the +Ad Server settings page first) plus line-item delivery defaults. + +Form fields use the impl_ prefix and are parsed by +_improvedigital_implementation_config() in src/admin/blueprints/products.py. + +Expected context: +- tenant_id: current tenant ID +- product (edit page only): dict with implementation_config +#} + +{% set impl = (product.implementation_config if product is defined and product and product.implementation_config else {}) %} + +
+

Improve Digital Configuration

+

+ Pin this product's Classic line items to 360Yield inventory. Pickers load + from the synced inventory cache — if they're empty, run + Sync Inventory Now on the Ad Server settings page. +

+ +
+ + + + Placements the line item is pinned to. Hold Cmd/Ctrl to select multiple. + +
+ +
+ + + + Placements explicitly excluded from delivery. + +
+ +
+ + + + Reusable placement groupings assigned to the line item. + +
+ +
+ + + + Creative sizes the line item targets. + +
+ +

Line-item Defaults

+ +
+ + + + CPM is confirmed; further models are pending platform confirmation (gap G2). + +
+ +
+ + +
+ +
+
+ + +
+
+ + +
+
+ +
+ + +
+
+ + diff --git a/templates/tenant_settings.html b/templates/tenant_settings.html index a10126f7b8..1a372241c7 100644 --- a/templates/tenant_settings.html +++ b/templates/tenant_settings.html @@ -764,6 +764,15 @@

FreeWheel

SpringServe (Magnite)

Direct-sold CTV, online video, and audio via Magnite SpringServe

+ +
+ {% if active_adapter == 'improvedigital' %} + Current + {% endif %} +

Improve Digital

+

Classic direct campaigns on Azerion's 360Yield Marketplace

+
diff --git a/tests/unit/test_improvedigital_adapter.py b/tests/unit/test_improvedigital_adapter.py new file mode 100644 index 0000000000..af832aa7ad --- /dev/null +++ b/tests/unit/test_improvedigital_adapter.py @@ -0,0 +1,214 @@ +"""Tests for the Improve Digital adapter — registry wiring + Classic dry-run behaviour.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from unittest.mock import MagicMock + +import pytest + +from src.adapters import get_adapter_default_channels, get_adapter_schemas +from src.adapters.improvedigital import ImproveDigitalAdapter +from src.adapters.improvedigital.schemas import ImproveDigitalConnectionConfig, ImproveDigitalProductConfig +from tests.helpers.adapter_test_helpers import ( + invoke_create_media_buy, + make_sample_create_request, + make_sample_video_package, +) + + +@pytest.fixture +def mock_principal(): + principal = MagicMock() + principal.name = "classic_advertiser" + principal.principal_id = "principal_impd_1" + # Advertiser IDs are integers in the live API; a numeric string keeps + # casts clean on the CampaignDto advertiserId field. + principal.get_adapter_id.return_value = "5001" + principal.platform_mappings = {"improvedigital": {"advertiser_id": "5001"}} + return principal + + +@pytest.fixture +def sample_request(): + return make_sample_create_request() + + +@pytest.fixture +def sample_packages(): + return [make_sample_video_package()] + + +def make_dry_run_adapter(mock_principal, config: dict | None = None) -> ImproveDigitalAdapter: + return ImproveDigitalAdapter( + config=config or {}, + principal=mock_principal, + dry_run=True, + tenant_id="tenant_impd_1", + ) + + +class TestRegistry: + def test_get_adapter_schemas_returns_improvedigital_classes(self): + schemas = get_adapter_schemas("improvedigital") + assert schemas is not None + assert schemas.connection_config is ImproveDigitalConnectionConfig + assert schemas.product_config is ImproveDigitalProductConfig + assert schemas.capabilities.inventory_entity_label == "Placements" + + def test_sync_capabilities_match_implementation_state(self): + # Inventory sync landed with Phase 2; reporting flips alongside the + # Phase 3 Report API cache — the scheduler must not call its stub. + schemas = get_adapter_schemas("improvedigital") + assert schemas.capabilities.supports_inventory_sync is True + assert schemas.capabilities.supports_reporting_sync is False + + def test_default_channels_cover_classic_media_types(self): + channels = get_adapter_default_channels("improvedigital") + assert "display" in channels + assert "olv" in channels + + +class TestAdapterConstruction: + def test_dry_run_defers_client_construction(self, mock_principal): + adapter = make_dry_run_adapter(mock_principal) + assert adapter._client is None + assert adapter.advertiser_id == "5001" + + def test_live_mode_without_credentials_raises(self, mock_principal): + with pytest.raises(ValueError, match="client_id \\+ client_secret"): + ImproveDigitalAdapter( + config={"improve_demand_contact_id": 7}, + principal=mock_principal, + dry_run=False, + tenant_id="tenant_impd_1", + ) + + def test_live_mode_without_advertiser_raises(self, mock_principal): + mock_principal.get_adapter_id.return_value = None + with pytest.raises(ValueError, match="advertiser ID"): + ImproveDigitalAdapter( + config={"client_id": "app-1", "client_secret": "s", "improve_demand_contact_id": 7}, + principal=mock_principal, + dry_run=False, + tenant_id="tenant_impd_1", + ) + + def test_live_mode_without_demand_contact_raises(self, mock_principal): + # The Classic campaign API rejects campaigns without + # improve_demand_contact_id — fail at construction, not at create. + with pytest.raises(ValueError, match="improve_demand_contact_id"): + ImproveDigitalAdapter( + config={"client_id": "app-1", "client_secret": "s"}, + principal=mock_principal, + dry_run=False, + tenant_id="tenant_impd_1", + ) + + def test_supported_pricing_models(self, mock_principal): + adapter = make_dry_run_adapter(mock_principal) + assert adapter.get_supported_pricing_models() == {"cpm"} + + def test_targeting_capabilities_reject_postal(self, mock_principal): + capabilities = make_dry_run_adapter(mock_principal).get_targeting_capabilities() + assert capabilities.geo_countries is True + assert capabilities.geo_regions is True + # 360Yield location targeting stops at city level — no postal systems. + assert capabilities.us_zip is False + assert capabilities.de_plz is False + + def test_creative_formats_cover_display_and_video(self, mock_principal): + formats = make_dry_run_adapter(mock_principal).get_creative_formats() + format_ids = {fmt["format_id"]["id"] for fmt in formats} + assert "display_image" in format_ids + assert "video_vast" in format_ids + + +class TestAdapterDryRun: + def test_dry_run_creates_buy_without_calling_client(self, mock_principal, sample_request, sample_packages): + adapter = make_dry_run_adapter(mock_principal, config={"improve_demand_contact_id": 7}) + response = invoke_create_media_buy(adapter, sample_request, sample_packages) + assert response.packages is not None + assert len(response.packages) == 1 + assert adapter._client is None + + def test_dry_run_rejects_postal_targeting(self, mock_principal, sample_request, sample_packages): + postal = MagicMock() + postal.geo_postal_areas = ["1012"] + sample_packages[0].targeting_overlay = postal + adapter = make_dry_run_adapter(mock_principal) + response = invoke_create_media_buy(adapter, sample_request, sample_packages) + assert response.errors[0].code == "unsupported_targeting" + + def test_dry_run_status_is_active(self, mock_principal): + adapter = make_dry_run_adapter(mock_principal) + response = adapter.check_media_buy_status("improvedigital_adcp_1", today=datetime.now(UTC)) + assert response.status == "active" + + def test_unsupported_update_action_rejected(self, mock_principal): + adapter = make_dry_run_adapter(mock_principal) + response = adapter.update_media_buy( + "improvedigital_adcp_1", + action="do_something_weird", + package_id=None, + budget=None, + today=datetime.now(UTC), + ) + assert response.errors + + def test_dry_run_inventory_sync_soft_fails(self, mock_principal): + # No credentials in dry-run — the sync reports a failed run instead + # of raising, so the shared scheduler records it as a failed SyncJob. + adapter = make_dry_run_adapter(mock_principal) + result = adapter.run_inventory_sync() + assert result.succeeded is False + assert "dry-run" in result.errors["adapter"] + + def test_pause_media_buy_dry_run_succeeds(self, mock_principal): + adapter = make_dry_run_adapter(mock_principal) + response = adapter.update_media_buy( + "improvedigital_adcp_1", + action="pause_media_buy", + package_id=None, + budget=None, + today=datetime.now(UTC), + ) + assert response.affected_packages == [] + + +class TestClassicCreatives: + def test_dry_run_tag_creative_approved(self, mock_principal): + adapter = make_dry_run_adapter(mock_principal) + statuses = adapter.add_creative_assets( + "improvedigital_123", + assets=[ + { + "creative_id": "cr_1", + "name": "Banner 300x250", + "asset_type": "banner", + "snippet": "", + "width": 300, + "height": 250, + } + ], + today=datetime.now(UTC), + ) + assert statuses[0].status == "approved" + assert statuses[0].creative_id == "cr_1" + + def test_asset_missing_size_rejected_explicitly(self, mock_principal): + # CreativeDto requires a size — partial assets fail loudly, they are + # never accepted and silently dropped later. + adapter = make_dry_run_adapter(mock_principal) + statuses = adapter.add_creative_assets( + "improvedigital_123", + assets=[{"creative_id": "cr_2", "snippet": ""}], + today=datetime.now(UTC), + ) + assert statuses[0].status == "failed" + assert "size" in statuses[0].message + + def test_dry_run_association_succeeds(self, mock_principal): + adapter = make_dry_run_adapter(mock_principal) + results = adapter.associate_creatives(["li_1"], ["9001"]) + assert results == [{"line_item_id": "li_1", "creative_id": "9001", "status": "success"}] diff --git a/tests/unit/test_improvedigital_inventory_sync.py b/tests/unit/test_improvedigital_inventory_sync.py new file mode 100644 index 0000000000..87a380ec7a --- /dev/null +++ b/tests/unit/test_improvedigital_inventory_sync.py @@ -0,0 +1,143 @@ +"""Tests for the Improve Digital inventory sync — pagination, dedup, partial failure.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from src.adapters.improvedigital.inventory_sync import ( + PAGE_SIZE, + ImproveDigitalInventorySync, + SyncResult, + _extract_items, +) + + +def make_sync(client: MagicMock) -> tuple[ImproveDigitalInventorySync, MagicMock]: + session = MagicMock() + sync = ImproveDigitalInventorySync(client=client, session=session, tenant_id="tenant_impd_1") + return sync, session + + +def make_client(*, placements=None, packages=None, sizes=None) -> MagicMock: + """Client whose list endpoints return a single short page each.""" + client = MagicMock() + client.inventory.search_placements.return_value = {"placements": placements or []} + client.inventory.list_packages.return_value = {"packages": packages or []} + client.lookups.sizes.return_value = {"sizes": sizes or []} + return client + + +class TestSyncResult: + def test_total_and_success(self): + result = SyncResult(counts={"placement": 3, "size": 2}) + assert result.total_synced == 5 + assert result.succeeded is True + + def test_errors_mean_failure(self): + result = SyncResult(counts={"size": 2}, errors={"placement": "boom"}) + assert result.succeeded is False + + +class TestExtractItems: + def test_prefers_documented_key(self): + body = {"sizes": [{"id": 1}], "other": [{"id": 99}]} + assert _extract_items(body, "sizes") == [{"id": 1}] + + def test_falls_back_to_first_list_value(self): + body = {"totalNumberOfElemements": 1, "renamed_key": [{"id": 7}]} + assert _extract_items(body, "sizes") == [{"id": 7}] + + def test_non_envelope_bodies_yield_nothing(self): + assert _extract_items(None, "sizes") == [] + assert _extract_items({"count": 3}, "sizes") == [] + + +class TestPlacementSync: + def test_placements_and_publishers_derived(self): + client = make_client( + placements=[ + {"id": 11, "name": "Homepage ATF", "publisher_id": 5, "publisher_name": "Pub Five"}, + {"id": 12, "name": "Article BTF", "publisher_id": 5, "publisher_name": "Pub Five"}, + {"id": 13, "name": "Run of Site", "publisher_id": 6, "publisher_name": "Pub Six"}, + ] + ) + sync, session = make_sync(client) + result = sync.run() + + assert result.succeeded is True + assert result.counts["placement"] == 3 + assert result.counts["publisher"] == 2 # deduped across placements + assert session.execute.call_count >= 1 + + def test_pagination_stops_on_short_page(self): + full_page = [{"id": i, "name": f"p{i}", "publisher_id": 1, "publisher_name": "Pub"} for i in range(PAGE_SIZE)] + short_page = [{"id": 9999, "name": "last", "publisher_id": 1, "publisher_name": "Pub"}] + client = make_client() + client.inventory.search_placements.side_effect = [ + {"placements": full_page, "totalNumberOfElemements": PAGE_SIZE + 1}, + {"placements": short_page, "totalNumberOfElemements": PAGE_SIZE + 1}, + ] + sync, _session = make_sync(client) + result = sync.run() + + assert result.counts["placement"] == PAGE_SIZE + 1 + assert client.inventory.search_placements.call_count == 2 + + def test_rows_without_id_skipped(self): + client = make_client(placements=[{"name": "no id"}, {"id": 1, "name": "ok"}]) + sync, _session = make_sync(client) + result = sync.run() + assert result.counts["placement"] == 1 + + +class TestPartialFailure: + def test_placement_failure_does_not_block_other_families(self): + client = make_client( + packages=[{"id": 3, "name": "Premium Bundle"}], sizes=[{"id": 4, "width": 300, "height": 250}] + ) + client.inventory.search_placements.side_effect = RuntimeError("upstream 500") + sync, _session = make_sync(client) + result = sync.run() + + assert result.succeeded is False + assert "upstream 500" in result.errors["placement"] + assert result.counts["package"] == 1 + assert result.counts["size"] == 1 + assert "placement" not in result.counts + + +class TestUpsertRows: + def test_size_name_falls_back_to_dimensions(self): + client = make_client( + sizes=[{"id": 4, "width": 300, "height": 250}, {"id": 5, "name": "Skin", "width": 1, "height": 1}] + ) + sync, _session = make_sync(client) + sync._repo = MagicMock() + result = sync.run() + + assert result.counts["size"] == 2 + size_rows = [ + row + for call in sync._repo.bulk_upsert.call_args_list + for row in call.args[0] + if row["entity_type"] == "size" + ] + by_id = {row["entity_id"]: row for row in size_rows} + assert by_id["4"]["name"] == "300x250" + assert by_id["5"]["name"] == "Skin" + + def test_placement_rows_carry_publisher_parent(self): + client = make_client( + placements=[{"id": 11, "name": "Homepage", "publisher_id": 5, "publisher_name": "Pub Five"}] + ) + sync, _session = make_sync(client) + sync._repo = MagicMock() + sync.run() + + rows = [row for call in sync._repo.bulk_upsert.call_args_list for row in call.args[0]] + placement = next(row for row in rows if row["entity_type"] == "placement") + publisher = next(row for row in rows if row["entity_type"] == "publisher") + assert placement["entity_id"] == "11" + assert placement["parent_id"] == "5" + assert publisher["entity_id"] == "5" + assert publisher["name"] == "Pub Five" diff --git a/tests/unit/test_improvedigital_schemas.py b/tests/unit/test_improvedigital_schemas.py new file mode 100644 index 0000000000..89a1b2583e --- /dev/null +++ b/tests/unit/test_improvedigital_schemas.py @@ -0,0 +1,124 @@ +"""Tests for Improve Digital adapter schemas — encryption round-trip + validation.""" + +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest +from pydantic import ValidationError + +from src.adapters.improvedigital.schemas import ImproveDigitalConnectionConfig, ImproveDigitalProductConfig +from src.core.utils.encryption import generate_encryption_key, is_encrypted + + +@pytest.fixture +def encryption_key(): + key = generate_encryption_key() + with patch.dict(os.environ, {"ENCRYPTION_KEY": key}): + yield key + + +class TestConnectionConfig: + """OAuth2 client_credentials is the only auth path.""" + + def test_accepts_client_credentials(self): + cfg = ImproveDigitalConnectionConfig(client_id="app-1", client_secret="hunter2") + assert cfg.client_id == "app-1" + assert cfg.client_secret == "hunter2" + assert cfg.api_base_url == "https://api.360yield.com" + assert cfg.currency == "EUR" + assert cfg.timezone == "UTC" + assert cfg.improve_demand_contact_id is None + assert cfg.default_advertiser_id is None + assert cfg.agency_id is None + + def test_client_secret_serializes_to_ciphertext(self, encryption_key): + cfg = ImproveDigitalConnectionConfig(client_id="app-1", client_secret="super-secret") + dumped = cfg.model_dump() + assert dumped["client_secret"] != "super-secret" + assert is_encrypted(dumped["client_secret"]) + + def test_client_secret_round_trips_through_dump_and_validate(self, encryption_key): + original = ImproveDigitalConnectionConfig(client_id="app-1", client_secret="super-secret") + persisted = original.model_dump() + rehydrated = ImproveDigitalConnectionConfig.model_validate(persisted) + assert rehydrated.client_secret == "super-secret" + + def test_already_encrypted_secret_not_double_encrypted(self, encryption_key): + cfg = ImproveDigitalConnectionConfig(client_id="app-1", client_secret="super-secret") + ciphertext = cfg.model_dump()["client_secret"] + rehydrated = ImproveDigitalConnectionConfig.model_validate({"client_id": "app-1", "client_secret": ciphertext}) + assert rehydrated.client_secret == "super-secret" + + def test_missing_credentials_rejected(self): + with pytest.raises(ValidationError, match="client_id \\+ client_secret"): + ImproveDigitalConnectionConfig() + + def test_partial_credentials_rejected(self): + with pytest.raises(ValidationError, match="client_id \\+ client_secret"): + ImproveDigitalConnectionConfig(client_id="app-1") + + def test_non_https_base_url_rejected(self): + with pytest.raises(ValidationError, match="https"): + ImproveDigitalConnectionConfig( + client_id="app-1", + client_secret="s", + api_base_url="http://api.360yield.com", + ) + + def test_secret_flag_visible_in_json_schema(self): + schema = ImproveDigitalConnectionConfig.model_json_schema() + assert schema["properties"]["client_secret"]["secret"] is True + + def test_classic_defaults_round_trip(self, encryption_key): + cfg = ImproveDigitalConnectionConfig( + client_id="app-1", + client_secret="s", + improve_demand_contact_id=7, + default_advertiser_id=42, + agency_id=9, + currency="USD", + timezone="Europe/Amsterdam", + ) + rehydrated = ImproveDigitalConnectionConfig.model_validate(cfg.model_dump()) + assert rehydrated.improve_demand_contact_id == 7 + assert rehydrated.default_advertiser_id == 42 + assert rehydrated.agency_id == 9 + assert rehydrated.currency == "USD" + assert rehydrated.timezone == "Europe/Amsterdam" + + +class TestProductConfig: + def test_defaults_are_empty(self): + cfg = ImproveDigitalProductConfig() + assert cfg.placement_ids == [] + assert cfg.excluded_placement_ids == [] + assert cfg.package_ids == [] + assert cfg.size_ids == [] + assert cfg.pricing_model is None + assert cfg.frequency_cap is None + assert cfg.seller_types == [] + + def test_full_envelope_round_trips(self): + cfg = ImproveDigitalProductConfig( + placement_ids=[1, 2], + excluded_placement_ids=[3], + package_ids=[4], + size_ids=[5, 6], + pricing_model="CPM", + frequency_cap=3, + frequency_interval=1, + frequency_interval_type="days", + delivery_schedule="even", + azerion_owned=True, + seller_types=["PUBLISHER"], + iab_categories=[100], + tier_ids=[2], + ) + rehydrated = ImproveDigitalProductConfig.model_validate(cfg.model_dump()) + assert rehydrated == cfg + + def test_unknown_fields_rejected(self): + with pytest.raises(ValidationError): + ImproveDigitalProductConfig(not_a_field=True) From 64a9700e2a9c4a3a5987103a723e0bc798d81282 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 28 Jul 2026 15:56:07 +0600 Subject: [PATCH 61/90] feat: show tenant adapter type in admin nav header Inject nav_adapter_type in the inject_context processor (tenant is detached from the session at render time, so the relationship can't be lazy-loaded in Jinja) and render it after the workspace name in the top bar. Display labels come from ADAPTER_LABELS, moved from tenant_signals.py into src/admin/utils/helpers.py (extended with triton and improvedigital) and shared between the nav and the signals bulk-map UI. Co-Authored-By: Claude Fable 5 --- src/admin/app.py | 13 ++++++++++++- src/admin/blueprints/tenant_signals.py | 13 ++++--------- src/admin/utils/helpers.py | 13 +++++++++++++ templates/base.html | 7 +++++-- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/admin/app.py b/src/admin/app.py index 4d304124f1..7cd428aff8 100644 --- a/src/admin/app.py +++ b/src/admin/app.py @@ -615,8 +615,9 @@ def inject_context(): from flask import g, session from sqlalchemy import func, select + from src.admin.utils.helpers import ADAPTER_LABELS from src.core.database.database_session import get_db_session - from src.core.database.models import Product, Tenant + from src.core.database.models import AdapterConfig, Product, Tenant from src.core.domain_config import get_sales_agent_domain, get_support_email from src.core.version import get_build_info @@ -669,6 +670,16 @@ def inject_context(): db_session.scalar(select(func.count()).select_from(Product).filter_by(tenant_id=tenant_id)) or 0 ) + # Adapter type shown next to the workspace name in the + # top bar; fetched here because ``tenant`` is detached + # once the session closes, so the relationship can't + # be lazy-loaded from the template. + adapter_type = db_session.scalar( + select(AdapterConfig.adapter_type).filter_by(tenant_id=tenant_id) + ) + context["nav_adapter_type"] = ( + ADAPTER_LABELS.get(adapter_type, adapter_type) if adapter_type else None + ) except Exception as e: logger.warning(f"Could not load tenant {tenant_id} for context: {e}") diff --git a/src/admin/blueprints/tenant_signals.py b/src/admin/blueprints/tenant_signals.py index 9ad74083d8..6b6f646bff 100644 --- a/src/admin/blueprints/tenant_signals.py +++ b/src/admin/blueprints/tenant_signals.py @@ -41,6 +41,7 @@ from src.admin.services.catalog_webhook_events import publish_signal_catalog_changes from src.admin.utils import require_tenant_access from src.admin.utils.audit_decorator import log_admin_action +from src.admin.utils.helpers import ADAPTER_LABELS from src.admin.utils.signal_id import unique_signal_id from src.core.database.database_session import get_db_session from src.core.database.models import Tenant, TenantSignal @@ -61,14 +62,8 @@ _MAX_SIGNAL_NAME_LENGTH = 200 _MAX_BULK_SIGNAL_IDS = 500 # Display labels for the (multi-)adapter source list on the bulk-map UI -# (#480). Keys match ``tenant.ad_server`` values. -_ADAPTER_LABELS = { - "google_ad_manager": "Google Ad Manager", - "freewheel": "Freewheel", - "broadstreet": "Broadstreet", - "springserve": "SpringServe", - "mock": "Mock", -} +# (#480) now live in ``src.admin.utils.helpers.ADAPTER_LABELS``, shared +# with the nav header. def _notify_signal_catalog_changes( @@ -389,7 +384,7 @@ def _mapped_payload(signal: TenantSignal) -> dict[str, Any]: has_inventory = bool(segments or keys or composites) adapter_key = tenant.ad_server or "mock" - adapter_label = _ADAPTER_LABELS.get(adapter_key, adapter_key) + adapter_label = ADAPTER_LABELS.get(adapter_key, adapter_key) return render_template( "tenant_signals_list.html", tenant_id=tenant_id, diff --git a/src/admin/utils/helpers.py b/src/admin/utils/helpers.py index 4985aea234..30027f395c 100644 --- a/src/admin/utils/helpers.py +++ b/src/admin/utils/helpers.py @@ -22,6 +22,19 @@ logger = logging.getLogger(__name__) +# Display labels for ad-server adapter types. Keys match +# ``adapter_config.adapter_type`` / ``tenant.ad_server`` values. +ADAPTER_LABELS: dict[str, str] = { + "google_ad_manager": "Google Ad Manager", + "gam": "Google Ad Manager", + "freewheel": "Freewheel", + "broadstreet": "Broadstreet", + "springserve": "SpringServe", + "triton": "Triton", + "improvedigital": "Improve Digital", + "mock": "Mock", +} + def is_admin_production() -> bool: """Return True when admin should behave in production-safe mode. diff --git a/templates/base.html b/templates/base.html index 6787e7689a..7817f2641b 100644 --- a/templates/base.html +++ b/templates/base.html @@ -115,8 +115,8 @@

{% if session.role == 'super_admin' %}Sales Agent{% else %}Sales Agent{% end

+
+ +
+ + +
+ + + Discovery needs admin-scoped credentials; otherwise + enter the IDs below manually + (ask your Improve Digital account team). + +
+ +
+ + +
+ +
+ + + + Offices are per-currency — pick the one matching the default currency below. + +
+
+ placeholder="Optional — office default applies when empty" + {% if adapter_config and adapter_config.get('improve_demand_contact_id') %}readonly{% endif %}> - The Classic campaign API requires a demand contact on every campaign — ask your - Improve Digital account team for the ID. + Optional override — the selected office carries a default demand contact.
@@ -149,12 +198,16 @@

Inventory Sync

diff --git a/tests/unit/test_improvedigital_adapter.py b/tests/unit/test_improvedigital_adapter.py index 5668f6bcc2..c535387675 100644 --- a/tests/unit/test_improvedigital_adapter.py +++ b/tests/unit/test_improvedigital_adapter.py @@ -57,11 +57,11 @@ def test_get_adapter_schemas_returns_improvedigital_classes(self): assert schemas.capabilities.inventory_entity_label == "Placements" def test_sync_capabilities_match_implementation_state(self): - # Inventory sync landed with Phase 2; reporting landed with the - # Phase 3 Report API cache (improvedigital_line_item_stats). + # Inventory sync landed with Phase 2; reporting flips alongside the + # Phase 3 Report API cache — the scheduler must not call its stub. schemas = get_adapter_schemas("improvedigital") assert schemas.capabilities.supports_inventory_sync is True - assert schemas.capabilities.supports_reporting_sync is True + assert schemas.capabilities.supports_reporting_sync is False def test_default_channels_cover_classic_media_types(self): channels = get_adapter_default_channels("improvedigital") @@ -84,17 +84,16 @@ def test_live_mode_without_credentials_raises(self, mock_principal): tenant_id="tenant_impd_1", ) - def test_live_mode_without_advertiser_is_allowed(self, mock_principal): - # The Classic campaign API accepts campaigns without an advertiser - # (validated live: every dev campaign carries advertiserId=null), so - # a missing mapping must not block adapter construction. + def test_live_mode_without_advertiser_constructs(self, mock_principal): + # advertiserId is not part of the Classic campaign create schema + # (sandbox-confirmed) — a missing advertiser mapping must not block. mock_principal.get_adapter_id.return_value = None adapter = ImproveDigitalAdapter( config={ "client_id": "app-1", "client_secret": "s", - "improve_demand_contact_id": 7, "buying_entity_id": 421, + "buying_entity_office_id": 635, }, principal=mock_principal, dry_run=False, @@ -102,24 +101,16 @@ def test_live_mode_without_advertiser_is_allowed(self, mock_principal): ) assert adapter.advertiser_id is None - def test_live_mode_without_booking_identity_constructs_but_blocks_create(self, mock_principal): - # Credentials alone must be enough to construct (Test Connection and - # inventory sync only need reads). The booking-identity fields are - # enforced at create time with a typed, actionable error instead — - # a half-configured tenant could otherwise never sync inventory. - from tests.helpers.adapter_test_helpers import invoke_create_media_buy, make_sample_create_request - - adapter = ImproveDigitalAdapter( - config={"client_id": "app-1", "client_secret": "s"}, - principal=mock_principal, - dry_run=False, - tenant_id="tenant_impd_1", - ) - response = invoke_create_media_buy(adapter, make_sample_create_request(), []) - assert response.errors[0].code == "incomplete_adapter_config" - assert "improve_demand_contact_id" in response.errors[0].message - assert "buying_entity_id" in response.errors[0].message - assert "business_unit_id" in response.errors[0].message + def test_live_mode_without_buying_entity_raises(self, mock_principal): + # The Classic campaign API rejects campaigns without a buying entity + # + office (sandbox-confirmed) — fail at construction, not at create. + with pytest.raises(ValueError, match="buying_entity"): + ImproveDigitalAdapter( + config={"client_id": "app-1", "client_secret": "s"}, + principal=mock_principal, + dry_run=False, + tenant_id="tenant_impd_1", + ) def test_supported_pricing_models(self, mock_principal): adapter = make_dry_run_adapter(mock_principal) @@ -205,7 +196,6 @@ def test_dry_run_tag_creative_approved(self, mock_principal): "snippet": "", "width": 300, "height": 250, - "click_url": "https://www.brand.example.com/landing", } ], today=datetime.now(UTC), diff --git a/tests/unit/test_improvedigital_inventory_sync.py b/tests/unit/test_improvedigital_inventory_sync.py index 87a380ec7a..2494947db0 100644 --- a/tests/unit/test_improvedigital_inventory_sync.py +++ b/tests/unit/test_improvedigital_inventory_sync.py @@ -89,6 +89,31 @@ def test_rows_without_id_skipped(self): result = sync.run() assert result.counts["placement"] == 1 + def test_live_v3_wire_shape(self): + # Sandbox-confirmed: the v3 placement search returns placement_id / + # placement_name / site_name and a snake_case total — the original + # id/name mapping silently stored zero rows. + client = make_client() + client.inventory.search_placements.return_value = { + "placements": [ + { + "placement_id": 22289819, + "placement_name": "Toppbanner", + "site_name": "Avisa Sør-Trøndelag", + "publisher_id": 1574, + "publisher_name": "PM Tjenestesenter AS", + }, + ], + "total_number_of_elements": 1, + } + sync, _session = make_sync(client) + result = sync.run() + + assert result.counts["placement"] == 1 + assert result.counts["publisher"] == 1 + # snake_case total honored: one page, no second request + assert client.inventory.search_placements.call_count == 1 + class TestPartialFailure: def test_placement_failure_does_not_block_other_families(self): From ec10e3bcec350b6e8abd17a7da8e19bebd5c25a1 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Fri, 7 Aug 2026 10:22:21 +0600 Subject: [PATCH 64/90] Feature/improve delivery reports (#56) * Feat: Improve digital report page * feat: implement Improve Digital geo targeting end-to-end --- .../improvedigital/live-wire-shapes.md | 6 + scripts/ops/seed_improvedigital_demo.py | 164 ----------- src/adapters/improvedigital/adapter.py | 272 ++++++++++++++++-- src/adapters/improvedigital/client.py | 20 ++ src/adapters/improvedigital/schemas.py | 15 + src/adapters/improvedigital/targeting.py | 149 +++++++++- src/admin/blueprints/adapters.py | 164 ++++++++++- src/admin/blueprints/operations.py | 12 +- .../improvedigital_line_item_stats.py | 10 + src/core/database/repositories/media_buy.py | 19 ++ templates/improvedigital_reporting.html | 187 ++++++++++++ tests/unit/test_improvedigital_adapter.py | 9 +- tests/unit/test_improvedigital_live_paths.py | 239 +++++++++++++++ .../test_improvedigital_reporting_page.py | 76 +++++ tests/unit/test_improvedigital_targeting.py | 133 +++++++++ 15 files changed, 1272 insertions(+), 203 deletions(-) delete mode 100644 scripts/ops/seed_improvedigital_demo.py create mode 100644 templates/improvedigital_reporting.html create mode 100644 tests/unit/test_improvedigital_reporting_page.py create mode 100644 tests/unit/test_improvedigital_targeting.py diff --git a/docs/adapters/improvedigital/live-wire-shapes.md b/docs/adapters/improvedigital/live-wire-shapes.md index a8d39e9271..3084afb946 100644 --- a/docs/adapters/improvedigital/live-wire-shapes.md +++ b/docs/adapters/improvedigital/live-wire-shapes.md @@ -23,6 +23,12 @@ incomplete in several places. - **Advertiser is optional** on Classic campaigns (`advertiserId` is null on every live dev campaign); metadata advertisers (`/api/metadata-advertisers`) are UUIDs and do not fit `CampaignDto.advertiserId` (integer). +- **Geo targeting** (`PUT .../line-items/{id}/geo-targeting`) requires + `exclude` AND `region` on every `geo_targeting` entry — includes too — + despite the spec marking all `Geo` fields optional (400 "object has + missing required properties [\"exclude\",\"region\"]"). Country entries + must carry their region, resolved from `/rtb/v1/regions` + + `/rtb/v1/regions/{name}/countries`. ## Inventory diff --git a/scripts/ops/seed_improvedigital_demo.py b/scripts/ops/seed_improvedigital_demo.py deleted file mode 100644 index 30076d2850..0000000000 --- a/scripts/ops/seed_improvedigital_demo.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python3 -"""Seed everything an MCP buyer needs to book an Improve Digital campaign. - -Idempotent — safe to re-run. Creates on the target tenant (default: 'default'): - - - EUR currency limit (products need one) - - 'all_inventory' property tag + a verified authorized property - - product 'improvedigital_display_300x250' (CPM EUR, five live dev - placements that accept 300x250) + its pricing option - - principal 'ci-test-principal' with MCP token 'ci-test-token' - - approval_mode='auto' so create_media_buy books immediately (no HITL) - -Run AFTER configure_improvedigital_tenant.py (which stores the OAuth -credentials). Full local bring-up: - - eval $(.claude/skills/agent-db/agent-db.sh up) # or any DATABASE_URL - uv run python scripts/ops/migrate.py - export IMPROVEDIGITAL_CLIENT_ID=... IMPROVEDIGITAL_CLIENT_SECRET=... - uv run python scripts/ops/configure_improvedigital_tenant.py \ - --tenant default --demand-contact-id 15663 \ - --api-base-url https://api.360yielddev.com \ - --buying-entity-id 421 --buying-entity-office-id 635 \ - --business-unit-id 33 --timezone Europe/Amsterdam - uv run python scripts/ops/seed_improvedigital_demo.py -""" - -import argparse -import os -import sys - -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - -from sqlalchemy import select - -from src.core.database.database_session import get_db_session -from src.core.database.models import ( - AuthorizedProperty, - CurrencyLimit, - PricingOption, - Principal, - Product, - PropertyTag, - Tenant, -) - -PRODUCT_ID = "improvedigital_display_300x250" -# 300x250-capable placements on the dev platform (size_id 4); refresh via -# GET /rtb/v3/placements?size_ids=4 if these ever disappear. -DEV_PLACEMENT_IDS = [22349458, 22349460, 22349654, 22349657, 22349659] - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--tenant", default="tenant_azerion_gaming") - parser.add_argument("--principal-token", default="tok_TsDB5qiLETTMCZab_UAHmUES1cI3tInHt69sxf14TWE", help="MCP x-adcp-auth token to provision") - args = parser.parse_args() - tenant_id = args.tenant - - with get_db_session() as s: - tenant = s.get(Tenant, tenant_id) - if tenant is None: - print(f"Tenant {tenant_id!r} not found — run scripts/ops/migrate.py first.", file=sys.stderr) - return 1 - - if not s.get(CurrencyLimit, (tenant_id, "EUR")): - s.add(CurrencyLimit(tenant_id=tenant_id, currency_code="EUR")) - print("+ EUR currency limit") - - if not s.get(PropertyTag, ("all_inventory", tenant_id)): - s.add( - PropertyTag( - tag_id="all_inventory", - tenant_id=tenant_id, - name="All Inventory", - description="All publisher inventory", - ) - ) - print("+ all_inventory property tag") - - if not s.get(AuthorizedProperty, ("azerion_network", tenant_id)): - s.add( - AuthorizedProperty( - property_id="azerion_network", - tenant_id=tenant_id, - property_type="website", - name="Azerion 360Yield Network", - identifiers=[{"type": "domain", "value": "azerion.com"}], - tags=["all_inventory"], - publisher_domain="azerion.com", - verification_status="verified", - ) - ) - print("+ authorized property azerion_network") - - if not s.scalars(select(Product).filter_by(tenant_id=tenant_id, product_id=PRODUCT_ID)).first(): - s.add( - Product( - tenant_id=tenant_id, - product_id=PRODUCT_ID, - name="Improve Digital Display 300x250", - description=( - "Run-of-network 300x250 display on 360Yield marketplace placements (Improve Digital Classic)." - ), - format_ids=[{"id": "display_300x250", "agent_url": "https://creative.adcontextprotocol.org"}], - targeting_template={}, - delivery_type="guaranteed", - property_tags=["all_inventory"], - delivery_measurement={"provider": "improvedigital"}, - reporting_capabilities={ - "timezone": "UTC", - "available_metrics": ["impressions"], - "supports_webhooks": False, - "date_range_support": "date_range", - "available_reporting_frequencies": ["daily"], - "expected_delay_minutes": 0, - }, - implementation_config={ - "improvedigital": { - "placement_ids": DEV_PLACEMENT_IDS, - "size_ids": [4], - "pricing_model": "CPM", - } - }, - property_targeting_allowed=False, - signal_targeting_allowed=False, - ) - ) - s.add( - PricingOption( - tenant_id=tenant_id, - product_id=PRODUCT_ID, - pricing_model="cpm", - rate=2.50, - currency="EUR", - is_fixed=True, - ) - ) - print(f"+ product {PRODUCT_ID} (CPM 2.50 EUR, {len(DEV_PLACEMENT_IDS)} placements)") - - if not s.scalars(select(Principal).filter_by(tenant_id=tenant_id, principal_id="prin_960d6568")).first(): - s.add( - Principal( - tenant_id=tenant_id, - principal_id="prin_960d6568", - name="CI Test Principal", - platform_mappings={"improvedigital": {"advertiser_id": None}}, - access_token=args.principal_token, - ) - ) - print(f"+ principal ci-test-principal (token: {args.principal_token})") - - if tenant.approval_mode != "auto": - tenant.approval_mode = "auto" - tenant.human_review_required = False - print("+ tenant approval_mode -> auto (no HITL gate on create_media_buy)") - - s.commit() - - print(f"Tenant {tenant_id!r} ready for MCP booking.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/adapters/improvedigital/adapter.py b/src/adapters/improvedigital/adapter.py index 689abffd2c..a8b29e2c4c 100644 --- a/src/adapters/improvedigital/adapter.py +++ b/src/adapters/improvedigital/adapter.py @@ -398,10 +398,19 @@ def create_media_buy( for package in packages: rate, rate_type = self._resolve_pricing_rate(package, package_pricing_info) payload = self._line_item_payload(package, rate, rate_type, start_time, end_time) + # Geo travels via its own per-line-item endpoint, not the + # create body (LineItemGeoTargetingDto — see targeting.py). + geo_targeting = payload.pop("geo_targeting", None) line_item = self._client.campaigns.create_line_item(campaign_id, payload) line_item_id = int(line_item["id"]) self._line_item_campaigns[str(line_item_id)] = campaign_id self._assign_inventory(campaign_id, line_item_id, package) + if geo_targeting: + self._client.campaigns.set_line_item_geo_targeting( + campaign_id, + line_item_id, + {"filter": True, "geo_targeting": self._resolve_geo_regions(geo_targeting)}, + ) platform_line_item_ids[package.package_id] = str(line_item_id) package_responses.append( ResponsePackage( @@ -475,6 +484,123 @@ def _assign_inventory(self, campaign_id: int, line_item_id: int, package: MediaP {"line_item_packages": [{"id": int(pid), "assigned": True} for pid in package_ids]}, ) + def _resolve_geo_regions(self, geo_targeting: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Complete geo entries to the platform's required shape. + + The live geo-targeting endpoint requires ``exclude`` AND ``region`` + on every entry (validated live: HTTP 400 'object has missing + required properties ["exclude","region"]'), and geo values must be + the platform's own display names. Country tokens are resolved + against the platform geo dictionary — accepting ISO alpha-2 codes + (all AdCP buyer overlays: ``GeoCountry`` is ``^[A-Z]{2}$``), CLDR + English names, and case/diacritic variants — and rewritten to the + platform's exact spelling with their region attached. Region tokens + are matched against the platform region list the same way. An + unresolvable token fails the booking loudly — sending it would 400 + upstream anyway. + """ + from src.adapters.improvedigital.targeting import candidate_country_names, normalize_geo_name + + countries, regions = self._geo_dictionary() + resolved = [] + for entry in geo_targeting: + entry = dict(entry) + entry.setdefault("exclude", False) + if entry.get("country"): + match = next( + ( + countries[normalize_geo_name(candidate)] + for candidate in candidate_country_names(str(entry["country"])) + if normalize_geo_name(candidate) in countries + ), + None, + ) + if match is None: + raise ValueError( + f"could not resolve country {entry['country']!r} in the 360Yield geo " + "dictionary — use an ISO alpha-2 code or a platform country name " + "(pick them from the product page dropdowns)" + ) + entry["country"], entry["region"] = match + elif entry.get("region"): + region_name = regions.get(normalize_geo_name(str(entry["region"]))) + if region_name is None: + raise ValueError( + f"could not resolve region {entry['region']!r} in the 360Yield geo " + f"dictionary — valid regions: {', '.join(sorted(regions.values()))}" + ) + entry["region"] = region_name + resolved.append(entry) + + # Post-resolution dedup: distinct input tokens ('NL' from the buyer, + # 'The Netherlands' from the product config) resolve to identical + # rows — send each geo once. When include and exclude collide on the + # same geo, the exclude wins: AdCP overlay semantics are that the + # buyer's exclusion narrows the product default, and sending both + # contradictory rows would delegate the outcome to undocumented + # platform precedence. + deduped: dict[tuple[str | None, str | None], dict[str, Any]] = {} + for entry in resolved: + key = (entry.get("country"), entry.get("region")) + existing = deduped.get(key) + if existing is None or (entry["exclude"] and not existing["exclude"]): + deduped[key] = entry + return list(deduped.values()) + + def _geo_dictionary(self) -> tuple[dict[str, tuple[str, str]], dict[str, str]]: + """The platform geo dictionary, keyed by normalized display name. + + Returns ``(countries, regions)`` where ``countries`` maps a + normalized country name to ``(platform_country_name, region_name)`` + and ``regions`` maps a normalized region name to the platform + region name. Built from ``/rtb/v1/regions`` + + ``/rtb/v1/regions/{name}/countries`` (both paginated), once per + adapter instance. + """ + from src.adapters.improvedigital.targeting import normalize_geo_name + + assert self._client is not None + if not hasattr(self, "_geo_dictionary_cache"): + client = self._client + countries: dict[str, tuple[str, str]] = {} + regions: dict[str, str] = {} + for region_name in self._paginated_geo_names(client.lookups.regions, "regions"): + regions.setdefault(normalize_geo_name(region_name), region_name) + for country_name in self._paginated_geo_names( + lambda **params: client.lookups.region_countries(region_name, **params), # noqa: B023 + "countries", + ): + countries.setdefault(normalize_geo_name(country_name), (country_name, region_name)) + self._geo_dictionary_cache: tuple[dict[str, tuple[str, str]], dict[str, str]] = (countries, regions) + return self._geo_dictionary_cache + + @staticmethod + def _paginated_geo_names(fetch: Any, envelope_key: str) -> list[str]: + """Exhaust an offset/limit-paginated geo dictionary endpoint and + return the row names. Stops on an empty page, a page with no unseen + names (server ignoring ``offset``), or a 10k backstop.""" + names: list[str] = [] + seen: set[str] = set() + offset = 0 + while offset <= 10_000: + body = fetch(limit=100, offset=offset) + rows = body.get(envelope_key) if isinstance(body, dict) else body + rows = rows if isinstance(rows, list) else [] + if not rows: + break + page_names = [ + str(row.get("name")) if isinstance(row, dict) else str(row) + for row in rows + if (isinstance(row, dict) and row.get("name")) or not isinstance(row, dict) + ] + fresh = [name for name in page_names if name not in seen] + if not fresh: + break + seen.update(fresh) + names.extend(fresh) + offset += len(rows) + return names + @staticmethod def _format_datetime(value: datetime) -> str: """Platform datetime wire format (``YYYY-MM-DD HH:MM:SS``) — @@ -526,7 +652,7 @@ def _line_item_payload( business_unit_id, improve_demand_contact_id, and a goal for CPM line items. """ - product_config = self._product_config_from_package(package) + product_config = self._with_product_geo_defaults(package, self._product_config_from_package(package)) payload: dict[str, Any] = { "name": package.name or package.package_id, "type": "Standard", @@ -557,6 +683,31 @@ def _product_config_from_package(self, package: MediaPackage) -> dict[str, Any]: impl = getattr(package, "implementation_config", None) or {} return impl.get("improvedigital", impl) if isinstance(impl, dict) else {} + def _with_product_geo_defaults(self, package: MediaPackage, product_config: dict[str, Any]) -> dict[str, Any]: + """Fold the product's generic "Target Countries" selection + (``Product.countries``, ISO alpha-2 codes from the Channel & + Geographic Targeting section) into the adapter geo defaults, so the + product page's advertised geo is what the booked line items actually + target. + + The visible generic selection wins over any legacy + ``geo_countries`` stored in the adapter config; both lose to + nothing — an empty selection means no geo restriction. Live-only + (dry-run must stay DB-free). + """ + if self.dry_run or not getattr(package, "product_id", None): + return product_config + + from src.core.database.database_session import get_db_session + from src.core.database.repositories.product import ProductRepository + + with get_db_session() as session: + product = ProductRepository(session, self.tenant_id or "default").get_by_id(str(package.product_id)) + countries = list(product.countries) if product is not None and product.countries else None + if not countries: + return product_config + return {**product_config, "geo_countries": countries} + def _buy_name(self, request: CreateMediaBuyRequest) -> str: """Derive a human-readable buy name — po_number when present, timestamp fallback so buys without one never collide.""" @@ -630,10 +781,70 @@ def _create_live_creative(self, campaign_id: int, payload: dict[str, Any]) -> di if payload.get("type") == "Third Party Tag": body = {k: v for k, v in payload.items() if k != "type"} created = self._client.creatives.create_third_party_tag_creatives(campaign_id, [body]) - if isinstance(created, list): - return created[0] - return created["creatives"][0] - return self._client.creatives.create_creative(campaign_id, payload) + else: + created = self._client.creatives.create_creative(campaign_id, payload) + creative_id = self._created_creative_id(created) + if creative_id is None: + # The platform accepted the create but the response carried no + # usable ID (endpoint response shapes vary between the bulk + # servlets). Without the ID the creative would exist upstream but + # never get bound to a line item — recover it by name from the + # campaign's creative list. + creative_id = self._find_creative_id_by_name(campaign_id, str(payload.get("name") or "")) + if creative_id is None: + raise ValueError( + f"could not determine the platform creative ID (create response: {str(created)[:200]!r}); " + "the creative may exist on the platform but cannot be bound to a line item" + ) + return {"id": creative_id} + + @staticmethod + def _created_creative_id(created: Any) -> int | None: + """Extract the platform creative ID from a create response. + + Handles the shapes seen across the Classic creative endpoints: a bare + ``CreativeDto`` list (per the OpenAPI spec), an enveloped list + (``{"creatives"|"content"|"data": [...]}``), or a single entity dict. + Returns ``None`` when no row carries an ID (e.g. empty 200 body). + """ + rows: Any = created + if isinstance(created, dict): + for key in ("creatives", "content", "data"): + if isinstance(created.get(key), list): + rows = created[key] + break + else: + rows = [created] + if isinstance(rows, list): + for row in rows: + if isinstance(row, dict): + for id_key in ("id", "creative_id"): + if row.get(id_key) is not None: + return int(row[id_key]) + return None + + def _find_creative_id_by_name(self, campaign_id: int, name: str) -> int | None: + """Recover a just-created creative's platform ID by name lookup. + + Newest ID wins when names repeat — the creative created moments ago + is by construction the highest ID for that name.""" + if not name: + return None + assert self._client is not None + try: + body = self._client.creatives.list_creatives(campaign_id) + except ImproveDigitalError as exc: + logger.warning("Improve Digital creative recovery lookup failed for campaign %s: %s", campaign_id, exc) + return None + rows = body.get("creatives") if isinstance(body, dict) else body + matches = [ + row + for row in (rows if isinstance(rows, list) else []) + if isinstance(row, dict) and row.get("name") == name and str(row.get("id") or "").isdigit() + ] + if not matches: + return None + return max(int(row["id"]) for row in matches) def _resolve_size(self, width: int, height: int) -> dict[str, Any] | None: """Look up the platform size entry for a width×height pair. @@ -789,22 +1000,49 @@ def _campaign_id_for_line_item(self, line_item_id: str) -> int | None: with get_db_session() as session: repo = MediaBuyRepository(session, self.tenant_id or "default") - for buy in repo.get_active(): - for package in repo.get_packages(buy.media_buy_id): - platform_id = (package.package_config or {}).get("platform_line_item_id") - if platform_id is not None and str(platform_id) == str(line_item_id): - buy_ref = str(buy.external_id or buy.media_buy_id) - campaign_ref = buy_ref.removeprefix("improvedigital_") - if campaign_ref.isdigit(): - campaign_id = int(campaign_ref) - self._line_item_campaigns[str(line_item_id)] = campaign_id - return campaign_id + # No status filter — creative binding must also resolve buys that + # haven't started serving yet (pending_start, HITL states). + buy = repo.find_by_platform_line_item_id(str(line_item_id)) + if buy is not None: + for buy_ref in (buy.external_id, buy.media_buy_id): + campaign_ref = str(buy_ref or "").removeprefix("improvedigital_") + if campaign_ref.isdigit(): + campaign_id = int(campaign_ref) + self._line_item_campaigns[str(line_item_id)] = campaign_id + return campaign_id return None + def _resolve_campaign_id(self, media_buy_id: str) -> str: + """Classic campaign ID behind a media buy reference. + + The adapter returns ``improvedigital_`` at create time, + but delivery callers (the admin detail page, the MCP delivery tool) + pass the core layer's internal ID (``mb_*``) — the adapter reference + is stamped on ``media_buys.external_id``. Mirror the reporting + sync's write-side resolution so read paths hit the same cache rows. + """ + campaign_id = media_buy_id.removeprefix("improvedigital_") + if campaign_id.isdigit(): + return campaign_id + + from src.core.database.database_session import get_db_session + from src.core.database.repositories.media_buy import MediaBuyRepository + + with get_db_session() as session: + repo = MediaBuyRepository(session, self.tenant_id or "default") + buy = repo.get_by_id(media_buy_id) + external = str(buy.external_id or "") if buy is not None else "" + external_campaign = external.removeprefix("improvedigital_") + if external_campaign.isdigit(): + return external_campaign + return campaign_id + # ----- status / delivery ----- def check_media_buy_status(self, media_buy_id: str, today: datetime) -> CheckMediaBuyStatusResponse: - campaign_id = media_buy_id.removeprefix("improvedigital_") + campaign_id = ( + media_buy_id.removeprefix("improvedigital_") if self.dry_run else self._resolve_campaign_id(media_buy_id) + ) if self.dry_run: self.log(f"Would call: GET {self.base_url}/rtb/v1/classic/campaigns/{campaign_id}") return CheckMediaBuyStatusResponse(media_buy_id=media_buy_id, status="active") @@ -839,7 +1077,7 @@ def get_media_buy_delivery( ImproveDigitalLineItemStatsRepository, ) - campaign_id = media_buy_id.removeprefix("improvedigital_") + campaign_id = self._resolve_campaign_id(media_buy_id) with get_db_session() as session: repo = ImproveDigitalLineItemStatsRepository(session, self.tenant_id or "default") stat_rows = repo.list_by_campaign(campaign_id) diff --git a/src/adapters/improvedigital/client.py b/src/adapters/improvedigital/client.py index 3344f29833..357dce67a7 100644 --- a/src/adapters/improvedigital/client.py +++ b/src/adapters/improvedigital/client.py @@ -142,6 +142,16 @@ def set_line_item_placements(self, campaign_id: int, line_item_id: int, payload: f"/rtb/v1/classic/campaigns/{campaign_id}/line-items/{line_item_id}/placements", payload ) + def set_line_item_geo_targeting( + self, campaign_id: int, line_item_id: int, payload: dict[str, Any] + ) -> dict[str, Any]: + """``PUT /rtb/v1/classic/.../line-items/{id}/geo-targeting`` — + ``LineItemGeoTargetingDto``: ``{"filter": true, "geo_targeting": + [{"country"|"region"|"state"|"city": ..., "exclude": bool}]}``.""" + return self._transport.put_json( + f"/rtb/v1/classic/campaigns/{campaign_id}/line-items/{line_item_id}/geo-targeting", payload + ) + class ImproveDigitalCreativesClient: """Classic creatives — hosted by the Improve adserver. @@ -258,6 +268,16 @@ def creative_type_sizes(self, creative_type: str, **params: Any) -> Any: def countries(self, **params: Any) -> Any: return self._transport.get_json("/common/v1/countries", **params) + def regions(self, **params: Any) -> Any: + """``GET /rtb/v1/regions`` — geo region dictionary + (``RegionsDto``: ``{"regions": [{"name": ...}]}``).""" + return self._transport.get_json("/rtb/v1/regions", **params) + + def region_countries(self, region_name: str, **params: Any) -> Any: + """``GET /rtb/v1/regions/{regionName}/countries`` — country + dictionary per region (``CountriesDto``: ``{"countries": [{"name": ...}]}``).""" + return self._transport.get_json(f"/rtb/v1/regions/{region_name}/countries", **params) + def user_details(self) -> dict[str, Any]: """``GET /lookup/v1/user-details`` — identity behind the OAuth pair (user_id, name, business unit, buyers). Lookup-scoped, so it works diff --git a/src/adapters/improvedigital/schemas.py b/src/adapters/improvedigital/schemas.py index 3c85bf1b1f..aeb7e0a39e 100644 --- a/src/adapters/improvedigital/schemas.py +++ b/src/adapters/improvedigital/schemas.py @@ -184,6 +184,21 @@ class ImproveDigitalProductConfig(BaseProductConfig): ) # -- Classic line-item defaults -- + geo_countries: list[str] = Field( + default_factory=list, + description=( + "Default geo targeting: country names from the platform geo dictionary " + "(/common/v1/countries), applied to every line item booked from this " + "product (buyer overlays add on top)" + ), + ) + geo_regions: list[str] = Field( + default_factory=list, + description=( + "Default geo targeting: region names from the platform geo dictionary " + "(/rtb/v1/regions), applied to every line item booked from this product" + ), + ) pricing_model: str | None = Field( default=None, description="Line-item pricing model (CPM confirmed; further values pending platform confirmation — gap G2)", diff --git a/src/adapters/improvedigital/targeting.py b/src/adapters/improvedigital/targeting.py index c0284c55b9..94816b51c6 100644 --- a/src/adapters/improvedigital/targeting.py +++ b/src/adapters/improvedigital/targeting.py @@ -6,10 +6,10 @@ ``pixel-targeting``) plus flat fields on the line item itself (``size_ids``, frequency caps). -This module emits the line-item-creation subset only (inventory selection + -sizes); the per-dimension targeting PUTs land with the M2 buy path. The wire -shapes are exercised by dry-run logging until live calls are validated -against real credentials. +This module emits the line-item-creation subset (inventory selection + +sizes) plus the ``geo_targeting`` list consumed by the live per-line-item +``geo-targeting`` PUT (``LineItemGeoTargetingDto``: ``{"filter": true, +"geo_targeting": [{"country"|"region": ..., "exclude": bool}]}``). Hard platform constraint: location targeting supports region/country/state/ city (+ up to 10 IP ranges) — **no postal codes**. Postal targeting is @@ -18,9 +18,83 @@ from __future__ import annotations +import re +import unicodedata +from functools import cache from typing import Any +def _token(value: Any) -> str: + """Unwrap adcp RootModel tokens (``.root``) to their plain string.""" + return str(getattr(value, "root", value)) + + +# ISO 3166-1 alpha-2 → the platform's display name, for the countries whose +# 360Yield name diverges from the CLDR English name beyond what +# ``normalize_geo_name`` bridges. Curated empirically against the full live +# dev geo dictionary (2026-08-07): every other assigned code matches via +# babel + normalization. +_GEO_NAME_ALIASES: dict[str, str] = { + "AN": "Netherland Antilles", # deprecated ISO code, still on the platform + "BQ": "Bonaire, Sint Eustatius, and Saba", + "CD": "DR Congo", + "CG": "Congo Republic", + "CI": "Ivory Coast", + "CV": "Cabo Verde", + "FM": "Federated States of Micronesia", + "GS": "South Georgia and the South Sandwich Islands", + "HK": "Hong Kong", + "MM": "Myanmar", + "MO": "Macao", + "PS": "Palestine", +} + + +def normalize_geo_name(name: str) -> str: + """Fold a geo display name for dictionary matching. + + Lowercases, strips diacritics (``São Tomé`` ≡ ``Sao Tome``), folds + ``&``/punctuation, drops a leading ``the`` (``The Netherlands`` ≡ + ``Netherlands``) and expands ``St.`` → ``Saint`` — the divergences + observed between CLDR English names and the live 360Yield dictionary. + """ + folded = unicodedata.normalize("NFKD", name) + folded = "".join(c for c in folded if not unicodedata.combining(c)).lower() + folded = folded.replace("&", " and ") + folded = re.sub(r"[^a-z0-9 ]+", " ", folded) + folded = re.sub(r"\s+", " ", folded).strip() + folded = folded.removeprefix("the ") + return re.sub(r"\bst\b", "saint", folded) + + +@cache +def _cldr_country_name(code: str) -> str | None: + """ISO 3166-1 alpha-2 → CLDR English display name (via babel).""" + from babel import Locale + + return Locale("en").territories.get(code) + + +def candidate_country_names(token: str) -> list[str]: + """Display-name candidates for a country token, most specific first. + + AdCP buyers can only send ISO alpha-2 codes (``GeoCountry`` is + ``^[A-Z]{2}$``); operators may store either codes or platform names. + A bare token is tried verbatim; a two-letter token additionally tries + the curated platform alias and the CLDR English name. + """ + candidates = [token] + code = token.strip().upper() + if len(code) == 2 and code.isalpha(): + alias = _GEO_NAME_ALIASES.get(code) + if alias: + candidates.append(alias) + cldr = _cldr_country_name(code) + if cldr: + candidates.append(cldr) + return candidates + + def build_targeting( targeting_overlay: Any, product_config: dict[str, Any] | None = None, @@ -31,12 +105,16 @@ def build_targeting( Inputs: targeting_overlay: AdCP ``Targeting`` model (geo, device, custom). product_config: ``ImproveDigitalProductConfig`` as a dict — supplies - static inventory selection (placements/packages/sizes). + static inventory selection (placements/packages/sizes) and the + publisher's default geo targeting (``geo_countries`` / + ``geo_regions`` from the product-config page). tenant_id: reserved for signal resolution (M2+); unused for now. Returns a dict of line-item field values; only populated dimensions are - included. Geo lands under ``geo_targeting`` for the dry-run echo — the - live path PUTs it to the per-line-item ``geo-targeting`` endpoint (M2). + included. ``geo_targeting`` is the union of the product's default geo + and the buyer's overlay (includes and excludes), deduplicated — the + create path pops it off the payload and PUTs it to the per-line-item + ``geo-targeting`` endpoint. """ product_config = product_config or {} targeting: dict[str, Any] = {} @@ -46,14 +124,37 @@ def build_targeting( if values: targeting[config_key] = list(values) + geo: list[dict[str, Any]] = [] + seen: set[tuple[str, str, bool]] = set() + + def _add(kind: str, value: Any, exclude: bool = False) -> None: + token = _token(value) + key = (kind, token, exclude) + if token and key not in seen: + seen.add(key) + # ``exclude`` is required on every entry — the live geo-targeting + # endpoint 400s with 'missing required properties ["exclude", ...]' + # when it is omitted, even for plain includes. + geo.append({kind: token, "exclude": exclude}) + + for country in product_config.get("geo_countries") or []: + _add("country", country) + for region in product_config.get("geo_regions") or []: + _add("region", region) + if targeting_overlay is not None: - geo: list[dict[str, Any]] = [] - if getattr(targeting_overlay, "geo_countries", None): - geo.extend({"country": c.root} for c in targeting_overlay.geo_countries) - if getattr(targeting_overlay, "geo_regions", None): - geo.extend({"region": r.root} for r in targeting_overlay.geo_regions) - if geo: - targeting["geo_targeting"] = geo + # Overlay geo_regions are deliberately NOT mapped: AdCP GeoRegion + # tokens are ISO 3166-2 subdivisions (e.g. "US-NY") while the + # platform's region dimension is continental (APAC/EMEA/…) — the + # vocabularies cannot meet, so validate_targeting rejects them + # upfront before any campaign is created. + for country in getattr(targeting_overlay, "geo_countries", None) or []: + _add("country", country) + for country in getattr(targeting_overlay, "geo_countries_exclude", None) or []: + _add("country", country, exclude=True) + + if geo: + targeting["geo_targeting"] = geo return targeting @@ -78,9 +179,27 @@ def validate_targeting(targeting_overlay: Any) -> list[str]: ): unsupported.append( "Postal-area targeting is not supported on Improve Digital — location targeting " - "goes down to city level only. Use geo_regions or geo_countries instead." + "goes down to city level only. Use geo_countries instead." + ) + + if getattr(targeting_overlay, "geo_metros", None) or getattr(targeting_overlay, "geo_metros_exclude", None): + unsupported.append( + "Metro/DMA targeting is not supported on Improve Digital — the Classic geo " + "dimensions are country/region/state/city. Use geo_countries instead." ) + if getattr(targeting_overlay, "geo_regions", None) or getattr(targeting_overlay, "geo_regions_exclude", None): + unsupported.append( + "Region targeting is not supported on Improve Digital buyer overlays — AdCP " + "geo_regions are ISO 3166-2 subdivisions (e.g. 'US-NY') but the platform's " + "region dimension is continental (APAC/EMEA/…), and the platform's state " + "dimension is pending live validation. Use geo_countries; publishers can set " + "platform regions on the product configuration." + ) + + if getattr(targeting_overlay, "geo_proximity", None): + unsupported.append("Proximity (radius) targeting is not supported on Improve Digital.") + if getattr(targeting_overlay, "frequency_cap", None): unsupported.append( "Frequency cap targeting pending live validation against the 360Yield API — " diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index bc262f28d2..b13f17259f 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1057,6 +1057,13 @@ def _improvedigital_paginate(fetch_page, envelope_key: str, page_size: int = 100 no unseen ids (guards against a server that ignores ``offset``); ``max_rows`` backstops a runaway loop. """ + + def _row_key(row: dict) -> tuple: + # Dictionary rows (RegionDto/CountryDto) carry only ``name`` — keying + # on id alone would collapse them all to None and stop pagination + # after the first page. + return (row.get("id"), row.get("name")) + rows: list = [] seen_ids: set = set() offset = 0 @@ -1065,8 +1072,8 @@ def _improvedigital_paginate(fetch_page, envelope_key: str, page_size: int = 100 page = _improvedigital_rows(payload, envelope_key) if not page: break - fresh = [row for row in page if not isinstance(row, dict) or row.get("id") not in seen_ids] - seen_ids.update(row.get("id") for row in fresh if isinstance(row, dict)) + fresh = [row for row in page if not isinstance(row, dict) or _row_key(row) not in seen_ids] + seen_ids.update(_row_key(row) for row in fresh if isinstance(row, dict)) if not fresh: break rows.extend(fresh) @@ -1332,6 +1339,159 @@ def sync_improvedigital_inventory(tenant_id, **kwargs): return jsonify({"success": False, "error": "Sync failed (see server logs)"}), 500 +def _improvedigital_reporting_payload(stat_rows, buys_by_campaign: dict, currency: str) -> dict: + """Shape line-item stats cache rows into the reporting-page JSON. + + ``buys_by_campaign`` maps Classic campaign IDs to MediaBuy rows so each + stats row can carry the buy it belongs to; rows whose campaign has no + matching buy (e.g. booked outside salesagent) still render, unattributed. + Spend is stored as micros — converted to currency units here, once. + """ + rows = [] + total_impressions = 0 + total_clicks = 0 + total_spend = 0.0 + total_completed = 0 + for stat in stat_rows: + impressions = int(stat.impressions or 0) + clicks = int(stat.clicks) if stat.clicks is not None else None + spend = round((stat.spend_micros or 0) / 1_000_000, 2) + buy = buys_by_campaign.get(str(stat.campaign_id)) if stat.campaign_id else None + rows.append( + { + "campaign_id": stat.campaign_id, + "line_item_id": stat.line_item_id, + "media_buy_id": buy.media_buy_id if buy else None, + "order_name": buy.order_name if buy else None, + "advertiser_name": buy.advertiser_name if buy else None, + "impressions": impressions, + "clicks": clicks, + "ctr": round(clicks / impressions * 100, 2) if clicks and impressions else None, + "completed_views": int(stat.completed_views) if stat.completed_views is not None else None, + "spend": spend, + "currency": stat.currency or currency, + "as_of": stat.as_of.isoformat() if stat.as_of else None, + } + ) + total_impressions += impressions + total_clicks += clicks or 0 + total_spend += spend + total_completed += int(stat.completed_views or 0) + return { + "rows": rows, + "totals": { + "impressions": total_impressions, + "clicks": total_clicks, + "ctr": round(total_clicks / total_impressions * 100, 2) if total_impressions else None, + "completed_views": total_completed, + "spend": round(total_spend, 2), + }, + "currency": currency, + } + + +@adapters_bp.route("/api/tenant//adapters/improvedigital/reporting", methods=["GET"]) +@require_tenant_access(api_mode=True) +def get_improvedigital_reporting(tenant_id, **kwargs): + """Serve the Report-API stats cache for the reporting page. + + Reads ``improvedigital_line_item_stats`` (populated by the reporting + sync — no upstream call here, so the page loads instantly) and joins + campaigns to media buys via the ``improvedigital_`` + reference on ``external_id`` / ``media_buy_id``. + """ + from src.core.database.models import MediaBuy + from src.core.database.repositories.improvedigital_line_item_stats import ( + ImproveDigitalLineItemStatsRepository, + ) + + with get_db_session() as session: + repo = ImproveDigitalLineItemStatsRepository(session, tenant_id) + stat_rows = repo.list_all() + last_synced_at = repo.latest_sync_at() + + buys_by_campaign: dict = {} + for buy in session.scalars(select(MediaBuy).filter_by(tenant_id=tenant_id)).all(): + for candidate in (buy.external_id, buy.media_buy_id): + if not candidate: + continue + campaign_id = str(candidate).removeprefix("improvedigital_") + if campaign_id.isdigit(): + buys_by_campaign[campaign_id] = buy + break + + config_row = session.scalars(select(AdapterConfig).filter_by(tenant_id=tenant_id)).first() + currency = str((config_row.config_json or {}).get("currency") or "EUR") if config_row else "EUR" + + payload = _improvedigital_reporting_payload(stat_rows, buys_by_campaign, currency) + + payload["success"] = True + payload["last_synced_at"] = last_synced_at.isoformat() if last_synced_at else None + return jsonify(payload) + + +@adapters_bp.route("/api/tenant//adapters/improvedigital/sync-reporting", methods=["POST"]) +@require_tenant_access(role=("admin",), api_mode=True) +def sync_improvedigital_reporting(tenant_id, **kwargs): + """Pull fresh delivery metrics from the 360Yield Report API and upsert + the ``improvedigital_line_item_stats`` cache feeding the reporting page + and ``get_media_buy_delivery``. + + Returns 503 when the Report API scope is still pending for this OAuth2 + client (mirrors the FreeWheel sync-reporting contract). + """ + from src.services.adapter_sync_orchestration import SyncAlreadyRunning, execute_adapter_sync + + try: + result = execute_adapter_sync( + tenant_id=tenant_id, + adapter_type="improvedigital", + sync_kind="reporting", + triggered_by="admin_button", + ) + if result is None: + return ( + jsonify({"success": False, "error": "Improve Digital adapter is not configured for this tenant"}), + 400, + ) + if result.scope_pending: + return ( + jsonify( + { + "success": False, + "scope_pending": True, + "sync_id": result.sync_id, + "error": result.errors.get("scope", "Report API scope grant pending"), + } + ), + 503, + ) + return jsonify( + { + "success": result.succeeded, + "sync_id": result.sync_id, + "line_items_updated": result.counts.get("line_items", 0), + "campaigns_covered": result.counts.get("campaigns", 0), + "error": next(iter(result.errors.values()), None) if result.errors else None, + } + ) + except SyncAlreadyRunning as exc: + return ( + jsonify( + { + "success": False, + "error": f"A reporting sync is already running ({exc.sync_id}) — wait for it to finish", + } + ), + 409, + ) + except ValidationError as exc: + return jsonify({"success": False, "error": f"Stored config is invalid: {exc}"}), 400 + except Exception as e: + logger.error(f"Improve Digital reporting sync failed: {e}", exc_info=True) + return jsonify({"success": False, "error": "Sync failed (see server logs)"}), 500 + + @adapters_bp.route("/api/tenant//adapters/springserve/inventory", methods=["GET"]) @require_tenant_access() def list_springserve_inventory(tenant_id, **kwargs): diff --git a/src/admin/blueprints/operations.py b/src/admin/blueprints/operations.py index e30410b0c6..21496e401f 100644 --- a/src/admin/blueprints/operations.py +++ b/src/admin/blueprints/operations.py @@ -212,13 +212,21 @@ def reporting(tenant_id): "is_active": tenant_obj.is_active, } + # Improve Digital tenants get the Report-API-cache dashboard. + if tenant_obj.ad_server == "improvedigital": + adapter_config = db_session.scalars(select(AdapterConfig).filter_by(tenant_id=tenant_id)).first() + currency = "EUR" + if adapter_config and (adapter_config.config_json or {}).get("currency"): + currency = str(adapter_config.config_json["currency"]) + return render_template("improvedigital_reporting.html", tenant=tenant, currency=currency) + # Check if tenant is using Google Ad Manager if tenant_obj.ad_server != "google_ad_manager": return ( render_template( "error.html", - error_title="GAM Reporting Not Available", - error_message=f"This tenant is currently using {tenant_obj.ad_server or 'no ad server'}. GAM Reporting is only available for tenants using Google Ad Manager.", + error_title="Reporting Not Available", + error_message=f"This tenant is currently using {tenant_obj.ad_server or 'no ad server'}. Reporting is only available for tenants using Google Ad Manager or Improve Digital.", back_url=f"{request.script_root}/tenant/{tenant_id}", ), 400, diff --git a/src/core/database/repositories/improvedigital_line_item_stats.py b/src/core/database/repositories/improvedigital_line_item_stats.py index d6eb443fec..d657e9accb 100644 --- a/src/core/database/repositories/improvedigital_line_item_stats.py +++ b/src/core/database/repositories/improvedigital_line_item_stats.py @@ -39,6 +39,16 @@ def get_by_line_item_ids(self, line_item_ids: Iterable[str]) -> dict[str, Improv ) return {row.line_item_id: row for row in self._session.scalars(stmt).all()} + def list_all(self) -> list[ImproveDigitalLineItemStats]: + """Return every cached line-item stats row for this tenant, newest + campaigns first. Feeds the admin reporting page.""" + stmt = ( + select(ImproveDigitalLineItemStats) + .filter_by(tenant_id=self._tenant_id) + .order_by(ImproveDigitalLineItemStats.campaign_id.desc(), ImproveDigitalLineItemStats.line_item_id) + ) + return list(self._session.scalars(stmt).all()) + def list_by_campaign(self, campaign_id: str) -> list[ImproveDigitalLineItemStats]: """Return all cached line-item stats for one Classic campaign. Used by ``get_media_buy_delivery`` to aggregate totals across packages.""" diff --git a/src/core/database/repositories/media_buy.py b/src/core/database/repositories/media_buy.py index 3aa81902f3..47ac5309a3 100644 --- a/src/core/database/repositories/media_buy.py +++ b/src/core/database/repositories/media_buy.py @@ -289,6 +289,25 @@ def get_packages_for_ids(self, media_buy_ids: list[str]) -> dict[str, list[Media result.setdefault(pkg.media_buy_id, []).append(pkg) return result + def find_by_platform_line_item_id(self, platform_line_item_id: str) -> MediaBuy | None: + """Resolve the media buy owning the package whose ``package_config`` + carries this ad-server line-item reference. + + Deliberately no status filter — creative binding must resolve buys + that haven't started serving yet (``pending_start``, HITL approval + states), not just active ones. + """ + rows = self._session.execute( + select(MediaPackage, MediaBuy) + .join(MediaBuy, MediaPackage.media_buy_id == MediaBuy.media_buy_id) + .where(MediaBuy.tenant_id == self._tenant_id) + ).all() + for package, buy in rows: + platform_id = (package.package_config or {}).get("platform_line_item_id") + if platform_id is not None and str(platform_id) == str(platform_line_item_id): + return buy + return None + def find_package_with_media_buy( self, package_id: str, diff --git a/templates/improvedigital_reporting.html b/templates/improvedigital_reporting.html new file mode 100644 index 0000000000..4d17b85c32 --- /dev/null +++ b/templates/improvedigital_reporting.html @@ -0,0 +1,187 @@ +{% extends "base.html" %} +{% from '_macros.html' import currency_symbol %} + +{% block title %}Improve Digital Reporting - {{ tenant.name }}{% endblock %} + +{% block content %} +
+
+

Improve Digital Reporting - {{ tenant.name }}

+
+ + +
+
+

+ Definitive delivery metrics from the 360Yield Report API (last 31 days), refreshed by the + reporting sync. Numbers update when the sync runs — not in real time. +

+ + + + +
+
+
+
+
Total Impressions
+

-

+
+
+
+
+
+
+
Total Spend
+

-

+
+
+
+
+
+
+
Clicks / CTR
+

-

+
+
+
+
+
+
+
Completed Views
+

-

+
+
+
+
+ + +
+
+
+ +
+ + + + + + + + + + + + + + + + + +
+
+
+ + +{% endblock %} diff --git a/tests/unit/test_improvedigital_adapter.py b/tests/unit/test_improvedigital_adapter.py index c535387675..6ae6415d2a 100644 --- a/tests/unit/test_improvedigital_adapter.py +++ b/tests/unit/test_improvedigital_adapter.py @@ -57,11 +57,11 @@ def test_get_adapter_schemas_returns_improvedigital_classes(self): assert schemas.capabilities.inventory_entity_label == "Placements" def test_sync_capabilities_match_implementation_state(self): - # Inventory sync landed with Phase 2; reporting flips alongside the - # Phase 3 Report API cache — the scheduler must not call its stub. + # Inventory sync landed with Phase 2; reporting sync landed with the + # Report API cache (run_reporting_sync → improvedigital_line_item_stats). schemas = get_adapter_schemas("improvedigital") assert schemas.capabilities.supports_inventory_sync is True - assert schemas.capabilities.supports_reporting_sync is False + assert schemas.capabilities.supports_reporting_sync is True def test_default_channels_cover_classic_media_types(self): channels = get_adapter_default_channels("improvedigital") @@ -196,6 +196,9 @@ def test_dry_run_tag_creative_approved(self, mock_principal): "snippet": "", "width": 300, "height": 250, + # CreativeDto requires advertiser_domain (derived from the + # click URL when not given) — validated live on dev. + "click_url": "https://brand.example.com/landing", } ], today=datetime.now(UTC), diff --git a/tests/unit/test_improvedigital_live_paths.py b/tests/unit/test_improvedigital_live_paths.py index 11ab3d443c..f10bde01f7 100644 --- a/tests/unit/test_improvedigital_live_paths.py +++ b/tests/unit/test_improvedigital_live_paths.py @@ -33,6 +33,7 @@ "client_secret": "s3cret", "improve_demand_contact_id": 17918, "buying_entity_id": 421, + "buying_entity_office_id": 5068, "business_unit_id": 33, "api_base_url": "https://api.360yielddev.example", "currency": "EUR", @@ -70,6 +71,10 @@ def set_line_item_placements(self, campaign_id, line_item_id, payload): self.calls.append(("set_line_item_placements", campaign_id, line_item_id, payload)) return payload + def set_line_item_geo_targeting(self, campaign_id, line_item_id, payload): + self.calls.append(("set_line_item_geo_targeting", campaign_id, line_item_id, payload)) + return payload + def set_packages(self, campaign_id, line_item_id, payload): self.calls.append(("set_packages", campaign_id, line_item_id, payload)) return payload @@ -117,6 +122,16 @@ class FakeLookupsClient: def sizes(self, **params): return {"sizes": [{"id": 4, "width": 300, "height": 250, "name": "300x250 (Medium Rectangle)"}]} + def regions(self, **params): + if params.get("offset"): + return {"regions": []} + return {"regions": [{"name": "EMEA"}]} + + def region_countries(self, region_name, **params): + if params.get("offset"): + return {"countries": []} + return {"countries": [{"name": "The Netherlands"}, {"name": "Belgium"}]} + class FakeClient: def __init__(self) -> None: @@ -167,6 +182,133 @@ def test_creates_campaign_and_line_items_with_platform_ids(self): packages_call = next(c for c in calls if c[0] == "set_packages") assert packages_call[3] == {"line_item_packages": [{"id": 77, "assigned": True}]} + def test_product_geo_defaults_put_to_geo_targeting_endpoint(self): + """Geo travels via the per-line-item geo-targeting PUT, never in the + line-item create body (LineItemGeoTargetingDto wire shape).""" + adapter = make_live_adapter() + package = make_targeted_package() + package.implementation_config = {"improvedigital": {"placement_ids": [11, 12], "geo_countries": ["NL", "BE"]}} + invoke_create_media_buy(adapter, make_sample_create_request(), [package]) + + calls = adapter._client.campaigns.calls + geo_calls = [c for c in calls if c[0] == "set_line_item_geo_targeting"] + # Every entry carries region + exclude ('missing required properties + # ["exclude","region"]' otherwise — validated live), and ISO codes + # are rewritten to the platform's own display names. + assert geo_calls == [ + ( + "set_line_item_geo_targeting", + 101, + 202, + { + "filter": True, + "geo_targeting": [ + {"country": "The Netherlands", "exclude": False, "region": "EMEA"}, + {"country": "Belgium", "exclude": False, "region": "EMEA"}, + ], + }, + ) + ] + line_item_payload = next(c[2] for c in calls if c[0] == "create_line_item") + assert "geo_targeting" not in line_item_payload + + def test_region_tokens_and_platform_names_resolve_case_insensitively(self): + """'emea' and 'the netherlands' (free-text spellings) both resolve to + the platform's exact display names on the wire.""" + adapter = make_live_adapter() + package = make_targeted_package() + package.implementation_config = { + "improvedigital": { + "placement_ids": [11], + "geo_countries": ["the netherlands"], + "geo_regions": ["emea"], + } + } + invoke_create_media_buy(adapter, make_sample_create_request(), [package]) + + geo_call = next(c for c in adapter._client.campaigns.calls if c[0] == "set_line_item_geo_targeting") + assert geo_call[3]["geo_targeting"] == [ + {"country": "The Netherlands", "exclude": False, "region": "EMEA"}, + {"region": "EMEA", "exclude": False}, + ] + + def test_product_target_countries_fold_into_geo_targeting(self): + """The generic 'Channel & Geographic Targeting' selection + (Product.countries, ISO codes) is what booked line items geo-target — + it wins over legacy geo_countries in the adapter config.""" + adapter = make_live_adapter() + package = make_targeted_package() + package.product_id = "prod_1" + package.implementation_config = { + "improvedigital": {"placement_ids": [11], "geo_countries": ["BE"]} # legacy, must lose + } + with ( + patch("src.core.database.database_session.get_db_session"), + patch("src.core.database.repositories.product.ProductRepository") as repo_cls, + ): + repo_cls.return_value.get_by_id.return_value = SimpleNamespace(countries=["NL"]) + invoke_create_media_buy(adapter, make_sample_create_request(), [package]) + + geo_call = next(c for c in adapter._client.campaigns.calls if c[0] == "set_line_item_geo_targeting") + assert geo_call[3]["geo_targeting"] == [{"country": "The Netherlands", "exclude": False, "region": "EMEA"}] + + def test_product_without_countries_keeps_legacy_config_geo(self): + """Product.countries=None (All Countries) falls back to legacy + geo_countries stored in the adapter config.""" + adapter = make_live_adapter() + package = make_targeted_package() + package.product_id = "prod_1" + package.implementation_config = {"improvedigital": {"placement_ids": [11], "geo_countries": ["NL"]}} + with ( + patch("src.core.database.database_session.get_db_session"), + patch("src.core.database.repositories.product.ProductRepository") as repo_cls, + ): + repo_cls.return_value.get_by_id.return_value = SimpleNamespace(countries=None) + invoke_create_media_buy(adapter, make_sample_create_request(), [package]) + + geo_call = next(c for c in adapter._client.campaigns.calls if c[0] == "set_line_item_geo_targeting") + assert geo_call[3]["geo_targeting"] == [{"country": "The Netherlands", "exclude": False, "region": "EMEA"}] + + def test_resolved_duplicates_collapse_to_one_row(self): + """Config 'The Netherlands' + buyer 'NL' resolve to the same platform + geo — the PUT must carry it once, not twice.""" + adapter = make_live_adapter() + rows = adapter._resolve_geo_regions( + [ + {"country": "The Netherlands", "exclude": False}, + {"country": "NL", "exclude": False}, + ] + ) + assert rows == [{"country": "The Netherlands", "exclude": False, "region": "EMEA"}] + + def test_buyer_exclude_overrides_matching_include(self): + """AdCP overlay semantics: the buyer's exclusion narrows the product + default — never send contradictory include+exclude rows for the same geo.""" + adapter = make_live_adapter() + rows = adapter._resolve_geo_regions( + [ + {"country": "The Netherlands", "exclude": False}, + {"country": "Belgium", "exclude": False}, + {"country": "NL", "exclude": True}, + ] + ) + assert rows == [ + {"country": "The Netherlands", "exclude": True, "region": "EMEA"}, + {"country": "Belgium", "exclude": False, "region": "EMEA"}, + ] + + def test_unresolvable_geo_country_fails_booking_with_cleanup(self): + """A country outside the platform geo dictionary must fail the buy + (the PUT would 400 upstream anyway) and clean up the partial campaign.""" + adapter = make_live_adapter() + package = make_targeted_package() + package.implementation_config = {"improvedigital": {"placement_ids": [11], "geo_countries": ["Atlantis"]}} + response = invoke_create_media_buy(adapter, make_sample_create_request(), [package]) + + assert type(response).__name__.endswith("Error") + assert "Atlantis" in str(response.errors[0].message) + assert any(c[0] == "delete_campaign" for c in adapter._client.campaigns.calls) + def test_package_without_inventory_selection_fails_loudly(self): adapter = make_live_adapter() package = make_sample_video_package() @@ -225,6 +367,58 @@ def test_association_without_campaign_mapping_fails(self): assert results[0]["status"] == "failed" assert "No Classic campaign" in results[0]["message"] + def test_created_creative_id_handles_response_shape_variants(self): + extract = ImproveDigitalAdapter._created_creative_id + assert extract([{"id": 9001}]) == 9001 + assert extract({"creatives": [{"id": 9002}]}) == 9002 + assert extract({"id": 9003, "name": "x"}) == 9003 + assert extract({"content": [{"creative_id": 9004}]}) == 9004 + assert extract([]) is None + assert extract({}) is None + assert extract([{"name": "no id echo"}]) is None + + def test_upload_recovers_platform_id_by_name_when_response_has_no_id(self): + """The bulk servlet created the creative but echoed no ID — the + adapter must recover it from the campaign's creative list, or the + creative would exist upstream but never bind to a line item.""" + adapter = make_live_adapter() + creatives_client = adapter._client.creatives + creatives_client.create_third_party_tag_creatives = lambda campaign_id, creatives, **kw: [] + creatives_client.list_creatives = lambda campaign_id, **kw: { + "creatives": [{"id": 528059, "name": "other"}, {"id": 528060, "name": "Banner 300x250"}] + } + statuses = adapter.add_creative_assets( + "improvedigital_101", + assets=[ + { + "creative_id": "cr_1", + "name": "Banner 300x250", + "asset_type": "banner", + "snippet": "", + "width": 300, + "height": 250, + "advertiser_domain": "brand.example.com", + } + ], + today=datetime.now(UTC), + ) + assert statuses[0].status == "approved" + assert statuses[0].creative_id == "528060" + + def test_campaign_fallback_resolves_pending_start_buys(self): + """Cross-instance binding (e.g. sync_creatives after a HITL booking) + must resolve buys that haven't started serving yet.""" + adapter = make_live_adapter() + buy = SimpleNamespace(external_id=None, media_buy_id="improvedigital_314410") + with ( + patch("src.core.database.database_session.get_db_session"), + patch("src.core.database.repositories.media_buy.MediaBuyRepository") as repo_cls, + ): + repo_cls.return_value.find_by_platform_line_item_id.return_value = buy + campaign_id = adapter._campaign_id_for_line_item("585190") + assert campaign_id == 314410 + repo_cls.return_value.find_by_platform_line_item_id.assert_called_once_with("585190") + class TestUpdateMediaBuyLive: def test_pause_media_buy_pauses_every_line_item(self): @@ -288,6 +482,51 @@ def test_empty_cache_raises_delivery_unavailable(self): with pytest.raises(DeliveryDataUnavailable): adapter.get_media_buy_delivery("improvedigital_101", self._reporting_period(), datetime.now(UTC)) + def test_internal_media_buy_id_resolves_campaign_via_external_id(self): + """Callers (admin delivery sync, MCP delivery tool) pass the core + layer's internal ``mb_*`` ID; the Classic campaign reference lives on + ``media_buys.external_id``. The read path must resolve it the same + way the reporting sync's write path does.""" + adapter = make_live_adapter() + rows = [ + SimpleNamespace( + line_item_id="202", + impressions=1000, + clicks=10, + completed_views=None, + spend_micros=4_000_000, + currency="EUR", + ), + ] + with ( + patch("src.core.database.database_session.get_db_session"), + patch("src.core.database.repositories.media_buy.MediaBuyRepository") as buy_repo_cls, + patch( + "src.core.database.repositories.improvedigital_line_item_stats.ImproveDigitalLineItemStatsRepository" + ) as stats_repo_cls, + ): + buy_repo_cls.return_value.get_by_id.return_value = SimpleNamespace(external_id="improvedigital_101") + stats_repo_cls.return_value.list_by_campaign.return_value = rows + response = adapter.get_media_buy_delivery("mb_70aa67ac413f", self._reporting_period(), datetime.now(UTC)) + stats_repo_cls.return_value.list_by_campaign.assert_called_once_with("101") + assert response.totals.impressions == 1000 + + def test_internal_id_without_external_mapping_soft_fails(self): + """An internal ID whose buy row is missing (or has no external stamp) + must stay a soft DeliveryDataUnavailable, never a hard error.""" + adapter = make_live_adapter() + with ( + patch("src.core.database.database_session.get_db_session"), + patch("src.core.database.repositories.media_buy.MediaBuyRepository") as buy_repo_cls, + patch( + "src.core.database.repositories.improvedigital_line_item_stats.ImproveDigitalLineItemStatsRepository" + ) as stats_repo_cls, + ): + buy_repo_cls.return_value.get_by_id.return_value = None + stats_repo_cls.return_value.list_by_campaign.return_value = [] + with pytest.raises(DeliveryDataUnavailable): + adapter.get_media_buy_delivery("mb_70aa67ac413f", self._reporting_period(), datetime.now(UTC)) + def test_cache_rows_aggregate_to_delivery_totals(self): adapter = make_live_adapter() rows = [ diff --git a/tests/unit/test_improvedigital_reporting_page.py b/tests/unit/test_improvedigital_reporting_page.py new file mode 100644 index 0000000000..85f61f786b --- /dev/null +++ b/tests/unit/test_improvedigital_reporting_page.py @@ -0,0 +1,76 @@ +"""Payload shaping for the Improve Digital admin reporting page. + +Covers ``_improvedigital_reporting_payload`` — the pure transform between +``improvedigital_line_item_stats`` cache rows and the JSON the reporting +template renders. No Flask, no DB. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace + +from src.admin.blueprints.adapters import _improvedigital_reporting_payload + +AS_OF = datetime(2026, 8, 6, 9, 0, tzinfo=UTC) + + +def _stat(line_item_id="202", campaign_id="314403", impressions=1000, clicks=10, completed=None, micros=4_000_000): + return SimpleNamespace( + line_item_id=line_item_id, + campaign_id=campaign_id, + impressions=impressions, + clicks=clicks, + completed_views=completed, + spend_micros=micros, + currency="EUR", + as_of=AS_OF, + ) + + +def _buy(media_buy_id="mb_1", order_name="Summer Push", advertiser_name="Brand X"): + return SimpleNamespace(media_buy_id=media_buy_id, order_name=order_name, advertiser_name=advertiser_name) + + +class TestReportingPayload: + def test_rows_join_media_buys_and_convert_micros(self): + payload = _improvedigital_reporting_payload([_stat()], {"314403": _buy()}, "EUR") + + row = payload["rows"][0] + assert row["media_buy_id"] == "mb_1" + assert row["order_name"] == "Summer Push" + assert row["advertiser_name"] == "Brand X" + assert row["spend"] == 4.0 # micros → currency units + assert row["ctr"] == 1.0 # 10 clicks / 1000 impressions + assert row["as_of"] == AS_OF.isoformat() + + def test_unattributed_campaign_still_renders(self): + # Campaigns booked outside salesagent have no matching media buy — + # the row must render unattributed, never be dropped. + payload = _improvedigital_reporting_payload([_stat(campaign_id="999")], {}, "EUR") + + row = payload["rows"][0] + assert row["media_buy_id"] is None + assert row["order_name"] is None + assert row["impressions"] == 1000 + + def test_totals_aggregate_across_rows(self): + stats = [ + _stat(line_item_id="202", impressions=1000, clicks=10, micros=4_000_000), + _stat(line_item_id="203", impressions=500, clicks=None, completed=100, micros=2_000_000), + ] + payload = _improvedigital_reporting_payload(stats, {}, "EUR") + + totals = payload["totals"] + assert totals["impressions"] == 1500 + assert totals["clicks"] == 10 + assert totals["ctr"] == round(10 / 1500 * 100, 2) + assert totals["completed_views"] == 100 + assert totals["spend"] == 6.0 + + def test_zero_impressions_yields_null_ctr(self): + payload = _improvedigital_reporting_payload([_stat(impressions=0, clicks=0, micros=0)], {}, "EUR") + + assert payload["rows"][0]["ctr"] is None + assert payload["totals"]["ctr"] is None + assert payload["totals"]["spend"] == 0.0 diff --git a/tests/unit/test_improvedigital_targeting.py b/tests/unit/test_improvedigital_targeting.py new file mode 100644 index 0000000000..b974079d3a --- /dev/null +++ b/tests/unit/test_improvedigital_targeting.py @@ -0,0 +1,133 @@ +"""Geo targeting translation for the Improve Digital adapter. + +Covers ``build_targeting`` (product-config defaults + buyer overlay → +``LineItemGeoTargetingDto`` entries) and the loud rejection of geo +dimensions the Classic platform cannot express. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +from src.adapters.improvedigital.targeting import ( + build_targeting, + candidate_country_names, + normalize_geo_name, + validate_targeting, +) + + +class _Token: + """Mimics adcp RootModel tokens (value behind ``.root``).""" + + def __init__(self, root: str) -> None: + self.root = root + + +class TestBuildGeoTargeting: + def test_product_config_geo_defaults(self): + # ``exclude`` is present on every entry, includes too — the live + # geo-targeting endpoint 400s when it is omitted. + targeting = build_targeting(None, {"geo_countries": ["NL", "BE"], "geo_regions": ["Flanders"]}) + + assert targeting["geo_targeting"] == [ + {"country": "NL", "exclude": False}, + {"country": "BE", "exclude": False}, + {"region": "Flanders", "exclude": False}, + ] + + def test_overlay_adds_on_top_of_config_and_dedupes(self): + overlay = SimpleNamespace(geo_countries=[_Token("NL"), _Token("DE")], geo_regions=None) + targeting = build_targeting(overlay, {"geo_countries": ["NL"]}) + + assert targeting["geo_targeting"] == [ + {"country": "NL", "exclude": False}, + {"country": "DE", "exclude": False}, + ] + + def test_overlay_excludes_carry_the_exclude_flag(self): + overlay = SimpleNamespace( + geo_countries=[_Token("NL")], + geo_countries_exclude=[_Token("RU")], + ) + targeting = build_targeting(overlay, {}) + + assert targeting["geo_targeting"] == [ + {"country": "NL", "exclude": False}, + {"country": "RU", "exclude": True}, + ] + + def test_overlay_regions_are_not_mapped(self): + """AdCP geo_regions are ISO 3166-2 subdivisions — the platform's + continental region dimension cannot express them, so build_targeting + must not emit region entries from the overlay (validate_targeting + rejects them upfront).""" + overlay = SimpleNamespace(geo_regions=[_Token("US-NY")], geo_regions_exclude=[_Token("US-CA")]) + targeting = build_targeting(overlay, {}) + + assert "geo_targeting" not in targeting + + def test_no_geo_yields_no_geo_key(self): + targeting = build_targeting(None, {"placement_ids": [11]}) + + assert "geo_targeting" not in targeting + assert targeting["placement_ids"] == [11] + + +class TestGeoNameNormalization: + def test_folds_platform_vs_cldr_divergences(self): + # The observed divergence classes between CLDR English names and the + # live 360Yield dictionary — each pair must fold to the same key. + assert normalize_geo_name("The Netherlands") == normalize_geo_name("Netherlands") + assert normalize_geo_name("São Tomé and Príncipe") == normalize_geo_name("Sao Tome and Principe") + assert normalize_geo_name("St. Barthélemy") == normalize_geo_name("Saint Barthelemy") + assert normalize_geo_name("Ceuta & Melilla") == normalize_geo_name("Ceuta and Melilla") + assert normalize_geo_name("Côte d'Ivoire") == normalize_geo_name("Cote d Ivoire") + + def test_candidates_for_iso_code_include_cldr_name(self): + assert candidate_country_names("NL") == ["NL", "Netherlands"] + + def test_candidates_for_aliased_code_prefer_platform_alias(self): + candidates = candidate_country_names("CI") + assert candidates[0] == "CI" + assert candidates[1] == "Ivory Coast" # curated platform alias wins over CLDR + + def test_non_code_token_passes_verbatim_only(self): + assert candidate_country_names("The Netherlands") == ["The Netherlands"] + + +class TestValidateTargeting: + def test_metro_targeting_rejected_loudly(self): + overlay = SimpleNamespace(geo_metros=[_Token("501")]) + messages = validate_targeting(overlay) + + assert any("Metro" in m for m in messages) + + def test_proximity_targeting_rejected_loudly(self): + overlay = SimpleNamespace(geo_proximity=SimpleNamespace(radius=5)) + messages = validate_targeting(overlay) + + assert any("Proximity" in m for m in messages) + + def test_supported_geo_passes_validation(self): + overlay = SimpleNamespace(geo_countries=[_Token("NL")], geo_countries_exclude=[_Token("RU")]) + + assert validate_targeting(overlay) == [] + + def test_buyer_region_overlay_rejected_loudly(self): + """Spec-valid AdCP geo_regions ('US-NY' subdivisions) can never match + the platform's continental regions — reject before any campaign is + created, never after.""" + overlay = SimpleNamespace(geo_regions=[_Token("US-NY")]) + messages = validate_targeting(overlay) + + assert any("Region targeting" in m and "geo_countries" in m for m in messages) + + def test_rejection_advice_never_points_at_geo_regions(self): + """The postal/metro rejection messages must not steer buyers into the + (unsupported) geo_regions overlay path.""" + overlay = SimpleNamespace(geo_postal_areas=["1012"], geo_metros=[_Token("501")]) + messages = validate_targeting(overlay) + + assert messages, "postal + metro must be rejected" + assert not any("geo_regions" in m for m in messages) From d8b354d03867e9ca52b8079e0210fdadf8f6f57c Mon Sep 17 00:00:00 2001 From: Chinmoy Acharjee Date: Fri, 7 Aug 2026 12:30:56 +0600 Subject: [PATCH 65/90] feat(improvedigital): business unit id added (#57) --- src/admin/blueprints/adapters.py | 4 ++++ .../improvedigital/connection_config.html | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index b13f17259f..0f87e5afb0 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1125,6 +1125,10 @@ def test_improvedigital_connection(tenant_id, **kwargs): "user_id": details.get("user_id"), "name": f"{details.get('first_name', '')} {details.get('last_name', '')}".strip(), "business_unit": details.get("business_unit_name"), + # Campaign booking requires a business_unit_id (layer-2 rule, + # not in the create schema) — the API user's own unit is the + # right default, so the UI auto-fills it from here. + "business_unit_id": details.get("business_unit_id"), } except ImproveDigitalError: pass # identity display is optional — credentials are already verified diff --git a/templates/adapters/improvedigital/connection_config.html b/templates/adapters/improvedigital/connection_config.html index 209e3cd3e0..685848b9e7 100644 --- a/templates/adapters/improvedigital/connection_config.html +++ b/templates/adapters/improvedigital/connection_config.html @@ -102,6 +102,17 @@

Improve Digital 360Yield Configuration

+
+ + + + The Improve Digital business unit campaigns book under (your API user's own + unit — Test Connection fills it in automatically). + +
+
Inventory Sync

const buyingEntityEl = document.getElementById('improvedigital_buying_entity_id'); const officeEl = document.getElementById('improvedigital_buying_entity_office_id'); const demandContactEl = document.getElementById('improvedigital_demand_contact_id'); + const businessUnitEl = document.getElementById('improvedigital_business_unit_id'); const advertiserEl = document.getElementById('improvedigital_default_advertiser_id'); const agencyEl = document.getElementById('improvedigital_agency_id'); const baseUrlEl = document.getElementById('improvedigital_api_base_url'); @@ -209,6 +221,7 @@

Inventory Sync

buying_entity_id: buyingEntityEl.value ? parseInt(buyingEntityEl.value, 10) : null, buying_entity_office_id: officeEl.value ? parseInt(officeEl.value, 10) : null, improve_demand_contact_id: demandContactEl.value ? parseInt(demandContactEl.value, 10) : null, + business_unit_id: businessUnitEl.value ? parseInt(businessUnitEl.value, 10) : null, default_advertiser_id: advertiserEl.value ? parseInt(advertiserEl.value, 10) : null, agency_id: agencyEl.value ? parseInt(agencyEl.value, 10) : null, currency: document.getElementById('improvedigital_currency').value, @@ -451,6 +464,14 @@

Inventory Sync

(data.user.business_unit ? ' (' + data.user.business_unit + ')' : '') : ''; statusEl.innerHTML = '✓ Authenticated' + who + ' against ' + (data.base_url || '360Yield') + ''; + // Booking requires business_unit_id; the API user's own unit is + // the right value — fill it in when the operator hasn't set one. + const buEl = document.getElementById('improvedigital_business_unit_id'); + if (buEl && !buEl.value && data.user && data.user.business_unit_id) { + buEl.value = data.user.business_unit_id; + statusEl.innerHTML += ' — Business Unit ID filled in (' + + data.user.business_unit_id + '), remember to Save'; + } } else { statusEl.innerHTML = '✗ ' + (data.error || 'Connection failed') + ''; } From 5664765f269cf805da0b73eb1ba4d977d74e4810 Mon Sep 17 00:00:00 2001 From: Chinmoy Acharjee Date: Fri, 7 Aug 2026 17:31:08 +0600 Subject: [PATCH 66/90] Fix product (#58) * feat(improvedigital): fix targeting persistence issue in saving product * feat(improvedigital): enrich UI for selected ad server --- src/admin/blueprints/products.py | 28 ++++++++++++++++--------- src/core/json_validators.py | 6 ++++++ static/css/admin.css | 35 ++++++++++++++++++++++++++++++++ templates/base.html | 11 +++++++--- templates/edit_product.html | 4 +++- templates/edit_product_mock.html | 4 +++- 6 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/admin/blueprints/products.py b/src/admin/blueprints/products.py index d4805258b6..3293d786f9 100644 --- a/src/admin/blueprints/products.py +++ b/src/admin/blueprints/products.py @@ -1744,6 +1744,21 @@ def edit_product(tenant_id, product_id): elif line_item_type in ["PRICE_PRIORITY", "HOUSE"]: product.delivery_type = "non_guaranteed" + # Parse targeting template from form (custom targeting key-value + # pairs). The unified edit form posts this field for EVERY + # adapter — storing it must not be GAM-only, or non-GAM tenants + # silently lose their Custom Targeting edits on save. + targeting_template_json = form_data.get("targeting_template", "{}") + try: + targeting_template = json.loads(targeting_template_json) if targeting_template_json else {} + except json.JSONDecodeError: + targeting_template = {} + + product.targeting_template = targeting_template + from sqlalchemy.orm import attributes as _sa_attributes + + _sa_attributes.flag_modified(product, "targeting_template") + # Update implementation_config with GAM-specific fields # Note: This must run even if line_item_type is not present (automatic mode) if adapter_type == "google_ad_manager": @@ -1799,14 +1814,8 @@ def edit_product(tenant_id, product_id): if form_data.get("priority"): base_config["priority"] = int(form_data["priority"]) - # Parse targeting template from form (includes custom targeting key-value pairs) - targeting_template_json = form_data.get("targeting_template", "{}") - try: - targeting_template = json.loads(targeting_template_json) if targeting_template_json else {} - except json.JSONDecodeError: - targeting_template = {} - - # If targeting template has key_value_pairs, copy to implementation_config for GAM + # If targeting template (parsed above, stored for every + # adapter) has key_value_pairs, copy to implementation_config for GAM if targeting_template.get("key_value_pairs"): if "custom_targeting_keys" not in base_config: base_config["custom_targeting_keys"] = {} @@ -1822,8 +1831,7 @@ def edit_product(tenant_id, product_id): # Legacy format - merge as before base_config["custom_targeting_keys"].update(kv_pairs) - # Store targeting_template in product - product.targeting_template = targeting_template + # (targeting_template itself is stored above for every adapter) # Reject inconsistent GAM inventory configuration. Only applies to direct # targeting — profile-based products derive their inventory from the profile. diff --git a/src/core/json_validators.py b/src/core/json_validators.py index 6c116cfce8..a66e60d2ea 100644 --- a/src/core/json_validators.py +++ b/src/core/json_validators.py @@ -59,6 +59,12 @@ class TargetingTemplateModel(BaseModel): audience_segments: list[str] | None = None content_categories: list[str] | None = None custom_parameters: dict[str, Any] | None = None + # Custom targeting rules from the product-form targeting widget — + # {"groups": [{"criteria": [{"keyId", "values"}]}]} (grouped), + # {"include"/"exclude": ...} (enhanced), or a flat key→values dict + # (legacy). Without this field Pydantic silently dropped the key and + # every product save scrubbed the widget's targeting to {}. + key_value_pairs: dict[str, Any] | None = None class PolicySettingsModel(BaseModel): diff --git a/static/css/admin.css b/static/css/admin.css index d75640f6f2..e18531a5fb 100644 --- a/static/css/admin.css +++ b/static/css/admin.css @@ -316,6 +316,41 @@ pre { font-size: 9px; margin-left: 2px; } +/* Ad-server chip — identifies the tenant's configured adapter in the top + bar. Outlined mono lockup with a dim "AD SERVER" prefix and a neutral + dot: reads as metadata, not a control, and stays monochrome per the + header's ink/paper language (header vars keep it legible on the + inverted super-admin bar too). */ +.header .role-tag .adapter-chip { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 3px 9px; + border: 1px solid var(--sa-header-rule); + border-radius: 999px; + font-family: var(--sa-font-mono); + font-size: 10px; + font-weight: 500; + letter-spacing: 0.06em; + text-transform: uppercase; + line-height: 1; + color: var(--sa-header-fg); + white-space: nowrap; +} +.header .role-tag .adapter-chip::before { + content: ''; + width: 6px; + height: 6px; + border-radius: 50%; + background: var(--sa-header-fg-dim); + flex-shrink: 0; +} +.header .role-tag .adapter-chip .adapter-chip__label { + color: var(--sa-header-fg-dim); + padding-right: 6px; + border-right: 1px solid var(--sa-header-rule); +} + /* The role badge becomes the Ledger STOREFRONT / ADMIN pill — mono lockup, inverted from the bar background. */ .header .role-tag .status { diff --git a/templates/base.html b/templates/base.html index 7817f2641b..2f643d60f5 100644 --- a/templates/base.html +++ b/templates/base.html @@ -126,10 +126,15 @@

{% if session.role == 'super_admin' %}Sales Agent{% else %}Sales Agent{% end {% else %} {{ session.email or session.username }} {% endif %} + {% if nav_adapter_type %} + {# Configured ad-server chip — metadata, not a control + (deliberately NOT .workspace-name: that class carries + the ▾ switcher affordance). #} + + Ad server{{ nav_adapter_type }} + + {% endif %} - {% if nav_adapter_type %} - {{ nav_adapter_type }} - {% endif %} {% if session.role == 'super_admin' %} Accounts Scheduling diff --git a/templates/edit_product.html b/templates/edit_product.html index 0f51946c76..075ec9c10d 100644 --- a/templates/edit_product.html +++ b/templates/edit_product.html @@ -148,7 +148,9 @@

{# Enhanced targeting selector with OR/AND operators and include/exclude #} {% include 'components/targeting_selector_widget.html' %} - + {# Single-quoted attribute: |tojson emits double quotes (escapes <>&' but not "), + so a double-quoted value= truncates at the first key — same pattern as add_product_gam.html #} +

diff --git a/templates/edit_product_mock.html b/templates/edit_product_mock.html index b1ff809904..7b74adfd95 100644 --- a/templates/edit_product_mock.html +++ b/templates/edit_product_mock.html @@ -172,7 +172,9 @@

{# Enhanced targeting selector with OR/AND operators and include/exclude #} {% include 'components/targeting_selector_widget.html' %} - + {# Single-quoted attribute: |tojson emits double quotes (escapes <>&' but not "), + so a double-quoted value= truncates at the first key — same pattern as add_product_gam.html #} +

From 78ad1d3168d09e3ec75804a7d84d524cce2e02a6 Mon Sep 17 00:00:00 2001 From: Chinmoy Acharjee Date: Mon, 10 Aug 2026 13:48:10 +0600 Subject: [PATCH 67/90] Inventory improve digital (#59) * feat(improvedigital): adjust inventory page for improve digital * feat(improvedigital): fix inventory for improve digital and product targeting fix for editing product --- src/admin/blueprints/adapters.py | 36 +++++++++-- src/admin/blueprints/inventory.py | 38 +++++++++++- .../repositories/improvedigital_inventory.py | 10 +++ templates/edit_product.html | 40 ++++++++++-- templates/sync_inventory.html | 61 ++++++++++++++++++- 5 files changed, 169 insertions(+), 16 deletions(-) diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index 0f87e5afb0..0ad29c9efc 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1213,16 +1213,16 @@ def list_improvedigital_inventory(tenant_id, **kwargs): Filterable by ``entity_type`` (publisher, placement, package, size). Optional ``parent_id`` narrows placements to one publisher. Optional - ``q`` substring-matches the ``name`` field. - - Returns a flat list (no pagination — the cache is small enough that - sending the whole filtered set is fine for now). + ``q`` substring-matches the ``name`` field. Optional ``limit`` caps the + returned rows AFTER filtering (the browse page passes it; the product + pickers omit it and cache the full set client-side). """ from src.core.database.repositories.improvedigital_inventory import ImproveDigitalInventoryRepository entity_type = request.args.get("entity_type") parent_id = request.args.get("parent_id") q = request.args.get("q") + limit = request.args.get("limit", type=int) if not entity_type: return jsonify({"success": False, "error": "entity_type query param is required"}), 400 @@ -1236,7 +1236,33 @@ def list_improvedigital_inventory(tenant_id, **kwargs): for row in rows if not q or (row.name and q.lower() in row.name.lower()) ] - return jsonify({"success": True, "entity_type": entity_type, "count": len(items), "items": items}) + total = len(items) + if limit is not None and limit >= 0: + items = items[:limit] + return jsonify({"success": True, "entity_type": entity_type, "count": total, "items": items}) + + +@adapters_bp.route("/api/tenant//adapters/improvedigital/inventory-stats", methods=["GET"]) +@require_tenant_access() +def improvedigital_inventory_stats(tenant_id, **kwargs): + """Quick stats for the Improve Digital inventory cache — row counts per + entity type + last sync time. Feeds the Browse Inventory header and the + Sync Inventory page without materializing the 20k+ cached rows.""" + from src.core.database.repositories.improvedigital_inventory import ImproveDigitalInventoryRepository + + with get_db_session() as session: + repo = ImproveDigitalInventoryRepository(session, tenant_id) + counts = repo.counts_by_type() + last_synced = repo.latest_sync_at() + + return jsonify( + { + "success": True, + "counts": counts, + "total": sum(counts.values()), + "last_synced_at": last_synced.isoformat() if last_synced else None, + } + ) @adapters_bp.route( diff --git a/src/admin/blueprints/inventory.py b/src/admin/blueprints/inventory.py index 26d8b56695..1c0cccdfce 100644 --- a/src/admin/blueprints/inventory.py +++ b/src/admin/blueprints/inventory.py @@ -444,6 +444,19 @@ def inventory_browser(tenant_id): "is_embedded": False, } + # Capability-driven sync support for non-GAM adapters: the generic sync + # card renders whenever the adapter declares supports_inventory_sync + # (posts to /api/tenant//adapters//sync-inventory, the shared + # execute_adapter_sync orchestration). + from src.adapters import get_adapter_schemas + from src.admin.utils.helpers import ADAPTER_LABELS + + schemas = get_adapter_schemas(adapter_type) + capabilities = schemas.capabilities if schemas else None + supports_inventory_sync = bool(capabilities and capabilities.supports_inventory_sync) + inventory_entity_label = (capabilities.inventory_entity_label if capabilities else None) or "Inventory" + adapter_label = ADAPTER_LABELS.get(adapter_type, adapter_type) + return render_template( "sync_inventory.html", tenant=tenant_dict, @@ -451,6 +464,9 @@ def inventory_browser(tenant_id): tenant_name=tenant_dict["name"], is_gam=is_gam, adapter_type=adapter_type, + adapter_label=adapter_label, + supports_inventory_sync=supports_inventory_sync, + inventory_entity_label=inventory_entity_label, ) @@ -475,6 +491,18 @@ def inventory_browse(tenant_id): inventory_type = request.args.get("type", "all") + if adapter_type == "improvedigital": + # 360Yield inventory browser — publishers→placements tree plus + # packages and sizes, backed by the improvedigital_inventory cache + # (see /api/tenant//adapters/improvedigital/inventory). + return render_template( + "inventory_browser_improvedigital.html", + tenant=tenant_dict, + tenant_id=tenant_id, + tenant_name=tenant_dict["name"], + adapter_type=adapter_type, + ) + return render_template( "inventory_browser.html", tenant=tenant_dict, @@ -989,13 +1017,19 @@ def sync_inventory(tenant_id): if not tenant: return jsonify({"error": "Tenant not found"}), 404 - # Check adapter type - inventory sync is only for GAM + # This endpoint drives the GAM sync pipeline specifically. Other + # adapters sync through the shared orchestration at + # /api/tenant//adapters//sync-inventory — point callers + # there instead of (wrongly) claiming they need no sync. adapter_type = tenant.ad_server or "mock" if adapter_type != "google_ad_manager": return ( jsonify( { - "error": f"Inventory sync is only available for Google Ad Manager. Your tenant is using the '{adapter_type}' adapter which does not require inventory sync." + "error": ( + f"This endpoint syncs Google Ad Manager inventory only. The '{adapter_type}' " + f"adapter syncs via /api/tenant/{tenant_id}/adapters/{adapter_type}/sync-inventory." + ) } ), 400, diff --git a/src/core/database/repositories/improvedigital_inventory.py b/src/core/database/repositories/improvedigital_inventory.py index 538e56d5ae..87f5397965 100644 --- a/src/core/database/repositories/improvedigital_inventory.py +++ b/src/core/database/repositories/improvedigital_inventory.py @@ -93,6 +93,16 @@ def latest_sync_at(self) -> datetime | None: stmt = select(func.max(ImproveDigitalInventory.last_synced_at)).filter_by(tenant_id=self._tenant_id) return self._session.scalar(stmt) + def counts_by_type(self) -> dict[str, int]: + """Row counts per entity_type — feeds the browse page's quick stats + without materializing tens of thousands of rows.""" + stmt = ( + select(ImproveDigitalInventory.entity_type, func.count()) + .filter_by(tenant_id=self._tenant_id) + .group_by(ImproveDigitalInventory.entity_type) + ) + return dict(self._session.execute(stmt).tuples().all()) + def delete_all(self) -> int: """Wipe the tenant's inventory cache. Used when an operator triggers a full resync via the admin UI.""" diff --git a/templates/edit_product.html b/templates/edit_product.html index 075ec9c10d..ec62004b44 100644 --- a/templates/edit_product.html +++ b/templates/edit_product.html @@ -152,14 +152,22 @@

so a double-quoted value= truncates at the first key — same pattern as add_product_gam.html #} - +

- Advertising Channels + Channel & Geographic Targeting

-
- - {% set selected_channels = product.channels if product.channels else [] %} - {% include 'components/channel_selector.html' %} +
+
+ + {% set selected_channels = product.channels if product.channels else [] %} + {% include 'components/channel_selector.html' %} +
+ +
+ + {% set selected_countries = product.countries if product.countries else [] %} + {% include 'components/country_selector.html' %} +
@@ -508,6 +516,26 @@
Pricing Option #${index + 1}
addPricingOption(); } + // Handle country selection ("All Countries" is mutually exclusive with specific countries) + const countriesSelect = document.getElementById('countries'); + countriesSelect.addEventListener('change', function() { + const selectedOptions = Array.from(this.selectedOptions).map(opt => opt.value); + + // If "ALL" is selected, deselect all others + if (selectedOptions.includes('ALL') && selectedOptions.length > 1) { + Array.from(this.options).forEach(opt => opt.selected = false); + this.options[0].selected = true; // Select "All Countries" + } + // If specific countries are selected, deselect ALL + else if (!selectedOptions.includes('ALL') && selectedOptions.length > 0) { + this.options[0].selected = false; // Deselect "All Countries" + } + // If nothing is selected, select ALL by default + else if (selectedOptions.length === 0) { + this.options[0].selected = true; // Select "All Countries" + } + }); + // Initialize format template picker with existing formats window.formatPicker = new FormatTemplatePicker({ containerId: 'format-template-picker-container', diff --git a/templates/sync_inventory.html b/templates/sync_inventory.html index 55728b1996..97bb3a223f 100644 --- a/templates/sync_inventory.html +++ b/templates/sync_inventory.html @@ -15,11 +15,33 @@

Sync Inventory

- {% if not is_gam %} + {% if not is_gam and supports_inventory_sync %} + {# Capability-driven sync for non-GAM adapters — posts to the shared + adapter sync orchestration (execute_adapter_sync), which handles + concurrency (409 when a sync is in flight) and SyncJob bookkeeping. #} +
+
+
+
+
+
+
{{ adapter_label }} — {{ inventory_entity_label }}
+

+ Pull {{ inventory_entity_label|lower }} and related inventory from {{ adapter_label }} into the local cache. +

+
+ +
+
+
+
+
+ {% elif not is_gam %}
- Inventory sync is only available for Google Ad Manager tenants. Your tenant is configured to use the - {{ adapter_type }} adapter. + The {{ adapter_type }} adapter does not support inventory sync.
{% else %} @@ -98,6 +120,39 @@
When to use which sync
const tenantId = '{{ tenant_id }}'; const scriptRoot = '{{ request.script_root }}'; const isGam = {{ 'true' if is_gam else 'false' }}; +const adapterType = '{{ adapter_type }}'; + +// Non-GAM adapters with supports_inventory_sync: one-shot sync through the +// shared orchestration. Synchronous call — completes (or 409s) in-request. +function syncAdapterInventory() { + const btn = document.getElementById('adapterSyncBtn'); + const statusEl = document.getElementById('adapterSyncStatus'); + if (!btn) return; + const original = btn.innerHTML; + btn.disabled = true; + btn.innerHTML = ' Syncing…'; + statusEl.textContent = 'Syncing — large catalogs can take a minute (upstream rate limits apply).'; + + fetch(`${scriptRoot}/api/tenant/${tenantId}/adapters/${adapterType}/sync-inventory`, { + method: 'POST', + credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' } + }) + .then(r => r.json().then(data => ({ status: r.status, data }))) + .then(({ status, data }) => { + if (data.success) { + const summary = Object.entries(data.counts || {}).map(([k, v]) => `${k}=${v}`).join(', '); + statusEl.textContent = `✓ Synced ${(data.total_synced || 0).toLocaleString()} items (${summary})`; + } else if (status === 409) { + statusEl.textContent = data.error || 'A sync is already running — wait for it to finish.'; + } else { + const errors = data.errors ? Object.entries(data.errors).map(([k, v]) => `${k}: ${v}`).join('; ') : ''; + statusEl.textContent = `✗ ${data.error || errors || 'Sync failed'}`; + } + }) + .catch(err => { statusEl.textContent = '✗ ' + err.message; }) + .finally(() => { btn.disabled = false; btn.innerHTML = original; }); +} function syncInventory(mode) { if (!isGam) return; From c4e063709e0c9aad9e6b3e436f2a694e080ab5cb Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Mon, 10 Aug 2026 19:19:45 +0600 Subject: [PATCH 68/90] feat(improvedigital): Adapter settings page update, Line item payload updated --- .../improvedigital/live-wire-shapes.md | 14 ++ .../ops/configure_improvedigital_tenant.py | 7 + src/adapters/improvedigital/adapter.py | 57 +++++- src/adapters/improvedigital/schemas.py | 21 ++- src/admin/blueprints/adapters.py | 33 +++- src/admin/blueprints/products.py | 2 +- .../improvedigital/connection_config.html | 174 +++++++++++++++--- .../improvedigital/product_config.html | 19 +- 8 files changed, 298 insertions(+), 29 deletions(-) diff --git a/docs/adapters/improvedigital/live-wire-shapes.md b/docs/adapters/improvedigital/live-wire-shapes.md index 3084afb946..80c5fbe662 100644 --- a/docs/adapters/improvedigital/live-wire-shapes.md +++ b/docs/adapters/improvedigital/live-wire-shapes.md @@ -18,6 +18,20 @@ incomplete in several places. "Active"`, `business_unit_id` (33 = Azerion on dev), `improve_demand_contact_id`, and `goal` for CPM items (`IMPRESSION`/`BUDGET`). `pricing_model: "CPM"` confirmed (gap G2 closed). +- **Line item create — platform-parity fields** (from a payload captured on + `api-alpha.360yielddev.com`, not yet re-validated on `api.360yielddev.com`): + the platform's own create sends the money figure as `budget` plus a + `flight_details[]` row (`start_time`, `end_time`, `budget`, + `budget_is_daily`, `impression_cap`, `impression_cap_daily`) alongside + `buyer_id`, `invoice_type: "on_actuals"`, `pricing_model_type: "First Bid"`, + `delivery_schedule: "Evenly"`, `third_party_inventory`, `is_optimised` and + the compliance flags (`track_viewability`, `is_consentless`, + `is_coppa_compliant`, `conversion_tracking_enabled`, `keep_on_delivering`, + `optout_mechanism`). `flight_details` is absent from the committed + rtb-v3 OpenAPI spec, and `dynamic_optimization_kpi_value` is typed `string` + there but sent as `0` — both are alpha-side divergences. +- `frequency_interval_type` is a lowercase enum + (`months`/`weeks`/`days`/`hours`/`minutes`) — product configs must match. - **improve_demand_contact_id == the API user's user_id** (from `GET /lookup/v1/user-details`) — every recent dev campaign follows this. - **Advertiser is optional** on Classic campaigns (`advertiserId` is null on diff --git a/scripts/ops/configure_improvedigital_tenant.py b/scripts/ops/configure_improvedigital_tenant.py index 66bd65f8b0..8e903853fe 100644 --- a/scripts/ops/configure_improvedigital_tenant.py +++ b/scripts/ops/configure_improvedigital_tenant.py @@ -51,6 +51,12 @@ def main() -> int: required=True, help="Business unit ID for Classic line items (33 = Azerion on the dev platform)", ) + parser.add_argument( + "--buyer-id", + type=int, + default=None, + help="Buyer ID sent on Classic line items (omitted when unset)", + ) parser.add_argument("--currency", default="EUR") parser.add_argument("--timezone", default="Europe/Amsterdam") args = parser.parse_args() @@ -71,6 +77,7 @@ def main() -> int: buying_entity_id=args.buying_entity_id, buying_entity_office_id=args.buying_entity_office_id, business_unit_id=args.business_unit_id, + buyer_id=args.buyer_id, currency=args.currency, timezone=args.timezone, ) diff --git a/src/adapters/improvedigital/adapter.py b/src/adapters/improvedigital/adapter.py index a8b29e2c4c..57949e2eb0 100644 --- a/src/adapters/improvedigital/adapter.py +++ b/src/adapters/improvedigital/adapter.py @@ -138,6 +138,7 @@ def __init__( self.buying_entity_office_id = self.config.get("buying_entity_office_id") self.campaign_type = self.config.get("campaign_type") or "Improve" self.business_unit_id = self.config.get("business_unit_id") + self.buyer_id = self.config.get("buyer_id") self.improve_demand_contact_id = self.config.get("improve_demand_contact_id") self.agency_id = self.config.get("agency_id") @@ -651,13 +652,27 @@ def _line_item_payload( platform): name, type, line_item_status, start_date, business_unit_id, improve_demand_contact_id, and a goal for CPM line items. + + The booking goal comes from the product's ``goal`` config: ``BUDGET`` + (the default) paces against the money figure, ``IMPRESSION`` against + the impression cap. Both figures ride along either way — on the line + item itself and on a single ``flight_details`` row spanning the + flight — because the platform's own create sends both. The + delivery/compliance flags below mirror that same payload: the API + leaves them unset rather than defaulted when omitted. """ product_config = self._with_product_geo_defaults(package, self._product_config_from_package(package)) + goal = str(product_config.get("goal") or "BUDGET").upper() + # Packages booked from a buy-level budget carry no per-package figure; + # reverse the core layer's budget→goal-units conversion + # (media_buy_create._goal_units_from_budget) so a BUDGET-goal line item + # always books the money the impression cap was sized for. + budget = float(package.budget) if package.budget is not None else round(package.impressions * rate / 1000, 2) payload: dict[str, Any] = { "name": package.name or package.package_id, "type": "Standard", "line_item_status": "Active", - "goal": "IMPRESSION", + "goal": goal, "start_date": self._format_datetime(start_time), "end_date": self._format_datetime(end_time), "time_zone": self.timezone, @@ -667,18 +682,56 @@ def _line_item_payload( # match default campaign owner/publisher currency"). "cpm_bid": rate, "pricing_model": product_config.get("pricing_model") or rate_type, + "pricing_model_type": "First Bid", "impression_cap": package.impressions, + "impression_cap_daily": False, + "budget_is_daily": False, + "invoice_type": "on_actuals", + "delivery_schedule": product_config.get("delivery_schedule") or "Evenly", + "third_party_inventory": True, + "is_optimised": True, + "is_dynamic_optimization": False, + "dynamic_optimization_kpi_type": "", + "dynamic_optimization_kpi_value": 0, + "conversion_tracking_enabled": False, + "keep_on_delivering": False, + "track_viewability": False, + "is_consentless": False, + "is_coppa_compliant": False, + "optout_mechanism": [], "reference_number": package.package_id, "improve_demand_contact_id": self.improve_demand_contact_id, } + payload["budget"] = budget + payload["flight_details"] = [self._flight_detail(budget, package.impressions, start_time, end_time)] if self.business_unit_id: payload["business_unit_id"] = int(self.business_unit_id) - for field in ("frequency_cap", "frequency_interval", "frequency_interval_type", "delivery_schedule"): + if self.buyer_id: + payload["buyer_id"] = int(self.buyer_id) + for field in ("frequency_cap", "frequency_interval", "frequency_interval_type"): if product_config.get(field) is not None: payload[field] = product_config[field] payload.update(build_targeting(package.targeting_overlay, product_config, tenant_id=self.tenant_id)) return payload + def _flight_detail( + self, budget: float, impressions: int, start_time: datetime, end_time: datetime + ) -> dict[str, Any]: + """One flight row covering the whole booking window. + + The platform splits a line item's pacing into flights; AdCP buys have + a single flight, so the row mirrors the line item's own window, budget + and impression cap. + """ + return { + "start_time": self._format_datetime(start_time), + "end_time": self._format_datetime(end_time), + "budget": budget, + "budget_is_daily": False, + "impression_cap": impressions, + "impression_cap_daily": False, + } + def _product_config_from_package(self, package: MediaPackage) -> dict[str, Any]: impl = getattr(package, "implementation_config", None) or {} return impl.get("improvedigital", impl) if isinstance(impl, dict) else {} diff --git a/src/adapters/improvedigital/schemas.py b/src/adapters/improvedigital/schemas.py index aeb7e0a39e..91f46fb24e 100644 --- a/src/adapters/improvedigital/schemas.py +++ b/src/adapters/improvedigital/schemas.py @@ -99,11 +99,19 @@ class ImproveDigitalConnectionConfig(BaseConnectionConfig): description="Business unit ID — required on Classic line items (e.g. 33 = Azerion on the dev platform)", json_schema_extra={"ui_order": 10}, ) + buyer_id: int | None = Field( + default=None, + description=( + "Buyer the line items book under — sent as CommonDealLineItemDto.buyer_id " + "(omitted when unset; the platform then derives it from the campaign)" + ), + json_schema_extra={"ui_order": 11}, + ) currency: str = Field( default="EUR", description="Default campaign currency (ISO 4217)", json_schema_extra={ - "ui_order": 11, + "ui_order": 12, # CampaignDto currency enum from the rtb/v3 OpenAPI spec. "enum": [ "EUR", @@ -127,7 +135,7 @@ class ImproveDigitalConnectionConfig(BaseConnectionConfig): timezone: str = Field( default="UTC", description="Default campaign timezone (IANA name, e.g. Europe/Amsterdam) — required by the campaign API", - json_schema_extra={"ui_order": 12}, + json_schema_extra={"ui_order": 13}, ) @field_serializer("client_secret") @@ -203,6 +211,15 @@ class ImproveDigitalProductConfig(BaseProductConfig): default=None, description="Line-item pricing model (CPM confirmed; further values pending platform confirmation — gap G2)", ) + goal: str = Field( + default="BUDGET", + description=( + "Line-item delivery goal. BUDGET books against the package budget " + "(sent as budget + flight_details); IMPRESSION books against the " + "impression cap derived from budget ÷ rate" + ), + json_schema_extra={"enum": ["BUDGET", "IMPRESSION"]}, + ) frequency_cap: int | None = Field( default=None, description="Impressions per user per frequency interval", diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index 0f87e5afb0..af72937e29 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1047,6 +1047,31 @@ def _improvedigital_rows(payload, *keys: str) -> list: return [] +def _improvedigital_buyer_options(payload) -> list[dict]: + """Normalize 360Yield buyer rows into ``{id, name}`` picker options. + + Buyers surface as a ``buyers`` list on ``/lookup/v1/user-details`` and on + buying-entity office rows. ``BuyerDto`` carries both a numeric ``id`` and + a string ``buyer_id`` (the platform's external reference) — the line item + field is an integer, so the numeric id wins and a non-numeric fallback is + dropped rather than sent as garbage. + """ + rows = payload.get("buyers") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + return [] + options: list[dict] = [] + for row in rows: + if not isinstance(row, dict): + continue + identifier = row.get("id") + if identifier is None and str(row.get("buyer_id") or "").isdigit(): + identifier = int(row["buyer_id"]) + if not isinstance(identifier, int): + continue + options.append({"id": identifier, "name": row.get("name") or row.get("buyer_name") or str(identifier)}) + return options + + def _improvedigital_paginate(fetch_page, envelope_key: str, page_size: int = 100, max_rows: int = 10000) -> list: """Exhaust a 360Yield offset/limit-paginated list endpoint. @@ -1127,8 +1152,11 @@ def test_improvedigital_connection(tenant_id, **kwargs): "business_unit": details.get("business_unit_name"), # Campaign booking requires a business_unit_id (layer-2 rule, # not in the create schema) — the API user's own unit is the - # right default, so the UI auto-fills it from here. + # right default, so the UI auto-fills it from here. The + # user_id doubles as the improve_demand_contact_id, and the + # buyer list backs the Buyer ID picker. "business_unit_id": details.get("business_unit_id"), + "buyers": _improvedigital_buyer_options(details), } except ImproveDigitalError: pass # identity display is optional — credentials are already verified @@ -1174,6 +1202,9 @@ def discover_improvedigital_buying_entities(tenant_id, **kwargs): "improve_demand_contact_id": row.get("improve_demand_contact_id"), "billing_currency_code": row.get("billing_currency_code"), "buying_types": row.get("buying_types") or [], + # Offices that pin a buyer let the picker fill Buyer ID + # from the office selection instead of a second lookup. + "buyers": _improvedigital_buyer_options(row), } for row in rows if row.get("active") and "Classic" in (row.get("buying_types") or []) diff --git a/src/admin/blueprints/products.py b/src/admin/blueprints/products.py index 3293d786f9..ef83d83553 100644 --- a/src/admin/blueprints/products.py +++ b/src/admin/blueprints/products.py @@ -715,7 +715,7 @@ def _improvedigital_implementation_config(base_config: dict) -> dict: else: config.pop(field, None) - for field in ("pricing_model", "frequency_interval_type", "delivery_schedule"): + for field in ("pricing_model", "frequency_interval_type", "delivery_schedule", "goal"): raw = (request.form.get(f"impl_{field}") or "").strip() if raw: config[field] = raw diff --git a/templates/adapters/improvedigital/connection_config.html b/templates/adapters/improvedigital/connection_config.html index 685848b9e7..e62dfc1997 100644 --- a/templates/adapters/improvedigital/connection_config.html +++ b/templates/adapters/improvedigital/connection_config.html @@ -44,6 +44,24 @@

Improve Digital 360Yield Configuration

+
+ + + + + + + + + Every call — discovery, Test Connection, inventory sync, reporting and + booking — goes to this host, so set it before discovering anything. + Must be https; leave blank to use production. + +
+
@@ -113,6 +131,22 @@

Improve Digital 360Yield Configuration

+
+ + + + + Sent as buyer_id on every Classic line item. The picker fills + from Test Connection (your API user's buyers) or from the + selected office; leave empty to let the platform derive it from the campaign. + +
+
Improve Digital 360Yield Configuration

- + {% set current_timezone = adapter_config.get('timezone', 'UTC') if adapter_config else 'UTC' %} + {% set timezones = ['UTC', 'Europe/Amsterdam', 'Europe/London', 'Europe/Berlin', 'Europe/Paris', + 'Europe/Madrid', 'Europe/Rome', 'Europe/Stockholm', 'Europe/Warsaw', + 'Europe/Istanbul', 'America/New_York', 'America/Chicago', 'America/Denver', + 'America/Los_Angeles', 'America/Sao_Paulo', 'Asia/Dubai', 'Asia/Singapore', + 'Asia/Hong_Kong', 'Asia/Tokyo', 'Australia/Sydney'] %} + - IANA timezone applied to campaign flight dates (e.g. Europe/Amsterdam). + IANA timezone applied to campaign flight dates and line-item pacing.
-
- Advanced: API host override -
- - - - Optional — defaults to the production 360Yield host. Must be https. - -
-
-
+
+ Booking readiness +
+
+
+
+ + + + What the line item paces against. Budget uses the package budget; Impressions + uses the cap derived from budget ÷ rate. Both figures are sent either way. + +
+
Line-item Defaults + placeholder="Optional — defaults to Evenly"> + + Line-item pacing. Left blank, line items book with the platform's + Evenly schedule. +
From 35aebd6881183032477094166fc05ff55b3e02b1 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Mon, 10 Aug 2026 19:20:55 +0600 Subject: [PATCH 69/90] fix: improvement of slim_schemas for better token utilization during media buy --- core/main.py | 25 ++- src/core/slim_schemas.py | 367 ++++++++++++++++++++++++++++++++++----- 2 files changed, 347 insertions(+), 45 deletions(-) diff --git a/core/main.py b/core/main.py index abc4aadae8..648fb2bd9f 100644 --- a/core/main.py +++ b/core/main.py @@ -72,14 +72,27 @@ ) from adcp.server.spec_compat import _spec_compat_hooks_impl -from src.core.slim_schemas import CREATE_MEDIA_BUY_SLIM_SCHEMA, UPDATE_MEDIA_BUY_SLIM_SCHEMA +from src.core.slim_schemas import ( + CREATE_MEDIA_BUY_SLIM_SCHEMA, + GET_PRODUCTS_SLIM_SCHEMA, + SYNC_CREATIVES_SLIM_SCHEMA, + UPDATE_MEDIA_BUY_SLIM_SCHEMA, +) # Optionally replace large tool inputSchemas with compact versions. # -# adcp's _generate_pydantic_schemas() inlines all $refs, turning the -# CreateMediaBuyRequest schema into ~93 000 lines of JSON (~2.2 MB) and the -# UpdateMediaBuyRequest schema into an even larger blob (~4.2 MB). Either -# volume fills an LLM context window on tools/list, making the tool unusable. +# adcp's _generate_pydantic_schemas() inlines all $refs, so every model that +# carries creative assets explodes. Inlined size for the tools that make up +# one booking flow (~4 chars/token): +# +# update_media_buy ~4.2 MB (~1 045 000 tokens) +# create_media_buy ~2.2 MB (~547 000 tokens) +# sync_creatives ~1.9 MB (~480 000 tokens) +# get_products ~184 kB (~46 000 tokens) +# +# Any one of these fills an LLM context window on tools/list, making the tool +# unusable. Slimming all four takes the flow from ~1 092 000 to ~20 000 +# served tokens. # # Set ADCP_COMPACT_TOOL_SCHEMAS=true to activate the slim schemas. # When unset or false, the full adcp-generated schema is used (default). @@ -96,6 +109,8 @@ _SLIM_SCHEMAS = { "create_media_buy": CREATE_MEDIA_BUY_SLIM_SCHEMA, "update_media_buy": UPDATE_MEDIA_BUY_SLIM_SCHEMA, + "sync_creatives": SYNC_CREATIVES_SLIM_SCHEMA, + "get_products": GET_PRODUCTS_SLIM_SCHEMA, } for _tool in ADCP_TOOL_DEFINITIONS: _slim = _SLIM_SCHEMAS.get(_tool["name"]) diff --git a/src/core/slim_schemas.py b/src/core/slim_schemas.py index ae3ad86965..c602bf66b4 100644 --- a/src/core/slim_schemas.py +++ b/src/core/slim_schemas.py @@ -2,14 +2,29 @@ The adcp library generates input schemas from Pydantic models via ``_generate_pydantic_schemas()`` and then inlines all ``$ref`` nodes via -``_inline_refs()``. For ``create_media_buy`` this produces ~93 000 lines of -JSON that fills an LLM context window before any useful work can happen. -``update_media_buy`` is even larger (~4.2 MB inlined vs ~2.2 MB). +``_inline_refs()``. Because every asset-bearing model inlines the full +creative asset union, the result is enormous. Approximate served cost of +one booking flow (tokens, at ~4 chars/token) before and after slimming: + +===================== ============ =========== +tool full slim +===================== ============ =========== +sync_creatives 480 036 ~1 000 +create_media_buy 546 896 ~1 300 +update_media_buy 1 045 087 ~1 300 +get_products 45 972 ~1 000 +===================== ============ =========== + +Either volume alone fills an LLM context window on ``tools/list`` before any +useful work can happen. This module provides hand-crafted replacements that: * Cover every **required** field (enforced by ``test_slim_schema_guard.py``). * Include the most commonly used optional fields (ranked by test-corpus usage). * Stay under ~100 lines so the schema is a negligible context cost. +* Advertise **only** fields the request model actually accepts — a property + with no matching model field is not merely dead weight, it makes agents + send it and get rejected by ``mcp_compat_middleware`` outside production. Runtime behaviour is unchanged — the slim schema only affects what MCP clients see during tool discovery; the underlying function still validates @@ -60,10 +75,10 @@ "description": "Campaign end: ISO 8601 datetime.", }, # ── common optional (top-level) ──────────────────────────────────── - "name": { - "type": "string", - "description": "Human-readable campaign name.", - }, + # NOTE: no `name` field. CreateMediaBuyRequest has no `name`, so + # advertising one makes agents send it and get hard-rejected by + # mcp_compat_middleware ("Unknown field(s) for create_media_buy: name") + # outside production. Campaign naming is derived server-side. "po_number": { "type": "string", "description": "Purchase order number for tracking.", @@ -174,20 +189,13 @@ "creatives": { "type": "array", "description": ( - "Inline creative assets to upload and assign to this package (one-shot path). " - "Two approaches:\n" - " 1. One-shot: include creatives here with format_id or format_kind + assets.\n" - " 2. Two-step: call sync_creatives first, then reference creative_id here " - " (omit format_id/format_kind/assets).\n" - "format_id vs format_kind: use format_id {agent_url, id} when you need to " - "reference a named format from a specific creative agent. " - "Always call list_creative_formats first to discover the correct agent_url " - "(returned in creative_agents[].agent_url — typically " - "'https://creative.adcontextprotocol.org/'). " - "Use format_kind (enum string) for the simpler canonical-format path. " - "format_id and format_kind are mutually exclusive.\n" - "assets keys are slot names from the format (e.g. banner_image, click_url). " - "banner_image: {asset_type:'image', url, width, height}. " + "Inline creatives (one-shot upload + assign). Alternatively call " + "sync_creatives first and pass only creative_id + name here.\n" + "Supply exactly one of format_id or format_kind. Reuse the " + "format_id verbatim from the chosen product's format_ids " + "(get_products) — no list_creative_formats call needed.\n" + "assets keys are the format's slot names, e.g. " + "banner_image: {asset_type:'image', url, width, height}; " "click_url: {asset_type:'url', url, url_type:'clickthrough'}." ), "items": { @@ -196,30 +204,20 @@ "properties": { "creative_id": {"type": "string"}, "name": {"type": "string"}, - # format_id: legacy named-format path — agent_url discovered via - # list_creative_formats creative_agents[].agent_url + # format_id: legacy named-format path. Copy the object + # straight out of product.format_ids[] — it already + # carries agent_url, id and any width/height parameters. "format_id": { "type": "object", "description": ( - "Named-format path. Always {agent_url, id}. " - "agent_url MUST be discovered from list_creative_formats " - "response's creative_agents[].agent_url " - "(e.g. 'https://creative.adcontextprotocol.org/'). " - "Mutually exclusive with format_kind." + "Copy an entry from the product's format_ids[] as-is, " + "including width/height when present." ), "properties": { - "agent_url": { - "type": "string", - "description": ( - "URL of the agent that owns this format. " - "Discover via list_creative_formats creative_agents[].agent_url. " - "Example: 'https://creative.adcontextprotocol.org/'" - ), - }, - "id": { - "type": "string", - "description": "Format ID, e.g. 'display_300x250'.", - }, + "agent_url": {"type": "string"}, + "id": {"type": "string"}, + "width": {"type": "integer"}, + "height": {"type": "integer"}, }, "required": ["agent_url", "id"], }, @@ -272,6 +270,295 @@ } +# --------------------------------------------------------------------------- +# get_products +# --------------------------------------------------------------------------- +# Required fields (1): buying_mode +# `fields` is deliberately typed as a plain string array with `examples` +# rather than a hard enum: the enum has grown across adcp releases (30 -> 39 +# values), so inlining it risks advertising values the running server rejects +# with VALIDATION_ERROR. Unknown values are rejected, so prefer omitting +# `fields` entirely over guessing. +# --------------------------------------------------------------------------- + +GET_PRODUCTS_SLIM_SCHEMA: dict = { + "type": "object", + "required": ["buying_mode"], + "properties": { + # ── required ────────────────────────────────────────────────────── + "buying_mode": { + "type": "string", + "enum": ["brief", "wholesale", "refine"], + "description": ( + "'brief': publisher curates from a natural-language brief (requires brief). " + "'wholesale': raw product feed, no brief, returns no proposals. " + "'refine': iterate on a previous response (requires refine)." + ), + }, + # ── mode-specific ────────────────────────────────────────────────── + "brief": { + "type": "string", + "description": ( + "Natural-language campaign requirements. Required when " + "buying_mode='brief'; must be omitted for 'wholesale' and 'refine'." + ), + }, + "refine": { + "type": "array", + "description": ( + "Change requests against a previous response. Only valid when " + "buying_mode='refine'. Also the way to fetch a known product_id." + ), + "items": { + "type": "object", + "properties": { + "scope": { + "type": "string", + "enum": ["request", "product", "proposal"], + }, + "product_id": {"type": "string"}, + "proposal_id": {"type": "string"}, + "action": { + "type": "string", + "description": ( + "product scope: include|omit|more_like_this. " + "proposal scope: include|omit|finalize. " + "'finalize' must be the only action in the array." + ), + }, + "ask": {"type": "string", "description": "What to change."}, + }, + "required": ["scope"], + }, + }, + # ── common optional ──────────────────────────────────────────────── + "account": { + "type": "object", + "description": ( + "Account for account-specific rate-card pricing. " + "Either {account_id: str} or {brand: {domain: str}, operator: str}." + ), + }, + "brand": { + "type": "object", + "description": "Brand reference for discovery context: {domain: 'example.com'}.", + }, + "fields": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Response projection — omit to get all fields. Unknown values are " + "rejected with VALIDATION_ERROR, so omit rather than guess. " + "product_id and name are always returned." + ), + "examples": [ + [ + "product_id", + "name", + "channels", + "format_ids", + "pricing_options", + "delivery_type", + ] + ], + }, + "filters": { + "type": "object", + "description": "Narrow the catalog. Non-matching products are silently excluded.", + "properties": { + "channels": { + "type": "array", + "items": {"type": "string"}, + "description": "e.g. ['display', 'ctv', 'olv', 'streaming_audio'].", + }, + "delivery_type": { + "type": "string", + "enum": ["guaranteed", "non_guaranteed"], + }, + "is_fixed_price": {"type": "boolean"}, + "countries": { + "type": "array", + "items": {"type": "string"}, + "description": "ISO 3166-1 alpha-2, e.g. ['NL', 'DE'].", + }, + "pricing_currencies": { + "type": "array", + "items": {"type": "string"}, + "description": "ISO 4217, e.g. ['EUR'].", + }, + "start_date": {"type": "string", "format": "date"}, + "end_date": {"type": "string", "format": "date"}, + "budget_range": { + "type": "object", + "properties": { + "currency": {"type": "string"}, + "min": {"type": "number"}, + "max": {"type": "number"}, + }, + "required": ["currency"], + }, + }, + }, + "preferred_delivery_types": { + "type": "array", + "items": {"type": "string", "enum": ["guaranteed", "non_guaranteed"]}, + "description": "Preference hint (unlike filters.delivery_type, does not exclude).", + }, + "pagination": { + "type": "object", + "description": "Cursor pagination: {cursor?: str, max_results?: int (1-100, default 50)}.", + "properties": { + "cursor": {"type": "string"}, + "max_results": {"type": "integer"}, + }, + }, + }, +} + + +# --------------------------------------------------------------------------- +# sync_creatives +# --------------------------------------------------------------------------- +# Required fields (3): account, creatives, idempotency_key +# Only needed for the two-step creative path (upload to library, then pass +# creative_id to create_media_buy). The one-shot path — inline creatives in +# create_media_buy's packages[].creatives — skips this tool entirely. +# --------------------------------------------------------------------------- + +SYNC_CREATIVES_SLIM_SCHEMA: dict = { + "type": "object", + "required": ["account", "creatives", "idempotency_key"], + "properties": { + # ── required ────────────────────────────────────────────────────── + "account": { + "type": "object", + "description": ( + "Account that owns these creatives. Either {account_id: str} or " + "{brand: {domain: str}, operator: str, sandbox?: bool}." + ), + }, + "idempotency_key": { + "type": "string", + "description": ( + "Client-generated unique key (16-255 chars, alphanumeric + _.:-). " + "Re-send the same key to safely retry without syncing twice." + ), + }, + "creatives": { + "type": "array", + "description": ( + "Creatives to create or update (max 100). Idempotent per creative_id: " + "re-sending an existing creative_id updates it." + ), + "items": { + "type": "object", + "required": ["creative_id", "name", "assets"], + "properties": { + "creative_id": { + "type": "string", + "description": "Your stable identifier; the update key on re-sync.", + }, + "name": {"type": "string"}, + "format_id": { + "type": "object", + "description": ( + "Copy an entry from the target product's format_ids[] as-is, " + "including width/height when present. " + "Mutually exclusive with format_kind." + ), + "properties": { + "agent_url": {"type": "string"}, + "id": {"type": "string"}, + "width": {"type": "integer"}, + "height": {"type": "integer"}, + }, + "required": ["agent_url", "id"], + }, + "format_kind": { + "type": "string", + "description": "Canonical format name. Mutually exclusive with format_id.", + "enum": [ + "image", + "html5", + "display_tag", + "video_hosted", + "video_vast", + "audio_hosted", + "native_in_feed", + ], + }, + "assets": { + "type": "object", + "description": ( + "Slot values keyed by the format's slot names, e.g. " + "banner_image: {asset_type:'image', url, width, height}; " + "click_url: {asset_type:'url', url, url_type:'clickthrough'}." + ), + }, + "tags": { + "type": "array", + "items": {"type": "string"}, + "description": "Free-form tags for organisation and search.", + }, + }, + }, + }, + # ── common optional ──────────────────────────────────────────────── + "assignments": { + "type": "array", + "description": "Bulk-assign creatives to packages of an existing media buy.", + "items": { + "type": "object", + "required": ["creative_id", "package_id"], + "properties": { + "creative_id": {"type": "string"}, + "package_id": {"type": "string"}, + "weight": { + "type": "number", + "description": "Relative rotation weight 0-100. 0 = assigned but paused.", + }, + }, + }, + }, + "creative_ids": { + "type": "array", + "items": {"type": "string"}, + "description": ("Limit the sync to these creative_ids. Cannot be combined with delete_missing."), + }, + "validation_mode": { + "type": "string", + "enum": ["strict", "lenient"], + "description": ( + "'strict' (default) fails the whole sync on any error. " + "'lenient' processes valid creatives and reports the rest." + ), + }, + "dry_run": { + "type": "boolean", + "description": "Preview what would change without applying it.", + }, + "delete_missing": { + "type": "boolean", + "description": ( + "Archive every library creative absent from this call. Full-library replacement — use with care." + ), + }, + "push_notification_config": { + "type": "object", + "description": ( + "Webhook for async sync completion (large batches / manual review). " + "Provide {url, authentication: {schemes, credentials}}." + ), + "properties": { + "url": {"type": "string", "format": "uri"}, + "authentication": {"type": "object"}, + }, + "required": ["url"], + }, + }, +} + + # --------------------------------------------------------------------------- # update_media_buy # --------------------------------------------------------------------------- From 44fdcb3c242b2f6924b4e195bc5e90f9c947ef91 Mon Sep 17 00:00:00 2001 From: Chinmoy Acharjee Date: Wed, 12 Aug 2026 16:08:45 +0600 Subject: [PATCH 70/90] Inventory UI page (#60) * feat(improvedigital): adjust inventory page for improve digital * feat(improvedigital): fix inventory for improve digital and product targeting fix for editing product * feat(improvedigital): added inventory page --- .../inventory_browser_improvedigital.html | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 templates/inventory_browser_improvedigital.html diff --git a/templates/inventory_browser_improvedigital.html b/templates/inventory_browser_improvedigital.html new file mode 100644 index 0000000000..7fa24813ff --- /dev/null +++ b/templates/inventory_browser_improvedigital.html @@ -0,0 +1,297 @@ +{% extends "base.html" %} + +{% block title %}Browse Inventory - {{ tenant.name }} Sales Agent{% endblock %} + +{% block content %} + + +
+
+
+

Browse Inventory

+
+ Improve Digital 360Yield inventory for {{ tenant.name }} — + checking last sync… +
+
+
+ + +
+
+ +
+
–
Publishers
+
–
Placements
+
–
Packages
+
–
Creative Sizes
+
+ + + + +
+ + +{% endblock %} From bb4d26475435b9f9c8e7361f83bf7d0d241651c3 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Thu, 13 Aug 2026 16:40:30 +0600 Subject: [PATCH 71/90] postman update --- docs/adapters/improvedigital/api-requests.md | 443 ++++++++++++++++++ ...igital-marketplace.postman_collection.json | 272 ++++++++++- src/adapters/improvedigital/adapter.py | 4 +- 3 files changed, 703 insertions(+), 16 deletions(-) create mode 100644 docs/adapters/improvedigital/api-requests.md diff --git a/docs/adapters/improvedigital/api-requests.md b/docs/adapters/improvedigital/api-requests.md new file mode 100644 index 0000000000..772e49912c --- /dev/null +++ b/docs/adapters/improvedigital/api-requests.md @@ -0,0 +1,443 @@ +# Improve Digital (360Yield) — every request the adapter sends + +Complete inventory of the HTTP calls `src/adapters/improvedigital/` makes, +with the exact bodies we build, for cross-checking against the platform's +API documentation. + +Every request below has a runnable counterpart in +`improvedigital-marketplace.postman_collection.json` (same folder), with the +same bodies. + +Host comes from the tenant's `api_base_url` (`https://api.360yield.com` +production, `https://api.360yielddev.com` dev, `https://api-alpha.360yielddev.com` +alpha). Payload examples use the dev identity: demand contact `15917`, +buying entity `421`, office `5068`, business unit `33`, buyer `30`. + +## Contents + +| Flow | Trigger | Section | +|---|---|---| +| Auth | every call | [1](#1-authentication) | +| Permission probes | Test Connection, permissions report | [2](#2-permission-probes-read-only) | +| Adapter settings discovery | Admin UI buttons | [3](#3-admin-ui-discovery) | +| Booking | media buy approve | [4](#4-booking--create_media_buy) | +| Creatives | creative upload/assignment | [5](#5-creatives) | +| Buy updates | pause/resume/budget/archive | [6](#6-update_media_buy) | +| Status | delivery polling | [7](#7-status) | +| Reporting | reporting sync scheduler | [8](#8-reporting-sync) | +| Inventory | "Sync Inventory Now" + scheduler | [9](#9-inventory-sync) | + +Every request carries `Authorization: Bearer ` and +`accept: application/json`; JSON bodies add `Content-Type: application/json`. +A 401 re-mints the bearer and retries once; a 429 sleeps 61s and retries +(two attempts, at +20s and +65s). Default timeout 30s, 90s for booking. + +--- + +## 1. Authentication + +`src/adapters/improvedigital/_transport.py` + +### Mint a bearer + +``` +POST /oauth/token +Authorization: Basic base64(client_id:client_secret) +Content-Type: application/x-www-form-urlencoded + +grant_type=client_credentials +``` + +Bearer is read from the response's `value` field (not `access_token`), TTL +from `expiresIn`. Re-minted 2 minutes before expiry; there is no refresh +token. + +### Release a bearer (best effort, on client close) + +``` +DELETE /oauth/logout/{token} +``` + +--- + +## 2. Permission probes (read-only) + +`check_permissions()` — one GET per adapter concern, non-raising, no bodies. + +| Method | Path | Concern | Required | +|---|---|---|---| +| GET | `/rtb/v1/classic/campaigns?limit=1` | media buy lifecycle | yes | +| GET | `/rtb/v1/classic/line-items?limit=1` | media buy lifecycle | yes | +| GET | `/rtb/v3/placements?limit=1` | inventory | yes | +| GET | `/schema/rtb/v1/classic/campaigns/campaign` | create_media_buy | yes | +| GET | `/rtb/v1/sizes-all` | creative formats | no | + +--- + +## 3. Admin UI discovery + +`src/admin/blueprints/adapters.py` + +| Button | Requests | +|---|---| +| **Test Connection** | `GET /rtb/v1/classic/campaigns?limit=1`, then `GET /lookup/v1/user-details` | +| **Discover from API** (entities) | `GET /admin/v1/buying-entities-combo?limit=100&offset=N` (paginated) | +| **Discover from API** (offices) | `GET /admin/v1/buying-entities/{buying_entity_id}/buying-entity-offices?limit=100&offset=N` | +| **Expand a package** (product config) | `GET /rtb/v1/packages/{package_id}/placements` | + +`user-details` supplies `user_id` (→ `improve_demand_contact_id`), +`business_unit_id`, and the buyer list behind the Buyer ID picker. Office +rows are filtered to `active == true` and `"Classic" in buying_types`. + +--- + +## 4. Booking — `create_media_buy` + +Fires on media buy approval, in this order. Any failure triggers cleanup +(§4.6). + +### 4.1 Create the campaign + +``` +POST /rtb/v1/classic/campaigns +``` +```json +{ + "name": "adcp_PO-12345", + "type": "Improve", + "start_date": "2026-08-10 10:20:28", + "end_date": "2026-09-10 23:59:59", + "time_zone": "Europe/Amsterdam", + "currency": "EUR", + "improve_demand_contact_id": 15917, + "buying_entity_id": 421, + "advertiserId": null, + "buying_entity_office_id": 5068, + "buying_entity_office_ids": [5068], + "agencyId": 123 +} +``` + +- `name` is `adcp_`, falling back to `adcp_`. +- `buying_entity_office_id` / `buying_entity_office_ids` and `agencyId` are + omitted when not configured. +- `advertiserId` is the principal's platform mapping, but only when it is + numeric — metadata-advertiser UUIDs are sent as `null`. +- The response `id` becomes the media buy's external ID + (`improvedigital_`). + +### 4.2 Create one line item per package + +``` +POST /rtb/v1/classic/campaigns/{campaign_id}/line-items +``` +```json +{ + "name": "TEST_Valid_Flight_Budget_Aug26", + "type": "Standard", + "line_item_status": "Active", + "goal": "BUDGET", + "start_date": "2026-08-10 10:20:28", + "end_date": "2026-08-10 23:59:59", + "time_zone": "UTC", + "cpm_bid": 1.0, + "pricing_model": "CPM", + "pricing_model_type": "First Bid", + "impression_cap": 2000, + "impression_cap_daily": false, + "budget_is_daily": false, + "invoice_type": "on_actuals", + "delivery_schedule": "Evenly", + "third_party_inventory": true, + "is_optimised": true, + "is_dynamic_optimization": false, + "dynamic_optimization_kpi_type": "", + "dynamic_optimization_kpi_value": 0, + "conversion_tracking_enabled": false, + "keep_on_delivering": false, + "track_viewability": false, + "is_consentless": false, + "is_coppa_compliant": false, + "optout_mechanism": [], + "reference_number": "pkg_1", + "improve_demand_contact_id": 15917, + "budget": 2.0, + "flight_details": [ + { + "start_time": "2026-08-10 10:20:28", + "end_time": "2026-08-10 23:59:59", + "budget": 2.0, + "budget_is_daily": false, + "impression_cap": 2000, + "impression_cap_daily": false + } + ], + "business_unit_id": 33, + "buyer_id": 30, + "frequency_cap": 3, + "frequency_interval": 1, + "frequency_interval_type": "days", + "placement_ids": [98765], + "package_ids": [4321], + "size_ids": [4] +} +``` + +Field sources: + +| Field | Source | +|---|---| +| `name` | package name, falling back to package ID | +| `goal` | product config `goal` (default `BUDGET`) | +| `cpm_bid` | package pricing — `rate` when fixed, else `bid_price` | +| `budget` | package budget; when the buy carries budget only at buy level, derived as `impression_cap × cpm_bid ÷ 1000` | +| `impression_cap` | budget ÷ rate × 1000, computed by the core layer before dispatch | +| `pricing_model` | product config, else `CPM` (`FLAT_RATE` for flat-rate pricing) | +| `delivery_schedule`, `frequency_*` | product config; `delivery_schedule` defaults to `Evenly`, the frequency keys are omitted when unset | +| `reference_number` | our internal package ID, for reconciliation | +| `improve_demand_contact_id`, `business_unit_id`, `buyer_id`, `time_zone` | tenant adapter config (`business_unit_id`/`buyer_id` omitted when unset) | +| `placement_ids`, `package_ids`, `size_ids` | product config | + +Notes for cross-checking: + +- **No `currency`** — the platform requires it to match the campaign owner's + default and 400s when sent. +- **No `id` / `campaign_id`** in the body: `id` is server-assigned and the + campaign is already in the path. The platform's own UI sends `id: 0` and a + body `campaign_id`; add them if the API validates them. +- `flight_details` is **not** in the committed `rtb-v3-openapi.json` — the + closest documented fields are `custom_budgets` and `daily_impression_caps`. +- `dynamic_optimization_kpi_value` is typed `string` in that spec but sent as + the number `0`, matching the platform's own payload. +- `placement_ids` / `package_ids` / `size_ids` are not part of + `CommonDealLineItemDto`; the server ignores them and the real assignment + happens in §4.3–4.4. +- `frequency_interval_type` must be one of the spec's lowercase values: + `months`, `weeks`, `days`, `hours`, `minutes`. + +### 4.3 Assign placements (when the product pins any) + +``` +PUT /rtb/v1/classic/campaigns/{campaign_id}/line-items/{line_item_id}/placements +``` +```json +{"line_item_placements": [{"id": 98765, "assigned": true}, {"id": 98766, "assigned": true}]} +``` + +### 4.4 Assign packages (when the product pins any) + +``` +PUT /rtb/v1/classic/campaigns/{campaign_id}/line-items/{line_item_id}/packages +``` +```json +{"line_item_packages": [{"id": 4321, "assigned": true}]} +``` + +A package with neither placements nor packages aborts the booking — the line +item would target no inventory. + +### 4.5 Geo targeting (only when the buy or product sets geo) + +Resolving platform geo names first, once per adapter instance: + +``` +GET /rtb/v1/regions?limit=100&offset=N +GET /rtb/v1/regions/{regionName}/countries?limit=100&offset=N +``` + +Then, per line item: + +``` +PUT /rtb/v1/classic/campaigns/{campaign_id}/line-items/{line_item_id}/geo-targeting +``` +```json +{ + "filter": true, + "geo_targeting": [ + {"country": "Netherlands", "exclude": false, "region": "EMEA"}, + {"country": "Belgium", "exclude": true, "region": "EMEA"} + ] +} +``` + +`exclude` **and** `region` are required on every entry, includes included. +Country tokens (ISO alpha-2 from the buyer, or platform names from the +product) are rewritten to the platform's exact spelling; an unresolvable +token fails the booking rather than being dropped. + +### 4.6 Cleanup after a partial failure + +``` +DELETE /rtb/v1/classic/campaigns/{campaign_id} +PUT /rtb/v3/campaigns/{campaign_id}/archive # fallback when delete is refused +``` + +Delete is refused once the campaign has served impressions. + +--- + +## 5. Creatives + +### 5.1 Resolve the platform size (cached per adapter instance) + +``` +GET /rtb/v1/sizes-all +``` + +Matched on width × height; the resolved `name` + `id` are added to the +creative body. A bare `"300x250"` string is rejected by the platform. + +### 5.2a Tag creatives — plain JSON bulk upload + +``` +POST /rtb/v1/classic/campaign/creatives/third-party-tag/bulk-upload +``` +```json +{ + "campaign_id": 55501, + "creative_type": "Third Party Tag", + "creatives": [ + { + "name": "banner_300x250", + "size": "300x250", + "size_id": 4, + "width": 300, + "height": 250, + "status": "Active", + "tag": "", + "advertiser_domain": "example.com", + "third_party_type": "display", + "platform_types": ["Web"], + "tag_secure": true + } + ] +} +``` + +Hosted-image assets are wrapped into an `` (optionally inside an ``) +before being sent as a tag — the endpoint validates that the tag is real +HTML. + +### 5.2b Other creative types — multipart servlet + +``` +POST /rtb/v1/classic/campaigns/{campaign_id}/creatives +Content-Type: multipart/form-data +``` + +The `CreativeDto` travels as a `body` part with content type +`application/json`; same fields minus `third_party_type` / `platform_types` / +`tag_secure`. Type is derived from the asset: `VAST Audio`, `VAST`, `Native`, +otherwise `Third Party Tag`. Plain JSON to this path returns HTTP 500. + +### 5.3 Recover a creative ID (only when the create response carries none) + +``` +GET /rtb/v1/classic/campaigns/{campaign_id}/creatives +``` + +Matched by name, highest ID wins. + +### 5.4 Bind creatives to a line item + +``` +PUT /rtb/v1/classic/campaigns/{campaign_id}/line-items/{line_item_id}/creatives +``` +```json +{"line_item_creatives": [{"id": 778899, "assigned": true}]} +``` + +One call per line item, assigning every creative. Can take >30s on dev. + +--- + +## 6. `update_media_buy` + +| Action | Requests | +|---|---| +| `pause_media_buy` / `resume_media_buy` / `activate_order` | `GET /rtb/v1/classic/campaigns/{cid}/line-items`, then `PUT /rtb/v3/campaigns/{cid}/line-items/{lid}/status?active=true\|false` per line item | +| `pause_package` / `resume_package` | `PUT /rtb/v3/campaigns/{cid}/line-items/{lid}/status?active=true\|false` | +| `update_package_budget` | `GET /rtb/v1/classic/campaigns/{cid}/line-items/{lid}`, mutate `budget`, `PUT` the full DTO back to the same path | +| `update_package_impressions` | same read-modify-write, mutating `impression_cap` | +| `archive_order` | `PUT /rtb/v3/campaigns/{cid}/archive` | +| `submit_for_approval` / `approve_order` | **no request** — Classic campaigns have no approval workflow | + +Status toggles are query-param PUTs with no body. Budget updates are +read-modify-write because the platform's PUT expects the complete DTO. + +--- + +## 7. Status + +``` +GET /rtb/v1/classic/campaigns/{campaign_id} +``` + +`get_media_buy_delivery` makes **no** API call — it aggregates the local +`improvedigital_line_item_stats` cache filled by the reporting sync, and +raises `DeliveryDataUnavailable` while that cache is empty. + +--- + +## 8. Reporting sync + +``` +POST /report/ext/preview +``` +```json +{ + "rows": 500, + "report_generation_request": { + "title": "", + "report_type": "EXT_CONSOLIDATE", + "currency_id": 1, + "date_range": {"quick": "LAST_31_DAYS"}, + "dimensions": ["campaign_id", "line_item_id"], + "metrics": ["impressions", "clicks", "advertiser_payout", "complete"], + "filters": [{"column": "campaign_id", "operation": "IN", "value": [314417]}], + "timezone": "UTC", + "action": "PREVIEW_REPORT" + } +} +``` + +- Snake_case only — the camelCase shape in the OpenAPI spec deserializes to + a null request (HTTP 500). +- `currency_id`: 1 = EUR, 2 = USD. +- `filters.value` holds the campaign IDs of the tenant's active buys. +- Capped at 500 rows; a full page logs a truncation warning. The async + `/report/ext/generation` path is not wired up yet. + +--- + +## 9. Inventory sync + +Paginated sweeps, 1000 rows per page, up to 500 pages per family: + +``` +GET /rtb/v3/placements?offset=N&limit=1000 +GET /rtb/v1/packages?offset=N&limit=1000 +GET /rtb/v1/sizes-all?offset=N&limit=1000 +``` + +Publishers are derived from the inline `publisher_id` / `publisher_name` on +placement rows — there is no buy-side publishers endpoint. Package +membership is deliberately **not** swept (2k+ packages × one request each +would exhaust the 100-reads/60s quota); it is fetched on demand when an +operator expands a package (§3). + +--- + +## Client methods with no caller + +Defined in `client.py` but not reached by any adapter or admin flow today — +useful when scoping which endpoints actually need API permissions: + +`list_campaigns`, `delete_line_item`, `list_all_line_items`, +`archive_line_item`, `assign_placements` / `unassign_placements` (the v2 +`/placements/assign` surface — booking uses the v1 `/placements` PUT +instead), `list_placements`, `get_creative`, `update_creative`, +`delete_creative`, `set_creative_status`, `list_line_item_creatives`, +`unassign_all_creatives`, `validate_vast_url`, `creative_type_sizes`, +`countries` (`/common/v1/countries`), `submit_generation`, +`generation_status`, `allowed_filters`. diff --git a/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json b/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json index 360c2c9f7b..0ec48edf5a 100644 --- a/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json +++ b/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json @@ -1,7 +1,7 @@ { "info": { "name": "Improve Digital Marketplace — Direct (Classic) Campaign Booking", - "description": "End-to-end direct-campaign booking against the 360Yield Marketplace API (beta-deals.360yielddev.com / direct-campaigns).\n\nEvery request in this collection was validated live against the dev platform (2026-08-04) — the payloads encode wire requirements the OpenAPI spec gets wrong or omits (see docs/adapters/improvedigital/live-wire-shapes.md in the salesagent repo).\n\nSETUP\n1. Set collection variables `client_id` and `client_secret` (OAuth2 client credentials issued by Improve Digital).\n2. Confirm the account variables (`demand_contact_id`, `buying_entity_id`, `buying_entity_office_id`, `business_unit_id`) — the defaults are the dev-platform values for the 'AdCP API Acceptance' client. On another account, run `0.2 Who am I` and copy the values from an existing campaign (`2.4 List campaigns`).\n3. Run the folders top to bottom: 1 auth → 2 discovery → 3 booking → 4 lifecycle → 5 reporting → 6 cleanup. Requests chain automatically: campaign_id, line_item_id, placement_id and creative_id are captured into collection variables by test scripts.\n\nAUTH\nBearer tokens live ~30 minutes and there is NO refresh token. A collection-level pre-request script re-mints automatically when the cached token is missing or expiring, so you normally never need to run the token request by hand.\n\nRATE LIMIT\nThe API allows 100 read requests per 60s — heavy pagination (placement sweeps) will hit 429; wait out the window.", + "description": "End-to-end direct-campaign booking against the 360Yield Marketplace API (beta-deals.360yielddev.com / direct-campaigns).\n\nThis collection mirrors what the salesagent Improve Digital adapter sends — every endpoint it calls has a request here, and the booking bodies are the adapter's own payloads. The full request inventory with field-by-field sources lives at docs/adapters/improvedigital/api-requests.md; wire quirks the OpenAPI spec gets wrong are catalogued in docs/adapters/improvedigital/live-wire-shapes.md.\n\nSETUP\n1. Set collection variables `client_id` and `client_secret` (OAuth2 client credentials issued by Improve Digital).\n2. Confirm the account variables (`demand_contact_id`, `buying_entity_id`, `buying_entity_office_id`, `business_unit_id`, `buyer_id`) — the defaults are the dev-platform values for the 'AdCP API Acceptance' client. On another account, run `2.1 Who am I`, or discover them with `2.10` / `2.11`, or copy them from a working campaign (`2.4 List campaigns`).\n3. Run the folders top to bottom: 1 auth → 2 discovery → 3 booking → 4 lifecycle → 5 reporting → 6 cleanup. Requests chain automatically: campaign_id, line_item_id, placement_id, package_id and creative_id are captured into collection variables by test scripts.\n4. In folder 3, steps 3.4 (packages) and 3.5 (geo targeting) are optional — the adapter sends them only when the product config asks for them. 3.3 or 3.4 is mandatory: a line item with no inventory serves nowhere.\n\nAUTH\nBearer tokens live ~30 minutes and there is NO refresh token. A collection-level pre-request script re-mints automatically when the cached token is missing or expiring, so you normally never need to run the token request by hand.\n\nRATE LIMIT\nThe API allows 100 read requests per 60s — heavy pagination (placement sweeps) will hit 429; wait out the window.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "auth": { @@ -51,13 +51,17 @@ { "key": "buying_entity_id", "value": "421", "type": "string", "description": "421 = 'Improve Digital Marketplace' on dev" }, { "key": "buying_entity_office_id", "value": "5068", "type": "string", "description": "Buyer seat — at least one office is required on campaigns" }, { "key": "business_unit_id", "value": "33", "type": "string", "description": "33 = Azerion on dev; required on line items" }, + { "key": "buyer_id", "value": "30", "type": "string", "description": "Buyer the line items book under (see 2.1 Who am I)" }, { "key": "campaign_name", "value": "postman-demo-campaign", "type": "string" }, { "key": "cpm_bid", "value": "2.5", "type": "string" }, { "key": "impression_cap", "value": "10000", "type": "string" }, + { "key": "budget", "value": "25", "type": "string", "description": "impression_cap x cpm_bid / 1000 — keep the two consistent" }, { "key": "campaign_id", "value": "", "type": "string", "description": "Auto-filled by 3.1" }, { "key": "line_item_id", "value": "", "type": "string", "description": "Auto-filled by 3.2" }, { "key": "placement_id", "value": "", "type": "string", "description": "Auto-filled by 2.2 (first 300x250-capable placement)" }, - { "key": "creative_id", "value": "", "type": "string", "description": "Auto-filled by 3.4" } + { "key": "package_id", "value": "", "type": "string", "description": "Auto-filled by 2.6 (first reusable placement package)" }, + { "key": "region_name", "value": "EMEA", "type": "string", "description": "Geo region for 2.8 / 3.5 — see 2.7" }, + { "key": "creative_id", "value": "", "type": "string", "description": "Auto-filled by 3.6" } ], "item": [ { @@ -175,6 +179,117 @@ "url": { "raw": "{{base_url}}/common/v1/i18n/creative_type", "host": ["{{base_url}}"], "path": ["common", "v1", "i18n", "creative_type"] }, "description": "Canonical creative type names: Third Party Tag, Image (PNG, GIF, JPG), Html5, VAST, VAST_VPAID, Video File, Native, Ad Builder, Raw Video, VAST Audio. Creative create resolves types by exact name." } + }, + { + "name": "2.6 List placement packages", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const rows = pm.response.json().packages || [];", + "if (rows.length) {", + " pm.collectionVariables.set('package_id', String(rows[0].id));", + " console.log('package_id =', rows[0].id, rows[0].name);", + "}" + ] + } + } + ], + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/rtb/v1/packages?limit=10&offset=0", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "packages"], + "query": [ + { "key": "limit", "value": "10" }, + { "key": "offset", "value": "0" } + ] + }, + "description": "Reusable placement groupings — an alternative to pinning individual placements. The salesagent inventory sync sweeps this endpoint at limit=1000. Saves {{package_id}} for 3.4." + } + }, + { + "name": "2.7 Peek inside a package", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/rtb/v1/packages/{{package_id}}/placements?limit=20&offset=0", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "packages", "{{package_id}}", "placements"], + "query": [ + { "key": "limit", "value": "20" }, + { "key": "offset", "value": "0" } + ] + }, + "description": "Live membership of one package. Deliberately NOT part of the inventory sync — 2k+ packages x one request each would exhaust the 100-reads/60s quota; salesagent fetches it on demand when an operator expands a package in the product config UI." + } + }, + { + "name": "2.8 List geo regions", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/rtb/v1/regions?limit=100&offset=0", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "regions"], + "query": [ + { "key": "limit", "value": "100" }, + { "key": "offset", "value": "0" } + ] + }, + "description": "Geo region dictionary (`{\"regions\": [{\"name\": ...}]}`). The platform's region dimension is continental (EMEA / APAC / ...), NOT ISO 3166-2 subdivisions. Needed because every geo-targeting entry must carry its region (3.5)." + } + }, + { + "name": "2.9 List countries in a region", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/rtb/v1/regions/{{region_name}}/countries?limit=100&offset=0", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "regions", "{{region_name}}", "countries"], + "query": [ + { "key": "limit", "value": "100" }, + { "key": "offset", "value": "0" } + ] + }, + "description": "Country dictionary per region (`{\"countries\": [{\"name\": ...}]}`). Geo targeting matches on the platform's exact display names, so ISO alpha-2 codes from buyers must be resolved through 2.8 + 2.9 before they can be sent." + } + }, + { + "name": "2.10 List buying entities (admin scope)", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/admin/v1/buying-entities-combo?limit=100&offset=0", + "host": ["{{base_url}}"], + "path": ["admin", "v1", "buying-entities-combo"], + "query": [ + { "key": "limit", "value": "100" }, + { "key": "offset", "value": "0" } + ] + }, + "description": "Lightweight {id, name} rows backing the Buying Entity picker in the salesagent adapter settings page. Requires admin-scoped credentials — 403 otherwise, in which case the IDs are entered manually." + } + }, + { + "name": "2.11 List buying entity offices (admin scope)", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/admin/v1/buying-entities/{{buying_entity_id}}/buying-entity-offices?limit=100&offset=0", + "host": ["{{base_url}}"], + "path": ["admin", "v1", "buying-entities", "{{buying_entity_id}}", "buying-entity-offices"], + "query": [ + { "key": "limit", "value": "100" }, + { "key": "offset", "value": "0" } + ] + }, + "description": "Office rows carry `improve_demand_contact_id`, `billing_currency_code` and `buying_types`. Only offices with `active: true` and 'Classic' in `buying_types` can host a direct campaign — that is the filter the salesagent office picker applies." + } } ] }, @@ -254,14 +369,14 @@ "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", - "raw": "{\n \"name\": \"{{campaign_name}}-li\",\n \"type\": \"Standard\",\n \"line_item_status\": \"Active\",\n \"goal\": \"IMPRESSION\",\n \"start_date\": \"{{start_date}}\",\n \"end_date\": \"{{end_date}}\",\n \"time_zone\": \"Europe/Amsterdam\",\n \"pricing_model\": \"CPM\",\n \"cpm_bid\": {{cpm_bid}},\n \"impression_cap\": {{impression_cap}},\n \"improve_demand_contact_id\": {{demand_contact_id}},\n \"business_unit_id\": {{business_unit_id}}\n}" + "raw": "{\n \"name\": \"{{campaign_name}}-li\",\n \"type\": \"Standard\",\n \"line_item_status\": \"Active\",\n \"goal\": \"BUDGET\",\n \"start_date\": \"{{start_date}}\",\n \"end_date\": \"{{end_date}}\",\n \"time_zone\": \"Europe/Amsterdam\",\n \"cpm_bid\": {{cpm_bid}},\n \"pricing_model\": \"CPM\",\n \"pricing_model_type\": \"First Bid\",\n \"impression_cap\": {{impression_cap}},\n \"impression_cap_daily\": false,\n \"budget\": {{budget}},\n \"budget_is_daily\": false,\n \"flight_details\": [\n {\n \"start_time\": \"{{start_date}}\",\n \"end_time\": \"{{end_date}}\",\n \"budget\": {{budget}},\n \"budget_is_daily\": false,\n \"impression_cap\": {{impression_cap}},\n \"impression_cap_daily\": false\n }\n ],\n \"invoice_type\": \"on_actuals\",\n \"delivery_schedule\": \"Evenly\",\n \"third_party_inventory\": true,\n \"is_optimised\": true,\n \"is_dynamic_optimization\": false,\n \"dynamic_optimization_kpi_type\": \"\",\n \"dynamic_optimization_kpi_value\": 0,\n \"conversion_tracking_enabled\": false,\n \"keep_on_delivering\": false,\n \"track_viewability\": false,\n \"is_consentless\": false,\n \"is_coppa_compliant\": false,\n \"optout_mechanism\": [],\n \"reference_number\": \"{{campaign_name}}-pkg\",\n \"improve_demand_contact_id\": {{demand_contact_id}},\n \"business_unit_id\": {{business_unit_id}},\n \"buyer_id\": {{buyer_id}}\n}" }, "url": { "raw": "{{base_url}}/rtb/v1/classic/campaigns/{{campaign_id}}/line-items", "host": ["{{base_url}}"], "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}", "line-items"] }, - "description": "Required (validated live): `type: \"Standard\"`, `line_item_status: \"Active\"`, `goal` for CPM items (IMPRESSION or BUDGET), `business_unit_id`, `improve_demand_contact_id`. Do NOT send `currency` — it must match the campaign owner's default and 400s otherwise; it inherits from the campaign when omitted. Saves {{line_item_id}}." + "description": "This is the exact body the salesagent adapter sends (src/adapters/improvedigital/adapter.py::_line_item_payload) — see docs/adapters/improvedigital/api-requests.md.\n\nRequired (validated live): `type: \"Standard\"`, `line_item_status: \"Active\"`, `goal` (BUDGET or IMPRESSION), `business_unit_id`, `improve_demand_contact_id`.\n\nBUDGET pacing needs `budget` plus a `flight_details` row spanning the flight; both the money figure and the impression cap are sent either way, and they must stay consistent (budget = impression_cap x cpm_bid / 1000).\n\nDo NOT send `currency` — it must match the campaign owner's default and 400s otherwise; it inherits from the campaign when omitted.\n\nThe delivery/compliance block (pricing_model_type, invoice_type, third_party_inventory, is_optimised, keep_on_delivering, track_viewability, is_consentless, is_coppa_compliant, conversion_tracking_enabled, optout_mechanism, dynamic_optimization_*) mirrors the platform's own create payload; the API leaves these unset rather than defaulted when omitted.\n\nCaveats to verify against the API docs: `flight_details` is absent from the committed rtb-v3 OpenAPI spec (which documents `custom_budgets` / `daily_impression_caps` instead), and `dynamic_optimization_kpi_value` is typed string there but sent as the number 0. `frequency_interval_type`, when used, must be lowercase (months/weeks/days/hours/minutes).\n\nSaves {{line_item_id}}." } }, { @@ -287,11 +402,63 @@ "host": ["{{base_url}}"], "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}", "line-items", "{{line_item_id}}", "placements"] }, - "description": "Explicit placement assignment — envelope `{\"line_item_placements\": [{id, assigned: true}]}`. Add more entries for more placements. (Reusable packages use the same pattern at `.../packages` with `line_item_packages`.)" + "description": "Explicit placement assignment — envelope `{\"line_item_placements\": [{id, assigned: true}]}`. Add more entries for more placements. A line item with neither placements (3.3) nor packages (3.4) serves nowhere, and the salesagent adapter treats that as a hard product-config error." + } + }, + { + "name": "3.4 Assign packages to line item (optional)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": ["pm.test('packages assigned (2xx)', () => pm.expect(pm.response.code).to.be.within(200, 299));"] + } + } + ], + "request": { + "method": "PUT", + "header": [{ "key": "Content-Type", "value": "application/json" }], + "body": { + "mode": "raw", + "raw": "{\n \"line_item_packages\": [\n { \"id\": {{package_id}}, \"assigned\": true }\n ]\n}" + }, + "url": { + "raw": "{{base_url}}/rtb/v1/classic/campaigns/{{campaign_id}}/line-items/{{line_item_id}}/packages", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}", "line-items", "{{line_item_id}}", "packages"] + }, + "description": "Reusable placement packages, same envelope shape as 3.3 but keyed `line_item_packages`. Run 2.6 first to fill {{package_id}}. Products can pin placements, packages, or both — the adapter sends whichever the product config lists." + } + }, + { + "name": "3.5 Set geo targeting (optional)", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": ["pm.test('geo targeting set (2xx)', () => pm.expect(pm.response.code).to.be.within(200, 299));"] + } + } + ], + "request": { + "method": "PUT", + "header": [{ "key": "Content-Type", "value": "application/json" }], + "body": { + "mode": "raw", + "raw": "{\n \"filter\": true,\n \"geo_targeting\": [\n { \"country\": \"Netherlands\", \"region\": \"{{region_name}}\", \"exclude\": false }\n ]\n}" + }, + "url": { + "raw": "{{base_url}}/rtb/v1/classic/campaigns/{{campaign_id}}/line-items/{{line_item_id}}/geo-targeting", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}", "line-items", "{{line_item_id}}", "geo-targeting"] + }, + "description": "Geo travels on its own endpoint, never in the line-item create body. Live-validated: `exclude` AND `region` are required on EVERY entry — includes too — despite the spec marking all Geo fields optional (400 'object has missing required properties [\"exclude\",\"region\"]'). Country values must be the platform's own display names from 2.9, not ISO codes. Location targeting reaches city level; there is no postal-code support." } }, { - "name": "3.4 Create creative (Third Party Tag)", + "name": "3.6 Create creative (Third Party Tag)", "event": [ { "listen": "test", @@ -325,7 +492,7 @@ } }, { - "name": "3.5 Bind creative to line item", + "name": "3.7 Bind creative to line item", "event": [ { "listen": "test", @@ -403,20 +570,20 @@ } }, { - "name": "4.5 Update line item (impression cap)", + "name": "4.5 Update line item (budget / impression cap)", "request": { "method": "PUT", "header": [{ "key": "Content-Type", "value": "application/json" }], "body": { "mode": "raw", - "raw": "{\n \"name\": \"{{campaign_name}}-li\",\n \"type\": \"Standard\",\n \"line_item_status\": \"Active\",\n \"goal\": \"IMPRESSION\",\n \"pricing_model\": \"CPM\",\n \"cpm_bid\": {{cpm_bid}},\n \"impression_cap\": 20000,\n \"improve_demand_contact_id\": {{demand_contact_id}},\n \"business_unit_id\": {{business_unit_id}}\n}" + "raw": "{\n \"name\": \"{{campaign_name}}-li\",\n \"type\": \"Standard\",\n \"line_item_status\": \"Active\",\n \"goal\": \"BUDGET\",\n \"pricing_model\": \"CPM\",\n \"cpm_bid\": {{cpm_bid}},\n \"budget\": 50,\n \"budget_is_daily\": false,\n \"impression_cap\": 20000,\n \"impression_cap_daily\": false,\n \"improve_demand_contact_id\": {{demand_contact_id}},\n \"business_unit_id\": {{business_unit_id}},\n \"buyer_id\": {{buyer_id}}\n}" }, "url": { "raw": "{{base_url}}/rtb/v1/classic/campaigns/{{campaign_id}}/line-items/{{line_item_id}}", "host": ["{{base_url}}"], "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}", "line-items", "{{line_item_id}}"] }, - "description": "PUT expects the full DTO — read-modify-write: GET the line item (4.2), change the field, PUT it back. This body is a minimal example; safest is to echo everything 4.2 returned with your change applied." + "description": "PUT expects the full DTO — read-modify-write: GET the line item (4.2), change the field, PUT it back. That is exactly what the adapter does for update_package_budget (`budget`) and update_package_impressions (`impression_cap`): it echoes the GET response with the one field replaced. This body is a minimal illustration; echoing everything 4.2 returned is safer." } } ] @@ -444,10 +611,87 @@ ] }, { - "name": "6. Cleanup", + "name": "6. Reference & probes", + "item": [ + { + "name": "6.1 Permission probe — classic line items", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/rtb/v1/classic/line-items?limit=1", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "classic", "line-items"], + "query": [{ "key": "limit", "value": "1" }] + }, + "description": "Account-wide line-item read. One of the five endpoints salesagent probes in check_permissions (with /rtb/v1/classic/campaigns, /rtb/v3/placements, /schema/rtb/v1/classic/campaigns/campaign and /rtb/v1/sizes-all) to report which adapter concerns the credentials can actually serve." + } + }, + { + "name": "6.2 Campaign create schema", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/schema/rtb/v1/classic/campaigns/campaign", + "host": ["{{base_url}}"], + "path": ["schema", "rtb", "v1", "classic", "campaigns", "campaign"] + }, + "description": "The server-side JSON schema for campaign create — the authority when the OpenAPI spec and the API disagree. Probed by check_permissions as the create_media_buy readiness signal." + } + }, + { + "name": "6.3 List campaign creatives", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/rtb/v1/classic/campaigns/{{campaign_id}}/creatives", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}", "creatives"] + }, + "description": "Used as a recovery path: when a creative create returns 200 with no usable id in the body, the adapter re-finds the creative here by name (highest id wins) so it can still be bound to a line item." + } + }, + { + "name": "6.4 Create creative (multipart — non-tag types)", + "request": { + "method": "POST", + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "body", + "type": "text", + "contentType": "application/json", + "value": "{\n \"name\": \"{{campaign_name}}-vast\",\n \"type\": \"VAST\",\n \"size\": \"300x250 (Medium Rectangle)\",\n \"size_id\": 4,\n \"width\": 300,\n \"height\": 250,\n \"status\": \"Active\",\n \"tag\": \"https://example.com/vast.xml\",\n \"advertiser_domain\": \"example.com\"\n}" + } + ] + }, + "url": { + "raw": "{{base_url}}/rtb/v1/classic/campaigns/{{campaign_id}}/creatives", + "host": ["{{base_url}}"], + "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}", "creatives"] + }, + "description": "Single-create is a multipart servlet: the CreativeDto travels as a `body` part with content type application/json (binary image parts would ride alongside). Plain JSON to this path returns 500 'Failed to parse multipart servlet request'. Tag creatives should use 3.6 instead. Do not set Content-Type by hand — Postman writes the multipart boundary." + } + }, + { + "name": "6.5 Logout (invalidate bearer)", + "request": { + "method": "DELETE", + "url": { + "raw": "{{base_url}}/oauth/logout/{{access_token}}", + "host": ["{{base_url}}"], + "path": ["oauth", "logout", "{{access_token}}"] + }, + "description": "Best-effort server-side token invalidation, called when the adapter closes its client. Tokens expire on their own within minutes, so failures here are ignored. Running this invalidates {{access_token}}; the collection pre-request script mints a fresh one on the next call." + } + } + ] + }, + { + "name": "7. Cleanup", "item": [ { - "name": "6.1 Delete campaign (hard)", + "name": "7.1 Delete campaign (hard)", "request": { "method": "DELETE", "url": { @@ -455,11 +699,11 @@ "host": ["{{base_url}}"], "path": ["rtb", "v1", "classic", "campaigns", "{{campaign_id}}"] }, - "description": "Refused with 400 once the campaign has served impressions (the dev platform attributes simulated impressions within seconds of activation) — use 6.2 then." + "description": "Refused with 400 once the campaign has served impressions (the dev platform attributes simulated impressions within seconds of activation) — use 7.2 then. This is exactly the adapter's cleanup path when a booking fails part-way: delete, then archive." } }, { - "name": "6.2 Archive campaign (soft)", + "name": "7.2 Archive campaign (soft)", "request": { "method": "PUT", "url": { diff --git a/src/adapters/improvedigital/adapter.py b/src/adapters/improvedigital/adapter.py index 57949e2eb0..62619c0dec 100644 --- a/src/adapters/improvedigital/adapter.py +++ b/src/adapters/improvedigital/adapter.py @@ -366,7 +366,7 @@ def create_media_buy( product_config = self._product_config_from_package(package) if product_config.get("placement_ids") or product_config.get("package_ids"): self.log( - "Would call: PUT .../line-items//placements/assign " + "Would call: PUT .../line-items//placements + .../packages " f"placement_ids={product_config.get('placement_ids', [])} " f"package_ids={product_config.get('package_ids', [])}" ) @@ -671,7 +671,7 @@ def _line_item_payload( payload: dict[str, Any] = { "name": package.name or package.package_id, "type": "Standard", - "line_item_status": "Active", + "line_item_status": "Inactive", "goal": goal, "start_date": self._format_datetime(start_time), "end_date": self._format_datetime(end_time), From ce99ccbdb86b4957afff36cf95cea6a36eae3737 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Mon, 17 Aug 2026 15:05:10 +0600 Subject: [PATCH 72/90] feat(improvedigital): attach campaign metadata to booked campaigns (#62) --- .mcp.json | 2 +- docs/adapters/improvedigital/api-requests.md | 52 +++++ ...igital-marketplace.postman_collection.json | 147 ++++++++++++++ src/adapters/improvedigital/adapter.py | 100 ++++++++++ src/adapters/improvedigital/client.py | 50 +++++ src/adapters/improvedigital/schemas.py | 48 ++++- src/admin/blueprints/adapters.py | 60 ++++++ src/core/helpers/adapter_helpers.py | 12 ++ .../improvedigital/connection_config.html | 185 ++++++++++++++++-- .../test_adapter_helpers_config_forwarding.py | 100 ++++++++++ 10 files changed, 735 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_adapter_helpers_config_forwarding.py diff --git a/.mcp.json b/.mcp.json index 17f7740115..7fde327d9b 100644 --- a/.mcp.json +++ b/.mcp.json @@ -8,4 +8,4 @@ } } } -} \ No newline at end of file +} diff --git a/docs/adapters/improvedigital/api-requests.md b/docs/adapters/improvedigital/api-requests.md index 772e49912c..551fa5509e 100644 --- a/docs/adapters/improvedigital/api-requests.md +++ b/docs/adapters/improvedigital/api-requests.md @@ -83,6 +83,8 @@ DELETE /oauth/logout/{token} | **Test Connection** | `GET /rtb/v1/classic/campaigns?limit=1`, then `GET /lookup/v1/user-details` | | **Discover from API** (entities) | `GET /admin/v1/buying-entities-combo?limit=100&offset=N` (paginated) | | **Discover from API** (offices) | `GET /admin/v1/buying-entities/{buying_entity_id}/buying-entity-offices?limit=100&offset=N` | +| **Campaign Metadata** (advertiser picker) | `GET /api/metadata-advertisers?search=` | +| **Campaign Metadata** (agency picker) | `GET /api/metadata-agencies?search=` | | **Expand a package** (product config) | `GET /rtb/v1/packages/{package_id}/placements` | `user-details` supplies `user_id` (→ `improve_demand_contact_id`), @@ -126,6 +128,56 @@ POST /rtb/v1/classic/campaigns - The response `id` becomes the media buy's external ID (`improvedigital_`). +### 4.1b Attach campaign metadata (when configured) + +``` +POST /api/metadata-campaigns +``` +```json +{ + "campaignId": "370306", + "campaignName": "adcp_PO-12345", + "campaignStartDate": "2026-08-10T10:20:28.000Z", + "campaignEndDate": "2026-09-10T23:59:59.000Z", + "currencyCode": "EUR", + "entityType": "c", + "completed": true, + "isCompleted": true, + "advertiserUuid": "b0edd0c5-3fc7-4029-96f0-02e9ff6dca62", + "advertiserName": "Other", + "agencyId": 182, + "agencyName": "Other", + "businessUnitId": 33, + "buyerId": 30, + "integrationPlatformId": 1, + "seatId": "default", + "adOpsPersonId": 17373, + "salesPersonId": "f1b6846f-659b-426a-be89-c2c0b99be27c" +} +``` + +A surface parallel to booking: `CampaignMetadataDto` carries the commercial +attribution the Classic `CampaignDto` has no room for. Posted right after +campaign create so a rejection cleans up the campaign before any line item +exists. Skipped entirely when the tenant configured no metadata fields. + +Notes for cross-checking: + +- **camelCase**, unlike the snake_case booking API. +- Dates are **ISO-8601 UTC strings with milliseconds** + (`2026-08-14T14:45:18.407Z`), not the `YYYY-MM-DD HH:MM:SS` local strings + the booking API takes. +- `campaignId` is a **string** in the body, though the Classic campaign id + is an integer. It is also the only schema-required field — everything else + is omitted when unset. +- `entityType: "c"`, `completed` and `isCompleted` are fixed values pending + confirmation of what the platform derives on its own. +- Attribution is tenant-level today — every buy books under the same brand, + agency and owners. Per-buyer routing is plan item H2. + +Read the record back with +`GET /api/metadata-campaigns/integration-platform/{integrationPlatformId}/campaign/{campaignId}`. + ### 4.2 Create one line item per package ``` diff --git a/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json b/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json index 0ec48edf5a..b2c36e29cc 100644 --- a/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json +++ b/docs/adapters/improvedigital/improvedigital-marketplace.postman_collection.json @@ -61,6 +61,14 @@ { "key": "placement_id", "value": "", "type": "string", "description": "Auto-filled by 2.2 (first 300x250-capable placement)" }, { "key": "package_id", "value": "", "type": "string", "description": "Auto-filled by 2.6 (first reusable placement package)" }, { "key": "region_name", "value": "EMEA", "type": "string", "description": "Geo region for 2.8 / 3.5 — see 2.7" }, + { "key": "advertiser_uuid", "value": "", "type": "string", "description": "Metadata brand UUID for 3.1b — pick with 2.12" }, + { "key": "advertiser_name", "value": "Other", "type": "string" }, + { "key": "agency_id", "value": "182", "type": "string", "description": "Metadata agency (integer id) — pick with 2.13" }, + { "key": "agency_name", "value": "Other", "type": "string" }, + { "key": "integration_platform_id", "value": "1", "type": "string", "description": "1 = Improve Digital — see 2.14" }, + { "key": "seat_id", "value": "default", "type": "string", "description": "String, not a number" }, + { "key": "adops_person_id", "value": "", "type": "string", "description": "Integer ad-ops owner id" }, + { "key": "sales_person_id", "value": "", "type": "string", "description": "UUID string sales owner id" }, { "key": "creative_id", "value": "", "type": "string", "description": "Auto-filled by 3.6" } ], "item": [ @@ -290,6 +298,45 @@ }, "description": "Office rows carry `improve_demand_contact_id`, `billing_currency_code` and `buying_types`. Only offices with `active: true` and 'Classic' in `buying_types` can host a direct campaign — that is the filter the salesagent office picker applies." } + }, + { + "name": "2.12 Search metadata advertisers (brands)", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/api/metadata-advertisers?search=", + "host": ["{{base_url}}"], + "path": ["api", "metadata-advertisers"], + "query": [{ "key": "search", "value": "", "description": "Free-text filter; empty returns everything" }] + }, + "description": "The brand dimension for campaign metadata (3.1b). Rows are `MetadataAdvertiser`: `id` is a **UUID string**, not an integer — this is why CampaignDto.advertiserId (integer) cannot hold a metadata advertiser. Backs the Advertiser picker in the salesagent adapter settings page." + } + }, + { + "name": "2.13 Search metadata agencies", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/api/metadata-agencies?search=", + "host": ["{{base_url}}"], + "path": ["api", "metadata-agencies"], + "query": [{ "key": "search", "value": "", "description": "Free-text filter; empty returns everything" }] + }, + "description": "The agency dimension for campaign metadata (3.1b). Rows are `AgencyDto` with an **integer** `id` — the opposite of advertisers, so ids are never coerced between the two." + } + }, + { + "name": "2.14 List integration platforms (DSPs) and seats", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/api/metadata-integration-platforms?limit=50", + "host": ["{{base_url}}"], + "path": ["api", "metadata-integration-platforms"], + "query": [{ "key": "limit", "value": "50" }] + }, + "description": "`MetadataDSPDto` rows behind `integrationPlatformId` (1 = Improve Digital). Seats for one platform: `GET /api/metadata-integration-platforms/{id}/seats` — `MetadataDSPSeatDto.id` is a string (e.g. 'default'), which is why `seatId` is quoted in 3.1b." + } } ] }, @@ -379,6 +426,82 @@ "description": "This is the exact body the salesagent adapter sends (src/adapters/improvedigital/adapter.py::_line_item_payload) — see docs/adapters/improvedigital/api-requests.md.\n\nRequired (validated live): `type: \"Standard\"`, `line_item_status: \"Active\"`, `goal` (BUDGET or IMPRESSION), `business_unit_id`, `improve_demand_contact_id`.\n\nBUDGET pacing needs `budget` plus a `flight_details` row spanning the flight; both the money figure and the impression cap are sent either way, and they must stay consistent (budget = impression_cap x cpm_bid / 1000).\n\nDo NOT send `currency` — it must match the campaign owner's default and 400s otherwise; it inherits from the campaign when omitted.\n\nThe delivery/compliance block (pricing_model_type, invoice_type, third_party_inventory, is_optimised, keep_on_delivering, track_viewability, is_consentless, is_coppa_compliant, conversion_tracking_enabled, optout_mechanism, dynamic_optimization_*) mirrors the platform's own create payload; the API leaves these unset rather than defaulted when omitted.\n\nCaveats to verify against the API docs: `flight_details` is absent from the committed rtb-v3 OpenAPI spec (which documents `custom_budgets` / `daily_impression_caps` instead), and `dynamic_optimization_kpi_value` is typed string there but sent as the number 0. `frequency_interval_type`, when used, must be lowercase (months/weeks/days/hours/minutes).\n\nSaves {{line_item_id}}." } }, + { + "name": "3.1b Attach campaign metadata", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// CampaignMetadataDto takes ISO-8601 UTC strings, unlike the", + "// booking API's 'YYYY-MM-DD HH:MM:SS' local strings.", + "pm.variables.set('start_iso', new Date(Date.now() + 60 * 60 * 1000).toISOString());", + "pm.variables.set('end_iso', new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString());" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": ["pm.test('metadata attached (2xx)', () => pm.expect(pm.response.code).to.be.within(200, 299));"] + } + } + ], + "request": { + "method": "POST", + "header": [{ "key": "Content-Type", "value": "application/json" }], + "body": { + "mode": "raw", + "raw": "{\n \"campaignId\": \"{{campaign_id}}\",\n \"campaignName\": \"{{campaign_name}}\",\n \"campaignStartDate\": \"{{start_iso}}\",\n \"campaignEndDate\": \"{{end_iso}}\",\n \"currencyCode\": \"EUR\",\n \"entityType\": \"c\",\n \"completed\": true,\n \"isCompleted\": true,\n \"advertiserUuid\": \"{{advertiser_uuid}}\",\n \"advertiserName\": \"{{advertiser_name}}\",\n \"agencyId\": {{agency_id}},\n \"agencyName\": \"{{agency_name}}\",\n \"businessUnitId\": {{business_unit_id}},\n \"buyerId\": {{buyer_id}},\n \"integrationPlatformId\": {{integration_platform_id}},\n \"seatId\": \"{{seat_id}}\",\n \"adOpsPersonId\": {{adops_person_id}},\n \"salesPersonId\": \"{{sales_person_id}}\"\n}" + }, + "url": { + "raw": "{{base_url}}/api/metadata-campaigns", + "host": ["{{base_url}}"], + "path": ["api", "metadata-campaigns"] + }, + "description": "Commercial attribution for a booked campaign — brand, agency, business unit, buyer, DSP seat and the ad-ops / sales owners. A surface PARALLEL to booking: camelCase (booking is snake_case), ISO-8601 UTC dates (booking uses 'YYYY-MM-DD HH:MM:SS' local strings), `campaignId` as a string even though the Classic id is an integer, and only `campaignId` schema-required.\n\nsalesagent posts this immediately after 3.1 so a rejection cleans up the campaign before any line item exists. Read it back with 6.6.\n\nPick the advertiser and agency with 2.12 / 2.13. `entityType: \"c\"`, `completed` and `isCompleted` are fixed values pending confirmation of what the platform derives itself." + } + }, + { + "name": "3.1b Attach campaign metadata", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// CampaignMetadataDto takes ISO-8601 UTC strings, unlike the", + "// booking API's 'YYYY-MM-DD HH:MM:SS' local strings.", + "pm.variables.set('start_iso', new Date(Date.now() + 60 * 60 * 1000).toISOString());", + "pm.variables.set('end_iso', new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString());" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": ["pm.test('metadata attached (2xx)', () => pm.expect(pm.response.code).to.be.within(200, 299));"] + } + } + ], + "request": { + "method": "POST", + "header": [{ "key": "Content-Type", "value": "application/json" }], + "body": { + "mode": "raw", + "raw": "{\n \"campaignId\": \"{{campaign_id}}\",\n \"campaignName\": \"{{campaign_name}}\",\n \"campaignStartDate\": \"{{start_iso}}\",\n \"campaignEndDate\": \"{{end_iso}}\",\n \"currencyCode\": \"EUR\",\n \"entityType\": \"c\",\n \"completed\": true,\n \"isCompleted\": true,\n \"advertiserUuid\": \"{{advertiser_uuid}}\",\n \"advertiserName\": \"{{advertiser_name}}\",\n \"agencyId\": {{agency_id}},\n \"agencyName\": \"{{agency_name}}\",\n \"businessUnitId\": {{business_unit_id}},\n \"buyerId\": {{buyer_id}},\n \"integrationPlatformId\": {{integration_platform_id}},\n \"seatId\": \"{{seat_id}}\",\n \"adOpsPersonId\": {{adops_person_id}},\n \"salesPersonId\": \"{{sales_person_id}}\"\n}" + }, + "url": { + "raw": "{{base_url}}/api/metadata-campaigns", + "host": ["{{base_url}}"], + "path": ["api", "metadata-campaigns"] + }, + "description": "Commercial attribution for a booked campaign — brand, agency, business unit, buyer, DSP seat and the ad-ops / sales owners. A surface PARALLEL to booking: camelCase (booking is snake_case), ISO-8601 UTC dates (booking uses 'YYYY-MM-DD HH:MM:SS' local strings), `campaignId` as a string even though the Classic id is an integer, and only `campaignId` schema-required.\n\nsalesagent posts this immediately after 3.1 so a rejection cleans up the campaign before any line item exists. Read it back with 6.6.\n\nPick the advertiser and agency with 2.12 / 2.13. `entityType: \"c\"`, `completed` and `isCompleted` are fixed values pending confirmation of what the platform derives itself." + } + }, { "name": "3.3 Assign placements to line item", "event": [ @@ -673,6 +796,30 @@ "description": "Single-create is a multipart servlet: the CreativeDto travels as a `body` part with content type application/json (binary image parts would ride alongside). Plain JSON to this path returns 500 'Failed to parse multipart servlet request'. Tag creatives should use 3.6 instead. Do not set Content-Type by hand — Postman writes the multipart boundary." } }, + { + "name": "6.6 Read campaign metadata back", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/api/metadata-campaigns/integration-platform/{{integration_platform_id}}/campaign/{{campaign_id}}", + "host": ["{{base_url}}"], + "path": ["api", "metadata-campaigns", "integration-platform", "{{integration_platform_id}}", "campaign", "{{campaign_id}}"] + }, + "description": "Verify what 3.1b stored. The metadata record has its own `id` distinct from `campaignId` — it points at the Classic campaign rather than being part of it." + } + }, + { + "name": "6.6 Read campaign metadata back", + "request": { + "method": "GET", + "url": { + "raw": "{{base_url}}/api/metadata-campaigns/integration-platform/{{integration_platform_id}}/campaign/{{campaign_id}}", + "host": ["{{base_url}}"], + "path": ["api", "metadata-campaigns", "integration-platform", "{{integration_platform_id}}", "campaign", "{{campaign_id}}"] + }, + "description": "Verify what 3.1b stored. The metadata record has its own `id` distinct from `campaignId` — it points at the Classic campaign rather than being part of it." + } + }, { "name": "6.5 Logout (invalidate bearer)", "request": { diff --git a/src/adapters/improvedigital/adapter.py b/src/adapters/improvedigital/adapter.py index 62619c0dec..7945e7452c 100644 --- a/src/adapters/improvedigital/adapter.py +++ b/src/adapters/improvedigital/adapter.py @@ -143,6 +143,20 @@ def __init__( self.improve_demand_contact_id = self.config.get("improve_demand_contact_id") self.agency_id = self.config.get("agency_id") + # Campaign-metadata attribution (CampaignMetadataDto). Tenant-level + # for now, so every buy books under the same brand/agency/owners. + # TODO(H2): make these per-buyer — resolve from the principal's + # improvedigital platform mapping (and later the buyer-routing rules) + # with this config as the fallback default, the same way GAM routes + # buyer agents to advertisers. + self.advertiser_uuid = self.config.get("advertiser_uuid") + self.advertiser_name = self.config.get("advertiser_name") + self.agency_name = self.config.get("agency_name") + self.integration_platform_id = self.config.get("integration_platform_id") + self.seat_id = self.config.get("seat_id") + self.adops_person_id = self.config.get("adops_person_id") + self.sales_person_id = self.config.get("sales_person_id") + self.client_id = self.config.get("client_id") self.client_secret = self.config.get("client_secret") self.base_url = (self.config.get("api_base_url") or "https://api.360yield.com").rstrip("/") @@ -358,6 +372,9 @@ def create_media_buy( campaign_payload = self._campaign_payload(buy_name, start_time, end_time) self.log(f"Would call: POST {self.base_url}/rtb/v1/classic/campaigns") self.log(f" Campaign: {campaign_payload}") + if self._has_campaign_metadata(): + self.log(f"Would call: POST {self.base_url}/api/metadata-campaigns") + self.log(f" Metadata: {self._campaign_metadata_payload(0, buy_name, start_time, end_time)}") for package in packages: rate, rate_type = self._resolve_pricing_rate(package, package_pricing_info) payload = self._line_item_payload(package, rate, rate_type, start_time, end_time) @@ -394,6 +411,20 @@ def create_media_buy( try: campaign = self._client.campaigns.create_campaign(self._campaign_payload(buy_name, start_time, end_time)) campaign_id = int(campaign["id"]) + # Commercial attribution rides on its own record, posted before + # the line items so a rejection cleans up the campaign alone. + if self._has_campaign_metadata(): + self._client.metadata.upsert_campaign_metadata( + self._campaign_metadata_payload(campaign_id, buy_name, start_time, end_time) + ) + logger.info("Improve Digital: campaign metadata attached for campaign %s", campaign_id) + else: + logger.info( + "Improve Digital: campaign metadata skipped for campaign %s — no attribution " + "fields configured for tenant %s (fill them in the adapter settings page)", + campaign_id, + self.tenant_id, + ) platform_line_item_ids: dict[str, str] = {} package_responses: list[ResponsePackage] = [] for package in packages: @@ -608,6 +639,14 @@ def _format_datetime(value: datetime) -> str: confirmed against live dev campaigns/line items.""" return value.strftime("%Y-%m-%d %H:%M:%S") + @staticmethod + def _format_iso_datetime(value: datetime) -> str: + """Metadata API datetime format — ISO-8601 UTC with milliseconds + (``2026-08-14T14:45:18.407Z``). Naive values are read as UTC rather + than the host's local zone, which would silently shift the flight.""" + moment = (value if value.tzinfo else value.replace(tzinfo=UTC)).astimezone(UTC) + return f"{moment.strftime('%Y-%m-%dT%H:%M:%S')}.{moment.microsecond // 1000:03d}Z" + def _campaign_payload(self, buy_name: str, start_time: datetime, end_time: datetime) -> dict[str, Any]: """Build a ``CampaignDto`` body. @@ -638,6 +677,67 @@ def _campaign_payload(self, buy_name: str, start_time: datetime, end_time: datet payload["agencyId"] = int(self.agency_id) return payload + def _has_campaign_metadata(self) -> bool: + """Whether this tenant configured any campaign-metadata attribution. + + Metadata is required for production bookings but not for the + Classic API itself, so an unconfigured tenant (dev, smoke tests) + books without it rather than failing. + """ + return any( + ( + self.advertiser_uuid, + self.agency_id, + self.adops_person_id, + self.sales_person_id, + self.integration_platform_id, + ) + ) + + def _campaign_metadata_payload( + self, campaign_id: int, buy_name: str, start_time: datetime, end_time: datetime + ) -> dict[str, Any]: + """Build the ``CampaignMetadataDto`` for a freshly created campaign. + + The metadata API is a surface parallel to booking: the record is + keyed by ``campaignId`` and carries the commercial attribution + (brand, agency, owners, DSP seat) that the Classic ``CampaignDto`` + has no room for. Only ``campaignId`` is schema-required; everything + else comes from tenant config and is omitted when unset. + + Wire notes: the DTO is camelCase where booking is snake_case, and + its dates are ISO-8601 UTC strings (``2026-08-14T14:45:18.407Z``) + where booking uses ``YYYY-MM-DD HH:MM:SS`` in the campaign's own + timezone. ``campaignId`` is a string here even though the Classic + campaign id is an integer. + ``entityType``/``completed``/``isCompleted`` are fixed values pending + confirmation of what else the platform derives. + """ + payload: dict[str, Any] = { + "campaignId": str(campaign_id), + "campaignName": buy_name, + "campaignStartDate": self._format_iso_datetime(start_time), + "campaignEndDate": self._format_iso_datetime(end_time), + "currencyCode": self.currency, + "entityType": "c", + "completed": True, + "isCompleted": True, + } + optional: dict[str, Any] = { + "advertiserUuid": self.advertiser_uuid, + "advertiserName": self.advertiser_name, + "agencyId": int(self.agency_id) if self.agency_id else None, + "agencyName": self.agency_name, + "businessUnitId": int(self.business_unit_id) if self.business_unit_id else None, + "buyerId": int(self.buyer_id) if self.buyer_id else None, + "integrationPlatformId": (int(self.integration_platform_id) if self.integration_platform_id else None), + "seatId": self.seat_id, + "adOpsPersonId": int(self.adops_person_id) if self.adops_person_id else None, + "salesPersonId": self.sales_person_id, + } + payload.update({key: value for key, value in optional.items() if value is not None}) + return payload + def _line_item_payload( self, package: MediaPackage, diff --git a/src/adapters/improvedigital/client.py b/src/adapters/improvedigital/client.py index 357dce67a7..39922bdbe7 100644 --- a/src/adapters/improvedigital/client.py +++ b/src/adapters/improvedigital/client.py @@ -10,6 +10,7 @@ assignment, VAST validation - ``client.inventory`` — buy-side placement search + packages - ``client.lookups`` — dimension lookups (sizes, geo, creative types) +- ``client.metadata`` — campaign metadata API (brand / agency / owners) - ``client.reporting`` — Report API (preview / async generation / status) Endpoint paths come from the committed OpenAPI spec @@ -307,6 +308,54 @@ def list_buying_entity_offices(self, buying_entity_id: int, **params: Any) -> di return self._transport.get_json(f"/admin/v1/buying-entities/{buying_entity_id}/buying-entity-offices", **params) +class ImproveDigitalMetadataClient: + """Campaign metadata API (``/api/metadata-*``). + + A surface parallel to the booking API: ``CampaignMetadataDto`` records + are keyed by ``campaignId`` and carry the commercial attribution a + booked campaign needs in production — brand (``advertiserUuid``), + agency, business unit, buyer, DSP seat, and the ad-ops / sales owners. + The dimension endpoints back the pickers in the adapter settings page. + """ + + def __init__(self, transport: ImproveDigitalTransport): + self._transport = transport + + def list_advertisers(self, search: str | None = None) -> Any: + """``GET /api/metadata-advertisers`` — ``MetadataAdvertiser`` rows + (``id`` is a UUID string, not an integer).""" + return self._transport.get_json("/api/metadata-advertisers", **({"search": search} if search else {})) + + def list_agencies(self, search: str | None = None) -> Any: + """``GET /api/metadata-agencies`` — ``AgencyDto`` rows (integer ``id``).""" + return self._transport.get_json("/api/metadata-agencies", **({"search": search} if search else {})) + + def list_sales_persons(self, search: str | None = None) -> Any: + """``GET /api/metadata-sales-persons`` — sales owners (UUID ``id``).""" + return self._transport.get_json("/api/metadata-sales-persons", **({"search": search} if search else {})) + + def list_integration_platforms(self, **params: Any) -> Any: + """``GET /api/metadata-integration-platforms`` — ``MetadataDSPDto`` + rows (the DSP behind ``integrationPlatformId``).""" + return self._transport.get_json("/api/metadata-integration-platforms", **params) + + def list_platform_seats(self, platform_id: int) -> Any: + """``GET /api/metadata-integration-platforms/{id}/seats`` — + ``MetadataDSPSeatDto`` rows (``seatId`` is a string, e.g. "default").""" + return self._transport.get_json(f"/api/metadata-integration-platforms/{platform_id}/seats") + + def get_campaign_metadata(self, integration_platform_id: int, campaign_id: str) -> dict[str, Any]: + """``GET /api/metadata-campaigns/integration-platform/{ipId}/campaign/{campaignId}``.""" + return self._transport.get_json( + f"/api/metadata-campaigns/integration-platform/{integration_platform_id}/campaign/{campaign_id}" + ) + + def upsert_campaign_metadata(self, payload: dict[str, Any]) -> Any: + """``POST /api/metadata-campaigns`` — ``CampaignMetadataDto``; + ``campaignId`` is the only schema-required field.""" + return self._transport.post_json("/api/metadata-campaigns", payload) + + class ImproveDigitalReportingClient: """Improve Marketplace Report API (definitive delivery metrics).""" @@ -353,6 +402,7 @@ def __init__( self.lookups = ImproveDigitalLookupsClient(self._transport) self.reporting = ImproveDigitalReportingClient(self._transport) self.admin = ImproveDigitalAdminClient(self._transport) + self.metadata = ImproveDigitalMetadataClient(self._transport) def probe(self, method: str, path: str) -> tuple[int, str]: """Non-raising permission probe — see :meth:`ImproveDigitalTransport.probe`.""" diff --git a/src/adapters/improvedigital/schemas.py b/src/adapters/improvedigital/schemas.py index 91f46fb24e..cc283a2260 100644 --- a/src/adapters/improvedigital/schemas.py +++ b/src/adapters/improvedigital/schemas.py @@ -87,12 +87,54 @@ class ImproveDigitalConnectionConfig(BaseConnectionConfig): json_schema_extra={"ui_order": 8}, ) agency_id: int | None = Field( + default=None, + description="Metadata agency booking the campaign — CampaignMetadataDto.agencyId (pick via /api/metadata-agencies)", + json_schema_extra={"ui_order": 9}, + ) + agency_name: str | None = Field( + default=None, + description="Display name of the selected agency — CampaignMetadataDto.agencyName", + json_schema_extra={"ui_order": 9.1}, + ) + advertiser_uuid: str | None = Field( default=None, description=( - "Default agency ID — NOT part of the Classic campaign create schema; retained " - "pending the buyer-attribution decision" + "Metadata advertiser (brand) UUID — CampaignMetadataDto.advertiserUuid. " + "Distinct from default_advertiser_id: the metadata surface keys brands by " + "UUID, while CampaignDto.advertiserId is an integer" ), - json_schema_extra={"ui_order": 9}, + json_schema_extra={"ui_order": 9.2}, + ) + advertiser_name: str | None = Field( + default=None, + description="Display name of the selected advertiser — CampaignMetadataDto.advertiserName", + json_schema_extra={"ui_order": 9.3}, + ) + integration_platform_id: int | None = Field( + default=None, + description=( + "DSP the metadata record books under — CampaignMetadataDto.integrationPlatformId " + "(pick via /api/metadata-integration-platforms; 1 = Improve Digital)" + ), + json_schema_extra={"ui_order": 9.35}, + ) + seat_id: str | None = Field( + default=None, + description=( + "DSP seat on the metadata record — CampaignMetadataDto.seatId " + "(string, e.g. 'default'; see /api/metadata-integration-platforms/{id}/seats)" + ), + json_schema_extra={"ui_order": 9.36}, + ) + adops_person_id: int | None = Field( + default=None, + description="Ad-ops owner for booked campaigns — CampaignMetadataDto.adOpsPersonId (integer)", + json_schema_extra={"ui_order": 9.4}, + ) + sales_person_id: str | None = Field( + default=None, + description="Sales owner for booked campaigns — CampaignMetadataDto.salesPersonId (UUID string, not an integer)", + json_schema_extra={"ui_order": 9.5}, ) business_unit_id: int | None = Field( default=None, diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index ff1119bf35..27b8b96115 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1236,6 +1236,66 @@ def discover_improvedigital_buying_entities(tenant_id, **kwargs): return jsonify({"success": False, "error": "Discovery failed (see server logs)"}), 500 +@adapters_bp.route("/api/tenant//adapters/improvedigital/discover-metadata", methods=["POST"]) +@require_tenant_access(role=("admin",), allow_embedded_writes=True) +def discover_improvedigital_metadata(tenant_id, **kwargs): + """Search the campaign-metadata dimensions (advertisers / agencies). + + Backs the Campaign Metadata pickers in the adapter connection UI. Both + endpoints take a free-text ``search`` and return ``{id, name}`` rows — + advertiser ids are UUID strings, agency ids are integers, so ids are + passed through verbatim rather than coerced. + + Read-only — never writes to AdapterConfig — so it opts into the + embedded-write gate. + """ + try: + data = request.get_json() or {} + kind = str(data.get("kind") or "advertisers") + if kind not in ("advertisers", "agencies"): + return jsonify({"success": False, "error": f"Unknown metadata kind {kind!r}"}), 400 + + client_kwargs, cred_error = _resolve_improvedigital_credentials(tenant_id, data) + if cred_error: + return jsonify({"success": False, "error": cred_error}), 400 + + from src.adapters.improvedigital import ImproveDigitalClient, ImproveDigitalError + + client = ImproveDigitalClient(**client_kwargs) + search = (data.get("search") or "").strip() or None + try: + fetch = client.metadata.list_advertisers if kind == "advertisers" else client.metadata.list_agencies + rows = _improvedigital_rows(fetch(search), kind, "content", "data") + except ImproveDigitalError as exc: + if exc.status_code == 403: + return jsonify( + { + "success": False, + "discovery_available": False, + "error": "Credentials lack metadata API scope — enter the values manually", + } + ) + logger.warning( + "Improve Digital metadata discovery failed: tenant_id=%s kind=%s status=%s error=%s body_excerpt=%s", + tenant_id, + kind, + exc.status_code, + exc, + safe_upstream_body_excerpt(exc.body), + ) + return jsonify({"success": False, "error": "Improve Digital rejected the metadata lookup"}), 200 + + items = [ + {"id": row.get("id"), "name": row.get("name")} + for row in rows + if isinstance(row, dict) and row.get("id") is not None + ] + return jsonify({"success": True, "kind": kind, "items": items}) + except Exception as e: + logger.error(f"Improve Digital metadata discovery failed: {e}", exc_info=True) + return jsonify({"success": False, "error": "Metadata discovery failed (see server logs)"}), 500 + + @adapters_bp.route("/api/tenant//adapters/improvedigital/inventory", methods=["GET"]) @require_tenant_access() def list_improvedigital_inventory(tenant_id, **kwargs): diff --git a/src/core/helpers/adapter_helpers.py b/src/core/helpers/adapter_helpers.py index d51b386ef6..cf0555e776 100644 --- a/src/core/helpers/adapter_helpers.py +++ b/src/core/helpers/adapter_helpers.py @@ -222,7 +222,19 @@ def get_adapter( "agency_id": impd_validated.agency_id, "buying_entity_id": impd_validated.buying_entity_id, "buying_entity_office_id": impd_validated.buying_entity_office_id, + "campaign_type": impd_validated.campaign_type, "business_unit_id": impd_validated.business_unit_id, + "buyer_id": impd_validated.buyer_id, + # Campaign-metadata attribution (CampaignMetadataDto) — + # dropping any of these silently disables the + # POST /api/metadata-campaigns step in create_media_buy. + "agency_name": impd_validated.agency_name, + "advertiser_uuid": impd_validated.advertiser_uuid, + "advertiser_name": impd_validated.advertiser_name, + "integration_platform_id": impd_validated.integration_platform_id, + "seat_id": impd_validated.seat_id, + "adops_person_id": impd_validated.adops_person_id, + "sales_person_id": impd_validated.sales_person_id, "currency": impd_validated.currency, "timezone": impd_validated.timezone, "manual_approval_required": impd_validated.manual_approval_required, diff --git a/templates/adapters/improvedigital/connection_config.html b/templates/adapters/improvedigital/connection_config.html index e62dfc1997..9b1beebe50 100644 --- a/templates/adapters/improvedigital/connection_config.html +++ b/templates/adapters/improvedigital/connection_config.html @@ -113,10 +113,10 @@

Improve Digital 360Yield Configuration

+ placeholder="Your API user's user_id — office default applies when empty"> - Optional override — the selected office carries a default demand contact. + Editable. Test Connection fills it from your API user, and picking an office + overwrites it with that office's default contact — type over either. @@ -147,23 +147,105 @@

Improve Digital 360Yield Configuration

+
+ +

Campaign Metadata

+

+ Which brand and agency the campaign belongs to, and who owns it internally. +

+
- - + +
+ + +
+ +
+ + +
- Fallback advertiser ID for principals without explicit platform mappings. + Brands are keyed by UUID on the metadata API — this is not the numeric + Default Advertiser ID below.
- - + +
+ + +
+ +
+ + +
+
+
+ + + + integrationPlatformId — see /api/metadata-integration-platforms. + +
+
+ + + + seatId — a string, not a number. + +
+
+ +
+
+ + + Integer — adOpsPersonId. +
+
+ + + UUID string — salesPersonId. +
+
+ + +
+ + + + + + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + +
+
+ + + +
@@ -40,16 +93,16 @@

-

-
Clicks / CTR
-

-

+
Average CPM
+

-

-
Completed Views
-

-

+
Clicks / CTR
+

-

@@ -95,31 +148,106 @@

-

function fmtPct(v) { return v === null || v === undefined ? '—' : Number(v).toFixed(2) + '%'; } function esc(v) { const d = document.createElement('div'); d.textContent = v ?? ''; return d.innerHTML; } +const seenCampaigns = new Set(); +let currentRows = []; +let loadSeq = 0; + +function currentFilterParams() { + const params = new URLSearchParams(); + const q = document.getElementById('filter-q').value.trim(); + const campaign = document.getElementById('filter-campaign').value; + const range = document.getElementById('filter-range').value; + if (q) params.set('q', q); + if (campaign) params.set('campaign_id', campaign); + if (range) { + params.set('date_range', range); + params.set('timezone', document.getElementById('filter-timezone').value); + } + return params; +} + +function addCampaignOption(campaignId, label) { + if (!campaignId || seenCampaigns.has(campaignId)) return; + seenCampaigns.add(campaignId); + const opt = document.createElement('option'); + opt.value = campaignId; + opt.textContent = label || campaignId; + document.getElementById('filter-campaign').appendChild(opt); +} + +function clearFilters() { + document.getElementById('filter-q').value = ''; + document.getElementById('filter-campaign').value = ''; + document.getElementById('filter-range').value = ''; + document.getElementById('filter-timezone').disabled = true; + loadReporting(); +} + +function exportData() { + if (!currentRows.length) return; + const header = ['Media Buy', 'Order', 'Advertiser', 'Campaign', 'Line Item', + 'Impressions', 'Clicks', 'CTR %', 'Completed Views', 'Spend']; + const csvCell = (v) => { + let s = v === null || v === undefined ? '' : String(v); + // Neutralize spreadsheet formula injection — order/advertiser names + // are buyer-supplied and a leading =+-@ executes on open in Excel. + if (/^[=+\-@]/.test(s)) s = "'" + s; + return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s; + }; + const csv = [header.join(',')].concat(currentRows.map(row => [ + row.media_buy_id, row.order_name, row.advertiser_name, row.campaign_id, row.line_item_id, + row.impressions, row.clicks, row.ctr, row.completed_views, row.spend + ].map(csvCell).join(','))).join('\n'); + const blob = new Blob([csv], { type: 'text/csv' }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.setAttribute('hidden', ''); + a.setAttribute('href', url); + const range = document.getElementById('filter-range').value || 'cached'; + a.setAttribute('download', `improvedigital-reporting-${range.toLowerCase()}.csv`); + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); +} + async function loadReporting() { + // Sequence token: a slow live query resolving after a newer request + // (e.g. Clear during a 90-day fetch) must not overwrite fresher data. + const seq = ++loadSeq; document.getElementById('table-loading').style.display = ''; document.getElementById('report-table').style.display = 'none'; document.getElementById('table-empty').style.display = 'none'; try { - const response = await fetch(`${scriptRoot}/api/tenant/${tenantId}/adapters/improvedigital/reporting`, { + const params = currentFilterParams(); + const query = params.toString() ? `?${params.toString()}` : ''; + const response = await fetch(`${scriptRoot}/api/tenant/${tenantId}/adapters/improvedigital/reporting${query}`, { headers: {'Accept': 'application/json'} }); const data = await response.json(); + if (seq !== loadSeq) return; if (!response.ok || data.success === false) { throw new Error(data.error || `HTTP ${response.status}`); } + currentRows = data.rows || []; document.getElementById('total-impressions').textContent = fmtInt(data.totals.impressions); document.getElementById('total-spend').textContent = fmtMoney(data.totals.spend); + document.getElementById('avg-cpm').textContent = data.totals.impressions + ? fmtMoney(data.totals.spend / data.totals.impressions * 1000) + : '—'; document.getElementById('total-clicks').textContent = `${fmtInt(data.totals.clicks)}${data.totals.ctr !== null ? ' (' + fmtPct(data.totals.ctr) + ')' : ''}`; - document.getElementById('total-completed').textContent = fmtInt(data.totals.completed_views); - document.getElementById('last-synced').textContent = data.last_synced_at - ? `Last synced: ${new Date(data.last_synced_at).toLocaleString()}` - : 'Never synced'; + document.getElementById('last-synced').textContent = data.source === 'live' + ? `Live Report API query (${data.date_range})` + : (data.last_synced_at + ? `Last synced: ${new Date(data.last_synced_at).toLocaleString()}` + : 'Never synced'); const tbody = document.getElementById('report-rows'); tbody.innerHTML = ''; for (const row of data.rows) { + addCampaignOption(row.campaign_id, row.order_name ? `${row.campaign_id} — ${row.order_name}` : row.campaign_id); const buyCell = row.media_buy_id ? `${esc(row.order_name || row.media_buy_id)}` : 'not booked via salesagent'; @@ -140,7 +268,11 @@

-

if (data.rows.length) { document.getElementById('report-table').style.display = ''; } else { - document.getElementById('table-empty').style.display = ''; + const emptyEl = document.getElementById('table-empty'); + emptyEl.textContent = params.toString() + ? 'No rows match the current filters.' + : 'No delivery data in the reporting cache yet. Book a media buy, let it deliver, then press Sync Reporting Now.'; + emptyEl.style.display = ''; } } catch (err) { document.getElementById('table-loading').style.display = 'none'; @@ -159,28 +291,101 @@

-

const btn = document.getElementById('sync-btn'); btn.disabled = true; btn.textContent = 'Syncing…'; - showStatus('info', 'Pulling fresh metrics from the 360Yield Report API…'); + showStatus('info', 'Starting reporting sync…'); try { + // Only the campaign filter narrows the sync — date filters stay + // view-only (a partial-window sync would overwrite the lifetime + // cache and delivered_* columns with window-only totals). + const syncBody = {}; + const campaign = document.getElementById('filter-campaign').value; + if (campaign) syncBody.campaign_id = campaign; const response = await fetch(`${scriptRoot}/api/tenant/${tenantId}/adapters/improvedigital/sync-reporting`, { method: 'POST', - headers: {'Accept': 'application/json'} + headers: {'Accept': 'application/json', 'Content-Type': 'application/json'}, + body: JSON.stringify(syncBody) }); const data = await response.json(); - if (response.status === 503 && data.scope_pending) { - showStatus('warning', data.error); - } else if (!data.success) { - showStatus('danger', data.error || 'Sync failed'); + if (!data.success || !data.sync_id) { + showStatus('danger', data.error || 'Sync failed to start'); + resetSyncButton(); + return; + } + showStatus('info', 'Sync running — pulling fresh metrics from the 360Yield Report API…'); + pollReportingSync(data.sync_id, Date.now()); + } catch (err) { + showStatus('danger', `Sync request failed: ${err.message}`); + resetSyncButton(); + } +} + +function resetSyncButton() { + const btn = document.getElementById('sync-btn'); + btn.disabled = false; + btn.textContent = 'Sync Reporting Now'; +} + +async function pollReportingSync(syncId, startedMs) { + // The sync runs in a background thread server-side; poll the job row. + if (Date.now() - startedMs > 5 * 60 * 1000) { + showStatus('warning', 'Sync is still running in the background — refresh later for the result.'); + resetSyncButton(); + return; + } + try { + const response = await fetch(`${scriptRoot}/api/tenant/${tenantId}/adapters/improvedigital/sync-status/${encodeURIComponent(syncId)}`, { + headers: {'Accept': 'application/json'}, + credentials: 'same-origin' + }); + const job = await response.json(); + if (job.status !== 'completed' && job.status !== 'failed') { + setTimeout(() => pollReportingSync(syncId, startedMs), 2000); + return; + } + if ((job.metadata || {}).scope_pending) { + showStatus('warning', (job.errors || {}).scope || 'Report API scope grant pending'); + } else if (job.status === 'failed') { + showStatus('danger', job.error_message || 'Sync failed'); } else { - showStatus('success', `Synced ${data.line_items_updated} line items across ${data.campaigns_covered} campaigns.`); + const counts = job.counts || {}; + showStatus('success', `Synced ${counts.line_items ?? 0} line items across ${counts.campaigns ?? 0} campaigns.`); await loadReporting(); } } catch (err) { - showStatus('danger', `Sync request failed: ${err.message}`); - } finally { - btn.disabled = false; - btn.textContent = 'Sync Reporting Now'; + showStatus('danger', `Sync status check failed: ${err.message}`); } + resetSyncButton(); +} + +// Deep links: ?q=…&campaign_id=… pre-fill the filters (e.g. from a media +// buy details page linking to its campaign's reporting rows). +const initParams = new URLSearchParams(location.search); +document.getElementById('filter-q').value = initParams.get('q') || ''; +const initCampaign = (initParams.get('campaign_id') || '').trim(); +if (initCampaign) { + addCampaignOption(initCampaign, initCampaign); + document.getElementById('filter-campaign').value = initCampaign; } +// Match against the option list directly — interpolating the raw URL param +// into a querySelector string throws on quotes/brackets and would abort the +// whole script (dead controls, permanent spinner). +const initRange = (initParams.get('date_range') || '').trim().toUpperCase(); +const rangeSelect = document.getElementById('filter-range'); +if (initRange && Array.from(rangeSelect.options).some(opt => opt.value === initRange)) { + rangeSelect.value = initRange; + document.getElementById('filter-timezone').disabled = false; +} + +let filterDebounce = null; +document.getElementById('filter-q').addEventListener('input', () => { + clearTimeout(filterDebounce); + filterDebounce = setTimeout(loadReporting, 300); +}); +document.getElementById('filter-campaign').addEventListener('change', loadReporting); +document.getElementById('filter-range').addEventListener('change', () => { + document.getElementById('filter-timezone').disabled = !document.getElementById('filter-range').value; + loadReporting(); +}); +document.getElementById('filter-timezone').addEventListener('change', loadReporting); loadReporting(); diff --git a/tests/unit/test_adapter_delivery_aggregation.py b/tests/unit/test_adapter_delivery_aggregation.py new file mode 100644 index 0000000000..85f0e5708c --- /dev/null +++ b/tests/unit/test_adapter_delivery_aggregation.py @@ -0,0 +1,67 @@ +"""_aggregate_stat_rows_to_delivery_response must carry every metric the +stats cache holds. + +Regression: the helper populated impressions/spend/completed_views but +dropped clicks entirely, so the media buy details page (which reads +``totals.clicks`` / ``totals.ctr`` via adapter.get_media_buy_delivery) +showed empty click metrics while the reporting page — reading the same +cache rows directly — showed real ones. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace + +from src.adapters.base import AdServerAdapter +from src.core.schemas import ReportingPeriod + + +def make_row(line_item_id: str, impressions: int, clicks: int | None, spend_micros: int, completed: int | None): + return SimpleNamespace( + line_item_id=line_item_id, + impressions=impressions, + clicks=clicks, + spend_micros=spend_micros, + completed_views=completed, + currency="EUR", + ) + + +PERIOD = ReportingPeriod(start=datetime(2026, 8, 1, tzinfo=UTC), end=datetime(2026, 8, 18, tzinfo=UTC)) + + +class TestAggregateStatRows: + def test_clicks_and_ctr_are_aggregated(self): + rows = [ + make_row("li1", 28287, 30, 25_277_857, 277), + make_row("li2", 934, 0, 604_944, 20), + ] + response = AdServerAdapter._aggregate_stat_rows_to_delivery_response( + "improvedigital_314446", PERIOD, rows, package_id_attr="line_item_id", default_currency="EUR" + ) + assert response.totals.impressions == 29221.0 + assert response.totals.clicks == 30.0 + assert response.totals.ctr == 30 / 29221 + assert round(response.totals.spend, 2) == 25.88 + assert response.totals.completed_views == 297.0 + assert response.currency == "EUR" + + def test_ctr_clamps_when_clicks_exceed_impressions(self): + """Click trackers/companion clicks can report clicks > impressions; + DeliveryTotals.ctr enforces le=1, so the ratio must clamp instead of + raising ValidationError and killing the delivery response.""" + rows = [make_row("li1", 3, 5, 0, None)] + response = AdServerAdapter._aggregate_stat_rows_to_delivery_response( + "buy", PERIOD, rows, package_id_attr="line_item_id" + ) + assert response.totals.clicks == 5.0 + assert response.totals.ctr == 1.0 + + def test_clicks_stay_none_when_platform_reports_none(self): + rows = [make_row("li1", 100, None, 0, None)] + response = AdServerAdapter._aggregate_stat_rows_to_delivery_response( + "buy", PERIOD, rows, package_id_attr="line_item_id" + ) + assert response.totals.clicks is None + assert response.totals.ctr is None diff --git a/tests/unit/test_improvedigital_reporting_filters.py b/tests/unit/test_improvedigital_reporting_filters.py new file mode 100644 index 0000000000..680806ea9b --- /dev/null +++ b/tests/unit/test_improvedigital_reporting_filters.py @@ -0,0 +1,163 @@ +"""Filtering of the Improve Digital reporting page payload. + +The reporting endpoint accepts ``campaign_id`` / ``media_buy_id`` (exact) +and ``q`` (case-insensitive substring across order name, advertiser and +the campaign/line-item/media-buy ids). Filters apply before totals, so +the summary cards always match the visible table. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace + +from src.admin.blueprints.adapters import _improvedigital_reporting_payload + +AS_OF = datetime(2026, 8, 18, 12, 0, tzinfo=UTC) + + +def stat(campaign_id, line_item_id, impressions, clicks, spend_micros): + return SimpleNamespace( + campaign_id=campaign_id, + line_item_id=line_item_id, + impressions=impressions, + clicks=clicks, + completed_views=None, + spend_micros=spend_micros, + currency="EUR", + as_of=AS_OF, + ) + + +def buy(media_buy_id, order_name, advertiser_name): + return SimpleNamespace(media_buy_id=media_buy_id, order_name=order_name, advertiser_name=advertiser_name) + + +STATS = [ + stat("314446", "582321", 28287, 30, 25_277_857), + stat("314450", "582325", 934, 0, 604_944), + stat("314455", "585239", 500, 5, 1_250_000), +] +BUYS = { + "314446": buy("mb_aaa", "Acme Summer Push", "Acme Corp"), + "314450": buy("mb_bbb", "Globex Launch", "Globex"), +} + + +class TestReportingFilters: + def test_unfiltered_keeps_all_rows(self): + payload = _improvedigital_reporting_payload(STATS, BUYS, "EUR") + assert len(payload["rows"]) == 3 + assert payload["totals"]["impressions"] == 28287 + 934 + 500 + + def test_campaign_id_exact_match(self): + payload = _improvedigital_reporting_payload(STATS, BUYS, "EUR", campaign_id="314450") + assert [r["line_item_id"] for r in payload["rows"]] == ["582325"] + assert payload["totals"]["impressions"] == 934 + assert payload["totals"]["spend"] == 0.6 + + def test_media_buy_id_exact_match(self): + payload = _improvedigital_reporting_payload(STATS, BUYS, "EUR", media_buy_id="mb_aaa") + assert [r["campaign_id"] for r in payload["rows"]] == ["314446"] + assert payload["totals"]["clicks"] == 30 + + def test_q_matches_order_and_advertiser_case_insensitively(self): + payload = _improvedigital_reporting_payload(STATS, BUYS, "EUR", q="globex") + assert [r["campaign_id"] for r in payload["rows"]] == ["314450"] + + payload = _improvedigital_reporting_payload(STATS, BUYS, "EUR", q="ACME") + assert [r["campaign_id"] for r in payload["rows"]] == ["314446"] + + def test_q_matches_ids_including_unattributed_rows(self): + payload = _improvedigital_reporting_payload(STATS, BUYS, "EUR", q="585239") + assert [r["campaign_id"] for r in payload["rows"]] == ["314455"] + assert payload["rows"][0]["media_buy_id"] is None + + def test_filters_compose(self): + payload = _improvedigital_reporting_payload(STATS, BUYS, "EUR", campaign_id="314446", q="globex") + assert payload["rows"] == [] + assert payload["totals"]["impressions"] == 0 + + +class TestLiveDateFilteredRows: + """Date-range views query the Report API live — the cache has no time + dimension to slice — and shape rows exactly like cache rows.""" + + def _client(self, rows): + captured = {} + + class FakeReporting: + def preview(self, payload): + captured["payload"] = payload + return {"rows": rows} + + return SimpleNamespace(reporting=FakeReporting()), captured + + def test_builds_quick_range_request_and_stat_shaped_rows(self): + from src.admin.blueprints.adapters import _improvedigital_live_stat_rows + + client, captured = self._client( + [ + { + "campaign_id": 314446, + "line_item_id": 582321, + "impressions": "100", + "clicks": "3", + "advertiser_payout": "0.25", + "complete": "7", + } + ] + ) + rows = _improvedigital_live_stat_rows(client, [314446, 314450], "LAST_7_DAYS", "UTC", "EUR") + + request = captured["payload"]["report_generation_request"] + assert request["date_range"] == {"quick": "LAST_7_DAYS"} + assert request["timezone"] == "UTC" + assert request["currency_id"] == 1 + assert request["filters"] == [{"column": "campaign_id", "operation": "IN", "value": [314446, 314450]}] + + assert len(rows) == 1 + row = rows[0] + assert (row.campaign_id, row.line_item_id) == ("314446", "582321") + assert (row.impressions, row.clicks, row.completed_views) == (100, 3, 7) + assert row.spend_micros == 250_000 + assert row.currency == "EUR" + + def test_today_translates_to_the_relative_range_the_wire_accepts(self): + """quick TODAY 500s upstream (no same-day partition in the + consolidated warehouse, observed live) — the boundary maps it to a + 1-day relative range that returns today's data.""" + from src.admin.blueprints.adapters import _improvedigital_live_stat_rows + + client, captured = self._client([]) + _improvedigital_live_stat_rows(client, [314446], "TODAY", "UTC", "EUR") + assert captured["payload"]["report_generation_request"]["date_range"] == { + "relative": {"from_count": 1, "from_unit": "DAY", "to_count": 0, "to_unit": "DAY"} + } + + def test_no_campaigns_short_circuits_without_api_call(self): + from src.admin.blueprints.adapters import _improvedigital_live_stat_rows + + client, captured = self._client([]) + assert _improvedigital_live_stat_rows(client, [], "TODAY", "UTC", "EUR") == [] + assert "payload" not in captured + + def test_live_rows_flow_through_the_shared_payload_builder(self): + from src.admin.blueprints.adapters import _improvedigital_live_stat_rows + + client, _ = self._client( + [ + { + "campaign_id": 314446, + "line_item_id": 582321, + "impressions": "100", + "clicks": "3", + "advertiser_payout": "0.25", + "complete": None, + } + ] + ) + rows = _improvedigital_live_stat_rows(client, [314446], "TODAY", "UTC", "EUR") + payload = _improvedigital_reporting_payload(rows, BUYS, "EUR") + assert payload["rows"][0]["order_name"] == "Acme Summer Push" + assert payload["totals"]["spend"] == 0.25 diff --git a/tests/unit/test_improvedigital_reporting_window.py b/tests/unit/test_improvedigital_reporting_window.py new file mode 100644 index 0000000000..1fb30ba2f2 --- /dev/null +++ b/tests/unit/test_improvedigital_reporting_window.py @@ -0,0 +1,431 @@ +"""Reporting sync window and currency resolution (Report API OpenAPI spec). + +Spec-derived behaviour validated live on the dev platform (2026-08-18): +- ReportPreviewRequest.rows max is 2000 (was assumed 500) +- relative date ranges work on the wire; fixed ranges 400/500 +- /report/ext/currency/available serves the currency dictionary (15 rows) +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from unittest.mock import patch + +from src.adapters.improvedigital.client import ImproveDigitalError +from src.adapters.improvedigital.reporting_sync import ( + MAX_WINDOW_DAYS, + MIN_WINDOW_DAYS, + PREVIEW_ROW_LIMIT, + ImproveDigitalReportingSync, +) + + +class CapturingReporting: + def __init__(self, currencies=None): + self.payloads: list[dict] = [] + self._currencies = currencies + + def preview(self, payload): + self.payloads.append(payload) + return {"rows": []} + + def available_currencies(self): + if isinstance(self._currencies, Exception): + raise self._currencies + return self._currencies + + +def make_syncer(reporting, currency="EUR") -> ImproveDigitalReportingSync: + client = SimpleNamespace(reporting=reporting) + return ImproveDigitalReportingSync(client, "t1", session=SimpleNamespace(commit=lambda: None), currency=currency) + + +def fake_buy(campaign_id: int, start_days_ago: int): + return SimpleNamespace( + external_id=f"improvedigital_{campaign_id}", + media_buy_id=f"mb_{campaign_id}", + start_date=(datetime.now(UTC) - timedelta(days=start_days_ago)).date(), + ) + + +def run_with_buys(syncer, buys): + with ( + patch("src.adapters.improvedigital.reporting_sync.MediaBuyRepository") as repo_cls, + patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"), + ): + repo_cls.return_value.get_active.return_value = buys + return syncer.run() + + +class TestFlightAwareWindow: + def test_window_spans_the_oldest_active_flight(self): + reporting = CapturingReporting(currencies=[]) + run_with_buys(make_syncer(reporting), [fake_buy(101, 100), fake_buy(102, 5)]) + + date_range = reporting.payloads[0]["report_generation_request"]["date_range"] + assert date_range == {"relative": {"from_count": 101, "from_unit": "DAY", "to_count": 0, "to_unit": "DAY"}} + + def test_window_clamps_to_min_and_max(self): + reporting = CapturingReporting(currencies=[]) + run_with_buys(make_syncer(reporting), [fake_buy(101, 2)]) + assert reporting.payloads[0]["report_generation_request"]["date_range"]["relative"]["from_count"] == ( + MIN_WINDOW_DAYS + ) + + run_with_buys(make_syncer(reporting), [fake_buy(101, 900)]) + assert reporting.payloads[1]["report_generation_request"]["date_range"]["relative"]["from_count"] == ( + MAX_WINDOW_DAYS + ) + + def test_explicit_campaign_ids_keep_the_quick_window(self): + reporting = CapturingReporting(currencies=[]) + syncer = make_syncer(reporting) + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"]) + assert reporting.payloads[0]["report_generation_request"]["date_range"] == {"quick": "LAST_31_DAYS"} + + def test_preview_requests_the_spec_row_cap(self): + reporting = CapturingReporting(currencies=[]) + syncer = make_syncer(reporting) + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"]) + assert PREVIEW_ROW_LIMIT == 2000 + assert reporting.payloads[0]["rows"] == 2000 + + +class TestTargetedRunWindow: + def test_explicit_ids_with_earliest_start_use_flight_window(self): + reporting = CapturingReporting(currencies=[]) + syncer = make_syncer(reporting) + start = (datetime.now(UTC) - timedelta(days=100)).date() + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"], earliest_start=start) + assert reporting.payloads[0]["report_generation_request"]["date_range"] == { + "relative": {"from_count": 101, "from_unit": "DAY", "to_count": 0, "to_unit": "DAY"} + } + + +class TestDeliveryCacheMissFallback: + """get_media_buy_delivery pulls the Report API live on a cache miss — + the GAM-details-page behaviour — writing through the stats cache.""" + + def _adapter(self): + from src.adapters.improvedigital import ImproveDigitalAdapter + + class FakePrincipal: + name = "p" + principal_id = "p1" + platform_mappings: dict = {} + + def get_adapter_id(self, adapter): + return None + + adapter = ImproveDigitalAdapter( + config={ + "client_id": "app-1", + "client_secret": "s3cret", + "buying_entity_id": 421, + "buying_entity_office_id": 5068, + "api_base_url": "https://api.360yielddev.example", + }, + principal=FakePrincipal(), + dry_run=False, + tenant_id="t1", + ) + adapter._client = SimpleNamespace(reporting=object()) + return adapter + + def _period(self, days: int = 100): + from src.core.schemas import ReportingPeriod + + start = datetime.now(UTC) - timedelta(days=days) + return ReportingPeriod(start=start, end=datetime.now(UTC)) + + def test_classic_campaign_counters_answer_lifetime_requests(self): + """The details page (3-year 'all-time' window) reads lifetime + delivery straight off the Classic campaign entity — no cache, no + Report API — mirroring GAM's live per-view query. Windowed + requests (< 1 year) skip this path so lifetime counters are never + mislabeled as a period's delivery.""" + adapter = self._adapter() + adapter._client = SimpleNamespace( + campaigns=SimpleNamespace( + get_campaign=lambda cid: {"id": cid, "currency": "EUR"}, + list_line_items=lambda cid: { + "line_items": [ + {"id": 202, "impressions": 31, "clicks": 2, "completes": 1, "spent": 0.06}, + {"id": 203, "impressions": 9, "clicks": 0, "completes": 0, "spent": 0.01}, + ] + }, + ), + reporting=object(), + ) + response = adapter.get_media_buy_delivery("improvedigital_101", self._period(days=3 * 365), datetime.now(UTC)) + assert response.totals.impressions == 40 + assert response.totals.clicks == 2 + assert round(response.totals.spend, 2) == 0.07 + assert {p.package_id for p in response.by_package} == {"202", "203"} + assert response.currency == "EUR" + + def test_windowed_requests_skip_the_lifetime_counters(self): + """A 100-day window must not be answered with lifetime numbers — + it goes to the report-backed cache instead.""" + adapter = self._adapter() + campaign_reads: list[int] = [] + adapter._client = SimpleNamespace( + campaigns=SimpleNamespace( + get_campaign=lambda cid: campaign_reads.append(cid) or {"id": cid, "currency": "EUR"}, + list_line_items=lambda cid: {"line_items": [{"id": 202, "impressions": 999, "spent": 9.9}]}, + ), + reporting=object(), + ) + cached = [ + SimpleNamespace( + line_item_id="202", + impressions=10, + clicks=1, + completed_views=None, + spend_micros=100_000, + currency="EUR", + ) + ] + with ( + patch("src.core.database.database_session.get_db_session"), + patch( + "src.core.database.repositories.improvedigital_line_item_stats.ImproveDigitalLineItemStatsRepository" + ) as repo_cls, + ): + repo_cls.return_value.list_by_campaign.return_value = cached + response = adapter.get_media_buy_delivery("improvedigital_101", self._period(days=100), datetime.now(UTC)) + assert campaign_reads == [] # live path never touched + assert response.totals.impressions == 10 + + def test_cache_miss_triggers_targeted_pull_and_rereads(self): + adapter = self._adapter() + cached_after_pull = [ + SimpleNamespace( + line_item_id="202", + impressions=1000, + clicks=10, + completed_views=None, + spend_micros=4_000_000, + currency="EUR", + ) + ] + with ( + patch("src.core.database.database_session.get_db_session"), + patch( + "src.core.database.repositories.improvedigital_line_item_stats.ImproveDigitalLineItemStatsRepository" + ) as repo_cls, + patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalReportingSync") as sync_cls, + ): + repo_cls.return_value.list_by_campaign.side_effect = [[], cached_after_pull] + response = adapter.get_media_buy_delivery("improvedigital_101", self._period(), datetime.now(UTC)) + + run_kwargs = sync_cls.return_value.run.call_args + assert run_kwargs.args == (["101"],) or run_kwargs.kwargs.get("campaign_ids") == ["101"] + assert run_kwargs.kwargs["earliest_start"] == self._period().start.date() + assert response.totals.impressions == 1000 + assert response.totals.clicks == 10 + + def test_scope_pending_stays_soft_delivery_unavailable(self): + from src.adapters.base import DeliveryDataUnavailable + from src.adapters.improvedigital.reporting_sync import ReportingScopeNotGranted + + adapter = self._adapter() + with ( + patch("src.core.database.database_session.get_db_session"), + patch( + "src.core.database.repositories.improvedigital_line_item_stats.ImproveDigitalLineItemStatsRepository" + ) as repo_cls, + patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalReportingSync") as sync_cls, + ): + repo_cls.return_value.list_by_campaign.return_value = [] + sync_cls.return_value.run.side_effect = ReportingScopeNotGranted() + import pytest + + with pytest.raises(DeliveryDataUnavailable): + adapter.get_media_buy_delivery("improvedigital_101", self._period(), datetime.now(UTC)) + + +class TestGenerationFirstFlow: + def test_generation_submitted_before_preview_with_same_request(self): + calls: list[tuple[str, dict]] = [] + + class Reporting: + def submit_generation(self, body): + calls.append(("generation", body)) + return {"report_generation_id": "gen-1", "status_name": "ENQUEUED"} + + def preview(self, payload): + calls.append(("preview", payload)) + return {"rows": []} + + def available_currencies(self): + return [] + + syncer = make_syncer(Reporting()) + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"]) + + assert [name for name, _ in calls] == ["generation", "preview"] + gen_body = calls[0][1] + preview_request = calls[1][1]["report_generation_request"] + assert "action" not in gen_body # generation takes the bare request + assert preview_request["action"] == "PREVIEW_REPORT" + assert gen_body["filters"] == preview_request["filters"] + assert gen_body["date_range"] == preview_request["date_range"] + + def test_generation_failure_never_blocks_the_preview(self): + class Reporting: + def __init__(self): + self.previewed = False + + def submit_generation(self, body): + raise ImproveDigitalError("generation down") + + def preview(self, payload): + self.previewed = True + return {"rows": []} + + def available_currencies(self): + return [] + + reporting = Reporting() + syncer = make_syncer(reporting) + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + result = syncer.run(campaign_ids=["101"]) + assert reporting.previewed + assert result.error is None + + def test_earliest_start_for_scopes_to_requested_campaigns(self): + """Targeted syncs (admin campaign filter) derive a flight-aware + window from the selected buys — never a partial window that would + clobber lifetime cache totals.""" + syncer = make_syncer(CapturingReporting(currencies=[])) + with patch("src.adapters.improvedigital.reporting_sync.MediaBuyRepository") as repo_cls: + repo_cls.return_value.get_active.return_value = [fake_buy(101, 90), fake_buy(102, 300)] + assert syncer.earliest_start_for(["101"]) == fake_buy(101, 90).start_date + assert syncer.earliest_start_for(["101", "102"]) == fake_buy(102, 300).start_date + assert syncer.earliest_start_for(["999"]) is None + + +class TestMediaBuyDeliveryRollup: + def test_synced_rows_update_owning_buys_delivered_columns(self): + from decimal import Decimal + + reporting = CapturingReporting(currencies=[]) + + class FullReporting(CapturingReporting): + def preview(self, payload): + super().preview(payload) + return { + "rows": [ + { + "campaign_id": "101", + "line_item_id": "202", + "impressions": "1000", + "clicks": "10", + "advertiser_payout": "4.5", + "complete": "0", + }, + { + "campaign_id": "101", + "line_item_id": "203", + "impressions": "500", + "clicks": "0", + "advertiser_payout": "1.5", + "complete": "0", + }, + ] + } + + buy = SimpleNamespace( + external_id="improvedigital_101", + media_buy_id="mb_x", + delivered_amount=None, + delivered_impressions=None, + delivery_synced_at=None, + ) + syncer = make_syncer(FullReporting(currencies=[])) + with ( + patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"), + patch("src.adapters.improvedigital.reporting_sync.MediaBuyRepository") as buy_repo_cls, + ): + buy_repo_cls.return_value.get_active.return_value = [buy] + result = syncer.run(campaign_ids=["101"]) + + assert result.rows_updated == 2 + # Written through the repository (matching the GAM rollup), not raw ORM. + update_call = buy_repo_cls.return_value.update_fields.call_args + assert update_call.args == ("mb_x",) + assert update_call.kwargs["delivered_amount"] == Decimal("6.0") + assert update_call.kwargs["delivered_impressions"] == 1500 + assert update_call.kwargs["delivery_synced_at"] is not None + + def test_update_media_buys_false_skips_the_rollup(self): + """Read paths that warm the cache must never mutate delivered_*.""" + + class FullReporting(CapturingReporting): + def preview(self, payload): + super().preview(payload) + return { + "rows": [ + { + "campaign_id": "101", + "line_item_id": "202", + "impressions": "10", + "clicks": "0", + "advertiser_payout": "0.1", + "complete": "0", + } + ] + } + + syncer = make_syncer(FullReporting(currencies=[])) + with ( + patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"), + patch("src.adapters.improvedigital.reporting_sync.MediaBuyRepository") as buy_repo_cls, + ): + result = syncer.run(campaign_ids=["101"], update_media_buys=False) + assert result.rows_updated == 1 + buy_repo_cls.return_value.update_fields.assert_not_called() + + +class TestCurrencyResolution: + def test_static_map_answers_known_codes_without_a_lookup(self): + reporting = CapturingReporting(currencies=ImproveDigitalError("must not be called")) + syncer = make_syncer(reporting, currency="GBP") + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"]) + assert reporting.payloads[0]["report_generation_request"]["currency_id"] == 3 + + def test_unknown_code_resolves_from_live_dictionary(self): + reporting = CapturingReporting(currencies=[{"id": 99, "code": "XXX"}]) + syncer = make_syncer(reporting, currency="XXX") + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"]) + assert reporting.payloads[0]["report_generation_request"]["currency_id"] == 99 + + def test_falls_back_to_static_map_when_lookup_fails(self): + reporting = CapturingReporting(currencies=ImproveDigitalError("boom")) + syncer = make_syncer(reporting, currency="USD") + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"]) + assert reporting.payloads[0]["report_generation_request"]["currency_id"] == 2 + + def test_falls_back_when_client_lacks_the_method(self): + class BareReporting: + def __init__(self): + self.payloads = [] + + def preview(self, payload): + self.payloads.append(payload) + return {"rows": []} + + reporting = BareReporting() + syncer = make_syncer(reporting, currency="EUR") + with patch("src.adapters.improvedigital.reporting_sync.ImproveDigitalLineItemStatsRepository"): + syncer.run(campaign_ids=["101"]) + assert reporting.payloads[0]["report_generation_request"]["currency_id"] == 1 From 5a36df18c1a7d3851e1d38f02a9c254aeb22972c Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 18 Aug 2026 17:55:18 +0600 Subject: [PATCH 75/90] fix: fexisting campaigns check prior sync report --- src/adapters/improvedigital/reporting_sync.py | 60 +++++++++++++++++++ src/admin/blueprints/adapters.py | 9 ++- .../test_improvedigital_reporting_filters.py | 8 +-- .../test_improvedigital_reporting_window.py | 49 +++++++++++++++ 4 files changed, 121 insertions(+), 5 deletions(-) diff --git a/src/adapters/improvedigital/reporting_sync.py b/src/adapters/improvedigital/reporting_sync.py index 21235a9c6e..1513fda9de 100644 --- a/src/adapters/improvedigital/reporting_sync.py +++ b/src/adapters/improvedigital/reporting_sync.py @@ -32,6 +32,7 @@ ImproveDigitalClient, ImproveDigitalError, ImproveDigitalForbiddenError, + ImproveDigitalNotFoundError, ) from src.core.database.repositories.improvedigital_line_item_stats import ( ImproveDigitalLineItemStatsRepository, @@ -115,6 +116,55 @@ def resolve_currency_id(client: Any, currency: str | None) -> int: return 1 +# In-process campaign-existence verdicts: (tenant_id, campaign_id) -> +# (exists, checked_at_monotonic). Campaign ids are a global sequence and +# never reused, so verdicts are stable; the TTL only bounds staleness for +# campaigns deleted upstream after a positive verdict. +_EXISTENCE_TTL_SECONDS = 3600.0 +_campaign_existence: dict[tuple[str, str], tuple[bool, float]] = {} + + +def filter_to_existing_campaigns(client: Any, tenant_id: str, campaign_ids: list[str]) -> list[str]: + """Drop campaign ids the current environment's Classic API doesn't know. + + The Report API warehouse is NOT environment-scoped (observed live + 2026-08-18: prod ``/report/ext/preview`` returns delivery for dev + campaign ids that prod's own Classic API 404s), so a buy booked + against a different ``api_base_url`` would leak foreign metrics into + the cache and UI. The Classic campaign GET *is* scoped — use it as the + gate. Verdicts are cached in-process for an hour; transient upstream + errors (or clients without a campaigns surface, e.g. test fakes) keep + the campaign — better a retry next run than silently dropped data. + """ + kept: list[str] = [] + now = time.monotonic() + for campaign_id in campaign_ids: + key = (tenant_id, str(campaign_id)) + cached = _campaign_existence.get(key) + if cached is not None and now - cached[1] < _EXISTENCE_TTL_SECONDS: + exists = cached[0] + else: + try: + client.campaigns.get_campaign(int(campaign_id)) + exists = True + except ImproveDigitalNotFoundError: + exists = False + except (ImproveDigitalError, AttributeError, TypeError, ValueError): + kept.append(campaign_id) + continue + _campaign_existence[key] = (exists, now) + if exists: + kept.append(campaign_id) + else: + logger.warning( + "Improve Digital: campaign %s does not exist on this environment — excluded from reporting " + "(tenant %s; the buy was likely booked against a different api_base_url — close it out)", + campaign_id, + tenant_id, + ) + return kept + + def quick_date_range(quick: str) -> dict[str, Any]: """A quick range as the wire actually accepts it. @@ -262,6 +312,16 @@ def run( logger.info("Improve Digital reporting sync tenant=%s: no active campaigns, nothing to do", self._tenant_id) return ReportingSyncResult(rows_updated=0, campaigns_covered=0) + # Environment gate: never ask the (unscoped) Report API about + # campaigns this environment's Classic API doesn't know. + ids = filter_to_existing_campaigns(self._client, self._tenant_id, ids) + if not ids: + logger.info( + "Improve Digital reporting sync tenant=%s: no campaigns exist on this environment, nothing to do", + self._tenant_id, + ) + return ReportingSyncResult(rows_updated=0, campaigns_covered=0) + date_range = self._report_date_range(earliest_start) currency_id = self._currency_id() request_body = build_report_request( diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index f80afc3be9..9c09c56921 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1576,7 +1576,9 @@ def _improvedigital_reporting_payload( } -def _improvedigital_live_stat_rows(client, campaign_ids: list[int], quick_range: str, timezone: str, currency: str): +def _improvedigital_live_stat_rows( + client, tenant_id: str, campaign_ids: list[int], quick_range: str, timezone: str, currency: str +): """Query the Report API live for a date-filtered reporting-page view. The stats cache holds one aggregate row per line item with no time @@ -1594,10 +1596,14 @@ def _improvedigital_live_stat_rows(client, campaign_ids: list[int], quick_range: _as_float, _as_int, build_report_request, + filter_to_existing_campaigns, quick_date_range, resolve_currency_id, ) + # Environment gate — the Report API warehouse serves foreign-environment + # campaign ids; only query campaigns this environment actually knows. + campaign_ids = [int(cid) for cid in filter_to_existing_campaigns(client, tenant_id, [str(c) for c in campaign_ids])] if not campaign_ids: return [] # Shared request builder + currency resolver — one wire contract with @@ -1721,6 +1727,7 @@ def get_improvedigital_reporting(tenant_id, **kwargs): try: stat_rows = _improvedigital_live_stat_rows( client, + tenant_id, sorted(int(cid) for cid in buys_by_campaign), date_range, timezone_arg, diff --git a/tests/unit/test_improvedigital_reporting_filters.py b/tests/unit/test_improvedigital_reporting_filters.py index 680806ea9b..97979dc906 100644 --- a/tests/unit/test_improvedigital_reporting_filters.py +++ b/tests/unit/test_improvedigital_reporting_filters.py @@ -108,7 +108,7 @@ def test_builds_quick_range_request_and_stat_shaped_rows(self): } ] ) - rows = _improvedigital_live_stat_rows(client, [314446, 314450], "LAST_7_DAYS", "UTC", "EUR") + rows = _improvedigital_live_stat_rows(client, "t1", [314446, 314450], "LAST_7_DAYS", "UTC", "EUR") request = captured["payload"]["report_generation_request"] assert request["date_range"] == {"quick": "LAST_7_DAYS"} @@ -130,7 +130,7 @@ def test_today_translates_to_the_relative_range_the_wire_accepts(self): from src.admin.blueprints.adapters import _improvedigital_live_stat_rows client, captured = self._client([]) - _improvedigital_live_stat_rows(client, [314446], "TODAY", "UTC", "EUR") + _improvedigital_live_stat_rows(client, "t1", [314446], "TODAY", "UTC", "EUR") assert captured["payload"]["report_generation_request"]["date_range"] == { "relative": {"from_count": 1, "from_unit": "DAY", "to_count": 0, "to_unit": "DAY"} } @@ -139,7 +139,7 @@ def test_no_campaigns_short_circuits_without_api_call(self): from src.admin.blueprints.adapters import _improvedigital_live_stat_rows client, captured = self._client([]) - assert _improvedigital_live_stat_rows(client, [], "TODAY", "UTC", "EUR") == [] + assert _improvedigital_live_stat_rows(client, "t1", [], "TODAY", "UTC", "EUR") == [] assert "payload" not in captured def test_live_rows_flow_through_the_shared_payload_builder(self): @@ -157,7 +157,7 @@ def test_live_rows_flow_through_the_shared_payload_builder(self): } ] ) - rows = _improvedigital_live_stat_rows(client, [314446], "TODAY", "UTC", "EUR") + rows = _improvedigital_live_stat_rows(client, "t1", [314446], "TODAY", "UTC", "EUR") payload = _improvedigital_reporting_payload(rows, BUYS, "EUR") assert payload["rows"][0]["order_name"] == "Acme Summer Push" assert payload["totals"]["spend"] == 0.25 diff --git a/tests/unit/test_improvedigital_reporting_window.py b/tests/unit/test_improvedigital_reporting_window.py index 1fb30ba2f2..b00662dbc4 100644 --- a/tests/unit/test_improvedigital_reporting_window.py +++ b/tests/unit/test_improvedigital_reporting_window.py @@ -393,6 +393,55 @@ def preview(self, payload): buy_repo_cls.return_value.update_fields.assert_not_called() +class TestEnvironmentGate: + """The Report API warehouse serves foreign-environment campaign ids + (observed live); the Classic campaign GET is env-scoped and gates + which campaigns any report query may ask about.""" + + def _client(self, existing: set[int]): + from src.adapters.improvedigital.client import ImproveDigitalNotFoundError + + calls: list[int] = [] + + def get_campaign(cid): + calls.append(cid) + if cid in existing: + return {"id": cid} + raise ImproveDigitalNotFoundError("Unknown Classic Campaign") + + return SimpleNamespace(campaigns=SimpleNamespace(get_campaign=get_campaign)), calls + + def test_foreign_campaigns_are_dropped(self): + from src.adapters.improvedigital.reporting_sync import filter_to_existing_campaigns + + client, _ = self._client(existing={370320}) + kept = filter_to_existing_campaigns(client, "t-gate-1", ["314446", "370320"]) + assert kept == ["370320"] + + def test_verdicts_are_cached(self): + from src.adapters.improvedigital.reporting_sync import filter_to_existing_campaigns + + client, calls = self._client(existing={370320}) + filter_to_existing_campaigns(client, "t-gate-2", ["314446", "370320"]) + filter_to_existing_campaigns(client, "t-gate-2", ["314446", "370320"]) + assert calls == [314446, 370320] # second pass answered from cache + + def test_transient_errors_keep_the_campaign(self): + from src.adapters.improvedigital.reporting_sync import filter_to_existing_campaigns + + def get_campaign(cid): + raise ImproveDigitalError("upstream hiccup") + + client = SimpleNamespace(campaigns=SimpleNamespace(get_campaign=get_campaign)) + assert filter_to_existing_campaigns(client, "t-gate-3", ["370320"]) == ["370320"] + + def test_clients_without_campaign_surface_keep_everything(self): + from src.adapters.improvedigital.reporting_sync import filter_to_existing_campaigns + + client = SimpleNamespace(reporting=object()) # test fakes + assert filter_to_existing_campaigns(client, "t-gate-4", ["101", "102"]) == ["101", "102"] + + class TestCurrencyResolution: def test_static_map_answers_known_codes_without_a_lookup(self): reporting = CapturingReporting(currencies=ImproveDigitalError("must not be called")) From ebb2dda4bb48bdf471d51f713790d62e92ad5f04 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 18 Aug 2026 18:50:06 +0600 Subject: [PATCH 76/90] Fix media buy details label --- templates/media_buy_detail.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/templates/media_buy_detail.html b/templates/media_buy_detail.html index 706aff5f88..e396157f67 100644 --- a/templates/media_buy_detail.html +++ b/templates/media_buy_detail.html @@ -147,9 +147,14 @@

📊 Delivery Metrics

By Package

+ {# Improve Digital aggregates delivery per platform line item, so the + by_package id is the line item id — label it as such. #} + {% set pkg_label = 'Line Item' if media_buy.adapter_type == 'improvedigital' else 'Package' %} {% for pkg in delivery_metrics.by_package %}
-
{{ pkg.package_id }}
+
+ {{ pkg_label }}: {{ pkg.package_id }} +
Impressions: From 348b561c864ace605783d869a5dec73d8c3c6f8f Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Mon, 7 Sep 2026 14:53:51 +0600 Subject: [PATCH 77/90] Feature/tenant wise adserver (#63) * Feat: Adapter configuration locking * feat: store adapter config lock in DB and freeze connection identity fields --- ...b8c7d6e5f4_add_adapter_config_locked_at.py | 63 +++ src/adapters/gam/managers/sync.py | 10 + src/admin/blueprints/adapters.py | 38 ++ src/admin/blueprints/gam.py | 21 + src/admin/blueprints/settings.py | 34 ++ src/admin/blueprints/tenants.py | 8 + src/core/database/adapter_config_lock.py | 165 ++++++ src/core/database/models.py | 11 + .../database/repositories/adapter_config.py | 21 + src/services/adapter_sync_orchestration.py | 6 + src/services/background_sync_service.py | 7 + .../adapters/freewheel/connection_config.html | 6 +- .../google_ad_manager/connection_config.html | 21 +- .../improvedigital/connection_config.html | 9 +- templates/tenant_settings.html | 36 +- tests/integration/test_adapter_config_lock.py | 476 ++++++++++++++++++ 16 files changed, 908 insertions(+), 24 deletions(-) create mode 100644 alembic/versions/a9b8c7d6e5f4_add_adapter_config_locked_at.py create mode 100644 src/core/database/adapter_config_lock.py create mode 100644 tests/integration/test_adapter_config_lock.py diff --git a/alembic/versions/a9b8c7d6e5f4_add_adapter_config_locked_at.py b/alembic/versions/a9b8c7d6e5f4_add_adapter_config_locked_at.py new file mode 100644 index 0000000000..3e4a4cfbd0 --- /dev/null +++ b/alembic/versions/a9b8c7d6e5f4_add_adapter_config_locked_at.py @@ -0,0 +1,63 @@ +"""add adapter_config.config_locked_at + +Explicit stored lock state for the adapter-configuration lock +(src/core/database/adapter_config_lock.py): stamped when a tenant's first +inventory sync completes; while non-NULL the ad server configuration is +frozen and only credentials remain editable. Cleared only by platform ops +under super_admin_override. + +Backfill: tenants that already synced inventory (a terminal-success +sync_jobs row of sync_type='inventory', or legacy gam_inventory rows from +before sync history existed) are stamped locked at migration time, so the +lock applies retroactively. + +Revision ID: a9b8c7d6e5f4 +Revises: impd02b3c4d5 +Create Date: 2026-09-04 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "a9b8c7d6e5f4" +down_revision: str | Sequence[str] | None = "impd02b3c4d5" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "adapter_config", + sa.Column( + "config_locked_at", + sa.DateTime(timezone=True), + nullable=True, + comment=( + "Set when the tenant's first inventory sync completed; while non-NULL " + "the ad server configuration is frozen (only credentials editable)." + ), + ), + ) + + op.execute( + """ + UPDATE adapter_config + SET config_locked_at = NOW() + WHERE config_locked_at IS NULL + AND ( + tenant_id IN ( + SELECT DISTINCT tenant_id FROM sync_jobs + WHERE sync_type = 'inventory' AND status IN ('completed', 'success') + ) + OR tenant_id IN (SELECT DISTINCT tenant_id FROM gam_inventory) + ) + """ + ) + + +def downgrade() -> None: + op.drop_column("adapter_config", "config_locked_at") diff --git a/src/adapters/gam/managers/sync.py b/src/adapters/gam/managers/sync.py index cf4696a62d..7b90451cc7 100644 --- a/src/adapters/gam/managers/sync.py +++ b/src/adapters/gam/managers/sync.py @@ -134,6 +134,11 @@ def sync_inventory( # TODO: SyncJob.completed_at should use Mapped[datetime] not Mapped[DateTime] sync_job.completed_at = datetime.now(UTC) sync_job.summary = json.dumps(summary) + # First successful inventory sync freezes the tenant's ad server + # configuration (adapter_config_lock.py). + from src.core.database.repositories.adapter_config import AdapterConfigRepository + + AdapterConfigRepository(db_session, self.tenant_id).mark_config_locked() db_session.commit() logger.info(f"Inventory sync completed for tenant {self.tenant_id}: {summary}") @@ -366,6 +371,11 @@ def sync_selective( # TODO: SyncJob.completed_at should use Mapped[datetime] not Mapped[DateTime] sync_job.completed_at = datetime.now(UTC) sync_job.summary = json.dumps(summary) + # Selective syncs persist inventory too — same lock trigger as a + # full inventory sync (adapter_config_lock.py). + from src.core.database.repositories.adapter_config import AdapterConfigRepository + + AdapterConfigRepository(db_session, self.tenant_id).mark_config_locked() db_session.commit() logger.info(f"Selective sync completed for tenant {self.tenant_id}: {summary}") diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index 9c09c56921..b3d448cb9b 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -203,6 +203,13 @@ def save_adapter_config(tenant_id, **kwargs): "config": { ... adapter-specific config ... } } """ + from src.core.database.adapter_config_lock import ( + ADAPTER_LOCKED_MESSAGE, + LOCKED_CONFIG_JSON_FIELDS, + AdapterConfigLockedError, + is_adapter_config_locked, + ) + try: data = request.get_json() if not data: @@ -271,6 +278,33 @@ def save_adapter_config(tenant_id, **kwargs): stmt = select(AdapterConfig).filter_by(tenant_id=tenant_id) adapter_config = session.scalars(stmt).first() + # Once inventory has been synced, the connection identity is + # frozen: no adapter switch (saving a different adapter_type also + # flips tenant.ad_server below), and the client_id/client_secret + # keys inside config_json are read-only. Compared post-validation + # so both sides are plaintext — the schema's field validator + # decrypts stored ciphertext, which is non-deterministic at rest + # and useless to compare directly. Other config fields (passwords, + # tokens, environment, …) stay editable. + if adapter_config and is_adapter_config_locked(session, tenant_id): + if adapter_config.adapter_type != adapter_type: + return jsonify({"success": False, "error": ADAPTER_LOCKED_MESSAGE}), 403 + + if validated_config is not None and adapter_config.config_json: + locked_fields = [f for f in LOCKED_CONFIG_JSON_FIELDS if f in type(validated_config).model_fields] + stored_model = None + if locked_fields: + try: + stored_model = type(validated_config).model_validate(adapter_config.config_json) + except ValidationError: + # Legacy/partial stored config that no longer + # validates — nothing comparable to protect. + stored_model = None + if stored_model is not None: + for field_name in locked_fields: + if getattr(validated_config, field_name) != getattr(stored_model, field_name): + return jsonify({"success": False, "error": ADAPTER_LOCKED_MESSAGE}), 403 + if not adapter_config: adapter_config = AdapterConfig( tenant_id=tenant_id, @@ -327,6 +361,10 @@ def save_adapter_config(tenant_id, **kwargs): return jsonify({"success": True, "adapter_type": adapter_type}) + except AdapterConfigLockedError as e: + logger.info(f"Blocked adapter config change on locked tenant {tenant_id}: {e}") + return jsonify({"success": False, "error": ADAPTER_LOCKED_MESSAGE}), 403 + except Exception as e: logger.error(f"Error saving adapter config: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 diff --git a/src/admin/blueprints/gam.py b/src/admin/blueprints/gam.py index c98cbee00f..e7bc625cf6 100644 --- a/src/admin/blueprints/gam.py +++ b/src/admin/blueprints/gam.py @@ -342,6 +342,12 @@ def detect_gam_network(tenant_id): @require_tenant_access(role=("admin",)) def configure_gam(tenant_id): """Save GAM configuration for a tenant.""" + from src.core.database.adapter_config_lock import ( + ADAPTER_LOCKED_MESSAGE, + AdapterConfigLockedError, + is_adapter_config_locked, + ) + try: # Try to get JSON - use force=True to handle potential Content-Type issues data = request.get_json(force=True, silent=True) @@ -413,6 +419,17 @@ def configure_gam(tenant_id): adapter_config = db_session.scalars(select(AdapterConfig).filter_by(tenant_id=tenant_id)).first() + # Once inventory has been synced, the ad server configuration is + # frozen: no switching another adapter's tenant onto GAM, and no + # changing (or clearing) the GAM network code. Credential rotation + # for the same network code stays allowed; any other field change + # is caught at commit by the model guard (403 in the except below). + if is_adapter_config_locked(db_session, tenant_id): + current_adapter = tenant.ad_server or (adapter_config.adapter_type if adapter_config else None) + stored_network_code = adapter_config.gam_network_code if adapter_config else None + if (current_adapter and current_adapter != "google_ad_manager") or network_code != stored_network_code: + return jsonify({"success": False, "error": ADAPTER_LOCKED_MESSAGE}), 403 + if not adapter_config: adapter_config = AdapterConfig(tenant_id=tenant_id, adapter_type="google_ad_manager") db_session.add(adapter_config) @@ -478,6 +495,10 @@ def configure_gam(tenant_id): } ) + except AdapterConfigLockedError as e: + logger.info(f"Blocked GAM config change on locked tenant {tenant_id}: {e}") + return jsonify({"success": False, "error": ADAPTER_LOCKED_MESSAGE}), 403 + except Exception as e: logger.error(f"Error saving GAM configuration for tenant {tenant_id}: {e}") return ( diff --git a/src/admin/blueprints/settings.py b/src/admin/blueprints/settings.py index ca455e6415..b663910288 100644 --- a/src/admin/blueprints/settings.py +++ b/src/admin/blueprints/settings.py @@ -426,6 +426,12 @@ def update_general(tenant_id): ) def update_adapter(tenant_id): """Update the active adapter for a tenant.""" + from src.core.database.adapter_config_lock import ( + ADAPTER_LOCKED_MESSAGE, + AdapterConfigLockedError, + is_adapter_config_locked, + ) + try: # Support both JSON (from our frontend) and form data (from tests) if request.is_json: @@ -451,6 +457,25 @@ def update_adapter(tenant_id): flash("No adapter configured", "error") return redirect(url_for("tenants.tenant_settings", tenant_id=tenant_id, section="adapter")) + # Once inventory has been synced, the ad server configuration is + # frozen — switching adapters (or clearing the GAM config via + # edit_config) would orphan synced inventory, products, and + # media-buy history. Field-level changes (templates, AXE keys, …) + # are caught at commit by the model guard and returned as 403 in + # the except clause below. + adapter_locked = is_adapter_config_locked(db_session, tenant_id) + requested_action = request.json.get("action") if request.is_json and request.json else None + current_adapter = tenant.ad_server or ( + tenant.adapter_config.adapter_type if tenant.adapter_config else None + ) + if adapter_locked and ( + requested_action == "edit_config" or (current_adapter and new_adapter != current_adapter) + ): + if request.is_json: + return jsonify({"success": False, "error": ADAPTER_LOCKED_MESSAGE}), 403 + flash(ADAPTER_LOCKED_MESSAGE, "error") + return redirect(url_for("tenants.tenant_settings", tenant_id=tenant_id, section="adapter")) + # Update or create adapter config adapter_config_obj = tenant.adapter_config if adapter_config_obj: @@ -593,6 +618,15 @@ def update_adapter(tenant_id): flash(f"Adapter changed to {new_adapter}", "success") return redirect(url_for("tenants.tenant_settings", tenant_id=tenant_id, section="adapter")) + except AdapterConfigLockedError as e: + logger.info(f"Blocked adapter config change on locked tenant {tenant_id}: {e}") + + if request.is_json: + return jsonify({"success": False, "error": ADAPTER_LOCKED_MESSAGE}), 403 + + flash(ADAPTER_LOCKED_MESSAGE, "error") + return redirect(url_for("tenants.tenant_settings", tenant_id=tenant_id, section="adapter")) + except Exception as e: logger.error(f"Error updating adapter: {e}", exc_info=True) diff --git a/src/admin/blueprints/tenants.py b/src/admin/blueprints/tenants.py index baa72d9f9c..aa8ae6e27f 100644 --- a/src/admin/blueprints/tenants.py +++ b/src/admin/blueprints/tenants.py @@ -267,6 +267,13 @@ def tenant_settings(tenant_id, section=None): if adapter_config_obj and adapter_config_obj.adapter_type == "google_ad_manager": oauth_configured = bool(adapter_config_obj.gam_refresh_token) + # Ad server identity is frozen once inventory has been synced — + # the template disables the adapter picker and hides "Edit + # Configuration" when this is set. + from src.core.database.adapter_config_lock import is_adapter_config_locked + + adapter_locked = is_adapter_config_locked(db_session, tenant_id) + # Check if GAM OAuth environment variables are configured gam_oauth_configured = bool( os.environ.get("GAM_OAUTH_CLIENT_ID") and os.environ.get("GAM_OAUTH_CLIENT_SECRET") @@ -357,6 +364,7 @@ def tenant_settings(tenant_id, section=None): section=section or "general", active_adapter=active_adapter, adapter_config=adapter_config_dict, # Use dict format + adapter_locked=adapter_locked, oauth_configured=oauth_configured, gam_oauth_configured=gam_oauth_configured, # Environment check for GAM OAuth principals=principals, diff --git a/src/core/database/adapter_config_lock.py b/src/core/database/adapter_config_lock.py new file mode 100644 index 0000000000..8307c102cf --- /dev/null +++ b/src/core/database/adapter_config_lock.py @@ -0,0 +1,165 @@ +"""Model-layer write guard freezing the ad server connection identity after the first inventory sync. + +Once a tenant's first inventory sync completes, the fields that identify the ad +server connection are locked: + +- ``Tenant.ad_server`` and ``AdapterConfig.adapter_type`` — the adapter choice +- ``AdapterConfig.gam_network_code`` — the GAM network +- ``client_id`` / ``client_secret`` inside ``config_json`` — the network seat + for schema-driven adapters (Improve Digital, FreeWheel API-Access) + +Changing any of these after a sync would orphan the synced inventory, product +implementation configs, and media-buy history that reference the old network. A +publisher who needs a different ad server or network must create a new tenant. +Everything else on the adapter configuration (credentials such as the GAM +refresh token or FreeWheel password, naming templates, AXE keys, approval +flags, other ``config_json`` fields) stays editable. + +Lock state is stored explicitly in ``adapter_config.config_locked_at`` +(adapter-agnostic — every inventory-sync completion path stamps it via +:meth:`AdapterConfigRepository.mark_config_locked`, and the migration +backfilled already-synced tenants). ``config_locked_at`` is itself a locked +column: stamping it on an unlocked tenant is free, but clearing it once set +requires ``super_admin_override`` — that is the documented unlock procedure. + +Enforcement mirrors :mod:`src.core.database.embedded_tenant_guard`: SQLAlchemy +``before_update`` listeners compare each locked column's pending value against +the stored DB value and raise :class:`AdapterConfigLockedError` on a real +change. Same-value re-assignment (the settings forms resubmit stored values on +every save) passes. Callers holding one of the embedded-guard auth flags +(``management_api_caller``, ``super_admin_override``, +``platform_background_worker``) bypass the lock — the Tenant Management API and +platform workers remain the ops escape hatch. + +The ``client_id`` / ``client_secret`` keys inside ``config_json`` are enforced +by the ``save_adapter_config`` route rather than these listeners: +``client_secret`` is Fernet-encrypted at rest (non-deterministic ciphertext), +so only the route — which holds the schema that decrypts it — can compare +values meaningfully. The route is the sole UI write path for ``config_json``; +non-UI writers (management API, workers) carry auth flags anyway. + +Inserts are not guarded: a synced tenant always already has its AdapterConfig +row, and the only delete-and-recreate path is the management API, which carries +an auth flag. + +Importing this module attaches the listeners as a side effect; models.py +imports it at the bottom, next to embedded_tenant_guard. +""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import event, select +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import get_history + +# Plain module import (not ``from ... import ``) — this module and +# embedded_tenant_guard are both imported from the bottom of models.py, so a +# name import would raise against the partially-loaded sibling module. +from src.core.database import embedded_tenant_guard as _embedded_guard +from src.core.database.models import AdapterConfig, Tenant + +# Connection-identity columns frozen while config_locked_at is set. The lock +# stamp itself is in the set so that clearing it (unlocking) requires +# super_admin_override; stamping it on an unlocked tenant is unaffected. +LOCKED_TENANT_FIELDS: frozenset[str] = frozenset({"ad_server"}) +LOCKED_ADAPTER_CONFIG_FIELDS: frozenset[str] = frozenset({"adapter_type", "gam_network_code", "config_locked_at"}) + +# config_json keys enforced by the save_adapter_config route (see module +# docstring for why these can't be checked at the listener level). api_base_url +# is the 360Yield host (production vs dev seat) — as much a part of the +# connection identity as the credentials themselves. +LOCKED_CONFIG_JSON_FIELDS: tuple[str, ...] = ("client_id", "client_secret", "api_base_url") + +ADAPTER_LOCKED_MESSAGE = ( + "Ad server configuration is locked: inventory has already been synced with " + "this ad server. The ad server, network code, API base URL, and client " + "credentials cannot be changed. To connect a different ad server or network, " + "create a new tenant." +) + + +class AdapterConfigLockedError(Exception): + """Raised when a locked ad-server identity field is changed on a synced tenant.""" + + +def _tenant_is_locked(connection: Any, tenant_id: str | None) -> bool: + """True when the tenant's stored lock stamp is set. + + ``adapter_config.config_locked_at`` is the single source of truth: the + inventory-sync completion paths stamp it via + :meth:`AdapterConfigRepository.mark_config_locked`, and the migration + backfilled tenants that had already synced. Clearing it (platform ops, + under ``super_admin_override``) unlocks the tenant. + """ + if not tenant_id: + return False + + locked_at = connection.execute( + select(AdapterConfig.config_locked_at).where(AdapterConfig.tenant_id == tenant_id) + ).scalar() + return locked_at is not None + + +def is_adapter_config_locked(session: Session, tenant_id: str) -> bool: + """Route/template-layer predicate for the same lock the listeners enforce.""" + return _tenant_is_locked(session.connection(), tenant_id) + + +def _locked_fields_actually_changed(connection: Any, target: Any, locked: frozenset[str]) -> list[str]: + """Locked columns whose pending value differs from the stored DB value. + + Unlike the embedded guard, equality matters here: the adapter settings + forms re-assign the current adapter/network code on every save, and a + same-value write must not trip the lock. The comparison goes to the + database (not attribute history) because a write to an expired/unloaded + attribute carries no "old" value in its history. + """ + pending: dict[str, Any] = {} + for key in locked: + history = get_history(target, key) + if not history.has_changes(): + continue + pending[key] = history.added[0] if history.added else None + if not pending: + return [] + + cls = type(target) + keys = sorted(pending) + row = connection.execute( + select(*(getattr(cls, key) for key in keys)).where(cls.tenant_id == target.tenant_id) + ).first() + if row is None: + return keys + + stored = dict(zip(keys, row, strict=True)) + return [key for key in keys if pending[key] != stored[key]] + + +def _enforce_lock(connection: Any, target: Any, locked: frozenset[str]) -> None: + changed = _locked_fields_actually_changed(connection, target, locked) + if not changed: + return + + if not _tenant_is_locked(connection, getattr(target, "tenant_id", None)): + return + + if _embedded_guard._caller_is_authorized(target, connection): + return + + raise AdapterConfigLockedError( + f"{type(target).__name__} for tenant {getattr(target, 'tenant_id', '?')!r} " + f"has synced inventory; ad server identity fields {changed} are locked. " + f"{ADAPTER_LOCKED_MESSAGE}" + ) + + +@event.listens_for(Tenant, "before_update") +def _lock_tenant_ad_server(mapper, connection, target): + _enforce_lock(connection, target, LOCKED_TENANT_FIELDS) + + +@event.listens_for(AdapterConfig, "before_update") +def _lock_adapter_config_identity(mapper, connection, target): + _enforce_lock(connection, target, LOCKED_ADAPTER_CONFIG_FIELDS) diff --git a/src/core/database/models.py b/src/core/database/models.py index 4123e9d6fc..47f667f5f7 100644 --- a/src/core/database/models.py +++ b/src/core/database/models.py @@ -1477,6 +1477,13 @@ class AdapterConfig(Base): comment="Schema-validated adapter configuration", ) + # Stamped when the tenant's first inventory sync completes; while non-NULL + # the ad server configuration is frozen (adapter_config_lock.py) and only + # credentials remain editable. Cleared only by platform ops under + # super_admin_override — publishers needing a different ad server or + # network create a new tenant instead. + config_locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), onupdate=func.now() @@ -3206,4 +3213,8 @@ class Proposal(Base): # guard is imported first, it imports this module, which re-imports the guard # mid-load — a bare module binding resolves safely against the partially-loaded # module, whereas a name import would raise. +# adapter_config_lock shares the anchoring rationale: it freezes the ad-server +# identity columns (Tenant.ad_server, AdapterConfig.adapter_type / +# gam_network_code) once the tenant has successfully synced inventory. +from src.core.database import adapter_config_lock as _adapter_config_lock # noqa: E402,F401 from src.core.database import embedded_tenant_guard as _embedded_tenant_guard # noqa: E402,F401 diff --git a/src/core/database/repositories/adapter_config.py b/src/core/database/repositories/adapter_config.py index 1ad883c6c9..7fd468cf68 100644 --- a/src/core/database/repositories/adapter_config.py +++ b/src/core/database/repositories/adapter_config.py @@ -21,6 +21,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import UTC, datetime from typing import Any from sqlalchemy import select @@ -238,3 +239,23 @@ def update_custom_targeting_keys(self, keys: dict[str, str]) -> None: """ config = self.get_by_tenant() # raises if missing config.custom_targeting_keys = keys + + def mark_config_locked(self, locked_at: datetime | None = None) -> bool: + """Stamp ``config_locked_at`` on the tenant's first inventory sync. + + While the stamp is set, the ad server configuration is frozen (see + ``src/core/database/adapter_config_lock.py``). Idempotent: an + already-locked config is left untouched. Missing config rows are + tolerated (returns False) — a sync cannot have run without one, but + completion paths must not crash on that edge. + + Does not commit — caller handles the transaction boundary (the stamp + rides along with the sync-completion write). + + Returns True when the stamp was newly set. + """ + config = self.find_by_tenant() + if config is None or config.config_locked_at is not None: + return False + config.config_locked_at = locked_at or datetime.now(UTC) + return True diff --git a/src/services/adapter_sync_orchestration.py b/src/services/adapter_sync_orchestration.py index 67d755384b..98c866dcd7 100644 --- a/src/services/adapter_sync_orchestration.py +++ b/src/services/adapter_sync_orchestration.py @@ -623,6 +623,12 @@ def _finalize( job.summary = ( f"{result.sync_kind} sync — total={result.total_count} succeeded={result.succeeded} errors={len(result.errors)}" ) + if result.succeeded and job.sync_type == "inventory": + # First successful inventory sync freezes the tenant's ad server + # configuration (adapter_config_lock.py). + from src.core.database.repositories.adapter_config import AdapterConfigRepository + + AdapterConfigRepository(db, job.tenant_id).mark_config_locked() db.flush() if own_session: db.commit() diff --git a/src/services/background_sync_service.py b/src/services/background_sync_service.py index cc642076ff..773e8569ef 100644 --- a/src/services/background_sync_service.py +++ b/src/services/background_sync_service.py @@ -606,6 +606,7 @@ def update_progress(phase: str, phase_num: int, count: int = 0): try: from src.services.gam_advertisers_sync import _build_gam_client_for_tenant from src.services.gam_orders_service import GAMOrdersService + with _sync_session() as db: GAMOrdersService(db).sync_tenant_orders(tenant_id, _build_gam_client_for_tenant(tenant_id)) except Exception as _oe: @@ -651,6 +652,12 @@ def _mark_sync_complete(sync_id: str, summary: dict[str, Any]): sync_job.completed_at = datetime.now(UTC) # Convert summary dict to JSON string (summary field is Text, not JSON) sync_job.summary = json.dumps(summary) if summary else None + if sync_job.sync_type == "inventory": + # First successful inventory sync freezes the tenant's ad + # server configuration (adapter_config_lock.py). + from src.core.database.repositories.adapter_config import AdapterConfigRepository + + AdapterConfigRepository(db, sync_job.tenant_id).mark_config_locked() db.commit() except Exception as e: logger.error(f"Failed to mark sync complete: {e}") diff --git a/templates/adapters/freewheel/connection_config.html b/templates/adapters/freewheel/connection_config.html index 34e05872aa..ce327fe3a4 100644 --- a/templates/adapters/freewheel/connection_config.html +++ b/templates/adapters/freewheel/connection_config.html @@ -62,13 +62,15 @@

Sign-in Credentials (recom + placeholder="API-Access OAuth2 client ID" + {% if adapter_locked %}readonly style="background-color: #f3f4f6; color: #6b7280; cursor: not-allowed;" title="Locked after inventory sync"{% endif %}>

+ placeholder="{% if adapter_locked %}(locked after inventory sync){% elif adapter_config and adapter_config.get('client_id') %}(leave blank to keep existing){% else %}Enter client secret{% endif %}" + {% if adapter_locked %}readonly style="background-color: #f3f4f6; color: #6b7280; cursor: not-allowed;" title="Locked after inventory sync"{% endif %}> Stored encrypted at rest. Leave blank to keep the previously saved secret. diff --git a/templates/adapters/google_ad_manager/connection_config.html b/templates/adapters/google_ad_manager/connection_config.html index 5ad3f8547c..805d74551f 100644 --- a/templates/adapters/google_ad_manager/connection_config.html +++ b/templates/adapters/google_ad_manager/connection_config.html @@ -69,9 +69,11 @@

Configuration Complete

+ {% if not adapter_locked %} + {% endif %}
{% else %} @@ -130,9 +132,11 @@

Configuration Complete

+ placeholder="Enter network code or click auto-detect" + {% if adapter_locked %}readonly style="background-color: #f3f4f6; color: #6b7280; cursor: not-allowed;" title="Locked after inventory sync"{% endif %}>
-
@@ -190,9 +194,11 @@

Service Account Created

-
@@ -233,8 +239,9 @@

Service Account Created

GAM Network Code * + style="width: 100%; padding: 0.5rem; border: 1px solid #d1d5db; border-radius: 6px; font-family: monospace;{% if adapter_locked %} background-color: #f3f4f6; color: #6b7280; cursor: not-allowed;{% endif %}" + placeholder="e.g., 12345678" + {% if adapter_locked %}readonly title="Locked after inventory sync"{% endif %}>
diff --git a/templates/adapters/improvedigital/connection_config.html b/templates/adapters/improvedigital/connection_config.html index 360fa5a723..9fb7778752 100644 --- a/templates/adapters/improvedigital/connection_config.html +++ b/templates/adapters/improvedigital/connection_config.html @@ -28,7 +28,8 @@

Improve Digital 360Yield Configuration

+ placeholder="OAuth2 client ID issued by Improve Digital" + {% if adapter_locked %}readonly style="background-color: #f3f4f6; color: #6b7280; cursor: not-allowed;" title="Locked after inventory sync"{% endif %}> Issued per client application by Improve Digital — request via the platform team. @@ -38,7 +39,8 @@

Improve Digital 360Yield Configuration

+ placeholder="{% if adapter_locked %}(locked after inventory sync){% elif adapter_config and adapter_config.get('client_id') %}(leave blank to keep existing){% else %}Enter client secret{% endif %}" + {% if adapter_locked %}readonly style="background-color: #f3f4f6; color: #6b7280; cursor: not-allowed;" title="Locked after inventory sync"{% endif %}> Stored encrypted at rest. Leave blank to keep the previously saved secret. @@ -49,7 +51,8 @@

Improve Digital 360Yield Configuration

+ placeholder="https://api.360yield.com" + {% if adapter_locked %}readonly style="background-color: #f3f4f6; color: #6b7280; cursor: not-allowed;" title="Locked after inventory sync"{% endif %}> diff --git a/templates/tenant_settings.html b/templates/tenant_settings.html index 1a372241c7..0f7871fbc6 100644 --- a/templates/tenant_settings.html +++ b/templates/tenant_settings.html @@ -711,11 +711,23 @@

Ad Server Configuration

{% endif %} + {% if adapter_locked %} +
+ 🔒 Ad Server Connection Locked +

+ Inventory has been synced with this ad server, so the ad server, + network code, API base URL, and client credentials can no longer be + changed. Other settings remain editable. To connect a different ad + server or network, create a new tenant. +

+
+ {% endif %} +

Available Ad Servers

-
+
{% if active_adapter == 'mock' %} Current {% endif %} @@ -723,8 +735,8 @@

Mock (Testing)

Perfect for development and testing. No external setup required.

-
+
{% if active_adapter == 'google_ad_manager' %} Current {% endif %} @@ -735,8 +747,8 @@

Google Ad Manager

-
+
{% if active_adapter == 'broadstreet' %} Current {% endif %} @@ -747,8 +759,8 @@

Broadstreet Ads

{# Triton picker hidden — adapter parked while their APIs aren't production-ready. Source remains under src/adapters/triton/; restore the card + registry entry to bring it back. #} -
+
{% if active_adapter == 'freewheel' %} Current {% endif %} @@ -756,8 +768,8 @@

FreeWheel

Video and CTV advertising via Comcast/FreeWheel's Publisher API

-
+
{% if active_adapter == 'springserve' %} Current {% endif %} @@ -765,8 +777,8 @@

SpringServe (Magnite)

Direct-sold CTV, online video, and audio via Magnite SpringServe

-
+
{% if active_adapter == 'improvedigital' %} Current {% endif %} diff --git a/tests/integration/test_adapter_config_lock.py b/tests/integration/test_adapter_config_lock.py new file mode 100644 index 0000000000..6265297cbc --- /dev/null +++ b/tests/integration/test_adapter_config_lock.py @@ -0,0 +1,476 @@ +"""Integration tests for the adapter-configuration lock. + +Lock state is stored in ``adapter_config.config_locked_at``: stamped by the +inventory-sync completion paths, backfilled by migration for already-synced +tenants. While set, the connection identity is frozen — ``Tenant.ad_server``, +``AdapterConfig.adapter_type``, ``gam_network_code``, and the +``client_id``/``client_secret`` keys inside ``config_json``. Everything else +(credentials like the GAM refresh token, templates, AXE keys, other config +fields) stays editable; connecting a different ad server or GAM network +requires creating a new tenant. Clearing the stamp requires +``super_admin_override``. + +Covers the model-layer guard in ``src/core/database/adapter_config_lock.py``, +the sync-completion stamping, and the route-layer 403s on the admin write +endpoints. +""" + +import os +from datetime import UTC, datetime +from unittest.mock import patch + +import pytest +from cryptography.fernet import Fernet + +from src.core.database.adapter_config_lock import ( + AdapterConfigLockedError, + is_adapter_config_locked, +) +from src.core.database.repositories.adapter_config import AdapterConfigRepository +from tests.factories import AdapterConfigFactory, SyncJobFactory, TenantFactory + +pytestmark = pytest.mark.requires_db + +_TEST_ENCRYPTION_KEY = Fernet.generate_key().decode() + + +@pytest.fixture +def _encryption_key(): + with patch.dict(os.environ, {"ENCRYPTION_KEY": _TEST_ENCRYPTION_KEY}): + yield + + +def _gam_tenant(tenant_id: str, locked: bool = True): + tenant = TenantFactory(tenant_id=tenant_id, ad_server="google_ad_manager") + adapter_config = AdapterConfigFactory( + tenant=tenant, + adapter_type="google_ad_manager", + gam_network_code="111222333", + gam_refresh_token="tok_original", + config_locked_at=datetime.now(UTC) if locked else None, + ) + return tenant, adapter_config + + +class TestLockStateAndStamping: + """config_locked_at is the stored lock state, stamped on inventory-sync success.""" + + def test_unlocked_by_default(self, factory_session): + tenant = TenantFactory(tenant_id="lock_state_none") + AdapterConfigFactory(tenant=tenant) + assert is_adapter_config_locked(factory_session, tenant.tenant_id) is False + + def test_locked_when_stamp_set(self, factory_session): + _gam_tenant("lock_state_set", locked=True) + assert is_adapter_config_locked(factory_session, "lock_state_set") is True + + def test_sync_row_alone_does_not_lock(self, factory_session): + """The stored stamp is the single source of truth — history rows don't lock.""" + tenant, _ = _gam_tenant("lock_state_rows", locked=False) + SyncJobFactory(tenant=tenant, status="completed", sync_type="inventory") + assert is_adapter_config_locked(factory_session, tenant.tenant_id) is False + + def test_missing_adapter_config_is_unlocked(self, factory_session): + tenant = TenantFactory(tenant_id="lock_state_missing") + assert is_adapter_config_locked(factory_session, tenant.tenant_id) is False + + def test_repository_stamp_is_idempotent(self, factory_session): + tenant, adapter_config = _gam_tenant("lock_state_stamp", locked=False) + repo = AdapterConfigRepository(factory_session, tenant.tenant_id) + + assert repo.mark_config_locked() is True + factory_session.commit() + first_stamp = adapter_config.config_locked_at + assert first_stamp is not None + + assert repo.mark_config_locked() is False + factory_session.commit() + assert adapter_config.config_locked_at == first_stamp + + def test_inventory_sync_completion_stamps_lock(self, factory_session): + """The GAM background-sync completion path locks the tenant.""" + from src.services.background_sync_service import _mark_sync_complete + + tenant, _ = _gam_tenant("lock_state_sync", locked=False) + job = SyncJobFactory(tenant=tenant, status="running", sync_type="inventory") + + _mark_sync_complete(job.sync_id, {"ad_units": {"total": 1}}) + + assert is_adapter_config_locked(factory_session, tenant.tenant_id) is True + + def test_non_inventory_sync_completion_does_not_lock(self, factory_session): + from src.services.background_sync_service import _mark_sync_complete + + tenant, _ = _gam_tenant("lock_state_kind", locked=False) + job = SyncJobFactory(tenant=tenant, status="running", sync_type="custom_targeting") + + _mark_sync_complete(job.sync_id, {}) + + assert is_adapter_config_locked(factory_session, tenant.tenant_id) is False + + +class TestModelGuard: + """The before_update listeners block config changes on locked tenants.""" + + def test_adapter_type_change_blocked_when_locked(self, factory_session): + _, adapter_config = _gam_tenant("lock_guard_type") + + adapter_config.adapter_type = "mock" + with pytest.raises(AdapterConfigLockedError): + factory_session.commit() + factory_session.rollback() + + def test_gam_network_code_change_blocked_when_locked(self, factory_session): + _, adapter_config = _gam_tenant("lock_guard_code") + + adapter_config.gam_network_code = "999888777" + with pytest.raises(AdapterConfigLockedError): + factory_session.commit() + factory_session.rollback() + + def test_clearing_gam_network_code_blocked_when_locked(self, factory_session): + _, adapter_config = _gam_tenant("lock_guard_clear") + + adapter_config.gam_network_code = None + with pytest.raises(AdapterConfigLockedError): + factory_session.commit() + factory_session.rollback() + + def test_tenant_ad_server_change_blocked_when_locked(self, factory_session): + tenant, _ = _gam_tenant("lock_guard_server") + + tenant.ad_server = "mock" + with pytest.raises(AdapterConfigLockedError): + factory_session.commit() + factory_session.rollback() + + def test_changes_allowed_before_lock(self, factory_session): + tenant, adapter_config = _gam_tenant("lock_guard_free", locked=False) + + adapter_config.adapter_type = "mock" + adapter_config.gam_network_code = None + tenant.ad_server = "mock" + factory_session.commit() + + assert adapter_config.adapter_type == "mock" + assert tenant.ad_server == "mock" + + def test_same_value_reassignment_allowed_when_locked(self, factory_session): + """The settings forms resubmit the stored values on every save.""" + tenant, adapter_config = _gam_tenant("lock_guard_same") + + adapter_config.adapter_type = "google_ad_manager" + adapter_config.gam_network_code = "111222333" + tenant.ad_server = "google_ad_manager" + factory_session.commit() + + def test_credentials_and_mirror_flags_writable_when_locked(self, factory_session): + """Credential rotation and business-rule mirrors stay editable after lock.""" + _, adapter_config = _gam_tenant("lock_guard_writable") + + adapter_config.gam_refresh_token = "tok_rotated" + adapter_config.gam_auth_method = "service_account" + adapter_config.gam_manual_approval_required = True + adapter_config.gam_network_currency = "EUR" + factory_session.commit() + + assert adapter_config.gam_refresh_token == "tok_rotated" + + def test_other_settings_writable_when_locked(self, factory_session): + """Only the connection identity is frozen — templates, AXE keys, and + non-client config fields stay editable.""" + _, adapter_config = _gam_tenant("lock_guard_settings") + + adapter_config.gam_order_name_template = "Order {po_number}" + adapter_config.axe_include_key = "hb_pb" + adapter_config.gam_trafficker_id = "trafficker_2" + adapter_config.config_json = {"username": "new_user"} + factory_session.commit() + + assert adapter_config.gam_order_name_template == "Order {po_number}" + assert adapter_config.config_json == {"username": "new_user"} + + def test_clearing_lock_stamp_blocked_without_override(self, factory_session): + """config_locked_at is itself locked — unlocking requires super_admin_override.""" + _, adapter_config = _gam_tenant("lock_guard_unlock_deny") + + adapter_config.config_locked_at = None + with pytest.raises(AdapterConfigLockedError): + factory_session.commit() + factory_session.rollback() + + def test_super_admin_override_unlocks(self, factory_session): + """The documented unlock procedure: clear the stamp under super_admin_override.""" + _, adapter_config = _gam_tenant("lock_guard_unlock") + + factory_session.info["super_admin_override"] = True + try: + adapter_config.config_locked_at = None + factory_session.commit() + finally: + factory_session.info.pop("super_admin_override", None) + + assert is_adapter_config_locked(factory_session, "lock_guard_unlock") is False + + # Fully unlocked: identity changes work again without any flag. + adapter_config.adapter_type = "mock" + factory_session.commit() + assert adapter_config.adapter_type == "mock" + + def test_super_admin_override_bypasses_lock(self, factory_session): + _, adapter_config = _gam_tenant("lock_guard_override") + + factory_session.info["super_admin_override"] = True + try: + adapter_config.gam_network_code = "444555666" + factory_session.commit() + finally: + factory_session.info.pop("super_admin_override", None) + + assert adapter_config.gam_network_code == "444555666" + + +class TestAdminRoutes: + """The admin write endpoints reject config changes with a clean 403.""" + + def test_update_adapter_switch_returns_403_when_locked(self, authenticated_admin_client, factory_session): + tenant, _ = _gam_tenant("lock_route_switch") + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/settings/adapter", + json={"adapter": "mock"}, + ) + assert resp.status_code == 403 + assert "locked" in resp.get_json()["error"].lower() + + def test_update_adapter_edit_config_returns_403_when_locked(self, authenticated_admin_client, factory_session): + tenant, _ = _gam_tenant("lock_route_edit") + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/settings/adapter", + json={"adapter": "google_ad_manager", "action": "edit_config"}, + ) + assert resp.status_code == 403 + + def test_update_adapter_network_code_change_returns_403_when_locked( + self, authenticated_admin_client, factory_session + ): + tenant, _ = _gam_tenant("lock_route_code") + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/settings/adapter", + json={"adapter": "google_ad_manager", "gam_network_code": "999888777"}, + ) + assert resp.status_code == 403 + + def test_update_adapter_same_values_resubmit_allowed_when_locked(self, authenticated_admin_client, factory_session): + """Forms resubmit stored values on save — a no-op write must pass.""" + tenant, _ = _gam_tenant("lock_route_ok") + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/settings/adapter", + json={"adapter": "google_ad_manager", "gam_network_code": "111222333"}, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + + def test_update_adapter_template_change_allowed_when_locked(self, authenticated_admin_client, factory_session): + """Naming templates and AXE keys are not part of the connection identity.""" + tenant, _ = _gam_tenant("lock_route_template") + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/settings/adapter", + json={ + "adapter": "google_ad_manager", + "gam_network_code": "111222333", + "order_name_template": "Order {po_number}", + "axe_include_key": "hb_pb_new", + }, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + + def test_update_adapter_switch_allowed_when_not_locked(self, authenticated_admin_client, factory_session): + tenant, _ = _gam_tenant("lock_route_free", locked=False) + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/settings/adapter", + json={"adapter": "mock"}, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + + def test_save_adapter_config_switch_returns_403_when_locked(self, authenticated_admin_client, factory_session): + tenant, _ = _gam_tenant("lock_route_cfg") + + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={"adapter_type": "mock", "config": {}}, + ) + assert resp.status_code == 403 + assert "locked" in resp.get_json()["error"].lower() + + def test_save_adapter_config_other_fields_allowed_when_locked(self, authenticated_admin_client, factory_session): + """Config fields outside client_id/client_secret stay editable after lock.""" + tenant = TenantFactory(tenant_id="lock_route_mock", ad_server="mock") + AdapterConfigFactory(tenant=tenant, adapter_type="mock") + + # Seed config_json through the route while unlocked, then lock. + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={"adapter_type": "mock", "config": {"dry_run": False}}, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + AdapterConfigRepository(factory_session, tenant.tenant_id).mark_config_locked() + factory_session.commit() + + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={"adapter_type": "mock", "config": {"dry_run": True}}, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + + def test_save_adapter_config_client_credentials_locked( + self, authenticated_admin_client, factory_session, _encryption_key + ): + """client_id/client_secret identify the network seat — read-only after lock.""" + tenant = TenantFactory(tenant_id="lock_route_fw", ad_server="freewheel") + AdapterConfigFactory(tenant=tenant, adapter_type="freewheel") + + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={"adapter_type": "freewheel", "config": {"client_id": "cid_original", "client_secret": "cs_original"}}, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + AdapterConfigRepository(factory_session, tenant.tenant_id).mark_config_locked() + factory_session.commit() + + # Changing the client_id is rejected. + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={"adapter_type": "freewheel", "config": {"client_id": "cid_other", "client_secret": "cs_original"}}, + ) + assert resp.status_code == 403 + assert "locked" in resp.get_json()["error"].lower() + + # Changing the client_secret is rejected. + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={"adapter_type": "freewheel", "config": {"client_id": "cid_original", "client_secret": "cs_other"}}, + ) + assert resp.status_code == 403 + + # Editing other fields — client_secret omitted (preserved) — is allowed. + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={ + "adapter_type": "freewheel", + "config": {"client_id": "cid_original", "environment": "staging"}, + }, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + + def test_save_adapter_config_api_base_url_locked( + self, authenticated_admin_client, factory_session, _encryption_key + ): + """The Improve Digital API base URL is part of the connection identity — read-only after lock.""" + tenant = TenantFactory(tenant_id="lock_route_impd", ad_server="improvedigital") + AdapterConfigFactory(tenant=tenant, adapter_type="improvedigital") + + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={ + "adapter_type": "improvedigital", + "config": { + "client_id": "cid_impd", + "client_secret": "cs_impd", + "api_base_url": "https://api.360yield.com", + }, + }, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + AdapterConfigRepository(factory_session, tenant.tenant_id).mark_config_locked() + factory_session.commit() + + # Changing the API base URL (e.g. production -> dev host) is rejected. + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={ + "adapter_type": "improvedigital", + "config": { + "client_id": "cid_impd", + "api_base_url": "https://api.360yielddev.com", + }, + }, + ) + assert resp.status_code == 403 + assert "locked" in resp.get_json()["error"].lower() + + # Editing another field with the same base URL (secret omitted/preserved) is allowed. + resp = authenticated_admin_client.post( + f"/api/tenant/{tenant.tenant_id}/adapter-config", + json={ + "adapter_type": "improvedigital", + "config": { + "client_id": "cid_impd", + "api_base_url": "https://api.360yield.com", + "buying_entity_id": 421, + }, + }, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) + + def test_settings_page_renders_locked_client_credentials(self, authenticated_admin_client, factory_session): + """The Improve Digital client_id/client_secret inputs render read-only when locked.""" + tenant = TenantFactory(tenant_id="lock_ui_impd", ad_server="improvedigital") + AdapterConfigFactory( + tenant=tenant, + adapter_type="improvedigital", + config_json={"client_id": "cid_locked"}, + config_locked_at=datetime.now(UTC), + ) + + resp = authenticated_admin_client.get(f"/tenant/{tenant.tenant_id}/settings/adapter") + html = resp.get_data(as_text=True) + assert resp.status_code == 200 + assert "Ad Server Connection Locked" in html + client_id_input = html.split('id="improvedigital_client_id"')[1].split(">")[0] + assert "readonly" in client_id_input + client_secret_input = html.split('id="improvedigital_client_secret"')[1].split(">")[0] + assert "readonly" in client_secret_input + api_base_url_input = html.split('id="improvedigital_api_base_url"')[1].split(">")[0] + assert "readonly" in api_base_url_input + + def test_settings_page_renders_locked_gam_network_code(self, authenticated_admin_client, factory_session): + """The GAM wizard's editable network-code input renders read-only when + locked (the input only appears when a refresh token exists but no + network code is stored — fully-configured tenants get display-only text).""" + tenant = TenantFactory(tenant_id="lock_ui_gam", ad_server="google_ad_manager") + AdapterConfigFactory( + tenant=tenant, + adapter_type="google_ad_manager", + gam_network_code=None, + gam_refresh_token="tok_x", + config_locked_at=datetime.now(UTC), + ) + + resp = authenticated_admin_client.get(f"/tenant/{tenant.tenant_id}/settings/adapter") + html = resp.get_data(as_text=True) + assert resp.status_code == 200 + network_code_input = html.split('id="gam_network_code"')[1].split(">")[0] + assert "readonly" in network_code_input + + def test_configure_gam_network_change_returns_403_when_locked(self, authenticated_admin_client, factory_session): + tenant, _ = _gam_tenant("lock_route_gam") + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/gam/configure", + json={"auth_method": "oauth", "network_code": "999888777", "refresh_token": "tok_new"}, + ) + assert resp.status_code == 403 + + def test_configure_gam_token_rotation_allowed_when_locked(self, authenticated_admin_client, factory_session): + tenant, _ = _gam_tenant("lock_route_gam_rotate") + + resp = authenticated_admin_client.post( + f"/tenant/{tenant.tenant_id}/gam/configure", + json={"auth_method": "oauth", "network_code": "111222333", "refresh_token": "tok_new"}, + ) + assert resp.status_code == 200, resp.get_data(as_text=True) From 7d6c6f4f1251df8a67b23138bba5cd13a1eba3b5 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Wed, 9 Sep 2026 14:59:49 +0600 Subject: [PATCH 78/90] Commented model_dump to fix completed delivery metrics --- .mcp.json | 2 +- src/core/schemas/delivery.py | 30 ++++++++++++------- .../test_delivery_poll_behavioral.py | 2 +- tests/unit/test_delivery_poll_behavioral.py | 26 ++++++++-------- tests/unit/test_delivery_schema_contracts.py | 8 +++-- 5 files changed, 39 insertions(+), 29 deletions(-) diff --git a/.mcp.json b/.mcp.json index 9e77e09b97..7fde327d9b 100644 --- a/.mcp.json +++ b/.mcp.json @@ -4,7 +4,7 @@ "type": "http", "url": "http://0.0.0.0:8080/mcp", "headers": { - "Authorization": "Bearer tok_JQOJr84VHKMZGD7R6xJe6YIQFToBK8fKVUIJojtxOHI" + "Authorization": "Bearer tok_TsDB5qiLETTMCZab_UAHmUES1cI3tInHt69sxf14TWE" } } } diff --git a/src/core/schemas/delivery.py b/src/core/schemas/delivery.py index 8523f880a4..c544354b49 100644 --- a/src/core/schemas/delivery.py +++ b/src/core/schemas/delivery.py @@ -271,17 +271,25 @@ class GetMediaBuyDeliveryResponse(NestedModelSerializerMixin, LibraryGetMediaBuy None, description="True when any requested geo package breakdown was truncated" ) - def model_dump(self, **kwargs: Any) -> dict[str, Any]: - """Override to ensure webhook metadata fields are present when notification_type is set. - - The base AdCPBaseModel excludes None values, but the AdCP protocol requires - next_expected_at to be explicitly present (as null) when notification_type - is 'final' so consumers know no further reports are expected. - """ - result = super().model_dump(**kwargs) - if self.notification_type is not None and "next_expected_at" not in result: - result["next_expected_at"] = None - return result + # Todo: Check it if model_dump has any use case + # next_expected_at is intentionally NOT forced to null. Every AdCP schema + # (2.5, 3.0, 3.1) types it as a non-nullable date-time string and does not + # list it in ``required``; the response schema says it is "only present in + # webhook deliveries when notification_type is not 'final'" and the webhook + # result schema says "Omitted on final notifications". The base model's + # exclude-None behaviour already yields the correct wire shape. + + # def model_dump(self, **kwargs: Any) -> dict[str, Any]: + # """Override to ensure webhook metadata fields are present when notification_type is set. + + # The base AdCPBaseModel excludes None values, but the AdCP protocol requires + # next_expected_at to be explicitly present (as null) when notification_type + # is 'final' so consumers know no further reports are expected. + # """ + # result = super().model_dump(**kwargs) + # if self.notification_type is not None and "next_expected_at" not in result: + # result["next_expected_at"] = None + # return result def __str__(self) -> str: """Return human-readable summary message for protocol envelope.""" diff --git a/tests/integration/test_delivery_poll_behavioral.py b/tests/integration/test_delivery_poll_behavioral.py index e13b1b9a0b..2933f40603 100644 --- a/tests/integration/test_delivery_poll_behavioral.py +++ b/tests/integration/test_delivery_poll_behavioral.py @@ -93,7 +93,7 @@ def test_completed_campaign_sets_final_type(self, integration_db): dumped = response.model_dump(mode="json") assert dumped["notification_type"] == "final" - assert dumped["next_expected_at"] is None + assert "next_expected_at" not in dumped, "final notifications must omit next_expected_at, never null" # --------------------------------------------------------------------------- diff --git a/tests/unit/test_delivery_poll_behavioral.py b/tests/unit/test_delivery_poll_behavioral.py index dd66721250..fd27baa503 100644 --- a/tests/unit/test_delivery_poll_behavioral.py +++ b/tests/unit/test_delivery_poll_behavioral.py @@ -692,35 +692,35 @@ def test_many_creatives(self): class TestNextExpectedAtSerialization: - """model_dump() forces next_expected_at=null when notification_type is set. + """next_expected_at is omitted when unset -- never emitted as null. - The AdCP protocol requires next_expected_at to be explicitly present - (as null) when notification_type is 'final', so consumers know no - further reports are expected. The base model excludes None values, - so the override on line 304 re-injects it. + AdCP schemas (2.5, 3.0, 3.1) type the field as a non-nullable date-time + string and never list it in ``required``. The response schema says it is + "only present in webhook deliveries when notification_type is not 'final'" + and the webhook result schema says "Omitted on final notifications". A + null fails the framework's response validation, which turned every + completed-buy delivery response into VALIDATION_ERROR[/next_expected_at]. Covers: UC-004-SERIAL-01 """ - def test_final_notification_includes_null_next_expected_at(self): - """notification_type='final' forces next_expected_at=null in JSON output. + def test_final_notification_omits_next_expected_at(self): + """notification_type='final' must omit next_expected_at (not null). Covers: UC-004-SERIAL-01 """ resp = _make_media_buy_delivery_response(0, notification_type="final") dumped = resp.model_dump(mode="json") - assert "next_expected_at" in dumped - assert dumped["next_expected_at"] is None + assert "next_expected_at" not in dumped - def test_scheduled_notification_includes_null_next_expected_at(self): - """Any notification_type (not just 'final') forces next_expected_at into JSON. + def test_scheduled_notification_without_timestamp_omits_next_expected_at(self): + """notification_type='scheduled' with no timestamp omits the key rather than emitting null. Covers: UC-004-SERIAL-01 """ resp = _make_media_buy_delivery_response(0, notification_type="scheduled") dumped = resp.model_dump(mode="json") - assert "next_expected_at" in dumped - assert dumped["next_expected_at"] is None + assert "next_expected_at" not in dumped def test_no_notification_type_excludes_next_expected_at(self): """Without notification_type, next_expected_at is excluded from JSON (base behavior). diff --git a/tests/unit/test_delivery_schema_contracts.py b/tests/unit/test_delivery_schema_contracts.py index 84e950177d..00d9af533c 100644 --- a/tests/unit/test_delivery_schema_contracts.py +++ b/tests/unit/test_delivery_schema_contracts.py @@ -280,11 +280,13 @@ def test_str_multiple_deliveries(self): resp = _make_delivery_response(media_buy_deliveries=deliveries) assert str(resp) == "Retrieved delivery data for 3 media buys." - def test_model_dump_includes_next_expected_at_when_notification_type_set(self): + def test_model_dump_omits_next_expected_at_on_final_notification(self): + # AdCP types next_expected_at as a non-nullable date-time string that is + # "only present ... when notification_type is not 'final'"; a null fails + # response schema validation, so the key must be absent, never null. resp = _make_delivery_response(notification_type="final") dumped = resp.model_dump() - assert "next_expected_at" in dumped - assert dumped["next_expected_at"] is None + assert "next_expected_at" not in dumped def test_model_dump_omits_next_expected_at_when_no_notification_type(self): resp = _make_delivery_response() From 0bd3ee07779eec6759fe9444f18c87ef67e91624 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Thu, 10 Sep 2026 17:18:41 +0600 Subject: [PATCH 79/90] product flow log --- src/admin/blueprints/products.py | 95 +++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/src/admin/blueprints/products.py b/src/admin/blueprints/products.py index ef83d83553..d69c804ae7 100644 --- a/src/admin/blueprints/products.py +++ b/src/admin/blueprints/products.py @@ -1,8 +1,10 @@ """Products management blueprint for admin UI.""" import asyncio +import functools import json import logging +import time import uuid from dataclasses import dataclass from typing import Any @@ -31,6 +33,89 @@ logger = logging.getLogger(__name__) + +def _log_product_flow(action: str): + """Log entry, exit, duration and any escaping exception of a product form handler. + + Sits *inside* ``require_tenant_access`` so the tenant is already resolved. + The inner handlers already catch and flash most errors; this wrapper adds + the outer envelope: a START line, a DONE line with wall-clock duration and + the response status/redirect, or a FAILED line with the full traceback for + anything that escapes (which Flask turns into a 500). A request that logs + START but never DONE/FAILED was cut off upstream (proxy timeout, worker + killed) — that is the signature of a 502 at the gateway. + """ + + def decorator(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + started = time.monotonic() + tenant_id = kwargs.get("tenant_id") or (args[0] if args else None) + product_id = kwargs.get("product_id") or (args[1] if len(args) > 1 else None) + logger.info( + "[product_flow] %s START method=%s path=%s tenant=%s product=%s", + action, + request.method, + request.path, + tenant_id, + product_id, + ) + try: + response = f(*args, **kwargs) + except Exception: + logger.exception( + "[product_flow] %s FAILED after %.0f ms method=%s path=%s tenant=%s product=%s " + "(unhandled exception — Flask will return 500)", + action, + (time.monotonic() - started) * 1000, + request.method, + request.path, + tenant_id, + product_id, + ) + raise + status = getattr(response, "status_code", 200 if isinstance(response, str) else None) + headers = getattr(response, "headers", None) + location = headers.get("Location") if headers is not None else None + logger.info( + "[product_flow] %s DONE in %.0f ms method=%s tenant=%s product=%s status=%s location=%s", + action, + (time.monotonic() - started) * 1000, + request.method, + tenant_id, + product_id, + status, + location, + ) + return response + + return wrapper + + return decorator + + +def _publish_product_change_logged(step: str, publish_fn, **kwargs) -> None: + """Run a post-commit catalog webhook publication with timing and isolation. + + The product row is already committed when this runs, so a webhook failure + must not turn a successful save into an error page — it is logged with the + traceback and the request continues. The duration is logged because the + Tenant Management webhook delivery posts synchronously (10 s HTTP timeout + per subscriber) and can hold the request long enough for a gateway timeout. + """ + started = time.monotonic() + try: + publish_fn(**kwargs) + except Exception: + logger.exception( + "[product_flow] %s: webhook publication FAILED after %.0f ms (product already committed)", + step, + (time.monotonic() - started) * 1000, + ) + return + logger.info("[product_flow] %s: webhook publication done in %.0f ms", step, (time.monotonic() - started) * 1000) + + # Create Blueprint products_bp = Blueprint("products", __name__) @@ -834,6 +919,7 @@ def _render_add_product_form(tenant_id, tenant, adapter_type, currencies, form_d @products_bp.route("/add", methods=["GET", "POST"]) @log_admin_action("add_product") @require_tenant_access(role=("admin", "member"), allow_embedded_writes=True) +@_log_product_flow("add_product") def add_product(tenant_id): """Add a new product - adapter-specific form.""" if not publisher_owns("compose_products"): @@ -1429,7 +1515,9 @@ def add_product(tenant_id): db_session.commit() - publish_product_catalog_change( + _publish_product_change_logged( + "add_product", + publish_product_catalog_change, tenant_id=tenant_id, action="created", product_id=product.product_id, @@ -1453,6 +1541,7 @@ def add_product(tenant_id): @products_bp.route("//edit", methods=["GET", "POST"]) @log_admin_action("edit_product") @require_tenant_access(role=("admin", "member"), allow_embedded_writes=True) +@_log_product_flow("edit_product") def edit_product(tenant_id, product_id): """Edit an existing product.""" from sqlalchemy import select @@ -2053,7 +2142,9 @@ def edit_product(tenant_id, product_id): db_session.refresh(product) logger.info(f"[DEBUG] After commit - product.format_ids from DB: {product.format_ids}") - publish_product_record_update_catalog_change( + _publish_product_change_logged( + "edit_product", + publish_product_record_update_catalog_change, tenant_id=tenant_id, product=product, previous_allowed_principal_ids=previous_allowed_principal_ids, From cdac604af4c2e2016c8c10c55103a9ad6ed2047f Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Thu, 10 Sep 2026 17:43:49 +0600 Subject: [PATCH 80/90] Run server logs --- core/middleware/admin_mount.py | 45 +++++++++++++++++++++++++++++++--- scripts/run_server.py | 5 ++++ src/admin/app.py | 38 ++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/core/middleware/admin_mount.py b/core/middleware/admin_mount.py index 93c8a6bf9d..a1625657a4 100644 --- a/core/middleware/admin_mount.py +++ b/core/middleware/admin_mount.py @@ -39,10 +39,14 @@ from __future__ import annotations +import logging +import time from typing import Any from src.core.domain_config import get_sales_agent_domain, is_admin_domain +logger = logging.getLogger(__name__) + # Path prefixes the Flask admin claims. Anything under one of these # segments dispatches to Flask; everything else falls through to A2A # (which serves /.well-known/agent-card.json + the A2A RPC endpoint at @@ -118,7 +122,7 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: if self._is_admin_host(scope): new_scope = dict(scope) new_scope["root_path"] = scope.get("root_path", "") + "/admin" - await self.wsgi_app(new_scope, receive, send) + await self._dispatch_wsgi(new_scope, receive, send) return path = scope.get("path", "") @@ -182,12 +186,47 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: # str length for ASCII prefixes. new_scope["raw_path"] = raw[len(prefix) :] or b"/" new_scope["root_path"] = scope.get("root_path", "") + prefix - await self.wsgi_app(new_scope, receive, send) + await self._dispatch_wsgi(new_scope, receive, send) return - await self.wsgi_app(scope, receive, send) + await self._dispatch_wsgi(scope, receive, send) return await self.app(scope, receive, send) + async def _dispatch_wsgi(self, scope: dict, receive: Any, send: Any) -> None: + """Hand an HTTP request to the Flask WSGI app with boundary logging. + + Logs one START line before the request enters the WSGI bridge and one + DONE/FAILED line when it comes back. A START with no matching DONE means + the request died inside the bridge or the worker thread — the case a + reverse proxy reports as 502 while Flask itself never logs anything. + Only unsafe methods are logged at INFO to keep GET noise down. + """ + method = scope.get("method", "?") + path = scope.get("path", "") + noisy = method in ("GET", "HEAD", "OPTIONS") + level = logging.DEBUG if noisy else logging.INFO + content_length = None + for raw_name, raw_value in scope.get("headers", ()): + if raw_name.lower() == b"content-length": + content_length = raw_value.decode("latin-1") + logger.log( + level, + "[admin_mount] START %s %s host=%s content_length=%s", + method, + path, + self._resolve_host(scope), + content_length, + ) + started = time.monotonic() + try: + await self.wsgi_app(scope, receive, send) + except BaseException: + logger.exception( + "[admin_mount] FAILED %s %s after %.0f ms", method, path, (time.monotonic() - started) * 1000 + ) + raise + logger.log(level, "[admin_mount] DONE %s %s in %.0f ms", method, path, (time.monotonic() - started) * 1000) + @staticmethod def _resolve_host(scope: dict) -> str | None: """Pick the externally-visible host from ASGI scope headers. diff --git a/scripts/run_server.py b/scripts/run_server.py index 33447458b8..6a2674eb4f 100755 --- a/scripts/run_server.py +++ b/scripts/run_server.py @@ -5,12 +5,17 @@ A2A at /, and Flask admin via WSGI middleware. """ +import faulthandler import os import sys def main(): """Run the server with configurable port.""" + # Dump Python tracebacks of every thread to stderr on a hard crash + # (segfault, abort, fatal signal) — otherwise the process just vanishes + # and the proxy reports 502 with nothing in the application log. + faulthandler.enable() try: sys.path.insert(0, ".") from src.core.startup import initialize_application diff --git a/src/admin/app.py b/src/admin/app.py index 7cd428aff8..c340cfe043 100644 --- a/src/admin/app.py +++ b/src/admin/app.py @@ -5,6 +5,7 @@ import logging import os import secrets +import time from datetime import timedelta import markdown @@ -342,6 +343,43 @@ def csrf_token() -> str: return {"csrf_token": csrf_token} + @app.before_request + def log_admin_request_start(): + """Boundary log for every unsafe admin request, registered before the + CSRF guard so it fires even when that guard aborts. Pairs with + ``log_admin_request_teardown`` below: START without END means the + request never completed inside Flask.""" + if request.method in _CSRF_SAFE_METHODS: + return None + g._admin_request_started = time.monotonic() + logger.info( + "[admin_request] START %s %s content_length=%s endpoint=%s", + request.method, + request.path, + request.content_length, + request.endpoint, + ) + return None + + @app.teardown_request + def log_admin_request_teardown(exc): + started = getattr(g, "_admin_request_started", None) + if started is None: + return + elapsed_ms = (time.monotonic() - started) * 1000 + if exc is not None: + logger.error( + "[admin_request] END %s %s after %.0f ms with unhandled %s: %s", + request.method, + request.path, + elapsed_ms, + type(exc).__name__, + exc, + exc_info=exc, + ) + else: + logger.info("[admin_request] END %s %s in %.0f ms", request.method, request.path, elapsed_ms) + @app.before_request def enforce_admin_csrf(): from flask import abort, request From 0f73d1142617ca975fed40603b48a40f18cafa9b Mon Sep 17 00:00:00 2001 From: chinmoy Date: Thu, 10 Sep 2026 18:50:12 +0600 Subject: [PATCH 81/90] enrich logs --- scripts/deploy/run_all_services.py | 44 ++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/deploy/run_all_services.py b/scripts/deploy/run_all_services.py index b32d05a6cb..260a0502d1 100644 --- a/scripts/deploy/run_all_services.py +++ b/scripts/deploy/run_all_services.py @@ -20,6 +20,12 @@ # Store process references for cleanup processes = [] +# Exit code of the MCP/A2A/Admin server child once it has stopped. ``None`` +# while it is running. The main loop watches this so the container exits +# (and the orchestrator restarts it) instead of staying alive with nothing +# listening on the app port, which the load balancer reports as 502. +_mcp_exit_code: int | None = None + def validate_required_env(): """Validate required environment variables.""" @@ -181,9 +187,8 @@ def init_database(): sys.exit(1) -def cleanup(signum=None, frame=None): - """Clean up all processes on exit.""" - print("\nShutting down all services...") +def _terminate_children() -> None: + """Terminate every child process we started (nginx, cron, server).""" for proc in processes: if proc and proc.poll() is None: proc.terminate() @@ -191,6 +196,12 @@ def cleanup(signum=None, frame=None): proc.wait(timeout=5) except subprocess.TimeoutExpired: proc.kill() + + +def cleanup(signum=None, frame=None): + """Clean up all processes on exit.""" + print("\nShutting down all services...") + _terminate_children() sys.exit(0) @@ -240,7 +251,16 @@ def run_mcp_server(): for line in iter(proc.stdout.readline, b""): if line: print(f"[MCP] {line.decode().rstrip()}") - print("MCP server stopped") + rc = proc.wait() + # A negative return code means the child was killed by a signal: + # -9 = SIGKILL (kernel OOM killer), -11 = SIGSEGV (native crash). + if rc < 0: + hint = " (SIGKILL — likely out of memory)" if rc == -9 else " (SIGSEGV — native crash)" if rc == -11 else "" + print(f"MCP server stopped: killed by signal {-rc}{hint}") + else: + print(f"MCP server stopped with exit code {rc}") + global _mcp_exit_code + _mcp_exit_code = rc def exec_mcp_server(): @@ -424,10 +444,24 @@ def main(): print("\nℹ️ Nginx reverse proxy skipped (SKIP_NGINX=true)") print("Press Ctrl+C to stop all services") - # Keep the main thread alive + # Keep the main thread alive while the server child is running. If the + # child dies (OOM kill, native crash, unhandled exit) we must exit too: + # otherwise this wrapper keeps the container "alive" with nothing on the + # app port, the load balancer serves 502s until its unhealthy threshold + # finally kills the task, and the stopped-task reason only says "failed + # ELB health checks" with exit code 0, hiding the real cause. Exiting + # with the child's status surfaces it (137 = SIGKILL/OOM, 139 = SIGSEGV) + # and lets the orchestrator restart the task immediately. try: while True: time.sleep(1) + if _mcp_exit_code is not None: + rc = _mcp_exit_code + print(f"❌ MCP server process exited (code {rc}); stopping container so it can be restarted") + _terminate_children() + if rc < 0: + sys.exit(128 + (-rc)) # shell convention for signal deaths + sys.exit(rc or 1) # a clean exit of the server is still a failure here except KeyboardInterrupt: print("\n\nShutting down all services...") sys.exit(0) From 0dd6df8a22423ce21d8bd206609c2ed5a947e446 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Thu, 10 Sep 2026 19:43:50 +0600 Subject: [PATCH 82/90] fix boot --- scripts/deploy/run_all_services.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/scripts/deploy/run_all_services.py b/scripts/deploy/run_all_services.py index 260a0502d1..90828bf614 100644 --- a/scripts/deploy/run_all_services.py +++ b/scripts/deploy/run_all_services.py @@ -171,17 +171,38 @@ def check_schema_issues(): def init_database(): - """Initialize database schema and default data.""" + """Initialize database schema and default data. + + Runs in a subprocess (like ``run_migrations``) rather than in this + wrapper process on purpose: ``src.core.database.database`` imports the + ORM models, which import ``adcp.types``, and importing the ``adcp`` + library builds native pydantic validators for ~1,900 models — about + 1.2 GB of resident memory. This wrapper stays alive for the life of + the container, so doing that import here permanently doubled the + task's memory footprint (wrapper ~1.2 GB + server child ~1.5 GB) and + left a 4 GB Fargate task a few hundred MB from the OOM killer. + """ print("📦 Initializing database schema and default data...") print( "ℹ️ Note: init_db() is safe - it only creates tables (IF NOT EXISTS) and default tenant (if no tenants exist)" ) try: - from src.core.database.database import init_db - - init_db(exit_on_error=True) + result = subprocess.run( + [ + sys.executable, + "-c", + "from src.core.database.database import init_db; init_db(exit_on_error=True)", + ], + timeout=300, + ) + if result.returncode != 0: + print(f"❌ Database initialization failed (exit code {result.returncode})") + sys.exit(1) print("✅ Database initialization complete") + except subprocess.TimeoutExpired: + print("❌ Database initialization timed out after 300 seconds") + sys.exit(1) except Exception as e: print(f"❌ Database initialization failed: {e}") sys.exit(1) From d8bb0994bd93c015a380bff00651baca2e108501 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Fri, 11 Sep 2026 13:16:21 +0600 Subject: [PATCH 83/90] enrich logs, added memwatch for memory utilization --- scripts/run_server.py | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/scripts/run_server.py b/scripts/run_server.py index 6a2674eb4f..3df0c89f33 100755 --- a/scripts/run_server.py +++ b/scripts/run_server.py @@ -8,6 +8,59 @@ import faulthandler import os import sys +import threading +import time + + +def _rss_mb() -> float: + """Current resident set size in MB (Linux: /proc; elsewhere: peak RSS).""" + try: + with open("/proc/self/statm") as f: + resident_pages = int(f.read().split()[1]) + return resident_pages * os.sysconf("SC_PAGE_SIZE") / (1024 * 1024) + except (OSError, ValueError, IndexError): + import resource + + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak / (1024 * 1024) if sys.platform == "darwin" else peak / 1024 + + +def start_memory_watchdog() -> None: + """Log RSS milestones and dump every thread's stack when memory runs away. + + The Fargate task gets OOM-killed (SIGKILL) within a second or two of an + admin save while Container Insights never shows more than ~70% memory — + the burst is faster than the 1-minute metric. SIGKILL cannot be caught, + so ``faulthandler.enable()`` never fires for it. This thread polls RSS + every 200 ms, logs each ±200 MB move, and when RSS crosses + ``ADCP_MEM_DUMP_MB`` (default 2200) — and again every 500 MB above it — + dumps all thread stacks to stderr so the log shows exactly which code + path is allocating at the moment the process runs away. + Set ``ADCP_MEM_DUMP_MB=0`` to disable. + """ + threshold = int(os.environ.get("ADCP_MEM_DUMP_MB", "2200")) + if threshold <= 0: + return + + def run() -> None: + next_dump = float(threshold) + last_logged = 0.0 + while True: + time.sleep(0.2) + mb = _rss_mb() + if abs(mb - last_logged) >= 200: + print(f"[memwatch] rss={mb:.0f} MB", file=sys.stderr, flush=True) + last_logged = mb + if mb >= next_dump: + print( + f"[memwatch] rss={mb:.0f} MB crossed {next_dump:.0f} MB — dumping all thread stacks", + file=sys.stderr, + flush=True, + ) + faulthandler.dump_traceback(all_threads=True) + next_dump += 500 + + threading.Thread(target=run, name="memwatch", daemon=True).start() def main(): @@ -16,6 +69,7 @@ def main(): # (segfault, abort, fatal signal) — otherwise the process just vanishes # and the proxy reports 502 with nothing in the application log. faulthandler.enable() + start_memory_watchdog() try: sys.path.insert(0, ".") from src.core.startup import initialize_application From a872a4f21ae68b1be5dd600faac1287b45bb1ee7 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Fri, 11 Sep 2026 14:55:35 +0600 Subject: [PATCH 84/90] inventory fetch query optimization --- src/adapters/improvedigital/adapter.py | 13 ++++++---- src/admin/blueprints/adapters.py | 10 ++++--- .../repositories/improvedigital_inventory.py | 26 +++++++++++++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/adapters/improvedigital/adapter.py b/src/adapters/improvedigital/adapter.py index 19d2cf33cb..77dd58b678 100644 --- a/src/adapters/improvedigital/adapter.py +++ b/src/adapters/improvedigital/adapter.py @@ -277,13 +277,16 @@ async def get_available_inventory(self) -> dict[str, Any]: with get_db_session() as session: repo = ImproveDigitalInventoryRepository(session, self.tenant_id or "default") + # Column projection, not full rows: the placement set runs to + # hundreds of thousands of rows and loading them as ORM objects + # (with raw_json) costs gigabytes — see list_picker_rows. placements = [ - {"id": row.entity_id, "name": row.name, "publisher_id": row.parent_id} - for row in repo.list_by_type("placement") + {"id": entity_id, "name": name, "publisher_id": parent_id} + for entity_id, name, parent_id in repo.list_picker_rows("placement") ] - packages = [{"id": row.entity_id, "name": row.name} for row in repo.list_by_type("package")] - sizes = [{"id": row.entity_id, "name": row.name} for row in repo.list_by_type("size")] - publishers = [{"id": row.entity_id, "name": row.name} for row in repo.list_by_type("publisher")] + packages = [{"id": entity_id, "name": name} for entity_id, name, _ in repo.list_picker_rows("package")] + sizes = [{"id": entity_id, "name": name} for entity_id, name, _ in repo.list_picker_rows("size")] + publishers = [{"id": entity_id, "name": name} for entity_id, name, _ in repo.list_picker_rows("publisher")] return { "placements": placements, diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index b3d448cb9b..8811a430c0 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1358,12 +1358,14 @@ def list_improvedigital_inventory(tenant_id, **kwargs): with get_db_session() as session: repo = ImproveDigitalInventoryRepository(session, tenant_id) - rows = repo.list_by_type(entity_type, parent_id=parent_id) + # Column projection only — see ImproveDigitalInventoryRepository.list_picker_rows + # for why loading full rows (with raw_json) here OOM-killed the dev task. + rows = repo.list_picker_rows(entity_type, parent_id=parent_id) items = [ - {"entity_id": row.entity_id, "name": row.name, "parent_id": row.parent_id} - for row in rows - if not q or (row.name and q.lower() in row.name.lower()) + {"entity_id": entity_id, "name": name, "parent_id": row_parent_id} + for entity_id, name, row_parent_id in rows + if not q or (name and q.lower() in name.lower()) ] total = len(items) if limit is not None and limit >= 0: diff --git a/src/core/database/repositories/improvedigital_inventory.py b/src/core/database/repositories/improvedigital_inventory.py index 87f5397965..f04af7f506 100644 --- a/src/core/database/repositories/improvedigital_inventory.py +++ b/src/core/database/repositories/improvedigital_inventory.py @@ -42,6 +42,32 @@ def list_by_type( stmt = stmt.filter(ImproveDigitalInventory.parent_id == parent_id) return list(self._session.scalars(stmt).all()) + def list_picker_rows( + self, + entity_type: str, + *, + parent_id: str | None = None, + ) -> list[tuple[str, str | None, str | None]]: + """Return ``(entity_id, name, parent_id)`` tuples for one entity_type. + + Column projection, not ORM instances: the product-form pickers pull + the full placement set (tens to hundreds of thousands of rows) and + only need these three fields. Loading ``ImproveDigitalInventory`` + objects here also decodes each row's ``raw_json`` JSONB payload and + builds an ORM instance per row — measured at ~2 GB of resident + memory per request in dev, which OOM-killed the 4 GB Fargate task + whenever two product pages overlapped. Use :meth:`list_by_type` + only when the raw payload is actually needed. + """ + stmt = select( + ImproveDigitalInventory.entity_id, + ImproveDigitalInventory.name, + ImproveDigitalInventory.parent_id, + ).filter_by(tenant_id=self._tenant_id, entity_type=entity_type) + if parent_id is not None: + stmt = stmt.filter(ImproveDigitalInventory.parent_id == parent_id) + return list(self._session.execute(stmt).tuples().all()) + def search( self, entity_type: str, From 38f94f645365cc515b4f817c6c08fbf9b5cdd567 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Fri, 11 Sep 2026 15:42:37 +0600 Subject: [PATCH 85/90] implemented paginated search for inventories --- src/admin/blueprints/adapters.py | 73 ++++--- .../repositories/improvedigital_inventory.py | 69 +++++-- .../improvedigital/product_config.html | 187 ++++++++++++------ 3 files changed, 241 insertions(+), 88 deletions(-) diff --git a/src/admin/blueprints/adapters.py b/src/admin/blueprints/adapters.py index 8811a430c0..4423c2901f 100644 --- a/src/admin/blueprints/adapters.py +++ b/src/admin/blueprints/adapters.py @@ -1334,43 +1334,72 @@ def discover_improvedigital_metadata(tenant_id, **kwargs): return jsonify({"success": False, "error": "Metadata discovery failed (see server logs)"}), 500 +# Picker page sizes for the Improve Digital inventory endpoint (see below). +_IMPD_PICKER_PAGE_SIZE = 50 +_IMPD_PICKER_MAX_PAGE_SIZE = 200 + + @adapters_bp.route("/api/tenant//adapters/improvedigital/inventory", methods=["GET"]) @require_tenant_access() def list_improvedigital_inventory(tenant_id, **kwargs): - """Return locally-cached Improve Digital inventory entries for the - product setup UI. - - Filterable by ``entity_type`` (publisher, placement, package, size). - Optional ``parent_id`` narrows placements to one publisher. Optional - ``q`` substring-matches the ``name`` field. Optional ``limit`` caps the - returned rows AFTER filtering (the browse page passes it; the product - pickers omit it and cache the full set client-side). + """Return a page of locally-cached Improve Digital inventory entries for + the product setup UI. + + Query params: + + * ``entity_type`` (required) — publisher, placement, package, size. + * ``ids`` — comma-separated entity ids; returns just those rows (chip + label lookup for already-attached ids). Ignores paging params. + * ``q`` — case-insensitive substring match on name or id, in SQL. + * ``parent_id`` — narrows placements to one publisher. + * ``offset`` / ``limit`` — page window; ``limit`` defaults to 50 and is + capped at 200. ``count`` in the response is the TOTAL matching rows, + ``has_more`` says whether another page exists. + + Everything is filtered and paged in SQL through a three-column + projection — the full placement set is hundreds of thousands of rows, + and materialising it as ORM objects OOM-killed the dev task. """ from src.core.database.repositories.improvedigital_inventory import ImproveDigitalInventoryRepository entity_type = request.args.get("entity_type") parent_id = request.args.get("parent_id") - q = request.args.get("q") - limit = request.args.get("limit", type=int) + q = (request.args.get("q") or "").strip() or None + ids_param = request.args.get("ids") + offset = max(request.args.get("offset", default=0, type=int) or 0, 0) + limit = request.args.get("limit", default=_IMPD_PICKER_PAGE_SIZE, type=int) or _IMPD_PICKER_PAGE_SIZE + limit = max(1, min(limit, _IMPD_PICKER_MAX_PAGE_SIZE)) if not entity_type: return jsonify({"success": False, "error": "entity_type query param is required"}), 400 + def _item(row: tuple[str, str | None, str | None]) -> dict[str, str | None]: + entity_id, name, row_parent_id = row + return {"entity_id": entity_id, "name": name, "parent_id": row_parent_id} + with get_db_session() as session: repo = ImproveDigitalInventoryRepository(session, tenant_id) - # Column projection only — see ImproveDigitalInventoryRepository.list_picker_rows - # for why loading full rows (with raw_json) here OOM-killed the dev task. - rows = repo.list_picker_rows(entity_type, parent_id=parent_id) + if ids_param is not None: + ids = [part.strip() for part in ids_param.split(",") if part.strip()] + items = [_item(row) for row in repo.list_picker_rows_by_ids(entity_type, ids)] + return jsonify( + {"success": True, "entity_type": entity_type, "count": len(items), "items": items, "has_more": False} + ) + total = repo.count_picker_rows(entity_type, parent_id=parent_id, q=q) + rows = repo.list_picker_rows(entity_type, parent_id=parent_id, q=q, offset=offset, limit=limit) - items = [ - {"entity_id": entity_id, "name": name, "parent_id": row_parent_id} - for entity_id, name, row_parent_id in rows - if not q or (name and q.lower() in name.lower()) - ] - total = len(items) - if limit is not None and limit >= 0: - items = items[:limit] - return jsonify({"success": True, "entity_type": entity_type, "count": total, "items": items}) + items = [_item(row) for row in rows] + return jsonify( + { + "success": True, + "entity_type": entity_type, + "count": total, + "items": items, + "offset": offset, + "limit": limit, + "has_more": offset + len(items) < total, + } + ) @adapters_bp.route("/api/tenant//adapters/improvedigital/inventory-stats", methods=["GET"]) diff --git a/src/core/database/repositories/improvedigital_inventory.py b/src/core/database/repositories/improvedigital_inventory.py index f04af7f506..faef1a4d7d 100644 --- a/src/core/database/repositories/improvedigital_inventory.py +++ b/src/core/database/repositories/improvedigital_inventory.py @@ -42,30 +42,77 @@ def list_by_type( stmt = stmt.filter(ImproveDigitalInventory.parent_id == parent_id) return list(self._session.scalars(stmt).all()) + def _picker_filter(self, entity_type: str, *, parent_id: str | None, q: str | None): + """Shared WHERE clause for the picker projections: tenant scope, + entity_type, optional parent and optional case-insensitive substring + match on name or entity_id (same semantics as :meth:`search`).""" + stmt = select( + ImproveDigitalInventory.entity_id, + ImproveDigitalInventory.name, + ImproveDigitalInventory.parent_id, + ).filter_by(tenant_id=self._tenant_id, entity_type=entity_type) + if parent_id is not None: + stmt = stmt.filter(ImproveDigitalInventory.parent_id == parent_id) + if q: + pattern = f"%{q}%" + stmt = stmt.where( + (ImproveDigitalInventory.name.ilike(pattern)) | (ImproveDigitalInventory.entity_id.ilike(pattern)) + ) + return stmt + def list_picker_rows( self, entity_type: str, *, parent_id: str | None = None, + q: str | None = None, + offset: int = 0, + limit: int | None = None, ) -> list[tuple[str, str | None, str | None]]: - """Return ``(entity_id, name, parent_id)`` tuples for one entity_type. + """Return ``(entity_id, name, parent_id)`` tuples for one entity_type, + ordered by name then id so offset pagination is stable. - Column projection, not ORM instances: the product-form pickers pull - the full placement set (tens to hundreds of thousands of rows) and + Column projection, not ORM instances: the product-form pickers page + through the placement set (tens to hundreds of thousands of rows) and only need these three fields. Loading ``ImproveDigitalInventory`` objects here also decodes each row's ``raw_json`` JSONB payload and builds an ORM instance per row — measured at ~2 GB of resident memory per request in dev, which OOM-killed the 4 GB Fargate task whenever two product pages overlapped. Use :meth:`list_by_type` - only when the raw payload is actually needed. + only when the raw payload is actually needed. ``limit=None`` returns + every matching row (adapter-internal callers); the HTTP endpoint + always passes a bounded limit. """ - stmt = select( - ImproveDigitalInventory.entity_id, - ImproveDigitalInventory.name, - ImproveDigitalInventory.parent_id, - ).filter_by(tenant_id=self._tenant_id, entity_type=entity_type) - if parent_id is not None: - stmt = stmt.filter(ImproveDigitalInventory.parent_id == parent_id) + stmt = self._picker_filter(entity_type, parent_id=parent_id, q=q).order_by( + ImproveDigitalInventory.name.asc(), ImproveDigitalInventory.entity_id.asc() + ) + if offset: + stmt = stmt.offset(offset) + if limit is not None: + stmt = stmt.limit(limit) + return list(self._session.execute(stmt).tuples().all()) + + def count_picker_rows(self, entity_type: str, *, parent_id: str | None = None, q: str | None = None) -> int: + """Total rows :meth:`list_picker_rows` would return without a limit — + lets the picker show "N of M" and decide whether to offer Load more.""" + stmt = select(func.count()).select_from(self._picker_filter(entity_type, parent_id=parent_id, q=q).subquery()) + return int(self._session.scalar(stmt) or 0) + + def list_picker_rows_by_ids(self, entity_type: str, ids: Iterable[str]) -> list[tuple[str, str | None, str | None]]: + """Resolve already-attached ids to ``(entity_id, name, parent_id)`` so + the picker can label its chips without downloading the whole set.""" + wanted = [str(i) for i in ids if str(i).strip()] + if not wanted: + return [] + stmt = ( + select( + ImproveDigitalInventory.entity_id, + ImproveDigitalInventory.name, + ImproveDigitalInventory.parent_id, + ) + .filter_by(tenant_id=self._tenant_id, entity_type=entity_type) + .where(ImproveDigitalInventory.entity_id.in_(wanted)) + ) return list(self._session.execute(stmt).tuples().all()) def search( diff --git a/templates/adapters/improvedigital/product_config.html b/templates/adapters/improvedigital/product_config.html index 532856296d..3e63c98f95 100644 --- a/templates/adapters/improvedigital/product_config.html +++ b/templates/adapters/improvedigital/product_config.html @@ -62,6 +62,10 @@ .impd-result__add { color: #0369a1; font-size: 0.78rem; white-space: nowrap; } .impd-note { padding: 0.5rem 0.6rem; font-size: 0.8rem; color: #6b7280; } .impd-note--warn { color: #b45309; } +.impd-load-more { display: block; width: 100%; padding: 0.45rem 0.6rem; background: #f9fafb; border: 0; + border-top: 1px solid #e5e7eb; color: #0369a1; font-size: 0.8rem; cursor: pointer; text-align: center; } +.impd-load-more:hover { background: #f0f9ff; } +.impd-load-more[disabled] { color: #9ca3af; cursor: default; } .impd-badge { font-size: 0.68rem; font-weight: 600; text-transform: uppercase; letter-spacing: 0.03em; background: #ecfdf5; color: #047857; border: 1px solid #a7f3d0; border-radius: 4px; padding: 0.1rem 0.35rem; vertical-align: middle; } @@ -239,25 +243,19 @@

Line-item Defaults

const apiBase = scriptRoot + '/api/tenant/' + tenantId + '/adapters/improvedigital/inventory'; const settingsUrl = scriptRoot + '/tenant/' + tenantId + '/settings#adserver'; - // The placement cache runs to tens of thousands of rows and two pickers - // share it — fetch each entity type once and hand the same array to both. - const inventoryCache = {}; - function loadInventory(entityType) { - if (!inventoryCache[entityType]) { - inventoryCache[entityType] = fetch(apiBase + '?entity_type=' + encodeURIComponent(entityType), - {credentials: 'same-origin'}) - .then(response => response.json()) - .then(data => (data.success && data.items ? data.items : []).slice().sort( - (a, b) => String(a.name || a.entity_id).localeCompare(String(b.name || b.entity_id)))) - .catch(() => []); - } - return inventoryCache[entityType]; + // Inventory is paged from the server: the placement set runs to hundreds of + // thousands of rows, so the picker never downloads it whole. Each picker + // fetches one page for the current search and appends the next page on + // "Load more". Already-attached ids are labelled through an ids= lookup. + const PAGE_SIZE = 50; + const DEBOUNCE_MS = 250; + + function fetchInventory(entityType, params) { + const query = new URLSearchParams(Object.assign({entity_type: entityType}, params)); + return fetch(apiBase + '?' + query.toString(), {credentials: 'same-origin'}) + .then(response => response.json()); } - // Results are capped so a 25k-row cache can't stall the page; the count - // line tells the operator when a search still hides matches. - const RESULT_LIMIT = 200; - function initPicker(root) { const field = root.dataset.field; const noun = root.dataset.noun; @@ -304,16 +302,24 @@

Line-item Defaults

searchRow.insertAdjacentElement('afterend', results); results.insertAdjacentElement('afterend', hidden); - let items = []; + const entityType = root.dataset.entityType; + const labels = {}; // entity_id -> display name (from any page or the ids lookup) + let page = freshPage(''); // the result list for the current search + let requestSeq = 0; // drops responses from superseded searches let activeIndex = -1; + let debounceTimer = null; + + function freshPage(query) { + return {query: query, items: [], total: 0, offset: 0, hasMore: false, + loading: false, loaded: false, error: false}; + } - function byId(id) { - return items.find(item => String(item.entity_id) === String(id)); + function rememberLabels(list) { + list.forEach(item => { labels[String(item.entity_id)] = item.name || item.entity_id; }); } function labelFor(id) { - const item = byId(id); - return item ? (item.name || item.entity_id) : id; + return labels[String(id)] || String(id); } function syncHiddenInputs() { @@ -353,7 +359,7 @@

Line-item Defaults

remove.addEventListener('click', () => detach(id)); chip.appendChild(label); chip.appendChild(idTag); - if (root.dataset.entityType === 'package') { + if (entityType === 'package') { const peekBtn = document.createElement('button'); peekBtn.type = 'button'; peekBtn.className = 'impd-chip__peek'; @@ -373,30 +379,41 @@

Line-item Defaults

clearAll.style.display = selected.length ? '' : 'none'; } - function currentMatches() { - const needle = search.value.trim().toLowerCase(); - return items.filter(item => { - if (selected.includes(String(item.entity_id))) return false; - if (!needle) return true; - return String(item.name || item.entity_id).toLowerCase().includes(needle) - || String(item.entity_id).includes(needle); - }); + // Rows already attached are hidden from the list; the server total + // still counts them, so "Showing X of N" is by matching rows. + function visibleItems() { + return page.items.filter(item => !selected.includes(String(item.entity_id))); + } + + function note(text, warn) { + const div = document.createElement('div'); + div.className = 'impd-note' + (warn ? ' impd-note--warn' : ''); + div.textContent = text; + return div; } function renderResults() { - const matches = currentMatches(); - const shown = matches.slice(0, RESULT_LIMIT); + const shown = visibleItems(); activeIndex = -1; results.innerHTML = ''; - if (!items.length) { + if (page.error) { + results.appendChild(note('Lookup failed — is the API reachable?', true)); + return; + } + if (!page.loaded && !page.items.length) { + results.appendChild(note(page.loading ? 'Searching…' : '')); + return; + } + if (page.total === 0 && !page.query) { results.innerHTML = '
No ' + noun + 's in the inventory cache — run Sync Inventory Now.
'; return; } - if (!shown.length) { - results.innerHTML = '
No ' + noun + 's match “' + - search.value.replace(/'; + if (!shown.length && !page.hasMore) { + results.appendChild(note(page.query + ? 'No ' + noun + 's match “' + page.query + '”.' + : 'Every ' + noun + ' in the cache is already attached.')); return; } @@ -429,19 +446,69 @@

Line-item Defaults

results.appendChild(row); }); - const hiddenCount = matches.length - shown.length; - if (hiddenCount > 0) { - const note = document.createElement('div'); - note.className = 'impd-note'; - note.textContent = 'Showing ' + shown.length + ' of ' + matches.length + - ' matches — keep typing to narrow the remaining ' + hiddenCount + '.'; - results.appendChild(note); + if (page.hasMore || page.loading) { + const more = document.createElement('button'); + more.type = 'button'; + more.className = 'impd-load-more'; + more.disabled = page.loading; + more.textContent = page.loading + ? 'Loading…' + : 'Load more (showing ' + page.items.length + ' of ' + page.total + ' matches)'; + more.addEventListener('mousedown', event => { + // mousedown keeps the search input focused so the panel stays open. + event.preventDefault(); + if (!page.loading) loadPage(false); + }); + results.appendChild(more); + } else if (page.items.length) { + results.appendChild(note('Showing all ' + page.total + ' matching ' + noun + (page.total === 1 ? '' : 's') + '.')); } } - function openResults() { + // reset=true starts a new search (page 0 for the current input); + // reset=false appends the next page to the current search. + function loadPage(reset) { + const query = search.value.trim(); + if (reset) page = freshPage(query); + page.loading = true; + const seq = ++requestSeq; renderResults(); + fetchInventory(entityType, {q: query, offset: page.offset, limit: PAGE_SIZE}) + .then(data => { + if (seq !== requestSeq) return; // a newer search superseded this page + const items = data.success && data.items ? data.items : []; + rememberLabels(items); + page.items = page.items.concat(items); + page.total = data.count || 0; + page.offset += items.length; + page.hasMore = !!data.has_more; + page.loading = false; + page.loaded = true; + renderResults(); + }) + .catch(() => { + if (seq !== requestSeq) return; + page.loading = false; + page.loaded = true; + page.error = true; + renderResults(); + }); + } + + function openResults() { + results.hidden = false; + const query = search.value.trim(); + if (!page.loaded || page.query !== query) { + loadPage(true); + } else { + renderResults(); + } + } + + function scheduleSearch() { results.hidden = false; + clearTimeout(debounceTimer); + debounceTimer = setTimeout(() => loadPage(true), DEBOUNCE_MS); } function closeResults() { @@ -453,7 +520,6 @@

Line-item Defaults

if (!selected.includes(key)) selected.push(key); syncHiddenInputs(); renderChips(); - search.value = ''; renderResults(); search.focus(); } @@ -546,7 +612,7 @@

Line-item Defaults

}); search.addEventListener('focus', openResults); - search.addEventListener('input', openResults); + search.addEventListener('input', scheduleSearch); search.addEventListener('blur', () => setTimeout(closeResults, 120)); search.addEventListener('keydown', event => { const rows = Array.from(results.querySelectorAll('.impd-result')); @@ -570,14 +636,25 @@

Line-item Defaults

syncHiddenInputs(); renderChips(); - loadInventory(root.dataset.entityType).then(loaded => { - items = loaded; - renderChips(); // chip labels resolve once names are known - if (!items.length) { - helper.insertAdjacentHTML('beforeend', - ' Cache empty — run Sync Inventory Now.'); - } - }); + // Label the already-attached chips without downloading the whole set. + if (selected.length) { + fetchInventory(entityType, {ids: selected.join(',')}) + .then(data => { + rememberLabels(data.success && data.items ? data.items : []); + renderChips(); + }) + .catch(() => { /* chips keep showing raw ids */ }); + } + + // One cheap probe tells the operator up front when the cache is empty. + fetchInventory(entityType, {limit: 1}) + .then(data => { + if (data.success && data.count === 0) { + helper.insertAdjacentHTML('beforeend', + ' Cache empty — run Sync Inventory Now.'); + } + }) + .catch(() => { /* the results panel reports lookup failures */ }); } document.querySelectorAll('.impd-picker').forEach(initPicker); From 0bed89b99f61470951aae08ace100c494440c769 Mon Sep 17 00:00:00 2001 From: chinmoy Date: Fri, 11 Sep 2026 15:55:41 +0600 Subject: [PATCH 86/90] added migration for pg_trgm index --- ...add_improvedigital_inventory_trgm_index.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 alembic/versions/impd03_add_improvedigital_inventory_trgm_index.py diff --git a/alembic/versions/impd03_add_improvedigital_inventory_trgm_index.py b/alembic/versions/impd03_add_improvedigital_inventory_trgm_index.py new file mode 100644 index 0000000000..2fc473f2c7 --- /dev/null +++ b/alembic/versions/impd03_add_improvedigital_inventory_trgm_index.py @@ -0,0 +1,77 @@ +"""add trigram index for Improve Digital inventory picker search + +The product-form pickers page through ``improvedigital_inventory`` with a +case-insensitive substring match on ``name`` OR ``entity_id`` (see +``ImproveDigitalInventoryRepository.list_picker_rows`` / +``count_picker_rows``). Without an index every keystroke is a sequential +scan of the tenant's placement rows — hundreds of thousands in dev. A GIN +trigram index lets PostgreSQL answer ``ILIKE '%term%'`` on both columns +with a bitmap scan. + +``pg_trgm`` ships with PostgreSQL (and RDS) and is a *trusted* extension on +PostgreSQL 13+, so the application role can install it. The migration is +defensive anyway: if the extension is unavailable or cannot be created, it +logs a warning and leaves the table unindexed — the search still works, +just slower — rather than failing the boot-time migration and taking the +service down over a performance index. + +Revision ID: impd03c4d5e6 +Revises: a9b8c7d6e5f4 +Create Date: 2026-09-11 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "impd03c4d5e6" +down_revision: str | Sequence[str] | None = "a9b8c7d6e5f4" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +INDEX_NAME = "ix_improvedigital_inventory_trgm" + + +def _ensure_pg_trgm(conn: sa.Connection) -> bool: + """Install ``pg_trgm`` if needed. Returns False when it cannot be used.""" + installed = conn.execute(sa.text("SELECT 1 FROM pg_extension WHERE extname = 'pg_trgm'")).scalar() + if installed: + return True + available = conn.execute(sa.text("SELECT 1 FROM pg_available_extensions WHERE name = 'pg_trgm'")).scalar() + if not available: + print("WARNING: pg_trgm is not available on this PostgreSQL server; skipping trigram index") + return False + # Savepoint so a permission failure does not abort the surrounding + # migration transaction. + try: + with conn.begin_nested(): + conn.execute(sa.text("CREATE EXTENSION IF NOT EXISTS pg_trgm")) + except sa.exc.DBAPIError as exc: + print(f"WARNING: could not create pg_trgm extension ({exc.orig}); skipping trigram index") + return False + return True + + +def upgrade() -> None: + conn = op.get_bind() + if conn.dialect.name != "postgresql": + return + if not _ensure_pg_trgm(conn): + return + conn.execute( + sa.text( + f"CREATE INDEX IF NOT EXISTS {INDEX_NAME} " + "ON improvedigital_inventory USING gin (name gin_trgm_ops, entity_id gin_trgm_ops)" + ) + ) + + +def downgrade() -> None: + conn = op.get_bind() + if conn.dialect.name != "postgresql": + return + # The extension is left installed: other objects may come to rely on it. + conn.execute(sa.text(f"DROP INDEX IF EXISTS {INDEX_NAME}")) From c18f2bdbed13dc348b334504f98a9351261b7c96 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 15 Sep 2026 06:18:15 +0600 Subject: [PATCH 87/90] Feature/inventory (#64) * fix: wire Improve Digital into inventory bundles and fix Sync Now feedback * feat: stamp adapter config lock in the same transaction as inventory writes * fix: make Improve Digital inventory bundles authorable end to end --- src/adapters/improvedigital/inventory_sync.py | 7 + src/admin/blueprints/inventory_profiles.py | 95 ++++- src/core/database/adapter_config_lock.py | 15 +- .../repositories/improvedigital_inventory.py | 20 +- src/services/bundle_adapter.py | 245 ++++++++++- src/services/gam_inventory_service.py | 17 + static/css/tenant_inventory_profiles.css | 30 ++ templates/edit_inventory_profile.html | 394 ++++++++++++++++-- .../inventory_browser_improvedigital.html | 56 ++- templates/sync_inventory.html | 66 ++- .../test_inventory_profiles_improvedigital.py | 207 +++++++++ tests/integration/test_adapter_config_lock.py | 42 ++ tests/unit/test_bundle_adapter.py | 123 +++++- 13 files changed, 1243 insertions(+), 74 deletions(-) create mode 100644 tests/admin/test_inventory_profiles_improvedigital.py diff --git a/src/adapters/improvedigital/inventory_sync.py b/src/adapters/improvedigital/inventory_sync.py index 78e3c91f0a..a7419a1dcf 100644 --- a/src/adapters/improvedigital/inventory_sync.py +++ b/src/adapters/improvedigital/inventory_sync.py @@ -115,6 +115,7 @@ class ImproveDigitalInventorySync: def __init__(self, client: ImproveDigitalClient, session: Session, tenant_id: str): self._client = client self._session = session + self._tenant_id = tenant_id self._repo = ImproveDigitalInventoryRepository(session, tenant_id) def _persist_page(self, rows: list[dict[str, Any]]) -> None: @@ -124,6 +125,12 @@ def _persist_page(self, rows: list[dict[str, Any]]) -> None: if not rows: return self._repo.bulk_upsert(rows) + # The first inventory rows freeze the tenant's ad server configuration + # (adapter_config_lock.py) — stamp in the same per-page transaction as + # the rows, not only at sync completion. Idempotent after the first page. + from src.core.database.repositories.adapter_config import AdapterConfigRepository + + AdapterConfigRepository(self._session, self._tenant_id).mark_config_locked() self._session.commit() def run(self) -> SyncResult: diff --git a/src/admin/blueprints/inventory_profiles.py b/src/admin/blueprints/inventory_profiles.py index 841a0ba7f5..45e35ff095 100644 --- a/src/admin/blueprints/inventory_profiles.py +++ b/src/admin/blueprints/inventory_profiles.py @@ -1026,13 +1026,18 @@ def _build_inventory_picker_payload( selected_ad_unit_ids = list((inventory_config or {}).get("ad_units") or []) selected_placement_ids = list((inventory_config or {}).get("placements") or []) + # Paged adapters (Improve Digital) search/page rows from the server via + # ``search_inventory_picker_api``; the embedded payload only needs to + # label the rows already selected on this bundle. + paged = bool(getattr(adapter, "picker_paged", False)) + placement_rows = _merge_picker_rows( - adapter.list_inventory(session, tenant_id, "placement", limit=limit), + [] if paged else adapter.list_inventory(session, tenant_id, "placement", limit=limit), adapter.list_inventory_by_ids(session, tenant_id, "placement", selected_placement_ids), ) child_ad_unit_ids = sorted({child_id for row in placement_rows for child_id in _placement_child_ids(row)}) - ad_unit_rows = adapter.list_inventory(session, tenant_id, "ad_unit", limit=limit) + ad_unit_rows = [] if paged else adapter.list_inventory(session, tenant_id, "ad_unit", limit=limit) selected_ad_unit_rows = adapter.list_inventory_by_ids(session, tenant_id, "ad_unit", selected_ad_unit_ids) child_ad_unit_rows = adapter.list_inventory_by_ids(session, tenant_id, "ad_unit", child_ad_unit_ids) ad_unit_rows = _merge_picker_rows(_merge_picker_rows(ad_unit_rows, selected_ad_unit_rows), child_ad_unit_rows) @@ -1057,6 +1062,40 @@ def _build_inventory_picker_payload( return payload +INVENTORY_PICKER_PAGE_SIZE = 50 +INVENTORY_PICKER_MAX_PAGE_SIZE = 200 +_DEFAULT_PICKER_VOCAB = {"ad_units": "ad units", "placements": "placements"} + + +def _singular(noun: str) -> str: + return noun[:-1] if noun.endswith("s") else noun + + +def _inventory_picker_context(session, tenant_id: str, adapter) -> dict: + """Adapter vocabulary, paging flag and creative-size options for the + bundle editor template. + + GAM keeps its "ad units / placements" wording and derives formats from + the selected inventory; Improve Digital renders "placements / packages", + pages inventory from the server, and — because 360Yield placements carry + no sizes — offers the synced size catalogue as an explicit picker. + """ + vocab = ( + {"ad_units": adapter.vocab["primary"], "placements": adapter.vocab["secondary"]} + if adapter is not None + else dict(_DEFAULT_PICKER_VOCAB) + ) + explicit_sizes = bool(adapter is not None and getattr(adapter, "explicit_creative_sizes", False)) + return { + "adapter_vocab": vocab, + "adapter_vocab_singular": {key: _singular(value) for key, value in vocab.items()}, + "inventory_picker_paged": bool(adapter is not None and getattr(adapter, "picker_paged", False)), + "inventory_picker_page_size": INVENTORY_PICKER_PAGE_SIZE, + "explicit_creative_sizes": explicit_sizes, + "creative_size_options": adapter.list_creative_sizes(session, tenant_id) if explicit_sizes else [], + } + + def _list_products_using(session, tenant_id: str, profile_id: int) -> list[dict]: """Products that reference this bundle, with name + id for sidebar rendering. @@ -1302,6 +1341,7 @@ def add_inventory_profile(tenant_id: str): ) inventory_picker = _build_inventory_picker_payload(session, tenant_id, adapter, profile.inventory_config) known_property_tags = sorted({tag for prop in authorized_properties for tag in (prop.tags or [])}) + picker_context = _inventory_picker_context(session, tenant_id, adapter) return render_template( "edit_inventory_profile.html", @@ -1321,6 +1361,7 @@ def add_inventory_profile(tenant_id: str): products_using=[], form_mode="create", active_tab="inventory_profiles", + **picker_context, ) @@ -1593,6 +1634,7 @@ def edit_inventory_profile(tenant_id: str, profile_id: int): known_property_tags=known_property_tags, products_using=products_using, active_tab="inventory_profiles", + **_inventory_picker_context(session, tenant_id, adapter), ) @@ -2014,6 +2056,55 @@ def preview_inventory_profile_api(tenant_id: str, profile_id: int): ) +@inventory_profiles_bp.route("/api/inventory") +@require_tenant_access(api_mode=True) +def search_inventory_picker_api(tenant_id: str): + """One page of synced inventory for the bundle editor's picker. + + Mirrors the Improve Digital product-form picker: the placement set is + too large to embed, so the editor searches and pages here. + + Query params: ``kind`` (``placements`` | ``ad_units`` — bundle slots, + mapped by the adapter), ``q`` (substring on name or id), ``offset``, + ``limit`` (default 50, max 200). ``count`` is the total matching rows + (``null`` when the adapter cannot count cheaply); ``has_more`` says + whether another page exists. Items use the same shape as the embedded + ``INVENTORY_PICKER`` rows. + """ + kind = request.args.get("kind", "ad_units") + if kind not in {"ad_units", "placements"}: + return jsonify({"success": False, "error": "kind must be 'ad_units' or 'placements'"}), 400 + entity_type = "placement" if kind == "placements" else "ad_unit" + q = (request.args.get("q") or "").strip() or None + offset = max(request.args.get("offset", default=0, type=int) or 0, 0) + limit = request.args.get("limit", default=INVENTORY_PICKER_PAGE_SIZE, type=int) or INVENTORY_PICKER_PAGE_SIZE + limit = max(1, min(limit, INVENTORY_PICKER_MAX_PAGE_SIZE)) + + with get_db_session() as session: + tenant = session.get(Tenant, tenant_id) + adapter = adapter_for_tenant(tenant.ad_server) if tenant else None + if adapter is None: + return jsonify({"success": True, "kind": kind, "items": [], "count": 0, "has_more": False}) + rows, total = adapter.search_inventory(session, tenant_id, entity_type, q=q, offset=offset, limit=limit) + ids_by_key: dict[str, set[str]] = {"ad_units": set(), "placements": set()} + ids_by_key[kind] = {row.external_id for row in rows} + membership = _bundle_membership_counts(session, tenant_id, ids_by_key)[kind] + items = [_inventory_picker_row(row, membership.get(row.external_id, 0)) for row in rows] + + has_more = offset + len(items) < total if isinstance(total, int) else len(items) >= limit + return jsonify( + { + "success": True, + "kind": kind, + "items": items, + "count": total, + "offset": offset, + "limit": limit, + "has_more": has_more, + } + ) + + @inventory_profiles_bp.route("/api/list") @require_tenant_access(api_mode=True) def list_inventory_profiles_api(tenant_id: str): diff --git a/src/core/database/adapter_config_lock.py b/src/core/database/adapter_config_lock.py index 8307c102cf..507cf5c4a3 100644 --- a/src/core/database/adapter_config_lock.py +++ b/src/core/database/adapter_config_lock.py @@ -16,11 +16,16 @@ flags, other ``config_json`` fields) stays editable. Lock state is stored explicitly in ``adapter_config.config_locked_at`` -(adapter-agnostic — every inventory-sync completion path stamps it via -:meth:`AdapterConfigRepository.mark_config_locked`, and the migration -backfilled already-synced tenants). ``config_locked_at`` is itself a locked -column: stamping it on an unlocked tenant is free, but clearing it once set -requires ``super_admin_override`` — that is the documented unlock procedure. +(adapter-agnostic). The inventory *persistence* paths stamp it via +:meth:`AdapterConfigRepository.mark_config_locked` in the same transaction as +the first inventory rows (GAM batch writers, Improve Digital per-page upsert; +FreeWheel/SpringServe write in the single orchestration transaction), so the +tenant is locked the moment inventory lands in the DB — even if the sync dies +before completion. The sync-completion paths re-stamp as an idempotent +backstop, and the migration backfilled already-synced tenants. +``config_locked_at`` is itself a locked column: stamping it on an unlocked +tenant is free, but clearing it once set requires ``super_admin_override`` — +that is the documented unlock procedure. Enforcement mirrors :mod:`src.core.database.embedded_tenant_guard`: SQLAlchemy ``before_update`` listeners compare each locked column's pending value against diff --git a/src/core/database/repositories/improvedigital_inventory.py b/src/core/database/repositories/improvedigital_inventory.py index faef1a4d7d..60725e1921 100644 --- a/src/core/database/repositories/improvedigital_inventory.py +++ b/src/core/database/repositories/improvedigital_inventory.py @@ -42,10 +42,18 @@ def list_by_type( stmt = stmt.filter(ImproveDigitalInventory.parent_id == parent_id) return list(self._session.scalars(stmt).all()) - def _picker_filter(self, entity_type: str, *, parent_id: str | None, q: str | None): + def _picker_filter( + self, + entity_type: str, + *, + parent_id: str | None, + q: str | None, + exclude_ids: Iterable[str] | None = None, + ): """Shared WHERE clause for the picker projections: tenant scope, - entity_type, optional parent and optional case-insensitive substring - match on name or entity_id (same semantics as :meth:`search`).""" + entity_type, optional parent, optional case-insensitive substring + match on name or entity_id (same semantics as :meth:`search`) and an + optional id exclusion set (the bundle page's "not yet bundled" rail).""" stmt = select( ImproveDigitalInventory.entity_id, ImproveDigitalInventory.name, @@ -58,6 +66,9 @@ def _picker_filter(self, entity_type: str, *, parent_id: str | None, q: str | No stmt = stmt.where( (ImproveDigitalInventory.name.ilike(pattern)) | (ImproveDigitalInventory.entity_id.ilike(pattern)) ) + excluded = [str(i) for i in (exclude_ids or ()) if str(i).strip()] + if excluded: + stmt = stmt.where(ImproveDigitalInventory.entity_id.not_in(excluded)) return stmt def list_picker_rows( @@ -68,6 +79,7 @@ def list_picker_rows( q: str | None = None, offset: int = 0, limit: int | None = None, + exclude_ids: Iterable[str] | None = None, ) -> list[tuple[str, str | None, str | None]]: """Return ``(entity_id, name, parent_id)`` tuples for one entity_type, ordered by name then id so offset pagination is stable. @@ -83,7 +95,7 @@ def list_picker_rows( every matching row (adapter-internal callers); the HTTP endpoint always passes a bounded limit. """ - stmt = self._picker_filter(entity_type, parent_id=parent_id, q=q).order_by( + stmt = self._picker_filter(entity_type, parent_id=parent_id, q=q, exclude_ids=exclude_ids).order_by( ImproveDigitalInventory.name.asc(), ImproveDigitalInventory.entity_id.asc() ) if offset: diff --git a/src/services/bundle_adapter.py b/src/services/bundle_adapter.py index a0300729a7..ded67c3bc8 100644 --- a/src/services/bundle_adapter.py +++ b/src/services/bundle_adapter.py @@ -12,7 +12,7 @@ adapter-agnostic ``BundleInventoryRow`` records so templates don't need to know which adapter owns a row. -GAM is fully implemented. FW + SS are honest stubs — they return empty +GAM and Improve Digital are implemented. FW + SS are honest stubs — they return empty results today (their inventory sync surfaces don't carry the same data shape yet) but ship with their canonical labels + vocab so a tenant on either of them sees the right copy in the page header. @@ -67,6 +67,14 @@ class BundleInventoryAdapter(Protocol): label: str # "Google Ad Manager" vocab: dict[str, str] # {"primary": "ad units", "secondary": "placements"} matches_tenant_ad_server: set[str] # values of ``tenant.ad_server`` to claim + # True when the bundle editor should page inventory from the server + # (via :meth:`search_inventory`) instead of embedding a bounded list. + picker_paged: bool + # True when the adapter's inventory rows carry no creative sizes, so the + # bundle editor offers an explicit size picker (fed by + # :meth:`list_creative_sizes`) instead of deriving formats from the + # selected inventory. + explicit_creative_sizes: bool def has_synced_inventory(self, session: Session, tenant_id: str) -> bool: ... @@ -96,6 +104,26 @@ def find_inventory_item( def coverage_for_bundle(self, session: Session, tenant_id: str, inventory_config: dict) -> int: ... + def search_inventory( + self, + session: Session, + tenant_id: str, + entity_type: str, + *, + q: str | None = None, + offset: int = 0, + limit: int = 50, + ) -> tuple[list[BundleInventoryRow], int | None]: + """One page of rows matching ``q`` plus the total match count + (``None`` when the adapter cannot count cheaply).""" + ... + + def list_creative_sizes(self, session: Session, tenant_id: str) -> list[dict[str, Any]]: + """Options for the explicit creative-size picker — ``{label, width, + height, kind}`` with ``kind`` ``display`` | ``video``. Empty for + adapters that derive sizes from the selected inventory.""" + ... + # --------------------------------------------------------------------------- # Registry @@ -164,6 +192,8 @@ class _GAMAdapter: label = "Google Ad Manager" vocab = {"primary": "ad units", "secondary": "placements"} matches_tenant_ad_server = {"google_ad_manager", "gam"} + picker_paged = False + explicit_creative_sizes = False def _row_from_gam_inventory(self, row) -> BundleInventoryRow: return BundleInventoryRow( @@ -249,6 +279,201 @@ def coverage_for_bundle(self, session: Session, tenant_id: str, inventory_config synced_ad_unit_ids = {row.inventory_id for row in repo.list_inventory("ad_unit")} return len(covered.intersection(synced_ad_unit_ids)) + def search_inventory( + self, + session: Session, + tenant_id: str, + entity_type: str, + *, + q: str | None = None, + offset: int = 0, + limit: int = 50, + ) -> tuple[list[BundleInventoryRow], int | None]: + from src.core.database.repositories.gam_sync import GAMSyncRepository + + repo = GAMSyncRepository(session, tenant_id) + rows = repo.search_inventory(entity_type, q=q, offset=offset, limit=limit) + total = repo.count_inventory(entity_type) if not q else None + return [self._row_from_gam_inventory(r) for r in rows], total + + def list_creative_sizes(self, session: Session, tenant_id: str) -> list[dict[str, Any]]: + return [] + + +# --------------------------------------------------------------------------- +# Improve Digital adapter +# --------------------------------------------------------------------------- + + +class _ImproveDigitalAdapter: + """Improve Digital (360Yield) bundle-inventory adapter. + + Reads the ``improvedigital_inventory`` cache filled by + ``ImproveDigitalInventorySync``. The bundle model has two slots — a + leaf entity (``ad_unit``) and a wrapper (``placement``) — which map onto + 360Yield's placements and placement packages respectively: + + * bundle ``ad_unit`` → cache ``placement`` (the bookable leaf) + * bundle ``placement`` → cache ``package`` (reusable placement grouping) + + Package membership is not cached (it is a live per-package lookup), so + packages never expand into child placements here and coverage counts + only directly-picked placements. + + 360Yield placements carry no creative sizes (sizes are a search filter + and a separate ``size`` lookup), so ``explicit_creative_sizes`` is on: + the editor offers the synced size catalogue and derives the bundle's + canonical formats from the sizes the operator picks. + + ``picker_paged`` is on: the placement set runs to hundreds of thousands + of rows, so the editor searches/pages through :meth:`search_inventory` + instead of embedding a bounded list. + """ + + adapter_id = "improvedigital" + label = "Improve Digital" + vocab = {"primary": "placements", "secondary": "packages"} + matches_tenant_ad_server = {"improvedigital", "improve_digital"} + picker_paged = True + explicit_creative_sizes = True + + _CACHE_ENTITY = {"ad_unit": "placement", "placement": "package"} + # 360Yield size ``type`` → AdCP creative family. ``vast_audio`` has no + # display/video equivalent and is skipped. + _SIZE_KIND = {"display": "display", "mobile_app": "display", "text": "display", "vast": "video"} + + def _repo(self, session: Session, tenant_id: str): + from src.core.database.repositories.improvedigital_inventory import ImproveDigitalInventoryRepository + + return ImproveDigitalInventoryRepository(session, tenant_id) + + def _cache_type(self, entity_type: str) -> str: + try: + return self._CACHE_ENTITY[entity_type] + except KeyError as exc: + raise ValueError(f"Unknown bundle entity_type {entity_type!r}") from exc + + def _rows( + self, repo, entity_type: str, tuples: list[tuple[str, str | None, str | None]] + ) -> list[BundleInventoryRow]: + """Normalize ``(entity_id, name, parent_id)`` tuples. Placements carry + their publisher as ``parent_id``; one extra lookup labels them.""" + parent_ids = sorted({parent for _, _, parent in tuples if parent}) + publishers: dict[str, str | None] = {} + if parent_ids: + publishers = {pid: name for pid, name, _ in repo.list_picker_rows_by_ids("publisher", parent_ids)} + rows: list[BundleInventoryRow] = [] + for entity_id, name, parent_id in tuples: + publisher = publishers.get(parent_id) if parent_id else None + rows.append( + BundleInventoryRow( + external_id=str(entity_id), + name=name or str(entity_id), + entity_type=entity_type, + meta=publisher or "—", + raw={"metadata": {"parent_id": parent_id}, "publisher": publisher}, + ) + ) + return rows + + def has_synced_inventory(self, session: Session, tenant_id: str) -> bool: + return ( + self.count_inventory(session, tenant_id, "ad_unit") + self.count_inventory(session, tenant_id, "placement") + ) > 0 + + def count_inventory(self, session: Session, tenant_id: str, entity_type: str) -> int: + return self._repo(session, tenant_id).count_picker_rows(self._cache_type(entity_type)) + + def list_inventory_by_ids( + self, session: Session, tenant_id: str, entity_type: str, ids: list[str] + ) -> list[BundleInventoryRow]: + if not ids: + return [] + repo = self._repo(session, tenant_id) + return self._rows(repo, entity_type, repo.list_picker_rows_by_ids(self._cache_type(entity_type), ids)) + + def list_inventory( + self, session: Session, tenant_id: str, entity_type: str, limit: int | None = None + ) -> list[BundleInventoryRow]: + repo = self._repo(session, tenant_id) + return self._rows(repo, entity_type, repo.list_picker_rows(self._cache_type(entity_type), limit=limit)) + + def list_unbundled( + self, + session: Session, + tenant_id: str, + bundled_ids_by_type: dict[str, set[str]], + limit: int, + ) -> list[BundleInventoryRow]: + # Packages first (they group placements), then placements fill the rest. + repo = self._repo(session, tenant_id) + out: list[BundleInventoryRow] = [] + for entity_type in ("placement", "ad_unit"): + remaining = limit - len(out) + if remaining <= 0: + break + tuples = repo.list_picker_rows( + self._cache_type(entity_type), + limit=remaining, + exclude_ids=bundled_ids_by_type.get(entity_type) or (), + ) + out.extend(self._rows(repo, entity_type, tuples)) + return out + + def list_top_level_placements(self, session: Session, tenant_id: str, limit: int) -> list[BundleInventoryRow]: + return self.list_inventory(session, tenant_id, "placement", limit=limit) + + def find_inventory_item( + self, session: Session, tenant_id: str, entity_type: str, external_id: str + ) -> BundleInventoryRow | None: + rows = self.list_inventory_by_ids(session, tenant_id, entity_type, [external_id]) + return rows[0] if rows else None + + def coverage_for_bundle(self, session: Session, tenant_id: str, inventory_config: dict) -> int: + placement_ids = [str(i) for i in (inventory_config.get("ad_units") or []) if i] + if not placement_ids: + return 0 + return len(self._repo(session, tenant_id).list_picker_rows_by_ids("placement", placement_ids)) + + def search_inventory( + self, + session: Session, + tenant_id: str, + entity_type: str, + *, + q: str | None = None, + offset: int = 0, + limit: int = 50, + ) -> tuple[list[BundleInventoryRow], int | None]: + repo = self._repo(session, tenant_id) + cache_type = self._cache_type(entity_type) + total = repo.count_picker_rows(cache_type, q=q) + tuples = repo.list_picker_rows(cache_type, q=q, offset=offset, limit=limit) + return self._rows(repo, entity_type, tuples), total + + def list_creative_sizes(self, session: Session, tenant_id: str) -> list[dict[str, Any]]: + """Distinct ``(kind, width, height)`` options from the synced size + catalogue (~800 rows, so loading the raw payloads is cheap). 1x1 and + 2x1 "text" placeholders have no AdCP display equivalent and are + dropped.""" + options: dict[tuple[str, int, int], dict[str, Any]] = {} + for row in self._repo(session, tenant_id).list_by_type("size"): + raw = row.raw_json if isinstance(row.raw_json, dict) else {} + kind = self._SIZE_KIND.get(str(raw.get("type") or "display")) + width_raw, height_raw = raw.get("width"), raw.get("height") + if kind is None or width_raw is None or height_raw is None: + continue + try: + width, height = int(width_raw), int(height_raw) + except (TypeError, ValueError): + continue + if width <= 1 or height <= 1: + continue + options.setdefault( + (kind, width, height), {"label": f"{width}x{height}", "width": width, "height": height, "kind": kind} + ) + return [options[key] for key in sorted(options)] + # --------------------------------------------------------------------------- # FreeWheel + SpringServe stubs @@ -271,6 +496,8 @@ def __init__(self, *, adapter_id: str, label: str, vocab: dict[str, str], ad_ser self.label = label self.vocab = vocab self.matches_tenant_ad_server = ad_server_aliases + self.picker_paged = False + self.explicit_creative_sizes = False def has_synced_inventory(self, session: Session, tenant_id: str) -> bool: return False @@ -308,12 +535,28 @@ def find_inventory_item( def coverage_for_bundle(self, session: Session, tenant_id: str, inventory_config: dict) -> int: return 0 + def search_inventory( + self, + session: Session, + tenant_id: str, + entity_type: str, + *, + q: str | None = None, + offset: int = 0, + limit: int = 50, + ) -> tuple[list[BundleInventoryRow], int | None]: + return [], 0 + + def list_creative_sizes(self, session: Session, tenant_id: str) -> list[dict[str, Any]]: + return [] + # --------------------------------------------------------------------------- # Module-level registration # --------------------------------------------------------------------------- register_adapter(_GAMAdapter()) +register_adapter(_ImproveDigitalAdapter()) register_adapter( _NullInventoryAdapter( adapter_id="freewheel", diff --git a/src/services/gam_inventory_service.py b/src/services/gam_inventory_service.py index 37547612c2..a112b690e2 100644 --- a/src/services/gam_inventory_service.py +++ b/src/services/gam_inventory_service.py @@ -98,6 +98,14 @@ def flush_batch(): """Flush current batch to database with error handling.""" nonlocal total_inserted, total_updated try: + if to_insert or to_update: + # The first inventory rows freeze the tenant's ad server + # configuration (adapter_config_lock.py) — stamp in the + # same transaction as the rows. Idempotent after the + # first batch. + from src.core.database.repositories.adapter_config import AdapterConfigRepository + + AdapterConfigRepository(self.db, tenant_id).mark_config_locked() if to_insert: self.db.bulk_insert_mappings(GAMInventory, to_insert) batch_inserted = len(to_insert) @@ -484,6 +492,15 @@ def _write_inventory_batch(self, tenant_id: str, inventory_type: str, items: lis if not items: return + # The first inventory rows freeze the tenant's ad server configuration + # (adapter_config_lock.py). Stamped here — not only at sync completion — + # so config_locked_at commits atomically with the first batch of + # inventory rows, even if the sync dies mid-way. Idempotent after the + # first call. + from src.core.database.repositories.adapter_config import AdapterConfigRepository + + AdapterConfigRepository(self.db, tenant_id).mark_config_locked() + logger.info(f"📊 Writing {len(items)} {inventory_type} items to database...") BATCH_SIZE = 500 diff --git a/static/css/tenant_inventory_profiles.css b/static/css/tenant_inventory_profiles.css index 17facb5687..c63491c01a 100644 --- a/static/css/tenant_inventory_profiles.css +++ b/static/css/tenant_inventory_profiles.css @@ -1092,6 +1092,36 @@ letter-spacing: 0; font-weight: 450; } +.size-picker { + margin-bottom: 12px; +} + +.size-picker .size-picker__list { + max-height: 280px; +} + +.tree-picker__more { + display: block; + width: 100%; + padding: 10px 18px; + background: var(--sig-bg-subtle, #f9fafb); + border: 0; + border-top: 1px solid var(--sig-bd-hairline); + color: var(--sig-fg-link, #0369a1); + font-size: 12.5px; + cursor: pointer; + text-align: center; +} + +.tree-picker__more:hover { + background: var(--sig-bg-hover, #f0f9ff); +} + +.tree-picker__more[disabled] { + color: var(--sig-fg-tertiary); + cursor: default; +} + .tree-picker__empty { padding: 32px 20px; text-align: center; diff --git a/templates/edit_inventory_profile.html b/templates/edit_inventory_profile.html index 7d47cfdfe3..b9ebe3d146 100644 --- a/templates/edit_inventory_profile.html +++ b/templates/edit_inventory_profile.html @@ -8,6 +8,12 @@ {% set is_create = form_mode|default('edit') == 'create' %} {% set s = bundle_summary %} +{# Adapter vocabulary for the inventory picker: GAM reads "ad units / placements", + Improve Digital "placements / packages". Defaults keep legacy callers working. #} +{% set vocab = adapter_vocab|default({'ad_units': 'ad units', 'placements': 'placements'}) %} +{% set vocab1 = adapter_vocab_singular|default({'ad_units': 'ad unit', 'placements': 'placement'}) %} +{% set picker_paged = inventory_picker_paged|default(false) %} +{% set explicit_sizes = explicit_creative_sizes|default(false) %}
@@ -232,14 +238,15 @@

Basics

Inventory

- Pick placements or ad units from your synced {{ s.adapter_label }} inventory. - Placements wrap ad units — picking a placement covers all its children. + Pick {{ vocab.placements }} or {{ vocab.ad_units }} from your synced {{ s.adapter_label }} inventory. + {{ vocab.placements|capitalize }} wrap {{ vocab.ad_units }} — picking a {{ vocab1.placements }} covers all its children. + {% if picker_paged %}Results are searched and paged from the server — type to filter, then load more.{% endif %}

- - + +
@@ -250,7 +257,7 @@

Inventory

@@ -261,7 +268,7 @@

Inventory

@@ -269,8 +276,8 @@

Inventory

- Picking a placement includes its child ad units automatically. - Flip to Flat ad units to pick individual units only. + Picking a {{ vocab1.placements }} includes its child {{ vocab.ad_units }} automatically. + Flip to Flat {{ vocab.ad_units }} to pick individual units only.
@@ -285,11 +292,32 @@

Inventory

Creative formats

+ {% if explicit_sizes %} + {{ s.adapter_label }} {{ vocab.ad_units }} don't carry creative sizes, so pick the sizes this + bundle sells. Formats are derived from the sizes you select here. + {% else %} Automatically derived from selected GAM inventory sizes and rendering settings. + {% endif %}

+ {% if explicit_sizes %} +
+
+
+
+ + +
+ +
+
+
+ {% endif %}
Pick inventory to detect creative formats. @@ -437,6 +465,20 @@

Properties

const INVENTORY_PICKER = {{ inventory_picker | tojson | safe }}; const INVENTORY_PICKER_LIMIT = {{ inventory_picker_limit | tojson }}; +// Server-paged picker (Improve Digital): rows are searched/paged through +// the inventory-profiles API instead of the embedded INVENTORY_PICKER list, +// which only seeds already-selected rows. Mirrors the product-form picker. +const INVENTORY_PICKER_PAGED = {{ picker_paged | tojson }}; +const INVENTORY_PICKER_PAGE_SIZE = {{ inventory_picker_page_size|default(50) | tojson }}; +const INVENTORY_PICKER_API = '{{ url_for("inventory_profiles.search_inventory_picker_api", tenant_id=tenant_id) }}'; +const INVENTORY_VOCAB = {{ vocab | tojson | safe }}; +const INVENTORY_VOCAB_SINGULAR = {{ vocab1 | tojson | safe }}; +const INVENTORY_PICKER_DEBOUNCE_MS = 250; +// Explicit creative sizes (Improve Digital): the inventory rows carry no +// sizes, so formats are derived from sizes the operator picks out of the +// synced size catalogue instead of from the selected inventory. +const INVENTORY_EXPLICIT_SIZES = {{ explicit_sizes | tojson }}; +const CREATIVE_SIZE_OPTIONS = {{ creative_size_options|default([]) | tojson | safe }}; const INVENTORY_CHIP_REUSE_ENABLED = {{ (not is_create) | tojson }}; const BUNDLE_ADAPTER_ID = {{ (s.adapter_id or '') | tojson }}; const KNOWN_PROPERTY_TAGS = new Set({{ known_property_tags | tojson | safe }}); @@ -477,6 +519,10 @@

Properties

} function inventoryKindLabel(row) { + if (BUNDLE_ADAPTER_ID !== 'gam') { + const key = row.kind === 'placement' ? 'placements' : 'ad_units'; + return (INVENTORY_VOCAB_SINGULAR[key] || key).toUpperCase(); + } if (row.kind === 'placement') return row.subkind === 'tag' ? 'TAG' : 'SITE'; return 'AD UNIT'; } @@ -567,7 +613,7 @@

Properties

}); if (!items.length) { - strip.innerHTML = '
No inventory selected yet. Pick placements or ad units below.
'; + strip.innerHTML = `
No inventory selected yet. Pick ${_escAttr(INVENTORY_VOCAB.placements)} or ${_escAttr(INVENTORY_VOCAB.ad_units)} below.
`; return; } @@ -609,10 +655,203 @@

Properties

} function inventoryPickerMayBeCapped() { + if (INVENTORY_PICKER_PAGED) return false; return inventoryRows('ad_units').length >= INVENTORY_PICKER_LIMIT || inventoryRows('placements').length >= INVENTORY_PICKER_LIMIT; } +// ---- explicit creative-size picker (INVENTORY_EXPLICIT_SIZES only) ---- +// Keys are `${kind}:${width}x${height}`; the selection is persisted only +// through the derived formats (width/height on display_* / video_* ids), +// so edit pages rebuild it from existingProfile.formats. +const creativeSizeSelection = new Set(); + +function creativeSizeKey(option) { + return `${option.kind}:${option.width}x${option.height}`; +} + +function selectedCreativeSizeOptions() { + return CREATIVE_SIZE_OPTIONS.filter(option => creativeSizeSelection.has(creativeSizeKey(option))); +} + +function explicitSizeFormats() { + const formats = []; + selectedCreativeSizeOptions().forEach(option => { + const ids = option.kind === 'video' ? GAM_CANONICAL_VIDEO_FORMAT_IDS : GAM_CANONICAL_DISPLAY_FORMAT_IDS; + ids.forEach(formatId => { + const ref = canonicalFormatRef(formatId); + formats.push({ agent_url: ref.agent_url, id: ref.id, width: option.width, height: option.height }); + }); + }); + return formats; +} + +function prePopulateCreativeSizes() { + if (!INVENTORY_EXPLICIT_SIZES) return; + (existingProfile.formats || []).forEach(format => { + if (!format || !format.width || !format.height) return; + const kind = String(format.id || '').startsWith('video') ? 'video' : 'display'; + creativeSizeSelection.add(`${kind}:${Number(format.width)}x${Number(format.height)}`); + }); +} + +function toggleCreativeSize(key) { + if (creativeSizeSelection.has(key)) creativeSizeSelection.delete(key); + else creativeSizeSelection.add(key); + renderCreativeSizePicker(); + updateDerivedFormats(); + if (typeof recomputeValidity === 'function') recomputeValidity(); +} + +function renderCreativeSizePicker() { + const list = document.getElementById('creative-size-list'); + if (!list) return; + const strip = document.getElementById('creative-size-selected'); + const status = document.getElementById('creative-size-status'); + const query = (document.getElementById('creative-size-search')?.value || '').toLowerCase().trim(); + const selected = selectedCreativeSizeOptions(); + + if (strip) { + strip.innerHTML = selected.length + ? selected.map(option => `` + + `${option.kind === 'video' ? 'VIDEO' : 'DISPLAY'}` + + `${_escAttr(option.label)}` + + `` + + ``).join('') + : '
No creative sizes selected yet. Pick sizes below.
'; + } + + const visible = CREATIVE_SIZE_OPTIONS.filter(option => !query + || option.label.includes(query) + || option.kind.includes(query) + || (query === 'display' && option.kind === 'display')); + if (status) { + status.innerHTML = `${selected.length} selected · ${visible.length} of ${CREATIVE_SIZE_OPTIONS.length} shown`; + } + if (!CREATIVE_SIZE_OPTIONS.length) { + list.innerHTML = '
No creative sizes in the inventory cache yet — run Sync Inventory first.
'; + return; + } + const renderRow = option => { + const key = creativeSizeKey(option); + const checked = creativeSizeSelection.has(key); + return `
` + + `` + + `` + + `` + + `${option.kind === 'video' ? 'video_standard · video_vast' : 'image · HTML5 · JS'}` + + `
`; + }; + const groups = [ + ['display', 'Display sizes', 'image, HTML5 and JS creatives'], + ['video', 'Video sizes', 'VAST creatives'], + ].map(([kind, label, note]) => { + const rows = visible.filter(option => option.kind === kind); + return rows.length ? treeSection(label, note) + rows.map(renderRow).join('') : ''; + }).join(''); + list.innerHTML = groups || '
No sizes match.
'; +} + +// ---- server-paged picker state (INVENTORY_PICKER_PAGED only) ---- +// One page list per bundle slot. ``ids`` is the display order for the +// current query; row payloads are registered into INVENTORY_PICKER so the +// existing name lookups (chips, selected strip) keep working unchanged. +const inventoryPages = { + placements: freshInventoryPage(''), + ad_units: freshInventoryPage(''), +}; +let inventoryPageSeq = 0; +let inventorySearchTimer = null; + +function freshInventoryPage(query) { + return { query, ids: [], total: null, offset: 0, hasMore: false, loading: false, loaded: false, error: false }; +} + +function registerInventoryRows(key, rows) { + if (!INVENTORY_PICKER[key]) INVENTORY_PICKER[key] = []; + const known = new Set(INVENTORY_PICKER[key].map(row => row.id)); + rows.forEach(row => { + if (known.has(row.id)) return; + INVENTORY_PICKER[key].push(row); + known.add(row.id); + if (!INVENTORY_NAMES[key]) INVENTORY_NAMES[key] = {}; + INVENTORY_NAMES[key][row.id] = { name: row.name, id: row.id, subkind: row.subkind }; + }); +} + +function activeInventoryKey() { + return inventoryPickerMode === 'flat' ? 'ad_units' : 'placements'; +} + +// reset=true starts a new search for the current input (page 0); +// reset=false appends the next page to the current search. +function loadInventoryPage(key, reset) { + if (!INVENTORY_PICKER_PAGED) return; + const query = inventoryQuery(); + if (reset) inventoryPages[key] = freshInventoryPage(query); + const page = inventoryPages[key]; + page.loading = true; + const seq = ++inventoryPageSeq; + page.seq = seq; + renderInventoryPicker(); + const params = new URLSearchParams({ kind: key, q: query, offset: String(page.offset), limit: String(INVENTORY_PICKER_PAGE_SIZE) }); + fetch(`${INVENTORY_PICKER_API}?${params.toString()}`, { credentials: 'same-origin' }) + .then(response => response.json()) + .then(data => { + if (page.seq !== seq || inventoryPages[key] !== page) return; // superseded by a newer search + const items = data.success && Array.isArray(data.items) ? data.items : []; + registerInventoryRows(key, items); + const seen = new Set(page.ids); + items.forEach(item => { if (!seen.has(item.id)) { page.ids.push(item.id); seen.add(item.id); } }); + page.total = typeof data.count === 'number' ? data.count : null; + page.offset += items.length; + page.hasMore = !!data.has_more; + page.loading = false; + page.loaded = true; + page.error = !data.success; + renderInventoryPicker(); + }) + .catch(() => { + if (page.seq !== seq) return; + page.loading = false; + page.loaded = true; + page.error = true; + renderInventoryPicker(); + }); +} + +function pagedInventoryRows(key) { + return inventoryPages[key].ids.map(id => inventoryRowById(key, id)).filter(Boolean); +} + +function renderInventoryLoadMore(key) { + const page = inventoryPages[key]; + const noun = INVENTORY_VOCAB[key] || key; + if (page.error) { + return `
Lookup failed — is the API reachable?
`; + } + if (page.hasMore || page.loading) { + const shown = page.ids.length; + const label = page.loading + ? 'Loading…' + : `Load more (showing ${shown.toLocaleString()} of ${page.total === null ? 'many' : page.total.toLocaleString()} matching ${_escAttr(noun)})`; + return ``; + } + if (page.loaded && page.ids.length && page.total !== null) { + return `
Showing all ${page.total.toLocaleString()} matching ${_escAttr(noun)}
`; + } + return ''; +} + +function scheduleInventorySearch() { + if (!INVENTORY_PICKER_PAGED) { renderInventoryPicker(); return; } + clearTimeout(inventorySearchTimer); + inventorySearchTimer = setTimeout(() => loadInventoryPage(activeInventoryKey(), true), INVENTORY_PICKER_DEBOUNCE_MS); +} + function cssEscapeValue(value) { if (window.CSS && typeof window.CSS.escape === 'function') return window.CSS.escape(value); return String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"'); @@ -640,8 +879,13 @@

Properties

if (!status) return; const placementCount = inventorySelection.placements.size; const adUnitCount = inventorySelection.ad_units.size; + let shown = `${visibleCount} shown`; + if (INVENTORY_PICKER_PAGED) { + const page = inventoryPages[activeInventoryKey()]; + if (page.total !== null) shown = `${visibleCount.toLocaleString()} of ${page.total.toLocaleString()} shown`; + } const capped = inventoryPickerMayBeCapped() ? ` of first ${INVENTORY_PICKER_LIMIT} synced rows` : ''; - status.innerHTML = `${placementCount} placements, ${adUnitCount} standalone ad units · ${visibleCount} shown${capped}`; + status.innerHTML = `${placementCount} ${_escAttr(INVENTORY_VOCAB.placements)}, ${adUnitCount} standalone ${_escAttr(INVENTORY_VOCAB.ad_units)} · ${shown}${capped}`; } function treeSection(label, note) { @@ -672,7 +916,9 @@

Properties

+ `${_escAttr(row.name)}` + `#${_escAttr(row.id)}` + `` - + `${row.child_count || children.length}${(row.child_count || children.length) === 1 ? 'ad unit' : 'ad units'}` + + ((row.child_count || children.length || BUNDLE_ADAPTER_ID === 'gam') + ? `${row.child_count || children.length}${(row.child_count || children.length) === 1 ? _escAttr(INVENTORY_VOCAB_SINGULAR.ad_units) : _escAttr(INVENTORY_VOCAB.ad_units)}` + : `${_escAttr(row.meta || '—')}`) + inventoryRowSecondaryAction(row) + `
`; @@ -707,40 +953,62 @@

Properties

const help = document.getElementById('inventory-picker-help'); if (help) { help.hidden = inventoryPickerMode !== 'tree'; + const one = _escAttr(INVENTORY_VOCAB_SINGULAR.placements); + const units = _escAttr(INVENTORY_VOCAB.ad_units); const helpText = inventoryIncludesDescendants() - ? 'Picking a placement includes its child ad units automatically.' - : 'Picking a placement targets only that placement. Child ad units stay standalone.'; + ? `Picking a ${one} includes its child ${units} automatically.` + : `Picking a ${one} targets only that ${one}. Child ${units} stay standalone.`; help.innerHTML = '' - + `${helpText} Flip to Flat ad units to pick individual units only.`; + + `${helpText} Flip to Flat ${units} to pick individual units only.`; } if (inventoryPickerMode === 'flat') { - const rows = inventoryRows('ad_units').filter(row => inventoryRowMatches(row, query)); + const rows = INVENTORY_PICKER_PAGED + ? pagedInventoryRows('ad_units') + : inventoryRows('ad_units').filter(row => inventoryRowMatches(row, query)); renderInventoryPickerStatus(rows.length); - list.innerHTML = rows.length + const pending = INVENTORY_PICKER_PAGED && !inventoryPages.ad_units.loaded; + const errored = INVENTORY_PICKER_PAGED && inventoryPages.ad_units.error; + list.innerHTML = (rows.length ? rows.map(row => renderAdUnitRow(row)).join('') - : '
No ad units match.
'; + : (errored ? '' : `
${pending ? 'Loading…' : `No ${_escAttr(INVENTORY_VOCAB.ad_units)} match.`}
`)) + + (INVENTORY_PICKER_PAGED ? renderInventoryLoadMore('ad_units') : ''); hydratePartialCheckboxes(); restoreInventoryFocus(focusSelector); return; } - const placements = inventoryRows('placements'); - const sitePlacements = placements.filter(row => row.subkind !== 'tag'); - const tagPlacements = placements.filter(row => row.subkind === 'tag'); + const placements = INVENTORY_PICKER_PAGED ? pagedInventoryRows('placements') : inventoryRows('placements'); const renderGroup = (rows, label, note) => { - const visible = rows.filter(row => inventoryRowMatches(row, query) || - childRowsForPlacement(row).some(child => inventoryRowMatches(child, query))); + const visible = INVENTORY_PICKER_PAGED + ? rows + : rows.filter(row => inventoryRowMatches(row, query) || + childRowsForPlacement(row).some(child => inventoryRowMatches(child, query))); if (!visible.length) return { html: '', count: 0 }; return { html: treeSection(label, note) + visible.map(row => renderPlacementRow(row, query)).join(''), count: visible.length, }; }; - const site = renderGroup(sitePlacements, 'Site placements', 'wrap a set of ad units'); - const tag = renderGroup(tagPlacements, 'Tag-level placements', 'cross-section groups'); - renderInventoryPickerStatus(site.count + tag.count); - list.innerHTML = (site.html + tag.html) || '
No placements match.
'; + let groups; + if (BUNDLE_ADAPTER_ID === 'gam') { + const sitePlacements = placements.filter(row => row.subkind !== 'tag'); + const tagPlacements = placements.filter(row => row.subkind === 'tag'); + groups = [ + renderGroup(sitePlacements, 'Site placements', 'wrap a set of ad units'), + renderGroup(tagPlacements, 'Tag-level placements', 'cross-section groups'), + ]; + } else { + const label = INVENTORY_VOCAB.placements.charAt(0).toUpperCase() + INVENTORY_VOCAB.placements.slice(1); + groups = [renderGroup(placements, label, `group ${INVENTORY_VOCAB.ad_units}`)]; + } + const count = groups.reduce((sum, group) => sum + group.count, 0); + renderInventoryPickerStatus(count); + const pending = INVENTORY_PICKER_PAGED && !inventoryPages.placements.loaded; + const errored = INVENTORY_PICKER_PAGED && inventoryPages.placements.error; + list.innerHTML = (groups.map(group => group.html).join('') + || (errored ? '' : `
${pending ? 'Loading…' : `No ${_escAttr(INVENTORY_VOCAB.placements)} match.`}
`)) + + (INVENTORY_PICKER_PAGED ? renderInventoryLoadMore('placements') : ''); hydratePartialCheckboxes(); restoreInventoryFocus(focusSelector); } @@ -1110,7 +1378,24 @@

Properties

document.addEventListener('DOMContentLoaded', function() { seedInventoryNameMaps(); + prePopulateCreativeSizes(); prePopulateInventoryConfig(); + if (INVENTORY_EXPLICIT_SIZES) { + renderCreativeSizePicker(); + updateDerivedFormats(); + const sizeList = document.getElementById('creative-size-list'); + const sizeStrip = document.getElementById('creative-size-selected'); + const sizeSearch = document.getElementById('creative-size-search'); + if (sizeList) sizeList.addEventListener('click', (e) => { + const toggle = e.target.closest('[data-toggle-size]'); + if (toggle) toggleCreativeSize(toggle.dataset.toggleSize); + }); + if (sizeStrip) sizeStrip.addEventListener('click', (e) => { + const remove = e.target.closest('[data-remove-size]'); + if (remove) toggleCreativeSize(remove.dataset.removeSize); + }); + if (sizeSearch) sizeSearch.addEventListener('input', renderCreativeSizePicker); + } const pickerList = document.getElementById('inventory-picker-list'); const pickerSearch = document.getElementById('inventory-picker-search'); @@ -1122,6 +1407,11 @@

Properties

pickerList.addEventListener('click', (e) => { const reuse = e.target.closest('a.tree__use'); if (reuse) return; + const more = e.target.closest('[data-load-more]'); + if (more) { + if (!more.disabled) loadInventoryPage(more.dataset.loadMore, false); + return; + } const open = e.target.closest('[data-toggle-open]'); if (open && !e.target.closest('[data-toggle-placement]')) { const id = open.dataset.toggleOpen; @@ -1139,7 +1429,7 @@

Properties

if (adUnit && !adUnit.disabled) toggleAdUnit(adUnit.dataset.toggleAdUnit); }); } - if (pickerSearch) pickerSearch.addEventListener('input', renderInventoryPicker); + if (pickerSearch) pickerSearch.addEventListener('input', scheduleInventorySearch); if (modeSwitch) { modeSwitch.addEventListener('click', (e) => { const button = e.target.closest('[data-picker-mode]'); @@ -1149,9 +1439,17 @@

Properties

btn.classList.toggle('active', btn === button); btn.setAttribute('aria-pressed', btn === button ? 'true' : 'false'); }); + // Paged mode: (re)load the slot when its query is stale or it has + // never been fetched; otherwise the cached page renders as is. + if (INVENTORY_PICKER_PAGED) { + const key = activeInventoryKey(); + const page = inventoryPages[key]; + if (!page.loaded || page.query !== inventoryQuery()) { loadInventoryPage(key, true); return; } + } renderInventoryPicker(); }); } + if (INVENTORY_PICKER_PAGED) loadInventoryPage(activeInventoryKey(), true); if (selectedStrip) { selectedStrip.addEventListener('click', (e) => { const button = e.target.closest('[data-remove-kind][data-remove-id]'); @@ -1413,6 +1711,7 @@

Properties

function selectedInventorySizes() { const sizes = new Set(); + if (INVENTORY_EXPLICIT_SIZES) selectedCreativeSizeOptions().forEach(option => sizes.add(option.label)); inventorySelection.ad_units.forEach(id => addRowSizes(inventoryRowById('ad_units', id), sizes)); inventorySelection.placements.forEach(id => { const placement = inventoryRowById('placements', id); @@ -1438,7 +1737,7 @@

Properties

} function deriveCanonicalFormatsFromInventory() { - const formats = []; + const formats = INVENTORY_EXPLICIT_SIZES ? explicitSizeFormats() : []; inventorySelection.ad_units.forEach(id => addRowCanonicalFormats(inventoryRowById('ad_units', id), formats)); inventorySelection.placements.forEach(id => { const placement = inventoryRowById('placements', id); @@ -1501,23 +1800,30 @@

Properties

const sizes = selectedInventorySizes(); const specialRows = selectedInventorySpecialSizes(); - if ((inventorySelection.ad_units.size + inventorySelection.placements.size) === 0) { - status.textContent = 'Pick placements or ad units to detect creative formats.'; + if (INVENTORY_EXPLICIT_SIZES) { + if (sizes.length === 0) { + status.textContent = 'Pick at least one creative size above to set the formats this bundle sells.'; + list.innerHTML = ''; + return; + } + } else if ((inventorySelection.ad_units.size + inventorySelection.placements.size) === 0) { + status.textContent = `Pick ${INVENTORY_VOCAB.placements} or ${INVENTORY_VOCAB.ad_units} to detect creative formats.`; list.innerHTML = ''; return; - } - if (sizes.length === 0 && specialRows.length === 0) { + } else if (sizes.length === 0 && specialRows.length === 0) { status.textContent = 'No creative sizes were found on the selected inventory. Choose synced inventory with sizes.'; list.innerHTML = ''; return; } const fixedText = sizes.length - ? `${sizes.length} fixed ${sizes.length === 1 ? 'size' : 'sizes'} detected.` + ? `${sizes.length} fixed ${sizes.length === 1 ? 'size' : 'sizes'} ${INVENTORY_EXPLICIT_SIZES ? 'selected' : 'detected'}.` : 'No fixed creative sizes detected.'; - const specialText = specialRows.length - ? ` ${specialRows.length} GAM 1x1 ${specialRows.length === 1 ? 'slot needs' : 'slots need'} capability setup before saving.` - : ' Formats use display creative types supported by GAM.'; + const specialText = INVENTORY_EXPLICIT_SIZES + ? ' Formats follow the sizes picked above.' + : (specialRows.length + ? ` ${specialRows.length} GAM 1x1 ${specialRows.length === 1 ? 'slot needs' : 'slots need'} capability setup before saving.` + : ' Formats use display creative types supported by GAM.'); status.textContent = fixedText + specialText; const formatChips = derivedFormatChips().map(group => ` @@ -1644,8 +1950,8 @@

Properties

const placements = _placementIds(); const invSize = adUnits.length + placements.length; const invParts = []; - if (placements.length) invParts.push(placements.length + (placements.length === 1 ? ' placement' : ' placements')); - if (adUnits.length) invParts.push(adUnits.length + (adUnits.length === 1 ? ' ad unit' : ' ad units')); + if (placements.length) invParts.push(placements.length + ' ' + (placements.length === 1 ? INVENTORY_VOCAB_SINGULAR.placements : INVENTORY_VOCAB.placements)); + if (adUnits.length) invParts.push(adUnits.length + ' ' + (adUnits.length === 1 ? INVENTORY_VOCAB_SINGULAR.ad_units : INVENTORY_VOCAB.ad_units)); const formatCount = derivedFormats.length; const formatSizeCount = selectedInventorySizes().length; @@ -1779,16 +2085,18 @@

Properties

inventory: item({ ok: invSize > 0, text: invParts.length ? invParts.join(' · ') : 'none yet', - message: 'Add at least one placement or ad unit to save.', + message: `Add at least one ${INVENTORY_VOCAB_SINGULAR.placements} or ${INVENTORY_VOCAB_SINGULAR.ad_units} to save.`, focusId: 'inventory-picker-search', }), formats: item({ ok: formatCount > 0 && specialSizeCount === 0, text: formatSizeCount > 0 ? `${formatSizeCount} ${formatSizeCount === 1 ? 'size' : 'sizes'}` : 'none', - message: specialSizeCount > 0 - ? 'Classify GAM 1x1 special inventory before saving.' - : 'Pick inventory with synced creative sizes to save.', - focusId: 'inventory-picker-search', + message: INVENTORY_EXPLICIT_SIZES + ? 'Pick at least one creative size to save.' + : (specialSizeCount > 0 + ? 'Classify GAM 1x1 special inventory before saving.' + : 'Pick inventory with synced creative sizes to save.'), + focusId: INVENTORY_EXPLICIT_SIZES ? 'creative-size-search' : 'inventory-picker-search', }), properties: item({ ok: propertyOk, diff --git a/templates/inventory_browser_improvedigital.html b/templates/inventory_browser_improvedigital.html index 7fa24813ff..ae43ea434b 100644 --- a/templates/inventory_browser_improvedigital.html +++ b/templates/inventory_browser_improvedigital.html @@ -260,26 +260,66 @@

Browse Inventory

}); // ---- sync ---- + // sync-inventory enqueues a background sweep and answers 202 with a + // sync_id (no counts yet) — poll sync-status/ until it reaches a + // terminal state, then show the per-family counts and reload the panes. + const SYNC_POLL_MS = 3000; + + function describeSyncJob(data) { + const counts = data.counts || {}; + const total = Object.values(counts).reduce((sum, v) => sum + Number(v || 0), 0); + const summary = Object.entries(counts).map(([k, v]) => k + '=' + Number(v).toLocaleString()).join(', '); + const errors = Object.entries(data.errors || {}).map(([k, v]) => k + ': ' + v).join('; '); + if (data.status === 'failed') { + return '✗ ' + (data.error_message || errors || 'sync failed'); + } + return '✓ Synced ' + total.toLocaleString() + ' items' + (summary ? ' (' + summary + ')' : '') + + (errors ? ' — partial failures: ' + errors : ''); + } + + function pollSync(syncId, status, done) { + fetch(apiBase + '/sync-status/' + encodeURIComponent(syncId), {credentials: 'same-origin'}) + .then(r => r.json()) + .then(data => { + if (data.status === 'completed' || data.status === 'failed') { + status.textContent = describeSyncJob(data); + publishers = []; + loadAll(); + done(); + return; + } + if (data.success === false) { + status.textContent = '✗ ' + (data.error || 'could not read sync status'); + done(); + return; + } + status.textContent = 'Syncing in the background (' + (data.status || 'running') + ')… rows appear as pages land.'; + setTimeout(() => pollSync(syncId, status, done), SYNC_POLL_MS); + }) + .catch(err => { status.textContent = '✗ ' + err.message; done(); }); + } + window.impdSyncNow = function() { const btn = document.getElementById('impd-sync-btn'); const status = document.getElementById('impd-sync-status'); btn.disabled = true; status.textContent = 'Syncing… (can take a minute — rate-limited at 100 requests/min)'; + const done = () => { btn.disabled = false; }; fetch(apiBase + '/sync-inventory', {method: 'POST', headers: {'Content-Type': 'application/json'}, credentials: 'same-origin'}) .then(r => r.json().then(data => ({status: r.status, data}))) - .then(({status, data}) => { - if (data.success) { - status.textContent = '✓ Synced ' + (data.total_synced || 0).toLocaleString() + ' items'; - publishers = []; - loadAll(); - } else if (status === 409) { + .then(({status: httpStatus, data}) => { + if (data.success && data.sync_id) { + pollSync(data.sync_id, status, done); + return; + } + if (httpStatus === 409) { status.textContent = data.error || 'A sync is already running'; } else { status.textContent = '✗ ' + (data.error || JSON.stringify(data.errors || 'sync failed')); } + done(); }) - .catch(err => { status.textContent = '✗ ' + err.message; }) - .finally(() => { btn.disabled = false; }); + .catch(err => { status.textContent = '✗ ' + err.message; done(); }); }; // ---- boot ---- diff --git a/templates/sync_inventory.html b/templates/sync_inventory.html index 97bb3a223f..a2fad107fb 100644 --- a/templates/sync_inventory.html +++ b/templates/sync_inventory.html @@ -122,8 +122,55 @@
When to use which sync
const isGam = {{ 'true' if is_gam else 'false' }}; const adapterType = '{{ adapter_type }}'; -// Non-GAM adapters with supports_inventory_sync: one-shot sync through the -// shared orchestration. Synchronous call — completes (or 409s) in-request. +// Non-GAM adapters with supports_inventory_sync: sync through the shared +// orchestration. Some adapters finish in-request and return counts; others +// (Improve Digital) enqueue a background job and answer 202 with a sync_id, +// in which case we poll sync-status/ until it reaches a terminal +// state — the 202 body carries no counts, so reading them there shows "0". +const ADAPTER_SYNC_POLL_MS = 3000; + +function adapterSyncSummary(counts) { + return Object.entries(counts || {}).map(([k, v]) => `${k}=${Number(v).toLocaleString()}`).join(', '); +} + +function adapterSyncErrors(errors) { + return errors ? Object.entries(errors).map(([k, v]) => `${k}: ${v}`).join('; ') : ''; +} + +function showAdapterSyncResult(statusEl, data) { + const counts = data.counts || {}; + const total = data.total_synced !== undefined + ? data.total_synced + : Object.values(counts).reduce((sum, v) => sum + Number(v || 0), 0); + const errors = adapterSyncErrors(data.errors); + if (data.status === 'failed' || (data.success === false && !Object.keys(counts).length)) { + statusEl.textContent = `✗ ${data.error_message || data.error || errors || 'Sync failed'}`; + return; + } + statusEl.textContent = `✓ Synced ${Number(total).toLocaleString()} items (${adapterSyncSummary(counts)})` + + (errors ? ` — partial failures: ${errors}` : ''); +} + +function pollAdapterSync(syncId, statusEl, done) { + fetch(`${scriptRoot}/api/tenant/${tenantId}/adapters/${adapterType}/sync-status/${syncId}`, { credentials: 'same-origin' }) + .then(r => r.json()) + .then(data => { + if (data.status === 'completed' || data.status === 'failed') { + showAdapterSyncResult(statusEl, data); + done(); + return; + } + if (data.success === false) { + statusEl.textContent = `✗ ${data.error || 'Could not read sync status'}`; + done(); + return; + } + statusEl.textContent = `Syncing in the background (${data.status || 'running'})… this page updates when it finishes.`; + setTimeout(() => pollAdapterSync(syncId, statusEl, done), ADAPTER_SYNC_POLL_MS); + }) + .catch(err => { statusEl.textContent = '✗ ' + err.message; done(); }); +} + function syncAdapterInventory() { const btn = document.getElementById('adapterSyncBtn'); const statusEl = document.getElementById('adapterSyncStatus'); @@ -132,6 +179,7 @@
When to use which sync
btn.disabled = true; btn.innerHTML = ' Syncing…'; statusEl.textContent = 'Syncing — large catalogs can take a minute (upstream rate limits apply).'; + const done = () => { btn.disabled = false; btn.innerHTML = original; }; fetch(`${scriptRoot}/api/tenant/${tenantId}/adapters/${adapterType}/sync-inventory`, { method: 'POST', @@ -140,18 +188,20 @@
When to use which sync
}) .then(r => r.json().then(data => ({ status: r.status, data }))) .then(({ status, data }) => { + if (data.success && (status === 202 || data.status === 'queued') && data.sync_id) { + pollAdapterSync(data.sync_id, statusEl, done); + return; + } if (data.success) { - const summary = Object.entries(data.counts || {}).map(([k, v]) => `${k}=${v}`).join(', '); - statusEl.textContent = `✓ Synced ${(data.total_synced || 0).toLocaleString()} items (${summary})`; + showAdapterSyncResult(statusEl, data); } else if (status === 409) { statusEl.textContent = data.error || 'A sync is already running — wait for it to finish.'; } else { - const errors = data.errors ? Object.entries(data.errors).map(([k, v]) => `${k}: ${v}`).join('; ') : ''; - statusEl.textContent = `✗ ${data.error || errors || 'Sync failed'}`; + statusEl.textContent = `✗ ${data.error || adapterSyncErrors(data.errors) || 'Sync failed'}`; } + done(); }) - .catch(err => { statusEl.textContent = '✗ ' + err.message; }) - .finally(() => { btn.disabled = false; btn.innerHTML = original; }); + .catch(err => { statusEl.textContent = '✗ ' + err.message; done(); }); } function syncInventory(mode) { diff --git a/tests/admin/test_inventory_profiles_improvedigital.py b/tests/admin/test_inventory_profiles_improvedigital.py new file mode 100644 index 0000000000..e629419025 --- /dev/null +++ b/tests/admin/test_inventory_profiles_improvedigital.py @@ -0,0 +1,207 @@ +"""Inventory bundle authoring for an Improve Digital tenant. + +Drives the bundle editor through the Flask test client against real +PostgreSQL: the picker API pages placements/packages out of the +``improvedigital_inventory`` cache, the editor embeds the synced creative +size catalogue (360Yield placements carry no sizes), and a bundle saves +with the formats derived from those sizes. +""" + +import json +from datetime import UTC, datetime + +import pytest +from sqlalchemy import delete, select + +from src.admin.app import create_app +from src.core.database.database_session import get_db_session +from src.core.database.models import ImproveDigitalInventory, InventoryProfile, Tenant +from src.core.database.repositories.improvedigital_inventory import ImproveDigitalInventoryRepository +from tests.utils.database_helpers import create_tenant_with_timestamps + +app = create_app() + +pytestmark = [pytest.mark.admin, pytest.mark.requires_db] + +_TENANT_ID = "inv_prof_impd_tenant" + + +@pytest.fixture +def client(): + app.config["TESTING"] = True + app.config["WTF_CSRF_ENABLED"] = False + app.config["SESSION_COOKIE_PATH"] = "/" + with app.test_client() as client: + yield client + + +@pytest.fixture +def impd_tenant(integration_db): + """Improve Digital tenant with a small synced inventory cache.""" + synced_at = datetime.now(UTC) + with get_db_session() as session: + session.execute(delete(InventoryProfile).where(InventoryProfile.tenant_id == _TENANT_ID)) + session.execute(delete(ImproveDigitalInventory).where(ImproveDigitalInventory.tenant_id == _TENANT_ID)) + session.execute(delete(Tenant).where(Tenant.tenant_id == _TENANT_ID)) + session.commit() + + tenant = create_tenant_with_timestamps( + tenant_id=_TENANT_ID, + name="Improve Digital Bundle Tenant", + subdomain="inv-prof-impd", + ad_server="improvedigital", + is_active=True, + ) + session.add(tenant) + session.commit() + + ImproveDigitalInventoryRepository(session, _TENANT_ID).bulk_upsert( + [ + { + "entity_type": "publisher", + "entity_id": "7", + "name": "Jeep Community", + "parent_id": None, + "raw_json": {"id": 7, "name": "Jeep Community"}, + "last_synced_at": synced_at, + }, + { + "entity_type": "placement", + "entity_id": "23390686", + "name": "jeepcommunity.de-desktop-300x250", + "parent_id": "7", + "raw_json": {"placement_id": 23390686, "publisher_id": 7}, + "last_synced_at": synced_at, + }, + { + "entity_type": "placement", + "entity_id": "23366919", + "name": "ford-forum.de-mobile-300x250", + "parent_id": "7", + "raw_json": {"placement_id": 23366919, "publisher_id": 7}, + "last_synced_at": synced_at, + }, + { + "entity_type": "package", + "entity_id": "2832", + "name": "Automotive Premium", + "parent_id": None, + "raw_json": {"id": 2832, "name": "Automotive Premium", "sizes": []}, + "last_synced_at": synced_at, + }, + { + "entity_type": "size", + "entity_id": "171", + "name": "300x250", + "parent_id": None, + "raw_json": {"id": 171, "name": "300x250", "type": "display", "width": 300, "height": 250}, + "last_synced_at": synced_at, + }, + { + "entity_type": "size", + "entity_id": "893", + "name": "308x173 Video", + "parent_id": None, + "raw_json": {"id": 893, "name": "308x173 Video", "type": "vast", "width": 308, "height": 173}, + "last_synced_at": synced_at, + }, + ] + ) + session.commit() + return _TENANT_ID + + +def _auth_session(client, tenant_id): + with client.session_transaction() as sess: + sess["authenticated"] = True + sess["user"] = {"email": "test@example.com", "is_super_admin": True} + sess["email"] = "test@example.com" + sess["tenant_id"] = tenant_id + sess["test_user"] = "test@example.com" + sess["test_user_role"] = "super_admin" + sess["test_user_name"] = "Test User" + sess["test_tenant_id"] = tenant_id + + +class TestImproveDigitalBundleEditor: + def test_create_form_uses_improvedigital_vocabulary_and_paged_picker(self, client, impd_tenant): + _auth_session(client, impd_tenant) + response = client.get(f"/tenant/{impd_tenant}/inventory-profiles/add") + assert response.status_code == 200 + html = response.get_data(as_text=True) + assert "Flat placements" in html + assert "INVENTORY_PICKER_PAGED = true" in html + assert "INVENTORY_EXPLICIT_SIZES = true" in html + # The synced size catalogue is embedded for the explicit size picker. + assert '"label": "300x250"' in html + assert '"kind": "video"' in html + assert 'id="creative-size-picker"' in html + + def test_picker_api_pages_packages_and_placements(self, client, impd_tenant): + _auth_session(client, impd_tenant) + + response = client.get(f"/tenant/{impd_tenant}/inventory-profiles/api/inventory?kind=placements") + assert response.status_code == 200 + body = response.get_json() + assert body["success"] is True + assert body["count"] == 1 + assert body["has_more"] is False + assert [item["id"] for item in body["items"]] == ["2832"] + assert body["items"][0]["kind"] == "placement" + + response = client.get(f"/tenant/{impd_tenant}/inventory-profiles/api/inventory?kind=ad_units&q=jeep&limit=1") + assert response.status_code == 200 + body = response.get_json() + assert body["count"] == 1 + assert body["items"][0]["id"] == "23390686" + assert body["items"][0]["kind"] == "ad_unit" + assert body["items"][0]["meta"] == "Jeep Community" + assert body["items"][0]["bundle_count"] == 0 + + response = client.get(f"/tenant/{impd_tenant}/inventory-profiles/api/inventory?kind=ad_units&limit=1") + body = response.get_json() + assert body["count"] == 2 + assert body["has_more"] is True + + def test_create_bundle_saves_placements_and_size_derived_formats(self, client, impd_tenant): + _auth_session(client, impd_tenant) + formats = [ + {"agent_url": "https://creative.adcontextprotocol.org", "id": "display_image", "width": 300, "height": 250}, + {"agent_url": "https://creative.adcontextprotocol.org", "id": "display_html", "width": 300, "height": 250}, + ] + response = client.post( + f"/tenant/{impd_tenant}/inventory-profiles/add", + data={ + "name": "Automotive 300x250", + "profile_id": "automotive_300x250", + "description": "Created via test", + "targeted_ad_unit_ids": json.dumps(["23390686", "23366919"]), + "targeted_placement_ids": json.dumps(["2832"]), + "formats": json.dumps(formats), + "property_mode": "tags", + "property_tags": "all_inventory", + }, + follow_redirects=False, + ) + assert response.status_code in (302, 303), response.get_data(as_text=True)[:500] + + with get_db_session() as session: + profile = session.scalars( + select(InventoryProfile).where( + InventoryProfile.tenant_id == impd_tenant, + InventoryProfile.profile_id == "automotive_300x250", + ) + ).first() + assert profile is not None + assert profile.inventory_config["ad_units"] == ["23390686", "23366919"] + assert profile.inventory_config["placements"] == ["2832"] + assert profile.format_ids == formats + profile_pk = profile.id + + # The edit page resolves the saved ids to names and re-embeds the sizes. + response = client.get(f"/tenant/{impd_tenant}/inventory-profiles/{profile_pk}/edit") + assert response.status_code == 200 + html = response.get_data(as_text=True) + assert "jeepcommunity.de-desktop-300x250" in html + assert "Automotive Premium" in html + assert "INVENTORY_EXPLICIT_SIZES = true" in html diff --git a/tests/integration/test_adapter_config_lock.py b/tests/integration/test_adapter_config_lock.py index 6265297cbc..4b36c68dd5 100644 --- a/tests/integration/test_adapter_config_lock.py +++ b/tests/integration/test_adapter_config_lock.py @@ -108,6 +108,48 @@ def test_non_inventory_sync_completion_does_not_lock(self, factory_session): assert is_adapter_config_locked(factory_session, tenant.tenant_id) is False + def test_gam_inventory_write_stamps_lock_before_completion(self, factory_session): + """The lock commits with the first inventory rows themselves — a sync + that persists inventory and then dies before completion still locks.""" + from types import SimpleNamespace + + from src.services.gam_inventory_service import GAMInventoryService + + tenant, _ = _gam_tenant("lock_write_gam", locked=False) + item = SimpleNamespace( + id="au_1", + name="Ad Unit 1", + path=["Ad Unit 1"], + status=SimpleNamespace(value="ACTIVE"), + ad_unit_code="code_au_1", + parent_id=None, + description=None, + target_window=None, + explicitly_targeted=False, + has_children=False, + sizes=[], + effective_applied_labels=[], + ) + + GAMInventoryService(factory_session)._write_inventory_batch( + tenant.tenant_id, "ad_unit", [item], datetime.now(UTC) + ) + + # No sync job was ever completed — the inventory write alone locks. + assert is_adapter_config_locked(factory_session, tenant.tenant_id) is True + + def test_improvedigital_inventory_write_stamps_lock_before_completion(self, factory_session): + """Improve Digital commits inventory per page — the first page locks.""" + from src.adapters.improvedigital.inventory_sync import ImproveDigitalInventorySync + + tenant = TenantFactory(tenant_id="lock_write_impd", ad_server="improvedigital") + AdapterConfigFactory(tenant=tenant, adapter_type="improvedigital") + + sync = ImproveDigitalInventorySync(client=None, session=factory_session, tenant_id=tenant.tenant_id) + sync._persist_page([{"entity_type": "placement", "entity_id": "pl_1", "raw_json": {"id": "pl_1"}}]) + + assert is_adapter_config_locked(factory_session, tenant.tenant_id) is True + class TestModelGuard: """The before_update listeners block config changes on locked tenants.""" diff --git a/tests/unit/test_bundle_adapter.py b/tests/unit/test_bundle_adapter.py index c8ea515f4a..dbf1bac1a0 100644 --- a/tests/unit/test_bundle_adapter.py +++ b/tests/unit/test_bundle_adapter.py @@ -7,6 +7,8 @@ from __future__ import annotations +from unittest.mock import MagicMock, patch + import pytest from src.services.bundle_adapter import ( @@ -21,9 +23,9 @@ class TestRegistry: """Registry resolution + tenant matching.""" - def test_three_adapters_registered_at_import(self): + def test_four_adapters_registered_at_import(self): ids = {a.adapter_id for a in iter_adapters()} - assert ids == {"gam", "freewheel", "springserve"} + assert ids == {"gam", "improvedigital", "freewheel", "springserve"} def test_get_adapter_returns_registered_instance(self): gam = get_adapter("gam") @@ -43,6 +45,8 @@ def test_get_adapter_unknown_returns_none(self): ("fw", "freewheel"), ("springserve", "springserve"), ("ss", "springserve"), + ("improvedigital", "improvedigital"), + ("improve_digital", "improvedigital"), ], ) def test_adapter_for_tenant_matches_ad_server_aliases(self, ad_server, expected): @@ -109,7 +113,120 @@ def test_stub_methods_return_empty_safely(self): class TestProtocolConformance: """Every registered adapter must satisfy the runtime-checked Protocol.""" - @pytest.mark.parametrize("adapter_id", ["gam", "freewheel", "springserve"]) + @pytest.mark.parametrize("adapter_id", ["gam", "improvedigital", "freewheel", "springserve"]) def test_adapter_isinstance_protocol(self, adapter_id): adapter = get_adapter(adapter_id) assert isinstance(adapter, BundleInventoryAdapter) + + def test_only_improvedigital_pages_the_picker(self): + paged = {a.adapter_id for a in iter_adapters() if a.picker_paged} + assert paged == {"improvedigital"} + + +REPO_PATH = "src.core.database.repositories.improvedigital_inventory.ImproveDigitalInventoryRepository" + + +class TestImproveDigitalAdapter: + """Improve Digital maps the bundle slots onto the 360Yield cache: + bundle ``ad_unit`` → cached ``placement``, bundle ``placement`` → + cached ``package``. Reads go through the picker projections so the + editor never materializes ORM rows for the full placement set.""" + + def _repo(self): + repo = MagicMock() + repo.list_picker_rows_by_ids.side_effect = lambda entity_type, ids: ( + [("7", "Pub Seven", None)] + if entity_type == "publisher" + else [(str(i), f"{entity_type}-{i}", "7") for i in ids] + ) + return repo + + def test_label_and_vocab(self): + adapter = get_adapter("improvedigital") + assert adapter.label == "Improve Digital" + assert adapter.vocab == {"primary": "placements", "secondary": "packages"} + + def test_search_maps_bundle_slots_to_cache_entity_types(self): + adapter = get_adapter("improvedigital") + repo = self._repo() + repo.count_picker_rows.return_value = 3 + repo.list_picker_rows.return_value = [("11", "Home top", "7"), ("12", "Home side", "7")] + with patch(REPO_PATH, return_value=repo): + rows, total = adapter.search_inventory(None, "t1", "ad_unit", q="home", offset=0, limit=2) + repo.count_picker_rows.assert_called_once_with("placement", q="home") + repo.list_picker_rows.assert_called_once_with("placement", q="home", offset=0, limit=2) + assert total == 3 + assert [r.external_id for r in rows] == ["11", "12"] + assert all(r.entity_type == "ad_unit" for r in rows) + # Placements are labelled with their publisher (one extra lookup, not per row). + assert rows[0].meta == "Pub Seven" + assert rows[0].raw["metadata"]["parent_id"] == "7" + repo.list_picker_rows_by_ids.assert_called_once_with("publisher", ["7"]) + + def test_bundle_placement_slot_reads_packages(self): + adapter = get_adapter("improvedigital") + repo = self._repo() + repo.list_picker_rows.return_value = [("900", "Premium pack", None)] + with patch(REPO_PATH, return_value=repo): + rows = adapter.list_inventory(None, "t1", "placement", limit=5) + repo.list_picker_rows.assert_called_once_with("package", limit=5) + assert rows[0].entity_type == "placement" + assert rows[0].name == "Premium pack" + + def test_coverage_counts_only_directly_picked_placements(self): + """Package membership is a live lookup, not cached — packages must + not inflate coverage.""" + adapter = get_adapter("improvedigital") + repo = self._repo() + with patch(REPO_PATH, return_value=repo): + covered = adapter.coverage_for_bundle(None, "t1", {"ad_units": ["1", "2"], "placements": ["900"]}) + assert covered == 2 + repo.list_picker_rows_by_ids.assert_called_once_with("placement", ["1", "2"]) + + def test_unbundled_excludes_bundled_ids_and_lists_packages_first(self): + adapter = get_adapter("improvedigital") + repo = self._repo() + repo.list_picker_rows.side_effect = lambda entity_type, **kw: ( + [("900", "Pack", None)] if entity_type == "package" else [("11", "Home top", "7")] + ) + with patch(REPO_PATH, return_value=repo): + rows = adapter.list_unbundled(None, "t1", {"ad_unit": {"12"}, "placement": {"901"}}, limit=10) + assert [(r.entity_type, r.external_id) for r in rows] == [("placement", "900"), ("ad_unit", "11")] + calls = {c.args[0]: c.kwargs for c in repo.list_picker_rows.call_args_list} + assert set(calls["package"]["exclude_ids"]) == {"901"} + assert set(calls["placement"]["exclude_ids"]) == {"12"} + + def test_unknown_entity_type_is_rejected(self): + adapter = get_adapter("improvedigital") + with patch(REPO_PATH, return_value=self._repo()), pytest.raises(ValueError): + adapter.count_inventory(None, "t1", "size") + + def test_only_improvedigital_uses_explicit_creative_sizes(self): + explicit = {a.adapter_id for a in iter_adapters() if a.explicit_creative_sizes} + assert explicit == {"improvedigital"} + assert get_adapter("gam").list_creative_sizes(None, "t1") == [] + + def test_creative_sizes_map_360yield_types_and_skip_placeholders(self): + """Placements carry no sizes, so the editor offers the synced size + catalogue: display-ish types collapse to one display option per + WxH, vast becomes video, audio and 1x1/2x1 text placeholders drop.""" + adapter = get_adapter("improvedigital") + repo = self._repo() + size_rows = [ + {"id": 1, "name": "300x250", "type": "display", "width": 300, "height": 250}, + {"id": 2, "name": "300x250 app", "type": "mobile_app", "width": 300, "height": 250}, + {"id": 3, "name": "320x50", "type": "display", "width": 320, "height": 50}, + {"id": 4, "name": "308x173 Video", "type": "vast", "width": 308, "height": 173}, + {"id": 5, "name": "1x1 (Text Ad)", "type": "text", "width": 1, "height": 1}, + {"id": 6, "name": "audio", "type": "vast_audio", "width": 1, "height": 1}, + {"id": 7, "name": "broken", "type": "display", "width": None, "height": 90}, + ] + repo.list_by_type.return_value = [MagicMock(raw_json=raw) for raw in size_rows] + with patch(REPO_PATH, return_value=repo): + options = adapter.list_creative_sizes(None, "t1") + repo.list_by_type.assert_called_once_with("size") + assert options == [ + {"label": "300x250", "width": 300, "height": 250, "kind": "display"}, + {"label": "320x50", "width": 320, "height": 50, "kind": "display"}, + {"label": "308x173", "width": 308, "height": 173, "kind": "video"}, + ] From 0a41202e855480993439e88e47cb09d930abea36 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 15 Sep 2026 13:24:22 +0600 Subject: [PATCH 88/90] fix: adapter default currency for wholesale bundle pricing --- src/admin/tenant_management_api.py | 8 +-- src/core/inventory_profile_projection.py | 29 +++++--- src/core/tools/products.py | 10 +-- .../test_wholesale_currency_preference.py | 69 +++++++++++++++++++ 4 files changed, 94 insertions(+), 22 deletions(-) create mode 100644 tests/unit/test_wholesale_currency_preference.py diff --git a/src/admin/tenant_management_api.py b/src/admin/tenant_management_api.py index 52727ff7fb..30d15e8cca 100644 --- a/src/admin/tenant_management_api.py +++ b/src/admin/tenant_management_api.py @@ -207,6 +207,7 @@ inventory_profile_to_product_model, is_complete_inventory_profile, is_wholesale_owned_inventory_profile, + preferred_wholesale_currency, ) from src.core.security.url_validator import check_url_ssrf from src.services.aao_lookup_service import get_publisher_partner_status @@ -1720,14 +1721,9 @@ def _default_wholesale_currency_for_authoring( tenant_id: str, adapter: AdapterConfig | None, ) -> str: - preferred_currency = ( - adapter.gam_network_currency - if adapter is not None and adapter.adapter_type == "google_ad_manager" and adapter.gam_network_currency - else None - ) return default_wholesale_currency( CurrencyLimitRepository(session, tenant_id).list_all(), - preferred=preferred_currency, + preferred=preferred_wholesale_currency(adapter), ) diff --git a/src/core/inventory_profile_projection.py b/src/core/inventory_profile_projection.py index 2200cef965..f4db17b3c4 100644 --- a/src/core/inventory_profile_projection.py +++ b/src/core/inventory_profile_projection.py @@ -83,6 +83,26 @@ def default_wholesale_currency( return currency_codes[0] if currency_codes else fallback_code +def preferred_wholesale_currency(adapter_config: Any) -> str | None: + """Return the ad server's configured currency for bundle-backed pricing. + + GAM reports its network currency on connection test; every other adapter + (Improve Digital, etc.) stores an operator-chosen default currency in + ``config_json["currency"]``. Returns ``None`` when neither is set so the + caller falls back to the tenant's currency limits. + """ + if adapter_config is None: + return None + if adapter_config.adapter_type == "google_ad_manager": + network_currency = adapter_config.gam_network_currency + return str(network_currency).upper() if network_currency else None + config_json = adapter_config.config_json if isinstance(adapter_config.config_json, dict) else {} + configured = config_json.get("currency") + if isinstance(configured, str) and configured.strip(): + return configured.strip().upper() + return None + + def inventory_profile_to_product_model(profile: InventoryProfile, *, default_currency: str) -> Product: """Build a transient Product model for a wholesale inventory bundle. @@ -152,16 +172,9 @@ def project_visible_inventory_profile_product( if not isinstance(profile, InventoryProfile) or not is_buyer_visible_inventory_profile(profile): return None adapter_config = AdapterConfigRepository(session, tenant_id).find_by_tenant() - preferred_currency = ( - adapter_config.gam_network_currency - if adapter_config is not None - and adapter_config.adapter_type == "google_ad_manager" - and adapter_config.gam_network_currency - else None - ) currency = default_currency or default_wholesale_currency( CurrencyLimitRepository(session, tenant_id).list_all(), - preferred=preferred_currency, + preferred=preferred_wholesale_currency(adapter_config), ) return inventory_profile_to_product_model(profile, default_currency=currency) diff --git a/src/core/tools/products.py b/src/core/tools/products.py index 0915134c15..9bc36ad85d 100644 --- a/src/core/tools/products.py +++ b/src/core/tools/products.py @@ -418,25 +418,19 @@ async def _get_products_impl( from src.core.inventory_profile_projection import ( default_wholesale_currency, inventory_profiles_to_resolved_products, + preferred_wholesale_currency, ) assert uow.currency_limits is not None assert uow.inventory_profiles is not None assert uow.adapter_configs is not None adapter_config = uow.adapter_configs.find_by_tenant() - preferred_currency = ( - adapter_config.gam_network_currency - if adapter_config is not None - and adapter_config.adapter_type == "google_ad_manager" - and adapter_config.gam_network_currency - else None - ) products = inventory_profiles_to_resolved_products( uow.inventory_profiles.list_all(), adapter_type=tenant_adapter_type, default_currency=default_wholesale_currency( uow.currency_limits.list_all(), - preferred=preferred_currency, + preferred=preferred_wholesale_currency(adapter_config), ), ) else: diff --git a/tests/unit/test_wholesale_currency_preference.py b/tests/unit/test_wholesale_currency_preference.py new file mode 100644 index 0000000000..e2db1ab102 --- /dev/null +++ b/tests/unit/test_wholesale_currency_preference.py @@ -0,0 +1,69 @@ +"""Wholesale bundle pricing must honour the ad server's configured currency. + +Inventory bundles carry no pricing options of their own; get_products projects +each one into a single auction-CPM option in a tenant-wide default currency. +That currency has to follow the adapter's configured currency (GAM network +currency, or ``config_json["currency"]`` for Improve Digital and friends), +otherwise buyers are quoted a currency the ad server will never book in. +""" + +from src.core.database.models import AdapterConfig, CurrencyLimit, InventoryProfile +from src.core.inventory_profile_projection import ( + default_wholesale_currency, + inventory_profile_to_product_model, + preferred_wholesale_currency, +) + + +def _limits(*codes: str) -> list[CurrencyLimit]: + return [CurrencyLimit(tenant_id="t", currency_code=code) for code in codes] + + +def test_improvedigital_config_currency_is_preferred(): + adapter = AdapterConfig(tenant_id="t", adapter_type="improvedigital", config_json={"currency": "EUR"}) + + assert preferred_wholesale_currency(adapter) == "EUR" + assert default_wholesale_currency(_limits("USD", "EUR"), preferred=preferred_wholesale_currency(adapter)) == "EUR" + + +def test_gam_network_currency_is_preferred_over_config_json(): + adapter = AdapterConfig( + tenant_id="t", + adapter_type="google_ad_manager", + gam_network_currency="GBP", + config_json={"currency": "EUR"}, + ) + + assert preferred_wholesale_currency(adapter) == "GBP" + + +def test_no_configured_currency_falls_back_to_currency_limits(): + assert preferred_wholesale_currency(None) is None + assert preferred_wholesale_currency(AdapterConfig(tenant_id="t", adapter_type="mock", config_json={})) is None + assert preferred_wholesale_currency(AdapterConfig(tenant_id="t", adapter_type="mock", config_json=None)) is None + assert default_wholesale_currency(_limits("USD", "EUR"), preferred=None) == "USD" + + +def test_configured_currency_without_matching_limit_falls_back(): + adapter = AdapterConfig(tenant_id="t", adapter_type="improvedigital", config_json={"currency": "GBP"}) + + assert default_wholesale_currency(_limits("USD", "EUR"), preferred=preferred_wholesale_currency(adapter)) == "USD" + + +def test_projected_bundle_pricing_option_uses_preferred_currency(): + profile = InventoryProfile( + tenant_id="t", + profile_id="improve_test_bundle", + name="Improve Test Bundle", + inventory_config={"ad_units": [], "placements": [], "include_descendants": False}, + format_ids=[{"agent_url": "https://creative.adcontextprotocol.org/", "id": "display_html"}], + publisher_properties=[ + {"publisher_domain": "spel.nl", "selection_type": "by_tag", "property_tags": ["all_inventory"]} + ], + ) + adapter = AdapterConfig(tenant_id="t", adapter_type="improvedigital", config_json={"currency": "eur"}) + currency = default_wholesale_currency(_limits("USD", "EUR"), preferred=preferred_wholesale_currency(adapter)) + + product = inventory_profile_to_product_model(profile, default_currency=currency) + + assert [po.currency for po in product.pricing_options] == ["EUR"] From f383a7d72b01acc90f12917c36c93f4ab3cd9652 Mon Sep 17 00:00:00 2001 From: Sadrul Islam Toaha Date: Tue, 15 Sep 2026 14:43:38 +0600 Subject: [PATCH 89/90] fix(improvedigital): book bundle-backed packages against their picked placements --- src/adapters/improvedigital/adapter.py | 23 +++++++++++++- tests/unit/test_improvedigital_live_paths.py | 32 ++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/adapters/improvedigital/adapter.py b/src/adapters/improvedigital/adapter.py index 77dd58b678..212f127436 100644 --- a/src/adapters/improvedigital/adapter.py +++ b/src/adapters/improvedigital/adapter.py @@ -836,8 +836,29 @@ def _flight_detail( } def _product_config_from_package(self, package: MediaPackage) -> dict[str, Any]: + """Return the Improve Digital product config carried by a package. + + Legacy products store ``placement_ids``/``package_ids`` directly + (optionally nested under ``"improvedigital"``). Inventory bundles + instead overlay their picked inventory in the generic bundle + vocabulary — ``targeted_ad_unit_ids`` (bundle ``ad_units`` = 360Yield + placements) and ``targeted_placement_ids`` (bundle ``placements`` = + 360Yield packages) — so translate those when the adapter keys are + absent, otherwise a bundle-backed line item would target nothing. + """ impl = getattr(package, "implementation_config", None) or {} - return impl.get("improvedigital", impl) if isinstance(impl, dict) else {} + if not isinstance(impl, dict): + return {} + config = dict(impl.get("improvedigital", impl)) + if not config.get("placement_ids") and impl.get("targeted_ad_unit_ids"): + config["placement_ids"] = self._numeric_ids(impl["targeted_ad_unit_ids"]) + if not config.get("package_ids") and impl.get("targeted_placement_ids"): + config["package_ids"] = self._numeric_ids(impl["targeted_placement_ids"]) + return config + + @staticmethod + def _numeric_ids(values: Any) -> list[int]: + return [int(v) for v in values if str(v).strip().isdigit()] if isinstance(values, list | tuple) else [] def _with_product_geo_defaults(self, package: MediaPackage, product_config: dict[str, Any]) -> dict[str, Any]: """Fold the product's generic "Target Countries" selection diff --git a/tests/unit/test_improvedigital_live_paths.py b/tests/unit/test_improvedigital_live_paths.py index f10bde01f7..cc39f646ab 100644 --- a/tests/unit/test_improvedigital_live_paths.py +++ b/tests/unit/test_improvedigital_live_paths.py @@ -319,6 +319,38 @@ def test_package_without_inventory_selection_fails_loudly(self): # The partially created campaign must not be left orphaned upstream. assert ("delete_campaign", 101) in adapter._client.campaigns.calls + def test_bundle_backed_package_translates_generic_inventory_keys(self): + """Inventory bundles overlay picked inventory as ``targeted_ad_unit_ids`` + (360Yield placements) / ``targeted_placement_ids`` (360Yield packages); + the adapter must book those, not fail as if nothing was selected.""" + adapter = make_live_adapter() + package = make_sample_video_package() + package.implementation_config = { + "source": "inventory_profile", + "status": "active", + "targeted_ad_unit_ids": ["11", "12"], + "targeted_placement_ids": ["77"], + } + response = invoke_create_media_buy(adapter, make_sample_create_request(), [package]) + + assert not getattr(response, "errors", None) + calls = adapter._client.campaigns.calls + placements_call = next(c for c in calls if c[0] == "set_line_item_placements") + assert placements_call[3] == { + "line_item_placements": [{"id": 11, "assigned": True}, {"id": 12, "assigned": True}] + } + packages_call = next(c for c in calls if c[0] == "set_packages") + assert packages_call[3] == {"line_item_packages": [{"id": 77, "assigned": True}]} + + def test_explicit_adapter_inventory_keys_win_over_generic_ones(self): + adapter = make_live_adapter() + package = make_sample_video_package() + package.implementation_config = { + "improvedigital": {"placement_ids": [5]}, + "targeted_ad_unit_ids": ["11"], + } + assert adapter._product_config_from_package(package)["placement_ids"] == [5] + class TestCreativesLive: def test_upload_echoes_platform_creative_id(self): From 2dbc5218845d6c1eb9afa0bdef8d134f10c27c5c Mon Sep 17 00:00:00 2001 From: Chinmoy Acharjee Date: Tue, 15 Sep 2026 15:51:07 +0600 Subject: [PATCH 90/90] Fms 1929 fix webhooks (#65) * fix(slack): pass tenant_id not tenant_name to webhook delivery records * fix(webhooks): only legacy auth schemes select hmac signing mode * fix(webhooks): send create_media_buy decision webhook after admin approval * fix(webhooks): map internal step statuses to AdCP task statuses * fix(webhooks): send rejected webhook from Workflows page reject * fix(workflows): move media buy to rejected on Workflows page reject --- scripts/deploy/run_all_services.py | 3 +- src/admin/blueprints/operations.py | 137 ++++++------------ src/admin/blueprints/workflows.py | 48 ++++++ src/core/audit_logger.py | 2 + src/core/context_manager.py | 34 ++++- src/services/protocol_webhook_service.py | 82 ++++++++++- .../push_notification_registration.py | 13 +- src/services/slack_notifier.py | 6 +- tests/unit/test_media_buy_decision_webhook.py | 110 ++++++++++++++ .../test_push_notification_signing_mode.py | 50 +++++++ .../unit/test_webhook_task_status_mapping.py | 36 +++++ tests/unit/test_workflow_decision_webhook.py | 90 ++++++++++++ 12 files changed, 505 insertions(+), 106 deletions(-) create mode 100644 tests/unit/test_media_buy_decision_webhook.py create mode 100644 tests/unit/test_push_notification_signing_mode.py create mode 100644 tests/unit/test_webhook_task_status_mapping.py create mode 100644 tests/unit/test_workflow_decision_webhook.py diff --git a/scripts/deploy/run_all_services.py b/scripts/deploy/run_all_services.py index 90828bf614..5086b370cf 100644 --- a/scripts/deploy/run_all_services.py +++ b/scripts/deploy/run_all_services.py @@ -22,8 +22,7 @@ # Exit code of the MCP/A2A/Admin server child once it has stopped. ``None`` # while it is running. The main loop watches this so the container exits -# (and the orchestrator restarts it) instead of staying alive with nothing -# listening on the app port, which the load balancer reports as 502. +# (and the orchestrator restarts it) instead of leaving nginx serving 502s. _mcp_exit_code: int | None = None diff --git a/src/admin/blueprints/operations.py b/src/admin/blueprints/operations.py index 21496e401f..a0a4144969 100644 --- a/src/admin/blueprints/operations.py +++ b/src/admin/blueprints/operations.py @@ -4,9 +4,6 @@ import logging from datetime import UTC, datetime -from adcp import create_a2a_webhook_payload, create_mcp_webhook_payload -from adcp.types import CreateMediaBuySuccessResponse, Package -from adcp.types import GeneratedTaskStatus as AdcpTaskStatus from flask import Blueprint, request from sqlalchemy import select @@ -14,7 +11,10 @@ from src.admin.utils.embedded_capabilities import capability_owned_response, publisher_owns from src.core.database.models import PushNotificationConfig from src.core.database.repositories.media_buy import MediaBuyRepository -from src.services.protocol_webhook_service import get_protocol_webhook_service +from src.services.protocol_webhook_service import ( + build_request_scoped_config, + send_create_media_buy_decision, +) logger = logging.getLogger(__name__) @@ -469,6 +469,9 @@ def approve_media_buy(tenant_id, media_buy_id, **kwargs): media_buy_data = { "principal_id": media_buy.principal_id, "push_notification_url": push_config.get("url"), + "push_notification_config": push_config, + "confirmed_at": media_buy.confirmed_at, + "revision": media_buy.revision, } if action == "approve": @@ -618,54 +621,27 @@ def approve_media_buy(tenant_id, media_buy_id, **kwargs): ) webhook_config = db_session.scalars(stmt_webhook).first() + if webhook_config is None and media_buy_data: + webhook_config = build_request_scoped_config( + tenant_id=tenant_id, + principal_id=media_buy_data["principal_id"], + push_config=media_buy_data["push_notification_config"], + ) if webhook_config and media_buy_data: - approve_repo = MediaBuyRepository(db_session, tenant_id) - all_packages = approve_repo.get_packages(media_buy_id) - - create_media_buy_approved_result = CreateMediaBuySuccessResponse( + send_create_media_buy_decision( + config=webhook_config, + step_id=step_data["step_id"], + context_id=step_data["context_id"], + protocol=step_data["request_data"].get("protocol", "mcp"), media_buy_id=media_buy_id, - packages=[Package(package_id=x.package_id) for x in all_packages], - context={}, # TODO: @yusuf - please fix this, like we've fixed in the creative approval + package_ids=[ + x.package_id + for x in MediaBuyRepository(db_session, tenant_id).get_packages(media_buy_id) + ], + status="completed", + confirmed_at=media_buy_data["confirmed_at"], + revision=media_buy_data["revision"], ) - metadata = { - "task_type": step_data["tool_name"], - # TODO: @yusuf - check if we were passing principal_id and tenant to this previously - # TODO: @yusuf - check if we want to make metadata typed - } - - # Determine protocol type from workflow step request_data - protocol = step_data["request_data"].get( - "protocol", "mcp" - ) # Default to MCP for backward compatibility - - # Create appropriate webhook payload based on protocol - if protocol == "a2a": - create_media_buy_approved_payload = create_a2a_webhook_payload( - task_id=step_data["step_id"], - status=AdcpTaskStatus.completed, - result=create_media_buy_approved_result, - context_id=step_data["context_id"], - ) - else: - create_media_buy_approved_payload = create_mcp_webhook_payload( - task_id=step_data["step_id"], - status=AdcpTaskStatus.completed, - task_type="create_media_buy", - result=create_media_buy_approved_result, - ) - - try: - service = get_protocol_webhook_service() - asyncio.run( - service.send_notification( - push_notification_config=webhook_config, - payload=create_media_buy_approved_payload, - metadata=metadata, - ) - ) - logger.info(f"Sent webhook notification for approved media buy {media_buy_id}") - except Exception as webhook_err: - logger.warning(f"Failed to send webhook notification: {webhook_err}") flash("Media buy approved and order created successfully", "success") else: @@ -709,55 +685,26 @@ def approve_media_buy(tenant_id, media_buy_id, **kwargs): ) webhook_config = db_session.scalars(stmt_webhook).first() + if webhook_config is None and media_buy_data: + webhook_config = build_request_scoped_config( + tenant_id=tenant_id, + principal_id=media_buy_data["principal_id"], + push_config=media_buy_data["push_notification_config"], + ) if webhook_config and media_buy_data: - reject_repo = MediaBuyRepository(db_session, tenant_id) - all_packages = reject_repo.get_packages(media_buy_id) - - create_media_buy_rejected_result = CreateMediaBuySuccessResponse( + send_create_media_buy_decision( + config=webhook_config, + step_id=step_data["step_id"], + context_id=step_data["context_id"], + protocol=step_data["request_data"].get("protocol", "mcp"), media_buy_id=media_buy_id, - packages=[Package(package_id=x.package_id) for x in all_packages], - context={}, # TODO: @yusuf - please fix this, like we've fixed in the creative approval + package_ids=[ + x.package_id for x in MediaBuyRepository(db_session, tenant_id).get_packages(media_buy_id) + ], + status="rejected", + confirmed_at=media_buy_data["confirmed_at"], + revision=media_buy_data["revision"], ) - metadata = { - "task_type": step_data["tool_name"], - # TODO: @yusuf - check if we were passing principal_id and tenant to this previously - # TODO: @yusuf - check if we want to make metadata typed - } - - # Determine protocol type from workflow step request_data - protocol = step_data["request_data"].get( - "protocol", "mcp" - ) # Default to MCP for backward compatibility - - # Create appropriate webhook payload based on protocol - if protocol == "a2a": - create_media_buy_rejected_payload = create_a2a_webhook_payload( - task_id=step_data["step_id"], - status=AdcpTaskStatus.rejected, - result=create_media_buy_rejected_result, - context_id=step_data["context_id"], - ) - else: - create_media_buy_rejected_payload = create_mcp_webhook_payload( - task_id=step_data["step_id"], - status=AdcpTaskStatus.rejected, - task_type="create_media_buy", - result=create_media_buy_rejected_result, - ) - - try: - service = get_protocol_webhook_service() - asyncio.run( - service.send_notification( - push_notification_config=webhook_config, - payload=create_media_buy_rejected_payload, - metadata=metadata, - ) - ) - logger.info(f"Sent webhook notification for rejected media buy {media_buy_id}") - - except Exception as webhook_err: - logger.warning(f"Failed to send webhook notification: {webhook_err}") flash("Media buy rejected", "info") diff --git a/src/admin/blueprints/workflows.py b/src/admin/blueprints/workflows.py index a7b49886ed..3f55d82e6c 100644 --- a/src/admin/blueprints/workflows.py +++ b/src/admin/blueprints/workflows.py @@ -15,6 +15,7 @@ from src.core.database.models import Principal as ModelPrincipal from src.core.database.repositories import MediaBuyRepository from src.core.database.repositories.workflow import WorkflowRepository +from src.services.protocol_webhook_service import build_request_scoped_config, send_create_media_buy_decision logger = logging.getLogger(__name__) @@ -164,6 +165,34 @@ def review_workflow_step(tenant_id, workflow_id, step_id): ) +def _notify_media_buy_decision(step, tenant_id: str, media_buy, media_buy_repo, *, status: str) -> bool: + """Send the terminal ``create_media_buy`` task webhook after an operator decision. + + The buyer's ``push_notification_config`` is request-scoped: it lives on + the workflow step's ``request_data``, never in ``push_notification_configs``. + Returns ``False`` when the request registered no webhook. + """ + request_data = step.request_data or {} + webhook_config = build_request_scoped_config( + tenant_id=tenant_id, + principal_id=media_buy.principal_id, + push_config=request_data.get("push_notification_config"), + ) + if webhook_config is None: + return False + return send_create_media_buy_decision( + config=webhook_config, + step_id=step.step_id, + context_id=step.context_id, + protocol=request_data.get("protocol", "mcp"), + media_buy_id=media_buy.media_buy_id, + package_ids=[x.package_id for x in media_buy_repo.get_packages(media_buy.media_buy_id)], + status=status, + confirmed_at=media_buy.confirmed_at, + revision=media_buy.revision, + ) + + def _replay_update_media_buy(step, tenant_id: str, db) -> tuple[bool, str | None]: """Re-enter `_update_media_buy_impl` with the persisted request payload. @@ -349,6 +378,10 @@ def approve_workflow_step(tenant_id, workflow_id, step_id): db.commit() logger.info(f"[APPROVAL] Media buy {media_buy_id} successfully created in adapter") + + # Buyer registered push_notification_config on create_media_buy: + # send the terminal task-status webhook (spec: completed). + _notify_media_buy_decision(step, tenant_id, media_buy, media_buy_repo, status="completed") flash("Workflow step approved and media buy created successfully", "success") else: logger.warning( @@ -392,6 +425,21 @@ def reject_workflow_step(tenant_id, workflow_id, step_id): db.commit() + # Buyer registered push_notification_config on create_media_buy: + # send the terminal task-status webhook (spec: rejected). + mappings = workflow_repo.get_mappings_for_step(step_id) + mapping = next((m for m in mappings if m.object_type == "media_buy"), None) + if mapping: + media_buy_repo = MediaBuyRepository(db, tenant_id) + media_buy = media_buy_repo.get_by_id(mapping.object_id) + if media_buy: + # Mirror operations.approve_media_buy(action="reject"): the buy + # itself must leave pending_approval, not just the step. + if media_buy.status == "pending_approval": + media_buy_repo.update_status(media_buy.media_buy_id, "rejected") + db.commit() + _notify_media_buy_decision(step, tenant_id, media_buy, media_buy_repo, status="rejected") + flash("Workflow step rejected", "info") return jsonify({"success": True}), 200 diff --git a/src/core/audit_logger.py b/src/core/audit_logger.py index 72e38394c6..df8507fabf 100644 --- a/src/core/audit_logger.py +++ b/src/core/audit_logger.py @@ -249,6 +249,7 @@ def log_operation( error_message=error, details=details, security_alert=security_alert, + tenant_id=tenant_id, ) except Exception: # Don't let Slack failures affect core functionality @@ -322,6 +323,7 @@ def log_security_violation( error_message=f"Security violation: {reason}", details={"resource_id": resource_id, "violation_type": "unauthorized_access"}, security_alert=True, + tenant_id=tenant_id, ) except Exception: # Don't let Slack failures affect core functionality diff --git a/src/core/context_manager.py b/src/core/context_manager.py index 77374d9327..4f4d9ddb85 100644 --- a/src/core/context_manager.py +++ b/src/core/context_manager.py @@ -29,6 +29,35 @@ def _coerce_task_type(raw: str | None) -> TaskType | None: return None +# Internal ``WorkflowStep.status`` values that are not AdCP task statuses. +# Anything already a ``GeneratedTaskStatus`` value passes through unchanged. +_INTERNAL_STATUS_TO_ADCP: dict[str, GeneratedTaskStatus] = { + "requires_approval": GeneratedTaskStatus.input_required, + "pending_approval": GeneratedTaskStatus.input_required, + "pending": GeneratedTaskStatus.submitted, + "in_progress": GeneratedTaskStatus.working, + "approved": GeneratedTaskStatus.working, # approved by operator; adapter creation still pending +} + + +def _coerce_task_status(raw: str | None) -> GeneratedTaskStatus: + """Map a workflow-step status to the AdCP ``GeneratedTaskStatus`` for webhooks. + + Steps use internal vocabulary (``requires_approval`` ...). Buyers only + understand the AdCP enum; ``unknown`` is the last resort, not the answer + for a step that is simply waiting on the publisher. + """ + if not raw: + return GeneratedTaskStatus.unknown + mapped = _INTERNAL_STATUS_TO_ADCP.get(raw) + if mapped is not None: + return mapped + try: + return GeneratedTaskStatus(raw) + except ValueError: + return GeneratedTaskStatus.unknown + + from sqlalchemy import select from src.core.database.database_session import DatabaseManager @@ -700,10 +729,7 @@ def _send_push_notifications(self, step: WorkflowStep, new_status: str, session: raw_task_type = step.tool_name or mapping.action or "" task_type_enum = _coerce_task_type(raw_task_type) protocol = (step.request_data or {}).get("protocol", "mcp") - try: - status_enum = GeneratedTaskStatus(new_status) - except ValueError: - status_enum = GeneratedTaskStatus.unknown + status_enum = _coerce_task_status(new_status) payload: Task | TaskStatusUpdateEvent | McpWebhookPayload if protocol == "a2a": diff --git a/src/services/protocol_webhook_service.py b/src/services/protocol_webhook_service.py index 8d32f59664..93140728f8 100644 --- a/src/services/protocol_webhook_service.py +++ b/src/services/protocol_webhook_service.py @@ -23,8 +23,8 @@ import requests from a2a.types import Task, TaskStatusUpdateEvent -from adcp import extract_webhook_result_data -from adcp.types import McpWebhookPayload +from adcp import create_a2a_webhook_payload, create_mcp_webhook_payload, extract_webhook_result_data +from adcp.types import CreateMediaBuySuccessResponse, GeneratedTaskStatus, McpWebhookPayload, Package from adcp.webhooks import generate_webhook_idempotency_key, sign_legacy_webhook from google.protobuf.json_format import MessageToDict @@ -558,6 +558,84 @@ async def close(self): _webhook_service: ProtocolWebhookService | None = None +def build_request_scoped_config( + *, tenant_id: str, principal_id: str | None, push_config: dict[str, Any] | None +) -> PushNotificationConfig | None: + """Rebuild a delivery config from a request's ``push_notification_config``. + + ``create_media_buy`` stores the buyer's webhook config on the workflow step + (request-scoped); it is never persisted to ``push_notification_configs``. + Admin approve/reject handlers use this to send the terminal task-status + webhook. Mirrors ``ContextManager._send_push_notifications``. Returns + ``None`` when no URL was registered. + """ + if not push_config: + return None + url = push_config.get("url") + if not url: + return None + authentication = push_config.get("authentication") or {} + schemes = authentication.get("schemes") or [] + auth_type = schemes[0] if isinstance(schemes, list) and schemes else None + return PushNotificationConfig( + id=push_config.get("id") or f"pnc_{uuid4().hex[:16]}", + tenant_id=tenant_id, + principal_id=principal_id, + url=str(url), + authentication_type=auth_type, + authentication_token=authentication.get("credentials"), + purpose="async_task", + is_active=True, + ) + + +def send_create_media_buy_decision( + *, + config: PushNotificationConfig, + step_id: str, + context_id: str, + protocol: str, + media_buy_id: str, + package_ids: list[str], + status: str, + confirmed_at: datetime | None = None, + revision: int = 1, +) -> bool: + """Send the terminal ``create_media_buy`` task webhook after an admin decision. + + ``status`` is an AdCP task status (``completed`` on approve, ``rejected`` on + reject). Synchronous: intended for Flask admin handlers. Never raises — + delivery failure is logged and reported as ``False``. + """ + result = CreateMediaBuySuccessResponse( + media_buy_id=media_buy_id, + status="completed", + packages=[Package(package_id=pid) for pid in package_ids], + confirmed_at=confirmed_at or datetime.now(UTC), + revision=revision, + ) + task_status = GeneratedTaskStatus(status) + payload: Task | TaskStatusUpdateEvent | McpWebhookPayload + if protocol == "a2a": + payload = create_a2a_webhook_payload(task_id=step_id, status=task_status, result=result, context_id=context_id) + else: + payload = create_mcp_webhook_payload( + task_id=step_id, status=task_status, task_type="create_media_buy", result=result + ) + metadata = {"task_type": "create_media_buy", "tenant_id": config.tenant_id, "principal_id": config.principal_id} + try: + ok = asyncio.run( + get_protocol_webhook_service().send_notification( + push_notification_config=config, payload=payload, metadata=metadata + ) + ) + except Exception as exc: + logger.warning("Failed to send %s webhook for media buy %s: %s", status, media_buy_id, exc) + return False + logger.info("Sent %s webhook for media buy %s", status, media_buy_id) + return bool(ok) + + def get_protocol_webhook_service() -> ProtocolWebhookService: """Get or create global webhook service instance.""" global _webhook_service diff --git a/src/services/push_notification_registration.py b/src/services/push_notification_registration.py index 43eb1f4ca0..30c13ad985 100644 --- a/src/services/push_notification_registration.py +++ b/src/services/push_notification_registration.py @@ -27,11 +27,18 @@ from src.services.protocol_webhook_service import _normalize_localhost_for_docker from src.services.webhook_signing import ( SIGNING_MODE_BOTH, + SIGNING_MODE_HMAC, SIGNING_MODE_RFC9421, SigningConfigurationError, load_active_signing_credential, ) +# Deprecated AdCP 3.x authentication schemes that select the legacy signing +# path (shared-secret HMAC or bare Bearer token). Anything else — including +# the spec's ``HTTP_MESSAGE_SIGNATURES`` marker — resolves to RFC 9421 so a +# webhook is never emitted unsigned. Compared case-insensitively. +_LEGACY_AUTH_SCHEMES = frozenset({"hmac-sha256", "bearer"}) + @dataclass(frozen=True, kw_only=True) class PushNotificationRegistration(PushNotificationConfigWebhookFields): @@ -65,7 +72,11 @@ def normalize_push_notification_config( str(config_dict["subscriber_id"]) if config_dict.get("subscriber_id") is not None else default_subscriber_id ) event_types = _normalize_event_types(config_dict.get("event_types")) - signing_mode = "hmac" if auth_type is not None else "rfc9421" + signing_mode = ( + SIGNING_MODE_HMAC + if auth_type is not None and auth_type.lower() in _LEGACY_AUTH_SCHEMES + else SIGNING_MODE_RFC9421 + ) return PushNotificationRegistration( config_id=config_id, url=str(url), diff --git a/src/services/slack_notifier.py b/src/services/slack_notifier.py index eaba949ee1..a6f2a27839 100644 --- a/src/services/slack_notifier.py +++ b/src/services/slack_notifier.py @@ -356,6 +356,7 @@ def notify_audit_log( error_message: str | None = None, details: dict[str, Any] | None = None, security_alert: bool = False, + tenant_id: str | None = None, ) -> bool: """ Send audit log entry to Slack audit channel. @@ -369,6 +370,7 @@ def notify_audit_log( error_message: Error message if operation failed details: Additional operation details security_alert: Whether this is a security-related event + tenant_id: Tenant ID for delivery tracking (must be the ID, not the display name) Returns: True if notification sent successfully @@ -453,7 +455,7 @@ def notify_audit_log( max_retries=3, timeout=10, event_type="slack.audit_log", - tenant_id=tenant_name, # Use tenant_name as identifier + tenant_id=tenant_id, ) success_delivery, result = deliver_webhook_with_retry(delivery) @@ -728,7 +730,7 @@ def notify_media_buy_event( max_retries=3, timeout=10, event_type="slack.media_buy_event", - tenant_id=tenant_name, + tenant_id=tenant_id, object_id=media_buy_id, ) diff --git a/tests/unit/test_media_buy_decision_webhook.py b/tests/unit/test_media_buy_decision_webhook.py new file mode 100644 index 0000000000..6c2816d1cd --- /dev/null +++ b/tests/unit/test_media_buy_decision_webhook.py @@ -0,0 +1,110 @@ +"""Approval / rejection webhooks for media buys created with push_notification_config. + +A ``create_media_buy`` request stores its ``push_notification_config`` on the +workflow step (request-scoped), never in ``push_notification_configs``. The +admin approve/reject handlers must therefore be able to rebuild a delivery +config from the step and send the terminal task-status webhook. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +from adcp.types import McpWebhookPayload + +from src.services.protocol_webhook_service import ( + build_request_scoped_config, + send_create_media_buy_decision, +) + +_PUSH_CONFIG = { + "url": "http://127.0.0.1:9999/webhook", + "operation_id": "op-1", + "authentication": {"schemes": ["HMAC-SHA256"], "credentials": "s" * 40}, +} + + +def test_build_request_scoped_config_from_step_config(): + cfg = build_request_scoped_config(tenant_id="t1", principal_id="p1", push_config=_PUSH_CONFIG) + assert cfg is not None + assert cfg.tenant_id == "t1" + assert cfg.principal_id == "p1" + assert cfg.url == _PUSH_CONFIG["url"] + assert cfg.authentication_type == "HMAC-SHA256" + assert cfg.authentication_token == "s" * 40 + assert cfg.purpose == "async_task" + assert cfg.is_active is True + + +def test_build_request_scoped_config_without_authentication(): + cfg = build_request_scoped_config(tenant_id="t1", principal_id="p1", push_config={"url": "https://b.example/h"}) + assert cfg is not None + assert cfg.authentication_type is None + assert cfg.authentication_token is None + + +def test_build_request_scoped_config_returns_none_when_no_url(): + assert build_request_scoped_config(tenant_id="t1", principal_id="p1", push_config=None) is None + assert build_request_scoped_config(tenant_id="t1", principal_id="p1", push_config={}) is None + assert build_request_scoped_config(tenant_id="t1", principal_id="p1", push_config={"operation_id": "x"}) is None + + +def _run_decision(protocol: str, status: str): + cfg = build_request_scoped_config(tenant_id="t1", principal_id="p1", push_config=_PUSH_CONFIG) + service = MagicMock() + service.send_notification = AsyncMock(return_value=True) + with patch("src.services.protocol_webhook_service.get_protocol_webhook_service", return_value=service): + ok = send_create_media_buy_decision( + config=cfg, + step_id="step_1", + context_id="ctx_1", + protocol=protocol, + media_buy_id="mb_1", + package_ids=["pkg_a", "pkg_b"], + status=status, + ) + assert ok is True + service.send_notification.assert_awaited_once() + return service.send_notification.await_args.kwargs + + +def test_send_decision_mcp_completed_payload(): + kwargs = _run_decision("mcp", "completed") + payload = kwargs["payload"] + assert isinstance(payload, McpWebhookPayload) + assert payload.task_id == "step_1" + assert payload.task_type.value == "create_media_buy" + assert payload.status.value == "completed" + dumped = payload.model_dump(mode="json", exclude_none=True) + assert dumped["result"]["media_buy_id"] == "mb_1" + assert [p["package_id"] for p in dumped["result"]["packages"]] == ["pkg_a", "pkg_b"] + assert kwargs["metadata"]["task_type"] == "create_media_buy" + assert kwargs["push_notification_config"].url == _PUSH_CONFIG["url"] + + +def test_send_decision_mcp_rejected_status(): + kwargs = _run_decision("mcp", "rejected") + assert kwargs["payload"].status.value == "rejected" + + +def test_send_decision_a2a_payload_is_task_object(): + kwargs = _run_decision("a2a", "completed") + payload = kwargs["payload"] + assert not isinstance(payload, McpWebhookPayload) + assert payload.id == "step_1" + assert payload.context_id == "ctx_1" + + +def test_send_decision_returns_false_when_send_raises(): + cfg = build_request_scoped_config(tenant_id="t1", principal_id="p1", push_config=_PUSH_CONFIG) + service = MagicMock() + service.send_notification = AsyncMock(side_effect=RuntimeError("boom")) + with patch("src.services.protocol_webhook_service.get_protocol_webhook_service", return_value=service): + ok = send_create_media_buy_decision( + config=cfg, + step_id="step_1", + context_id="ctx_1", + protocol="mcp", + media_buy_id="mb_1", + package_ids=[], + status="completed", + ) + assert ok is False diff --git a/tests/unit/test_push_notification_signing_mode.py b/tests/unit/test_push_notification_signing_mode.py new file mode 100644 index 0000000000..7d0ff91a35 --- /dev/null +++ b/tests/unit/test_push_notification_signing_mode.py @@ -0,0 +1,50 @@ +"""Signing-mode inference for buyer push_notification_config registrations. + +AdCP 3.x makes RFC 9421 the baseline. Only the deprecated legacy schemes +(``HMAC-SHA256`` shared secret, ``Bearer`` token) may select the legacy +``hmac`` signing mode. Any other scheme — including the spec's +``HTTP_MESSAGE_SIGNATURES`` marker — must resolve to ``rfc9421`` so the +webhook is never sent unsigned. +""" + +import pytest + +from src.services.push_notification_registration import normalize_push_notification_config + +_URL = "https://buyer.example.com/webhooks/adcp" + + +def _normalize(authentication: dict | None): + config: dict = {"url": _URL, "operation_id": "op-1"} + if authentication is not None: + config["authentication"] = authentication + registration = normalize_push_notification_config(config) + assert registration is not None + return registration + + +def test_http_message_signatures_scheme_selects_rfc9421(): + registration = _normalize({"schemes": ["HTTP_MESSAGE_SIGNATURES"]}) + assert registration.signing_mode == "rfc9421" + assert registration.authentication_type == "HTTP_MESSAGE_SIGNATURES" + + +def test_unknown_scheme_defaults_to_rfc9421(): + registration = _normalize({"schemes": ["SomethingNew"]}) + assert registration.signing_mode == "rfc9421" + + +def test_no_authentication_block_selects_rfc9421(): + registration = _normalize(None) + assert registration.signing_mode == "rfc9421" + assert registration.authentication_type is None + + +@pytest.mark.parametrize( + "scheme", + ["HMAC-SHA256", "hmac-sha256", "Bearer", "bearer"], +) +def test_legacy_schemes_select_hmac(scheme: str): + registration = _normalize({"schemes": [scheme], "credentials": "x" * 40}) + assert registration.signing_mode == "hmac" + assert registration.authentication_type == scheme diff --git a/tests/unit/test_webhook_task_status_mapping.py b/tests/unit/test_webhook_task_status_mapping.py new file mode 100644 index 0000000000..965c72f22a --- /dev/null +++ b/tests/unit/test_webhook_task_status_mapping.py @@ -0,0 +1,36 @@ +"""Internal workflow-step statuses must map to AdCP task statuses in webhooks. + +``WorkflowStep.status`` uses internal vocabulary (``requires_approval``, +``pending``, ``in_progress`` ...). Task-status webhooks must carry the AdCP +``GeneratedTaskStatus`` value the buyer understands; a buyer waiting on +``input-required`` cannot act on ``unknown``. +""" + +import pytest +from adcp.webhooks import GeneratedTaskStatus + +from src.core.context_manager import _coerce_task_status + + +@pytest.mark.parametrize( + ("internal", "expected"), + [ + ("requires_approval", GeneratedTaskStatus.input_required), + ("pending_approval", GeneratedTaskStatus.input_required), + ("pending", GeneratedTaskStatus.submitted), + ("in_progress", GeneratedTaskStatus.working), + ("approved", GeneratedTaskStatus.working), + ], +) +def test_internal_statuses_map_to_adcp(internal: str, expected: GeneratedTaskStatus): + assert _coerce_task_status(internal) is expected + + +@pytest.mark.parametrize("value", [s.value for s in GeneratedTaskStatus]) +def test_adcp_values_pass_through(value: str): + assert _coerce_task_status(value) is GeneratedTaskStatus(value) + + +@pytest.mark.parametrize("raw", ["", None, "something_internal"]) +def test_unmapped_falls_back_to_unknown(raw: str | None): + assert _coerce_task_status(raw) is GeneratedTaskStatus.unknown diff --git a/tests/unit/test_workflow_decision_webhook.py b/tests/unit/test_workflow_decision_webhook.py new file mode 100644 index 0000000000..a9ce2d6622 --- /dev/null +++ b/tests/unit/test_workflow_decision_webhook.py @@ -0,0 +1,90 @@ +"""Workflows-page approve/reject must notify the buyer of the terminal task status. + +``_notify_media_buy_decision`` is the shared helper behind both handlers in +``src/admin/blueprints/workflows.py``. It rebuilds the delivery config from +the step's request-scoped ``push_notification_config`` and sends the +``completed`` / ``rejected`` webhook. +""" + +from datetime import UTC, datetime +from unittest.mock import MagicMock, patch + +from src.admin.blueprints.workflows import _notify_media_buy_decision + +_PUSH_CONFIG = { + "url": "http://127.0.0.1:9999/webhook", + "authentication": {"schemes": ["Bearer"], "credentials": "t" * 40}, +} + + +def _make_step(request_data): + step = MagicMock() + step.step_id = "step_1" + step.context_id = "ctx_1" + step.request_data = request_data + return step + + +def _make_media_buy(): + mb = MagicMock() + mb.media_buy_id = "mb_1" + mb.principal_id = "p_1" + mb.confirmed_at = datetime(2026, 9, 1, tzinfo=UTC) + mb.revision = 3 + return mb + + +def _make_repo(package_ids): + repo = MagicMock() + repo.get_packages.return_value = [MagicMock(package_id=pid) for pid in package_ids] + return repo + + +def test_rejected_decision_sends_webhook_from_step_config(): + step = _make_step({"push_notification_config": _PUSH_CONFIG, "protocol": "a2a"}) + with patch("src.admin.blueprints.workflows.send_create_media_buy_decision", return_value=True) as send: + ok = _notify_media_buy_decision(step, "t_1", _make_media_buy(), _make_repo(["pkg_1"]), status="rejected") + + assert ok is True + send.assert_called_once() + kwargs = send.call_args.kwargs + assert kwargs["status"] == "rejected" + assert kwargs["protocol"] == "a2a" + assert kwargs["step_id"] == "step_1" + assert kwargs["context_id"] == "ctx_1" + assert kwargs["media_buy_id"] == "mb_1" + assert kwargs["package_ids"] == ["pkg_1"] + assert kwargs["revision"] == 3 + assert kwargs["confirmed_at"] == datetime(2026, 9, 1, tzinfo=UTC) + config = kwargs["config"] + assert config.tenant_id == "t_1" + assert config.principal_id == "p_1" + assert config.url == _PUSH_CONFIG["url"] + assert config.authentication_type == "Bearer" + + +def test_completed_decision_defaults_protocol_to_mcp(): + step = _make_step({"push_notification_config": _PUSH_CONFIG}) + with patch("src.admin.blueprints.workflows.send_create_media_buy_decision", return_value=True) as send: + _notify_media_buy_decision(step, "t_1", _make_media_buy(), _make_repo([]), status="completed") + + assert send.call_args.kwargs["protocol"] == "mcp" + assert send.call_args.kwargs["status"] == "completed" + + +def test_no_push_config_sends_nothing(): + step = _make_step({"protocol": "mcp"}) + with patch("src.admin.blueprints.workflows.send_create_media_buy_decision") as send: + ok = _notify_media_buy_decision(step, "t_1", _make_media_buy(), _make_repo([]), status="rejected") + + assert ok is False + send.assert_not_called() + + +def test_none_request_data_sends_nothing(): + step = _make_step(None) + with patch("src.admin.blueprints.workflows.send_create_media_buy_decision") as send: + ok = _notify_media_buy_decision(step, "t_1", _make_media_buy(), _make_repo([]), status="rejected") + + assert ok is False + send.assert_not_called()