Skip to content

Commit 29a2bcf

Browse files
Merge pull request #546 from nodease/feature/mba-326
fix: Model Routing 런타임 계약 및 PR 537 회귀 안정화
2 parents 0bc5eaf + 4b3407c commit 29a2bcf

31 files changed

Lines changed: 972 additions & 100 deletions

‎apps/client/app/features/workflow/api/workflowApi.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import {
4848
ModelRoutingBootstrapPreview,
4949
ModelRoutingBootstrapRequest,
5050
ModelRoutingBootstrapResponse,
51+
ModelRoutingBootstrapWriteResponse,
5152
WorkflowPermissionResponse,
5253
LLMTraceListResponse,
5354
WorkflowResponse,
@@ -501,7 +502,7 @@ export const workflowApi = {
501502
workflowId: string,
502503
nodeId: string,
503504
data: ModelRoutingBootstrapRequest,
504-
): Promise<ModelRoutingBootstrapResponse> => {
505+
): Promise<ModelRoutingBootstrapWriteResponse> => {
505506
const response = await api.post(
506507
`/workflows/${workflowId}/llm-nodes/${nodeId}/model-routing/bootstrap`,
507508
data,

‎apps/client/app/features/workflow/components/agentBuilder/AgentBuilderPanel.test.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2630,7 +2630,9 @@ describe('AgentBuilderPanel', () => {
26302630
screen.getByRole('button', { name: '선택한 Knowledge Base로 생성' }),
26312631
);
26322632

2633-
expect(await screen.findByRole('status')).toHaveTextContent('Workflow 확인 중');
2633+
await waitFor(() =>
2634+
expect(screen.getByRole('status')).toHaveTextContent('Workflow 확인 중'),
2635+
);
26342636
expect(screen.getByRole('checkbox', { name: '휴가 정책' })).toBeDisabled();
26352637
expect(
26362638
screen.getByRole('button', { name: '선택한 Knowledge Base로 생성' }),

‎apps/client/app/features/workflow/tests/costOptimizer/fr14-dashboard-automatic-optimization-management.test.tsx‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,12 @@ describe('FR-014 내 모듈 자동 최적화 관리', () => {
186186

187187
const card = await screen.findByText('예상 월 비용');
188188
const container = card.parentElement?.parentElement;
189-
expect(container).toHaveTextContent('$303.000');
190-
expect(container).toHaveTextContent('101개 배포 workflow');
191-
expect(container).toHaveTextContent('워크플로 실행 $202.000');
192-
expect(container).toHaveTextContent('Agent Builder $101.000');
189+
await waitFor(() => {
190+
expect(container).toHaveTextContent('$303.000');
191+
expect(container).toHaveTextContent('101개 배포 workflow');
192+
expect(container).toHaveTextContent('워크플로 실행 $202.000');
193+
expect(container).toHaveTextContent('Agent Builder $101.000');
194+
});
193195
});
194196

195197
it('전체 활성 workflow 요약을 불러오지 못하면 목록 페이지 비용으로 대체하지 않는다', async () => {

‎apps/client/app/features/workflow/types/Api.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,8 @@ export interface ModelRoutingPolicyPatchRequest extends WorkflowGraphCASExpectat
313313
fallback_model_id?: string | null;
314314
}
315315

316-
export interface ModelRoutingBootstrapRequest {
316+
export interface ModelRoutingBootstrapRequest
317+
extends WorkflowGraphCASExpectation {
317318
task_description: string;
318319
default_model_id: string;
319320
fallback_model_id?: string | null;
@@ -332,6 +333,12 @@ export interface ModelRoutingBootstrapResponse {
332333
created_at?: string | null;
333334
}
334335

336+
export interface ModelRoutingBootstrapWriteResponse
337+
extends ModelRoutingBootstrapResponse {
338+
graph_hash: string;
339+
updated_at: string;
340+
}
341+
335342
export interface ModelRoutingBootstrapPreview {
336343
task_fingerprint: string;
337344
history_mode: 'history' | 'judge_first';

‎apps/gateway/api/v1/endpoints/workflow.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,8 @@ class ModelRoutingBootstrapRequest(BaseModel):
327327
task_description: str = Field(min_length=10, max_length=4000)
328328
default_model_id: str = Field(min_length=1, max_length=255)
329329
fallback_model_id: str | None = Field(default=None, max_length=255)
330+
expected_graph_hash: str = Field(min_length=64, max_length=64)
331+
expected_updated_at: datetime
330332

331333

332334
class ModelRoutingPolicyRefreshRequest(BaseModel):
@@ -3850,7 +3852,16 @@ def create_model_routing_bootstrap_endpoint(
38503852
"""Planner 호출 없이 초안 단계의 Judge-first 정책을 준비한다."""
38513853
# 초기 기준 생성은 초안 편집 단계의 작업이다. 실제 배포 권한은 이후 배포 API에서
38523854
# 별도로 검사하므로, 여기서는 workflow 수정 권한만 요구한다.
3853-
workflow = ensure_workflow_permission(db, current_user, workflow_id, "write")
3855+
authorized_workflow = ensure_workflow_permission(
3856+
db, current_user, workflow_id, "write"
3857+
)
3858+
workflow = _lock_workflow_for_cas_graph_write(
3859+
db,
3860+
current_user,
3861+
workflow_id,
3862+
request_body,
3863+
authorized_workflow,
3864+
)
38543865
next_graph = copy.deepcopy(workflow.graph or {})
38553866
node = _ensure_cost_optimizer_llm_node(
38563867
SimpleNamespace(id=workflow.id, graph=next_graph), node_id
@@ -3911,10 +3922,11 @@ def create_model_routing_bootstrap_endpoint(
39113922
)
39123923
node["data"] = node_data
39133924
workflow.graph = next_graph
3914-
db.commit()
3915-
return PersistedModelRoutingBootstrapStore.public_summary(
3925+
summary = PersistedModelRoutingBootstrapStore.public_summary(
39163926
bootstrap, include_samples=True, db=db
39173927
)
3928+
graph_metadata = _commit_graph_write_with_canonical_metadata(db, workflow)
3929+
return {**(summary or {}), **graph_metadata}
39183930

39193931

39203932
@router.get("/{workflow_id}/llm-nodes/{node_id}/model-routing/policy")

‎apps/gateway/services/cost_optimizer_output_quality_service.py‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -331,23 +331,29 @@ def _rag_enabled(result: dict[str, Any]) -> bool:
331331
trace = result.get("trace") if isinstance(result, dict) else None
332332
if not isinstance(trace, dict):
333333
return False
334-
return bool(trace.get("rag_summary"))
334+
return bool(
335+
CostOptimizerOutputQualityService._safe_rag_summary(
336+
trace.get("rag_summary")
337+
)
338+
)
335339

336340
@classmethod
337341
def _variant_from_result(cls, result: dict[str, Any]) -> dict[str, Any]:
338342
output = result.get("output") if isinstance(result, dict) else {}
339343
output = output if isinstance(output, dict) else {"text": output}
340344
trace = result.get("trace") if isinstance(result, dict) else {}
341345
trace = trace if isinstance(trace, dict) else {}
342-
rag_summary = trace.get("rag_summary")
346+
rag_summary = cls._safe_rag_summary(trace.get("rag_summary"))
343347
return {
344348
"input": cls._judge_visible_value(
345349
result.get("input") if isinstance(result, dict) else None
346350
),
347351
"output": cls._judge_visible_value(output),
348352
"rag_enabled": bool(rag_summary),
349-
"authoritative_evidence_available": bool(rag_summary),
350-
"rag_summary": cls._safe_rag_summary(rag_summary),
353+
"authoritative_evidence_available": cls._authoritative_evidence_available(
354+
rag_summary
355+
),
356+
"rag_summary": rag_summary,
351357
}
352358

353359
@classmethod
@@ -429,6 +435,13 @@ def _safe_rag_summary(value: Any) -> dict[str, Any] | None:
429435
allowed = ("retrieved_chunk_count", "context_token_estimate", "evidence_sufficient")
430436
return {key: value.get(key) for key in allowed if key in value}
431437

438+
@staticmethod
439+
def _authoritative_evidence_available(value: Any) -> bool:
440+
if not isinstance(value, dict) or value.get("evidence_sufficient") is not True:
441+
return False
442+
count = value.get("retrieved_chunk_count")
443+
return not isinstance(count, bool) and isinstance(count, int) and count > 0
444+
432445
@staticmethod
433446
def _pair_order(baseline: dict[str, Any], candidate_result: dict[str, Any]) -> str:
434447
# 호출마다 baseline이 항상 왼쪽에 놓이는 위치 편향을 피한다.

‎apps/gateway/services/model_routing_preview_service.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from apps.workflow_engine.services.llm_service import LLMService as WorkflowRuntimeLLMService
1414
from apps.workflow_engine.services.model_router import (
1515
ModelRouter,
16+
ModelRoutingPromptRenderError,
1617
ModelRoutingUnavailableError,
1718
)
1819
from apps.workflow_engine.workflow.nodes.llm.entities import LLMNodeData
@@ -85,12 +86,16 @@ def preview(
8586
"active_policy": policy.active_policy,
8687
}
8788
try:
89+
routing_feature_text = ModelRouter.routing_feature_text(inputs, node_data)
8890
decision = ModelRouter.resolve_policy(
8991
policy_payload,
9092
inputs=inputs,
9193
node_data=node_data,
9294
available_model_ids=available_model_ids,
95+
routing_feature_text=routing_feature_text,
9396
)
97+
except ModelRoutingPromptRenderError as exc:
98+
raise ModelRoutingPreviewBlockedError(exc.args[0]) from exc
9499
except ModelRoutingUnavailableError as exc:
95100
raise ModelRoutingPreviewBlockedError(
96101
"model_routing.no_available_model"
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
from datetime import datetime, timezone
2+
from types import SimpleNamespace
3+
from unittest.mock import MagicMock, patch
4+
from uuid import uuid4
5+
6+
import pytest
7+
from fastapi import HTTPException
8+
from pydantic import ValidationError
9+
10+
from apps.gateway.api.v1.endpoints import workflow as workflow_endpoint
11+
12+
13+
def _workflow():
14+
return SimpleNamespace(
15+
id=uuid4(),
16+
organization_id=uuid4(),
17+
graph={
18+
"nodes": [
19+
{
20+
"id": "llm-answer",
21+
"type": "llmNode",
22+
"data": {
23+
"title": "답변",
24+
"model_id": "gpt-4.1-mini",
25+
"auto_model_routing": False,
26+
},
27+
}
28+
],
29+
"edges": [],
30+
},
31+
)
32+
33+
34+
def _request():
35+
return workflow_endpoint.ModelRoutingBootstrapRequest(
36+
task_description="회사 정책 근거를 비교해 안전한 답변을 작성합니다.",
37+
default_model_id="gpt-4.1-mini",
38+
fallback_model_id="gpt-4.1",
39+
expected_graph_hash="a" * 64,
40+
expected_updated_at=datetime(2026, 7, 18, tzinfo=timezone.utc),
41+
)
42+
43+
44+
def test_model_routing_bootstrap_request_requires_graph_cas_expectation():
45+
with pytest.raises(ValidationError):
46+
workflow_endpoint.ModelRoutingBootstrapRequest(
47+
task_description="회사 정책 근거를 비교해 안전한 답변을 작성합니다.",
48+
default_model_id="gpt-4.1-mini",
49+
)
50+
51+
52+
def test_model_routing_bootstrap_writes_artifact_and_graph_through_cas():
53+
workflow = _workflow()
54+
current_user = SimpleNamespace(id=uuid4())
55+
bootstrap = SimpleNamespace(id=uuid4(), task_fingerprint="fingerprint-v1")
56+
candidate_models = [
57+
SimpleNamespace(model_id="gpt-4.1-mini"),
58+
SimpleNamespace(model_id="gpt-4.1"),
59+
]
60+
graph_metadata = {
61+
"graph_hash": "b" * 64,
62+
"updated_at": "2026-07-18T00:00:01+00:00",
63+
}
64+
65+
with (
66+
patch.object(
67+
workflow_endpoint,
68+
"ensure_workflow_permission",
69+
return_value=workflow,
70+
),
71+
patch.object(
72+
workflow_endpoint,
73+
"_lock_workflow_for_cas_graph_write",
74+
return_value=workflow,
75+
) as lock_workflow,
76+
patch.object(
77+
workflow_endpoint,
78+
"_model_routing_candidates_for_user",
79+
return_value=candidate_models,
80+
),
81+
patch.object(
82+
workflow_endpoint.PersistedModelRoutingBootstrapStore,
83+
"create_ready",
84+
return_value=bootstrap,
85+
) as create_ready,
86+
patch.object(
87+
workflow_endpoint.PersistedModelRoutingBootstrapStore,
88+
"public_summary",
89+
return_value={"id": str(bootstrap.id), "status": "ready"},
90+
),
91+
patch.object(
92+
workflow_endpoint,
93+
"_commit_graph_write_with_canonical_metadata",
94+
return_value=graph_metadata,
95+
) as commit_graph,
96+
):
97+
result = workflow_endpoint.create_model_routing_bootstrap_endpoint(
98+
str(workflow.id),
99+
"llm-answer",
100+
_request(),
101+
db=MagicMock(),
102+
current_user=current_user,
103+
)
104+
105+
lock_workflow.assert_called_once()
106+
create_ready.assert_called_once()
107+
commit_graph.assert_called_once()
108+
node_data = workflow.graph["nodes"][0]["data"]
109+
assert node_data["auto_model_routing"] is True
110+
assert node_data["model_routing_bootstrap_id"] == str(bootstrap.id)
111+
assert node_data["model_routing_task_description"].startswith("회사 정책")
112+
assert result == {"id": str(bootstrap.id), "status": "ready", **graph_metadata}
113+
114+
115+
def test_model_routing_bootstrap_stale_graph_stops_before_artifact_creation():
116+
workflow = _workflow()
117+
conflict = HTTPException(status_code=409, detail="workflow.graph_conflict")
118+
119+
with (
120+
patch.object(
121+
workflow_endpoint,
122+
"ensure_workflow_permission",
123+
return_value=workflow,
124+
),
125+
patch.object(
126+
workflow_endpoint,
127+
"_lock_workflow_for_cas_graph_write",
128+
side_effect=conflict,
129+
),
130+
patch.object(
131+
workflow_endpoint.PersistedModelRoutingBootstrapStore,
132+
"create_ready",
133+
) as create_ready,
134+
pytest.raises(HTTPException) as exc_info,
135+
):
136+
workflow_endpoint.create_model_routing_bootstrap_endpoint(
137+
str(workflow.id),
138+
"llm-answer",
139+
_request(),
140+
db=MagicMock(),
141+
current_user=SimpleNamespace(id=uuid4()),
142+
)
143+
144+
assert exc_info.value.status_code == 409
145+
assert exc_info.value.detail == "workflow.graph_conflict"
146+
create_ready.assert_not_called()

0 commit comments

Comments
 (0)