Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,7 @@ API는 이 필드 전송과 별개로 실제 DB 상태 검사·생성 차단을
"intention": "...",
"userAnswer": "...",
"modelAnswer": "...",
"followUpQuestion": "다른 대안과 비교한 근거는 무엇인가요?",
"strengths": [],
"improvements": [],
"comment": "..."
Expand Down Expand Up @@ -1002,3 +1003,17 @@ PDF 프로세스 간 복사 비용과 기존 그림 의미 분석의 중복 호
정상 문자로 잘못 매핑된 글자와 일부 작은 문구·읽기 순서 문제는 초기 탐지 규칙이 놓칠 수 있다.
제목·연락처만 남은 경우의 유용성도 이 평가에 포함한다. 정제 후 급감은 진단만 남기며 재호출하지 않는다.
실문서 평가 전에는 기본 비활성 상태를 유지한다. 자동 테스트 통과는 OCR 품질 검증을 대신하지 않는다.

### 피드백 선택 필드와 기업 평가 복구

- solo/multi의 문항별 콜백은 선택 필드인 `followUpQuestion`을 제공한다. 실제 답변에 기반한 복습용 질문 한 문장이며 실제 면접에 삽입하지 않는다.
- 앞뒤 공백 제거 후 1~200 Unicode code point만 허용한다. 누락·공백·타입 오류·길이 초과는 null로 두고 채점은 유지한다.
- API 서버도 해당 필드의 저장·조회 매핑을 추가해야 하며 웹의 고정 안내 문구를 응답 필드에 연결해야 표시된다.
- 생성 문장의 Q 번호는 콜백 `feedbacks` 순서를 따른다. 확실한 내부 ID 참조만 교정하며 질문·답변 원문과 식별자는 바꾸지 않는다.
- SOLO/MULTI 채점과 기업 평가의 strict 도구 스키마에는 `minLength`, `maxLength`, `maxItems`를 보내지 않는다.
길이·개수 상한은 필드 설명으로 안내하고 서버에서 검증한다. 예상 꼬리질문 1~200자, 기업 총평 1~300자,
근거·제안 1~150자, 각 목록 최대 3개 정책은 유지한다. `question_ids.minItems=1`은 지원되는 제약이라 유지한다.
지원 범위는 [Anthropic structured outputs 문서](https://platform.claude.com/docs/en/build-with-claude/structured-outputs#json-schema-limitations)를 따른다.
- 기업 평가는 strict 도구 제출을 사용하고 잘못된 제출만 최대 한 번 다시 생성한다. 두 호출은 하나의 `COMPANY_FIT_TIMEOUT_SECONDS` 예산을 공유한다.
- API 오류·토큰 잘림·시간 초과에는 추가 재시도하지 않는다. 기업 평가가 실패해도 일반 채점은 유지한다.
- `company_fit.*` 로그의 `job_id`로 요청을 구분한다. 완료 로그에는 적용 여부·시도·종료 사유·출력 토큰·경과 시간이 있고 원문은 기록하지 않는다.
24 changes: 19 additions & 5 deletions src/app/core/commands/dispatch_feedback_multi.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from app.core.common.company.fit.dto import CompanyFitPayload
from app.core.common.company.fit.evaluate import CompanyFitEvaluation
from app.core.common.company.fit.logging_context import company_fit_job
from app.core.common.feedback.concurrent import run_with_company_fit
from app.core.common.feedback.dto import FeedbackCallbackFailure, FeedbackErrorDetail
from app.core.common.feedback.multi.answer_grading import MultiAnswerGrading
Expand All @@ -17,6 +18,8 @@
MultiFeedbackCallbackSuccess,
MultiInterviewFeedbackResult,
)
from app.core.common.feedback.presentation import normalize_follow_up_question, normalize_generated_fields
from app.core.common.feedback.question_label import FEEDBACK_TEXT_FIELDS, OVERALL_TEXT_FIELDS, PERSONA_TEXT_FIELDS
from app.core.common.feedback.scoring import (
AxisLevels,
SessionScores,
Expand Down Expand Up @@ -105,7 +108,8 @@ async def _build_payload(self, job_id: str, job_request: FeedbackMultiRequest) -
),
)
fit = self._evaluate_company_fit(assembled, job_request)
raw_result, company_fit = await run_with_company_fit(grading, fit)
with company_fit_job(job_id):
raw_result, company_fit = await run_with_company_fit(grading, fit)
levels = _question_levels(assembled, persona_by_question, raw_result)
scores = _session_scores(levels, raw_result)
result = self._build_result(job_request, assembled, persona_by_question, raw_result, levels, scores)
Expand Down Expand Up @@ -157,7 +161,7 @@ async def _evaluate_company_fit(
)
return await self._company_fit.execute(assembled.targets, job_request.company, tone)
except Exception as exc:
logger.warning("feedback_multi.company_fit.failed", extra={"error_type": type(exc).__name__})
logger.warning("feedback_multi.company_fit.failed error_type=%s", type(exc).__name__)
return None

def _build_result(
Expand All @@ -169,8 +173,15 @@ def _build_result(
levels: dict[str, AxisLevels],
scores: SessionScores,
) -> MultiInterviewFeedbackResult:
graded: dict[str, Any] = raw_result["feedbacks"]
graded_personas: dict[str, Any] = raw_result["personas"]
question_ids = tuple(target.question_id for target in assembled.targets)
graded = {
qid: normalize_generated_fields(entry, (*FEEDBACK_TEXT_FIELDS, "follow_up_question"), question_ids)
for qid, entry in raw_result["feedbacks"].items()
}
graded_personas = {
pid: normalize_generated_fields(entry, PERSONA_TEXT_FIELDS, question_ids)
for pid, entry in raw_result["personas"].items()
}

# 문항 순서는 LLM 응답 순서가 아니라 실제 면접 진행 순서를 따른다.
# 질문 본문/의도/사용자 답변/면접관은 요청 body 값을 그대로 되돌려준다(LLM 생성분이 아니다).
Expand All @@ -181,6 +192,9 @@ def _build_result(
"question_content": target.content,
"intention": target.intention,
"user_answer": target.answer,
"follow_up_question": normalize_follow_up_question(
graded[target.question_id].get("follow_up_question")
),
"model_answer": graded[target.question_id].get("model_answer", ""),
"strengths": graded[target.question_id].get("strengths", []),
"improvements": graded[target.question_id].get("improvements", []),
Expand All @@ -202,7 +216,7 @@ def _build_result(
for persona in job_request.personas
]

graded_overall: dict[str, Any] = raw_result["overall"]
graded_overall = normalize_generated_fields(raw_result["overall"], OVERALL_TEXT_FIELDS, question_ids)
# 3지표는 LLM 이 아니라 서버가 축 등급으로 계산한 값이다(scoring.py).
overall: dict[str, Any] = {
"total_score": scores.total_score,
Expand Down
19 changes: 15 additions & 4 deletions src/app/core/commands/dispatch_feedback_solo.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@

from app.core.common.company.fit.dto import CompanyFitPayload
from app.core.common.company.fit.evaluate import CompanyFitEvaluation
from app.core.common.company.fit.logging_context import company_fit_job
from app.core.common.feedback.concurrent import run_with_company_fit
from app.core.common.feedback.dto import FeedbackCallbackFailure, FeedbackErrorDetail
from app.core.common.feedback.presentation import normalize_follow_up_question, normalize_generated_fields
from app.core.common.feedback.question_label import FEEDBACK_TEXT_FIELDS, OVERALL_TEXT_FIELDS
from app.core.common.feedback.scoring import SessionScores, summarize_session
from app.core.common.feedback.solo.answer_assembly import AnswerAssembly
from app.core.common.feedback.solo.answer_grading import AnswerGrading
Expand Down Expand Up @@ -83,7 +86,8 @@ async def _build_payload(self, job_id: str, job_request: FeedbackSoloRequest) ->
),
)
fit = self._evaluate_company_fit(assembled, job_request)
raw_result, company_fit = await run_with_company_fit(grading, fit)
with company_fit_job(job_id):
raw_result, company_fit = await run_with_company_fit(grading, fit)
scores = _session_scores(assembled, raw_result)
result = self._build_result(assembled, raw_result, scores)
result.company_fit = company_fit
Expand Down Expand Up @@ -132,7 +136,7 @@ async def _evaluate_company_fit(
tone = job_request.persona_tone
return await self._company_fit.execute(assembled.targets, job_request.company, tone)
except Exception as exc:
logger.warning("feedback_solo.company_fit.failed", extra={"error_type": type(exc).__name__})
logger.warning("feedback_solo.company_fit.failed error_type=%s", type(exc).__name__)
return None

def _build_result(
Expand All @@ -141,7 +145,11 @@ def _build_result(
raw_result: dict[str, Any],
scores: SessionScores,
) -> InterviewFeedbackResult:
graded: dict[str, Any] = raw_result["feedbacks"]
question_ids = tuple(target.question_id for target in assembled.targets)
graded = {
qid: normalize_generated_fields(entry, (*FEEDBACK_TEXT_FIELDS, "follow_up_question"), question_ids)
for qid, entry in raw_result["feedbacks"].items()
}

# 문항 순서는 LLM 응답 순서가 아니라 실제 면접 진행 순서를 따른다.
# 질문 본문/의도/사용자 답변은 요청 body 값을 그대로 되돌려준다(LLM 생성분이 아니다).
Expand All @@ -151,6 +159,9 @@ def _build_result(
"question_content": target.content,
"intention": target.intention,
"user_answer": target.answer,
"follow_up_question": normalize_follow_up_question(
graded[target.question_id].get("follow_up_question")
),
"model_answer": graded[target.question_id].get("model_answer", ""),
"strengths": graded[target.question_id].get("strengths", []),
"improvements": graded[target.question_id].get("improvements", []),
Expand All @@ -159,7 +170,7 @@ def _build_result(
for target in assembled.targets
]

graded_overall: dict[str, Any] = raw_result["overall"]
graded_overall = normalize_generated_fields(raw_result["overall"], OVERALL_TEXT_FIELDS, question_ids)
# 3지표는 LLM 이 아니라 서버가 축 등급으로 계산한 값이다(scoring.py).
overall: dict[str, Any] = {
"total_score": scores.total_score,
Expand Down
126 changes: 75 additions & 51 deletions src/app/core/common/company/fit/evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,16 @@
CompanyFitItem,
CompanyFitPayload,
)
from app.core.common.company.fit.logging_context import CompanyFitLogger
from app.core.common.company.fit.prompt import SYSTEM_PROMPT, build_fit_user_message
from app.core.common.company.fit.tools import SUBMIT_COMPANY_FIT_TOOL, TOOL_NAME
from app.core.common.feedback.presentation import normalize_generated_fields
from app.core.common.feedback.question_label import warn_out_of_range_labels
from app.core.common.feedback.solo.dto import GradingTarget
from app.core.common.interview_qa.ports.anthropic_text_client import AnthropicTextClient
from app.core.common.interview_qa.ports.anthropic_text_client import AnthropicTextClient, AnthropicTextClientError
from app.core.common.tool_use import extract_tool_input

logger = logging.getLogger(__name__)
logger = CompanyFitLogger(logging.getLogger(__name__), {})


class CompanyFitEvaluation:
Expand Down Expand Up @@ -52,58 +54,80 @@ async def execute(
persona_tone: str | None,
) -> CompanyFitPayload | None:
started = time.monotonic()
message = build_fit_user_message(targets, company, persona_tone, self._answer_max_chars)
question_ids = tuple(target.question_id for target in targets)
try:
# SDK 내부 대기·재시도를 포함해 fit 시작부터 전체 논리 호출을 제한한다.
# SDK 내부 재시도 및 형식 보정 1회를 포함한 전체 예산이다.
async with asyncio.timeout(self._timeout_seconds):
response = await self._client.call(
model=self._model,
system=SYSTEM_PROMPT,
messages=[
{
"role": "user",
"content": build_fit_user_message(targets, company, persona_tone, self._answer_max_chars),
}
],
tools=[SUBMIT_COMPANY_FIT_TOOL],
tool_choice={"type": "tool", "name": TOOL_NAME},
max_tokens=self._max_tokens,
temperature=self._temperature,
)
for attempt in (1, 2):
response = await self._client.call(
model=self._model,
system=SYSTEM_PROMPT,
messages=[{"role": "user", "content": message}],
tools=[SUBMIT_COMPANY_FIT_TOOL],
tool_choice={"type": "tool", "name": TOOL_NAME},
max_tokens=self._max_tokens,
temperature=self._temperature,
)
tool_blocks = [block for block in response.content_blocks if block.get("type") == "tool_use"]
raw = extract_tool_input(response.content_blocks, TOOL_NAME)
result = None
if raw is None or response.stop_reason != "tool_use" or len(tool_blocks) != 1:
logger.warning(
"company_fit.submission_missing_or_truncated stop_reason=%s attempt=%d",
response.stop_reason,
attempt,
)
else:
result = _parse_submission(raw, company, set(question_ids))
if result is not None:
_normalize_references(result, question_ids)
logger.info(
"company_fit.completed applied=%s attempt=%d stop_reason=%s output_tokens=%d elapsed_ms=%.0f",
result is not None,
attempt,
response.stop_reason,
response.output_tokens,
(time.monotonic() - started) * 1000,
extra={
"applied": result is not None,
"matched_count": len(result.matched_focus) if result else 0,
"gap_count": len(result.gaps) if result else 0,
"input_tokens": response.input_tokens,
"output_tokens": response.output_tokens,
"attempt": attempt,
"elapsed_ms": (time.monotonic() - started) * 1000,
},
)
if result is not None or response.stop_reason == "max_tokens":
return result
if attempt == 1:
logger.warning("company_fit.retry reason=invalid_submission attempt=1")
message += (
"\n직전 응답이 제출 형식을 충족하지 못했습니다. "
"지정된 도구로 필수 필드와 길이를 지키고 제공된 fact_id·question_id만 사용해 제출하세요."
)
except TimeoutError:
logger.warning("company_fit.timeout", extra={"elapsed_ms": (time.monotonic() - started) * 1000})
return None
tool_blocks = [block for block in response.content_blocks if block.get("type") == "tool_use"]
raw = extract_tool_input(response.content_blocks, TOOL_NAME)
result = None
if raw is None or response.stop_reason != "tool_use" or len(tool_blocks) != 1:
logger.warning("company_fit.submission_missing_or_truncated")
else:
result = _parse_submission(raw, company, {target.question_id for target in targets})
if result is not None:
# 문장에 블록 번호(Q1…) 대신 question_id 를 쓴 흔적을 찾는다. 로그만 남기고 결과는 그대로 쓴다.
# 블록 번호는 targets 순서로 매기므로 대상 수도 targets 기준이다.
warn_out_of_range_labels(
logger,
"company_fit.question_id_label",
[
result.comment,
*(item.evidence for item in result.matched_focus),
*(gap.suggestion for gap in result.gaps),
],
len(targets),
)
logger.info(
"company_fit.completed",
extra={
"applied": result is not None,
"matched_count": len(result.matched_focus) if result else 0,
"gap_count": len(result.gaps) if result else 0,
"input_tokens": response.input_tokens,
"output_tokens": response.output_tokens,
"elapsed_ms": (time.monotonic() - started) * 1000,
},
)
return result
logger.warning("company_fit.timeout elapsed_ms=%.0f", (time.monotonic() - started) * 1000)
except AnthropicTextClientError as exc:
logger.warning("company_fit.api_failed reason=%s", exc.kind)
except Exception as exc:
logger.warning("company_fit.failed error_type=%s", type(exc).__name__)
return None


def _normalize_references(result: CompanyFitPayload, question_ids: tuple[str, ...]) -> None:
warn_out_of_range_labels(
logger,
"company_fit.question_id_label",
[result.comment, *(item.evidence for item in result.matched_focus), *(gap.suggestion for gap in result.gaps)],
len(question_ids),
)
result.comment = normalize_generated_fields({"comment": result.comment}, ("comment",), question_ids)["comment"]
for item in result.matched_focus:
item.evidence = normalize_generated_fields({"text": item.evidence}, ("text",), question_ids)["text"]
for gap in result.gaps:
gap.suggestion = normalize_generated_fields({"text": gap.suggestion}, ("text",), question_ids)["text"]


def _parse_submission(
Expand Down
28 changes: 28 additions & 0 deletions src/app/core/common/company/fit/logging_context.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
from __future__ import annotations

import logging
from collections.abc import Iterator, MutableMapping
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any

_job_id: ContextVar[str | None] = ContextVar("company_fit_job_id", default=None)


@contextmanager
def company_fit_job(job_id: str) -> Iterator[None]:
token = _job_id.set(job_id)
try:
yield
finally:
_job_id.reset(token)


class CompanyFitLogger(logging.LoggerAdapter[logging.Logger]):
def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> tuple[Any, MutableMapping[str, Any]]:
job_id = _job_id.get()
if job_id is not None:
# job_id는 HTTP 진입점에서 발급한 UUID다. 모델 응답·사용자 입력은 넣지 않는다.
msg = f"{msg} job_id={job_id}"
kwargs["extra"] = {**kwargs.get("extra", {}), "job_id": job_id}
return msg, kwargs
Loading
Loading